From ee3e4fea140b061bfb3cc0f863b9660913c8e9e6 Mon Sep 17 00:00:00 2001 From: Evan Hubinger Date: Sat, 29 Jul 2023 00:44:11 -0700 Subject: [PATCH 001/121] Reenable develop --- coconut/root.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/coconut/root.py b/coconut/root.py index 32cd33428..ff0868cab 100644 --- a/coconut/root.py +++ b/coconut/root.py @@ -26,7 +26,7 @@ VERSION = "3.0.3" VERSION_NAME = None # False for release, int >= 1 for develop -DEVELOP = False +DEVELOP = 1 ALPHA = False # for pre releases rather than post releases assert DEVELOP is False or DEVELOP >= 1, "DEVELOP must be False or an int >= 1" From 200deded781bc3b18021cc22603339e4ef14e4c8 Mon Sep 17 00:00:00 2001 From: Evan Hubinger Date: Sat, 29 Jul 2023 01:12:40 -0700 Subject: [PATCH 002/121] Remove --history-file Resolves #778. --- DOCS.md | 10 ++++------ coconut/command/cli.py | 9 --------- coconut/command/command.py | 2 -- coconut/constants.py | 12 ++++++++++-- coconut/root.py | 2 +- 5 files changed, 15 insertions(+), 20 deletions(-) diff --git a/DOCS.md b/DOCS.md index 9cf16df75..eb6f40df4 100644 --- a/DOCS.md +++ b/DOCS.md @@ -123,10 +123,10 @@ depth: 1 #### Usage ``` -coconut [-h] [--and source [dest ...]] [-v] [-t version] [-i] [-p] [-a] [-l] [-k] [-w] - [-r] [-n] [-d] [-q] [-s] [--no-tco] [--no-wrap-types] [-c code] [-j processes] - [-f] [--minify] [--jupyter ...] [--mypy ...] [--argv ...] [--tutorial] - [--docs] [--style name] [--history-file path] [--vi-mode] +coconut [-h] [--and source [dest ...]] [-v] [-t version] [-i] [-p] [-a] [-l] + [--no-line-numbers] [-k] [-w] [-r] [-n] [-d] [-q] [-s] [--no-tco] + [--no-wrap-types] [-c code] [-j processes] [-f] [--minify] [--jupyter ...] + [--mypy ...] [--argv ...] [--tutorial] [--docs] [--style name] [--vi-mode] [--recursion-limit limit] [--stack-size kbs] [--site-install] [--site-uninstall] [--verbose] [--trace] [--profile] [source] [dest] @@ -196,8 +196,6 @@ dest destination directory for compiled files (defaults to --style name set Pygments syntax highlighting style (or 'list' to list styles) (defaults to COCONUT_STYLE environment variable if it exists, otherwise 'default') ---history-file path set history file (or '' for no file) (can be modified by setting - COCONUT_HOME environment variable) --vi-mode, --vimode enable vi mode in the interpreter (currently set to False) (can be modified by setting COCONUT_VI_MODE environment variable) --recursion-limit limit, --recursionlimit limit diff --git a/coconut/command/cli.py b/coconut/command/cli.py index 5087e52d0..0281513b0 100644 --- a/coconut/command/cli.py +++ b/coconut/command/cli.py @@ -31,8 +31,6 @@ default_style, vi_mode_env_var, prompt_vi_mode, - prompt_histfile, - home_env_var, py_version_str, default_jobs, ) @@ -245,13 +243,6 @@ + style_env_var + " environment variable if it exists, otherwise '" + default_style + "')", ) -arguments.add_argument( - "--history-file", - metavar="path", - type=str, - help="set history file (or '' for no file) (currently set to " + ascii(prompt_histfile) + ") (can be modified by setting " + home_env_var + " environment variable)", -) - arguments.add_argument( "--vi-mode", "--vimode", action="store_true", diff --git a/coconut/command/command.py b/coconut/command/command.py index fee072a41..58acb6db0 100644 --- a/coconut/command/command.py +++ b/coconut/command/command.py @@ -278,8 +278,6 @@ def execute_args(self, args, interact=True, original_args=None): self.prompt.vi_mode = args.vi_mode if args.style is not None: self.prompt.set_style(args.style) - if args.history_file is not None: - self.prompt.set_history_file(args.history_file) if args.argv is not None: self.argv_args = list(args.argv) diff --git a/coconut/constants.py b/coconut/constants.py index 38f1c671d..c12c461ea 100644 --- a/coconut/constants.py +++ b/coconut/constants.py @@ -50,6 +50,11 @@ def get_bool_env_var(env_var, default=False): return default +def get_path_env_var(env_var, default): + """Get a path from an environment variable.""" + return fixpath(os.getenv(env_var, default)) + + # ----------------------------------------------------------------------------------------------------------------------- # VERSION CONSTANTS: # ----------------------------------------------------------------------------------------------------------------------- @@ -604,14 +609,17 @@ def get_bool_env_var(env_var, default=False): force_verbose_logger = get_bool_env_var("COCONUT_FORCE_VERBOSE", False) -coconut_home = fixpath(os.getenv(home_env_var, "~")) +coconut_home = get_path_env_var(home_env_var, "~") use_color = get_bool_env_var("COCONUT_USE_COLOR", None) error_color_code = "31" log_color_code = "93" default_style = "default" -prompt_histfile = os.path.join(coconut_home, ".coconut_history") +prompt_histfile = get_path_env_var( + "COCONUT_HISTORY_FILE", + os.path.join(coconut_home, ".coconut_history"), +) prompt_multiline = False prompt_vi_mode = get_bool_env_var(vi_mode_env_var, False) prompt_wrap_lines = True diff --git a/coconut/root.py b/coconut/root.py index ff0868cab..067ee2734 100644 --- a/coconut/root.py +++ b/coconut/root.py @@ -26,7 +26,7 @@ VERSION = "3.0.3" VERSION_NAME = None # False for release, int >= 1 for develop -DEVELOP = 1 +DEVELOP = 2 ALPHA = False # for pre releases rather than post releases assert DEVELOP is False or DEVELOP >= 1, "DEVELOP must be False or an int >= 1" From 24bcd15edc3e317ef8c71c8e9229b62b7739348e Mon Sep 17 00:00:00 2001 From: Evan Hubinger Date: Sun, 30 Jul 2023 21:24:25 -0700 Subject: [PATCH 003/121] Add --incremental Refs #772. --- DOCS.md | 8 ++-- Makefile | 20 ++++----- coconut/_pyparsing.py | 6 +++ coconut/command/cli.py | 6 +++ coconut/command/command.py | 22 ++++++++-- coconut/compiler/compiler.py | 74 ++++++++++++++++++++++--------- coconut/compiler/util.py | 83 ++++++++++++++++++++++++++++------- coconut/constants.py | 4 +- coconut/root.py | 2 +- coconut/terminal.py | 12 ++++- coconut/tests/main_test.py | 12 ++++- coconut/tests/src/extras.coco | 1 + coconut/util.py | 9 +++- 13 files changed, 199 insertions(+), 60 deletions(-) diff --git a/DOCS.md b/DOCS.md index eb6f40df4..14d9d23cc 100644 --- a/DOCS.md +++ b/DOCS.md @@ -125,9 +125,9 @@ depth: 1 ``` coconut [-h] [--and source [dest ...]] [-v] [-t version] [-i] [-p] [-a] [-l] [--no-line-numbers] [-k] [-w] [-r] [-n] [-d] [-q] [-s] [--no-tco] - [--no-wrap-types] [-c code] [-j processes] [-f] [--minify] [--jupyter ...] - [--mypy ...] [--argv ...] [--tutorial] [--docs] [--style name] [--vi-mode] - [--recursion-limit limit] [--stack-size kbs] [--site-install] + [--no-wrap-types] [-c code] [--incremental] [-j processes] [-f] [--minify] + [--jupyter ...] [--mypy ...] [--argv ...] [--tutorial] [--docs] [--style name] + [--vi-mode] [--recursion-limit limit] [--stack-size kbs] [--site-install] [--site-uninstall] [--verbose] [--trace] [--profile] [source] [dest] ``` @@ -176,6 +176,8 @@ dest destination directory for compiled files (defaults to disable wrapping type annotations in strings and turn off 'from __future__ import annotations' behavior -c code, --code code run Coconut passed in as a string (can also be piped into stdin) +--incremental enable incremental compilation mode (caches previous parses to + improve recompilation performance for slightly modified files) -j processes, --jobs processes number of additional processes to use (defaults to 'sys') (0 is no additional processes; 'sys' uses machine default) diff --git a/Makefile b/Makefile index 99ddd3752..a664fa94e 100644 --- a/Makefile +++ b/Makefile @@ -91,7 +91,7 @@ test-univ: clean .PHONY: test-univ-tests test-univ-tests: export COCONUT_USE_COLOR=TRUE test-univ-tests: clean-no-tests - python ./coconut/tests --strict --keep-lines + python ./coconut/tests --strict --keep-lines --incremental python ./coconut/tests/dest/runner.py python ./coconut/tests/dest/extras.py @@ -131,7 +131,7 @@ test-pypy3: clean .PHONY: test-mypy-univ test-mypy-univ: export COCONUT_USE_COLOR=TRUE test-mypy-univ: clean - python ./coconut/tests --strict --force --keep-lines --mypy --follow-imports silent --ignore-missing-imports --allow-redefinition + python ./coconut/tests --strict --keep-lines --force --mypy --follow-imports silent --ignore-missing-imports --allow-redefinition python ./coconut/tests/dest/runner.py python ./coconut/tests/dest/extras.py @@ -139,7 +139,7 @@ test-mypy-univ: clean .PHONY: test-mypy test-mypy: export COCONUT_USE_COLOR=TRUE test-mypy: clean - python ./coconut/tests --strict --force --target sys --keep-lines --mypy --follow-imports silent --ignore-missing-imports --allow-redefinition + python ./coconut/tests --strict --keep-lines --force --target sys --mypy --follow-imports silent --ignore-missing-imports --allow-redefinition python ./coconut/tests/dest/runner.py python ./coconut/tests/dest/extras.py @@ -147,7 +147,7 @@ test-mypy: clean .PHONY: test-mypy-tests test-mypy-tests: export COCONUT_USE_COLOR=TRUE test-mypy-tests: clean-no-tests - python ./coconut/tests --strict --target sys --keep-lines --mypy --follow-imports silent --ignore-missing-imports --allow-redefinition + python ./coconut/tests --strict --keep-lines --incremental --target sys --mypy --follow-imports silent --ignore-missing-imports --allow-redefinition python ./coconut/tests/dest/runner.py python ./coconut/tests/dest/extras.py @@ -171,7 +171,7 @@ test-verbose-sync: clean .PHONY: test-mypy-verbose test-mypy-verbose: export COCONUT_USE_COLOR=TRUE test-mypy-verbose: clean - python ./coconut/tests --strict --force --target sys --verbose --keep-lines --mypy --follow-imports silent --ignore-missing-imports --allow-redefinition + python ./coconut/tests --strict --keep-lines --force --target sys --verbose --mypy --follow-imports silent --ignore-missing-imports --allow-redefinition python ./coconut/tests/dest/runner.py python ./coconut/tests/dest/extras.py @@ -179,7 +179,7 @@ test-mypy-verbose: clean .PHONY: test-mypy-all test-mypy-all: export COCONUT_USE_COLOR=TRUE test-mypy-all: clean - python ./coconut/tests --strict --force --target sys --keep-lines --mypy --follow-imports silent --ignore-missing-imports --allow-redefinition --check-untyped-defs + python ./coconut/tests --strict --keep-lines --force --target sys --mypy --follow-imports silent --ignore-missing-imports --allow-redefinition --check-untyped-defs python ./coconut/tests/dest/runner.py python ./coconut/tests/dest/extras.py @@ -270,10 +270,10 @@ clean: clean-no-tests .PHONY: wipe wipe: clean rm -rf ./coconut/tests/dest vprof.json profile.log *.egg-info - -find . -name "__pycache__" -delete - -C:/GnuWin32/bin/find.exe . -name "__pycache__" -delete - -find . -name "__coconut_cache__" -delete - -C:/GnuWin32/bin/find.exe . -name "__coconut_cache__" -delete + -find . -name "__pycache__" -type d -prune -exec rm -rf '{}' + + -C:/GnuWin32/bin/find.exe . -name "__pycache__" -type d -prune -exec rm -rf '{}' + + -find . -name "__coconut_cache__" -type d -prune -exec rm -rf '{}' + + -C:/GnuWin32/bin/find.exe . -name "__coconut_cache__" -type d -prune -exec rm -rf '{}' + -find . -name "*.pyc" -delete -C:/GnuWin32/bin/find.exe . -name "*.pyc" -delete -python -m coconut --site-uninstall diff --git a/coconut/_pyparsing.py b/coconut/_pyparsing.py index 936b8b6a7..de21afa3a 100644 --- a/coconut/_pyparsing.py +++ b/coconut/_pyparsing.py @@ -210,6 +210,11 @@ def enableIncremental(*args, **kwargs): Keyword.setDefaultKeywordChars(varchars) +if SUPPORTS_INCREMENTAL: + all_parse_elements = ParserElement.collectParseElements() +else: + all_parse_elements = None + # ----------------------------------------------------------------------------------------------------------------------- # MISSING OBJECTS: @@ -258,6 +263,7 @@ def unset_fast_pyparsing_reprs(): for obj, (repr_method, str_method) in _old_pyparsing_reprs: obj.__repr__ = repr_method obj.__str__ = str_method + _old_pyparsing_reprs[:] = [] if use_fast_pyparsing_reprs: diff --git a/coconut/command/cli.py b/coconut/command/cli.py index 0281513b0..2bd237a13 100644 --- a/coconut/command/cli.py +++ b/coconut/command/cli.py @@ -183,6 +183,12 @@ help="run Coconut passed in as a string (can also be piped into stdin)", ) +arguments.add_argument( + "--incremental", + action="store_true", + help="enable incremental compilation mode (caches previous parses to improve recompilation performance for slightly modified files)", +) + arguments.add_argument( "-j", "--jobs", metavar="processes", diff --git a/coconut/command/command.py b/coconut/command/command.py index 58acb6db0..010a1ddb8 100644 --- a/coconut/command/command.py +++ b/coconut/command/command.py @@ -80,6 +80,7 @@ install_custom_kernel, get_clock_time, first_import_time, + ensure_dir, ) from coconut.command.util import ( writefile, @@ -129,6 +130,7 @@ class Command(object): mypy_args = None # corresponds to --mypy flag argv_args = None # corresponds to --argv flag stack_size = 0 # corresponds to --stack-size flag + incremental = False # corresponds to --incremental flag _prompt = None @@ -280,6 +282,7 @@ def execute_args(self, args, interact=True, original_args=None): self.prompt.set_style(args.style) if args.argv is not None: self.argv_args = list(args.argv) + self.incremental = args.incremental # execute non-compilation tasks if args.docs: @@ -563,8 +566,7 @@ def compile(self, codepath, destpath=None, package=False, run=False, force=False if destpath is not None: destpath = fixpath(destpath) destdir = os.path.dirname(destpath) - if not os.path.exists(destdir): - os.makedirs(destdir) + ensure_dir(destdir) if package is True: package_level = self.get_package_level(codepath) if package_level == 0: @@ -597,10 +599,22 @@ def callback(compiled): else: self.execute_file(destpath, argv_source_path=codepath) + parse_kwargs = dict( + filename=os.path.basename(codepath), + ) + if self.incremental: + code_dir, code_fname = os.path.split(codepath) + + cache_dir = os.path.join(code_dir, coconut_cache_dir) + ensure_dir(cache_dir) + + pickle_fname = code_fname + ".pickle" + parse_kwargs["incremental_cache_filename"] = os.path.join(cache_dir, pickle_fname) + if package is True: - self.submit_comp_job(codepath, callback, "parse_package", code, package_level=package_level, filename=os.path.basename(codepath)) + self.submit_comp_job(codepath, callback, "parse_package", code, package_level=package_level, **parse_kwargs) elif package is False: - self.submit_comp_job(codepath, callback, "parse_file", code, filename=os.path.basename(codepath)) + self.submit_comp_job(codepath, callback, "parse_file", code, **parse_kwargs) else: raise CoconutInternalException("invalid value for package", package) diff --git a/coconut/compiler/compiler.py b/coconut/compiler/compiler.py index 9abf5dd15..46e7c532b 100644 --- a/coconut/compiler/compiler.py +++ b/coconut/compiler/compiler.py @@ -171,6 +171,8 @@ get_psf_target, move_loc_to_non_whitespace, move_endpt_to_non_whitespace, + unpickle_incremental_cache, + pickle_incremental_cache, ) from coconut.compiler.header import ( minify_header, @@ -862,10 +864,14 @@ def reformat_locs(self, snip, loc, endpt=None, **kwargs): if endpt is None: return new_snip, new_loc - new_endpt = move_endpt_to_non_whitespace( - new_snip, - len(self.reformat(snip[:endpt], **kwargs)), + new_endpt = clip( + move_endpt_to_non_whitespace( + new_snip, + len(self.reformat(snip[:endpt], **kwargs)), + ), + min=new_loc, ) + return new_snip, new_loc, new_endpt def reformat_without_adding_code_before(self, code, **kwargs): @@ -1235,28 +1241,54 @@ def run_final_checks(self, original, keep_state=False): loc, ) - def parse(self, inputstring, parser, preargs, postargs, streamline=True, keep_state=False, filename=None): + def parse( + self, + inputstring, + parser, + preargs, + postargs, + streamline=True, + keep_state=False, + filename=None, + incremental_cache_filename=None, + ): """Use the parser to parse the inputstring with appropriate setup and teardown.""" with self.parsing(keep_state, filename): if streamline: self.streamline(parser, inputstring) - with logger.gather_parsing_stats(): - pre_procd = None - try: - pre_procd = self.pre(inputstring, keep_state=keep_state, **preargs) - parsed = parse(parser, pre_procd, inner=False) - out = self.post(parsed, keep_state=keep_state, **postargs) - except ParseBaseException as err: - raise self.make_parse_err(err) - except CoconutDeferredSyntaxError as err: - internal_assert(pre_procd is not None, "invalid deferred syntax error in pre-processing", err) - raise self.make_syntax_err(err, pre_procd) - # RuntimeError, not RecursionError, for Python < 3.5 - except RuntimeError as err: - raise CoconutException( - str(err), extra="try again with --recursion-limit greater than the current " - + str(sys.getrecursionlimit()) + " (you may also need to increase --stack-size)", - ) + # unpickling must happen after streamlining and must occur in the + # compiler so that it happens in the same process as compilation + if incremental_cache_filename is not None: + incremental_enabled = enable_incremental_parsing() + if not incremental_enabled: + raise CoconutException("--incremental mode requires cPyparsing (run '{python} -m pip install --upgrade cPyparsing' to fix)".format(python=sys.executable)) + did_load_cache = unpickle_incremental_cache(incremental_cache_filename) + logger.log("{Loaded} incremental cache for {filename!r} from {incremental_cache_filename!r}.".format( + Loaded="Loaded" if did_load_cache else "Failed to load", + filename=filename, + incremental_cache_filename=incremental_cache_filename, + )) + pre_procd = None + try: + with logger.gather_parsing_stats(): + try: + pre_procd = self.pre(inputstring, keep_state=keep_state, **preargs) + parsed = parse(parser, pre_procd, inner=False) + out = self.post(parsed, keep_state=keep_state, **postargs) + except ParseBaseException as err: + raise self.make_parse_err(err) + except CoconutDeferredSyntaxError as err: + internal_assert(pre_procd is not None, "invalid deferred syntax error in pre-processing", err) + raise self.make_syntax_err(err, pre_procd) + # RuntimeError, not RecursionError, for Python < 3.5 + except RuntimeError as err: + raise CoconutException( + str(err), extra="try again with --recursion-limit greater than the current " + + str(sys.getrecursionlimit()) + " (you may also need to increase --stack-size)", + ) + finally: + if incremental_cache_filename is not None and pre_procd is not None: + pickle_incremental_cache(pre_procd, incremental_cache_filename) self.run_final_checks(pre_procd, keep_state) return out diff --git a/coconut/compiler/util.py b/coconut/compiler/util.py index 7605cecae..47a705639 100644 --- a/coconut/compiler/util.py +++ b/coconut/compiler/util.py @@ -39,6 +39,11 @@ from contextlib import contextmanager from pprint import pformat, pprint +if sys.version_info >= (3,): + import pickle +else: + import cPickle as pickle + from coconut._pyparsing import ( USE_COMPUTATION_GRAPH, SUPPORTS_INCREMENTAL, @@ -62,6 +67,7 @@ _trim_arity, _ParseResultsWithOffset, line as _line, + all_parse_elements, ) from coconut.integrations import embed @@ -70,6 +76,7 @@ get_name, get_target_info, memoize, + univ_open, ) from coconut.terminal import ( logger, @@ -102,6 +109,7 @@ incremental_cache_size, repeatedly_clear_incremental_cache, py_vers_with_eols, + unwrapper, ) from coconut.exceptions import ( CoconutException, @@ -525,13 +533,6 @@ def get_pyparsing_cache(): return {} -def add_to_cache(new_cache_items): - """Add the given items directly to the pyparsing packrat cache.""" - packrat_cache = ParserElement.packrat_cache - for lookup, value in new_cache_items: - packrat_cache.set(lookup, value) - - def get_cache_items_for(original): """Get items from the pyparsing cache filtered to only from parsing original.""" cache = get_pyparsing_cache() @@ -545,8 +546,8 @@ def get_highest_parse_loc(original): """Get the highest observed parse location.""" # find the highest observed parse location highest_loc = 0 - for item, _ in get_cache_items_for(original): - loc = item[2] + for lookup, _ in get_cache_items_for(original): + loc = lookup[2] if loc > highest_loc: highest_loc = loc return highest_loc @@ -559,8 +560,54 @@ def enable_incremental_parsing(force=False): ParserElement.enableIncremental(incremental_cache_size, still_reset_cache=False) except ImportError as err: raise CoconutException(str(err)) - else: - logger.log("Incremental parsing mode enabled.") + logger.log("Incremental parsing mode enabled.") + return True + else: + return False + + +def pickle_incremental_cache(original, filename, protocol=pickle.HIGHEST_PROTOCOL): + """Pickle the pyparsing cache for original to filename. """ + internal_assert(all_parse_elements is not None, "pickle_incremental_cache requires cPyparsing") + pickleable_cache_items = [] + for lookup, value in get_cache_items_for(original): + pickleable_lookup = (lookup[0].parse_element_index,) + lookup[1:] + pickleable_cache_items.append((pickleable_lookup, value)) + logger.log("Saving {num_items} incremental cache items to {filename!r}.".format( + num_items=len(pickleable_cache_items), + filename=filename, + )) + pickle_info_obj = { + "VERSION": VERSION, + "pickleable_cache_items": pickleable_cache_items, + } + with univ_open(filename, "wb") as pickle_file: + pickle.dump(pickle_info_obj, pickle_file, protocol=protocol) + + +def unpickle_incremental_cache(filename): + """Unpickle and load the given incremental cache file.""" + internal_assert(all_parse_elements is not None, "unpickle_incremental_cache requires cPyparsing") + if not os.path.exists(filename): + return False + try: + with univ_open(filename, "rb") as pickle_file: + pickle_info_obj = pickle.load(pickle_file) + except Exception: + logger.log_exc() + return False + if pickle_info_obj["VERSION"] != VERSION: + return False + pickleable_cache_items = pickle_info_obj["pickleable_cache_items"] + logger.log("Loaded {num_items} incremental cache items from {filename!r}.".format( + num_items=len(pickleable_cache_items), + filename=filename, + )) + packrat_cache = ParserElement.packrat_cache + for pickleable_lookup, value in pickleable_cache_items: + lookup = (all_parse_elements[pickleable_lookup[0]],) + pickleable_lookup[1:] + packrat_cache.set(lookup, value) + return True # ----------------------------------------------------------------------------------------------------------------------- @@ -646,6 +693,7 @@ def get_target_info_smart(target, mode="lowest"): class Wrap(ParseElementEnhance): """PyParsing token that wraps the given item in the given context manager.""" + global_instance_counter = 0 inside = False def __init__(self, item, wrapper, greedy=False, include_in_packrat_context=False): @@ -653,6 +701,8 @@ def __init__(self, item, wrapper, greedy=False, include_in_packrat_context=False self.wrapper = wrapper self.greedy = greedy self.include_in_packrat_context = include_in_packrat_context and hasattr(ParserElement, "packrat_context") + self.identifier = Wrap.global_instance_counter + Wrap.global_instance_counter += 1 @property def wrapped_name(self): @@ -666,12 +716,13 @@ def wrapped_context(self): and unwrapped parses. Only supported natively on cPyparsing.""" was_inside, self.inside = self.inside, True if self.include_in_packrat_context: - ParserElement.packrat_context.append(self.wrapper) + ParserElement.packrat_context.append(self.identifier) try: yield finally: if self.include_in_packrat_context: - ParserElement.packrat_context.pop() + popped = ParserElement.packrat_context.pop() + internal_assert(popped == self.identifier, "invalid popped Wrap identifier", self.identifier) self.inside = was_inside @override @@ -1476,9 +1527,9 @@ def move_loc_to_non_whitespace(original, loc, backwards=False): original, loc, chars_to_move_forwards={ - default_whitespace_chars: not backwards, # for loc, move backwards on newlines/indents, which we can do safely without removing anything from the error indchars: False, + default_whitespace_chars: not backwards, }, ) @@ -1489,8 +1540,10 @@ def move_endpt_to_non_whitespace(original, loc, backwards=False): original, loc, chars_to_move_forwards={ - default_whitespace_chars: not backwards, # for endpt, ignore newlines/indents to avoid introducing unnecessary lines into the error + default_whitespace_chars: not backwards, + # always move forwards on unwrapper to ensure we don't cut wrapped objects in the middle + unwrapper: True, }, ) diff --git a/coconut/constants.py b/coconut/constants.py index c12c461ea..41647ae3b 100644 --- a/coconut/constants.py +++ b/coconut/constants.py @@ -713,6 +713,8 @@ def get_path_env_var(env_var, default): create_package_retries = 1 +max_orig_lines_in_log_loc = 2 + # ----------------------------------------------------------------------------------------------------------------------- # HIGHLIGHTER CONSTANTS: # ----------------------------------------------------------------------------------------------------------------------- @@ -954,7 +956,7 @@ def get_path_env_var(env_var, default): # min versions are inclusive min_versions = { - "cPyparsing": (2, 4, 7, 2, 2, 1), + "cPyparsing": (2, 4, 7, 2, 2, 2), ("pre-commit", "py3"): (3,), ("psutil", "py>=27"): (5,), "jupyter": (1, 0), diff --git a/coconut/root.py b/coconut/root.py index 067ee2734..a1768b59b 100644 --- a/coconut/root.py +++ b/coconut/root.py @@ -26,7 +26,7 @@ VERSION = "3.0.3" VERSION_NAME = None # False for release, int >= 1 for develop -DEVELOP = 2 +DEVELOP = 3 ALPHA = False # for pre releases rather than post releases assert DEVELOP is False or DEVELOP >= 1, "DEVELOP must be False or an int >= 1" diff --git a/coconut/terminal.py b/coconut/terminal.py index 30db0ecf4..309504385 100644 --- a/coconut/terminal.py +++ b/coconut/terminal.py @@ -51,6 +51,7 @@ log_color_code, ansii_escape, force_verbose_logger, + max_orig_lines_in_log_loc, ) from coconut.util import ( get_clock_time, @@ -352,7 +353,16 @@ def log_loc(self, name, original, loc): """Log a location in source code.""" if self.verbose: if isinstance(loc, int): - self.printlog("in error construction:", str(name), "=", repr(original[:loc]), "|", repr(original[loc:])) + pre_loc_orig, post_loc_orig = original[:loc], original[loc:] + if pre_loc_orig.count("\n") > max_orig_lines_in_log_loc: + pre_loc_orig_repr = "... " + repr(pre_loc_orig.rsplit("\n", 1)[-1]) + else: + pre_loc_orig_repr = repr(pre_loc_orig) + if post_loc_orig.count("\n") > max_orig_lines_in_log_loc: + post_loc_orig_repr = repr(post_loc_orig.split("\n", 1)[0]) + " ..." + else: + post_loc_orig_repr = repr(post_loc_orig) + self.printlog("in error construction:", str(name), "=", pre_loc_orig_repr, "|", post_loc_orig_repr) else: self.printlog("in error construction:", str(name), "=", repr(loc)) diff --git a/coconut/tests/main_test.py b/coconut/tests/main_test.py index d30e9b793..4bfa1fe10 100644 --- a/coconut/tests/main_test.py +++ b/coconut/tests/main_test.py @@ -128,6 +128,11 @@ "*** glibc detected ***", "INTERNAL ERROR", ) +ignore_error_lines_with = ( + # ignore SyntaxWarnings containing assert_raises + "assert_raises(", + " raise ", +) mypy_snip = "a: str = count()[0]" mypy_snip_err_2 = '''error: Incompatible types in assignment (expression has type\n"int", variable has type "unicode")''' @@ -331,8 +336,7 @@ def call( for line in lines: for errstr in always_err_strs: assert errstr not in line, "{errstr!r} in {line!r}".format(errstr=errstr, line=line) - # ignore SyntaxWarnings containing assert_raises - if check_errors and "assert_raises(" not in line: + if check_errors and not any(ignore in line for ignore in ignore_error_lines_with): assert "Traceback (most recent call last):" not in line, "Traceback in " + repr(line) assert "Exception" not in line, "Exception in " + repr(line) assert "Error" not in line, "Error in " + repr(line) @@ -915,6 +919,10 @@ def test_strict(self): def test_and(self): run(["--and"]) # src and dest built by comp + def test_incremental(self): + run(["--incremental"]) + run(["--incremental", "--force"]) + if PY35: def test_no_wrap(self): run(["--no-wrap"]) diff --git a/coconut/tests/src/extras.coco b/coconut/tests/src/extras.coco index 854903912..c3d7d96d2 100644 --- a/coconut/tests/src/extras.coco +++ b/coconut/tests/src/extras.coco @@ -344,6 +344,7 @@ else: assert_raises(-> parse("obj."), CoconutStyleError, err_has="getattr") assert_raises(-> parse("def x -> pass, 1"), CoconutStyleError, err_has="statement lambda") assert_raises(-> parse("abc = f'abc'"), CoconutStyleError, err_has="\n ^") + assert_raises(-> parse('f"{f"{f"infinite"}"}"'), CoconutStyleError, err_has=" ^~~~~~~~~~~|") setup(line_numbers=False, strict=True, target="sys") assert_raises(-> parse("await f x"), CoconutParseError, err_has='invalid use of the keyword "await"') diff --git a/coconut/util.py b/coconut/util.py index 69e0e2f3c..62e2fcfa0 100644 --- a/coconut/util.py +++ b/coconut/util.py @@ -254,6 +254,12 @@ def assert_remove_prefix(inputstr, prefix): return inputstr[len(prefix):] +def ensure_dir(dirpath): + """Ensure that a directory exists.""" + if not os.path.exists(dirpath): + os.makedirs(dirpath) + + # ----------------------------------------------------------------------------------------------------------------------- # VERSIONING: # ----------------------------------------------------------------------------------------------------------------------- @@ -327,8 +333,7 @@ def install_custom_kernel(executable=None, logger=None): kernel_dest = fixpath(os.path.join(sys.exec_prefix, icoconut_custom_kernel_install_loc)) try: make_custom_kernel(executable) - if not os.path.exists(kernel_dest): - os.makedirs(kernel_dest) + ensure_dir(kernel_dest) shutil.copy(kernel_source, kernel_dest) except OSError: existing_kernel = os.path.join(kernel_dest, "kernel.json") From 27a15c75908e183c0d703d6ffee541a4a3457f74 Mon Sep 17 00:00:00 2001 From: Evan Hubinger Date: Tue, 1 Aug 2023 02:03:54 -0700 Subject: [PATCH 004/121] Fix --incremental --- Makefile | 26 +++++++++++++-- coconut/_pyparsing.py | 4 +-- coconut/compiler/util.py | 70 ++++++++++++++++++++++++++-------------- coconut/constants.py | 7 +++- coconut/root.py | 2 +- coconut/terminal.py | 18 ++++++----- 6 files changed, 88 insertions(+), 39 deletions(-) diff --git a/Makefile b/Makefile index a664fa94e..1f789a68d 100644 --- a/Makefile +++ b/Makefile @@ -152,6 +152,7 @@ test-mypy-tests: clean-no-tests python ./coconut/tests/dest/extras.py # same as test-univ but includes verbose output for better debugging +# regex for getting non-timing lines: ^(?!Time|\s+Packrat|Loaded|Saving).* .PHONY: test-verbose test-verbose: export COCONUT_USE_COLOR=TRUE test-verbose: clean @@ -183,6 +184,22 @@ test-mypy-all: clean python ./coconut/tests/dest/runner.py python ./coconut/tests/dest/extras.py +# same as test-univ-tests, but forces recompilation for testing --incremental +.PHONY: test-incremental +test-incremental: export COCONUT_USE_COLOR=TRUE +test-incremental: clean-no-tests + python ./coconut/tests --strict --keep-lines --incremental --force + python ./coconut/tests/dest/runner.py + python ./coconut/tests/dest/extras.py + +# same as test-incremental, but uses --verbose +.PHONY: test-incremental-verbose +test-incremental-verbose: export COCONUT_USE_COLOR=TRUE +test-incremental-verbose: clean-no-tests + python ./coconut/tests --strict --keep-lines --incremental --force --verbose + python ./coconut/tests/dest/runner.py + python ./coconut/tests/dest/extras.py + # same as test-univ but also tests easter eggs .PHONY: test-easter-eggs test-easter-eggs: export COCONUT_USE_COLOR=TRUE @@ -267,13 +284,16 @@ clean-no-tests: clean: clean-no-tests rm -rf ./coconut/tests/dest +.PHONY: clean-cache +clean-cache: clean + -find . -name "__coconut_cache__" -type d -prune -exec rm -rf '{}' + + -C:/GnuWin32/bin/find.exe . -name "__coconut_cache__" -type d -prune -exec rm -rf '{}' + + .PHONY: wipe -wipe: clean +wipe: clean-cache rm -rf ./coconut/tests/dest vprof.json profile.log *.egg-info -find . -name "__pycache__" -type d -prune -exec rm -rf '{}' + -C:/GnuWin32/bin/find.exe . -name "__pycache__" -type d -prune -exec rm -rf '{}' + - -find . -name "__coconut_cache__" -type d -prune -exec rm -rf '{}' + - -C:/GnuWin32/bin/find.exe . -name "__coconut_cache__" -type d -prune -exec rm -rf '{}' + -find . -name "*.pyc" -delete -C:/GnuWin32/bin/find.exe . -name "*.pyc" -delete -python -m coconut --site-uninstall diff --git a/coconut/_pyparsing.py b/coconut/_pyparsing.py index de21afa3a..6f290d3cb 100644 --- a/coconut/_pyparsing.py +++ b/coconut/_pyparsing.py @@ -42,7 +42,7 @@ get_bool_env_var, use_computation_graph_env_var, use_incremental_if_available, - incremental_cache_size, + default_incremental_cache_size, never_clear_incremental_cache, warn_on_multiline_regex, ) @@ -202,7 +202,7 @@ def enableIncremental(*args, **kwargs): if MODERN_PYPARSING and use_left_recursion_if_available: ParserElement.enable_left_recursion() elif SUPPORTS_INCREMENTAL and use_incremental_if_available: - ParserElement.enableIncremental(incremental_cache_size, still_reset_cache=not never_clear_incremental_cache) + ParserElement.enableIncremental(default_incremental_cache_size, still_reset_cache=not never_clear_incremental_cache) elif use_packrat_parser: ParserElement.enablePackrat(packrat_cache_size) diff --git a/coconut/compiler/util.py b/coconut/compiler/util.py index 47a705639..221268958 100644 --- a/coconut/compiler/util.py +++ b/coconut/compiler/util.py @@ -98,7 +98,6 @@ specific_targets, pseudo_targets, reserved_vars, - use_packrat_parser, packrat_cache_size, temp_grammar_item_ref_count, indchars, @@ -106,10 +105,12 @@ non_syntactic_newline, allow_explicit_keyword_vars, reserved_prefix, - incremental_cache_size, + incremental_mode_cache_size, + default_incremental_cache_size, repeatedly_clear_incremental_cache, py_vers_with_eols, unwrapper, + incremental_cache_limit, ) from coconut.exceptions import ( CoconutException, @@ -357,16 +358,16 @@ def attach(item, action, ignore_no_tokens=None, ignore_one_token=None, ignore_to def should_clear_cache(): """Determine if we should be clearing the packrat cache.""" - return ( - use_packrat_parser - and ( - not ParserElement._incrementalEnabled - or ( - ParserElement._incrementalWithResets - and repeatedly_clear_incremental_cache - ) - ) - ) + if not ParserElement._packratEnabled: + internal_assert(not ParserElement._incrementalEnabled) + return False + if not ParserElement._incrementalEnabled: + return True + if ParserElement._incrementalWithResets and repeatedly_clear_incremental_cache: + return True + if incremental_cache_limit is not None and len(ParserElement.packrat_cache) > incremental_cache_limit: + return True + return False def final_evaluate_tokens(tokens): @@ -403,7 +404,10 @@ def force_reset_packrat_cache(): """Forcibly reset the packrat cache and all packrat stats.""" if ParserElement._incrementalEnabled: ParserElement._incrementalEnabled = False - ParserElement.enableIncremental(incremental_cache_size, still_reset_cache=ParserElement._incrementalWithResets) + if ParserElement._incrementalWithResets: + ParserElement.enableIncremental(default_incremental_cache_size, still_reset_cache=True) + else: + ParserElement.enableIncremental(incremental_mode_cache_size, still_reset_cache=False) else: ParserElement._packratEnabled = False ParserElement.enablePackrat(packrat_cache_size) @@ -553,26 +557,35 @@ def get_highest_parse_loc(original): return highest_loc -def enable_incremental_parsing(force=False): - """Enable incremental parsing mode where prefix parses are reused.""" - if SUPPORTS_INCREMENTAL or force: - try: - ParserElement.enableIncremental(incremental_cache_size, still_reset_cache=False) - except ImportError as err: - raise CoconutException(str(err)) - logger.log("Incremental parsing mode enabled.") - return True - else: +def enable_incremental_parsing(): + """Enable incremental parsing mode where prefix/suffix parses are reused.""" + if not SUPPORTS_INCREMENTAL: return False + if ParserElement._incrementalEnabled and not ParserElement._incrementalWithResets: # incremental mode is already enabled + return True + ParserElement._incrementalEnabled = False + try: + ParserElement.enableIncremental(incremental_mode_cache_size, still_reset_cache=False) + except ImportError as err: + raise CoconutException(str(err)) + logger.log("Incremental parsing mode enabled.") + return True def pickle_incremental_cache(original, filename, protocol=pickle.HIGHEST_PROTOCOL): - """Pickle the pyparsing cache for original to filename. """ + """Pickle the pyparsing cache for original to filename.""" internal_assert(all_parse_elements is not None, "pickle_incremental_cache requires cPyparsing") + pickleable_cache_items = [] for lookup, value in get_cache_items_for(original): + if incremental_mode_cache_size is not None and len(pickleable_cache_items) > incremental_mode_cache_size: + complain("got too large incremental cache: " + str(len(get_pyparsing_cache())) + " > " + str(incremental_mode_cache_size)) + break + if len(pickleable_cache_items) >= incremental_cache_limit: + break pickleable_lookup = (lookup[0].parse_element_index,) + lookup[1:] pickleable_cache_items.append((pickleable_lookup, value)) + logger.log("Saving {num_items} incremental cache items to {filename!r}.".format( num_items=len(pickleable_cache_items), filename=filename, @@ -588,6 +601,7 @@ def pickle_incremental_cache(original, filename, protocol=pickle.HIGHEST_PROTOCO def unpickle_incremental_cache(filename): """Unpickle and load the given incremental cache file.""" internal_assert(all_parse_elements is not None, "unpickle_incremental_cache requires cPyparsing") + if not os.path.exists(filename): return False try: @@ -603,6 +617,14 @@ def unpickle_incremental_cache(filename): num_items=len(pickleable_cache_items), filename=filename, )) + + max_cache_size = min( + incremental_mode_cache_size or float("inf"), + incremental_cache_limit or float("inf"), + ) + if max_cache_size != float("inf"): + pickleable_cache_items = pickleable_cache_items[-max_cache_size:] + packrat_cache = ParserElement.packrat_cache for pickleable_lookup, value in pickleable_cache_items: lookup = (all_parse_elements[pickleable_lookup[0]],) + pickleable_lookup[1:] diff --git a/coconut/constants.py b/coconut/constants.py index 41647ae3b..aee27380b 100644 --- a/coconut/constants.py +++ b/coconut/constants.py @@ -127,11 +127,16 @@ def get_path_env_var(env_var, default): # note that _parseIncremental produces much smaller caches use_incremental_if_available = True -incremental_cache_size = None + # these only apply to use_incremental_if_available, not compiler.util.enable_incremental_parsing() +default_incremental_cache_size = None repeatedly_clear_incremental_cache = True never_clear_incremental_cache = False +# this is what gets used in compiler.util.enable_incremental_parsing() +incremental_mode_cache_size = None +incremental_cache_limit = 524288 # clear cache when it gets this large + use_left_recursion_if_available = False # ----------------------------------------------------------------------------------------------------------------------- diff --git a/coconut/root.py b/coconut/root.py index a1768b59b..714a7124a 100644 --- a/coconut/root.py +++ b/coconut/root.py @@ -26,7 +26,7 @@ VERSION = "3.0.3" VERSION_NAME = None # False for release, int >= 1 for develop -DEVELOP = 3 +DEVELOP = 4 ALPHA = False # for pre releases rather than post releases assert DEVELOP is False or DEVELOP >= 1, "DEVELOP must be False or an int >= 1" diff --git a/coconut/terminal.py b/coconut/terminal.py index 309504385..8a5f7cde0 100644 --- a/coconut/terminal.py +++ b/coconut/terminal.py @@ -97,22 +97,24 @@ def format_error(err_value, err_type=None, err_trace=None): return "".join(traceback.format_exception(err_type, err_value, err_trace)).strip() -def complain(error): +def complain(error_or_msg, *args, **kwargs): """Raises in develop; warns in release.""" - if callable(error): + if callable(error_or_msg): if DEVELOP: - error = error() + error_or_msg = error_or_msg() else: return - if not isinstance(error, BaseException) or (not isinstance(error, CoconutInternalException) and isinstance(error, CoconutException)): - error = CoconutInternalException(str(error)) + if not isinstance(error_or_msg, BaseException) or (not isinstance(error_or_msg, CoconutInternalException) and isinstance(error_or_msg, CoconutException)): + error_or_msg = CoconutInternalException(str(error_or_msg), *args, **kwargs) + else: + internal_assert(not args and not kwargs, "if error_or_msg is an error, args and kwargs must be empty, not", (args, kwargs)) if not DEVELOP: - logger.warn_err(error) + logger.warn_err(error_or_msg) elif embed_on_internal_exc: - logger.warn_err(error) + logger.warn_err(error_or_msg) embed(depth=1) else: - raise error + raise error_or_msg def internal_assert(condition, message=None, item=None, extra=None, exc_maker=None): From f7e4fbcf5fe30d6594cd9260df5fc75e07614023 Mon Sep 17 00:00:00 2001 From: Evan Hubinger Date: Tue, 1 Aug 2023 14:52:36 -0700 Subject: [PATCH 005/121] Fix incremental test --- coconut/tests/main_test.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/coconut/tests/main_test.py b/coconut/tests/main_test.py index 4bfa1fe10..7429d46e2 100644 --- a/coconut/tests/main_test.py +++ b/coconut/tests/main_test.py @@ -921,7 +921,8 @@ def test_and(self): def test_incremental(self): run(["--incremental"]) - run(["--incremental", "--force"]) + # includes "Error" because exceptions include the whole file + run(["--incremental", "--force"], check_errors=False) if PY35: def test_no_wrap(self): From 1c4a14f794f0389504e3b516a91447782306f446 Mon Sep 17 00:00:00 2001 From: Evan Hubinger Date: Wed, 2 Aug 2023 02:09:30 -0700 Subject: [PATCH 006/121] Slightly improve incremental mode --- coconut/compiler/util.py | 14 +++++++++++--- 1 file changed, 11 insertions(+), 3 deletions(-) diff --git a/coconut/compiler/util.py b/coconut/compiler/util.py index 221268958..83df71384 100644 --- a/coconut/compiler/util.py +++ b/coconut/compiler/util.py @@ -579,12 +579,19 @@ def pickle_incremental_cache(original, filename, protocol=pickle.HIGHEST_PROTOCO pickleable_cache_items = [] for lookup, value in get_cache_items_for(original): if incremental_mode_cache_size is not None and len(pickleable_cache_items) > incremental_mode_cache_size: - complain("got too large incremental cache: " + str(len(get_pyparsing_cache())) + " > " + str(incremental_mode_cache_size)) + complain( + "got too large incremental cache: " + + str(len(get_pyparsing_cache())) + " > " + str(incremental_mode_cache_size) + ) break if len(pickleable_cache_items) >= incremental_cache_limit: break - pickleable_lookup = (lookup[0].parse_element_index,) + lookup[1:] - pickleable_cache_items.append((pickleable_lookup, value)) + loc = lookup[2] + # only include cache items that aren't at the start or end, since those + # are the only ones that parseIncremental will reuse + if 0 < loc < len(original) - 1: + pickleable_lookup = (lookup[0].parse_element_index,) + lookup[1:] + pickleable_cache_items.append((pickleable_lookup, value)) logger.log("Saving {num_items} incremental cache items to {filename!r}.".format( num_items=len(pickleable_cache_items), @@ -612,6 +619,7 @@ def unpickle_incremental_cache(filename): return False if pickle_info_obj["VERSION"] != VERSION: return False + pickleable_cache_items = pickle_info_obj["pickleable_cache_items"] logger.log("Loaded {num_items} incremental cache items from {filename!r}.".format( num_items=len(pickleable_cache_items), From aa884076fe087c2cf782254b4caaa78cee58e76a Mon Sep 17 00:00:00 2001 From: Evan Hubinger Date: Wed, 2 Aug 2023 22:47:16 -0700 Subject: [PATCH 007/121] Further fix --incremental --- Makefile | 2 +- coconut/command/command.py | 21 +++++++----- coconut/compiler/compiler.py | 14 ++++---- coconut/compiler/util.py | 63 +++++++++++++++++++++++------------ coconut/constants.py | 8 +++-- coconut/integrations.py | 2 +- coconut/root.py | 2 +- coconut/tests/main_test.py | 2 ++ coconut/tests/src/extras.coco | 5 ++- 9 files changed, 76 insertions(+), 43 deletions(-) diff --git a/Makefile b/Makefile index 1f789a68d..bf2008bc1 100644 --- a/Makefile +++ b/Makefile @@ -152,7 +152,7 @@ test-mypy-tests: clean-no-tests python ./coconut/tests/dest/extras.py # same as test-univ but includes verbose output for better debugging -# regex for getting non-timing lines: ^(?!Time|\s+Packrat|Loaded|Saving).* +# regex for getting non-timing lines: ^(?!Time|\s+Packrat|Loaded|Saving)[^\n]* .PHONY: test-verbose test-verbose: export COCONUT_USE_COLOR=TRUE test-verbose: clean diff --git a/coconut/command/command.py b/coconut/command/command.py index 010a1ddb8..480a31ef5 100644 --- a/coconut/command/command.py +++ b/coconut/command/command.py @@ -73,6 +73,7 @@ coconut_cache_dir, coconut_run_kwargs, interpreter_uses_incremental, + disable_incremental_for_len, ) from coconut.util import ( univ_open, @@ -603,13 +604,16 @@ def callback(compiled): filename=os.path.basename(codepath), ) if self.incremental: - code_dir, code_fname = os.path.split(codepath) + if disable_incremental_for_len is not None and len(code) > disable_incremental_for_len: + logger.warn("--incremental mode is not currently supported for files as large as {codepath!r}") + else: + code_dir, code_fname = os.path.split(codepath) - cache_dir = os.path.join(code_dir, coconut_cache_dir) - ensure_dir(cache_dir) + cache_dir = os.path.join(code_dir, coconut_cache_dir) + ensure_dir(cache_dir) - pickle_fname = code_fname + ".pickle" - parse_kwargs["incremental_cache_filename"] = os.path.join(cache_dir, pickle_fname) + pickle_fname = code_fname + ".pickle" + parse_kwargs["incremental_cache_filename"] = os.path.join(cache_dir, pickle_fname) if package is True: self.submit_comp_job(codepath, callback, "parse_package", code, package_level=package_level, **parse_kwargs) @@ -822,9 +826,10 @@ def execute(self, compiled=None, path=None, use_eval=False, allow_show=True): if path is None: # header is not included if not self.mypy: no_str_code = self.comp.remove_strs(compiled) - result = mypy_builtin_regex.search(no_str_code) - if result: - logger.warn("found mypy-only built-in " + repr(result.group(0)) + "; pass --mypy to use mypy-only built-ins at the interpreter") + if no_str_code is not None: + result = mypy_builtin_regex.search(no_str_code) + if result: + logger.warn("found mypy-only built-in " + repr(result.group(0)) + "; pass --mypy to use mypy-only built-ins at the interpreter") else: # header is included compiled = rem_encoding(compiled) diff --git a/coconut/compiler/compiler.py b/coconut/compiler/compiler.py index 46e7c532b..4889b9f97 100644 --- a/coconut/compiler/compiler.py +++ b/coconut/compiler/compiler.py @@ -934,12 +934,14 @@ def complain_on_err(self): except CoconutException as err: complain(err) - def remove_strs(self, inputstring, inner_environment=True): - """Remove strings/comments from the given input.""" - with self.complain_on_err(): + def remove_strs(self, inputstring, inner_environment=True, **kwargs): + """Remove strings/comments from the given input if possible.""" + try: with (self.inner_environment() if inner_environment else noop_ctx()): - return self.str_proc(inputstring) - return inputstring + return self.str_proc(inputstring, **kwargs) + except Exception: + logger.log_exc() + return None def get_matcher(self, original, loc, check_var, name_list=None): """Get a Matcher object.""" @@ -1213,7 +1215,7 @@ def parsing(self, keep_state=False, filename=None): def streamline(self, grammar, inputstring="", force=False): """Streamline the given grammar for the given inputstring.""" - if force or (streamline_grammar_for_len is not None and len(inputstring) >= streamline_grammar_for_len): + if force or (streamline_grammar_for_len is not None and len(inputstring) > streamline_grammar_for_len): start_time = get_clock_time() prep_grammar(grammar, streamline=True) logger.log_lambda( diff --git a/coconut/compiler/util.py b/coconut/compiler/util.py index 83df71384..8ff8a3279 100644 --- a/coconut/compiler/util.py +++ b/coconut/compiler/util.py @@ -66,8 +66,9 @@ ParserElement, _trim_arity, _ParseResultsWithOffset, - line as _line, all_parse_elements, + line as _line, + __version__ as pyparsing_version, ) from coconut.integrations import embed @@ -111,6 +112,7 @@ py_vers_with_eols, unwrapper, incremental_cache_limit, + incremental_mode_cache_successes, ) from coconut.exceptions import ( CoconutException, @@ -356,26 +358,9 @@ def attach(item, action, ignore_no_tokens=None, ignore_one_token=None, ignore_to return add_action(item, action, make_copy) -def should_clear_cache(): - """Determine if we should be clearing the packrat cache.""" - if not ParserElement._packratEnabled: - internal_assert(not ParserElement._incrementalEnabled) - return False - if not ParserElement._incrementalEnabled: - return True - if ParserElement._incrementalWithResets and repeatedly_clear_incremental_cache: - return True - if incremental_cache_limit is not None and len(ParserElement.packrat_cache) > incremental_cache_limit: - return True - return False - - def final_evaluate_tokens(tokens): """Same as evaluate_tokens but should only be used once a parse is assured.""" - # don't clear the cache in incremental mode - if should_clear_cache(): - # clear cache without resetting stats - ParserElement.packrat_cache.clear() + clear_packrat_cache() return evaluate_tokens(tokens) @@ -537,6 +522,39 @@ def get_pyparsing_cache(): return {} +def should_clear_cache(): + """Determine if we should be clearing the packrat cache.""" + if not ParserElement._packratEnabled: + return False + if SUPPORTS_INCREMENTAL: + if not ParserElement._incrementalEnabled: + return True + if ParserElement._incrementalWithResets and repeatedly_clear_incremental_cache: + return True + if incremental_cache_limit is not None and len(ParserElement.packrat_cache) > incremental_cache_limit: + # only clear the second half of the cache, since the first + # half is what will help us next time we recompile + return "second half" + return False + + +def clear_packrat_cache(): + """Clear the packrat cache if applicable.""" + clear_cache = should_clear_cache() + if not clear_cache: + return + if clear_cache == "second half": + cache_items = list(get_pyparsing_cache().items()) + restore_items = cache_items[:len(cache_items) // 2] + else: + restore_items = () + # clear cache without resetting stats + ParserElement.packrat_cache.clear() + # restore any items we want to keep + for lookup, value in restore_items: + ParserElement.packrat_cache.set(lookup, value) + + def get_cache_items_for(original): """Get items from the pyparsing cache filtered to only from parsing original.""" cache = get_pyparsing_cache() @@ -561,6 +579,7 @@ def enable_incremental_parsing(): """Enable incremental parsing mode where prefix/suffix parses are reused.""" if not SUPPORTS_INCREMENTAL: return False + ParserElement._should_cache_incremental_success = incremental_mode_cache_successes if ParserElement._incrementalEnabled and not ParserElement._incrementalWithResets: # incremental mode is already enabled return True ParserElement._incrementalEnabled = False @@ -599,6 +618,7 @@ def pickle_incremental_cache(original, filename, protocol=pickle.HIGHEST_PROTOCO )) pickle_info_obj = { "VERSION": VERSION, + "pyparsing_version": pyparsing_version, "pickleable_cache_items": pickleable_cache_items, } with univ_open(filename, "wb") as pickle_file: @@ -617,7 +637,7 @@ def unpickle_incremental_cache(filename): except Exception: logger.log_exc() return False - if pickle_info_obj["VERSION"] != VERSION: + if pickle_info_obj["VERSION"] != VERSION or pickle_info_obj["pyparsing_version"] != pyparsing_version: return False pickleable_cache_items = pickle_info_obj["pickleable_cache_items"] @@ -633,10 +653,9 @@ def unpickle_incremental_cache(filename): if max_cache_size != float("inf"): pickleable_cache_items = pickleable_cache_items[-max_cache_size:] - packrat_cache = ParserElement.packrat_cache for pickleable_lookup, value in pickleable_cache_items: lookup = (all_parse_elements[pickleable_lookup[0]],) + pickleable_lookup[1:] - packrat_cache.set(lookup, value) + ParserElement.packrat_cache.set(lookup, value) return True diff --git a/coconut/constants.py b/coconut/constants.py index aee27380b..84d7a3d3b 100644 --- a/coconut/constants.py +++ b/coconut/constants.py @@ -120,7 +120,8 @@ def get_path_env_var(env_var, default): # below constants are experimentally determined to maximize performance -streamline_grammar_for_len = 4000 +streamline_grammar_for_len = 4096 +disable_incremental_for_len = streamline_grammar_for_len # disables --incremental use_packrat_parser = True # True also gives us better error messages packrat_cache_size = None # only works because final() clears the cache @@ -135,7 +136,8 @@ def get_path_env_var(env_var, default): # this is what gets used in compiler.util.enable_incremental_parsing() incremental_mode_cache_size = None -incremental_cache_limit = 524288 # clear cache when it gets this large +incremental_cache_limit = 1048576 # clear cache when it gets this large +incremental_mode_cache_successes = False use_left_recursion_if_available = False @@ -961,7 +963,7 @@ def get_path_env_var(env_var, default): # min versions are inclusive min_versions = { - "cPyparsing": (2, 4, 7, 2, 2, 2), + "cPyparsing": (2, 4, 7, 2, 2, 3), ("pre-commit", "py3"): (3,), ("psutil", "py>=27"): (5,), "jupyter": (1, 0), diff --git a/coconut/integrations.py b/coconut/integrations.py index 8d2fec811..f2a3537ee 100644 --- a/coconut/integrations.py +++ b/coconut/integrations.py @@ -173,7 +173,7 @@ def new_ctxvisit(self, ctxtransformer, node, inp, ctx, mode="exec", *args, **kwa # we handle our own inner_environment rather than have remove_strs do it so that we can reformat with self.compiler.inner_environment(): line_no_strs = self.compiler.remove_strs(line, inner_environment=False) - if ";" in line_no_strs: + if line_no_strs is not None and ";" in line_no_strs: remaining_pieces = [ self.compiler.reformat(piece, ignore_errors=True) for piece in line_no_strs.split(";") diff --git a/coconut/root.py b/coconut/root.py index 714a7124a..d184e8e2c 100644 --- a/coconut/root.py +++ b/coconut/root.py @@ -26,7 +26,7 @@ VERSION = "3.0.3" VERSION_NAME = None # False for release, int >= 1 for develop -DEVELOP = 4 +DEVELOP = 5 ALPHA = False # for pre releases rather than post releases assert DEVELOP is False or DEVELOP >= 1, "DEVELOP must be False or an int >= 1" diff --git a/coconut/tests/main_test.py b/coconut/tests/main_test.py index 7429d46e2..50cf27d86 100644 --- a/coconut/tests/main_test.py +++ b/coconut/tests/main_test.py @@ -812,6 +812,8 @@ def test_xontrib(self): p.sendline('echo f"{$ENV_VAR}"; echo f"{$ENV_VAR}"') p.expect("ABC") p.expect("ABC") + p.sendline('len("""1\n3\n5""")') + p.expect("5") if not PYPY or PY39: if PY36: p.sendline("echo 123;; 123") diff --git a/coconut/tests/src/extras.coco b/coconut/tests/src/extras.coco index c3d7d96d2..79de0f5f7 100644 --- a/coconut/tests/src/extras.coco +++ b/coconut/tests/src/extras.coco @@ -325,7 +325,10 @@ line 6''') assert_raises(-> parse("a=1;"), CoconutStyleError, err_has="\n ^") assert_raises(-> parse("class derp(object)"), CoconutStyleError) assert_raises(-> parse("def f(a.b) = True"), CoconutStyleError, err_has="\n ^") - assert_raises(-> parse("match def kwd_only_x_is_int_def_0(*, x is int = 0) = x"), CoconutStyleError, err_has="\n ^~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~|") + assert_raises(-> parse("match def kwd_only_x_is_int_def_0(*, x is int = 0) = x"), CoconutStyleError, err_has=( + "\n ^~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~|", + "\n ^~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~/", + )) try: parse(""" try: From 779f9dc7b3062265bbd97edbb8de1a5966a078a1 Mon Sep 17 00:00:00 2001 From: Evan Hubinger Date: Thu, 3 Aug 2023 01:27:59 -0700 Subject: [PATCH 008/121] Attempt to fix tests --- coconut/command/command.py | 3 +++ coconut/compiler/compiler.py | 2 +- coconut/tests/main_test.py | 34 ++++++++++++++++------------------ 3 files changed, 20 insertions(+), 19 deletions(-) diff --git a/coconut/command/command.py b/coconut/command/command.py index 480a31ef5..214832bac 100644 --- a/coconut/command/command.py +++ b/coconut/command/command.py @@ -31,6 +31,7 @@ unset_fast_pyparsing_reprs, collect_timing_info, print_timing_info, + SUPPORTS_INCREMENTAL, ) from coconut.compiler import Compiler @@ -264,6 +265,8 @@ def execute_args(self, args, interact=True, original_args=None): raise CoconutException("cannot compile with both --line-numbers and --no-line-numbers") if args.site_install and args.site_uninstall: raise CoconutException("cannot --site-install and --site-uninstall simultaneously") + if args.incremental and not SUPPORTS_INCREMENTAL: + raise CoconutException("--incremental mode requires cPyparsing (run '{python} -m pip install --upgrade cPyparsing' to fix)".format(python=sys.executable)) for and_args in getattr(args, "and") or []: if len(and_args) > 2: raise CoconutException( diff --git a/coconut/compiler/compiler.py b/coconut/compiler/compiler.py index 4889b9f97..0222b066a 100644 --- a/coconut/compiler/compiler.py +++ b/coconut/compiler/compiler.py @@ -1263,7 +1263,7 @@ def parse( if incremental_cache_filename is not None: incremental_enabled = enable_incremental_parsing() if not incremental_enabled: - raise CoconutException("--incremental mode requires cPyparsing (run '{python} -m pip install --upgrade cPyparsing' to fix)".format(python=sys.executable)) + raise CoconutException("incremental_cache_filename requires cPyparsing (run '{python} -m pip install --upgrade cPyparsing' to fix)".format(python=sys.executable)) did_load_cache = unpickle_incremental_cache(incremental_cache_filename) logger.log("{Loaded} incremental cache for {filename!r} from {incremental_cache_filename!r}.".format( Loaded="Loaded" if did_load_cache else "Failed to load", diff --git a/coconut/tests/main_test.py b/coconut/tests/main_test.py index 50cf27d86..b47195e7b 100644 --- a/coconut/tests/main_test.py +++ b/coconut/tests/main_test.py @@ -812,7 +812,7 @@ def test_xontrib(self): p.sendline('echo f"{$ENV_VAR}"; echo f"{$ENV_VAR}"') p.expect("ABC") p.expect("ABC") - p.sendline('len("""1\n3\n5""")') + p.sendline('len("""1\n3\n5""") |> print') p.expect("5") if not PYPY or PY39: if PY36: @@ -910,6 +910,10 @@ def test_package(self): def test_no_tco(self): run(["--no-tco"]) + if PY35: + def test_no_wrap(self): + run(["--no-wrap"]) + # run fewer tests on Windows so appveyor doesn't time out if not WINDOWS: def test_keep_lines(self): @@ -921,14 +925,18 @@ def test_strict(self): def test_and(self): run(["--and"]) # src and dest built by comp - def test_incremental(self): - run(["--incremental"]) - # includes "Error" because exceptions include the whole file - run(["--incremental", "--force"], check_errors=False) + def test_run_arg(self): + run(use_run_arg=True) - if PY35: - def test_no_wrap(self): - run(["--no-wrap"]) + if not PYPY and not PY26: + def test_jobs_zero(self): + run(["--jobs", "0"]) + + if not PYPY: + def test_incremental(self): + run(["--incremental"]) + # includes "Error" because exceptions include the whole file + run(["--incremental", "--force"], check_errors=False) if get_bool_env_var("COCONUT_TEST_VERBOSE"): def test_verbose(self): @@ -938,16 +946,6 @@ def test_verbose(self): def test_trace(self): run(["--jobs", "0", "--trace"], check_errors=False) - # avoids a strange, unreproducable failure on appveyor - if not (WINDOWS and sys.version_info[:2] == (3, 8)): - def test_run_arg(self): - run(use_run_arg=True) - - # not WINDOWS is for appveyor timeout prevention - if not WINDOWS and not PYPY and not PY26: - def test_jobs_zero(self): - run(["--jobs", "0"]) - # more appveyor timeout prevention if not WINDOWS: From 22bcb28a3ca4239e36ecd676fb773c5f8ccdefaf Mon Sep 17 00:00:00 2001 From: Evan Hubinger Date: Thu, 3 Aug 2023 16:45:58 -0700 Subject: [PATCH 009/121] Fix tests, incremental log message --- coconut/command/command.py | 2 +- coconut/tests/main_test.py | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/coconut/command/command.py b/coconut/command/command.py index 214832bac..dcdae0b12 100644 --- a/coconut/command/command.py +++ b/coconut/command/command.py @@ -608,7 +608,7 @@ def callback(compiled): ) if self.incremental: if disable_incremental_for_len is not None and len(code) > disable_incremental_for_len: - logger.warn("--incremental mode is not currently supported for files as large as {codepath!r}") + logger.warn("--incremental mode is not currently supported for files as large as {codepath!r}".format(codepath=codepath)) else: code_dir, code_fname = os.path.split(codepath) diff --git a/coconut/tests/main_test.py b/coconut/tests/main_test.py index b47195e7b..22a5f1733 100644 --- a/coconut/tests/main_test.py +++ b/coconut/tests/main_test.py @@ -812,7 +812,7 @@ def test_xontrib(self): p.sendline('echo f"{$ENV_VAR}"; echo f"{$ENV_VAR}"') p.expect("ABC") p.expect("ABC") - p.sendline('len("""1\n3\n5""") |> print') + p.sendline('len("""1\n3\n5""")\n') p.expect("5") if not PYPY or PY39: if PY36: From 7d95dc19f9e04a53aff2e2ff83691f80ef63b96c Mon Sep 17 00:00:00 2001 From: Evan Hubinger Date: Sun, 6 Aug 2023 12:20:23 -0700 Subject: [PATCH 010/121] Fix unused import errors --- coconut/compiler/compiler.py | 1 + coconut/root.py | 2 +- 2 files changed, 2 insertions(+), 1 deletion(-) diff --git a/coconut/compiler/compiler.py b/coconut/compiler/compiler.py index 0222b066a..c1235e27b 100644 --- a/coconut/compiler/compiler.py +++ b/coconut/compiler/compiler.py @@ -1241,6 +1241,7 @@ def run_final_checks(self, original, keep_state=False): "found unused import " + repr(self.reformat(name, ignore_errors=True)) + " (add '# NOQA' to suppress)", original, loc, + endpoint=False, ) def parse( diff --git a/coconut/root.py b/coconut/root.py index d184e8e2c..c8bb08392 100644 --- a/coconut/root.py +++ b/coconut/root.py @@ -26,7 +26,7 @@ VERSION = "3.0.3" VERSION_NAME = None # False for release, int >= 1 for develop -DEVELOP = 5 +DEVELOP = 6 ALPHA = False # for pre releases rather than post releases assert DEVELOP is False or DEVELOP >= 1, "DEVELOP must be False or an int >= 1" From 6473ac8b3ed75faa5523a6abb7fff0813db54add Mon Sep 17 00:00:00 2001 From: Evan Hubinger Date: Sun, 6 Aug 2023 12:32:03 -0700 Subject: [PATCH 011/121] Add missing import test --- coconut/tests/src/extras.coco | 10 ++++++++++ 1 file changed, 10 insertions(+) diff --git a/coconut/tests/src/extras.coco b/coconut/tests/src/extras.coco index 79de0f5f7..b3ba40208 100644 --- a/coconut/tests/src/extras.coco +++ b/coconut/tests/src/extras.coco @@ -348,6 +348,16 @@ else: assert_raises(-> parse("def x -> pass, 1"), CoconutStyleError, err_has="statement lambda") assert_raises(-> parse("abc = f'abc'"), CoconutStyleError, err_has="\n ^") assert_raises(-> parse('f"{f"{f"infinite"}"}"'), CoconutStyleError, err_has=" ^~~~~~~~~~~|") + try: + parse(""" +import abc +1 +2 +3 + """.strip()) + except CoconutStyleError as err: + assert str(err) == """found unused import 'abc' (add '# NOQA' to suppress) (remove --strict to downgrade to a warning) (line 1) + import abc""" setup(line_numbers=False, strict=True, target="sys") assert_raises(-> parse("await f x"), CoconutParseError, err_has='invalid use of the keyword "await"') From e402c580844edd2dc12cd5ac2fd6f2516423fe1b Mon Sep 17 00:00:00 2001 From: Evan Hubinger Date: Sat, 2 Sep 2023 01:48:15 -0700 Subject: [PATCH 012/121] Make where use temp vars Resolves #784. --- DOCS.md | 12 +- coconut/compiler/compiler.py | 160 ++++++++++++++---- coconut/compiler/grammar.py | 29 ++-- coconut/compiler/util.py | 10 +- coconut/root.py | 2 +- .../tests/src/cocotest/agnostic/primary.coco | 9 + coconut/tests/src/cocotest/agnostic/util.coco | 6 +- 7 files changed, 168 insertions(+), 60 deletions(-) diff --git a/DOCS.md b/DOCS.md index 14d9d23cc..bcf7acb1a 100644 --- a/DOCS.md +++ b/DOCS.md @@ -1543,27 +1543,27 @@ _Can't be done without a series of method definitions for each data type. See th ### `where` -Coconut's `where` statement is extremely straightforward. The syntax for a `where` statement is just +Coconut's `where` statement is fairly straightforward. The syntax for a `where` statement is just ``` where: ``` -which just executes `` followed by ``. +which executes `` followed by ``, with the exception that any new variables defined in `` are available _only_ in `` (though they are only mangled, not deleted, such that e.g. lambdas can still capture them). ##### Example **Coconut:** ```coconut -c = a + b where: +result = a + b where: a = 1 b = 2 ``` **Python:** ```coconut_python -a = 1 -b = 2 -c = a + b +_a = 1 +_b = 2 +result = _a + _b ``` ### `async with for` diff --git a/coconut/compiler/compiler.py b/coconut/compiler/compiler.py index c1235e27b..02afd2a28 100644 --- a/coconut/compiler/compiler.py +++ b/coconut/compiler/compiler.py @@ -150,7 +150,6 @@ append_it, interleaved_join, handle_indentation, - Wrap, tuple_str_of, join_args, parse_where, @@ -173,6 +172,7 @@ move_endpt_to_non_whitespace, unpickle_incremental_cache, pickle_incremental_cache, + handle_and_manage, ) from coconut.compiler.header import ( minify_header, @@ -596,6 +596,8 @@ def reset(self, keep_state=False, filename=None): @contextmanager def inner_environment(self, ln=None): """Set up compiler to evaluate inner expressions.""" + if ln is None: + ln = self.outer_ln outer_ln, self.outer_ln = self.outer_ln, ln line_numbers, self.line_numbers = self.line_numbers, False keep_lines, self.keep_lines = self.keep_lines, False @@ -628,6 +630,15 @@ def current_parsing_context(self, name, default=None): else: return default + @contextmanager + def add_to_parsing_context(self, name, obj): + """Add the given object to the parsing context for the given name.""" + self.parsing_context[name].append(obj) + try: + yield + finally: + self.parsing_context[name].pop() + @contextmanager def disable_checks(self): """Run the block without checking names or strict errors.""" @@ -694,38 +705,63 @@ def method(original, loc, tokens): def bind(cls): """Binds reference objects to the proper parse actions.""" # handle parsing_context for class definitions - new_classdef = attach(cls.classdef_ref, cls.method("classdef_handle")) - cls.classdef <<= Wrap(new_classdef, cls.method("class_manage"), greedy=True) - - new_datadef = attach(cls.datadef_ref, cls.method("datadef_handle")) - cls.datadef <<= Wrap(new_datadef, cls.method("class_manage"), greedy=True) - - new_match_datadef = attach(cls.match_datadef_ref, cls.method("match_datadef_handle")) - cls.match_datadef <<= Wrap(new_match_datadef, cls.method("class_manage"), greedy=True) + cls.classdef <<= handle_and_manage( + cls.classdef_ref, + cls.method("classdef_handle"), + cls.method("class_manage"), + ) + cls.datadef <<= handle_and_manage( + cls.datadef_ref, + cls.method("datadef_handle"), + cls.method("class_manage"), + ) + cls.match_datadef <<= handle_and_manage( + cls.match_datadef_ref, + cls.method("match_datadef_handle"), + cls.method("class_manage"), + ) # handle parsing_context for function definitions - new_stmt_lambdef = attach(cls.stmt_lambdef_ref, cls.method("stmt_lambdef_handle")) - cls.stmt_lambdef <<= Wrap(new_stmt_lambdef, cls.method("func_manage"), greedy=True) - - new_decoratable_normal_funcdef_stmt = attach( + cls.stmt_lambdef <<= handle_and_manage( + cls.stmt_lambdef_ref, + cls.method("stmt_lambdef_handle"), + cls.method("func_manage"), + ) + cls.decoratable_normal_funcdef_stmt <<= handle_and_manage( cls.decoratable_normal_funcdef_stmt_ref, cls.method("decoratable_funcdef_stmt_handle"), + cls.method("func_manage"), ) - cls.decoratable_normal_funcdef_stmt <<= Wrap(new_decoratable_normal_funcdef_stmt, cls.method("func_manage"), greedy=True) - - new_decoratable_async_funcdef_stmt = attach( + cls.decoratable_async_funcdef_stmt <<= handle_and_manage( cls.decoratable_async_funcdef_stmt_ref, cls.method("decoratable_funcdef_stmt_handle", is_async=True), + cls.method("func_manage"), ) - cls.decoratable_async_funcdef_stmt <<= Wrap(new_decoratable_async_funcdef_stmt, cls.method("func_manage"), greedy=True) # handle parsing_context for type aliases - new_type_alias_stmt = attach(cls.type_alias_stmt_ref, cls.method("type_alias_stmt_handle")) - cls.type_alias_stmt <<= Wrap(new_type_alias_stmt, cls.method("type_alias_stmt_manage"), greedy=True) + cls.type_alias_stmt <<= handle_and_manage( + cls.type_alias_stmt_ref, + cls.method("type_alias_stmt_handle"), + cls.method("type_alias_stmt_manage"), + ) + + # handle parsing_context for where statements + cls.where_stmt <<= handle_and_manage( + cls.where_stmt_ref, + cls.method("where_stmt_handle"), + cls.method("where_stmt_manage"), + ) + cls.implicit_return_where <<= handle_and_manage( + cls.implicit_return_where_ref, + cls.method("where_stmt_handle"), + cls.method("where_stmt_manage"), + ) # greedy handlers (we need to know about them even if suppressed and/or they use the parsing_context) cls.comment <<= attach(cls.comment_tokens, cls.method("comment_handle"), greedy=True) cls.type_param <<= attach(cls.type_param_ref, cls.method("type_param_handle"), greedy=True) + cls.where_item <<= attach(cls.where_item_ref, cls.method("where_item_handle"), greedy=True) + cls.implicit_return_where_item <<= attach(cls.implicit_return_where_item_ref, cls.method("where_item_handle"), greedy=True) # name handlers cls.refname <<= attach(cls.name_ref, cls.method("name_handle")) @@ -880,6 +916,14 @@ def reformat_without_adding_code_before(self, code, **kwargs): reformatted_code = self.reformat(code, put_code_to_add_before_in=got_code_to_add_before, **kwargs) return reformatted_code, tuple(got_code_to_add_before.keys()), got_code_to_add_before.values() + def extract_deferred_code(self, code): + """Extract the code to be added before in code.""" + got_code_to_add_before = {} + procd_out = self.deferred_code_proc(code, put_code_to_add_before_in=got_code_to_add_before) + added_names = tuple(got_code_to_add_before.keys()) + add_code_before = "\n".join(got_code_to_add_before.values()) + return procd_out, added_names, add_code_before + def literal_eval(self, code): """Version of ast.literal_eval that reformats first.""" return literal_eval(self.reformat(code, ignore_errors=False)) @@ -1122,7 +1166,10 @@ def make_err(self, errtype, message, original, loc=0, ln=None, extra=None, refor # get line number if ln is None: - ln = self.outer_ln or self.adjust(lineno(loc, original)) + if self.outer_ln is None: + ln = self.adjust(lineno(loc, original)) + else: + ln = self.outer_ln # get line indices for the error locs original_lines = tuple(logical_lines(original, True)) @@ -1199,7 +1246,10 @@ def inner_parse_eval( """Parse eval code in an inner environment.""" if parser is None: parser = self.eval_parser - with self.inner_environment(ln=self.adjust(lineno(loc, original))): + outer_ln = self.outer_ln + if outer_ln is None: + outer_ln = self.adjust(lineno(loc, original)) + with self.inner_environment(ln=outer_ln): self.streamline(parser, inputstring) pre_procd = self.pre(inputstring, **preargs) parsed = parse(parser, pre_procd) @@ -3980,17 +4030,13 @@ def get_generic_for_typevars(self): @contextmanager def type_alias_stmt_manage(self, item=None, original=None, loc=None): """Manage the typevars parsing context.""" - typevars_stack = self.parsing_context["typevars"] prev_typevar_info = self.current_parsing_context("typevars") - typevars_stack.append({ + with self.add_to_parsing_context("typevars", { "all_typevars": {} if prev_typevar_info is None else prev_typevar_info["all_typevars"].copy(), "new_typevars": [], "typevar_locs": {}, - }) - try: + }): yield - finally: - typevars_stack.pop() def type_alias_stmt_handle(self, tokens): """Handle type alias statements.""" @@ -4008,6 +4054,53 @@ def type_alias_stmt_handle(self, tokens): self.wrap_typedef(typedef, for_py_typedef=False), ]) + def where_item_handle(self, tokens): + """Manage where items.""" + where_context = self.current_parsing_context("where") + internal_assert(not where_context["assigns"], "invalid where_context", where_context) + where_context["assigns"] = set() + return tokens + + @contextmanager + def where_stmt_manage(self, item, original, loc): + """Manage where statements.""" + with self.add_to_parsing_context("where", { + "assigns": None, + }): + yield + + def where_stmt_handle(self, loc, tokens): + """Process where statements.""" + final_stmt, init_stmts = tokens + + where_assigns = self.current_parsing_context("where")["assigns"] + internal_assert(lambda: where_assigns is not None, "missing where_assigns") + + out = "".join(init_stmts) + final_stmt + "\n" + if not where_assigns: + return out + + name_regexes = { + name: compile_regex(r"\b" + name + r"\b") + for name in where_assigns + } + where_temp_vars = { + name: self.get_temp_var("where_" + name, loc) + for name in where_assigns + } + + out, ignore_names, add_code_before = self.extract_deferred_code(out) + + for name in where_assigns: + out = name_regexes[name].sub(lambda match: where_temp_vars[name], out) + add_code_before = name_regexes[name].sub(lambda match: where_temp_vars[name], add_code_before) + + return self.add_code_before_marker_with_replacement( + out, + add_code_before, + ignore_names=ignore_names, + ) + def with_stmt_handle(self, tokens): """Process with statements.""" withs, body = tokens @@ -4521,14 +4614,21 @@ def name_handle(self, original, loc, tokens, assign=False, classname=False): else: escaped = False + if self.disable_name_check: + return name + + if assign: + where_context = self.current_parsing_context("where") + if where_context is not None: + where_assigns = where_context["assigns"] + if where_assigns is not None: + where_assigns.add(name) + if classname: cls_context = self.current_parsing_context("class") self.internal_assert(cls_context is not None, original, loc, "found classname outside of class", tokens) cls_context["name"] = name - if self.disable_name_check: - return name - # raise_or_wrap_error for all errors here to make sure we don't # raise spurious errors if not using the computation graph diff --git a/coconut/compiler/grammar.py b/coconut/compiler/grammar.py index bbd3bd902..adff7a9c2 100644 --- a/coconut/compiler/grammar.py +++ b/coconut/compiler/grammar.py @@ -518,12 +518,6 @@ def join_match_funcdef(tokens): ) -def where_handle(tokens): - """Process where statements.""" - final_stmt, init_stmts = tokens - return "".join(init_stmts) + final_stmt + "\n" - - def kwd_err_msg_handle(tokens): """Handle keyword parse error messages.""" kwd, = tokens @@ -2089,27 +2083,26 @@ class Grammar(object): ) match_funcdef = addspace(match_def_modifiers + def_match_funcdef) - where_stmt = attach( - unsafe_simple_stmt_item - + keyword("where").suppress() - - full_suite, - where_handle, - ) + where_suite = keyword("where").suppress() - full_suite + + where_stmt = Forward() + where_item = Forward() + where_item_ref = unsafe_simple_stmt_item + where_stmt_ref = where_item + where_suite implicit_return = ( invalid_syntax(return_stmt, "expected expression but got return statement") | attach(new_testlist_star_expr, implicit_return_handle) ) - implicit_return_where = attach( - implicit_return - + keyword("where").suppress() - - full_suite, - where_handle, - ) + implicit_return_where = Forward() + implicit_return_where_item = Forward() + implicit_return_where_item_ref = implicit_return + implicit_return_where_ref = implicit_return_where_item + where_suite implicit_return_stmt = ( condense(implicit_return + newline) | implicit_return_where ) + math_funcdef_body = condense(ZeroOrMore(~(implicit_return_stmt + dedent) + stmt) - implicit_return_stmt) math_funcdef_suite = ( attach(implicit_return_stmt, make_suite_handle) diff --git a/coconut/compiler/util.py b/coconut/compiler/util.py index 8ff8a3279..1a97b7ca5 100644 --- a/coconut/compiler/util.py +++ b/coconut/compiler/util.py @@ -796,6 +796,12 @@ def __repr__(self): return self.wrapped_name +def handle_and_manage(item, handler, manager): + """Attach a handler and a manager to the given parse item.""" + new_item = attach(item, handler) + return Wrap(new_item, manager, greedy=True) + + def disable_inside(item, *elems, **kwargs): """Prevent elems from matching inside of item. @@ -871,6 +877,7 @@ def longest(*args): return matcher +@memoize(64) def compile_regex(regex, options=None): """Compiles the given regex to support unicode.""" if options is None: @@ -880,9 +887,6 @@ def compile_regex(regex, options=None): return re.compile(regex, options) -memoized_compile_regex = memoize(64)(compile_regex) - - def regex_item(regex, options=None): """pyparsing.Regex except it always uses unicode.""" if options is None: diff --git a/coconut/root.py b/coconut/root.py index c8bb08392..be751cb2b 100644 --- a/coconut/root.py +++ b/coconut/root.py @@ -26,7 +26,7 @@ VERSION = "3.0.3" VERSION_NAME = None # False for release, int >= 1 for develop -DEVELOP = 6 +DEVELOP = 7 ALPHA = False # for pre releases rather than post releases assert DEVELOP is False or DEVELOP >= 1, "DEVELOP must be False or an int >= 1" diff --git a/coconut/tests/src/cocotest/agnostic/primary.coco b/coconut/tests/src/cocotest/agnostic/primary.coco index b4db19453..26192f408 100644 --- a/coconut/tests/src/cocotest/agnostic/primary.coco +++ b/coconut/tests/src/cocotest/agnostic/primary.coco @@ -1644,4 +1644,13 @@ def primary_test() -> bool: }___" == '___1___' == f"___{( 1 )}___" + x = 10 + assert x == 5 where: + x = 5 + assert x == 10 + def nested() = f where: + f = def -> g where: + def g() = x where: + x = 5 + assert nested()()() == 5 return True diff --git a/coconut/tests/src/cocotest/agnostic/util.coco b/coconut/tests/src/cocotest/agnostic/util.coco index 427245454..fbbcc2e4d 100644 --- a/coconut/tests/src/cocotest/agnostic/util.coco +++ b/coconut/tests/src/cocotest/agnostic/util.coco @@ -207,10 +207,12 @@ operator !! # bool operator lol lols = [-1] -match def lol = "lol" where: +match def lol = lols[0] += 1 -addpattern def (s) lol = s + "ol" where: # type: ignore + "lol" +addpattern def (s) lol = # type: ignore lols[0] += 1 + s + "ol" lol operator *** From 479c6aa0e0de8308f8047fe49f80bf2fb1a7627e Mon Sep 17 00:00:00 2001 From: Evan Hubinger Date: Sat, 2 Sep 2023 13:36:42 -0700 Subject: [PATCH 013/121] Fix where statements --- coconut/compiler/compiler.py | 38 ++++++++++++++++++++++-------------- coconut/compiler/util.py | 7 +++++++ 2 files changed, 30 insertions(+), 15 deletions(-) diff --git a/coconut/compiler/compiler.py b/coconut/compiler/compiler.py index 02afd2a28..a74568749 100644 --- a/coconut/compiler/compiler.py +++ b/coconut/compiler/compiler.py @@ -173,6 +173,7 @@ unpickle_incremental_cache, pickle_incremental_cache, handle_and_manage, + sub_all, ) from coconut.compiler.header import ( minify_header, @@ -465,6 +466,7 @@ class Compiler(Grammar, pickleable_obj): reformatprocs = [ # deferred_code_proc must come first lambda self: self.deferred_code_proc, + lambda self: partial(self.base_passthrough_repl, wrap_char=early_passthrough_wrapper), lambda self: self.reind_proc, lambda self: self.endline_repl, lambda self: partial(self.base_passthrough_repl, wrap_char="\\"), @@ -1056,6 +1058,9 @@ def wrap_passthrough(self, text, multiline=True, early=False): if not multiline: text = text.lstrip() if early: + # early passthroughs can be nested, so un-nest them + while early_passthrough_wrapper in text: + text = self.base_passthrough_repl(text, wrap_char=early_passthrough_wrapper) out = early_passthrough_wrapper elif multiline: out = "\\" @@ -2566,6 +2571,14 @@ def {mock_var}({mock_paramdef}): internal_assert(not decorators, "unhandled decorators", decorators) return "".join(out) + def modify_add_code_before(self, add_code_before_names, code_modifier): + """Apply code_modifier to all the code corresponding to add_code_before_names.""" + for name in add_code_before_names: + self.add_code_before[name] = code_modifier(self.add_code_before[name]) + replacement = self.add_code_before_replacements.get(name) + if replacement is not None: + self.add_code_before_replacements[name] = code_modifier(replacement) + def add_code_before_marker_with_replacement(self, replacement, add_code_before, add_spaces=True, ignore_names=None): """Add code before a marker that will later be replaced.""" # temp_marker will be set back later, but needs to be a unique name until then for add_code_before @@ -2596,9 +2609,6 @@ def deferred_code_proc(self, inputstring, add_code_at_start=False, ignore_names= for raw_line in inputstring.splitlines(True): bef_ind, line, aft_ind = split_leading_trailing_indent(raw_line) - # handle early passthroughs - line = self.base_passthrough_repl(line, wrap_char=early_passthrough_wrapper, **kwargs) - # look for deferred errors while errwrapper in raw_line: pre_err_line, err_line = raw_line.split(errwrapper, 1) @@ -4071,12 +4081,14 @@ def where_stmt_manage(self, item, original, loc): def where_stmt_handle(self, loc, tokens): """Process where statements.""" - final_stmt, init_stmts = tokens + main_stmt, body_stmts = tokens where_assigns = self.current_parsing_context("where")["assigns"] internal_assert(lambda: where_assigns is not None, "missing where_assigns") - out = "".join(init_stmts) + final_stmt + "\n" + where_init = "".join(body_stmts) + where_final = main_stmt + "\n" + out = where_init + where_final if not where_assigns: return out @@ -4084,22 +4096,18 @@ def where_stmt_handle(self, loc, tokens): name: compile_regex(r"\b" + name + r"\b") for name in where_assigns } - where_temp_vars = { + name_replacements = { name: self.get_temp_var("where_" + name, loc) for name in where_assigns } - out, ignore_names, add_code_before = self.extract_deferred_code(out) + where_init = self.deferred_code_proc(where_init) + where_final = self.deferred_code_proc(where_final) + out = where_init + where_final - for name in where_assigns: - out = name_regexes[name].sub(lambda match: where_temp_vars[name], out) - add_code_before = name_regexes[name].sub(lambda match: where_temp_vars[name], add_code_before) + out = sub_all(out, name_regexes, name_replacements) - return self.add_code_before_marker_with_replacement( - out, - add_code_before, - ignore_names=ignore_names, - ) + return self.wrap_passthrough(out, early=True) def with_stmt_handle(self, tokens): """Process with statements.""" diff --git a/coconut/compiler/util.py b/coconut/compiler/util.py index 1a97b7ca5..5efb771d8 100644 --- a/coconut/compiler/util.py +++ b/coconut/compiler/util.py @@ -1601,6 +1601,13 @@ def move_endpt_to_non_whitespace(original, loc, backwards=False): ) +def sub_all(inputstr, regexes, replacements): + """Sub all regexes for replacements in inputstr.""" + for key, regex in regexes.items(): + inputstr = regex.sub(lambda match: replacements[key], inputstr) + return inputstr + + # ----------------------------------------------------------------------------------------------------------------------- # PYTEST: # ----------------------------------------------------------------------------------------------------------------------- From 00ef984b0f70c281407b2e11670103179fde91eb Mon Sep 17 00:00:00 2001 From: Evan Hubinger Date: Sat, 2 Sep 2023 14:57:59 -0700 Subject: [PATCH 014/121] Add numpy install extra --- DOCS.md | 1 + coconut/compiler/compiler.py | 8 +++++--- coconut/constants.py | 8 +++++--- coconut/requirements.py | 2 ++ 4 files changed, 13 insertions(+), 6 deletions(-) diff --git a/DOCS.md b/DOCS.md index bcf7acb1a..9045fb812 100644 --- a/DOCS.md +++ b/DOCS.md @@ -96,6 +96,7 @@ The full list of optional dependencies is: - Installs [`async_generator`](https://github.com/python-trio/async_generator) to backport [`async` generators](https://peps.python.org/pep-0525/) and [`asynccontextmanager`](https://docs.python.org/3/library/contextlib.html#contextlib.asynccontextmanager). - Installs [`trollius`](https://pypi.python.org/pypi/trollius) to backport [`async`/`await`](https://docs.python.org/3/library/asyncio-task.html) and [`asyncio`](https://docs.python.org/3/library/asyncio.html). - `xonsh`: enables use of Coconut's [`xonsh` support](#xonsh-support). +- `numpy`: installs everything necessary for making use of Coconut's [`numpy` integration](#numpy-integration). - `kernel`: lightweight subset of `jupyter` that only includes the dependencies that are strictly necessary for Coconut's [Jupyter kernel](#kernel). - `tests`: everything necessary to test the Coconut language itself. - `docs`: everything necessary to build Coconut's documentation. diff --git a/coconut/compiler/compiler.py b/coconut/compiler/compiler.py index a74568749..f3dae9c7e 100644 --- a/coconut/compiler/compiler.py +++ b/coconut/compiler/compiler.py @@ -661,6 +661,8 @@ def post_transform(self, grammar, text): def get_temp_var(self, base_name="temp", loc=None): """Get a unique temporary variable name.""" + if isinstance(base_name, tuple): + base_name = "_".join(base_name) if loc is None: key = None else: @@ -2323,7 +2325,7 @@ def proc_funcdef(self, original, loc, decorators, funcdef, is_async, in_method, undotted_name = None if func_name is not None and "." in func_name: undotted_name = func_name.rsplit(".", 1)[-1] - def_name = self.get_temp_var("dotted_" + undotted_name, loc) + def_name = self.get_temp_var(("dotted", undotted_name), loc) # detect pattern-matching functions is_match_func = func_paramdef == match_func_paramdef @@ -4007,7 +4009,7 @@ def type_param_handle(self, original, loc, tokens): else: if name in typevar_info["all_typevars"]: raise CoconutDeferredSyntaxError("type variable {name!r} already defined".format(name=name), loc) - temp_name = self.get_temp_var("typevar_" + name, name_loc) + temp_name = self.get_temp_var(("typevar", name), name_loc) typevar_info["all_typevars"][name] = temp_name typevar_info["new_typevars"].append((TypeVarFunc, temp_name)) typevar_info["typevar_locs"][name] = name_loc @@ -4097,7 +4099,7 @@ def where_stmt_handle(self, loc, tokens): for name in where_assigns } name_replacements = { - name: self.get_temp_var("where_" + name, loc) + name: self.get_temp_var(("where", name), loc) for name in where_assigns } diff --git a/coconut/constants.py b/coconut/constants.py index 84d7a3d3b..f3cab0ac7 100644 --- a/coconut/constants.py +++ b/coconut/constants.py @@ -951,13 +951,15 @@ def get_path_env_var(env_var, default): "myst-parser", "pydata-sphinx-theme", ), + "numpy": ( + ("numpy", "py34"), + ("numpy", "py<3;cpy"), + ("pandas", "py36"), + ), "tests": ( ("pytest", "py<36"), ("pytest", "py36"), "pexpect", - ("numpy", "py34"), - ("numpy", "py<3;cpy"), - ("pandas", "py36"), ), } diff --git a/coconut/requirements.py b/coconut/requirements.py index 6ead04b53..cac2b82d2 100644 --- a/coconut/requirements.py +++ b/coconut/requirements.py @@ -222,6 +222,7 @@ def everything_in(req_dict): "mypy": get_reqs("mypy"), "backports": get_reqs("backports"), "xonsh": get_reqs("xonsh"), + "numpy": get_reqs("numpy"), } extras["jupyter"] = uniqueify_all( @@ -237,6 +238,7 @@ def everything_in(req_dict): "tests": uniqueify_all( get_reqs("tests"), extras["backports"], + extras["numpy"], extras["jupyter"] if IPY else [], extras["mypy"] if MYPY else [], extras["xonsh"] if XONSH else [], From f4e3ebdedbe0222ae448ed22f7046c8510520615 Mon Sep 17 00:00:00 2001 From: Evan Hubinger Date: Fri, 22 Sep 2023 22:46:09 -0700 Subject: [PATCH 015/121] Add attritemgetter partials Resolves #787. --- .pre-commit-config.yaml | 4 +- __coconut__/__init__.pyi | 6 +++ coconut/__coconut__.pyi | 2 +- coconut/compiler/compiler.py | 21 +++++--- coconut/compiler/grammar.py | 42 +++++++++------ coconut/compiler/header.py | 2 +- coconut/compiler/templates/header.py_template | 53 ++++++++++++------- coconut/root.py | 2 +- .../tests/src/cocotest/agnostic/primary.coco | 5 ++ .../tests/src/cocotest/agnostic/suite.coco | 12 +++++ coconut/tests/src/cocotest/agnostic/util.coco | 19 +++++++ 11 files changed, 120 insertions(+), 48 deletions(-) diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index df224ace7..8f79c7238 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -24,13 +24,13 @@ repos: args: - --autofix - repo: https://github.com/pycqa/flake8 - rev: 6.0.0 + rev: 6.1.0 hooks: - id: flake8 args: - --ignore=W503,E501,E265,E402,F405,E305,E126 - repo: https://github.com/pre-commit/mirrors-autopep8 - rev: v2.0.2 + rev: v2.0.4 hooks: - id: autopep8 args: diff --git a/__coconut__/__init__.pyi b/__coconut__/__init__.pyi index d4b4ff4a6..c75480c00 100644 --- a/__coconut__/__init__.pyi +++ b/__coconut__/__init__.pyi @@ -640,6 +640,12 @@ def _coconut_iter_getitem( ... +def _coconut_attritemgetter( + attr: _t.Optional[_t.Text], + *is_iter_and_items: _t.Tuple[_t.Tuple[bool, _t.Any], ...], +) -> _t.Callable[[_t.Any], _t.Any]: ... + + def _coconut_base_compose( func: _t.Callable[[_T], _t.Any], *func_infos: _t.Tuple[_Callable, int, bool], diff --git a/coconut/__coconut__.pyi b/coconut/__coconut__.pyi index 45d413ea3..91be385cb 100644 --- a/coconut/__coconut__.pyi +++ b/coconut/__coconut__.pyi @@ -1,2 +1,2 @@ from __coconut__ import * -from __coconut__ import _coconut_tail_call, _coconut_tco, _coconut_call_set_names, _coconut_handle_cls_kwargs, _coconut_handle_cls_stargs, _namedtuple_of, _coconut, _coconut_Expected, _coconut_MatchError, _coconut_SupportsAdd, _coconut_SupportsMinus, _coconut_SupportsMul, _coconut_SupportsPow, _coconut_SupportsTruediv, _coconut_SupportsFloordiv, _coconut_SupportsMod, _coconut_SupportsAnd, _coconut_SupportsXor, _coconut_SupportsOr, _coconut_SupportsLshift, _coconut_SupportsRshift, _coconut_SupportsMatmul, _coconut_SupportsInv, _coconut_Expected, _coconut_MatchError, _coconut_iter_getitem, _coconut_base_compose, _coconut_forward_compose, _coconut_back_compose, _coconut_forward_star_compose, _coconut_back_star_compose, _coconut_forward_dubstar_compose, _coconut_back_dubstar_compose, _coconut_pipe, _coconut_star_pipe, _coconut_dubstar_pipe, _coconut_back_pipe, _coconut_back_star_pipe, _coconut_back_dubstar_pipe, _coconut_none_pipe, _coconut_none_star_pipe, _coconut_none_dubstar_pipe, _coconut_bool_and, _coconut_bool_or, _coconut_none_coalesce, _coconut_minus, _coconut_map, _coconut_partial, _coconut_get_function_match_error, _coconut_base_pattern_func, _coconut_addpattern, _coconut_sentinel, _coconut_assert, _coconut_raise, _coconut_mark_as_match, _coconut_reiterable, _coconut_self_match_types, _coconut_dict_merge, _coconut_exec, _coconut_comma_op, _coconut_multi_dim_arr, _coconut_mk_anon_namedtuple, _coconut_matmul, _coconut_py_str, _coconut_flatten, _coconut_multiset, _coconut_back_none_pipe, _coconut_back_none_star_pipe, _coconut_back_none_dubstar_pipe, _coconut_forward_none_compose, _coconut_back_none_compose, _coconut_forward_none_star_compose, _coconut_back_none_star_compose, _coconut_forward_none_dubstar_compose, _coconut_back_none_dubstar_compose, _coconut_call_or_coefficient, _coconut_in, _coconut_not_in +from __coconut__ import _coconut_tail_call, _coconut_tco, _coconut_call_set_names, _coconut_handle_cls_kwargs, _coconut_handle_cls_stargs, _namedtuple_of, _coconut, _coconut_Expected, _coconut_MatchError, _coconut_SupportsAdd, _coconut_SupportsMinus, _coconut_SupportsMul, _coconut_SupportsPow, _coconut_SupportsTruediv, _coconut_SupportsFloordiv, _coconut_SupportsMod, _coconut_SupportsAnd, _coconut_SupportsXor, _coconut_SupportsOr, _coconut_SupportsLshift, _coconut_SupportsRshift, _coconut_SupportsMatmul, _coconut_SupportsInv, _coconut_Expected, _coconut_MatchError, _coconut_iter_getitem, _coconut_base_compose, _coconut_forward_compose, _coconut_back_compose, _coconut_forward_star_compose, _coconut_back_star_compose, _coconut_forward_dubstar_compose, _coconut_back_dubstar_compose, _coconut_pipe, _coconut_star_pipe, _coconut_dubstar_pipe, _coconut_back_pipe, _coconut_back_star_pipe, _coconut_back_dubstar_pipe, _coconut_none_pipe, _coconut_none_star_pipe, _coconut_none_dubstar_pipe, _coconut_bool_and, _coconut_bool_or, _coconut_none_coalesce, _coconut_minus, _coconut_map, _coconut_partial, _coconut_get_function_match_error, _coconut_base_pattern_func, _coconut_addpattern, _coconut_sentinel, _coconut_assert, _coconut_raise, _coconut_mark_as_match, _coconut_reiterable, _coconut_self_match_types, _coconut_dict_merge, _coconut_exec, _coconut_comma_op, _coconut_multi_dim_arr, _coconut_mk_anon_namedtuple, _coconut_matmul, _coconut_py_str, _coconut_flatten, _coconut_multiset, _coconut_back_none_pipe, _coconut_back_none_star_pipe, _coconut_back_none_dubstar_pipe, _coconut_forward_none_compose, _coconut_back_none_compose, _coconut_forward_none_star_compose, _coconut_back_none_star_compose, _coconut_forward_none_dubstar_compose, _coconut_back_none_dubstar_compose, _coconut_call_or_coefficient, _coconut_in, _coconut_not_in, _coconut_attritemgetter diff --git a/coconut/compiler/compiler.py b/coconut/compiler/compiler.py index f3dae9c7e..fa7f611d9 100644 --- a/coconut/compiler/compiler.py +++ b/coconut/compiler/compiler.py @@ -2747,7 +2747,7 @@ def pipe_item_split(self, tokens, loc): - (expr,) for expression - (func, pos_args, kwd_args) for partial - (name, args) for attr/method - - (op, args)+ for itemgetter + - (attr, [(op, args)]) for itemgetter - (op, arg) for right op partial """ # list implies artificial tokens, which must be expr @@ -2762,8 +2762,12 @@ def pipe_item_split(self, tokens, loc): name, args = attrgetter_atom_split(tokens) return "attrgetter", (name, args) elif "itemgetter" in tokens: - internal_assert(len(tokens) >= 2, "invalid itemgetter pipe item tokens", tokens) - return "itemgetter", tokens + if len(tokens) == 1: + attr = None + ops_and_args, = tokens + else: + attr, ops_and_args = tokens + return "itemgetter", (attr, ops_and_args) elif "op partial" in tokens: inner_toks, = tokens if "left partial" in inner_toks: @@ -2853,12 +2857,13 @@ def pipe_handle(self, original, loc, tokens, **kwargs): elif name == "itemgetter": if stars: raise CoconutDeferredSyntaxError("cannot star pipe into item getting", loc) - self.internal_assert(len(split_item) % 2 == 0, original, loc, "invalid itemgetter pipe tokens", split_item) - out = subexpr - for i in range(0, len(split_item), 2): - op, args = split_item[i:i + 2] + attr, ops_and_args = split_item + out = "(" + subexpr + ")" + if attr is not None: + out += "." + attr + for op, args in ops_and_args: if op == "[": - fmtstr = "({x})[{args}]" + fmtstr = "{x}[{args}]" elif op == "$[": fmtstr = "_coconut_iter_getitem({x}, ({args}))" else: diff --git a/coconut/compiler/grammar.py b/coconut/compiler/grammar.py index adff7a9c2..48e22713d 100644 --- a/coconut/compiler/grammar.py +++ b/coconut/compiler/grammar.py @@ -437,22 +437,24 @@ def subscriptgroup_handle(tokens): def itemgetter_handle(tokens): """Process implicit itemgetter partials.""" - if len(tokens) == 2: - op, args = tokens + if len(tokens) == 1: + attr = None + ops_and_args, = tokens + else: + attr, ops_and_args = tokens + if attr is None and len(ops_and_args) == 1: + (op, args), = ops_and_args if op == "[": return "_coconut.operator.itemgetter((" + args + "))" elif op == "$[": return "_coconut.functools.partial(_coconut_iter_getitem, index=(" + args + "))" else: raise CoconutInternalException("invalid implicit itemgetter type", op) - elif len(tokens) > 2: - internal_assert(len(tokens) % 2 == 0, "invalid itemgetter composition tokens", tokens) - itemgetters = [] - for i in range(0, len(tokens), 2): - itemgetters.append(itemgetter_handle(tokens[i:i + 2])) - return "_coconut_forward_compose(" + ", ".join(itemgetters) + ")" else: - raise CoconutInternalException("invalid implicit itemgetter tokens", tokens) + return "_coconut_attritemgetter({attr}, {is_iter_and_items})".format( + attr=repr(attr), + is_iter_and_items=", ".join("({is_iter}, ({item}))".format(is_iter=op == "$[", item=args) for op, args in ops_and_args), + ) def class_suite_handle(tokens): @@ -1300,11 +1302,21 @@ class Grammar(object): lparen + Optional(methodcaller_args) + rparen.suppress() ) attrgetter_atom = attach(attrgetter_atom_tokens, attrgetter_atom_handle) - itemgetter_atom_tokens = dot.suppress() + OneOrMore(condense(Optional(dollar) + lbrack) + subscriptgrouplist + rbrack.suppress()) + + itemgetter_atom_tokens = ( + dot.suppress() + + Optional(unsafe_dotted_name) + + Group(OneOrMore(Group( + condense(Optional(dollar) + lbrack) + + subscriptgrouplist + + rbrack.suppress() + ))) + ) itemgetter_atom = attach(itemgetter_atom_tokens, itemgetter_handle) + implicit_partial_atom = ( - attrgetter_atom - | itemgetter_atom + itemgetter_atom + | attrgetter_atom | fixto(dot + lbrack + rbrack, "_coconut.operator.getitem") | fixto(dot + dollar + lbrack + rbrack, "_coconut_iter_getitem") ) @@ -1485,8 +1497,8 @@ class Grammar(object): pipe_item = ( # we need the pipe_op since any of the atoms could otherwise be the start of an expression labeled_group(keyword("await"), "await") + pipe_op - | labeled_group(attrgetter_atom_tokens, "attrgetter") + pipe_op | labeled_group(itemgetter_atom_tokens, "itemgetter") + pipe_op + | labeled_group(attrgetter_atom_tokens, "attrgetter") + pipe_op | labeled_group(partial_atom_tokens, "partial") + pipe_op | labeled_group(partial_op_atom_tokens, "op partial") + pipe_op # expr must come at end @@ -1495,8 +1507,8 @@ class Grammar(object): pipe_augassign_item = ( # should match pipe_item but with pipe_op -> end_simple_stmt_item and no expr labeled_group(keyword("await"), "await") + end_simple_stmt_item - | labeled_group(attrgetter_atom_tokens, "attrgetter") + end_simple_stmt_item | labeled_group(itemgetter_atom_tokens, "itemgetter") + end_simple_stmt_item + | labeled_group(attrgetter_atom_tokens, "attrgetter") + end_simple_stmt_item | labeled_group(partial_atom_tokens, "partial") + end_simple_stmt_item | labeled_group(partial_op_atom_tokens, "op partial") + end_simple_stmt_item ) @@ -1505,8 +1517,8 @@ class Grammar(object): # we need longest here because there's no following pipe_op we can use as above | longest( keyword("await")("await"), - attrgetter_atom_tokens("attrgetter"), itemgetter_atom_tokens("itemgetter"), + attrgetter_atom_tokens("attrgetter"), partial_atom_tokens("partial"), partial_op_atom_tokens("op partial"), comp_pipe_expr("expr"), diff --git a/coconut/compiler/header.py b/coconut/compiler/header.py index 3be75c8fd..409929e50 100644 --- a/coconut/compiler/header.py +++ b/coconut/compiler/header.py @@ -586,7 +586,7 @@ async def __anext__(self): # (extra_format_dict is to keep indentation levels matching) extra_format_dict = dict( # when anything is added to this list it must also be added to *both* __coconut__ stub files - underscore_imports="{tco_comma}{call_set_names_comma}{handle_cls_args_comma}_namedtuple_of, _coconut, _coconut_Expected, _coconut_MatchError, _coconut_SupportsAdd, _coconut_SupportsMinus, _coconut_SupportsMul, _coconut_SupportsPow, _coconut_SupportsTruediv, _coconut_SupportsFloordiv, _coconut_SupportsMod, _coconut_SupportsAnd, _coconut_SupportsXor, _coconut_SupportsOr, _coconut_SupportsLshift, _coconut_SupportsRshift, _coconut_SupportsMatmul, _coconut_SupportsInv, _coconut_iter_getitem, _coconut_base_compose, _coconut_forward_compose, _coconut_back_compose, _coconut_forward_star_compose, _coconut_back_star_compose, _coconut_forward_dubstar_compose, _coconut_back_dubstar_compose, _coconut_pipe, _coconut_star_pipe, _coconut_dubstar_pipe, _coconut_back_pipe, _coconut_back_star_pipe, _coconut_back_dubstar_pipe, _coconut_none_pipe, _coconut_none_star_pipe, _coconut_none_dubstar_pipe, _coconut_bool_and, _coconut_bool_or, _coconut_none_coalesce, _coconut_minus, _coconut_map, _coconut_partial, _coconut_get_function_match_error, _coconut_base_pattern_func, _coconut_addpattern, _coconut_sentinel, _coconut_assert, _coconut_raise, _coconut_mark_as_match, _coconut_reiterable, _coconut_self_match_types, _coconut_dict_merge, _coconut_exec, _coconut_comma_op, _coconut_multi_dim_arr, _coconut_mk_anon_namedtuple, _coconut_matmul, _coconut_py_str, _coconut_flatten, _coconut_multiset, _coconut_back_none_pipe, _coconut_back_none_star_pipe, _coconut_back_none_dubstar_pipe, _coconut_forward_none_compose, _coconut_back_none_compose, _coconut_forward_none_star_compose, _coconut_back_none_star_compose, _coconut_forward_none_dubstar_compose, _coconut_back_none_dubstar_compose, _coconut_call_or_coefficient, _coconut_in, _coconut_not_in".format(**format_dict), + underscore_imports="{tco_comma}{call_set_names_comma}{handle_cls_args_comma}_namedtuple_of, _coconut, _coconut_Expected, _coconut_MatchError, _coconut_SupportsAdd, _coconut_SupportsMinus, _coconut_SupportsMul, _coconut_SupportsPow, _coconut_SupportsTruediv, _coconut_SupportsFloordiv, _coconut_SupportsMod, _coconut_SupportsAnd, _coconut_SupportsXor, _coconut_SupportsOr, _coconut_SupportsLshift, _coconut_SupportsRshift, _coconut_SupportsMatmul, _coconut_SupportsInv, _coconut_iter_getitem, _coconut_base_compose, _coconut_forward_compose, _coconut_back_compose, _coconut_forward_star_compose, _coconut_back_star_compose, _coconut_forward_dubstar_compose, _coconut_back_dubstar_compose, _coconut_pipe, _coconut_star_pipe, _coconut_dubstar_pipe, _coconut_back_pipe, _coconut_back_star_pipe, _coconut_back_dubstar_pipe, _coconut_none_pipe, _coconut_none_star_pipe, _coconut_none_dubstar_pipe, _coconut_bool_and, _coconut_bool_or, _coconut_none_coalesce, _coconut_minus, _coconut_map, _coconut_partial, _coconut_get_function_match_error, _coconut_base_pattern_func, _coconut_addpattern, _coconut_sentinel, _coconut_assert, _coconut_raise, _coconut_mark_as_match, _coconut_reiterable, _coconut_self_match_types, _coconut_dict_merge, _coconut_exec, _coconut_comma_op, _coconut_multi_dim_arr, _coconut_mk_anon_namedtuple, _coconut_matmul, _coconut_py_str, _coconut_flatten, _coconut_multiset, _coconut_back_none_pipe, _coconut_back_none_star_pipe, _coconut_back_none_dubstar_pipe, _coconut_forward_none_compose, _coconut_back_none_compose, _coconut_forward_none_star_compose, _coconut_back_none_star_compose, _coconut_forward_none_dubstar_compose, _coconut_back_none_dubstar_compose, _coconut_call_or_coefficient, _coconut_in, _coconut_not_in, _coconut_attritemgetter".format(**format_dict), import_typing=pycondition( (3, 5), if_ge=''' diff --git a/coconut/compiler/templates/header.py_template b/coconut/compiler/templates/header.py_template index 845f6265b..0f8e4e58c 100644 --- a/coconut/compiler/templates/header.py_template +++ b/coconut/compiler/templates/header.py_template @@ -103,6 +103,12 @@ class _coconut_baseclass{object}: if getitem is None: raise _coconut.NotImplementedError return getitem(index) +class _coconut_base_callable(_coconut_baseclass): + __slots__ = () + def __get__(self, obj, objtype=None): + if obj is None: + return self +{return_method_of_self} class _coconut_Sentinel(_coconut_baseclass): __slots__ = () def __reduce__(self): @@ -354,7 +360,26 @@ def _coconut_iter_getitem(iterable, index): return () iterable = _coconut.itertools.islice(iterable, 0, n) return _coconut.tuple(iterable)[i::step] -class _coconut_compostion_baseclass(_coconut_baseclass):{COMMENT.no_slots_to_allow_update_wrapper}{COMMENT.must_use_coconut_attrs_to_avoid_interacting_with_update_wrapper} +class _coconut_attritemgetter(_coconut_base_callable): + __slots__ = ("attr", "is_iter_and_items") + def __init__(self, attr, *is_iter_and_items): + self.attr = attr + self.is_iter_and_items = is_iter_and_items + def __call__(self, obj): + out = obj + if self.attr is not None: + out = _coconut.getattr(out, self.attr) + for is_iter, item in self.is_iter_and_items: + if is_iter: + out = _coconut_iter_getitem(out, item) + else: + out = out[item] + return out + def __repr__(self): + return "." + (self.attr or "") + "".join(("$" if is_iter else "") + "[" + _coconut.repr(item) + "]" for is_iter, item in self.is_iter_and_items) + def __reduce__(self): + return (self.__class__, (self.attr,) + self.is_iter_and_items) +class _coconut_compostion_baseclass(_coconut_base_callable):{COMMENT.no_slots_to_allow_update_wrapper}{COMMENT.must_use_coconut_attrs_to_avoid_interacting_with_update_wrapper} def __init__(self, func, *func_infos): try: _coconut.functools.update_wrapper(self, func) @@ -376,10 +401,6 @@ class _coconut_compostion_baseclass(_coconut_baseclass):{COMMENT.no_slots_to_all self._coconut_func_infos = _coconut.tuple(self._coconut_func_infos) def __reduce__(self): return (self.__class__, (self._coconut_func,) + self._coconut_func_infos) - def __get__(self, obj, objtype=None): - if obj is None: - return self -{return_method_of_self} class _coconut_base_compose(_coconut_compostion_baseclass): __slots__ = () def __call__(self, *args, **kwargs): @@ -1278,7 +1299,7 @@ class groupsof(_coconut_has_iter): return (self.__class__, (self.group_size, self.iter)) def __copy__(self): return self.__class__(self.group_size, self.get_new_iter()) -class recursive_iterator(_coconut_baseclass): +class recursive_iterator(_coconut_base_callable): """Decorator that memoizes a recursive function that returns an iterator (e.g. a recursive generator).""" __slots__ = ("func", "reit_store", "backup_reit_store") def __init__(self, func): @@ -1312,10 +1333,6 @@ class recursive_iterator(_coconut_baseclass): return "recursive_iterator(%r)" % (self.func,) def __reduce__(self): return (self.__class__, (self.func,)) - def __get__(self, obj, objtype=None): - if obj is None: - return self -{return_method_of_self} class _coconut_FunctionMatchErrorContext(_coconut_baseclass): __slots__ = ("exc_class", "taken") threadlocal_ns = _coconut.threading.local() @@ -1340,7 +1357,7 @@ def _coconut_get_function_match_error(): return {_coconut_}MatchError ctx.taken = True return ctx.exc_class -class _coconut_base_pattern_func(_coconut_baseclass):{COMMENT.no_slots_to_allow_func_attrs} +class _coconut_base_pattern_func(_coconut_base_callable):{COMMENT.no_slots_to_allow_func_attrs} _coconut_is_match = True def __init__(self, *funcs): self.FunctionMatchError = _coconut.type(_coconut_py_str("MatchError"), ({_coconut_}MatchError,), {empty_py_dict}) @@ -1378,10 +1395,6 @@ class _coconut_base_pattern_func(_coconut_baseclass):{COMMENT.no_slots_to_allow_ return "addpattern(%r)(*%r)" % (self.patterns[0], self.patterns[1:]) def __reduce__(self): return (self.__class__, _coconut.tuple(self.patterns)) - def __get__(self, obj, objtype=None): - if obj is None: - return self -{return_method_of_self} def _coconut_mark_as_match(base_func):{COMMENT._coconut_is_match_is_used_above_and_in_main_coco} base_func._coconut_is_match = True return base_func @@ -1401,7 +1414,7 @@ def addpattern(base_func, *add_funcs, **kwargs): return _coconut.functools.partial(_coconut_base_pattern_func, base_func) _coconut_addpattern = addpattern {def_prepattern} -class _coconut_partial(_coconut_baseclass): +class _coconut_partial(_coconut_base_callable): __slots__ = ("func", "_argdict", "_arglen", "_pos_kwargs", "_stargs", "keywords") def __init__(self, _coconut_func, _coconut_argdict, _coconut_arglen, _coconut_pos_kwargs, *args, **kwargs): self.func = _coconut_func @@ -1779,7 +1792,7 @@ class Expected(_coconut.collections.namedtuple("Expected", ("result", "error")){ if not self: raise self.error return self.result -class flip(_coconut_baseclass): +class flip(_coconut_base_callable): """Given a function, return a new function with inverse argument order. If nargs is passed, only the first nargs arguments are reversed.""" __slots__ = ("func", "nargs") @@ -1801,7 +1814,7 @@ class flip(_coconut_baseclass): return self.func(*(args[self.nargs-1::-1] + args[self.nargs:]), **kwargs) def __repr__(self): return "flip(%r%s)" % (self.func, "" if self.nargs is None else ", " + _coconut.repr(self.nargs)) -class const(_coconut_baseclass): +class const(_coconut_base_callable): """Create a function that, whatever its arguments, just returns the given value.""" __slots__ = ("value",) def __init__(self, value): @@ -1812,7 +1825,7 @@ class const(_coconut_baseclass): return self.value def __repr__(self): return "const(%s)" % (_coconut.repr(self.value),) -class _coconut_lifted(_coconut_baseclass): +class _coconut_lifted(_coconut_base_callable): __slots__ = ("func", "func_args", "func_kwargs") def __init__(self, _coconut_func, *func_args, **func_kwargs): self.func = _coconut_func @@ -1824,7 +1837,7 @@ class _coconut_lifted(_coconut_baseclass): return self.func(*(g(*args, **kwargs) for g in self.func_args), **_coconut_py_dict((k, h(*args, **kwargs)) for k, h in self.func_kwargs.items())) def __repr__(self): return "lift(%r)(%s%s)" % (self.func, ", ".join(_coconut.repr(g) for g in self.func_args), ", ".join(k + "=" + _coconut.repr(h) for k, h in self.func_kwargs.items())) -class lift(_coconut_baseclass): +class lift(_coconut_base_callable): """Lifts a function up so that all of its arguments are functions. For a binary function f(x, y) and two unary functions g(z) and h(z), lift works as the S' combinator: diff --git a/coconut/root.py b/coconut/root.py index be751cb2b..f30ef27ae 100644 --- a/coconut/root.py +++ b/coconut/root.py @@ -26,7 +26,7 @@ VERSION = "3.0.3" VERSION_NAME = None # False for release, int >= 1 for develop -DEVELOP = 7 +DEVELOP = 8 ALPHA = False # for pre releases rather than post releases assert DEVELOP is False or DEVELOP >= 1, "DEVELOP must be False or an int >= 1" diff --git a/coconut/tests/src/cocotest/agnostic/primary.coco b/coconut/tests/src/cocotest/agnostic/primary.coco index 26192f408..20218a28b 100644 --- a/coconut/tests/src/cocotest/agnostic/primary.coco +++ b/coconut/tests/src/cocotest/agnostic/primary.coco @@ -1653,4 +1653,9 @@ def primary_test() -> bool: def g() = x where: x = 5 assert nested()()() == 5 + class HasPartial: + def f(self, x) = (self, x) + g = f$(?, 1) + has_partial = HasPartial() + assert has_partial.g() == (has_partial, 1) return True diff --git a/coconut/tests/src/cocotest/agnostic/suite.coco b/coconut/tests/src/cocotest/agnostic/suite.coco index 7e0440630..89db9b0a4 100644 --- a/coconut/tests/src/cocotest/agnostic/suite.coco +++ b/coconut/tests/src/cocotest/agnostic/suite.coco @@ -1053,6 +1053,18 @@ forward 2""") == 900 assert ret_args_kwargs(123, ...=really_long_var, abc="abc") == ((123,), {"really_long_var": 10, "abc": "abc"}) == ret_args_kwargs$(123, ...=really_long_var, abc="abc")() assert "Coconut version of typing" in typing.__doc__ numlist: NumList = [1, 2.3, 5] + assert hasloc([[1, 2]]).loc[0][1] == 2 == hasloc([[1, 2]]) |> .loc[0][1] + locgetter = .loc[0][1] + assert hasloc([[1, 2]]) |> locgetter == 2 == (hasloc([[1, 2]]) |> .loc[0])[1] + haslocobj = hasloc([[1, 2]]) + haslocobj |>= .loc[0][1] + assert haslocobj == 2 + assert hasloc([[1, 2]]).iloc$[0]$[1] == 2 == hasloc([[1, 2]]) |> .iloc$[0]$[1] + locgetter = .iloc$[0]$[1] + assert hasloc([[1, 2]]) |> locgetter == 2 == (hasloc([[1, 2]]) |> .iloc$[0])$[1] + haslocobj = hasloc([[1, 2]]) + haslocobj |>= .iloc$[0]$[1] + assert haslocobj == 2 # must come at end assert fibs_calls[0] == 1 diff --git a/coconut/tests/src/cocotest/agnostic/util.coco b/coconut/tests/src/cocotest/agnostic/util.coco index fbbcc2e4d..c2c1c8553 100644 --- a/coconut/tests/src/cocotest/agnostic/util.coco +++ b/coconut/tests/src/cocotest/agnostic/util.coco @@ -1059,6 +1059,25 @@ class unrepresentable: def __repr__(self): raise Fail("unrepresentable") +class hasloc: + def __init__(self, arr): + self.arr = arr + class Loc: + def __init__(inner, outer): + inner.outer = outer + def __getitem__(inner, item) = + inner.outer.arr[item] + @property + def loc(self) = self.Loc(self) + class ILoc: + def __init__(inner, outer): + inner.outer = outer + def __iter_getitem__(inner, item) = + inner.outer.arr$[item] + @property + def iloc(self) = self.ILoc(self) + + # Typing if TYPE_CHECKING or sys.version_info >= (3, 5): # test from typing import *, but that doesn't actually get us From af257bc15fba5368e7017b0bc29726978d24bac6 Mon Sep 17 00:00:00 2001 From: Evan Hubinger Date: Sat, 23 Sep 2023 01:26:56 -0700 Subject: [PATCH 016/121] Fix docs, tests --- DOCS.md | 2 +- coconut/tests/src/extras.coco | 5 +++-- 2 files changed, 4 insertions(+), 3 deletions(-) diff --git a/DOCS.md b/DOCS.md index 9045fb812..cf7f5ac6e 100644 --- a/DOCS.md +++ b/DOCS.md @@ -1818,7 +1818,7 @@ iter$[] => # the equivalent of seq[] for iterators .$[a:b:c] => # the equivalent of .[a:b:c] for iterators ``` -Additionally, `.attr.method(args)`, `.[x][y]`, and `.$[x]$[y]` are also supported. +Additionally, `.attr.method(args)`, `.[x][y]`, `.$[x]$[y]`, and `.method[x]` are also supported. In addition, for every Coconut [operator function](#operator-functions), Coconut supports syntax for implicitly partially applying that operator function as ``` diff --git a/coconut/tests/src/extras.coco b/coconut/tests/src/extras.coco index b3ba40208..fed27375d 100644 --- a/coconut/tests/src/extras.coco +++ b/coconut/tests/src/extras.coco @@ -455,11 +455,12 @@ def test_kernel() -> bool: k = CoconutKernel() fake_session = FakeSession() + assert k.shell is not None k.shell.displayhook.session = fake_session exec_result = k.do_execute("derp = pow$(?, 2)", False, True, {"two": "(+)(1, 1)"}, True) |> unwrap_future$(loop) - assert exec_result["status"] == "ok" - assert exec_result["user_expressions"]["two"]["data"]["text/plain"] == "2" + assert exec_result["status"] == "ok", exec_result + assert exec_result["user_expressions"]["two"]["data"]["text/plain"] == "2", exec_result assert k.do_execute("operator ++", False, True, {}, True) |> unwrap_future$(loop) |> .["status"] == "ok" assert k.do_execute("(++) = 1", False, True, {}, True) |> unwrap_future$(loop) |> .["status"] == "ok" From f55e5524739ba30a5b621c7d763595a36dc7ed10 Mon Sep 17 00:00:00 2001 From: Evan Hubinger Date: Sat, 23 Sep 2023 15:28:31 -0700 Subject: [PATCH 017/121] Fix jupyter console --- coconut/icoconut/root.py | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/coconut/icoconut/root.py b/coconut/icoconut/root.py index babd03616..f89935eb9 100644 --- a/coconut/icoconut/root.py +++ b/coconut/icoconut/root.py @@ -165,7 +165,11 @@ def _coconut_compile(self, source, *args, **kwargs): """Version of _compile that checks Coconut code. None means that the code should not be run as is. Any other value means that it can.""" - if not source.endswith("\n\n") and should_indent(source): + if source.replace(" ", "").endswith("\n\n"): + return True + elif should_indent(source): + return None + elif "\n" in source.rstrip(): return None else: return True From 0c1cada2981fd1897b4503ab9097485d7e7bb87a Mon Sep 17 00:00:00 2001 From: Evan Hubinger Date: Tue, 26 Sep 2023 19:49:08 -0700 Subject: [PATCH 018/121] Bump IPython --- coconut/constants.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/coconut/constants.py b/coconut/constants.py index f3cab0ac7..af3f4ce1b 100644 --- a/coconut/constants.py +++ b/coconut/constants.py @@ -987,7 +987,7 @@ def get_path_env_var(env_var, default): ("jupyter-console", "py37"): (6, 6), ("typing", "py<35"): (3, 10), ("typing_extensions", "py>=37"): (4, 7), - ("ipython", "py38"): (8,), + ("ipython", "py38"): (8, 15), ("ipykernel", "py38"): (6,), ("jedi", "py39"): (0, 19), ("pygments", "py>=39"): (2, 15), From 440334b3f8faf65c9702eb3efe81b20a9d2f61ce Mon Sep 17 00:00:00 2001 From: Evan Hubinger Date: Mon, 9 Oct 2023 18:59:42 -0700 Subject: [PATCH 019/121] Backport ExceptionGroup Resolves #789. --- DOCS.md | 1 + Makefile | 4 +++ coconut/compiler/header.py | 4 ++- coconut/compiler/util.py | 10 +++--- coconut/constants.py | 9 ++++- coconut/highlighter.py | 5 ++- coconut/root.py | 34 ++++++++++++++++--- .../tests/src/cocotest/agnostic/specific.coco | 4 ++- coconut/util.py | 14 +++----- 9 files changed, 58 insertions(+), 27 deletions(-) diff --git a/DOCS.md b/DOCS.md index cf7f5ac6e..e72cd01bf 100644 --- a/DOCS.md +++ b/DOCS.md @@ -90,6 +90,7 @@ The full list of optional dependencies is: - `watch`: enables use of the `--watch` flag. - `mypy`: enables use of the `--mypy` flag. - `backports`: installs libraries that backport newer Python features to older versions, which Coconut will automatically use instead of the standard library if the standard library is not available. Specifically: + - Installs [`exceptiongroup`](https://pypi.org/project/exceptiongroup/) to backport [`ExceptionGroup` and `BaseExceptionGroup`](https://docs.python.org/3/library/exceptions.html#ExceptionGroup). - Installs [`dataclasses`](https://pypi.org/project/dataclasses/) to backport [`dataclasses`](https://docs.python.org/3/library/dataclasses.html). - Installs [`typing`](https://pypi.org/project/typing/) to backport [`typing`](https://docs.python.org/3/library/typing.html) ([`typing_extensions`](https://pypi.org/project/typing-extensions/) is always installed for backporting individual `typing` objects). - Installs [`aenum`](https://pypi.org/project/aenum) to backport [`enum`](https://docs.python.org/3/library/enum.html). diff --git a/Makefile b/Makefile index bf2008bc1..5ff886a84 100644 --- a/Makefile +++ b/Makefile @@ -52,6 +52,10 @@ setup-pypy3: install: setup python -m pip install -e .[tests] +.PHONY: install-purepy +install-purepy: setup + python -m pip install --no-deps --upgrade -e . "pyparsing<3" + .PHONY: install-py2 install-py2: setup-py2 python2 -m pip install -e .[tests] diff --git a/coconut/compiler/header.py b/coconut/compiler/header.py index 409929e50..b4b68ee78 100644 --- a/coconut/compiler/header.py +++ b/coconut/compiler/header.py @@ -934,7 +934,9 @@ def getheader(which, use_hash, target, no_tco, strict, no_wrap): newline=True, ).format(**format_dict) - if target_info >= (3, 9): + if target_info >= (3, 11): + header += _get_root_header("311") + elif target_info >= (3, 9): header += _get_root_header("39") if target_info >= (3, 7): header += _get_root_header("37") diff --git a/coconut/compiler/util.py b/coconut/compiler/util.py index 5efb771d8..9db8bf782 100644 --- a/coconut/compiler/util.py +++ b/coconut/compiler/util.py @@ -214,7 +214,7 @@ def evaluate_tokens(tokens, **kwargs): class ComputationNode(object): """A single node in the computation graph.""" - __slots__ = ("action", "original", "loc", "tokens") + (("been_called",) if DEVELOP else ()) + __slots__ = ("action", "original", "loc", "tokens") pprinting = False def __new__(cls, action, original, loc, tokens, ignore_no_tokens=False, ignore_one_token=False, greedy=False, trim_arity=True): @@ -236,8 +236,6 @@ def __new__(cls, action, original, loc, tokens, ignore_no_tokens=False, ignore_o self.original = original self.loc = loc self.tokens = tokens - if DEVELOP: - self.been_called = False if greedy: return self.evaluate() else: @@ -253,9 +251,9 @@ def name(self): def evaluate(self): """Get the result of evaluating the computation graph at this node. Very performance sensitive.""" - if DEVELOP: # avoid the overhead of the call if not develop - internal_assert(not self.been_called, "inefficient reevaluation of action " + self.name + " with tokens", self.tokens) - self.been_called = True + # note that this should never cache, since if a greedy Wrap that doesn't add to the packrat context + # hits the cache, it'll get the same ComputationNode object, but since it's greedy that object needs + # to actually be reevaluated evaluated_toks = evaluate_tokens(self.tokens) if logger.tracing: # avoid the overhead of the call if not tracing logger.log_trace(self.name, self.original, self.loc, evaluated_toks, self.tokens) diff --git a/coconut/constants.py b/coconut/constants.py index af3f4ce1b..1965516a2 100644 --- a/coconut/constants.py +++ b/coconut/constants.py @@ -583,7 +583,9 @@ def get_path_env_var(env_var, default): '__file__', '__annotations__', '__debug__', - # # don't include builtins that aren't always made available by Coconut: + # we treat these as coconut_exceptions so the highlighter will always know about them: + # 'ExceptionGroup', 'BaseExceptionGroup', + # don't include builtins that aren't always made available by Coconut: # 'BlockingIOError', 'ChildProcessError', 'ConnectionError', # 'BrokenPipeError', 'ConnectionAbortedError', 'ConnectionRefusedError', # 'ConnectionResetError', 'FileExistsError', 'FileNotFoundError', @@ -807,8 +809,11 @@ def get_path_env_var(env_var, default): coconut_exceptions = ( "MatchError", + "ExceptionGroup", + "BaseExceptionGroup", ) +highlight_builtins = coconut_specific_builtins + interp_only_builtins all_builtins = frozenset(python_builtins + coconut_specific_builtins + coconut_exceptions) magic_methods = ( @@ -938,6 +943,7 @@ def get_path_env_var(env_var, default): ("dataclasses", "py==36"), ("typing", "py<35"), ("async_generator", "py35"), + ("exceptiongroup", "py37"), ), "dev": ( ("pre-commit", "py3"), @@ -994,6 +1000,7 @@ def get_path_env_var(env_var, default): ("xonsh", "py38"): (0, 14), ("pytest", "py36"): (7,), ("async_generator", "py35"): (1, 10), + ("exceptiongroup", "py37"): (1,), # pinned reqs: (must be added to pinned_reqs below) diff --git a/coconut/highlighter.py b/coconut/highlighter.py index aef74f588..a12686a06 100644 --- a/coconut/highlighter.py +++ b/coconut/highlighter.py @@ -25,8 +25,7 @@ from pygments.util import shebang_matches from coconut.constants import ( - coconut_specific_builtins, - interp_only_builtins, + highlight_builtins, new_operators, tabideal, default_encoding, @@ -94,7 +93,7 @@ class CoconutLexer(Python3Lexer): (words(reserved_vars, prefix=r"(?= 1 for develop -DEVELOP = 8 +DEVELOP = 9 ALPHA = False # for pre releases rather than post releases assert DEVELOP is False or DEVELOP >= 1, "DEVELOP must be False or an int >= 1" @@ -45,6 +45,16 @@ def _indent(code, by=1, tabsize=4, strip=False, newline=False, initial_newline=F ) + ("\n" if newline else "") +def _get_target_info(target): + """Return target information as a version tuple.""" + if not target or target == "universal": + return () + elif len(target) == 1: + return (int(target),) + else: + return (int(target[0]), int(target[1:])) + + # ----------------------------------------------------------------------------------------------------------------------- # HEADER: # ----------------------------------------------------------------------------------------------------------------------- @@ -264,15 +274,24 @@ def _coconut_reduce_partial(self): _coconut_copy_reg.pickle(_coconut_functools.partial, _coconut_reduce_partial) ''' +_py3_before_py311_extras = '''try: + from exceptiongroup import ExceptionGroup, BaseExceptionGroup +except ImportError: + class you_need_to_install_exceptiongroup(object): + __slots__ = () + ExceptionGroup = BaseExceptionGroup = you_need_to_install_exceptiongroup() +''' + # whenever new versions are added here, header.py must be updated to use them ROOT_HEADER_VERSIONS = ( "universal", "2", - "3", "27", + "3", "37", "39", + "311", ) @@ -284,6 +303,7 @@ def _get_root_header(version="universal"): ''' + _indent(_get_root_header("2")) + '''else: ''' + _indent(_get_root_header("3")) + version_info = _get_target_info(version) header = "" if version.startswith("3"): @@ -293,7 +313,7 @@ def _get_root_header(version="universal"): # if a new assignment is added below, a new builtins import should be added alongside it header += _base_py2_header - if version in ("37", "39"): + if version_info >= (3, 7): header += r'''py_breakpoint = breakpoint ''' elif version == "3": @@ -311,7 +331,7 @@ def _get_root_header(version="universal"): header += r'''if _coconut_sys.version_info < (3, 7): ''' + _indent(_below_py37_extras) + r'''elif _coconut_sys.version_info < (3, 9): ''' + _indent(_py37_py38_extras) - elif version == "37": + elif (3, 7) <= version_info < (3, 9): header += r'''if _coconut_sys.version_info < (3, 9): ''' + _indent(_py37_py38_extras) elif version.startswith("2"): @@ -320,7 +340,11 @@ def _get_root_header(version="universal"): dict.items = _coconut_OrderedDict.viewitems ''' else: - assert version == "39", version + assert version_info >= (3, 9), version + + if (3,) <= version_info < (3, 11): + header += r'''if _coconut_sys.version_info < (3, 11): +''' + _indent(_py3_before_py311_extras) return header diff --git a/coconut/tests/src/cocotest/agnostic/specific.coco b/coconut/tests/src/cocotest/agnostic/specific.coco index 1a3b8ba6f..cbb1eefbe 100644 --- a/coconut/tests/src/cocotest/agnostic/specific.coco +++ b/coconut/tests/src/cocotest/agnostic/specific.coco @@ -2,7 +2,7 @@ from io import StringIO if TYPE_CHECKING: from typing import Any -from .util import mod # NOQA +from .util import mod, assert_raises # NOQA def non_py26_test() -> bool: @@ -181,6 +181,8 @@ def py37_spec_test() -> bool: class HasVarGen[*Ts] # type: ignore assert HasVarGen `issubclass` object assert typing.Protocol.__module__ == "typing_extensions" + assert_raises((def -> raise ExceptionGroup("derp", [Exception("herp")])), ExceptionGroup) + assert_raises((def -> raise BaseExceptionGroup("derp", [BaseException("herp")])), BaseExceptionGroup) return True diff --git a/coconut/util.py b/coconut/util.py index 62e2fcfa0..128c4afd3 100644 --- a/coconut/util.py +++ b/coconut/util.py @@ -39,6 +39,7 @@ except ImportError: lru_cache = None +from coconut.root import _get_target_info from coconut.constants import ( fixpath, default_encoding, @@ -265,6 +266,9 @@ def ensure_dir(dirpath): # ----------------------------------------------------------------------------------------------------------------------- +get_target_info = _get_target_info + + def ver_tuple_to_str(req_ver): """Converts a requirement version tuple into a version string.""" return ".".join(str(x) for x in req_ver) @@ -287,16 +291,6 @@ def get_next_version(req_ver, point_to_increment=-1): return req_ver[:point_to_increment] + (req_ver[point_to_increment] + 1,) -def get_target_info(target): - """Return target information as a version tuple.""" - if not target: - return () - elif len(target) == 1: - return (int(target),) - else: - return (int(target[0]), int(target[1:])) - - def get_displayable_target(target): """Get a displayable version of the target.""" try: From b1a5f43e348fdeb8ff080b4137f385815c3835dc Mon Sep 17 00:00:00 2001 From: Evan Hubinger Date: Mon, 9 Oct 2023 23:20:30 -0700 Subject: [PATCH 020/121] Improve reqs, tests --- coconut/constants.py | 12 ++++++++---- coconut/tests/main_test.py | 8 ++++++++ coconut/tests/src/cocotest/agnostic/main.coco | 3 +++ .../tests/src/cocotest/target_311/py311_test.coco | 10 ++++++++++ 4 files changed, 29 insertions(+), 4 deletions(-) create mode 100644 coconut/tests/src/cocotest/target_311/py311_test.coco diff --git a/coconut/constants.py b/coconut/constants.py index 1965516a2..672703edc 100644 --- a/coconut/constants.py +++ b/coconut/constants.py @@ -904,7 +904,8 @@ def get_path_env_var(env_var, default): ("ipython", "py<3"), ("ipython", "py3;py<37"), ("ipython", "py==37"), - ("ipython", "py38"), + ("ipython", "py==38"), + ("ipython", "py>=39"), ("ipykernel", "py<3"), ("ipykernel", "py3;py<38"), ("ipykernel", "py38"), @@ -920,9 +921,10 @@ def get_path_env_var(env_var, default): ("jupyter-console", "py<35"), ("jupyter-console", "py>=35;py<37"), ("jupyter-console", "py37"), - ("jupyterlab", "py35"), - ("jupytext", "py3"), "papermill", + # these are fully optional, so no need to pull them in here + # ("jupyterlab", "py35"), + # ("jupytext", "py3"), ), "mypy": ( "mypy[python2]", @@ -993,7 +995,6 @@ def get_path_env_var(env_var, default): ("jupyter-console", "py37"): (6, 6), ("typing", "py<35"): (3, 10), ("typing_extensions", "py>=37"): (4, 7), - ("ipython", "py38"): (8, 15), ("ipykernel", "py38"): (6,), ("jedi", "py39"): (0, 19), ("pygments", "py>=39"): (2, 15), @@ -1001,9 +1002,12 @@ def get_path_env_var(env_var, default): ("pytest", "py36"): (7,), ("async_generator", "py35"): (1, 10), ("exceptiongroup", "py37"): (1,), + ("ipython", "py>=39"): (8, 15), # pinned reqs: (must be added to pinned_reqs below) + # don't upgrade these; they breaks on Python 3.8 + ("ipython", "py==38"): (8, 12), # don't upgrade these; they breaks on Python 3.7 ("ipython", "py==37"): (7, 34), # don't upgrade these; they breaks on Python 3.6 diff --git a/coconut/tests/main_test.py b/coconut/tests/main_test.py index 22a5f1733..bd094f15f 100644 --- a/coconut/tests/main_test.py +++ b/coconut/tests/main_test.py @@ -562,6 +562,11 @@ def comp_38(args=[], always_sys=False, **kwargs): comp(path="cocotest", folder="target_38", args=["--target", "38" if not always_sys else "sys"] + args, **kwargs) +def comp_311(args=[], always_sys=False, **kwargs): + """Compiles target_311.""" + comp(path="cocotest", folder="target_311", args=["--target", "311" if not always_sys else "sys"] + args, **kwargs) + + def comp_sys(args=[], **kwargs): """Compiles target_sys.""" comp(path="cocotest", folder="target_sys", args=["--target", "sys"] + args, **kwargs) @@ -605,6 +610,8 @@ def run(args=[], agnostic_target=None, use_run_arg=False, convert_to_import=Fals comp_36(args, **spec_kwargs) if sys.version_info >= (3, 8): comp_38(args, **spec_kwargs) + if sys.version_info >= (3, 11): + comp_311(args, **spec_kwargs) comp_agnostic(agnostic_args, **kwargs) comp_sys(args, **kwargs) @@ -646,6 +653,7 @@ def comp_all(args=[], agnostic_target=None, **kwargs): comp_35(args, **kwargs) comp_36(args, **kwargs) comp_38(args, **kwargs) + comp_311(args, **kwargs) comp_sys(args, **kwargs) comp_non_strict(args, **kwargs) diff --git a/coconut/tests/src/cocotest/agnostic/main.coco b/coconut/tests/src/cocotest/agnostic/main.coco index 2e5402122..b6bdbfa59 100644 --- a/coconut/tests/src/cocotest/agnostic/main.coco +++ b/coconut/tests/src/cocotest/agnostic/main.coco @@ -104,6 +104,9 @@ def run_main(outer_MatchError, test_easter_eggs=False) -> bool: if sys.version_info >= (3, 8): from .py38_test import py38_test assert py38_test() is True + if sys.version_info >= (3, 11): + from .py311_test import py311_test + assert py311_test() is True print_dot() # ....... from .target_sys_test import TEST_ASYNCIO, target_sys_test diff --git a/coconut/tests/src/cocotest/target_311/py311_test.coco b/coconut/tests/src/cocotest/target_311/py311_test.coco new file mode 100644 index 000000000..a2c655815 --- /dev/null +++ b/coconut/tests/src/cocotest/target_311/py311_test.coco @@ -0,0 +1,10 @@ +def py311_test() -> bool: + """Performs Python-3.11-specific tests.""" + multi_err = ExceptionGroup("herp", [ValueError("a"), ValueError("b")]) + got_err = None + try: + raise multi_err + except* ValueError as err: + got_err = err + assert repr(got_err) == repr(multi_err), (got_err, multi_err) + return True From 87a53250c2c45f62eede5cb94bc548f828ea1849 Mon Sep 17 00:00:00 2001 From: Evan Hubinger Date: Tue, 10 Oct 2023 13:40:27 -0700 Subject: [PATCH 021/121] Improve reqs, tests --- .github/workflows/run-tests.yml | 2 +- coconut/constants.py | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/run-tests.yml b/.github/workflows/run-tests.yml index 2927a0edb..ad6f69ca3 100644 --- a/.github/workflows/run-tests.yml +++ b/.github/workflows/run-tests.yml @@ -25,7 +25,7 @@ jobs: steps: - uses: actions/checkout@v3 - name: Setup python - uses: MatteoH2O1999/setup-python@v1.3.1 + uses: MatteoH2O1999/setup-python@v2 with: python-version: ${{ matrix.python-version }} cache: pip diff --git a/coconut/constants.py b/coconut/constants.py index 672703edc..49c8cfdd2 100644 --- a/coconut/constants.py +++ b/coconut/constants.py @@ -945,7 +945,7 @@ def get_path_env_var(env_var, default): ("dataclasses", "py==36"), ("typing", "py<35"), ("async_generator", "py35"), - ("exceptiongroup", "py37"), + ("exceptiongroup", "py37;py<311"), ), "dev": ( ("pre-commit", "py3"), From 021ad3443236ceef43bb73fe57a134c63d6b387c Mon Sep 17 00:00:00 2001 From: Evan Hubinger Date: Tue, 10 Oct 2023 20:38:04 -0700 Subject: [PATCH 022/121] Fix reqs --- coconut/constants.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/coconut/constants.py b/coconut/constants.py index 49c8cfdd2..b0f4c539e 100644 --- a/coconut/constants.py +++ b/coconut/constants.py @@ -1001,7 +1001,7 @@ def get_path_env_var(env_var, default): ("xonsh", "py38"): (0, 14), ("pytest", "py36"): (7,), ("async_generator", "py35"): (1, 10), - ("exceptiongroup", "py37"): (1,), + ("exceptiongroup", "py37;py<311"): (1,), ("ipython", "py>=39"): (8, 15), # pinned reqs: (must be added to pinned_reqs below) From e819bed3b35bf9bd716f2c97813e4a87d9a246c2 Mon Sep 17 00:00:00 2001 From: Evan Hubinger Date: Wed, 11 Oct 2023 00:10:03 -0700 Subject: [PATCH 023/121] Bump reqs --- coconut/constants.py | 9 +++++---- 1 file changed, 5 insertions(+), 4 deletions(-) diff --git a/coconut/constants.py b/coconut/constants.py index b0f4c539e..f86d133db 100644 --- a/coconut/constants.py +++ b/coconut/constants.py @@ -988,16 +988,16 @@ def get_path_env_var(env_var, default): ("numpy", "py<3;cpy"): (1,), ("dataclasses", "py==36"): (0, 8), ("aenum", "py<34"): (3, 1, 15), - "pydata-sphinx-theme": (0, 13), + "pydata-sphinx-theme": (0, 14), "myst-parser": (2,), "sphinx": (7,), - "mypy[python2]": (1, 4), + "mypy[python2]": (1, 6), ("jupyter-console", "py37"): (6, 6), ("typing", "py<35"): (3, 10), - ("typing_extensions", "py>=37"): (4, 7), + ("typing_extensions", "py>=37"): (4, 8), ("ipykernel", "py38"): (6,), ("jedi", "py39"): (0, 19), - ("pygments", "py>=39"): (2, 15), + ("pygments", "py>=39"): (2, 16), ("xonsh", "py38"): (0, 14), ("pytest", "py36"): (7,), ("async_generator", "py35"): (1, 10), @@ -1049,6 +1049,7 @@ def get_path_env_var(env_var, default): # should match the reqs with comments above pinned_reqs = ( + ("ipython", "py==38"), ("ipython", "py==37"), ("xonsh", "py>=36;py<38"), ("pandas", "py36"), From 595487ac21135d2d9cfd70a420a114f345f883fb Mon Sep 17 00:00:00 2001 From: Evan Hubinger Date: Wed, 11 Oct 2023 01:05:34 -0700 Subject: [PATCH 024/121] Fix py37 --- coconut/constants.py | 44 ++++++--------------------------- coconut/requirements.py | 4 +-- coconut/tests/constants_test.py | 4 +-- 3 files changed, 12 insertions(+), 40 deletions(-) diff --git a/coconut/constants.py b/coconut/constants.py index f86d133db..d89a6723e 100644 --- a/coconut/constants.py +++ b/coconut/constants.py @@ -892,7 +892,8 @@ def get_path_env_var(env_var, default): ("pygments", "py>=39"), ("typing_extensions", "py<36"), ("typing_extensions", "py==36"), - ("typing_extensions", "py>=37"), + ("typing_extensions", "py==37"), + ("typing_extensions", "py>=38"), ), "cpython": ( "cPyparsing", @@ -972,7 +973,7 @@ def get_path_env_var(env_var, default): } # min versions are inclusive -min_versions = { +unpinned_min_versions = { "cPyparsing": (2, 4, 7, 2, 2, 3), ("pre-commit", "py3"): (3,), ("psutil", "py>=27"): (5,), @@ -994,7 +995,7 @@ def get_path_env_var(env_var, default): "mypy[python2]": (1, 6), ("jupyter-console", "py37"): (6, 6), ("typing", "py<35"): (3, 10), - ("typing_extensions", "py>=37"): (4, 8), + ("typing_extensions", "py>=38"): (4, 8), ("ipykernel", "py38"): (6,), ("jedi", "py39"): (0, 19), ("pygments", "py>=39"): (2, 16), @@ -1003,13 +1004,14 @@ def get_path_env_var(env_var, default): ("async_generator", "py35"): (1, 10), ("exceptiongroup", "py37;py<311"): (1,), ("ipython", "py>=39"): (8, 15), +} - # pinned reqs: (must be added to pinned_reqs below) - +pinned_min_versions = { # don't upgrade these; they breaks on Python 3.8 ("ipython", "py==38"): (8, 12), # don't upgrade these; they breaks on Python 3.7 ("ipython", "py==37"): (7, 34), + ("typing_extensions", "py==37"): (4, 7), # don't upgrade these; they breaks on Python 3.6 ("xonsh", "py>=36;py<38"): (0, 11), ("pandas", "py36"): (1,), @@ -1047,37 +1049,7 @@ def get_path_env_var(env_var, default): "pyparsing": (2, 4, 7), } -# should match the reqs with comments above -pinned_reqs = ( - ("ipython", "py==38"), - ("ipython", "py==37"), - ("xonsh", "py>=36;py<38"), - ("pandas", "py36"), - ("jupyter-client", "py36"), - ("typing_extensions", "py==36"), - ("jupyter-client", "py<35"), - ("ipykernel", "py3;py<38"), - ("ipython", "py3;py<37"), - ("jupyter-console", "py>=35;py<37"), - ("jupyter-client", "py==35"), - ("jupytext", "py3"), - ("jupyterlab", "py35"), - ("xonsh", "py<36"), - ("typing_extensions", "py<36"), - ("prompt_toolkit", "py>=3"), - ("pytest", "py<36"), - "vprof", - ("pygments", "py<39"), - ("pywinpty", "py<3;windows"), - ("jupyter-console", "py<35"), - ("ipython", "py<3"), - ("ipykernel", "py<3"), - ("prompt_toolkit", "py<3"), - "watchdog", - "papermill", - ("jedi", "py<39"), - "pyparsing", -) +min_versions = pinned_min_versions | unpinned_min_versions # max versions are exclusive; None implies that the max version should # be generated by incrementing the min version; multiple Nones implies diff --git a/coconut/requirements.py b/coconut/requirements.py index cac2b82d2..55c293471 100644 --- a/coconut/requirements.py +++ b/coconut/requirements.py @@ -33,7 +33,7 @@ all_reqs, min_versions, max_versions, - pinned_reqs, + pinned_min_versions, requests_sleep_times, embed_on_internal_exc, ) @@ -342,7 +342,7 @@ def print_new_versions(strict=False): + " = " + ver_tuple_to_str(min_versions[req]) + " -> " + ", ".join(new_versions + ["(" + v + ")" for v in same_versions]) ) - if req in pinned_reqs: + if req in pinned_min_versions: pinned_updates.append(update_str) elif new_versions: new_updates.append(update_str) diff --git a/coconut/tests/constants_test.py b/coconut/tests/constants_test.py index 2df0da3ba..65ae8beea 100644 --- a/coconut/tests/constants_test.py +++ b/coconut/tests/constants_test.py @@ -108,8 +108,8 @@ def test_imports(self): assert is_importable(old_imp), "Failed to import " + old_imp def test_reqs(self): - assert set(constants.pinned_reqs) <= set(constants.min_versions), "found old pinned requirement" - assert set(constants.max_versions) <= set(constants.pinned_reqs) | set(("cPyparsing",)), "found unlisted constrained but unpinned requirements" + assert not set(constants.unpinned_min_versions) & set(constants.pinned_min_versions), "found pinned and unpinned requirements" + assert set(constants.max_versions) <= set(constants.pinned_min_versions) | set(("cPyparsing",)), "found unlisted constrained but unpinned requirements" for maxed_ver in constants.max_versions: assert isinstance(maxed_ver, tuple) or maxed_ver in ("pyparsing", "cPyparsing"), "maxed versions must be tagged to a specific Python version" From 1967d14bde71903a01d0b0db42b194794b7e20bd Mon Sep 17 00:00:00 2001 From: Evan Hubinger Date: Wed, 11 Oct 2023 20:55:48 -0700 Subject: [PATCH 025/121] Fix py<=38 --- coconut/constants.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/coconut/constants.py b/coconut/constants.py index d89a6723e..db58f1531 100644 --- a/coconut/constants.py +++ b/coconut/constants.py @@ -1049,7 +1049,9 @@ def get_path_env_var(env_var, default): "pyparsing": (2, 4, 7), } -min_versions = pinned_min_versions | unpinned_min_versions +min_versions = {} +min_versions.update(pinned_min_versions) +min_versions.update(unpinned_min_versions) # max versions are exclusive; None implies that the max version should # be generated by incrementing the min version; multiple Nones implies From 40ee5589e816369407e88a3d5048dda665d3a0dc Mon Sep 17 00:00:00 2001 From: Evan Hubinger Date: Thu, 12 Oct 2023 21:54:16 -0700 Subject: [PATCH 026/121] Fix py37, docs --- DOCS.md | 2 +- coconut/constants.py | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/DOCS.md b/DOCS.md index e72cd01bf..f7c33cf7e 100644 --- a/DOCS.md +++ b/DOCS.md @@ -1586,7 +1586,7 @@ This is especially true when using [`trio`](https://github.com/python-trio/trio) Since this pattern can often be quite syntactically cumbersome, Coconut provides the shortcut syntax ``` -async with for aclosing(my_generator()) as values: +async with for value in aclosing(my_generator()): ... ``` which compiles to exactly the pattern above. diff --git a/coconut/constants.py b/coconut/constants.py index db58f1531..f765fe26b 100644 --- a/coconut/constants.py +++ b/coconut/constants.py @@ -89,7 +89,7 @@ def get_path_env_var(env_var, default): and sys.version_info[:2] != (3, 7) ) MYPY = ( - PY37 + PY38 and not WINDOWS and not PYPY ) From f0b4a60e8dbed4308e76f803387b1a1a79ff9f15 Mon Sep 17 00:00:00 2001 From: Evan Hubinger Date: Fri, 13 Oct 2023 00:44:08 -0700 Subject: [PATCH 027/121] Fix mypy errors --- __coconut__/__init__.pyi | 138 +++++++++++++++++++-------------------- 1 file changed, 69 insertions(+), 69 deletions(-) diff --git a/__coconut__/__init__.pyi b/__coconut__/__init__.pyi index c75480c00..636f7e37b 100644 --- a/__coconut__/__init__.pyi +++ b/__coconut__/__init__.pyi @@ -271,30 +271,30 @@ def call( _y: _U, _z: _V, ) -> _W: ... -@_t.overload -def call( - _func: _t.Callable[_t.Concatenate[_T, _P], _U], - _x: _T, - *args: _t.Any, - **kwargs: _t.Any, -) -> _U: ... -@_t.overload -def call( - _func: _t.Callable[_t.Concatenate[_T, _U, _P], _V], - _x: _T, - _y: _U, - *args: _t.Any, - **kwargs: _t.Any, -) -> _V: ... -@_t.overload -def call( - _func: _t.Callable[_t.Concatenate[_T, _U, _V, _P], _W], - _x: _T, - _y: _U, - _z: _V, - *args: _t.Any, - **kwargs: _t.Any, -) -> _W: ... +# @_t.overload +# def call( +# _func: _t.Callable[_t.Concatenate[_T, _P], _U], +# _x: _T, +# *args: _t.Any, +# **kwargs: _t.Any, +# ) -> _U: ... +# @_t.overload +# def call( +# _func: _t.Callable[_t.Concatenate[_T, _U, _P], _V], +# _x: _T, +# _y: _U, +# *args: _t.Any, +# **kwargs: _t.Any, +# ) -> _V: ... +# @_t.overload +# def call( +# _func: _t.Callable[_t.Concatenate[_T, _U, _V, _P], _W], +# _x: _T, +# _y: _U, +# _z: _V, +# *args: _t.Any, +# **kwargs: _t.Any, +# ) -> _W: ... @_t.overload def call( _func: _t.Callable[..., _T], @@ -439,30 +439,30 @@ def safe_call( _y: _U, _z: _V, ) -> Expected[_W]: ... -@_t.overload -def safe_call( - _func: _t.Callable[_t.Concatenate[_T, _P], _U], - _x: _T, - *args: _t.Any, - **kwargs: _t.Any, -) -> Expected[_U]: ... -@_t.overload -def safe_call( - _func: _t.Callable[_t.Concatenate[_T, _U, _P], _V], - _x: _T, - _y: _U, - *args: _t.Any, - **kwargs: _t.Any, -) -> Expected[_V]: ... -@_t.overload -def safe_call( - _func: _t.Callable[_t.Concatenate[_T, _U, _V, _P], _W], - _x: _T, - _y: _U, - _z: _V, - *args: _t.Any, - **kwargs: _t.Any, -) -> Expected[_W]: ... +# @_t.overload +# def safe_call( +# _func: _t.Callable[_t.Concatenate[_T, _P], _U], +# _x: _T, +# *args: _t.Any, +# **kwargs: _t.Any, +# ) -> Expected[_U]: ... +# @_t.overload +# def safe_call( +# _func: _t.Callable[_t.Concatenate[_T, _U, _P], _V], +# _x: _T, +# _y: _U, +# *args: _t.Any, +# **kwargs: _t.Any, +# ) -> Expected[_V]: ... +# @_t.overload +# def safe_call( +# _func: _t.Callable[_t.Concatenate[_T, _U, _V, _P], _W], +# _x: _T, +# _y: _U, +# _z: _V, +# *args: _t.Any, +# **kwargs: _t.Any, +# ) -> Expected[_W]: ... @_t.overload def safe_call( _func: _t.Callable[..., _T], @@ -501,27 +501,27 @@ def _coconut_call_or_coefficient( _y: _U, _z: _V, ) -> _W: ... -@_t.overload -def _coconut_call_or_coefficient( - _func: _t.Callable[_t.Concatenate[_T, _P], _U], - _x: _T, - *args: _t.Any, -) -> _U: ... -@_t.overload -def _coconut_call_or_coefficient( - _func: _t.Callable[_t.Concatenate[_T, _U, _P], _V], - _x: _T, - _y: _U, - *args: _t.Any, -) -> _V: ... -@_t.overload -def _coconut_call_or_coefficient( - _func: _t.Callable[_t.Concatenate[_T, _U, _V, _P], _W], - _x: _T, - _y: _U, - _z: _V, - *args: _t.Any, -) -> _W: ... +# @_t.overload +# def _coconut_call_or_coefficient( +# _func: _t.Callable[_t.Concatenate[_T, _P], _U], +# _x: _T, +# *args: _t.Any, +# ) -> _U: ... +# @_t.overload +# def _coconut_call_or_coefficient( +# _func: _t.Callable[_t.Concatenate[_T, _U, _P], _V], +# _x: _T, +# _y: _U, +# *args: _t.Any, +# ) -> _V: ... +# @_t.overload +# def _coconut_call_or_coefficient( +# _func: _t.Callable[_t.Concatenate[_T, _U, _V, _P], _W], +# _x: _T, +# _y: _U, +# _z: _V, +# *args: _t.Any, +# ) -> _W: ... @_t.overload def _coconut_call_or_coefficient( _func: _t.Callable[..., _T], From ef54243b09165d87cfa40807ce186687097df495 Mon Sep 17 00:00:00 2001 From: Evan Hubinger Date: Sat, 14 Oct 2023 17:59:56 -0700 Subject: [PATCH 028/121] Fix lots of tests --- coconut/constants.py | 16 +++++++++------- coconut/root.py | 2 +- coconut/tests/main_test.py | 4 ++-- coconut/tests/src/extras.coco | 2 +- 4 files changed, 13 insertions(+), 11 deletions(-) diff --git a/coconut/constants.py b/coconut/constants.py index f765fe26b..c2ad6eea1 100644 --- a/coconut/constants.py +++ b/coconut/constants.py @@ -961,8 +961,8 @@ def get_path_env_var(env_var, default): "pydata-sphinx-theme", ), "numpy": ( - ("numpy", "py34"), ("numpy", "py<3;cpy"), + ("numpy", "py34;py<39"), ("pandas", "py36"), ), "tests": ( @@ -985,8 +985,7 @@ def get_path_env_var(env_var, default): "pexpect": (4,), ("trollius", "py<3;cpy"): (2, 2), "requests": (2, 31), - ("numpy", "py34"): (1,), - ("numpy", "py<3;cpy"): (1,), + ("numpy", "py39"): (1, 26), ("dataclasses", "py==36"): (0, 8), ("aenum", "py<34"): (3, 1, 15), "pydata-sphinx-theme": (0, 14), @@ -1003,16 +1002,18 @@ def get_path_env_var(env_var, default): ("pytest", "py36"): (7,), ("async_generator", "py35"): (1, 10), ("exceptiongroup", "py37;py<311"): (1,), - ("ipython", "py>=39"): (8, 15), + ("ipython", "py>=39"): (8, 16), } pinned_min_versions = { - # don't upgrade these; they breaks on Python 3.8 + # don't upgrade these; they break on Python 3.9 + ("numpy", "py34;py<39"): (1, 18), + # don't upgrade these; they break on Python 3.8 ("ipython", "py==38"): (8, 12), - # don't upgrade these; they breaks on Python 3.7 + # don't upgrade these; they break on Python 3.7 ("ipython", "py==37"): (7, 34), ("typing_extensions", "py==37"): (4, 7), - # don't upgrade these; they breaks on Python 3.6 + # don't upgrade these; they break on Python 3.6 ("xonsh", "py>=36;py<38"): (0, 11), ("pandas", "py36"): (1,), ("jupyter-client", "py36"): (7, 1, 2), @@ -1043,6 +1044,7 @@ def get_path_env_var(env_var, default): ("prompt_toolkit", "py<3"): (1,), "watchdog": (0, 10), "papermill": (1, 2), + ("numpy", "py<3;cpy"): (1, 16), # don't upgrade this; it breaks with old IPython versions ("jedi", "py<39"): (0, 17), # Coconut requires pyparsing 2 diff --git a/coconut/root.py b/coconut/root.py index 3800eda59..3f47c749a 100644 --- a/coconut/root.py +++ b/coconut/root.py @@ -26,7 +26,7 @@ VERSION = "3.0.3" VERSION_NAME = None # False for release, int >= 1 for develop -DEVELOP = 9 +DEVELOP = 10 ALPHA = False # for pre releases rather than post releases assert DEVELOP is False or DEVELOP >= 1, "DEVELOP must be False or an int >= 1" diff --git a/coconut/tests/main_test.py b/coconut/tests/main_test.py index bd094f15f..edc424bc5 100644 --- a/coconut/tests/main_test.py +++ b/coconut/tests/main_test.py @@ -89,8 +89,8 @@ # ----------------------------------------------------------------------------------------------------------------------- -default_recursion_limit = "6144" -default_stack_size = "6144" +default_recursion_limit = "7168" +default_stack_size = "7168" jupyter_timeout = 120 diff --git a/coconut/tests/src/extras.coco b/coconut/tests/src/extras.coco index fed27375d..4b08eb743 100644 --- a/coconut/tests/src/extras.coco +++ b/coconut/tests/src/extras.coco @@ -241,7 +241,7 @@ def f() = assert_raises(-> parse('"a" 10'), CoconutParseError, err_has=" \\~~~^") assert_raises(-> parse("A. ."), CoconutParseError, err_has=" \\~~^") assert_raises(-> parse('''f"""{ -}"""'''), CoconutSyntaxError, err_has=(" ~~~~|", "\n ^~~/")) +}"""'''), CoconutSyntaxError, err_has="parsing failed for format string expression") assert_raises(-> parse("f([] {})"), CoconutParseError, err_has=" \\~~~~^") assert_raises(-> parse("return = 1"), CoconutParseError, err_has='invalid use of the keyword "return"') From 5b51c4cbbabd3773895b749c55a06e2da9584736 Mon Sep 17 00:00:00 2001 From: Evan Hubinger Date: Sat, 14 Oct 2023 18:11:38 -0700 Subject: [PATCH 029/121] Update pre-commit --- .pre-commit-config.yaml | 22 +++++++++++----------- 1 file changed, 11 insertions(+), 11 deletions(-) diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index 8f79c7238..5764b616b 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -1,6 +1,16 @@ repos: +- repo: https://github.com/pre-commit/mirrors-autopep8 + rev: v2.0.4 + hooks: + - id: autopep8 + args: + - --in-place + - --aggressive + - --aggressive + - --experimental + - --ignore=W503,E501,E722,E402 - repo: https://github.com/pre-commit/pre-commit-hooks.git - rev: v4.4.0 + rev: v4.5.0 hooks: - id: check-added-large-files - id: fix-byte-order-marker @@ -29,13 +39,3 @@ repos: - id: flake8 args: - --ignore=W503,E501,E265,E402,F405,E305,E126 -- repo: https://github.com/pre-commit/mirrors-autopep8 - rev: v2.0.4 - hooks: - - id: autopep8 - args: - - --in-place - - --aggressive - - --aggressive - - --experimental - - --ignore=W503,E501,E722,E402 From c2165d0ad154ecf3397d35b379b17bbef7335aef Mon Sep 17 00:00:00 2001 From: Evan Hubinger Date: Sat, 14 Oct 2023 22:20:00 -0700 Subject: [PATCH 030/121] Fix more tests --- coconut/command/command.py | 2 +- coconut/constants.py | 1 + coconut/tests/main_test.py | 2 +- 3 files changed, 3 insertions(+), 2 deletions(-) diff --git a/coconut/command/command.py b/coconut/command/command.py index dcdae0b12..9548f1ed3 100644 --- a/coconut/command/command.py +++ b/coconut/command/command.py @@ -266,7 +266,7 @@ def execute_args(self, args, interact=True, original_args=None): if args.site_install and args.site_uninstall: raise CoconutException("cannot --site-install and --site-uninstall simultaneously") if args.incremental and not SUPPORTS_INCREMENTAL: - raise CoconutException("--incremental mode requires cPyparsing (run '{python} -m pip install --upgrade cPyparsing' to fix)".format(python=sys.executable)) + raise CoconutException("--incremental mode not supported in current environment (try '{python} -m pip install --upgrade cPyparsing' to fix)".format(python=sys.executable)) for and_args in getattr(args, "and") or []: if len(and_args) > 2: raise CoconutException( diff --git a/coconut/constants.py b/coconut/constants.py index c2ad6eea1..dc142f320 100644 --- a/coconut/constants.py +++ b/coconut/constants.py @@ -963,6 +963,7 @@ def get_path_env_var(env_var, default): "numpy": ( ("numpy", "py<3;cpy"), ("numpy", "py34;py<39"), + ("numpy", "py39"), ("pandas", "py36"), ), "tests": ( diff --git a/coconut/tests/main_test.py b/coconut/tests/main_test.py index edc424bc5..690bca953 100644 --- a/coconut/tests/main_test.py +++ b/coconut/tests/main_test.py @@ -940,7 +940,7 @@ def test_run_arg(self): def test_jobs_zero(self): run(["--jobs", "0"]) - if not PYPY: + if not PYPY and PY38: def test_incremental(self): run(["--incremental"]) # includes "Error" because exceptions include the whole file From b8665d60622227c028d2b7bb2a9b10959bd0c77f Mon Sep 17 00:00:00 2001 From: Evan Hubinger Date: Mon, 16 Oct 2023 19:26:14 -0700 Subject: [PATCH 031/121] Increase stack --- coconut/tests/main_test.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/coconut/tests/main_test.py b/coconut/tests/main_test.py index 690bca953..2b01c8196 100644 --- a/coconut/tests/main_test.py +++ b/coconut/tests/main_test.py @@ -89,8 +89,8 @@ # ----------------------------------------------------------------------------------------------------------------------- -default_recursion_limit = "7168" -default_stack_size = "7168" +default_recursion_limit = "8192" +default_stack_size = "8192" jupyter_timeout = 120 From 29a834516c2cf93838dd4c8a275104e8fad70046 Mon Sep 17 00:00:00 2001 From: Evan Hubinger Date: Sat, 21 Oct 2023 22:42:45 -0700 Subject: [PATCH 032/121] 3.12 prep --- .github/workflows/run-tests.yml | 1 + coconut/compiler/compiler.py | 9 +++------ coconut/compiler/grammar.py | 1 + coconut/root.py | 2 +- coconut/tests/main_test.py | 4 ++-- coconut/tests/src/cocotest/agnostic/primary.coco | 3 ++- 6 files changed, 10 insertions(+), 10 deletions(-) diff --git a/.github/workflows/run-tests.yml b/.github/workflows/run-tests.yml index ad6f69ca3..900d71c89 100644 --- a/.github/workflows/run-tests.yml +++ b/.github/workflows/run-tests.yml @@ -15,6 +15,7 @@ jobs: - '3.9' - '3.10' - '3.11' + - '3.12' - 'pypy-2.7' - 'pypy-3.6' - 'pypy-3.7' diff --git a/coconut/compiler/compiler.py b/coconut/compiler/compiler.py index fa7f611d9..74ba9feea 100644 --- a/coconut/compiler/compiler.py +++ b/coconut/compiler/compiler.py @@ -2355,10 +2355,7 @@ def proc_funcdef(self, original, loc, decorators, funcdef, is_async, in_method, # modify function definition to use def_name if def_name != func_name: - def_stmt_pre_lparen, def_stmt_post_lparen = def_stmt.split("(", 1) - def_stmt_def, def_stmt_name = def_stmt_pre_lparen.rsplit(" ", 1) - def_stmt_name = def_stmt_name.replace(func_name, def_name) - def_stmt = def_stmt_def + " " + def_stmt_name + "(" + def_stmt_post_lparen + def_stmt = compile_regex(r"\b" + re.escape(func_name) + r"\b").sub(def_name, def_stmt) # detect generators is_gen = self.detect_is_gen(raw_lines) @@ -3985,8 +3982,8 @@ def type_param_handle(self, original, loc, tokens): kwargs = "" if bound_op is not None: self.internal_assert(bound_op_type in ("bound", "constraint"), original, loc, "invalid type_param bound_op", bound_op) - # # uncomment this line whenever mypy adds support for infer_variance in TypeVar - # # (and remove the warning about it in the DOCS) + # uncomment this line whenever mypy adds support for infer_variance in TypeVar + # (and remove the warning about it in the DOCS) # kwargs = ", infer_variance=True" if bound_op == "<=": self.strict_err_or_warn( diff --git a/coconut/compiler/grammar.py b/coconut/compiler/grammar.py index 48e22713d..3f0220c45 100644 --- a/coconut/compiler/grammar.py +++ b/coconut/compiler/grammar.py @@ -2495,6 +2495,7 @@ class Grammar(object): start_marker - keyword("def").suppress() - unsafe_dotted_name + - Optional(brackets).suppress() - lparen.suppress() - parameters_tokens - rparen.suppress() ) diff --git a/coconut/root.py b/coconut/root.py index 3f47c749a..972a7a864 100644 --- a/coconut/root.py +++ b/coconut/root.py @@ -26,7 +26,7 @@ VERSION = "3.0.3" VERSION_NAME = None # False for release, int >= 1 for develop -DEVELOP = 10 +DEVELOP = 11 ALPHA = False # for pre releases rather than post releases assert DEVELOP is False or DEVELOP >= 1, "DEVELOP must be False or an int >= 1" diff --git a/coconut/tests/main_test.py b/coconut/tests/main_test.py index 2b01c8196..485596103 100644 --- a/coconut/tests/main_test.py +++ b/coconut/tests/main_test.py @@ -89,8 +89,8 @@ # ----------------------------------------------------------------------------------------------------------------------- -default_recursion_limit = "8192" -default_stack_size = "8192" +default_recursion_limit = "6144" +default_stack_size = "6144" jupyter_timeout = 120 diff --git a/coconut/tests/src/cocotest/agnostic/primary.coco b/coconut/tests/src/cocotest/agnostic/primary.coco index 20218a28b..7cc794c2d 100644 --- a/coconut/tests/src/cocotest/agnostic/primary.coco +++ b/coconut/tests/src/cocotest/agnostic/primary.coco @@ -572,8 +572,9 @@ def primary_test() -> bool: a = A() f = 10 def a.f(x) = x # type: ignore + def a.a(x) = x # type: ignore assert f == 10 - assert a.f 1 == 1 + assert a.f 1 == 1 == a.a 1 def f(x, y) = (x, y) # type: ignore assert f 1 2 == (1, 2) def f(0) = 'a' # type: ignore From f34759ccd54580ba75d3bed6bca427da11d259e4 Mon Sep 17 00:00:00 2001 From: Starwort Date: Sun, 22 Oct 2023 21:28:12 +0100 Subject: [PATCH 033/121] Fix typo in pure-Python example for scan() --- DOCS.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/DOCS.md b/DOCS.md index 9cf16df75..bfd659f6b 100644 --- a/DOCS.md +++ b/DOCS.md @@ -3839,7 +3839,7 @@ max_so_far = input_data[0] for x in input_data: if x > max_so_far: max_so_far = x - running_max.append(x) + running_max.append(max_so_far) ``` #### `count` From 66e5254a321cfe1a09c8d7fe0f16d0bc28c6e275 Mon Sep 17 00:00:00 2001 From: Evan Hubinger Date: Fri, 27 Oct 2023 22:19:45 -0700 Subject: [PATCH 034/121] Rename process/thread maps Resolves #792. --- DOCS.md | 45 +++++++------ __coconut__/__init__.pyi | 2 +- coconut/compiler/header.py | 41 +++++++----- coconut/compiler/templates/header.py_template | 45 +++++++------ coconut/constants.py | 4 +- coconut/root.py | 2 +- coconut/tests/main_test.py | 13 ++-- .../tests/src/cocotest/agnostic/primary.coco | 67 +++++++++++++------ .../tests/src/cocotest/agnostic/suite.coco | 10 +-- coconut/tests/src/cocotest/agnostic/util.coco | 2 +- .../cocotest/non_strict/non_strict_test.coco | 1 + .../cocotest/target_sys/target_sys_test.coco | 10 +-- 12 files changed, 141 insertions(+), 101 deletions(-) diff --git a/DOCS.md b/DOCS.md index d3b205a60..23f94814d 100644 --- a/DOCS.md +++ b/DOCS.md @@ -3506,11 +3506,12 @@ depth: 1 Coconut's `map`, `zip`, `filter`, `reversed`, and `enumerate` objects are enhanced versions of their Python equivalents that support: +- The ability to be iterated over multiple times if the underlying iterators can be iterated over multiple times. + - _Note: This can lead to different behavior between Coconut built-ins and Python built-ins. Use `py_` versions if the Python behavior is necessary._ - `reversed` - `repr` - Optimized normal (and iterator) indexing/slicing (`map`, `zip`, `reversed`, and `enumerate` but not `filter`). - `len` (all but `filter`) (though `bool` will still always yield `True`). -- The ability to be iterated over multiple times if the underlying iterators can be iterated over multiple times. - [PEP 618](https://www.python.org/dev/peps/pep-0618) `zip(..., strict=True)` support on all Python versions. - Added `strict=True` support to `map` as well (enforces that iterables are the same length in the multi-iterable case; uses `zip` under the hood such that errors will show up as `zip(..., strict=True)` errors). - Added attributes which subclasses can make use of to get at the original arguments to the object: @@ -3848,9 +3849,9 @@ for x in input_data: **count**(_start_=`0`, _step_=`1`) -Coconut provides a modified version of `itertools.count` that supports `in`, normal slicing, optimized iterator slicing, the standard `count` and `index` sequence methods, `repr`, and `start`/`step` attributes as a built-in under the name `count`. +Coconut provides a modified version of `itertools.count` that supports `in`, normal slicing, optimized iterator slicing, the standard `count` and `index` sequence methods, `repr`, and `start`/`step` attributes as a built-in under the name `count`. If the _step_ parameter is set to `None`, `count` will behave like `itertools.repeat` instead. -Additionally, if the _step_ parameter is set to `None`, `count` will behave like `itertools.repeat` instead. +Since `count` supports slicing, `count()` can be used as a version of `range` that can in some cases be more readable. In particular, it is easy to accidentally write `range(10, 2)` when you meant `range(0, 10, 2)`, but it is hard to accidentally write `count()[10:2]` when you mean `count()[:10:2]`. ##### Python Docs @@ -4120,33 +4121,35 @@ all_equal([1, 1, 1]) all_equal([1, 1, 2]) ``` -#### `parallel_map` +#### `process_map` -**parallel\_map**(_function_, *_iterables_, *, _chunksize_=`1`, _strict_=`False`) +**process\_map**(_function_, *_iterables_, *, _chunksize_=`1`, _strict_=`False`) -Coconut provides a parallel version of `map` under the name `parallel_map`. `parallel_map` makes use of multiple processes, and is therefore much faster than `map` for CPU-bound tasks. `parallel_map` never loads the entire input iterator into memory, though it does consume the entire input iterator as soon as a single output is requested. If any exceptions are raised inside of `parallel_map`, a traceback will be printed as soon as they are encountered. +Coconut provides a `multiprocessing`-based version of `map` under the name `process_map`. `process_map` makes use of multiple processes, and is therefore much faster than `map` for CPU-bound tasks. `process_map` never loads the entire input iterator into memory, though it does consume the entire input iterator as soon as a single output is requested. If any exceptions are raised inside of `process_map`, a traceback will be printed as soon as they are encountered. -Because `parallel_map` uses multiple processes for its execution, it is necessary that all of its arguments be pickleable. Only objects defined at the module level, and not lambdas, objects defined inside of a function, or objects defined inside of the interpreter, are pickleable. Furthermore, on Windows, it is necessary that all calls to `parallel_map` occur inside of an `if __name__ == "__main__"` guard. +Because `process_map` uses multiple processes for its execution, it is necessary that all of its arguments be pickleable. Only objects defined at the module level, and not lambdas, objects defined inside of a function, or objects defined inside of the interpreter, are pickleable. Furthermore, on Windows, it is necessary that all calls to `process_map` occur inside of an `if __name__ == "__main__"` guard. -`parallel_map` supports a `chunksize` argument, which determines how many items are passed to each process at a time. Larger values of _chunksize_ are recommended when dealing with very long iterables. Additionally, in the multi-iterable case, _strict_ can be set to `True` to ensure that all iterables are the same length. +`process_map` supports a `chunksize` argument, which determines how many items are passed to each process at a time. Larger values of _chunksize_ are recommended when dealing with very long iterables. Additionally, in the multi-iterable case, _strict_ can be set to `True` to ensure that all iterables are the same length. -If multiple sequential calls to `parallel_map` need to be made, it is highly recommended that they be done inside of a `with parallel_map.multiple_sequential_calls():` block, which will cause the different calls to use the same process pool and result in `parallel_map` immediately returning a list rather than a `parallel_map` object. If multiple sequential calls are necessary and the laziness of parallel_map is required, then the `parallel_map` objects should be constructed before the `multiple_sequential_calls` block and then only iterated over once inside the block. +If multiple sequential calls to `process_map` need to be made, it is highly recommended that they be done inside of a `with process_map.multiple_sequential_calls():` block, which will cause the different calls to use the same process pool and result in `process_map` immediately returning a list rather than a `process_map` object. If multiple sequential calls are necessary and the laziness of process_map is required, then the `process_map` objects should be constructed before the `multiple_sequential_calls` block and then only iterated over once inside the block. -`parallel_map.multiple_sequential_calls` also supports a `max_workers` argument to set the number of processes. +`process_map.multiple_sequential_calls` also supports a `max_workers` argument to set the number of processes. + +_Deprecated: `parallel_map` is available as a deprecated alias for `process_map`. Note that deprecated features are disabled in `--strict` mode._ ##### Python Docs -**parallel_map**(_func, \*iterables_, _chunksize_=`1`) +**process_map**(_func, \*iterables_, _chunksize_=`1`) Equivalent to `map(func, *iterables)` except _func_ is executed asynchronously and several calls to _func_ may be made concurrently. If a call raises an exception, then that exception will be raised when its value is retrieved from the iterator. -`parallel_map` chops the iterable into a number of chunks which it submits to the process pool as separate tasks. The (approximate) size of these chunks can be specified by setting _chunksize_ to a positive integer. For very long iterables using a large value for _chunksize_ can make the job complete **much** faster than using the default value of `1`. +`process_map` chops the iterable into a number of chunks which it submits to the process pool as separate tasks. The (approximate) size of these chunks can be specified by setting _chunksize_ to a positive integer. For very long iterables using a large value for _chunksize_ can make the job complete **much** faster than using the default value of `1`. ##### Example **Coconut:** ```coconut -parallel_map(pow$(2), range(100)) |> list |> print +process_map(pow$(2), range(100)) |> list |> print ``` **Python:** @@ -4157,25 +4160,27 @@ with Pool() as pool: print(list(pool.imap(functools.partial(pow, 2), range(100)))) ``` -#### `concurrent_map` +#### `thread_map` + +**thread\_map**(_function_, *_iterables_, *, _chunksize_=`1`, _strict_=`False`) -**concurrent\_map**(_function_, *_iterables_, *, _chunksize_=`1`, _strict_=`False`) +Coconut provides a `multithreading`-based version of [`process_map`](#process_map) under the name `thread_map`. `thread_map` behaves identically to `process_map` (including support for `thread_map.multiple_sequential_calls`) except that it uses multithreading instead of multiprocessing, and is therefore primarily useful only for IO-bound tasks due to CPython's Global Interpreter Lock. -Coconut provides a concurrent version of [`parallel_map`](#parallel_map) under the name `concurrent_map`. `concurrent_map` behaves identically to `parallel_map` (including support for `concurrent_map.multiple_sequential_calls`) except that it uses multithreading instead of multiprocessing, and is therefore primarily useful only for IO-bound tasks due to CPython's Global Interpreter Lock. +_Deprecated: `concurrent_map` is available as a deprecated alias for `thread_map`. Note that deprecated features are disabled in `--strict` mode._ ##### Python Docs -**concurrent_map**(_func, \*iterables_, _chunksize_=`1`) +**thread_map**(_func, \*iterables_, _chunksize_=`1`) Equivalent to `map(func, *iterables)` except _func_ is executed asynchronously and several calls to _func_ may be made concurrently. If a call raises an exception, then that exception will be raised when its value is retrieved from the iterator. -`concurrent_map` chops the iterable into a number of chunks which it submits to the thread pool as separate tasks. The (approximate) size of these chunks can be specified by setting _chunksize_ to a positive integer. For very long iterables using a large value for _chunksize_ can make the job complete **much** faster than using the default value of `1`. +`thread_map` chops the iterable into a number of chunks which it submits to the thread pool as separate tasks. The (approximate) size of these chunks can be specified by setting _chunksize_ to a positive integer. For very long iterables using a large value for _chunksize_ can make the job complete **much** faster than using the default value of `1`. ##### Example **Coconut:** ```coconut -concurrent_map(get_data_for_user, get_all_users()) |> list |> print +thread_map(get_data_for_user, get_all_users()) |> list |> print ``` **Python:** @@ -4546,5 +4551,5 @@ All Coconut built-ins are accessible from `coconut.__coconut__`. The recommended ##### Example ```coconut_python -from coconut.__coconut__ import parallel_map +from coconut.__coconut__ import process_map ``` diff --git a/__coconut__/__init__.pyi b/__coconut__/__init__.pyi index 636f7e37b..3388e597f 100644 --- a/__coconut__/__init__.pyi +++ b/__coconut__/__init__.pyi @@ -215,7 +215,7 @@ _coconut_cartesian_product = cartesian_product _coconut_multiset = multiset -parallel_map = concurrent_map = _coconut_map = map +process_map = thread_map = parallel_map = concurrent_map = _coconut_map = map TYPE_CHECKING = _t.TYPE_CHECKING diff --git a/coconut/compiler/header.py b/coconut/compiler/header.py index b4b68ee78..79c5dc093 100644 --- a/coconut/compiler/header.py +++ b/coconut/compiler/header.py @@ -300,31 +300,36 @@ def process_header_args(which, use_hash, target, no_tco, strict, no_wrap): ), # disabled mocks must have different docstrings so the # interpreter can tell them apart from the real thing - def_prepattern=( - r'''def prepattern(base_func, **kwargs): + def_aliases=prepare( + r''' +def prepattern(base_func, **kwargs): """DEPRECATED: use addpattern instead.""" def pattern_prepender(func): return addpattern(func, base_func, **kwargs) - return pattern_prepender''' - if not strict else - r'''def prepattern(*args, **kwargs): - """Deprecated Coconut built-in 'prepattern' disabled by --strict compilation; use 'addpattern' instead.""" - raise _coconut.NameError("deprecated Coconut built-in 'prepattern' disabled by --strict compilation; use 'addpattern' instead")''' - ), - def_datamaker=( - r'''def datamaker(data_type): + return pattern_prepender +def datamaker(data_type): """DEPRECATED: use makedata instead.""" - return _coconut.functools.partial(makedata, data_type)''' + return _coconut.functools.partial(makedata, data_type) +of, parallel_map, concurrent_map = call, process_map, thread_map + ''' if not strict else - r'''def datamaker(*args, **kwargs): + r''' +def prepattern(*args, **kwargs): + """Deprecated Coconut built-in 'prepattern' disabled by --strict compilation; use 'addpattern' instead.""" + raise _coconut.NameError("deprecated Coconut built-in 'prepattern' disabled by --strict compilation; use 'addpattern' instead") +def datamaker(*args, **kwargs): """Deprecated Coconut built-in 'datamaker' disabled by --strict compilation; use 'makedata' instead.""" - raise _coconut.NameError("deprecated Coconut built-in 'datamaker' disabled by --strict compilation; use 'makedata' instead")''' - ), - of_is_call=( - "of = call" if not strict else - r'''def of(*args, **kwargs): + raise _coconut.NameError("deprecated Coconut built-in 'datamaker' disabled by --strict compilation; use 'makedata' instead") +def of(*args, **kwargs): """Deprecated Coconut built-in 'of' disabled by --strict compilation; use 'call' instead.""" - raise _coconut.NameError("deprecated Coconut built-in 'of' disabled by --strict compilation; use 'call' instead")''' + raise _coconut.NameError("deprecated Coconut built-in 'of' disabled by --strict compilation; use 'call' instead") +def parallel_map(*args, **kwargs): + """Deprecated Coconut built-in 'parallel_map' disabled by --strict compilation; use 'process_map' instead.""" + raise _coconut.NameError("deprecated Coconut built-in 'parallel_map' disabled by --strict compilation; use 'process_map' instead") +def concurrent_map(*args, **kwargs): + """Deprecated Coconut built-in 'concurrent_map' disabled by --strict compilation; use 'thread_map' instead.""" + raise _coconut.NameError("deprecated Coconut built-in 'concurrent_map' disabled by --strict compilation; use 'thread_map' instead") + ''' ), return_method_of_self=pycondition( (3,), diff --git a/coconut/compiler/templates/header.py_template b/coconut/compiler/templates/header.py_template index 0f8e4e58c..0e346b2b3 100644 --- a/coconut/compiler/templates/header.py_template +++ b/coconut/compiler/templates/header.py_template @@ -836,7 +836,7 @@ class map(_coconut_baseclass, _coconut.map): return _coconut.iter(_coconut.map(self.func, *self.iters)) def __fmap__(self, func): return self.__class__(_coconut_forward_compose(self.func, func), *self.iters) -class _coconut_parallel_concurrent_map_func_wrapper(_coconut_baseclass): +class _coconut_parallel_map_func_wrapper(_coconut_baseclass): __slots__ = ("map_cls", "func", "star") def __init__(self, map_cls, func, star): self.map_cls = map_cls @@ -848,7 +848,7 @@ class _coconut_parallel_concurrent_map_func_wrapper(_coconut_baseclass): self.map_cls.get_pool_stack().append(None) try: if self.star: - assert _coconut.len(args) == 1, "internal parallel/concurrent map error {report_this_text}" + assert _coconut.len(args) == 1, "internal process/thread map error {report_this_text}" return self.func(*args[0], **kwargs) else: return self.func(*args, **kwargs) @@ -857,14 +857,14 @@ class _coconut_parallel_concurrent_map_func_wrapper(_coconut_baseclass): _coconut.traceback.print_exc() raise finally: - assert self.map_cls.get_pool_stack().pop() is None, "internal parallel/concurrent map error {report_this_text}" -class _coconut_base_parallel_concurrent_map(map): + assert self.map_cls.get_pool_stack().pop() is None, "internal process/thread map error {report_this_text}" +class _coconut_base_parallel_map(map): __slots__ = ("result", "chunksize", "strict") @classmethod def get_pool_stack(cls): return cls.threadlocal_ns.__dict__.setdefault("pool_stack", [None]) def __new__(cls, function, *iterables, **kwargs): - self = _coconut.super(_coconut_base_parallel_concurrent_map, cls).__new__(cls, function, *iterables) + self = _coconut.super(_coconut_base_parallel_map, cls).__new__(cls, function, *iterables) self.result = None self.chunksize = kwargs.pop("chunksize", 1) self.strict = kwargs.pop("strict", False) @@ -876,35 +876,38 @@ class _coconut_base_parallel_concurrent_map(map): @classmethod @_coconut.contextlib.contextmanager def multiple_sequential_calls(cls, max_workers=None): - """Context manager that causes nested calls to use the same pool.""" + """Context manager that causes nested calls to use the same pool. + Yields True if this is at the top-level otherwise False.""" if cls.get_pool_stack()[-1] is None: cls.get_pool_stack()[-1] = cls.make_pool(max_workers) try: - yield + yield True finally: cls.get_pool_stack()[-1].terminate() cls.get_pool_stack()[-1] = None else: - yield + yield False + def execute_map_method(self, map_method="imap"): + if _coconut.len(self.iters) == 1: + return _coconut.getattr(self.get_pool_stack()[-1], map_method)(_coconut_parallel_map_func_wrapper(self.__class__, self.func, False), self.iters[0], self.chunksize) + elif self.strict: + return _coconut.getattr(self.get_pool_stack()[-1], map_method)(_coconut_parallel_map_func_wrapper(self.__class__, self.func, True), {_coconut_}zip(*self.iters, strict=True), self.chunksize) + else: + return _coconut.getattr(self.get_pool_stack()[-1], map_method)(_coconut_parallel_map_func_wrapper(self.__class__, self.func, True), _coconut.zip(*self.iters), self.chunksize) def get_list(self): if self.result is None: with self.multiple_sequential_calls(): - if _coconut.len(self.iters) == 1: - self.result = _coconut.list(self.get_pool_stack()[-1].imap(_coconut_parallel_concurrent_map_func_wrapper(self.__class__, self.func, False), self.iters[0], self.chunksize)) - elif self.strict: - self.result = _coconut.list(self.get_pool_stack()[-1].imap(_coconut_parallel_concurrent_map_func_wrapper(self.__class__, self.func, True), {_coconut_}zip(*self.iters, strict=True), self.chunksize)) - else: - self.result = _coconut.list(self.get_pool_stack()[-1].imap(_coconut_parallel_concurrent_map_func_wrapper(self.__class__, self.func, True), _coconut.zip(*self.iters), self.chunksize)) + self.result = _coconut.tuple(self.execute_map_method()) self.func = {_coconut_}ident self.iters = (self.result,) return self.result def __iter__(self): - return _coconut.iter(self.get_list()) -class parallel_map(_coconut_base_parallel_concurrent_map): + return _coconut.iter(self.get_list()){COMMENT.have_to_get_list_so_finishes_before_return_else_cant_manage_context} +class process_map(_coconut_base_parallel_map): """Multi-process implementation of map. Requires arguments to be pickleable. For multiple sequential calls, use: - with parallel_map.multiple_sequential_calls(): + with process_map.multiple_sequential_calls(): ... """ __slots__ = () @@ -912,11 +915,11 @@ class parallel_map(_coconut_base_parallel_concurrent_map): @staticmethod def make_pool(max_workers=None): return _coconut.multiprocessing.Pool(max_workers) -class concurrent_map(_coconut_base_parallel_concurrent_map): +class thread_map(_coconut_base_parallel_map): """Multi-thread implementation of map. For multiple sequential calls, use: - with concurrent_map.multiple_sequential_calls(): + with thread_map.multiple_sequential_calls(): ... """ __slots__ = () @@ -1413,7 +1416,6 @@ def addpattern(base_func, *add_funcs, **kwargs): return _coconut_base_pattern_func(base_func, *add_funcs) return _coconut.functools.partial(_coconut_base_pattern_func, base_func) _coconut_addpattern = addpattern -{def_prepattern} class _coconut_partial(_coconut_base_callable): __slots__ = ("func", "_argdict", "_arglen", "_pos_kwargs", "_stargs", "keywords") def __init__(self, _coconut_func, _coconut_argdict, _coconut_arglen, _coconut_pos_kwargs, *args, **kwargs): @@ -1562,7 +1564,6 @@ def makedata(data_type, *args, **kwargs): if kwargs: raise _coconut.TypeError("makedata() got unexpected keyword arguments " + _coconut.repr(kwargs)) return _coconut_base_makedata(data_type, args, fallback_to_init=fallback_to_init) -{def_datamaker} {class_amap} def fmap(func, obj, **kwargs): """fmap(func, obj) creates a copy of obj with func applied to its contents. @@ -1676,7 +1677,6 @@ def call(_coconut_f{comma_slash}, *args, **kwargs): def call(f, /, *args, **kwargs) = f(*args, **kwargs). """ return _coconut_f(*args, **kwargs) -{of_is_call} def safe_call(_coconut_f{comma_slash}, *args, **kwargs): """safe_call is a version of call that catches any Exceptions and returns an Expected containing either the result or the error. @@ -2104,5 +2104,6 @@ class _coconut_SupportsInv(_coconut.typing.Protocol): """ def __invert__(self): raise NotImplementedError("Protocol methods cannot be called at runtime ((~) in a typing context is a Protocol)") +{def_aliases} _coconut_self_match_types = {self_match_types} _coconut_Expected, _coconut_MatchError, _coconut_cartesian_product, _coconut_count, _coconut_cycle, _coconut_enumerate, _coconut_flatten, _coconut_filter, _coconut_groupsof, _coconut_ident, _coconut_lift, _coconut_map, _coconut_multiset, _coconut_range, _coconut_reiterable, _coconut_reversed, _coconut_scan, _coconut_starmap, _coconut_tee, _coconut_windowsof, _coconut_zip, _coconut_zip_longest, TYPE_CHECKING, reduce, takewhile, dropwhile = Expected, MatchError, cartesian_product, count, cycle, enumerate, flatten, filter, groupsof, ident, lift, map, multiset, range, reiterable, reversed, scan, starmap, tee, windowsof, zip, zip_longest, False, _coconut.functools.reduce, _coconut.itertools.takewhile, _coconut.itertools.dropwhile{COMMENT.anything_added_here_should_be_copied_to_stub_file} diff --git a/coconut/constants.py b/coconut/constants.py index dc142f320..d58704ab5 100644 --- a/coconut/constants.py +++ b/coconut/constants.py @@ -748,10 +748,10 @@ def get_path_env_var(env_var, default): "count", "makedata", "consume", - "parallel_map", + "process_map", + "thread_map", "addpattern", "recursive_iterator", - "concurrent_map", "fmap", "starmap", "reiterable", diff --git a/coconut/root.py b/coconut/root.py index 972a7a864..2753a0a28 100644 --- a/coconut/root.py +++ b/coconut/root.py @@ -26,7 +26,7 @@ VERSION = "3.0.3" VERSION_NAME = None # False for release, int >= 1 for develop -DEVELOP = 11 +DEVELOP = 12 ALPHA = False # for pre releases rather than post releases assert DEVELOP is False or DEVELOP >= 1, "DEVELOP must be False or an int >= 1" diff --git a/coconut/tests/main_test.py b/coconut/tests/main_test.py index 485596103..fc4cff555 100644 --- a/coconut/tests/main_test.py +++ b/coconut/tests/main_test.py @@ -615,7 +615,6 @@ def run(args=[], agnostic_target=None, use_run_arg=False, convert_to_import=Fals comp_agnostic(agnostic_args, **kwargs) comp_sys(args, **kwargs) - comp_non_strict(args, **kwargs) if use_run_arg: _kwargs = kwargs.copy() @@ -635,6 +634,9 @@ def run(args=[], agnostic_target=None, use_run_arg=False, convert_to_import=Fals comp_extras(agnostic_args, **kwargs) run_extras(convert_to_import=convert_to_import) # **kwargs are for comp, not run + # do non-strict at the end so we get the non-strict header + comp_non_strict(args, **kwargs) + def comp_all(args=[], agnostic_target=None, **kwargs): """Compile Coconut tests.""" @@ -648,6 +650,10 @@ def comp_all(args=[], agnostic_target=None, **kwargs): except Exception: pass + comp_agnostic(agnostic_args, **kwargs) + comp_runner(agnostic_args, **kwargs) + comp_extras(agnostic_args, **kwargs) + comp_2(args, **kwargs) comp_3(args, **kwargs) comp_35(args, **kwargs) @@ -655,12 +661,9 @@ def comp_all(args=[], agnostic_target=None, **kwargs): comp_38(args, **kwargs) comp_311(args, **kwargs) comp_sys(args, **kwargs) + # do non-strict at the end so we get the non-strict header comp_non_strict(args, **kwargs) - comp_agnostic(agnostic_args, **kwargs) - comp_runner(agnostic_args, **kwargs) - comp_extras(agnostic_args, **kwargs) - def comp_pyston(args=[], **kwargs): """Compiles evhub/pyston.""" diff --git a/coconut/tests/src/cocotest/agnostic/primary.coco b/coconut/tests/src/cocotest/agnostic/primary.coco index 7cc794c2d..dad76b379 100644 --- a/coconut/tests/src/cocotest/agnostic/primary.coco +++ b/coconut/tests/src/cocotest/agnostic/primary.coco @@ -212,19 +212,19 @@ def primary_test() -> bool: assert map((-), range(5)).iters[0] |> tuple == range(5) |> tuple == zip(range(5), range(6)).iters[0] |> tuple # type: ignore assert repr(zip((0,1), (1,2))) == "zip((0, 1), (1, 2))" assert repr(map((-), range(5))).startswith("map(") # type: ignore - assert repr(parallel_map((-), range(5))).startswith("parallel_map(") # type: ignore - assert parallel_map((-), range(5)) |> tuple == (0, -1, -2, -3, -4) == parallel_map(map$((-)), (range(5),))$[0] |> tuple # type: ignore - assert parallel_map(zip, (range(2),), (range(2),)) |> map$(tuple) |> tuple == (((0,0), (1,1)),) # type: ignore - assert parallel_map(zip_longest$(fillvalue=10), (range(1),), (range(2),)) |> map$(tuple) |> tuple == (((0,0), (10,1)),) # type: ignore + assert repr(process_map((-), range(5))).startswith("process_map(") # type: ignore + assert process_map((-), range(5)) |> tuple == (0, -1, -2, -3, -4) == process_map(map$((-)), (range(5),))$[0] |> tuple # type: ignore + assert process_map(zip, (range(2),), (range(2),)) |> map$(tuple) |> tuple == (((0,0), (1,1)),) # type: ignore + assert process_map(zip_longest$(fillvalue=10), (range(1),), (range(2),)) |> map$(tuple) |> tuple == (((0,0), (10,1)),) # type: ignore assert (range(0, 5), range(5, 10)) |*> map$(+) |> tuple == (5, 7, 9, 11, 13) - assert parallel_map((*)$(2)..(+)$(1), range(5)) |> tuple == (2, 4, 6, 8, 10) - assert parallel_map((+), range(5), range(5), chunksize=2) |> list == map((*)$(2), range(5)) |> list == concurrent_map((+), range(5), range(5), chunksize=2) |> list # type: ignore - assert repr(concurrent_map((-), range(5))).startswith("concurrent_map(") # type: ignore - with concurrent_map.multiple_sequential_calls(max_workers=4): # type: ignore - assert concurrent_map((-), range(5)) |> tuple == (0, -1, -2, -3, -4) == concurrent_map(map$((-)), (range(5),))$[0] |> tuple # type: ignore - assert concurrent_map(zip, (range(2),), (range(2),)) |> map$(tuple) |> tuple == (((0,0), (1,1)),) # type: ignore + assert process_map((*)$(2)..(+)$(1), range(5)) |> tuple == (2, 4, 6, 8, 10) + assert process_map((+), range(5), range(5), chunksize=2) |> list == map((*)$(2), range(5)) |> list == thread_map((+), range(5), range(5), chunksize=2) |> list # type: ignore + assert repr(thread_map((-), range(5))).startswith("thread_map(") # type: ignore + with thread_map.multiple_sequential_calls(max_workers=4): # type: ignore + assert thread_map((-), range(5)) |> tuple == (0, -1, -2, -3, -4) == thread_map(map$((-)), (range(5),))$[0] |> tuple # type: ignore + assert thread_map(zip, (range(2),), (range(2),)) |> map$(tuple) |> tuple == (((0,0), (1,1)),) # type: ignore assert (range(0, 5), range(5, 10)) |*> map$(+) |> tuple == (5, 7, 9, 11, 13) - assert concurrent_map((*)$(2)..(+)$(1), range(5)) |> tuple == (2, 4, 6, 8, 10) + assert thread_map((*)$(2)..(+)$(1), range(5)) |> tuple == (2, 4, 6, 8, 10) assert 0 in range(1) assert range(1).count(0) == 1 assert 2 in range(5) @@ -320,7 +320,7 @@ def primary_test() -> bool: assert pow$(?, 2)(3) == 9 assert [] |> reduce$((+), ?, ()) == () assert pow$(?, 2) |> repr == "$(?, 2)" - assert parallel_map(pow$(?, 2), range(10)) |> tuple == (0, 1, 4, 9, 16, 25, 36, 49, 64, 81) + assert process_map(pow$(?, 2), range(10)) |> tuple == (0, 1, 4, 9, 16, 25, 36, 49, 64, 81) assert pow$(?, 2).args == (None, 2) assert range(20) |> filter$((x) -> x < 5) |> reversed |> tuple == (4,3,2,1,0) == (0,1,2,3,4,5,6,7,8,9) |> filter$((x) -> x < 5) |> reversed |> tuple # type: ignore assert (range(10) |> map$((x) -> x) |> reversed) `isinstance` map # type: ignore @@ -624,15 +624,15 @@ def primary_test() -> bool: it3 = iter(it2) item3 = next(it3) assert item3 != item2 - for map_func in (parallel_map, concurrent_map): + for map_func in (process_map, thread_map): m1 = map_func((+)$(1), range(5)) assert m1 `isinstance` map_func with map_func.multiple_sequential_calls(): # type: ignore m2 = map_func((+)$(1), range(5)) - assert m2 `isinstance` list + assert m2 `isinstance` tuple assert m1.result is None - assert m2 == [1, 2, 3, 4, 5] == list(m1) - assert m1.result == [1, 2, 3, 4, 5] == list(m1) + assert m2 == (1, 2, 3, 4, 5) == tuple(m1) + assert m1.result == (1, 2, 3, 4, 5) == tuple(m1) for it in ((), [], (||)): assert_raises(-> it$[0], IndexError) assert_raises(-> it$[-1], IndexError) @@ -1145,7 +1145,7 @@ def primary_test() -> bool: def __call__(self) = super().__call__() HasSuper assert HasSuper5.HasHasSuper.HasSuper()() == 10 == HasSuper6().get_HasSuper()()() - assert parallel_map((.+(10,)), [ + assert process_map((.+(10,)), [ (a=1, b=2), (x=3, y=4), ]) |> list == [(1, 2, 10), (3, 4, 10)] @@ -1409,8 +1409,8 @@ def primary_test() -> bool: assert flatten([[[1,2]], [[3], [4]]], 2) |> list == [1, 2, 3, 4] assert flatten([[[1,2]], [[3], [4]]], 2) |> reversed |> list == [4, 3, 2, 1] assert_raises(-> map((+), range(3), range(4), strict=True) |> list, ValueError) - assert map((+), range(3), range(4)$[:-1], strict=True) |> list == [0, 2, 4] == parallel_map((+), range(3), range(4)$[:-1], strict=True) |> list - assert range(3) |> map$((.+1), strict=True) |> list == [1, 2, 3] == range(3) |> parallel_map$((.+1), strict=True) |> list + assert map((+), range(3), range(4)$[:-1], strict=True) |> list == [0, 2, 4] == process_map((+), range(3), range(4)$[:-1], strict=True) |> list + assert range(3) |> map$((.+1), strict=True) |> list == [1, 2, 3] == range(3) |> process_map$((.+1), strict=True) |> list assert cartesian_product((1, 2), (3, 4), repeat=0) |> list == [()] assert (a=1, b=2)[1] == 2 obj = object() @@ -1418,9 +1418,9 @@ def primary_test() -> bool: hardref = map((.+1), [1,2,3]) assert weakref.ref(hardref)() |> list == [2, 3, 4] my_match_err = MatchError("my match error", 123) - assert parallel_map(ident, [my_match_err]) |> list |> str == str([my_match_err]) + assert process_map(ident, [my_match_err]) |> list |> str == str([my_match_err]) # repeat the same thing again now that my_match_err.str has been called - assert parallel_map(ident, [my_match_err]) |> list |> str == str([my_match_err]) + assert process_map(ident, [my_match_err]) |> list |> str == str([my_match_err]) match data tuple(1, 2) in (1, 2, 3): assert False data TestDefaultMatching(x="x default", y="y default") @@ -1659,4 +1659,29 @@ def primary_test() -> bool: g = f$(?, 1) has_partial = HasPartial() assert has_partial.g() == (has_partial, 1) + xs = zip([1, 2], [3, 4]) + py_xs = py_zip([1, 2], [3, 4]) + assert list(xs) == [(1, 3), (2, 4)] == list(xs) + assert list(py_xs) == [(1, 3), (2, 4)] + assert list(py_xs) == [] + xs = map((+), [1, 2], [3, 4]) + py_xs = py_map((+), [1, 2], [3, 4]) + assert list(xs) == [4, 6] == list(xs) + assert list(py_xs) == [4, 6] + assert list(py_xs) == [] + for xs in [ + zip((x for x in range(5)), (x for x in range(10))), + py_zip((x for x in range(5)), (x for x in range(10))), + map((,), (x for x in range(5)), (x for x in range(10))), + py_map((,), (x for x in range(5)), (x for x in range(10))), + ]: + assert list(xs) == list(zip(range(5), range(5))) + assert list(xs) == [] + xs = map((.+1), range(5)) + py_xs = py_map((.+1), range(5)) + assert list(xs) == list(range(1, 6)) == list(xs) + assert list(py_xs) == list(range(1, 6)) + assert list(py_xs) == [] + assert count()[:10:2] == range(0, 10, 2) + assert count()[10:2] == range(10, 2) return True diff --git a/coconut/tests/src/cocotest/agnostic/suite.coco b/coconut/tests/src/cocotest/agnostic/suite.coco index 89db9b0a4..a13d6c538 100644 --- a/coconut/tests/src/cocotest/agnostic/suite.coco +++ b/coconut/tests/src/cocotest/agnostic/suite.coco @@ -44,12 +44,12 @@ def suite_test() -> bool: def test_sqplus1_plus1sq(sqplus1, plus1sq, parallel=True): assert sqplus1(3) == 10 == (plus1..square)(3), sqplus1 if parallel: - assert parallel_map(sqplus1, range(3)) |> tuple == (1, 2, 5), sqplus1 # type: ignore + assert process_map(sqplus1, range(3)) |> tuple == (1, 2, 5), sqplus1 # type: ignore assert 3 `plus1sq` == 16, plus1sq assert 3 `sqplus1` == 10, sqplus1 test_sqplus1_plus1sq(sqplus1_1, plus1sq_1) test_sqplus1_plus1sq(sqplus1_2, plus1sq_2, parallel=False) - with parallel_map.multiple_sequential_calls(max_workers=2): # type: ignore + with process_map.multiple_sequential_calls(max_workers=2): # type: ignore test_sqplus1_plus1sq(sqplus1_3, plus1sq_3) test_sqplus1_plus1sq(sqplus1_4, plus1sq_4) test_sqplus1_plus1sq(sqplus1_5, plus1sq_5) @@ -67,7 +67,7 @@ def suite_test() -> bool: to_sort = rand_list(10) assert to_sort |> qsort |> tuple == to_sort |> sorted |> tuple, qsort # type: ignore to_sort = rand_list(10) - assert parallel_map(tuple <.. (|>)$(to_sort), qsorts) |> list == [to_sort |> sorted |> tuple] * len(qsorts) + assert process_map(tuple <.. (|>)$(to_sort), qsorts) |> list == [to_sort |> sorted |> tuple] * len(qsorts) assert repeat(3)$[2] == 3 == repeat_(3)$[2] assert sum_(repeat(1)$[:5]) == 5 == sum_(repeat_(1)$[:5]) assert (sum_(takewhile((x)-> x<5, N())) @@ -279,7 +279,7 @@ def suite_test() -> bool: assert fibs()$[:10] |> list == [1,1,2,3,5,8,13,21,34,55] == fibs_()$[:10] |> list assert fibs() |> takewhile$((i) -> i < 4000000 ) |> filter$((i) -> i % 2 == 0 ) |> sum == 4613732 == fibs_() |> takewhile$((i) -> i < 4000000 ) |> filter$((i) -> i % 2 == 0 ) |> sum # type: ignore assert loop([1,2])$[:4] |> list == [1, 2] * 2 - assert parallel_map(list .. .$[:2] .. loop, ([1], [2]))$[:2] |> tuple == ([1, 1], [2, 2]) + assert process_map(list .. .$[:2] .. loop, ([1], [2]))$[:2] |> tuple == ([1, 1], [2, 2]) assert nest("a") |> .$[1] |> .$[1] |> .$[0] == "a" assert (def -> mod)()(5, 3) == 2 assert sieve((2, 3, 4, 5)) |> list == [2, 3, 5] @@ -748,7 +748,7 @@ def suite_test() -> bool: class inh_A() `isinstance` clsA `isinstance` object = inh_A() for maxdiff in (maxdiff1, maxdiff2, maxdiff3, maxdiff_): assert maxdiff([7,1,4,5]) == 4, "failed for " + repr(maxdiff) - assert all(r == 4 for r in parallel_map(call$(?, [7,1,4,5]), [maxdiff1, maxdiff2, maxdiff3])) + assert all(r == 4 for r in process_map(call$(?, [7,1,4,5]), [maxdiff1, maxdiff2, maxdiff3])) assert ret_ret_func(1) == ((), {"func": 1}) == ret_args_kwargs$(func=1)() # type: ignore assert ret_args_kwargs$(?, func=2)(1) == ((1,), {"func": 2}) assert lift(ret_args_kwargs)(ident, plus1, times2, sq=square)(3) == ((3, 4, 6), {"sq": 9}) diff --git a/coconut/tests/src/cocotest/agnostic/util.coco b/coconut/tests/src/cocotest/agnostic/util.coco index c2c1c8553..09c3430fe 100644 --- a/coconut/tests/src/cocotest/agnostic/util.coco +++ b/coconut/tests/src/cocotest/agnostic/util.coco @@ -924,7 +924,7 @@ def grid_map(func, gridsample): def parallel_grid_map(func, gridsample): """Map a function over every point in a grid in parallel.""" - return gridsample |> parallel_map$(parallel_map$(func)) + return gridsample |> process_map$(process_map$(func)) def grid_trim(gridsample, xmax, ymax): """Convert a grid to a list of lists up to xmax and ymax.""" diff --git a/coconut/tests/src/cocotest/non_strict/non_strict_test.coco b/coconut/tests/src/cocotest/non_strict/non_strict_test.coco index 5550ee1f5..195230ede 100644 --- a/coconut/tests/src/cocotest/non_strict/non_strict_test.coco +++ b/coconut/tests/src/cocotest/non_strict/non_strict_test.coco @@ -89,6 +89,7 @@ def non_strict_test() -> bool: assert f"a" r"b" fr"c" rf"d" == "abcd" assert "a" fr"b" == "ab" == "a" rf"b" assert f"{f"{f"infinite"}"}" + " " + f"{f"nesting!!!"}" == "infinite nesting!!!" + assert parallel_map((.+1), range(5)) |> tuple == tuple(range(1, 6)) return True if __name__ == "__main__": diff --git a/coconut/tests/src/cocotest/target_sys/target_sys_test.coco b/coconut/tests/src/cocotest/target_sys/target_sys_test.coco index 012c4a6eb..03320c62d 100644 --- a/coconut/tests/src/cocotest/target_sys/target_sys_test.coco +++ b/coconut/tests/src/cocotest/target_sys/target_sys_test.coco @@ -47,11 +47,11 @@ def asyncio_test() -> bool: def toa(f) = async def (*args, **kwargs) -> f(*args, **kwargs) async def async_map_0(args): - return parallel_map(args[0], *args[1:]) - async def async_map_1(args) = parallel_map(args[0], *args[1:]) - async def async_map_2([func] + iters) = parallel_map(func, *iters) - async match def async_map_3([func] + iters) = parallel_map(func, *iters) - match async def async_map_4([func] + iters) = parallel_map(func, *iters) + return process_map(args[0], *args[1:]) + async def async_map_1(args) = process_map(args[0], *args[1:]) + async def async_map_2([func] + iters) = process_map(func, *iters) + async match def async_map_3([func] + iters) = process_map(func, *iters) + match async def async_map_4([func] + iters) = process_map(func, *iters) async def async_map_test() = for async_map in (async_map_0, async_map_1, async_map_2, async_map_3, async_map_4): assert (await ((pow$(2), range(5)) |> async_map)) |> tuple == (1, 2, 4, 8, 16) From 586dd5e2ef92f4e61d6c57e173183beb662231d0 Mon Sep 17 00:00:00 2001 From: Evan Hubinger Date: Sat, 28 Oct 2023 00:47:25 -0700 Subject: [PATCH 035/121] Add mapreduce, improve collectby, process/thread maps Resolves #793. --- DOCS.md | 45 ++++---- __coconut__/__init__.pyi | 56 +++++++++- coconut/compiler/templates/header.py_template | 102 +++++++++++------- coconut/constants.py | 1 + coconut/root.py | 2 +- .../tests/src/cocotest/agnostic/primary.coco | 19 ++-- 6 files changed, 151 insertions(+), 74 deletions(-) diff --git a/DOCS.md b/DOCS.md index 23f94814d..77f1d6ac2 100644 --- a/DOCS.md +++ b/DOCS.md @@ -4042,37 +4042,24 @@ assert "12345" |> windowsof$(3) |> list == [("1", "2", "3"), ("2", "3", "4"), (" **Python:** _Can't be done without the definition of `windowsof`; see the compiled header for the full definition._ -#### `collectby` +#### `collectby` and `mapreduce` -**collectby**(_key\_func_, _iterable_, _value\_func_=`None`, _reduce\_func_=`None`) +**collectby**(_key\_func_, _iterable_, _value\_func_=`None`, \*, _reduce\_func_=`None`, _map\_using_=`None`) `collectby(key_func, iterable)` collects the items in `iterable` into a dictionary of lists keyed by `key_func(item)`. -If `value_func` is passed, `collectby(key_func, iterable, value_func=value_func)` instead collects value_func(item) into each list instead of item. +If `value_func` is passed, `collectby(key_func, iterable, value_func=value_func)` instead collects `value_func(item)` into each list instead of `item`. -If `reduce_func` is passed, `collectby(key_func, iterable, reduce_func=reduce_func)`, instead of collecting the items into lists, reduces over the items of each key with reduce_func, effectively implementing a MapReduce operation. +If `reduce_func` is passed, `collectby(key_func, iterable, reduce_func=reduce_func)`, instead of collecting the items into lists, reduces over the items of each key with `reduce_func`, effectively implementing a MapReduce operation. -`collectby` is effectively equivalent to: -```coconut_python -from collections import defaultdict - -def collectby(key_func, iterable, value_func=ident, reduce_func=None): - collection = defaultdict(list) if reduce_func is None else {} - for item in iterable: - key = key_func(item) - value = value_func(item) - if reduce_func is None: - collection[key].append(value) - else: - old_value = collection.get(key, sentinel) - if old_value is not sentinel: - value = reduce_func(old_value, value) - collection[key] = value - return collection -``` +If `map_using` is passed, calculate `key_func` and `value_func` by mapping them over the iterable using `map_using` as `map`. Useful with [`process_map`](#process_map)/[`thread_map`](#thread_map). To immediately start calling `reduce_func` as soon as results arrive, pass `map_using=process_map$(stream=True)` (though note that `stream=True` requires the use of `process_map.multiple_sequential_calls`). `collectby` is similar to [`itertools.groupby`](https://docs.python.org/3/library/itertools.html#itertools.groupby) except that `collectby` aggregates common elements regardless of their order in the input iterable, whereas `groupby` only aggregates common elements that are adjacent in the input iterable. +**mapreduce**(_key\_value\_func_, _iterable_, \*, _reduce\_func_=`None`, _map\_using_=`None`) + +`mapreduce(key_value_func, iterable)` functions the same as `collectby`, but allows calculating the keys and values together in one function. _key\_value\_func_ must return a 2-tuple of `(key, value)`. + ##### Example **Coconut:** @@ -4123,20 +4110,24 @@ all_equal([1, 1, 2]) #### `process_map` -**process\_map**(_function_, *_iterables_, *, _chunksize_=`1`, _strict_=`False`) +**process\_map**(_function_, *_iterables_, *, _chunksize_=`1`, _strict_=`False`, _stream_=`False`, _ordered_=`True`) + +Coconut provides a `multiprocessing`-based version of `map` under the name `process_map`. `process_map` makes use of multiple processes, and is therefore much faster than `map` for CPU-bound tasks. If any exceptions are raised inside of `process_map`, a traceback will be printed as soon as they are encountered. Results will be in the same order as the input unless _ordered_=`False`. -Coconut provides a `multiprocessing`-based version of `map` under the name `process_map`. `process_map` makes use of multiple processes, and is therefore much faster than `map` for CPU-bound tasks. `process_map` never loads the entire input iterator into memory, though it does consume the entire input iterator as soon as a single output is requested. If any exceptions are raised inside of `process_map`, a traceback will be printed as soon as they are encountered. +`process_map` never loads the entire input iterator into memory, though by default it does consume the entire input iterator as soon as a single output is requested. Results can be streamed one at a time when iterating by passing _stream_=`True`, however note that _stream_=`True` requires that the resulting iterator only be iterated over inside of a `process_map.multiple_sequential_calls` block (see below). Because `process_map` uses multiple processes for its execution, it is necessary that all of its arguments be pickleable. Only objects defined at the module level, and not lambdas, objects defined inside of a function, or objects defined inside of the interpreter, are pickleable. Furthermore, on Windows, it is necessary that all calls to `process_map` occur inside of an `if __name__ == "__main__"` guard. `process_map` supports a `chunksize` argument, which determines how many items are passed to each process at a time. Larger values of _chunksize_ are recommended when dealing with very long iterables. Additionally, in the multi-iterable case, _strict_ can be set to `True` to ensure that all iterables are the same length. +_Deprecated: `parallel_map` is available as a deprecated alias for `process_map`. Note that deprecated features are disabled in `--strict` mode._ + +**process\_map\.multiple\_sequential\_calls**(_max\_workers_=`None`) + If multiple sequential calls to `process_map` need to be made, it is highly recommended that they be done inside of a `with process_map.multiple_sequential_calls():` block, which will cause the different calls to use the same process pool and result in `process_map` immediately returning a list rather than a `process_map` object. If multiple sequential calls are necessary and the laziness of process_map is required, then the `process_map` objects should be constructed before the `multiple_sequential_calls` block and then only iterated over once inside the block. `process_map.multiple_sequential_calls` also supports a `max_workers` argument to set the number of processes. -_Deprecated: `parallel_map` is available as a deprecated alias for `process_map`. Note that deprecated features are disabled in `--strict` mode._ - ##### Python Docs **process_map**(_func, \*iterables_, _chunksize_=`1`) @@ -4162,7 +4153,7 @@ with Pool() as pool: #### `thread_map` -**thread\_map**(_function_, *_iterables_, *, _chunksize_=`1`, _strict_=`False`) +**thread\_map**(_function_, *_iterables_, *, _chunksize_=`1`, _strict_=`False`, _stream_=`False`, _ordered_=`True`) Coconut provides a `multithreading`-based version of [`process_map`](#process_map) under the name `thread_map`. `thread_map` behaves identically to `process_map` (including support for `thread_map.multiple_sequential_calls`) except that it uses multithreading instead of multiprocessing, and is therefore primarily useful only for IO-bound tasks due to CPython's Global Interpreter Lock. diff --git a/__coconut__/__init__.pyi b/__coconut__/__init__.pyi index 3388e597f..0932bd0fc 100644 --- a/__coconut__/__init__.pyi +++ b/__coconut__/__init__.pyi @@ -1593,23 +1593,75 @@ def all_equal(iterable: _Iterable) -> bool: def collectby( key_func: _t.Callable[[_T], _U], iterable: _t.Iterable[_T], + *, + map_using: _t.Callable | None = None, ) -> _t.DefaultDict[_U, _t.List[_T]]: ... @_t.overload def collectby( key_func: _t.Callable[[_T], _U], iterable: _t.Iterable[_T], + *, reduce_func: _t.Callable[[_T, _T], _V], + map_using: _t.Callable | None = None, +) -> _t.DefaultDict[_U, _V]: ... +@_t.overload +def collectby( + key_func: _t.Callable[[_T], _U], + iterable: _t.Iterable[_T], + value_func: _t.Callable[[_T], _W], + *, + map_using: _t.Callable | None = None, +) -> _t.DefaultDict[_U, _t.List[_W]]: ... +@_t.overload +def collectby( + key_func: _t.Callable[[_T], _U], + iterable: _t.Iterable[_T], + value_func: _t.Callable[[_T], _W], + *, + reduce_func: _t.Callable[[_W, _W], _V], + map_using: _t.Callable | None = None, ) -> _t.DefaultDict[_U, _V]: """Collect the items in iterable into a dictionary of lists keyed by key_func(item). - if value_func is passed, collect value_func(item) into each list instead of item. + If value_func is passed, collect value_func(item) into each list instead of item. If reduce_func is passed, instead of collecting the items into lists, reduce over - the items of each key with reduce_func, effectively implementing a MapReduce operation. + the items for each key with reduce_func, effectively implementing a MapReduce operation. + + If map_using is passed, calculate key_func and value_func by mapping them over + the iterable using map_using as map. Useful with process_map/thread_map. """ ... +@_t.overload +def mapreduce( + key_value_func: _t.Callable[[_T], _t.Tuple[_U, _W]], + iterable: _t.Iterable[_T], + *, + map_using: _t.Callable | None = None, +) -> _t.DefaultDict[_U, _t.List[_W]]: ... +@_t.overload +def mapreduce( + key_value_func: _t.Callable[[_T], _t.Tuple[_U, _W]], + iterable: _t.Iterable[_T], + *, + reduce_func: _t.Callable[[_W, _W], _V], + map_using: _t.Callable | None = None, +) -> _t.DefaultDict[_U, _V]: + """Map key_value_func over iterable, then collect the values into a dictionary of lists keyed by the keys. + + If reduce_func is passed, instead of collecting the values into lists, reduce over + the values for each key with reduce_func, effectively implementing a MapReduce operation. + + If map_using is passed, calculate key_value_func by mapping them over + the iterable using map_using as map. Useful with process_map/thread_map. + """ + ... + +_coconut_mapreduce = mapreduce + + @_t.overload def _namedtuple_of(**kwargs: _t.Dict[_t.Text, _T]) -> _t.Tuple[_T, ...]: ... @_t.overload diff --git a/coconut/compiler/templates/header.py_template b/coconut/compiler/templates/header.py_template index 0e346b2b3..29f56cfb0 100644 --- a/coconut/compiler/templates/header.py_template +++ b/coconut/compiler/templates/header.py_template @@ -845,7 +845,7 @@ class _coconut_parallel_map_func_wrapper(_coconut_baseclass): def __reduce__(self): return (self.__class__, (self.map_cls, self.func, self.star)) def __call__(self, *args, **kwargs): - self.map_cls.get_pool_stack().append(None) + self.map_cls._get_pool_stack().append(None) try: if self.star: assert _coconut.len(args) == 1, "internal process/thread map error {report_this_text}" @@ -857,52 +857,65 @@ class _coconut_parallel_map_func_wrapper(_coconut_baseclass): _coconut.traceback.print_exc() raise finally: - assert self.map_cls.get_pool_stack().pop() is None, "internal process/thread map error {report_this_text}" + assert self.map_cls._get_pool_stack().pop() is None, "internal process/thread map error {report_this_text}" class _coconut_base_parallel_map(map): - __slots__ = ("result", "chunksize", "strict") + __slots__ = ("result", "chunksize", "strict", "stream", "ordered") @classmethod - def get_pool_stack(cls): + def _get_pool_stack(cls): return cls.threadlocal_ns.__dict__.setdefault("pool_stack", [None]) def __new__(cls, function, *iterables, **kwargs): self = _coconut.super(_coconut_base_parallel_map, cls).__new__(cls, function, *iterables) self.result = None self.chunksize = kwargs.pop("chunksize", 1) self.strict = kwargs.pop("strict", False) + self.stream = kwargs.pop("stream", False) + self.ordered = kwargs.pop("ordered", True) if kwargs: raise _coconut.TypeError(cls.__name__ + "() got unexpected keyword arguments " + _coconut.repr(kwargs)) - if cls.get_pool_stack()[-1] is not None: - return self.get_list() + if not self.stream and cls._get_pool_stack()[-1] is not None: + return self.to_tuple() return self + def __reduce__(self): + return (self.__class__, (self.func,) + self.iters, {lbrace}"chunksize": self.chunksize, "strict": self.strict, "stream": self.stream, "ordered": self.ordered{rbrace}) @classmethod @_coconut.contextlib.contextmanager def multiple_sequential_calls(cls, max_workers=None): - """Context manager that causes nested calls to use the same pool. - Yields True if this is at the top-level otherwise False.""" - if cls.get_pool_stack()[-1] is None: - cls.get_pool_stack()[-1] = cls.make_pool(max_workers) + """Context manager that causes nested calls to use the same pool.""" + if cls._get_pool_stack()[-1] is None: + cls._get_pool_stack()[-1] = cls.make_pool(max_workers) try: - yield True + yield finally: - cls.get_pool_stack()[-1].terminate() - cls.get_pool_stack()[-1] = None + cls._get_pool_stack()[-1].terminate() + cls._get_pool_stack()[-1] = None else: - yield False - def execute_map_method(self, map_method="imap"): + yield + def _execute_map(self): + map_func = self._get_pool_stack()[-1].imap if self.ordered else self._get_pool_stack()[-1].imap_unordered if _coconut.len(self.iters) == 1: - return _coconut.getattr(self.get_pool_stack()[-1], map_method)(_coconut_parallel_map_func_wrapper(self.__class__, self.func, False), self.iters[0], self.chunksize) + return map_func(_coconut_parallel_map_func_wrapper(self.__class__, self.func, False), self.iters[0], self.chunksize) elif self.strict: - return _coconut.getattr(self.get_pool_stack()[-1], map_method)(_coconut_parallel_map_func_wrapper(self.__class__, self.func, True), {_coconut_}zip(*self.iters, strict=True), self.chunksize) + return map_func(_coconut_parallel_map_func_wrapper(self.__class__, self.func, True), {_coconut_}zip(*self.iters, strict=True), self.chunksize) else: - return _coconut.getattr(self.get_pool_stack()[-1], map_method)(_coconut_parallel_map_func_wrapper(self.__class__, self.func, True), _coconut.zip(*self.iters), self.chunksize) - def get_list(self): + return map_func(_coconut_parallel_map_func_wrapper(self.__class__, self.func, True), _coconut.zip(*self.iters), self.chunksize) + def to_tuple(self): + """Execute the map operation and return the results as a tuple.""" if self.result is None: with self.multiple_sequential_calls(): - self.result = _coconut.tuple(self.execute_map_method()) + self.result = _coconut.tuple(self._execute_map()) self.func = {_coconut_}ident self.iters = (self.result,) return self.result + def to_stream(self): + """Stream the map operation, yielding results one at a time.""" + if self._get_pool_stack()[-1] is None: + raise _coconut.RuntimeError("cannot stream outside of " + cls.__name__ + ".multiple_sequential_calls context") + return self._execute_map() def __iter__(self): - return _coconut.iter(self.get_list()){COMMENT.have_to_get_list_so_finishes_before_return_else_cant_manage_context} + if self.stream: + return self.to_stream() + else: + return _coconut.iter(self.to_tuple()){COMMENT.have_to_to_tuple_so_finishes_before_return_else_cant_manage_context} class process_map(_coconut_base_parallel_map): """Multi-process implementation of map. Requires arguments to be pickleable. @@ -1303,7 +1316,8 @@ class groupsof(_coconut_has_iter): def __copy__(self): return self.__class__(self.group_size, self.get_new_iter()) class recursive_iterator(_coconut_base_callable): - """Decorator that memoizes a recursive function that returns an iterator (e.g. a recursive generator).""" + """Decorator that memoizes a generator (or any function that returns an iterator). + Particularly useful for recursive generators, which may require recursive_iterator to function properly.""" __slots__ = ("func", "reit_store", "backup_reit_store") def __init__(self, func): self.func = func @@ -1879,27 +1893,41 @@ def all_equal(iterable): elif first_item != item: return False return True -def collectby(key_func, iterable, value_func=None, reduce_func=None): - """Collect the items in iterable into a dictionary of lists keyed by key_func(item). +def mapreduce(key_value_func, iterable, **kwargs): + """Map key_value_func over iterable, then collect the values into a dictionary of lists keyed by the keys. - if value_func is passed, collect value_func(item) into each list instead of item. + If reduce_func is passed, instead of collecting the values into lists, reduce over + the values for each key with reduce_func, effectively implementing a MapReduce operation. - If reduce_func is passed, instead of collecting the items into lists, reduce over - the items of each key with reduce_func, effectively implementing a MapReduce operation. + If map_using is passed, calculate key_value_func by mapping them over + the iterable using map_using as map. Useful with process_map/thread_map. """ + reduce_func = kwargs.pop("reduce_func", None) + map_using = kwargs.pop("map_using", _coconut.map) + if kwargs: + raise _coconut.TypeError("mapreduce()/collectby() got unexpected keyword arguments " + _coconut.repr(kwargs)) collection = _coconut.collections.defaultdict(_coconut.list) if reduce_func is None else {empty_dict} - for item in iterable: - key = key_func(item) - if value_func is not None: - item = value_func(item) + for key, val in map_using(key_value_func, iterable): if reduce_func is None: - collection[key].append(item) + collection[key].append(val) else: - old_item = collection.get(key, _coconut_sentinel) - if old_item is not _coconut_sentinel: - item = reduce_func(old_item, item) - collection[key] = item + old_val = collection.get(key, _coconut_sentinel) + if old_val is not _coconut_sentinel: + val = reduce_func(old_val, val) + collection[key] = val return collection +def collectby(key_func, iterable, value_func=None, **kwargs): + """Collect the items in iterable into a dictionary of lists keyed by key_func(item). + + If value_func is passed, collect value_func(item) into each list instead of item. + + If reduce_func is passed, instead of collecting the items into lists, reduce over + the items for each key with reduce_func, effectively implementing a MapReduce operation. + + If map_using is passed, calculate key_func and value_func by mapping them over + the iterable using map_using as map. Useful with process_map/thread_map. + """ + return {_coconut_}mapreduce(_coconut_lifted(_coconut_comma_op, key_func, {_coconut_}ident if value_func is None else value_func), iterable, **kwargs) def _namedtuple_of(**kwargs): """Construct an anonymous namedtuple of the given keyword arguments.""" {namedtuple_of_implementation} @@ -2106,4 +2134,4 @@ class _coconut_SupportsInv(_coconut.typing.Protocol): raise NotImplementedError("Protocol methods cannot be called at runtime ((~) in a typing context is a Protocol)") {def_aliases} _coconut_self_match_types = {self_match_types} -_coconut_Expected, _coconut_MatchError, _coconut_cartesian_product, _coconut_count, _coconut_cycle, _coconut_enumerate, _coconut_flatten, _coconut_filter, _coconut_groupsof, _coconut_ident, _coconut_lift, _coconut_map, _coconut_multiset, _coconut_range, _coconut_reiterable, _coconut_reversed, _coconut_scan, _coconut_starmap, _coconut_tee, _coconut_windowsof, _coconut_zip, _coconut_zip_longest, TYPE_CHECKING, reduce, takewhile, dropwhile = Expected, MatchError, cartesian_product, count, cycle, enumerate, flatten, filter, groupsof, ident, lift, map, multiset, range, reiterable, reversed, scan, starmap, tee, windowsof, zip, zip_longest, False, _coconut.functools.reduce, _coconut.itertools.takewhile, _coconut.itertools.dropwhile{COMMENT.anything_added_here_should_be_copied_to_stub_file} +_coconut_Expected, _coconut_MatchError, _coconut_cartesian_product, _coconut_count, _coconut_cycle, _coconut_enumerate, _coconut_flatten, _coconut_filter, _coconut_groupsof, _coconut_ident, _coconut_lift, _coconut_map, _coconut_mapreduce, _coconut_multiset, _coconut_range, _coconut_reiterable, _coconut_reversed, _coconut_scan, _coconut_starmap, _coconut_tee, _coconut_windowsof, _coconut_zip, _coconut_zip_longest, TYPE_CHECKING, reduce, takewhile, dropwhile = Expected, MatchError, cartesian_product, count, cycle, enumerate, flatten, filter, groupsof, ident, lift, map, mapreduce, multiset, range, reiterable, reversed, scan, starmap, tee, windowsof, zip, zip_longest, False, _coconut.functools.reduce, _coconut.itertools.takewhile, _coconut.itertools.dropwhile{COMMENT.anything_added_here_should_be_copied_to_stub_file} diff --git a/coconut/constants.py b/coconut/constants.py index d58704ab5..7fdbd7ed9 100644 --- a/coconut/constants.py +++ b/coconut/constants.py @@ -769,6 +769,7 @@ def get_path_env_var(env_var, default): "lift", "all_equal", "collectby", + "mapreduce", "multi_enumerate", "cartesian_product", "multiset", diff --git a/coconut/root.py b/coconut/root.py index 2753a0a28..9480ffd12 100644 --- a/coconut/root.py +++ b/coconut/root.py @@ -26,7 +26,7 @@ VERSION = "3.0.3" VERSION_NAME = None # False for release, int >= 1 for develop -DEVELOP = 12 +DEVELOP = 13 ALPHA = False # for pre releases rather than post releases assert DEVELOP is False or DEVELOP >= 1, "DEVELOP must be False or an int >= 1" diff --git a/coconut/tests/src/cocotest/agnostic/primary.coco b/coconut/tests/src/cocotest/agnostic/primary.coco index dad76b379..38acc52e6 100644 --- a/coconut/tests/src/cocotest/agnostic/primary.coco +++ b/coconut/tests/src/cocotest/agnostic/primary.coco @@ -221,7 +221,7 @@ def primary_test() -> bool: assert process_map((+), range(5), range(5), chunksize=2) |> list == map((*)$(2), range(5)) |> list == thread_map((+), range(5), range(5), chunksize=2) |> list # type: ignore assert repr(thread_map((-), range(5))).startswith("thread_map(") # type: ignore with thread_map.multiple_sequential_calls(max_workers=4): # type: ignore - assert thread_map((-), range(5)) |> tuple == (0, -1, -2, -3, -4) == thread_map(map$((-)), (range(5),))$[0] |> tuple # type: ignore + assert thread_map((-), range(5), stream=True) |> tuple == (0, -1, -2, -3, -4) == thread_map(map$((-)), (range(5),))$[0] |> tuple # type: ignore assert thread_map(zip, (range(2),), (range(2),)) |> map$(tuple) |> tuple == (((0,0), (1,1)),) # type: ignore assert (range(0, 5), range(5, 10)) |*> map$(+) |> tuple == (5, 7, 9, 11, 13) assert thread_map((*)$(2)..(+)$(1), range(5)) |> tuple == (2, 4, 6, 8, 10) @@ -632,6 +632,11 @@ def primary_test() -> bool: assert m2 `isinstance` tuple assert m1.result is None assert m2 == (1, 2, 3, 4, 5) == tuple(m1) + m3 = tuple(map_func((.+1), range(5), stream=True)) + assert m3 == (1, 2, 3, 4, 5) + m4 = set(map_func((.+1), range(5), ordered=False)) + m5 = set(map_func((.+1), range(5), ordered=False, stream=True)) + assert m4 == {1, 2, 3, 4, 5} == m5 assert m1.result == (1, 2, 3, 4, 5) == tuple(m1) for it in ((), [], (||)): assert_raises(-> it$[0], IndexError) @@ -945,13 +950,13 @@ def primary_test() -> bool: assert 1 `(,)` 2 == (1, 2) == (,) 1 2 assert (-1+.)(2) == 1 ==-1 = -1 - assert collectby((def -> assert False), [], (def (x,y) -> assert False)) == {} - assert collectby(ident, range(5)) == {0: [0], 1: [1], 2: [2], 3: [3], 4: [4]} - assert collectby(.[1], zip(range(5), reversed(range(5)))) == {0: [(4, 0)], 1: [(3, 1)], 2: [(2, 2)], 3: [(1, 3)], 4: [(0, 4)]} - assert collectby(ident, range(5) :: range(5)) == {0: [0, 0], 1: [1, 1], 2: [2, 2], 3: [3, 3], 4: [4, 4]} - assert collectby(ident, range(5) :: range(5), reduce_func=(+)) == {0: 0, 1: 2, 2: 4, 3: 6, 4: 8} == collectby(ident, range(5) :: range(5), ident, (+)) + assert collectby((def -> assert False), [], (def (x,y) -> assert False)) == {} == collectby((def -> assert False), [], (def (x,y) -> assert False), map_using=map) + assert collectby(ident, range(5)) == {0: [0], 1: [1], 2: [2], 3: [3], 4: [4]} == collectby(ident, range(5), map_using=map) + assert collectby(.[1], zip(range(5), reversed(range(5)))) == {0: [(4, 0)], 1: [(3, 1)], 2: [(2, 2)], 3: [(1, 3)], 4: [(0, 4)]} == collectby(.[1], zip(range(5), reversed(range(5))), map_using=map) + assert collectby(ident, range(5) :: range(5)) == {0: [0, 0], 1: [1, 1], 2: [2, 2], 3: [3, 3], 4: [4, 4]} == collectby(ident, range(5) :: range(5), map_using=map) + assert collectby(ident, range(5) :: range(5), reduce_func=(+)) == {0: 0, 1: 2, 2: 4, 3: 6, 4: 8} == collectby(ident, range(5) :: range(5), reduce_func=(+), map_using=map) def dub(xs) = xs :: xs - assert collectby(.[0], dub <| zip(range(5), reversed(range(5))), value_func=.[1], reduce_func=(+)) == {0: 8, 1: 6, 2: 4, 3: 2, 4: 0} + assert collectby(.[0], dub <| zip(range(5), reversed(range(5))), value_func=.[1], reduce_func=(+)) == {0: 8, 1: 6, 2: 4, 3: 2, 4: 0} == mapreduce(ident, dub <| zip(range(5), reversed(range(5))), reduce_func=(+)) assert int(1e9) in range(2**31-1) assert (a=1, b=2) == (1, 2) == (a:int=1, b=2) assert (a=1, b: int = 2) == (1, 2) == (a: int=1, b: int=2) From 2e03ba092e3d33c205602fb0a98e7b4d4c511a43 Mon Sep 17 00:00:00 2001 From: Evan Hubinger Date: Sat, 28 Oct 2023 01:46:24 -0700 Subject: [PATCH 036/121] recursive_iterator to recursive_generator Resolves #749. --- DOCS.md | 22 +++--- FAQ.md | 4 +- __coconut__/__init__.pyi | 5 +- coconut/compiler/header.py | 6 +- coconut/compiler/templates/header.py_template | 68 ++++++++----------- coconut/constants.py | 4 +- coconut/root.py | 2 +- coconut/tests/main_test.py | 5 +- coconut/tests/src/cocotest/agnostic/util.coco | 16 ++--- .../cocotest/non_strict/non_strict_test.coco | 5 +- 10 files changed, 68 insertions(+), 69 deletions(-) diff --git a/DOCS.md b/DOCS.md index 77f1d6ac2..a32e27b13 100644 --- a/DOCS.md +++ b/DOCS.md @@ -2316,8 +2316,6 @@ Coconut will perform automatic [tail call](https://en.wikipedia.org/wiki/Tail_ca Tail call optimization (though not tail recursion elimination) will work even for 1) mutual recursion and 2) pattern-matching functions split across multiple definitions using [`addpattern`](#addpattern). -If you are encountering a `RuntimeError` due to maximum recursion depth, it is highly recommended that you rewrite your function to meet either the criteria above for tail call optimization, or the corresponding criteria for [`recursive_iterator`](#recursive-iterator), either of which should prevent such errors. - ##### Example **Coconut:** @@ -2960,6 +2958,8 @@ Coconut provides `functools.lru_cache` as a built-in under the name `memoize` wi Use of `memoize` requires `functools.lru_cache`, which exists in the Python 3 standard library, but under Python 2 will require `pip install backports.functools_lru_cache` to function. Additionally, if on Python 2 and `backports.functools_lru_cache` is present, Coconut will patch `functools` such that `functools.lru_cache = backports.functools_lru_cache.lru_cache`. +Note that, if the function to be memoized is a generator or otherwise returns an iterator, [`recursive_generator`](#recursive_generator) can also be used to achieve a similar effect, the use of which is required for recursive generators. + ##### Python Docs @**memoize**(_user\_function_) @@ -3060,36 +3060,36 @@ class B: **Python:** _Can't be done without a long decorator definition. The full definition of the decorator in Python can be found in the Coconut header._ -#### `recursive_iterator` +#### `recursive_generator` -**recursive\_iterator**(_func_) +**recursive\_generator**(_func_) -Coconut provides a `recursive_iterator` decorator that memoizes any stateless, recursive function that returns an iterator. To use `recursive_iterator` on a function, it must meet the following criteria: +Coconut provides a `recursive_generator` decorator that memoizes and makes [`reiterable`](#reiterable) any generator or other stateless function that returns an iterator. To use `recursive_generator` on a function, it must meet the following criteria: 1. your function either always `return`s an iterator or generates an iterator using `yield`, 2. when called multiple times with arguments that are equal, your function produces the same iterator (your function is stateless), and 3. your function gets called (usually calls itself) multiple times with the same arguments. -If you are encountering a `RuntimeError` due to maximum recursion depth, it is highly recommended that you rewrite your function to meet either the criteria above for `recursive_iterator`, or the corresponding criteria for Coconut's [tail call optimization](#tail-call-optimization), either of which should prevent such errors. - -Furthermore, `recursive_iterator` also allows the resolution of a [nasty segmentation fault in Python's iterator logic that has never been fixed](http://bugs.python.org/issue14010). Specifically, instead of writing +Importantly, `recursive_generator` also allows the resolution of a [nasty segmentation fault in Python's iterator logic that has never been fixed](http://bugs.python.org/issue14010). Specifically, instead of writing ```coconut seq = get_elem() :: seq ``` which will crash due to the aforementioned Python issue, write ```coconut -@recursive_iterator +@recursive_generator def seq() = get_elem() :: seq() ``` which will work just fine. -One pitfall to keep in mind working with `recursive_iterator` is that it shouldn't be used in contexts where the function can potentially be called multiple times with the same iterator object as an input, but with that object not actually corresponding to the same items (e.g. because the first time the object hasn't been iterated over yet and the second time it has been). +One pitfall to keep in mind working with `recursive_generator` is that it shouldn't be used in contexts where the function can potentially be called multiple times with the same iterator object as an input, but with that object not actually corresponding to the same items (e.g. because the first time the object hasn't been iterated over yet and the second time it has been). + +_Deprecated: `recursive_iterator` is available as a deprecated alias for `recursive_generator`. Note that deprecated features are disabled in `--strict` mode._ ##### Example **Coconut:** ```coconut -@recursive_iterator +@recursive_generator def fib() = (1, 1) :: map((+), fib(), fib()$[1:]) ``` diff --git a/FAQ.md b/FAQ.md index 201885b2e..d197f42ed 100644 --- a/FAQ.md +++ b/FAQ.md @@ -34,9 +34,9 @@ Information on every Coconut release is chronicled on the [GitHub releases page] Yes! Coconut compiles the [newest](https://www.python.org/dev/peps/pep-0526/), [fanciest](https://www.python.org/dev/peps/pep-0484/) type annotation syntax into version-independent type comments which can then by checked using Coconut's built-in [MyPy Integration](./DOCS.md#mypy-integration). -### Help! I tried to write a recursive iterator and my Python segfaulted! +### Help! I tried to write a recursive generator and my Python segfaulted! -No problem—just use Coconut's [`recursive_iterator`](./DOCS.md#recursive-iterator) decorator and you should be fine. This is a [known Python issue](http://bugs.python.org/issue14010) but `recursive_iterator` will fix it for you. +No problem—just use Coconut's [`recursive_generator`](./DOCS.md#recursive_generator) decorator and you should be fine. This is a [known Python issue](http://bugs.python.org/issue14010) but `recursive_generator` will fix it for you. ### How do I split an expression across multiple lines in Coconut? diff --git a/__coconut__/__init__.pyi b/__coconut__/__init__.pyi index 0932bd0fc..67f784cf4 100644 --- a/__coconut__/__init__.pyi +++ b/__coconut__/__init__.pyi @@ -534,9 +534,10 @@ def _coconut_call_or_coefficient( ) -> _T: ... -def recursive_iterator(func: _T_iter_func) -> _T_iter_func: +def recursive_generator(func: _T_iter_func) -> _T_iter_func: """Decorator that memoizes a recursive function that returns an iterator (e.g. a recursive generator).""" return func +recursive_iterator = recursive_generator # if sys.version_info >= (3, 12): @@ -590,7 +591,7 @@ def addpattern( *add_funcs: _Callable, allow_any_func: bool=False, ) -> _t.Callable[..., _t.Any]: - """Decorator to add a new case to a pattern-matching function (where the new case is checked last). + """Decorator to add new cases to a pattern-matching function (where the new case is checked last). Pass allow_any_func=True to allow any object as the base_func rather than just pattern-matching functions. If add_funcs are passed, addpattern(base_func, add_func) is equivalent to addpattern(base_func)(add_func). diff --git a/coconut/compiler/header.py b/coconut/compiler/header.py index 79c5dc093..3910880b7 100644 --- a/coconut/compiler/header.py +++ b/coconut/compiler/header.py @@ -230,6 +230,7 @@ def process_header_args(which, use_hash, target, no_tco, strict, no_wrap): comma_object="" if target.startswith("3") else ", object", comma_slash=", /" if target_info >= (3, 8) else "", report_this_text=report_this_text, + from_None=" from None" if target.startswith("3") else "", numpy_modules=tuple_str_of(numpy_modules, add_quotes=True), pandas_numpy_modules=tuple_str_of(pandas_numpy_modules, add_quotes=True), jax_numpy_modules=tuple_str_of(jax_numpy_modules, add_quotes=True), @@ -310,7 +311,7 @@ def pattern_prepender(func): def datamaker(data_type): """DEPRECATED: use makedata instead.""" return _coconut.functools.partial(makedata, data_type) -of, parallel_map, concurrent_map = call, process_map, thread_map +of, parallel_map, concurrent_map, recursive_iterator = call, process_map, thread_map, recursive_generator ''' if not strict else r''' @@ -329,6 +330,9 @@ def parallel_map(*args, **kwargs): def concurrent_map(*args, **kwargs): """Deprecated Coconut built-in 'concurrent_map' disabled by --strict compilation; use 'thread_map' instead.""" raise _coconut.NameError("deprecated Coconut built-in 'concurrent_map' disabled by --strict compilation; use 'thread_map' instead") +def recursive_iterator(*args, **kwargs): + """Deprecated Coconut built-in 'recursive_iterator' disabled by --strict compilation; use 'recursive_generator' instead.""" + raise _coconut.NameError("deprecated Coconut built-in 'recursive_iterator' disabled by --strict compilation; use 'recursive_generator' instead") ''' ), return_method_of_self=pycondition( diff --git a/coconut/compiler/templates/header.py_template b/coconut/compiler/templates/header.py_template index 29f56cfb0..b4e3a1165 100644 --- a/coconut/compiler/templates/header.py_template +++ b/coconut/compiler/templates/header.py_template @@ -13,7 +13,7 @@ def _coconut_super(type=None, object_or_type=None): try: cls = frame.f_locals["__class__"] except _coconut.AttributeError: - raise _coconut.RuntimeError("super(): __class__ cell not found") + raise _coconut.RuntimeError("super(): __class__ cell not found"){from_None} self = frame.f_locals[frame.f_code.co_varnames[0]] return _coconut_py_super(cls, self) return _coconut_py_super(type, object_or_type) @@ -1315,39 +1315,29 @@ class groupsof(_coconut_has_iter): return (self.__class__, (self.group_size, self.iter)) def __copy__(self): return self.__class__(self.group_size, self.get_new_iter()) -class recursive_iterator(_coconut_base_callable): +class recursive_generator(_coconut_base_callable): """Decorator that memoizes a generator (or any function that returns an iterator). - Particularly useful for recursive generators, which may require recursive_iterator to function properly.""" - __slots__ = ("func", "reit_store", "backup_reit_store") + Particularly useful for recursive generators, which may require recursive_generator to function properly.""" + __slots__ = ("func", "reit_store") def __init__(self, func): self.func = func self.reit_store = {empty_dict} - self.backup_reit_store = [] def __call__(self, *args, **kwargs): - key = (args, _coconut.frozenset(kwargs.items())) - use_backup = False + key = (0, args, _coconut.frozenset(kwargs.items())) try: _coconut.hash(key) - except _coconut.Exception: + except _coconut.TypeError: try: - key = _coconut.pickle.dumps(key, -1) + key = (1, _coconut.pickle.dumps(key, -1)) except _coconut.Exception: - use_backup = True - if use_backup: - for k, v in self.backup_reit_store: - if k == key: - return reit + raise _coconut.TypeError("recursive_generator() requires function arguments to be hashable or pickleable"){from_None} + reit = self.reit_store.get(key) + if reit is None: reit = {_coconut_}reiterable(self.func(*args, **kwargs)) - self.backup_reit_store.append([key, reit]) - return reit - else: - reit = self.reit_store.get(key) - if reit is None: - reit = {_coconut_}reiterable(self.func(*args, **kwargs)) - self.reit_store[key] = reit - return reit + self.reit_store[key] = reit + return reit def __repr__(self): - return "recursive_iterator(%r)" % (self.func,) + return "recursive_generator(%r)" % (self.func,) def __reduce__(self): return (self.__class__, (self.func,)) class _coconut_FunctionMatchErrorContext(_coconut_baseclass): @@ -1416,7 +1406,7 @@ def _coconut_mark_as_match(base_func):{COMMENT._coconut_is_match_is_used_above_a base_func._coconut_is_match = True return base_func def addpattern(base_func, *add_funcs, **kwargs): - """Decorator to add a new case to a pattern-matching function (where the new case is checked last). + """Decorator to add new cases to a pattern-matching function (where the new case is checked last). Pass allow_any_func=True to allow any object as the base_func rather than just pattern-matching functions. If add_funcs are passed, addpattern(base_func, add_func) is equivalent to addpattern(base_func)(add_func). @@ -2010,7 +2000,7 @@ class _coconut_SupportsAdd(_coconut.typing.Protocol): raise NotImplementedError(...) """ def __add__(self, other): - raise NotImplementedError("Protocol methods cannot be called at runtime ((+) in a typing context is a Protocol)") + raise _coconut.NotImplementedError("Protocol methods cannot be called at runtime ((+) in a typing context is a Protocol)") class _coconut_SupportsMinus(_coconut.typing.Protocol): """Coconut (-) Protocol. Equivalent to: @@ -2021,9 +2011,9 @@ class _coconut_SupportsMinus(_coconut.typing.Protocol): raise NotImplementedError """ def __sub__(self, other): - raise NotImplementedError("Protocol methods cannot be called at runtime ((-) in a typing context is a Protocol)") + raise _coconut.NotImplementedError("Protocol methods cannot be called at runtime ((-) in a typing context is a Protocol)") def __neg__(self): - raise NotImplementedError("Protocol methods cannot be called at runtime ((-) in a typing context is a Protocol)") + raise _coconut.NotImplementedError("Protocol methods cannot be called at runtime ((-) in a typing context is a Protocol)") class _coconut_SupportsMul(_coconut.typing.Protocol): """Coconut (*) Protocol. Equivalent to: @@ -2032,7 +2022,7 @@ class _coconut_SupportsMul(_coconut.typing.Protocol): raise NotImplementedError(...) """ def __mul__(self, other): - raise NotImplementedError("Protocol methods cannot be called at runtime ((*) in a typing context is a Protocol)") + raise _coconut.NotImplementedError("Protocol methods cannot be called at runtime ((*) in a typing context is a Protocol)") class _coconut_SupportsPow(_coconut.typing.Protocol): """Coconut (**) Protocol. Equivalent to: @@ -2041,7 +2031,7 @@ class _coconut_SupportsPow(_coconut.typing.Protocol): raise NotImplementedError(...) """ def __pow__(self, other): - raise NotImplementedError("Protocol methods cannot be called at runtime ((**) in a typing context is a Protocol)") + raise _coconut.NotImplementedError("Protocol methods cannot be called at runtime ((**) in a typing context is a Protocol)") class _coconut_SupportsTruediv(_coconut.typing.Protocol): """Coconut (/) Protocol. Equivalent to: @@ -2050,7 +2040,7 @@ class _coconut_SupportsTruediv(_coconut.typing.Protocol): raise NotImplementedError(...) """ def __truediv__(self, other): - raise NotImplementedError("Protocol methods cannot be called at runtime ((/) in a typing context is a Protocol)") + raise _coconut.NotImplementedError("Protocol methods cannot be called at runtime ((/) in a typing context is a Protocol)") class _coconut_SupportsFloordiv(_coconut.typing.Protocol): """Coconut (//) Protocol. Equivalent to: @@ -2059,7 +2049,7 @@ class _coconut_SupportsFloordiv(_coconut.typing.Protocol): raise NotImplementedError(...) """ def __floordiv__(self, other): - raise NotImplementedError("Protocol methods cannot be called at runtime ((//) in a typing context is a Protocol)") + raise _coconut.NotImplementedError("Protocol methods cannot be called at runtime ((//) in a typing context is a Protocol)") class _coconut_SupportsMod(_coconut.typing.Protocol): """Coconut (%) Protocol. Equivalent to: @@ -2068,7 +2058,7 @@ class _coconut_SupportsMod(_coconut.typing.Protocol): raise NotImplementedError(...) """ def __mod__(self, other): - raise NotImplementedError("Protocol methods cannot be called at runtime ((%) in a typing context is a Protocol)") + raise _coconut.NotImplementedError("Protocol methods cannot be called at runtime ((%) in a typing context is a Protocol)") class _coconut_SupportsAnd(_coconut.typing.Protocol): """Coconut (&) Protocol. Equivalent to: @@ -2077,7 +2067,7 @@ class _coconut_SupportsAnd(_coconut.typing.Protocol): raise NotImplementedError(...) """ def __and__(self, other): - raise NotImplementedError("Protocol methods cannot be called at runtime ((&) in a typing context is a Protocol)") + raise _coconut.NotImplementedError("Protocol methods cannot be called at runtime ((&) in a typing context is a Protocol)") class _coconut_SupportsXor(_coconut.typing.Protocol): """Coconut (^) Protocol. Equivalent to: @@ -2086,7 +2076,7 @@ class _coconut_SupportsXor(_coconut.typing.Protocol): raise NotImplementedError(...) """ def __xor__(self, other): - raise NotImplementedError("Protocol methods cannot be called at runtime ((^) in a typing context is a Protocol)") + raise _coconut.NotImplementedError("Protocol methods cannot be called at runtime ((^) in a typing context is a Protocol)") class _coconut_SupportsOr(_coconut.typing.Protocol): """Coconut (|) Protocol. Equivalent to: @@ -2095,7 +2085,7 @@ class _coconut_SupportsOr(_coconut.typing.Protocol): raise NotImplementedError(...) """ def __or__(self, other): - raise NotImplementedError("Protocol methods cannot be called at runtime ((|) in a typing context is a Protocol)") + raise _coconut.NotImplementedError("Protocol methods cannot be called at runtime ((|) in a typing context is a Protocol)") class _coconut_SupportsLshift(_coconut.typing.Protocol): """Coconut (<<) Protocol. Equivalent to: @@ -2104,7 +2094,7 @@ class _coconut_SupportsLshift(_coconut.typing.Protocol): raise NotImplementedError(...) """ def __lshift__(self, other): - raise NotImplementedError("Protocol methods cannot be called at runtime ((<<) in a typing context is a Protocol)") + raise _coconut.NotImplementedError("Protocol methods cannot be called at runtime ((<<) in a typing context is a Protocol)") class _coconut_SupportsRshift(_coconut.typing.Protocol): """Coconut (>>) Protocol. Equivalent to: @@ -2113,7 +2103,7 @@ class _coconut_SupportsRshift(_coconut.typing.Protocol): raise NotImplementedError(...) """ def __rshift__(self, other): - raise NotImplementedError("Protocol methods cannot be called at runtime ((>>) in a typing context is a Protocol)") + raise _coconut.NotImplementedError("Protocol methods cannot be called at runtime ((>>) in a typing context is a Protocol)") class _coconut_SupportsMatmul(_coconut.typing.Protocol): """Coconut (@) Protocol. Equivalent to: @@ -2122,7 +2112,7 @@ class _coconut_SupportsMatmul(_coconut.typing.Protocol): raise NotImplementedError(...) """ def __matmul__(self, other): - raise NotImplementedError("Protocol methods cannot be called at runtime ((@) in a typing context is a Protocol)") + raise _coconut.NotImplementedError("Protocol methods cannot be called at runtime ((@) in a typing context is a Protocol)") class _coconut_SupportsInv(_coconut.typing.Protocol): """Coconut (~) Protocol. Equivalent to: @@ -2131,7 +2121,7 @@ class _coconut_SupportsInv(_coconut.typing.Protocol): raise NotImplementedError(...) """ def __invert__(self): - raise NotImplementedError("Protocol methods cannot be called at runtime ((~) in a typing context is a Protocol)") + raise _coconut.NotImplementedError("Protocol methods cannot be called at runtime ((~) in a typing context is a Protocol)") {def_aliases} _coconut_self_match_types = {self_match_types} _coconut_Expected, _coconut_MatchError, _coconut_cartesian_product, _coconut_count, _coconut_cycle, _coconut_enumerate, _coconut_flatten, _coconut_filter, _coconut_groupsof, _coconut_ident, _coconut_lift, _coconut_map, _coconut_mapreduce, _coconut_multiset, _coconut_range, _coconut_reiterable, _coconut_reversed, _coconut_scan, _coconut_starmap, _coconut_tee, _coconut_windowsof, _coconut_zip, _coconut_zip_longest, TYPE_CHECKING, reduce, takewhile, dropwhile = Expected, MatchError, cartesian_product, count, cycle, enumerate, flatten, filter, groupsof, ident, lift, map, mapreduce, multiset, range, reiterable, reversed, scan, starmap, tee, windowsof, zip, zip_longest, False, _coconut.functools.reduce, _coconut.itertools.takewhile, _coconut.itertools.dropwhile{COMMENT.anything_added_here_should_be_copied_to_stub_file} diff --git a/coconut/constants.py b/coconut/constants.py index 7fdbd7ed9..7f5976c8f 100644 --- a/coconut/constants.py +++ b/coconut/constants.py @@ -751,7 +751,7 @@ def get_path_env_var(env_var, default): "process_map", "thread_map", "addpattern", - "recursive_iterator", + "recursive_generator", "fmap", "starmap", "reiterable", @@ -1127,6 +1127,7 @@ def get_path_env_var(env_var, default): "recursion", "call", "recursive", + "recursive_iterator", "infix", "function", "composition", @@ -1149,6 +1150,7 @@ def get_path_env_var(env_var, default): "datamaker", "prepattern", "iterator", + "generator", "none", "coalesce", "coalescing", diff --git a/coconut/root.py b/coconut/root.py index 9480ffd12..33140f22b 100644 --- a/coconut/root.py +++ b/coconut/root.py @@ -26,7 +26,7 @@ VERSION = "3.0.3" VERSION_NAME = None # False for release, int >= 1 for develop -DEVELOP = 13 +DEVELOP = 14 ALPHA = False # for pre releases rather than post releases assert DEVELOP is False or DEVELOP >= 1, "DEVELOP must be False or an int >= 1" diff --git a/coconut/tests/main_test.py b/coconut/tests/main_test.py index fc4cff555..c4cc4108f 100644 --- a/coconut/tests/main_test.py +++ b/coconut/tests/main_test.py @@ -615,6 +615,8 @@ def run(args=[], agnostic_target=None, use_run_arg=False, convert_to_import=Fals comp_agnostic(agnostic_args, **kwargs) comp_sys(args, **kwargs) + # do non-strict at the end so we get the non-strict header + comp_non_strict(args, **kwargs) if use_run_arg: _kwargs = kwargs.copy() @@ -634,9 +636,6 @@ def run(args=[], agnostic_target=None, use_run_arg=False, convert_to_import=Fals comp_extras(agnostic_args, **kwargs) run_extras(convert_to_import=convert_to_import) # **kwargs are for comp, not run - # do non-strict at the end so we get the non-strict header - comp_non_strict(args, **kwargs) - def comp_all(args=[], agnostic_target=None, **kwargs): """Compile Coconut tests.""" diff --git a/coconut/tests/src/cocotest/agnostic/util.coco b/coconut/tests/src/cocotest/agnostic/util.coco index 09c3430fe..0b67954b9 100644 --- a/coconut/tests/src/cocotest/agnostic/util.coco +++ b/coconut/tests/src/cocotest/agnostic/util.coco @@ -417,7 +417,7 @@ def partition(items, pivot, lprefix=[], rprefix=[]): return partition(tail, pivot, lprefix, [head]::rprefix) match []::_: return lprefix, rprefix -partition_ = recursive_iterator(partition) +partition_ = recursive_generator(partition) def myreduce(func, items): match [first]::tail1 in items: @@ -965,21 +965,21 @@ addpattern def `pattern_abs_` (x) = x # type: ignore # Recursive iterator -@recursive_iterator +@recursive_generator def fibs() = fibs_calls[0] += 1 (1, 1) :: map((+), fibs(), fibs()$[1:]) fibs_calls = [0] -@recursive_iterator +@recursive_generator def fibs_() = map((+), (1, 1) :: fibs_(), (0, 0) :: fibs_()$[1:]) # use separate name for base func for pickle def _loop(it) = it :: loop(it) -loop = recursive_iterator(_loop) +loop = recursive_generator(_loop) -@recursive_iterator +@recursive_generator def nest(x) = (|x, nest(x)|) # Sieve Example @@ -1294,7 +1294,7 @@ def fib(n if n < 2) = n @memoize() # type: ignore addpattern def fib(n) = fib(n-1) + fib(n-2) # type: ignore -@recursive_iterator +@recursive_generator def Fibs() = (0, 1) :: map((+), Fibs(), Fibs()$[1:]) fib_ = reiterable(Fibs())$[] @@ -1353,11 +1353,11 @@ class descriptor_test: lam = self -> self comp = tuplify .. ident - @recursive_iterator + @recursive_generator def N(self, i=0) = [(self, i)] :: self.N(i+1) - @recursive_iterator + @recursive_generator match def N_(self, *, i=0) = [(self, i)] :: self.N_(i=i+1) diff --git a/coconut/tests/src/cocotest/non_strict/non_strict_test.coco b/coconut/tests/src/cocotest/non_strict/non_strict_test.coco index 195230ede..a21b8a155 100644 --- a/coconut/tests/src/cocotest/non_strict/non_strict_test.coco +++ b/coconut/tests/src/cocotest/non_strict/non_strict_test.coco @@ -89,7 +89,10 @@ def non_strict_test() -> bool: assert f"a" r"b" fr"c" rf"d" == "abcd" assert "a" fr"b" == "ab" == "a" rf"b" assert f"{f"{f"infinite"}"}" + " " + f"{f"nesting!!!"}" == "infinite nesting!!!" - assert parallel_map((.+1), range(5)) |> tuple == tuple(range(1, 6)) + assert range(100) |> parallel_map$(.**2) |> list |> .$[-1] == 9801 + @recursive_iterator + def fib() = (1, 1) :: map((+), fib(), fib()$[1:]) + assert fib()$[:5] |> list == [1, 1, 2, 3, 5] return True if __name__ == "__main__": From d034af39cdcfc5a5511028700b2137748dc6fc5a Mon Sep 17 00:00:00 2001 From: Evan Hubinger Date: Sat, 28 Oct 2023 01:49:19 -0700 Subject: [PATCH 037/121] Fix readthedocs --- .readthedocs.yaml | 1 - 1 file changed, 1 deletion(-) diff --git a/.readthedocs.yaml b/.readthedocs.yaml index 56e6e605a..fe3e5c3b8 100644 --- a/.readthedocs.yaml +++ b/.readthedocs.yaml @@ -32,4 +32,3 @@ python: path: . extra_requirements: - docs - system_packages: true From 746bf5846362ec8fe187e2f96804200a7badb8da Mon Sep 17 00:00:00 2001 From: Evan Hubinger Date: Sun, 29 Oct 2023 01:22:41 -0700 Subject: [PATCH 038/121] Improve header --- coconut/compiler/templates/header.py_template | 22 +++++++++---------- 1 file changed, 11 insertions(+), 11 deletions(-) diff --git a/coconut/compiler/templates/header.py_template b/coconut/compiler/templates/header.py_template index b4e3a1165..2d77e7720 100644 --- a/coconut/compiler/templates/header.py_template +++ b/coconut/compiler/templates/header.py_template @@ -833,7 +833,7 @@ class map(_coconut_baseclass, _coconut.map): self.iters = _coconut.tuple({_coconut_}reiterable(it) for it in self.iters) return self.__class__(self.func, *self.iters) def __iter__(self): - return _coconut.iter(_coconut.map(self.func, *self.iters)) + return _coconut.map(self.func, *self.iters) def __fmap__(self, func): return self.__class__(_coconut_forward_compose(self.func, func), *self.iters) class _coconut_parallel_map_func_wrapper(_coconut_baseclass): @@ -848,7 +848,7 @@ class _coconut_parallel_map_func_wrapper(_coconut_baseclass): self.map_cls._get_pool_stack().append(None) try: if self.star: - assert _coconut.len(args) == 1, "internal process/thread map error {report_this_text}" + assert _coconut.len(args) == 1, "internal process_map/thread_map error {report_this_text}" return self.func(*args[0], **kwargs) else: return self.func(*args, **kwargs) @@ -857,12 +857,12 @@ class _coconut_parallel_map_func_wrapper(_coconut_baseclass): _coconut.traceback.print_exc() raise finally: - assert self.map_cls._get_pool_stack().pop() is None, "internal process/thread map error {report_this_text}" + assert self.map_cls._get_pool_stack().pop() is None, "internal process_map/thread_map error {report_this_text}" class _coconut_base_parallel_map(map): __slots__ = ("result", "chunksize", "strict", "stream", "ordered") @classmethod def _get_pool_stack(cls): - return cls.threadlocal_ns.__dict__.setdefault("pool_stack", [None]) + return cls._threadlocal_ns.__dict__.setdefault("pool_stack", [None]) def __new__(cls, function, *iterables, **kwargs): self = _coconut.super(_coconut_base_parallel_map, cls).__new__(cls, function, *iterables) self.result = None @@ -882,7 +882,7 @@ class _coconut_base_parallel_map(map): def multiple_sequential_calls(cls, max_workers=None): """Context manager that causes nested calls to use the same pool.""" if cls._get_pool_stack()[-1] is None: - cls._get_pool_stack()[-1] = cls.make_pool(max_workers) + cls._get_pool_stack()[-1] = cls._make_pool(max_workers) try: yield finally: @@ -924,9 +924,9 @@ class process_map(_coconut_base_parallel_map): ... """ __slots__ = () - threadlocal_ns = _coconut.threading.local() + _threadlocal_ns = _coconut.threading.local() @staticmethod - def make_pool(max_workers=None): + def _make_pool(max_workers=None): return _coconut.multiprocessing.Pool(max_workers) class thread_map(_coconut_base_parallel_map): """Multi-thread implementation of map. @@ -936,9 +936,9 @@ class thread_map(_coconut_base_parallel_map): ... """ __slots__ = () - threadlocal_ns = _coconut.threading.local() + _threadlocal_ns = _coconut.threading.local() @staticmethod - def make_pool(max_workers=None): + def _make_pool(max_workers=None): return _coconut.multiprocessing_dummy.Pool(_coconut.multiprocessing.cpu_count() * 5 if max_workers is None else max_workers) class zip(_coconut_baseclass, _coconut.zip): __slots__ = ("iters", "strict") @@ -1342,13 +1342,13 @@ class recursive_generator(_coconut_base_callable): return (self.__class__, (self.func,)) class _coconut_FunctionMatchErrorContext(_coconut_baseclass): __slots__ = ("exc_class", "taken") - threadlocal_ns = _coconut.threading.local() + _threadlocal_ns = _coconut.threading.local() def __init__(self, exc_class): self.exc_class = exc_class self.taken = False @classmethod def get_contexts(cls): - return cls.threadlocal_ns.__dict__.setdefault("contexts", []) + return cls._threadlocal_ns.__dict__.setdefault("contexts", []) def __enter__(self): self.get_contexts().append(self) def __exit__(self, type, value, traceback): From 06e1586ad0af989c85be48eac00ac9378f5ff260 Mon Sep 17 00:00:00 2001 From: Evan Hubinger Date: Mon, 30 Oct 2023 16:47:31 -0700 Subject: [PATCH 039/121] Add process/thread versions of collectby/mapreduce --- DOCS.md | 127 ++++++++++-------- coconut/compiler/templates/header.py_template | 18 +++ coconut/root.py | 2 +- .../tests/src/cocotest/agnostic/primary.coco | 11 ++ 4 files changed, 104 insertions(+), 54 deletions(-) diff --git a/DOCS.md b/DOCS.md index a32e27b13..c727ad5d0 100644 --- a/DOCS.md +++ b/DOCS.md @@ -4042,43 +4042,6 @@ assert "12345" |> windowsof$(3) |> list == [("1", "2", "3"), ("2", "3", "4"), (" **Python:** _Can't be done without the definition of `windowsof`; see the compiled header for the full definition._ -#### `collectby` and `mapreduce` - -**collectby**(_key\_func_, _iterable_, _value\_func_=`None`, \*, _reduce\_func_=`None`, _map\_using_=`None`) - -`collectby(key_func, iterable)` collects the items in `iterable` into a dictionary of lists keyed by `key_func(item)`. - -If `value_func` is passed, `collectby(key_func, iterable, value_func=value_func)` instead collects `value_func(item)` into each list instead of `item`. - -If `reduce_func` is passed, `collectby(key_func, iterable, reduce_func=reduce_func)`, instead of collecting the items into lists, reduces over the items of each key with `reduce_func`, effectively implementing a MapReduce operation. - -If `map_using` is passed, calculate `key_func` and `value_func` by mapping them over the iterable using `map_using` as `map`. Useful with [`process_map`](#process_map)/[`thread_map`](#thread_map). To immediately start calling `reduce_func` as soon as results arrive, pass `map_using=process_map$(stream=True)` (though note that `stream=True` requires the use of `process_map.multiple_sequential_calls`). - -`collectby` is similar to [`itertools.groupby`](https://docs.python.org/3/library/itertools.html#itertools.groupby) except that `collectby` aggregates common elements regardless of their order in the input iterable, whereas `groupby` only aggregates common elements that are adjacent in the input iterable. - -**mapreduce**(_key\_value\_func_, _iterable_, \*, _reduce\_func_=`None`, _map\_using_=`None`) - -`mapreduce(key_value_func, iterable)` functions the same as `collectby`, but allows calculating the keys and values together in one function. _key\_value\_func_ must return a 2-tuple of `(key, value)`. - -##### Example - -**Coconut:** -```coconut -user_balances = ( - balance_data - |> collectby$(.user, value_func=.balance, reduce_func=(+)) -) -``` - -**Python:** -```coconut_python -from collections import defaultdict - -user_balances = defaultdict(int) -for item in balance_data: - user_balances[item.user] += item.balance -``` - #### `all_equal` **all\_equal**(_iterable_) @@ -4108,7 +4071,7 @@ all_equal([1, 1, 1]) all_equal([1, 1, 2]) ``` -#### `process_map` +#### `process_map` and `thread_map` **process\_map**(_function_, *_iterables_, *, _chunksize_=`1`, _strict_=`False`, _stream_=`False`, _ordered_=`True`) @@ -4126,7 +4089,13 @@ _Deprecated: `parallel_map` is available as a deprecated alias for `process_map` If multiple sequential calls to `process_map` need to be made, it is highly recommended that they be done inside of a `with process_map.multiple_sequential_calls():` block, which will cause the different calls to use the same process pool and result in `process_map` immediately returning a list rather than a `process_map` object. If multiple sequential calls are necessary and the laziness of process_map is required, then the `process_map` objects should be constructed before the `multiple_sequential_calls` block and then only iterated over once inside the block. -`process_map.multiple_sequential_calls` also supports a `max_workers` argument to set the number of processes. +`process_map.multiple_sequential_calls` also supports a _max\_workers_ argument to set the number of processes. If `max_workers=None`, Coconut will pick a suitable _max\_workers_, including reusing worker pools from higher up in the call stack. + +**thread\_map**(_function_, *_iterables_, *, _chunksize_=`1`, _strict_=`False`, _stream_=`False`, _ordered_=`True`) + +Coconut provides a `multithreading`-based version of [`process_map`](#process_map) under the name `thread_map`. `thread_map` behaves identically to `process_map` (including support for `thread_map.multiple_sequential_calls`) except that it uses multithreading instead of multiprocessing, and is therefore primarily useful only for IO-bound tasks due to CPython's Global Interpreter Lock. + +_Deprecated: `concurrent_map` is available as a deprecated alias for `thread_map`. Note that deprecated features are disabled in `--strict` mode._ ##### Python Docs @@ -4136,7 +4105,13 @@ Equivalent to `map(func, *iterables)` except _func_ is executed asynchronously a `process_map` chops the iterable into a number of chunks which it submits to the process pool as separate tasks. The (approximate) size of these chunks can be specified by setting _chunksize_ to a positive integer. For very long iterables using a large value for _chunksize_ can make the job complete **much** faster than using the default value of `1`. -##### Example +**thread_map**(_func, \*iterables_, _chunksize_=`1`) + +Equivalent to `map(func, *iterables)` except _func_ is executed asynchronously and several calls to _func_ may be made concurrently. If a call raises an exception, then that exception will be raised when its value is retrieved from the iterator. + +`thread_map` chops the iterable into a number of chunks which it submits to the thread pool as separate tasks. The (approximate) size of these chunks can be specified by setting _chunksize_ to a positive integer. For very long iterables using a large value for _chunksize_ can make the job complete **much** faster than using the default value of `1`. + +##### Examples **Coconut:** ```coconut @@ -4151,35 +4126,81 @@ with Pool() as pool: print(list(pool.imap(functools.partial(pow, 2), range(100)))) ``` -#### `thread_map` +**Coconut:** +```coconut +thread_map(get_data_for_user, get_all_users()) |> list |> print +``` + +**Python:** +```coconut_python +import functools +import concurrent.futures +with concurrent.futures.ThreadPoolExecutor() as executor: + print(list(executor.map(get_data_for_user, get_all_users()))) +``` -**thread\_map**(_function_, *_iterables_, *, _chunksize_=`1`, _strict_=`False`, _stream_=`False`, _ordered_=`True`) -Coconut provides a `multithreading`-based version of [`process_map`](#process_map) under the name `thread_map`. `thread_map` behaves identically to `process_map` (including support for `thread_map.multiple_sequential_calls`) except that it uses multithreading instead of multiprocessing, and is therefore primarily useful only for IO-bound tasks due to CPython's Global Interpreter Lock. +#### `collectby` and `mapreduce` -_Deprecated: `concurrent_map` is available as a deprecated alias for `thread_map`. Note that deprecated features are disabled in `--strict` mode._ +**collectby**(_key\_func_, _iterable_, _value\_func_=`None`, \*, _reduce\_func_=`None`, _map\_using_=`None`) -##### Python Docs +`collectby(key_func, iterable)` collects the items in `iterable` into a dictionary of lists keyed by `key_func(item)`. -**thread_map**(_func, \*iterables_, _chunksize_=`1`) +If `value_func` is passed, `collectby(key_func, iterable, value_func=value_func)` instead collects `value_func(item)` into each list instead of `item`. -Equivalent to `map(func, *iterables)` except _func_ is executed asynchronously and several calls to _func_ may be made concurrently. If a call raises an exception, then that exception will be raised when its value is retrieved from the iterator. +If `reduce_func` is passed, `collectby(key_func, iterable, reduce_func=reduce_func)`, instead of collecting the items into lists, reduces over the items of each key with `reduce_func`, effectively implementing a MapReduce operation. -`thread_map` chops the iterable into a number of chunks which it submits to the thread pool as separate tasks. The (approximate) size of these chunks can be specified by setting _chunksize_ to a positive integer. For very long iterables using a large value for _chunksize_ can make the job complete **much** faster than using the default value of `1`. +If `map_using` is passed, calculate `key_func` and `value_func` by mapping them over the iterable using `map_using` as `map`. Useful with [`process_map`](#process_map)/[`thread_map`](#thread_map). See `.using_threads` and `.using_processes` methods below for simple shortcut methods that make use of `map_using` internally. + +`collectby` is similar to [`itertools.groupby`](https://docs.python.org/3/library/itertools.html#itertools.groupby) except that `collectby` aggregates common elements regardless of their order in the input iterable, whereas `groupby` only aggregates common elements that are adjacent in the input iterable. + +**mapreduce**(_key\_value\_func_, _iterable_, \*, _reduce\_func_=`None`, _map\_using_=`None`) + +`mapreduce(key_value_func, iterable)` functions the same as `collectby`, but allows calculating the keys and values together in one function. _key\_value\_func_ must return a 2-tuple of `(key, value)`. + +**collectby.using_threads**(_key\_func_, _iterable_, _value\_func_=`None`, \*, _reduce\_func_=`None`, _ordered_=`False`, _chunksize_=`1`, _max\_workers_=`None`) + +**collectby.using_processes**(_key\_func_, _iterable_, _value\_func_=`None`, \*, _reduce\_func_=`None`, _ordered_=`False`, _chunksize_=`1`, _max\_workers_=`None`) + +**mapreduce.using_threads**(_key\_value\_func_, _iterable_, \*, _reduce\_func_=`None`, _ordered_=`False`, _chunksize_=`1`, _max\_workers_=`None`) + +**mapreduce.using_processes**(_key\_value\_func_, _iterable_, \*, _reduce\_func_=`None`, _ordered_=`False`, _chunksize_=`1`, _max\_workers_=`None`) + +These shortcut methods call `collectby`/`mapreduce` with `map_using` set to [`process_map`](#process_map)/[`thread_map`](#thread_map), properly managed using the `.multiple_sequential_calls` method and the `stream=True` argument of [`process_map`](#process_map)/[`thread_map`](#thread_map). `reduce_func` will be called as soon as results arrive, and by default in whatever order they arrive in (to enforce the original order, pass _ordered_=`True`). Note that, for very long iterables, it is highly recommended to pass a value other than the default `1` for _chunksize_. + +As an example, `mapreduce.using_processes` is effectively equivalent to: +```coconut +def mapreduce.using_processes(key_value_func, iterable, *, reduce_func=None, ordered=False, chunksize=1, max_workers=None): + with process_map.multiple_sequential_calls(max_workers=max_workers): + return mapreduce( + key_value_func, + iterable, + reduce_func=reduce_func, + map_using=process_map$( + stream=True, + ordered=ordered, + chunksize=chunksize, + ), + ) +``` ##### Example **Coconut:** ```coconut -thread_map(get_data_for_user, get_all_users()) |> list |> print +user_balances = ( + balance_data + |> collectby$(.user, value_func=.balance, reduce_func=(+)) +) ``` **Python:** ```coconut_python -import functools -import concurrent.futures -with concurrent.futures.ThreadPoolExecutor() as executor: - print(list(executor.map(get_data_for_user, get_all_users()))) +from collections import defaultdict + +user_balances = defaultdict(int) +for item in balance_data: + user_balances[item.user] += item.balance ``` #### `tee` diff --git a/coconut/compiler/templates/header.py_template b/coconut/compiler/templates/header.py_template index 2d77e7720..5bb639d8b 100644 --- a/coconut/compiler/templates/header.py_template +++ b/coconut/compiler/templates/header.py_template @@ -888,6 +888,13 @@ class _coconut_base_parallel_map(map): finally: cls._get_pool_stack()[-1].terminate() cls._get_pool_stack()[-1] = None + elif max_workers is not None: + self.map_cls._get_pool_stack().append(cls._make_pool(max_workers)) + try: + yield + finally: + cls._get_pool_stack()[-1].terminate() + cls._get_pool_stack().pop() else: yield def _execute_map(self): @@ -1906,6 +1913,15 @@ def mapreduce(key_value_func, iterable, **kwargs): val = reduce_func(old_val, val) collection[key] = val return collection +def _coconut_parallel_mapreduce(mapreduce_func, map_cls, *args, **kwargs): + """Run collectby/mapreduce in parallel using threads or processes.""" + if "map_using" in kwargs: + raise _coconut.TypeError("redundant map_using argument to process/thread mapreduce/collectby") + kwargs["map_using"] = _coconut.functools.partial(map_cls, stream=True, ordered=kwargs.pop("ordered", False), chunksize=kwargs.pop("chunksize", 1)) + with map_cls.multiple_sequential_calls(max_workers=kwargs.pop("max_workers", None)): + return mapreduce_func(*args, **kwargs) +mapreduce.using_processes = _coconut.functools.partial(_coconut_parallel_mapreduce, mapreduce, process_map) +mapreduce.using_threads = _coconut.functools.partial(_coconut_parallel_mapreduce, mapreduce, thread_map) def collectby(key_func, iterable, value_func=None, **kwargs): """Collect the items in iterable into a dictionary of lists keyed by key_func(item). @@ -1918,6 +1934,8 @@ def collectby(key_func, iterable, value_func=None, **kwargs): the iterable using map_using as map. Useful with process_map/thread_map. """ return {_coconut_}mapreduce(_coconut_lifted(_coconut_comma_op, key_func, {_coconut_}ident if value_func is None else value_func), iterable, **kwargs) +collectby.using_processes = _coconut.functools.partial(_coconut_parallel_mapreduce, collectby, process_map) +collectby.using_threads = _coconut.functools.partial(_coconut_parallel_mapreduce, collectby, thread_map) def _namedtuple_of(**kwargs): """Construct an anonymous namedtuple of the given keyword arguments.""" {namedtuple_of_implementation} diff --git a/coconut/root.py b/coconut/root.py index 33140f22b..5d49b4a17 100644 --- a/coconut/root.py +++ b/coconut/root.py @@ -26,7 +26,7 @@ VERSION = "3.0.3" VERSION_NAME = None # False for release, int >= 1 for develop -DEVELOP = 14 +DEVELOP = 15 ALPHA = False # for pre releases rather than post releases assert DEVELOP is False or DEVELOP >= 1, "DEVELOP must be False or an int >= 1" diff --git a/coconut/tests/src/cocotest/agnostic/primary.coco b/coconut/tests/src/cocotest/agnostic/primary.coco index 38acc52e6..7db998c2c 100644 --- a/coconut/tests/src/cocotest/agnostic/primary.coco +++ b/coconut/tests/src/cocotest/agnostic/primary.coco @@ -1689,4 +1689,15 @@ def primary_test() -> bool: assert list(py_xs) == [] assert count()[:10:2] == range(0, 10, 2) assert count()[10:2] == range(10, 2) + some_data = [ + (name="a", val="123"), + (name="b", val="567"), + ] + for mapreducer in ( + mapreduce.using_processes$(lift(,)(.name, .val)), # type: ignore + mapreduce.using_threads$(lift(,)(.name, .val)), # type: ignore + collectby.using_processes$(.name, value_func=.val), # type: ignore + collectby.using_threads$(.name, value_func=.val), # type: ignore + ): + assert some_data |> mapreducer == {"a": ["123"], "b": ["567"]} return True From 4e11b6ded2c23f8f5a13e43f36d31aac1697bb3c Mon Sep 17 00:00:00 2001 From: Evan Hubinger Date: Mon, 30 Oct 2023 16:47:57 -0700 Subject: [PATCH 040/121] Remove docstring --- coconut/compiler/templates/header.py_template | 1 - 1 file changed, 1 deletion(-) diff --git a/coconut/compiler/templates/header.py_template b/coconut/compiler/templates/header.py_template index 5bb639d8b..d05f720b1 100644 --- a/coconut/compiler/templates/header.py_template +++ b/coconut/compiler/templates/header.py_template @@ -1914,7 +1914,6 @@ def mapreduce(key_value_func, iterable, **kwargs): collection[key] = val return collection def _coconut_parallel_mapreduce(mapreduce_func, map_cls, *args, **kwargs): - """Run collectby/mapreduce in parallel using threads or processes.""" if "map_using" in kwargs: raise _coconut.TypeError("redundant map_using argument to process/thread mapreduce/collectby") kwargs["map_using"] = _coconut.functools.partial(map_cls, stream=True, ordered=kwargs.pop("ordered", False), chunksize=kwargs.pop("chunksize", 1)) From 57c0386519867204769445b0ef9a6dcc7d328718 Mon Sep 17 00:00:00 2001 From: Evan Hubinger Date: Mon, 30 Oct 2023 16:53:04 -0700 Subject: [PATCH 041/121] Improve docs --- DOCS.md | 22 ++++++++++++---------- 1 file changed, 12 insertions(+), 10 deletions(-) diff --git a/DOCS.md b/DOCS.md index c727ad5d0..55971217e 100644 --- a/DOCS.md +++ b/DOCS.md @@ -4073,7 +4073,7 @@ all_equal([1, 1, 2]) #### `process_map` and `thread_map` -**process\_map**(_function_, *_iterables_, *, _chunksize_=`1`, _strict_=`False`, _stream_=`False`, _ordered_=`True`) +##### **process\_map**(_function_, *_iterables_, *, _chunksize_=`1`, _strict_=`False`, _stream_=`False`, _ordered_=`True`) Coconut provides a `multiprocessing`-based version of `map` under the name `process_map`. `process_map` makes use of multiple processes, and is therefore much faster than `map` for CPU-bound tasks. If any exceptions are raised inside of `process_map`, a traceback will be printed as soon as they are encountered. Results will be in the same order as the input unless _ordered_=`False`. @@ -4085,15 +4085,17 @@ Because `process_map` uses multiple processes for its execution, it is necessary _Deprecated: `parallel_map` is available as a deprecated alias for `process_map`. Note that deprecated features are disabled in `--strict` mode._ -**process\_map\.multiple\_sequential\_calls**(_max\_workers_=`None`) +##### **process\_map\.multiple\_sequential\_calls**(_max\_workers_=`None`) If multiple sequential calls to `process_map` need to be made, it is highly recommended that they be done inside of a `with process_map.multiple_sequential_calls():` block, which will cause the different calls to use the same process pool and result in `process_map` immediately returning a list rather than a `process_map` object. If multiple sequential calls are necessary and the laziness of process_map is required, then the `process_map` objects should be constructed before the `multiple_sequential_calls` block and then only iterated over once inside the block. `process_map.multiple_sequential_calls` also supports a _max\_workers_ argument to set the number of processes. If `max_workers=None`, Coconut will pick a suitable _max\_workers_, including reusing worker pools from higher up in the call stack. -**thread\_map**(_function_, *_iterables_, *, _chunksize_=`1`, _strict_=`False`, _stream_=`False`, _ordered_=`True`) +##### **thread\_map**(_function_, *_iterables_, *, _chunksize_=`1`, _strict_=`False`, _stream_=`False`, _ordered_=`True`) -Coconut provides a `multithreading`-based version of [`process_map`](#process_map) under the name `thread_map`. `thread_map` behaves identically to `process_map` (including support for `thread_map.multiple_sequential_calls`) except that it uses multithreading instead of multiprocessing, and is therefore primarily useful only for IO-bound tasks due to CPython's Global Interpreter Lock. +##### **thread\_map\.multiple\_sequential\_calls**(_max\_workers_=`None`) + +Coconut provides a `multithreading`-based version of [`process_map`](#process_map) under the name `thread_map`. `thread_map` and `thread_map.multiple_sequential_calls` behave identically to `process_map` except that they use multithreading instead of multiprocessing, and are therefore primarily useful only for IO-bound tasks due to CPython's Global Interpreter Lock. _Deprecated: `concurrent_map` is available as a deprecated alias for `thread_map`. Note that deprecated features are disabled in `--strict` mode._ @@ -4142,7 +4144,7 @@ with concurrent.futures.ThreadPoolExecutor() as executor: #### `collectby` and `mapreduce` -**collectby**(_key\_func_, _iterable_, _value\_func_=`None`, \*, _reduce\_func_=`None`, _map\_using_=`None`) +##### **collectby**(_key\_func_, _iterable_, _value\_func_=`None`, \*, _reduce\_func_=`None`, _map\_using_=`None`) `collectby(key_func, iterable)` collects the items in `iterable` into a dictionary of lists keyed by `key_func(item)`. @@ -4154,17 +4156,17 @@ If `map_using` is passed, calculate `key_func` and `value_func` by mapping them `collectby` is similar to [`itertools.groupby`](https://docs.python.org/3/library/itertools.html#itertools.groupby) except that `collectby` aggregates common elements regardless of their order in the input iterable, whereas `groupby` only aggregates common elements that are adjacent in the input iterable. -**mapreduce**(_key\_value\_func_, _iterable_, \*, _reduce\_func_=`None`, _map\_using_=`None`) +##### **mapreduce**(_key\_value\_func_, _iterable_, \*, _reduce\_func_=`None`, _map\_using_=`None`) `mapreduce(key_value_func, iterable)` functions the same as `collectby`, but allows calculating the keys and values together in one function. _key\_value\_func_ must return a 2-tuple of `(key, value)`. -**collectby.using_threads**(_key\_func_, _iterable_, _value\_func_=`None`, \*, _reduce\_func_=`None`, _ordered_=`False`, _chunksize_=`1`, _max\_workers_=`None`) +##### **collectby.using_threads**(_key\_func_, _iterable_, _value\_func_=`None`, \*, _reduce\_func_=`None`, _ordered_=`False`, _chunksize_=`1`, _max\_workers_=`None`) -**collectby.using_processes**(_key\_func_, _iterable_, _value\_func_=`None`, \*, _reduce\_func_=`None`, _ordered_=`False`, _chunksize_=`1`, _max\_workers_=`None`) +##### **collectby.using_processes**(_key\_func_, _iterable_, _value\_func_=`None`, \*, _reduce\_func_=`None`, _ordered_=`False`, _chunksize_=`1`, _max\_workers_=`None`) -**mapreduce.using_threads**(_key\_value\_func_, _iterable_, \*, _reduce\_func_=`None`, _ordered_=`False`, _chunksize_=`1`, _max\_workers_=`None`) +##### **mapreduce.using_threads**(_key\_value\_func_, _iterable_, \*, _reduce\_func_=`None`, _ordered_=`False`, _chunksize_=`1`, _max\_workers_=`None`) -**mapreduce.using_processes**(_key\_value\_func_, _iterable_, \*, _reduce\_func_=`None`, _ordered_=`False`, _chunksize_=`1`, _max\_workers_=`None`) +##### **mapreduce.using_processes**(_key\_value\_func_, _iterable_, \*, _reduce\_func_=`None`, _ordered_=`False`, _chunksize_=`1`, _max\_workers_=`None`) These shortcut methods call `collectby`/`mapreduce` with `map_using` set to [`process_map`](#process_map)/[`thread_map`](#thread_map), properly managed using the `.multiple_sequential_calls` method and the `stream=True` argument of [`process_map`](#process_map)/[`thread_map`](#thread_map). `reduce_func` will be called as soon as results arrive, and by default in whatever order they arrive in (to enforce the original order, pass _ordered_=`True`). Note that, for very long iterables, it is highly recommended to pass a value other than the default `1` for _chunksize_. From f24588eb49ad36e35c20fb75722aa24cb7447e05 Mon Sep 17 00:00:00 2001 From: Evan Hubinger Date: Tue, 31 Oct 2023 00:36:33 -0700 Subject: [PATCH 042/121] Minor doc cleanup --- DOCS.md | 14 +++++++------- 1 file changed, 7 insertions(+), 7 deletions(-) diff --git a/DOCS.md b/DOCS.md index 55971217e..c95aeb25d 100644 --- a/DOCS.md +++ b/DOCS.md @@ -13,7 +13,7 @@ depth: 2 ## Overview -This documentation covers all the features of the [Coconut Programming Language](http://evhub.github.io/coconut/), and is intended as a reference/specification, not a tutorialized introduction. For a full introduction and tutorial of Coconut, see [the tutorial](./HELP.md). +This documentation covers all the features of the [Coconut Programming Language](http://evhub.github.io/coconut/), and is intended as a reference/specification, not a tutorialized introduction. For an introduction to and tutorial of Coconut, see [the tutorial](./HELP.md). Coconut is a variant of [Python](https://www.python.org/) built for **simple, elegant, Pythonic functional programming**. Coconut syntax is a strict superset of the latest Python 3 syntax. Thus, users familiar with Python will already be familiar with most of Coconut. @@ -407,7 +407,7 @@ Simply installing Coconut should add a `Coconut` kernel to your Jupyter/IPython The Coconut kernel will always compile using the parameters: `--target sys --line-numbers --keep-lines --no-wrap-types`. -Coconut also provides the following api commands: +Coconut also provides the following commands: - `coconut --jupyter notebook` will ensure that the Coconut kernel is available and launch a Jupyter/IPython notebook. - `coconut --jupyter console` will launch a Jupyter/IPython console using the Coconut kernel. @@ -517,11 +517,11 @@ f x n/a +, - left <<, >> left & left -&: left +&: yes ^ left | left -:: n/a (lazy) -.. n/a +:: yes (lazy) +.. yes a `b` c, left (captures lambda) all custom operators ?? left (short-circuits) @@ -536,7 +536,7 @@ a `b` c, left (captures lambda) not unary and left (short-circuits) or left (short-circuits) -x if c else y, ternary left (short-circuits) +x if c else y, ternary (short-circuits) if c then x else y => right ====================== ========================== @@ -3851,7 +3851,7 @@ for x in input_data: Coconut provides a modified version of `itertools.count` that supports `in`, normal slicing, optimized iterator slicing, the standard `count` and `index` sequence methods, `repr`, and `start`/`step` attributes as a built-in under the name `count`. If the _step_ parameter is set to `None`, `count` will behave like `itertools.repeat` instead. -Since `count` supports slicing, `count()` can be used as a version of `range` that can in some cases be more readable. In particular, it is easy to accidentally write `range(10, 2)` when you meant `range(0, 10, 2)`, but it is hard to accidentally write `count()[10:2]` when you mean `count()[:10:2]`. +Since `count` supports slicing, `count()[...]` can be used as a version of `range` that can in some cases be more readable. In particular, it is easy to accidentally write `range(10, 2)` when you meant `range(0, 10, 2)`, but it is hard to accidentally write `count()[10:2]` when you mean `count()[:10:2]`. ##### Python Docs From 5baa6fc5a27b1673ccb1054760d58ffcc5e29a80 Mon Sep 17 00:00:00 2001 From: Evan Hubinger Date: Tue, 31 Oct 2023 02:32:02 -0700 Subject: [PATCH 043/121] Improve mapreduce --- DOCS.md | 20 ++++++++------- __coconut__/__init__.pyi | 25 ++++++++++++++++--- coconut/compiler/templates/header.py_template | 5 +++- coconut/root.py | 2 +- .../tests/src/cocotest/agnostic/primary.coco | 1 + coconut/tests/src/extras.coco | 5 ++++ 6 files changed, 44 insertions(+), 14 deletions(-) diff --git a/DOCS.md b/DOCS.md index c95aeb25d..3da099ebf 100644 --- a/DOCS.md +++ b/DOCS.md @@ -4144,29 +4144,31 @@ with concurrent.futures.ThreadPoolExecutor() as executor: #### `collectby` and `mapreduce` -##### **collectby**(_key\_func_, _iterable_, _value\_func_=`None`, \*, _reduce\_func_=`None`, _map\_using_=`None`) +##### **collectby**(_key\_func_, _iterable_, _value\_func_=`None`, \*, _reduce\_func_=`None`, _init\_collection_=`None`, _map\_using_=`None`) `collectby(key_func, iterable)` collects the items in `iterable` into a dictionary of lists keyed by `key_func(item)`. -If `value_func` is passed, `collectby(key_func, iterable, value_func=value_func)` instead collects `value_func(item)` into each list instead of `item`. +If _value\_func_ is passed, instead collects `value_func(item)` into each list instead of `item`. -If `reduce_func` is passed, `collectby(key_func, iterable, reduce_func=reduce_func)`, instead of collecting the items into lists, reduces over the items of each key with `reduce_func`, effectively implementing a MapReduce operation. +If _reduce\_func_ is passed, instead of collecting the items into lists, reduces over the items of each key with `reduce_func`, effectively implementing a MapReduce operation. If keys are intended to be unique, set `reduce_func=False`. -If `map_using` is passed, calculate `key_func` and `value_func` by mapping them over the iterable using `map_using` as `map`. Useful with [`process_map`](#process_map)/[`thread_map`](#thread_map). See `.using_threads` and `.using_processes` methods below for simple shortcut methods that make use of `map_using` internally. +If _init\_collection_ is passed, initializes the collection from _init\_collection_ rather than as a `collections.defaultdict` (if `reduce_func=None`) or an empty `dict` (otherwise). Useful when you want to collect the results into a `pandas.DataFrame`. + +If _map_using_ is passed, calculates `key_func` and `value_func` by mapping them over the iterable using `map_using` as `map`. Useful with [`process_map`](#process_map)/[`thread_map`](#thread_map). See `.using_threads` and `.using_processes` methods below for simple shortcut methods that make use of `map_using` internally. `collectby` is similar to [`itertools.groupby`](https://docs.python.org/3/library/itertools.html#itertools.groupby) except that `collectby` aggregates common elements regardless of their order in the input iterable, whereas `groupby` only aggregates common elements that are adjacent in the input iterable. -##### **mapreduce**(_key\_value\_func_, _iterable_, \*, _reduce\_func_=`None`, _map\_using_=`None`) +##### **mapreduce**(_key\_value\_func_, _iterable_, \*, _reduce\_func_=`None`, _init\_collection_=`None`, _map\_using_=`None`) `mapreduce(key_value_func, iterable)` functions the same as `collectby`, but allows calculating the keys and values together in one function. _key\_value\_func_ must return a 2-tuple of `(key, value)`. -##### **collectby.using_threads**(_key\_func_, _iterable_, _value\_func_=`None`, \*, _reduce\_func_=`None`, _ordered_=`False`, _chunksize_=`1`, _max\_workers_=`None`) +##### **collectby.using_threads**(_key\_func_, _iterable_, _value\_func_=`None`, \*, _reduce\_func_=`None`, _init\_collection_=`None`, _ordered_=`False`, _chunksize_=`1`, _max\_workers_=`None`) -##### **collectby.using_processes**(_key\_func_, _iterable_, _value\_func_=`None`, \*, _reduce\_func_=`None`, _ordered_=`False`, _chunksize_=`1`, _max\_workers_=`None`) +##### **collectby.using_processes**(_key\_func_, _iterable_, _value\_func_=`None`, \*, _reduce\_func_=`None`, _init\_collection_=`None`, _ordered_=`False`, _chunksize_=`1`, _max\_workers_=`None`) -##### **mapreduce.using_threads**(_key\_value\_func_, _iterable_, \*, _reduce\_func_=`None`, _ordered_=`False`, _chunksize_=`1`, _max\_workers_=`None`) +##### **mapreduce.using_threads**(_key\_value\_func_, _iterable_, \*, _reduce\_func_=`None`, _init\_collection_=`None`, _ordered_=`False`, _chunksize_=`1`, _max\_workers_=`None`) -##### **mapreduce.using_processes**(_key\_value\_func_, _iterable_, \*, _reduce\_func_=`None`, _ordered_=`False`, _chunksize_=`1`, _max\_workers_=`None`) +##### **mapreduce.using_processes**(_key\_value\_func_, _iterable_, \*, _reduce\_func_=`None`, _init\_collection_=`None`, _ordered_=`False`, _chunksize_=`1`, _max\_workers_=`None`) These shortcut methods call `collectby`/`mapreduce` with `map_using` set to [`process_map`](#process_map)/[`thread_map`](#thread_map), properly managed using the `.multiple_sequential_calls` method and the `stream=True` argument of [`process_map`](#process_map)/[`thread_map`](#thread_map). `reduce_func` will be called as soon as results arrive, and by default in whatever order they arrive in (to enforce the original order, pass _ordered_=`True`). Note that, for very long iterables, it is highly recommended to pass a value other than the default `1` for _chunksize_. diff --git a/__coconut__/__init__.pyi b/__coconut__/__init__.pyi index 67f784cf4..6aabf8c14 100644 --- a/__coconut__/__init__.pyi +++ b/__coconut__/__init__.pyi @@ -1604,7 +1604,7 @@ def collectby( *, reduce_func: _t.Callable[[_T, _T], _V], map_using: _t.Callable | None = None, -) -> _t.DefaultDict[_U, _V]: ... +) -> _t.Dict[_U, _V]: ... @_t.overload def collectby( key_func: _t.Callable[[_T], _U], @@ -1621,7 +1621,17 @@ def collectby( *, reduce_func: _t.Callable[[_W, _W], _V], map_using: _t.Callable | None = None, -) -> _t.DefaultDict[_U, _V]: +) -> _t.Dict[_U, _V]: ... +@_t.overload +def collectby( + key_func: _t.Callable, + iterable: _t.Iterable, + value_func: _t.Callable | None = None, + *, + reduce_func: _t.Callable | None | _t.Literal[False] = None, + map_using: _t.Callable | None = None, + init_collection: _T +) -> _T: """Collect the items in iterable into a dictionary of lists keyed by key_func(item). If value_func is passed, collect value_func(item) into each list instead of item. @@ -1649,7 +1659,16 @@ def mapreduce( *, reduce_func: _t.Callable[[_W, _W], _V], map_using: _t.Callable | None = None, -) -> _t.DefaultDict[_U, _V]: +) -> _t.Dict[_U, _V]: ... +@_t.overload +def mapreduce( + key_value_func: _t.Callable, + iterable: _t.Iterable, + *, + reduce_func: _t.Callable | None | _t.Literal[False] = None, + map_using: _t.Callable | None = None, + init_collection: _T +) -> _T: """Map key_value_func over iterable, then collect the values into a dictionary of lists keyed by the keys. If reduce_func is passed, instead of collecting the values into lists, reduce over diff --git a/coconut/compiler/templates/header.py_template b/coconut/compiler/templates/header.py_template index d05f720b1..39c8974e6 100644 --- a/coconut/compiler/templates/header.py_template +++ b/coconut/compiler/templates/header.py_template @@ -1900,16 +1900,19 @@ def mapreduce(key_value_func, iterable, **kwargs): the iterable using map_using as map. Useful with process_map/thread_map. """ reduce_func = kwargs.pop("reduce_func", None) + init_collection = kwargs.pop("init_collection", None) map_using = kwargs.pop("map_using", _coconut.map) if kwargs: raise _coconut.TypeError("mapreduce()/collectby() got unexpected keyword arguments " + _coconut.repr(kwargs)) - collection = _coconut.collections.defaultdict(_coconut.list) if reduce_func is None else {empty_dict} + collection = init_collection if init_collection is not None else _coconut.collections.defaultdict(_coconut.list) if reduce_func is None else {empty_dict} for key, val in map_using(key_value_func, iterable): if reduce_func is None: collection[key].append(val) else: old_val = collection.get(key, _coconut_sentinel) if old_val is not _coconut_sentinel: + if reduce_func is False: + raise ValueError("duplicate key " + repr(key) + " with reduce_func=False") val = reduce_func(old_val, val) collection[key] = val return collection diff --git a/coconut/root.py b/coconut/root.py index 5d49b4a17..b78fdbac9 100644 --- a/coconut/root.py +++ b/coconut/root.py @@ -26,7 +26,7 @@ VERSION = "3.0.3" VERSION_NAME = None # False for release, int >= 1 for develop -DEVELOP = 15 +DEVELOP = 16 ALPHA = False # for pre releases rather than post releases assert DEVELOP is False or DEVELOP >= 1, "DEVELOP must be False or an int >= 1" diff --git a/coconut/tests/src/cocotest/agnostic/primary.coco b/coconut/tests/src/cocotest/agnostic/primary.coco index 7db998c2c..c8dccb37a 100644 --- a/coconut/tests/src/cocotest/agnostic/primary.coco +++ b/coconut/tests/src/cocotest/agnostic/primary.coco @@ -1700,4 +1700,5 @@ def primary_test() -> bool: collectby.using_threads$(.name, value_func=.val), # type: ignore ): assert some_data |> mapreducer == {"a": ["123"], "b": ["567"]} + assert_raises(-> collectby(.[0], [(0, 1), (0, 2)], reduce_func=False), ValueError) return True diff --git a/coconut/tests/src/extras.coco b/coconut/tests/src/extras.coco index 4b08eb743..7bb529f9b 100644 --- a/coconut/tests/src/extras.coco +++ b/coconut/tests/src/extras.coco @@ -605,6 +605,11 @@ def test_pandas() -> bool: ], dtype=object) # type: ignore d4 = d1 |> fmap$(def r -> r["nums2"] = r["nums"]*2; r) assert (d4["nums"] * 2 == d4["nums2"]).all() + df = pd.DataFrame({"123": [1, 2, 3]}) + mapreduce(ident, [("123", [4, 5, 6])], init_collection=df) + assert df["123"] |> list == [4, 5, 6] + mapreduce(ident, [("789", [7, 8, 9])], init_collection=df, reduce_func=False) + assert df["789"] |> list == [7, 8, 9] return True From 959ce347c46b42c2f1a26698515d3484236d0da0 Mon Sep 17 00:00:00 2001 From: Evan Hubinger Date: Tue, 31 Oct 2023 12:51:51 -0700 Subject: [PATCH 044/121] Improve mapreduce/collectby --- DOCS.md | 16 ++++++++-------- __coconut__/__init__.pyi | 4 ++-- coconut/compiler/templates/header.py_template | 6 +++--- coconut/tests/src/extras.coco | 6 +++--- 4 files changed, 16 insertions(+), 16 deletions(-) diff --git a/DOCS.md b/DOCS.md index 3da099ebf..ec7f4158e 100644 --- a/DOCS.md +++ b/DOCS.md @@ -4144,31 +4144,31 @@ with concurrent.futures.ThreadPoolExecutor() as executor: #### `collectby` and `mapreduce` -##### **collectby**(_key\_func_, _iterable_, _value\_func_=`None`, \*, _reduce\_func_=`None`, _init\_collection_=`None`, _map\_using_=`None`) +##### **collectby**(_key\_func_, _iterable_, _value\_func_=`None`, \*, _reduce\_func_=`None`, _collect\_in_=`None`, _map\_using_=`None`) `collectby(key_func, iterable)` collects the items in `iterable` into a dictionary of lists keyed by `key_func(item)`. If _value\_func_ is passed, instead collects `value_func(item)` into each list instead of `item`. -If _reduce\_func_ is passed, instead of collecting the items into lists, reduces over the items of each key with `reduce_func`, effectively implementing a MapReduce operation. If keys are intended to be unique, set `reduce_func=False`. +If _reduce\_func_ is passed, instead of collecting the items into lists, reduces over the items of each key with `reduce_func`, effectively implementing a MapReduce operation. If keys are intended to be unique, set `reduce_func=False` (`reduce_func=False` is also the default if _collect\_in_ is passed). -If _init\_collection_ is passed, initializes the collection from _init\_collection_ rather than as a `collections.defaultdict` (if `reduce_func=None`) or an empty `dict` (otherwise). Useful when you want to collect the results into a `pandas.DataFrame`. +If _collect\_in_ is passed, initializes the collection from _collect\_in_ rather than as a `collections.defaultdict` (if `reduce_func=None`) or an empty `dict` (otherwise). Additionally, _reduce\_func_ defaults to `False` rather than `None` when _collect\_in_ is passed. Useful when you want to collect the results into a `pandas.DataFrame`. If _map_using_ is passed, calculates `key_func` and `value_func` by mapping them over the iterable using `map_using` as `map`. Useful with [`process_map`](#process_map)/[`thread_map`](#thread_map). See `.using_threads` and `.using_processes` methods below for simple shortcut methods that make use of `map_using` internally. `collectby` is similar to [`itertools.groupby`](https://docs.python.org/3/library/itertools.html#itertools.groupby) except that `collectby` aggregates common elements regardless of their order in the input iterable, whereas `groupby` only aggregates common elements that are adjacent in the input iterable. -##### **mapreduce**(_key\_value\_func_, _iterable_, \*, _reduce\_func_=`None`, _init\_collection_=`None`, _map\_using_=`None`) +##### **mapreduce**(_key\_value\_func_, _iterable_, \*, _reduce\_func_=`None`, _collect\_in_=`None`, _map\_using_=`None`) `mapreduce(key_value_func, iterable)` functions the same as `collectby`, but allows calculating the keys and values together in one function. _key\_value\_func_ must return a 2-tuple of `(key, value)`. -##### **collectby.using_threads**(_key\_func_, _iterable_, _value\_func_=`None`, \*, _reduce\_func_=`None`, _init\_collection_=`None`, _ordered_=`False`, _chunksize_=`1`, _max\_workers_=`None`) +##### **collectby.using_threads**(_key\_func_, _iterable_, _value\_func_=`None`, \*, _reduce\_func_=`None`, _collect\_in_=`None`, _ordered_=`False`, _chunksize_=`1`, _max\_workers_=`None`) -##### **collectby.using_processes**(_key\_func_, _iterable_, _value\_func_=`None`, \*, _reduce\_func_=`None`, _init\_collection_=`None`, _ordered_=`False`, _chunksize_=`1`, _max\_workers_=`None`) +##### **collectby.using_processes**(_key\_func_, _iterable_, _value\_func_=`None`, \*, _reduce\_func_=`None`, _collect\_in_=`None`, _ordered_=`False`, _chunksize_=`1`, _max\_workers_=`None`) -##### **mapreduce.using_threads**(_key\_value\_func_, _iterable_, \*, _reduce\_func_=`None`, _init\_collection_=`None`, _ordered_=`False`, _chunksize_=`1`, _max\_workers_=`None`) +##### **mapreduce.using_threads**(_key\_value\_func_, _iterable_, \*, _reduce\_func_=`None`, _collect\_in_=`None`, _ordered_=`False`, _chunksize_=`1`, _max\_workers_=`None`) -##### **mapreduce.using_processes**(_key\_value\_func_, _iterable_, \*, _reduce\_func_=`None`, _init\_collection_=`None`, _ordered_=`False`, _chunksize_=`1`, _max\_workers_=`None`) +##### **mapreduce.using_processes**(_key\_value\_func_, _iterable_, \*, _reduce\_func_=`None`, _collect\_in_=`None`, _ordered_=`False`, _chunksize_=`1`, _max\_workers_=`None`) These shortcut methods call `collectby`/`mapreduce` with `map_using` set to [`process_map`](#process_map)/[`thread_map`](#thread_map), properly managed using the `.multiple_sequential_calls` method and the `stream=True` argument of [`process_map`](#process_map)/[`thread_map`](#thread_map). `reduce_func` will be called as soon as results arrive, and by default in whatever order they arrive in (to enforce the original order, pass _ordered_=`True`). Note that, for very long iterables, it is highly recommended to pass a value other than the default `1` for _chunksize_. diff --git a/__coconut__/__init__.pyi b/__coconut__/__init__.pyi index 6aabf8c14..804609004 100644 --- a/__coconut__/__init__.pyi +++ b/__coconut__/__init__.pyi @@ -1628,9 +1628,9 @@ def collectby( iterable: _t.Iterable, value_func: _t.Callable | None = None, *, + collect_in: _T, reduce_func: _t.Callable | None | _t.Literal[False] = None, map_using: _t.Callable | None = None, - init_collection: _T ) -> _T: """Collect the items in iterable into a dictionary of lists keyed by key_func(item). @@ -1665,9 +1665,9 @@ def mapreduce( key_value_func: _t.Callable, iterable: _t.Iterable, *, + collect_in: _T, reduce_func: _t.Callable | None | _t.Literal[False] = None, map_using: _t.Callable | None = None, - init_collection: _T ) -> _T: """Map key_value_func over iterable, then collect the values into a dictionary of lists keyed by the keys. diff --git a/coconut/compiler/templates/header.py_template b/coconut/compiler/templates/header.py_template index 39c8974e6..0eb8e1417 100644 --- a/coconut/compiler/templates/header.py_template +++ b/coconut/compiler/templates/header.py_template @@ -1899,12 +1899,12 @@ def mapreduce(key_value_func, iterable, **kwargs): If map_using is passed, calculate key_value_func by mapping them over the iterable using map_using as map. Useful with process_map/thread_map. """ - reduce_func = kwargs.pop("reduce_func", None) - init_collection = kwargs.pop("init_collection", None) + collect_in = kwargs.pop("collect_in", None) + reduce_func = kwargs.pop("reduce_func", None if collect_in is None else False) map_using = kwargs.pop("map_using", _coconut.map) if kwargs: raise _coconut.TypeError("mapreduce()/collectby() got unexpected keyword arguments " + _coconut.repr(kwargs)) - collection = init_collection if init_collection is not None else _coconut.collections.defaultdict(_coconut.list) if reduce_func is None else {empty_dict} + collection = collect_in if collect_in is not None else _coconut.collections.defaultdict(_coconut.list) if reduce_func is None else {empty_dict} for key, val in map_using(key_value_func, iterable): if reduce_func is None: collection[key].append(val) diff --git a/coconut/tests/src/extras.coco b/coconut/tests/src/extras.coco index 7bb529f9b..2e012d6c0 100644 --- a/coconut/tests/src/extras.coco +++ b/coconut/tests/src/extras.coco @@ -606,9 +606,9 @@ def test_pandas() -> bool: d4 = d1 |> fmap$(def r -> r["nums2"] = r["nums"]*2; r) assert (d4["nums"] * 2 == d4["nums2"]).all() df = pd.DataFrame({"123": [1, 2, 3]}) - mapreduce(ident, [("123", [4, 5, 6])], init_collection=df) - assert df["123"] |> list == [4, 5, 6] - mapreduce(ident, [("789", [7, 8, 9])], init_collection=df, reduce_func=False) + mapreduce(ident, [("456", [4, 5, 6])], collect_in=df) + assert df["456"] |> list == [4, 5, 6] + mapreduce(ident, [("789", [7, 8, 9])], collect_in=df, reduce_func=False) assert df["789"] |> list == [7, 8, 9] return True From 0c7a5af2260cf840d9a13cfe5a984b5aea8d154f Mon Sep 17 00:00:00 2001 From: Evan Hubinger Date: Tue, 31 Oct 2023 14:30:34 -0700 Subject: [PATCH 045/121] Add Expected.handle --- DOCS.md | 6 ++++++ __coconut__/__init__.pyi | 7 +++++++ coconut/compiler/templates/header.py_template | 11 ++++++++++- coconut/tests/src/cocotest/agnostic/suite.coco | 2 ++ coconut/tests/src/cocotest/agnostic/util.coco | 4 ++++ 5 files changed, 29 insertions(+), 1 deletion(-) diff --git a/DOCS.md b/DOCS.md index ec7f4158e..f4e6d8153 100644 --- a/DOCS.md +++ b/DOCS.md @@ -3186,6 +3186,10 @@ data Expected[T](result: T? = None, error: BaseException? = None): if not self: raise self.error return self.result + def handle(self, err_type, handler: BaseException -> T) -> Expected[T]: + if not self and _coconut.isinstance(self.error, err_type): + return self.__class__(handler(self.error)) + return self ``` `Expected` is primarily used as the return type for [`safe_call`](#safe_call). Generally, the best way to use `Expected` is with [`fmap`](#fmap), which will apply a function to the result if it exists, or otherwise retain the error. If you want to sequence multiple `Expected`-returning operations, `.and_then` should be used instead of `fmap`. @@ -3336,6 +3340,8 @@ def safe_call(f, /, *args, **kwargs): return Expected(error=err) ``` +To define a function that always returns an `Expected` rather than raising any errors, simply decorate it with `@safe_call$`. + ##### Example **Coconut:** diff --git a/__coconut__/__init__.pyi b/__coconut__/__init__.pyi index 804609004..def115f7b 100644 --- a/__coconut__/__init__.pyi +++ b/__coconut__/__init__.pyi @@ -354,6 +354,10 @@ class Expected(_BaseExpected[_T]): if not self: raise self.error return self.result + def handle(self, err_type, handler: BaseException -> T) -> Expected[T]: + if not self and _coconut.isinstance(self.error, err_type): + return self.__class__(handler(self.error)) + return self ''' __slots__ = () _coconut_is_data = True @@ -416,6 +420,9 @@ class Expected(_BaseExpected[_T]): def unwrap(self) -> _T: """Unwrap the result or raise the error.""" ... + def handle(self, err_type: _t.Type[BaseException], handler: _t.Callable[[BaseException], _T]) -> Expected[_T]: + """Recover from the given err_type by calling handler on the error to determine the result.""" + ... _coconut_Expected = Expected diff --git a/coconut/compiler/templates/header.py_template b/coconut/compiler/templates/header.py_template index 0eb8e1417..4edc7ce34 100644 --- a/coconut/compiler/templates/header.py_template +++ b/coconut/compiler/templates/header.py_template @@ -1741,6 +1741,10 @@ class Expected(_coconut.collections.namedtuple("Expected", ("result", "error")){ if not self: raise self.error return self.result + def handle(self, err_type, handler: BaseException -> T) -> Expected[T]: + if not self and _coconut.isinstance(self.error, err_type): + return self.__class__(handler(self.error)) + return self ''' __slots__ = () {is_data_var} = True @@ -1803,6 +1807,11 @@ class Expected(_coconut.collections.namedtuple("Expected", ("result", "error")){ if not self: raise self.error return self.result + def handle(self, err_type, handler): + """Recover from the given err_type by calling handler on the error to determine the result.""" + if not self and _coconut.isinstance(self.error, err_type): + return self.__class__(handler(self.error)) + return self class flip(_coconut_base_callable): """Given a function, return a new function with inverse argument order. If nargs is passed, only the first nargs arguments are reversed.""" @@ -1912,7 +1921,7 @@ def mapreduce(key_value_func, iterable, **kwargs): old_val = collection.get(key, _coconut_sentinel) if old_val is not _coconut_sentinel: if reduce_func is False: - raise ValueError("duplicate key " + repr(key) + " with reduce_func=False") + raise ValueError("mapreduce()/collectby() got duplicate key " + repr(key) + " with reduce_func=False") val = reduce_func(old_val, val) collection[key] = val return collection diff --git a/coconut/tests/src/cocotest/agnostic/suite.coco b/coconut/tests/src/cocotest/agnostic/suite.coco index a13d6c538..64da67027 100644 --- a/coconut/tests/src/cocotest/agnostic/suite.coco +++ b/coconut/tests/src/cocotest/agnostic/suite.coco @@ -1065,6 +1065,8 @@ forward 2""") == 900 haslocobj = hasloc([[1, 2]]) haslocobj |>= .iloc$[0]$[1] assert haslocobj == 2 + assert safe_raise_exc().error `isinstance` Exception + assert safe_raise_exc().handle(Exception, const 10).result == 10 # must come at end assert fibs_calls[0] == 1 diff --git a/coconut/tests/src/cocotest/agnostic/util.coco b/coconut/tests/src/cocotest/agnostic/util.coco index 0b67954b9..efa5b681d 100644 --- a/coconut/tests/src/cocotest/agnostic/util.coco +++ b/coconut/tests/src/cocotest/agnostic/util.coco @@ -1014,6 +1014,10 @@ def minus(a, b) = b - a def raise_exc(): raise Exception("raise_exc") +@safe_call$ +def safe_raise_exc() = + raise_exc() + def does_raise_exc(func): try: return func() From c13fa4cab4704696e145fe70c00b49e1a12ed4c8 Mon Sep 17 00:00:00 2001 From: Evan Hubinger Date: Thu, 2 Nov 2023 01:48:43 -0700 Subject: [PATCH 046/121] Add Expected.expect_error --- DOCS.md | 46 ++++++++---- __coconut__/__init__.pyi | 63 ++++++++++------ coconut/compiler/templates/header.py_template | 75 ++++++++++++------- .../tests/src/cocotest/agnostic/suite.coco | 4 +- coconut/tests/src/cocotest/agnostic/util.coco | 4 +- 5 files changed, 128 insertions(+), 64 deletions(-) diff --git a/DOCS.md b/DOCS.md index f4e6d8153..a514f1999 100644 --- a/DOCS.md +++ b/DOCS.md @@ -3157,6 +3157,10 @@ data Expected[T](result: T? = None, error: BaseException? = None): def __bool__(self) -> bool: return self.error is None def __fmap__[U](self, func: T -> U) -> Expected[U]: + """Maps func over the result if it exists. + + __fmap__ should be used directly only when fmap is not available (e.g. when consuming an Expected in vanilla Python). + """ return self.__class__(func(self.result)) if self else self def and_then[U](self, func: T -> Expected[U]) -> Expected[U]: """Maps a T -> Expected[U] over an Expected[T] to produce an Expected[U]. @@ -3172,27 +3176,43 @@ data Expected[T](result: T? = None, error: BaseException? = None): def map_error(self, func: BaseException -> BaseException) -> Expected[T]: """Maps func over the error if it exists.""" return self if self else self.__class__(error=func(self.error)) + def handle(self, err_type, handler: BaseException -> T) -> Expected[T]: + """Recover from the given err_type by calling handler on the error to determine the result.""" + if not self and isinstance(self.error, err_type): + return self.__class__(handler(self.error)) + return self + def expect_error(self, *err_types: BaseException) -> Expected[T]: + """Raise any errors that do not match the given error types.""" + if not self and not isinstance(self.error, err_types): + raise self.error + return self + def unwrap(self) -> T: + """Unwrap the result or raise the error.""" + if not self: + raise self.error + return self.result def or_else[U](self, func: BaseException -> Expected[U]) -> Expected[T | U]: """Return self if no error, otherwise return the result of evaluating func on the error.""" return self if self else func(self.error) - def result_or[U](self, default: U) -> T | U: - """Return the result if it exists, otherwise return the default.""" - return self.result if self else default def result_or_else[U](self, func: BaseException -> U) -> T | U: """Return the result if it exists, otherwise return the result of evaluating func on the error.""" return self.result if self else func(self.error) - def unwrap(self) -> T: - """Unwrap the result or raise the error.""" - if not self: - raise self.error - return self.result - def handle(self, err_type, handler: BaseException -> T) -> Expected[T]: - if not self and _coconut.isinstance(self.error, err_type): - return self.__class__(handler(self.error)) - return self + def result_or[U](self, default: U) -> T | U: + """Return the result if it exists, otherwise return the default. + + Since .result_or() completely silences errors, it is highly recommended that you + call .expect_error() first to explicitly declare what errors you are okay silencing. + """ + return self.result if self else default ``` -`Expected` is primarily used as the return type for [`safe_call`](#safe_call). Generally, the best way to use `Expected` is with [`fmap`](#fmap), which will apply a function to the result if it exists, or otherwise retain the error. If you want to sequence multiple `Expected`-returning operations, `.and_then` should be used instead of `fmap`. +`Expected` is primarily used as the return type for [`safe_call`](#safe_call). + +Generally, the best way to use `Expected` is with [`fmap`](#fmap), which will apply a function to the result if it exists, or otherwise retain the error. If you want to sequence multiple `Expected`-returning operations, `.and_then` should be used instead of `fmap`. To handle specific errors, the following patterns are equivalent: +``` +safe_call(might_raise_IOError).handle(IOError, const 10).unwrap() +safe_call(might_raise_IOError).expect_error(IOError).result_or(10) +``` To match against an `Expected`, just: ``` diff --git a/__coconut__/__init__.pyi b/__coconut__/__init__.pyi index def115f7b..8f60cc7e7 100644 --- a/__coconut__/__init__.pyi +++ b/__coconut__/__init__.pyi @@ -325,6 +325,10 @@ class Expected(_BaseExpected[_T]): def __bool__(self) -> bool: return self.error is None def __fmap__[U](self, func: T -> U) -> Expected[U]: + """Maps func over the result if it exists. + + __fmap__ should be used directly only when fmap is not available (e.g. when consuming an Expected in vanilla Python). + """ return self.__class__(func(self.result)) if self else self def and_then[U](self, func: T -> Expected[U]) -> Expected[U]: """Maps a T -> Expected[U] over an Expected[T] to produce an Expected[U]. @@ -340,24 +344,34 @@ class Expected(_BaseExpected[_T]): def map_error(self, func: BaseException -> BaseException) -> Expected[T]: """Maps func over the error if it exists.""" return self if self else self.__class__(error=func(self.error)) + def handle(self, err_type, handler: BaseException -> T) -> Expected[T]: + """Recover from the given err_type by calling handler on the error to determine the result.""" + if not self and isinstance(self.error, err_type): + return self.__class__(handler(self.error)) + return self + def expect_error(self, *err_types: BaseException) -> Expected[T]: + """Raise any errors that do not match the given error types.""" + if not self and not isinstance(self.error, err_types): + raise self.error + return self + def unwrap(self) -> T: + """Unwrap the result or raise the error.""" + if not self: + raise self.error + return self.result def or_else[U](self, func: BaseException -> Expected[U]) -> Expected[T | U]: """Return self if no error, otherwise return the result of evaluating func on the error.""" return self if self else func(self.error) - def result_or[U](self, default: U) -> T | U: - """Return the result if it exists, otherwise return the default.""" - return self.result if self else default def result_or_else[U](self, func: BaseException -> U) -> T | U: """Return the result if it exists, otherwise return the result of evaluating func on the error.""" return self.result if self else func(self.error) - def unwrap(self) -> T: - """Unwrap the result or raise the error.""" - if not self: - raise self.error - return self.result - def handle(self, err_type, handler: BaseException -> T) -> Expected[T]: - if not self and _coconut.isinstance(self.error, err_type): - return self.__class__(handler(self.error)) - return self + def result_or[U](self, default: U) -> T | U: + """Return the result if it exists, otherwise return the default. + + Since .result_or() completely silences errors, it is highly recommended that you + call .expect_error() first to explicitly declare what errors you are okay silencing. + """ + return self.result if self else default ''' __slots__ = () _coconut_is_data = True @@ -408,20 +422,27 @@ class Expected(_BaseExpected[_T]): def map_error(self, func: _t.Callable[[BaseException], BaseException]) -> Expected[_T]: """Maps func over the error if it exists.""" ... + def handle(self, err_type: _t.Type[BaseException], handler: _t.Callable[[BaseException], _T]) -> Expected[_T]: + """Recover from the given err_type by calling handler on the error to determine the result.""" + ... + def expect_error(self, *err_types: BaseException) -> Expected[_T]: + """Raise any errors that do not match the given error types.""" + ... + def unwrap(self) -> _T: + """Unwrap the result or raise the error.""" + ... def or_else(self, func: _t.Callable[[BaseException], Expected[_U]]) -> Expected[_T | _U]: """Return self if no error, otherwise return the result of evaluating func on the error.""" ... - def result_or(self, default: _U) -> _T | _U: - """Return the result if it exists, otherwise return the default.""" - ... def result_or_else(self, func: _t.Callable[[BaseException], _U]) -> _T | _U: """Return the result if it exists, otherwise return the result of evaluating func on the error.""" ... - def unwrap(self) -> _T: - """Unwrap the result or raise the error.""" - ... - def handle(self, err_type: _t.Type[BaseException], handler: _t.Callable[[BaseException], _T]) -> Expected[_T]: - """Recover from the given err_type by calling handler on the error to determine the result.""" + def result_or(self, default: _U) -> _T | _U: + """Return the result if it exists, otherwise return the default. + + Since .result_or() completely silences errors, it is highly recommended that you + call .expect_error() first to explicitly declare what errors you are okay silencing. + """ ... _coconut_Expected = Expected @@ -1574,7 +1595,7 @@ def lift(func: _t.Callable[[_T, _U], _W]) -> _coconut_lifted_2[_T, _U, _W]: ... def lift(func: _t.Callable[[_T, _U, _V], _W]) -> _coconut_lifted_3[_T, _U, _V, _W]: ... @_t.overload def lift(func: _t.Callable[..., _W]) -> _t.Callable[..., _t.Callable[..., _W]]: - """Lifts a function up so that all of its arguments are functions. + """Lift a function up so that all of its arguments are functions. For a binary function f(x, y) and two unary functions g(z) and h(z), lift works as the S' combinator: lift(f)(g, h)(z) == f(g(z), h(z)) diff --git a/coconut/compiler/templates/header.py_template b/coconut/compiler/templates/header.py_template index 4edc7ce34..bde083643 100644 --- a/coconut/compiler/templates/header.py_template +++ b/coconut/compiler/templates/header.py_template @@ -1712,6 +1712,10 @@ class Expected(_coconut.collections.namedtuple("Expected", ("result", "error")){ def __bool__(self) -> bool: return self.error is None def __fmap__[U](self, func: T -> U) -> Expected[U]: + """Maps func over the result if it exists. + + __fmap__ should be used directly only when fmap is not available (e.g. when consuming an Expected in vanilla Python). + """ return self.__class__(func(self.result)) if self else self def and_then[U](self, func: T -> Expected[U]) -> Expected[U]: """Maps a T -> Expected[U] over an Expected[T] to produce an Expected[U]. @@ -1727,24 +1731,34 @@ class Expected(_coconut.collections.namedtuple("Expected", ("result", "error")){ def map_error(self, func: BaseException -> BaseException) -> Expected[T]: """Maps func over the error if it exists.""" return self if self else self.__class__(error=func(self.error)) + def handle(self, err_type, handler: BaseException -> T) -> Expected[T]: + """Recover from the given err_type by calling handler on the error to determine the result.""" + if not self and isinstance(self.error, err_type): + return self.__class__(handler(self.error)) + return self + def expect_error(self, *err_types: BaseException) -> Expected[T]: + """Raise any errors that do not match the given error types.""" + if not self and not isinstance(self.error, err_types): + raise self.error + return self + def unwrap(self) -> T: + """Unwrap the result or raise the error.""" + if not self: + raise self.error + return self.result def or_else[U](self, func: BaseException -> Expected[U]) -> Expected[T | U]: """Return self if no error, otherwise return the result of evaluating func on the error.""" return self if self else func(self.error) - def result_or[U](self, default: U) -> T | U: - """Return the result if it exists, otherwise return the default.""" - return self.result if self else default def result_or_else[U](self, func: BaseException -> U) -> T | U: """Return the result if it exists, otherwise return the result of evaluating func on the error.""" return self.result if self else func(self.error) - def unwrap(self) -> T: - """Unwrap the result or raise the error.""" - if not self: - raise self.error - return self.result - def handle(self, err_type, handler: BaseException -> T) -> Expected[T]: - if not self and _coconut.isinstance(self.error, err_type): - return self.__class__(handler(self.error)) - return self + def result_or[U](self, default: U) -> T | U: + """Return the result if it exists, otherwise return the default. + + Since .result_or() completely silences errors, it is highly recommended that you + call .expect_error() first to explicitly declare what errors you are okay silencing. + """ + return self.result if self else default ''' __slots__ = () {is_data_var} = True @@ -1788,6 +1802,21 @@ class Expected(_coconut.collections.namedtuple("Expected", ("result", "error")){ def map_error(self, func): """Maps func over the error if it exists.""" return self if self else self.__class__(error=func(self.error)) + def handle(self, err_type, handler): + """Recover from the given err_type by calling handler on the error to determine the result.""" + if not self and _coconut.isinstance(self.error, err_type): + return self.__class__(handler(self.error)) + return self + def expect_error(self, *err_types): + """Raise any errors that do not match the given error types.""" + if not self and not _coconut.isinstance(self.error, err_types): + raise self.error + return self + def unwrap(self): + """Unwrap the result or raise the error.""" + if not self: + raise self.error + return self.result def or_else(self, func): """Return self if no error, otherwise return the result of evaluating func on the error.""" if self: @@ -1796,22 +1825,16 @@ class Expected(_coconut.collections.namedtuple("Expected", ("result", "error")){ if not _coconut.isinstance(got, {_coconut_}Expected): raise _coconut.TypeError("Expected.or_else() requires a function that returns an Expected") return got - def result_or(self, default): - """Return the result if it exists, otherwise return the default.""" - return self.result if self else default def result_or_else(self, func): """Return the result if it exists, otherwise return the result of evaluating func on the error.""" return self.result if self else func(self.error) - def unwrap(self): - """Unwrap the result or raise the error.""" - if not self: - raise self.error - return self.result - def handle(self, err_type, handler): - """Recover from the given err_type by calling handler on the error to determine the result.""" - if not self and _coconut.isinstance(self.error, err_type): - return self.__class__(handler(self.error)) - return self + def result_or(self, default): + """Return the result if it exists, otherwise return the default. + + Since .result_or() completely silences errors, it is highly recommended that you + call .expect_error() first to explicitly declare what errors you are okay silencing. + """ + return self.result if self else default class flip(_coconut_base_callable): """Given a function, return a new function with inverse argument order. If nargs is passed, only the first nargs arguments are reversed.""" @@ -1858,7 +1881,7 @@ class _coconut_lifted(_coconut_base_callable): def __repr__(self): return "lift(%r)(%s%s)" % (self.func, ", ".join(_coconut.repr(g) for g in self.func_args), ", ".join(k + "=" + _coconut.repr(h) for k, h in self.func_kwargs.items())) class lift(_coconut_base_callable): - """Lifts a function up so that all of its arguments are functions. + """Lift a function up so that all of its arguments are functions. For a binary function f(x, y) and two unary functions g(z) and h(z), lift works as the S' combinator: lift(f)(g, h)(z) == f(g(z), h(z)) diff --git a/coconut/tests/src/cocotest/agnostic/suite.coco b/coconut/tests/src/cocotest/agnostic/suite.coco index 64da67027..1ac462e6d 100644 --- a/coconut/tests/src/cocotest/agnostic/suite.coco +++ b/coconut/tests/src/cocotest/agnostic/suite.coco @@ -1065,8 +1065,8 @@ forward 2""") == 900 haslocobj = hasloc([[1, 2]]) haslocobj |>= .iloc$[0]$[1] assert haslocobj == 2 - assert safe_raise_exc().error `isinstance` Exception - assert safe_raise_exc().handle(Exception, const 10).result == 10 + assert safe_raise_exc(IOError).error `isinstance` IOError + assert safe_raise_exc(IOError).handle(IOError, const 10).unwrap() == 10 == safe_raise_exc(IOError).expect_error(IOError).result_or(10) # must come at end assert fibs_calls[0] == 1 diff --git a/coconut/tests/src/cocotest/agnostic/util.coco b/coconut/tests/src/cocotest/agnostic/util.coco index efa5b681d..0cb370c59 100644 --- a/coconut/tests/src/cocotest/agnostic/util.coco +++ b/coconut/tests/src/cocotest/agnostic/util.coco @@ -1015,8 +1015,8 @@ def raise_exc(): raise Exception("raise_exc") @safe_call$ -def safe_raise_exc() = - raise_exc() +def safe_raise_exc(exc_cls = Exception): + raise exc_cls() def does_raise_exc(func): try: From 6c9c113faa7598d82b0e64c40ddb634b57be9060 Mon Sep 17 00:00:00 2001 From: Evan Hubinger Date: Thu, 2 Nov 2023 22:21:41 -0700 Subject: [PATCH 047/121] Fix typing --- __coconut__/__init__.pyi | 36 +- coconut/tests/src/cocotest/agnostic/main.coco | 7 +- .../agnostic/{primary.coco => primary_1.coco} | 404 +---------------- .../src/cocotest/agnostic/primary_2.coco | 410 ++++++++++++++++++ 4 files changed, 447 insertions(+), 410 deletions(-) rename coconut/tests/src/cocotest/agnostic/{primary.coco => primary_1.coco} (74%) create mode 100644 coconut/tests/src/cocotest/agnostic/primary_2.coco diff --git a/__coconut__/__init__.pyi b/__coconut__/__init__.pyi index 8f60cc7e7..066d10c20 100644 --- a/__coconut__/__init__.pyi +++ b/__coconut__/__init__.pyi @@ -207,12 +207,10 @@ dropwhile = _coconut.itertools.dropwhile tee = _coconut.itertools.tee starmap = _coconut.itertools.starmap cartesian_product = _coconut.itertools.product -multiset = _coconut.collections.Counter _coconut_tee = tee _coconut_starmap = starmap _coconut_cartesian_product = cartesian_product -_coconut_multiset = multiset process_map = thread_map = parallel_map = concurrent_map = _coconut_map = map @@ -254,6 +252,10 @@ def _coconut_tco(func: _Tfunc) -> _Tfunc: # any changes here should also be made to safe_call and call_or_coefficient below @_t.overload +def call( + _func: _t.Callable[[], _U], +) -> _U: ... +@_t.overload def call( _func: _t.Callable[[_T], _U], _x: _T, @@ -450,6 +452,10 @@ _coconut_Expected = Expected # should match call above but with Expected @_t.overload +def safe_call( + _func: _t.Callable[[], _U], +) -> Expected[_U]: ... +@_t.overload def safe_call( _func: _t.Callable[[_T], _U], _x: _T, @@ -510,7 +516,11 @@ def safe_call( ... -# based on call above +# based on call above@_t.overload +@_t.overload +def _coconut_call_or_coefficient( + _func: _t.Callable[[], _U], +) -> _U: ... @_t.overload def _coconut_call_or_coefficient( _func: _t.Callable[[_T], _U], @@ -678,7 +688,7 @@ def _coconut_attritemgetter( def _coconut_base_compose( func: _t.Callable[[_T], _t.Any], *func_infos: _t.Tuple[_Callable, int, bool], - ) -> _t.Callable[[_T], _t.Any]: ... +) -> _t.Callable[[_T], _t.Any]: ... def and_then( @@ -1303,6 +1313,7 @@ class groupsof(_t.Generic[_T]): cls, n: _SupportsIndex, iterable: _t.Iterable[_T], + fillvalue: _T = ..., ) -> groupsof[_T]: ... def __iter__(self) -> _t.Iterator[_t.Tuple[_T, ...]]: ... def __hash__(self) -> int: ... @@ -1322,8 +1333,8 @@ class windowsof(_t.Generic[_T]): cls, size: _SupportsIndex, iterable: _t.Iterable[_T], - fillvalue: _T=..., - step: _SupportsIndex=1, + fillvalue: _T = ..., + step: _SupportsIndex = 1, ) -> windowsof[_T]: ... def __iter__(self) -> _t.Iterator[_t.Tuple[_T, ...]]: ... def __hash__(self) -> int: ... @@ -1339,7 +1350,7 @@ class flatten(_t.Iterable[_T]): def __new__( cls, iterable: _t.Iterable[_t.Iterable[_T]], - levels: _t.Optional[_SupportsIndex]=1, + levels: _t.Optional[_SupportsIndex] = 1, ) -> flatten[_T]: ... def __iter__(self) -> _t.Iterator[_T]: ... @@ -1380,6 +1391,17 @@ def consume( ... +class multiset(_t.Generic[_T], _coconut.collections.Counter[_T]): + def add(self, item: _T) -> None: ... + def discard(self, item: _T) -> None: ... + def remove(self, item: _T) -> None: ... + def isdisjoint(self, other: _coconut.collections.Counter[_T]) -> bool: ... + def __xor__(self, other: _coconut.collections.Counter[_T]) -> multiset[_T]: ... + def count(self, item: _T) -> int: ... + def __fmap__(self, func: _t.Callable[[_T], _U]) -> multiset[_U]: ... +_coconut_multiset = multiset + + class _FMappable(_t.Protocol[_Tfunc_contra, _Tco]): def __fmap__(self, func: _Tfunc_contra) -> _Tco: ... diff --git a/coconut/tests/src/cocotest/agnostic/main.coco b/coconut/tests/src/cocotest/agnostic/main.coco index b6bdbfa59..78c0baa5f 100644 --- a/coconut/tests/src/cocotest/agnostic/main.coco +++ b/coconut/tests/src/cocotest/agnostic/main.coco @@ -1,6 +1,8 @@ import sys -from .primary import assert_raises, primary_test +from .util import assert_raises +from .primary_1 import primary_test_1 +from .primary_2 import primary_test_2 def test_asyncio() -> bool: @@ -49,7 +51,8 @@ def run_main(outer_MatchError, test_easter_eggs=False) -> bool: using_tco = "_coconut_tco" in globals() or "_coconut_tco" in locals() print_dot() # .. - assert primary_test() is True + assert primary_test_1() is True + assert primary_test_2() is True print_dot() # ... from .specific import ( diff --git a/coconut/tests/src/cocotest/agnostic/primary.coco b/coconut/tests/src/cocotest/agnostic/primary_1.coco similarity index 74% rename from coconut/tests/src/cocotest/agnostic/primary.coco rename to coconut/tests/src/cocotest/agnostic/primary_1.coco index c8dccb37a..42a056e1b 100644 --- a/coconut/tests/src/cocotest/agnostic/primary.coco +++ b/coconut/tests/src/cocotest/agnostic/primary_1.coco @@ -1,7 +1,6 @@ import itertools import collections import collections.abc -import weakref import platform from copy import copy @@ -12,11 +11,11 @@ from importlib import reload # NOQA if platform.python_implementation() == "CPython": # fixes weird aenum issue on pypy from enum import Enum # noqa -from .util import assert_raises, typed_eq +from .util import assert_raises -def primary_test() -> bool: - """Basic no-dependency tests.""" +def primary_test_1() -> bool: + """Basic no-dependency tests (1/2).""" # must come at start so that local sys binding is correct import sys import queue as q, builtins, email.mime.base @@ -1304,401 +1303,4 @@ def primary_test() -> bool: assert err is some_err assert Expected(error=TypeError()).map_error(const some_err) == Expected(error=some_err) assert Expected(10).map_error(const some_err) == Expected(10) - - recit = ([1,2,3] :: recit) |> map$(.+1) - assert tee(recit) - rawit = (_ for _ in (0, 1)) - t1, t2 = tee(rawit) - t1a, t1b = tee(t1) - assert (list(t1a), list(t1b), list(t2)) == ([0, 1], [0, 1], [0, 1]) - assert m{1, 3, 1}[1] == 2 - assert m{1, 2} |> repr in ("multiset({1: 1, 2: 1})", "multiset({2: 1, 1: 1})") - m = m{} - m.add(1) - m.add(1) - m.add(2) - assert m == m{1, 1, 2} - assert m != m{1, 2} - m.discard(2) - m.discard(2) - assert m == m{1, 1} - assert m != m{1} - m.remove(1) - assert m == m{1} - m.remove(1) - assert m == m{} - assert_raises(-> m.remove(1), KeyError) - assert 1 not in m - assert 2 not in m - assert m{1, 2}.isdisjoint(m{3, 4}) - assert not m{1, 2}.isdisjoint(m{2, 3}) - assert m{1, 2} ^ m{2, 3} `typed_eq` m{1, 3} - m = m{1, 2} - m ^= m{2, 3} - assert m `typed_eq` m{1, 3} - assert m{1, 1} ^ m{1} `typed_eq` m{1} - assert multiset((1, 2)) == m{1, 2} == multiset(m{1, 2}) - assert multiset({1: 2, 2: 1}) == m{1, 1, 2} - assert m{} `isinstance` multiset - assert m{} `isinstance` collections.abc.Set - assert m{} `isinstance` collections.abc.MutableSet - assert True `isinstance` bool - class HasBool: - def __bool__(self) = False - assert not HasBool() - assert m{1}.count(2) == 0 - assert m{1, 1}.count(1) == 2 - bad_m = m{} - bad_m[1] = -1 - assert_raises(-> bad_m.count(1), ValueError) - assert len(m{1, 1}) == 1 - assert m{1, 1}.total() == 2 == m{1, 2}.total() - weird_m = m{1, 2} - weird_m[3] = 0 - assert weird_m == m{1, 2} - assert not (weird_m != m{1, 2}) - assert m{} <= m{} < m{1, 2} < m{1, 1, 2} <= m{1, 1, 2} - assert m{1, 1, 2} >= m{1, 1, 2} > m{1, 2} > m{} >= m{} - assert m{1} != {1:1, 2:0} - assert not (m{1} == {1:1, 2:0}) - assert s{1, 2, *(2, 3, 4), *(4, 5)} == s{1, 2, 3, 4, 5} - assert m{1, 2, *(2, 3, 4), *(4, 5)} == m{1, 2, 2, 3, 4, 4, 5} - assert {*(1, 2)} == {1, 2} - assert cycle(range(3))[:5] |> list == [0, 1, 2, 0, 1] == cycle(range(3)) |> iter |> .$[:5] |> list - assert cycle(range(2), 2)[:5] |> list == [0, 1, 0, 1] == cycle(range(2), 2) |> iter |> .$[:5] |> list - assert 2 in cycle(range(3)) - assert reversed(cycle(range(2), 2)) |> list == [1, 0, 1, 0] - assert cycle((_ for _ in range(2)), 2) |> list == [0, 1, 0, 1] - assert cycle(range(3)).count(0) == float("inf") - assert cycle(range(3), 3).index(2) == 2 - assert zip(range(2), range(2))[0] == (0, 0) == enumerate(range(2))[0] - assert reversed([0,1,3])[0] == 3 - assert cycle((), 0) |> list == [] - assert "1234" |> windowsof$(2) |> map$("".join) |> list == ["12", "23", "34"] - assert len(windowsof(2, "1234")) == 3 - assert windowsof(3, "12345", None) |> map$("".join) |> list == ["123", "234", "345"] - assert len(windowsof(3, "12345", None)) == 3 - assert windowsof(3, "1") |> list == [] == windowsof(2, "1", step=2) |> list - assert len(windowsof(2, "1")) == 0 == len(windowsof(2, "1", step=2)) - assert windowsof(2, "1", None) |> list == [("1", None)] == windowsof(2, "1", None, 2) |> list - assert len(windowsof(2, "1", None)) == 1 == len(windowsof(2, "1", None, 2)) - assert windowsof(2, "1234", step=2) |> map$("".join) |> list == ["12", "34"] == windowsof(2, "1234", fillvalue=None, step=2) |> map$("".join) |> list - assert len(windowsof(2, "1234", step=2)) == 2 == len(windowsof(2, "1234", fillvalue=None, step=2)) - assert repr(windowsof(2, "1234", None, 3)) == "windowsof(2, '1234', fillvalue=None, step=3)" - assert lift(,)((+), (*))(2, 3) == (5, 6) - assert "abac" |> windowsof$(2) |> filter$(addpattern( - (def (("a", b) if b != "b") -> True), - (def ((_, _)) -> False), - )) |> list == [("a", "c")] - assert "[A], [B]" |> windowsof$(3) |> map$(addpattern( - (def (("[","A","]")) -> "A"), - (def (("[","B","]")) -> "B"), - (def ((_,_,_)) -> None), - )) |> filter$((.is None) ..> (not)) |> list == ["A", "B"] - assert windowsof(3, "abcdefg", step=3) |> map$("".join) |> list == ["abc", "def"] - assert windowsof(3, "abcdefg", step=3) |> len == 2 - assert windowsof(3, "abcdefg", step=3, fillvalue="") |> map$("".join) |> list == ["abc", "def", "g"] - assert windowsof(3, "abcdefg", step=3, fillvalue="") |> len == 3 - assert windowsof(3, "abcdefg", step=2, fillvalue="") |> map$("".join) |> list == ["abc", "cde", "efg", "g"] - assert windowsof(3, "abcdefg", step=2, fillvalue="") |> len == 4 - assert windowsof(5, "abc", step=2, fillvalue="") |> map$("".join) |> list == ["abc"] - assert windowsof(5, "abc", step=2, fillvalue="") |> len == 1 - assert groupsof(2, "123", fillvalue="") |> map$("".join) |> list == ["12", "3"] - assert groupsof(2, "123", fillvalue="") |> len == 2 - assert groupsof(2, "123", fillvalue="") |> repr == "groupsof(2, '123', fillvalue='')" - assert flip((,), 0)(1, 2) == (1, 2) - assert flatten([1, 2, [3, 4]], 0) == [1, 2, [3, 4]] - assert flatten([1, 2, [3, 4]], None) |> list == [1, 2, 3, 4] - assert flatten([1, 2, [3, 4]], None) |> reversed |> list == [4, 3, 2, 1] - assert_raises(-> flatten([1, 2, [3, 4]]) |> list, TypeError) - assert flatten([[[1,2]], [[3], [4]]], 2) |> list == [1, 2, 3, 4] - assert flatten([[[1,2]], [[3], [4]]], 2) |> reversed |> list == [4, 3, 2, 1] - assert_raises(-> map((+), range(3), range(4), strict=True) |> list, ValueError) - assert map((+), range(3), range(4)$[:-1], strict=True) |> list == [0, 2, 4] == process_map((+), range(3), range(4)$[:-1], strict=True) |> list - assert range(3) |> map$((.+1), strict=True) |> list == [1, 2, 3] == range(3) |> process_map$((.+1), strict=True) |> list - assert cartesian_product((1, 2), (3, 4), repeat=0) |> list == [()] - assert (a=1, b=2)[1] == 2 - obj = object() - assert_raises((def -> obj.abc = 123), AttributeError) # type: ignore - hardref = map((.+1), [1,2,3]) - assert weakref.ref(hardref)() |> list == [2, 3, 4] - my_match_err = MatchError("my match error", 123) - assert process_map(ident, [my_match_err]) |> list |> str == str([my_match_err]) - # repeat the same thing again now that my_match_err.str has been called - assert process_map(ident, [my_match_err]) |> list |> str == str([my_match_err]) - match data tuple(1, 2) in (1, 2, 3): - assert False - data TestDefaultMatching(x="x default", y="y default") - TestDefaultMatching(got_x) = TestDefaultMatching(1) - assert got_x == 1 - TestDefaultMatching(y=got_y) = TestDefaultMatching(y=10) - assert got_y == 10 - TestDefaultMatching() = TestDefaultMatching() - data HasStar(x, y, *zs) - HasStar(x, *ys) = HasStar(1, 2, 3, 4) - assert x == 1 - assert ys == (2, 3, 4) - HasStar(x, y, z) = HasStar(1, 2, 3) - assert (x, y, z) == (1, 2, 3) - HasStar(5, y=10) = HasStar(5, 10) - HasStar(1, 2, 3, zs=(3,)) = HasStar(1, 2, 3) - HasStar(x=1, y=2) = HasStar(1, 2) - match HasStar(x) in HasStar(1, 2): - assert False - match HasStar(x, y) in HasStar(1, 2, 3): - assert False - data HasStarAndDef(x, y="y", *zs) - HasStarAndDef(1, "y") = HasStarAndDef(1) - HasStarAndDef(1) = HasStarAndDef(1) - HasStarAndDef(x=1) = HasStarAndDef(1) - HasStarAndDef(1, 2, 3, zs=(3,)) = HasStarAndDef(1, 2, 3) - HasStarAndDef(1, y=2) = HasStarAndDef(1, 2) - match HasStarAndDef(x, y) in HasStarAndDef(1, 2, 3): - assert False - - assert (.+1) kwargs) <**?| None is None - assert ((**kwargs) -> kwargs) <**?| {"a": 1, "b": 2} == {"a": 1, "b": 2} - assert (<**?|)((**kwargs) -> kwargs, None) is None - assert (<**?|)((**kwargs) -> kwargs, {"a": 1, "b": 2}) == {"a": 1, "b": 2} - optx = (**kwargs) -> kwargs - optx <**?|= None - assert optx is None - optx = (**kwargs) -> kwargs - optx <**?|= {"a": 1, "b": 2} - assert optx == {"a": 1, "b": 2} - - assert `const None ..?> (.+1)` is None is (..?>)(const None, (.+1))() - assert `(.+1) (.+1)` == 6 == (..?>)(const 5, (.+1))() - assert `(.+1) (+)` is None is (..?*>)(const None, (+))() - assert `(+) <*?.. const None` is None is (<*?..)((+), const None)() - assert `const((5, 2)) ..?*> (+)` == 7 == (..?*>)(const((5, 2)), (+))() - assert `(+) <*?.. const((5, 2))` == 7 == (<*?..)((+), const((5, 2)))() - assert `const None ..?**> (**kwargs) -> kwargs` is None is (..?**>)(const None, (**kwargs) -> kwargs)() - assert `((**kwargs) -> kwargs) <**?.. const None` is None is (<**?..)((**kwargs) -> kwargs, const None)() - assert `const({"a": 1}) ..?**> (**kwargs) -> kwargs` == {"a": 1} == (..?**>)(const({"a": 1}), (**kwargs) -> kwargs)() - assert `((**kwargs) -> kwargs) <**?.. const({"a": 1})` == {"a": 1} == (<**?..)((**kwargs) -> kwargs, const({"a": 1}))() - optx = const None - optx ..?>= (.+1) - optx ..?*>= (+) - optx ..?**>= (,) - assert optx() is None - optx = (.+1) - optx five (two + three), TypeError) - assert_raises(-> 5 (10), TypeError) - assert_raises(-> 5 [0], TypeError) - assert five ** 2 two == 50 - assert 2i x == 20i - some_str = "some" - assert_raises(-> some_str five, TypeError) - assert (not in)("a", "bcd") - assert not (not in)("a", "abc") - assert ("a" not in .)("bcd") - assert (. not in "abc")("d") - assert (is not)(1, True) - assert not (is not)(False, False) - assert (True is not .)(1) - assert (. is not True)(1) - a_dict = {} - a_dict[1] = 1 - a_dict[3] = 2 - a_dict[2] = 3 - assert a_dict |> str == "{1: 1, 3: 2, 2: 3}" == a_dict |> repr, a_dict - assert a_dict.keys() |> tuple == (1, 3, 2) - assert not a_dict.keys() `isinstance` list - assert not a_dict.values() `isinstance` list - assert not a_dict.items() `isinstance` list - assert len(a_dict.keys()) == len(a_dict.values()) == len(a_dict.items()) == 3 - assert {1: 1, 3: 2, 2: 3}.keys() |> tuple == (1, 3, 2) - assert {**{1: 0, 3: 0}, 2: 0}.keys() |> tuple == (1, 3, 2) == {**dict([(1, 1), (3, 2), (2, 3)])}.keys() |> tuple - assert a_dict == {1: 1, 2: 3, 3: 2} - assert {1: 1} |> str == "{1: 1}" == {1: 1} |> repr - assert py_dict `issubclass` dict - assert py_dict() `isinstance` dict - assert {5:0, 3:0, **{2:0, 6:0}, 8:0}.keys() |> tuple == (5, 3, 2, 6, 8) - a_multiset = m{1,1,2} - assert not a_multiset.keys() `isinstance` list - assert not a_multiset.values() `isinstance` list - assert not a_multiset.items() `isinstance` list - assert len(a_multiset.keys()) == len(a_multiset.values()) == len(a_multiset.items()) == 2 - assert (in)(1, [1, 2]) - assert not (1 not in .)([1, 2]) - assert not (in)([[]], []) - assert ("{a}" . .)("format")(a=1) == "1" - a_dict = {"a": 1, "b": 2} - a_dict |= {"a": 10, "c": 20} - assert a_dict == {"a": 10, "b": 2, "c": 20} == {"a": 1, "b": 2} | {"a": 10, "c": 20} - assert ["abc" ; "def"] == ['abc', 'def'] - assert ["abc" ;; "def"] == [['abc'], ['def']] - assert {"a":0, "b":1}$[0] == "a" - assert (|0, NotImplemented, 2|)$[1] is NotImplemented - assert m{1, 1, 2} |> fmap$(.+1) == m{2, 2, 3} - assert (+) ..> ((*) ..> (/)) == (+) ..> (*) ..> (/) == ((+) ..> (*)) ..> (/) - def f(x, y=1) = x, y # type: ignore - f.is_f = True # type: ignore - assert (f ..*> (+)).is_f # type: ignore - really_long_var = 10 - assert (...=really_long_var) == (10,) - assert (...=really_long_var, abc="abc") == (10, "abc") - assert (abc="abc", ...=really_long_var) == ("abc", 10) - assert (...=really_long_var).really_long_var == 10 - n = [0] - assert n[0] == 0 - assert_raises(-> m{{1:2,2:3}}, TypeError) - assert_raises((def -> from typing import blah), ImportError) # NOQA - assert type(m{1, 2}) is multiset - assert multiset(collections.Counter({1: 1, 2: 1})) `typed_eq` m{1, 2} - assert +m{-1, 1} `typed_eq` m{-1, 1} - assert -m{-1, 1} `typed_eq` m{} - assert m{1, 1, 2} + m{1, 3} `typed_eq` m{1, 1, 1, 2, 3} - assert m{1, 1, 2} | m{1, 3} `typed_eq` m{1, 1, 2, 3} - assert m{1, 1, 2} & m{1, 3} `typed_eq` m{1} - assert m{1, 1, 2} - m{1, 3} `typed_eq` m{1, 2} - assert (.+1) `and_then` (.*2) `and_then_await` (./3) |> repr == "$(?, 1) `and_then` $(?, 2) `and_then_await` $(?, 3)" - assert 5.5⏨3 == 5.5 * 10**3 - assert (x => x)(5) == 5 == (def x => x)(5) - assert (=> _)(5) == 5 == (def => _)(5) - assert ((x, y) => (x, y))(1, 2) == (1, 2) == (def (x, y) => (x, y))(1, 2) - assert (def (int(x)) => x)(5) == 5 == (def (int -> x) => x)("5") - assert (def (x: int) -> int => x)(5) == 5 == (def (int(x)) -> int => x)(5) - assert (x ⇒ x)(5) == 5 == (def x ⇒ x)(5) - assert f"a: { "abc" }" == "a: abc" == f'a: { 'abc' }' - assert f"1 + {"a" + "b"} + 2 + {"c" + "d"}" == "1 + ab + 2 + cd" == f'1 + {'a' + 'b'} + 2 + {'c' + 'd'}' - assert f"{"a" + "b"} + c + {"d" + "e"}" == "ab + c + de" == f'{'a' + 'b'} + c + {'d' + 'e'}' - assert f"""{""" -"""}""" == """ -""" == f"""{''' -'''}""" - assert f"""{( - )}""" == "()" == f'''{( - )}''' - assert f"{'\n'.join(["", ""])}" == "\n" - assert f"""{f'''{f'{f"{1+1}"}'}'''}""" == "2" == f"{f"{f"{f"{f"{f"{1+1}"}"}"}"}"}" - assert f"___{ - 1 -}___" == '___1___' == f"___{( - 1 -)}___" - x = 10 - assert x == 5 where: - x = 5 - assert x == 10 - def nested() = f where: - f = def -> g where: - def g() = x where: - x = 5 - assert nested()()() == 5 - class HasPartial: - def f(self, x) = (self, x) - g = f$(?, 1) - has_partial = HasPartial() - assert has_partial.g() == (has_partial, 1) - xs = zip([1, 2], [3, 4]) - py_xs = py_zip([1, 2], [3, 4]) - assert list(xs) == [(1, 3), (2, 4)] == list(xs) - assert list(py_xs) == [(1, 3), (2, 4)] - assert list(py_xs) == [] - xs = map((+), [1, 2], [3, 4]) - py_xs = py_map((+), [1, 2], [3, 4]) - assert list(xs) == [4, 6] == list(xs) - assert list(py_xs) == [4, 6] - assert list(py_xs) == [] - for xs in [ - zip((x for x in range(5)), (x for x in range(10))), - py_zip((x for x in range(5)), (x for x in range(10))), - map((,), (x for x in range(5)), (x for x in range(10))), - py_map((,), (x for x in range(5)), (x for x in range(10))), - ]: - assert list(xs) == list(zip(range(5), range(5))) - assert list(xs) == [] - xs = map((.+1), range(5)) - py_xs = py_map((.+1), range(5)) - assert list(xs) == list(range(1, 6)) == list(xs) - assert list(py_xs) == list(range(1, 6)) - assert list(py_xs) == [] - assert count()[:10:2] == range(0, 10, 2) - assert count()[10:2] == range(10, 2) - some_data = [ - (name="a", val="123"), - (name="b", val="567"), - ] - for mapreducer in ( - mapreduce.using_processes$(lift(,)(.name, .val)), # type: ignore - mapreduce.using_threads$(lift(,)(.name, .val)), # type: ignore - collectby.using_processes$(.name, value_func=.val), # type: ignore - collectby.using_threads$(.name, value_func=.val), # type: ignore - ): - assert some_data |> mapreducer == {"a": ["123"], "b": ["567"]} - assert_raises(-> collectby(.[0], [(0, 1), (0, 2)], reduce_func=False), ValueError) return True diff --git a/coconut/tests/src/cocotest/agnostic/primary_2.coco b/coconut/tests/src/cocotest/agnostic/primary_2.coco new file mode 100644 index 000000000..9b36abe9d --- /dev/null +++ b/coconut/tests/src/cocotest/agnostic/primary_2.coco @@ -0,0 +1,410 @@ +import collections +import collections.abc +import weakref + +if TYPE_CHECKING: + from typing import Any, Iterable +from importlib import reload # NOQA + +from .util import assert_raises, typed_eq + + +def primary_test_2() -> bool: + """Basic no-dependency tests (2/2).""" + recit: Iterable[int] = ([1,2,3] :: recit) |> map$(.+1) + assert tee(recit) + rawit = (_ for _ in (0, 1)) + t1, t2 = tee(rawit) + t1a, t1b = tee(t1) + assert (list(t1a), list(t1b), list(t2)) == ([0, 1], [0, 1], [0, 1]) + assert m{1, 3, 1}[1] == 2 + assert m{1, 2} |> repr in ("multiset({1: 1, 2: 1})", "multiset({2: 1, 1: 1})") + m: multiset = m{} + m.add(1) + m.add(1) + m.add(2) + assert m == m{1, 1, 2} + assert m != m{1, 2} + m.discard(2) + m.discard(2) + assert m == m{1, 1} + assert m != m{1} + m.remove(1) + assert m == m{1} + m.remove(1) + assert m == m{} + assert_raises(-> m.remove(1), KeyError) + assert 1 not in m + assert 2 not in m + assert m{1, 2}.isdisjoint(m{3, 4}) + assert not m{1, 2}.isdisjoint(m{2, 3}) + assert m{1, 2} ^ m{2, 3} `typed_eq` m{1, 3} + m = m{1, 2} + m ^= m{2, 3} + assert m `typed_eq` m{1, 3} + assert m{1, 1} ^ m{1} `typed_eq` m{1} + assert multiset((1, 2)) == m{1, 2} == multiset(m{1, 2}) + assert multiset({1: 2, 2: 1}) == m{1, 1, 2} + assert m{} `isinstance` multiset + assert m{} `isinstance` collections.abc.Set + assert m{} `isinstance` collections.abc.MutableSet + assert True `isinstance` bool + class HasBool: + def __bool__(self) = False + assert not HasBool() + assert m{1}.count(2) == 0 + assert m{1, 1}.count(1) == 2 + bad_m: multiset = m{} + bad_m[1] = -1 + assert_raises(-> bad_m.count(1), ValueError) + assert len(m{1, 1}) == 1 + assert m{1, 1}.total() == 2 == m{1, 2}.total() + weird_m = m{1, 2} + weird_m[3] = 0 + assert weird_m == m{1, 2} + assert not (weird_m != m{1, 2}) + assert m{} <= m{} < m{1, 2} < m{1, 1, 2} <= m{1, 1, 2} + assert m{1, 1, 2} >= m{1, 1, 2} > m{1, 2} > m{} >= m{} + assert m{1} != {1:1, 2:0} + assert not (m{1} == {1:1, 2:0}) + assert s{1, 2, *(2, 3, 4), *(4, 5)} == s{1, 2, 3, 4, 5} + assert m{1, 2, *(2, 3, 4), *(4, 5)} == m{1, 2, 2, 3, 4, 4, 5} + assert {*(1, 2)} == {1, 2} + assert cycle(range(3))[:5] |> list == [0, 1, 2, 0, 1] == cycle(range(3)) |> iter |> .$[:5] |> list + assert cycle(range(2), 2)[:5] |> list == [0, 1, 0, 1] == cycle(range(2), 2) |> iter |> .$[:5] |> list + assert 2 in cycle(range(3)) + assert reversed(cycle(range(2), 2)) |> list == [1, 0, 1, 0] + assert cycle((_ for _ in range(2)), 2) |> list == [0, 1, 0, 1] + assert cycle(range(3)).count(0) == float("inf") + assert cycle(range(3), 3).index(2) == 2 + assert zip(range(2), range(2))[0] == (0, 0) == enumerate(range(2))[0] # type: ignore + assert reversed([0,1,3])[0] == 3 # type: ignore + assert cycle((), 0) |> list == [] + assert "1234" |> windowsof$(2) |> map$("".join) |> list == ["12", "23", "34"] + assert len(windowsof(2, "1234")) == 3 + assert windowsof(3, "12345", None) |> map$("".join) |> list == ["123", "234", "345"] # type: ignore + assert len(windowsof(3, "12345", None)) == 3 + assert windowsof(3, "1") |> list == [] == windowsof(2, "1", step=2) |> list + assert len(windowsof(2, "1")) == 0 == len(windowsof(2, "1", step=2)) + assert windowsof(2, "1", None) |> list == [("1", None)] == windowsof(2, "1", None, 2) |> list + assert len(windowsof(2, "1", None)) == 1 == len(windowsof(2, "1", None, 2)) + assert windowsof(2, "1234", step=2) |> map$("".join) |> list == ["12", "34"] == windowsof(2, "1234", fillvalue=None, step=2) |> map$("".join) |> list # type: ignore + assert len(windowsof(2, "1234", step=2)) == 2 == len(windowsof(2, "1234", fillvalue=None, step=2)) + assert repr(windowsof(2, "1234", None, 3)) == "windowsof(2, '1234', fillvalue=None, step=3)" + assert lift(,)((+), (*))(2, 3) == (5, 6) + assert "abac" |> windowsof$(2) |> filter$(addpattern( + (def (("a", b) if b != "b") -> True), + (def ((_, _)) -> False), + )) |> list == [("a", "c")] + assert "[A], [B]" |> windowsof$(3) |> map$(addpattern( + (def (("[","A","]")) -> "A"), + (def (("[","B","]")) -> "B"), + (def ((_,_,_)) -> None), + )) |> filter$((.is None) ..> (not)) |> list == ["A", "B"] + assert windowsof(3, "abcdefg", step=3) |> map$("".join) |> list == ["abc", "def"] + assert windowsof(3, "abcdefg", step=3) |> len == 2 + assert windowsof(3, "abcdefg", step=3, fillvalue="") |> map$("".join) |> list == ["abc", "def", "g"] + assert windowsof(3, "abcdefg", step=3, fillvalue="") |> len == 3 + assert windowsof(3, "abcdefg", step=2, fillvalue="") |> map$("".join) |> list == ["abc", "cde", "efg", "g"] + assert windowsof(3, "abcdefg", step=2, fillvalue="") |> len == 4 + assert windowsof(5, "abc", step=2, fillvalue="") |> map$("".join) |> list == ["abc"] + assert windowsof(5, "abc", step=2, fillvalue="") |> len == 1 + assert groupsof(2, "123", fillvalue="") |> map$("".join) |> list == ["12", "3"] + assert groupsof(2, "123", fillvalue="") |> len == 2 + assert groupsof(2, "123", fillvalue="") |> repr == "groupsof(2, '123', fillvalue='')" + assert flip((,), 0)(1, 2) == (1, 2) + assert flatten([1, 2, [3, 4]], 0) == [1, 2, [3, 4]] # type: ignore + assert flatten([1, 2, [3, 4]], None) |> list == [1, 2, 3, 4] # type: ignore + assert flatten([1, 2, [3, 4]], None) |> reversed |> list == [4, 3, 2, 1] # type: ignore + assert_raises(-> flatten([1, 2, [3, 4]]) |> list, TypeError) # type: ignore + assert flatten([[[1,2]], [[3], [4]]], 2) |> list == [1, 2, 3, 4] + assert flatten([[[1,2]], [[3], [4]]], 2) |> reversed |> list == [4, 3, 2, 1] + assert_raises(-> map((+), range(3), range(4), strict=True) |> list, ValueError) # type: ignore + assert map((+), range(3), range(4)$[:-1], strict=True) |> list == [0, 2, 4] == process_map((+), range(3), range(4)$[:-1], strict=True) |> list # type: ignore + assert range(3) |> map$((.+1), strict=True) |> list == [1, 2, 3] == range(3) |> process_map$((.+1), strict=True) |> list # type: ignore + assert cartesian_product((1, 2), (3, 4), repeat=0) |> list == [()] + assert (a=1, b=2)[1] == 2 + obj = object() + assert_raises((def -> obj.abc = 123), AttributeError) # type: ignore + hardref = map((.+1), [1,2,3]) + assert weakref.ref(hardref)() |> list == [2, 3, 4] # type: ignore + my_match_err = MatchError("my match error", 123) + assert process_map(ident, [my_match_err]) |> list |> str == str([my_match_err]) # type: ignore + # repeat the same thing again now that my_match_err.str has been called + assert process_map(ident, [my_match_err]) |> list |> str == str([my_match_err]) # type: ignore + match data tuple(1, 2) in (1, 2, 3): + assert False + data TestDefaultMatching(x="x default", y="y default") + TestDefaultMatching(got_x) = TestDefaultMatching(1) + assert got_x == 1 + TestDefaultMatching(y=got_y) = TestDefaultMatching(y=10) + assert got_y == 10 + TestDefaultMatching() = TestDefaultMatching() + data HasStar(x, y, *zs) + HasStar(x, *ys) = HasStar(1, 2, 3, 4) + assert x == 1 + assert ys == (2, 3, 4) + HasStar(x, y, z) = HasStar(1, 2, 3) + assert (x, y, z) == (1, 2, 3) + HasStar(5, y=10) = HasStar(5, 10) + HasStar(1, 2, 3, zs=(3,)) = HasStar(1, 2, 3) + HasStar(x=1, y=2) = HasStar(1, 2) + match HasStar(x) in HasStar(1, 2): + assert False + match HasStar(x, y) in HasStar(1, 2, 3): + assert False + data HasStarAndDef(x, y="y", *zs) + HasStarAndDef(1, "y") = HasStarAndDef(1) + HasStarAndDef(1) = HasStarAndDef(1) + HasStarAndDef(x=1) = HasStarAndDef(1) + HasStarAndDef(1, 2, 3, zs=(3,)) = HasStarAndDef(1, 2, 3) + HasStarAndDef(1, y=2) = HasStarAndDef(1, 2) + match HasStarAndDef(x, y) in HasStarAndDef(1, 2, 3): + assert False + + assert (.+1) kwargs) <**?| None is None # type: ignore + assert ((**kwargs) -> kwargs) <**?| {"a": 1, "b": 2} == {"a": 1, "b": 2} # type: ignore + assert (<**?|)((**kwargs) -> kwargs, None) is None # type: ignore + assert (<**?|)((**kwargs) -> kwargs, {"a": 1, "b": 2}) == {"a": 1, "b": 2} # type: ignore + optx = (**kwargs) -> kwargs + optx <**?|= None + assert optx is None + optx = (**kwargs) -> kwargs + optx <**?|= {"a": 1, "b": 2} + assert optx == {"a": 1, "b": 2} + + assert `const None ..?> (.+1)` is None is (..?>)(const None, (.+1))() # type: ignore + assert `(.+1) (.+1)` == 6 == (..?>)(const 5, (.+1))() # type: ignore + assert `(.+1) (+)` is None is (..?*>)(const None, (+))() # type: ignore + assert `(+) <*?.. const None` is None is (<*?..)((+), const None)() # type: ignore + assert `const((5, 2)) ..?*> (+)` == 7 == (..?*>)(const((5, 2)), (+))() # type: ignore + assert `(+) <*?.. const((5, 2))` == 7 == (<*?..)((+), const((5, 2)))() # type: ignore + assert `const None ..?**> (**kwargs) -> kwargs` is None is (..?**>)(const None, (**kwargs) -> kwargs)() # type: ignore + assert `((**kwargs) -> kwargs) <**?.. const None` is None is (<**?..)((**kwargs) -> kwargs, const None)() # type: ignore + assert `const({"a": 1}) ..?**> (**kwargs) -> kwargs` == {"a": 1} == (..?**>)(const({"a": 1}), (**kwargs) -> kwargs)() # type: ignore + assert `((**kwargs) -> kwargs) <**?.. const({"a": 1})` == {"a": 1} == (<**?..)((**kwargs) -> kwargs, const({"a": 1}))() # type: ignore + optx = const None + optx ..?>= (.+1) + optx ..?*>= (+) + optx ..?**>= (,) + assert optx() is None + optx = (.+1) + optx five (two + three), TypeError) # type: ignore + assert_raises(-> 5 (10), TypeError) # type: ignore + assert_raises(-> 5 [0], TypeError) # type: ignore + assert five ** 2 two == 50 + assert 2i x == 20i + some_str = "some" + assert_raises(-> some_str five, TypeError) + assert (not in)("a", "bcd") + assert not (not in)("a", "abc") + assert ("a" not in .)("bcd") + assert (. not in "abc")("d") + assert (is not)(1, True) + assert not (is not)(False, False) + assert (True is not .)(1) + assert (. is not True)(1) + a_dict = {} + a_dict[1] = 1 + a_dict[3] = 2 + a_dict[2] = 3 + assert a_dict |> str == "{1: 1, 3: 2, 2: 3}" == a_dict |> repr, a_dict + assert a_dict.keys() |> tuple == (1, 3, 2) + assert not a_dict.keys() `isinstance` list + assert not a_dict.values() `isinstance` list + assert not a_dict.items() `isinstance` list + assert len(a_dict.keys()) == len(a_dict.values()) == len(a_dict.items()) == 3 + assert {1: 1, 3: 2, 2: 3}.keys() |> tuple == (1, 3, 2) + assert {**{1: 0, 3: 0}, 2: 0}.keys() |> tuple == (1, 3, 2) == {**dict([(1, 1), (3, 2), (2, 3)])}.keys() |> tuple + assert a_dict == {1: 1, 2: 3, 3: 2} + assert {1: 1} |> str == "{1: 1}" == {1: 1} |> repr + assert py_dict `issubclass` dict + assert py_dict() `isinstance` dict + assert {5:0, 3:0, **{2:0, 6:0}, 8:0}.keys() |> tuple == (5, 3, 2, 6, 8) + a_multiset = m{1,1,2} + assert not a_multiset.keys() `isinstance` list + assert not a_multiset.values() `isinstance` list + assert not a_multiset.items() `isinstance` list + assert len(a_multiset.keys()) == len(a_multiset.values()) == len(a_multiset.items()) == 2 + assert (in)(1, [1, 2]) + assert not (1 not in .)([1, 2]) + assert not (in)([[]], []) + assert ("{a}" . .)("format")(a=1) == "1" + a_dict = {"a": 1, "b": 2} + a_dict |= {"a": 10, "c": 20} + assert a_dict == {"a": 10, "b": 2, "c": 20} == {"a": 1, "b": 2} | {"a": 10, "c": 20} + assert ["abc" ; "def"] == ['abc', 'def'] + assert ["abc" ;; "def"] == [['abc'], ['def']] + assert {"a":0, "b":1}$[0] == "a" + assert (|0, NotImplemented, 2|)$[1] is NotImplemented + assert m{1, 1, 2} |> fmap$(.+1) == m{2, 2, 3} + assert (+) ..> ((*) ..> (/)) == (+) ..> (*) ..> (/) == ((+) ..> (*)) ..> (/) # type: ignore + def f(x, y=1) = x, y # type: ignore + f.is_f = True # type: ignore + assert (f ..*> (+)).is_f # type: ignore + really_long_var = 10 + assert (...=really_long_var) == (10,) + assert (...=really_long_var, abc="abc") == (10, "abc") + assert (abc="abc", ...=really_long_var) == ("abc", 10) + assert (...=really_long_var).really_long_var == 10 # type: ignore + n = [0] + assert n[0] == 0 + assert_raises(-> m{{1:2,2:3}}, TypeError) + assert_raises((def -> from typing import blah), ImportError) # NOQA + assert type(m{1, 2}) is multiset + assert multiset(collections.Counter({1: 1, 2: 1})) `typed_eq` m{1, 2} + assert +m{-1, 1} `typed_eq` m{-1, 1} + assert -m{-1, 1} `typed_eq` m{} + assert m{1, 1, 2} + m{1, 3} `typed_eq` m{1, 1, 1, 2, 3} + assert m{1, 1, 2} | m{1, 3} `typed_eq` m{1, 1, 2, 3} + assert m{1, 1, 2} & m{1, 3} `typed_eq` m{1} + assert m{1, 1, 2} - m{1, 3} `typed_eq` m{1, 2} + assert (.+1) `and_then` (.*2) `and_then_await` (./3) |> repr == "$(?, 1) `and_then` $(?, 2) `and_then_await` $(?, 3)" + assert 5.5⏨3 == 5.5 * 10**3 + assert (x => x)(5) == 5 == (def x => x)(5) + assert (=> _)(5) == 5 == (def => _)(5) # type: ignore + assert ((x, y) => (x, y))(1, 2) == (1, 2) == (def (x, y) => (x, y))(1, 2) + assert (def (int(x)) => x)(5) == 5 == (def (int -> x) => x)("5") + assert (def (x: int) -> int => x)(5) == 5 == (def (int(x)) -> int => x)(5) + assert (x ⇒ x)(5) == 5 == (def x ⇒ x)(5) + assert f"a: { "abc" }" == "a: abc" == f'a: { 'abc' }' + assert f"1 + {"a" + "b"} + 2 + {"c" + "d"}" == "1 + ab + 2 + cd" == f'1 + {'a' + 'b'} + 2 + {'c' + 'd'}' + assert f"{"a" + "b"} + c + {"d" + "e"}" == "ab + c + de" == f'{'a' + 'b'} + c + {'d' + 'e'}' + assert f"""{""" +"""}""" == """ +""" == f"""{''' +'''}""" + assert f"""{( + )}""" == "()" == f'''{( + )}''' + assert f"{'\n'.join(["", ""])}" == "\n" + assert f"""{f'''{f'{f"{1+1}"}'}'''}""" == "2" == f"{f"{f"{f"{f"{f"{1+1}"}"}"}"}"}" + assert f"___{ + 1 +}___" == '___1___' == f"___{( + 1 +)}___" + x = 10 + assert x == 5 where: + x = 5 + assert x == 10 + def nested() = f where: + f = def -> g where: + def g() = x where: + x = 5 + assert nested()()() == 5 + class HasPartial: + def f(self, x) = (self, x) + g = f$(?, 1) + has_partial = HasPartial() + assert has_partial.g() == (has_partial, 1) + xs = zip([1, 2], [3, 4]) + py_xs = py_zip([1, 2], [3, 4]) + assert list(xs) == [(1, 3), (2, 4)] == list(xs) + assert list(py_xs) == [(1, 3), (2, 4)] + assert list(py_xs) == [] + xs = map((+), [1, 2], [3, 4]) + py_xs = py_map((+), [1, 2], [3, 4]) + assert list(xs) == [4, 6] == list(xs) + assert list(py_xs) == [4, 6] + assert list(py_xs) == [] + for xs in [ + zip((x for x in range(5)), (x for x in range(10))), + py_zip((x for x in range(5)), (x for x in range(10))), + map((,), (x for x in range(5)), (x for x in range(10))), + py_map((,), (x for x in range(5)), (x for x in range(10))), + ]: # type: ignore + assert list(xs) == list(zip(range(5), range(5))) + assert list(xs) == [] + xs = map((.+1), range(5)) + py_xs = py_map((.+1), range(5)) + assert list(xs) == list(range(1, 6)) == list(xs) + assert list(py_xs) == list(range(1, 6)) + assert list(py_xs) == [] + assert count()[:10:2] == range(0, 10, 2) + assert count()[10:2] == range(10, 2) + some_data = [ + (name="a", val="123"), + (name="b", val="567"), + ] + for mapreducer in ( + mapreduce.using_processes$(lift(,)(.name, .val)), # type: ignore + mapreduce.using_threads$(lift(,)(.name, .val)), # type: ignore + collectby.using_processes$(.name, value_func=.val), # type: ignore + collectby.using_threads$(.name, value_func=.val), # type: ignore + ): + assert some_data |> mapreducer == {"a": ["123"], "b": ["567"]} + assert_raises(-> collectby(.[0], [(0, 1), (0, 2)], reduce_func=False), ValueError) # type: ignore + return True From 061e67f147a25248e5aa78048ca26e79ad541639 Mon Sep 17 00:00:00 2001 From: Evan Hubinger Date: Fri, 3 Nov 2023 00:36:22 -0700 Subject: [PATCH 048/121] Fix no wrap test --- coconut/tests/src/cocotest/agnostic/primary_2.coco | 2 ++ 1 file changed, 2 insertions(+) diff --git a/coconut/tests/src/cocotest/agnostic/primary_2.coco b/coconut/tests/src/cocotest/agnostic/primary_2.coco index 9b36abe9d..83385cbec 100644 --- a/coconut/tests/src/cocotest/agnostic/primary_2.coco +++ b/coconut/tests/src/cocotest/agnostic/primary_2.coco @@ -4,6 +4,8 @@ import weakref if TYPE_CHECKING: from typing import Any, Iterable +else: + Any = Iterable = None from importlib import reload # NOQA from .util import assert_raises, typed_eq From 77daeb9a31e2af3bd9a9bbb2c7c21bb2564aac3d Mon Sep 17 00:00:00 2001 From: Evan Hubinger Date: Fri, 3 Nov 2023 02:13:42 -0700 Subject: [PATCH 049/121] Add pyspy --- .gitignore | 1 + Makefile | 28 +++++++++++++------ coconut/constants.py | 2 ++ .../src/cocotest/agnostic/primary_2.coco | 2 +- 4 files changed, 23 insertions(+), 10 deletions(-) diff --git a/.gitignore b/.gitignore index 243d558fd..26f176d2b 100644 --- a/.gitignore +++ b/.gitignore @@ -141,3 +141,4 @@ index.rst vprof.json /coconut/icoconut/coconut/ __coconut_cache__/ +profile.svg diff --git a/Makefile b/Makefile index 5ff886a84..1128c283b 100644 --- a/Makefile +++ b/Makefile @@ -217,6 +217,11 @@ test-easter-eggs: clean test-pyparsing: export COCONUT_PURE_PYTHON=TRUE test-pyparsing: test-univ +# same as test-univ but disables the computation graph +.PHONY: test-no-computation-graph +test-no-computation-graph: export COCONUT_USE_COMPUTATION_GRAPH=FALSE +test-no-computation-graph: test-univ + # same as test-univ but uses --minify .PHONY: test-minify test-minify: export COCONUT_USE_COLOR=TRUE @@ -330,16 +335,21 @@ check-reqs: profile-parser: export COCONUT_USE_COLOR=TRUE profile-parser: export COCONUT_PURE_PYTHON=TRUE profile-parser: - coconut ./coconut/tests/src/cocotest/agnostic ./coconut/tests/dest/cocotest --force --profile --verbose --stack-size 4096 --recursion-limit 4096 2>&1 | tee ./profile.log + coconut ./coconut/tests/src/cocotest/agnostic ./coconut/tests/dest/cocotest --force --jobs 0 --profile --verbose --stack-size 4096 --recursion-limit 4096 2>&1 | tee ./profile.log + +.PHONY: pyspy +pyspy: + py-spy record -o profile.svg --native -- python -m coconut ./coconut/tests/src/cocotest/agnostic ./coconut/tests/dest/cocotest --force --jobs 0 + open profile.svg -.PHONY: profile-time -profile-time: - vprof -c h "./coconut ./coconut/tests/src/cocotest/agnostic ./coconut/tests/dest/cocotest --force --stack-size 4096 --recursion-limit 4096" --output-file ./vprof.json +.PHONY: vprof-time +vprof-time: + vprof -c h "./coconut ./coconut/tests/src/cocotest/agnostic ./coconut/tests/dest/cocotest --force --jobs 0 --stack-size 4096 --recursion-limit 4096" --output-file ./vprof.json -.PHONY: profile-memory -profile-memory: - vprof -c m "./coconut ./coconut/tests/src/cocotest/agnostic ./coconut/tests/dest/cocotest --force --stack-size 4096 --recursion-limit 4096" --output-file ./vprof.json +.PHONY: vprof-memory +vprof-memory: + vprof -c m "./coconut ./coconut/tests/src/cocotest/agnostic ./coconut/tests/dest/cocotest --force --jobs 0 --stack-size 4096 --recursion-limit 4096" --output-file ./vprof.json -.PHONY: view-profile -view-profile: +.PHONY: view-vprof +view-vprof: vprof --input-file ./vprof.json diff --git a/coconut/constants.py b/coconut/constants.py index 7f5976c8f..ed699fc91 100644 --- a/coconut/constants.py +++ b/coconut/constants.py @@ -953,6 +953,7 @@ def get_path_env_var(env_var, default): ("pre-commit", "py3"), "requests", "vprof", + "py-spy", ), "docs": ( "sphinx", @@ -1005,6 +1006,7 @@ def get_path_env_var(env_var, default): ("async_generator", "py35"): (1, 10), ("exceptiongroup", "py37;py<311"): (1,), ("ipython", "py>=39"): (8, 16), + "py-spy": (0, 3), } pinned_min_versions = { diff --git a/coconut/tests/src/cocotest/agnostic/primary_2.coco b/coconut/tests/src/cocotest/agnostic/primary_2.coco index 83385cbec..9e27b0f6f 100644 --- a/coconut/tests/src/cocotest/agnostic/primary_2.coco +++ b/coconut/tests/src/cocotest/agnostic/primary_2.coco @@ -361,7 +361,7 @@ def primary_test_2() -> bool: x = 10 assert x == 5 where: x = 5 - assert x == 10 + assert x == 10, x def nested() = f where: f = def -> g where: def g() = x where: From fb04ee69d72f8eeef5d5a7c15898b5c883bc0f0d Mon Sep 17 00:00:00 2001 From: Evan Hubinger Date: Fri, 3 Nov 2023 21:14:27 -0700 Subject: [PATCH 050/121] Improve exception formatting Resolves #794. --- coconut/compiler/compiler.py | 8 ++++---- coconut/exceptions.py | 7 +++++-- coconut/root.py | 2 +- .../tests/src/cocotest/agnostic/primary_2.coco | 5 ++--- coconut/tests/src/extras.coco | 15 +++++++++++++++ 5 files changed, 27 insertions(+), 10 deletions(-) diff --git a/coconut/compiler/compiler.py b/coconut/compiler/compiler.py index 74ba9feea..8ad08f02f 100644 --- a/coconut/compiler/compiler.py +++ b/coconut/compiler/compiler.py @@ -1218,10 +1218,10 @@ def make_err(self, errtype, message, original, loc=0, ln=None, extra=None, refor kwargs["extra"] = extra return errtype(message, snippet, loc_in_snip, ln, endpoint=endpt_in_snip, filename=self.filename, **kwargs) - def make_syntax_err(self, err, original): + def make_syntax_err(self, err, original, after_parsing=False): """Make a CoconutSyntaxError from a CoconutDeferredSyntaxError.""" msg, loc = err.args - return self.make_err(CoconutSyntaxError, msg, original, loc) + return self.make_err(CoconutSyntaxError, msg, original, loc, endpoint=not after_parsing) def make_parse_err(self, err, msg=None, include_ln=True, **kwargs): """Make a CoconutParseError from a ParseBaseException.""" @@ -1328,7 +1328,7 @@ def parse( filename=filename, incremental_cache_filename=incremental_cache_filename, )) - pre_procd = None + pre_procd = parsed = None try: with logger.gather_parsing_stats(): try: @@ -1339,7 +1339,7 @@ def parse( raise self.make_parse_err(err) except CoconutDeferredSyntaxError as err: internal_assert(pre_procd is not None, "invalid deferred syntax error in pre-processing", err) - raise self.make_syntax_err(err, pre_procd) + raise self.make_syntax_err(err, pre_procd, after_parsing=parsed is not None) # RuntimeError, not RecursionError, for Python < 3.5 except RuntimeError as err: raise CoconutException( diff --git a/coconut/exceptions.py b/coconut/exceptions.py index 61319e4ed..341ef3831 100644 --- a/coconut/exceptions.py +++ b/coconut/exceptions.py @@ -180,11 +180,14 @@ def message(self, message, source, point, ln, extra=None, endpoint=None, filenam point_ind = clip(point_ind, 0, len(lines[0])) endpoint_ind = clip(endpoint_ind, 0, len(lines[-1])) + max_line_len = max(len(line) for line in lines) + message += "\n" + " " * (taberrfmt + point_ind) if point_ind >= len(lines[0]): - message += "|\n" + message += "|" else: - message += "/" + "~" * (len(lines[0]) - point_ind - 1) + "\n" + message += "/" + "~" * (len(lines[0]) - point_ind - 1) + message += "~" * (max_line_len - len(lines[0])) + "\n" for line in lines: message += "\n" + " " * taberrfmt + line message += ( diff --git a/coconut/root.py b/coconut/root.py index b78fdbac9..26a67c81e 100644 --- a/coconut/root.py +++ b/coconut/root.py @@ -26,7 +26,7 @@ VERSION = "3.0.3" VERSION_NAME = None # False for release, int >= 1 for develop -DEVELOP = 16 +DEVELOP = 17 ALPHA = False # for pre releases rather than post releases assert DEVELOP is False or DEVELOP >= 1, "DEVELOP must be False or an int >= 1" diff --git a/coconut/tests/src/cocotest/agnostic/primary_2.coco b/coconut/tests/src/cocotest/agnostic/primary_2.coco index 9e27b0f6f..e49eae5c6 100644 --- a/coconut/tests/src/cocotest/agnostic/primary_2.coco +++ b/coconut/tests/src/cocotest/agnostic/primary_2.coco @@ -1,11 +1,10 @@ import collections import collections.abc import weakref +import sys -if TYPE_CHECKING: +if TYPE_CHECKING or sys.version_info >= (3, 5): from typing import Any, Iterable -else: - Any = Iterable = None from importlib import reload # NOQA from .util import assert_raises, typed_eq diff --git a/coconut/tests/src/extras.coco b/coconut/tests/src/extras.coco index 2e012d6c0..bf5ded5f1 100644 --- a/coconut/tests/src/extras.coco +++ b/coconut/tests/src/extras.coco @@ -277,6 +277,21 @@ def gam_eps_rate(bitarr) = ( else: assert False + try: + parse(""" +def f(x=1, y) = x, y + +class A + +def g(x) = x + """.strip()) + except CoconutSyntaxError as err: + err_str = str(err) + assert "non-default arguments must come first" in err_str, err_str + assert "class A" not in err_str, err_str + else: + assert False + assert parse("def f(x):\n ${var}", "xonsh") == "def f(x):\n ${var}\n" assert "data ABC" not in parse("data ABC:\n ${var}", "xonsh") From 61bd78755f11ccbbdd06e53f6fcb0248720c880c Mon Sep 17 00:00:00 2001 From: Evan Hubinger Date: Fri, 3 Nov 2023 22:59:44 -0700 Subject: [PATCH 051/121] Improve profiling --- .gitignore | 5 ++++- Makefile | 29 ++++++++++++++++++++--------- coconut/compiler/header.py | 2 ++ coconut/compiler/util.py | 2 +- coconut/requirements.py | 1 + 5 files changed, 28 insertions(+), 11 deletions(-) diff --git a/.gitignore b/.gitignore index 26f176d2b..96d716fb7 100644 --- a/.gitignore +++ b/.gitignore @@ -138,7 +138,10 @@ pyprover/ bbopt/ coconut-prelude/ index.rst -vprof.json /coconut/icoconut/coconut/ __coconut_cache__/ + +# Profiling +vprof.json profile.svg +profile.speedscope diff --git a/Makefile b/Makefile index 1128c283b..35da6ac15 100644 --- a/Makefile +++ b/Makefile @@ -25,27 +25,27 @@ dev-py3: clean setup-py3 .PHONY: setup setup: - python -m ensurepip + -python -m ensurepip python -m pip install --upgrade setuptools wheel pip pytest_remotedata cython .PHONY: setup-py2 setup-py2: - python2 -m ensurepip + -python2 -m ensurepip python2 -m pip install --upgrade "setuptools<58" wheel pip pytest_remotedata cython .PHONY: setup-py3 setup-py3: - python3 -m ensurepip + -python3 -m ensurepip python3 -m pip install --upgrade setuptools wheel pip pytest_remotedata cython .PHONY: setup-pypy setup-pypy: - pypy -m ensurepip + -pypy -m ensurepip pypy -m pip install --upgrade "setuptools<58" wheel pip pytest_remotedata .PHONY: setup-pypy3 setup-pypy3: - pypy3 -m ensurepip + -pypy3 -m ensurepip pypy3 -m pip install --upgrade setuptools wheel pip pytest_remotedata .PHONY: install @@ -337,10 +337,21 @@ profile-parser: export COCONUT_PURE_PYTHON=TRUE profile-parser: coconut ./coconut/tests/src/cocotest/agnostic ./coconut/tests/dest/cocotest --force --jobs 0 --profile --verbose --stack-size 4096 --recursion-limit 4096 2>&1 | tee ./profile.log -.PHONY: pyspy -pyspy: - py-spy record -o profile.svg --native -- python -m coconut ./coconut/tests/src/cocotest/agnostic ./coconut/tests/dest/cocotest --force --jobs 0 - open profile.svg +.PHONY: open-speedscope +open-speedscope: + npm install -g speedscope + speedscope ./profile.speedscope + +.PHONY: pyspy-purepy +pyspy-purepy: export COCONUT_PURE_PYTHON=TRUE +pyspy-purepy: + py-spy record -o profile.speedscope --format speedscope --subprocesses -- python -m coconut ./coconut/tests/src/cocotest/agnostic ./coconut/tests/dest/cocotest --force + open-speedscope + +.PHONY: pyspy-native +pyspy-native: + py-spy record -o profile.speedscope --format speedscope --native -- python -m coconut ./coconut/tests/src/cocotest/agnostic ./coconut/tests/dest/cocotest --force --jobs 0 + open-speedscope .PHONY: vprof-time vprof-time: diff --git a/coconut/compiler/header.py b/coconut/compiler/header.py index 3910880b7..5b1a68686 100644 --- a/coconut/compiler/header.py +++ b/coconut/compiler/header.py @@ -44,6 +44,7 @@ univ_open, get_target_info, assert_remove_prefix, + memoize, ) from coconut.compiler.util import ( split_comment, @@ -96,6 +97,7 @@ def minify_header(compiled): template_dir = os.path.join(os.path.dirname(os.path.abspath(__file__)), "templates") +@memoize() def get_template(template): """Read the given template file.""" with univ_open(os.path.join(template_dir, template) + template_ext, "r") as template_file: diff --git a/coconut/compiler/util.py b/coconut/compiler/util.py index 9db8bf782..1dfbe7d34 100644 --- a/coconut/compiler/util.py +++ b/coconut/compiler/util.py @@ -514,7 +514,7 @@ def get_pyparsing_cache(): else: # on pyparsing we have to do this try: # this is sketchy, so errors should only be complained - return get_func_closure(packrat_cache.get.__func__)["cache"] + return get_func_closure(packrat_cache.set.__func__)["cache"] except Exception as err: complain(err) return {} diff --git a/coconut/requirements.py b/coconut/requirements.py index 55c293471..c2e9668a0 100644 --- a/coconut/requirements.py +++ b/coconut/requirements.py @@ -247,6 +247,7 @@ def everything_in(req_dict): extras["dev"] = uniqueify_all( everything_in(extras), + get_reqs("purepython"), get_reqs("dev"), ) From f1759b1413c12942d65b60648e67cd81671a5ebd Mon Sep 17 00:00:00 2001 From: Evan Hubinger Date: Sat, 4 Nov 2023 02:27:06 -0700 Subject: [PATCH 052/121] Add async_map Resolves #795. --- DOCS.md | 152 ++++++++++++++++-- __coconut__/__init__.pyi | 20 ++- coconut/compiler/header.py | 46 +++++- coconut/compiler/templates/header.py_template | 20 +-- coconut/constants.py | 26 +-- coconut/requirements.py | 11 +- coconut/root.py | 2 +- .../src/cocotest/target_36/py36_test.coco | 14 ++ .../cocotest/target_sys/target_sys_test.coco | 4 +- 9 files changed, 252 insertions(+), 43 deletions(-) diff --git a/DOCS.md b/DOCS.md index a514f1999..bce108fcf 100644 --- a/DOCS.md +++ b/DOCS.md @@ -11,6 +11,7 @@ depth: 2 --- ``` + ## Overview This documentation covers all the features of the [Coconut Programming Language](http://evhub.github.io/coconut/), and is intended as a reference/specification, not a tutorialized introduction. For an introduction to and tutorial of Coconut, see [the tutorial](./HELP.md). @@ -25,6 +26,7 @@ Thought Coconut syntax is primarily based on that of Python, other languages tha If you want to try Coconut in your browser, check out the [online interpreter](https://cs121-team-panda.github.io/coconut-interpreter). Note, however, that it may be running an outdated version of Coconut. + ## Installation ```{contents} @@ -85,23 +87,15 @@ pip install coconut[opt_dep_1,opt_dep_2] The full list of optional dependencies is: -- `all`: alias for `jupyter,watch,mypy,backports,xonsh` (this is the recommended way to install a feature-complete version of Coconut). +- `all`: alias for everything below (this is the recommended way to install a feature-complete version of Coconut). - `jupyter`/`ipython`: enables use of the `--jupyter` / `--ipython` flag. +- `kernel`: lightweight subset of `jupyter` that only includes the dependencies that are strictly necessary for Coconut's [Jupyter kernel](#kernel). - `watch`: enables use of the `--watch` flag. - `mypy`: enables use of the `--mypy` flag. -- `backports`: installs libraries that backport newer Python features to older versions, which Coconut will automatically use instead of the standard library if the standard library is not available. Specifically: - - Installs [`exceptiongroup`](https://pypi.org/project/exceptiongroup/) to backport [`ExceptionGroup` and `BaseExceptionGroup`](https://docs.python.org/3/library/exceptions.html#ExceptionGroup). - - Installs [`dataclasses`](https://pypi.org/project/dataclasses/) to backport [`dataclasses`](https://docs.python.org/3/library/dataclasses.html). - - Installs [`typing`](https://pypi.org/project/typing/) to backport [`typing`](https://docs.python.org/3/library/typing.html) ([`typing_extensions`](https://pypi.org/project/typing-extensions/) is always installed for backporting individual `typing` objects). - - Installs [`aenum`](https://pypi.org/project/aenum) to backport [`enum`](https://docs.python.org/3/library/enum.html). - - Installs [`async_generator`](https://github.com/python-trio/async_generator) to backport [`async` generators](https://peps.python.org/pep-0525/) and [`asynccontextmanager`](https://docs.python.org/3/library/contextlib.html#contextlib.asynccontextmanager). - - Installs [`trollius`](https://pypi.python.org/pypi/trollius) to backport [`async`/`await`](https://docs.python.org/3/library/asyncio-task.html) and [`asyncio`](https://docs.python.org/3/library/asyncio.html). - `xonsh`: enables use of Coconut's [`xonsh` support](#xonsh-support). - `numpy`: installs everything necessary for making use of Coconut's [`numpy` integration](#numpy-integration). -- `kernel`: lightweight subset of `jupyter` that only includes the dependencies that are strictly necessary for Coconut's [Jupyter kernel](#kernel). -- `tests`: everything necessary to test the Coconut language itself. -- `docs`: everything necessary to build Coconut's documentation. -- `dev`: everything necessary to develop on the Coconut language itself, including all of the dependencies above. +- `jupyterlab`: installs everything necessary to use [JupyterLab](https://github.com/jupyterlab/jupyterlab) with Coconut. +- `jupytext`: installs everything necessary to use [Jupytext](https://github.com/mwouts/jupytext) with Coconut. #### Develop Version @@ -113,6 +107,7 @@ which will install the most recent working version from Coconut's [`develop` bra _Note: if you have an existing release version of `coconut` installed, you'll need to `pip uninstall coconut` before installing `coconut-develop`._ + ## Compilation ```{contents} @@ -291,7 +286,7 @@ Finally, while Coconut will try to compile Python-3-specific syntax to its unive - the `nonlocal` keyword, - keyword-only function parameters (use [pattern-matching function definition](#pattern-matching-functions) for universal code), -- `async` and `await` statements (requires a specific target; Coconut will attempt different backports based on the targeted version), +- `async` and `await` statements (requires a specific target; Coconut will attempt different [backports](#backports) based on the targeted version), - `:=` assignment expressions (requires `--target 3.8`), - positional-only function parameters (use [pattern-matching function definition](#pattern-matching-functions) for universal code) (requires `--target 3.8`), - `a[x, *y]` variadic generic syntax (use [type parameter syntax](#type-parameter-syntax) for universal code) (requires `--target 3.11`), and @@ -351,6 +346,21 @@ The style issues which will cause `--strict` to throw an error are: Note that many of the above style issues will still show a warning if `--strict` is not present. +#### Backports + +In addition to the newer Python features that Coconut can backport automatically itself to older Python versions, Coconut will also automatically compile code to make use of a variety of external backports as well. These backports are automatically installed with Coconut if needed and Coconut will automatically use them instead of the standard library if the standard library is not available. These backports are: +- [`typing`](https://pypi.org/project/typing/) for backporting [`typing`](https://docs.python.org/3/library/typing.html). +- [`typing_extensions`](https://pypi.org/project/typing-extensions/) for backporting individual `typing` objects. +- [`backports.functools-lru-cache`](https://pypi.org/project/backports.functools-lru-cache/) for backporting [`functools.lru_cache`](https://docs.python.org/3/library/functools.html#functools.lru_cache). +- [`exceptiongroup`](https://pypi.org/project/exceptiongroup/) for backporting [`ExceptionGroup` and `BaseExceptionGroup`](https://docs.python.org/3/library/exceptions.html#ExceptionGroup). +- [`dataclasses`](https://pypi.org/project/dataclasses/) for backporting [`dataclasses`](https://docs.python.org/3/library/dataclasses.html). +- [`aenum`](https://pypi.org/project/aenum) for backporting [`enum`](https://docs.python.org/3/library/enum.html). +- [`async_generator`](https://github.com/python-trio/async_generator) for backporting [`async` generators](https://peps.python.org/pep-0525/) and [`asynccontextmanager`](https://docs.python.org/3/library/contextlib.html#contextlib.asynccontextmanager). +- [`trollius`](https://pypi.python.org/pypi/trollius) for backporting [`async`/`await`](https://docs.python.org/3/library/asyncio-task.html) and [`asyncio`](https://docs.python.org/3/library/asyncio.html). + +Note that, when distributing compiled Coconut code, if you use any of these backports, you'll need to make sure that the requisite backport module is included as a dependency. + + ## Integrations ```{contents} @@ -493,6 +503,7 @@ Compilation always uses the same parameters as in the [Coconut Jupyter kernel](# Note that the way that Coconut integrates with `xonsh`, `@()` syntax and the `execx` command will only work with Python code, not Coconut code. Additionally, Coconut will only compile individual commands—Coconut will not touch the `.xonshrc` or any other `.xsh` files. + ## Operators ```{contents} @@ -502,6 +513,7 @@ depth: 1 --- ``` + ### Precedence In order of precedence, highest first, the operators supported in Coconut are: @@ -544,6 +556,7 @@ x if c else y, ternary (short-circuits) For example, since addition has a higher precedence than piping, expressions of the form `x |> y + z` are equivalent to `x |> (y + z)`. + ### Lambdas Coconut provides the simple, clean `=>` operator as an alternative to Python's `lambda` statements. The syntax for the `=>` operator is `(parameters) => expression` (or `parameter => expression` for one-argument lambdas). The operator has the same precedence as the old statement, which means it will often be necessary to surround the lambda in parentheses, and is right-associative. @@ -601,6 +614,7 @@ get_random_number = (=> random.random()) _Note: Nesting implicit lambdas can lead to problems with the scope of the `_` parameter to each lambda. It is recommended that nesting implicit lambdas be avoided._ + ### Partial Application Coconut uses a `$` sign right after a function's name but before the open parenthesis used to call the function to denote partial application. @@ -649,6 +663,7 @@ expnums = map(lambda x: pow(x, 2), range(5)) print(list(expnums)) ``` + ### Pipes Coconut uses pipe operators for pipeline-style function application. All the operators have a precedence in-between function composition pipes and comparisons, and are left-associative. All operators also support in-place versions. The different operators are: @@ -720,6 +735,7 @@ async def do_stuff(some_data): return post_proc(await async_func(some_data)) ``` + ### Function Composition Coconut has three basic function composition operators: `..`, `..>`, and `<..`. Both `..` and `<..` use math-style "backwards" function composition, where the first function is called last, while `..>` uses "forwards" function composition, where the first function is called first. Forwards and backwards function composition pipes cannot be used together in the same expression (unlike normal pipes) and have precedence in-between `None`-coalescing and normal pipes. @@ -765,6 +781,7 @@ fog = lambda *args, **kwargs: f(g(*args, **kwargs)) f_into_g = lambda *args, **kwargs: g(f(*args, **kwargs)) ``` + ### Iterator Slicing Coconut uses a `$` sign right after an iterator before a slice to perform iterator slicing, as in `it$[:5]`. Coconut's iterator slicing works much the same as Python's sequence slicing, and looks much the same as Coconut's partial application, but with brackets instead of parentheses. @@ -783,6 +800,7 @@ map(x => x*2, range(10**100))$[-1] |> print **Python:** _Can't be done without a complicated iterator slicing function and inspection of custom objects. The necessary definitions in Python can be found in the Coconut header._ + ### Iterator Chaining Coconut uses the `::` operator for iterator chaining. Coconut's iterator chaining is done lazily, in that the arguments are not evaluated until they are needed. It has a precedence in-between bitwise or and infix calls. Chains are reiterable (can be iterated over multiple times and get the same result) only when the iterators passed in are reiterable. The in-place operator is `::=`. @@ -816,6 +834,7 @@ def N(n=0) = (n,) :: N(n+1) # no infinite loop because :: is lazy **Python:** _Can't be done without a complicated iterator comprehension in place of the lazy chaining. See the compiled code for the Python syntax._ + ### Infix Functions Coconut allows for infix function calling, where an expression that evaluates to a function is surrounded by backticks and then can have arguments placed in front of or behind it. Infix calling has a precedence in-between chaining and `None`-coalescing, and is left-associative. @@ -856,6 +875,7 @@ def mod(a, b): return a % b print(mod(x, 2)) ``` + ### Custom Operators Coconut allows you to declare your own custom operators with the syntax @@ -926,6 +946,7 @@ print(bool(0)) print(math.log10(100)) ``` + ### None Coalescing Coconut provides `??` as a `None`-coalescing operator, similar to the `??` null-coalescing operator in C# and Swift. Additionally, Coconut implements all of the `None`-aware operators proposed in [PEP 505](https://www.python.org/dev/peps/pep-0505/). @@ -997,6 +1018,7 @@ import functools (lambda result: None if result is None else result.attr[index].method())(could_be_none()) ``` + ### Protocol Intersection Coconut uses the `&:` operator to indicate protocol intersection. That is, for two [`typing.Protocol`s](https://docs.python.org/3/library/typing.html#typing.Protocol) `Protocol1` and `Protocol1`, `Protocol1 &: Protocol2` is equivalent to a `Protocol` that combines the requirements of both `Protocol1` and `Protocol2`. @@ -1052,6 +1074,7 @@ class CanAddAndSub(Protocol, Generic[T, U, V]): raise NotImplementedError ``` + ### Unicode Alternatives Coconut supports Unicode alternatives to many different operator symbols. The Unicode alternatives are relatively straightforward, and chosen to reflect the look and/or meaning of the original symbol. @@ -1107,6 +1130,7 @@ _Note: these are only the default, built-in unicode operators. Coconut supports ⏨ (\u23e8) => "e" (in scientific notation) ``` + ## Keywords ```{contents} @@ -1116,6 +1140,7 @@ depth: 1 --- ``` + ### `match` Coconut provides fully-featured, functional pattern-matching through its `match` statements. @@ -1329,6 +1354,7 @@ _Showcases the use of an iterable search pattern and a view pattern to construct **Python:** _Can't be done without a long series of checks for each `match` statement. See the compiled code for the Python syntax._ + ### `case` Coconut's `case` blocks serve as an extension of Coconut's `match` statement for performing multiple `match` statements against the same value, where only one of them should succeed. Unlike lone `match` statements, only one match statement inside of a `case` block will ever succeed, and thus more general matches should be put below more specific ones. @@ -1392,6 +1418,7 @@ _Example of the `cases` keyword instead._ **Python:** _Can't be done without a long series of checks for each `match` statement. See the compiled code for the Python syntax._ + ### `match for` Coconut supports pattern-matching in for loops, where the pattern is matched against each item in the iterable. The syntax is @@ -1423,6 +1450,7 @@ for user_data in get_data(): print(uid) ``` + ### `data` Coconut's `data` keyword is used to create immutable, algebraic data types, including built-in support for destructuring [pattern-matching](#match) and [`fmap`](#fmap). @@ -1543,6 +1571,7 @@ data namedpt(name `isinstance` str, x `isinstance` int, y `isinstance` int): **Python:** _Can't be done without a series of method definitions for each data type. See the compiled code for the Python syntax._ + ### `where` Coconut's `where` statement is fairly straightforward. The syntax for a `where` statement is just @@ -1568,6 +1597,7 @@ _b = 2 result = _a + _b ``` + ### `async with for` In modern Python `async` code, such as when using [`contextlib.aclosing`](https://docs.python.org/3/library/contextlib.html#contextlib.aclosing), it is often recommended to use a pattern like @@ -1622,6 +1652,7 @@ async with my_generator() as agen: print(value) ``` + ### Handling Keyword/Variable Name Overlap In Coconut, the following keywords are also valid variable names: @@ -1666,6 +1697,7 @@ print(data) x, y = input_list ``` + ## Expressions ```{contents} @@ -1675,6 +1707,7 @@ depth: 1 --- ``` + ### Statement Lambdas The statement lambda syntax is an extension of the [normal lambda syntax](#lambdas) to support statements, not just expressions. @@ -1722,6 +1755,7 @@ g = def (a: int, b: int) -> int => a ** b _Deprecated: if the deprecated `->` is used in place of `=>`, then return type annotations will not be available._ + ### Operator Functions Coconut uses a simple operator function short-hand: surround an operator with parentheses to retrieve its function. Similarly to iterator comprehensions, if the operator function is the only argument to a function, the parentheses of the function call can also serve as the parentheses for the operator function. @@ -1806,6 +1840,7 @@ import operator print(list(map(operator.add, range(0, 5), range(5, 10)))) ``` + ### Implicit Partial Application Coconut supports a number of different syntactical aliases for common partial application use cases. These are: @@ -1853,6 +1888,7 @@ mod(5, 3) (3 * 2) + 1 ``` + ### Enhanced Type Annotation Since Coconut syntax is a superset of the latest Python 3 syntax, it supports [Python 3 function type annotation syntax](https://www.python.org/dev/peps/pep-0484/) and [Python 3.6 variable type annotation syntax](https://www.python.org/dev/peps/pep-0526/). By default, Coconut compiles all type annotations into Python-2-compatible type comments. If you want to keep the type annotations instead, simply pass a `--target` that supports them. @@ -1992,6 +2028,7 @@ class CanAddAndSub(typing.Protocol, typing.Generic[T, U, V]): raise NotImplementedError ``` + ### Multidimensional Array Literal/Concatenation Syntax Coconut supports multidimensional array literal and array [concatenation](https://numpy.org/doc/stable/reference/generated/numpy.concatenate.html)/[stack](https://numpy.org/doc/stable/reference/generated/numpy.stack.html) syntax. @@ -2067,6 +2104,7 @@ _General showcase of how the different concatenation operators work using `numpy **Python:** _The equivalent Python array literals can be seen in the printed representations in each example._ + ### Lazy Lists Coconut supports the creation of lazy lists, where the contents in the list will be treated as an iterator and not evaluated until they are needed. Unlike normal iterators, however, lazy lists can be iterated over multiple times and still return the same result. Lazy lists can be created in Coconut simply by surrounding a comma-separated list of items with `(|` and `|)` (so-called "banana brackets") instead of `[` and `]` for a list or `(` and `)` for a tuple. @@ -2087,6 +2125,7 @@ Lazy lists, where sequences are only evaluated when their contents are requested **Python:** _Can't be done without a complicated iterator comprehension in place of the lazy list. See the compiled code for the Python syntax._ + ### Implicit Function Application and Coefficients Coconut supports implicit function application of the form `f x y`, which is compiled to `f(x, y)` (note: **not** `f(x)(y)` as is common in many languages with automatic currying). @@ -2144,6 +2183,7 @@ print(p1(5)) quad = 5 * x**2 + 3 * x + 1 ``` + ### Keyword Argument Name Elision When passing in long variable names as keyword arguments of the same name, Coconut supports the syntax @@ -2179,6 +2219,7 @@ main_func( ) ``` + ### Anonymous Namedtuples Coconut supports anonymous [`namedtuple`](https://docs.python.org/3/library/collections.html#collections.namedtuple) literals, such that `(a=1, b=2)` can be used just as `(1, 2)`, but with added names. Anonymous `namedtuple`s are always pickleable. @@ -2217,6 +2258,7 @@ users = [ ] ``` + ### Set Literals Coconut allows an optional `s` to be prepended in front of Python set literals. While in most cases this does nothing, in the case of the empty set it lets Coconut know that it is an empty set and not an empty dictionary. Set literals also support unpacking syntax (e.g. `s{*xs}`). @@ -2235,6 +2277,7 @@ empty_frozen_set = f{} empty_frozen_set = frozenset() ``` + ### Imaginary Literals In addition to Python's `j` or `J` notation for imaginary literals, Coconut also supports `i` or `I`, to make imaginary literals more readable if used in a mathematical context. @@ -2262,6 +2305,7 @@ An imaginary literal yields a complex number with a real part of 0.0. Complex nu print(abs(3 + 4j)) ``` + ### Alternative Ternary Operator Python supports the ternary operator syntax @@ -2298,6 +2342,7 @@ value = ( ) ``` + ## Function Definition ```{contents} @@ -2307,6 +2352,7 @@ depth: 1 --- ``` + ### Tail Call Optimization Coconut will perform automatic [tail call](https://en.wikipedia.org/wiki/Tail_call) optimization and tail recursion elimination on any function that meets the following criteria: @@ -2370,6 +2416,7 @@ print(foo()) # 2 (!) Because this could have unintended and potentially damaging consequences, Coconut opts to not perform TRE on any function with a lambda or inner function. + ### Assignment Functions Coconut allows for assignment function definition that automatically returns the last line of the function body. An assignment function is constructed by substituting `=` for `:` after the function definition line. Thus, the syntax for assignment function definition is either @@ -2404,6 +2451,7 @@ def binexp(x): return 2**x print(binexp(5)) ``` + ### Pattern-Matching Functions Coconut pattern-matching functions are just normal functions, except where the arguments are patterns to be matched against instead of variables to be assigned to. The syntax for pattern-matching function definition is @@ -2441,6 +2489,7 @@ range(5) |> last_two |> print **Python:** _Can't be done without a long series of checks at the top of the function. See the compiled code for the Python syntax._ + ### `addpattern` Functions Coconut provides the `addpattern def` syntax as a shortcut for the full @@ -2466,6 +2515,7 @@ addpattern def factorial(n) = n * factorial(n - 1) **Python:** _Can't be done without a complicated decorator definition and a long series of checks for each pattern-matching. See the compiled code for the Python syntax._ + ### `copyclosure` Functions Coconut supports the syntax @@ -2517,6 +2567,7 @@ def outer_func(): return funcs ``` + ### Explicit Generators Coconut supports the syntax @@ -2542,6 +2593,7 @@ def empty_it(): yield ``` + ### Dotted Function Definition Coconut allows for function definition using a dotted name to assign a function as a method of an object as specified in [PEP 542](https://www.python.org/dev/peps/pep-0542/). Dotted function definition can be combined with all other types of function definition above. @@ -2561,6 +2613,7 @@ def my_method(self): MyClass.my_method = my_method ``` + ## Statements ```{contents} @@ -2570,6 +2623,7 @@ depth: 1 --- ``` + ### Destructuring Assignment Coconut supports significantly enhanced destructuring assignment, similar to Python's tuple/list destructuring, but much more powerful. The syntax for Coconut's destructuring assignment is @@ -2599,6 +2653,7 @@ print(a, b) **Python:** _Can't be done without a long series of checks in place of the destructuring assignment statement. See the compiled code for the Python syntax._ + ### Type Parameter Syntax Coconut fully supports [Python 3.12 PEP 695](https://peps.python.org/pep-0695/) type parameter syntax on all Python versions. @@ -2682,6 +2737,7 @@ def my_ident[T](x: T) -> T = x **Python:** _Can't be done without a complex definition for the data type. See the compiled code for the Python syntax._ + ### Implicit `pass` Coconut supports the simple `class name(base)` and `data name(args)` as aliases for `class name(base): pass` and `data name(args): pass`. @@ -2699,6 +2755,7 @@ data Node(left, right) from Tree **Python:** _Can't be done without a series of method definitions for each data type. See the compiled code for the Python syntax._ + ### Statement Nesting Coconut supports the nesting of compound statements on the same line. This allows the mixing of `match` and `if` statements together, as well as compound `try` statements. @@ -2727,6 +2784,7 @@ else: print(input_list) ``` + ### `except` Statements Python 3 requires that if multiple exceptions are to be caught, they must be placed inside of parentheses, so as to disallow Python 2's use of a comma instead of `as`. Coconut allows commas in except statements to translate to catching multiple exceptions without the need for parentheses, since, as in Python 3, `as` is always required to bind the exception to a name. @@ -2749,6 +2807,7 @@ except (SyntaxError, ValueError) as err: handle(err) ``` + ### In-line `global` And `nonlocal` Assignment Coconut allows for `global` or `nonlocal` to precede assignment to a list of variables or (augmented) assignment to a variable to make that assignment `global` or `nonlocal`, respectively. @@ -2767,6 +2826,7 @@ global state_a, state_b; state_a, state_b = 10, 100 global state_c; state_c += 1 ``` + ### Code Passthrough Coconut supports the ability to pass arbitrary code through the compiler without being touched, for compatibility with other variants of Python, such as [Cython](http://cython.org/) or [Mython](http://mython.org/). When using Coconut to compile to another variant of Python, make sure you [name your source file properly](#naming-source-files) to ensure the resulting compiled code has the right file extension for the intended usage. @@ -2787,6 +2847,7 @@ cdef f(x): return g(x) ``` + ### Enhanced Parenthetical Continuation Since Coconut syntax is a superset of the latest Python 3 syntax, Coconut supports the same line continuation syntax as Python. That means both backslash line continuation and implied line continuation inside of parentheses, brackets, or braces will all work. @@ -2816,6 +2877,7 @@ with open('/path/to/some/file/you/want/to/read') as file_1: file_2.write(file_1.read()) ``` + ### Assignment Expression Chaining Unlike Python, Coconut allows assignment expressions to be chained, as in `a := b := c`. Note, however, that assignment expressions in general are currently only supported on `--target 3.8` or higher. @@ -2832,6 +2894,7 @@ Unlike Python, Coconut allows assignment expressions to be chained, as in `a := (a := (b := 1)) ``` + ## Built-Ins ```{contents} @@ -2841,6 +2904,7 @@ depth: 2 --- ``` + ### Built-In Function Decorators ```{contents} @@ -3096,6 +3160,7 @@ def fib() = (1, 1) :: map((+), fib(), fib()$[1:]) **Python:** _Can't be done without a long decorator definition. The full definition of the decorator in Python can be found in the Coconut header._ + ### Built-In Types ```{contents} @@ -3245,6 +3310,7 @@ Additionally, if you are using [view patterns](#match), you might need to raise In some cases where there are multiple Coconut packages installed at the same time, there may be multiple `MatchError`s defined in different packages. Coconut can perform some magic under the hood to make sure that all these `MatchError`s will seamlessly interoperate, but only if all such packages are compiled in [`--package` mode rather than `--standalone` mode](#compilation-modes). + ### Generic Built-In Functions ```{contents} @@ -3519,6 +3585,7 @@ async def load_and_send_data(): return await send_data(proc_data(await load_data_async())) ``` + ### Built-Ins for Working with Iterators ```{contents} @@ -4167,7 +4234,6 @@ with concurrent.futures.ThreadPoolExecutor() as executor: print(list(executor.map(get_data_for_user, get_all_users()))) ``` - #### `collectby` and `mapreduce` ##### **collectby**(_key\_func_, _iterable_, _value\_func_=`None`, \*, _reduce\_func_=`None`, _collect\_in_=`None`, _map\_using_=`None`) @@ -4233,6 +4299,57 @@ for item in balance_data: user_balances[item.user] += item.balance ``` +#### `async_map` + +**async\_map**(_async\_func_, *_iters_, _strict_=`False`) + +`async_map` maps _async\_func_ over _iters_ asynchronously using [`anyio`](https://anyio.readthedocs.io/en/stable/), which must be installed for _async\_func_ to work. _strict_ functions as in [`map`/`zip`](#enhanced-built-ins), enforcing that all the _iters_ must have the same length. + +Equivalent to: +```coconut +async def async_map[T, U]( + async_func: async T -> U, + *iters: T$[], + strict: bool = False +) -> U[]: + """Map async_func over iters asynchronously using anyio.""" + import anyio + results = [] + async def store_func_in_of(i, args): + got = await async_func(*args) + results.extend([None] * (1 + i - len(results))) + results[i] = got + async with anyio.create_task_group() as nursery: + for i, args in enumerate(zip(*iters, strict=strict)): + nursery.start_soon(store_func_in_of, i, args) + return results +``` + +##### Example + +**Coconut:** +```coconut +async def load_pages(urls) = ( + urls + |> async_map$(load_page) + |> await +) +``` + +**Python:** +```coconut_python +import anyio + +async def load_pages(urls): + results = [None] * len(urls) + async def proc_url(i, url): + results[i] = await load_page(url) + async with anyio.create_task_group() as nursery: + for i, url in enumerate(urls) + nursery.start_soon(proc_url, i, url) + return results +``` + #### `tee` **tee**(_iterable_, _n_=`2`) @@ -4309,6 +4426,7 @@ range(10) |> map$((x) => x**2) |> map$(print) |> consume collections.deque(map(print, map(lambda x: x**2, range(10))), maxlen=0) ``` + ### Typing-Specific Built-Ins ```{contents} @@ -4410,6 +4528,7 @@ from coconut.__coconut__ import fmap reveal_type(fmap) ``` + ## Coconut API ```{contents} @@ -4419,6 +4538,7 @@ depth: 2 --- ``` + ### `coconut.embed` **coconut.embed**(_kernel_=`None`, _depth_=`0`, \*\*_kwargs_) @@ -4427,6 +4547,7 @@ If _kernel_=`False` (default), embeds a Coconut Jupyter console initialized from Recommended usage is as a debugging tool, where the code `from coconut import embed; embed()` can be inserted to launch an interactive Coconut shell initialized from that point. + ### Automatic Compilation Automatic compilation lets you simply import Coconut files directly without having to go through a compilation step first. Automatic compilation can be enabled either by importing [`coconut.api`](#coconut-api) before you import anything else, or by running `coconut --site-install`. @@ -4439,6 +4560,7 @@ Automatic compilation is always available in the Coconut interpreter or when usi If using the Coconut interpreter, a `reload` built-in is always provided to easily reload (and thus recompile) imported modules. + ### Coconut Encoding While automatic compilation is the preferred method for dynamically compiling Coconut files, as it caches the compiled code as a `.py` file to prevent recompilation, Coconut also supports a special @@ -4447,6 +4569,7 @@ While automatic compilation is the preferred method for dynamically compiling Co ``` declaration which can be added to `.py` files to have them treated as Coconut files instead. To use such a coding declaration, you'll need to either run `coconut --site-install` or `import coconut.api` at some point before you first attempt to import a file with a `# coding: coconut` declaration. Like automatic compilation, the Coconut encoding is always available from the Coconut interpreter. Compilation always uses the same parameters as in the [Coconut Jupyter kernel](#kernel). + ### `coconut.api` In addition to enabling automatic compilation, `coconut.api` can also be used to call the Coconut compiler from code instead of from the command line. See below for specifications of the different api functions. @@ -4584,6 +4707,7 @@ Switches the [`breakpoint` built-in](https://www.python.org/dev/peps/pep-0553/) If an error is encountered in a api function, a `CoconutException` instance may be raised. `coconut.api.CoconutException` is provided to allow catching such errors. + ### `coconut.__coconut__` It is sometimes useful to be able to access Coconut built-ins from pure Python. To accomplish this, Coconut provides `coconut.__coconut__`, which behaves exactly like the `__coconut__.py` header file included when Coconut is compiled in package mode. diff --git a/__coconut__/__init__.pyi b/__coconut__/__init__.pyi index 066d10c20..c35050497 100644 --- a/__coconut__/__init__.pyi +++ b/__coconut__/__init__.pyi @@ -1230,6 +1230,22 @@ def reiterable(iterable: _t.Iterable[_T]) -> _t.Iterable[_T]: _coconut_reiterable = reiterable +@_t.overload +def async_map( + async_func: _t.Callable[[_T], _t.Awaitable[_U]], + iter: _t.Iterable[_T], + strict: bool = False, +) -> _t.Awaitable[_t.List[_U]]: ... +@_t.overload +def async_map( + async_func: _t.Callable[..., _t.Awaitable[_U]], + *iters: _t.Iterable, + strict: bool = False, +) -> _t.Awaitable[_t.List[_U]]: + """Map async_func over iters asynchronously using anyio.""" + ... + + def multi_enumerate(iterable: _Iterable) -> _t.Iterable[_t.Tuple[_t.Tuple[int, ...], _t.Any]]: """Enumerate an iterable of iterables. Works like enumerate, but indexes through inner iterables and produces a tuple index representing the index @@ -1694,6 +1710,8 @@ def collectby( """ ... +collectby.using_processes = collectby.using_threads = collectby # type: ignore + @_t.overload def mapreduce( @@ -1729,7 +1747,7 @@ def mapreduce( """ ... -_coconut_mapreduce = mapreduce +_coconut_mapreduce = mapreduce.using_processes = mapreduce.using_threads = mapreduce # type: ignore @_t.overload diff --git a/coconut/compiler/header.py b/coconut/compiler/header.py index 5b1a68686..8dff65b8b 100644 --- a/coconut/compiler/header.py +++ b/coconut/compiler/header.py @@ -233,6 +233,8 @@ def process_header_args(which, use_hash, target, no_tco, strict, no_wrap): comma_slash=", /" if target_info >= (3, 8) else "", report_this_text=report_this_text, from_None=" from None" if target.startswith("3") else "", + process_="process_" if target_info >= (3, 13) else "", + numpy_modules=tuple_str_of(numpy_modules, add_quotes=True), pandas_numpy_modules=tuple_str_of(pandas_numpy_modules, add_quotes=True), jax_numpy_modules=tuple_str_of(jax_numpy_modules, add_quotes=True), @@ -716,10 +718,10 @@ def Return(self, obj): ), class_amap=pycondition( (3, 3), - if_lt=r''' + if_lt=''' _coconut_amap = None ''', - if_ge=r''' + if_ge=''' class _coconut_amap(_coconut_baseclass): __slots__ = ("func", "aiter") def __init__(self, func, aiter): @@ -787,6 +789,46 @@ def __neg__(self): '''.format(**format_dict), indent=1, ), + def_async_map=prepare( + ''' +async def async_map(async_func, *iters, strict=False): + """Map async_func over iters asynchronously using anyio.""" + import anyio + results = [] + async def store_func_in_of(i, args): + got = await async_func(*args) + results.extend([None] * (1 + i - _coconut.len(results))) + results[i] = got + async with anyio.create_task_group() as nursery: + for i, args in _coconut.enumerate({_coconut_}zip(*iters, strict=strict)): + nursery.start_soon(store_func_in_of, i, args) + return results + '''.format(**format_dict) if target_info >= (3, 5) else + pycondition( + (3, 5), + if_ge=''' +_coconut_async_map_ns = {lbrace}"_coconut": _coconut, "zip": zip{rbrace} +_coconut_exec("""async def async_map(async_func, *iters, strict=False): + \'''Map async_func over iters asynchronously using anyio.\''' + import anyio + results = [] + async def store_func_in_of(i, args): + got = await async_func(*args) + results.extend([None] * (1 + i - _coconut.len(results))) + results[i] = got + async with anyio.create_task_group() as nursery: + for i, args in _coconut.enumerate({_coconut_}zip(*iters, strict=strict)): + nursery.start_soon(store_func_in_of, i, args) + return results""", _coconut_async_map_ns) +async_map = _coconut_async_map_ns["async_map"] + '''.format(**format_dict), + if_lt=''' +def async_map(*args, **kwargs): + """async_map not available on Python < 3.5""" + raise _coconut.NameError("async_map not available on Python < 3.5") + ''', + ), + ), ) format_dict.update(extra_format_dict) diff --git a/coconut/compiler/templates/header.py_template b/coconut/compiler/templates/header.py_template index bde083643..1135a87d0 100644 --- a/coconut/compiler/templates/header.py_template +++ b/coconut/compiler/templates/header.py_template @@ -946,7 +946,7 @@ class thread_map(_coconut_base_parallel_map): _threadlocal_ns = _coconut.threading.local() @staticmethod def _make_pool(max_workers=None): - return _coconut.multiprocessing_dummy.Pool(_coconut.multiprocessing.cpu_count() * 5 if max_workers is None else max_workers) + return _coconut.multiprocessing_dummy.Pool(_coconut.multiprocessing.{process_}cpu_count() * 5 if max_workers is None else max_workers) class zip(_coconut_baseclass, _coconut.zip): __slots__ = ("iters", "strict") __doc__ = getattr(_coconut.zip, "__doc__", "") @@ -978,6 +978,7 @@ class zip(_coconut_baseclass, _coconut.zip): {zip_iter} def __fmap__(self, func): return {_coconut_}map(func, self) +{def_async_map} class zip_longest(zip): __slots__ = ("fillvalue",) __doc__ = getattr(_coconut.zip_longest, "__doc__", "Version of zip that fills in missing values with fillvalue.") @@ -1520,22 +1521,21 @@ class multiset(_coconut.collections.Counter{comma_object}): def add(self, item): """Add an element to a multiset.""" self[item] += 1 - def discard(self, item): - """Remove an element from a multiset if it is a member.""" - item_count = self[item] - if item_count > 0: - self[item] = item_count - 1 - if item_count - 1 <= 0: - del self[item] - def remove(self, item): + def remove(self, item, **kwargs): """Remove an element from a multiset; it must be a member.""" + allow_missing = kwargs.pop("allow_missing", False) + if kwargs: + raise _coconut.TypeError("multiset.remove() got unexpected keyword arguments " + _coconut.repr(kwargs)) item_count = self[item] if item_count > 0: self[item] = item_count - 1 if item_count - 1 <= 0: del self[item] - else: + elif not allow_missing: raise _coconut.KeyError(item) + def discard(self, item): + """Remove an element from a multiset if it is a member.""" + return self.remove(item, allow_missing=True) def isdisjoint(self, other): """Return True if two multisets have a null intersection.""" return not self & other diff --git a/coconut/constants.py b/coconut/constants.py index ed699fc91..7605d4561 100644 --- a/coconut/constants.py +++ b/coconut/constants.py @@ -777,6 +777,7 @@ def get_path_env_var(env_var, default): "windowsof", "and_then", "and_then_await", + "async_map", "py_chr", "py_dict", "py_hex", @@ -895,6 +896,13 @@ def get_path_env_var(env_var, default): ("typing_extensions", "py==36"), ("typing_extensions", "py==37"), ("typing_extensions", "py>=38"), + ("trollius", "py<3;cpy"), + ("aenum", "py<34"), + ("dataclasses", "py==36"), + ("typing", "py<35"), + ("async_generator", "py35"), + ("exceptiongroup", "py37;py<311"), + ("anyio", "py36"), ), "cpython": ( "cPyparsing", @@ -924,9 +932,12 @@ def get_path_env_var(env_var, default): ("jupyter-console", "py>=35;py<37"), ("jupyter-console", "py37"), "papermill", - # these are fully optional, so no need to pull them in here - # ("jupyterlab", "py35"), - # ("jupytext", "py3"), + ), + "jupyterlab": ( + ("jupyterlab", "py35"), + ), + "jupytext": ( + ("jupytext", "py3"), ), "mypy": ( "mypy[python2]", @@ -941,14 +952,6 @@ def get_path_env_var(env_var, default): ("xonsh", "py>=36;py<38"), ("xonsh", "py38"), ), - "backports": ( - ("trollius", "py<3;cpy"), - ("aenum", "py<34"), - ("dataclasses", "py==36"), - ("typing", "py<35"), - ("async_generator", "py35"), - ("exceptiongroup", "py37;py<311"), - ), "dev": ( ("pre-commit", "py3"), "requests", @@ -1007,6 +1010,7 @@ def get_path_env_var(env_var, default): ("exceptiongroup", "py37;py<311"): (1,), ("ipython", "py>=39"): (8, 16), "py-spy": (0, 3), + ("anyio", "py36"): (3,), } pinned_min_versions = { diff --git a/coconut/requirements.py b/coconut/requirements.py index c2e9668a0..3035c8440 100644 --- a/coconut/requirements.py +++ b/coconut/requirements.py @@ -220,7 +220,6 @@ def everything_in(req_dict): "kernel": get_reqs("kernel"), "watch": get_reqs("watch"), "mypy": get_reqs("mypy"), - "backports": get_reqs("backports"), "xonsh": get_reqs("xonsh"), "numpy": get_reqs("numpy"), } @@ -230,6 +229,15 @@ def everything_in(req_dict): get_reqs("jupyter"), ) +extras["jupyterlab"] = uniqueify_all( + extras["jupyter"], + get_reqs("jupyterlab"), +) +extras["jupytext"] = uniqueify_all( + extras["jupyter"], + get_reqs("jupytext"), +) + extras["all"] = everything_in(extras) extras.update({ @@ -237,7 +245,6 @@ def everything_in(req_dict): "docs": unique_wrt(get_reqs("docs"), requirements), "tests": uniqueify_all( get_reqs("tests"), - extras["backports"], extras["numpy"], extras["jupyter"] if IPY else [], extras["mypy"] if MYPY else [], diff --git a/coconut/root.py b/coconut/root.py index 26a67c81e..6a58d3015 100644 --- a/coconut/root.py +++ b/coconut/root.py @@ -26,7 +26,7 @@ VERSION = "3.0.3" VERSION_NAME = None # False for release, int >= 1 for develop -DEVELOP = 17 +DEVELOP = 18 ALPHA = False # for pre releases rather than post releases assert DEVELOP is False or DEVELOP >= 1, "DEVELOP must be False or an int >= 1" diff --git a/coconut/tests/src/cocotest/target_36/py36_test.coco b/coconut/tests/src/cocotest/target_36/py36_test.coco index c7645db71..47d4bb4d1 100644 --- a/coconut/tests/src/cocotest/target_36/py36_test.coco +++ b/coconut/tests/src/cocotest/target_36/py36_test.coco @@ -21,6 +21,20 @@ def py36_test() -> bool: |> map$(call) |> await_all |> await + ) == range(5) |> list == ( + outer_func() + |> await + |> async_map$(call) + |> await + ) + assert ( + range(5) + |> map$(./10) + |> reversed + |> async_map$(lift(asyncio.sleep)(ident, result=ident)) + |> await + |> reversed + |> map$(.*10) ) == range(5) |> list loop.run_until_complete(atest()) diff --git a/coconut/tests/src/cocotest/target_sys/target_sys_test.coco b/coconut/tests/src/cocotest/target_sys/target_sys_test.coco index 03320c62d..ce878926e 100644 --- a/coconut/tests/src/cocotest/target_sys/target_sys_test.coco +++ b/coconut/tests/src/cocotest/target_sys/target_sys_test.coco @@ -53,8 +53,8 @@ def asyncio_test() -> bool: async match def async_map_3([func] + iters) = process_map(func, *iters) match async def async_map_4([func] + iters) = process_map(func, *iters) async def async_map_test() = - for async_map in (async_map_0, async_map_1, async_map_2, async_map_3, async_map_4): - assert (await ((pow$(2), range(5)) |> async_map)) |> tuple == (1, 2, 4, 8, 16) + for async_map_ in (async_map_0, async_map_1, async_map_2, async_map_3, async_map_4): + assert (await ((pow$(2), range(5)) |> async_map_)) |> tuple == (1, 2, 4, 8, 16) True async def aplus(x) = y -> x + y From a99a76d09b86c96e7f62bf2f04979fbb4e6ae50a Mon Sep 17 00:00:00 2001 From: Evan Hubinger Date: Sat, 4 Nov 2023 13:59:13 -0700 Subject: [PATCH 053/121] Fix tests --- coconut/icoconut/root.py | 5 ----- coconut/tests/src/cocotest/target_36/py36_test.coco | 1 + 2 files changed, 1 insertion(+), 5 deletions(-) diff --git a/coconut/icoconut/root.py b/coconut/icoconut/root.py index f89935eb9..5d658e28e 100644 --- a/coconut/icoconut/root.py +++ b/coconut/icoconut/root.py @@ -33,8 +33,6 @@ CoconutInternalException, ) from coconut.constants import ( - WINDOWS, - PY38, PY311, py_syntax_version, mimetype, @@ -51,9 +49,6 @@ from coconut.compiler.util import should_indent from coconut.command.util import Runner -if WINDOWS and PY38 and asyncio is not None: # attempt to fix zmq warning - asyncio.set_event_loop_policy(asyncio.WindowsSelectorEventLoopPolicy()) - try: from IPython.core.inputsplitter import IPythonInputSplitter from IPython.core.interactiveshell import InteractiveShellABC diff --git a/coconut/tests/src/cocotest/target_36/py36_test.coco b/coconut/tests/src/cocotest/target_36/py36_test.coco index 47d4bb4d1..255dd1084 100644 --- a/coconut/tests/src/cocotest/target_36/py36_test.coco +++ b/coconut/tests/src/cocotest/target_36/py36_test.coco @@ -35,6 +35,7 @@ def py36_test() -> bool: |> await |> reversed |> map$(.*10) + |> list ) == range(5) |> list loop.run_until_complete(atest()) From c3b027318d48472d77b26f69957aa85151dcbbb2 Mon Sep 17 00:00:00 2001 From: Evan Hubinger Date: Sat, 4 Nov 2023 15:15:08 -0700 Subject: [PATCH 054/121] Improve partials Resolves #797. --- DOCS.md | 8 ++++++- __coconut__/__init__.pyi | 5 +++- coconut/__coconut__.pyi | 2 +- coconut/compiler/compiler.py | 14 +++++------ coconut/compiler/grammar.py | 8 +++---- coconut/compiler/header.py | 4 ++-- coconut/compiler/templates/header.py_template | 24 ++++++++++++------- coconut/root.py | 2 +- .../src/cocotest/agnostic/primary_2.coco | 1 + 9 files changed, 42 insertions(+), 26 deletions(-) diff --git a/DOCS.md b/DOCS.md index bce108fcf..ee1eb8b4b 100644 --- a/DOCS.md +++ b/DOCS.md @@ -628,6 +628,8 @@ def new_f(x, *args, **kwargs): return f(*args, **kwargs) ``` +Unlike `functools.partial`, Coconut's partial application will preserve the `__name__` of the wrapped function. + ##### Rationale Partial application, or currying, is a mainstay of functional programming, and for good reason: it allows the dynamic customization of functions to fit the needs of where they are being used. Partial application allows a new function to be created out of an old function with some of its arguments pre-specified. @@ -4262,7 +4264,11 @@ If _map_using_ is passed, calculates `key_func` and `value_func` by mapping them ##### **mapreduce.using_processes**(_key\_value\_func_, _iterable_, \*, _reduce\_func_=`None`, _collect\_in_=`None`, _ordered_=`False`, _chunksize_=`1`, _max\_workers_=`None`) -These shortcut methods call `collectby`/`mapreduce` with `map_using` set to [`process_map`](#process_map)/[`thread_map`](#thread_map), properly managed using the `.multiple_sequential_calls` method and the `stream=True` argument of [`process_map`](#process_map)/[`thread_map`](#thread_map). `reduce_func` will be called as soon as results arrive, and by default in whatever order they arrive in (to enforce the original order, pass _ordered_=`True`). Note that, for very long iterables, it is highly recommended to pass a value other than the default `1` for _chunksize_. +These shortcut methods call `collectby`/`mapreduce` with `map_using` set to [`process_map`](#process_map)/[`thread_map`](#thread_map), properly managed using the `.multiple_sequential_calls` method and the `stream=True` argument of [`process_map`](#process_map)/[`thread_map`](#thread_map). `reduce_func` will be called as soon as results arrive, and by default in whatever order they arrive in (to enforce the original order, pass _ordered_=`True`). + +To make multiple sequential calls to `collectby.using_threads()`/`mapreduce.using_threads()`, manage them using `thread_map.multiple_sequential_calls()`. Similarly, use `process_map.multiple_sequential_calls()` to manage `.using_processes()`. + +Note that, for very long iterables, it is highly recommended to pass a value other than the default `1` for _chunksize_. As an example, `mapreduce.using_processes` is effectively equivalent to: ```coconut diff --git a/__coconut__/__init__.pyi b/__coconut__/__init__.pyi index c35050497..5ee15b3e2 100644 --- a/__coconut__/__init__.pyi +++ b/__coconut__/__init__.pyi @@ -208,6 +208,7 @@ tee = _coconut.itertools.tee starmap = _coconut.itertools.starmap cartesian_product = _coconut.itertools.product +_coconut_partial = _coconut.functools.partial _coconut_tee = tee _coconut_starmap = starmap _coconut_cartesian_product = cartesian_product @@ -644,7 +645,8 @@ def _coconut_mark_as_match(func: _Tfunc) -> _Tfunc: return func -class _coconut_partial(_t.Generic[_T]): +class _coconut_complex_partial(_t.Generic[_T]): + func: _t.Callable[..., _T] = ... args: _Tuple = ... required_nargs: int = ... keywords: _t.Dict[_t.Text, _t.Any] = ... @@ -658,6 +660,7 @@ class _coconut_partial(_t.Generic[_T]): **kwargs: _t.Any, ) -> None: ... def __call__(self, *args: _t.Any, **kwargs: _t.Any) -> _T: ... + __name__: str | None = ... @_t.overload diff --git a/coconut/__coconut__.pyi b/coconut/__coconut__.pyi index 91be385cb..e56d0e55e 100644 --- a/coconut/__coconut__.pyi +++ b/coconut/__coconut__.pyi @@ -1,2 +1,2 @@ from __coconut__ import * -from __coconut__ import _coconut_tail_call, _coconut_tco, _coconut_call_set_names, _coconut_handle_cls_kwargs, _coconut_handle_cls_stargs, _namedtuple_of, _coconut, _coconut_Expected, _coconut_MatchError, _coconut_SupportsAdd, _coconut_SupportsMinus, _coconut_SupportsMul, _coconut_SupportsPow, _coconut_SupportsTruediv, _coconut_SupportsFloordiv, _coconut_SupportsMod, _coconut_SupportsAnd, _coconut_SupportsXor, _coconut_SupportsOr, _coconut_SupportsLshift, _coconut_SupportsRshift, _coconut_SupportsMatmul, _coconut_SupportsInv, _coconut_Expected, _coconut_MatchError, _coconut_iter_getitem, _coconut_base_compose, _coconut_forward_compose, _coconut_back_compose, _coconut_forward_star_compose, _coconut_back_star_compose, _coconut_forward_dubstar_compose, _coconut_back_dubstar_compose, _coconut_pipe, _coconut_star_pipe, _coconut_dubstar_pipe, _coconut_back_pipe, _coconut_back_star_pipe, _coconut_back_dubstar_pipe, _coconut_none_pipe, _coconut_none_star_pipe, _coconut_none_dubstar_pipe, _coconut_bool_and, _coconut_bool_or, _coconut_none_coalesce, _coconut_minus, _coconut_map, _coconut_partial, _coconut_get_function_match_error, _coconut_base_pattern_func, _coconut_addpattern, _coconut_sentinel, _coconut_assert, _coconut_raise, _coconut_mark_as_match, _coconut_reiterable, _coconut_self_match_types, _coconut_dict_merge, _coconut_exec, _coconut_comma_op, _coconut_multi_dim_arr, _coconut_mk_anon_namedtuple, _coconut_matmul, _coconut_py_str, _coconut_flatten, _coconut_multiset, _coconut_back_none_pipe, _coconut_back_none_star_pipe, _coconut_back_none_dubstar_pipe, _coconut_forward_none_compose, _coconut_back_none_compose, _coconut_forward_none_star_compose, _coconut_back_none_star_compose, _coconut_forward_none_dubstar_compose, _coconut_back_none_dubstar_compose, _coconut_call_or_coefficient, _coconut_in, _coconut_not_in, _coconut_attritemgetter +from __coconut__ import _coconut_tail_call, _coconut_tco, _coconut_call_set_names, _coconut_handle_cls_kwargs, _coconut_handle_cls_stargs, _namedtuple_of, _coconut, _coconut_Expected, _coconut_MatchError, _coconut_SupportsAdd, _coconut_SupportsMinus, _coconut_SupportsMul, _coconut_SupportsPow, _coconut_SupportsTruediv, _coconut_SupportsFloordiv, _coconut_SupportsMod, _coconut_SupportsAnd, _coconut_SupportsXor, _coconut_SupportsOr, _coconut_SupportsLshift, _coconut_SupportsRshift, _coconut_SupportsMatmul, _coconut_SupportsInv, _coconut_Expected, _coconut_MatchError, _coconut_iter_getitem, _coconut_base_compose, _coconut_forward_compose, _coconut_back_compose, _coconut_forward_star_compose, _coconut_back_star_compose, _coconut_forward_dubstar_compose, _coconut_back_dubstar_compose, _coconut_pipe, _coconut_star_pipe, _coconut_dubstar_pipe, _coconut_back_pipe, _coconut_back_star_pipe, _coconut_back_dubstar_pipe, _coconut_none_pipe, _coconut_none_star_pipe, _coconut_none_dubstar_pipe, _coconut_bool_and, _coconut_bool_or, _coconut_none_coalesce, _coconut_minus, _coconut_map, _coconut_partial, _coconut_complex_partial, _coconut_get_function_match_error, _coconut_base_pattern_func, _coconut_addpattern, _coconut_sentinel, _coconut_assert, _coconut_raise, _coconut_mark_as_match, _coconut_reiterable, _coconut_self_match_types, _coconut_dict_merge, _coconut_exec, _coconut_comma_op, _coconut_multi_dim_arr, _coconut_mk_anon_namedtuple, _coconut_matmul, _coconut_py_str, _coconut_flatten, _coconut_multiset, _coconut_back_none_pipe, _coconut_back_none_star_pipe, _coconut_back_none_dubstar_pipe, _coconut_forward_none_compose, _coconut_back_none_compose, _coconut_forward_none_star_compose, _coconut_back_none_star_compose, _coconut_forward_none_dubstar_compose, _coconut_back_none_dubstar_compose, _coconut_call_or_coefficient, _coconut_in, _coconut_not_in, _coconut_attritemgetter diff --git a/coconut/compiler/compiler.py b/coconut/compiler/compiler.py index 8ad08f02f..fee1d3d42 100644 --- a/coconut/compiler/compiler.py +++ b/coconut/compiler/compiler.py @@ -2797,7 +2797,7 @@ def pipe_handle(self, original, loc, tokens, **kwargs): return expr elif name == "partial": self.internal_assert(len(split_item) == 3, original, loc) - return "_coconut.functools.partial(" + join_args(split_item) + ")" + return "_coconut_partial(" + join_args(split_item) + ")" elif name == "attrgetter": return attrgetter_atom_handle(loc, item) elif name == "itemgetter": @@ -2891,14 +2891,14 @@ def item_handle(self, original, loc, tokens): out += trailer elif len(trailer) == 1: if trailer[0] == "$[]": - out = "_coconut.functools.partial(_coconut_iter_getitem, " + out + ")" + out = "_coconut_partial(_coconut_iter_getitem, " + out + ")" elif trailer[0] == "$": - out = "_coconut.functools.partial(_coconut.functools.partial, " + out + ")" + out = "_coconut_partial(_coconut_partial, " + out + ")" elif trailer[0] == "[]": - out = "_coconut.functools.partial(_coconut.operator.getitem, " + out + ")" + out = "_coconut_partial(_coconut.operator.getitem, " + out + ")" elif trailer[0] == ".": self.strict_err_or_warn("'obj.' as a shorthand for 'getattr$(obj)' is deprecated (just use the getattr partial)", original, loc) - out = "_coconut.functools.partial(_coconut.getattr, " + out + ")" + out = "_coconut_partial(_coconut.getattr, " + out + ")" elif trailer[0] == "type:[]": out = "_coconut.typing.Sequence[" + out + "]" elif trailer[0] == "type:$[]": @@ -2931,7 +2931,7 @@ def item_handle(self, original, loc, tokens): args = trailer[1][1:-1] if not args: raise CoconutDeferredSyntaxError("a partial application argument is required", loc) - out = "_coconut.functools.partial(" + out + ", " + args + ")" + out = "_coconut_partial(" + out + ", " + args + ")" elif trailer[0] == "$[": out = "_coconut_iter_getitem(" + out + ", " + trailer[1] + ")" elif trailer[0] == "$(?": @@ -2959,7 +2959,7 @@ def item_handle(self, original, loc, tokens): raise CoconutInternalException("no question mark in question mark partial", trailer[1]) elif argdict_pairs or pos_kwargs or extra_args_str: out = ( - "_coconut_partial(" + "_coconut_complex_partial(" + out + ", {" + ", ".join(argdict_pairs) + "}" + ", " + str(len(pos_args)) diff --git a/coconut/compiler/grammar.py b/coconut/compiler/grammar.py index 3f0220c45..0abed4963 100644 --- a/coconut/compiler/grammar.py +++ b/coconut/compiler/grammar.py @@ -447,7 +447,7 @@ def itemgetter_handle(tokens): if op == "[": return "_coconut.operator.itemgetter((" + args + "))" elif op == "$[": - return "_coconut.functools.partial(_coconut_iter_getitem, index=(" + args + "))" + return "_coconut_partial(_coconut_iter_getitem, index=(" + args + "))" else: raise CoconutInternalException("invalid implicit itemgetter type", op) else: @@ -540,10 +540,10 @@ def partial_op_item_handle(tokens): tok_grp, = tokens if "left partial" in tok_grp: arg, op = tok_grp - return "_coconut.functools.partial(" + op + ", " + arg + ")" + return "_coconut_partial(" + op + ", " + arg + ")" elif "right partial" in tok_grp: op, arg = tok_grp - return "_coconut_partial(" + op + ", {1: " + arg + "}, 2, ())" + return "_coconut_complex_partial(" + op + ", {1: " + arg + "}, 2, ())" else: raise CoconutInternalException("invalid operator function implicit partial token group", tok_grp) @@ -1013,7 +1013,7 @@ class Grammar(object): | fixto(dubquestion, "_coconut_none_coalesce") | fixto(dot, "_coconut.getattr") | fixto(unsafe_dubcolon, "_coconut.itertools.chain") - | fixto(dollar, "_coconut.functools.partial") + | fixto(dollar, "_coconut_partial") | fixto(exp_dubstar, "_coconut.operator.pow") | fixto(mul_star, "_coconut.operator.mul") | fixto(div_dubslash, "_coconut.operator.floordiv") diff --git a/coconut/compiler/header.py b/coconut/compiler/header.py index 8dff65b8b..49f4864c4 100644 --- a/coconut/compiler/header.py +++ b/coconut/compiler/header.py @@ -314,7 +314,7 @@ def pattern_prepender(func): return pattern_prepender def datamaker(data_type): """DEPRECATED: use makedata instead.""" - return _coconut.functools.partial(makedata, data_type) + return _coconut_partial(makedata, data_type) of, parallel_map, concurrent_map, recursive_iterator = call, process_map, thread_map, recursive_generator ''' if not strict else @@ -599,7 +599,7 @@ async def __anext__(self): # (extra_format_dict is to keep indentation levels matching) extra_format_dict = dict( # when anything is added to this list it must also be added to *both* __coconut__ stub files - underscore_imports="{tco_comma}{call_set_names_comma}{handle_cls_args_comma}_namedtuple_of, _coconut, _coconut_Expected, _coconut_MatchError, _coconut_SupportsAdd, _coconut_SupportsMinus, _coconut_SupportsMul, _coconut_SupportsPow, _coconut_SupportsTruediv, _coconut_SupportsFloordiv, _coconut_SupportsMod, _coconut_SupportsAnd, _coconut_SupportsXor, _coconut_SupportsOr, _coconut_SupportsLshift, _coconut_SupportsRshift, _coconut_SupportsMatmul, _coconut_SupportsInv, _coconut_iter_getitem, _coconut_base_compose, _coconut_forward_compose, _coconut_back_compose, _coconut_forward_star_compose, _coconut_back_star_compose, _coconut_forward_dubstar_compose, _coconut_back_dubstar_compose, _coconut_pipe, _coconut_star_pipe, _coconut_dubstar_pipe, _coconut_back_pipe, _coconut_back_star_pipe, _coconut_back_dubstar_pipe, _coconut_none_pipe, _coconut_none_star_pipe, _coconut_none_dubstar_pipe, _coconut_bool_and, _coconut_bool_or, _coconut_none_coalesce, _coconut_minus, _coconut_map, _coconut_partial, _coconut_get_function_match_error, _coconut_base_pattern_func, _coconut_addpattern, _coconut_sentinel, _coconut_assert, _coconut_raise, _coconut_mark_as_match, _coconut_reiterable, _coconut_self_match_types, _coconut_dict_merge, _coconut_exec, _coconut_comma_op, _coconut_multi_dim_arr, _coconut_mk_anon_namedtuple, _coconut_matmul, _coconut_py_str, _coconut_flatten, _coconut_multiset, _coconut_back_none_pipe, _coconut_back_none_star_pipe, _coconut_back_none_dubstar_pipe, _coconut_forward_none_compose, _coconut_back_none_compose, _coconut_forward_none_star_compose, _coconut_back_none_star_compose, _coconut_forward_none_dubstar_compose, _coconut_back_none_dubstar_compose, _coconut_call_or_coefficient, _coconut_in, _coconut_not_in, _coconut_attritemgetter".format(**format_dict), + underscore_imports="{tco_comma}{call_set_names_comma}{handle_cls_args_comma}_namedtuple_of, _coconut, _coconut_Expected, _coconut_MatchError, _coconut_SupportsAdd, _coconut_SupportsMinus, _coconut_SupportsMul, _coconut_SupportsPow, _coconut_SupportsTruediv, _coconut_SupportsFloordiv, _coconut_SupportsMod, _coconut_SupportsAnd, _coconut_SupportsXor, _coconut_SupportsOr, _coconut_SupportsLshift, _coconut_SupportsRshift, _coconut_SupportsMatmul, _coconut_SupportsInv, _coconut_iter_getitem, _coconut_base_compose, _coconut_forward_compose, _coconut_back_compose, _coconut_forward_star_compose, _coconut_back_star_compose, _coconut_forward_dubstar_compose, _coconut_back_dubstar_compose, _coconut_pipe, _coconut_star_pipe, _coconut_dubstar_pipe, _coconut_back_pipe, _coconut_back_star_pipe, _coconut_back_dubstar_pipe, _coconut_none_pipe, _coconut_none_star_pipe, _coconut_none_dubstar_pipe, _coconut_bool_and, _coconut_bool_or, _coconut_none_coalesce, _coconut_minus, _coconut_map, _coconut_partial, _coconut_complex_partial, _coconut_get_function_match_error, _coconut_base_pattern_func, _coconut_addpattern, _coconut_sentinel, _coconut_assert, _coconut_raise, _coconut_mark_as_match, _coconut_reiterable, _coconut_self_match_types, _coconut_dict_merge, _coconut_exec, _coconut_comma_op, _coconut_multi_dim_arr, _coconut_mk_anon_namedtuple, _coconut_matmul, _coconut_py_str, _coconut_flatten, _coconut_multiset, _coconut_back_none_pipe, _coconut_back_none_star_pipe, _coconut_back_none_dubstar_pipe, _coconut_forward_none_compose, _coconut_back_none_compose, _coconut_forward_none_star_compose, _coconut_back_none_star_compose, _coconut_forward_none_dubstar_compose, _coconut_back_none_dubstar_compose, _coconut_call_or_coefficient, _coconut_in, _coconut_not_in, _coconut_attritemgetter".format(**format_dict), import_typing=pycondition( (3, 5), if_ge=''' diff --git a/coconut/compiler/templates/header.py_template b/coconut/compiler/templates/header.py_template index 1135a87d0..721978c81 100644 --- a/coconut/compiler/templates/header.py_template +++ b/coconut/compiler/templates/header.py_template @@ -61,6 +61,11 @@ class _coconut{object}:{COMMENT.EVERYTHING_HERE_MUST_BE_COPIED_TO_STUB_FILE} fmappables = list, tuple, dict, set, frozenset abc.Sequence.register(collections.deque) Ellipsis, NotImplemented, NotImplementedError, Exception, AttributeError, ImportError, IndexError, KeyError, NameError, TypeError, ValueError, StopIteration, RuntimeError, all, any, bool, bytes, callable, classmethod, complex, dict, enumerate, filter, float, frozenset, getattr, hasattr, hash, id, int, isinstance, issubclass, iter, len, list, locals, globals, map, min, max, next, object, property, range, reversed, set, setattr, slice, str, sum, super, tuple, type, vars, zip, repr, print{comma_bytearray} = Ellipsis, NotImplemented, NotImplementedError, Exception, AttributeError, ImportError, IndexError, KeyError, NameError, TypeError, ValueError, StopIteration, RuntimeError, all, any, bool, bytes, callable, classmethod, complex, dict, enumerate, filter, float, frozenset, getattr, hasattr, hash, id, int, isinstance, issubclass, iter, len, list, locals, globals, map, min, max, next, object, property, range, reversed, set, setattr, slice, str, sum, {lstatic}super{rstatic}, tuple, type, vars, zip, {lstatic}repr{rstatic}, {lstatic}print{rstatic}{comma_bytearray} +@_coconut.functools.wraps(_coconut.functools.partial) +def _coconut_partial(_coconut_func, *args, **kwargs): + partial_func = _coconut.functools.partial(_coconut_func, *args, **kwargs) + partial_func.__name__ = _coconut.getattr(_coconut_func, "__name__", None) + return partial_func def _coconut_handle_cls_kwargs(**kwargs): """Some code taken from six under the terms of its MIT license.""" metaclass = kwargs.pop("metaclass", None) @@ -171,7 +176,7 @@ def _coconut_tco(func): if call_func.__self__ is None: call_func = call_func._coconut_tco_func else: - call_func = _coconut.functools.partial(call_func._coconut_tco_func, call_func.__self__) + call_func = _coconut_partial(call_func._coconut_tco_func, call_func.__self__) else: wkref = _coconut_tco_func_dict.get(_coconut.id(call_func)) wkref_func = None if wkref is None else wkref() @@ -731,7 +736,7 @@ class flatten(_coconut_has_iter):{COMMENT.cant_implement_len_else_list_calls_bec raise ValueError("%r not in %r" % (elem, self)) def __fmap__(self, func): if self.levels == 1: - return self.__class__({_coconut_}map(_coconut.functools.partial({_coconut_}map, func), self.get_new_iter())) + return self.__class__({_coconut_}map(_coconut_partial({_coconut_}map, func), self.get_new_iter())) return {_coconut_}map(func, self) class cartesian_product(_coconut_baseclass): __slots__ = ("iters", "repeat") @@ -1426,10 +1431,10 @@ def addpattern(base_func, *add_funcs, **kwargs): raise _coconut.TypeError("addpattern() got unexpected keyword arguments " + _coconut.repr(kwargs)) if add_funcs: return _coconut_base_pattern_func(base_func, *add_funcs) - return _coconut.functools.partial(_coconut_base_pattern_func, base_func) + return _coconut_partial(_coconut_base_pattern_func, base_func) _coconut_addpattern = addpattern -class _coconut_partial(_coconut_base_callable): - __slots__ = ("func", "_argdict", "_arglen", "_pos_kwargs", "_stargs", "keywords") +class _coconut_complex_partial(_coconut_base_callable): + __slots__ = ("func", "_argdict", "_arglen", "_pos_kwargs", "_stargs", "keywords", "__name__") def __init__(self, _coconut_func, _coconut_argdict, _coconut_arglen, _coconut_pos_kwargs, *args, **kwargs): self.func = _coconut_func self._argdict = _coconut_argdict @@ -1437,6 +1442,7 @@ class _coconut_partial(_coconut_base_callable): self._pos_kwargs = _coconut_pos_kwargs self._stargs = args self.keywords = kwargs + self.__name__ = _coconut.getattr(_coconut_func, "__name__", None) def __reduce__(self): return (self.__class__, (self.func, self._argdict, self._arglen, self._pos_kwargs) + self._stargs, {lbrace}"keywords": self.keywords{rbrace}) @property @@ -1954,8 +1960,8 @@ def _coconut_parallel_mapreduce(mapreduce_func, map_cls, *args, **kwargs): kwargs["map_using"] = _coconut.functools.partial(map_cls, stream=True, ordered=kwargs.pop("ordered", False), chunksize=kwargs.pop("chunksize", 1)) with map_cls.multiple_sequential_calls(max_workers=kwargs.pop("max_workers", None)): return mapreduce_func(*args, **kwargs) -mapreduce.using_processes = _coconut.functools.partial(_coconut_parallel_mapreduce, mapreduce, process_map) -mapreduce.using_threads = _coconut.functools.partial(_coconut_parallel_mapreduce, mapreduce, thread_map) +mapreduce.using_processes = _coconut_partial(_coconut_parallel_mapreduce, mapreduce, process_map) +mapreduce.using_threads = _coconut_partial(_coconut_parallel_mapreduce, mapreduce, thread_map) def collectby(key_func, iterable, value_func=None, **kwargs): """Collect the items in iterable into a dictionary of lists keyed by key_func(item). @@ -1968,8 +1974,8 @@ def collectby(key_func, iterable, value_func=None, **kwargs): the iterable using map_using as map. Useful with process_map/thread_map. """ return {_coconut_}mapreduce(_coconut_lifted(_coconut_comma_op, key_func, {_coconut_}ident if value_func is None else value_func), iterable, **kwargs) -collectby.using_processes = _coconut.functools.partial(_coconut_parallel_mapreduce, collectby, process_map) -collectby.using_threads = _coconut.functools.partial(_coconut_parallel_mapreduce, collectby, thread_map) +collectby.using_processes = _coconut_partial(_coconut_parallel_mapreduce, collectby, process_map) +collectby.using_threads = _coconut_partial(_coconut_parallel_mapreduce, collectby, thread_map) def _namedtuple_of(**kwargs): """Construct an anonymous namedtuple of the given keyword arguments.""" {namedtuple_of_implementation} diff --git a/coconut/root.py b/coconut/root.py index 6a58d3015..1e9792141 100644 --- a/coconut/root.py +++ b/coconut/root.py @@ -26,7 +26,7 @@ VERSION = "3.0.3" VERSION_NAME = None # False for release, int >= 1 for develop -DEVELOP = 18 +DEVELOP = 19 ALPHA = False # for pre releases rather than post releases assert DEVELOP is False or DEVELOP >= 1, "DEVELOP must be False or an int >= 1" diff --git a/coconut/tests/src/cocotest/agnostic/primary_2.coco b/coconut/tests/src/cocotest/agnostic/primary_2.coco index e49eae5c6..e44a94e8a 100644 --- a/coconut/tests/src/cocotest/agnostic/primary_2.coco +++ b/coconut/tests/src/cocotest/agnostic/primary_2.coco @@ -408,4 +408,5 @@ def primary_test_2() -> bool: ): assert some_data |> mapreducer == {"a": ["123"], "b": ["567"]} assert_raises(-> collectby(.[0], [(0, 1), (0, 2)], reduce_func=False), ValueError) # type: ignore + assert ident$(x=?).__name__ == "ident" == ident$(1).__name__ # type: ignore return True From fd5a17f312ea8a384347b72f10488d5588906ff6 Mon Sep 17 00:00:00 2001 From: Evan Hubinger Date: Sat, 4 Nov 2023 15:28:10 -0700 Subject: [PATCH 055/121] Reduce jobs --- coconut/tests/main_test.py | 5 +++++ coconut/tests/src/cocotest/agnostic/suite.coco | 1 + coconut/tests/src/cocotest/agnostic/util.coco | 7 +++++++ 3 files changed, 13 insertions(+) diff --git a/coconut/tests/main_test.py b/coconut/tests/main_test.py index c4cc4108f..e60783c52 100644 --- a/coconut/tests/main_test.py +++ b/coconut/tests/main_test.py @@ -92,6 +92,9 @@ default_recursion_limit = "6144" default_stack_size = "6144" +# fix EOM on GitHub actions +default_jobs = None if PY36 and not PYPY else "4" + jupyter_timeout = 120 base = os.path.dirname(os.path.relpath(__file__)) @@ -375,6 +378,8 @@ def call_coconut(args, **kwargs): args = ["--recursion-limit", default_recursion_limit] + args if default_stack_size is not None and "--stack-size" not in args: args = ["--stack-size", default_stack_size] + args + if default_jobs is not None and "--jobs" not in args: + args = ["--jobs", default_jobs] + args if "--mypy" in args and "check_mypy" not in kwargs: kwargs["check_mypy"] = True if PY26: diff --git a/coconut/tests/src/cocotest/agnostic/suite.coco b/coconut/tests/src/cocotest/agnostic/suite.coco index 1ac462e6d..61691cae5 100644 --- a/coconut/tests/src/cocotest/agnostic/suite.coco +++ b/coconut/tests/src/cocotest/agnostic/suite.coco @@ -1067,6 +1067,7 @@ forward 2""") == 900 assert haslocobj == 2 assert safe_raise_exc(IOError).error `isinstance` IOError assert safe_raise_exc(IOError).handle(IOError, const 10).unwrap() == 10 == safe_raise_exc(IOError).expect_error(IOError).result_or(10) + assert pickle_round_trip(ident$(1))() == 1 == pickle_round_trip(ident$(x=?))(1) # must come at end assert fibs_calls[0] == 1 diff --git a/coconut/tests/src/cocotest/agnostic/util.coco b/coconut/tests/src/cocotest/agnostic/util.coco index 0cb370c59..b6fd84fc1 100644 --- a/coconut/tests/src/cocotest/agnostic/util.coco +++ b/coconut/tests/src/cocotest/agnostic/util.coco @@ -1,6 +1,7 @@ # Imports: import sys import random +import pickle import operator # NOQA from contextlib import contextmanager from functools import wraps @@ -45,6 +46,12 @@ except NameError, TypeError: def x `typed_eq` y = (type(x), x) == (type(y), y) +def pickle_round_trip(obj) = ( + obj + |> pickle.dumps + |> pickle.loads +) + # Old functions: old_fmap = fmap$(starmap_over_mappings=True) From e093c9bee9fcf3aee6c9c464a7e96a7a7c05340f Mon Sep 17 00:00:00 2001 From: Evan Hubinger Date: Sat, 4 Nov 2023 16:39:35 -0700 Subject: [PATCH 056/121] Improve tests perf --- .gitignore | 2 ++ Makefile | 11 ++++++-- __coconut__/__init__.pyi | 3 ++- coconut/command/command.py | 9 +------ coconut/command/util.py | 15 +++++++++-- coconut/compiler/header.py | 1 + coconut/constants.py | 3 +-- coconut/tests/main_test.py | 9 ++++--- .../src/cocotest/agnostic/primary_1.coco | 26 ++++++++++--------- .../src/cocotest/agnostic/primary_2.coco | 15 ++++++----- .../tests/src/cocotest/agnostic/suite.coco | 8 +++--- .../cocotest/target_sys/target_sys_test.coco | 10 +++---- 12 files changed, 68 insertions(+), 44 deletions(-) diff --git a/.gitignore b/.gitignore index 96d716fb7..ed62891af 100644 --- a/.gitignore +++ b/.gitignore @@ -145,3 +145,5 @@ __coconut_cache__/ vprof.json profile.svg profile.speedscope +runtime_profile.svg +runtime_profile.speedscope diff --git a/Makefile b/Makefile index 35da6ac15..24481ed4d 100644 --- a/Makefile +++ b/Makefile @@ -346,20 +346,27 @@ open-speedscope: pyspy-purepy: export COCONUT_PURE_PYTHON=TRUE pyspy-purepy: py-spy record -o profile.speedscope --format speedscope --subprocesses -- python -m coconut ./coconut/tests/src/cocotest/agnostic ./coconut/tests/dest/cocotest --force - open-speedscope + make open-speedscope .PHONY: pyspy-native pyspy-native: py-spy record -o profile.speedscope --format speedscope --native -- python -m coconut ./coconut/tests/src/cocotest/agnostic ./coconut/tests/dest/cocotest --force --jobs 0 - open-speedscope + make open-speedscope + +.PHONY: pyspy-runtime +pyspy-runtime: + py-spy record -o runtime_profile.speedscope --format speedscope --subprocesses -- python ./coconut/tests/dest/runner.py + speedscope ./runtime_profile.speedscope .PHONY: vprof-time vprof-time: vprof -c h "./coconut ./coconut/tests/src/cocotest/agnostic ./coconut/tests/dest/cocotest --force --jobs 0 --stack-size 4096 --recursion-limit 4096" --output-file ./vprof.json + make view-vprof .PHONY: vprof-memory vprof-memory: vprof -c m "./coconut ./coconut/tests/src/cocotest/agnostic ./coconut/tests/dest/cocotest --force --jobs 0 --stack-size 4096 --recursion-limit 4096" --output-file ./vprof.json + make view-vprof .PHONY: view-vprof view-vprof: diff --git a/__coconut__/__init__.pyi b/__coconut__/__init__.pyi index 5ee15b3e2..edd630917 100644 --- a/__coconut__/__init__.pyi +++ b/__coconut__/__init__.pyi @@ -1750,7 +1750,8 @@ def mapreduce( """ ... -_coconut_mapreduce = mapreduce.using_processes = mapreduce.using_threads = mapreduce # type: ignore +mapreduce.using_processes = mapreduce.using_threads = mapreduce # type: ignore +_coconut_mapreduce = mapreduce @_t.overload diff --git a/coconut/command/command.py b/coconut/command/command.py index 9548f1ed3..9848c7fc1 100644 --- a/coconut/command/command.py +++ b/coconut/command/command.py @@ -134,14 +134,7 @@ class Command(object): stack_size = 0 # corresponds to --stack-size flag incremental = False # corresponds to --incremental flag - _prompt = None - - @property - def prompt(self): - """Delay creation of a Prompt() until it's needed.""" - if self._prompt is None: - self._prompt = Prompt() - return self._prompt + prompt = Prompt() def start(self, run=False): """Endpoint for coconut and coconut-run.""" diff --git a/coconut/command/util.py b/coconut/command/util.py index 57d2872d6..2f57f7fd3 100644 --- a/coconut/command/util.py +++ b/coconut/command/util.py @@ -495,14 +495,24 @@ class Prompt(object): session = None style = None runner = None + lexer = None + suggester = None if prompt_use_suggester else False - def __init__(self, use_suggester=prompt_use_suggester): + def __init__(self, setup_now=False): """Set up the prompt.""" if prompt_toolkit is not None: self.set_style(os.getenv(style_env_var, default_style)) self.set_history_file(prompt_histfile) + if setup_now: + self.setup() + + def setup(self): + """Actually initialize the underlying Prompt. + We do this lazily since it's expensive.""" + if self.lexer is None: self.lexer = PygmentsLexer(CoconutLexer) - self.suggester = AutoSuggestFromHistory() if use_suggester else None + if self.suggester is None: + self.suggester = AutoSuggestFromHistory() def set_style(self, style): """Set pygments syntax highlighting style.""" @@ -555,6 +565,7 @@ def input(self, more=False): def prompt(self, msg): """Get input using prompt_toolkit.""" + self.setup() try: # prompt_toolkit v2 if self.session is None: diff --git a/coconut/compiler/header.py b/coconut/compiler/header.py index 49f4864c4..076068b06 100644 --- a/coconut/compiler/header.py +++ b/coconut/compiler/header.py @@ -840,6 +840,7 @@ def async_map(*args, **kwargs): # ----------------------------------------------------------------------------------------------------------------------- +@memoize() def getheader(which, use_hash, target, no_tco, strict, no_wrap): """Generate the specified header. diff --git a/coconut/constants.py b/coconut/constants.py index 7605d4561..66717a961 100644 --- a/coconut/constants.py +++ b/coconut/constants.py @@ -82,10 +82,9 @@ def get_path_env_var(env_var, default): PY310 = sys.version_info >= (3, 10) PY311 = sys.version_info >= (3, 11) IPY = ( - ((PY2 and not PY26) or PY35) + PY35 and (PY37 or not PYPY) and not (PYPY and WINDOWS) - and not (PY2 and WINDOWS) and sys.version_info[:2] != (3, 7) ) MYPY = ( diff --git a/coconut/tests/main_test.py b/coconut/tests/main_test.py index e60783c52..0a5c94ca7 100644 --- a/coconut/tests/main_test.py +++ b/coconut/tests/main_test.py @@ -91,9 +91,12 @@ default_recursion_limit = "6144" default_stack_size = "6144" - -# fix EOM on GitHub actions -default_jobs = None if PY36 and not PYPY else "4" +default_jobs = ( + # fix EOMs on GitHub actions + "2" if PYPY + else "4" if not PY36 + else None +) jupyter_timeout = 120 diff --git a/coconut/tests/src/cocotest/agnostic/primary_1.coco b/coconut/tests/src/cocotest/agnostic/primary_1.coco index 42a056e1b..7418c4e89 100644 --- a/coconut/tests/src/cocotest/agnostic/primary_1.coco +++ b/coconut/tests/src/cocotest/agnostic/primary_1.coco @@ -211,13 +211,6 @@ def primary_test_1() -> bool: assert map((-), range(5)).iters[0] |> tuple == range(5) |> tuple == zip(range(5), range(6)).iters[0] |> tuple # type: ignore assert repr(zip((0,1), (1,2))) == "zip((0, 1), (1, 2))" assert repr(map((-), range(5))).startswith("map(") # type: ignore - assert repr(process_map((-), range(5))).startswith("process_map(") # type: ignore - assert process_map((-), range(5)) |> tuple == (0, -1, -2, -3, -4) == process_map(map$((-)), (range(5),))$[0] |> tuple # type: ignore - assert process_map(zip, (range(2),), (range(2),)) |> map$(tuple) |> tuple == (((0,0), (1,1)),) # type: ignore - assert process_map(zip_longest$(fillvalue=10), (range(1),), (range(2),)) |> map$(tuple) |> tuple == (((0,0), (10,1)),) # type: ignore - assert (range(0, 5), range(5, 10)) |*> map$(+) |> tuple == (5, 7, 9, 11, 13) - assert process_map((*)$(2)..(+)$(1), range(5)) |> tuple == (2, 4, 6, 8, 10) - assert process_map((+), range(5), range(5), chunksize=2) |> list == map((*)$(2), range(5)) |> list == thread_map((+), range(5), range(5), chunksize=2) |> list # type: ignore assert repr(thread_map((-), range(5))).startswith("thread_map(") # type: ignore with thread_map.multiple_sequential_calls(max_workers=4): # type: ignore assert thread_map((-), range(5), stream=True) |> tuple == (0, -1, -2, -3, -4) == thread_map(map$((-)), (range(5),))$[0] |> tuple # type: ignore @@ -319,7 +312,6 @@ def primary_test_1() -> bool: assert pow$(?, 2)(3) == 9 assert [] |> reduce$((+), ?, ()) == () assert pow$(?, 2) |> repr == "$(?, 2)" - assert process_map(pow$(?, 2), range(10)) |> tuple == (0, 1, 4, 9, 16, 25, 36, 49, 64, 81) assert pow$(?, 2).args == (None, 2) assert range(20) |> filter$((x) -> x < 5) |> reversed |> tuple == (4,3,2,1,0) == (0,1,2,3,4,5,6,7,8,9) |> filter$((x) -> x < 5) |> reversed |> tuple # type: ignore assert (range(10) |> map$((x) -> x) |> reversed) `isinstance` map # type: ignore @@ -1149,10 +1141,6 @@ def primary_test_1() -> bool: def __call__(self) = super().__call__() HasSuper assert HasSuper5.HasHasSuper.HasSuper()() == 10 == HasSuper6().get_HasSuper()()() - assert process_map((.+(10,)), [ - (a=1, b=2), - (x=3, y=4), - ]) |> list == [(1, 2, 10), (3, 4, 10)] assert f"{'a' + 'b'}" == "ab" int_str_tup: (int; str) = (1, "a") key = "abc" @@ -1303,4 +1291,18 @@ def primary_test_1() -> bool: assert err is some_err assert Expected(error=TypeError()).map_error(const some_err) == Expected(error=some_err) assert Expected(10).map_error(const some_err) == Expected(10) + assert repr(process_map((-), range(5))).startswith("process_map(") # type: ignore + + with process_map.multiple_sequential_calls(): # type: ignore + assert process_map((-), range(5)) |> tuple == (0, -1, -2, -3, -4) == process_map(map$((-)), (range(5),))$[0] |> tuple # type: ignore + assert process_map(zip, (range(2),), (range(2),)) |> map$(tuple) |> tuple == (((0,0), (1,1)),) # type: ignore + assert process_map(zip_longest$(fillvalue=10), (range(1),), (range(2),)) |> map$(tuple) |> tuple == (((0,0), (10,1)),) # type: ignore + assert (range(0, 5), range(5, 10)) |*> map$(+) |> tuple == (5, 7, 9, 11, 13) + assert process_map((*)$(2)..(+)$(1), range(5)) |> tuple == (2, 4, 6, 8, 10) + assert process_map((+), range(5), range(5), chunksize=2) |> list == map((*)$(2), range(5)) |> list == thread_map((+), range(5), range(5), chunksize=2) |> list # type: ignore + assert process_map(pow$(?, 2), range(10)) |> tuple == (0, 1, 4, 9, 16, 25, 36, 49, 64, 81) + assert process_map((.+(10,)), [ + (a=1, b=2), + (x=3, y=4), + ]) |> list == [(1, 2, 10), (3, 4, 10)] return True diff --git a/coconut/tests/src/cocotest/agnostic/primary_2.coco b/coconut/tests/src/cocotest/agnostic/primary_2.coco index e44a94e8a..3d4b31417 100644 --- a/coconut/tests/src/cocotest/agnostic/primary_2.coco +++ b/coconut/tests/src/cocotest/agnostic/primary_2.coco @@ -121,18 +121,12 @@ def primary_test_2() -> bool: assert flatten([[[1,2]], [[3], [4]]], 2) |> list == [1, 2, 3, 4] assert flatten([[[1,2]], [[3], [4]]], 2) |> reversed |> list == [4, 3, 2, 1] assert_raises(-> map((+), range(3), range(4), strict=True) |> list, ValueError) # type: ignore - assert map((+), range(3), range(4)$[:-1], strict=True) |> list == [0, 2, 4] == process_map((+), range(3), range(4)$[:-1], strict=True) |> list # type: ignore - assert range(3) |> map$((.+1), strict=True) |> list == [1, 2, 3] == range(3) |> process_map$((.+1), strict=True) |> list # type: ignore assert cartesian_product((1, 2), (3, 4), repeat=0) |> list == [()] assert (a=1, b=2)[1] == 2 obj = object() assert_raises((def -> obj.abc = 123), AttributeError) # type: ignore hardref = map((.+1), [1,2,3]) assert weakref.ref(hardref)() |> list == [2, 3, 4] # type: ignore - my_match_err = MatchError("my match error", 123) - assert process_map(ident, [my_match_err]) |> list |> str == str([my_match_err]) # type: ignore - # repeat the same thing again now that my_match_err.str has been called - assert process_map(ident, [my_match_err]) |> list |> str == str([my_match_err]) # type: ignore match data tuple(1, 2) in (1, 2, 3): assert False data TestDefaultMatching(x="x default", y="y default") @@ -409,4 +403,13 @@ def primary_test_2() -> bool: assert some_data |> mapreducer == {"a": ["123"], "b": ["567"]} assert_raises(-> collectby(.[0], [(0, 1), (0, 2)], reduce_func=False), ValueError) # type: ignore assert ident$(x=?).__name__ == "ident" == ident$(1).__name__ # type: ignore + + with process_map.multiple_sequential_calls(): # type: ignore + assert map((+), range(3), range(4)$[:-1], strict=True) |> list == [0, 2, 4] == process_map((+), range(3), range(4)$[:-1], strict=True) |> list # type: ignore + assert range(3) |> map$((.+1), strict=True) |> list == [1, 2, 3] == range(3) |> process_map$((.+1), strict=True) |> list # type: ignore + my_match_err = MatchError("my match error", 123) + assert process_map(ident, [my_match_err]) |> list |> str == str([my_match_err]) # type: ignore + # repeat the same thing again now that my_match_err.str has been called + assert process_map(ident, [my_match_err]) |> list |> str == str([my_match_err]) # type: ignore + return True diff --git a/coconut/tests/src/cocotest/agnostic/suite.coco b/coconut/tests/src/cocotest/agnostic/suite.coco index 61691cae5..91c12d3b4 100644 --- a/coconut/tests/src/cocotest/agnostic/suite.coco +++ b/coconut/tests/src/cocotest/agnostic/suite.coco @@ -67,7 +67,6 @@ def suite_test() -> bool: to_sort = rand_list(10) assert to_sort |> qsort |> tuple == to_sort |> sorted |> tuple, qsort # type: ignore to_sort = rand_list(10) - assert process_map(tuple <.. (|>)$(to_sort), qsorts) |> list == [to_sort |> sorted |> tuple] * len(qsorts) assert repeat(3)$[2] == 3 == repeat_(3)$[2] assert sum_(repeat(1)$[:5]) == 5 == sum_(repeat_(1)$[:5]) assert (sum_(takewhile((x)-> x<5, N())) @@ -279,7 +278,6 @@ def suite_test() -> bool: assert fibs()$[:10] |> list == [1,1,2,3,5,8,13,21,34,55] == fibs_()$[:10] |> list assert fibs() |> takewhile$((i) -> i < 4000000 ) |> filter$((i) -> i % 2 == 0 ) |> sum == 4613732 == fibs_() |> takewhile$((i) -> i < 4000000 ) |> filter$((i) -> i % 2 == 0 ) |> sum # type: ignore assert loop([1,2])$[:4] |> list == [1, 2] * 2 - assert process_map(list .. .$[:2] .. loop, ([1], [2]))$[:2] |> tuple == ([1, 1], [2, 2]) assert nest("a") |> .$[1] |> .$[1] |> .$[0] == "a" assert (def -> mod)()(5, 3) == 2 assert sieve((2, 3, 4, 5)) |> list == [2, 3, 5] @@ -748,7 +746,6 @@ def suite_test() -> bool: class inh_A() `isinstance` clsA `isinstance` object = inh_A() for maxdiff in (maxdiff1, maxdiff2, maxdiff3, maxdiff_): assert maxdiff([7,1,4,5]) == 4, "failed for " + repr(maxdiff) - assert all(r == 4 for r in process_map(call$(?, [7,1,4,5]), [maxdiff1, maxdiff2, maxdiff3])) assert ret_ret_func(1) == ((), {"func": 1}) == ret_args_kwargs$(func=1)() # type: ignore assert ret_args_kwargs$(?, func=2)(1) == ((1,), {"func": 2}) assert lift(ret_args_kwargs)(ident, plus1, times2, sq=square)(3) == ((3, 4, 6), {"sq": 9}) @@ -1069,6 +1066,11 @@ forward 2""") == 900 assert safe_raise_exc(IOError).handle(IOError, const 10).unwrap() == 10 == safe_raise_exc(IOError).expect_error(IOError).result_or(10) assert pickle_round_trip(ident$(1))() == 1 == pickle_round_trip(ident$(x=?))(1) + with process_map.multiple_sequential_calls(): # type: ignore + assert process_map(tuple <.. (|>)$(to_sort), qsorts) |> list == [to_sort |> sorted |> tuple] * len(qsorts) + assert process_map(list .. .$[:2] .. loop, ([1], [2]))$[:2] |> tuple == ([1, 1], [2, 2]) + assert all(r == 4 for r in process_map(call$(?, [7,1,4,5]), [maxdiff1, maxdiff2, maxdiff3])) + # must come at end assert fibs_calls[0] == 1 assert lols[0] == 5 diff --git a/coconut/tests/src/cocotest/target_sys/target_sys_test.coco b/coconut/tests/src/cocotest/target_sys/target_sys_test.coco index ce878926e..c65bc4125 100644 --- a/coconut/tests/src/cocotest/target_sys/target_sys_test.coco +++ b/coconut/tests/src/cocotest/target_sys/target_sys_test.coco @@ -47,11 +47,11 @@ def asyncio_test() -> bool: def toa(f) = async def (*args, **kwargs) -> f(*args, **kwargs) async def async_map_0(args): - return process_map(args[0], *args[1:]) - async def async_map_1(args) = process_map(args[0], *args[1:]) - async def async_map_2([func] + iters) = process_map(func, *iters) - async match def async_map_3([func] + iters) = process_map(func, *iters) - match async def async_map_4([func] + iters) = process_map(func, *iters) + return thread_map(args[0], *args[1:]) + async def async_map_1(args) = thread_map(args[0], *args[1:]) + async def async_map_2([func] + iters) = thread_map(func, *iters) + async match def async_map_3([func] + iters) = thread_map(func, *iters) + match async def async_map_4([func] + iters) = thread_map(func, *iters) async def async_map_test() = for async_map_ in (async_map_0, async_map_1, async_map_2, async_map_3, async_map_4): assert (await ((pow$(2), range(5)) |> async_map_)) |> tuple == (1, 2, 4, 8, 16) From f5eb7fd7de0d4829aa1f2e5ec277e9a6303c883d Mon Sep 17 00:00:00 2001 From: Evan Hubinger Date: Sat, 4 Nov 2023 17:08:46 -0700 Subject: [PATCH 057/121] Disable jobs on pypy --- coconut/tests/main_test.py | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/coconut/tests/main_test.py b/coconut/tests/main_test.py index 0a5c94ca7..6c5b44701 100644 --- a/coconut/tests/main_test.py +++ b/coconut/tests/main_test.py @@ -93,8 +93,7 @@ default_stack_size = "6144" default_jobs = ( # fix EOMs on GitHub actions - "2" if PYPY - else "4" if not PY36 + "0" if PYPY else None ) From 827c06c64293493a3903673227da9ab3773aafa3 Mon Sep 17 00:00:00 2001 From: Evan Hubinger Date: Sat, 4 Nov 2023 22:11:37 -0700 Subject: [PATCH 058/121] Improve header --- .github/workflows/run-tests.yml | 2 +- DOCS.md | 169 +++++++++--------- __coconut__/__init__.pyi | 33 +++- _coconut/__init__.pyi | 3 + coconut/compiler/header.py | 156 +++++++++------- coconut/compiler/templates/header.py_template | 18 +- .../src/cocotest/agnostic/primary_2.coco | 1 + .../tests/src/cocotest/agnostic/suite.coco | 1 + coconut/tests/src/cocotest/agnostic/util.coco | 4 +- .../src/cocotest/target_36/py36_test.coco | 37 ++++ coconut/util.py | 11 +- 11 files changed, 269 insertions(+), 166 deletions(-) diff --git a/.github/workflows/run-tests.yml b/.github/workflows/run-tests.yml index 900d71c89..3065514a1 100644 --- a/.github/workflows/run-tests.yml +++ b/.github/workflows/run-tests.yml @@ -7,7 +7,6 @@ jobs: matrix: python-version: - '2.7' - - '3.4' - '3.5' - '3.6' - '3.7' @@ -21,6 +20,7 @@ jobs: - 'pypy-3.7' - 'pypy-3.8' - 'pypy-3.9' + - 'pypy-3.10' fail-fast: false name: Python ${{ matrix.python-version }} steps: diff --git a/DOCS.md b/DOCS.md index ee1eb8b4b..a745d5f50 100644 --- a/DOCS.md +++ b/DOCS.md @@ -4166,6 +4166,85 @@ all_equal([1, 1, 1]) all_equal([1, 1, 2]) ``` +#### `tee` + +**tee**(_iterable_, _n_=`2`) + +Coconut provides an optimized version of `itertools.tee` as a built-in under the name `tee`. + +Though `tee` is not deprecated, [`reiterable`](#reiterable) is generally recommended over `tee`. + +Custom `tee`/`reiterable` implementations for custom [Containers/Collections](https://docs.python.org/3/library/collections.abc.html) should be put in the `__copy__` method. Note that all [Sequences/Mappings/Sets](https://docs.python.org/3/library/collections.abc.html) are always assumed to be reiterable even without calling `__copy__`. + +##### Python Docs + +**tee**(_iterable, n=2_) + +Return _n_ independent iterators from a single iterable. Equivalent to: +```coconut_python +def tee(iterable, n=2): + it = iter(iterable) + deques = [collections.deque() for i in range(n)] + def gen(mydeque): + while True: + if not mydeque: # when the local deque is empty + newval = next(it) # fetch a new value and + for d in deques: # load it to all the deques + d.append(newval) + yield mydeque.popleft() + return tuple(gen(d) for d in deques) +``` +Once `tee()` has made a split, the original _iterable_ should not be used anywhere else; otherwise, the _iterable_ could get advanced without the tee objects being informed. + +This itertool may require significant auxiliary storage (depending on how much temporary data needs to be stored). In general, if one iterator uses most or all of the data before another iterator starts, it is faster to use `list()` instead of `tee()`. + +##### Example + +**Coconut:** +```coconut +original, temp = tee(original) +sliced = temp$[5:] +``` + +**Python:** +```coconut_python +import itertools +original, temp = itertools.tee(original) +sliced = itertools.islice(temp, 5, None) +``` + +#### `consume` + +**consume**(_iterable_, _keep\_last_=`0`) + +Coconut provides the `consume` function to efficiently exhaust an iterator and thus perform any lazy evaluation contained within it. `consume` takes one optional argument, `keep_last`, that defaults to 0 and specifies how many, if any, items from the end to return as a sequence (`None` will keep all elements). + +Equivalent to: +```coconut +def consume(iterable, keep_last=0): + """Fully exhaust iterable and return the last keep_last elements.""" + return collections.deque(iterable, maxlen=keep_last) # fastest way to exhaust an iterator +``` + +##### Rationale + +In the process of lazily applying operations to iterators, eventually a point is reached where evaluation of the iterator is necessary. To do this efficiently, Coconut provides the `consume` function, which will fully exhaust the iterator given to it. + +##### Example + +**Coconut:** +```coconut +range(10) |> map$((x) => x**2) |> map$(print) |> consume +``` + +**Python:** +```coconut_python +collections.deque(map(print, map(lambda x: x**2, range(10))), maxlen=0) +``` + + +### Built-Ins for Parallelization + #### `process_map` and `thread_map` ##### **process\_map**(_function_, *_iterables_, *, _chunksize_=`1`, _strict_=`False`, _stream_=`False`, _ordered_=`True`) @@ -4238,13 +4317,13 @@ with concurrent.futures.ThreadPoolExecutor() as executor: #### `collectby` and `mapreduce` -##### **collectby**(_key\_func_, _iterable_, _value\_func_=`None`, \*, _reduce\_func_=`None`, _collect\_in_=`None`, _map\_using_=`None`) +##### **collectby**(_key\_func_, _iterable_, _value\_func_=`None`, \*, _reduce\_func_=`None`, _collect\_in_=`None`, _reduce\_func\_init_=`...`, _map\_using_=`None`) `collectby(key_func, iterable)` collects the items in `iterable` into a dictionary of lists keyed by `key_func(item)`. If _value\_func_ is passed, instead collects `value_func(item)` into each list instead of `item`. -If _reduce\_func_ is passed, instead of collecting the items into lists, reduces over the items of each key with `reduce_func`, effectively implementing a MapReduce operation. If keys are intended to be unique, set `reduce_func=False` (`reduce_func=False` is also the default if _collect\_in_ is passed). +If _reduce\_func_ is passed, instead of collecting the items into lists, [`reduce`](#reduce) over the items of each key with _reduce\_func_, effectively implementing a MapReduce operation. If keys are intended to be unique, set `reduce_func=False` (`reduce_func=False` is also the default if _collect\_in_ is passed). If _reduce\_func_ is passed, then _reduce\_func\_init_ may also be passed, and will determine the initial value when reducing with _reduce\_func_. If _collect\_in_ is passed, initializes the collection from _collect\_in_ rather than as a `collections.defaultdict` (if `reduce_func=None`) or an empty `dict` (otherwise). Additionally, _reduce\_func_ defaults to `False` rather than `None` when _collect\_in_ is passed. Useful when you want to collect the results into a `pandas.DataFrame`. @@ -4252,17 +4331,17 @@ If _map_using_ is passed, calculates `key_func` and `value_func` by mapping them `collectby` is similar to [`itertools.groupby`](https://docs.python.org/3/library/itertools.html#itertools.groupby) except that `collectby` aggregates common elements regardless of their order in the input iterable, whereas `groupby` only aggregates common elements that are adjacent in the input iterable. -##### **mapreduce**(_key\_value\_func_, _iterable_, \*, _reduce\_func_=`None`, _collect\_in_=`None`, _map\_using_=`None`) +##### **mapreduce**(_key\_value\_func_, _iterable_, \*, _reduce\_func_=`None`, _collect\_in_=`None`, _reduce\_func\_init_=`...`, _map\_using_=`None`) `mapreduce(key_value_func, iterable)` functions the same as `collectby`, but allows calculating the keys and values together in one function. _key\_value\_func_ must return a 2-tuple of `(key, value)`. -##### **collectby.using_threads**(_key\_func_, _iterable_, _value\_func_=`None`, \*, _reduce\_func_=`None`, _collect\_in_=`None`, _ordered_=`False`, _chunksize_=`1`, _max\_workers_=`None`) +##### **collectby.using_threads**(_key\_func_, _iterable_, _value\_func_=`None`, \*, _reduce\_func_=`None`, _collect\_in_=`None`, _reduce\_func\_init_=`...`, _ordered_=`False`, _chunksize_=`1`, _max\_workers_=`None`) -##### **collectby.using_processes**(_key\_func_, _iterable_, _value\_func_=`None`, \*, _reduce\_func_=`None`, _collect\_in_=`None`, _ordered_=`False`, _chunksize_=`1`, _max\_workers_=`None`) +##### **collectby.using_processes**(_key\_func_, _iterable_, _value\_func_=`None`, \*, _reduce\_func_=`None`, _collect\_in_=`None`, _reduce\_func\_init_=`...`, _ordered_=`False`, _chunksize_=`1`, _max\_workers_=`None`) -##### **mapreduce.using_threads**(_key\_value\_func_, _iterable_, \*, _reduce\_func_=`None`, _collect\_in_=`None`, _ordered_=`False`, _chunksize_=`1`, _max\_workers_=`None`) +##### **mapreduce.using_threads**(_key\_value\_func_, _iterable_, \*, _reduce\_func_=`None`, _collect\_in_=`None`, _reduce\_func\_init_=`...`, _ordered_=`False`, _chunksize_=`1`, _max\_workers_=`None`) -##### **mapreduce.using_processes**(_key\_value\_func_, _iterable_, \*, _reduce\_func_=`None`, _collect\_in_=`None`, _ordered_=`False`, _chunksize_=`1`, _max\_workers_=`None`) +##### **mapreduce.using_processes**(_key\_value\_func_, _iterable_, \*, _reduce\_func_=`None`, _collect\_in_=`None`, _reduce\_func\_init_=`...`, _ordered_=`False`, _chunksize_=`1`, _max\_workers_=`None`) These shortcut methods call `collectby`/`mapreduce` with `map_using` set to [`process_map`](#process_map)/[`thread_map`](#thread_map), properly managed using the `.multiple_sequential_calls` method and the `stream=True` argument of [`process_map`](#process_map)/[`thread_map`](#thread_map). `reduce_func` will be called as soon as results arrive, and by default in whatever order they arrive in (to enforce the original order, pass _ordered_=`True`). @@ -4356,82 +4435,6 @@ async def load_pages(urls): return results ``` -#### `tee` - -**tee**(_iterable_, _n_=`2`) - -Coconut provides an optimized version of `itertools.tee` as a built-in under the name `tee`. - -Though `tee` is not deprecated, [`reiterable`](#reiterable) is generally recommended over `tee`. - -Custom `tee`/`reiterable` implementations for custom [Containers/Collections](https://docs.python.org/3/library/collections.abc.html) should be put in the `__copy__` method. Note that all [Sequences/Mappings/Sets](https://docs.python.org/3/library/collections.abc.html) are always assumed to be reiterable even without calling `__copy__`. - -##### Python Docs - -**tee**(_iterable, n=2_) - -Return _n_ independent iterators from a single iterable. Equivalent to: -```coconut_python -def tee(iterable, n=2): - it = iter(iterable) - deques = [collections.deque() for i in range(n)] - def gen(mydeque): - while True: - if not mydeque: # when the local deque is empty - newval = next(it) # fetch a new value and - for d in deques: # load it to all the deques - d.append(newval) - yield mydeque.popleft() - return tuple(gen(d) for d in deques) -``` -Once `tee()` has made a split, the original _iterable_ should not be used anywhere else; otherwise, the _iterable_ could get advanced without the tee objects being informed. - -This itertool may require significant auxiliary storage (depending on how much temporary data needs to be stored). In general, if one iterator uses most or all of the data before another iterator starts, it is faster to use `list()` instead of `tee()`. - -##### Example - -**Coconut:** -```coconut -original, temp = tee(original) -sliced = temp$[5:] -``` - -**Python:** -```coconut_python -import itertools -original, temp = itertools.tee(original) -sliced = itertools.islice(temp, 5, None) -``` - -#### `consume` - -**consume**(_iterable_, _keep\_last_=`0`) - -Coconut provides the `consume` function to efficiently exhaust an iterator and thus perform any lazy evaluation contained within it. `consume` takes one optional argument, `keep_last`, that defaults to 0 and specifies how many, if any, items from the end to return as a sequence (`None` will keep all elements). - -Equivalent to: -```coconut -def consume(iterable, keep_last=0): - """Fully exhaust iterable and return the last keep_last elements.""" - return collections.deque(iterable, maxlen=keep_last) # fastest way to exhaust an iterator -``` - -##### Rationale - -In the process of lazily applying operations to iterators, eventually a point is reached where evaluation of the iterator is necessary. To do this efficiently, Coconut provides the `consume` function, which will fully exhaust the iterator given to it. - -##### Example - -**Coconut:** -```coconut -range(10) |> map$((x) => x**2) |> map$(print) |> consume -``` - -**Python:** -```coconut_python -collections.deque(map(print, map(lambda x: x**2, range(10))), maxlen=0) -``` - ### Typing-Specific Built-Ins diff --git a/__coconut__/__init__.pyi b/__coconut__/__init__.pyi index edd630917..d501ea9f1 100644 --- a/__coconut__/__init__.pyi +++ b/__coconut__/__init__.pyi @@ -1672,6 +1672,7 @@ def collectby( iterable: _t.Iterable[_T], *, reduce_func: _t.Callable[[_T, _T], _V], + reduce_func_init: _T = ..., map_using: _t.Callable | None = None, ) -> _t.Dict[_U, _V]: ... @_t.overload @@ -1689,16 +1690,27 @@ def collectby( value_func: _t.Callable[[_T], _W], *, reduce_func: _t.Callable[[_W, _W], _V], + reduce_func_init: _W = ..., map_using: _t.Callable | None = None, ) -> _t.Dict[_U, _V]: ... @_t.overload def collectby( - key_func: _t.Callable, - iterable: _t.Iterable, - value_func: _t.Callable | None = None, + key_func: _t.Callable[[_T], _U], + iterable: _t.Iterable[_T], + *, + reduce_func: _t.Callable[[_T, _T], _V], + reduce_func_init: _T = ..., + map_using: _t.Callable | None = None, +) -> _t.Dict[_U, _V]: ... +@_t.overload +def collectby( + key_func: _t.Callable[[_U], _t.Any], + iterable: _t.Iterable[_U], + value_func: _t.Callable[[_U], _t.Any] | None = None, *, collect_in: _T, reduce_func: _t.Callable | None | _t.Literal[False] = None, + reduce_func_init: _t.Any = ..., map_using: _t.Callable | None = None, ) -> _T: """Collect the items in iterable into a dictionary of lists keyed by key_func(item). @@ -1729,15 +1741,26 @@ def mapreduce( iterable: _t.Iterable[_T], *, reduce_func: _t.Callable[[_W, _W], _V], + reduce_func_init: _W = ..., map_using: _t.Callable | None = None, ) -> _t.Dict[_U, _V]: ... @_t.overload def mapreduce( - key_value_func: _t.Callable, - iterable: _t.Iterable, + key_value_func: _t.Callable[[_T], _t.Tuple[_U, _W]], + iterable: _t.Iterable[_T], + *, + reduce_func: _t.Callable[[_X, _W], _V], + reduce_func_init: _X = ..., + map_using: _t.Callable | None = None, +) -> _t.Dict[_U, _V]: ... +@_t.overload +def mapreduce( + key_value_func: _t.Callable[[_U], _t.Tuple[_t.Any, _t.Any]], + iterable: _t.Iterable[_U], *, collect_in: _T, reduce_func: _t.Callable | None | _t.Literal[False] = None, + reduce_func_init: _t.Any = ..., map_using: _t.Callable | None = None, ) -> _T: """Map key_value_func over iterable, then collect the values into a dictionary of lists keyed by the keys. diff --git a/_coconut/__init__.pyi b/_coconut/__init__.pyi index c00dfdcb1..31d9fd411 100644 --- a/_coconut/__init__.pyi +++ b/_coconut/__init__.pyi @@ -29,6 +29,7 @@ import traceback as _traceback import weakref as _weakref import multiprocessing as _multiprocessing import pickle as _pickle +import inspect as _inspect from multiprocessing import dummy as _multiprocessing_dummy if sys.version_info >= (3,): @@ -86,6 +87,8 @@ contextlib = _contextlib traceback = _traceback weakref = _weakref multiprocessing = _multiprocessing +inspect = _inspect + multiprocessing_dummy = _multiprocessing_dummy copyreg = _copyreg diff --git a/coconut/compiler/header.py b/coconut/compiler/header.py index 076068b06..590f21498 100644 --- a/coconut/compiler/header.py +++ b/coconut/compiler/header.py @@ -125,7 +125,16 @@ def prepare(code, indent=0, **kwargs): return _indent(code, by=indent, strip=True, **kwargs) -def base_pycondition(target, ver, if_lt=None, if_ge=None, indent=None, newline=False, initial_newline=False, fallback=""): +def base_pycondition( + target, + ver, + if_lt=None, + if_ge=None, + indent=None, + newline=False, + initial_newline=False, + fallback="", +): """Produce code that depends on the Python version for the given target.""" internal_assert(isinstance(ver, tuple), "invalid pycondition version") internal_assert(if_lt or if_ge, "either if_lt or if_ge must be specified") @@ -179,6 +188,52 @@ def base_pycondition(target, ver, if_lt=None, if_ge=None, indent=None, newline=F return out +def def_in_exec(name, code, needs_vars={}, decorator=None): + """Get code that runs code in an exec and extracts name.""" + return ''' +_coconut_{name}_ns = {lbrace}"_coconut": _coconut{needs_vars}{rbrace} +_coconut_exec({code}, _coconut_{name}_ns) +{name} = {open_decorator}_coconut_{name}_ns["{name}"]{close_decorator} + '''.format( + lbrace="{", + rbrace="}", + name=name, + code=repr(code.strip()), + needs_vars=( + ", " + ", ".join( + repr(var_in_def) + ": " + var_out_def + for var_in_def, var_out_def in needs_vars.items() + ) + if needs_vars else "" + ), + open_decorator=decorator + "(" if decorator is not None else "", + close_decorator=")" if decorator is not None else "", + ) + + +def base_async_def( + target, + func_name, + async_def, + no_async_def, + needs_vars={}, + decorator=None, + **kwargs, +): + """Build up a universal async function definition.""" + target_info = get_target_info(target) + if target_info >= (3, 5): + out = async_def + else: + out = base_pycondition( + target, + (3, 5), + if_ge=def_in_exec(func_name, async_def, needs_vars=needs_vars, decorator=decorator), + if_lt=no_async_def, + ) + return prepare(out, **kwargs) + + def make_py_str(str_contents, target, after_py_str_defined=False): """Get code that effectively wraps the given code in py_str.""" return ( @@ -209,6 +264,7 @@ def process_header_args(which, use_hash, target, no_tco, strict, no_wrap): """Create the dictionary passed to str.format in the header.""" target_info = get_target_info(target) pycondition = partial(base_pycondition, target) + async_def = partial(base_async_def, target) format_dict = dict( COMMENT=COMMENT, @@ -503,8 +559,9 @@ def __bool__(self): indent=1, newline=True, ), - def_async_compose_call=prepare( - r''' + def_async_compose_call=async_def( + "__call__", + async_def=r''' async def __call__(self, *args, **kwargs): arg = await self._coconut_func(*args, **kwargs) for f, await_f in self._coconut_func_infos: @@ -512,34 +569,23 @@ async def __call__(self, *args, **kwargs): if await_f: arg = await arg return arg - ''' if target_info >= (3, 5) else - pycondition( - (3, 5), - if_ge=r''' -_coconut_call_ns = {"_coconut": _coconut} -_coconut_exec("""async def __call__(self, *args, **kwargs): - arg = await self._coconut_func(*args, **kwargs) - for f, await_f in self._coconut_func_infos: - arg = f(arg) - if await_f: - arg = await arg - return arg""", _coconut_call_ns) -__call__ = _coconut_call_ns["__call__"] - ''', - if_lt=pycondition( - (3, 4), - if_ge=r''' -_coconut_call_ns = {"_coconut": _coconut} -_coconut_exec("""def __call__(self, *args, **kwargs): + ''', + no_async_def=pycondition( + (3, 4), + if_ge=def_in_exec( + "__call__", + r''' +def __call__(self, *args, **kwargs): arg = yield from self._coconut_func(*args, **kwargs) for f, await_f in self._coconut_func_infos: arg = f(arg) if await_f: arg = yield from arg - raise _coconut.StopIteration(arg)""", _coconut_call_ns) -__call__ = _coconut.asyncio.coroutine(_coconut_call_ns["__call__"]) + raise _coconut.StopIteration(arg) ''', - if_lt=''' + decorator="_coconut.asyncio.coroutine", + ), + if_lt=''' @_coconut.asyncio.coroutine def __call__(self, *args, **kwargs): arg = yield _coconut.asyncio.From(self._coconut_func(*args, **kwargs)) @@ -549,7 +595,6 @@ def __call__(self, *args, **kwargs): arg = yield _coconut.asyncio.From(arg) raise _coconut.asyncio.Return(arg) ''', - ), ), indent=1 ), @@ -558,26 +603,20 @@ def __call__(self, *args, **kwargs): tco_comma="_coconut_tail_call, _coconut_tco, " if not no_tco else "", call_set_names_comma="_coconut_call_set_names, " if target_info < (3, 6) else "", handle_cls_args_comma="_coconut_handle_cls_kwargs, _coconut_handle_cls_stargs, " if not target.startswith("3") else "", - async_def_anext=prepare( - r''' + async_def_anext=async_def( + "__anext__", + async_def=r''' async def __anext__(self): return self.func(await self.aiter.__anext__()) - ''' if target_info >= (3, 5) else - pycondition( - (3, 5), - if_ge=r''' -_coconut_anext_ns = {"_coconut": _coconut} -_coconut_exec("""async def __anext__(self): - return self.func(await self.aiter.__anext__())""", _coconut_anext_ns) -__anext__ = _coconut_anext_ns["__anext__"] - ''', - if_lt=r''' -_coconut_anext_ns = {"_coconut": _coconut} -_coconut_exec("""def __anext__(self): + ''', + no_async_def=def_in_exec( + "__anext__", + r''' +def __anext__(self): result = yield from self.aiter.__anext__() - return self.func(result)""", _coconut_anext_ns) -__anext__ = _coconut.asyncio.coroutine(_coconut_anext_ns["__anext__"]) + return self.func(result) ''', + decorator="_coconut.asyncio.coroutine", ), indent=1, ), @@ -789,8 +828,9 @@ def __neg__(self): '''.format(**format_dict), indent=1, ), - def_async_map=prepare( - ''' + def_async_map=async_def( + "async_map", + async_def=''' async def async_map(async_func, *iters, strict=False): """Map async_func over iters asynchronously using anyio.""" import anyio @@ -803,31 +843,15 @@ async def store_func_in_of(i, args): for i, args in _coconut.enumerate({_coconut_}zip(*iters, strict=strict)): nursery.start_soon(store_func_in_of, i, args) return results - '''.format(**format_dict) if target_info >= (3, 5) else - pycondition( - (3, 5), - if_ge=''' -_coconut_async_map_ns = {lbrace}"_coconut": _coconut, "zip": zip{rbrace} -_coconut_exec("""async def async_map(async_func, *iters, strict=False): - \'''Map async_func over iters asynchronously using anyio.\''' - import anyio - results = [] - async def store_func_in_of(i, args): - got = await async_func(*args) - results.extend([None] * (1 + i - _coconut.len(results))) - results[i] = got - async with anyio.create_task_group() as nursery: - for i, args in _coconut.enumerate({_coconut_}zip(*iters, strict=strict)): - nursery.start_soon(store_func_in_of, i, args) - return results""", _coconut_async_map_ns) -async_map = _coconut_async_map_ns["async_map"] - '''.format(**format_dict), - if_lt=''' + '''.format(**format_dict), + no_async_def=''' def async_map(*args, **kwargs): """async_map not available on Python < 3.5""" raise _coconut.NameError("async_map not available on Python < 3.5") - ''', - ), + ''', + needs_vars={ + "{_coconut_}zip".format(**format_dict): "zip", + }, ), ) format_dict.update(extra_format_dict) diff --git a/coconut/compiler/templates/header.py_template b/coconut/compiler/templates/header.py_template index 721978c81..dc3ffff00 100644 --- a/coconut/compiler/templates/header.py_template +++ b/coconut/compiler/templates/header.py_template @@ -19,7 +19,7 @@ def _coconut_super(type=None, object_or_type=None): return _coconut_py_super(type, object_or_type) {set_super} class _coconut{object}:{COMMENT.EVERYTHING_HERE_MUST_BE_COPIED_TO_STUB_FILE} - import collections, copy, functools, types, itertools, operator, threading, os, warnings, contextlib, traceback, weakref, multiprocessing + import collections, copy, functools, types, itertools, operator, threading, os, warnings, contextlib, traceback, weakref, multiprocessing, inspect from multiprocessing import dummy as multiprocessing_dummy {maybe_bind_lru_cache}{import_copyreg} {import_asyncio} @@ -195,7 +195,7 @@ def _coconut_tco(func): @_coconut.functools.wraps(_coconut.itertools.tee) def tee(iterable, n=2): if n < 0: - raise ValueError("tee: n cannot be negative") + raise _coconut.ValueError("tee: n cannot be negative") elif n == 0: return () elif n == 1: @@ -733,7 +733,7 @@ class flatten(_coconut_has_iter):{COMMENT.cant_implement_len_else_list_calls_bec return ind + it.index(elem) except _coconut.ValueError: ind += _coconut.len(it) - raise ValueError("%r not in %r" % (elem, self)) + raise _coconut.ValueError("%r not in %r" % (elem, self)) def __fmap__(self, func): if self.levels == 1: return self.__class__({_coconut_}map(_coconut_partial({_coconut_}map, func), self.get_new_iter())) @@ -983,7 +983,6 @@ class zip(_coconut_baseclass, _coconut.zip): {zip_iter} def __fmap__(self, func): return {_coconut_}map(func, self) -{def_async_map} class zip_longest(zip): __slots__ = ("fillvalue",) __doc__ = getattr(_coconut.zip_longest, "__doc__", "Version of zip that fills in missing values with fillvalue.") @@ -1934,11 +1933,13 @@ def mapreduce(key_value_func, iterable, **kwargs): If reduce_func is passed, instead of collecting the values into lists, reduce over the values for each key with reduce_func, effectively implementing a MapReduce operation. - If map_using is passed, calculate key_value_func by mapping them over - the iterable using map_using as map. Useful with process_map/thread_map. + If collect_in is passed, initialize the collection from . """ collect_in = kwargs.pop("collect_in", None) reduce_func = kwargs.pop("reduce_func", None if collect_in is None else False) + reduce_func_init = kwargs.pop("reduce_func_init", _coconut_sentinel) + if reduce_func_init is not _coconut_sentinel and not reduce_func: + raise _coconut.TypeError("reduce_func_init requires reduce_func") map_using = kwargs.pop("map_using", _coconut.map) if kwargs: raise _coconut.TypeError("mapreduce()/collectby() got unexpected keyword arguments " + _coconut.repr(kwargs)) @@ -1947,10 +1948,10 @@ def mapreduce(key_value_func, iterable, **kwargs): if reduce_func is None: collection[key].append(val) else: - old_val = collection.get(key, _coconut_sentinel) + old_val = collection.get(key, reduce_func_init) if old_val is not _coconut_sentinel: if reduce_func is False: - raise ValueError("mapreduce()/collectby() got duplicate key " + repr(key) + " with reduce_func=False") + raise _coconut.ValueError("mapreduce()/collectby() got duplicate key " + repr(key) + " with reduce_func=False") val = reduce_func(old_val, val) collection[key] = val return collection @@ -2180,6 +2181,7 @@ class _coconut_SupportsInv(_coconut.typing.Protocol): """ def __invert__(self): raise _coconut.NotImplementedError("Protocol methods cannot be called at runtime ((~) in a typing context is a Protocol)") +{def_async_map} {def_aliases} _coconut_self_match_types = {self_match_types} _coconut_Expected, _coconut_MatchError, _coconut_cartesian_product, _coconut_count, _coconut_cycle, _coconut_enumerate, _coconut_flatten, _coconut_filter, _coconut_groupsof, _coconut_ident, _coconut_lift, _coconut_map, _coconut_mapreduce, _coconut_multiset, _coconut_range, _coconut_reiterable, _coconut_reversed, _coconut_scan, _coconut_starmap, _coconut_tee, _coconut_windowsof, _coconut_zip, _coconut_zip_longest, TYPE_CHECKING, reduce, takewhile, dropwhile = Expected, MatchError, cartesian_product, count, cycle, enumerate, flatten, filter, groupsof, ident, lift, map, mapreduce, multiset, range, reiterable, reversed, scan, starmap, tee, windowsof, zip, zip_longest, False, _coconut.functools.reduce, _coconut.itertools.takewhile, _coconut.itertools.dropwhile{COMMENT.anything_added_here_should_be_copied_to_stub_file} diff --git a/coconut/tests/src/cocotest/agnostic/primary_2.coco b/coconut/tests/src/cocotest/agnostic/primary_2.coco index 3d4b31417..7ad4819a0 100644 --- a/coconut/tests/src/cocotest/agnostic/primary_2.coco +++ b/coconut/tests/src/cocotest/agnostic/primary_2.coco @@ -403,6 +403,7 @@ def primary_test_2() -> bool: assert some_data |> mapreducer == {"a": ["123"], "b": ["567"]} assert_raises(-> collectby(.[0], [(0, 1), (0, 2)], reduce_func=False), ValueError) # type: ignore assert ident$(x=?).__name__ == "ident" == ident$(1).__name__ # type: ignore + assert collectby(.[0], [(0, 1), (0, 2)], value_func=.[1], reduce_func=(+), reduce_func_init=1) == {0: 4} with process_map.multiple_sequential_calls(): # type: ignore assert map((+), range(3), range(4)$[:-1], strict=True) |> list == [0, 2, 4] == process_map((+), range(3), range(4)$[:-1], strict=True) |> list # type: ignore diff --git a/coconut/tests/src/cocotest/agnostic/suite.coco b/coconut/tests/src/cocotest/agnostic/suite.coco index 91c12d3b4..569418ee2 100644 --- a/coconut/tests/src/cocotest/agnostic/suite.coco +++ b/coconut/tests/src/cocotest/agnostic/suite.coco @@ -1065,6 +1065,7 @@ forward 2""") == 900 assert safe_raise_exc(IOError).error `isinstance` IOError assert safe_raise_exc(IOError).handle(IOError, const 10).unwrap() == 10 == safe_raise_exc(IOError).expect_error(IOError).result_or(10) assert pickle_round_trip(ident$(1))() == 1 == pickle_round_trip(ident$(x=?))(1) + assert x_or_y(x=1) == (1, 1) == x_or_y(y=1) with process_map.multiple_sequential_calls(): # type: ignore assert process_map(tuple <.. (|>)$(to_sort), qsorts) |> list == [to_sort |> sorted |> tuple] * len(qsorts) diff --git a/coconut/tests/src/cocotest/agnostic/util.coco b/coconut/tests/src/cocotest/agnostic/util.coco index b6fd84fc1..ae0a9bfef 100644 --- a/coconut/tests/src/cocotest/agnostic/util.coco +++ b/coconut/tests/src/cocotest/agnostic/util.coco @@ -962,7 +962,7 @@ def still_ident(x) = @prepattern(ident, allow_any_func=True) def not_ident(x) = "bar" -# Pattern-matching functions with guards +# Pattern-matching functions def pattern_abs(x if x < 0) = -x addpattern def pattern_abs(x) = x # type: ignore @@ -970,6 +970,8 @@ addpattern def pattern_abs(x) = x # type: ignore def `pattern_abs_` (x) if x < 0 = -x addpattern def `pattern_abs_` (x) = x # type: ignore +def x_or_y(x and y) = (x, y) + # Recursive iterator @recursive_generator diff --git a/coconut/tests/src/cocotest/target_36/py36_test.coco b/coconut/tests/src/cocotest/target_36/py36_test.coco index 255dd1084..2d7afee34 100644 --- a/coconut/tests/src/cocotest/target_36/py36_test.coco +++ b/coconut/tests/src/cocotest/target_36/py36_test.coco @@ -14,6 +14,14 @@ def py36_test() -> bool: funcs.append(async copyclosure def -> x) return funcs async def await_all(xs) = [await x for x in xs] + async def aplus1(x) = x + 1 + async def async_mapreduce(func, iterable, **kwargs) = ( + iterable + |> async_map$(func) + |> await + |> mapreduce$(ident, **kwargs) + ) + async def atest(): assert ( outer_func() @@ -37,6 +45,35 @@ def py36_test() -> bool: |> map$(.*10) |> list ) == range(5) |> list + assert ( + {"a": 0, "b": 1} + |> .items() + |> async_mapreduce$( + (async def ((k, v)) => + (key=k, value=await aplus1(v))), + collect_in={"c": 0}, + ) + |> await + ) == {"a": 1, "b": 2, "c": 0} + assert ( + [0, 2, 0] + |> async_mapreduce$( + (async def x => + (key=x, value=await aplus1(x))), + reduce_func=(+), + ) + |> await + ) == {0: 2, 2: 3} + assert ( + [0, 2, 0] + |> async_mapreduce$( + (async def x => + (key=x, value=await aplus1(x))), + reduce_func=(+), + reduce_func_init=10, + ) + |> await + ) == {0: 12, 2: 13} loop.run_until_complete(atest()) loop.close() diff --git a/coconut/util.py b/coconut/util.py index 128c4afd3..739645214 100644 --- a/coconut/util.py +++ b/coconut/util.py @@ -30,6 +30,7 @@ from types import MethodType from contextlib import contextmanager from collections import defaultdict +from functools import partial if sys.version_info >= (3, 2): from functools import lru_cache @@ -249,12 +250,18 @@ def add(self, item): self[item] = True -def assert_remove_prefix(inputstr, prefix): +def assert_remove_prefix(inputstr, prefix, allow_no_prefix=False): """Remove prefix asserting that inputstr starts with it.""" - assert inputstr.startswith(prefix), inputstr + if not allow_no_prefix: + assert inputstr.startswith(prefix), inputstr + elif not inputstr.startswith(prefix): + return inputstr return inputstr[len(prefix):] +remove_prefix = partial(assert_remove_prefix, allow_no_prefix=True) + + def ensure_dir(dirpath): """Ensure that a directory exists.""" if not os.path.exists(dirpath): From d1b1cde1dd6736d206514e0dc1ddb0e29d6c5df2 Mon Sep 17 00:00:00 2001 From: Evan Hubinger Date: Sun, 5 Nov 2023 00:51:20 -0700 Subject: [PATCH 059/121] Add find_packages, improve override Resolves #798, #800. --- DOCS.md | 55 ++++++++++++++----- coconut/api.py | 47 ++++++++++++++-- coconut/api.pyi | 7 +++ coconut/command/command.py | 10 +++- coconut/command/command.pyi | 5 +- coconut/compiler/templates/header.py_template | 6 ++ coconut/constants.py | 4 +- coconut/integrations.py | 3 +- coconut/root.py | 2 +- coconut/tests/main_test.py | 28 ++++++++++ .../tests/src/cocotest/agnostic/suite.coco | 2 + coconut/tests/src/cocotest/agnostic/util.coco | 14 +++++ coconut/util.py | 5 ++ 13 files changed, 164 insertions(+), 24 deletions(-) diff --git a/DOCS.md b/DOCS.md index a745d5f50..b3fa538cf 100644 --- a/DOCS.md +++ b/DOCS.md @@ -3106,7 +3106,7 @@ def fib(n): **override**(_func_) -Coconut provides the `@override` decorator to allow declaring a method definition in a subclass as an override of some parent class method. When `@override` is used on a method, if a method of the same name does not exist on some parent class, the class definition will raise a `RuntimeError`. +Coconut provides the `@override` decorator to allow declaring a method definition in a subclass as an override of some parent class method. When `@override` is used on a method, if a method of the same name does not exist on some parent class, the class definition will raise a `RuntimeError`. `@override` works with other decorators such as `@classmethod` and `@staticmethod`, but only if `@override` is the outer-most decorator. Additionally, `override` will present to type checkers as [`typing_extensions.override`](https://pypi.org/project/typing-extensions/). @@ -4672,6 +4672,12 @@ Executes the given _args_ as if they were fed to `coconut` on the command-line, Has the same effect of setting the command-line flags on the given _state_ object as `setup` (with the global `state` object used when _state_ is `False`). +#### `cmd_sys` + +**coconut.api.cmd_sys**(_args_=`None`, *, _argv_=`None`, _interact_=`False`, _default\_target_=`"sys"`, _state_=`False`) + +Same as `coconut.api.cmd` but _default\_target_ is `"sys"` rather than `None` (universal). + #### `coconut_exec` **coconut.api.coconut_exec**(_expression_, _globals_=`None`, _locals_=`None`, _state_=`False`, _keep\_internal\_state_=`None`) @@ -4684,18 +4690,6 @@ Version of [`exec`](https://docs.python.org/3/library/functions.html#exec) which Version of [`eval`](https://docs.python.org/3/library/functions.html#eval) which can evaluate Coconut code. -#### `version` - -**coconut.api.version**(**[**_which_**]**) - -Retrieves a string containing information about the Coconut version. The optional argument _which_ is the type of version information desired. Possible values of _which_ are: - -- `"num"`: the numerical version (the default) -- `"name"`: the version codename -- `"spec"`: the numerical version with the codename attached -- `"tag"`: the version tag used in GitHub and documentation URLs -- `"-v"`: the full string printed by `coconut -v` - #### `auto_compilation` **coconut.api.auto_compilation**(_on_=`True`, _args_=`None`, _use\_cache\_dir_=`None`) @@ -4712,6 +4706,41 @@ If _use\_cache\_dir_ is passed, it will turn on or off the usage of a `__coconut Switches the [`breakpoint` built-in](https://www.python.org/dev/peps/pep-0553/) which Coconut makes universally available to use [`coconut.embed`](#coconut-embed) instead of [`pdb.set_trace`](https://docs.python.org/3/library/pdb.html#pdb.set_trace) (or undoes that switch if `on=False`). This function is called automatically when `coconut.api` is imported. +#### `find_and_compile_packages` + +**coconut.api.find_and_compile_packages**(_where_=`"."`, _exclude_=`()`, _include_=`("*",)`) + +Behaves similarly to [`setuptools.find_packages`](https://setuptools.pypa.io/en/latest/userguide/quickstart.html#package-discovery) except that it finds Coconut packages rather than Python packages, and compiles any Coconut packages that it finds in-place. + +Note that if you want to use `find_and_compile_packages` in your `setup.py`, you'll need to include `coconut` as a [build-time dependency in your `pyproject.toml`](https://pip.pypa.io/en/stable/reference/build-system/pyproject-toml/#build-time-dependencies). + +##### Example + +```coconut_python +# if you put this in your setup.py, your Coconut package will be compiled in-place whenever it is installed + +from setuptools import setup +from coconut.api import find_and_compile_packages + +setup( + name=..., + version=..., + packages=find_and_compile_packages(), +) +``` + +#### `version` + +**coconut.api.version**(**[**_which_**]**) + +Retrieves a string containing information about the Coconut version. The optional argument _which_ is the type of version information desired. Possible values of _which_ are: + +- `"num"`: the numerical version (the default) +- `"name"`: the version codename +- `"spec"`: the numerical version with the codename attached +- `"tag"`: the version tag used in GitHub and documentation URLs +- `"-v"`: the full string printed by `coconut -v` + #### `CoconutException` If an error is encountered in a api function, a `CoconutException` instance may be raised. `coconut.api.CoconutException` is provided to allow catching such errors. diff --git a/coconut/api.py b/coconut/api.py index c8a8bb995..71d74773e 100644 --- a/coconut/api.py +++ b/coconut/api.py @@ -23,15 +23,17 @@ import os.path import codecs from functools import partial +from setuptools import PackageFinder try: from encodings import utf_8 except ImportError: utf_8 = None from coconut.root import _coconut_exec +from coconut.util import override from coconut.integrations import embed from coconut.exceptions import CoconutException -from coconut.command import Command +from coconut.command.command import Command from coconut.command.cli import cli_version from coconut.command.util import proc_run_args from coconut.compiler import Compiler @@ -42,7 +44,6 @@ coconut_kernel_kwargs, default_use_cache_dir, coconut_cache_dir, - coconut_run_kwargs, ) # ----------------------------------------------------------------------------------------------------------------------- @@ -68,9 +69,16 @@ def get_state(state=None): def cmd(cmd_args, **kwargs): """Process command-line arguments.""" state = kwargs.pop("state", False) + cmd_func = kwargs.pop("_cmd_func", "cmd") if isinstance(cmd_args, (str, bytes)): cmd_args = cmd_args.split() - return get_state(state).cmd(cmd_args, **kwargs) + return getattr(get_state(state), cmd_func)(cmd_args, **kwargs) + + +def cmd_sys(*args, **kwargs): + """Same as api.cmd() but defaults to --target sys.""" + kwargs["_cmd_func"] = "cmd_sys" + return cmd(*args, **kwargs) VERSIONS = { @@ -214,7 +222,7 @@ def cmd(self, *args): """Run the Coconut compiler with the given args.""" if self.command is None: self.command = Command() - return self.command.cmd(list(args) + self.args, interact=False, **coconut_run_kwargs) + return self.command.cmd_sys(list(args) + self.args, interact=False) def compile(self, path, package): """Compile a path to a file or package.""" @@ -315,6 +323,7 @@ def compile_coconut(cls, source): cls.coconut_compiler = Compiler(**coconut_kernel_kwargs) return cls.coconut_compiler.parse_sys(source) + @override @classmethod def decode(cls, input_bytes, errors="strict"): """Decode and compile the given Coconut source bytes.""" @@ -347,3 +356,33 @@ def get_coconut_encoding(encoding="coconut"): codecs.register(get_coconut_encoding) + + +# ----------------------------------------------------------------------------------------------------------------------- +# SETUPTOOLS: +# ----------------------------------------------------------------------------------------------------------------------- + +class CoconutPackageFinder(PackageFinder, object): + + _coconut_command = None + + @classmethod + def _coconut_compile(cls, path): + """Run the Coconut compiler with the given args.""" + if cls._coconut_command is None: + cls._coconut_command = Command() + return cls._coconut_command.cmd_sys([path], interact=False) + + @override + @classmethod + def _looks_like_package(cls, path, _package_name): + is_coconut_package = any( + os.path.isfile(os.path.join(path, "__init__" + ext)) + for ext in code_exts + ) + if is_coconut_package: + cls._coconut_compile(path) + return is_coconut_package + + +find_and_compile_packages = CoconutPackageFinder.find diff --git a/coconut/api.pyi b/coconut/api.pyi index 97f6fbf80..5078d8206 100644 --- a/coconut/api.pyi +++ b/coconut/api.pyi @@ -21,6 +21,8 @@ from typing import ( Text, ) +from setuptools import find_packages as _find_packages + from coconut.command.command import Command class CoconutException(Exception): @@ -50,6 +52,8 @@ def cmd( """Process command-line arguments.""" ... +cmd_sys = cmd + VERSIONS: Dict[Text, Text] = ... @@ -150,3 +154,6 @@ def auto_compilation( def get_coconut_encoding(encoding: Text = ...) -> Any: """Get a CodecInfo for the given Coconut encoding.""" ... + + +find_and_compile_packages = _find_packages diff --git a/coconut/command/command.py b/coconut/command/command.py index 9848c7fc1..d427fc79e 100644 --- a/coconut/command/command.py +++ b/coconut/command/command.py @@ -72,7 +72,7 @@ create_package_retries, default_use_cache_dir, coconut_cache_dir, - coconut_run_kwargs, + coconut_sys_kwargs, interpreter_uses_incremental, disable_incremental_for_len, ) @@ -165,10 +165,16 @@ def start(self, run=False): dest = os.path.join(os.path.dirname(source), coconut_cache_dir) else: dest = os.path.join(source, coconut_cache_dir) - self.cmd(args, argv=argv, use_dest=dest, **coconut_run_kwargs) + self.cmd_sys(args, argv=argv, use_dest=dest) else: self.cmd() + def cmd_sys(self, *args, **in_kwargs): + """Same as .cmd(), but uses defaults from coconut_sys_kwargs.""" + out_kwargs = coconut_sys_kwargs.copy() + out_kwargs.update(in_kwargs) + return self.cmd(*args, **out_kwargs) + # new external parameters should be updated in api.pyi and DOCS def cmd(self, args=None, argv=None, interact=True, default_target=None, use_dest=None): """Process command-line arguments.""" diff --git a/coconut/command/command.pyi b/coconut/command/command.pyi index 3f1d4ba40..f69b9ec2b 100644 --- a/coconut/command/command.pyi +++ b/coconut/command/command.pyi @@ -15,7 +15,10 @@ Description: MyPy stub file for command.py. # MAIN: # ----------------------------------------------------------------------------------------------------------------------- +from typing import Callable + class Command: """Coconut command-line interface.""" - ... + cmd: Callable + cmd_sys: Callable diff --git a/coconut/compiler/templates/header.py_template b/coconut/compiler/templates/header.py_template index dc3ffff00..3a742eee9 100644 --- a/coconut/compiler/templates/header.py_template +++ b/coconut/compiler/templates/header.py_template @@ -1649,6 +1649,12 @@ class override(_coconut_baseclass): def __init__(self, func): self.func = func def __get__(self, obj, objtype=None): + self_func_get = _coconut.getattr(self.func, "__get__", None) + if self_func_get is not None: + if objtype is None: + return self_func_get(obj) + else: + return self_func_get(obj, objtype) if obj is None: return self.func {return_method_of_self_func} diff --git a/coconut/constants.py b/coconut/constants.py index 66717a961..8b7399e8d 100644 --- a/coconut/constants.py +++ b/coconut/constants.py @@ -669,7 +669,7 @@ def get_path_env_var(env_var, default): # always use atomic --xxx=yyy rather than --xxx yyy # and don't include --run, --quiet, or --target as they're added separately coconut_base_run_args = ("--keep-lines",) -coconut_run_kwargs = dict(default_target="sys") # passed to Command.cmd +coconut_sys_kwargs = dict(default_target="sys") # passed to Command.cmd default_mypy_args = ( "--pretty", @@ -902,6 +902,7 @@ def get_path_env_var(env_var, default): ("async_generator", "py35"), ("exceptiongroup", "py37;py<311"), ("anyio", "py36"), + "setuptools", ), "cpython": ( "cPyparsing", @@ -1043,6 +1044,7 @@ def get_path_env_var(env_var, default): # don't upgrade this; it breaks on Python 3.4 ("pygments", "py<39"): (2, 3), # don't upgrade these; they break on Python 2 + "setuptools": (44,), ("jupyter-client", "py<35"): (5, 3), ("pywinpty", "py<3;windows"): (0, 5), ("jupyter-console", "py<35"): (5, 2), diff --git a/coconut/integrations.py b/coconut/integrations.py index f2a3537ee..6ca1c377c 100644 --- a/coconut/integrations.py +++ b/coconut/integrations.py @@ -23,7 +23,6 @@ from coconut.constants import ( coconut_kernel_kwargs, - coconut_run_kwargs, enabled_xonsh_modes, interpreter_uses_incremental, ) @@ -77,7 +76,7 @@ def magic(line, cell=None): # first line in block is cmd, rest is code line = line.strip() if line: - api.cmd(line, state=magic_state, **coconut_run_kwargs) + api.cmd_sys(line, state=magic_state) code = cell compiled = api.parse(code, state=magic_state) except CoconutException: diff --git a/coconut/root.py b/coconut/root.py index 1e9792141..e6538d3ed 100644 --- a/coconut/root.py +++ b/coconut/root.py @@ -26,7 +26,7 @@ VERSION = "3.0.3" VERSION_NAME = None # False for release, int >= 1 for develop -DEVELOP = 19 +DEVELOP = 20 ALPHA = False # for pre releases rather than post releases assert DEVELOP is False or DEVELOP >= 1, "DEVELOP must be False or an int >= 1" diff --git a/coconut/tests/main_test.py b/coconut/tests/main_test.py index 6c5b44701..5f6ec7b30 100644 --- a/coconut/tests/main_test.py +++ b/coconut/tests/main_test.py @@ -105,6 +105,8 @@ additional_dest = os.path.join(base, "dest", "additional_dest") src_cache_dir = os.path.join(src, coconut_cache_dir) +cocotest_dir = os.path.join(src, "cocotest") +agnostic_dir = os.path.join(cocotest_dir, "agnostic") runnable_coco = os.path.join(src, "runnable.coco") runnable_py = os.path.join(src, "runnable.py") @@ -472,6 +474,26 @@ def using_coconut(fresh_logger=True, fresh_api=False): logger.copy_from(saved_logger) +def remove_pys_in(dirpath): + removed_pys = 0 + for fname in os.listdir(dirpath): + if fname.endswith(".py"): + rm_path(os.path.join(dirpath, fname)) + removed_pys += 1 + return removed_pys + + +@contextmanager +def using_pys_in(dirpath): + """Remove *.py in dirpath at start and finish.""" + remove_pys_in(dirpath) + try: + yield + finally: + removed_pys = remove_pys_in(dirpath) + assert removed_pys > 0, os.listdir(dirpath) + + @contextmanager def using_sys_path(path, prepend=False): """Adds a path to sys.path.""" @@ -797,6 +819,12 @@ def test_import_hook(self): reload(runnable) assert runnable.success == "" + def test_find_packages(self): + with using_pys_in(agnostic_dir): + with using_coconut(): + from coconut.api import find_and_compile_packages + assert find_and_compile_packages(cocotest_dir) == ["agnostic"] + def test_runnable(self): run_runnable() diff --git a/coconut/tests/src/cocotest/agnostic/suite.coco b/coconut/tests/src/cocotest/agnostic/suite.coco index 569418ee2..beba7d22d 100644 --- a/coconut/tests/src/cocotest/agnostic/suite.coco +++ b/coconut/tests/src/cocotest/agnostic/suite.coco @@ -1066,6 +1066,8 @@ forward 2""") == 900 assert safe_raise_exc(IOError).handle(IOError, const 10).unwrap() == 10 == safe_raise_exc(IOError).expect_error(IOError).result_or(10) assert pickle_round_trip(ident$(1))() == 1 == pickle_round_trip(ident$(x=?))(1) assert x_or_y(x=1) == (1, 1) == x_or_y(y=1) + assert DerivedWithMeths().cls_meth() + assert DerivedWithMeths().static_meth() with process_map.multiple_sequential_calls(): # type: ignore assert process_map(tuple <.. (|>)$(to_sort), qsorts) |> list == [to_sort |> sorted |> tuple] * len(qsorts) diff --git a/coconut/tests/src/cocotest/agnostic/util.coco b/coconut/tests/src/cocotest/agnostic/util.coco index ae0a9bfef..517168152 100644 --- a/coconut/tests/src/cocotest/agnostic/util.coco +++ b/coconut/tests/src/cocotest/agnostic/util.coco @@ -881,6 +881,20 @@ class inh_inh_A(inh_A): @override def true(self) = False +class BaseWithMeths: + @classmethod + def cls_meth(cls) = False + @staticmethod + def static_meth() = False + +class DerivedWithMeths(BaseWithMeths): + @override + @classmethod + def cls_meth(cls) = True + @override + @staticmethod + def static_meth() = True + class MyExc(Exception): def __init__(self, m): super().__init__(m) diff --git a/coconut/util.py b/coconut/util.py index 739645214..a5d68f39c 100644 --- a/coconut/util.py +++ b/coconut/util.py @@ -107,6 +107,11 @@ def __init__(self, func): self.func = func def __get__(self, obj, objtype=None): + if hasattr(self.func, "__get__"): + if objtype is None: + return self.func.__get__(obj) + else: + return self.func.__get__(obj, objtype) if obj is None: return self.func if PY2: From 484965f0b485ce54965b0cbaf6b92b1d3c8c449e Mon Sep 17 00:00:00 2001 From: Evan Hubinger Date: Sun, 5 Nov 2023 01:26:18 -0700 Subject: [PATCH 060/121] Fix syntax --- coconut/compiler/header.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/coconut/compiler/header.py b/coconut/compiler/header.py index 590f21498..a81a37f10 100644 --- a/coconut/compiler/header.py +++ b/coconut/compiler/header.py @@ -218,7 +218,7 @@ def base_async_def( no_async_def, needs_vars={}, decorator=None, - **kwargs, + **kwargs # no comma; breaks on <=3.5 ): """Build up a universal async function definition.""" target_info = get_target_info(target) From cedb148e5ea9dad7dca7b600b5b91a5da0091394 Mon Sep 17 00:00:00 2001 From: Evan Hubinger Date: Sun, 5 Nov 2023 01:58:14 -0700 Subject: [PATCH 061/121] Prevent unwanted multiprocessing --- DOCS.md | 6 +++--- coconut/api.pyi | 1 + coconut/command/cli.py | 4 ++-- coconut/command/command.py | 8 +++++--- coconut/constants.py | 4 ++-- coconut/root.py | 2 +- 6 files changed, 14 insertions(+), 11 deletions(-) diff --git a/DOCS.md b/DOCS.md index b3fa538cf..a78927800 100644 --- a/DOCS.md +++ b/DOCS.md @@ -4666,7 +4666,7 @@ Can optionally be called to warm up the compiler and get it ready for parsing. P #### `cmd` -**coconut.api.cmd**(_args_=`None`, *, _argv_=`None`, _interact_=`False`, _default\_target_=`None`, _state_=`False`) +**coconut.api.cmd**(_args_=`None`, *, _argv_=`None`, _interact_=`False`, _default\_target_=`None`, _default\_jobs_=`None`, _state_=`False`) Executes the given _args_ as if they were fed to `coconut` on the command-line, with the exception that unless _interact_ is true or `-i` is passed, the interpreter will not be started. Additionally, _argv_ can be used to pass in arguments as in `--argv` and _default\_target_ can be used to set the default `--target`. @@ -4674,9 +4674,9 @@ Has the same effect of setting the command-line flags on the given _state_ objec #### `cmd_sys` -**coconut.api.cmd_sys**(_args_=`None`, *, _argv_=`None`, _interact_=`False`, _default\_target_=`"sys"`, _state_=`False`) +**coconut.api.cmd_sys**(_args_=`None`, *, _argv_=`None`, _interact_=`False`, _default\_target_=`"sys"`, _default\_jobs_=`"0"`, _state_=`False`) -Same as `coconut.api.cmd` but _default\_target_ is `"sys"` rather than `None` (universal). +Same as `coconut.api.cmd` but _default\_target_ is `"sys"` rather than `None` (universal) and _default\_jobs_=`"0"` rather than `None` (`"sys"`)`. #### `coconut_exec` diff --git a/coconut/api.pyi b/coconut/api.pyi index 5078d8206..511831c60 100644 --- a/coconut/api.pyi +++ b/coconut/api.pyi @@ -48,6 +48,7 @@ def cmd( argv: Iterable[Text] | None = None, interact: bool = False, default_target: Text | None = None, + default_jobs: Text | None = None, ) -> None: """Process command-line arguments.""" ... diff --git a/coconut/command/cli.py b/coconut/command/cli.py index 2bd237a13..38f51a8b7 100644 --- a/coconut/command/cli.py +++ b/coconut/command/cli.py @@ -32,7 +32,7 @@ vi_mode_env_var, prompt_vi_mode, py_version_str, - default_jobs, + base_default_jobs, ) # ----------------------------------------------------------------------------------------------------------------------- @@ -193,7 +193,7 @@ "-j", "--jobs", metavar="processes", type=str, - help="number of additional processes to use (defaults to " + ascii(default_jobs) + ") (0 is no additional processes; 'sys' uses machine default)", + help="number of additional processes to use (defaults to " + ascii(base_default_jobs) + ") (0 is no additional processes; 'sys' uses machine default)", ) arguments.add_argument( diff --git a/coconut/command/command.py b/coconut/command/command.py index d427fc79e..5c0aafd32 100644 --- a/coconut/command/command.py +++ b/coconut/command/command.py @@ -68,7 +68,7 @@ coconut_pth_file, error_color_code, jupyter_console_commands, - default_jobs, + base_default_jobs, create_package_retries, default_use_cache_dir, coconut_cache_dir, @@ -176,7 +176,7 @@ def cmd_sys(self, *args, **in_kwargs): return self.cmd(*args, **out_kwargs) # new external parameters should be updated in api.pyi and DOCS - def cmd(self, args=None, argv=None, interact=True, default_target=None, use_dest=None): + def cmd(self, args=None, argv=None, interact=True, default_target=None, default_jobs=None, use_dest=None): """Process command-line arguments.""" result = None with self.handling_exceptions(): @@ -190,6 +190,8 @@ def cmd(self, args=None, argv=None, interact=True, default_target=None, use_dest parsed_args.argv = argv if parsed_args.target is None: parsed_args.target = default_target + if parsed_args.jobs is None: + parsed_args.jobs = default_jobs if use_dest is not None and not parsed_args.no_write: internal_assert(parsed_args.dest is None, "coconut-run got passed a dest", parsed_args) parsed_args.dest = use_dest @@ -706,7 +708,7 @@ def disable_jobs(self): def get_max_workers(self): """Get the max_workers to use for creating ProcessPoolExecutor.""" - jobs = self.jobs if self.jobs is not None else default_jobs + jobs = self.jobs if self.jobs is not None else base_default_jobs if jobs == "sys": return None else: diff --git a/coconut/constants.py b/coconut/constants.py index 8b7399e8d..6b6d7f9b0 100644 --- a/coconut/constants.py +++ b/coconut/constants.py @@ -669,7 +669,7 @@ def get_path_env_var(env_var, default): # always use atomic --xxx=yyy rather than --xxx yyy # and don't include --run, --quiet, or --target as they're added separately coconut_base_run_args = ("--keep-lines",) -coconut_sys_kwargs = dict(default_target="sys") # passed to Command.cmd +coconut_sys_kwargs = dict(default_target="sys", default_jobs="0") # passed to Command.cmd default_mypy_args = ( "--pretty", @@ -701,7 +701,7 @@ def get_path_env_var(env_var, default): kilobyte = 1024 min_stack_size_kbs = 160 -default_jobs = "sys" if not PY26 else 0 +base_default_jobs = "sys" if not PY26 else 0 mypy_install_arg = "install" jupyter_install_arg = "install" diff --git a/coconut/root.py b/coconut/root.py index e6538d3ed..3f7331836 100644 --- a/coconut/root.py +++ b/coconut/root.py @@ -26,7 +26,7 @@ VERSION = "3.0.3" VERSION_NAME = None # False for release, int >= 1 for develop -DEVELOP = 20 +DEVELOP = 21 ALPHA = False # for pre releases rather than post releases assert DEVELOP is False or DEVELOP >= 1, "DEVELOP must be False or an int >= 1" From c83609f59a0a581c502cb204713beda2afdb4d9c Mon Sep 17 00:00:00 2001 From: Evan Hubinger Date: Sun, 5 Nov 2023 01:05:43 -0800 Subject: [PATCH 062/121] Clean up docs --- DOCS.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/DOCS.md b/DOCS.md index a78927800..8a484efa0 100644 --- a/DOCS.md +++ b/DOCS.md @@ -4676,7 +4676,7 @@ Has the same effect of setting the command-line flags on the given _state_ objec **coconut.api.cmd_sys**(_args_=`None`, *, _argv_=`None`, _interact_=`False`, _default\_target_=`"sys"`, _default\_jobs_=`"0"`, _state_=`False`) -Same as `coconut.api.cmd` but _default\_target_ is `"sys"` rather than `None` (universal) and _default\_jobs_=`"0"` rather than `None` (`"sys"`)`. +Same as `coconut.api.cmd` but _default\_target_ is `"sys"` rather than `None` (universal) and _default\_jobs_=`"0"` rather than `None` (`"sys"`). Since `cmd_sys` defaults to not using `multiprocessing`, it is preferred whenever that might be a problem, e.g. [if you're not inside an `if __name__ == "__main__"` block on Windows](https://stackoverflow.com/questions/20360686/compulsory-usage-of-if-name-main-in-windows-while-using-multiprocessi). #### `coconut_exec` From 352b9429423537cfe146fbcd18cbea820098cf90 Mon Sep 17 00:00:00 2001 From: Evan Hubinger Date: Sun, 5 Nov 2023 15:54:58 -0800 Subject: [PATCH 063/121] Fix test errors --- coconut/api.py | 2 +- coconut/tests/src/cocotest/agnostic/primary_2.coco | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/coconut/api.py b/coconut/api.py index 71d74773e..562784f64 100644 --- a/coconut/api.py +++ b/coconut/api.py @@ -375,7 +375,7 @@ def _coconut_compile(cls, path): @override @classmethod - def _looks_like_package(cls, path, _package_name): + def _looks_like_package(cls, path, _package_name=None): is_coconut_package = any( os.path.isfile(os.path.join(path, "__init__" + ext)) for ext in code_exts diff --git a/coconut/tests/src/cocotest/agnostic/primary_2.coco b/coconut/tests/src/cocotest/agnostic/primary_2.coco index 7ad4819a0..9c589e99e 100644 --- a/coconut/tests/src/cocotest/agnostic/primary_2.coco +++ b/coconut/tests/src/cocotest/agnostic/primary_2.coco @@ -369,12 +369,12 @@ def primary_test_2() -> bool: py_xs = py_zip([1, 2], [3, 4]) assert list(xs) == [(1, 3), (2, 4)] == list(xs) assert list(py_xs) == [(1, 3), (2, 4)] - assert list(py_xs) == [] + assert list(py_xs) == [] if sys.version_info >= (3,) else [(1, 3), (2, 4)] xs = map((+), [1, 2], [3, 4]) py_xs = py_map((+), [1, 2], [3, 4]) assert list(xs) == [4, 6] == list(xs) assert list(py_xs) == [4, 6] - assert list(py_xs) == [] + assert list(py_xs) == [] if sys.version_info >= (3,) else [4, 6] for xs in [ zip((x for x in range(5)), (x for x in range(10))), py_zip((x for x in range(5)), (x for x in range(10))), From 27b8c7ab25e2b838a6579cc2cfc7221a1077a6a9 Mon Sep 17 00:00:00 2001 From: Evan Hubinger Date: Sun, 5 Nov 2023 20:59:05 -0800 Subject: [PATCH 064/121] Fix more tests --- coconut/_pyparsing.py | 4 ++-- coconut/tests/src/cocotest/agnostic/primary_2.coco | 8 ++++++-- 2 files changed, 8 insertions(+), 4 deletions(-) diff --git a/coconut/_pyparsing.py b/coconut/_pyparsing.py index 6f290d3cb..54790c18b 100644 --- a/coconut/_pyparsing.py +++ b/coconut/_pyparsing.py @@ -28,7 +28,6 @@ from collections import defaultdict from coconut.constants import ( - PYPY, PURE_PYTHON, use_fast_pyparsing_reprs, use_packrat_parser, @@ -187,7 +186,8 @@ def enableIncremental(*args, **kwargs): use_computation_graph_env_var, default=( not MODERN_PYPARSING # not yet supported - and not PYPY # experimentally determined + # commented out to minimize memory footprint when running tests: + # and not PYPY # experimentally determined ), ) diff --git a/coconut/tests/src/cocotest/agnostic/primary_2.coco b/coconut/tests/src/cocotest/agnostic/primary_2.coco index 9c589e99e..3f9d63a65 100644 --- a/coconut/tests/src/cocotest/agnostic/primary_2.coco +++ b/coconut/tests/src/cocotest/agnostic/primary_2.coco @@ -377,12 +377,16 @@ def primary_test_2() -> bool: assert list(py_xs) == [] if sys.version_info >= (3,) else [4, 6] for xs in [ zip((x for x in range(5)), (x for x in range(10))), - py_zip((x for x in range(5)), (x for x in range(10))), map((,), (x for x in range(5)), (x for x in range(10))), - py_map((,), (x for x in range(5)), (x for x in range(10))), ]: # type: ignore assert list(xs) == list(zip(range(5), range(5))) assert list(xs) == [] + for xs in [ + py_zip((x for x in range(5)), (x for x in range(10))), + py_map((,), (x for x in range(5)), (x for x in range(10))), + ]: # type: ignore + assert list(xs) == list(zip(range(5), range(5))) + assert list(xs) == [] if sys.version_info >= (3,) else list(zip(range(5), range(5))) xs = map((.+1), range(5)) py_xs = py_map((.+1), range(5)) assert list(xs) == list(range(1, 6)) == list(xs) From 143fa614d6efb95e21c831a0a8cdd669f64d2c21 Mon Sep 17 00:00:00 2001 From: Evan Hubinger Date: Mon, 6 Nov 2023 00:55:41 -0800 Subject: [PATCH 065/121] Improve pyparsing usage --- Makefile | 12 ++++++------ coconut/_pyparsing.py | 13 +++++++------ 2 files changed, 13 insertions(+), 12 deletions(-) diff --git a/Makefile b/Makefile index 24481ed4d..e59803d2c 100644 --- a/Makefile +++ b/Makefile @@ -213,9 +213,9 @@ test-easter-eggs: clean python ./coconut/tests/dest/extras.py # same as test-univ but uses python pyparsing -.PHONY: test-pyparsing -test-pyparsing: export COCONUT_PURE_PYTHON=TRUE -test-pyparsing: test-univ +.PHONY: test-purepy +test-purepy: export COCONUT_PURE_PYTHON=TRUE +test-purepy: test-univ # same as test-univ but disables the computation graph .PHONY: test-no-computation-graph @@ -264,9 +264,9 @@ test-mini-debug: python -X dev -m coconut ./coconut/tests/src/cocotest/agnostic ./coconut/tests/dest/cocotest --strict --keep-lines --force --jobs 0 --stack-size 4096 --recursion-limit 4096 # same as test-mini-debug but uses vanilla pyparsing -.PHONY: test-mini-debug-pyparsing -test-mini-debug-pyparsing: export COCONUT_PURE_PYTHON=TRUE -test-mini-debug-pyparsing: test-mini-debug +.PHONY: test-mini-debug-purepy +test-mini-debug-purepy: export COCONUT_PURE_PYTHON=TRUE +test-mini-debug-purepy: test-mini-debug .PHONY: debug-test-crash debug-test-crash: diff --git a/coconut/_pyparsing.py b/coconut/_pyparsing.py index 54790c18b..6fbf6a4b0 100644 --- a/coconut/_pyparsing.py +++ b/coconut/_pyparsing.py @@ -20,6 +20,7 @@ from coconut.root import * # NOQA import os +import re import sys import traceback import functools @@ -220,13 +221,13 @@ def enableIncremental(*args, **kwargs): # MISSING OBJECTS: # ----------------------------------------------------------------------------------------------------------------------- -if not hasattr(_pyparsing, "python_quoted_string"): - import re as _re +python_quoted_string = getattr(_pyparsing, "python_quoted_string", None) +if python_quoted_string is None: python_quoted_string = _pyparsing.Combine( - (_pyparsing.Regex(r'"""(?:[^"\\]|""(?!")|"(?!"")|\\.)*', flags=_re.MULTILINE) + '"""').setName("multiline double quoted string") - ^ (_pyparsing.Regex(r"'''(?:[^'\\]|''(?!')|'(?!'')|\\.)*", flags=_re.MULTILINE) + "'''").setName("multiline single quoted string") - ^ (_pyparsing.Regex(r'"(?:[^"\n\r\\]|(?:\\")|(?:\\(?:[^x]|x[0-9a-fA-F]+)))*') + '"').setName("double quoted string") - ^ (_pyparsing.Regex(r"'(?:[^'\n\r\\]|(?:\\')|(?:\\(?:[^x]|x[0-9a-fA-F]+)))*") + "'").setName("single quoted string") + (_pyparsing.Regex(r'"""(?:[^"\\]|""(?!")|"(?!"")|\\.)*', flags=re.MULTILINE) + '"""').setName("multiline double quoted string") + | (_pyparsing.Regex(r"'''(?:[^'\\]|''(?!')|'(?!'')|\\.)*", flags=re.MULTILINE) + "'''").setName("multiline single quoted string") + | (_pyparsing.Regex(r'"(?:[^"\n\r\\]|(?:\\")|(?:\\(?:[^x]|x[0-9a-fA-F]+)))*') + '"').setName("double quoted string") + | (_pyparsing.Regex(r"'(?:[^'\n\r\\]|(?:\\')|(?:\\(?:[^x]|x[0-9a-fA-F]+)))*") + "'").setName("single quoted string") ).setName("Python quoted string") _pyparsing.python_quoted_string = python_quoted_string From 0e6f193e14e336420f38ae7b462a9d9c915b7d4b Mon Sep 17 00:00:00 2001 From: Evan Hubinger Date: Mon, 6 Nov 2023 02:14:47 -0800 Subject: [PATCH 066/121] Add more profiling --- Makefile | 8 ++-- coconut/_pyparsing.py | 93 +++++++++++++++++++++++++++++++++++-- coconut/command/command.py | 13 ++++-- coconut/compiler/grammar.py | 4 +- 4 files changed, 103 insertions(+), 15 deletions(-) diff --git a/Makefile b/Makefile index e59803d2c..af318d0bf 100644 --- a/Makefile +++ b/Makefile @@ -331,10 +331,10 @@ upload: wipe dev just-upload check-reqs: python ./coconut/requirements.py -.PHONY: profile-parser -profile-parser: export COCONUT_USE_COLOR=TRUE -profile-parser: export COCONUT_PURE_PYTHON=TRUE -profile-parser: +.PHONY: profile +profile: export COCONUT_USE_COLOR=TRUE +profile: export COCONUT_PURE_PYTHON=TRUE +profile: coconut ./coconut/tests/src/cocotest/agnostic ./coconut/tests/dest/cocotest --force --jobs 0 --profile --verbose --stack-size 4096 --recursion-limit 4096 2>&1 | tee ./profile.log .PHONY: open-speedscope diff --git a/coconut/_pyparsing.py b/coconut/_pyparsing.py index 6fbf6a4b0..3863ca7d1 100644 --- a/coconut/_pyparsing.py +++ b/coconut/_pyparsing.py @@ -287,7 +287,10 @@ def add_timing_to_method(cls, method_name, method): It's a monstrosity, but it's only used for profiling.""" from coconut.terminal import internal_assert # hide to avoid circular import - args, varargs, keywords, defaults = inspect.getargspec(method) + if hasattr(inspect, "getargspec"): + args, varargs, varkw, defaults = inspect.getargspec(method) + else: + args, varargs, varkw, defaults, kwonlyargs, kwonlydefaults, annotations = inspect.getfullargspec(method) internal_assert(args[:1] == ["self"], "cannot add timing to method", method_name) if not defaults: @@ -316,9 +319,9 @@ def add_timing_to_method(cls, method_name, method): if varargs: def_args.append("*" + varargs) call_args.append("*" + varargs) - if keywords: - def_args.append("**" + keywords) - call_args.append("**" + keywords) + if varkw: + def_args.append("**" + varkw) + call_args.append("**" + varkw) new_method_name = "new_" + method_name + "_func" _exec_dict = globals().copy() @@ -391,6 +394,7 @@ def collect_timing_info(): added_timing |= add_timing_to_method(obj, attr_name, attr) if added_timing: logger.log("\tadded timing to", obj) + return _timing_info def print_timing_info(): @@ -408,3 +412,84 @@ def print_timing_info(): sorted_timing_info = sorted(_timing_info[0].items(), key=lambda kv: kv[1]) for method_name, total_time in sorted_timing_info: print("{method_name}:\t{total_time}".format(method_name=method_name, total_time=total_time)) + + +_profiled_MatchFirst_objs = {} + + +def add_profiling_to_MatchFirsts(): + """Add profiling to MatchFirst objects to look for possible reorderings.""" + + def new_parseImpl(self, instring, loc, doActions=True): + if id(self) not in _profiled_MatchFirst_objs: + _profiled_MatchFirst_objs[id(self)] = self + self.expr_usage_stats = [0] * len(self.exprs) + self.expr_timing_stats = [[] for _ in range(len(self.exprs))] + maxExcLoc = -1 + maxException = None + for i, e in enumerate(self.exprs): + try: + start_time = get_clock_time() + try: + ret = e._parse(instring, loc, doActions) + finally: + self.expr_timing_stats[i].append(get_clock_time() - start_time) + self.expr_usage_stats[i] += 1 + return ret + except _pyparsing.ParseException as err: + if err.loc > maxExcLoc: + maxException = err + maxExcLoc = err.loc + except IndexError: + if len(instring) > maxExcLoc: + maxException = _pyparsing.ParseException(instring, len(instring), e.errmsg, self) + maxExcLoc = len(instring) + else: + if maxException is not None: + maxException.msg = self.errmsg + raise maxException + else: + raise _pyparsing.ParseException(instring, loc, "no defined alternatives to match", self) + _pyparsing.MatchFirst.parseImpl = new_parseImpl + return _profiled_MatchFirst_objs + + +def time_for_ordering(expr_usage_stats, expr_timing_aves): + """Get the total time for a given MatchFirst ordering.""" + total_time = 0 + for i, n in enumerate(expr_usage_stats): + total_time += n * sum(expr_timing_aves[:i + 1]) + return total_time + + +def naive_timing_improvement(expr_usage_stats, expr_timing_aves): + """Get the expected timing improvement for a better MatchFirst ordering.""" + usage_ordered_expr_usage_stats, usage_ordered_expr_timing_aves = zip(*sorted( + zip(expr_usage_stats, expr_timing_aves), + reverse=True, + )) + return time_for_ordering(usage_ordered_expr_usage_stats, usage_ordered_expr_timing_aves) - time_for_ordering(expr_usage_stats, expr_timing_aves) + + +def print_poorly_ordered_MatchFirsts(): + """Print poorly ordered MatchFirsts.""" + for obj in _profiled_MatchFirst_objs.values(): + obj.expr_timing_aves = [sum(ts) / len(ts) if ts else 0 for ts in obj.expr_timing_stats] + obj.naive_timing_improvement = naive_timing_improvement(obj.expr_usage_stats, obj.expr_timing_aves) + most_improveable = sorted(_profiled_MatchFirst_objs.values(), key=lambda obj: obj.naive_timing_improvement)[-100:] + for obj in most_improveable: + print(obj, ":", obj.naive_timing_improvement) + print("\t" + repr(obj.expr_usage_stats)) + print("\t" + repr(obj.expr_timing_aves)) + + +def start_profiling(): + """Do all the setup to begin profiling.""" + collect_timing_info() + add_profiling_to_MatchFirsts() + + +def print_profiling_results(): + """Print all profiling results.""" + print_timing_info() + print_poorly_ordered_MatchFirsts() diff --git a/coconut/command/command.py b/coconut/command/command.py index 5c0aafd32..b053dc418 100644 --- a/coconut/command/command.py +++ b/coconut/command/command.py @@ -29,8 +29,8 @@ from coconut._pyparsing import ( unset_fast_pyparsing_reprs, - collect_timing_info, - print_timing_info, + start_profiling, + print_profiling_results, SUPPORTS_INCREMENTAL, ) @@ -249,7 +249,7 @@ def execute_args(self, args, interact=True, original_args=None): if args.trace or args.profile: unset_fast_pyparsing_reprs() if args.profile: - collect_timing_info() + start_profiling() logger.enable_colors() logger.log(cli_version) @@ -358,7 +358,10 @@ def execute_args(self, args, interact=True, original_args=None): self.disable_jobs() # do compilation - with self.running_jobs(exit_on_error=not args.watch): + with self.running_jobs(exit_on_error=not ( + args.watch + or args.profile + )): for source, dest, package in src_dest_package_triples: filepaths += self.compile_path(source, dest, package, run=args.run or args.interact, force=args.force) self.run_mypy(filepaths) @@ -406,7 +409,7 @@ def execute_args(self, args, interact=True, original_args=None): # src_dest_package_triples is always available here self.watch(src_dest_package_triples, args.run, args.force) if args.profile: - print_timing_info() + print_profiling_results() # make sure to return inside handling_exceptions to ensure filepaths is available return filepaths diff --git a/coconut/compiler/grammar.py b/coconut/compiler/grammar.py index 0abed4963..9a857a4dc 100644 --- a/coconut/compiler/grammar.py +++ b/coconut/compiler/grammar.py @@ -1122,9 +1122,9 @@ class Grammar(object): )) call_item = ( - dubstar + test + unsafe_name + default + | dubstar + test | star + test - | unsafe_name + default | ellipsis_tokens + equals.suppress() + refname | namedexpr_test ) From b001630ded686463ad34e4d781526d91d6155021 Mon Sep 17 00:00:00 2001 From: Evan Hubinger Date: Mon, 6 Nov 2023 02:22:14 -0800 Subject: [PATCH 067/121] Clean up profiling --- coconut/_pyparsing.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/coconut/_pyparsing.py b/coconut/_pyparsing.py index 3863ca7d1..a65faa5e2 100644 --- a/coconut/_pyparsing.py +++ b/coconut/_pyparsing.py @@ -450,6 +450,7 @@ def new_parseImpl(self, instring, loc, doActions=True): raise maxException else: raise _pyparsing.ParseException(instring, loc, "no defined alternatives to match", self) + _pyparsing.MatchFirst.parseImpl = new_parseImpl return _profiled_MatchFirst_objs @@ -485,8 +486,8 @@ def print_poorly_ordered_MatchFirsts(): def start_profiling(): """Do all the setup to begin profiling.""" - collect_timing_info() add_profiling_to_MatchFirsts() + collect_timing_info() def print_profiling_results(): From 78542b8d23410ccc1107bbbd30f36213b3f1b709 Mon Sep 17 00:00:00 2001 From: Evan Hubinger Date: Tue, 7 Nov 2023 01:50:15 -0800 Subject: [PATCH 068/121] Improve --profile --- Makefile | 3 +- coconut/_pyparsing.py | 142 ++++++++---------- coconut/compiler/grammar.py | 2 +- coconut/compiler/templates/header.py_template | 24 ++- coconut/constants.py | 2 + 5 files changed, 78 insertions(+), 95 deletions(-) diff --git a/Makefile b/Makefile index af318d0bf..903f017f8 100644 --- a/Makefile +++ b/Makefile @@ -333,9 +333,8 @@ check-reqs: .PHONY: profile profile: export COCONUT_USE_COLOR=TRUE -profile: export COCONUT_PURE_PYTHON=TRUE profile: - coconut ./coconut/tests/src/cocotest/agnostic ./coconut/tests/dest/cocotest --force --jobs 0 --profile --verbose --stack-size 4096 --recursion-limit 4096 2>&1 | tee ./profile.log + coconut ./coconut/tests/src/cocotest/agnostic/util.coco ./coconut/tests/dest/cocotest --force --jobs 0 --profile --verbose --stack-size 4096 --recursion-limit 4096 2>&1 | tee ./profile.log .PHONY: open-speedscope open-speedscope: diff --git a/coconut/_pyparsing.py b/coconut/_pyparsing.py index a65faa5e2..94895a1b4 100644 --- a/coconut/_pyparsing.py +++ b/coconut/_pyparsing.py @@ -24,9 +24,11 @@ import sys import traceback import functools -import inspect from warnings import warn from collections import defaultdict +from itertools import permutations +from functools import wraps +from pprint import pprint from coconut.constants import ( PURE_PYTHON, @@ -45,6 +47,7 @@ default_incremental_cache_size, never_clear_incremental_cache, warn_on_multiline_regex, + num_displayed_timing_items, ) from coconut.util import get_clock_time # NOQA from coconut.util import ( @@ -285,69 +288,15 @@ class _timing_sentinel(object): def add_timing_to_method(cls, method_name, method): """Add timing collection to the given method. It's a monstrosity, but it's only used for profiling.""" - from coconut.terminal import internal_assert # hide to avoid circular import - - if hasattr(inspect, "getargspec"): - args, varargs, varkw, defaults = inspect.getargspec(method) - else: - args, varargs, varkw, defaults, kwonlyargs, kwonlydefaults, annotations = inspect.getfullargspec(method) - internal_assert(args[:1] == ["self"], "cannot add timing to method", method_name) - - if not defaults: - defaults = [] - num_undefaulted_args = len(args) - len(defaults) - def_args = [] - call_args = [] - fix_arg_defaults = [] - defaults_dict = {} - for i, arg in enumerate(args): - if i >= num_undefaulted_args: - default = defaults[i - num_undefaulted_args] - def_args.append(arg + "=_timing_sentinel") - defaults_dict[arg] = default - fix_arg_defaults.append( - """ - if {arg} is _timing_sentinel: - {arg} = _exec_dict["defaults_dict"]["{arg}"] -""".strip("\n").format( - arg=arg, - ), - ) - else: - def_args.append(arg) - call_args.append(arg) - if varargs: - def_args.append("*" + varargs) - call_args.append("*" + varargs) - if varkw: - def_args.append("**" + varkw) - call_args.append("**" + varkw) - - new_method_name = "new_" + method_name + "_func" - _exec_dict = globals().copy() - _exec_dict.update(locals()) - new_method_code = """ -def {new_method_name}({def_args}): -{fix_arg_defaults} - - _all_args = (lambda *args, **kwargs: args + tuple(kwargs.values()))({call_args}) - _exec_dict["internal_assert"](not any(_arg is _timing_sentinel for _arg in _all_args), "error handling arguments in timed method {new_method_name}({def_args}); got", _all_args) - - _start_time = _exec_dict["get_clock_time"]() - try: - return _exec_dict["method"]({call_args}) - finally: - _timing_info[0][str(self)] += _exec_dict["get_clock_time"]() - _start_time -{new_method_name}._timed = True - """.format( - fix_arg_defaults="\n".join(fix_arg_defaults), - new_method_name=new_method_name, - def_args=", ".join(def_args), - call_args=", ".join(call_args), - ) - exec(new_method_code, _exec_dict) - - setattr(cls, method_name, _exec_dict[new_method_name]) + @wraps(method) + def new_method(self, *args, **kwargs): + start_time = get_clock_time() + try: + return method(self, *args, **kwargs) + finally: + _timing_info[0][ascii(self)] += get_clock_time() - start_time + new_method._timed = True + setattr(cls, method_name, new_method) return True @@ -409,7 +358,7 @@ def print_timing_info(): num=len(_timing_info[0]), ), ) - sorted_timing_info = sorted(_timing_info[0].items(), key=lambda kv: kv[1]) + sorted_timing_info = sorted(_timing_info[0].items(), key=lambda kv: kv[1])[-num_displayed_timing_items:] for method_name, total_time in sorted_timing_info: print("{method_name}:\t{total_time}".format(method_name=method_name, total_time=total_time)) @@ -420,11 +369,15 @@ def print_timing_info(): def add_profiling_to_MatchFirsts(): """Add profiling to MatchFirst objects to look for possible reorderings.""" + @wraps(MatchFirst.parseImpl) def new_parseImpl(self, instring, loc, doActions=True): if id(self) not in _profiled_MatchFirst_objs: _profiled_MatchFirst_objs[id(self)] = self - self.expr_usage_stats = [0] * len(self.exprs) - self.expr_timing_stats = [[] for _ in range(len(self.exprs))] + self.expr_usage_stats = [] + self.expr_timing_stats = [] + while len(self.expr_usage_stats) < len(self.exprs): + self.expr_usage_stats.append(0) + self.expr_timing_stats.append([]) maxExcLoc = -1 maxException = None for i, e in enumerate(self.exprs): @@ -463,25 +416,58 @@ def time_for_ordering(expr_usage_stats, expr_timing_aves): return total_time -def naive_timing_improvement(expr_usage_stats, expr_timing_aves): +def find_best_ordering(obj, num_perms_to_eval=None): + """Get the best ordering of the MatchFirst.""" + if num_perms_to_eval is None: + num_perms_to_eval = True if len(obj.exprs) <= 10 else 100000 + best_exprs = None + best_time = float("inf") + stats_zip = tuple(zip(obj.expr_usage_stats, obj.expr_timing_aves, obj.exprs)) + if num_perms_to_eval is True: + perms_to_eval = permutations(stats_zip) + else: + perms_to_eval = [ + stats_zip, + sorted(stats_zip, key=lambda u_t_e: (-u_t_e[0], u_t_e[1])), + sorted(stats_zip, key=lambda u_t_e: (u_t_e[1], -u_t_e[0])), + ] + if num_perms_to_eval: + max_usage = max(obj.expr_usage_stats) + max_time = max(obj.expr_timing_aves) + for i in range(1, num_perms_to_eval): + a = i / num_perms_to_eval + perms_to_eval.append(sorted( + stats_zip, + key=lambda u_t_e: + -a * u_t_e[0] / max_usage + + (1 - a) * u_t_e[1] / max_time, + )) + for perm in perms_to_eval: + perm_expr_usage_stats, perm_expr_timing_aves = zip(*[(usage, timing) for usage, timing, expr in perm]) + perm_time = time_for_ordering(perm_expr_usage_stats, perm_expr_timing_aves) + if perm_time < best_time: + best_time = perm_time + best_exprs = [expr for usage, timing, expr in perm] + return best_exprs, best_time + + +def naive_timing_improvement(obj): """Get the expected timing improvement for a better MatchFirst ordering.""" - usage_ordered_expr_usage_stats, usage_ordered_expr_timing_aves = zip(*sorted( - zip(expr_usage_stats, expr_timing_aves), - reverse=True, - )) - return time_for_ordering(usage_ordered_expr_usage_stats, usage_ordered_expr_timing_aves) - time_for_ordering(expr_usage_stats, expr_timing_aves) + _, best_time = find_best_ordering(obj, num_perms_to_eval=False) + return time_for_ordering(obj.expr_usage_stats, obj.expr_timing_aves) - best_time def print_poorly_ordered_MatchFirsts(): """Print poorly ordered MatchFirsts.""" for obj in _profiled_MatchFirst_objs.values(): obj.expr_timing_aves = [sum(ts) / len(ts) if ts else 0 for ts in obj.expr_timing_stats] - obj.naive_timing_improvement = naive_timing_improvement(obj.expr_usage_stats, obj.expr_timing_aves) - most_improveable = sorted(_profiled_MatchFirst_objs.values(), key=lambda obj: obj.naive_timing_improvement)[-100:] + obj.naive_timing_improvement = naive_timing_improvement(obj) + most_improveable = sorted(_profiled_MatchFirst_objs.values(), key=lambda obj: obj.naive_timing_improvement)[-num_displayed_timing_items:] for obj in most_improveable: - print(obj, ":", obj.naive_timing_improvement) - print("\t" + repr(obj.expr_usage_stats)) - print("\t" + repr(obj.expr_timing_aves)) + print(ascii(obj), ":", obj.naive_timing_improvement) + pprint(list(zip(obj.exprs, obj.expr_usage_stats, obj.expr_timing_aves))) + best_ordering, best_time = find_best_ordering(obj) + print("\tbest (" + str(best_time) + "):", ascii(best_ordering)) def start_profiling(): diff --git a/coconut/compiler/grammar.py b/coconut/compiler/grammar.py index 9a857a4dc..600fbb0df 100644 --- a/coconut/compiler/grammar.py +++ b/coconut/compiler/grammar.py @@ -1290,7 +1290,7 @@ class Grammar(object): ) + ~questionmark partial_trailer_tokens = Group(dollar.suppress() + function_call_tokens) - no_call_trailer = simple_trailer | partial_trailer | known_trailer + no_call_trailer = simple_trailer | known_trailer | partial_trailer no_partial_complex_trailer = call_trailer | known_trailer no_partial_trailer = simple_trailer | no_partial_complex_trailer diff --git a/coconut/compiler/templates/header.py_template b/coconut/compiler/templates/header.py_template index 3a742eee9..0e7a698fd 100644 --- a/coconut/compiler/templates/header.py_template +++ b/coconut/compiler/templates/header.py_template @@ -216,16 +216,14 @@ def tee(iterable, n=2): return _coconut.tuple(existing_copies) return _coconut.itertools.tee(iterable, n) class _coconut_has_iter(_coconut_baseclass): - __slots__ = ("lock", "iter") + __slots__ = ("iter",) def __new__(cls, iterable): self = _coconut.super(_coconut_has_iter, cls).__new__(cls) - self.lock = _coconut.threading.Lock() self.iter = iterable return self def get_new_iter(self): """Tee the underlying iterator.""" - with self.lock: - self.iter = {_coconut_}reiterable(self.iter) + self.iter = {_coconut_}reiterable(self.iter) return self.iter def __fmap__(self, func): return {_coconut_}map(func, self) @@ -238,8 +236,7 @@ class reiterable(_coconut_has_iter): return _coconut.super({_coconut_}reiterable, cls).__new__(cls, iterable) def get_new_iter(self): """Tee the underlying iterator.""" - with self.lock: - self.iter, new_iter = {_coconut_}tee(self.iter) + self.iter, new_iter = {_coconut_}tee(self.iter) return new_iter def __iter__(self): return _coconut.iter(self.get_new_iter()) @@ -674,14 +671,13 @@ class flatten(_coconut_has_iter):{COMMENT.cant_implement_len_else_list_calls_bec return self def get_new_iter(self): """Tee the underlying iterator.""" - with self.lock: - if not self._made_reit: - for i in _coconut.reversed(_coconut.range(0 if self.levels is None else self.levels + 1)): - mapper = {_coconut_}reiterable - for _ in _coconut.range(i): - mapper = _coconut.functools.partial({_coconut_}map, mapper) - self.iter = mapper(self.iter) - self._made_reit = True + if not self._made_reit: + for i in _coconut.reversed(_coconut.range(0 if self.levels is None else self.levels + 1)): + mapper = {_coconut_}reiterable + for _ in _coconut.range(i): + mapper = _coconut.functools.partial({_coconut_}map, mapper) + self.iter = mapper(self.iter) + self._made_reit = True return self.iter def __iter__(self): if self.levels is None: diff --git a/coconut/constants.py b/coconut/constants.py index 6b6d7f9b0..fceabdfdf 100644 --- a/coconut/constants.py +++ b/coconut/constants.py @@ -117,6 +117,8 @@ def get_path_env_var(env_var, default): use_computation_graph_env_var = "COCONUT_USE_COMPUTATION_GRAPH" +num_displayed_timing_items = 100 + # below constants are experimentally determined to maximize performance streamline_grammar_for_len = 4096 From f496a4d98c0eb0222b8937d1fa9361f52bb72a91 Mon Sep 17 00:00:00 2001 From: Evan Hubinger Date: Tue, 7 Nov 2023 02:00:45 -0800 Subject: [PATCH 069/121] Further fix profiling --- coconut/_pyparsing.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/coconut/_pyparsing.py b/coconut/_pyparsing.py index 94895a1b4..ef606fd36 100644 --- a/coconut/_pyparsing.py +++ b/coconut/_pyparsing.py @@ -465,7 +465,7 @@ def print_poorly_ordered_MatchFirsts(): most_improveable = sorted(_profiled_MatchFirst_objs.values(), key=lambda obj: obj.naive_timing_improvement)[-num_displayed_timing_items:] for obj in most_improveable: print(ascii(obj), ":", obj.naive_timing_improvement) - pprint(list(zip(obj.exprs, obj.expr_usage_stats, obj.expr_timing_aves))) + pprint(list(zip(map(ascii, obj.exprs), obj.expr_usage_stats, obj.expr_timing_aves))) best_ordering, best_time = find_best_ordering(obj) print("\tbest (" + str(best_time) + "):", ascii(best_ordering)) From 42281fea8c5807989461a65031934b7cf68435b7 Mon Sep 17 00:00:00 2001 From: Evan Hubinger Date: Wed, 8 Nov 2023 00:28:58 -0800 Subject: [PATCH 070/121] More profiling improvements --- DOCS.md | 4 ++-- coconut/_pyparsing.py | 18 ++++++++++++------ coconut/api.pyi | 2 +- coconut/command/command.py | 10 +++++----- coconut/compiler/compiler.py | 8 +++++--- coconut/compiler/grammar.py | 18 +++++++++--------- 6 files changed, 34 insertions(+), 26 deletions(-) diff --git a/DOCS.md b/DOCS.md index 8a484efa0..685ecb0f2 100644 --- a/DOCS.md +++ b/DOCS.md @@ -4660,9 +4660,9 @@ If _state_ is `False`, the global state object is used. #### `warm_up` -**coconut.api.warm_up**(_force_=`True`, _enable\_incremental\_mode_=`False`, *, _state_=`False`) +**coconut.api.warm_up**(_streamline_=`True`, _enable\_incremental\_mode_=`False`, *, _state_=`False`) -Can optionally be called to warm up the compiler and get it ready for parsing. Passing _force_ will cause the warm up to take longer but will substantially reduce parsing times (by default, this level of warm up is only done when the compiler encounters a large file). Passing _enable\_incremental\_mode_ will enable the compiler's incremental mdoe, where parsing some string, then later parsing a continuation of that string, will yield substantial performance improvements. +Can optionally be called to warm up the compiler and get it ready for parsing. Passing _streamline_ will cause the warm up to take longer but will substantially reduce parsing times (by default, this level of warm up is only done when the compiler encounters a large file). Passing _enable\_incremental\_mode_ will enable the compiler's incremental mdoe, where parsing some string, then later parsing a continuation of that string, will yield substantial performance improvements. #### `cmd` diff --git a/coconut/_pyparsing.py b/coconut/_pyparsing.py index ef606fd36..6921a099c 100644 --- a/coconut/_pyparsing.py +++ b/coconut/_pyparsing.py @@ -420,7 +420,7 @@ def find_best_ordering(obj, num_perms_to_eval=None): """Get the best ordering of the MatchFirst.""" if num_perms_to_eval is None: num_perms_to_eval = True if len(obj.exprs) <= 10 else 100000 - best_exprs = None + best_ordering = None best_time = float("inf") stats_zip = tuple(zip(obj.expr_usage_stats, obj.expr_timing_aves, obj.exprs)) if num_perms_to_eval is True: @@ -447,8 +447,8 @@ def find_best_ordering(obj, num_perms_to_eval=None): perm_time = time_for_ordering(perm_expr_usage_stats, perm_expr_timing_aves) if perm_time < best_time: best_time = perm_time - best_exprs = [expr for usage, timing, expr in perm] - return best_exprs, best_time + best_ordering = [(obj.exprs.index(expr), parse_expr_repr(expr)) for usage, timing, expr in perm] + return best_ordering, best_time def naive_timing_improvement(obj): @@ -457,6 +457,11 @@ def naive_timing_improvement(obj): return time_for_ordering(obj.expr_usage_stats, obj.expr_timing_aves) - best_time +def parse_expr_repr(obj): + """Get a clean repr of a parse expression for displaying.""" + return getattr(obj, "name", None) or ascii(obj) + + def print_poorly_ordered_MatchFirsts(): """Print poorly ordered MatchFirsts.""" for obj in _profiled_MatchFirst_objs.values(): @@ -464,10 +469,11 @@ def print_poorly_ordered_MatchFirsts(): obj.naive_timing_improvement = naive_timing_improvement(obj) most_improveable = sorted(_profiled_MatchFirst_objs.values(), key=lambda obj: obj.naive_timing_improvement)[-num_displayed_timing_items:] for obj in most_improveable: - print(ascii(obj), ":", obj.naive_timing_improvement) - pprint(list(zip(map(ascii, obj.exprs), obj.expr_usage_stats, obj.expr_timing_aves))) + print("\n" + parse_expr_repr(obj), "(" + str(obj.naive_timing_improvement) + "):") + pprint(list(zip(map(parse_expr_repr, obj.exprs), obj.expr_usage_stats, obj.expr_timing_aves))) best_ordering, best_time = find_best_ordering(obj) - print("\tbest (" + str(best_time) + "):", ascii(best_ordering)) + print("\tbest (" + str(best_time) + "):") + pprint(best_ordering) def start_profiling(): diff --git a/coconut/api.pyi b/coconut/api.pyi index 511831c60..27210efa3 100644 --- a/coconut/api.pyi +++ b/coconut/api.pyi @@ -85,7 +85,7 @@ def setup( def warm_up( - force: bool = False, + streamline: bool = False, enable_incremental_mode: bool = False, *, state: Optional[Command] = ..., diff --git a/coconut/command/command.py b/coconut/command/command.py index b053dc418..69e713641 100644 --- a/coconut/command/command.py +++ b/coconut/command/command.py @@ -111,7 +111,6 @@ get_target_info_smart, ) from coconut.compiler.header import gethash -from coconut.compiler.grammar import set_grammar_names from coconut.command.cli import arguments, cli_version # ----------------------------------------------------------------------------------------------------------------------- @@ -244,8 +243,6 @@ def execute_args(self, args, interact=True, original_args=None): verbose=args.verbose, tracing=args.trace, ) - if args.verbose or args.trace or args.profile: - set_grammar_names() if args.trace or args.profile: unset_fast_pyparsing_reprs() if args.profile: @@ -318,8 +315,11 @@ def execute_args(self, args, interact=True, original_args=None): no_tco=args.no_tco, no_wrap=args.no_wrap_types, ) - if args.watch: - self.comp.warm_up(enable_incremental_mode=True) + self.comp.warm_up( + streamline=args.watch or args.profile, + enable_incremental_mode=args.watch, + set_debug_names=args.verbose or args.trace or args.profile, + ) # process mypy args and print timing info (must come after compiler setup) if args.mypy is not None: diff --git a/coconut/compiler/compiler.py b/coconut/compiler/compiler.py index fee1d3d42..551462807 100644 --- a/coconut/compiler/compiler.py +++ b/coconut/compiler/compiler.py @@ -4809,10 +4809,12 @@ def parse_xonsh(self, inputstring, **kwargs): """Parse xonsh code.""" return self.parse(inputstring, self.xonsh_parser, {"strip": True}, {"header": "none", "initial": "none"}, streamline=False, **kwargs) - def warm_up(self, force=False, enable_incremental_mode=False): + def warm_up(self, streamline=False, enable_incremental_mode=False, set_debug_names=False): """Warm up the compiler by streamlining the file_parser.""" - self.streamline(self.file_parser, force=force) - self.streamline(self.eval_parser, force=force) + if set_debug_names: + self.set_grammar_names() + self.streamline(self.file_parser, force=streamline) + self.streamline(self.eval_parser, force=streamline) if enable_incremental_mode: enable_incremental_parsing() diff --git a/coconut/compiler/grammar.py b/coconut/compiler/grammar.py index 600fbb0df..cc02add8c 100644 --- a/coconut/compiler/grammar.py +++ b/coconut/compiler/grammar.py @@ -1476,8 +1476,8 @@ class Grammar(object): comp_pipe_handle, ) comp_pipe_expr = ( - comp_pipe_item - | none_coalesce_expr + none_coalesce_expr + ~comp_pipe_op + | comp_pipe_item ) pipe_op = ( @@ -2308,8 +2308,8 @@ class Grammar(object): compound_stmt = ( decoratable_class_stmt | decoratable_func_stmt - | for_stmt | while_stmt + | for_stmt | with_stmt | async_stmt | match_for_stmt @@ -2567,12 +2567,12 @@ def add_to_grammar_init_time(cls): finally: cls.grammar_init_time += get_clock_time() - start_time - -def set_grammar_names(): - """Set names of grammar elements to their variable names.""" - for varname, val in vars(Grammar).items(): - if isinstance(val, ParserElement): - val.setName(varname) + @staticmethod + def set_grammar_names(): + """Set names of grammar elements to their variable names.""" + for varname, val in vars(Grammar).items(): + if isinstance(val, ParserElement): + val.setName(varname) # end: TRACING From e5bfd254ae72254fb67adfc3d9862e6218f6342a Mon Sep 17 00:00:00 2001 From: Evan Hubinger Date: Wed, 8 Nov 2023 01:59:10 -0800 Subject: [PATCH 071/121] Add adaptive parsing support --- Makefile | 2 +- coconut/_pyparsing.py | 7 ++++-- coconut/compiler/util.py | 47 +++++++++++++++++++++++++++++++++++----- coconut/constants.py | 3 +++ coconut/root.py | 2 +- coconut/terminal.py | 12 ++++++++++ 6 files changed, 64 insertions(+), 9 deletions(-) diff --git a/Makefile b/Makefile index 903f017f8..96b630508 100644 --- a/Makefile +++ b/Makefile @@ -156,7 +156,7 @@ test-mypy-tests: clean-no-tests python ./coconut/tests/dest/extras.py # same as test-univ but includes verbose output for better debugging -# regex for getting non-timing lines: ^(?!Time|\s+Packrat|Loaded|Saving)[^\n]* +# regex for getting non-timing lines: ^(?!Time|\s+Packrat|Loaded|Saving|Adaptive)[^\n]* .PHONY: test-verbose test-verbose: export COCONUT_USE_COLOR=TRUE test-verbose: clean diff --git a/coconut/_pyparsing.py b/coconut/_pyparsing.py index 6921a099c..e9830c828 100644 --- a/coconut/_pyparsing.py +++ b/coconut/_pyparsing.py @@ -48,6 +48,7 @@ never_clear_incremental_cache, warn_on_multiline_regex, num_displayed_timing_items, + use_adaptive_if_available, ) from coconut.util import get_clock_time # NOQA from coconut.util import ( @@ -174,6 +175,8 @@ def enableIncremental(*args, **kwargs): + " (run '{python} -m pip install --upgrade cPyparsing' to fix)".format(python=sys.executable) ) +USE_ADAPTIVE = hasattr(MatchFirst, "setAdaptiveMode") and use_adaptive_if_available + # ----------------------------------------------------------------------------------------------------------------------- # SETUP: @@ -459,7 +462,7 @@ def naive_timing_improvement(obj): def parse_expr_repr(obj): """Get a clean repr of a parse expression for displaying.""" - return getattr(obj, "name", None) or ascii(obj) + return ascii(getattr(obj, "name", None) or obj) def print_poorly_ordered_MatchFirsts(): @@ -469,7 +472,7 @@ def print_poorly_ordered_MatchFirsts(): obj.naive_timing_improvement = naive_timing_improvement(obj) most_improveable = sorted(_profiled_MatchFirst_objs.values(), key=lambda obj: obj.naive_timing_improvement)[-num_displayed_timing_items:] for obj in most_improveable: - print("\n" + parse_expr_repr(obj), "(" + str(obj.naive_timing_improvement) + "):") + print("\n" + parse_expr_repr(obj) + " (" + str(obj.naive_timing_improvement) + "):") pprint(list(zip(map(parse_expr_repr, obj.exprs), obj.expr_usage_stats, obj.expr_timing_aves))) best_ordering, best_time = find_best_ordering(obj) print("\tbest (" + str(best_time) + "):") diff --git a/coconut/compiler/util.py b/coconut/compiler/util.py index 1dfbe7d34..828bc7010 100644 --- a/coconut/compiler/util.py +++ b/coconut/compiler/util.py @@ -47,6 +47,7 @@ from coconut._pyparsing import ( USE_COMPUTATION_GRAPH, SUPPORTS_INCREMENTAL, + USE_ADAPTIVE, replaceWith, ZeroOrMore, OneOrMore, @@ -64,6 +65,7 @@ CaselessLiteral, Group, ParserElement, + MatchFirst, _trim_arity, _ParseResultsWithOffset, all_parse_elements, @@ -113,6 +115,7 @@ unwrapper, incremental_cache_limit, incremental_mode_cache_successes, + adaptive_reparse_usage_weight, ) from coconut.exceptions import ( CoconutException, @@ -362,8 +365,36 @@ def final_evaluate_tokens(tokens): return evaluate_tokens(tokens) +@contextmanager +def adaptive_manager(item, original, loc, reparse=False): + """Manage the use of MatchFirst.setAdaptiveMode.""" + if reparse: + item.include_in_packrat_context = True + MatchFirst.setAdaptiveMode(False, usage_weight=adaptive_reparse_usage_weight) + try: + yield + finally: + MatchFirst.setAdaptiveMode(False, usage_weight=1) + item.include_in_packrat_context = False + else: + MatchFirst.setAdaptiveMode(True) + try: + yield + except Exception as exc: + if DEVELOP: + logger.log("reparsing due to:", exc) + logger.record_adaptive_stat(False) + else: + if DEVELOP: + logger.record_adaptive_stat(True) + finally: + MatchFirst.setAdaptiveMode(False) + + def final(item): """Collapse the computation graph upon parsing the given item.""" + if USE_ADAPTIVE: + item = Wrap(item, adaptive_manager, greedy=True) # evaluate_tokens expects a computation graph, so we just call add_action directly return add_action(trace(item), final_evaluate_tokens) @@ -778,11 +809,17 @@ def parseImpl(self, original, loc, *args, **kwargs): if logger.tracing: # avoid the overhead of the call if not tracing logger.log_trace(self.wrapped_name, original, loc) with logger.indent_tracing(): - with self.wrapper(self, original, loc): - with self.wrapped_context(): - parse_loc, tokens = super(Wrap, self).parseImpl(original, loc, *args, **kwargs) - if self.greedy: - tokens = evaluate_tokens(tokens) + reparse = False + parse_loc = None + while parse_loc is None: # lets wrapper catch errors to trigger a reparse + with self.wrapper(self, original, loc, **(dict(reparse=True) if reparse else {})): + with self.wrapped_context(): + parse_loc, tokens = super(Wrap, self).parseImpl(original, loc, *args, **kwargs) + if self.greedy: + tokens = evaluate_tokens(tokens) + if reparse and parse_loc is None: + raise CoconutInternalException("illegal double reparse in", self) + reparse = True if logger.tracing: # avoid the overhead of the call if not tracing logger.log_trace(self.wrapped_name, original, loc, tokens) return parse_loc, tokens diff --git a/coconut/constants.py b/coconut/constants.py index fceabdfdf..199b96d57 100644 --- a/coconut/constants.py +++ b/coconut/constants.py @@ -130,6 +130,9 @@ def get_path_env_var(env_var, default): # note that _parseIncremental produces much smaller caches use_incremental_if_available = True +use_adaptive_if_available = True +adaptive_reparse_usage_weight = 10 + # these only apply to use_incremental_if_available, not compiler.util.enable_incremental_parsing() default_incremental_cache_size = None repeatedly_clear_incremental_cache = True diff --git a/coconut/root.py b/coconut/root.py index 3f7331836..c9bcc020b 100644 --- a/coconut/root.py +++ b/coconut/root.py @@ -26,7 +26,7 @@ VERSION = "3.0.3" VERSION_NAME = None # False for release, int >= 1 for develop -DEVELOP = 21 +DEVELOP = 22 ALPHA = False # for pre releases rather than post releases assert DEVELOP is False or DEVELOP >= 1, "DEVELOP must be False or an int >= 1" diff --git a/coconut/terminal.py b/coconut/terminal.py index 8a5f7cde0..c0fcf1809 100644 --- a/coconut/terminal.py +++ b/coconut/terminal.py @@ -522,10 +522,19 @@ def trace(self, item): item.debug = True return item + adaptive_stats = None + + def record_adaptive_stat(self, success): + if self.verbose: + if self.adaptive_stats is None: + self.adaptive_stats = [0, 0] + self.adaptive_stats[success] += 1 + @contextmanager def gather_parsing_stats(self): """Times parsing if --verbose.""" if self.verbose: + self.adaptive_stats = None start_time = get_clock_time() try: yield @@ -538,6 +547,9 @@ def gather_parsing_stats(self): # reset stats after printing if in incremental mode if ParserElement._incrementalEnabled: ParserElement.packrat_cache_stats[:] = [0] * len(ParserElement.packrat_cache_stats) + if self.adaptive_stats: + failures, successes = self.adaptive_stats + self.printlog("\tAdaptive parsing stats:", successes, "successes;", failures, "failures") else: yield From dd0ba2a77a86ceb81da68c3ea8b8795ea7eac183 Mon Sep 17 00:00:00 2001 From: Evan Hubinger Date: Wed, 8 Nov 2023 23:47:05 -0800 Subject: [PATCH 072/121] More adaptive improvements --- Makefile | 2 +- coconut/compiler/util.py | 64 ++++++++++++++++++++++++++-------------- coconut/constants.py | 2 +- 3 files changed, 44 insertions(+), 24 deletions(-) diff --git a/Makefile b/Makefile index 96b630508..aa9dd6d5c 100644 --- a/Makefile +++ b/Makefile @@ -156,7 +156,7 @@ test-mypy-tests: clean-no-tests python ./coconut/tests/dest/extras.py # same as test-univ but includes verbose output for better debugging -# regex for getting non-timing lines: ^(?!Time|\s+Packrat|Loaded|Saving|Adaptive)[^\n]* +# regex for getting non-timing lines: ^(?!Time|\s+Packrat|Loaded|Saving|Adaptive)[^\n]*\n* .PHONY: test-verbose test-verbose: export COCONUT_USE_COLOR=TRUE test-verbose: clean diff --git a/coconut/compiler/util.py b/coconut/compiler/util.py index 828bc7010..db97100e8 100644 --- a/coconut/compiler/util.py +++ b/coconut/compiler/util.py @@ -369,13 +369,16 @@ def final_evaluate_tokens(tokens): def adaptive_manager(item, original, loc, reparse=False): """Manage the use of MatchFirst.setAdaptiveMode.""" if reparse: - item.include_in_packrat_context = True + cleared_cache = clear_packrat_cache(force=True) + if cleared_cache is not True: + item.include_in_packrat_context = True MatchFirst.setAdaptiveMode(False, usage_weight=adaptive_reparse_usage_weight) try: yield finally: MatchFirst.setAdaptiveMode(False, usage_weight=1) - item.include_in_packrat_context = False + if cleared_cache is not True: + item.include_in_packrat_context = False else: MatchFirst.setAdaptiveMode(True) try: @@ -551,37 +554,43 @@ def get_pyparsing_cache(): return {} -def should_clear_cache(): +def should_clear_cache(force=False): """Determine if we should be clearing the packrat cache.""" if not ParserElement._packratEnabled: return False if SUPPORTS_INCREMENTAL: - if not ParserElement._incrementalEnabled: + if ( + not ParserElement._incrementalEnabled + or ParserElement._incrementalWithResets and repeatedly_clear_incremental_cache + ): return True - if ParserElement._incrementalWithResets and repeatedly_clear_incremental_cache: - return True - if incremental_cache_limit is not None and len(ParserElement.packrat_cache) > incremental_cache_limit: + if force or ( + incremental_cache_limit is not None + and len(ParserElement.packrat_cache) > incremental_cache_limit + ): # only clear the second half of the cache, since the first # half is what will help us next time we recompile return "second half" - return False + return False + else: + return True -def clear_packrat_cache(): +def clear_packrat_cache(force=False): """Clear the packrat cache if applicable.""" - clear_cache = should_clear_cache() - if not clear_cache: - return - if clear_cache == "second half": - cache_items = list(get_pyparsing_cache().items()) - restore_items = cache_items[:len(cache_items) // 2] - else: - restore_items = () - # clear cache without resetting stats - ParserElement.packrat_cache.clear() - # restore any items we want to keep - for lookup, value in restore_items: - ParserElement.packrat_cache.set(lookup, value) + clear_cache = should_clear_cache(force=force) + if clear_cache: + if clear_cache == "second half": + cache_items = list(get_pyparsing_cache().items()) + restore_items = cache_items[:len(cache_items) // 2] + else: + restore_items = () + # clear cache without resetting stats + ParserElement.packrat_cache.clear() + # restore any items we want to keep + for lookup, value in restore_items: + ParserElement.packrat_cache.set(lookup, value) + return clear_cache def get_cache_items_for(original): @@ -769,6 +778,17 @@ def get_target_info_smart(target, mode="lowest"): # PARSE ELEMENTS: # ----------------------------------------------------------------------------------------------------------------------- +class MatchAny(MatchFirst): + """Version of MatchFirst that always uses adaptive parsing.""" + adaptive_mode = True + + +def any_of(match_first): + """Build a MatchAny of the given MatchFirst.""" + internal_assert(isinstance(match_first, MatchFirst), "invalid any_of target", match_first) + return MatchAny(match_first.exprs) + + class Wrap(ParseElementEnhance): """PyParsing token that wraps the given item in the given context manager.""" global_instance_counter = 0 diff --git a/coconut/constants.py b/coconut/constants.py index 199b96d57..e9379f00a 100644 --- a/coconut/constants.py +++ b/coconut/constants.py @@ -130,7 +130,7 @@ def get_path_env_var(env_var, default): # note that _parseIncremental produces much smaller caches use_incremental_if_available = True -use_adaptive_if_available = True +use_adaptive_if_available = False adaptive_reparse_usage_weight = 10 # these only apply to use_incremental_if_available, not compiler.util.enable_incremental_parsing() From 1cfbe068fdf3171455cd6a21b178575cbb2856ab Mon Sep 17 00:00:00 2001 From: Evan Hubinger Date: Fri, 10 Nov 2023 02:02:14 -0800 Subject: [PATCH 073/121] Use fast grammar methods --- coconut/command/command.py | 9 +- coconut/compiler/compiler.py | 34 +- coconut/compiler/grammar.py | 3600 +++++++++-------- coconut/compiler/util.py | 71 +- coconut/constants.py | 4 +- coconut/terminal.py | 21 +- .../src/cocotest/agnostic/primary_2.coco | 1 + coconut/util.py | 19 + 8 files changed, 1928 insertions(+), 1831 deletions(-) diff --git a/coconut/command/command.py b/coconut/command/command.py index 69e713641..49ac9c4e6 100644 --- a/coconut/command/command.py +++ b/coconut/command/command.py @@ -83,6 +83,7 @@ get_clock_time, first_import_time, ensure_dir, + assert_remove_prefix, ) from coconut.command.util import ( writefile, @@ -324,7 +325,13 @@ def execute_args(self, args, interact=True, original_args=None): # process mypy args and print timing info (must come after compiler setup) if args.mypy is not None: self.set_mypy_args(args.mypy) - logger.log("Grammar init time: " + str(self.comp.grammar_init_time) + " secs / Total init time: " + str(get_clock_time() - first_import_time) + " secs") + if logger.verbose: + logger.log("Grammar init time: " + str(self.comp.grammar_init_time) + " secs / Total init time: " + str(get_clock_time() - first_import_time) + " secs") + for stat_name, (no_copy, yes_copy) in logger.recorded_stats.items(): + if not stat_name.startswith("maybe_copy_"): + continue + name = assert_remove_prefix(stat_name, "maybe_copy_") + logger.printlog("\tGrammar copying stats (" + name + "):", no_copy, "not copied;", yes_copy, "copied") # do compilation, keeping track of compiled filepaths filepaths = [] diff --git a/coconut/compiler/compiler.py b/coconut/compiler/compiler.py index 551462807..1f1ce3c39 100644 --- a/coconut/compiler/compiler.py +++ b/coconut/compiler/compiler.py @@ -2936,13 +2936,18 @@ def item_handle(self, original, loc, tokens): out = "_coconut_iter_getitem(" + out + ", " + trailer[1] + ")" elif trailer[0] == "$(?": pos_args, star_args, base_kwd_args, dubstar_args = self.split_function_call(trailer[1], loc) + has_question_mark = False + needs_complex_partial = False argdict_pairs = [] + last_pos_i = -1 for i, arg in enumerate(pos_args): if arg == "?": has_question_mark = True else: + if last_pos_i != i - 1: + needs_complex_partial = True argdict_pairs.append(str(i) + ": " + arg) pos_kwargs = [] @@ -2950,25 +2955,36 @@ def item_handle(self, original, loc, tokens): for i, arg in enumerate(base_kwd_args): if arg.endswith("=?"): has_question_mark = True + needs_complex_partial = True pos_kwargs.append(arg[:-2]) else: kwd_args.append(arg) - extra_args_str = join_args(star_args, kwd_args, dubstar_args) if not has_question_mark: raise CoconutInternalException("no question mark in question mark partial", trailer[1]) - elif argdict_pairs or pos_kwargs or extra_args_str: + + if needs_complex_partial: + extra_args_str = join_args(star_args, kwd_args, dubstar_args) + if argdict_pairs or pos_kwargs or extra_args_str: + out = ( + "_coconut_complex_partial(" + + out + + ", {" + ", ".join(argdict_pairs) + "}" + + ", " + str(len(pos_args)) + + ", " + tuple_str_of(pos_kwargs, add_quotes=True) + + (", " if extra_args_str else "") + extra_args_str + + ")" + ) + else: + raise CoconutDeferredSyntaxError("a non-? partial application argument is required", loc) + else: out = ( - "_coconut_complex_partial(" + "_coconut_partial(" + out - + ", {" + ", ".join(argdict_pairs) + "}" - + ", " + str(len(pos_args)) - + ", " + tuple_str_of(pos_kwargs, add_quotes=True) - + (", " if extra_args_str else "") + extra_args_str + + ", " + + join_args([arg for arg in pos_args if arg != "?"], star_args, kwd_args, dubstar_args) + ")" ) - else: - raise CoconutDeferredSyntaxError("a non-? partial application argument is required", loc) else: raise CoconutInternalException("invalid special trailer", trailer[0]) diff --git a/coconut/compiler/grammar.py b/coconut/compiler/grammar.py index cc02add8c..e6f3b6bdb 100644 --- a/coconut/compiler/grammar.py +++ b/coconut/compiler/grammar.py @@ -116,6 +116,7 @@ compile_regex, always_match, caseless_literal, + using_fast_grammar_methods, ) @@ -614,1941 +615,1942 @@ def typedef_op_item_handle(loc, tokens): class Grammar(object): """Coconut grammar specification.""" grammar_init_time = get_clock_time() + with using_fast_grammar_methods(): + + comma = Literal(",") + dubstar = Literal("**") + star = ~dubstar + Literal("*") + at = Literal("@") + arrow = Literal("->") | fixto(Literal("\u2192"), "->") + unsafe_fat_arrow = Literal("=>") | fixto(Literal("\u21d2"), "=>") + colon_eq = Literal(":=") + unsafe_dubcolon = Literal("::") + unsafe_colon = Literal(":") + colon = ~unsafe_dubcolon + ~colon_eq + unsafe_colon + lt_colon = Literal("<:") + semicolon = Literal(";") | invalid_syntax("\u037e", "invalid Greek question mark instead of semicolon", greedy=True) + multisemicolon = combine(OneOrMore(semicolon)) + eq = Literal("==") + equals = ~eq + ~Literal("=>") + Literal("=") + lbrack = Literal("[") + rbrack = Literal("]") + lbrace = Literal("{") + rbrace = Literal("}") + lbanana = ~Literal("(|)") + ~Literal("(|>") + ~Literal("(|*") + ~Literal("(|?") + Literal("(|") + rbanana = Literal("|)") + lparen = ~lbanana + Literal("(") + rparen = Literal(")") + unsafe_dot = Literal(".") + dot = ~Literal("..") + unsafe_dot + plus = Literal("+") + minus = ~Literal("->") + Literal("-") + dubslash = Literal("//") + slash = ~dubslash + Literal("/") + pipe = Literal("|>") | fixto(Literal("\u21a6"), "|>") + star_pipe = Literal("|*>") | fixto(Literal("*\u21a6"), "|*>") + dubstar_pipe = Literal("|**>") | fixto(Literal("**\u21a6"), "|**>") + back_pipe = Literal("<|") | fixto(Literal("\u21a4"), "<|") + back_star_pipe = Literal("<*|") | ~Literal("\u21a4**") + fixto(Literal("\u21a4*"), "<*|") + back_dubstar_pipe = Literal("<**|") | fixto(Literal("\u21a4**"), "<**|") + none_pipe = Literal("|?>") | fixto(Literal("?\u21a6"), "|?>") + none_star_pipe = ( + Literal("|?*>") + | fixto(Literal("?*\u21a6"), "|?*>") + | invalid_syntax("|*?>", "Coconut's None-aware forward multi-arg pipe is '|?*>', not '|*?>'") + ) + none_dubstar_pipe = ( + Literal("|?**>") + | fixto(Literal("?**\u21a6"), "|?**>") + | invalid_syntax("|**?>", "Coconut's None-aware forward keyword pipe is '|?**>', not '|**?>'") + ) + back_none_pipe = Literal("") + ~Literal("..*") + ~Literal("..?") + Literal("..") + | ~Literal("\u2218>") + ~Literal("\u2218*") + ~Literal("\u2218?") + fixto(Literal("\u2218"), "..") + ) + comp_pipe = Literal("..>") | fixto(Literal("\u2218>"), "..>") + comp_back_pipe = Literal("<..") | fixto(Literal("<\u2218"), "<..") + comp_star_pipe = Literal("..*>") | fixto(Literal("\u2218*>"), "..*>") + comp_back_star_pipe = Literal("<*..") | fixto(Literal("<*\u2218"), "<*..") + comp_dubstar_pipe = Literal("..**>") | fixto(Literal("\u2218**>"), "..**>") + comp_back_dubstar_pipe = Literal("<**..") | fixto(Literal("<**\u2218"), "<**..") + comp_none_pipe = Literal("..?>") | fixto(Literal("\u2218?>"), "..?>") + comp_back_none_pipe = Literal("") + | fixto(Literal("\u2218?*>"), "..?*>") + | invalid_syntax("..*?>", "Coconut's None-aware forward multi-arg composition pipe is '..?*>', not '..*?>'") + ) + comp_back_none_star_pipe = ( + Literal("<*?..") + | fixto(Literal("<*?\u2218"), "<*?..") + | invalid_syntax("") + | fixto(Literal("\u2218?**>"), "..?**>") + | invalid_syntax("..**?>", "Coconut's None-aware forward keyword composition pipe is '..?**>', not '..**?>'") + ) + comp_back_none_dubstar_pipe = ( + Literal("<**?..") + | fixto(Literal("<**?\u2218"), "<**?..") + | invalid_syntax("") + ~Literal("|*") + Literal("|") | fixto(Literal("\u222a"), "|") + bar = ~rbanana + unsafe_bar | invalid_syntax("\xa6", "invalid broken bar character", greedy=True) + percent = Literal("%") + dollar = Literal("$") + lshift = Literal("<<") | fixto(Literal("\xab"), "<<") + rshift = Literal(">>") | fixto(Literal("\xbb"), ">>") + tilde = Literal("~") + underscore = Literal("_") + pound = Literal("#") + unsafe_backtick = Literal("`") + dubbackslash = Literal("\\\\") + backslash = ~dubbackslash + Literal("\\") + dubquestion = Literal("??") + questionmark = ~dubquestion + Literal("?") + bang = ~Literal("!=") + Literal("!") + + kwds = keydefaultdict(partial(base_keyword, explicit_prefix=colon)) + keyword = kwds.__getitem__ + + except_star_kwd = combine(keyword("except") + star) + kwds["except"] = ~except_star_kwd + keyword("except") + kwds["lambda"] = keyword("lambda") | fixto(keyword("\u03bb"), "lambda") + kwds["operator"] = base_keyword("operator", explicit_prefix=colon, require_whitespace=True) + + ellipsis = Forward() + ellipsis_tokens = Literal("...") | fixto(Literal("\u2026"), "...") + + lt = ( + ~Literal("<<") + + ~Literal("<=") + + ~Literal("<|") + + ~Literal("<..") + + ~Literal("<*") + + ~Literal("<:") + + Literal("<") + | fixto(Literal("\u228a"), "<") + ) + gt = ( + ~Literal(">>") + + ~Literal(">=") + + Literal(">") + | fixto(Literal("\u228b"), ">") + ) + le = Literal("<=") | fixto(Literal("\u2264") | Literal("\u2286"), "<=") + ge = Literal(">=") | fixto(Literal("\u2265") | Literal("\u2287"), ">=") + ne = Literal("!=") | fixto(Literal("\xac=") | Literal("\u2260"), "!=") + + mul_star = star | fixto(Literal("\xd7"), "*") + exp_dubstar = dubstar | fixto(Literal("\u2191"), "**") + neg_minus = ( + minus + | fixto(Literal("\u207b"), "-") + ) + sub_minus = ( + minus + | invalid_syntax("\u207b", "U+207b is only for negation, not subtraction") + ) + div_slash = slash | fixto(Literal("\xf7") + ~slash, "/") + div_dubslash = dubslash | fixto(combine(Literal("\xf7") + slash), "//") + matrix_at = at + + test = Forward() + test_no_chain, dubcolon = disable_inside(test, unsafe_dubcolon) + test_no_infix, backtick = disable_inside(test, unsafe_backtick) + + base_name_regex = r"" + for no_kwd in keyword_vars + const_vars: + base_name_regex += r"(?!" + no_kwd + r"\b)" + # we disallow ['"{] after to not match the "b" in b"" or the "s" in s{} + base_name_regex += r"(?![0-9])\w+\b(?![{" + strwrapper + r"])" + base_name = regex_item(base_name_regex) + + refname = Forward() + setname = Forward() + classname = Forward() + name_ref = combine(Optional(backslash) + base_name) + unsafe_name = combine(Optional(backslash.suppress()) + base_name) + + # use unsafe_name for dotted components since name should only be used for base names + dotted_refname = condense(refname + ZeroOrMore(dot + unsafe_name)) + dotted_setname = condense(setname + ZeroOrMore(dot + unsafe_name)) + unsafe_dotted_name = condense(unsafe_name + ZeroOrMore(dot + unsafe_name)) + must_be_dotted_name = condense(refname + OneOrMore(dot + unsafe_name)) + + integer = combine(Word(nums) + ZeroOrMore(underscore.suppress() + Word(nums))) + binint = combine(Word("01") + ZeroOrMore(underscore.suppress() + Word("01"))) + octint = combine(Word("01234567") + ZeroOrMore(underscore.suppress() + Word("01234567"))) + hexint = combine(Word(hexnums) + ZeroOrMore(underscore.suppress() + Word(hexnums))) + + imag_j = caseless_literal("j") | fixto(caseless_literal("i", suppress=True), "j") + basenum = combine( + integer + dot + Optional(integer) + | Optional(integer) + dot + integer + ) | integer + sci_e = combine((caseless_literal("e") | fixto(Literal("\u23e8"), "e")) + Optional(plus | neg_minus)) + numitem = ~(Literal("0") + Word(nums + "_", exact=1)) + combine(basenum + Optional(sci_e + integer)) + imag_num = combine(numitem + imag_j) + bin_num = combine(caseless_literal("0b") + Optional(underscore.suppress()) + binint) + oct_num = combine(caseless_literal("0o") + Optional(underscore.suppress()) + octint) + hex_num = combine(caseless_literal("0x") + Optional(underscore.suppress()) + hexint) + number = ( + bin_num + | oct_num + | hex_num + | imag_num + | numitem + ) + # make sure that this gets addspaced not condensed so it doesn't produce a SyntaxError + num_atom = addspace(number + Optional(condense(dot + unsafe_name))) + + moduledoc_item = Forward() + unwrap = Literal(unwrapper) + comment = Forward() + comment_tokens = combine(pound + integer + unwrap) + string_item = ( + combine(Literal(strwrapper) + integer + unwrap) + | invalid_syntax(("\u201c", "\u201d", "\u2018", "\u2019"), "invalid unicode quotation mark; strings must use \" or '", greedy=True) + ) - comma = Literal(",") - dubstar = Literal("**") - star = ~dubstar + Literal("*") - at = Literal("@") - arrow = Literal("->") | fixto(Literal("\u2192"), "->") - unsafe_fat_arrow = Literal("=>") | fixto(Literal("\u21d2"), "=>") - colon_eq = Literal(":=") - unsafe_dubcolon = Literal("::") - unsafe_colon = Literal(":") - colon = ~unsafe_dubcolon + ~colon_eq + unsafe_colon - lt_colon = Literal("<:") - semicolon = Literal(";") | invalid_syntax("\u037e", "invalid Greek question mark instead of semicolon", greedy=True) - multisemicolon = combine(OneOrMore(semicolon)) - eq = Literal("==") - equals = ~eq + ~Literal("=>") + Literal("=") - lbrack = Literal("[") - rbrack = Literal("]") - lbrace = Literal("{") - rbrace = Literal("}") - lbanana = ~Literal("(|)") + ~Literal("(|>") + ~Literal("(|*") + ~Literal("(|?") + Literal("(|") - rbanana = Literal("|)") - lparen = ~lbanana + Literal("(") - rparen = Literal(")") - unsafe_dot = Literal(".") - dot = ~Literal("..") + unsafe_dot - plus = Literal("+") - minus = ~Literal("->") + Literal("-") - dubslash = Literal("//") - slash = ~dubslash + Literal("/") - pipe = Literal("|>") | fixto(Literal("\u21a6"), "|>") - star_pipe = Literal("|*>") | fixto(Literal("*\u21a6"), "|*>") - dubstar_pipe = Literal("|**>") | fixto(Literal("**\u21a6"), "|**>") - back_pipe = Literal("<|") | fixto(Literal("\u21a4"), "<|") - back_star_pipe = Literal("<*|") | ~Literal("\u21a4**") + fixto(Literal("\u21a4*"), "<*|") - back_dubstar_pipe = Literal("<**|") | fixto(Literal("\u21a4**"), "<**|") - none_pipe = Literal("|?>") | fixto(Literal("?\u21a6"), "|?>") - none_star_pipe = ( - Literal("|?*>") - | fixto(Literal("?*\u21a6"), "|?*>") - | invalid_syntax("|*?>", "Coconut's None-aware forward multi-arg pipe is '|?*>', not '|*?>'") - ) - none_dubstar_pipe = ( - Literal("|?**>") - | fixto(Literal("?**\u21a6"), "|?**>") - | invalid_syntax("|**?>", "Coconut's None-aware forward keyword pipe is '|?**>', not '|**?>'") - ) - back_none_pipe = Literal("") + ~Literal("..*") + ~Literal("..?") + Literal("..") - | ~Literal("\u2218>") + ~Literal("\u2218*") + ~Literal("\u2218?") + fixto(Literal("\u2218"), "..") - ) - comp_pipe = Literal("..>") | fixto(Literal("\u2218>"), "..>") - comp_back_pipe = Literal("<..") | fixto(Literal("<\u2218"), "<..") - comp_star_pipe = Literal("..*>") | fixto(Literal("\u2218*>"), "..*>") - comp_back_star_pipe = Literal("<*..") | fixto(Literal("<*\u2218"), "<*..") - comp_dubstar_pipe = Literal("..**>") | fixto(Literal("\u2218**>"), "..**>") - comp_back_dubstar_pipe = Literal("<**..") | fixto(Literal("<**\u2218"), "<**..") - comp_none_pipe = Literal("..?>") | fixto(Literal("\u2218?>"), "..?>") - comp_back_none_pipe = Literal("") - | fixto(Literal("\u2218?*>"), "..?*>") - | invalid_syntax("..*?>", "Coconut's None-aware forward multi-arg composition pipe is '..?*>', not '..*?>'") - ) - comp_back_none_star_pipe = ( - Literal("<*?..") - | fixto(Literal("<*?\u2218"), "<*?..") - | invalid_syntax("") - | fixto(Literal("\u2218?**>"), "..?**>") - | invalid_syntax("..**?>", "Coconut's None-aware forward keyword composition pipe is '..?**>', not '..**?>'") - ) - comp_back_none_dubstar_pipe = ( - Literal("<**?..") - | fixto(Literal("<**?\u2218"), "<**?..") - | invalid_syntax("") + ~Literal("|*") + Literal("|") | fixto(Literal("\u222a"), "|") - bar = ~rbanana + unsafe_bar | invalid_syntax("\xa6", "invalid broken bar character", greedy=True) - percent = Literal("%") - dollar = Literal("$") - lshift = Literal("<<") | fixto(Literal("\xab"), "<<") - rshift = Literal(">>") | fixto(Literal("\xbb"), ">>") - tilde = Literal("~") - underscore = Literal("_") - pound = Literal("#") - unsafe_backtick = Literal("`") - dubbackslash = Literal("\\\\") - backslash = ~dubbackslash + Literal("\\") - dubquestion = Literal("??") - questionmark = ~dubquestion + Literal("?") - bang = ~Literal("!=") + Literal("!") - - kwds = keydefaultdict(partial(base_keyword, explicit_prefix=colon)) - keyword = kwds.__getitem__ - - except_star_kwd = combine(keyword("except") + star) - kwds["except"] = ~except_star_kwd + keyword("except") - kwds["lambda"] = keyword("lambda") | fixto(keyword("\u03bb"), "lambda") - kwds["operator"] = base_keyword("operator", explicit_prefix=colon, require_whitespace=True) - - ellipsis = Forward() - ellipsis_tokens = Literal("...") | fixto(Literal("\u2026"), "...") - - lt = ( - ~Literal("<<") - + ~Literal("<=") - + ~Literal("<|") - + ~Literal("<..") - + ~Literal("<*") - + ~Literal("<:") - + Literal("<") - | fixto(Literal("\u228a"), "<") - ) - gt = ( - ~Literal(">>") - + ~Literal(">=") - + Literal(">") - | fixto(Literal("\u228b"), ">") - ) - le = Literal("<=") | fixto(Literal("\u2264") | Literal("\u2286"), "<=") - ge = Literal(">=") | fixto(Literal("\u2265") | Literal("\u2287"), ">=") - ne = Literal("!=") | fixto(Literal("\xac=") | Literal("\u2260"), "!=") - - mul_star = star | fixto(Literal("\xd7"), "*") - exp_dubstar = dubstar | fixto(Literal("\u2191"), "**") - neg_minus = ( - minus - | fixto(Literal("\u207b"), "-") - ) - sub_minus = ( - minus - | invalid_syntax("\u207b", "U+207b is only for negation, not subtraction") - ) - div_slash = slash | fixto(Literal("\xf7") + ~slash, "/") - div_dubslash = dubslash | fixto(combine(Literal("\xf7") + slash), "//") - matrix_at = at - - test = Forward() - test_no_chain, dubcolon = disable_inside(test, unsafe_dubcolon) - test_no_infix, backtick = disable_inside(test, unsafe_backtick) - - base_name_regex = r"" - for no_kwd in keyword_vars + const_vars: - base_name_regex += r"(?!" + no_kwd + r"\b)" - # we disallow ['"{] after to not match the "b" in b"" or the "s" in s{} - base_name_regex += r"(?![0-9])\w+\b(?![{" + strwrapper + r"])" - base_name = regex_item(base_name_regex) - - refname = Forward() - setname = Forward() - classname = Forward() - name_ref = combine(Optional(backslash) + base_name) - unsafe_name = combine(Optional(backslash.suppress()) + base_name) - - # use unsafe_name for dotted components since name should only be used for base names - dotted_refname = condense(refname + ZeroOrMore(dot + unsafe_name)) - dotted_setname = condense(setname + ZeroOrMore(dot + unsafe_name)) - unsafe_dotted_name = condense(unsafe_name + ZeroOrMore(dot + unsafe_name)) - must_be_dotted_name = condense(refname + OneOrMore(dot + unsafe_name)) - - integer = combine(Word(nums) + ZeroOrMore(underscore.suppress() + Word(nums))) - binint = combine(Word("01") + ZeroOrMore(underscore.suppress() + Word("01"))) - octint = combine(Word("01234567") + ZeroOrMore(underscore.suppress() + Word("01234567"))) - hexint = combine(Word(hexnums) + ZeroOrMore(underscore.suppress() + Word(hexnums))) - - imag_j = caseless_literal("j") | fixto(caseless_literal("i", suppress=True), "j") - basenum = combine( - integer + dot + Optional(integer) - | Optional(integer) + dot + integer - ) | integer - sci_e = combine((caseless_literal("e") | fixto(Literal("\u23e8"), "e")) + Optional(plus | neg_minus)) - numitem = ~(Literal("0") + Word(nums + "_", exact=1)) + combine(basenum + Optional(sci_e + integer)) - imag_num = combine(numitem + imag_j) - bin_num = combine(caseless_literal("0b") + Optional(underscore.suppress()) + binint) - oct_num = combine(caseless_literal("0o") + Optional(underscore.suppress()) + octint) - hex_num = combine(caseless_literal("0x") + Optional(underscore.suppress()) + hexint) - number = ( - bin_num - | oct_num - | hex_num - | imag_num - | numitem - ) - # make sure that this gets addspaced not condensed so it doesn't produce a SyntaxError - num_atom = addspace(number + Optional(condense(dot + unsafe_name))) - - moduledoc_item = Forward() - unwrap = Literal(unwrapper) - comment = Forward() - comment_tokens = combine(pound + integer + unwrap) - string_item = ( - combine(Literal(strwrapper) + integer + unwrap) - | invalid_syntax(("\u201c", "\u201d", "\u2018", "\u2019"), "invalid unicode quotation mark; strings must use \" or '", greedy=True) - ) - - xonsh_command = Forward() - passthrough_item = combine((backslash | Literal(early_passthrough_wrapper)) + integer + unwrap) | xonsh_command - passthrough_block = combine(fixto(dubbackslash, "\\") + integer + unwrap) - - endline = Forward() - endline_ref = condense(OneOrMore(Literal("\n"))) - lineitem = ZeroOrMore(comment) + endline - newline = condense(OneOrMore(lineitem)) - # rparen handles simple stmts ending parenthesized stmt lambdas - end_simple_stmt_item = FollowedBy(semicolon | newline | rparen) - - start_marker = StringStart() - moduledoc_marker = condense(ZeroOrMore(lineitem) - Optional(moduledoc_item)) - end_marker = StringEnd() - indent = Literal(openindent) - dedent = Literal(closeindent) - - u_string = Forward() - f_string = Forward() - - bit_b = caseless_literal("b") - raw_r = caseless_literal("r") - unicode_u = caseless_literal("u", suppress=True) - format_f = caseless_literal("f", suppress=True) - - string = combine(Optional(raw_r) + string_item) - # Python 2 only supports br"..." not rb"..." - b_string = combine((bit_b + Optional(raw_r) | fixto(raw_r + bit_b, "br")) + string_item) - # ur"..."/ru"..." strings are not suppored in Python 3 - u_string_ref = combine(unicode_u + string_item) - f_string_tokens = combine((format_f + Optional(raw_r) | raw_r + format_f) + string_item) - nonbf_string = string | u_string - nonb_string = nonbf_string | f_string - any_string = nonb_string | b_string - moduledoc = any_string + newline - docstring = condense(moduledoc) - - pipe_augassign = ( - combine(pipe + equals) - | combine(star_pipe + equals) - | combine(dubstar_pipe + equals) - | combine(back_pipe + equals) - | combine(back_star_pipe + equals) - | combine(back_dubstar_pipe + equals) - | combine(none_pipe + equals) - | combine(none_star_pipe + equals) - | combine(none_dubstar_pipe + equals) - | combine(back_none_pipe + equals) - | combine(back_none_star_pipe + equals) - | combine(back_none_dubstar_pipe + equals) - ) - augassign = ( - pipe_augassign - | combine(comp_pipe + equals) - | combine(dotdot + equals) - | combine(comp_back_pipe + equals) - | combine(comp_star_pipe + equals) - | combine(comp_back_star_pipe + equals) - | combine(comp_dubstar_pipe + equals) - | combine(comp_back_dubstar_pipe + equals) - | combine(comp_none_pipe + equals) - | combine(comp_back_none_pipe + equals) - | combine(comp_none_star_pipe + equals) - | combine(comp_back_none_star_pipe + equals) - | combine(comp_none_dubstar_pipe + equals) - | combine(comp_back_none_dubstar_pipe + equals) - | combine(unsafe_dubcolon + equals) - | combine(div_dubslash + equals) - | combine(div_slash + equals) - | combine(exp_dubstar + equals) - | combine(mul_star + equals) - | combine(plus + equals) - | combine(sub_minus + equals) - | combine(percent + equals) - | combine(amp + equals) - | combine(bar + equals) - | combine(caret + equals) - | combine(lshift + equals) - | combine(rshift + equals) - | combine(matrix_at + equals) - | combine(dubquestion + equals) - ) + xonsh_command = Forward() + passthrough_item = combine((backslash | Literal(early_passthrough_wrapper)) + integer + unwrap) | xonsh_command + passthrough_block = combine(fixto(dubbackslash, "\\") + integer + unwrap) + + endline = Forward() + endline_ref = condense(OneOrMore(Literal("\n"))) + lineitem = ZeroOrMore(comment) + endline + newline = condense(OneOrMore(lineitem)) + # rparen handles simple stmts ending parenthesized stmt lambdas + end_simple_stmt_item = FollowedBy(semicolon | newline | rparen) + + start_marker = StringStart() + moduledoc_marker = condense(ZeroOrMore(lineitem) - Optional(moduledoc_item)) + end_marker = StringEnd() + indent = Literal(openindent) + dedent = Literal(closeindent) + + u_string = Forward() + f_string = Forward() + + bit_b = caseless_literal("b") + raw_r = caseless_literal("r") + unicode_u = caseless_literal("u", suppress=True) + format_f = caseless_literal("f", suppress=True) + + string = combine(Optional(raw_r) + string_item) + # Python 2 only supports br"..." not rb"..." + b_string = combine((bit_b + Optional(raw_r) | fixto(raw_r + bit_b, "br")) + string_item) + # ur"..."/ru"..." strings are not suppored in Python 3 + u_string_ref = combine(unicode_u + string_item) + f_string_tokens = combine((format_f + Optional(raw_r) | raw_r + format_f) + string_item) + nonbf_string = string | u_string + nonb_string = nonbf_string | f_string + any_string = nonb_string | b_string + moduledoc = any_string + newline + docstring = condense(moduledoc) + + pipe_augassign = ( + combine(pipe + equals) + | combine(star_pipe + equals) + | combine(dubstar_pipe + equals) + | combine(back_pipe + equals) + | combine(back_star_pipe + equals) + | combine(back_dubstar_pipe + equals) + | combine(none_pipe + equals) + | combine(none_star_pipe + equals) + | combine(none_dubstar_pipe + equals) + | combine(back_none_pipe + equals) + | combine(back_none_star_pipe + equals) + | combine(back_none_dubstar_pipe + equals) + ) + augassign = ( + pipe_augassign + | combine(comp_pipe + equals) + | combine(dotdot + equals) + | combine(comp_back_pipe + equals) + | combine(comp_star_pipe + equals) + | combine(comp_back_star_pipe + equals) + | combine(comp_dubstar_pipe + equals) + | combine(comp_back_dubstar_pipe + equals) + | combine(comp_none_pipe + equals) + | combine(comp_back_none_pipe + equals) + | combine(comp_none_star_pipe + equals) + | combine(comp_back_none_star_pipe + equals) + | combine(comp_none_dubstar_pipe + equals) + | combine(comp_back_none_dubstar_pipe + equals) + | combine(unsafe_dubcolon + equals) + | combine(div_dubslash + equals) + | combine(div_slash + equals) + | combine(exp_dubstar + equals) + | combine(mul_star + equals) + | combine(plus + equals) + | combine(sub_minus + equals) + | combine(percent + equals) + | combine(amp + equals) + | combine(bar + equals) + | combine(caret + equals) + | combine(lshift + equals) + | combine(rshift + equals) + | combine(matrix_at + equals) + | combine(dubquestion + equals) + ) - comp_op = ( - le | ge | ne | lt | gt | eq - | addspace(keyword("not") + keyword("in")) - | keyword("in") - | addspace(keyword("is") + keyword("not")) - | keyword("is") - ) + comp_op = ( + le | ge | ne | lt | gt | eq + | addspace(keyword("not") + keyword("in")) + | keyword("in") + | addspace(keyword("is") + keyword("not")) + | keyword("is") + ) - atom_item = Forward() - expr = Forward() - star_expr = Forward() - dubstar_expr = Forward() - comp_for = Forward() - test_no_cond = Forward() - infix_op = Forward() - namedexpr_test = Forward() - # for namedexpr locations only supported in Python 3.10 - new_namedexpr_test = Forward() - lambdef = Forward() - - typedef = Forward() - typedef_default = Forward() - unsafe_typedef_default = Forward() - typedef_test = Forward() - typedef_tuple = Forward() - typedef_ellipsis = Forward() - typedef_op_item = Forward() - - negable_atom_item = condense(Optional(neg_minus) + atom_item) - - testlist = itemlist(test, comma, suppress_trailing=False) - testlist_has_comma = addspace(OneOrMore(condense(test + comma)) + Optional(test)) - new_namedexpr_testlist_has_comma = addspace(OneOrMore(condense(new_namedexpr_test + comma)) + Optional(test)) - - testlist_star_expr = Forward() - testlist_star_expr_ref = tokenlist(Group(test) | star_expr, comma, suppress=False) - testlist_star_namedexpr = Forward() - testlist_star_namedexpr_tokens = tokenlist(Group(namedexpr_test) | star_expr, comma, suppress=False) - # for testlist_star_expr locations only supported in Python 3.9 - new_testlist_star_expr = Forward() - new_testlist_star_expr_ref = testlist_star_expr - - yield_from = Forward() - dict_comp = Forward() - dict_literal = Forward() - yield_classic = addspace(keyword("yield") + Optional(new_testlist_star_expr)) - yield_from_ref = keyword("yield").suppress() + keyword("from").suppress() + test - yield_expr = yield_from | yield_classic - dict_comp_ref = lbrace.suppress() + ( - test + colon.suppress() + test + comp_for - | invalid_syntax(dubstar_expr + comp_for, "dict unpacking cannot be used in dict comprehension") - ) + rbrace.suppress() - dict_literal_ref = ( - lbrace.suppress() - + Optional(tokenlist( - Group(test + colon + test) - | dubstar_expr, - comma, - )) - + rbrace.suppress() - ) - test_expr = yield_expr | testlist_star_expr - - base_op_item = ( - # must go dubstar then star then no star - fixto(dubstar_pipe, "_coconut_dubstar_pipe") - | fixto(back_dubstar_pipe, "_coconut_back_dubstar_pipe") - | fixto(none_dubstar_pipe, "_coconut_none_dubstar_pipe") - | fixto(back_none_dubstar_pipe, "_coconut_back_none_dubstar_pipe") - | fixto(star_pipe, "_coconut_star_pipe") - | fixto(back_star_pipe, "_coconut_back_star_pipe") - | fixto(none_star_pipe, "_coconut_none_star_pipe") - | fixto(back_none_star_pipe, "_coconut_back_none_star_pipe") - | fixto(pipe, "_coconut_pipe") - | fixto(back_pipe, "_coconut_back_pipe") - | fixto(none_pipe, "_coconut_none_pipe") - | fixto(back_none_pipe, "_coconut_back_none_pipe") - - # must go dubstar then star then no star - | fixto(comp_dubstar_pipe, "_coconut_forward_dubstar_compose") - | fixto(comp_back_dubstar_pipe, "_coconut_back_dubstar_compose") - | fixto(comp_none_dubstar_pipe, "_coconut_forward_none_dubstar_compose") - | fixto(comp_back_none_dubstar_pipe, "_coconut_back_none_dubstar_compose") - | fixto(comp_star_pipe, "_coconut_forward_star_compose") - | fixto(comp_back_star_pipe, "_coconut_back_star_compose") - | fixto(comp_none_star_pipe, "_coconut_forward_none_star_compose") - | fixto(comp_back_none_star_pipe, "_coconut_back_none_star_compose") - | fixto(comp_pipe, "_coconut_forward_compose") - | fixto(dotdot | comp_back_pipe, "_coconut_back_compose") - | fixto(comp_none_pipe, "_coconut_forward_none_compose") - | fixto(comp_back_none_pipe, "_coconut_back_none_compose") - - # neg_minus must come after minus - | fixto(minus, "_coconut_minus") - | fixto(neg_minus, "_coconut.operator.neg") - - | fixto(keyword("assert"), "_coconut_assert") - | fixto(keyword("raise"), "_coconut_raise") - | fixto(keyword("and"), "_coconut_bool_and") - | fixto(keyword("or"), "_coconut_bool_or") - | fixto(comma, "_coconut_comma_op") - | fixto(dubquestion, "_coconut_none_coalesce") - | fixto(dot, "_coconut.getattr") - | fixto(unsafe_dubcolon, "_coconut.itertools.chain") - | fixto(dollar, "_coconut_partial") - | fixto(exp_dubstar, "_coconut.operator.pow") - | fixto(mul_star, "_coconut.operator.mul") - | fixto(div_dubslash, "_coconut.operator.floordiv") - | fixto(div_slash, "_coconut.operator.truediv") - | fixto(percent, "_coconut.operator.mod") - | fixto(plus, "_coconut.operator.add") - | fixto(amp, "_coconut.operator.and_") - | fixto(caret, "_coconut.operator.xor") - | fixto(unsafe_bar, "_coconut.operator.or_") - | fixto(lshift, "_coconut.operator.lshift") - | fixto(rshift, "_coconut.operator.rshift") - | fixto(lt, "_coconut.operator.lt") - | fixto(gt, "_coconut.operator.gt") - | fixto(eq, "_coconut.operator.eq") - | fixto(le, "_coconut.operator.le") - | fixto(ge, "_coconut.operator.ge") - | fixto(ne, "_coconut.operator.ne") - | fixto(tilde, "_coconut.operator.inv") - | fixto(matrix_at, "_coconut_matmul") - | fixto(keyword("is") + keyword("not"), "_coconut.operator.is_not") - | fixto(keyword("not") + keyword("in"), "_coconut_not_in") - - # must come after is not / not in - | fixto(keyword("not"), "_coconut.operator.not_") - | fixto(keyword("is"), "_coconut.operator.is_") - | fixto(keyword("in"), "_coconut_in") - ) - partialable_op = base_op_item | infix_op - partial_op_item_tokens = ( - labeled_group(dot.suppress() + partialable_op + test_no_infix, "right partial") - | labeled_group(test_no_infix + partialable_op + dot.suppress(), "left partial") - ) - partial_op_item = attach(partial_op_item_tokens, partial_op_item_handle) - op_item = ( - typedef_op_item - | partial_op_item - | base_op_item - ) + atom_item = Forward() + expr = Forward() + star_expr = Forward() + dubstar_expr = Forward() + comp_for = Forward() + test_no_cond = Forward() + infix_op = Forward() + namedexpr_test = Forward() + # for namedexpr locations only supported in Python 3.10 + new_namedexpr_test = Forward() + lambdef = Forward() + + typedef = Forward() + typedef_default = Forward() + unsafe_typedef_default = Forward() + typedef_test = Forward() + typedef_tuple = Forward() + typedef_ellipsis = Forward() + typedef_op_item = Forward() + + negable_atom_item = condense(Optional(neg_minus) + atom_item) + + testlist = itemlist(test, comma, suppress_trailing=False) + testlist_has_comma = addspace(OneOrMore(condense(test + comma)) + Optional(test)) + new_namedexpr_testlist_has_comma = addspace(OneOrMore(condense(new_namedexpr_test + comma)) + Optional(test)) + + testlist_star_expr = Forward() + testlist_star_expr_ref = tokenlist(Group(test) | star_expr, comma, suppress=False) + testlist_star_namedexpr = Forward() + testlist_star_namedexpr_tokens = tokenlist(Group(namedexpr_test) | star_expr, comma, suppress=False) + # for testlist_star_expr locations only supported in Python 3.9 + new_testlist_star_expr = Forward() + new_testlist_star_expr_ref = testlist_star_expr + + yield_from = Forward() + dict_comp = Forward() + dict_literal = Forward() + yield_classic = addspace(keyword("yield") + Optional(new_testlist_star_expr)) + yield_from_ref = keyword("yield").suppress() + keyword("from").suppress() + test + yield_expr = yield_from | yield_classic + dict_comp_ref = lbrace.suppress() + ( + test + colon.suppress() + test + comp_for + | invalid_syntax(dubstar_expr + comp_for, "dict unpacking cannot be used in dict comprehension") + ) + rbrace.suppress() + dict_literal_ref = ( + lbrace.suppress() + + Optional(tokenlist( + Group(test + colon + test) + | dubstar_expr, + comma, + )) + + rbrace.suppress() + ) + test_expr = yield_expr | testlist_star_expr + + base_op_item = ( + # must go dubstar then star then no star + fixto(dubstar_pipe, "_coconut_dubstar_pipe") + | fixto(back_dubstar_pipe, "_coconut_back_dubstar_pipe") + | fixto(none_dubstar_pipe, "_coconut_none_dubstar_pipe") + | fixto(back_none_dubstar_pipe, "_coconut_back_none_dubstar_pipe") + | fixto(star_pipe, "_coconut_star_pipe") + | fixto(back_star_pipe, "_coconut_back_star_pipe") + | fixto(none_star_pipe, "_coconut_none_star_pipe") + | fixto(back_none_star_pipe, "_coconut_back_none_star_pipe") + | fixto(pipe, "_coconut_pipe") + | fixto(back_pipe, "_coconut_back_pipe") + | fixto(none_pipe, "_coconut_none_pipe") + | fixto(back_none_pipe, "_coconut_back_none_pipe") + + # must go dubstar then star then no star + | fixto(comp_dubstar_pipe, "_coconut_forward_dubstar_compose") + | fixto(comp_back_dubstar_pipe, "_coconut_back_dubstar_compose") + | fixto(comp_none_dubstar_pipe, "_coconut_forward_none_dubstar_compose") + | fixto(comp_back_none_dubstar_pipe, "_coconut_back_none_dubstar_compose") + | fixto(comp_star_pipe, "_coconut_forward_star_compose") + | fixto(comp_back_star_pipe, "_coconut_back_star_compose") + | fixto(comp_none_star_pipe, "_coconut_forward_none_star_compose") + | fixto(comp_back_none_star_pipe, "_coconut_back_none_star_compose") + | fixto(comp_pipe, "_coconut_forward_compose") + | fixto(dotdot | comp_back_pipe, "_coconut_back_compose") + | fixto(comp_none_pipe, "_coconut_forward_none_compose") + | fixto(comp_back_none_pipe, "_coconut_back_none_compose") + + # neg_minus must come after minus + | fixto(minus, "_coconut_minus") + | fixto(neg_minus, "_coconut.operator.neg") + + | fixto(keyword("assert"), "_coconut_assert") + | fixto(keyword("raise"), "_coconut_raise") + | fixto(keyword("and"), "_coconut_bool_and") + | fixto(keyword("or"), "_coconut_bool_or") + | fixto(comma, "_coconut_comma_op") + | fixto(dubquestion, "_coconut_none_coalesce") + | fixto(dot, "_coconut.getattr") + | fixto(unsafe_dubcolon, "_coconut.itertools.chain") + | fixto(dollar, "_coconut_partial") + | fixto(exp_dubstar, "_coconut.operator.pow") + | fixto(mul_star, "_coconut.operator.mul") + | fixto(div_dubslash, "_coconut.operator.floordiv") + | fixto(div_slash, "_coconut.operator.truediv") + | fixto(percent, "_coconut.operator.mod") + | fixto(plus, "_coconut.operator.add") + | fixto(amp, "_coconut.operator.and_") + | fixto(caret, "_coconut.operator.xor") + | fixto(unsafe_bar, "_coconut.operator.or_") + | fixto(lshift, "_coconut.operator.lshift") + | fixto(rshift, "_coconut.operator.rshift") + | fixto(lt, "_coconut.operator.lt") + | fixto(gt, "_coconut.operator.gt") + | fixto(eq, "_coconut.operator.eq") + | fixto(le, "_coconut.operator.le") + | fixto(ge, "_coconut.operator.ge") + | fixto(ne, "_coconut.operator.ne") + | fixto(tilde, "_coconut.operator.inv") + | fixto(matrix_at, "_coconut_matmul") + | fixto(keyword("is") + keyword("not"), "_coconut.operator.is_not") + | fixto(keyword("not") + keyword("in"), "_coconut_not_in") + + # must come after is not / not in + | fixto(keyword("not"), "_coconut.operator.not_") + | fixto(keyword("is"), "_coconut.operator.is_") + | fixto(keyword("in"), "_coconut_in") + ) + partialable_op = base_op_item | infix_op + partial_op_item_tokens = ( + labeled_group(dot.suppress() + partialable_op + test_no_infix, "right partial") + | labeled_group(test_no_infix + partialable_op + dot.suppress(), "left partial") + ) + partial_op_item = attach(partial_op_item_tokens, partial_op_item_handle) + op_item = ( + typedef_op_item + | partial_op_item + | base_op_item + ) - partial_op_atom_tokens = lparen.suppress() + partial_op_item_tokens + rparen.suppress() - - # we include (var)arg_comma to ensure the pattern matches the whole arg - arg_comma = comma | fixto(FollowedBy(rparen), "") - setarg_comma = arg_comma | fixto(FollowedBy(colon), "") - typedef_ref = setname + colon.suppress() + typedef_test + arg_comma - default = condense(equals + test) - unsafe_typedef_default_ref = setname + colon.suppress() + typedef_test + Optional(default) - typedef_default_ref = unsafe_typedef_default_ref + arg_comma - tfpdef = typedef | condense(setname + arg_comma) - tfpdef_default = typedef_default | condense(setname + Optional(default) + arg_comma) - - star_sep_arg = Forward() - star_sep_arg_ref = condense(star + arg_comma) - star_sep_setarg = Forward() - star_sep_setarg_ref = condense(star + setarg_comma) - - slash_sep_arg = Forward() - slash_sep_arg_ref = condense(slash + arg_comma) - slash_sep_setarg = Forward() - slash_sep_setarg_ref = condense(slash + setarg_comma) - - just_star = star + rparen - just_slash = slash + rparen - just_op = just_star | just_slash - - match = Forward() - args_list = ( - ~just_op - + addspace( - ZeroOrMore( - condense( - # everything here must end with arg_comma - (star | dubstar) + tfpdef - | star_sep_arg - | slash_sep_arg - | tfpdef_default + partial_op_atom_tokens = lparen.suppress() + partial_op_item_tokens + rparen.suppress() + + # we include (var)arg_comma to ensure the pattern matches the whole arg + arg_comma = comma | fixto(FollowedBy(rparen), "") + setarg_comma = arg_comma | fixto(FollowedBy(colon), "") + typedef_ref = setname + colon.suppress() + typedef_test + arg_comma + default = condense(equals + test) + unsafe_typedef_default_ref = setname + colon.suppress() + typedef_test + Optional(default) + typedef_default_ref = unsafe_typedef_default_ref + arg_comma + tfpdef = typedef | condense(setname + arg_comma) + tfpdef_default = typedef_default | condense(setname + Optional(default) + arg_comma) + + star_sep_arg = Forward() + star_sep_arg_ref = condense(star + arg_comma) + star_sep_setarg = Forward() + star_sep_setarg_ref = condense(star + setarg_comma) + + slash_sep_arg = Forward() + slash_sep_arg_ref = condense(slash + arg_comma) + slash_sep_setarg = Forward() + slash_sep_setarg_ref = condense(slash + setarg_comma) + + just_star = star + rparen + just_slash = slash + rparen + just_op = just_star | just_slash + + match = Forward() + args_list = ( + ~just_op + + addspace( + ZeroOrMore( + condense( + # everything here must end with arg_comma + (star | dubstar) + tfpdef + | star_sep_arg + | slash_sep_arg + | tfpdef_default + ) ) ) ) - ) - parameters = condense(lparen + args_list + rparen) - set_args_list = ( - ~just_op - + addspace( - ZeroOrMore( - condense( - # everything here must end with setarg_comma - (star | dubstar) + setname + setarg_comma - | star_sep_setarg - | slash_sep_setarg - | setname + Optional(default) + setarg_comma + parameters = condense(lparen + args_list + rparen) + set_args_list = ( + ~just_op + + addspace( + ZeroOrMore( + condense( + # everything here must end with setarg_comma + (star | dubstar) + setname + setarg_comma + | star_sep_setarg + | slash_sep_setarg + | setname + Optional(default) + setarg_comma + ) ) ) ) - ) - match_args_list = Group(Optional( - tokenlist( - Group( - (star | dubstar) + match - | star # not star_sep because pattern-matching can handle star separators on any Python version - | slash # not slash_sep as above - | match + Optional(equals.suppress() + test) - ), - comma, + match_args_list = Group(Optional( + tokenlist( + Group( + (star | dubstar) + match + | star # not star_sep because pattern-matching can handle star separators on any Python version + | slash # not slash_sep as above + | match + Optional(equals.suppress() + test) + ), + comma, + ) + )) + + call_item = ( + unsafe_name + default + | dubstar + test + | star + test + | ellipsis_tokens + equals.suppress() + refname + | namedexpr_test + ) + function_call_tokens = lparen.suppress() + ( + # everything here must end with rparen + rparen.suppress() + | tokenlist(Group(call_item), comma) + rparen.suppress() + | Group(attach(addspace(test + comp_for), add_parens_handle)) + rparen.suppress() + | Group(op_item) + rparen.suppress() + ) + function_call = Forward() + questionmark_call_tokens = Group( + tokenlist( + Group( + questionmark + | unsafe_name + condense(equals + questionmark) + | call_item + ), + comma, + ) + ) + methodcaller_args = ( + itemlist(condense(call_item), comma) + | op_item ) - )) - call_item = ( - unsafe_name + default - | dubstar + test - | star + test - | ellipsis_tokens + equals.suppress() + refname - | namedexpr_test - ) - function_call_tokens = lparen.suppress() + ( - # everything here must end with rparen - rparen.suppress() - | tokenlist(Group(call_item), comma) + rparen.suppress() - | Group(attach(addspace(test + comp_for), add_parens_handle)) + rparen.suppress() - | Group(op_item) + rparen.suppress() - ) - function_call = Forward() - questionmark_call_tokens = Group( - tokenlist( + subscript_star = Forward() + subscript_star_ref = star + slicetest = Optional(test_no_chain) + sliceop = condense(unsafe_colon + slicetest) + subscript = condense( + slicetest + sliceop + Optional(sliceop) + | Optional(subscript_star) + test + ) + subscriptlist = itemlist(subscript, comma, suppress_trailing=False) | new_namedexpr_test + + slicetestgroup = Optional(test_no_chain, default="") + sliceopgroup = unsafe_colon.suppress() + slicetestgroup + subscriptgroup = attach( + slicetestgroup + sliceopgroup + Optional(sliceopgroup) + | test, + subscriptgroup_handle, + ) + subscriptgrouplist = itemlist(subscriptgroup, comma) + + anon_namedtuple = Forward() + maybe_typedef = Optional(colon.suppress() + typedef_test) + anon_namedtuple_ref = tokenlist( Group( - questionmark - | unsafe_name + condense(equals + questionmark) - | call_item + unsafe_name + maybe_typedef + equals.suppress() + test + | ellipsis_tokens + maybe_typedef + equals.suppress() + refname ), comma, ) - ) - methodcaller_args = ( - itemlist(condense(call_item), comma) - | op_item - ) - - subscript_star = Forward() - subscript_star_ref = star - slicetest = Optional(test_no_chain) - sliceop = condense(unsafe_colon + slicetest) - subscript = condense( - slicetest + sliceop + Optional(sliceop) - | Optional(subscript_star) + test - ) - subscriptlist = itemlist(subscript, comma, suppress_trailing=False) | new_namedexpr_test - - slicetestgroup = Optional(test_no_chain, default="") - sliceopgroup = unsafe_colon.suppress() + slicetestgroup - subscriptgroup = attach( - slicetestgroup + sliceopgroup + Optional(sliceopgroup) - | test, - subscriptgroup_handle, - ) - subscriptgrouplist = itemlist(subscriptgroup, comma) - - anon_namedtuple = Forward() - maybe_typedef = Optional(colon.suppress() + typedef_test) - anon_namedtuple_ref = tokenlist( - Group( - unsafe_name + maybe_typedef + equals.suppress() + test - | ellipsis_tokens + maybe_typedef + equals.suppress() + refname - ), - comma, - ) - comprehension_expr = ( - addspace(namedexpr_test + comp_for) - | invalid_syntax(star_expr + comp_for, "iterable unpacking cannot be used in comprehension") - ) - paren_atom = condense( - lparen + ( - # everything here must end with rparen - rparen - | yield_expr + rparen - | comprehension_expr + rparen - | testlist_star_namedexpr + rparen - | op_item + rparen - | anon_namedtuple + rparen - ) | ( - lparen.suppress() - + typedef_tuple - + rparen.suppress() + comprehension_expr = ( + addspace(namedexpr_test + comp_for) + | invalid_syntax(star_expr + comp_for, "iterable unpacking cannot be used in comprehension") + ) + paren_atom = condense( + lparen + ( + # everything here must end with rparen + rparen + | yield_expr + rparen + | comprehension_expr + rparen + | testlist_star_namedexpr + rparen + | op_item + rparen + | anon_namedtuple + rparen + ) | ( + lparen.suppress() + + typedef_tuple + + rparen.suppress() + ) ) - ) - list_expr = Forward() - list_expr_ref = testlist_star_namedexpr_tokens - array_literal = attach( - lbrack.suppress() + OneOrMore( - multisemicolon - | attach(comprehension_expr, add_bracks_handle) - | namedexpr_test + ~comma - | list_expr - ) + rbrack.suppress(), - array_literal_handle, - ) - list_item = ( - condense(lbrack + Optional(comprehension_expr) + rbrack) - | lbrack.suppress() + list_expr + rbrack.suppress() - | array_literal - ) + list_expr = Forward() + list_expr_ref = testlist_star_namedexpr_tokens + array_literal = attach( + lbrack.suppress() + OneOrMore( + multisemicolon + | attach(comprehension_expr, add_bracks_handle) + | namedexpr_test + ~comma + | list_expr + ) + rbrack.suppress(), + array_literal_handle, + ) + list_item = ( + condense(lbrack + Optional(comprehension_expr) + rbrack) + | lbrack.suppress() + list_expr + rbrack.suppress() + | array_literal + ) - string_atom = Forward() - string_atom_ref = OneOrMore(nonb_string) | OneOrMore(b_string) - fixed_len_string_tokens = OneOrMore(nonbf_string) | OneOrMore(b_string) - f_string_atom = Forward() - f_string_atom_ref = ZeroOrMore(nonbf_string) + f_string + ZeroOrMore(nonb_string) - - keyword_atom = any_keyword_in(const_vars) - passthrough_atom = addspace(OneOrMore(passthrough_item)) - - set_literal = Forward() - set_letter_literal = Forward() - set_s = caseless_literal("s") - set_f = caseless_literal("f") - set_m = caseless_literal("m") - set_letter = set_s | set_f | set_m - setmaker = Group( - (new_namedexpr_test + FollowedBy(rbrace))("test") - | (new_namedexpr_testlist_has_comma + FollowedBy(rbrace))("list") - | addspace(new_namedexpr_test + comp_for + FollowedBy(rbrace))("comp") - | (testlist_star_namedexpr + FollowedBy(rbrace))("testlist_star_expr") - ) - set_literal_ref = lbrace.suppress() + setmaker + rbrace.suppress() - set_letter_literal_ref = combine(set_letter + lbrace.suppress()) + Optional(setmaker) + rbrace.suppress() - - lazy_items = Optional(tokenlist(test, comma)) - lazy_list = attach(lbanana.suppress() + lazy_items + rbanana.suppress(), lazy_list_handle) - - known_atom = ( - keyword_atom - | string_atom - | num_atom - | list_item - | dict_comp - | dict_literal - | set_literal - | set_letter_literal - | lazy_list - | typedef_ellipsis - | ellipsis - ) - atom = ( - # known_atom must come before name to properly parse string prefixes - known_atom - | refname - | paren_atom - | passthrough_atom - ) + string_atom = Forward() + string_atom_ref = OneOrMore(nonb_string) | OneOrMore(b_string) + fixed_len_string_tokens = OneOrMore(nonbf_string) | OneOrMore(b_string) + f_string_atom = Forward() + f_string_atom_ref = ZeroOrMore(nonbf_string) + f_string + ZeroOrMore(nonb_string) + + keyword_atom = any_keyword_in(const_vars) + passthrough_atom = addspace(OneOrMore(passthrough_item)) + + set_literal = Forward() + set_letter_literal = Forward() + set_s = caseless_literal("s") + set_f = caseless_literal("f") + set_m = caseless_literal("m") + set_letter = set_s | set_f | set_m + setmaker = Group( + (new_namedexpr_test + FollowedBy(rbrace))("test") + | (new_namedexpr_testlist_has_comma + FollowedBy(rbrace))("list") + | addspace(new_namedexpr_test + comp_for + FollowedBy(rbrace))("comp") + | (testlist_star_namedexpr + FollowedBy(rbrace))("testlist_star_expr") + ) + set_literal_ref = lbrace.suppress() + setmaker + rbrace.suppress() + set_letter_literal_ref = combine(set_letter + lbrace.suppress()) + Optional(setmaker) + rbrace.suppress() + + lazy_items = Optional(tokenlist(test, comma)) + lazy_list = attach(lbanana.suppress() + lazy_items + rbanana.suppress(), lazy_list_handle) + + known_atom = ( + keyword_atom + | string_atom + | num_atom + | list_item + | dict_comp + | dict_literal + | set_literal + | set_letter_literal + | lazy_list + | typedef_ellipsis + | ellipsis + ) + atom = ( + # known_atom must come before name to properly parse string prefixes + known_atom + | refname + | paren_atom + | passthrough_atom + ) - typedef_trailer = Forward() - typedef_or_expr = Forward() + typedef_trailer = Forward() + typedef_or_expr = Forward() - simple_trailer = ( - condense(dot + unsafe_name) - | condense(lbrack + subscriptlist + rbrack) - ) - call_trailer = ( - function_call - | invalid_syntax(dollar + questionmark, "'?' must come before '$' in None-coalescing partial application") - | Group(dollar + ~lparen + ~lbrack) # keep $ for item_handle - ) - known_trailer = typedef_trailer | ( - Group(condense(dollar + lbrack) + subscriptgroup + rbrack.suppress()) # $[ - | Group(condense(dollar + lbrack + rbrack)) # $[] - | Group(condense(lbrack + rbrack)) # [] - | Group(questionmark) # ? - | Group(dot + ~unsafe_name + ~lbrack + ~dot) # . - ) + ~questionmark - partial_trailer = ( - Group(fixto(dollar, "$(") + function_call) # $( - | Group(fixto(dollar + lparen, "$(?") + questionmark_call_tokens) + rparen.suppress() # $(? - ) + ~questionmark - partial_trailer_tokens = Group(dollar.suppress() + function_call_tokens) - - no_call_trailer = simple_trailer | known_trailer | partial_trailer - - no_partial_complex_trailer = call_trailer | known_trailer - no_partial_trailer = simple_trailer | no_partial_complex_trailer - - complex_trailer = no_partial_complex_trailer | partial_trailer - trailer = simple_trailer | complex_trailer - - attrgetter_atom_tokens = dot.suppress() + unsafe_dotted_name + Optional( - lparen + Optional(methodcaller_args) + rparen.suppress() - ) - attrgetter_atom = attach(attrgetter_atom_tokens, attrgetter_atom_handle) - - itemgetter_atom_tokens = ( - dot.suppress() - + Optional(unsafe_dotted_name) - + Group(OneOrMore(Group( - condense(Optional(dollar) + lbrack) - + subscriptgrouplist - + rbrack.suppress() - ))) - ) - itemgetter_atom = attach(itemgetter_atom_tokens, itemgetter_handle) + simple_trailer = ( + condense(dot + unsafe_name) + | condense(lbrack + subscriptlist + rbrack) + ) + call_trailer = ( + function_call + | invalid_syntax(dollar + questionmark, "'?' must come before '$' in None-coalescing partial application") + | Group(dollar + ~lparen + ~lbrack) # keep $ for item_handle + ) + known_trailer = typedef_trailer | ( + Group(condense(dollar + lbrack) + subscriptgroup + rbrack.suppress()) # $[ + | Group(condense(dollar + lbrack + rbrack)) # $[] + | Group(condense(lbrack + rbrack)) # [] + | Group(questionmark) # ? + | Group(dot + ~unsafe_name + ~lbrack + ~dot) # . + ) + ~questionmark + partial_trailer = ( + Group(fixto(dollar, "$(") + function_call) # $( + | Group(fixto(dollar + lparen, "$(?") + questionmark_call_tokens) + rparen.suppress() # $(? + ) + ~questionmark + partial_trailer_tokens = Group(dollar.suppress() + function_call_tokens) + + no_call_trailer = simple_trailer | known_trailer | partial_trailer + + no_partial_complex_trailer = call_trailer | known_trailer + no_partial_trailer = simple_trailer | no_partial_complex_trailer + + complex_trailer = no_partial_complex_trailer | partial_trailer + trailer = simple_trailer | complex_trailer + + attrgetter_atom_tokens = dot.suppress() + unsafe_dotted_name + Optional( + lparen + Optional(methodcaller_args) + rparen.suppress() + ) + attrgetter_atom = attach(attrgetter_atom_tokens, attrgetter_atom_handle) + + itemgetter_atom_tokens = ( + dot.suppress() + + Optional(unsafe_dotted_name) + + Group(OneOrMore(Group( + condense(Optional(dollar) + lbrack) + + subscriptgrouplist + + rbrack.suppress() + ))) + ) + itemgetter_atom = attach(itemgetter_atom_tokens, itemgetter_handle) - implicit_partial_atom = ( - itemgetter_atom - | attrgetter_atom - | fixto(dot + lbrack + rbrack, "_coconut.operator.getitem") - | fixto(dot + dollar + lbrack + rbrack, "_coconut_iter_getitem") - ) + implicit_partial_atom = ( + itemgetter_atom + | attrgetter_atom + | fixto(dot + lbrack + rbrack, "_coconut.operator.getitem") + | fixto(dot + dollar + lbrack + rbrack, "_coconut_iter_getitem") + ) - trailer_atom = Forward() - trailer_atom_ref = atom + ZeroOrMore(trailer) - atom_item <<= ( - trailer_atom - | implicit_partial_atom - ) + trailer_atom = Forward() + trailer_atom_ref = atom + ZeroOrMore(trailer) + atom_item <<= ( + trailer_atom + | implicit_partial_atom + ) - no_partial_trailer_atom = Forward() - no_partial_trailer_atom_ref = atom + ZeroOrMore(no_partial_trailer) - partial_atom_tokens = no_partial_trailer_atom + partial_trailer_tokens + no_partial_trailer_atom = Forward() + no_partial_trailer_atom_ref = atom + ZeroOrMore(no_partial_trailer) + partial_atom_tokens = no_partial_trailer_atom + partial_trailer_tokens - simple_assign = Forward() - simple_assign_ref = maybeparens( - lparen, - (setname | passthrough_atom) - + ZeroOrMore(ZeroOrMore(complex_trailer) + OneOrMore(simple_trailer)), - rparen, - ) - simple_assignlist = maybeparens(lparen, itemlist(simple_assign, comma, suppress_trailing=False), rparen) - - assignlist = Forward() - star_assign_item = Forward() - base_assign_item = condense( - simple_assign - | lparen + assignlist + rparen - | lbrack + assignlist + rbrack - ) - star_assign_item_ref = condense(star + base_assign_item) - assign_item = star_assign_item | base_assign_item - assignlist <<= itemlist(assign_item, comma, suppress_trailing=False) - - typed_assign_stmt = Forward() - typed_assign_stmt_ref = simple_assign + colon.suppress() + typedef_test + Optional(equals.suppress() + test_expr) - basic_stmt = addspace(ZeroOrMore(assignlist + equals) + test_expr) - - type_param = Forward() - type_param_bound_op = lt_colon | colon | le - type_var_name = stores_loc_item + setname - type_param_constraint = lparen.suppress() + Group(tokenlist(typedef_test, comma, require_sep=True)) + rparen.suppress() - type_param_ref = ( - (type_var_name + type_param_bound_op + type_param_constraint)("TypeVar constraint") - | (type_var_name + Optional(type_param_bound_op + typedef_test))("TypeVar") - | (star.suppress() + type_var_name)("TypeVarTuple") - | (dubstar.suppress() + type_var_name)("ParamSpec") - ) - type_params = Group(lbrack.suppress() + tokenlist(type_param, comma) + rbrack.suppress()) - - type_alias_stmt = Forward() - type_alias_stmt_ref = keyword("type").suppress() + setname + Optional(type_params) + equals.suppress() + typedef_test - - await_expr = Forward() - await_expr_ref = keyword("await").suppress() + atom_item - await_item = await_expr | atom_item - - factor = Forward() - unary = plus | neg_minus | tilde - - power = condense(exp_dubstar + ZeroOrMore(unary) + await_item) - power_in_impl_call = Forward() - - impl_call_arg = condense(( - keyword_atom - | number - | disallow_keywords(reserved_vars) + dotted_refname - ) + Optional(power_in_impl_call)) - impl_call_item = condense( - disallow_keywords(reserved_vars) - + ~any_string - + atom_item - + Optional(power_in_impl_call) - ) - impl_call = Forward() - # we need to disable this inside the xonsh parser - impl_call_ref = Forward() - unsafe_impl_call_ref = ( - impl_call_item + OneOrMore(impl_call_arg) - ) + simple_assign = Forward() + simple_assign_ref = maybeparens( + lparen, + (setname | passthrough_atom) + + ZeroOrMore(ZeroOrMore(complex_trailer) + OneOrMore(simple_trailer)), + rparen, + ) + simple_assignlist = maybeparens(lparen, itemlist(simple_assign, comma, suppress_trailing=False), rparen) + + assignlist = Forward() + star_assign_item = Forward() + base_assign_item = condense( + simple_assign + | lparen + assignlist + rparen + | lbrack + assignlist + rbrack + ) + star_assign_item_ref = condense(star + base_assign_item) + assign_item = star_assign_item | base_assign_item + assignlist <<= itemlist(assign_item, comma, suppress_trailing=False) + + typed_assign_stmt = Forward() + typed_assign_stmt_ref = simple_assign + colon.suppress() + typedef_test + Optional(equals.suppress() + test_expr) + basic_stmt = addspace(ZeroOrMore(assignlist + equals) + test_expr) + + type_param = Forward() + type_param_bound_op = lt_colon | colon | le + type_var_name = stores_loc_item + setname + type_param_constraint = lparen.suppress() + Group(tokenlist(typedef_test, comma, require_sep=True)) + rparen.suppress() + type_param_ref = ( + (type_var_name + type_param_bound_op + type_param_constraint)("TypeVar constraint") + | (type_var_name + Optional(type_param_bound_op + typedef_test))("TypeVar") + | (star.suppress() + type_var_name)("TypeVarTuple") + | (dubstar.suppress() + type_var_name)("ParamSpec") + ) + type_params = Group(lbrack.suppress() + tokenlist(type_param, comma) + rbrack.suppress()) + + type_alias_stmt = Forward() + type_alias_stmt_ref = keyword("type").suppress() + setname + Optional(type_params) + equals.suppress() + typedef_test + + await_expr = Forward() + await_expr_ref = keyword("await").suppress() + atom_item + await_item = await_expr | atom_item + + factor = Forward() + unary = plus | neg_minus | tilde + + power = condense(exp_dubstar + ZeroOrMore(unary) + await_item) + power_in_impl_call = Forward() + + impl_call_arg = condense(( + keyword_atom + | number + | disallow_keywords(reserved_vars) + dotted_refname + ) + Optional(power_in_impl_call)) + impl_call_item = condense( + disallow_keywords(reserved_vars) + + ~any_string + + atom_item + + Optional(power_in_impl_call) + ) + impl_call = Forward() + # we need to disable this inside the xonsh parser + impl_call_ref = Forward() + unsafe_impl_call_ref = ( + impl_call_item + OneOrMore(impl_call_arg) + ) - factor <<= condense( - ZeroOrMore(unary) + ( - impl_call - | await_item + Optional(power) + factor <<= condense( + ZeroOrMore(unary) + ( + impl_call + | await_item + Optional(power) + ) ) - ) - mulop = mul_star | div_slash | div_dubslash | percent | matrix_at - addop = plus | sub_minus - shift = lshift | rshift - - term = Forward() - term_ref = tokenlist(factor, mulop, allow_trailing=False, suppress=False) - - # we condense all of these down, since Python handles the precedence, not Coconut - # arith_expr = exprlist(term, addop) - # shift_expr = exprlist(arith_expr, shift) - # and_expr = exprlist(shift_expr, amp) - and_expr = exprlist( - term, - addop - | shift - | amp, - ) + mulop = mul_star | div_slash | div_dubslash | percent | matrix_at + addop = plus | sub_minus + shift = lshift | rshift + + term = Forward() + term_ref = tokenlist(factor, mulop, allow_trailing=False, suppress=False) + + # we condense all of these down, since Python handles the precedence, not Coconut + # arith_expr = exprlist(term, addop) + # shift_expr = exprlist(arith_expr, shift) + # and_expr = exprlist(shift_expr, amp) + and_expr = exprlist( + term, + addop + | shift + | amp, + ) - protocol_intersect_expr = Forward() - protocol_intersect_expr_ref = tokenlist(and_expr, amp_colon, allow_trailing=False) + protocol_intersect_expr = Forward() + protocol_intersect_expr_ref = tokenlist(and_expr, amp_colon, allow_trailing=False) - xor_expr = exprlist(protocol_intersect_expr, caret) + xor_expr = exprlist(protocol_intersect_expr, caret) - or_expr = typedef_or_expr | exprlist(xor_expr, bar) + or_expr = typedef_or_expr | exprlist(xor_expr, bar) - chain_expr = attach(tokenlist(or_expr, dubcolon, allow_trailing=False), chain_handle) + chain_expr = attach(tokenlist(or_expr, dubcolon, allow_trailing=False), chain_handle) - compose_expr = attach( - tokenlist( - chain_expr, - dotdot + Optional(invalid_syntax(lambdef, "lambdas only allowed after composition pipe operators '..>' and '<..', not '..' (replace '..' with '<..' to fix)")), - allow_trailing=False, - ), compose_expr_handle, - ) + compose_expr = attach( + tokenlist( + chain_expr, + dotdot + Optional(invalid_syntax(lambdef, "lambdas only allowed after composition pipe operators '..>' and '<..', not '..' (replace '..' with '<..' to fix)")), + allow_trailing=False, + ), compose_expr_handle, + ) - infix_op <<= backtick.suppress() + test_no_infix + backtick.suppress() - infix_expr = Forward() - infix_item = attach( - Group(Optional(compose_expr)) - + OneOrMore( - infix_op + Group(Optional(lambdef | compose_expr)) - ), - infix_handle, - ) - infix_expr <<= ( - compose_expr + ~backtick - | infix_item - ) + infix_op <<= backtick.suppress() + test_no_infix + backtick.suppress() + infix_expr = Forward() + infix_item = attach( + Group(Optional(compose_expr)) + + OneOrMore( + infix_op + Group(Optional(lambdef | compose_expr)) + ), + infix_handle, + ) + infix_expr <<= ( + compose_expr + ~backtick + | infix_item + ) - none_coalesce_expr = attach(tokenlist(infix_expr, dubquestion, allow_trailing=False), none_coalesce_handle) - - comp_pipe_op = ( - comp_pipe - | comp_star_pipe - | comp_back_pipe - | comp_back_star_pipe - | comp_dubstar_pipe - | comp_back_dubstar_pipe - | comp_none_dubstar_pipe - | comp_back_none_dubstar_pipe - | comp_none_star_pipe - | comp_back_none_star_pipe - | comp_none_pipe - | comp_back_none_pipe - ) - comp_pipe_item = attach( - OneOrMore(none_coalesce_expr + comp_pipe_op) + (lambdef | none_coalesce_expr), - comp_pipe_handle, - ) - comp_pipe_expr = ( - none_coalesce_expr + ~comp_pipe_op - | comp_pipe_item - ) + none_coalesce_expr = attach(tokenlist(infix_expr, dubquestion, allow_trailing=False), none_coalesce_handle) + + comp_pipe_op = ( + comp_pipe + | comp_star_pipe + | comp_back_pipe + | comp_back_star_pipe + | comp_dubstar_pipe + | comp_back_dubstar_pipe + | comp_none_dubstar_pipe + | comp_back_none_dubstar_pipe + | comp_none_star_pipe + | comp_back_none_star_pipe + | comp_none_pipe + | comp_back_none_pipe + ) + comp_pipe_item = attach( + OneOrMore(none_coalesce_expr + comp_pipe_op) + (lambdef | none_coalesce_expr), + comp_pipe_handle, + ) + comp_pipe_expr = ( + none_coalesce_expr + ~comp_pipe_op + | comp_pipe_item + ) - pipe_op = ( - pipe - | star_pipe - | dubstar_pipe - | back_pipe - | back_star_pipe - | back_dubstar_pipe - | none_pipe - | none_star_pipe - | none_dubstar_pipe - | back_none_pipe - | back_none_star_pipe - | back_none_dubstar_pipe - ) - pipe_item = ( - # we need the pipe_op since any of the atoms could otherwise be the start of an expression - labeled_group(keyword("await"), "await") + pipe_op - | labeled_group(itemgetter_atom_tokens, "itemgetter") + pipe_op - | labeled_group(attrgetter_atom_tokens, "attrgetter") + pipe_op - | labeled_group(partial_atom_tokens, "partial") + pipe_op - | labeled_group(partial_op_atom_tokens, "op partial") + pipe_op - # expr must come at end - | labeled_group(comp_pipe_expr, "expr") + pipe_op - ) - pipe_augassign_item = ( - # should match pipe_item but with pipe_op -> end_simple_stmt_item and no expr - labeled_group(keyword("await"), "await") + end_simple_stmt_item - | labeled_group(itemgetter_atom_tokens, "itemgetter") + end_simple_stmt_item - | labeled_group(attrgetter_atom_tokens, "attrgetter") + end_simple_stmt_item - | labeled_group(partial_atom_tokens, "partial") + end_simple_stmt_item - | labeled_group(partial_op_atom_tokens, "op partial") + end_simple_stmt_item - ) - last_pipe_item = Group( - lambdef("expr") - # we need longest here because there's no following pipe_op we can use as above - | longest( - keyword("await")("await"), - itemgetter_atom_tokens("itemgetter"), - attrgetter_atom_tokens("attrgetter"), - partial_atom_tokens("partial"), - partial_op_atom_tokens("op partial"), - comp_pipe_expr("expr"), + pipe_op = ( + pipe + | star_pipe + | dubstar_pipe + | back_pipe + | back_star_pipe + | back_dubstar_pipe + | none_pipe + | none_star_pipe + | none_dubstar_pipe + | back_none_pipe + | back_none_star_pipe + | back_none_dubstar_pipe ) - ) - normal_pipe_expr = Forward() - normal_pipe_expr_tokens = OneOrMore(pipe_item) + last_pipe_item + pipe_item = ( + # we need the pipe_op since any of the atoms could otherwise be the start of an expression + labeled_group(keyword("await"), "await") + pipe_op + | labeled_group(itemgetter_atom_tokens, "itemgetter") + pipe_op + | labeled_group(attrgetter_atom_tokens, "attrgetter") + pipe_op + | labeled_group(partial_atom_tokens, "partial") + pipe_op + | labeled_group(partial_op_atom_tokens, "op partial") + pipe_op + # expr must come at end + | labeled_group(comp_pipe_expr, "expr") + pipe_op + ) + pipe_augassign_item = ( + # should match pipe_item but with pipe_op -> end_simple_stmt_item and no expr + labeled_group(keyword("await"), "await") + end_simple_stmt_item + | labeled_group(itemgetter_atom_tokens, "itemgetter") + end_simple_stmt_item + | labeled_group(attrgetter_atom_tokens, "attrgetter") + end_simple_stmt_item + | labeled_group(partial_atom_tokens, "partial") + end_simple_stmt_item + | labeled_group(partial_op_atom_tokens, "op partial") + end_simple_stmt_item + ) + last_pipe_item = Group( + lambdef("expr") + # we need longest here because there's no following pipe_op we can use as above + | longest( + keyword("await")("await"), + itemgetter_atom_tokens("itemgetter"), + attrgetter_atom_tokens("attrgetter"), + partial_atom_tokens("partial"), + partial_op_atom_tokens("op partial"), + comp_pipe_expr("expr"), + ) + ) + normal_pipe_expr = Forward() + normal_pipe_expr_tokens = OneOrMore(pipe_item) + last_pipe_item - pipe_expr = ( - comp_pipe_expr + ~pipe_op - | normal_pipe_expr - ) + pipe_expr = ( + comp_pipe_expr + ~pipe_op + | normal_pipe_expr + ) - expr <<= pipe_expr - - # though 3.9 allows tests in the grammar here, they still raise a SyntaxError later - star_expr <<= Group(star + expr) - dubstar_expr <<= Group(dubstar + expr) - - comparison = exprlist(expr, comp_op) - not_test = addspace(ZeroOrMore(keyword("not")) + comparison) - # we condense "and" and "or" into one, since Python handles the precedence, not Coconut - # and_test = exprlist(not_test, keyword("and")) - # test_item = exprlist(and_test, keyword("or")) - test_item = exprlist(not_test, keyword("and") | keyword("or")) - - simple_stmt_item = Forward() - unsafe_simple_stmt_item = Forward() - simple_stmt = Forward() - stmt = Forward() - suite = Forward() - nocolon_suite = Forward() - base_suite = Forward() - - fat_arrow = Forward() - lambda_arrow = Forward() - unsafe_lambda_arrow = fat_arrow | arrow - - keyword_lambdef_params = maybeparens(lparen, set_args_list, rparen) - arrow_lambdef_params = lparen.suppress() + set_args_list + rparen.suppress() | setname - - keyword_lambdef = Forward() - keyword_lambdef_ref = addspace(keyword("lambda") + condense(keyword_lambdef_params + colon)) - arrow_lambdef = attach(arrow_lambdef_params + lambda_arrow.suppress(), lambdef_handle) - implicit_lambdef = fixto(lambda_arrow, "lambda _=None:") - lambdef_base = keyword_lambdef | arrow_lambdef | implicit_lambdef - - stmt_lambdef = Forward() - match_guard = Optional(keyword("if").suppress() + namedexpr_test) - closing_stmt = longest(new_testlist_star_expr("tests"), unsafe_simple_stmt_item) - stmt_lambdef_match_params = Group(lparen.suppress() + match_args_list + match_guard + rparen.suppress()) - stmt_lambdef_params = Optional( - attach(setname, add_parens_handle) - | parameters - | stmt_lambdef_match_params, - default="(_=None)", - ) - stmt_lambdef_body = Group( - Group(OneOrMore(simple_stmt_item + semicolon.suppress())) + Optional(closing_stmt) - | Group(ZeroOrMore(simple_stmt_item + semicolon.suppress())) + closing_stmt, - ) + expr <<= pipe_expr + + # though 3.9 allows tests in the grammar here, they still raise a SyntaxError later + star_expr <<= Group(star + expr) + dubstar_expr <<= Group(dubstar + expr) + + comparison = exprlist(expr, comp_op) + not_test = addspace(ZeroOrMore(keyword("not")) + comparison) + # we condense "and" and "or" into one, since Python handles the precedence, not Coconut + # and_test = exprlist(not_test, keyword("and")) + # test_item = exprlist(and_test, keyword("or")) + test_item = exprlist(not_test, keyword("and") | keyword("or")) + + simple_stmt_item = Forward() + unsafe_simple_stmt_item = Forward() + simple_stmt = Forward() + stmt = Forward() + suite = Forward() + nocolon_suite = Forward() + base_suite = Forward() + + fat_arrow = Forward() + lambda_arrow = Forward() + unsafe_lambda_arrow = fat_arrow | arrow + + keyword_lambdef_params = maybeparens(lparen, set_args_list, rparen) + arrow_lambdef_params = lparen.suppress() + set_args_list + rparen.suppress() | setname + + keyword_lambdef = Forward() + keyword_lambdef_ref = addspace(keyword("lambda") + condense(keyword_lambdef_params + colon)) + arrow_lambdef = attach(arrow_lambdef_params + lambda_arrow.suppress(), lambdef_handle) + implicit_lambdef = fixto(lambda_arrow, "lambda _=None:") + lambdef_base = keyword_lambdef | arrow_lambdef | implicit_lambdef + + stmt_lambdef = Forward() + match_guard = Optional(keyword("if").suppress() + namedexpr_test) + closing_stmt = longest(new_testlist_star_expr("tests"), unsafe_simple_stmt_item) + stmt_lambdef_match_params = Group(lparen.suppress() + match_args_list + match_guard + rparen.suppress()) + stmt_lambdef_params = Optional( + attach(setname, add_parens_handle) + | parameters + | stmt_lambdef_match_params, + default="(_=None)", + ) + stmt_lambdef_body = Group( + Group(OneOrMore(simple_stmt_item + semicolon.suppress())) + Optional(closing_stmt) + | Group(ZeroOrMore(simple_stmt_item + semicolon.suppress())) + closing_stmt, + ) - no_fat_arrow_stmt_lambdef_body, _fat_arrow = disable_inside(stmt_lambdef_body, unsafe_fat_arrow) - fat_arrow <<= _fat_arrow - stmt_lambdef_suite = ( - arrow.suppress() + no_fat_arrow_stmt_lambdef_body + ~fat_arrow - | Optional(arrow.suppress() + typedef_test) + fat_arrow.suppress() + stmt_lambdef_body - ) + no_fat_arrow_stmt_lambdef_body, _fat_arrow = disable_inside(stmt_lambdef_body, unsafe_fat_arrow) + fat_arrow <<= _fat_arrow + stmt_lambdef_suite = ( + arrow.suppress() + no_fat_arrow_stmt_lambdef_body + ~fat_arrow + | Optional(arrow.suppress() + typedef_test) + fat_arrow.suppress() + stmt_lambdef_body + ) - general_stmt_lambdef = ( - Group(any_len_perm( - keyword("async"), - keyword("copyclosure"), - )) + keyword("def").suppress() - + stmt_lambdef_params - + stmt_lambdef_suite - ) - match_stmt_lambdef = ( - Group(any_len_perm( - keyword("match").suppress(), - keyword("async"), - keyword("copyclosure"), - )) + keyword("def").suppress() - + stmt_lambdef_match_params - + stmt_lambdef_suite - ) - stmt_lambdef_ref = trace( - general_stmt_lambdef - | match_stmt_lambdef - ) + ( - fixto(FollowedBy(comma), ",") - | fixto(always_match, "") - ) + general_stmt_lambdef = ( + Group(any_len_perm( + keyword("async"), + keyword("copyclosure"), + )) + keyword("def").suppress() + + stmt_lambdef_params + + stmt_lambdef_suite + ) + match_stmt_lambdef = ( + Group(any_len_perm( + keyword("match").suppress(), + keyword("async"), + keyword("copyclosure"), + )) + keyword("def").suppress() + + stmt_lambdef_match_params + + stmt_lambdef_suite + ) + stmt_lambdef_ref = trace( + general_stmt_lambdef + | match_stmt_lambdef + ) + ( + fixto(FollowedBy(comma), ",") + | fixto(always_match, "") + ) - lambdef <<= addspace(lambdef_base + test) | stmt_lambdef - lambdef_no_cond = addspace(lambdef_base + test_no_cond) + lambdef <<= addspace(lambdef_base + test) | stmt_lambdef + lambdef_no_cond = addspace(lambdef_base + test_no_cond) - typedef_callable_arg = Group( - test("arg") - | (dubstar.suppress() + refname)("paramspec") - ) - typedef_callable_params = Optional(Group( - labeled_group(maybeparens(lparen, ellipsis_tokens, rparen), "ellipsis") - | lparen.suppress() + Optional(tokenlist(typedef_callable_arg, comma)) + rparen.suppress() - | labeled_group(negable_atom_item, "arg") - )) - unsafe_typedef_callable = attach( - Optional(keyword("async"), default="") - + typedef_callable_params - + arrow.suppress() - + typedef_test, - typedef_callable_handle, - ) + typedef_callable_arg = Group( + test("arg") + | (dubstar.suppress() + refname)("paramspec") + ) + typedef_callable_params = Optional(Group( + labeled_group(maybeparens(lparen, ellipsis_tokens, rparen), "ellipsis") + | lparen.suppress() + Optional(tokenlist(typedef_callable_arg, comma)) + rparen.suppress() + | labeled_group(negable_atom_item, "arg") + )) + unsafe_typedef_callable = attach( + Optional(keyword("async"), default="") + + typedef_callable_params + + arrow.suppress() + + typedef_test, + typedef_callable_handle, + ) - unsafe_typedef_trailer = ( # use special type signifier for item_handle - Group(fixto(lbrack + rbrack, "type:[]")) - | Group(fixto(dollar + lbrack + rbrack, "type:$[]")) - | Group(fixto(questionmark + ~questionmark, "type:?")) - ) + unsafe_typedef_trailer = ( # use special type signifier for item_handle + Group(fixto(lbrack + rbrack, "type:[]")) + | Group(fixto(dollar + lbrack + rbrack, "type:$[]")) + | Group(fixto(questionmark + ~questionmark, "type:?")) + ) - unsafe_typedef_or_expr = Forward() - unsafe_typedef_or_expr_ref = tokenlist(xor_expr, bar, allow_trailing=False, at_least_two=True) + unsafe_typedef_or_expr = Forward() + unsafe_typedef_or_expr_ref = tokenlist(xor_expr, bar, allow_trailing=False, at_least_two=True) - unsafe_typedef_tuple = Forward() - # should mimic testlist_star_namedexpr but with require_sep=True - unsafe_typedef_tuple_ref = tokenlist(Group(namedexpr_test) | star_expr, fixto(semicolon, ","), suppress=False, require_sep=True) + unsafe_typedef_tuple = Forward() + # should mimic testlist_star_namedexpr but with require_sep=True + unsafe_typedef_tuple_ref = tokenlist(Group(namedexpr_test) | star_expr, fixto(semicolon, ","), suppress=False, require_sep=True) - unsafe_typedef_ellipsis = ellipsis_tokens + unsafe_typedef_ellipsis = ellipsis_tokens - unsafe_typedef_op_item = attach(base_op_item, typedef_op_item_handle) + unsafe_typedef_op_item = attach(base_op_item, typedef_op_item_handle) - unsafe_typedef_test, typedef_callable, _typedef_trailer, _typedef_or_expr, _typedef_tuple, _typedef_ellipsis, _typedef_op_item = disable_outside( - test, - unsafe_typedef_callable, - unsafe_typedef_trailer, - unsafe_typedef_or_expr, - unsafe_typedef_tuple, - unsafe_typedef_ellipsis, - unsafe_typedef_op_item, - ) - typedef_trailer <<= _typedef_trailer - typedef_or_expr <<= _typedef_or_expr - typedef_tuple <<= _typedef_tuple - typedef_ellipsis <<= _typedef_ellipsis - typedef_op_item <<= _typedef_op_item - - _typedef_test, _lambda_arrow = disable_inside( - unsafe_typedef_test, - unsafe_lambda_arrow, - ) - typedef_test <<= _typedef_test - lambda_arrow <<= _lambda_arrow - - alt_ternary_expr = attach(keyword("if").suppress() + test_item + keyword("then").suppress() + test_item + keyword("else").suppress() + test, alt_ternary_handle) - test <<= ( - typedef_callable - | lambdef - | alt_ternary_expr - | addspace(test_item + Optional(keyword("if") + test_item + keyword("else") + test)) # must come last since it includes plain test_item - ) - test_no_cond <<= lambdef_no_cond | test_item + unsafe_typedef_test, typedef_callable, _typedef_trailer, _typedef_or_expr, _typedef_tuple, _typedef_ellipsis, _typedef_op_item = disable_outside( + test, + unsafe_typedef_callable, + unsafe_typedef_trailer, + unsafe_typedef_or_expr, + unsafe_typedef_tuple, + unsafe_typedef_ellipsis, + unsafe_typedef_op_item, + ) + typedef_trailer <<= _typedef_trailer + typedef_or_expr <<= _typedef_or_expr + typedef_tuple <<= _typedef_tuple + typedef_ellipsis <<= _typedef_ellipsis + typedef_op_item <<= _typedef_op_item + + _typedef_test, _lambda_arrow = disable_inside( + unsafe_typedef_test, + unsafe_lambda_arrow, + ) + typedef_test <<= _typedef_test + lambda_arrow <<= _lambda_arrow + + alt_ternary_expr = attach(keyword("if").suppress() + test_item + keyword("then").suppress() + test_item + keyword("else").suppress() + test, alt_ternary_handle) + test <<= ( + typedef_callable + | lambdef + | alt_ternary_expr + | addspace(test_item + Optional(keyword("if") + test_item + keyword("else") + test)) # must come last since it includes plain test_item + ) + test_no_cond <<= lambdef_no_cond | test_item - namedexpr = Forward() - namedexpr_ref = addspace( - setname + colon_eq + ( + namedexpr = Forward() + namedexpr_ref = addspace( + setname + colon_eq + ( + test + ~colon_eq + | attach(namedexpr, add_parens_handle) + ) + ) + namedexpr_test <<= ( test + ~colon_eq - | attach(namedexpr, add_parens_handle) + | namedexpr ) - ) - namedexpr_test <<= ( - test + ~colon_eq - | namedexpr - ) - new_namedexpr = Forward() - new_namedexpr_ref = namedexpr_ref - new_namedexpr_test <<= ( - test + ~colon_eq - | new_namedexpr - ) - - classdef = Forward() - decorators = Forward() - classlist = Group( - Optional(function_call_tokens) - + ~equals, # don't match class destructuring assignment - ) - class_suite = suite | attach(newline, class_suite_handle) - classdef_ref = ( - Optional(decorators, default="") - + keyword("class").suppress() - + classname - + Optional(type_params, default=()) - + classlist - + class_suite - ) - - async_comp_for = Forward() - comp_iter = Forward() - comp_it_item = ( - invalid_syntax(maybeparens(lparen, namedexpr, rparen), "PEP 572 disallows assignment expressions in comprehension iterable expressions") - | test_item - ) - base_comp_for = addspace(keyword("for") + assignlist + keyword("in") + comp_it_item + Optional(comp_iter)) - async_comp_for_ref = addspace(keyword("async") + base_comp_for) - comp_for <<= async_comp_for | base_comp_for - comp_if = addspace(keyword("if") + test_no_cond + Optional(comp_iter)) - comp_iter <<= comp_for | comp_if - - return_stmt = addspace(keyword("return") - Optional(new_testlist_star_expr)) - - complex_raise_stmt = Forward() - pass_stmt = keyword("pass") - break_stmt = keyword("break") - continue_stmt = keyword("continue") - simple_raise_stmt = addspace(keyword("raise") + Optional(test)) - complex_raise_stmt_ref = keyword("raise").suppress() + test + keyword("from").suppress() - test - raise_stmt = complex_raise_stmt | simple_raise_stmt - flow_stmt = ( - return_stmt - | raise_stmt - | break_stmt - | yield_expr - | continue_stmt - ) + new_namedexpr = Forward() + new_namedexpr_ref = namedexpr_ref + new_namedexpr_test <<= ( + test + ~colon_eq + | new_namedexpr + ) - imp_name = ( - # maybeparens allows for using custom operator names here - maybeparens(lparen, setname, rparen) - | passthrough_item - ) - unsafe_imp_name = ( - # should match imp_name except with unsafe_name instead of setname - maybeparens(lparen, unsafe_name, rparen) - | passthrough_item - ) - dotted_imp_name = ( - dotted_setname - | passthrough_item - ) - unsafe_dotted_imp_name = ( - # should match dotted_imp_name except with unsafe_dotted_name - unsafe_dotted_name - | passthrough_item - ) - imp_as = keyword("as").suppress() - imp_name - import_item = Group( - unsafe_dotted_imp_name + imp_as - | dotted_imp_name - ) - from_import_item = Group( - unsafe_imp_name + imp_as - | imp_name - ) + classdef = Forward() + decorators = Forward() + classlist = Group( + Optional(function_call_tokens) + + ~equals, # don't match class destructuring assignment + ) + class_suite = suite | attach(newline, class_suite_handle) + classdef_ref = ( + Optional(decorators, default="") + + keyword("class").suppress() + + classname + + Optional(type_params, default=()) + + classlist + + class_suite + ) - import_names = Group( - maybeparens(lparen, tokenlist(import_item, comma), rparen) - | star - ) - from_import_names = Group( - maybeparens(lparen, tokenlist(from_import_item, comma), rparen) - | star - ) - basic_import = keyword("import").suppress() - import_names - import_from_name = condense( - ZeroOrMore(unsafe_dot) + unsafe_dotted_name - | OneOrMore(unsafe_dot) - | star - ) - from_import = ( - keyword("from").suppress() - - import_from_name - - keyword("import").suppress() - from_import_names - ) - import_stmt = Forward() - import_stmt_ref = from_import | basic_import + async_comp_for = Forward() + comp_iter = Forward() + comp_it_item = ( + invalid_syntax(maybeparens(lparen, namedexpr, rparen), "PEP 572 disallows assignment expressions in comprehension iterable expressions") + | test_item + ) + base_comp_for = addspace(keyword("for") + assignlist + keyword("in") + comp_it_item + Optional(comp_iter)) + async_comp_for_ref = addspace(keyword("async") + base_comp_for) + comp_for <<= async_comp_for | base_comp_for + comp_if = addspace(keyword("if") + test_no_cond + Optional(comp_iter)) + comp_iter <<= comp_for | comp_if + + return_stmt = addspace(keyword("return") - Optional(new_testlist_star_expr)) + + complex_raise_stmt = Forward() + pass_stmt = keyword("pass") + break_stmt = keyword("break") + continue_stmt = keyword("continue") + simple_raise_stmt = addspace(keyword("raise") + Optional(test)) + complex_raise_stmt_ref = keyword("raise").suppress() + test + keyword("from").suppress() - test + raise_stmt = complex_raise_stmt | simple_raise_stmt + flow_stmt = ( + return_stmt + | raise_stmt + | break_stmt + | yield_expr + | continue_stmt + ) - augassign_stmt = Forward() - augassign_rhs = ( - labeled_group(pipe_augassign + pipe_augassign_item, "pipe") - | labeled_group(augassign + test_expr, "simple") - ) - augassign_stmt_ref = simple_assign + augassign_rhs + imp_name = ( + # maybeparens allows for using custom operator names here + maybeparens(lparen, setname, rparen) + | passthrough_item + ) + unsafe_imp_name = ( + # should match imp_name except with unsafe_name instead of setname + maybeparens(lparen, unsafe_name, rparen) + | passthrough_item + ) + dotted_imp_name = ( + dotted_setname + | passthrough_item + ) + unsafe_dotted_imp_name = ( + # should match dotted_imp_name except with unsafe_dotted_name + unsafe_dotted_name + | passthrough_item + ) + imp_as = keyword("as").suppress() - imp_name + import_item = Group( + unsafe_dotted_imp_name + imp_as + | dotted_imp_name + ) + from_import_item = Group( + unsafe_imp_name + imp_as + | imp_name + ) - simple_kwd_assign = attach( - maybeparens(lparen, itemlist(setname, comma), rparen) + Optional(equals.suppress() - test_expr), - simple_kwd_assign_handle, - ) - kwd_augassign = Forward() - kwd_augassign_ref = setname + augassign_rhs - kwd_assign = ( - kwd_augassign - | simple_kwd_assign - ) - global_stmt = addspace(keyword("global") - kwd_assign) - nonlocal_stmt = Forward() - nonlocal_stmt_ref = addspace(keyword("nonlocal") - kwd_assign) - - del_stmt = addspace(keyword("del") - simple_assignlist) - - matchlist_data_item = Group(Optional(star | Optional(dot) + unsafe_name + equals) + match) - matchlist_data = Group(Optional(tokenlist(matchlist_data_item, comma))) - - match_check_equals = Forward() - match_check_equals_ref = equals - - match_dotted_name_const = Forward() - complex_number = condense(Optional(neg_minus) + number + (plus | sub_minus) + Optional(neg_minus) + imag_num) - match_const = condense( - (eq | match_check_equals).suppress() + negable_atom_item - | string_atom - | complex_number - | Optional(neg_minus) + number - | match_dotted_name_const - ) - empty_const = fixto( - lparen + rparen - | lbrack + rbrack - | set_letter + lbrace + rbrace, - "()", - ) + import_names = Group( + maybeparens(lparen, tokenlist(import_item, comma), rparen) + | star + ) + from_import_names = Group( + maybeparens(lparen, tokenlist(from_import_item, comma), rparen) + | star + ) + basic_import = keyword("import").suppress() - import_names + import_from_name = condense( + ZeroOrMore(unsafe_dot) + unsafe_dotted_name + | OneOrMore(unsafe_dot) + | star + ) + from_import = ( + keyword("from").suppress() + - import_from_name + - keyword("import").suppress() - from_import_names + ) + import_stmt = Forward() + import_stmt_ref = from_import | basic_import - match_pair = Group(match_const + colon.suppress() + match) - matchlist_dict = Group(Optional(tokenlist(match_pair, comma))) - set_star = star.suppress() + (keyword(wildcard) | empty_const) + augassign_stmt = Forward() + augassign_rhs = ( + labeled_group(pipe_augassign + pipe_augassign_item, "pipe") + | labeled_group(augassign + test_expr, "simple") + ) + augassign_stmt_ref = simple_assign + augassign_rhs - matchlist_tuple_items = ( - match + OneOrMore(comma.suppress() + match) + Optional(comma.suppress()) - | match + comma.suppress() - ) - matchlist_tuple = Group(Optional(matchlist_tuple_items)) - matchlist_list = Group(Optional(tokenlist(match, comma))) - match_list = Group(lbrack + matchlist_list + rbrack.suppress()) - match_tuple = Group(lparen + matchlist_tuple + rparen.suppress()) - match_lazy = Group(lbanana + matchlist_list + rbanana.suppress()) - - interior_name_match = labeled_group(setname, "var") - match_string = interleaved_tokenlist( - # f_string_atom must come first - f_string_atom("f_string") | fixed_len_string_tokens("string"), - interior_name_match("capture"), - plus, - at_least_two=True, - )("string_sequence") - sequence_match = interleaved_tokenlist( - (match_list | match_tuple)("literal"), - interior_name_match("capture"), - plus, - )("sequence") - iter_match = interleaved_tokenlist( - (match_list | match_tuple | match_lazy)("literal"), - interior_name_match("capture"), - unsafe_dubcolon, - at_least_two=True, - )("iter") - matchlist_star = interleaved_tokenlist( - star.suppress() + match("capture"), - match("elem"), - comma, - allow_trailing=True, - ) - star_match = ( - lbrack.suppress() + matchlist_star + rbrack.suppress() - | lparen.suppress() + matchlist_star + rparen.suppress() - )("star") - - base_match = Group( - (negable_atom_item + arrow.suppress() + match)("view") - | match_string - | match_const("const") - | (keyword_atom | keyword("is").suppress() + negable_atom_item)("is") - | (keyword("in").suppress() + negable_atom_item)("in") - | iter_match - | match_lazy("lazy") - | sequence_match - | star_match - | (lparen.suppress() + match + rparen.suppress())("paren") - | (lbrace.suppress() + matchlist_dict + Optional(dubstar.suppress() + (setname | condense(lbrace + rbrace)) + Optional(comma.suppress())) + rbrace.suppress())("dict") - | ( - Group(Optional(set_letter)) - + lbrace.suppress() - + ( - Group(tokenlist(match_const, comma, allow_trailing=False)) + Optional(comma.suppress() + set_star + Optional(comma.suppress())) - | Group(always_match) + set_star + Optional(comma.suppress()) - | Group(Optional(tokenlist(match_const, comma))) - ) + rbrace.suppress() - )("set") - | (keyword("data").suppress() + dotted_refname + lparen.suppress() + matchlist_data + rparen.suppress())("data") - | (keyword("class").suppress() + dotted_refname + lparen.suppress() + matchlist_data + rparen.suppress())("class") - | (dotted_refname + lparen.suppress() + matchlist_data + rparen.suppress())("data_or_class") - | Optional(keyword("as").suppress()) + setname("var") - ) + simple_kwd_assign = attach( + maybeparens(lparen, itemlist(setname, comma), rparen) + Optional(equals.suppress() - test_expr), + simple_kwd_assign_handle, + ) + kwd_augassign = Forward() + kwd_augassign_ref = setname + augassign_rhs + kwd_assign = ( + kwd_augassign + | simple_kwd_assign + ) + global_stmt = addspace(keyword("global") - kwd_assign) + nonlocal_stmt = Forward() + nonlocal_stmt_ref = addspace(keyword("nonlocal") - kwd_assign) + + del_stmt = addspace(keyword("del") - simple_assignlist) + + matchlist_data_item = Group(Optional(star | Optional(dot) + unsafe_name + equals) + match) + matchlist_data = Group(Optional(tokenlist(matchlist_data_item, comma))) + + match_check_equals = Forward() + match_check_equals_ref = equals + + match_dotted_name_const = Forward() + complex_number = condense(Optional(neg_minus) + number + (plus | sub_minus) + Optional(neg_minus) + imag_num) + match_const = condense( + (eq | match_check_equals).suppress() + negable_atom_item + | string_atom + | complex_number + | Optional(neg_minus) + number + | match_dotted_name_const + ) + empty_const = fixto( + lparen + rparen + | lbrack + rbrack + | set_letter + lbrace + rbrace, + "()", + ) - matchlist_isinstance = base_match + OneOrMore(keyword("is").suppress() + negable_atom_item) - isinstance_match = labeled_group(matchlist_isinstance, "isinstance_is") | base_match + match_pair = Group(match_const + colon.suppress() + match) + matchlist_dict = Group(Optional(tokenlist(match_pair, comma))) + set_star = star.suppress() + (keyword(wildcard) | empty_const) - matchlist_bar_or = isinstance_match + OneOrMore(bar.suppress() + isinstance_match) - bar_or_match = labeled_group(matchlist_bar_or, "or") | isinstance_match + matchlist_tuple_items = ( + match + OneOrMore(comma.suppress() + match) + Optional(comma.suppress()) + | match + comma.suppress() + ) + matchlist_tuple = Group(Optional(matchlist_tuple_items)) + matchlist_list = Group(Optional(tokenlist(match, comma))) + match_list = Group(lbrack + matchlist_list + rbrack.suppress()) + match_tuple = Group(lparen + matchlist_tuple + rparen.suppress()) + match_lazy = Group(lbanana + matchlist_list + rbanana.suppress()) + + interior_name_match = labeled_group(setname, "var") + match_string = interleaved_tokenlist( + # f_string_atom must come first + f_string_atom("f_string") | fixed_len_string_tokens("string"), + interior_name_match("capture"), + plus, + at_least_two=True, + )("string_sequence") + sequence_match = interleaved_tokenlist( + (match_list | match_tuple)("literal"), + interior_name_match("capture"), + plus, + )("sequence") + iter_match = interleaved_tokenlist( + (match_list | match_tuple | match_lazy)("literal"), + interior_name_match("capture"), + unsafe_dubcolon, + at_least_two=True, + )("iter") + matchlist_star = interleaved_tokenlist( + star.suppress() + match("capture"), + match("elem"), + comma, + allow_trailing=True, + ) + star_match = ( + lbrack.suppress() + matchlist_star + rbrack.suppress() + | lparen.suppress() + matchlist_star + rparen.suppress() + )("star") + + base_match = Group( + (negable_atom_item + arrow.suppress() + match)("view") + | match_string + | match_const("const") + | (keyword_atom | keyword("is").suppress() + negable_atom_item)("is") + | (keyword("in").suppress() + negable_atom_item)("in") + | iter_match + | match_lazy("lazy") + | sequence_match + | star_match + | (lparen.suppress() + match + rparen.suppress())("paren") + | (lbrace.suppress() + matchlist_dict + Optional(dubstar.suppress() + (setname | condense(lbrace + rbrace)) + Optional(comma.suppress())) + rbrace.suppress())("dict") + | ( + Group(Optional(set_letter)) + + lbrace.suppress() + + ( + Group(tokenlist(match_const, comma, allow_trailing=False)) + Optional(comma.suppress() + set_star + Optional(comma.suppress())) + | Group(always_match) + set_star + Optional(comma.suppress()) + | Group(Optional(tokenlist(match_const, comma))) + ) + rbrace.suppress() + )("set") + | (keyword("data").suppress() + dotted_refname + lparen.suppress() + matchlist_data + rparen.suppress())("data") + | (keyword("class").suppress() + dotted_refname + lparen.suppress() + matchlist_data + rparen.suppress())("class") + | (dotted_refname + lparen.suppress() + matchlist_data + rparen.suppress())("data_or_class") + | Optional(keyword("as").suppress()) + setname("var") + ) - matchlist_infix = bar_or_match + OneOrMore(Group(infix_op + Optional(negable_atom_item))) - infix_match = labeled_group(matchlist_infix, "infix") | bar_or_match + matchlist_isinstance = base_match + OneOrMore(keyword("is").suppress() + negable_atom_item) + isinstance_match = labeled_group(matchlist_isinstance, "isinstance_is") | base_match - matchlist_as = infix_match + OneOrMore(keyword("as").suppress() + setname) - as_match = labeled_group(matchlist_as, "as") | infix_match + matchlist_bar_or = isinstance_match + OneOrMore(bar.suppress() + isinstance_match) + bar_or_match = labeled_group(matchlist_bar_or, "or") | isinstance_match - matchlist_and = as_match + OneOrMore(keyword("and").suppress() + as_match) - and_match = labeled_group(matchlist_and, "and") | as_match + matchlist_infix = bar_or_match + OneOrMore(Group(infix_op + Optional(negable_atom_item))) + infix_match = labeled_group(matchlist_infix, "infix") | bar_or_match - matchlist_kwd_or = and_match + OneOrMore(keyword("or").suppress() + and_match) - kwd_or_match = labeled_group(matchlist_kwd_or, "or") | and_match + matchlist_as = infix_match + OneOrMore(keyword("as").suppress() + setname) + as_match = labeled_group(matchlist_as, "as") | infix_match - match <<= kwd_or_match + matchlist_and = as_match + OneOrMore(keyword("and").suppress() + as_match) + and_match = labeled_group(matchlist_and, "and") | as_match - many_match = ( - labeled_group(matchlist_star, "star") - | labeled_group(matchlist_tuple_items, "implicit_tuple") - | match - ) + matchlist_kwd_or = and_match + OneOrMore(keyword("or").suppress() + and_match) + kwd_or_match = labeled_group(matchlist_kwd_or, "or") | and_match - else_stmt = condense(keyword("else") - suite) - full_suite = colon.suppress() - Group((newline.suppress() - indent.suppress() - OneOrMore(stmt) - dedent.suppress()) | simple_stmt) - full_match = Forward() - full_match_ref = ( - keyword("match").suppress() - + many_match - + addspace(Optional(keyword("not")) + keyword("in")) - + testlist_star_namedexpr - + match_guard - # avoid match match-case blocks - + ~FollowedBy(colon + newline + indent + keyword("case")) - - full_suite - ) - match_stmt = condense(full_match - Optional(else_stmt)) - - destructuring_stmt = Forward() - base_destructuring_stmt = Optional(keyword("match").suppress()) + many_match + equals.suppress() + test_expr - destructuring_stmt_ref, match_dotted_name_const_ref = disable_inside(base_destructuring_stmt, must_be_dotted_name + ~lparen) - - # both syntaxes here must be kept the same except for the keywords - case_match_co_syntax = Group( - (keyword("match") | keyword("case")).suppress() - + stores_loc_item - + many_match - + Optional(keyword("if").suppress() + namedexpr_test) - - full_suite - ) - cases_stmt_co_syntax = ( - (keyword("cases") | keyword("case")) + testlist_star_namedexpr + colon.suppress() + newline.suppress() - + indent.suppress() + Group(OneOrMore(case_match_co_syntax)) - + dedent.suppress() + Optional(keyword("else").suppress() + suite) - ) - case_match_py_syntax = Group( - keyword("case").suppress() - + stores_loc_item - + many_match - + Optional(keyword("if").suppress() + namedexpr_test) - - full_suite - ) - cases_stmt_py_syntax = ( - keyword("match") + testlist_star_namedexpr + colon.suppress() + newline.suppress() - + indent.suppress() + Group(OneOrMore(case_match_py_syntax)) - + dedent.suppress() + Optional(keyword("else").suppress() - suite) - ) - cases_stmt = Forward() - cases_stmt_ref = cases_stmt_co_syntax | cases_stmt_py_syntax + match <<= kwd_or_match - assert_stmt = addspace( - keyword("assert") - - ( - lparen.suppress() + testlist + rparen.suppress() + end_simple_stmt_item - | testlist + many_match = ( + labeled_group(matchlist_star, "star") + | labeled_group(matchlist_tuple_items, "implicit_tuple") + | match ) - ) - if_stmt = condense( - addspace(keyword("if") + condense(namedexpr_test + suite)) - - ZeroOrMore(addspace(keyword("elif") - condense(namedexpr_test - suite))) - - Optional(else_stmt) - ) - while_stmt = addspace(keyword("while") - condense(namedexpr_test - suite - Optional(else_stmt))) - for_stmt = addspace(keyword("for") + assignlist + keyword("in") - condense(new_testlist_star_expr - suite - Optional(else_stmt))) + else_stmt = condense(keyword("else") - suite) + full_suite = colon.suppress() - Group((newline.suppress() - indent.suppress() - OneOrMore(stmt) - dedent.suppress()) | simple_stmt) + full_match = Forward() + full_match_ref = ( + keyword("match").suppress() + + many_match + + addspace(Optional(keyword("not")) + keyword("in")) + + testlist_star_namedexpr + + match_guard + # avoid match match-case blocks + + ~FollowedBy(colon + newline + indent + keyword("case")) + - full_suite + ) + match_stmt = condense(full_match - Optional(else_stmt)) + + destructuring_stmt = Forward() + base_destructuring_stmt = Optional(keyword("match").suppress()) + many_match + equals.suppress() + test_expr + destructuring_stmt_ref, match_dotted_name_const_ref = disable_inside(base_destructuring_stmt, must_be_dotted_name + ~lparen) + + # both syntaxes here must be kept the same except for the keywords + case_match_co_syntax = Group( + (keyword("match") | keyword("case")).suppress() + + stores_loc_item + + many_match + + Optional(keyword("if").suppress() + namedexpr_test) + - full_suite + ) + cases_stmt_co_syntax = ( + (keyword("cases") | keyword("case")) + testlist_star_namedexpr + colon.suppress() + newline.suppress() + + indent.suppress() + Group(OneOrMore(case_match_co_syntax)) + + dedent.suppress() + Optional(keyword("else").suppress() + suite) + ) + case_match_py_syntax = Group( + keyword("case").suppress() + + stores_loc_item + + many_match + + Optional(keyword("if").suppress() + namedexpr_test) + - full_suite + ) + cases_stmt_py_syntax = ( + keyword("match") + testlist_star_namedexpr + colon.suppress() + newline.suppress() + + indent.suppress() + Group(OneOrMore(case_match_py_syntax)) + + dedent.suppress() + Optional(keyword("else").suppress() - suite) + ) + cases_stmt = Forward() + cases_stmt_ref = cases_stmt_co_syntax | cases_stmt_py_syntax + + assert_stmt = addspace( + keyword("assert") + - ( + lparen.suppress() + testlist + rparen.suppress() + end_simple_stmt_item + | testlist + ) + ) + if_stmt = condense( + addspace(keyword("if") + condense(namedexpr_test + suite)) + - ZeroOrMore(addspace(keyword("elif") - condense(namedexpr_test - suite))) + - Optional(else_stmt) + ) + while_stmt = addspace(keyword("while") - condense(namedexpr_test - suite - Optional(else_stmt))) - suite_with_else_tokens = colon.suppress() + condense(nocolon_suite + Optional(else_stmt)) + for_stmt = addspace(keyword("for") + assignlist + keyword("in") - condense(new_testlist_star_expr - suite - Optional(else_stmt))) - base_match_for_stmt = Forward() - base_match_for_stmt_ref = ( - keyword("for").suppress() - + many_match - + keyword("in").suppress() - - new_testlist_star_expr - - suite_with_else_tokens - ) - match_for_stmt = Optional(keyword("match").suppress()) + base_match_for_stmt + suite_with_else_tokens = colon.suppress() + condense(nocolon_suite + Optional(else_stmt)) - except_item = ( - testlist_has_comma("list") - | test("test") - ) - Optional( - keyword("as").suppress() - setname - ) - except_clause = attach(keyword("except") + except_item, except_handle) - except_star_clause = Forward() - except_star_clause_ref = attach(except_star_kwd + except_item, except_handle) - try_stmt = condense( - keyword("try") - suite + ( - keyword("finally") - suite - | ( - OneOrMore(except_clause - suite) - Optional(keyword("except") - suite) - | keyword("except") - suite - | OneOrMore(except_star_clause - suite) - ) - Optional(else_stmt) - Optional(keyword("finally") - suite) + base_match_for_stmt = Forward() + base_match_for_stmt_ref = ( + keyword("for").suppress() + + many_match + + keyword("in").suppress() + - new_testlist_star_expr + - suite_with_else_tokens ) - ) - - with_item = addspace(test + Optional(keyword("as") + base_assign_item)) - with_item_list = Group(maybeparens(lparen, tokenlist(with_item, comma), rparen)) - with_stmt_ref = keyword("with").suppress() - with_item_list - suite - with_stmt = Forward() - - funcname_typeparams = Forward() - funcname_typeparams_ref = dotted_setname + Optional(type_params) - name_funcdef = condense(funcname_typeparams + parameters) - op_tfpdef = unsafe_typedef_default | condense(setname + Optional(default)) - op_funcdef_arg = setname | condense(lparen.suppress() + op_tfpdef + rparen.suppress()) - op_funcdef_name = unsafe_backtick.suppress() + funcname_typeparams + unsafe_backtick.suppress() - op_funcdef = attach( - Group(Optional(op_funcdef_arg)) - + op_funcdef_name - + Group(Optional(op_funcdef_arg)), - op_funcdef_handle, - ) + match_for_stmt = Optional(keyword("match").suppress()) + base_match_for_stmt - return_typedef = Forward() - return_typedef_ref = arrow.suppress() + typedef_test - end_func_colon = return_typedef + colon.suppress() | colon - base_funcdef = op_funcdef | name_funcdef - funcdef = addspace(keyword("def") + condense(base_funcdef + end_func_colon + nocolon_suite)) - - name_match_funcdef = Forward() - op_match_funcdef = Forward() - op_match_funcdef_arg = Group(Optional( - Group( - ( - lparen.suppress() - + match - + Optional(equals.suppress() + test) - + rparen.suppress() - ) | interior_name_match - ) - )) - name_match_funcdef_ref = keyword("def").suppress() + funcname_typeparams + lparen.suppress() + match_args_list + match_guard + rparen.suppress() - op_match_funcdef_ref = keyword("def").suppress() + op_match_funcdef_arg + op_funcdef_name + op_match_funcdef_arg + match_guard - base_match_funcdef = op_match_funcdef | name_match_funcdef - func_suite = ( - attach(simple_stmt, make_suite_handle) - | ( - newline.suppress() - - indent.suppress() - - Optional(docstring) - - attach(condense(OneOrMore(stmt)), make_suite_handle) - - dedent.suppress() + except_item = ( + testlist_has_comma("list") + | test("test") + ) - Optional( + keyword("as").suppress() - setname + ) + except_clause = attach(keyword("except") + except_item, except_handle) + except_star_clause = Forward() + except_star_clause_ref = attach(except_star_kwd + except_item, except_handle) + try_stmt = condense( + keyword("try") - suite + ( + keyword("finally") - suite + | ( + OneOrMore(except_clause - suite) - Optional(keyword("except") - suite) + | keyword("except") - suite + | OneOrMore(except_star_clause - suite) + ) - Optional(else_stmt) - Optional(keyword("finally") - suite) + ) ) - ) - def_match_funcdef = attach( - base_match_funcdef - + end_func_colon - - func_suite, - join_match_funcdef, - ) - match_def_modifiers = any_len_perm( - keyword("match").suppress(), - # addpattern is detected later - keyword("addpattern"), - ) - match_funcdef = addspace(match_def_modifiers + def_match_funcdef) - - where_suite = keyword("where").suppress() - full_suite - where_stmt = Forward() - where_item = Forward() - where_item_ref = unsafe_simple_stmt_item - where_stmt_ref = where_item + where_suite + with_item = addspace(test + Optional(keyword("as") + base_assign_item)) + with_item_list = Group(maybeparens(lparen, tokenlist(with_item, comma), rparen)) + with_stmt_ref = keyword("with").suppress() - with_item_list - suite + with_stmt = Forward() + + funcname_typeparams = Forward() + funcname_typeparams_ref = dotted_setname + Optional(type_params) + name_funcdef = condense(funcname_typeparams + parameters) + op_tfpdef = unsafe_typedef_default | condense(setname + Optional(default)) + op_funcdef_arg = setname | condense(lparen.suppress() + op_tfpdef + rparen.suppress()) + op_funcdef_name = unsafe_backtick.suppress() + funcname_typeparams + unsafe_backtick.suppress() + op_funcdef = attach( + Group(Optional(op_funcdef_arg)) + + op_funcdef_name + + Group(Optional(op_funcdef_arg)), + op_funcdef_handle, + ) - implicit_return = ( - invalid_syntax(return_stmt, "expected expression but got return statement") - | attach(new_testlist_star_expr, implicit_return_handle) - ) - implicit_return_where = Forward() - implicit_return_where_item = Forward() - implicit_return_where_item_ref = implicit_return - implicit_return_where_ref = implicit_return_where_item + where_suite - implicit_return_stmt = ( - condense(implicit_return + newline) - | implicit_return_where - ) + return_typedef = Forward() + return_typedef_ref = arrow.suppress() + typedef_test + end_func_colon = return_typedef + colon.suppress() | colon + base_funcdef = op_funcdef | name_funcdef + funcdef = addspace(keyword("def") + condense(base_funcdef + end_func_colon + nocolon_suite)) - math_funcdef_body = condense(ZeroOrMore(~(implicit_return_stmt + dedent) + stmt) - implicit_return_stmt) - math_funcdef_suite = ( - attach(implicit_return_stmt, make_suite_handle) - | condense(newline - indent - math_funcdef_body - dedent) - ) - end_func_equals = return_typedef + equals.suppress() | fixto(equals, ":") - math_funcdef = attach( - condense(addspace(keyword("def") + base_funcdef) + end_func_equals) - math_funcdef_suite, - math_funcdef_handle, - ) - math_match_funcdef = addspace( - match_def_modifiers - + attach( + name_match_funcdef = Forward() + op_match_funcdef = Forward() + op_match_funcdef_arg = Group(Optional( + Group( + ( + lparen.suppress() + + match + + Optional(equals.suppress() + test) + + rparen.suppress() + ) | interior_name_match + ) + )) + name_match_funcdef_ref = keyword("def").suppress() + funcname_typeparams + lparen.suppress() + match_args_list + match_guard + rparen.suppress() + op_match_funcdef_ref = keyword("def").suppress() + op_match_funcdef_arg + op_funcdef_name + op_match_funcdef_arg + match_guard + base_match_funcdef = op_match_funcdef | name_match_funcdef + func_suite = ( + attach(simple_stmt, make_suite_handle) + | ( + newline.suppress() + - indent.suppress() + - Optional(docstring) + - attach(condense(OneOrMore(stmt)), make_suite_handle) + - dedent.suppress() + ) + ) + def_match_funcdef = attach( base_match_funcdef - + end_func_equals - + ( - attach(implicit_return_stmt, make_suite_handle) - | ( - newline.suppress() - indent.suppress() - + Optional(docstring) - + attach(math_funcdef_body, make_suite_handle) - + dedent.suppress() - ) - ), + + end_func_colon + - func_suite, join_match_funcdef, ) - ) - - async_stmt = Forward() - async_with_for_stmt = Forward() - async_with_for_stmt_ref = ( - labeled_group( - (keyword("async") + keyword("with") + keyword("for")).suppress() - + assignlist + keyword("in").suppress() - - test - - suite_with_else_tokens, - "normal", - ) - | labeled_group( - (any_len_perm( - keyword("match"), - required=(keyword("async"), keyword("with")), - ) + keyword("for")).suppress() - + many_match + keyword("in").suppress() - - test - - suite_with_else_tokens, - "match", - ) - ) - async_stmt_ref = addspace( - keyword("async") + (with_stmt | for_stmt | match_for_stmt) # handles async [match] for - | keyword("match").suppress() + keyword("async") + base_match_for_stmt # handles match async for - | async_with_for_stmt - ) - - async_funcdef = keyword("async").suppress() + (funcdef | math_funcdef) - async_match_funcdef = addspace( - any_len_perm( + match_def_modifiers = any_len_perm( keyword("match").suppress(), # addpattern is detected later keyword("addpattern"), - required=(keyword("async").suppress(),), - ) + (def_match_funcdef | math_match_funcdef), - ) + ) + match_funcdef = addspace(match_def_modifiers + def_match_funcdef) - async_keyword_normal_funcdef = Group( - any_len_perm_at_least_one( - keyword("yield"), - keyword("copyclosure"), - required=(keyword("async").suppress(),), - ) - ) + (funcdef | math_funcdef) - async_keyword_match_funcdef = Group( - any_len_perm_at_least_one( - keyword("yield"), - keyword("copyclosure"), - keyword("match").suppress(), - # addpattern is detected later - keyword("addpattern"), - required=(keyword("async").suppress(),), + where_suite = keyword("where").suppress() - full_suite + + where_stmt = Forward() + where_item = Forward() + where_item_ref = unsafe_simple_stmt_item + where_stmt_ref = where_item + where_suite + + implicit_return = ( + invalid_syntax(return_stmt, "expected expression but got return statement") + | attach(new_testlist_star_expr, implicit_return_handle) + ) + implicit_return_where = Forward() + implicit_return_where_item = Forward() + implicit_return_where_item_ref = implicit_return + implicit_return_where_ref = implicit_return_where_item + where_suite + implicit_return_stmt = ( + condense(implicit_return + newline) + | implicit_return_where ) - ) + (def_match_funcdef | math_match_funcdef) - async_keyword_funcdef = Forward() - async_keyword_funcdef_ref = async_keyword_normal_funcdef | async_keyword_match_funcdef - async_funcdef_stmt = ( - async_funcdef - | async_match_funcdef - | async_keyword_funcdef - ) + math_funcdef_body = condense(ZeroOrMore(~(implicit_return_stmt + dedent) + stmt) - implicit_return_stmt) + math_funcdef_suite = ( + attach(implicit_return_stmt, make_suite_handle) + | condense(newline - indent - math_funcdef_body - dedent) + ) + end_func_equals = return_typedef + equals.suppress() | fixto(equals, ":") + math_funcdef = attach( + condense(addspace(keyword("def") + base_funcdef) + end_func_equals) - math_funcdef_suite, + math_funcdef_handle, + ) + math_match_funcdef = addspace( + match_def_modifiers + + attach( + base_match_funcdef + + end_func_equals + + ( + attach(implicit_return_stmt, make_suite_handle) + | ( + newline.suppress() - indent.suppress() + + Optional(docstring) + + attach(math_funcdef_body, make_suite_handle) + + dedent.suppress() + ) + ), + join_match_funcdef, + ) + ) - keyword_normal_funcdef = Group( - any_len_perm_at_least_one( - keyword("yield"), - keyword("copyclosure"), + async_stmt = Forward() + async_with_for_stmt = Forward() + async_with_for_stmt_ref = ( + labeled_group( + (keyword("async") + keyword("with") + keyword("for")).suppress() + + assignlist + keyword("in").suppress() + - test + - suite_with_else_tokens, + "normal", + ) + | labeled_group( + (any_len_perm( + keyword("match"), + required=(keyword("async"), keyword("with")), + ) + keyword("for")).suppress() + + many_match + keyword("in").suppress() + - test + - suite_with_else_tokens, + "match", + ) ) - ) + (funcdef | math_funcdef) - keyword_match_funcdef = Group( - any_len_perm_at_least_one( - keyword("yield"), - keyword("copyclosure"), - keyword("match").suppress(), - # addpattern is detected later - keyword("addpattern"), + async_stmt_ref = addspace( + keyword("async") + (with_stmt | for_stmt | match_for_stmt) # handles async [match] for + | keyword("match").suppress() + keyword("async") + base_match_for_stmt # handles match async for + | async_with_for_stmt ) - ) + (def_match_funcdef | math_match_funcdef) - keyword_funcdef = Forward() - keyword_funcdef_ref = keyword_normal_funcdef | keyword_match_funcdef - normal_funcdef_stmt = ( - funcdef - | math_funcdef - | math_match_funcdef - | match_funcdef - | keyword_funcdef - ) + async_funcdef = keyword("async").suppress() + (funcdef | math_funcdef) + async_match_funcdef = addspace( + any_len_perm( + keyword("match").suppress(), + # addpattern is detected later + keyword("addpattern"), + required=(keyword("async").suppress(),), + ) + (def_match_funcdef | math_match_funcdef), + ) - datadef = Forward() - data_args = Group(Optional( - lparen.suppress() + ZeroOrMore(Group( - # everything here must end with arg_comma - (unsafe_name + arg_comma.suppress())("name") - | (unsafe_name + equals.suppress() + test + arg_comma.suppress())("default") - | (star.suppress() + unsafe_name + arg_comma.suppress())("star") - | (unsafe_name + colon.suppress() + typedef_test + equals.suppress() + test + arg_comma.suppress())("type default") - | (unsafe_name + colon.suppress() + typedef_test + arg_comma.suppress())("type") - )) + rparen.suppress() - )) - data_inherit = Optional(keyword("from").suppress() + testlist) - data_suite = Group( - colon.suppress() - ( - (newline.suppress() + indent.suppress() + Optional(docstring) + Group(OneOrMore(stmt)) - dedent.suppress())("complex") - | (newline.suppress() + indent.suppress() + docstring - dedent.suppress() | docstring)("docstring") - | simple_stmt("simple") - ) | newline("empty") - ) - datadef_ref = ( - Optional(decorators, default="") - + keyword("data").suppress() - + classname - + Optional(type_params, default=()) - + data_args - + data_inherit - + data_suite - ) + async_keyword_normal_funcdef = Group( + any_len_perm_at_least_one( + keyword("yield"), + keyword("copyclosure"), + required=(keyword("async").suppress(),), + ) + ) + (funcdef | math_funcdef) + async_keyword_match_funcdef = Group( + any_len_perm_at_least_one( + keyword("yield"), + keyword("copyclosure"), + keyword("match").suppress(), + # addpattern is detected later + keyword("addpattern"), + required=(keyword("async").suppress(),), + ) + ) + (def_match_funcdef | math_match_funcdef) + async_keyword_funcdef = Forward() + async_keyword_funcdef_ref = async_keyword_normal_funcdef | async_keyword_match_funcdef + + async_funcdef_stmt = ( + async_funcdef + | async_match_funcdef + | async_keyword_funcdef + ) - match_datadef = Forward() - match_data_args = lparen.suppress() + Group( - match_args_list + match_guard - ) + rparen.suppress() - # we don't support type_params here since we don't support types - match_datadef_ref = ( - Optional(decorators, default="") - + Optional(keyword("match").suppress()) - + keyword("data").suppress() - + classname - + match_data_args - + data_inherit - + data_suite - ) + keyword_normal_funcdef = Group( + any_len_perm_at_least_one( + keyword("yield"), + keyword("copyclosure"), + ) + ) + (funcdef | math_funcdef) + keyword_match_funcdef = Group( + any_len_perm_at_least_one( + keyword("yield"), + keyword("copyclosure"), + keyword("match").suppress(), + # addpattern is detected later + keyword("addpattern"), + ) + ) + (def_match_funcdef | math_match_funcdef) + keyword_funcdef = Forward() + keyword_funcdef_ref = keyword_normal_funcdef | keyword_match_funcdef + + normal_funcdef_stmt = ( + funcdef + | math_funcdef + | math_match_funcdef + | match_funcdef + | keyword_funcdef + ) - simple_decorator = condense(dotted_refname + Optional(function_call) + newline)("simple") - complex_decorator = condense(namedexpr_test + newline)("complex") - decorators_ref = OneOrMore( - at.suppress() - - Group( - simple_decorator - | complex_decorator + datadef = Forward() + data_args = Group(Optional( + lparen.suppress() + ZeroOrMore(Group( + # everything here must end with arg_comma + (unsafe_name + arg_comma.suppress())("name") + | (unsafe_name + equals.suppress() + test + arg_comma.suppress())("default") + | (star.suppress() + unsafe_name + arg_comma.suppress())("star") + | (unsafe_name + colon.suppress() + typedef_test + equals.suppress() + test + arg_comma.suppress())("type default") + | (unsafe_name + colon.suppress() + typedef_test + arg_comma.suppress())("type") + )) + rparen.suppress() + )) + data_inherit = Optional(keyword("from").suppress() + testlist) + data_suite = Group( + colon.suppress() - ( + (newline.suppress() + indent.suppress() + Optional(docstring) + Group(OneOrMore(stmt)) - dedent.suppress())("complex") + | (newline.suppress() + indent.suppress() + docstring - dedent.suppress() | docstring)("docstring") + | simple_stmt("simple") + ) | newline("empty") + ) + datadef_ref = ( + Optional(decorators, default="") + + keyword("data").suppress() + + classname + + Optional(type_params, default=()) + + data_args + + data_inherit + + data_suite ) - ) - decoratable_normal_funcdef_stmt = Forward() - decoratable_normal_funcdef_stmt_ref = Optional(decorators) + normal_funcdef_stmt + match_datadef = Forward() + match_data_args = lparen.suppress() + Group( + match_args_list + match_guard + ) + rparen.suppress() + # we don't support type_params here since we don't support types + match_datadef_ref = ( + Optional(decorators, default="") + + Optional(keyword("match").suppress()) + + keyword("data").suppress() + + classname + + match_data_args + + data_inherit + + data_suite + ) - decoratable_async_funcdef_stmt = Forward() - decoratable_async_funcdef_stmt_ref = Optional(decorators) + async_funcdef_stmt + simple_decorator = condense(dotted_refname + Optional(function_call) + newline)("simple") + complex_decorator = condense(namedexpr_test + newline)("complex") + decorators_ref = OneOrMore( + at.suppress() + - Group( + simple_decorator + | complex_decorator + ) + ) - decoratable_func_stmt = decoratable_normal_funcdef_stmt | decoratable_async_funcdef_stmt + decoratable_normal_funcdef_stmt = Forward() + decoratable_normal_funcdef_stmt_ref = Optional(decorators) + normal_funcdef_stmt - # decorators are integrated into the definitions of each item here - decoratable_class_stmt = classdef | datadef | match_datadef + decoratable_async_funcdef_stmt = Forward() + decoratable_async_funcdef_stmt_ref = Optional(decorators) + async_funcdef_stmt - passthrough_stmt = condense(passthrough_block - (base_suite | newline)) + decoratable_func_stmt = decoratable_normal_funcdef_stmt | decoratable_async_funcdef_stmt - simple_compound_stmt = ( - if_stmt - | try_stmt - | match_stmt - | passthrough_stmt - ) - compound_stmt = ( - decoratable_class_stmt - | decoratable_func_stmt - | while_stmt - | for_stmt - | with_stmt - | async_stmt - | match_for_stmt - | simple_compound_stmt - | where_stmt - ) - endline_semicolon = Forward() - endline_semicolon_ref = semicolon.suppress() + newline - keyword_stmt = ( - flow_stmt - | import_stmt - | assert_stmt - | pass_stmt - | del_stmt - | global_stmt - | nonlocal_stmt - ) - special_stmt = ( - keyword_stmt - | augassign_stmt - | typed_assign_stmt - | type_alias_stmt - ) - unsafe_simple_stmt_item <<= special_stmt | longest(basic_stmt, destructuring_stmt) - simple_stmt_item <<= ( - special_stmt - | basic_stmt + end_simple_stmt_item - | destructuring_stmt + end_simple_stmt_item - ) - simple_stmt <<= condense( - simple_stmt_item - + ZeroOrMore(fixto(semicolon, "\n") + simple_stmt_item) - + (newline | endline_semicolon) - ) - anything_stmt = Forward() - stmt <<= final( - compound_stmt - | simple_stmt - # must be after destructuring due to ambiguity - | cases_stmt - # at the very end as a fallback case for the anything parser - | anything_stmt - ) - base_suite <<= condense(newline + indent - OneOrMore(stmt) - dedent) - simple_suite = attach(stmt, make_suite_handle) - nocolon_suite <<= base_suite | simple_suite - suite <<= condense(colon + nocolon_suite) - line = newline | stmt - - single_input = condense(Optional(line) - ZeroOrMore(newline)) - file_input = condense(moduledoc_marker - ZeroOrMore(line)) - eval_input = condense(testlist - ZeroOrMore(newline)) - - single_parser = start_marker - single_input - end_marker - file_parser = start_marker - file_input - end_marker - eval_parser = start_marker - eval_input - end_marker - some_eval_parser = start_marker + eval_input - - parens = originalTextFor(nestedExpr("(", ")", ignoreExpr=None)) - brackets = originalTextFor(nestedExpr("[", "]", ignoreExpr=None)) - braces = originalTextFor(nestedExpr("{", "}", ignoreExpr=None)) - - unsafe_anything_stmt = originalTextFor(regex_item("[^\n]+\n+")) - unsafe_xonsh_command = originalTextFor( - (Optional(at) + dollar | bang) - + ~(lparen + rparen | lbrack + rbrack | lbrace + rbrace) - + (parens | brackets | braces | unsafe_name) - ) - unsafe_xonsh_parser, _impl_call_ref = disable_inside( - single_parser, - unsafe_impl_call_ref, - ) - impl_call_ref <<= _impl_call_ref - xonsh_parser, _anything_stmt, _xonsh_command = disable_outside( - unsafe_xonsh_parser, - unsafe_anything_stmt, - unsafe_xonsh_command, - ) - anything_stmt <<= _anything_stmt - xonsh_command <<= _xonsh_command + # decorators are integrated into the definitions of each item here + decoratable_class_stmt = classdef | datadef | match_datadef + + passthrough_stmt = condense(passthrough_block - (base_suite | newline)) + + simple_compound_stmt = ( + if_stmt + | try_stmt + | match_stmt + | passthrough_stmt + ) + compound_stmt = ( + decoratable_class_stmt + | decoratable_func_stmt + | while_stmt + | for_stmt + | with_stmt + | async_stmt + | match_for_stmt + | simple_compound_stmt + | where_stmt + ) + endline_semicolon = Forward() + endline_semicolon_ref = semicolon.suppress() + newline + keyword_stmt = ( + flow_stmt + | import_stmt + | assert_stmt + | pass_stmt + | del_stmt + | global_stmt + | nonlocal_stmt + ) + special_stmt = ( + keyword_stmt + | augassign_stmt + | typed_assign_stmt + | type_alias_stmt + ) + unsafe_simple_stmt_item <<= special_stmt | longest(basic_stmt, destructuring_stmt) + simple_stmt_item <<= ( + special_stmt + | basic_stmt + end_simple_stmt_item + | destructuring_stmt + end_simple_stmt_item + ) + simple_stmt <<= condense( + simple_stmt_item + + ZeroOrMore(fixto(semicolon, "\n") + simple_stmt_item) + + (newline | endline_semicolon) + ) + anything_stmt = Forward() + stmt <<= final( + compound_stmt + | simple_stmt + # must be after destructuring due to ambiguity + | cases_stmt + # at the very end as a fallback case for the anything parser + | anything_stmt + ) + base_suite <<= condense(newline + indent - OneOrMore(stmt) - dedent) + simple_suite = attach(stmt, make_suite_handle) + nocolon_suite <<= base_suite | simple_suite + suite <<= condense(colon + nocolon_suite) + line = newline | stmt + + single_input = condense(Optional(line) - ZeroOrMore(newline)) + file_input = condense(moduledoc_marker - ZeroOrMore(line)) + eval_input = condense(testlist - ZeroOrMore(newline)) + + single_parser = start_marker - single_input - end_marker + file_parser = start_marker - file_input - end_marker + eval_parser = start_marker - eval_input - end_marker + some_eval_parser = start_marker + eval_input + + parens = originalTextFor(nestedExpr("(", ")", ignoreExpr=None)) + brackets = originalTextFor(nestedExpr("[", "]", ignoreExpr=None)) + braces = originalTextFor(nestedExpr("{", "}", ignoreExpr=None)) + + unsafe_anything_stmt = originalTextFor(regex_item("[^\n]+\n+")) + unsafe_xonsh_command = originalTextFor( + (Optional(at) + dollar | bang) + + ~(lparen + rparen | lbrack + rbrack | lbrace + rbrace) + + (parens | brackets | braces | unsafe_name) + ) + unsafe_xonsh_parser, _impl_call_ref = disable_inside( + single_parser, + unsafe_impl_call_ref, + ) + impl_call_ref <<= _impl_call_ref + xonsh_parser, _anything_stmt, _xonsh_command = disable_outside( + unsafe_xonsh_parser, + unsafe_anything_stmt, + unsafe_xonsh_command, + ) + anything_stmt <<= _anything_stmt + xonsh_command <<= _xonsh_command # end: MAIN GRAMMAR # ----------------------------------------------------------------------------------------------------------------------- # EXTRA GRAMMAR: # ----------------------------------------------------------------------------------------------------------------------- - # we don't need to include opens/closes here because those are explicitly disallowed - existing_operator_regex = compile_regex(r"([.;\\]|([+-=@%^&|*:,/<>~]|\*\*|//|>>|<<)=?|!=|" + r"|".join(new_operators) + r")$") + # we don't need to include opens/closes here because those are explicitly disallowed + existing_operator_regex = compile_regex(r"([.;\\]|([+-=@%^&|*:,/<>~]|\*\*|//|>>|<<)=?|!=|" + r"|".join(new_operators) + r")$") - whitespace_regex = compile_regex(r"\s") + whitespace_regex = compile_regex(r"\s") - def_regex = compile_regex(r"\b((async|addpattern|copyclosure)\s+)*def\b") - yield_regex = compile_regex(r"\byield(?!\s+_coconut\.asyncio\.From)\b") - yield_from_regex = compile_regex(r"\byield\s+from\b") + def_regex = compile_regex(r"\b((async|addpattern|copyclosure)\s+)*def\b") + yield_regex = compile_regex(r"\byield(?!\s+_coconut\.asyncio\.From)\b") + yield_from_regex = compile_regex(r"\byield\s+from\b") - tco_disable_regex = compile_regex(r"\b(try\b|(async\s+)?(with\b|for\b)|while\b)") - return_regex = compile_regex(r"\breturn\b") + tco_disable_regex = compile_regex(r"\b(try\b|(async\s+)?(with\b|for\b)|while\b)") + return_regex = compile_regex(r"\breturn\b") - noqa_regex = compile_regex(r"\b[Nn][Oo][Qq][Aa]\b") + noqa_regex = compile_regex(r"\b[Nn][Oo][Qq][Aa]\b") - just_non_none_atom = start_marker + ~keyword("None") + known_atom + end_marker + just_non_none_atom = start_marker + ~keyword("None") + known_atom + end_marker - original_function_call_tokens = ( - lparen.suppress() + rparen.suppress() - # we need to keep the parens here, since f(x for x in y) is fine but tail_call(f, x for x in y) is not - | condense(lparen + originalTextFor(test + comp_for) + rparen) - | attach(parens, strip_parens_handle) - ) + original_function_call_tokens = ( + lparen.suppress() + rparen.suppress() + # we need to keep the parens here, since f(x for x in y) is fine but tail_call(f, x for x in y) is not + | condense(lparen + originalTextFor(test + comp_for) + rparen) + | attach(parens, strip_parens_handle) + ) - tre_func_name = Forward() - tre_return = ( - start_marker - + keyword("return").suppress() - + maybeparens( - lparen, - tre_func_name + original_function_call_tokens, - rparen, - ) + end_marker - ) + tre_func_name = Forward() + tre_return = ( + start_marker + + keyword("return").suppress() + + maybeparens( + lparen, + tre_func_name + original_function_call_tokens, + rparen, + ) + end_marker + ) - tco_return = attach( - start_marker - + keyword("return").suppress() - + maybeparens( - lparen, - disallow_keywords(untcoable_funcs, with_suffix="(") - + condense( - (unsafe_name | parens | brackets | braces | string_atom) - + ZeroOrMore( - dot + unsafe_name - | brackets - # don't match the last set of parentheses - | parens + ~end_marker + ~rparen - ), - ) - + original_function_call_tokens, - rparen, - ) + end_marker, - tco_return_handle, - # this is the root in what it's used for, so might as well evaluate greedily - greedy=True, - ) + tco_return = attach( + start_marker + + keyword("return").suppress() + + maybeparens( + lparen, + disallow_keywords(untcoable_funcs, with_suffix="(") + + condense( + (unsafe_name | parens | brackets | braces | string_atom) + + ZeroOrMore( + dot + unsafe_name + | brackets + # don't match the last set of parentheses + | parens + ~end_marker + ~rparen + ), + ) + + original_function_call_tokens, + rparen, + ) + end_marker, + tco_return_handle, + # this is the root in what it's used for, so might as well evaluate greedily + greedy=True, + ) - rest_of_lambda = Forward() - lambdas = keyword("lambda") - rest_of_lambda - colon - rest_of_lambda <<= ZeroOrMore( - # handle anything that could capture colon - parens - | brackets - | braces - | lambdas - | ~colon + any_char - ) - rest_of_tfpdef = originalTextFor( - ZeroOrMore( - # handle anything that could capture comma, rparen, or equals + rest_of_lambda = Forward() + lambdas = keyword("lambda") - rest_of_lambda - colon + rest_of_lambda <<= ZeroOrMore( + # handle anything that could capture colon parens | brackets | braces | lambdas - | ~comma + ~rparen + ~equals + any_char + | ~colon + any_char + ) + rest_of_tfpdef = originalTextFor( + ZeroOrMore( + # handle anything that could capture comma, rparen, or equals + parens + | brackets + | braces + | lambdas + | ~comma + ~rparen + ~equals + any_char + ) + ) + tfpdef_tokens = unsafe_name - Optional(colon - rest_of_tfpdef).suppress() + tfpdef_default_tokens = tfpdef_tokens - Optional(equals - rest_of_tfpdef) + type_comment = Optional( + comment_tokens + | passthrough_item + ).suppress() + parameters_tokens = Group( + Optional(tokenlist( + Group( + dubstar - tfpdef_tokens + | star - Optional(tfpdef_tokens) + | slash + | tfpdef_default_tokens + ) + type_comment, + comma + type_comment, + )) ) - ) - tfpdef_tokens = unsafe_name - Optional(colon - rest_of_tfpdef).suppress() - tfpdef_default_tokens = tfpdef_tokens - Optional(equals - rest_of_tfpdef) - type_comment = Optional( - comment_tokens - | passthrough_item - ).suppress() - parameters_tokens = Group( - Optional(tokenlist( - Group( - dubstar - tfpdef_tokens - | star - Optional(tfpdef_tokens) - | slash - | tfpdef_default_tokens - ) + type_comment, - comma + type_comment, - )) - ) - split_func = ( - start_marker - - keyword("def").suppress() - - unsafe_dotted_name - - Optional(brackets).suppress() - - lparen.suppress() - parameters_tokens - rparen.suppress() - ) + split_func = ( + start_marker + - keyword("def").suppress() + - unsafe_dotted_name + - Optional(brackets).suppress() + - lparen.suppress() - parameters_tokens - rparen.suppress() + ) - stores_scope = boundary + ( - keyword("lambda") - # match comprehensions but not for loops - | ~indent + ~dedent + any_char + keyword("for") + unsafe_name + keyword("in") - ) + stores_scope = boundary + ( + keyword("lambda") + # match comprehensions but not for loops + | ~indent + ~dedent + any_char + keyword("for") + unsafe_name + keyword("in") + ) - just_a_string = start_marker + string_atom + end_marker + just_a_string = start_marker + string_atom + end_marker - end_of_line = end_marker | Literal("\n") | pound + end_of_line = end_marker | Literal("\n") | pound - unsafe_equals = Literal("=") + unsafe_equals = Literal("=") - kwd_err_msg = attach(any_keyword_in(keyword_vars + reserved_vars), kwd_err_msg_handle) - parse_err_msg = ( - start_marker + ( - fixto(end_of_line, "misplaced newline (maybe missing ':')") - | fixto(Optional(keyword("if") + skip_to_in_line(unsafe_equals)) + equals, "misplaced assignment (maybe should be '==')") - | kwd_err_msg - ) - | fixto( - questionmark - + ~dollar - + ~lparen - + ~lbrack - + ~dot, - "misplaced '?' (naked '?' is only supported inside partial application arguments)", + kwd_err_msg = attach(any_keyword_in(keyword_vars + reserved_vars), kwd_err_msg_handle) + parse_err_msg = ( + start_marker + ( + fixto(end_of_line, "misplaced newline (maybe missing ':')") + | fixto(Optional(keyword("if") + skip_to_in_line(unsafe_equals)) + equals, "misplaced assignment (maybe should be '==')") + | kwd_err_msg + ) + | fixto( + questionmark + + ~dollar + + ~lparen + + ~lbrack + + ~dot, + "misplaced '?' (naked '?' is only supported inside partial application arguments)", + ) ) - ) - end_f_str_expr = combine(start_marker + (bang | colon | rbrace)) + end_f_str_expr = combine(start_marker + (bang | colon | rbrace)) - string_start = start_marker + python_quoted_string + string_start = start_marker + python_quoted_string - no_unquoted_newlines = start_marker + ZeroOrMore(python_quoted_string | ~Literal("\n") + any_char) + end_marker + no_unquoted_newlines = start_marker + ZeroOrMore(python_quoted_string | ~Literal("\n") + any_char) + end_marker - operator_stmt = ( - start_marker - + keyword("operator").suppress() - + restOfLine - ) + operator_stmt = ( + start_marker + + keyword("operator").suppress() + + restOfLine + ) - unsafe_import_from_name = condense(ZeroOrMore(unsafe_dot) + unsafe_dotted_name | OneOrMore(unsafe_dot)) - from_import_operator = ( - start_marker - + keyword("from").suppress() - + unsafe_import_from_name - + keyword("import").suppress() - + keyword("operator").suppress() - + restOfLine - ) + unsafe_import_from_name = condense(ZeroOrMore(unsafe_dot) + unsafe_dotted_name | OneOrMore(unsafe_dot)) + from_import_operator = ( + start_marker + + keyword("from").suppress() + + unsafe_import_from_name + + keyword("import").suppress() + + keyword("operator").suppress() + + restOfLine + ) # end: EXTRA GRAMMAR # ----------------------------------------------------------------------------------------------------------------------- diff --git a/coconut/compiler/util.py b/coconut/compiler/util.py index db97100e8..4e8a138ee 100644 --- a/coconut/compiler/util.py +++ b/coconut/compiler/util.py @@ -45,6 +45,7 @@ import cPickle as pickle from coconut._pyparsing import ( + MODERN_PYPARSING, USE_COMPUTATION_GRAPH, SUPPORTS_INCREMENTAL, USE_ADAPTIVE, @@ -66,6 +67,7 @@ Group, ParserElement, MatchFirst, + And, _trim_arity, _ParseResultsWithOffset, all_parse_elements, @@ -324,13 +326,65 @@ def postParse(self, original, loc, tokens): combine = Combine +def maybe_copy_elem(item, name): + """Copy the given grammar element if it's referenced somewhere else.""" + item_ref_count = sys.getrefcount(item) if CPYTHON and not on_new_python else float("inf") + internal_assert(lambda: item_ref_count >= temp_grammar_item_ref_count, "add_action got item with too low ref count", (item, type(item), item_ref_count)) + if item_ref_count <= temp_grammar_item_ref_count: + if DEVELOP: + logger.record_stat("maybe_copy_" + name, False) + return item + else: + if DEVELOP: + logger.record_stat("maybe_copy_" + name, True) + return item.copy() + + +def hasaction(elem): + """Determine if the given grammar element has any actions associated with it.""" + return ( + MODERN_PYPARSING + or elem.parseAction + or elem.resultsName is not None + or elem.debug + ) + + +@contextmanager +def using_fast_grammar_methods(): + """Enables grammar methods that modify their operands when they aren't referenced elsewhere.""" + if MODERN_PYPARSING: + yield + return + + def fast_add(self, other): + if hasaction(self): + return old_add(self, other) + self = maybe_copy_elem(self, "add") + self += other + return self + old_add, And.__add__ = And.__add__, fast_add + + def fast_or(self, other): + if hasaction(self): + return old_or(self, other) + self = maybe_copy_elem(self, "or") + self |= other + return self + old_or, MatchFirst.__or__ = MatchFirst.__or__, fast_or + + try: + yield + finally: + And.__add__ = old_add + MatchFirst.__or__ = old_or + + def add_action(item, action, make_copy=None): """Add a parse action to the given item.""" if make_copy is None: - item_ref_count = sys.getrefcount(item) if CPYTHON and not on_new_python else float("inf") - internal_assert(lambda: item_ref_count >= temp_grammar_item_ref_count, "add_action got item with too low ref count", (item, type(item), item_ref_count)) - make_copy = item_ref_count > temp_grammar_item_ref_count - if make_copy: + item = maybe_copy_elem(item, "attach") + elif make_copy: item = item.copy() return item.addParseAction(action) @@ -386,10 +440,10 @@ def adaptive_manager(item, original, loc, reparse=False): except Exception as exc: if DEVELOP: logger.log("reparsing due to:", exc) - logger.record_adaptive_stat(False) + logger.record_stat("adaptive", False) else: if DEVELOP: - logger.record_adaptive_stat(True) + logger.record_stat("adaptive", True) finally: MatchFirst.setAdaptiveMode(False) @@ -783,10 +837,9 @@ class MatchAny(MatchFirst): adaptive_mode = True -def any_of(match_first): +def any_of(*exprs): """Build a MatchAny of the given MatchFirst.""" - internal_assert(isinstance(match_first, MatchFirst), "invalid any_of target", match_first) - return MatchAny(match_first.exprs) + return MatchAny(exprs) class Wrap(ParseElementEnhance): diff --git a/coconut/constants.py b/coconut/constants.py index e9379f00a..7c0a8c11c 100644 --- a/coconut/constants.py +++ b/coconut/constants.py @@ -153,8 +153,8 @@ def get_path_env_var(env_var, default): embed_on_internal_exc = False assert not embed_on_internal_exc or DEVELOP, "embed_on_internal_exc should never be enabled on non-develop build" -# should be the minimal ref count observed by attach -temp_grammar_item_ref_count = 3 if PY311 else 5 +# should be the minimal ref count observed by maybe_copy_elem +temp_grammar_item_ref_count = 4 if PY311 else 5 minimum_recursion_limit = 128 # shouldn't be raised any higher to avoid stack overflows diff --git a/coconut/terminal.py b/coconut/terminal.py index c0fcf1809..2833558ce 100644 --- a/coconut/terminal.py +++ b/coconut/terminal.py @@ -183,14 +183,17 @@ def logging(self): class Logger(object): """Container object for various logger functions and variables.""" force_verbose = force_verbose_logger + colors_enabled = False + verbose = force_verbose quiet = False path = None name = None - colors_enabled = False tracing = False trace_ind = 0 + recorded_stats = defaultdict(lambda: [0, 0]) + def __init__(self, other=None): """Create a logger, optionally from another logger.""" if other is not None: @@ -522,19 +525,15 @@ def trace(self, item): item.debug = True return item - adaptive_stats = None - - def record_adaptive_stat(self, success): - if self.verbose: - if self.adaptive_stats is None: - self.adaptive_stats = [0, 0] - self.adaptive_stats[success] += 1 + def record_stat(self, stat_name, stat_bool): + """Record the given boolean statistic for the given stat_name.""" + self.recorded_stats[stat_name][stat_bool] += 1 @contextmanager def gather_parsing_stats(self): """Times parsing if --verbose.""" if self.verbose: - self.adaptive_stats = None + self.recorded_stats.pop("adaptive", None) start_time = get_clock_time() try: yield @@ -547,8 +546,8 @@ def gather_parsing_stats(self): # reset stats after printing if in incremental mode if ParserElement._incrementalEnabled: ParserElement.packrat_cache_stats[:] = [0] * len(ParserElement.packrat_cache_stats) - if self.adaptive_stats: - failures, successes = self.adaptive_stats + if "adaptive" in self.recorded_stats: + failures, successes = self.recorded_stats["adaptive"] self.printlog("\tAdaptive parsing stats:", successes, "successes;", failures, "failures") else: yield diff --git a/coconut/tests/src/cocotest/agnostic/primary_2.coco b/coconut/tests/src/cocotest/agnostic/primary_2.coco index 3f9d63a65..0bc92d989 100644 --- a/coconut/tests/src/cocotest/agnostic/primary_2.coco +++ b/coconut/tests/src/cocotest/agnostic/primary_2.coco @@ -408,6 +408,7 @@ def primary_test_2() -> bool: assert_raises(-> collectby(.[0], [(0, 1), (0, 2)], reduce_func=False), ValueError) # type: ignore assert ident$(x=?).__name__ == "ident" == ident$(1).__name__ # type: ignore assert collectby(.[0], [(0, 1), (0, 2)], value_func=.[1], reduce_func=(+), reduce_func_init=1) == {0: 4} + assert ident$(1, ?) |> type == ident$(1) |> type with process_map.multiple_sequential_calls(): # type: ignore assert map((+), range(3), range(4)$[:-1], strict=True) |> list == [0, 2, 4] == process_map((+), range(3), range(4)$[:-1], strict=True) |> list # type: ignore diff --git a/coconut/util.py b/coconut/util.py index a5d68f39c..4b4338a15 100644 --- a/coconut/util.py +++ b/coconut/util.py @@ -92,6 +92,20 @@ def __reduce_ex__(self, _): return self.__reduce__() +class const(pickleable_obj): + """Implementaiton of Coconut's const for use within Coconut.""" + __slots__ = ("value",) + + def __init__(self, value): + self.value = value + + def __reduce__(self): + return (self.__class__, (self.value,)) + + def __call__(self, *args, **kwargs): + return self.value + + class override(pickleable_obj): """Implementation of Coconut's @override for use within Coconut.""" __slots__ = ("func",) @@ -273,6 +287,11 @@ def ensure_dir(dirpath): os.makedirs(dirpath) +def without_keys(inputdict, rem_keys): + """Get a copy of inputdict without rem_keys.""" + return {k: v for k, v in inputdict.items() if k not in rem_keys} + + # ----------------------------------------------------------------------------------------------------------------------- # VERSIONING: # ----------------------------------------------------------------------------------------------------------------------- From 607c3b76a40ff5cf964d89f9544307b6054add64 Mon Sep 17 00:00:00 2001 From: Evan Hubinger Date: Fri, 10 Nov 2023 15:54:51 -0800 Subject: [PATCH 074/121] Fix fast parse methods --- coconut/compiler/grammar.py | 2 +- coconut/compiler/util.py | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/coconut/compiler/grammar.py b/coconut/compiler/grammar.py index e6f3b6bdb..b5cecb4aa 100644 --- a/coconut/compiler/grammar.py +++ b/coconut/compiler/grammar.py @@ -2037,7 +2037,7 @@ class Grammar(object): with_item = addspace(test + Optional(keyword("as") + base_assign_item)) with_item_list = Group(maybeparens(lparen, tokenlist(with_item, comma), rparen)) - with_stmt_ref = keyword("with").suppress() - with_item_list - suite + with_stmt_ref = keyword("with").suppress() + with_item_list + suite with_stmt = Forward() funcname_typeparams = Forward() diff --git a/coconut/compiler/util.py b/coconut/compiler/util.py index 4e8a138ee..50a3d33ca 100644 --- a/coconut/compiler/util.py +++ b/coconut/compiler/util.py @@ -353,7 +353,7 @@ def hasaction(elem): @contextmanager def using_fast_grammar_methods(): """Enables grammar methods that modify their operands when they aren't referenced elsewhere.""" - if MODERN_PYPARSING: + if True: # MODERN_PYPARSING: yield return From 9d2acaa2a7bc82bd10ba1bf1eeed74b2b898c35b Mon Sep 17 00:00:00 2001 From: Evan Hubinger Date: Fri, 10 Nov 2023 15:55:21 -0800 Subject: [PATCH 075/121] Actually enable fast parse methods --- coconut/compiler/util.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/coconut/compiler/util.py b/coconut/compiler/util.py index 50a3d33ca..4e8a138ee 100644 --- a/coconut/compiler/util.py +++ b/coconut/compiler/util.py @@ -353,7 +353,7 @@ def hasaction(elem): @contextmanager def using_fast_grammar_methods(): """Enables grammar methods that modify their operands when they aren't referenced elsewhere.""" - if True: # MODERN_PYPARSING: + if MODERN_PYPARSING: yield return From b8d15814347b4db7ce3451651059f5264273a012 Mon Sep 17 00:00:00 2001 From: Evan Hubinger Date: Fri, 10 Nov 2023 18:22:01 -0800 Subject: [PATCH 076/121] Improve exceptions --- coconut/terminal.py | 3 +++ 1 file changed, 3 insertions(+) diff --git a/coconut/terminal.py b/coconut/terminal.py index 2833558ce..574918292 100644 --- a/coconut/terminal.py +++ b/coconut/terminal.py @@ -232,6 +232,9 @@ def setup(self, quiet=None, verbose=None, tracing=None): if tracing is not None: self.tracing = tracing + if self.verbose: + ParserElement.verbose_stacktrace = True + def display( self, messages, From ec1c1dd9c6cead8540f7d24fe2ee5feb0e2a848e Mon Sep 17 00:00:00 2001 From: Evan Hubinger Date: Sat, 11 Nov 2023 23:32:16 -0800 Subject: [PATCH 077/121] Lots of optimizations --- Makefile | 2 +- coconut/_pyparsing.py | 32 +++ coconut/command/command.py | 11 +- coconut/compiler/grammar.py | 422 ++++++++++++++++++++---------------- coconut/compiler/util.py | 54 ++++- coconut/constants.py | 6 +- coconut/root.py | 2 +- coconut/terminal.py | 20 +- 8 files changed, 336 insertions(+), 213 deletions(-) diff --git a/Makefile b/Makefile index aa9dd6d5c..1313b1215 100644 --- a/Makefile +++ b/Makefile @@ -156,7 +156,7 @@ test-mypy-tests: clean-no-tests python ./coconut/tests/dest/extras.py # same as test-univ but includes verbose output for better debugging -# regex for getting non-timing lines: ^(?!Time|\s+Packrat|Loaded|Saving|Adaptive)[^\n]*\n* +# regex for getting non-timing lines: ^(?!Time|\s+Packrat|Loaded|Saving|Adaptive|tErrorless)[^\n]*\n* .PHONY: test-verbose test-verbose: export COCONUT_USE_COLOR=TRUE test-verbose: clean diff --git a/coconut/_pyparsing.py b/coconut/_pyparsing.py index e9830c828..182bded20 100644 --- a/coconut/_pyparsing.py +++ b/coconut/_pyparsing.py @@ -153,6 +153,36 @@ def _parseCache(self, instring, loc, doActions=True, callPreParse=True): ParserElement.packrat_context = [] ParserElement._parseCache = _parseCache + # [CPYPARSING] fix append + def append(self, other): + if (self.parseAction + or self.resultsName is not None + or self.debug): + return self.__class__([self, other]) + elif (other.__class__ == self.__class__ + and not other.parseAction + and other.resultsName is None + and not other.debug): + self.exprs += other.exprs + self.strRepr = None + self.saveAsList |= other.saveAsList + if isinstance(self, And): + self.mayReturnEmpty &= other.mayReturnEmpty + else: + self.mayReturnEmpty |= other.mayReturnEmpty + self.mayIndexError |= other.mayIndexError + else: + self.exprs.append(other) + self.strRepr = None + if isinstance(self, And): + self.mayReturnEmpty &= other.mayReturnEmpty + else: + self.mayReturnEmpty |= other.mayReturnEmpty + self.mayIndexError |= other.mayIndexError + self.saveAsList |= other.saveAsList + return self + ParseExpression.append = append + elif not hasattr(ParserElement, "packrat_context"): raise ImportError( "This version of Coconut requires cPyparsing>=" + ver_tuple_to_str(min_versions["cPyparsing"]) @@ -177,6 +207,8 @@ def enableIncremental(*args, **kwargs): USE_ADAPTIVE = hasattr(MatchFirst, "setAdaptiveMode") and use_adaptive_if_available +maybe_make_safe = getattr(_pyparsing, "maybe_make_safe", None) + # ----------------------------------------------------------------------------------------------------------------------- # SETUP: diff --git a/coconut/command/command.py b/coconut/command/command.py index 49ac9c4e6..0d19344ff 100644 --- a/coconut/command/command.py +++ b/coconut/command/command.py @@ -81,9 +81,8 @@ ver_tuple_to_str, install_custom_kernel, get_clock_time, - first_import_time, ensure_dir, - assert_remove_prefix, + first_import_time, ) from coconut.command.util import ( writefile, @@ -325,13 +324,7 @@ def execute_args(self, args, interact=True, original_args=None): # process mypy args and print timing info (must come after compiler setup) if args.mypy is not None: self.set_mypy_args(args.mypy) - if logger.verbose: - logger.log("Grammar init time: " + str(self.comp.grammar_init_time) + " secs / Total init time: " + str(get_clock_time() - first_import_time) + " secs") - for stat_name, (no_copy, yes_copy) in logger.recorded_stats.items(): - if not stat_name.startswith("maybe_copy_"): - continue - name = assert_remove_prefix(stat_name, "maybe_copy_") - logger.printlog("\tGrammar copying stats (" + name + "):", no_copy, "not copied;", yes_copy, "copied") + logger.log_compiler_stats(self.comp) # do compilation, keeping track of compiled filepaths filepaths = [] diff --git a/coconut/compiler/grammar.py b/coconut/compiler/grammar.py index b5cecb4aa..4f79c2c04 100644 --- a/coconut/compiler/grammar.py +++ b/coconut/compiler/grammar.py @@ -117,6 +117,8 @@ always_match, caseless_literal, using_fast_grammar_methods, + disambiguate_literal, + any_of, ) @@ -636,7 +638,7 @@ class Grammar(object): rbrack = Literal("]") lbrace = Literal("{") rbrace = Literal("}") - lbanana = ~Literal("(|)") + ~Literal("(|>") + ~Literal("(|*") + ~Literal("(|?") + Literal("(|") + lbanana = disambiguate_literal("(|", ["(|)", "(|>", "(|*", "(|?"]) rbanana = Literal("|)") lparen = ~lbanana + Literal("(") rparen = Literal(")") @@ -675,8 +677,8 @@ class Grammar(object): | invalid_syntax("") + ~Literal("..*") + ~Literal("..?") + Literal("..") - | ~Literal("\u2218>") + ~Literal("\u2218*") + ~Literal("\u2218?") + fixto(Literal("\u2218"), "..") + disambiguate_literal("..", ["...", "..>", "..*", "..?"]) + | fixto(disambiguate_literal("\u2218", ["\u2218>", "\u2218*", "\u2218?"]), "..") ) comp_pipe = Literal("..>") | fixto(Literal("\u2218>"), "..>") comp_back_pipe = Literal("<..") | fixto(Literal("<\u2218"), "<..") @@ -709,7 +711,10 @@ class Grammar(object): amp_colon = Literal("&:") amp = ~amp_colon + Literal("&") | fixto(Literal("\u2229"), "&") caret = Literal("^") - unsafe_bar = ~Literal("|>") + ~Literal("|*") + Literal("|") | fixto(Literal("\u222a"), "|") + unsafe_bar = ( + disambiguate_literal("|", ["|>", "|*"]) + | fixto(Literal("\u222a"), "|") + ) bar = ~rbanana + unsafe_bar | invalid_syntax("\xa6", "invalid broken bar character", greedy=True) percent = Literal("%") dollar = Literal("$") @@ -737,19 +742,11 @@ class Grammar(object): ellipsis_tokens = Literal("...") | fixto(Literal("\u2026"), "...") lt = ( - ~Literal("<<") - + ~Literal("<=") - + ~Literal("<|") - + ~Literal("<..") - + ~Literal("<*") - + ~Literal("<:") - + Literal("<") + disambiguate_literal("<", ["<<", "<=", "<|", "<..", "<*", "<:"]) | fixto(Literal("\u228a"), "<") ) gt = ( - ~Literal(">>") - + ~Literal(">=") - + Literal(">") + disambiguate_literal(">", [">>", ">="]) | fixto(Literal("\u228b"), ">") ) le = Literal("<=") | fixto(Literal("\u2264") | Literal("\u2286"), "<=") @@ -800,21 +797,21 @@ class Grammar(object): imag_j = caseless_literal("j") | fixto(caseless_literal("i", suppress=True), "j") basenum = combine( - integer + dot + Optional(integer) - | Optional(integer) + dot + integer - ) | integer + Optional(integer) + dot + integer + | integer + Optional(dot + Optional(integer)) + ) sci_e = combine((caseless_literal("e") | fixto(Literal("\u23e8"), "e")) + Optional(plus | neg_minus)) numitem = ~(Literal("0") + Word(nums + "_", exact=1)) + combine(basenum + Optional(sci_e + integer)) imag_num = combine(numitem + imag_j) + maybe_imag_num = combine(numitem + Optional(imag_j)) bin_num = combine(caseless_literal("0b") + Optional(underscore.suppress()) + binint) oct_num = combine(caseless_literal("0o") + Optional(underscore.suppress()) + octint) hex_num = combine(caseless_literal("0x") + Optional(underscore.suppress()) + hexint) number = ( - bin_num - | oct_num + maybe_imag_num | hex_num - | imag_num - | numitem + | bin_num + | oct_num ) # make sure that this gets addspaced not condensed so it doesn't produce a SyntaxError num_atom = addspace(number + Optional(condense(dot + unsafe_name))) @@ -829,7 +826,7 @@ class Grammar(object): ) xonsh_command = Forward() - passthrough_item = combine((backslash | Literal(early_passthrough_wrapper)) + integer + unwrap) | xonsh_command + passthrough_item = combine((Literal(early_passthrough_wrapper) | backslash) + integer + unwrap) | xonsh_command passthrough_block = combine(fixto(dubbackslash, "\\") + integer + unwrap) endline = Forward() @@ -837,7 +834,7 @@ class Grammar(object): lineitem = ZeroOrMore(comment) + endline newline = condense(OneOrMore(lineitem)) # rparen handles simple stmts ending parenthesized stmt lambdas - end_simple_stmt_item = FollowedBy(semicolon | newline | rparen) + end_simple_stmt_item = FollowedBy(newline | semicolon | rparen) start_marker = StringStart() moduledoc_marker = condense(ZeroOrMore(lineitem) - Optional(moduledoc_item)) @@ -865,58 +862,63 @@ class Grammar(object): moduledoc = any_string + newline docstring = condense(moduledoc) - pipe_augassign = ( - combine(pipe + equals) - | combine(star_pipe + equals) - | combine(dubstar_pipe + equals) - | combine(back_pipe + equals) - | combine(back_star_pipe + equals) - | combine(back_dubstar_pipe + equals) - | combine(none_pipe + equals) - | combine(none_star_pipe + equals) - | combine(none_dubstar_pipe + equals) - | combine(back_none_pipe + equals) - | combine(back_none_star_pipe + equals) - | combine(back_none_dubstar_pipe + equals) - ) - augassign = ( - pipe_augassign - | combine(comp_pipe + equals) - | combine(dotdot + equals) - | combine(comp_back_pipe + equals) - | combine(comp_star_pipe + equals) - | combine(comp_back_star_pipe + equals) - | combine(comp_dubstar_pipe + equals) - | combine(comp_back_dubstar_pipe + equals) - | combine(comp_none_pipe + equals) - | combine(comp_back_none_pipe + equals) - | combine(comp_none_star_pipe + equals) - | combine(comp_back_none_star_pipe + equals) - | combine(comp_none_dubstar_pipe + equals) - | combine(comp_back_none_dubstar_pipe + equals) - | combine(unsafe_dubcolon + equals) - | combine(div_dubslash + equals) - | combine(div_slash + equals) - | combine(exp_dubstar + equals) - | combine(mul_star + equals) - | combine(plus + equals) - | combine(sub_minus + equals) - | combine(percent + equals) - | combine(amp + equals) - | combine(bar + equals) - | combine(caret + equals) - | combine(lshift + equals) - | combine(rshift + equals) - | combine(matrix_at + equals) - | combine(dubquestion + equals) - ) - - comp_op = ( - le | ge | ne | lt | gt | eq - | addspace(keyword("not") + keyword("in")) - | keyword("in") - | addspace(keyword("is") + keyword("not")) - | keyword("is") + pipe_augassign = any_of( + combine(pipe + equals), + combine(star_pipe + equals), + combine(dubstar_pipe + equals), + combine(back_pipe + equals), + combine(back_star_pipe + equals), + combine(back_dubstar_pipe + equals), + combine(none_pipe + equals), + combine(none_star_pipe + equals), + combine(none_dubstar_pipe + equals), + combine(back_none_pipe + equals), + combine(back_none_star_pipe + equals), + combine(back_none_dubstar_pipe + equals), + ) + augassign = any_of( + pipe_augassign, + combine(comp_pipe + equals), + combine(dotdot + equals), + combine(comp_back_pipe + equals), + combine(comp_star_pipe + equals), + combine(comp_back_star_pipe + equals), + combine(comp_dubstar_pipe + equals), + combine(comp_back_dubstar_pipe + equals), + combine(comp_none_pipe + equals), + combine(comp_back_none_pipe + equals), + combine(comp_none_star_pipe + equals), + combine(comp_back_none_star_pipe + equals), + combine(comp_none_dubstar_pipe + equals), + combine(comp_back_none_dubstar_pipe + equals), + combine(unsafe_dubcolon + equals), + combine(div_dubslash + equals), + combine(div_slash + equals), + combine(exp_dubstar + equals), + combine(mul_star + equals), + combine(plus + equals), + combine(sub_minus + equals), + combine(percent + equals), + combine(amp + equals), + combine(bar + equals), + combine(caret + equals), + combine(lshift + equals), + combine(rshift + equals), + combine(matrix_at + equals), + combine(dubquestion + equals), + ) + + comp_op = any_of( + eq, + ne, + keyword("in"), + addspace(keyword("not") + keyword("in")), + lt, + gt, + le, + ge, + keyword("is") + ~keyword("not"), + addspace(keyword("is") + keyword("not")), ) atom_item = Forward() @@ -958,7 +960,11 @@ class Grammar(object): dict_literal = Forward() yield_classic = addspace(keyword("yield") + Optional(new_testlist_star_expr)) yield_from_ref = keyword("yield").suppress() + keyword("from").suppress() + test - yield_expr = yield_from | yield_classic + yield_expr = ( + # yield_from must come first + yield_from + | yield_classic + ) dict_comp_ref = lbrace.suppress() + ( test + colon.suppress() + test + comp_for | invalid_syntax(dubstar_expr + comp_for, "dict unpacking cannot be used in dict comprehension") @@ -972,7 +978,7 @@ class Grammar(object): )) + rbrace.suppress() ) - test_expr = yield_expr | testlist_star_expr + test_expr = testlist_star_expr | yield_expr base_op_item = ( # must go dubstar then star then no star @@ -1050,8 +1056,8 @@ class Grammar(object): ) partial_op_item = attach(partial_op_item_tokens, partial_op_item_handle) op_item = ( - typedef_op_item - | partial_op_item + partial_op_item + | typedef_op_item | base_op_item ) @@ -1064,8 +1070,8 @@ class Grammar(object): default = condense(equals + test) unsafe_typedef_default_ref = setname + colon.suppress() + typedef_test + Optional(default) typedef_default_ref = unsafe_typedef_default_ref + arg_comma - tfpdef = typedef | condense(setname + arg_comma) - tfpdef_default = typedef_default | condense(setname + Optional(default) + arg_comma) + tfpdef = condense(setname + arg_comma) | typedef + tfpdef_default = condense(setname + Optional(default) + arg_comma) | typedef_default star_sep_arg = Forward() star_sep_arg_ref = condense(star + arg_comma) @@ -1088,10 +1094,10 @@ class Grammar(object): ZeroOrMore( condense( # everything here must end with arg_comma - (star | dubstar) + tfpdef + tfpdef_default + | (star | dubstar) + tfpdef | star_sep_arg | slash_sep_arg - | tfpdef_default ) ) ) @@ -1103,10 +1109,10 @@ class Grammar(object): ZeroOrMore( condense( # everything here must end with setarg_comma - (star | dubstar) + setname + setarg_comma + setname + Optional(default) + setarg_comma + | (star | dubstar) + setname + setarg_comma | star_sep_setarg | slash_sep_setarg - | setname + Optional(default) + setarg_comma ) ) ) @@ -1125,10 +1131,10 @@ class Grammar(object): call_item = ( unsafe_name + default - | dubstar + test - | star + test | ellipsis_tokens + equals.suppress() + refname | namedexpr_test + | star + test + | dubstar + test ) function_call_tokens = lparen.suppress() + ( # everything here must end with rparen @@ -1187,14 +1193,14 @@ class Grammar(object): | invalid_syntax(star_expr + comp_for, "iterable unpacking cannot be used in comprehension") ) paren_atom = condense( - lparen + ( + lparen + any_of( # everything here must end with rparen - rparen - | yield_expr + rparen - | comprehension_expr + rparen - | testlist_star_namedexpr + rparen - | op_item + rparen - | anon_namedtuple + rparen + rparen, + testlist_star_namedexpr + rparen, + comprehension_expr + rparen, + op_item + rparen, + yield_expr + rparen, + anon_namedtuple + rparen, ) | ( lparen.suppress() + typedef_tuple @@ -1214,8 +1220,8 @@ class Grammar(object): array_literal_handle, ) list_item = ( - condense(lbrack + Optional(comprehension_expr) + rbrack) - | lbrack.suppress() + list_expr + rbrack.suppress() + lbrack.suppress() + list_expr + rbrack.suppress() + | condense(lbrack + Optional(comprehension_expr) + rbrack) | array_literal ) @@ -1251,11 +1257,12 @@ class Grammar(object): | string_atom | num_atom | list_item - | dict_comp | dict_literal + | dict_comp | set_literal | set_letter_literal | lazy_list + # typedef ellipsis must come before ellipsis | typedef_ellipsis | ellipsis ) @@ -1292,7 +1299,7 @@ class Grammar(object): ) + ~questionmark partial_trailer_tokens = Group(dollar.suppress() + function_call_tokens) - no_call_trailer = simple_trailer | known_trailer | partial_trailer + no_call_trailer = simple_trailer | partial_trailer | known_trailer no_partial_complex_trailer = call_trailer | known_trailer no_partial_trailer = simple_trailer | no_partial_complex_trailer @@ -1317,6 +1324,7 @@ class Grammar(object): itemgetter_atom = attach(itemgetter_atom_tokens, itemgetter_handle) implicit_partial_atom = ( + # itemgetter must come before attrgetter itemgetter_atom | attrgetter_atom | fixto(dot + lbrack + rbrack, "_coconut.operator.getitem") @@ -1351,7 +1359,7 @@ class Grammar(object): | lbrack + assignlist + rbrack ) star_assign_item_ref = condense(star + base_assign_item) - assign_item = star_assign_item | base_assign_item + assign_item = base_assign_item | star_assign_item assignlist <<= itemlist(assign_item, comma, suppress_trailing=False) typed_assign_stmt = Forward() @@ -1363,6 +1371,7 @@ class Grammar(object): type_var_name = stores_loc_item + setname type_param_constraint = lparen.suppress() + Group(tokenlist(typedef_test, comma, require_sep=True)) + rparen.suppress() type_param_ref = ( + # constraint must come before test (type_var_name + type_param_bound_op + type_param_constraint)("TypeVar constraint") | (type_var_name + Optional(type_param_bound_op + typedef_test))("TypeVar") | (star.suppress() + type_var_name)("TypeVarTuple") @@ -1378,15 +1387,15 @@ class Grammar(object): await_item = await_expr | atom_item factor = Forward() - unary = plus | neg_minus | tilde + unary = neg_minus | plus | tilde power = condense(exp_dubstar + ZeroOrMore(unary) + await_item) power_in_impl_call = Forward() impl_call_arg = condense(( - keyword_atom + disallow_keywords(reserved_vars) + dotted_refname | number - | disallow_keywords(reserved_vars) + dotted_refname + | keyword_atom ) + Optional(power_in_impl_call)) impl_call_item = condense( disallow_keywords(reserved_vars) @@ -1448,7 +1457,10 @@ class Grammar(object): infix_item = attach( Group(Optional(compose_expr)) + OneOrMore( - infix_op + Group(Optional(lambdef | compose_expr)) + infix_op + Group(Optional( + # lambdef must come first + lambdef | compose_expr + )) ), infix_handle, ) @@ -1459,22 +1471,25 @@ class Grammar(object): none_coalesce_expr = attach(tokenlist(infix_expr, dubquestion, allow_trailing=False), none_coalesce_handle) - comp_pipe_op = ( - comp_pipe - | comp_star_pipe - | comp_back_pipe - | comp_back_star_pipe - | comp_dubstar_pipe - | comp_back_dubstar_pipe - | comp_none_dubstar_pipe - | comp_back_none_dubstar_pipe - | comp_none_star_pipe - | comp_back_none_star_pipe - | comp_none_pipe - | comp_back_none_pipe + comp_pipe_op = any_of( + comp_pipe, + comp_star_pipe, + comp_back_pipe, + comp_back_star_pipe, + comp_dubstar_pipe, + comp_back_dubstar_pipe, + comp_none_dubstar_pipe, + comp_back_none_dubstar_pipe, + comp_none_star_pipe, + comp_back_none_star_pipe, + comp_none_pipe, + comp_back_none_pipe, ) comp_pipe_item = attach( - OneOrMore(none_coalesce_expr + comp_pipe_op) + (lambdef | none_coalesce_expr), + OneOrMore(none_coalesce_expr + comp_pipe_op) + ( + # lambdef must come first + lambdef | none_coalesce_expr + ), comp_pipe_handle, ) comp_pipe_expr = ( @@ -1482,26 +1497,27 @@ class Grammar(object): | comp_pipe_item ) - pipe_op = ( - pipe - | star_pipe - | dubstar_pipe - | back_pipe - | back_star_pipe - | back_dubstar_pipe - | none_pipe - | none_star_pipe - | none_dubstar_pipe - | back_none_pipe - | back_none_star_pipe - | back_none_dubstar_pipe + pipe_op = any_of( + pipe, + star_pipe, + dubstar_pipe, + back_pipe, + back_star_pipe, + back_dubstar_pipe, + none_pipe, + none_star_pipe, + none_dubstar_pipe, + back_none_pipe, + back_none_star_pipe, + back_none_dubstar_pipe, ) pipe_item = ( # we need the pipe_op since any of the atoms could otherwise be the start of an expression labeled_group(keyword("await"), "await") + pipe_op + | labeled_group(partial_atom_tokens, "partial") + pipe_op + # itemgetter must come before attrgetter | labeled_group(itemgetter_atom_tokens, "itemgetter") + pipe_op | labeled_group(attrgetter_atom_tokens, "attrgetter") + pipe_op - | labeled_group(partial_atom_tokens, "partial") + pipe_op | labeled_group(partial_op_atom_tokens, "op partial") + pipe_op # expr must come at end | labeled_group(comp_pipe_expr, "expr") + pipe_op @@ -1509,9 +1525,9 @@ class Grammar(object): pipe_augassign_item = ( # should match pipe_item but with pipe_op -> end_simple_stmt_item and no expr labeled_group(keyword("await"), "await") + end_simple_stmt_item + | labeled_group(partial_atom_tokens, "partial") + end_simple_stmt_item | labeled_group(itemgetter_atom_tokens, "itemgetter") + end_simple_stmt_item | labeled_group(attrgetter_atom_tokens, "attrgetter") + end_simple_stmt_item - | labeled_group(partial_atom_tokens, "partial") + end_simple_stmt_item | labeled_group(partial_op_atom_tokens, "op partial") + end_simple_stmt_item ) last_pipe_item = Group( @@ -1566,7 +1582,11 @@ class Grammar(object): keyword_lambdef_ref = addspace(keyword("lambda") + condense(keyword_lambdef_params + colon)) arrow_lambdef = attach(arrow_lambdef_params + lambda_arrow.suppress(), lambdef_handle) implicit_lambdef = fixto(lambda_arrow, "lambda _=None:") - lambdef_base = keyword_lambdef | arrow_lambdef | implicit_lambdef + lambdef_base = ( + arrow_lambdef + | implicit_lambdef + | keyword_lambdef + ) stmt_lambdef = Forward() match_guard = Optional(keyword("if").suppress() + namedexpr_test) @@ -1678,8 +1698,9 @@ class Grammar(object): test <<= ( typedef_callable | lambdef + # must come near end since it includes plain test_item + | addspace(test_item + Optional(keyword("if") + test_item + keyword("else") + test)) | alt_ternary_expr - | addspace(test_item + Optional(keyword("if") + test_item + keyword("else") + test)) # must come last since it includes plain test_item ) test_no_cond <<= lambdef_no_cond | test_item @@ -1726,7 +1747,7 @@ class Grammar(object): ) base_comp_for = addspace(keyword("for") + assignlist + keyword("in") + comp_it_item + Optional(comp_iter)) async_comp_for_ref = addspace(keyword("async") + base_comp_for) - comp_for <<= async_comp_for | base_comp_for + comp_for <<= base_comp_for | async_comp_for comp_if = addspace(keyword("if") + test_no_cond + Optional(comp_iter)) comp_iter <<= comp_for | comp_if @@ -1736,15 +1757,15 @@ class Grammar(object): pass_stmt = keyword("pass") break_stmt = keyword("break") continue_stmt = keyword("continue") - simple_raise_stmt = addspace(keyword("raise") + Optional(test)) + simple_raise_stmt = addspace(keyword("raise") + Optional(test)) + ~keyword("from") complex_raise_stmt_ref = keyword("raise").suppress() + test + keyword("from").suppress() - test - raise_stmt = complex_raise_stmt | simple_raise_stmt - flow_stmt = ( - return_stmt - | raise_stmt - | break_stmt - | yield_expr - | continue_stmt + flow_stmt = any_of( + return_stmt, + simple_raise_stmt, + break_stmt, + continue_stmt, + yield_expr, + complex_raise_stmt, ) imp_name = ( @@ -1911,26 +1932,44 @@ class Grammar(object): | (keyword("data").suppress() + dotted_refname + lparen.suppress() + matchlist_data + rparen.suppress())("data") | (keyword("class").suppress() + dotted_refname + lparen.suppress() + matchlist_data + rparen.suppress())("class") | (dotted_refname + lparen.suppress() + matchlist_data + rparen.suppress())("data_or_class") - | Optional(keyword("as").suppress()) + setname("var") + | Optional(keyword("as").suppress()) + setname("var"), ) matchlist_isinstance = base_match + OneOrMore(keyword("is").suppress() + negable_atom_item) - isinstance_match = labeled_group(matchlist_isinstance, "isinstance_is") | base_match + isinstance_match = ( + labeled_group(matchlist_isinstance, "isinstance_is") + | base_match + ) matchlist_bar_or = isinstance_match + OneOrMore(bar.suppress() + isinstance_match) - bar_or_match = labeled_group(matchlist_bar_or, "or") | isinstance_match + bar_or_match = ( + labeled_group(matchlist_bar_or, "or") + | isinstance_match + ) matchlist_infix = bar_or_match + OneOrMore(Group(infix_op + Optional(negable_atom_item))) - infix_match = labeled_group(matchlist_infix, "infix") | bar_or_match + infix_match = ( + labeled_group(matchlist_infix, "infix") + | bar_or_match + ) matchlist_as = infix_match + OneOrMore(keyword("as").suppress() + setname) - as_match = labeled_group(matchlist_as, "as") | infix_match + as_match = ( + labeled_group(matchlist_as, "as") + | infix_match + ) matchlist_and = as_match + OneOrMore(keyword("and").suppress() + as_match) - and_match = labeled_group(matchlist_and, "and") | as_match + and_match = ( + labeled_group(matchlist_and, "and") + | as_match + ) matchlist_kwd_or = and_match + OneOrMore(keyword("or").suppress() + and_match) - kwd_or_match = labeled_group(matchlist_kwd_or, "or") | and_match + kwd_or_match = ( + labeled_group(matchlist_kwd_or, "or") + | and_match + ) match <<= kwd_or_match @@ -2075,14 +2114,14 @@ class Grammar(object): op_match_funcdef_ref = keyword("def").suppress() + op_match_funcdef_arg + op_funcdef_name + op_match_funcdef_arg + match_guard base_match_funcdef = op_match_funcdef | name_match_funcdef func_suite = ( - attach(simple_stmt, make_suite_handle) - | ( + ( newline.suppress() - indent.suppress() - Optional(docstring) - attach(condense(OneOrMore(stmt)), make_suite_handle) - dedent.suppress() ) + | attach(simple_stmt, make_suite_handle) ) def_match_funcdef = attach( base_match_funcdef @@ -2119,8 +2158,8 @@ class Grammar(object): math_funcdef_body = condense(ZeroOrMore(~(implicit_return_stmt + dedent) + stmt) - implicit_return_stmt) math_funcdef_suite = ( - attach(implicit_return_stmt, make_suite_handle) - | condense(newline - indent - math_funcdef_body - dedent) + condense(newline - indent - math_funcdef_body - dedent) + | attach(implicit_return_stmt, make_suite_handle) ) end_func_equals = return_typedef + equals.suppress() | fixto(equals, ":") math_funcdef = attach( @@ -2203,6 +2242,7 @@ class Grammar(object): async_keyword_funcdef_ref = async_keyword_normal_funcdef | async_keyword_match_funcdef async_funcdef_stmt = ( + # match funcdefs must come after normal async_funcdef | async_match_funcdef | async_keyword_funcdef @@ -2227,6 +2267,7 @@ class Grammar(object): keyword_funcdef_ref = keyword_normal_funcdef | keyword_match_funcdef normal_funcdef_stmt = ( + # match funcdefs must come after normal funcdef | math_funcdef | math_match_funcdef @@ -2301,33 +2342,33 @@ class Grammar(object): passthrough_stmt = condense(passthrough_block - (base_suite | newline)) - simple_compound_stmt = ( - if_stmt - | try_stmt - | match_stmt - | passthrough_stmt - ) - compound_stmt = ( - decoratable_class_stmt - | decoratable_func_stmt - | while_stmt - | for_stmt - | with_stmt - | async_stmt - | match_for_stmt - | simple_compound_stmt - | where_stmt + simple_compound_stmt = any_of( + if_stmt, + try_stmt, + match_stmt, + passthrough_stmt, + ) + compound_stmt = any_of( + decoratable_class_stmt, + decoratable_func_stmt, + while_stmt, + for_stmt, + with_stmt, + async_stmt, + match_for_stmt, + simple_compound_stmt, + where_stmt, ) endline_semicolon = Forward() endline_semicolon_ref = semicolon.suppress() + newline - keyword_stmt = ( - flow_stmt - | import_stmt - | assert_stmt - | pass_stmt - | del_stmt - | global_stmt - | nonlocal_stmt + keyword_stmt = any_of( + flow_stmt, + import_stmt, + assert_stmt, + pass_stmt, + del_stmt, + global_stmt, + nonlocal_stmt, ) special_stmt = ( keyword_stmt @@ -2337,6 +2378,7 @@ class Grammar(object): ) unsafe_simple_stmt_item <<= special_stmt | longest(basic_stmt, destructuring_stmt) simple_stmt_item <<= ( + # destructuring stmt must come after basic special_stmt | basic_stmt + end_simple_stmt_item | destructuring_stmt + end_simple_stmt_item @@ -2439,13 +2481,19 @@ class Grammar(object): lparen, disallow_keywords(untcoable_funcs, with_suffix="(") + condense( - (unsafe_name | parens | brackets | braces | string_atom) - + ZeroOrMore( - dot + unsafe_name - | brackets + any_of( + unsafe_name, + parens, + string_atom, + brackets, + braces, + ) + + ZeroOrMore(any_of( + dot + unsafe_name, + brackets, # don't match the last set of parentheses - | parens + ~end_marker + ~rparen - ), + parens + ~end_marker + ~rparen, + )), ) + original_function_call_tokens, rparen, @@ -2484,10 +2532,10 @@ class Grammar(object): parameters_tokens = Group( Optional(tokenlist( Group( - dubstar - tfpdef_tokens + tfpdef_default_tokens | star - Optional(tfpdef_tokens) + | dubstar - tfpdef_tokens | slash - | tfpdef_default_tokens ) + type_comment, comma + type_comment, )) @@ -2530,7 +2578,7 @@ class Grammar(object): ) ) - end_f_str_expr = combine(start_marker + (bang | colon | rbrace)) + end_f_str_expr = combine(start_marker + (rbrace | colon | bang)) string_start = start_marker + python_quoted_string diff --git a/coconut/compiler/util.py b/coconut/compiler/util.py index 4e8a138ee..982957dc4 100644 --- a/coconut/compiler/util.py +++ b/coconut/compiler/util.py @@ -118,6 +118,7 @@ incremental_cache_limit, incremental_mode_cache_successes, adaptive_reparse_usage_weight, + use_adaptive_any_of, ) from coconut.exceptions import ( CoconutException, @@ -471,14 +472,16 @@ def unpack(tokens): return tokens +def in_incremental_mode(): + """Determine if we are using the --incremental parsing mode.""" + return ParserElement._incrementalEnabled and not ParserElement._incrementalWithResets + + def force_reset_packrat_cache(): """Forcibly reset the packrat cache and all packrat stats.""" if ParserElement._incrementalEnabled: ParserElement._incrementalEnabled = False - if ParserElement._incrementalWithResets: - ParserElement.enableIncremental(default_incremental_cache_size, still_reset_cache=True) - else: - ParserElement.enableIncremental(incremental_mode_cache_size, still_reset_cache=False) + ParserElement.enableIncremental(incremental_mode_cache_size if in_incremental_mode() else default_incremental_cache_size, still_reset_cache=False) else: ParserElement._packratEnabled = False ParserElement.enablePackrat(packrat_cache_size) @@ -487,7 +490,9 @@ def force_reset_packrat_cache(): @contextmanager def parsing_context(inner_parse=True): """Context to manage the packrat cache across parse calls.""" - if inner_parse and should_clear_cache(): + if not inner_parse: + yield + elif should_clear_cache(): # store old packrat cache old_cache = ParserElement.packrat_cache old_cache_stats = ParserElement.packrat_cache_stats[:] @@ -501,7 +506,8 @@ def parsing_context(inner_parse=True): if logger.verbose: ParserElement.packrat_cache_stats[0] += old_cache_stats[0] ParserElement.packrat_cache_stats[1] += old_cache_stats[1] - elif inner_parse and ParserElement._incrementalWithResets: + # if we shouldn't clear the cache, but we're using incrementalWithResets, then do this to avoid clearing it + elif ParserElement._incrementalWithResets: incrementalWithResets, ParserElement._incrementalWithResets = ParserElement._incrementalWithResets, False try: yield @@ -615,7 +621,7 @@ def should_clear_cache(force=False): if SUPPORTS_INCREMENTAL: if ( not ParserElement._incrementalEnabled - or ParserElement._incrementalWithResets and repeatedly_clear_incremental_cache + or not in_incremental_mode() and repeatedly_clear_incremental_cache ): return True if force or ( @@ -672,7 +678,7 @@ def enable_incremental_parsing(): if not SUPPORTS_INCREMENTAL: return False ParserElement._should_cache_incremental_success = incremental_mode_cache_successes - if ParserElement._incrementalEnabled and not ParserElement._incrementalWithResets: # incremental mode is already enabled + if in_incremental_mode(): # incremental mode is already enabled return True ParserElement._incrementalEnabled = False try: @@ -836,10 +842,30 @@ class MatchAny(MatchFirst): """Version of MatchFirst that always uses adaptive parsing.""" adaptive_mode = True + def __or__(self, other): + if hasaction(self): + return MatchFirst([self, other]) + self = maybe_copy_elem(self, "any_or") + if not isinstance(other, MatchAny): + self.__class__ = MatchFirst + self |= other + return self + -def any_of(*exprs): +def any_of(*exprs, **kwargs): """Build a MatchAny of the given MatchFirst.""" - return MatchAny(exprs) + use_adaptive = kwargs.pop("use_adaptive", use_adaptive_any_of) + internal_assert(not kwargs, "excess keyword arguments passed to any_of", kwargs) + + AnyOf = MatchAny if use_adaptive else MatchFirst + + flat_exprs = [] + for e in exprs: + if isinstance(e, AnyOf) and not hasaction(e): + flat_exprs.extend(e.exprs) + else: + flat_exprs.append(e) + return AnyOf(flat_exprs) class Wrap(ParseElementEnhance): @@ -1136,6 +1162,14 @@ def disallow_keywords(kwds, with_suffix=""): return regex_item(r"(?!" + "|".join(to_disallow) + r")").suppress() +def disambiguate_literal(literal, not_literals): + """Get an item that matchesl literal and not any of not_literals.""" + return regex_item( + r"(?!" + "|".join(re.escape(s) for s in not_literals) + ")" + + re.escape(literal) + ) + + def any_keyword_in(kwds): """Match any of the given keywords.""" return regex_item(r"|".join(k + r"\b" for k in kwds)) diff --git a/coconut/constants.py b/coconut/constants.py index 7c0a8c11c..b664a3c5e 100644 --- a/coconut/constants.py +++ b/coconut/constants.py @@ -130,7 +130,9 @@ def get_path_env_var(env_var, default): # note that _parseIncremental produces much smaller caches use_incremental_if_available = True -use_adaptive_if_available = False +use_adaptive_any_of = False + +use_adaptive_if_available = False # currently broken adaptive_reparse_usage_weight = 10 # these only apply to use_incremental_if_available, not compiler.util.enable_incremental_parsing() @@ -985,7 +987,7 @@ def get_path_env_var(env_var, default): # min versions are inclusive unpinned_min_versions = { - "cPyparsing": (2, 4, 7, 2, 2, 3), + "cPyparsing": (2, 4, 7, 2, 2, 4), ("pre-commit", "py3"): (3,), ("psutil", "py>=27"): (5,), "jupyter": (1, 0), diff --git a/coconut/root.py b/coconut/root.py index c9bcc020b..9be6a4efd 100644 --- a/coconut/root.py +++ b/coconut/root.py @@ -26,7 +26,7 @@ VERSION = "3.0.3" VERSION_NAME = None # False for release, int >= 1 for develop -DEVELOP = 22 +DEVELOP = 23 ALPHA = False # for pre releases rather than post releases assert DEVELOP is False or DEVELOP >= 1, "DEVELOP must be False or an int >= 1" diff --git a/coconut/terminal.py b/coconut/terminal.py index 574918292..c33d81877 100644 --- a/coconut/terminal.py +++ b/coconut/terminal.py @@ -36,6 +36,7 @@ lineno, col, ParserElement, + maybe_make_safe, ) from coconut.root import _indent @@ -57,6 +58,8 @@ get_clock_time, get_name, displayable, + first_import_time, + assert_remove_prefix, ) from coconut.exceptions import ( CoconutWarning, @@ -231,9 +234,7 @@ def setup(self, quiet=None, verbose=None, tracing=None): self.verbose = verbose if tracing is not None: self.tracing = tracing - - if self.verbose: - ParserElement.verbose_stacktrace = True + ParserElement.verbose_stacktrace = self.verbose def display( self, @@ -552,9 +553,22 @@ def gather_parsing_stats(self): if "adaptive" in self.recorded_stats: failures, successes = self.recorded_stats["adaptive"] self.printlog("\tAdaptive parsing stats:", successes, "successes;", failures, "failures") + if maybe_make_safe is not None: + hits, misses = maybe_make_safe.stats + self.printlog("\tErrorless parsing stats:", hits, "errorless;", misses, "with errors") else: yield + def log_compiler_stats(self, comp): + """Log stats for the given compiler.""" + if self.verbose: + self.log("Grammar init time: " + str(comp.grammar_init_time) + " secs / Total init time: " + str(get_clock_time() - first_import_time) + " secs") + for stat_name, (no_copy, yes_copy) in self.recorded_stats.items(): + if not stat_name.startswith("maybe_copy_"): + continue + name = assert_remove_prefix(stat_name, "maybe_copy_") + self.printlog("\tGrammar copying stats (" + name + "):", no_copy, "not copied;", yes_copy, "copied") + total_block_time = defaultdict(int) @contextmanager From bb83c9c7e51753eb425ba95f4076ad84b74e764f Mon Sep 17 00:00:00 2001 From: Evan Hubinger Date: Sun, 12 Nov 2023 00:48:18 -0800 Subject: [PATCH 078/121] Improve --incremental --- Makefile | 2 +- coconut/_pyparsing.py | 10 +++--- coconut/command/command.py | 14 +++----- coconut/compiler/compiler.py | 41 +++++++++++++++------- coconut/compiler/util.py | 66 +++++++++++++++++++++++------------- coconut/constants.py | 4 +-- 6 files changed, 85 insertions(+), 52 deletions(-) diff --git a/Makefile b/Makefile index 1313b1215..9988b6868 100644 --- a/Makefile +++ b/Makefile @@ -156,7 +156,7 @@ test-mypy-tests: clean-no-tests python ./coconut/tests/dest/extras.py # same as test-univ but includes verbose output for better debugging -# regex for getting non-timing lines: ^(?!Time|\s+Packrat|Loaded|Saving|Adaptive|tErrorless)[^\n]*\n* +# regex for getting non-timing lines: ^(?!\s*(Time|Packrat|Loaded|Saving|Adaptive|Errorless|Grammar))[^\n]*\n* .PHONY: test-verbose test-verbose: export COCONUT_USE_COLOR=TRUE test-verbose: clean diff --git a/coconut/_pyparsing.py b/coconut/_pyparsing.py index 182bded20..9d7ed9b2b 100644 --- a/coconut/_pyparsing.py +++ b/coconut/_pyparsing.py @@ -67,7 +67,7 @@ from cPyparsing import * # NOQA from cPyparsing import __version__ - PYPARSING_PACKAGE = "cPyparsing" + CPYPARSING = True PYPARSING_INFO = "Cython cPyparsing v" + __version__ except ImportError: @@ -77,13 +77,13 @@ from pyparsing import * # NOQA from pyparsing import __version__ - PYPARSING_PACKAGE = "pyparsing" + CPYPARSING = False PYPARSING_INFO = "Python pyparsing v" + __version__ except ImportError: traceback.print_exc() __version__ = None - PYPARSING_PACKAGE = "cPyparsing" + CPYPARSING = True PYPARSING_INFO = None @@ -91,6 +91,8 @@ # VERSIONING: # ----------------------------------------------------------------------------------------------------------------------- +PYPARSING_PACKAGE = "cPyparsing" if CPYPARSING else "pyparsing" + min_ver = min(min_versions["pyparsing"], min_versions["cPyparsing"][:3]) # inclusive max_ver = get_next_version(max(min_versions["pyparsing"], min_versions["cPyparsing"][:3])) # exclusive cur_ver = None if __version__ is None else ver_str_to_tuple(__version__) @@ -124,7 +126,7 @@ # OVERRIDES: # ----------------------------------------------------------------------------------------------------------------------- -if PYPARSING_PACKAGE != "cPyparsing": +if not CPYPARSING: if not MODERN_PYPARSING: HIT, MISS = 0, 1 diff --git a/coconut/command/command.py b/coconut/command/command.py index 0d19344ff..3fd5b1d70 100644 --- a/coconut/command/command.py +++ b/coconut/command/command.py @@ -74,7 +74,6 @@ coconut_cache_dir, coconut_sys_kwargs, interpreter_uses_incremental, - disable_incremental_for_len, ) from coconut.util import ( univ_open, @@ -611,16 +610,13 @@ def callback(compiled): filename=os.path.basename(codepath), ) if self.incremental: - if disable_incremental_for_len is not None and len(code) > disable_incremental_for_len: - logger.warn("--incremental mode is not currently supported for files as large as {codepath!r}".format(codepath=codepath)) - else: - code_dir, code_fname = os.path.split(codepath) + code_dir, code_fname = os.path.split(codepath) - cache_dir = os.path.join(code_dir, coconut_cache_dir) - ensure_dir(cache_dir) + cache_dir = os.path.join(code_dir, coconut_cache_dir) + ensure_dir(cache_dir) - pickle_fname = code_fname + ".pickle" - parse_kwargs["incremental_cache_filename"] = os.path.join(cache_dir, pickle_fname) + pickle_fname = code_fname + ".pickle" + parse_kwargs["cache_filename"] = os.path.join(cache_dir, pickle_fname) if package is True: self.submit_comp_job(codepath, callback, "parse_package", code, package_level=package_level, **parse_kwargs) diff --git a/coconut/compiler/compiler.py b/coconut/compiler/compiler.py index 1f1ce3c39..c0ee3717b 100644 --- a/coconut/compiler/compiler.py +++ b/coconut/compiler/compiler.py @@ -38,6 +38,7 @@ from coconut._pyparsing import ( USE_COMPUTATION_GRAPH, + CPYPARSING, ParseBaseException, ParseResults, col as getcol, @@ -86,6 +87,7 @@ in_place_op_funcs, match_first_arg_var, import_existing, + disable_incremental_for_len, ) from coconut.util import ( pickleable_obj, @@ -170,8 +172,8 @@ get_psf_target, move_loc_to_non_whitespace, move_endpt_to_non_whitespace, - unpickle_incremental_cache, - pickle_incremental_cache, + unpickle_cache, + pickle_cache, handle_and_manage, sub_all, ) @@ -1310,7 +1312,7 @@ def parse( streamline=True, keep_state=False, filename=None, - incremental_cache_filename=None, + cache_filename=None, ): """Use the parser to parse the inputstring with appropriate setup and teardown.""" with self.parsing(keep_state, filename): @@ -1318,15 +1320,30 @@ def parse( self.streamline(parser, inputstring) # unpickling must happen after streamlining and must occur in the # compiler so that it happens in the same process as compilation - if incremental_cache_filename is not None: - incremental_enabled = enable_incremental_parsing() - if not incremental_enabled: - raise CoconutException("incremental_cache_filename requires cPyparsing (run '{python} -m pip install --upgrade cPyparsing' to fix)".format(python=sys.executable)) - did_load_cache = unpickle_incremental_cache(incremental_cache_filename) - logger.log("{Loaded} incremental cache for {filename!r} from {incremental_cache_filename!r}.".format( + if cache_filename is not None: + if not CPYPARSING: + raise CoconutException("cache_filename requires cPyparsing (run '{python} -m pip install --upgrade cPyparsing' to fix)".format(python=sys.executable)) + if len(inputstring) < disable_incremental_for_len: + incremental_enabled = enable_incremental_parsing() + if incremental_enabled: + incremental_info = "incremental parsing mode enabled due to len == {input_len} < {max_len}".format( + input_len=len(inputstring), + max_len=disable_incremental_for_len, + ) + else: + incremental_info = "failed to enable incremental parsing mode" + else: + incremental_enabled = False + incremental_info = "not using incremental parsing mode due to len == {input_len} >= {max_len}".format( + input_len=len(inputstring), + max_len=disable_incremental_for_len, + ) + did_load_cache = unpickle_cache(cache_filename) + logger.log("{Loaded} cache for {filename!r} from {cache_filename!r} ({incremental_info}).".format( Loaded="Loaded" if did_load_cache else "Failed to load", filename=filename, - incremental_cache_filename=incremental_cache_filename, + cache_filename=cache_filename, + incremental_info=incremental_info, )) pre_procd = parsed = None try: @@ -1347,8 +1364,8 @@ def parse( + str(sys.getrecursionlimit()) + " (you may also need to increase --stack-size)", ) finally: - if incremental_cache_filename is not None and pre_procd is not None: - pickle_incremental_cache(pre_procd, incremental_cache_filename) + if cache_filename is not None and pre_procd is not None: + pickle_cache(pre_procd, cache_filename, include_incremental=incremental_enabled) self.run_final_checks(pre_procd, keep_state) return out diff --git a/coconut/compiler/util.py b/coconut/compiler/util.py index 982957dc4..d1050d56c 100644 --- a/coconut/compiler/util.py +++ b/coconut/compiler/util.py @@ -689,43 +689,50 @@ def enable_incremental_parsing(): return True -def pickle_incremental_cache(original, filename, protocol=pickle.HIGHEST_PROTOCOL): +def pickle_cache(original, filename, include_incremental=True, protocol=pickle.HIGHEST_PROTOCOL): """Pickle the pyparsing cache for original to filename.""" - internal_assert(all_parse_elements is not None, "pickle_incremental_cache requires cPyparsing") + internal_assert(all_parse_elements is not None, "pickle_cache requires cPyparsing") pickleable_cache_items = [] - for lookup, value in get_cache_items_for(original): - if incremental_mode_cache_size is not None and len(pickleable_cache_items) > incremental_mode_cache_size: - complain( - "got too large incremental cache: " - + str(len(get_pyparsing_cache())) + " > " + str(incremental_mode_cache_size) - ) - break - if len(pickleable_cache_items) >= incremental_cache_limit: - break - loc = lookup[2] - # only include cache items that aren't at the start or end, since those - # are the only ones that parseIncremental will reuse - if 0 < loc < len(original) - 1: - pickleable_lookup = (lookup[0].parse_element_index,) + lookup[1:] - pickleable_cache_items.append((pickleable_lookup, value)) - - logger.log("Saving {num_items} incremental cache items to {filename!r}.".format( - num_items=len(pickleable_cache_items), + if include_incremental: + for lookup, value in get_cache_items_for(original): + if incremental_mode_cache_size is not None and len(pickleable_cache_items) > incremental_mode_cache_size: + complain( + "got too large incremental cache: " + + str(len(get_pyparsing_cache())) + " > " + str(incremental_mode_cache_size) + ) + break + if len(pickleable_cache_items) >= incremental_cache_limit: + break + loc = lookup[2] + # only include cache items that aren't at the start or end, since those + # are the only ones that parseIncremental will reuse + if 0 < loc < len(original) - 1: + pickleable_lookup = (lookup[0].parse_element_index,) + lookup[1:] + pickleable_cache_items.append((pickleable_lookup, value)) + + all_adaptive_stats = {} + for match_any in MatchAny.all_match_anys: + all_adaptive_stats[match_any.parse_element_index] = match_any.adaptive_usage + + logger.log("Saving {num_inc} incremental and {num_adapt} adaptive cache items to {filename!r}.".format( + num_inc=len(pickleable_cache_items), + num_adapt=len(all_adaptive_stats), filename=filename, )) pickle_info_obj = { "VERSION": VERSION, "pyparsing_version": pyparsing_version, "pickleable_cache_items": pickleable_cache_items, + "all_adaptive_stats": all_adaptive_stats, } with univ_open(filename, "wb") as pickle_file: pickle.dump(pickle_info_obj, pickle_file, protocol=protocol) -def unpickle_incremental_cache(filename): +def unpickle_cache(filename): """Unpickle and load the given incremental cache file.""" - internal_assert(all_parse_elements is not None, "unpickle_incremental_cache requires cPyparsing") + internal_assert(all_parse_elements is not None, "unpickle_cache requires cPyparsing") if not os.path.exists(filename): return False @@ -739,11 +746,17 @@ def unpickle_incremental_cache(filename): return False pickleable_cache_items = pickle_info_obj["pickleable_cache_items"] - logger.log("Loaded {num_items} incremental cache items from {filename!r}.".format( - num_items=len(pickleable_cache_items), + all_adaptive_stats = pickle_info_obj["all_adaptive_stats"] + + logger.log("Loaded {num_inc} incremental and {num_adapt} adaptive cache items from {filename!r}.".format( + num_inc=len(pickleable_cache_items), + num_adapt=len(all_adaptive_stats), filename=filename, )) + for identifier, adaptive_usage in all_adaptive_stats.items(): + all_parse_elements[identifier].adaptive_usage = adaptive_usage + max_cache_size = min( incremental_mode_cache_size or float("inf"), incremental_cache_limit or float("inf"), @@ -841,6 +854,11 @@ def get_target_info_smart(target, mode="lowest"): class MatchAny(MatchFirst): """Version of MatchFirst that always uses adaptive parsing.""" adaptive_mode = True + all_match_anys = [] + + def __init__(self, *args, **kwargs): + super(MatchAny, self).__init__(*args, **kwargs) + self.all_match_anys.append(self) def __or__(self, other): if hasaction(self): diff --git a/coconut/constants.py b/coconut/constants.py index b664a3c5e..09a7a747d 100644 --- a/coconut/constants.py +++ b/coconut/constants.py @@ -128,9 +128,9 @@ def get_path_env_var(env_var, default): packrat_cache_size = None # only works because final() clears the cache # note that _parseIncremental produces much smaller caches -use_incremental_if_available = True +use_incremental_if_available = False -use_adaptive_any_of = False +use_adaptive_any_of = True use_adaptive_if_available = False # currently broken adaptive_reparse_usage_weight = 10 From 8ce71c8f956e9e93a82e75cc658e6545ad76b5ca Mon Sep 17 00:00:00 2001 From: Evan Hubinger Date: Sun, 12 Nov 2023 01:43:47 -0800 Subject: [PATCH 079/121] Further improve --incremental --- coconut/compiler/compiler.py | 29 +------ coconut/compiler/util.py | 143 +++++++++++++++++++++-------------- coconut/root.py | 2 +- coconut/terminal.py | 2 +- 4 files changed, 93 insertions(+), 83 deletions(-) diff --git a/coconut/compiler/compiler.py b/coconut/compiler/compiler.py index c0ee3717b..53487eb7e 100644 --- a/coconut/compiler/compiler.py +++ b/coconut/compiler/compiler.py @@ -38,7 +38,6 @@ from coconut._pyparsing import ( USE_COMPUTATION_GRAPH, - CPYPARSING, ParseBaseException, ParseResults, col as getcol, @@ -87,7 +86,6 @@ in_place_op_funcs, match_first_arg_var, import_existing, - disable_incremental_for_len, ) from coconut.util import ( pickleable_obj, @@ -172,7 +170,7 @@ get_psf_target, move_loc_to_non_whitespace, move_endpt_to_non_whitespace, - unpickle_cache, + load_cache_for, pickle_cache, handle_and_manage, sub_all, @@ -1321,30 +1319,11 @@ def parse( # unpickling must happen after streamlining and must occur in the # compiler so that it happens in the same process as compilation if cache_filename is not None: - if not CPYPARSING: - raise CoconutException("cache_filename requires cPyparsing (run '{python} -m pip install --upgrade cPyparsing' to fix)".format(python=sys.executable)) - if len(inputstring) < disable_incremental_for_len: - incremental_enabled = enable_incremental_parsing() - if incremental_enabled: - incremental_info = "incremental parsing mode enabled due to len == {input_len} < {max_len}".format( - input_len=len(inputstring), - max_len=disable_incremental_for_len, - ) - else: - incremental_info = "failed to enable incremental parsing mode" - else: - incremental_enabled = False - incremental_info = "not using incremental parsing mode due to len == {input_len} >= {max_len}".format( - input_len=len(inputstring), - max_len=disable_incremental_for_len, - ) - did_load_cache = unpickle_cache(cache_filename) - logger.log("{Loaded} cache for {filename!r} from {cache_filename!r} ({incremental_info}).".format( - Loaded="Loaded" if did_load_cache else "Failed to load", + incremental_enabled = load_cache_for( + inputstring=inputstring, filename=filename, cache_filename=cache_filename, - incremental_info=incremental_info, - )) + ) pre_procd = parsed = None try: with logger.gather_parsing_stats(): diff --git a/coconut/compiler/util.py b/coconut/compiler/util.py index d1050d56c..8d3bcefac 100644 --- a/coconut/compiler/util.py +++ b/coconut/compiler/util.py @@ -45,6 +45,7 @@ import cPickle as pickle from coconut._pyparsing import ( + CPYPARSING, MODERN_PYPARSING, USE_COMPUTATION_GRAPH, SUPPORTS_INCREMENTAL, @@ -119,6 +120,7 @@ incremental_mode_cache_successes, adaptive_reparse_usage_weight, use_adaptive_any_of, + disable_incremental_for_len, ) from coconut.exceptions import ( CoconutException, @@ -327,60 +329,6 @@ def postParse(self, original, loc, tokens): combine = Combine -def maybe_copy_elem(item, name): - """Copy the given grammar element if it's referenced somewhere else.""" - item_ref_count = sys.getrefcount(item) if CPYTHON and not on_new_python else float("inf") - internal_assert(lambda: item_ref_count >= temp_grammar_item_ref_count, "add_action got item with too low ref count", (item, type(item), item_ref_count)) - if item_ref_count <= temp_grammar_item_ref_count: - if DEVELOP: - logger.record_stat("maybe_copy_" + name, False) - return item - else: - if DEVELOP: - logger.record_stat("maybe_copy_" + name, True) - return item.copy() - - -def hasaction(elem): - """Determine if the given grammar element has any actions associated with it.""" - return ( - MODERN_PYPARSING - or elem.parseAction - or elem.resultsName is not None - or elem.debug - ) - - -@contextmanager -def using_fast_grammar_methods(): - """Enables grammar methods that modify their operands when they aren't referenced elsewhere.""" - if MODERN_PYPARSING: - yield - return - - def fast_add(self, other): - if hasaction(self): - return old_add(self, other) - self = maybe_copy_elem(self, "add") - self += other - return self - old_add, And.__add__ = And.__add__, fast_add - - def fast_or(self, other): - if hasaction(self): - return old_or(self, other) - self = maybe_copy_elem(self, "or") - self |= other - return self - old_or, MatchFirst.__or__ = MatchFirst.__or__, fast_or - - try: - yield - finally: - And.__add__ = old_add - MatchFirst.__or__ = old_or - - def add_action(item, action, make_copy=None): """Add a parse action to the given item.""" if make_copy is None: @@ -586,6 +534,59 @@ def transform(grammar, text, inner=True): # PARSING INTROSPECTION: # ----------------------------------------------------------------------------------------------------------------------- +def maybe_copy_elem(item, name): + """Copy the given grammar element if it's referenced somewhere else.""" + item_ref_count = sys.getrefcount(item) if CPYTHON and not on_new_python else float("inf") + internal_assert(lambda: item_ref_count >= temp_grammar_item_ref_count, "add_action got item with too low ref count", (item, type(item), item_ref_count)) + if item_ref_count <= temp_grammar_item_ref_count: + if DEVELOP: + logger.record_stat("maybe_copy_" + name, False) + return item + else: + if DEVELOP: + logger.record_stat("maybe_copy_" + name, True) + return item.copy() + + +def hasaction(elem): + """Determine if the given grammar element has any actions associated with it.""" + return ( + MODERN_PYPARSING + or elem.parseAction + or elem.resultsName is not None + or elem.debug + ) + + +@contextmanager +def using_fast_grammar_methods(): + """Enables grammar methods that modify their operands when they aren't referenced elsewhere.""" + if MODERN_PYPARSING: + yield + return + + def fast_add(self, other): + if hasaction(self): + return old_add(self, other) + self = maybe_copy_elem(self, "add") + self += other + return self + old_add, And.__add__ = And.__add__, fast_add + + def fast_or(self, other): + if hasaction(self): + return old_or(self, other) + self = maybe_copy_elem(self, "or") + self |= other + return self + old_or, MatchFirst.__or__ = MatchFirst.__or__, fast_or + + try: + yield + finally: + And.__add__ = old_add + MatchFirst.__or__ = old_or + def get_func_closure(func): """Get variables in func's closure.""" @@ -713,7 +714,7 @@ def pickle_cache(original, filename, include_incremental=True, protocol=pickle.H all_adaptive_stats = {} for match_any in MatchAny.all_match_anys: - all_adaptive_stats[match_any.parse_element_index] = match_any.adaptive_usage + all_adaptive_stats[match_any.parse_element_index] = (match_any.adaptive_usage, match_any.expr_order) logger.log("Saving {num_inc} incremental and {num_adapt} adaptive cache items to {filename!r}.".format( num_inc=len(pickleable_cache_items), @@ -754,8 +755,9 @@ def unpickle_cache(filename): filename=filename, )) - for identifier, adaptive_usage in all_adaptive_stats.items(): + for identifier, (adaptive_usage, expr_order) in all_adaptive_stats.items(): all_parse_elements[identifier].adaptive_usage = adaptive_usage + all_parse_elements[identifier].expr_order = expr_order max_cache_size = min( incremental_mode_cache_size or float("inf"), @@ -770,6 +772,35 @@ def unpickle_cache(filename): return True +def load_cache_for(inputstring, filename, cache_filename): + """Load cache_filename (for the given inputstring and filename).""" + if not CPYPARSING: + raise CoconutException("--incremental requires cPyparsing (run '{python} -m pip install --upgrade cPyparsing' to fix)".format(python=sys.executable)) + if len(inputstring) < disable_incremental_for_len: + incremental_enabled = enable_incremental_parsing() + if incremental_enabled: + incremental_info = "incremental parsing mode enabled due to len == {input_len} < {max_len}".format( + input_len=len(inputstring), + max_len=disable_incremental_for_len, + ) + else: + incremental_info = "failed to enable incremental parsing mode" + else: + incremental_enabled = False + incremental_info = "not using incremental parsing mode due to len == {input_len} >= {max_len}".format( + input_len=len(inputstring), + max_len=disable_incremental_for_len, + ) + did_load_cache = unpickle_cache(cache_filename) + logger.log("{Loaded} cache for {filename!r} from {cache_filename!r} ({incremental_info}).".format( + Loaded="Loaded" if did_load_cache else "Failed to load", + filename=filename, + cache_filename=cache_filename, + incremental_info=incremental_info, + )) + return incremental_enabled + + # ----------------------------------------------------------------------------------------------------------------------- # TARGETS: # ----------------------------------------------------------------------------------------------------------------------- diff --git a/coconut/root.py b/coconut/root.py index 9be6a4efd..323d8124b 100644 --- a/coconut/root.py +++ b/coconut/root.py @@ -26,7 +26,7 @@ VERSION = "3.0.3" VERSION_NAME = None # False for release, int >= 1 for develop -DEVELOP = 23 +DEVELOP = 24 ALPHA = False # for pre releases rather than post releases assert DEVELOP is False or DEVELOP >= 1, "DEVELOP must be False or an int >= 1" diff --git a/coconut/terminal.py b/coconut/terminal.py index c33d81877..8d96938b3 100644 --- a/coconut/terminal.py +++ b/coconut/terminal.py @@ -360,7 +360,7 @@ def log_vars(self, message, variables, rem_vars=("self",)): def log_loc(self, name, original, loc): """Log a location in source code.""" - if self.verbose: + if self.tracing: if isinstance(loc, int): pre_loc_orig, post_loc_orig = original[:loc], original[loc:] if pre_loc_orig.count("\n") > max_orig_lines_in_log_loc: From 0b2db7ed56a9949380a0cd1e22bf7dba1ec8fd6d Mon Sep 17 00:00:00 2001 From: Evan Hubinger Date: Sun, 12 Nov 2023 02:12:35 -0800 Subject: [PATCH 080/121] Always use cache --- Makefile | 30 +++++++++++------------------- coconut/command/cli.py | 12 ++++++------ coconut/command/command.py | 14 +++++++++----- coconut/compiler/util.py | 12 +++++++----- coconut/constants.py | 8 +++++++- coconut/root.py | 2 +- coconut/tests/main_test.py | 7 +++---- 7 files changed, 44 insertions(+), 41 deletions(-) diff --git a/Makefile b/Makefile index 9988b6868..229aae742 100644 --- a/Makefile +++ b/Makefile @@ -95,7 +95,7 @@ test-univ: clean .PHONY: test-univ-tests test-univ-tests: export COCONUT_USE_COLOR=TRUE test-univ-tests: clean-no-tests - python ./coconut/tests --strict --keep-lines --incremental + python ./coconut/tests --strict --keep-lines python ./coconut/tests/dest/runner.py python ./coconut/tests/dest/extras.py @@ -151,7 +151,7 @@ test-mypy: clean .PHONY: test-mypy-tests test-mypy-tests: export COCONUT_USE_COLOR=TRUE test-mypy-tests: clean-no-tests - python ./coconut/tests --strict --keep-lines --incremental --target sys --mypy --follow-imports silent --ignore-missing-imports --allow-redefinition + python ./coconut/tests --strict --keep-lines --target sys --mypy --follow-imports silent --ignore-missing-imports --allow-redefinition python ./coconut/tests/dest/runner.py python ./coconut/tests/dest/extras.py @@ -164,7 +164,15 @@ test-verbose: clean python ./coconut/tests/dest/runner.py python ./coconut/tests/dest/extras.py -# same as test-univ but includes verbose output for better debugging and is fully synchronous +# same as test-verbose but doesn't use the incremental cache +.PHONY: test-verbose-no-cache +test-verbose-no-cache: export COCONUT_USE_COLOR=TRUE +test-verbose-no-cache: clean + python ./coconut/tests --strict --keep-lines --force --verbose --no-cache + python ./coconut/tests/dest/runner.py + python ./coconut/tests/dest/extras.py + +# same as test-verbose but is fully synchronous .PHONY: test-verbose-sync test-verbose-sync: export COCONUT_USE_COLOR=TRUE test-verbose-sync: clean @@ -188,22 +196,6 @@ test-mypy-all: clean python ./coconut/tests/dest/runner.py python ./coconut/tests/dest/extras.py -# same as test-univ-tests, but forces recompilation for testing --incremental -.PHONY: test-incremental -test-incremental: export COCONUT_USE_COLOR=TRUE -test-incremental: clean-no-tests - python ./coconut/tests --strict --keep-lines --incremental --force - python ./coconut/tests/dest/runner.py - python ./coconut/tests/dest/extras.py - -# same as test-incremental, but uses --verbose -.PHONY: test-incremental-verbose -test-incremental-verbose: export COCONUT_USE_COLOR=TRUE -test-incremental-verbose: clean-no-tests - python ./coconut/tests --strict --keep-lines --incremental --force --verbose - python ./coconut/tests/dest/runner.py - python ./coconut/tests/dest/extras.py - # same as test-univ but also tests easter eggs .PHONY: test-easter-eggs test-easter-eggs: export COCONUT_USE_COLOR=TRUE diff --git a/coconut/command/cli.py b/coconut/command/cli.py index 38f51a8b7..a0e375d55 100644 --- a/coconut/command/cli.py +++ b/coconut/command/cli.py @@ -183,12 +183,6 @@ help="run Coconut passed in as a string (can also be piped into stdin)", ) -arguments.add_argument( - "--incremental", - action="store_true", - help="enable incremental compilation mode (caches previous parses to improve recompilation performance for slightly modified files)", -) - arguments.add_argument( "-j", "--jobs", metavar="processes", @@ -269,6 +263,12 @@ help="run the compiler in a separate thread with the given stack size in kilobytes", ) +arguments.add_argument( + "--no-cache", + action="store_true", + help="disables use of Coconut's incremental parsing cache (caches previous parses to improve recompilation performance for slightly modified files)", +) + arguments.add_argument( "--site-install", "--siteinstall", action="store_true", diff --git a/coconut/command/command.py b/coconut/command/command.py index 3fd5b1d70..f2ca9b635 100644 --- a/coconut/command/command.py +++ b/coconut/command/command.py @@ -130,7 +130,7 @@ class Command(object): mypy_args = None # corresponds to --mypy flag argv_args = None # corresponds to --argv flag stack_size = 0 # corresponds to --stack-size flag - incremental = False # corresponds to --incremental flag + use_cache = True # corresponds to --no-cache flag prompt = Prompt() @@ -262,8 +262,6 @@ def execute_args(self, args, interact=True, original_args=None): raise CoconutException("cannot compile with both --line-numbers and --no-line-numbers") if args.site_install and args.site_uninstall: raise CoconutException("cannot --site-install and --site-uninstall simultaneously") - if args.incremental and not SUPPORTS_INCREMENTAL: - raise CoconutException("--incremental mode not supported in current environment (try '{python} -m pip install --upgrade cPyparsing' to fix)".format(python=sys.executable)) for and_args in getattr(args, "and") or []: if len(and_args) > 2: raise CoconutException( @@ -283,7 +281,13 @@ def execute_args(self, args, interact=True, original_args=None): self.prompt.set_style(args.style) if args.argv is not None: self.argv_args = list(args.argv) - self.incremental = args.incremental + if args.no_cache: + self.use_cache = False + elif SUPPORTS_INCREMENTAL: + self.use_cache = True + else: + logger.log("incremental parsing mode not supported in current environment (try '{python} -m pip install --upgrade cPyparsing' to fix)".format(python=sys.executable)) + self.use_cache = False # execute non-compilation tasks if args.docs: @@ -609,7 +613,7 @@ def callback(compiled): parse_kwargs = dict( filename=os.path.basename(codepath), ) - if self.incremental: + if self.use_cache: code_dir, code_fname = os.path.split(codepath) cache_dir = os.path.join(code_dir, coconut_cache_dir) diff --git a/coconut/compiler/util.py b/coconut/compiler/util.py index 8d3bcefac..b681c082a 100644 --- a/coconut/compiler/util.py +++ b/coconut/compiler/util.py @@ -45,7 +45,6 @@ import cPickle as pickle from coconut._pyparsing import ( - CPYPARSING, MODERN_PYPARSING, USE_COMPUTATION_GRAPH, SUPPORTS_INCREMENTAL, @@ -421,7 +420,7 @@ def unpack(tokens): def in_incremental_mode(): - """Determine if we are using the --incremental parsing mode.""" + """Determine if we are using incremental parsing mode.""" return ParserElement._incrementalEnabled and not ParserElement._incrementalWithResets @@ -746,7 +745,10 @@ def unpickle_cache(filename): if pickle_info_obj["VERSION"] != VERSION or pickle_info_obj["pyparsing_version"] != pyparsing_version: return False - pickleable_cache_items = pickle_info_obj["pickleable_cache_items"] + if ParserElement._incrementalEnabled: + pickleable_cache_items = pickle_info_obj["pickleable_cache_items"] + else: + pickleable_cache_items = [] all_adaptive_stats = pickle_info_obj["all_adaptive_stats"] logger.log("Loaded {num_inc} incremental and {num_adapt} adaptive cache items from {filename!r}.".format( @@ -774,8 +776,8 @@ def unpickle_cache(filename): def load_cache_for(inputstring, filename, cache_filename): """Load cache_filename (for the given inputstring and filename).""" - if not CPYPARSING: - raise CoconutException("--incremental requires cPyparsing (run '{python} -m pip install --upgrade cPyparsing' to fix)".format(python=sys.executable)) + if not SUPPORTS_INCREMENTAL: + raise CoconutException("incremental parsing mode requires cPyparsing (run '{python} -m pip install --upgrade cPyparsing' to fix)".format(python=sys.executable)) if len(inputstring) < disable_incremental_for_len: incremental_enabled = enable_incremental_parsing() if incremental_enabled: diff --git a/coconut/constants.py b/coconut/constants.py index 09a7a747d..d34b931ba 100644 --- a/coconut/constants.py +++ b/coconut/constants.py @@ -122,7 +122,13 @@ def get_path_env_var(env_var, default): # below constants are experimentally determined to maximize performance streamline_grammar_for_len = 4096 -disable_incremental_for_len = streamline_grammar_for_len # disables --incremental + +# Current problems with this: +# - only actually helpful for tiny files (< streamline_grammar_for_len) +# - sets incremental mode for the whole process, which can really slow down some compilations +# - makes exceptions include the entire file +# disable_incremental_for_len = streamline_grammar_for_len +disable_incremental_for_len = 0 use_packrat_parser = True # True also gives us better error messages packrat_cache_size = None # only works because final() clears the cache diff --git a/coconut/root.py b/coconut/root.py index 323d8124b..b8030fc51 100644 --- a/coconut/root.py +++ b/coconut/root.py @@ -26,7 +26,7 @@ VERSION = "3.0.3" VERSION_NAME = None # False for release, int >= 1 for develop -DEVELOP = 24 +DEVELOP = 25 ALPHA = False # for pre releases rather than post releases assert DEVELOP is False or DEVELOP >= 1, "DEVELOP must be False or an int >= 1" diff --git a/coconut/tests/main_test.py b/coconut/tests/main_test.py index 5f6ec7b30..20916c79e 100644 --- a/coconut/tests/main_test.py +++ b/coconut/tests/main_test.py @@ -977,11 +977,10 @@ def test_run_arg(self): def test_jobs_zero(self): run(["--jobs", "0"]) - if not PYPY and PY38: + if not PYPY: def test_incremental(self): - run(["--incremental"]) - # includes "Error" because exceptions include the whole file - run(["--incremental", "--force"], check_errors=False) + run() + run(["--force"]) if get_bool_env_var("COCONUT_TEST_VERBOSE"): def test_verbose(self): From 5753c2b924a441f43e98589fc2a4af1a6a228684 Mon Sep 17 00:00:00 2001 From: Evan Hubinger Date: Sun, 12 Nov 2023 19:40:27 -0800 Subject: [PATCH 081/121] More performance tuning --- Makefile | 2 +- coconut/_pyparsing.py | 6 ++++-- coconut/command/command.py | 21 ++++----------------- coconut/command/util.py | 24 +++++++++++++++++++++--- coconut/compiler/compiler.py | 26 +++++++++++++++++--------- coconut/compiler/util.py | 36 +++++++++++++++++++++++++++--------- coconut/constants.py | 13 ++++++++----- coconut/root.py | 2 +- 8 files changed, 83 insertions(+), 47 deletions(-) diff --git a/Makefile b/Makefile index 229aae742..beceb6df8 100644 --- a/Makefile +++ b/Makefile @@ -156,7 +156,7 @@ test-mypy-tests: clean-no-tests python ./coconut/tests/dest/extras.py # same as test-univ but includes verbose output for better debugging -# regex for getting non-timing lines: ^(?!\s*(Time|Packrat|Loaded|Saving|Adaptive|Errorless|Grammar))[^\n]*\n* +# regex for getting non-timing lines: ^(?!\s*(Time|Packrat|Loaded|Saving|Adaptive|Errorless|Grammar|Failed)\s)[^\n]*\n* .PHONY: test-verbose test-verbose: export COCONUT_USE_COLOR=TRUE test-verbose: clean diff --git a/coconut/_pyparsing.py b/coconut/_pyparsing.py index 9d7ed9b2b..2c288ece3 100644 --- a/coconut/_pyparsing.py +++ b/coconut/_pyparsing.py @@ -48,7 +48,7 @@ never_clear_incremental_cache, warn_on_multiline_regex, num_displayed_timing_items, - use_adaptive_if_available, + use_cache_file, ) from coconut.util import get_clock_time # NOQA from coconut.util import ( @@ -152,6 +152,7 @@ def _parseCache(self, instring, loc, doActions=True, callPreParse=True): if isinstance(value, Exception): raise value return value[0], value[1].copy() + ParserElement.packrat_context = [] ParserElement._parseCache = _parseCache @@ -207,7 +208,8 @@ def enableIncremental(*args, **kwargs): + " (run '{python} -m pip install --upgrade cPyparsing' to fix)".format(python=sys.executable) ) -USE_ADAPTIVE = hasattr(MatchFirst, "setAdaptiveMode") and use_adaptive_if_available +SUPPORTS_ADAPTIVE = hasattr(MatchFirst, "setAdaptiveMode") +USE_CACHE = SUPPORTS_INCREMENTAL and use_cache_file maybe_make_safe = getattr(_pyparsing, "maybe_make_safe", None) diff --git a/coconut/command/command.py b/coconut/command/command.py index f2ca9b635..84abc4cf4 100644 --- a/coconut/command/command.py +++ b/coconut/command/command.py @@ -28,10 +28,10 @@ from subprocess import CalledProcessError from coconut._pyparsing import ( + USE_CACHE, unset_fast_pyparsing_reprs, start_profiling, print_profiling_results, - SUPPORTS_INCREMENTAL, ) from coconut.compiler import Compiler @@ -130,7 +130,7 @@ class Command(object): mypy_args = None # corresponds to --mypy flag argv_args = None # corresponds to --argv flag stack_size = 0 # corresponds to --stack-size flag - use_cache = True # corresponds to --no-cache flag + use_cache = USE_CACHE # corresponds to --no-cache flag prompt = Prompt() @@ -283,11 +283,6 @@ def execute_args(self, args, interact=True, original_args=None): self.argv_args = list(args.argv) if args.no_cache: self.use_cache = False - elif SUPPORTS_INCREMENTAL: - self.use_cache = True - else: - logger.log("incremental parsing mode not supported in current environment (try '{python} -m pip install --upgrade cPyparsing' to fix)".format(python=sys.executable)) - self.use_cache = False # execute non-compilation tasks if args.docs: @@ -611,17 +606,9 @@ def callback(compiled): self.execute_file(destpath, argv_source_path=codepath) parse_kwargs = dict( - filename=os.path.basename(codepath), + codepath=codepath, + use_cache=self.use_cache, ) - if self.use_cache: - code_dir, code_fname = os.path.split(codepath) - - cache_dir = os.path.join(code_dir, coconut_cache_dir) - ensure_dir(cache_dir) - - pickle_fname = code_fname + ".pickle" - parse_kwargs["cache_filename"] = os.path.join(cache_dir, pickle_fname) - if package is True: self.submit_comp_job(codepath, callback, "parse_package", code, package_level=package_level, **parse_kwargs) elif package is False: diff --git a/coconut/command/util.py b/coconut/command/util.py index 2f57f7fd3..1758b399b 100644 --- a/coconut/command/util.py +++ b/coconut/command/util.py @@ -81,6 +81,7 @@ kilobyte, min_stack_size_kbs, coconut_base_run_args, + high_proc_prio, ) if PY26: @@ -130,6 +131,11 @@ ), ) prompt_toolkit = None +try: + import psutil +except ImportError: + psutil = None + # ----------------------------------------------------------------------------------------------------------------------- # UTILITIES: @@ -222,9 +228,7 @@ def handling_broken_process_pool(): def kill_children(): """Terminate all child processes.""" - try: - import psutil - except ImportError: + if psutil is None: logger.warn( "missing psutil; --jobs may not properly terminate", extra="run '{python} -m pip install psutil' to fix".format(python=sys.executable), @@ -709,6 +713,19 @@ def was_run_code(self, get_all=True): return self.stored[-1] +def highten_process(): + """Set the current process to high priority.""" + if high_proc_prio and psutil is not None: + try: + p = psutil.Process() + if WINDOWS: + p.nice(psutil.HIGH_PRIORITY_CLASS) + else: + p.nice(-10) + except Exception: + logger.log_exc() + + class multiprocess_wrapper(pickleable_obj): """Wrapper for a method that needs to be multiprocessed.""" __slots__ = ("base", "method", "stack_size", "rec_limit", "logger", "argv") @@ -728,6 +745,7 @@ def __reduce__(self): def __call__(self, *args, **kwargs): """Call the method.""" + highten_process() sys.setrecursionlimit(self.rec_limit) logger.copy_from(self.logger) sys.argv = self.argv diff --git a/coconut/compiler/compiler.py b/coconut/compiler/compiler.py index 53487eb7e..a970e993e 100644 --- a/coconut/compiler/compiler.py +++ b/coconut/compiler/compiler.py @@ -30,6 +30,7 @@ from coconut.root import * # NOQA import sys +import os import re from contextlib import contextmanager from functools import partial, wraps @@ -38,6 +39,7 @@ from coconut._pyparsing import ( USE_COMPUTATION_GRAPH, + USE_CACHE, ParseBaseException, ParseResults, col as getcol, @@ -174,6 +176,7 @@ pickle_cache, handle_and_manage, sub_all, + get_cache_path, ) from coconut.compiler.header import ( minify_header, @@ -1257,7 +1260,7 @@ def inner_parse_eval( if outer_ln is None: outer_ln = self.adjust(lineno(loc, original)) with self.inner_environment(ln=outer_ln): - self.streamline(parser, inputstring) + self.streamline(parser, inputstring, inner=True) pre_procd = self.pre(inputstring, **preargs) parsed = parse(parser, pre_procd) return self.post(parsed, **postargs) @@ -1270,7 +1273,7 @@ def parsing(self, keep_state=False, filename=None): self.current_compiler[0] = self yield - def streamline(self, grammar, inputstring="", force=False): + def streamline(self, grammar, inputstring="", force=False, inner=False): """Streamline the given grammar for the given inputstring.""" if force or (streamline_grammar_for_len is not None and len(inputstring) > streamline_grammar_for_len): start_time = get_clock_time() @@ -1282,7 +1285,7 @@ def streamline(self, grammar, inputstring="", force=False): length=len(inputstring), ), ) - else: + elif not inner: logger.log("No streamlining done for input of length {length}.".format(length=len(inputstring))) def run_final_checks(self, original, keep_state=False): @@ -1309,20 +1312,25 @@ def parse( postargs, streamline=True, keep_state=False, - filename=None, - cache_filename=None, + codepath=None, + use_cache=None, ): """Use the parser to parse the inputstring with appropriate setup and teardown.""" + if use_cache is None: + use_cache = codepath is not None and USE_CACHE + if use_cache: + cache_path = get_cache_path(codepath) + filename = os.path.basename(codepath) if codepath is not None else None with self.parsing(keep_state, filename): if streamline: self.streamline(parser, inputstring) # unpickling must happen after streamlining and must occur in the # compiler so that it happens in the same process as compilation - if cache_filename is not None: + if use_cache: incremental_enabled = load_cache_for( inputstring=inputstring, filename=filename, - cache_filename=cache_filename, + cache_path=cache_path, ) pre_procd = parsed = None try: @@ -1343,8 +1351,8 @@ def parse( + str(sys.getrecursionlimit()) + " (you may also need to increase --stack-size)", ) finally: - if cache_filename is not None and pre_procd is not None: - pickle_cache(pre_procd, cache_filename, include_incremental=incremental_enabled) + if use_cache and pre_procd is not None: + pickle_cache(pre_procd, cache_path, include_incremental=incremental_enabled) self.run_final_checks(pre_procd, keep_state) return out diff --git a/coconut/compiler/util.py b/coconut/compiler/util.py index b681c082a..989c0df6a 100644 --- a/coconut/compiler/util.py +++ b/coconut/compiler/util.py @@ -28,6 +28,7 @@ from coconut.root import * # NOQA import sys +import os import re import ast import inspect @@ -48,7 +49,7 @@ MODERN_PYPARSING, USE_COMPUTATION_GRAPH, SUPPORTS_INCREMENTAL, - USE_ADAPTIVE, + SUPPORTS_ADAPTIVE, replaceWith, ZeroOrMore, OneOrMore, @@ -82,6 +83,7 @@ get_target_info, memoize, univ_open, + ensure_dir, ) from coconut.terminal import ( logger, @@ -120,6 +122,8 @@ adaptive_reparse_usage_weight, use_adaptive_any_of, disable_incremental_for_len, + coconut_cache_dir, + use_adaptive_if_available, ) from coconut.exceptions import ( CoconutException, @@ -398,7 +402,7 @@ def adaptive_manager(item, original, loc, reparse=False): def final(item): """Collapse the computation graph upon parsing the given item.""" - if USE_ADAPTIVE: + if SUPPORTS_ADAPTIVE and use_adaptive_if_available: item = Wrap(item, adaptive_manager, greedy=True) # evaluate_tokens expects a computation graph, so we just call add_action directly return add_action(trace(item), final_evaluate_tokens) @@ -774,8 +778,8 @@ def unpickle_cache(filename): return True -def load_cache_for(inputstring, filename, cache_filename): - """Load cache_filename (for the given inputstring and filename).""" +def load_cache_for(inputstring, filename, cache_path): + """Load cache_path (for the given inputstring and filename).""" if not SUPPORTS_INCREMENTAL: raise CoconutException("incremental parsing mode requires cPyparsing (run '{python} -m pip install --upgrade cPyparsing' to fix)".format(python=sys.executable)) if len(inputstring) < disable_incremental_for_len: @@ -793,16 +797,27 @@ def load_cache_for(inputstring, filename, cache_filename): input_len=len(inputstring), max_len=disable_incremental_for_len, ) - did_load_cache = unpickle_cache(cache_filename) - logger.log("{Loaded} cache for {filename!r} from {cache_filename!r} ({incremental_info}).".format( + did_load_cache = unpickle_cache(cache_path) + logger.log("{Loaded} cache for {filename!r} from {cache_path!r} ({incremental_info}).".format( Loaded="Loaded" if did_load_cache else "Failed to load", filename=filename, - cache_filename=cache_filename, + cache_path=cache_path, incremental_info=incremental_info, )) return incremental_enabled +def get_cache_path(codepath): + """Get the cache filename to use for the given codepath.""" + code_dir, code_fname = os.path.split(codepath) + + cache_dir = os.path.join(code_dir, coconut_cache_dir) + ensure_dir(cache_dir) + + pickle_fname = code_fname + ".pkl" + return os.path.join(cache_dir, pickle_fname) + + # ----------------------------------------------------------------------------------------------------------------------- # TARGETS: # ----------------------------------------------------------------------------------------------------------------------- @@ -886,7 +901,6 @@ def get_target_info_smart(target, mode="lowest"): class MatchAny(MatchFirst): """Version of MatchFirst that always uses adaptive parsing.""" - adaptive_mode = True all_match_anys = [] def __init__(self, *args, **kwargs): @@ -903,9 +917,13 @@ def __or__(self, other): return self +if SUPPORTS_ADAPTIVE: + MatchAny.setAdaptiveMode(True) + + def any_of(*exprs, **kwargs): """Build a MatchAny of the given MatchFirst.""" - use_adaptive = kwargs.pop("use_adaptive", use_adaptive_any_of) + use_adaptive = kwargs.pop("use_adaptive", use_adaptive_any_of) and SUPPORTS_ADAPTIVE internal_assert(not kwargs, "excess keyword arguments passed to any_of", kwargs) AnyOf = MatchAny if use_adaptive else MatchFirst diff --git a/coconut/constants.py b/coconut/constants.py index d34b931ba..32ea113be 100644 --- a/coconut/constants.py +++ b/coconut/constants.py @@ -121,6 +121,9 @@ def get_path_env_var(env_var, default): # below constants are experimentally determined to maximize performance +use_packrat_parser = True # True also gives us better error messages +packrat_cache_size = None # only works because final() clears the cache + streamline_grammar_for_len = 4096 # Current problems with this: @@ -130,14 +133,12 @@ def get_path_env_var(env_var, default): # disable_incremental_for_len = streamline_grammar_for_len disable_incremental_for_len = 0 -use_packrat_parser = True # True also gives us better error messages -packrat_cache_size = None # only works because final() clears the cache +use_cache_file = True +use_adaptive_any_of = True # note that _parseIncremental produces much smaller caches use_incremental_if_available = False -use_adaptive_any_of = True - use_adaptive_if_available = False # currently broken adaptive_reparse_usage_weight = 10 @@ -716,6 +717,8 @@ def get_path_env_var(env_var, default): base_default_jobs = "sys" if not PY26 else 0 +high_proc_prio = True + mypy_install_arg = "install" jupyter_install_arg = "install" @@ -993,7 +996,7 @@ def get_path_env_var(env_var, default): # min versions are inclusive unpinned_min_versions = { - "cPyparsing": (2, 4, 7, 2, 2, 4), + "cPyparsing": (2, 4, 7, 2, 2, 5), ("pre-commit", "py3"): (3,), ("psutil", "py>=27"): (5,), "jupyter": (1, 0), diff --git a/coconut/root.py b/coconut/root.py index b8030fc51..c509a4b28 100644 --- a/coconut/root.py +++ b/coconut/root.py @@ -26,7 +26,7 @@ VERSION = "3.0.3" VERSION_NAME = None # False for release, int >= 1 for develop -DEVELOP = 25 +DEVELOP = 26 ALPHA = False # for pre releases rather than post releases assert DEVELOP is False or DEVELOP >= 1, "DEVELOP must be False or an int >= 1" From 8679c8c91abdb50850a02ffefe7c1ea10794d713 Mon Sep 17 00:00:00 2001 From: Evan Hubinger Date: Sun, 12 Nov 2023 22:45:34 -0800 Subject: [PATCH 082/121] Robustify os operations --- Makefile | 4 ++-- coconut/command/command.py | 2 +- coconut/compiler/util.py | 2 +- coconut/root.py | 2 +- coconut/tests/src/cocotest/agnostic/util.coco | 2 +- coconut/util.py | 10 ++++++++-- 6 files changed, 14 insertions(+), 8 deletions(-) diff --git a/Makefile b/Makefile index beceb6df8..9d1f15177 100644 --- a/Makefile +++ b/Makefile @@ -336,12 +336,12 @@ open-speedscope: .PHONY: pyspy-purepy pyspy-purepy: export COCONUT_PURE_PYTHON=TRUE pyspy-purepy: - py-spy record -o profile.speedscope --format speedscope --subprocesses -- python -m coconut ./coconut/tests/src/cocotest/agnostic ./coconut/tests/dest/cocotest --force + py-spy record -o profile.speedscope --format speedscope --subprocesses --rate 75 -- python -m coconut ./coconut/tests/src/cocotest/agnostic ./coconut/tests/dest/cocotest --force make open-speedscope .PHONY: pyspy-native pyspy-native: - py-spy record -o profile.speedscope --format speedscope --native -- python -m coconut ./coconut/tests/src/cocotest/agnostic ./coconut/tests/dest/cocotest --force --jobs 0 + py-spy record -o profile.speedscope --format speedscope --native --subprocesses --rate 15 -- python -m coconut ./coconut/tests/src/cocotest/agnostic ./coconut/tests/dest/cocotest --force make open-speedscope .PHONY: pyspy-runtime diff --git a/coconut/command/command.py b/coconut/command/command.py index 84abc4cf4..8c3cbc08e 100644 --- a/coconut/command/command.py +++ b/coconut/command/command.py @@ -572,7 +572,7 @@ def compile(self, codepath, destpath=None, package=False, run=False, force=False if destpath is not None: destpath = fixpath(destpath) destdir = os.path.dirname(destpath) - ensure_dir(destdir) + ensure_dir(destdir, logger=logger) if package is True: package_level = self.get_package_level(codepath) if package_level == 0: diff --git a/coconut/compiler/util.py b/coconut/compiler/util.py index 989c0df6a..f2ae57545 100644 --- a/coconut/compiler/util.py +++ b/coconut/compiler/util.py @@ -812,7 +812,7 @@ def get_cache_path(codepath): code_dir, code_fname = os.path.split(codepath) cache_dir = os.path.join(code_dir, coconut_cache_dir) - ensure_dir(cache_dir) + ensure_dir(cache_dir, logger=logger) pickle_fname = code_fname + ".pkl" return os.path.join(cache_dir, pickle_fname) diff --git a/coconut/root.py b/coconut/root.py index c509a4b28..d47b49310 100644 --- a/coconut/root.py +++ b/coconut/root.py @@ -26,7 +26,7 @@ VERSION = "3.0.3" VERSION_NAME = None # False for release, int >= 1 for develop -DEVELOP = 26 +DEVELOP = 27 ALPHA = False # for pre releases rather than post releases assert DEVELOP is False or DEVELOP >= 1, "DEVELOP must be False or an int >= 1" diff --git a/coconut/tests/src/cocotest/agnostic/util.coco b/coconut/tests/src/cocotest/agnostic/util.coco index 517168152..c16ff70f1 100644 --- a/coconut/tests/src/cocotest/agnostic/util.coco +++ b/coconut/tests/src/cocotest/agnostic/util.coco @@ -134,7 +134,7 @@ product = reduce$((*), ?, 1) product_ = reduce$(*) def zipwith(f, *args) = map((items) -> f(*items), zip(*args)) def zipwith_(f, *args) = starmap$(f)..zip <*| args -zipsum = zip ..> map$(sum) +zipsum = zip ..> map$(sum) # type: ignore ident_ = (x) -> x @ ident .. ident # type: ignore def plus1_(x: int) -> int = x + 1 diff --git a/coconut/util.py b/coconut/util.py index 4b4338a15..5b2c60b05 100644 --- a/coconut/util.py +++ b/coconut/util.py @@ -281,10 +281,16 @@ def assert_remove_prefix(inputstr, prefix, allow_no_prefix=False): remove_prefix = partial(assert_remove_prefix, allow_no_prefix=True) -def ensure_dir(dirpath): +def ensure_dir(dirpath, logger=None): """Ensure that a directory exists.""" if not os.path.exists(dirpath): - os.makedirs(dirpath) + try: + os.makedirs(dirpath) + except OSError: + if logger is not None: + logger.log_exc() + return False + return True def without_keys(inputdict, rem_keys): From 807eb9ff4f88f5e21c3edcef6855a23f14f58582 Mon Sep 17 00:00:00 2001 From: Evan Hubinger Date: Mon, 13 Nov 2023 02:14:48 -0800 Subject: [PATCH 083/121] Improve any_of --- .pre-commit-config.yaml | 2 +- Makefile | 13 +-- coconut/_pyparsing.py | 1 + coconut/compiler/grammar.py | 162 ++++++++++++++++++++---------------- coconut/compiler/util.py | 12 ++- coconut/root.py | 2 +- 6 files changed, 108 insertions(+), 84 deletions(-) diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index 5764b616b..2df5155a4 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -8,7 +8,7 @@ repos: - --aggressive - --aggressive - --experimental - - --ignore=W503,E501,E722,E402 + - --ignore=W503,E501,E722,E402,E721 - repo: https://github.com/pre-commit/pre-commit-hooks.git rev: v4.5.0 hooks: diff --git a/Makefile b/Makefile index 9d1f15177..2b7bdcadd 100644 --- a/Makefile +++ b/Makefile @@ -333,20 +333,23 @@ open-speedscope: npm install -g speedscope speedscope ./profile.speedscope -.PHONY: pyspy-purepy -pyspy-purepy: export COCONUT_PURE_PYTHON=TRUE -pyspy-purepy: +.PHONY: pyspy +pyspy: py-spy record -o profile.speedscope --format speedscope --subprocesses --rate 75 -- python -m coconut ./coconut/tests/src/cocotest/agnostic ./coconut/tests/dest/cocotest --force make open-speedscope +.PHONY: pyspy-purepy +pyspy-purepy: export COCONUT_PURE_PYTHON=TRUE +pyspy-purepy: pyspy + .PHONY: pyspy-native pyspy-native: - py-spy record -o profile.speedscope --format speedscope --native --subprocesses --rate 15 -- python -m coconut ./coconut/tests/src/cocotest/agnostic ./coconut/tests/dest/cocotest --force + py-spy record -o profile.speedscope --format speedscope --native --rate 75 -- python -m coconut ./coconut/tests/src/cocotest/agnostic ./coconut/tests/dest/cocotest --force --jobs 0 make open-speedscope .PHONY: pyspy-runtime pyspy-runtime: - py-spy record -o runtime_profile.speedscope --format speedscope --subprocesses -- python ./coconut/tests/dest/runner.py + py-spy record -o runtime_profile.speedscope --format speedscope -- python ./coconut/tests/dest/runner.py speedscope ./runtime_profile.speedscope .PHONY: vprof-time diff --git a/coconut/_pyparsing.py b/coconut/_pyparsing.py index 2c288ece3..b3320a5c4 100644 --- a/coconut/_pyparsing.py +++ b/coconut/_pyparsing.py @@ -266,6 +266,7 @@ def enableIncremental(*args, **kwargs): python_quoted_string = getattr(_pyparsing, "python_quoted_string", None) if python_quoted_string is None: python_quoted_string = _pyparsing.Combine( + # multiline strings must come first (_pyparsing.Regex(r'"""(?:[^"\\]|""(?!")|"(?!"")|\\.)*', flags=re.MULTILINE) + '"""').setName("multiline double quoted string") | (_pyparsing.Regex(r"'''(?:[^'\\]|''(?!')|'(?!'')|\\.)*", flags=re.MULTILINE) + "'''").setName("multiline single quoted string") | (_pyparsing.Regex(r'"(?:[^"\n\r\\]|(?:\\")|(?:\\(?:[^x]|x[0-9a-fA-F]+)))*') + '"').setName("double quoted string") diff --git a/coconut/compiler/grammar.py b/coconut/compiler/grammar.py index 4f79c2c04..a40fb25e8 100644 --- a/coconut/compiler/grammar.py +++ b/coconut/compiler/grammar.py @@ -807,11 +807,12 @@ class Grammar(object): bin_num = combine(caseless_literal("0b") + Optional(underscore.suppress()) + binint) oct_num = combine(caseless_literal("0o") + Optional(underscore.suppress()) + octint) hex_num = combine(caseless_literal("0x") + Optional(underscore.suppress()) + hexint) - number = ( - maybe_imag_num - | hex_num - | bin_num - | oct_num + number = any_of( + maybe_imag_num, + hex_num, + bin_num, + oct_num, + use_adaptive=False, ) # make sure that this gets addspaced not condensed so it doesn't produce a SyntaxError num_atom = addspace(number + Optional(condense(dot + unsafe_name))) @@ -877,9 +878,21 @@ class Grammar(object): combine(back_none_dubstar_pipe + equals), ) augassign = any_of( - pipe_augassign, + combine(plus + equals), + combine(sub_minus + equals), + combine(bar + equals), + combine(amp + equals), + combine(mul_star + equals), + combine(div_slash + equals), + combine(div_dubslash + equals), + combine(percent + equals), + combine(lshift + equals), + combine(rshift + equals), + combine(matrix_at + equals), + combine(exp_dubstar + equals), + combine(caret + equals), + combine(dubquestion + equals), combine(comp_pipe + equals), - combine(dotdot + equals), combine(comp_back_pipe + equals), combine(comp_star_pipe + equals), combine(comp_back_star_pipe + equals), @@ -892,31 +905,19 @@ class Grammar(object): combine(comp_none_dubstar_pipe + equals), combine(comp_back_none_dubstar_pipe + equals), combine(unsafe_dubcolon + equals), - combine(div_dubslash + equals), - combine(div_slash + equals), - combine(exp_dubstar + equals), - combine(mul_star + equals), - combine(plus + equals), - combine(sub_minus + equals), - combine(percent + equals), - combine(amp + equals), - combine(bar + equals), - combine(caret + equals), - combine(lshift + equals), - combine(rshift + equals), - combine(matrix_at + equals), - combine(dubquestion + equals), + combine(dotdot + equals), + pipe_augassign, ) comp_op = any_of( eq, ne, keyword("in"), - addspace(keyword("not") + keyword("in")), lt, gt, le, ge, + addspace(keyword("not") + keyword("in")), keyword("is") + ~keyword("not"), addspace(keyword("is") + keyword("not")), ) @@ -981,7 +982,7 @@ class Grammar(object): test_expr = testlist_star_expr | yield_expr base_op_item = ( - # must go dubstar then star then no star + # pipes must come first, and must go dubstar then star then no star fixto(dubstar_pipe, "_coconut_dubstar_pipe") | fixto(back_dubstar_pipe, "_coconut_back_dubstar_pipe") | fixto(none_dubstar_pipe, "_coconut_none_dubstar_pipe") @@ -995,7 +996,7 @@ class Grammar(object): | fixto(none_pipe, "_coconut_none_pipe") | fixto(back_none_pipe, "_coconut_back_none_pipe") - # must go dubstar then star then no star + # comp pipes must come early, and must go dubstar then star then no star | fixto(comp_dubstar_pipe, "_coconut_forward_dubstar_compose") | fixto(comp_back_dubstar_pipe, "_coconut_back_dubstar_compose") | fixto(comp_none_dubstar_pipe, "_coconut_forward_none_dubstar_compose") @@ -1005,32 +1006,16 @@ class Grammar(object): | fixto(comp_none_star_pipe, "_coconut_forward_none_star_compose") | fixto(comp_back_none_star_pipe, "_coconut_back_none_star_compose") | fixto(comp_pipe, "_coconut_forward_compose") - | fixto(dotdot | comp_back_pipe, "_coconut_back_compose") | fixto(comp_none_pipe, "_coconut_forward_none_compose") | fixto(comp_back_none_pipe, "_coconut_back_none_compose") + # dotdot must come last + | fixto(dotdot | comp_back_pipe, "_coconut_back_compose") - # neg_minus must come after minus - | fixto(minus, "_coconut_minus") - | fixto(neg_minus, "_coconut.operator.neg") - - | fixto(keyword("assert"), "_coconut_assert") - | fixto(keyword("raise"), "_coconut_raise") - | fixto(keyword("and"), "_coconut_bool_and") - | fixto(keyword("or"), "_coconut_bool_or") - | fixto(comma, "_coconut_comma_op") - | fixto(dubquestion, "_coconut_none_coalesce") - | fixto(dot, "_coconut.getattr") - | fixto(unsafe_dubcolon, "_coconut.itertools.chain") - | fixto(dollar, "_coconut_partial") - | fixto(exp_dubstar, "_coconut.operator.pow") + | fixto(plus, "_coconut.operator.add") | fixto(mul_star, "_coconut.operator.mul") - | fixto(div_dubslash, "_coconut.operator.floordiv") | fixto(div_slash, "_coconut.operator.truediv") - | fixto(percent, "_coconut.operator.mod") - | fixto(plus, "_coconut.operator.add") - | fixto(amp, "_coconut.operator.and_") - | fixto(caret, "_coconut.operator.xor") | fixto(unsafe_bar, "_coconut.operator.or_") + | fixto(amp, "_coconut.operator.and_") | fixto(lshift, "_coconut.operator.lshift") | fixto(rshift, "_coconut.operator.rshift") | fixto(lt, "_coconut.operator.lt") @@ -1039,11 +1024,28 @@ class Grammar(object): | fixto(le, "_coconut.operator.le") | fixto(ge, "_coconut.operator.ge") | fixto(ne, "_coconut.operator.ne") - | fixto(tilde, "_coconut.operator.inv") | fixto(matrix_at, "_coconut_matmul") + | fixto(div_dubslash, "_coconut.operator.floordiv") + | fixto(caret, "_coconut.operator.xor") + | fixto(percent, "_coconut.operator.mod") + | fixto(exp_dubstar, "_coconut.operator.pow") + | fixto(tilde, "_coconut.operator.inv") + | fixto(dot, "_coconut.getattr") + | fixto(comma, "_coconut_comma_op") + | fixto(keyword("and"), "_coconut_bool_and") + | fixto(keyword("or"), "_coconut_bool_or") + | fixto(dubquestion, "_coconut_none_coalesce") + | fixto(unsafe_dubcolon, "_coconut.itertools.chain") + | fixto(dollar, "_coconut_partial") + | fixto(keyword("assert"), "_coconut_assert") + | fixto(keyword("raise"), "_coconut_raise") | fixto(keyword("is") + keyword("not"), "_coconut.operator.is_not") | fixto(keyword("not") + keyword("in"), "_coconut_not_in") + # neg_minus must come after minus + | fixto(minus, "_coconut_minus") + | fixto(neg_minus, "_coconut.operator.neg") + # must come after is not / not in | fixto(keyword("not"), "_coconut.operator.not_") | fixto(keyword("is"), "_coconut.operator.is_") @@ -1056,6 +1058,7 @@ class Grammar(object): ) partial_op_item = attach(partial_op_item_tokens, partial_op_item_handle) op_item = ( + # partial_op_item must come first, then typedef_op_item must come after base_op_item partial_op_item | typedef_op_item | base_op_item @@ -1131,6 +1134,7 @@ class Grammar(object): call_item = ( unsafe_name + default + # ellipsis must come before namedexpr_test | ellipsis_tokens + equals.suppress() + refname | namedexpr_test | star + test @@ -1193,9 +1197,9 @@ class Grammar(object): | invalid_syntax(star_expr + comp_for, "iterable unpacking cannot be used in comprehension") ) paren_atom = condense( - lparen + any_of( + lparen + rparen + | lparen + any_of( # everything here must end with rparen - rparen, testlist_star_namedexpr + rparen, comprehension_expr + rparen, op_item + rparen, @@ -1222,6 +1226,7 @@ class Grammar(object): list_item = ( lbrack.suppress() + list_expr + rbrack.suppress() | condense(lbrack + Optional(comprehension_expr) + rbrack) + # array_literal must come last | array_literal ) @@ -1277,9 +1282,10 @@ class Grammar(object): typedef_trailer = Forward() typedef_or_expr = Forward() - simple_trailer = ( - condense(dot + unsafe_name) - | condense(lbrack + subscriptlist + rbrack) + simple_trailer = any_of( + condense(dot + unsafe_name), + condense(lbrack + subscriptlist + rbrack), + use_adaptive=False, ) call_trailer = ( function_call @@ -1417,9 +1423,15 @@ class Grammar(object): ) ) - mulop = mul_star | div_slash | div_dubslash | percent | matrix_at - addop = plus | sub_minus - shift = lshift | rshift + mulop = any_of( + mul_star, + div_slash, + div_dubslash, + percent, + matrix_at, + ) + addop = any_of(plus, sub_minus) + shift = any_of(lshift, rshift) term = Forward() term_ref = tokenlist(factor, mulop, allow_trailing=False, suppress=False) @@ -1430,9 +1442,11 @@ class Grammar(object): # and_expr = exprlist(shift_expr, amp) and_expr = exprlist( term, - addop - | shift - | amp, + any_of( + addop, + shift, + amp, + ), ) protocol_intersect_expr = Forward() @@ -1484,6 +1498,7 @@ class Grammar(object): comp_back_none_star_pipe, comp_none_pipe, comp_back_none_pipe, + use_adaptive=False, ) comp_pipe_item = attach( OneOrMore(none_coalesce_expr + comp_pipe_op) + ( @@ -1510,6 +1525,7 @@ class Grammar(object): back_none_pipe, back_none_star_pipe, back_none_dubstar_pipe, + use_adaptive=False, ) pipe_item = ( # we need the pipe_op since any of the atoms could otherwise be the start of an expression @@ -1573,7 +1589,7 @@ class Grammar(object): fat_arrow = Forward() lambda_arrow = Forward() - unsafe_lambda_arrow = fat_arrow | arrow + unsafe_lambda_arrow = any_of(fat_arrow, arrow) keyword_lambdef_params = maybeparens(lparen, set_args_list, rparen) arrow_lambdef_params = lparen.suppress() + set_args_list + rparen.suppress() | setname @@ -1582,10 +1598,10 @@ class Grammar(object): keyword_lambdef_ref = addspace(keyword("lambda") + condense(keyword_lambdef_params + colon)) arrow_lambdef = attach(arrow_lambdef_params + lambda_arrow.suppress(), lambdef_handle) implicit_lambdef = fixto(lambda_arrow, "lambda _=None:") - lambdef_base = ( - arrow_lambdef - | implicit_lambdef - | keyword_lambdef + lambdef_base = any_of( + arrow_lambdef, + implicit_lambdef, + keyword_lambdef, ) stmt_lambdef = Forward() @@ -1749,7 +1765,7 @@ class Grammar(object): async_comp_for_ref = addspace(keyword("async") + base_comp_for) comp_for <<= base_comp_for | async_comp_for comp_if = addspace(keyword("if") + test_no_cond + Optional(comp_iter)) - comp_iter <<= comp_for | comp_if + comp_iter <<= any_of(comp_for, comp_if) return_stmt = addspace(keyword("return") - Optional(new_testlist_star_expr)) @@ -1821,6 +1837,7 @@ class Grammar(object): augassign_stmt = Forward() augassign_rhs = ( + # pipe_augassign must come first labeled_group(pipe_augassign + pipe_augassign_item, "pipe") | labeled_group(augassign + test_expr, "simple") ) @@ -2095,7 +2112,7 @@ class Grammar(object): return_typedef = Forward() return_typedef_ref = arrow.suppress() + typedef_test end_func_colon = return_typedef + colon.suppress() | colon - base_funcdef = op_funcdef | name_funcdef + base_funcdef = name_funcdef | op_funcdef funcdef = addspace(keyword("def") + condense(base_funcdef + end_func_colon + nocolon_suite)) name_match_funcdef = Forward() @@ -2112,7 +2129,7 @@ class Grammar(object): )) name_match_funcdef_ref = keyword("def").suppress() + funcname_typeparams + lparen.suppress() + match_args_list + match_guard + rparen.suppress() op_match_funcdef_ref = keyword("def").suppress() + op_match_funcdef_arg + op_funcdef_name + op_match_funcdef_arg + match_guard - base_match_funcdef = op_match_funcdef | name_match_funcdef + base_match_funcdef = name_match_funcdef | op_match_funcdef func_suite = ( ( newline.suppress() @@ -2370,11 +2387,11 @@ class Grammar(object): global_stmt, nonlocal_stmt, ) - special_stmt = ( - keyword_stmt - | augassign_stmt - | typed_assign_stmt - | type_alias_stmt + special_stmt = any_of( + keyword_stmt, + augassign_stmt, + typed_assign_stmt, + type_alias_stmt, ) unsafe_simple_stmt_item <<= special_stmt | longest(basic_stmt, destructuring_stmt) simple_stmt_item <<= ( @@ -2420,7 +2437,12 @@ class Grammar(object): unsafe_xonsh_command = originalTextFor( (Optional(at) + dollar | bang) + ~(lparen + rparen | lbrack + rbrack | lbrace + rbrace) - + (parens | brackets | braces | unsafe_name) + + any_of( + parens, + brackets, + braces, + unsafe_name, + ) ) unsafe_xonsh_parser, _impl_call_ref = disable_inside( single_parser, diff --git a/coconut/compiler/util.py b/coconut/compiler/util.py index f2ae57545..167dae31c 100644 --- a/coconut/compiler/util.py +++ b/coconut/compiler/util.py @@ -908,13 +908,11 @@ def __init__(self, *args, **kwargs): self.all_match_anys.append(self) def __or__(self, other): - if hasaction(self): + if isinstance(other, MatchAny): + self = maybe_copy_elem(self, "any_or") + return self.append(other) + else: return MatchFirst([self, other]) - self = maybe_copy_elem(self, "any_or") - if not isinstance(other, MatchAny): - self.__class__ = MatchFirst - self |= other - return self if SUPPORTS_ADAPTIVE: @@ -930,7 +928,7 @@ def any_of(*exprs, **kwargs): flat_exprs = [] for e in exprs: - if isinstance(e, AnyOf) and not hasaction(e): + if e.__class__ == AnyOf and not hasaction(e): flat_exprs.extend(e.exprs) else: flat_exprs.append(e) diff --git a/coconut/root.py b/coconut/root.py index d47b49310..7e4deedd9 100644 --- a/coconut/root.py +++ b/coconut/root.py @@ -26,7 +26,7 @@ VERSION = "3.0.3" VERSION_NAME = None # False for release, int >= 1 for develop -DEVELOP = 27 +DEVELOP = 28 ALPHA = False # for pre releases rather than post releases assert DEVELOP is False or DEVELOP >= 1, "DEVELOP must be False or an int >= 1" From 496181261044d8b70353e9235e8128ef3a98c615 Mon Sep 17 00:00:00 2001 From: Evan Hubinger Date: Mon, 13 Nov 2023 17:00:45 -0800 Subject: [PATCH 084/121] Improve incremental --- Makefile | 5 +++ coconut/compiler/util.py | 79 +++++++++++++++++++++++++++++----------- coconut/constants.py | 9 ++--- 3 files changed, 66 insertions(+), 27 deletions(-) diff --git a/Makefile b/Makefile index 2b7bdcadd..3ffe76983 100644 --- a/Makefile +++ b/Makefile @@ -244,6 +244,11 @@ test-watch: clean test-mini: coconut ./coconut/tests/src/cocotest/agnostic ./coconut/tests/dest/cocotest --force --jobs 0 --stack-size 4096 --recursion-limit 4096 +# same as test mini but allows parallelization and turns on verbose +.PHONY: test-mini-verbose +test-mini-verbose: + coconut ./coconut/tests/src/cocotest/agnostic ./coconut/tests/dest/cocotest --force --verbose --stack-size 4096 --recursion-limit 4096 + # same as test-univ but debugs crashes .PHONY: test-univ-debug test-univ-debug: export COCONUT_TEST_DEBUG_PYTHON=TRUE diff --git a/coconut/compiler/util.py b/coconut/compiler/util.py index 167dae31c..0420ab3db 100644 --- a/coconut/compiler/util.py +++ b/coconut/compiler/util.py @@ -632,36 +632,65 @@ def should_clear_cache(force=False): incremental_cache_limit is not None and len(ParserElement.packrat_cache) > incremental_cache_limit ): - # only clear the second half of the cache, since the first - # half is what will help us next time we recompile - return "second half" + if should_clear_cache.last_cache_clear_strat == "failed parents": + clear_strat = "second half" + else: + clear_strat = "failed parents" + should_clear_cache.last_cache_clear_strat = clear_strat + return clear_strat return False else: return True +should_clear_cache.last_cache_clear_strat = None + + def clear_packrat_cache(force=False): """Clear the packrat cache if applicable.""" clear_cache = should_clear_cache(force=force) if clear_cache: + cache_items = None if clear_cache == "second half": cache_items = list(get_pyparsing_cache().items()) restore_items = cache_items[:len(cache_items) // 2] + elif clear_cache == "failed parents": + cache_items = get_pyparsing_cache().items() + restore_items = [ + (lookup, value) + for lookup, value in cache_items + if value[2][0] is not False + ] else: restore_items = () + if DEVELOP and cache_items is not None: + logger.log("Pruned packrat cache from {orig_len} items to {new_len} items using {strat} strategy.".format( + orig_len=len(cache_items), + new_len=len(restore_items), + strat=clear_cache, + )) # clear cache without resetting stats ParserElement.packrat_cache.clear() # restore any items we want to keep - for lookup, value in restore_items: - ParserElement.packrat_cache.set(lookup, value) + if PY2: + for lookup, value in restore_items: + ParserElement.packrat_cache.set(lookup, value) + else: + ParserElement.packrat_cache.update(restore_items) return clear_cache -def get_cache_items_for(original): +def get_cache_items_for(original, no_failing_parents=False): """Get items from the pyparsing cache filtered to only from parsing original.""" cache = get_pyparsing_cache() for lookup, value in cache.items(): got_orig = lookup[1] + internal_assert(lambda: isinstance(got_orig, (bytes, str)), "failed to look up original in pyparsing cache item", (lookup, value)) + if no_failing_parents: + got_parent_success = value[2][0] + internal_assert(lambda: got_parent_success in (True, False, None), "failed to look up parent success in pyparsing cache item", (lookup, value)) + if got_parent_success is False: + continue if got_orig == original: yield lookup, value @@ -681,12 +710,11 @@ def enable_incremental_parsing(): """Enable incremental parsing mode where prefix/suffix parses are reused.""" if not SUPPORTS_INCREMENTAL: return False - ParserElement._should_cache_incremental_success = incremental_mode_cache_successes if in_incremental_mode(): # incremental mode is already enabled return True ParserElement._incrementalEnabled = False try: - ParserElement.enableIncremental(incremental_mode_cache_size, still_reset_cache=False) + ParserElement.enableIncremental(incremental_mode_cache_size, still_reset_cache=False, cache_successes=incremental_mode_cache_successes) except ImportError as err: raise CoconutException(str(err)) logger.log("Incremental parsing mode enabled.") @@ -699,7 +727,7 @@ def pickle_cache(original, filename, include_incremental=True, protocol=pickle.H pickleable_cache_items = [] if include_incremental: - for lookup, value in get_cache_items_for(original): + for lookup, value in get_cache_items_for(original, no_failing_parents=True): if incremental_mode_cache_size is not None and len(pickleable_cache_items) > incremental_mode_cache_size: complain( "got too large incremental cache: " @@ -755,12 +783,6 @@ def unpickle_cache(filename): pickleable_cache_items = [] all_adaptive_stats = pickle_info_obj["all_adaptive_stats"] - logger.log("Loaded {num_inc} incremental and {num_adapt} adaptive cache items from {filename!r}.".format( - num_inc=len(pickleable_cache_items), - num_adapt=len(all_adaptive_stats), - filename=filename, - )) - for identifier, (adaptive_usage, expr_order) in all_adaptive_stats.items(): all_parse_elements[identifier].adaptive_usage = adaptive_usage all_parse_elements[identifier].expr_order = expr_order @@ -775,7 +797,10 @@ def unpickle_cache(filename): for pickleable_lookup, value in pickleable_cache_items: lookup = (all_parse_elements[pickleable_lookup[0]],) + pickleable_lookup[1:] ParserElement.packrat_cache.set(lookup, value) - return True + + num_inc = len(pickleable_cache_items) + num_adapt = len(all_adaptive_stats) + return num_inc, num_adapt def load_cache_for(inputstring, filename, cache_path): @@ -797,13 +822,23 @@ def load_cache_for(inputstring, filename, cache_path): input_len=len(inputstring), max_len=disable_incremental_for_len, ) + did_load_cache = unpickle_cache(cache_path) - logger.log("{Loaded} cache for {filename!r} from {cache_path!r} ({incremental_info}).".format( - Loaded="Loaded" if did_load_cache else "Failed to load", - filename=filename, - cache_path=cache_path, - incremental_info=incremental_info, - )) + if did_load_cache: + num_inc, num_adapt = did_load_cache + logger.log("Loaded {num_inc} incremental and {num_adapt} adaptive cache items for {filename!r} ({incremental_info}).".format( + num_inc=num_inc, + num_adapt=num_adapt, + filename=filename, + incremental_info=incremental_info, + )) + else: + logger.log("Failed to load cache for {filename!r} from {cache_path!r} ({incremental_info}).".format( + filename=filename, + cache_path=cache_path, + incremental_info=incremental_info, + )) + return incremental_enabled diff --git a/coconut/constants.py b/coconut/constants.py index 32ea113be..ae973f557 100644 --- a/coconut/constants.py +++ b/coconut/constants.py @@ -128,10 +128,9 @@ def get_path_env_var(env_var, default): # Current problems with this: # - only actually helpful for tiny files (< streamline_grammar_for_len) -# - sets incremental mode for the whole process, which can really slow down some compilations -# - makes exceptions include the entire file -# disable_incremental_for_len = streamline_grammar_for_len -disable_incremental_for_len = 0 +# - sets incremental mode for the whole process, which can really slow down later compilations in that process +# - makes exceptions include the entire file when recompiling with --force +disable_incremental_for_len = streamline_grammar_for_len use_cache_file = True use_adaptive_any_of = True @@ -996,7 +995,7 @@ def get_path_env_var(env_var, default): # min versions are inclusive unpinned_min_versions = { - "cPyparsing": (2, 4, 7, 2, 2, 5), + "cPyparsing": (2, 4, 7, 2, 2, 6), ("pre-commit", "py3"): (3,), ("psutil", "py>=27"): (5,), "jupyter": (1, 0), From f9e3b4a102d78d0fbddd9fd2891b44a5c9e0ca73 Mon Sep 17 00:00:00 2001 From: Evan Hubinger Date: Mon, 13 Nov 2023 22:41:04 -0800 Subject: [PATCH 085/121] Use newest cpyparsing --- Makefile | 13 +-- coconut/compiler/grammar.py | 25 ++--- coconut/compiler/templates/header.py_template | 10 +- coconut/compiler/util.py | 98 ++++++++++++------- coconut/constants.py | 8 +- coconut/root.py | 2 +- coconut/terminal.py | 5 + 7 files changed, 91 insertions(+), 70 deletions(-) diff --git a/Makefile b/Makefile index 3ffe76983..6b0e6c1e9 100644 --- a/Makefile +++ b/Makefile @@ -156,7 +156,7 @@ test-mypy-tests: clean-no-tests python ./coconut/tests/dest/extras.py # same as test-univ but includes verbose output for better debugging -# regex for getting non-timing lines: ^(?!\s*(Time|Packrat|Loaded|Saving|Adaptive|Errorless|Grammar|Failed)\s)[^\n]*\n* +# regex for getting non-timing lines: ^(?!\s*(Time|Packrat|Loaded|Saving|Adaptive|Errorless|Grammar|Failed|Incremental)\s)[^\n]*\n* .PHONY: test-verbose test-verbose: export COCONUT_USE_COLOR=TRUE test-verbose: clean @@ -289,19 +289,16 @@ clean-no-tests: .PHONY: clean clean: clean-no-tests rm -rf ./coconut/tests/dest - -.PHONY: clean-cache -clean-cache: clean -find . -name "__coconut_cache__" -type d -prune -exec rm -rf '{}' + - -C:/GnuWin32/bin/find.exe . -name "__coconut_cache__" -type d -prune -exec rm -rf '{}' + + -powershell -Command "get-childitem -Include __coconut_cache__ -Recurse -force | Remove-Item -Force -Recurse" .PHONY: wipe -wipe: clean-cache +wipe: clean rm -rf ./coconut/tests/dest vprof.json profile.log *.egg-info -find . -name "__pycache__" -type d -prune -exec rm -rf '{}' + - -C:/GnuWin32/bin/find.exe . -name "__pycache__" -type d -prune -exec rm -rf '{}' + + -powershell -Command "get-childitem -Include __pycache__ -Recurse -force | Remove-Item -Force -Recurse" -find . -name "*.pyc" -delete - -C:/GnuWin32/bin/find.exe . -name "*.pyc" -delete + -powershell -Command "get-childitem -Include *.pyc -Recurse -force | Remove-Item -Force -Recurse" -python -m coconut --site-uninstall -python3 -m coconut --site-uninstall -python2 -m coconut --site-uninstall diff --git a/coconut/compiler/grammar.py b/coconut/compiler/grammar.py index a40fb25e8..08d24ba20 100644 --- a/coconut/compiler/grammar.py +++ b/coconut/compiler/grammar.py @@ -1196,21 +1196,16 @@ class Grammar(object): addspace(namedexpr_test + comp_for) | invalid_syntax(star_expr + comp_for, "iterable unpacking cannot be used in comprehension") ) - paren_atom = condense( - lparen + rparen - | lparen + any_of( - # everything here must end with rparen - testlist_star_namedexpr + rparen, - comprehension_expr + rparen, - op_item + rparen, - yield_expr + rparen, - anon_namedtuple + rparen, - ) | ( - lparen.suppress() - + typedef_tuple - + rparen.suppress() - ) - ) + paren_atom = condense(lparen + any_of( + # everything here must end with rparen + rparen, + testlist_star_namedexpr + rparen, + comprehension_expr + rparen, + op_item + rparen, + yield_expr + rparen, + anon_namedtuple + rparen, + typedef_tuple + rparen, + )) list_expr = Forward() list_expr_ref = testlist_star_namedexpr_tokens diff --git a/coconut/compiler/templates/header.py_template b/coconut/compiler/templates/header.py_template index 0e7a698fd..71657588f 100644 --- a/coconut/compiler/templates/header.py_template +++ b/coconut/compiler/templates/header.py_template @@ -161,7 +161,7 @@ class _coconut_tail_call(_coconut_baseclass): self.kwargs = kwargs def __reduce__(self): return (self.__class__, (self.func, self.args, self.kwargs)) -_coconut_tco_func_dict = {empty_dict} +_coconut_tco_func_dict = _coconut.weakref.WeakValueDictionary() def _coconut_tco(func): @_coconut.functools.wraps(func) def tail_call_optimized_func(*args, **kwargs): @@ -170,16 +170,14 @@ def _coconut_tco(func): if _coconut.isinstance(call_func, _coconut_base_pattern_func): call_func = call_func._coconut_tco_func elif _coconut.isinstance(call_func, _coconut.types.MethodType): - wkref = _coconut_tco_func_dict.get(_coconut.id(call_func.__func__)) - wkref_func = None if wkref is None else wkref() + wkref_func = _coconut_tco_func_dict.get(_coconut.id(call_func.__func__)) if wkref_func is call_func.__func__: if call_func.__self__ is None: call_func = call_func._coconut_tco_func else: call_func = _coconut_partial(call_func._coconut_tco_func, call_func.__self__) else: - wkref = _coconut_tco_func_dict.get(_coconut.id(call_func)) - wkref_func = None if wkref is None else wkref() + wkref_func = _coconut_tco_func_dict.get(_coconut.id(call_func)) if wkref_func is call_func: call_func = call_func._coconut_tco_func result = call_func(*args, **kwargs) # use 'coconut --no-tco' to clean up your traceback @@ -190,7 +188,7 @@ def _coconut_tco(func): tail_call_optimized_func.__module__ = _coconut.getattr(func, "__module__", None) tail_call_optimized_func.__name__ = _coconut.getattr(func, "__name__", None) tail_call_optimized_func.__qualname__ = _coconut.getattr(func, "__qualname__", None) - _coconut_tco_func_dict[_coconut.id(tail_call_optimized_func)] = _coconut.weakref.ref(tail_call_optimized_func) + _coconut_tco_func_dict[_coconut.id(tail_call_optimized_func)] = tail_call_optimized_func return tail_call_optimized_func @_coconut.functools.wraps(_coconut.itertools.tee) def tee(iterable, n=2): diff --git a/coconut/compiler/util.py b/coconut/compiler/util.py index 0420ab3db..90a5b0d66 100644 --- a/coconut/compiler/util.py +++ b/coconut/compiler/util.py @@ -34,6 +34,7 @@ import inspect import __future__ import itertools +import weakref import datetime as dt from functools import partial, reduce from collections import defaultdict @@ -612,6 +613,7 @@ def get_pyparsing_cache(): else: # on pyparsing we have to do this try: # this is sketchy, so errors should only be complained + # use .set instead of .get for the sake of MODERN_PYPARSING return get_func_closure(packrat_cache.set.__func__)["cache"] except Exception as err: complain(err) @@ -622,20 +624,21 @@ def should_clear_cache(force=False): """Determine if we should be clearing the packrat cache.""" if not ParserElement._packratEnabled: return False - if SUPPORTS_INCREMENTAL: + elif SUPPORTS_INCREMENTAL and ParserElement._incrementalEnabled: + if not in_incremental_mode(): + return repeatedly_clear_incremental_cache + if force: + # force is for when we know the recent cache is invalid, + # and second half is guaranteed to clear out recent entries + return "second half" if ( - not ParserElement._incrementalEnabled - or not in_incremental_mode() and repeatedly_clear_incremental_cache - ): - return True - if force or ( incremental_cache_limit is not None and len(ParserElement.packrat_cache) > incremental_cache_limit ): - if should_clear_cache.last_cache_clear_strat == "failed parents": + if should_clear_cache.last_cache_clear_strat == "useless": clear_strat = "second half" else: - clear_strat = "failed parents" + clear_strat = "useless" should_clear_cache.last_cache_clear_strat = clear_strat return clear_strat return False @@ -646,6 +649,16 @@ def should_clear_cache(force=False): should_clear_cache.last_cache_clear_strat = None +def add_packrat_cache_items(new_items): + """Add the given items to the packrat cache.""" + if new_items: + if PY2: + for lookup, value in new_items: + ParserElement.packrat_cache.set(lookup, value) + else: + ParserElement.packrat_cache.update(new_items) + + def clear_packrat_cache(force=False): """Clear the packrat cache if applicable.""" clear_cache = should_clear_cache(force=force) @@ -654,14 +667,15 @@ def clear_packrat_cache(force=False): if clear_cache == "second half": cache_items = list(get_pyparsing_cache().items()) restore_items = cache_items[:len(cache_items) // 2] - elif clear_cache == "failed parents": + elif clear_cache == "useless": cache_items = get_pyparsing_cache().items() restore_items = [ (lookup, value) for lookup, value in cache_items - if value[2][0] is not False + if value[2][0] ] else: + internal_assert(clear_cache is True, "invalid clear_cache strategy", clear_cache) restore_items = () if DEVELOP and cache_items is not None: logger.log("Pruned packrat cache from {orig_len} items to {new_len} items using {strat} strategy.".format( @@ -672,24 +686,21 @@ def clear_packrat_cache(force=False): # clear cache without resetting stats ParserElement.packrat_cache.clear() # restore any items we want to keep - if PY2: - for lookup, value in restore_items: - ParserElement.packrat_cache.set(lookup, value) - else: - ParserElement.packrat_cache.update(restore_items) + add_packrat_cache_items(restore_items) return clear_cache -def get_cache_items_for(original, no_failing_parents=False): +def get_cache_items_for(original, only_useful=False, exclude_stale=False): """Get items from the pyparsing cache filtered to only from parsing original.""" cache = get_pyparsing_cache() for lookup, value in cache.items(): got_orig = lookup[1] internal_assert(lambda: isinstance(got_orig, (bytes, str)), "failed to look up original in pyparsing cache item", (lookup, value)) - if no_failing_parents: - got_parent_success = value[2][0] - internal_assert(lambda: got_parent_success in (True, False, None), "failed to look up parent success in pyparsing cache item", (lookup, value)) - if got_parent_success is False: + if ParserElement._incrementalEnabled: + (is_useful,) = value[-1] + if only_useful and not is_useful: + continue + if exclude_stale and is_useful >= 2: continue if got_orig == original: yield lookup, value @@ -699,7 +710,7 @@ def get_highest_parse_loc(original): """Get the highest observed parse location.""" # find the highest observed parse location highest_loc = 0 - for lookup, _ in get_cache_items_for(original): + for lookup, _ in get_cache_items_for(original, exclude_stale=True): loc = lookup[2] if loc > highest_loc: highest_loc = loc @@ -726,11 +737,11 @@ def pickle_cache(original, filename, include_incremental=True, protocol=pickle.H internal_assert(all_parse_elements is not None, "pickle_cache requires cPyparsing") pickleable_cache_items = [] - if include_incremental: - for lookup, value in get_cache_items_for(original, no_failing_parents=True): + if ParserElement._incrementalEnabled and include_incremental: + for lookup, value in get_cache_items_for(original, only_useful=True): if incremental_mode_cache_size is not None and len(pickleable_cache_items) > incremental_mode_cache_size: - complain( - "got too large incremental cache: " + logger.log( + "Got too large incremental cache: " + str(len(get_pyparsing_cache())) + " > " + str(incremental_mode_cache_size) ) break @@ -744,8 +755,10 @@ def pickle_cache(original, filename, include_incremental=True, protocol=pickle.H pickleable_cache_items.append((pickleable_lookup, value)) all_adaptive_stats = {} - for match_any in MatchAny.all_match_anys: - all_adaptive_stats[match_any.parse_element_index] = (match_any.adaptive_usage, match_any.expr_order) + for wkref in MatchAny.all_match_anys: + match_any = wkref() + if match_any is not None: + all_adaptive_stats[match_any.parse_element_index] = (match_any.adaptive_usage, match_any.expr_order) logger.log("Saving {num_inc} incremental and {num_adapt} adaptive cache items to {filename!r}.".format( num_inc=len(pickleable_cache_items), @@ -774,7 +787,10 @@ def unpickle_cache(filename): except Exception: logger.log_exc() return False - if pickle_info_obj["VERSION"] != VERSION or pickle_info_obj["pyparsing_version"] != pyparsing_version: + if ( + pickle_info_obj["VERSION"] != VERSION + or pickle_info_obj["pyparsing_version"] != pyparsing_version + ): return False if ParserElement._incrementalEnabled: @@ -784,8 +800,10 @@ def unpickle_cache(filename): all_adaptive_stats = pickle_info_obj["all_adaptive_stats"] for identifier, (adaptive_usage, expr_order) in all_adaptive_stats.items(): - all_parse_elements[identifier].adaptive_usage = adaptive_usage - all_parse_elements[identifier].expr_order = expr_order + maybe_elem = all_parse_elements[identifier]() + if maybe_elem is not None: + maybe_elem.adaptive_usage = adaptive_usage + maybe_elem.expr_order = expr_order max_cache_size = min( incremental_mode_cache_size or float("inf"), @@ -794,9 +812,15 @@ def unpickle_cache(filename): if max_cache_size != float("inf"): pickleable_cache_items = pickleable_cache_items[-max_cache_size:] + new_cache_items = [] for pickleable_lookup, value in pickleable_cache_items: - lookup = (all_parse_elements[pickleable_lookup[0]],) + pickleable_lookup[1:] - ParserElement.packrat_cache.set(lookup, value) + maybe_elem = all_parse_elements[pickleable_lookup[0]]() + if maybe_elem is not None: + lookup = (maybe_elem,) + pickleable_lookup[1:] + internal_assert(value[-1], "loaded useless cache item", (lookup, value)) + stale_value = value[:-1] + ([value[-1][0] + 1],) + new_cache_items.append((lookup, stale_value)) + add_packrat_cache_items(new_cache_items) num_inc = len(pickleable_cache_items) num_adapt = len(all_adaptive_stats) @@ -940,7 +964,7 @@ class MatchAny(MatchFirst): def __init__(self, *args, **kwargs): super(MatchAny, self).__init__(*args, **kwargs) - self.all_match_anys.append(self) + self.all_match_anys.append(weakref.ref(self)) def __or__(self, other): if isinstance(other, MatchAny): @@ -995,13 +1019,15 @@ def wrapped_context(self): and unwrapped parses. Only supported natively on cPyparsing.""" was_inside, self.inside = self.inside, True if self.include_in_packrat_context: - ParserElement.packrat_context.append(self.identifier) + old_packrat_context = ParserElement.packrat_context + new_packrat_context = old_packrat_context + (self.identifier,) + ParserElement.packrat_context = new_packrat_context try: yield finally: if self.include_in_packrat_context: - popped = ParserElement.packrat_context.pop() - internal_assert(popped == self.identifier, "invalid popped Wrap identifier", self.identifier) + internal_assert(ParserElement.packrat_context == new_packrat_context, "invalid popped Wrap identifier", self.identifier) + ParserElement.packrat_context = old_packrat_context self.inside = was_inside @override diff --git a/coconut/constants.py b/coconut/constants.py index ae973f557..89f722122 100644 --- a/coconut/constants.py +++ b/coconut/constants.py @@ -129,8 +129,8 @@ def get_path_env_var(env_var, default): # Current problems with this: # - only actually helpful for tiny files (< streamline_grammar_for_len) # - sets incremental mode for the whole process, which can really slow down later compilations in that process -# - makes exceptions include the entire file when recompiling with --force -disable_incremental_for_len = streamline_grammar_for_len +# - recompilation for suite and util is currently broken for some reason +disable_incremental_for_len = 0 use_cache_file = True use_adaptive_any_of = True @@ -148,7 +148,7 @@ def get_path_env_var(env_var, default): # this is what gets used in compiler.util.enable_incremental_parsing() incremental_mode_cache_size = None -incremental_cache_limit = 1048576 # clear cache when it gets this large +incremental_cache_limit = 2097152 # clear cache when it gets this large incremental_mode_cache_successes = False use_left_recursion_if_available = False @@ -995,7 +995,7 @@ def get_path_env_var(env_var, default): # min versions are inclusive unpinned_min_versions = { - "cPyparsing": (2, 4, 7, 2, 2, 6), + "cPyparsing": (2, 4, 7, 2, 2, 7), ("pre-commit", "py3"): (3,), ("psutil", "py>=27"): (5,), "jupyter": (1, 0), diff --git a/coconut/root.py b/coconut/root.py index 7e4deedd9..a64949b26 100644 --- a/coconut/root.py +++ b/coconut/root.py @@ -26,7 +26,7 @@ VERSION = "3.0.3" VERSION_NAME = None # False for release, int >= 1 for develop -DEVELOP = 28 +DEVELOP = 29 ALPHA = False # for pre releases rather than post releases assert DEVELOP is False or DEVELOP >= 1, "DEVELOP must be False or an int >= 1" diff --git a/coconut/terminal.py b/coconut/terminal.py index 8d96938b3..b07f882ef 100644 --- a/coconut/terminal.py +++ b/coconut/terminal.py @@ -417,6 +417,11 @@ def warn_err(self, warning, force=False): except Exception: self.print_exc(warning=True) + def log_warn(self, *args, **kwargs): + """Log a warning.""" + if self.verbose: + return self.warn(*args, **kwargs) + def print_exc(self, err=None, show_tb=None, warning=False): """Properly prints an exception.""" self.print_formatted_error(self.get_error(err, show_tb), warning) From e1b9f80f62a9ebfab980895049f35f93c0df2744 Mon Sep 17 00:00:00 2001 From: Evan Hubinger Date: Tue, 14 Nov 2023 00:13:59 -0800 Subject: [PATCH 086/121] Improve grammar --- coconut/compiler/grammar.py | 22 +++++++++++----------- coconut/compiler/util.py | 6 ++++-- coconut/constants.py | 2 +- 3 files changed, 16 insertions(+), 14 deletions(-) diff --git a/coconut/compiler/grammar.py b/coconut/compiler/grammar.py index 08d24ba20..dca50dc50 100644 --- a/coconut/compiler/grammar.py +++ b/coconut/compiler/grammar.py @@ -621,19 +621,19 @@ class Grammar(object): comma = Literal(",") dubstar = Literal("**") - star = ~dubstar + Literal("*") + star = disambiguate_literal("*", ["**"]) at = Literal("@") arrow = Literal("->") | fixto(Literal("\u2192"), "->") unsafe_fat_arrow = Literal("=>") | fixto(Literal("\u21d2"), "=>") colon_eq = Literal(":=") unsafe_dubcolon = Literal("::") unsafe_colon = Literal(":") - colon = ~unsafe_dubcolon + ~colon_eq + unsafe_colon + colon = disambiguate_literal(":", ["::", ":="]) lt_colon = Literal("<:") semicolon = Literal(";") | invalid_syntax("\u037e", "invalid Greek question mark instead of semicolon", greedy=True) multisemicolon = combine(OneOrMore(semicolon)) eq = Literal("==") - equals = ~eq + ~Literal("=>") + Literal("=") + equals = disambiguate_literal("=", ["==", "=>"]) lbrack = Literal("[") rbrack = Literal("]") lbrace = Literal("{") @@ -643,11 +643,11 @@ class Grammar(object): lparen = ~lbanana + Literal("(") rparen = Literal(")") unsafe_dot = Literal(".") - dot = ~Literal("..") + unsafe_dot + dot = disambiguate_literal(".", [".."]) plus = Literal("+") - minus = ~Literal("->") + Literal("-") + minus = disambiguate_literal("-", ["->"]) dubslash = Literal("//") - slash = ~dubslash + Literal("/") + slash = disambiguate_literal("/", ["//"]) pipe = Literal("|>") | fixto(Literal("\u21a6"), "|>") star_pipe = Literal("|*>") | fixto(Literal("*\u21a6"), "|*>") dubstar_pipe = Literal("|**>") | fixto(Literal("**\u21a6"), "|**>") @@ -709,13 +709,13 @@ class Grammar(object): | invalid_syntax("", "|*"]) | fixto(Literal("\u222a"), "|") ) - bar = ~rbanana + unsafe_bar | invalid_syntax("\xa6", "invalid broken bar character", greedy=True) + bar = disambiguate_literal("|", ["|)"]) | invalid_syntax("\xa6", "invalid broken bar character", greedy=True) percent = Literal("%") dollar = Literal("$") lshift = Literal("<<") | fixto(Literal("\xab"), "<<") @@ -725,10 +725,10 @@ class Grammar(object): pound = Literal("#") unsafe_backtick = Literal("`") dubbackslash = Literal("\\\\") - backslash = ~dubbackslash + Literal("\\") + backslash = disambiguate_literal("\\", ["\\\\"]) dubquestion = Literal("??") - questionmark = ~dubquestion + Literal("?") - bang = ~Literal("!=") + Literal("!") + questionmark = disambiguate_literal("?", ["??"]) + bang = disambiguate_literal("!", ["!="]) kwds = keydefaultdict(partial(base_keyword, explicit_prefix=colon)) keyword = kwds.__getitem__ diff --git a/coconut/compiler/util.py b/coconut/compiler/util.py index 90a5b0d66..386a1690c 100644 --- a/coconut/compiler/util.py +++ b/coconut/compiler/util.py @@ -690,7 +690,7 @@ def clear_packrat_cache(force=False): return clear_cache -def get_cache_items_for(original, only_useful=False, exclude_stale=False): +def get_cache_items_for(original, only_useful=False, exclude_stale=True): """Get items from the pyparsing cache filtered to only from parsing original.""" cache = get_pyparsing_cache() for lookup, value in cache.items(): @@ -710,7 +710,7 @@ def get_highest_parse_loc(original): """Get the highest observed parse location.""" # find the highest observed parse location highest_loc = 0 - for lookup, _ in get_cache_items_for(original, exclude_stale=True): + for lookup, _ in get_cache_items_for(original): loc = lookup[2] if loc > highest_loc: highest_loc = loc @@ -738,6 +738,8 @@ def pickle_cache(original, filename, include_incremental=True, protocol=pickle.H pickleable_cache_items = [] if ParserElement._incrementalEnabled and include_incremental: + # note that exclude_stale is fine here because that means it was never used, + # since _parseIncremental sets usefullness to True when a cache item is used for lookup, value in get_cache_items_for(original, only_useful=True): if incremental_mode_cache_size is not None and len(pickleable_cache_items) > incremental_mode_cache_size: logger.log( diff --git a/coconut/constants.py b/coconut/constants.py index 89f722122..ded2b067e 100644 --- a/coconut/constants.py +++ b/coconut/constants.py @@ -130,7 +130,7 @@ def get_path_env_var(env_var, default): # - only actually helpful for tiny files (< streamline_grammar_for_len) # - sets incremental mode for the whole process, which can really slow down later compilations in that process # - recompilation for suite and util is currently broken for some reason -disable_incremental_for_len = 0 +disable_incremental_for_len = streamline_grammar_for_len use_cache_file = True use_adaptive_any_of = True From a079bf0f445fcf49e4cfbe44398a0f9ddfa714e3 Mon Sep 17 00:00:00 2001 From: Evan Hubinger Date: Tue, 14 Nov 2023 22:23:25 -0800 Subject: [PATCH 087/121] Improve incremental --- Makefile | 19 ++- coconut/_pyparsing.py | 127 ++++++++-------- coconut/command/cli.py | 6 + coconut/command/command.py | 17 ++- coconut/compiler/compiler.py | 6 +- coconut/compiler/util.py | 279 ++++++++++++++++++++--------------- coconut/constants.py | 12 +- coconut/root.py | 2 +- 8 files changed, 268 insertions(+), 200 deletions(-) diff --git a/Makefile b/Makefile index 6b0e6c1e9..a00cb2a94 100644 --- a/Makefile +++ b/Makefile @@ -239,22 +239,27 @@ test-watch: clean python ./coconut/tests/dest/runner.py python ./coconut/tests/dest/extras.py -# mini test that just compiles agnostic tests with fully synchronous output +# mini test that just compiles agnostic tests with verbose output .PHONY: test-mini test-mini: - coconut ./coconut/tests/src/cocotest/agnostic ./coconut/tests/dest/cocotest --force --jobs 0 --stack-size 4096 --recursion-limit 4096 - -# same as test mini but allows parallelization and turns on verbose -.PHONY: test-mini-verbose -test-mini-verbose: coconut ./coconut/tests/src/cocotest/agnostic ./coconut/tests/dest/cocotest --force --verbose --stack-size 4096 --recursion-limit 4096 +# same as test-mini but doesn't overwrite the cache +.PHONY: test-cache-mini +test-cache-mini: export COCONUT_ALLOW_SAVE_TO_CACHE=FALSE +test-cache-mini: test-mini + +# same as test-mini but with fully synchronous output and fast failing +.PHONY: test-mini-sync +test-mini-sync: + coconut ./coconut/tests/src/cocotest/agnostic ./coconut/tests/dest/cocotest --force --verbose --jobs 0 --fail-fast --stack-size 4096 --recursion-limit 4096 + # same as test-univ but debugs crashes .PHONY: test-univ-debug test-univ-debug: export COCONUT_TEST_DEBUG_PYTHON=TRUE test-univ-debug: test-univ -# same as test-mini but debugs crashes +# same as test-mini but debugs crashes, is fully synchronous, and doesn't use verbose output .PHONY: test-mini-debug test-mini-debug: export COCONUT_USE_COLOR=TRUE test-mini-debug: diff --git a/coconut/_pyparsing.py b/coconut/_pyparsing.py index b3320a5c4..cdb96cc2d 100644 --- a/coconut/_pyparsing.py +++ b/coconut/_pyparsing.py @@ -126,72 +126,75 @@ # OVERRIDES: # ----------------------------------------------------------------------------------------------------------------------- -if not CPYPARSING: - if not MODERN_PYPARSING: - HIT, MISS = 0, 1 - - def _parseCache(self, instring, loc, doActions=True, callPreParse=True): - # [CPYPARSING] include packrat_context - lookup = (self, instring, loc, callPreParse, doActions, tuple(self.packrat_context)) - with ParserElement.packrat_cache_lock: - cache = ParserElement.packrat_cache - value = cache.get(lookup) - if value is cache.not_in_cache: - ParserElement.packrat_cache_stats[MISS] += 1 - try: - value = self._parseNoCache(instring, loc, doActions, callPreParse) - except ParseBaseException as pe: - # cache a copy of the exception, without the traceback - cache.set(lookup, pe.__class__(*pe.args)) - raise - else: - cache.set(lookup, (value[0], value[1].copy())) - return value - else: - ParserElement.packrat_cache_stats[HIT] += 1 - if isinstance(value, Exception): - raise value - return value[0], value[1].copy() - - ParserElement.packrat_context = [] - ParserElement._parseCache = _parseCache - - # [CPYPARSING] fix append - def append(self, other): - if (self.parseAction - or self.resultsName is not None - or self.debug): - return self.__class__([self, other]) - elif (other.__class__ == self.__class__ - and not other.parseAction - and other.resultsName is None - and not other.debug): - self.exprs += other.exprs - self.strRepr = None - self.saveAsList |= other.saveAsList - if isinstance(self, And): - self.mayReturnEmpty &= other.mayReturnEmpty - else: - self.mayReturnEmpty |= other.mayReturnEmpty - self.mayIndexError |= other.mayIndexError - else: - self.exprs.append(other) - self.strRepr = None - if isinstance(self, And): - self.mayReturnEmpty &= other.mayReturnEmpty - else: - self.mayReturnEmpty |= other.mayReturnEmpty - self.mayIndexError |= other.mayIndexError - self.saveAsList |= other.saveAsList - return self - ParseExpression.append = append - -elif not hasattr(ParserElement, "packrat_context"): - raise ImportError( +if MODERN_PYPARSING: + SUPPORTS_PACKRAT_CONTEXT = False +elif CPYPARSING: + assert hasattr(ParserElement, "packrat_context"), ( "This version of Coconut requires cPyparsing>=" + ver_tuple_to_str(min_versions["cPyparsing"]) + "; got cPyparsing==" + __version__ + " (run '{python} -m pip install --upgrade cPyparsing' to fix)".format(python=sys.executable), ) + SUPPORTS_PACKRAT_CONTEXT = True +else: + SUPPORTS_PACKRAT_CONTEXT = True + + HIT, MISS = 0, 1 + + def _parseCache(self, instring, loc, doActions=True, callPreParse=True): + # [CPYPARSING] include packrat_context + lookup = (self, instring, loc, callPreParse, doActions, tuple(self.packrat_context)) + with ParserElement.packrat_cache_lock: + cache = ParserElement.packrat_cache + value = cache.get(lookup) + if value is cache.not_in_cache: + ParserElement.packrat_cache_stats[MISS] += 1 + try: + value = self._parseNoCache(instring, loc, doActions, callPreParse) + except ParseBaseException as pe: + # cache a copy of the exception, without the traceback + cache.set(lookup, pe.__class__(*pe.args)) + raise + else: + cache.set(lookup, (value[0], value[1].copy())) + return value + else: + ParserElement.packrat_cache_stats[HIT] += 1 + if isinstance(value, Exception): + raise value + return value[0], value[1].copy() + + ParserElement.packrat_context = [] + ParserElement._parseCache = _parseCache + + # [CPYPARSING] fix append + def append(self, other): + if (self.parseAction + or self.resultsName is not None + or self.debug): + return self.__class__([self, other]) + elif (other.__class__ == self.__class__ + and not other.parseAction + and other.resultsName is None + and not other.debug): + self.exprs += other.exprs + self.strRepr = None + self.saveAsList |= other.saveAsList + if isinstance(self, And): + self.mayReturnEmpty &= other.mayReturnEmpty + else: + self.mayReturnEmpty |= other.mayReturnEmpty + self.mayIndexError |= other.mayIndexError + else: + self.exprs.append(other) + self.strRepr = None + if isinstance(self, And): + self.mayReturnEmpty &= other.mayReturnEmpty + else: + self.mayReturnEmpty |= other.mayReturnEmpty + self.mayIndexError |= other.mayIndexError + self.saveAsList |= other.saveAsList + return self + ParseExpression.append = append if hasattr(ParserElement, "enableIncremental"): SUPPORTS_INCREMENTAL = sys.version_info >= (3, 8) # avoids stack overflows on py<=37 diff --git a/coconut/command/cli.py b/coconut/command/cli.py index a0e375d55..5ea28e199 100644 --- a/coconut/command/cli.py +++ b/coconut/command/cli.py @@ -263,6 +263,12 @@ help="run the compiler in a separate thread with the given stack size in kilobytes", ) +arguments.add_argument( + "--fail-fast", + action="store_true", + help="causes the compiler to fail immediately upon encountering a compilation error rather than attempting to continue compiling other files", +) + arguments.add_argument( "--no-cache", action="store_true", diff --git a/coconut/command/command.py b/coconut/command/command.py index 8c3cbc08e..1eec78f2c 100644 --- a/coconut/command/command.py +++ b/coconut/command/command.py @@ -131,6 +131,7 @@ class Command(object): argv_args = None # corresponds to --argv flag stack_size = 0 # corresponds to --stack-size flag use_cache = USE_CACHE # corresponds to --no-cache flag + fail_fast = False # corresponds to --fail-fast flag prompt = Prompt() @@ -177,7 +178,7 @@ def cmd_sys(self, *args, **in_kwargs): def cmd(self, args=None, argv=None, interact=True, default_target=None, default_jobs=None, use_dest=None): """Process command-line arguments.""" result = None - with self.handling_exceptions(): + with self.handling_exceptions(exit_on_error=True): if args is None: parsed_args = arguments.parse_args() else: @@ -196,7 +197,6 @@ def cmd(self, args=None, argv=None, interact=True, default_target=None, default_ self.exit_code = 0 self.stack_size = parsed_args.stack_size result = self.run_with_stack_size(self.execute_args, parsed_args, interact, original_args=args) - self.exit_on_error() return result def run_with_stack_size(self, func, *args, **kwargs): @@ -275,6 +275,7 @@ def execute_args(self, args, interact=True, original_args=None): self.set_jobs(args.jobs, args.profile) if args.recursion_limit is not None: set_recursion_limit(args.recursion_limit) + self.fail_fast = args.fail_fast self.display = args.display self.prompt.vi_mode = args.vi_mode if args.style is not None: @@ -315,7 +316,7 @@ def execute_args(self, args, interact=True, original_args=None): ) self.comp.warm_up( streamline=args.watch or args.profile, - enable_incremental_mode=args.watch, + enable_incremental_mode=self.use_cache and args.watch, set_debug_names=args.verbose or args.trace or args.profile, ) @@ -473,8 +474,10 @@ def register_exit_code(self, code=1, errmsg=None, err=None): self.exit_code = code or self.exit_code @contextmanager - def handling_exceptions(self): + def handling_exceptions(self, exit_on_error=None): """Perform proper exception handling.""" + if exit_on_error is None: + exit_on_error = self.fail_fast try: if self.using_jobs: with handling_broken_process_pool(): @@ -492,6 +495,8 @@ def handling_exceptions(self): logger.print_exc() logger.printerr(report_this_text) self.register_exit_code(err=err) + if exit_on_error: + self.exit_on_error() def compile_path(self, path, write=True, package=True, **kwargs): """Compile a path and return paths to compiled files.""" @@ -713,7 +718,7 @@ def using_jobs(self): @contextmanager def running_jobs(self, exit_on_error=True): """Initialize multiprocessing.""" - with self.handling_exceptions(): + with self.handling_exceptions(exit_on_error=exit_on_error): if self.using_jobs: from concurrent.futures import ProcessPoolExecutor try: @@ -723,8 +728,6 @@ def running_jobs(self, exit_on_error=True): self.executor = None else: yield - if exit_on_error: - self.exit_on_error() def has_hash_of(self, destpath, code, package_level): """Determine if a file has the hash of the code.""" diff --git a/coconut/compiler/compiler.py b/coconut/compiler/compiler.py index a970e993e..6c9dfa504 100644 --- a/coconut/compiler/compiler.py +++ b/coconut/compiler/compiler.py @@ -456,7 +456,7 @@ def call_decorators(decorators, func_name): class Compiler(Grammar, pickleable_obj): """The Coconut compiler.""" lock = Lock() - current_compiler = [None] # list for mutability + current_compiler = None preprocs = [ lambda self: self.prepare, @@ -692,7 +692,7 @@ def method(cls, method_name, is_action=None, **kwargs): @wraps(cls_method) def method(original, loc, tokens): - self_method = getattr(cls.current_compiler[0], method_name) + self_method = getattr(cls.current_compiler, method_name) if kwargs: self_method = partial(self_method, **kwargs) if trim_arity: @@ -1270,7 +1270,7 @@ def parsing(self, keep_state=False, filename=None): """Acquire the lock and reset the parser.""" with self.lock: self.reset(keep_state, filename) - self.current_compiler[0] = self + Compiler.current_compiler = self yield def streamline(self, grammar, inputstring="", force=False, inner=False): diff --git a/coconut/compiler/util.py b/coconut/compiler/util.py index 386a1690c..125e7df46 100644 --- a/coconut/compiler/util.py +++ b/coconut/compiler/util.py @@ -47,10 +47,12 @@ import cPickle as pickle from coconut._pyparsing import ( + CPYPARSING, MODERN_PYPARSING, USE_COMPUTATION_GRAPH, SUPPORTS_INCREMENTAL, SUPPORTS_ADAPTIVE, + SUPPORTS_PACKRAT_CONTEXT, replaceWith, ZeroOrMore, OneOrMore, @@ -85,6 +87,7 @@ memoize, univ_open, ensure_dir, + get_clock_time, ) from coconut.terminal import ( logger, @@ -125,6 +128,9 @@ disable_incremental_for_len, coconut_cache_dir, use_adaptive_if_available, + use_fast_pyparsing_reprs, + save_new_cache_items, + cache_validation_info, ) from coconut.exceptions import ( CoconutException, @@ -376,7 +382,7 @@ def final_evaluate_tokens(tokens): def adaptive_manager(item, original, loc, reparse=False): """Manage the use of MatchFirst.setAdaptiveMode.""" if reparse: - cleared_cache = clear_packrat_cache(force=True) + cleared_cache = clear_packrat_cache() if cleared_cache is not True: item.include_in_packrat_context = True MatchFirst.setAdaptiveMode(False, usage_weight=adaptive_reparse_usage_weight) @@ -534,6 +540,84 @@ def transform(grammar, text, inner=True): return result +# ----------------------------------------------------------------------------------------------------------------------- +# TARGETS: +# ----------------------------------------------------------------------------------------------------------------------- + +on_new_python = False + +raw_sys_target = str(sys.version_info[0]) + str(sys.version_info[1]) +if raw_sys_target in pseudo_targets: + sys_target = pseudo_targets[raw_sys_target] +elif raw_sys_target in specific_targets: + sys_target = raw_sys_target +elif sys.version_info > supported_py3_vers[-1]: + sys_target = "".join(str(i) for i in supported_py3_vers[-1]) + on_new_python = True +elif sys.version_info < supported_py2_vers[0]: + sys_target = "".join(str(i) for i in supported_py2_vers[0]) +elif sys.version_info < (3,): + sys_target = "".join(str(i) for i in supported_py2_vers[-1]) +else: + sys_target = "".join(str(i) for i in supported_py3_vers[0]) + + +def get_psf_target(): + """Get the oldest PSF-supported Python version target.""" + now = dt.datetime.now() + for ver, eol in py_vers_with_eols: + if now < eol: + break + return pseudo_targets.get(ver, ver) + + +def get_vers_for_target(target): + """Gets a list of the versions supported by the given target.""" + target_info = get_target_info(target) + if not target_info: + return supported_py2_vers + supported_py3_vers + elif len(target_info) == 1: + if target_info == (2,): + return supported_py2_vers + elif target_info == (3,): + return supported_py3_vers + else: + raise CoconutInternalException("invalid target info", target_info) + elif target_info[0] == 2: + return tuple(ver for ver in supported_py2_vers if ver >= target_info) + elif target_info[0] == 3: + return tuple(ver for ver in supported_py3_vers if ver >= target_info) + else: + raise CoconutInternalException("invalid target info", target_info) + + +def get_target_info_smart(target, mode="lowest"): + """Converts target into a length 2 Python version tuple. + + Modes: + - "lowest" (default): Gets the lowest version supported by the target. + - "highest": Gets the highest version supported by the target. + - "nearest": Gets the supported version that is nearest to the current one. + """ + supported_vers = get_vers_for_target(target) + if mode == "lowest": + return supported_vers[0] + elif mode == "highest": + return supported_vers[-1] + elif mode == "nearest": + sys_ver = sys.version_info[:2] + if sys_ver in supported_vers: + return sys_ver + elif sys_ver > supported_vers[-1]: + return supported_vers[-1] + elif sys_ver < supported_vers[0]: + return supported_vers[0] + else: + raise CoconutInternalException("invalid sys version", sys_ver) + else: + raise CoconutInternalException("unknown get_target_info_smart mode", mode) + + # ----------------------------------------------------------------------------------------------------------------------- # PARSING INTROSPECTION: # ----------------------------------------------------------------------------------------------------------------------- @@ -608,8 +692,8 @@ def get_pyparsing_cache(): packrat_cache = ParserElement.packrat_cache if isinstance(packrat_cache, dict): # if enablePackrat is never called return packrat_cache - elif hasattr(packrat_cache, "cache"): # cPyparsing adds this - return packrat_cache.cache + elif CPYPARSING: + return packrat_cache.cache # cPyparsing adds this else: # on pyparsing we have to do this try: # this is sketchy, so errors should only be complained @@ -622,15 +706,13 @@ def get_pyparsing_cache(): def should_clear_cache(force=False): """Determine if we should be clearing the packrat cache.""" - if not ParserElement._packratEnabled: + if force: + return True + elif not ParserElement._packratEnabled: return False elif SUPPORTS_INCREMENTAL and ParserElement._incrementalEnabled: if not in_incremental_mode(): return repeatedly_clear_incremental_cache - if force: - # force is for when we know the recent cache is invalid, - # and second half is guaranteed to clear out recent entries - return "second half" if ( incremental_cache_limit is not None and len(ParserElement.packrat_cache) > incremental_cache_limit @@ -649,10 +731,12 @@ def should_clear_cache(force=False): should_clear_cache.last_cache_clear_strat = None -def add_packrat_cache_items(new_items): +def add_packrat_cache_items(new_items, clear_first=False): """Add the given items to the packrat cache.""" + if clear_first: + ParserElement.packrat_cache.clear() if new_items: - if PY2: + if PY2 or not CPYPARSING: for lookup, value in new_items: ParserElement.packrat_cache.set(lookup, value) else: @@ -660,33 +744,44 @@ def add_packrat_cache_items(new_items): def clear_packrat_cache(force=False): - """Clear the packrat cache if applicable.""" + """Clear the packrat cache if applicable. + Very performance-sensitive for incremental parsing mode.""" clear_cache = should_clear_cache(force=force) + if clear_cache: - cache_items = None - if clear_cache == "second half": - cache_items = list(get_pyparsing_cache().items()) - restore_items = cache_items[:len(cache_items) // 2] - elif clear_cache == "useless": - cache_items = get_pyparsing_cache().items() - restore_items = [ - (lookup, value) - for lookup, value in cache_items - if value[2][0] - ] + if DEVELOP: + start_time = get_clock_time() + + orig_cache_len = None + if clear_cache is True: + # clear cache without resetting stats + ParserElement.packrat_cache.clear() else: - internal_assert(clear_cache is True, "invalid clear_cache strategy", clear_cache) - restore_items = () - if DEVELOP and cache_items is not None: - logger.log("Pruned packrat cache from {orig_len} items to {new_len} items using {strat} strategy.".format( - orig_len=len(cache_items), - new_len=len(restore_items), + internal_assert(CPYPARSING, "unsupported clear_cache strategy", clear_cache) + cache = get_pyparsing_cache() + orig_cache_len = len(cache) + if clear_cache == "second half": + all_keys = tuple(cache.keys()) + for del_key in all_keys[len(all_keys) // 2: len(all_keys)]: + del cache[del_key] + elif clear_cache == "useless": + keys_to_del = [] + for lookup, value in cache.items(): + if not value[-1][0]: + keys_to_del.append(lookup) + for del_key in keys_to_del: + del cache[del_key] + else: + raise CoconutInternalException("invalid clear_cache strategy", clear_cache) + + if DEVELOP and orig_cache_len is not None: + logger.log("Pruned packrat cache from {orig_len} items to {new_len} items using {strat} strategy ({time} secs).".format( + orig_len=orig_cache_len, + new_len=len(get_pyparsing_cache()), strat=clear_cache, + time=get_clock_time() - start_time, )) - # clear cache without resetting stats - ParserElement.packrat_cache.clear() - # restore any items we want to keep - add_packrat_cache_items(restore_items) + return clear_cache @@ -735,6 +830,11 @@ def enable_incremental_parsing(): def pickle_cache(original, filename, include_incremental=True, protocol=pickle.HIGHEST_PROTOCOL): """Pickle the pyparsing cache for original to filename.""" internal_assert(all_parse_elements is not None, "pickle_cache requires cPyparsing") + if not save_new_cache_items: + logger.log("Skipping saving cache items due to environment variable.") + return + + validation_dict = {} if cache_validation_info else None pickleable_cache_items = [] if ParserElement._incrementalEnabled and include_incremental: @@ -753,13 +853,18 @@ def pickle_cache(original, filename, include_incremental=True, protocol=pickle.H # only include cache items that aren't at the start or end, since those # are the only ones that parseIncremental will reuse if 0 < loc < len(original) - 1: - pickleable_lookup = (lookup[0].parse_element_index,) + lookup[1:] + elem = lookup[0] + if validation_dict is not None: + validation_dict[elem.parse_element_index] = elem.__class__.__name__ + pickleable_lookup = (elem.parse_element_index,) + lookup[1:] pickleable_cache_items.append((pickleable_lookup, value)) all_adaptive_stats = {} for wkref in MatchAny.all_match_anys: match_any = wkref() if match_any is not None: + if validation_dict is not None: + validation_dict[match_any.parse_element_index] = match_any.__class__.__name__ all_adaptive_stats[match_any.parse_element_index] = (match_any.adaptive_usage, match_any.expr_order) logger.log("Saving {num_inc} incremental and {num_adapt} adaptive cache items to {filename!r}.".format( @@ -770,12 +875,16 @@ def pickle_cache(original, filename, include_incremental=True, protocol=pickle.H pickle_info_obj = { "VERSION": VERSION, "pyparsing_version": pyparsing_version, + "validation_dict": validation_dict, "pickleable_cache_items": pickleable_cache_items, "all_adaptive_stats": all_adaptive_stats, } with univ_open(filename, "wb") as pickle_file: pickle.dump(pickle_info_obj, pickle_file, protocol=protocol) + # clear the packrat cache when we're done so we don't interfere with anything else happening in this process + clear_packrat_cache(force=True) + def unpickle_cache(filename): """Unpickle and load the given incremental cache file.""" @@ -795,6 +904,7 @@ def unpickle_cache(filename): ): return False + validation_dict = pickle_info_obj["validation_dict"] if ParserElement._incrementalEnabled: pickleable_cache_items = pickle_info_obj["pickleable_cache_items"] else: @@ -804,6 +914,8 @@ def unpickle_cache(filename): for identifier, (adaptive_usage, expr_order) in all_adaptive_stats.items(): maybe_elem = all_parse_elements[identifier]() if maybe_elem is not None: + if validation_dict is not None: + internal_assert(maybe_elem.__class__.__name__ == validation_dict[identifier], "adaptive cache pickle-unpickle inconsistency", (maybe_elem, validation_dict[identifier])) maybe_elem.adaptive_usage = adaptive_usage maybe_elem.expr_order = expr_order @@ -816,11 +928,15 @@ def unpickle_cache(filename): new_cache_items = [] for pickleable_lookup, value in pickleable_cache_items: - maybe_elem = all_parse_elements[pickleable_lookup[0]]() + identifier = pickleable_lookup[0] + maybe_elem = all_parse_elements[identifier]() if maybe_elem is not None: + if validation_dict is not None: + internal_assert(maybe_elem.__class__.__name__ == validation_dict[identifier], "incremental cache pickle-unpickle inconsistency", (maybe_elem, validation_dict[identifier])) lookup = (maybe_elem,) + pickleable_lookup[1:] - internal_assert(value[-1], "loaded useless cache item", (lookup, value)) - stale_value = value[:-1] + ([value[-1][0] + 1],) + usefullness = value[-1][0] + internal_assert(usefullness, "loaded useless cache item", (lookup, value)) + stale_value = value[:-1] + ([usefullness + 1],) new_cache_items.append((lookup, stale_value)) add_packrat_cache_items(new_cache_items) @@ -833,7 +949,11 @@ def load_cache_for(inputstring, filename, cache_path): """Load cache_path (for the given inputstring and filename).""" if not SUPPORTS_INCREMENTAL: raise CoconutException("incremental parsing mode requires cPyparsing (run '{python} -m pip install --upgrade cPyparsing' to fix)".format(python=sys.executable)) - if len(inputstring) < disable_incremental_for_len: + + if in_incremental_mode(): + incremental_enabled = True + incremental_info = "using incremental parsing mode since it was already enabled" + elif len(inputstring) < disable_incremental_for_len: incremental_enabled = enable_incremental_parsing() if incremental_enabled: incremental_info = "incremental parsing mode enabled due to len == {input_len} < {max_len}".format( @@ -879,83 +999,6 @@ def get_cache_path(codepath): return os.path.join(cache_dir, pickle_fname) -# ----------------------------------------------------------------------------------------------------------------------- -# TARGETS: -# ----------------------------------------------------------------------------------------------------------------------- -on_new_python = False - -raw_sys_target = str(sys.version_info[0]) + str(sys.version_info[1]) -if raw_sys_target in pseudo_targets: - sys_target = pseudo_targets[raw_sys_target] -elif raw_sys_target in specific_targets: - sys_target = raw_sys_target -elif sys.version_info > supported_py3_vers[-1]: - sys_target = "".join(str(i) for i in supported_py3_vers[-1]) - on_new_python = True -elif sys.version_info < supported_py2_vers[0]: - sys_target = "".join(str(i) for i in supported_py2_vers[0]) -elif sys.version_info < (3,): - sys_target = "".join(str(i) for i in supported_py2_vers[-1]) -else: - sys_target = "".join(str(i) for i in supported_py3_vers[0]) - - -def get_psf_target(): - """Get the oldest PSF-supported Python version target.""" - now = dt.datetime.now() - for ver, eol in py_vers_with_eols: - if now < eol: - break - return pseudo_targets.get(ver, ver) - - -def get_vers_for_target(target): - """Gets a list of the versions supported by the given target.""" - target_info = get_target_info(target) - if not target_info: - return supported_py2_vers + supported_py3_vers - elif len(target_info) == 1: - if target_info == (2,): - return supported_py2_vers - elif target_info == (3,): - return supported_py3_vers - else: - raise CoconutInternalException("invalid target info", target_info) - elif target_info[0] == 2: - return tuple(ver for ver in supported_py2_vers if ver >= target_info) - elif target_info[0] == 3: - return tuple(ver for ver in supported_py3_vers if ver >= target_info) - else: - raise CoconutInternalException("invalid target info", target_info) - - -def get_target_info_smart(target, mode="lowest"): - """Converts target into a length 2 Python version tuple. - - Modes: - - "lowest" (default): Gets the lowest version supported by the target. - - "highest": Gets the highest version supported by the target. - - "nearest": Gets the supported version that is nearest to the current one. - """ - supported_vers = get_vers_for_target(target) - if mode == "lowest": - return supported_vers[0] - elif mode == "highest": - return supported_vers[-1] - elif mode == "nearest": - sys_ver = sys.version_info[:2] - if sys_ver in supported_vers: - return sys_ver - elif sys_ver > supported_vers[-1]: - return supported_vers[-1] - elif sys_ver < supported_vers[0]: - return supported_vers[0] - else: - raise CoconutInternalException("invalid sys version", sys_ver) - else: - raise CoconutInternalException("unknown get_target_info_smart mode", mode) - - # ----------------------------------------------------------------------------------------------------------------------- # PARSE ELEMENTS: # ----------------------------------------------------------------------------------------------------------------------- @@ -975,6 +1018,10 @@ def __or__(self, other): else: return MatchFirst([self, other]) + if not use_fast_pyparsing_reprs: + def __str__(self): + return self.__class__.__name__ + ":" + super(MatchAny, self).__str__() + if SUPPORTS_ADAPTIVE: MatchAny.setAdaptiveMode(True) @@ -989,7 +1036,7 @@ def any_of(*exprs, **kwargs): flat_exprs = [] for e in exprs: - if e.__class__ == AnyOf and not hasaction(e): + if e.__class__ is AnyOf and not hasaction(e): flat_exprs.extend(e.exprs) else: flat_exprs.append(e) @@ -1005,7 +1052,7 @@ def __init__(self, item, wrapper, greedy=False, include_in_packrat_context=False super(Wrap, self).__init__(item) self.wrapper = wrapper self.greedy = greedy - self.include_in_packrat_context = include_in_packrat_context and hasattr(ParserElement, "packrat_context") + self.include_in_packrat_context = SUPPORTS_PACKRAT_CONTEXT and include_in_packrat_context self.identifier = Wrap.global_instance_counter Wrap.global_instance_counter += 1 diff --git a/coconut/constants.py b/coconut/constants.py index ded2b067e..8011bf898 100644 --- a/coconut/constants.py +++ b/coconut/constants.py @@ -119,18 +119,22 @@ def get_path_env_var(env_var, default): num_displayed_timing_items = 100 +save_new_cache_items = get_bool_env_var("COCONUT_ALLOW_SAVE_TO_CACHE", True) + +cache_validation_info = DEVELOP + # below constants are experimentally determined to maximize performance use_packrat_parser = True # True also gives us better error messages packrat_cache_size = None # only works because final() clears the cache -streamline_grammar_for_len = 4096 +streamline_grammar_for_len = 1536 # Current problems with this: -# - only actually helpful for tiny files (< streamline_grammar_for_len) +# - only actually helpful for tiny files (< ~4096) # - sets incremental mode for the whole process, which can really slow down later compilations in that process -# - recompilation for suite and util is currently broken for some reason -disable_incremental_for_len = streamline_grammar_for_len +# - currently breaks recompilation for suite and util for some reason +disable_incremental_for_len = 0 use_cache_file = True use_adaptive_any_of = True diff --git a/coconut/root.py b/coconut/root.py index a64949b26..f27c0037e 100644 --- a/coconut/root.py +++ b/coconut/root.py @@ -26,7 +26,7 @@ VERSION = "3.0.3" VERSION_NAME = None # False for release, int >= 1 for develop -DEVELOP = 29 +DEVELOP = 30 ALPHA = False # for pre releases rather than post releases assert DEVELOP is False or DEVELOP >= 1, "DEVELOP must be False or an int >= 1" From 11320e8998042bec935f7583551e0d068b823f2d Mon Sep 17 00:00:00 2001 From: Evan Hubinger Date: Tue, 14 Nov 2023 22:33:52 -0800 Subject: [PATCH 088/121] Update docs --- DOCS.md | 38 ++++++++++++++++++++------------------ 1 file changed, 20 insertions(+), 18 deletions(-) diff --git a/DOCS.md b/DOCS.md index 685ecb0f2..a00f9f696 100644 --- a/DOCS.md +++ b/DOCS.md @@ -146,16 +146,16 @@ dest destination directory for compiled files (defaults to -v, -V, --version print Coconut and Python version information -t version, --target version specify target Python version (defaults to universal) --i, --interact force the interpreter to start (otherwise starts if no other command - is given) (implies --run) +-i, --interact force the interpreter to start (otherwise starts if no other command is + given) (implies --run) -p, --package compile source as part of a package (defaults to only if source is a directory) -a, --standalone, --stand-alone - compile source as standalone files (defaults to only if source is a - single file) + compile source as standalone files (defaults to only if source is a single + file) -l, --line-numbers, --linenumbers - force enable line number comments (--line-numbers are enabled by - default unless --minify is passed) + force enable line number comments (--line-numbers are enabled by default + unless --minify is passed) --no-line-numbers, --nolinenumbers disable line number comments (opposite of --line-numbers) -k, --keep-lines, --keeplines @@ -170,11 +170,9 @@ dest destination directory for compiled files (defaults to -s, --strict enforce code cleanliness standards --no-tco, --notco disable tail call optimization --no-wrap-types, --nowraptypes - disable wrapping type annotations in strings and turn off 'from - __future__ import annotations' behavior + disable wrapping type annotations in strings and turn off 'from __future__ + import annotations' behavior -c code, --code code run Coconut passed in as a string (can also be piped into stdin) ---incremental enable incremental compilation mode (caches previous parses to - improve recompilation performance for slightly modified files) -j processes, --jobs processes number of additional processes to use (defaults to 'sys') (0 is no additional processes; 'sys' uses machine default) @@ -182,28 +180,32 @@ dest destination directory for compiled files (defaults to haven't changed --minify reduce size of compiled Python --jupyter ..., --ipython ... - run Jupyter/IPython with Coconut as the kernel (remaining args passed - to Jupyter) + run Jupyter/IPython with Coconut as the kernel (remaining args passed to + Jupyter) --mypy ... run MyPy on compiled Python (remaining args passed to MyPy) (implies --package --line-numbers) --argv ..., --args ... - set sys.argv to source plus remaining args for use in the Coconut - script being run + set sys.argv to source plus remaining args for use in the Coconut script + being run --tutorial open Coconut's tutorial in the default web browser --docs, --documentation open Coconut's documentation in the default web browser --style name set Pygments syntax highlighting style (or 'list' to list styles) - (defaults to COCONUT_STYLE environment variable if it exists, - otherwise 'default') + (defaults to COCONUT_STYLE environment variable if it exists, otherwise + 'default') --vi-mode, --vimode enable vi mode in the interpreter (currently set to False) (can be modified by setting COCONUT_VI_MODE environment variable) --recursion-limit limit, --recursionlimit limit set maximum recursion depth in compiler (defaults to 1920) (when - increasing --recursion-limit, you may also need to increase --stack- - size; setting them to approximately equal values is recommended) + increasing --recursion-limit, you may also need to increase --stack-size; + setting them to approximately equal values is recommended) --stack-size kbs, --stacksize kbs run the compiler in a separate thread with the given stack size in kilobytes +--fail-fast causes the compiler to fail immediately upon encountering a compilation + error rather than attempting to continue compiling other files +--no-cache disables use of Coconut's incremental parsing cache (caches previous + parses to improve recompilation performance for slightly modified files) --site-install, --siteinstall set up coconut.api to be imported on Python start --site-uninstall, --siteuninstall From 592bc7876780a73dc675fc3557a795f14c43f164 Mon Sep 17 00:00:00 2001 From: Evan Hubinger Date: Thu, 16 Nov 2023 22:47:48 -0800 Subject: [PATCH 089/121] Fix incremental parsing --- Makefile | 24 +++-- coconut/_pyparsing.py | 10 ++- coconut/command/command.py | 79 ++++++++++------- coconut/command/util.py | 12 +++ coconut/compiler/compiler.py | 15 ++-- coconut/compiler/grammar.py | 87 +++++++++++-------- coconut/compiler/util.py | 33 ++++--- coconut/constants.py | 12 +-- coconut/root.py | 2 +- coconut/terminal.py | 12 +-- .../tests/src/cocotest/agnostic/suite.coco | 2 +- 11 files changed, 177 insertions(+), 111 deletions(-) diff --git a/Makefile b/Makefile index a00cb2a94..c4fe8177d 100644 --- a/Makefile +++ b/Makefile @@ -156,7 +156,7 @@ test-mypy-tests: clean-no-tests python ./coconut/tests/dest/extras.py # same as test-univ but includes verbose output for better debugging -# regex for getting non-timing lines: ^(?!\s*(Time|Packrat|Loaded|Saving|Adaptive|Errorless|Grammar|Failed|Incremental)\s)[^\n]*\n* +# regex for getting non-timing lines: ^(?!\s*(Time|Packrat|Loaded|Saving|Adaptive|Errorless|Grammar|Failed|Incremental|Pruned)\s)[^\n]*\n* .PHONY: test-verbose test-verbose: export COCONUT_USE_COLOR=TRUE test-verbose: clean @@ -235,22 +235,36 @@ test-no-wrap: clean test-watch: export COCONUT_USE_COLOR=TRUE test-watch: clean python ./coconut/tests --strict --keep-lines --force - coconut ./coconut/tests/src/cocotest/agnostic ./coconut/tests/dest/cocotest --watch --strict --keep-lines --stack-size 4096 --recursion-limit 4096 + make just-watch python ./coconut/tests/dest/runner.py python ./coconut/tests/dest/extras.py +# just watches tests +.PHONY: just-watch +just-watch: export COCONUT_USE_COLOR=TRUE +just-watch: + coconut ./coconut/tests/src/cocotest/agnostic ./coconut/tests/dest/cocotest --watch --strict --keep-lines --stack-size 4096 --recursion-limit 4096 + +# same as just-watch but uses verbose output and is fully sychronous and doesn't use the cache +.PHONY: just-watch-verbose +just-watch-verbose: export COCONUT_USE_COLOR=TRUE +just-watch-verbose: + coconut ./coconut/tests/src/cocotest/agnostic ./coconut/tests/dest/cocotest --watch --strict --keep-lines --stack-size 4096 --recursion-limit 4096 --verbose --jobs 0 --no-cache + # mini test that just compiles agnostic tests with verbose output .PHONY: test-mini +test-mini: export COCONUT_USE_COLOR=TRUE test-mini: coconut ./coconut/tests/src/cocotest/agnostic ./coconut/tests/dest/cocotest --force --verbose --stack-size 4096 --recursion-limit 4096 # same as test-mini but doesn't overwrite the cache -.PHONY: test-cache-mini -test-cache-mini: export COCONUT_ALLOW_SAVE_TO_CACHE=FALSE -test-cache-mini: test-mini +.PHONY: test-mini-cache +test-mini-cache: export COCONUT_ALLOW_SAVE_TO_CACHE=FALSE +test-mini-cache: test-mini # same as test-mini but with fully synchronous output and fast failing .PHONY: test-mini-sync +test-mini-sync: export COCONUT_USE_COLOR=TRUE test-mini-sync: coconut ./coconut/tests/src/cocotest/agnostic ./coconut/tests/dest/cocotest --force --verbose --jobs 0 --fail-fast --stack-size 4096 --recursion-limit 4096 diff --git a/coconut/_pyparsing.py b/coconut/_pyparsing.py index cdb96cc2d..a94410361 100644 --- a/coconut/_pyparsing.py +++ b/coconut/_pyparsing.py @@ -128,6 +128,7 @@ if MODERN_PYPARSING: SUPPORTS_PACKRAT_CONTEXT = False + elif CPYPARSING: assert hasattr(ParserElement, "packrat_context"), ( "This version of Coconut requires cPyparsing>=" + ver_tuple_to_str(min_versions["cPyparsing"]) @@ -135,14 +136,15 @@ + " (run '{python} -m pip install --upgrade cPyparsing' to fix)".format(python=sys.executable), ) SUPPORTS_PACKRAT_CONTEXT = True + else: SUPPORTS_PACKRAT_CONTEXT = True - HIT, MISS = 0, 1 def _parseCache(self, instring, loc, doActions=True, callPreParse=True): - # [CPYPARSING] include packrat_context - lookup = (self, instring, loc, callPreParse, doActions, tuple(self.packrat_context)) + # [CPYPARSING] HIT, MISS are constants + # [CPYPARSING] include packrat_context, merge callPreParse and doActions + lookup = (self, instring, loc, callPreParse | doActions << 1, ParserElement.packrat_context) with ParserElement.packrat_cache_lock: cache = ParserElement.packrat_cache value = cache.get(lookup) @@ -163,7 +165,7 @@ def _parseCache(self, instring, loc, doActions=True, callPreParse=True): raise value return value[0], value[1].copy() - ParserElement.packrat_context = [] + ParserElement.packrat_context = frozenset() ParserElement._parseCache = _parseCache # [CPYPARSING] fix append diff --git a/coconut/command/command.py b/coconut/command/command.py index 1eec78f2c..f5c58e699 100644 --- a/coconut/command/command.py +++ b/coconut/command/command.py @@ -45,7 +45,6 @@ internal_assert, ) from coconut.constants import ( - PY32, PY35, fixpath, code_exts, @@ -104,6 +103,7 @@ invert_mypy_arg, run_with_stack_size, proc_run_args, + get_python_lib, ) from coconut.compiler.util import ( should_indent, @@ -315,9 +315,19 @@ def execute_args(self, args, interact=True, original_args=None): no_wrap=args.no_wrap_types, ) self.comp.warm_up( - streamline=args.watch or args.profile, - enable_incremental_mode=self.use_cache and args.watch, - set_debug_names=args.verbose or args.trace or args.profile, + streamline=( + not self.using_jobs + and (args.watch or args.profile) + ), + enable_incremental_mode=( + not self.using_jobs + and args.watch + ), + set_debug_names=( + args.verbose + or args.trace + or args.profile + ), ) # process mypy args and print timing info (must come after compiler setup) @@ -474,7 +484,7 @@ def register_exit_code(self, code=1, errmsg=None, err=None): self.exit_code = code or self.exit_code @contextmanager - def handling_exceptions(self, exit_on_error=None): + def handling_exceptions(self, exit_on_error=None, on_keyboard_interrupt=None): """Perform proper exception handling.""" if exit_on_error is None: exit_on_error = self.fail_fast @@ -486,19 +496,23 @@ def handling_exceptions(self, exit_on_error=None): yield except SystemExit as err: self.register_exit_code(err.code) + # make sure we don't catch GeneratorExit below + except GeneratorExit: + raise except BaseException as err: - if isinstance(err, GeneratorExit): - raise - elif isinstance(err, CoconutException): + if isinstance(err, CoconutException): logger.print_exc() - elif not isinstance(err, KeyboardInterrupt): + elif isinstance(err, KeyboardInterrupt): + if on_keyboard_interrupt is not None: + on_keyboard_interrupt() + else: logger.print_exc() logger.printerr(report_this_text) self.register_exit_code(err=err) if exit_on_error: self.exit_on_error() - def compile_path(self, path, write=True, package=True, **kwargs): + def compile_path(self, path, write=True, package=True, handling_exceptions_kwargs={}, **kwargs): """Compile a path and return paths to compiled files.""" if not isinstance(write, bool): write = fixpath(write) @@ -506,11 +520,11 @@ def compile_path(self, path, write=True, package=True, **kwargs): destpath = self.compile_file(path, write, package, **kwargs) return [destpath] if destpath is not None else [] elif os.path.isdir(path): - return self.compile_folder(path, write, package, **kwargs) + return self.compile_folder(path, write, package, handling_exceptions_kwargs=handling_exceptions_kwargs, **kwargs) else: raise CoconutException("could not find source path", path) - def compile_folder(self, directory, write=True, package=True, **kwargs): + def compile_folder(self, directory, write=True, package=True, handling_exceptions_kwargs={}, **kwargs): """Compile a directory and return paths to compiled files.""" if not isinstance(write, bool) and os.path.isfile(write): raise CoconutException("destination path cannot point to a file when compiling a directory") @@ -522,7 +536,7 @@ def compile_folder(self, directory, write=True, package=True, **kwargs): writedir = os.path.join(write, os.path.relpath(dirpath, directory)) for filename in filenames: if os.path.splitext(filename)[1] in code_exts: - with self.handling_exceptions(): + with self.handling_exceptions(**handling_exceptions_kwargs): destpath = self.compile_file(os.path.join(dirpath, filename), writedir, package, **kwargs) if destpath is not None: filepaths.append(destpath) @@ -642,7 +656,6 @@ def get_package_level(self, codepath): logger.warn("missing __init__" + code_exts[0] + " in package", check_dir, extra="remove --strict to dismiss") package_level = 0 return package_level - return 0 def create_package(self, dirpath, retries_left=create_package_retries): """Set up a package directory.""" @@ -1078,17 +1091,30 @@ def watch(self, src_dest_package_triples, run=False, force=False): logger.show() logger.show_tabulated("Watching", showpath(src), "(press Ctrl-C to end)...") + interrupted = [False] # in list to allow modification + + def interrupt(): + interrupted[0] = True + def recompile(path, src, dest, package): path = fixpath(path) if os.path.isfile(path) and os.path.splitext(path)[1] in code_exts: - with self.handling_exceptions(): + with self.handling_exceptions(on_keyboard_interrupt=interrupt): if dest is True or dest is None: writedir = dest else: # correct the compilation path based on the relative position of path to src dirpath = os.path.dirname(path) writedir = os.path.join(dest, os.path.relpath(dirpath, src)) - filepaths = self.compile_path(path, writedir, package, run=run, force=force, show_unchanged=False) + filepaths = self.compile_path( + path, + writedir, + package, + run=run, + force=force, + show_unchanged=False, + handling_exceptions_kwargs=dict(on_keyboard_interrupt=interrupt), + ) self.run_mypy(filepaths) observer = Observer() @@ -1101,37 +1127,28 @@ def recompile(path, src, dest, package): with self.running_jobs(): observer.start() try: - while True: + while not interrupted[0]: time.sleep(watch_interval) for wcher in watchers: wcher.keep_watching() except KeyboardInterrupt: - logger.show_sig("Got KeyboardInterrupt; stopping watcher.") + interrupt() finally: + if interrupted[0]: + logger.show_sig("Got KeyboardInterrupt; stopping watcher.") observer.stop() observer.join() - def get_python_lib(self): - """Get current Python lib location.""" - # these are expensive, so should only be imported here - if PY32: - from sysconfig import get_path - python_lib = get_path("purelib") - else: - from distutils import sysconfig - python_lib = sysconfig.get_python_lib() - return fixpath(python_lib) - def site_install(self): """Add Coconut's pth file to site-packages.""" - python_lib = self.get_python_lib() + python_lib = get_python_lib() shutil.copy(coconut_pth_file, python_lib) logger.show_sig("Added %s to %s" % (os.path.basename(coconut_pth_file), python_lib)) def site_uninstall(self): """Remove Coconut's pth file from site-packages.""" - python_lib = self.get_python_lib() + python_lib = get_python_lib() pth_file = os.path.join(python_lib, os.path.basename(coconut_pth_file)) if os.path.isfile(pth_file): diff --git a/coconut/command/util.py b/coconut/command/util.py index 1758b399b..3750b34bc 100644 --- a/coconut/command/util.py +++ b/coconut/command/util.py @@ -483,6 +483,18 @@ def proc_run_args(args=()): return args +def get_python_lib(): + """Get current Python lib location.""" + # these are expensive, so should only be imported here + if PY32: + from sysconfig import get_path + python_lib = get_path("purelib") + else: + from distutils import sysconfig + python_lib = sysconfig.get_python_lib() + return fixpath(python_lib) + + # ----------------------------------------------------------------------------------------------------------------------- # CLASSES: # ----------------------------------------------------------------------------------------------------------------------- diff --git a/coconut/compiler/compiler.py b/coconut/compiler/compiler.py index 6c9dfa504..9778fae45 100644 --- a/coconut/compiler/compiler.py +++ b/coconut/compiler/compiler.py @@ -1273,20 +1273,23 @@ def parsing(self, keep_state=False, filename=None): Compiler.current_compiler = self yield - def streamline(self, grammar, inputstring="", force=False, inner=False): + def streamline(self, grammar, inputstring=None, force=False, inner=False): """Streamline the given grammar for the given inputstring.""" - if force or (streamline_grammar_for_len is not None and len(inputstring) > streamline_grammar_for_len): + input_len = 0 if inputstring is None else len(inputstring) + if force or (streamline_grammar_for_len is not None and input_len > streamline_grammar_for_len): start_time = get_clock_time() prep_grammar(grammar, streamline=True) logger.log_lambda( - lambda: "Streamlined {grammar} in {time} seconds (streamlined due to receiving input of length {length}).".format( + lambda: "Streamlined {grammar} in {time} seconds{info}.".format( grammar=get_name(grammar), time=get_clock_time() - start_time, - length=len(inputstring), + info="" if inputstring is None else " (streamlined due to receiving input of length {length})".format( + length=input_len, + ), ), ) - elif not inner: - logger.log("No streamlining done for input of length {length}.".format(length=len(inputstring))) + elif inputstring is not None and not inner: + logger.log("No streamlining done for input of length {length}.".format(length=input_len)) def run_final_checks(self, original, keep_state=False): """Run post-parsing checks to raise any necessary errors/warnings.""" diff --git a/coconut/compiler/grammar.py b/coconut/compiler/grammar.py index dca50dc50..1ecbf147f 100644 --- a/coconut/compiler/grammar.py +++ b/coconut/compiler/grammar.py @@ -876,6 +876,7 @@ class Grammar(object): combine(back_none_pipe + equals), combine(back_none_star_pipe + equals), combine(back_none_dubstar_pipe + equals), + use_adaptive=False, ) augassign = any_of( combine(plus + equals), @@ -907,19 +908,21 @@ class Grammar(object): combine(unsafe_dubcolon + equals), combine(dotdot + equals), pipe_augassign, + use_adaptive=False, ) - comp_op = any_of( - eq, - ne, - keyword("in"), - lt, - gt, - le, - ge, - addspace(keyword("not") + keyword("in")), - keyword("is") + ~keyword("not"), - addspace(keyword("is") + keyword("not")), + comp_op = ( + eq + | ne + | keyword("in") + | lt + | gt + | le + | ge + | addspace(keyword("not") + keyword("in")) + # is not must come before is + | addspace(keyword("is") + keyword("not")) + | keyword("is") ) atom_item = Forward() @@ -1839,7 +1842,7 @@ class Grammar(object): augassign_stmt_ref = simple_assign + augassign_rhs simple_kwd_assign = attach( - maybeparens(lparen, itemlist(setname, comma), rparen) + Optional(equals.suppress() - test_expr), + maybeparens(lparen, itemlist(setname, comma), rparen) + Optional(equals.suppress() + test_expr), simple_kwd_assign_handle, ) kwd_augassign = Forward() @@ -1848,9 +1851,9 @@ class Grammar(object): kwd_augassign | simple_kwd_assign ) - global_stmt = addspace(keyword("global") - kwd_assign) + global_stmt = addspace(keyword("global") + kwd_assign) nonlocal_stmt = Forward() - nonlocal_stmt_ref = addspace(keyword("nonlocal") - kwd_assign) + nonlocal_stmt_ref = addspace(keyword("nonlocal") + kwd_assign) del_stmt = addspace(keyword("del") - simple_assignlist) @@ -2002,7 +2005,7 @@ class Grammar(object): + match_guard # avoid match match-case blocks + ~FollowedBy(colon + newline + indent + keyword("case")) - - full_suite + + full_suite ) match_stmt = condense(full_match - Optional(else_stmt)) @@ -2183,13 +2186,14 @@ class Grammar(object): + attach( base_match_funcdef + end_func_equals - + ( + - ( attach(implicit_return_stmt, make_suite_handle) | ( - newline.suppress() - indent.suppress() - + Optional(docstring) - + attach(math_funcdef_body, make_suite_handle) - + dedent.suppress() + newline.suppress() + - indent.suppress() + - Optional(docstring) + - attach(math_funcdef_body, make_suite_handle) + - dedent.suppress() ) ), join_match_funcdef, @@ -2282,8 +2286,8 @@ class Grammar(object): # match funcdefs must come after normal funcdef | math_funcdef - | math_match_funcdef | match_funcdef + | math_match_funcdef | keyword_funcdef ) @@ -2335,7 +2339,7 @@ class Grammar(object): complex_decorator = condense(namedexpr_test + newline)("complex") decorators_ref = OneOrMore( at.suppress() - - Group( + + Group( simple_decorator | complex_decorator ) @@ -2347,28 +2351,37 @@ class Grammar(object): decoratable_async_funcdef_stmt = Forward() decoratable_async_funcdef_stmt_ref = Optional(decorators) + async_funcdef_stmt - decoratable_func_stmt = decoratable_normal_funcdef_stmt | decoratable_async_funcdef_stmt + decoratable_func_stmt = any_of( + decoratable_normal_funcdef_stmt, + decoratable_async_funcdef_stmt, + ) + decoratable_data_stmt = ( + # match must come after + datadef + | match_datadef + ) - # decorators are integrated into the definitions of each item here - decoratable_class_stmt = classdef | datadef | match_datadef + any_for_stmt = ( + # match must come after + for_stmt + | match_for_stmt + ) passthrough_stmt = condense(passthrough_block - (base_suite | newline)) - simple_compound_stmt = any_of( - if_stmt, - try_stmt, - match_stmt, - passthrough_stmt, - ) compound_stmt = any_of( - decoratable_class_stmt, + # decorators should be integrated into the definitions of any items that need them + if_stmt, decoratable_func_stmt, + classdef, while_stmt, - for_stmt, + try_stmt, with_stmt, + any_for_stmt, async_stmt, - match_for_stmt, - simple_compound_stmt, + decoratable_data_stmt, + match_stmt, + passthrough_stmt, where_stmt, ) endline_semicolon = Forward() @@ -2563,7 +2576,9 @@ class Grammar(object): - keyword("def").suppress() - unsafe_dotted_name - Optional(brackets).suppress() - - lparen.suppress() - parameters_tokens - rparen.suppress() + - lparen.suppress() + - parameters_tokens + - rparen.suppress() ) stores_scope = boundary + ( diff --git a/coconut/compiler/util.py b/coconut/compiler/util.py index 125e7df46..3270c422f 100644 --- a/coconut/compiler/util.py +++ b/coconut/compiler/util.py @@ -827,8 +827,8 @@ def enable_incremental_parsing(): return True -def pickle_cache(original, filename, include_incremental=True, protocol=pickle.HIGHEST_PROTOCOL): - """Pickle the pyparsing cache for original to filename.""" +def pickle_cache(original, cache_path, include_incremental=True, protocol=pickle.HIGHEST_PROTOCOL): + """Pickle the pyparsing cache for original to cache_path.""" internal_assert(all_parse_elements is not None, "pickle_cache requires cPyparsing") if not save_new_cache_items: logger.log("Skipping saving cache items due to environment variable.") @@ -854,23 +854,28 @@ def pickle_cache(original, filename, include_incremental=True, protocol=pickle.H # are the only ones that parseIncremental will reuse if 0 < loc < len(original) - 1: elem = lookup[0] + identifier = elem.parse_element_index + internal_assert(lambda: elem == all_parse_elements[identifier](), "failed to look up parse element by identifier", (elem, all_parse_elements[identifier]())) if validation_dict is not None: - validation_dict[elem.parse_element_index] = elem.__class__.__name__ - pickleable_lookup = (elem.parse_element_index,) + lookup[1:] + validation_dict[identifier] = elem.__class__.__name__ + pickleable_lookup = (identifier,) + lookup[1:] pickleable_cache_items.append((pickleable_lookup, value)) all_adaptive_stats = {} for wkref in MatchAny.all_match_anys: match_any = wkref() if match_any is not None: + identifier = match_any.parse_element_index + internal_assert(lambda: match_any == all_parse_elements[identifier](), "failed to look up match_any by identifier", (match_any, all_parse_elements[identifier]())) if validation_dict is not None: - validation_dict[match_any.parse_element_index] = match_any.__class__.__name__ - all_adaptive_stats[match_any.parse_element_index] = (match_any.adaptive_usage, match_any.expr_order) + validation_dict[identifier] = match_any.__class__.__name__ + all_adaptive_stats[identifier] = (match_any.adaptive_usage, match_any.expr_order) + logger.log("Caching adaptive item:", match_any, "<-", all_adaptive_stats[identifier]) - logger.log("Saving {num_inc} incremental and {num_adapt} adaptive cache items to {filename!r}.".format( + logger.log("Saving {num_inc} incremental and {num_adapt} adaptive cache items to {cache_path!r}.".format( num_inc=len(pickleable_cache_items), num_adapt=len(all_adaptive_stats), - filename=filename, + cache_path=cache_path, )) pickle_info_obj = { "VERSION": VERSION, @@ -879,21 +884,21 @@ def pickle_cache(original, filename, include_incremental=True, protocol=pickle.H "pickleable_cache_items": pickleable_cache_items, "all_adaptive_stats": all_adaptive_stats, } - with univ_open(filename, "wb") as pickle_file: + with univ_open(cache_path, "wb") as pickle_file: pickle.dump(pickle_info_obj, pickle_file, protocol=protocol) # clear the packrat cache when we're done so we don't interfere with anything else happening in this process clear_packrat_cache(force=True) -def unpickle_cache(filename): +def unpickle_cache(cache_path): """Unpickle and load the given incremental cache file.""" internal_assert(all_parse_elements is not None, "unpickle_cache requires cPyparsing") - if not os.path.exists(filename): + if not os.path.exists(cache_path): return False try: - with univ_open(filename, "rb") as pickle_file: + with univ_open(cache_path, "rb") as pickle_file: pickle_info_obj = pickle.load(pickle_file) except Exception: logger.log_exc() @@ -1036,7 +1041,7 @@ def any_of(*exprs, **kwargs): flat_exprs = [] for e in exprs: - if e.__class__ is AnyOf and not hasaction(e): + if e.__class__ == AnyOf and not hasaction(e): flat_exprs.extend(e.exprs) else: flat_exprs.append(e) @@ -1069,7 +1074,7 @@ def wrapped_context(self): was_inside, self.inside = self.inside, True if self.include_in_packrat_context: old_packrat_context = ParserElement.packrat_context - new_packrat_context = old_packrat_context + (self.identifier,) + new_packrat_context = old_packrat_context | frozenset((self.identifier,)) ParserElement.packrat_context = new_packrat_context try: yield diff --git a/coconut/constants.py b/coconut/constants.py index 8011bf898..ed8ef69bb 100644 --- a/coconut/constants.py +++ b/coconut/constants.py @@ -105,7 +105,7 @@ def get_path_env_var(env_var, default): # ----------------------------------------------------------------------------------------------------------------------- # set this to False only ever temporarily for ease of debugging -use_fast_pyparsing_reprs = True +use_fast_pyparsing_reprs = False assert use_fast_pyparsing_reprs or DEVELOP, "use_fast_pyparsing_reprs should never be disabled on non-develop build" enable_pyparsing_warnings = DEVELOP @@ -130,14 +130,10 @@ def get_path_env_var(env_var, default): streamline_grammar_for_len = 1536 -# Current problems with this: -# - only actually helpful for tiny files (< ~4096) -# - sets incremental mode for the whole process, which can really slow down later compilations in that process -# - currently breaks recompilation for suite and util for some reason -disable_incremental_for_len = 0 +disable_incremental_for_len = float("inf") # always use use_cache_file = True -use_adaptive_any_of = True +use_adaptive_any_of = get_bool_env_var("COCONUT_ADAPTIVE_ANY_OF", True) # note that _parseIncremental produces much smaller caches use_incremental_if_available = False @@ -999,7 +995,7 @@ def get_path_env_var(env_var, default): # min versions are inclusive unpinned_min_versions = { - "cPyparsing": (2, 4, 7, 2, 2, 7), + "cPyparsing": (2, 4, 7, 2, 2, 8), ("pre-commit", "py3"): (3,), ("psutil", "py>=27"): (5,), "jupyter": (1, 0), diff --git a/coconut/root.py b/coconut/root.py index f27c0037e..932bfb04a 100644 --- a/coconut/root.py +++ b/coconut/root.py @@ -26,7 +26,7 @@ VERSION = "3.0.3" VERSION_NAME = None # False for release, int >= 1 for develop -DEVELOP = 30 +DEVELOP = 31 ALPHA = False # for pre releases rather than post releases assert DEVELOP is False or DEVELOP >= 1, "DEVELOP must be False or an int >= 1" diff --git a/coconut/terminal.py b/coconut/terminal.py index b07f882ef..9564ad982 100644 --- a/coconut/terminal.py +++ b/coconut/terminal.py @@ -131,6 +131,8 @@ def internal_assert(condition, message=None, item=None, extra=None, exc_maker=No item = condition elif callable(message): message = message() + # ensure the item is pickleable so that the exception can be transferred back across processes + item = str(item) if callable(extra): extra = extra() if exc_maker is None: @@ -473,17 +475,17 @@ def print_trace(self, *args): trace = " ".join(str(arg) for arg in args) self.printlog(_indent(trace, self.trace_ind)) - def log_tag(self, tag, code, multiline=False, force=False): + def log_tag(self, tag, block, multiline=False, force=False): """Logs a tagged message if tracing.""" if self.tracing or force: assert not (not DEVELOP and force), tag - if callable(code): - code = code() + if callable(block): + block = block() tagstr = "[" + str(tag) + "]" if multiline: - self.print_trace(tagstr + "\n" + displayable(code)) + self.print_trace(tagstr + "\n" + displayable(block)) else: - self.print_trace(tagstr, ascii(code)) + self.print_trace(tagstr, ascii(block)) def log_trace(self, expr, original, loc, item=None, extra=None): """Formats and displays a trace if tracing.""" diff --git a/coconut/tests/src/cocotest/agnostic/suite.coco b/coconut/tests/src/cocotest/agnostic/suite.coco index beba7d22d..992a0fce1 100644 --- a/coconut/tests/src/cocotest/agnostic/suite.coco +++ b/coconut/tests/src/cocotest/agnostic/suite.coco @@ -730,7 +730,7 @@ def suite_test() -> bool: only_match_if(1) -> _ = 1 match only_match_if(1) -> _ in 2: assert False - only_match_int -> _ = 1 + only_match_int -> _ = 10 match only_match_int -> _ in "abc": assert False only_match_abc -> _ = "abc" From cd7d9d8206243db2d94ba61898b19377199086b3 Mon Sep 17 00:00:00 2001 From: Evan Hubinger Date: Fri, 17 Nov 2023 00:32:51 -0800 Subject: [PATCH 090/121] Improve incremental parsing --- coconut/_pyparsing.py | 15 ++-- coconut/compiler/compiler.py | 22 ++--- coconut/compiler/grammar.py | 39 +++++---- coconut/compiler/util.py | 154 ++++++++++++++++++++--------------- coconut/constants.py | 10 ++- coconut/root.py | 2 +- 6 files changed, 134 insertions(+), 108 deletions(-) diff --git a/coconut/_pyparsing.py b/coconut/_pyparsing.py index a94410361..ce6aa061d 100644 --- a/coconut/_pyparsing.py +++ b/coconut/_pyparsing.py @@ -23,7 +23,6 @@ import re import sys import traceback -import functools from warnings import warn from collections import defaultdict from itertools import permutations @@ -284,10 +283,14 @@ def enableIncremental(*args, **kwargs): # FAST REPRS: # ----------------------------------------------------------------------------------------------------------------------- -if PY2: - def fast_repr(cls): +if DEVELOP: + def fast_repr(self): """A very simple, fast __repr__/__str__ implementation.""" - return "<" + cls.__name__ + ">" + return getattr(self, "name", self.__class__.__name__) +elif PY2: + def fast_repr(self): + """A very simple, fast __repr__/__str__ implementation.""" + return "<" + self.__class__.__name__ + ">" else: fast_repr = object.__repr__ @@ -301,8 +304,8 @@ def set_fast_pyparsing_reprs(): try: if issubclass(obj, ParserElement): _old_pyparsing_reprs.append((obj, (obj.__repr__, obj.__str__))) - obj.__repr__ = functools.partial(fast_repr, obj) - obj.__str__ = functools.partial(fast_repr, obj) + obj.__repr__ = fast_repr + obj.__str__ = fast_repr except TypeError: pass diff --git a/coconut/compiler/compiler.py b/coconut/compiler/compiler.py index 9778fae45..e82c3b42b 100644 --- a/coconut/compiler/compiler.py +++ b/coconut/compiler/compiler.py @@ -176,7 +176,6 @@ pickle_cache, handle_and_manage, sub_all, - get_cache_path, ) from coconut.compiler.header import ( minify_header, @@ -1266,8 +1265,9 @@ def inner_parse_eval( return self.post(parsed, **postargs) @contextmanager - def parsing(self, keep_state=False, filename=None): + def parsing(self, keep_state=False, codepath=None): """Acquire the lock and reset the parser.""" + filename = None if codepath is None else os.path.basename(codepath) with self.lock: self.reset(keep_state, filename) Compiler.current_compiler = self @@ -1320,21 +1320,17 @@ def parse( ): """Use the parser to parse the inputstring with appropriate setup and teardown.""" if use_cache is None: - use_cache = codepath is not None and USE_CACHE - if use_cache: - cache_path = get_cache_path(codepath) - filename = os.path.basename(codepath) if codepath is not None else None - with self.parsing(keep_state, filename): + use_cache = USE_CACHE + use_cache = use_cache and codepath is not None + with self.parsing(keep_state, codepath): if streamline: self.streamline(parser, inputstring) # unpickling must happen after streamlining and must occur in the # compiler so that it happens in the same process as compilation if use_cache: - incremental_enabled = load_cache_for( - inputstring=inputstring, - filename=filename, - cache_path=cache_path, - ) + cache_path, incremental_enabled = load_cache_for(inputstring, codepath) + else: + cache_path = None pre_procd = parsed = None try: with logger.gather_parsing_stats(): @@ -1354,7 +1350,7 @@ def parse( + str(sys.getrecursionlimit()) + " (you may also need to increase --stack-size)", ) finally: - if use_cache and pre_procd is not None: + if cache_path is not None and pre_procd is not None: pickle_cache(pre_procd, cache_path, include_incremental=incremental_enabled) self.run_final_checks(pre_procd, keep_state) return out diff --git a/coconut/compiler/grammar.py b/coconut/compiler/grammar.py index 1ecbf147f..e545d56f7 100644 --- a/coconut/compiler/grammar.py +++ b/coconut/compiler/grammar.py @@ -1773,14 +1773,6 @@ class Grammar(object): continue_stmt = keyword("continue") simple_raise_stmt = addspace(keyword("raise") + Optional(test)) + ~keyword("from") complex_raise_stmt_ref = keyword("raise").suppress() + test + keyword("from").suppress() - test - flow_stmt = any_of( - return_stmt, - simple_raise_stmt, - break_stmt, - continue_stmt, - yield_expr, - complex_raise_stmt, - ) imp_name = ( # maybeparens allows for using custom operator names here @@ -2068,6 +2060,11 @@ class Grammar(object): - suite_with_else_tokens ) match_for_stmt = Optional(keyword("match").suppress()) + base_match_for_stmt + any_for_stmt = ( + # match must come after + for_stmt + | match_for_stmt + ) except_item = ( testlist_has_comma("list") @@ -2222,7 +2219,7 @@ class Grammar(object): ) ) async_stmt_ref = addspace( - keyword("async") + (with_stmt | for_stmt | match_for_stmt) # handles async [match] for + keyword("async") + (with_stmt | any_for_stmt) # handles async [match] for | keyword("match").suppress() + keyword("async") + base_match_for_stmt # handles match async for | async_with_for_stmt ) @@ -2361,12 +2358,6 @@ class Grammar(object): | match_datadef ) - any_for_stmt = ( - # match must come after - for_stmt - | match_for_stmt - ) - passthrough_stmt = condense(passthrough_block - (base_suite | newline)) compound_stmt = any_of( @@ -2384,8 +2375,15 @@ class Grammar(object): passthrough_stmt, where_stmt, ) - endline_semicolon = Forward() - endline_semicolon_ref = semicolon.suppress() + newline + + flow_stmt = any_of( + return_stmt, + simple_raise_stmt, + break_stmt, + continue_stmt, + yield_expr, + complex_raise_stmt, + ) keyword_stmt = any_of( flow_stmt, import_stmt, @@ -2408,24 +2406,29 @@ class Grammar(object): | basic_stmt + end_simple_stmt_item | destructuring_stmt + end_simple_stmt_item ) + endline_semicolon = Forward() + endline_semicolon_ref = semicolon.suppress() + newline simple_stmt <<= condense( simple_stmt_item + ZeroOrMore(fixto(semicolon, "\n") + simple_stmt_item) + (newline | endline_semicolon) ) + anything_stmt = Forward() stmt <<= final( compound_stmt - | simple_stmt + | simple_stmt # includes destructuring # must be after destructuring due to ambiguity | cases_stmt # at the very end as a fallback case for the anything parser | anything_stmt ) + base_suite <<= condense(newline + indent - OneOrMore(stmt) - dedent) simple_suite = attach(stmt, make_suite_handle) nocolon_suite <<= base_suite | simple_suite suite <<= condense(colon + nocolon_suite) + line = newline | stmt single_input = condense(Optional(line) - ZeroOrMore(newline)) diff --git a/coconut/compiler/util.py b/coconut/compiler/util.py index 3270c422f..05c0365e5 100644 --- a/coconut/compiler/util.py +++ b/coconut/compiler/util.py @@ -131,6 +131,7 @@ use_fast_pyparsing_reprs, save_new_cache_items, cache_validation_info, + require_cache_clear_frac, ) from coconut.exceptions import ( CoconutException, @@ -717,20 +718,12 @@ def should_clear_cache(force=False): incremental_cache_limit is not None and len(ParserElement.packrat_cache) > incremental_cache_limit ): - if should_clear_cache.last_cache_clear_strat == "useless": - clear_strat = "second half" - else: - clear_strat = "useless" - should_clear_cache.last_cache_clear_strat = clear_strat - return clear_strat + return "smart" return False else: return True -should_clear_cache.last_cache_clear_strat = None - - def add_packrat_cache_items(new_items, clear_first=False): """Add the given items to the packrat cache.""" if clear_first: @@ -743,50 +736,58 @@ def add_packrat_cache_items(new_items, clear_first=False): ParserElement.packrat_cache.update(new_items) +def execute_clear_strat(clear_cache): + """Clear PyParsing cache using clear_cache.""" + orig_cache_len = None + if clear_cache is True: + # clear cache without resetting stats + ParserElement.packrat_cache.clear() + elif clear_cache == "smart": + orig_cache_len = execute_clear_strat("useless") + cleared_frac = (orig_cache_len - len(get_pyparsing_cache())) / orig_cache_len + if cleared_frac < require_cache_clear_frac: + logger.log("Packrat cache pruning using 'useless' strat failed; falling back to 'second half' strat.") + execute_clear_strat("second half") + else: + internal_assert(CPYPARSING, "unsupported clear_cache strategy", clear_cache) + cache = get_pyparsing_cache() + orig_cache_len = len(cache) + if clear_cache == "useless": + keys_to_del = [] + for lookup, value in cache.items(): + if not value[-1][0]: + keys_to_del.append(lookup) + for del_key in keys_to_del: + del cache[del_key] + elif clear_cache == "second half": + all_keys = tuple(cache.keys()) + for del_key in all_keys[len(all_keys) // 2: len(all_keys)]: + del cache[del_key] + else: + raise CoconutInternalException("invalid clear_cache strategy", clear_cache) + return orig_cache_len + + def clear_packrat_cache(force=False): """Clear the packrat cache if applicable. Very performance-sensitive for incremental parsing mode.""" clear_cache = should_clear_cache(force=force) - if clear_cache: if DEVELOP: start_time = get_clock_time() - - orig_cache_len = None - if clear_cache is True: - # clear cache without resetting stats - ParserElement.packrat_cache.clear() - else: - internal_assert(CPYPARSING, "unsupported clear_cache strategy", clear_cache) - cache = get_pyparsing_cache() - orig_cache_len = len(cache) - if clear_cache == "second half": - all_keys = tuple(cache.keys()) - for del_key in all_keys[len(all_keys) // 2: len(all_keys)]: - del cache[del_key] - elif clear_cache == "useless": - keys_to_del = [] - for lookup, value in cache.items(): - if not value[-1][0]: - keys_to_del.append(lookup) - for del_key in keys_to_del: - del cache[del_key] - else: - raise CoconutInternalException("invalid clear_cache strategy", clear_cache) - + orig_cache_len = execute_clear_strat(clear_cache) if DEVELOP and orig_cache_len is not None: - logger.log("Pruned packrat cache from {orig_len} items to {new_len} items using {strat} strategy ({time} secs).".format( + logger.log("Pruned packrat cache from {orig_len} items to {new_len} items using {strat!r} strategy ({time} secs).".format( orig_len=orig_cache_len, new_len=len(get_pyparsing_cache()), strat=clear_cache, time=get_clock_time() - start_time, )) - return clear_cache def get_cache_items_for(original, only_useful=False, exclude_stale=True): - """Get items from the pyparsing cache filtered to only from parsing original.""" + """Get items from the pyparsing cache filtered to only be from parsing original.""" cache = get_pyparsing_cache() for lookup, value in cache.items(): got_orig = lookup[1] @@ -864,7 +865,7 @@ def pickle_cache(original, cache_path, include_incremental=True, protocol=pickle all_adaptive_stats = {} for wkref in MatchAny.all_match_anys: match_any = wkref() - if match_any is not None: + if match_any is not None and match_any.adaptive_usage is not None: identifier = match_any.parse_element_index internal_assert(lambda: match_any == all_parse_elements[identifier](), "failed to look up match_any by identifier", (match_any, all_parse_elements[identifier]())) if validation_dict is not None: @@ -917,12 +918,13 @@ def unpickle_cache(cache_path): all_adaptive_stats = pickle_info_obj["all_adaptive_stats"] for identifier, (adaptive_usage, expr_order) in all_adaptive_stats.items(): - maybe_elem = all_parse_elements[identifier]() - if maybe_elem is not None: - if validation_dict is not None: - internal_assert(maybe_elem.__class__.__name__ == validation_dict[identifier], "adaptive cache pickle-unpickle inconsistency", (maybe_elem, validation_dict[identifier])) - maybe_elem.adaptive_usage = adaptive_usage - maybe_elem.expr_order = expr_order + if identifier < len(all_parse_elements): + maybe_elem = all_parse_elements[identifier]() + if maybe_elem is not None: + if validation_dict is not None: + internal_assert(maybe_elem.__class__.__name__ == validation_dict[identifier], "adaptive cache pickle-unpickle inconsistency", (maybe_elem, validation_dict[identifier])) + maybe_elem.adaptive_usage = adaptive_usage + maybe_elem.expr_order = expr_order max_cache_size = min( incremental_mode_cache_size or float("inf"), @@ -934,15 +936,16 @@ def unpickle_cache(cache_path): new_cache_items = [] for pickleable_lookup, value in pickleable_cache_items: identifier = pickleable_lookup[0] - maybe_elem = all_parse_elements[identifier]() - if maybe_elem is not None: - if validation_dict is not None: - internal_assert(maybe_elem.__class__.__name__ == validation_dict[identifier], "incremental cache pickle-unpickle inconsistency", (maybe_elem, validation_dict[identifier])) - lookup = (maybe_elem,) + pickleable_lookup[1:] - usefullness = value[-1][0] - internal_assert(usefullness, "loaded useless cache item", (lookup, value)) - stale_value = value[:-1] + ([usefullness + 1],) - new_cache_items.append((lookup, stale_value)) + if identifier < len(all_parse_elements): + maybe_elem = all_parse_elements[identifier]() + if maybe_elem is not None: + if validation_dict is not None: + internal_assert(maybe_elem.__class__.__name__ == validation_dict[identifier], "incremental cache pickle-unpickle inconsistency", (maybe_elem, validation_dict[identifier])) + lookup = (maybe_elem,) + pickleable_lookup[1:] + usefullness = value[-1][0] + internal_assert(usefullness, "loaded useless cache item", (lookup, value)) + stale_value = value[:-1] + ([usefullness + 1],) + new_cache_items.append((lookup, stale_value)) add_packrat_cache_items(new_cache_items) num_inc = len(pickleable_cache_items) @@ -950,10 +953,11 @@ def unpickle_cache(cache_path): return num_inc, num_adapt -def load_cache_for(inputstring, filename, cache_path): +def load_cache_for(inputstring, codepath): """Load cache_path (for the given inputstring and filename).""" if not SUPPORTS_INCREMENTAL: - raise CoconutException("incremental parsing mode requires cPyparsing (run '{python} -m pip install --upgrade cPyparsing' to fix)".format(python=sys.executable)) + raise CoconutException("the parsing cache requires cPyparsing (run '{python} -m pip install --upgrade cPyparsing' to fix)".format(python=sys.executable)) + filename = os.path.basename(codepath) if in_incremental_mode(): incremental_enabled = True @@ -974,23 +978,35 @@ def load_cache_for(inputstring, filename, cache_path): max_len=disable_incremental_for_len, ) - did_load_cache = unpickle_cache(cache_path) - if did_load_cache: - num_inc, num_adapt = did_load_cache - logger.log("Loaded {num_inc} incremental and {num_adapt} adaptive cache items for {filename!r} ({incremental_info}).".format( - num_inc=num_inc, - num_adapt=num_adapt, - filename=filename, - incremental_info=incremental_info, - )) + if ( + incremental_enabled + or use_adaptive_any_of + or use_adaptive_if_available + ): + cache_path = get_cache_path(codepath) + did_load_cache = unpickle_cache(cache_path) + if did_load_cache: + num_inc, num_adapt = did_load_cache + logger.log("Loaded {num_inc} incremental and {num_adapt} adaptive cache items for {filename!r} ({incremental_info}).".format( + num_inc=num_inc, + num_adapt=num_adapt, + filename=filename, + incremental_info=incremental_info, + )) + else: + logger.log("Failed to load cache for {filename!r} from {cache_path!r} ({incremental_info}).".format( + filename=filename, + cache_path=cache_path, + incremental_info=incremental_info, + )) else: - logger.log("Failed to load cache for {filename!r} from {cache_path!r} ({incremental_info}).".format( + cache_path = None + logger.log("Declined to load cache for {filename!r} ({incremental_info}).".format( filename=filename, - cache_path=cache_path, incremental_info=incremental_info, )) - return incremental_enabled + return cache_path, incremental_enabled def get_cache_path(codepath): @@ -1023,6 +1039,12 @@ def __or__(self, other): else: return MatchFirst([self, other]) + @override + def copy(self): + self = super(MatchAny, self).copy() + self.all_match_anys.append(weakref.ref(self)) + return self + if not use_fast_pyparsing_reprs: def __str__(self): return self.__class__.__name__ + ":" + super(MatchAny, self).__str__() diff --git a/coconut/constants.py b/coconut/constants.py index ed8ef69bb..bebceea91 100644 --- a/coconut/constants.py +++ b/coconut/constants.py @@ -105,7 +105,7 @@ def get_path_env_var(env_var, default): # ----------------------------------------------------------------------------------------------------------------------- # set this to False only ever temporarily for ease of debugging -use_fast_pyparsing_reprs = False +use_fast_pyparsing_reprs = True assert use_fast_pyparsing_reprs or DEVELOP, "use_fast_pyparsing_reprs should never be disabled on non-develop build" enable_pyparsing_warnings = DEVELOP @@ -130,10 +130,11 @@ def get_path_env_var(env_var, default): streamline_grammar_for_len = 1536 -disable_incremental_for_len = float("inf") # always use - use_cache_file = True -use_adaptive_any_of = get_bool_env_var("COCONUT_ADAPTIVE_ANY_OF", True) +disable_incremental_for_len = 45875 +# this is disabled by default for now because it doesn't improve performance +# by very much but is very hard to test, so it's hard to be confident in it +use_adaptive_any_of = get_bool_env_var("COCONUT_ADAPTIVE_ANY_OF", False) # note that _parseIncremental produces much smaller caches use_incremental_if_available = False @@ -150,6 +151,7 @@ def get_path_env_var(env_var, default): incremental_mode_cache_size = None incremental_cache_limit = 2097152 # clear cache when it gets this large incremental_mode_cache_successes = False +require_cache_clear_frac = 0.25 # require that at least this much of the cache must be cleared on each cache clear use_left_recursion_if_available = False diff --git a/coconut/root.py b/coconut/root.py index 932bfb04a..e0d21ddb9 100644 --- a/coconut/root.py +++ b/coconut/root.py @@ -26,7 +26,7 @@ VERSION = "3.0.3" VERSION_NAME = None # False for release, int >= 1 for develop -DEVELOP = 31 +DEVELOP = 32 ALPHA = False # for pre releases rather than post releases assert DEVELOP is False or DEVELOP >= 1, "DEVELOP must be False or an int >= 1" From 71c26590791249dd4e9d52a3003f780a6f8521af Mon Sep 17 00:00:00 2001 From: Evan Hubinger Date: Fri, 17 Nov 2023 00:54:27 -0800 Subject: [PATCH 091/121] Fix dependencies --- coconut/constants.py | 13 ++++++++----- 1 file changed, 8 insertions(+), 5 deletions(-) diff --git a/coconut/constants.py b/coconut/constants.py index bebceea91..45113ba35 100644 --- a/coconut/constants.py +++ b/coconut/constants.py @@ -81,6 +81,7 @@ def get_path_env_var(env_var, default): PY39 = sys.version_info >= (3, 9) PY310 = sys.version_info >= (3, 10) PY311 = sys.version_info >= (3, 11) +PY312 = sys.version_info >= (3, 12) IPY = ( PY35 and (PY37 or not PYPY) @@ -91,6 +92,8 @@ def get_path_env_var(env_var, default): PY38 and not WINDOWS and not PYPY + # disabled until MyPy supports PEP 695 + and not PY312 ) XONSH = ( PY35 @@ -997,7 +1000,7 @@ def get_path_env_var(env_var, default): # min versions are inclusive unpinned_min_versions = { - "cPyparsing": (2, 4, 7, 2, 2, 8), + "cPyparsing": (2, 4, 7, 2, 2, 9), ("pre-commit", "py3"): (3,), ("psutil", "py>=27"): (5,), "jupyter": (1, 0), @@ -1014,7 +1017,7 @@ def get_path_env_var(env_var, default): "pydata-sphinx-theme": (0, 14), "myst-parser": (2,), "sphinx": (7,), - "mypy[python2]": (1, 6), + "mypy[python2]": (1, 7), ("jupyter-console", "py37"): (6, 6), ("typing", "py<35"): (3, 10), ("typing_extensions", "py>=38"): (4, 8), @@ -1025,9 +1028,8 @@ def get_path_env_var(env_var, default): ("pytest", "py36"): (7,), ("async_generator", "py35"): (1, 10), ("exceptiongroup", "py37;py<311"): (1,), - ("ipython", "py>=39"): (8, 16), + ("ipython", "py>=39"): (8, 17), "py-spy": (0, 3), - ("anyio", "py36"): (3,), } pinned_min_versions = { @@ -1039,6 +1041,7 @@ def get_path_env_var(env_var, default): ("ipython", "py==37"): (7, 34), ("typing_extensions", "py==37"): (4, 7), # don't upgrade these; they break on Python 3.6 + ("anyio", "py36"): (3,), ("xonsh", "py>=36;py<38"): (0, 11), ("pandas", "py36"): (1,), ("jupyter-client", "py36"): (7, 1, 2), @@ -1088,7 +1091,7 @@ def get_path_env_var(env_var, default): max_versions = { ("jupyter-client", "py==35"): _, "pyparsing": _, - "cPyparsing": (_, _, _), + "cPyparsing": (_, _, _, _, _,), ("prompt_toolkit", "py<3"): _, ("jedi", "py<39"): _, ("pywinpty", "py<3;windows"): _, From a14a8292a99d61e8f8de9c6c96f0872ee82ab381 Mon Sep 17 00:00:00 2001 From: Evan Hubinger Date: Fri, 17 Nov 2023 02:28:24 -0800 Subject: [PATCH 092/121] Fix tests --- coconut/_pyparsing.py | 47 +++++++++++++------ coconut/compiler/util.py | 17 +++++-- coconut/constants.py | 8 ++-- coconut/requirements.py | 2 +- coconut/root.py | 2 +- coconut/tests/constants_test.py | 3 ++ coconut/tests/main_test.py | 6 ++- .../tests/src/cocotest/agnostic/specific.coco | 4 +- 8 files changed, 61 insertions(+), 28 deletions(-) diff --git a/coconut/_pyparsing.py b/coconut/_pyparsing.py index ce6aa061d..1c7344735 100644 --- a/coconut/_pyparsing.py +++ b/coconut/_pyparsing.py @@ -37,6 +37,7 @@ default_whitespace_chars, varchars, min_versions, + max_versions, pure_python_env_var, enable_pyparsing_warnings, use_left_recursion_if_available, @@ -92,32 +93,48 @@ PYPARSING_PACKAGE = "cPyparsing" if CPYPARSING else "pyparsing" -min_ver = min(min_versions["pyparsing"], min_versions["cPyparsing"][:3]) # inclusive -max_ver = get_next_version(max(min_versions["pyparsing"], min_versions["cPyparsing"][:3])) # exclusive -cur_ver = None if __version__ is None else ver_str_to_tuple(__version__) +if CPYPARSING: + min_ver = min_versions["cPyparsing"] # inclusive + max_ver = get_next_version(min_versions["cPyparsing"], point_to_increment=len(max_versions["cPyparsing"]) - 1) # exclusive +else: + min_ver = min_versions["pyparsing"] # inclusive + max_ver = get_next_version(min_versions["pyparsing"]) # exclusive -min_ver_str = ver_tuple_to_str(min_ver) -max_ver_str = ver_tuple_to_str(max_ver) +cur_ver = None if __version__ is None else ver_str_to_tuple(__version__) if cur_ver is None or cur_ver < min_ver: raise ImportError( - "This version of Coconut requires pyparsing/cPyparsing version >= " + min_ver_str - + ("; got " + PYPARSING_INFO if PYPARSING_INFO is not None else "") - + " (run '{python} -m pip install --upgrade {package}' to fix)".format(python=sys.executable, package=PYPARSING_PACKAGE), + ( + "This version of Coconut requires {package} version >= {min_ver}" + + ("; got " + PYPARSING_INFO if PYPARSING_INFO is not None else "") + + " (run '{python} -m pip install --upgrade {package}' to fix)" + ).format( + python=sys.executable, + package=PYPARSING_PACKAGE, + min_ver=ver_tuple_to_str(min_ver), + ) ) elif cur_ver >= max_ver: warn( - "This version of Coconut was built for pyparsing/cPyparsing versions < " + max_ver_str - + ("; got " + PYPARSING_INFO if PYPARSING_INFO is not None else "") - + " (run '{python} -m pip install {package}<{max_ver}' to fix)".format(python=sys.executable, package=PYPARSING_PACKAGE, max_ver=max_ver_str), + ( + "This version of Coconut was built for {package} versions < {max_ver}" + + ("; got " + PYPARSING_INFO if PYPARSING_INFO is not None else "") + + " (run '{python} -m pip install {package}<{max_ver}' to fix)" + ).format( + python=sys.executable, + package=PYPARSING_PACKAGE, + max_ver=ver_tuple_to_str(max_ver), + ) ) MODERN_PYPARSING = cur_ver >= (3,) if MODERN_PYPARSING: warn( - "This version of Coconut is not built for pyparsing v3; some syntax features WILL NOT WORK" - + " (run either '{python} -m pip install cPyparsing<{max_ver}' or '{python} -m pip install pyparsing<{max_ver}' to fix)".format(python=sys.executable, max_ver=max_ver_str), + "This version of Coconut is not built for pyparsing v3; some syntax features WILL NOT WORK (run either '{python} -m pip install cPyparsing<{max_ver}' or '{python} -m pip install pyparsing<{max_ver}' to fix)".format( + python=sys.executable, + max_ver=ver_tuple_to_str(max_ver), + ) ) @@ -164,7 +181,6 @@ def _parseCache(self, instring, loc, doActions=True, callPreParse=True): raise value return value[0], value[1].copy() - ParserElement.packrat_context = frozenset() ParserElement._parseCache = _parseCache # [CPYPARSING] fix append @@ -197,6 +213,9 @@ def append(self, other): return self ParseExpression.append = append +if SUPPORTS_PACKRAT_CONTEXT: + ParserElement.packrat_context = frozenset() + if hasattr(ParserElement, "enableIncremental"): SUPPORTS_INCREMENTAL = sys.version_info >= (3, 8) # avoids stack overflows on py<=37 else: diff --git a/coconut/compiler/util.py b/coconut/compiler/util.py index 05c0365e5..33a861343 100644 --- a/coconut/compiler/util.py +++ b/coconut/compiler/util.py @@ -885,11 +885,17 @@ def pickle_cache(original, cache_path, include_incremental=True, protocol=pickle "pickleable_cache_items": pickleable_cache_items, "all_adaptive_stats": all_adaptive_stats, } - with univ_open(cache_path, "wb") as pickle_file: - pickle.dump(pickle_info_obj, pickle_file, protocol=protocol) - - # clear the packrat cache when we're done so we don't interfere with anything else happening in this process - clear_packrat_cache(force=True) + try: + with univ_open(cache_path, "wb") as pickle_file: + pickle.dump(pickle_info_obj, pickle_file, protocol=protocol) + except Exception: + logger.log_exc() + return False + else: + return True + finally: + # clear the packrat cache when we're done so we don't interfere with anything else happening in this process + clear_packrat_cache(force=True) def unpickle_cache(cache_path): @@ -979,6 +985,7 @@ def load_cache_for(inputstring, codepath): ) if ( + # only load the cache if we're using anything that makes use of it incremental_enabled or use_adaptive_any_of or use_adaptive_if_available diff --git a/coconut/constants.py b/coconut/constants.py index 45113ba35..a581e5fd6 100644 --- a/coconut/constants.py +++ b/coconut/constants.py @@ -134,7 +134,7 @@ def get_path_env_var(env_var, default): streamline_grammar_for_len = 1536 use_cache_file = True -disable_incremental_for_len = 45875 +disable_incremental_for_len = 46080 # this is disabled by default for now because it doesn't improve performance # by very much but is very hard to test, so it's hard to be confident in it use_adaptive_any_of = get_bool_env_var("COCONUT_ADAPTIVE_ANY_OF", False) @@ -154,7 +154,7 @@ def get_path_env_var(env_var, default): incremental_mode_cache_size = None incremental_cache_limit = 2097152 # clear cache when it gets this large incremental_mode_cache_successes = False -require_cache_clear_frac = 0.25 # require that at least this much of the cache must be cleared on each cache clear +require_cache_clear_frac = 0.3125 # require that at least this much of the cache must be cleared on each cache clear use_left_recursion_if_available = False @@ -483,8 +483,7 @@ def get_path_env_var(env_var, default): "itertools.zip_longest": ("itertools./izip_longest", (3,)), "math.gcd": ("fractions./gcd", (3, 5)), "time.process_time": ("time./clock", (3, 3)), - # # _dummy_thread was removed in Python 3.9, so this no longer works - # "_dummy_thread": ("dummy_thread", (3,)), + "_dummy_thread": ("dummy_thread", (3,)), # third-party backports "asyncio": ("trollius", (3, 4)), @@ -1125,6 +1124,7 @@ def get_path_env_var(env_var, default): "Programming Language :: Python :: 3.9", "Programming Language :: Python :: 3.10", "Programming Language :: Python :: 3.11", + "Programming Language :: Python :: 3.12", "Programming Language :: Other", "Programming Language :: Other Scripting Engines", "Programming Language :: Python :: Implementation :: CPython", diff --git a/coconut/requirements.py b/coconut/requirements.py index 3035c8440..56b92cee7 100644 --- a/coconut/requirements.py +++ b/coconut/requirements.py @@ -130,7 +130,7 @@ def get_req_str(req): max_ver = get_next_version(min_versions[req]) if None in max_ver: assert all(v is None for v in max_ver), "invalid max version " + repr(max_ver) - max_ver = get_next_version(min_versions[req], len(max_ver) - 1) + max_ver = get_next_version(min_versions[req], point_to_increment=len(max_ver) - 1) req_str += ",<" + ver_tuple_to_str(max_ver) return req_str diff --git a/coconut/root.py b/coconut/root.py index e0d21ddb9..d5c3fe2a9 100644 --- a/coconut/root.py +++ b/coconut/root.py @@ -26,7 +26,7 @@ VERSION = "3.0.3" VERSION_NAME = None # False for release, int >= 1 for develop -DEVELOP = 32 +DEVELOP = 33 ALPHA = False # for pre releases rather than post releases assert DEVELOP is False or DEVELOP >= 1, "DEVELOP must be False or an int >= 1" diff --git a/coconut/tests/constants_test.py b/coconut/tests/constants_test.py index 65ae8beea..fc32ed6c8 100644 --- a/coconut/tests/constants_test.py +++ b/coconut/tests/constants_test.py @@ -31,6 +31,7 @@ from coconut.constants import ( WINDOWS, PYPY, + PY39, fixpath, ) @@ -100,6 +101,8 @@ def test_imports(self): or PYPY and old_imp in ("trollius", "aenum") # don't test typing_extensions, async_generator or old_imp.startswith(("typing_extensions", "async_generator")) + # don't test _dummy_thread on Py3.9 + or PY39 and new_imp == "_dummy_thread" ): pass elif sys.version_info >= ver_cutoff: diff --git a/coconut/tests/main_test.py b/coconut/tests/main_test.py index 20916c79e..3804f1fc8 100644 --- a/coconut/tests/main_test.py +++ b/coconut/tests/main_test.py @@ -31,7 +31,6 @@ import imp import pytest -import pexpect from coconut.util import noop_ctx, get_target_info from coconut.terminal import ( @@ -64,6 +63,8 @@ get_bool_env_var, coconut_cache_dir, default_use_cache_dir, + base_dir, + fixpath, ) from coconut.api import ( @@ -412,6 +413,8 @@ def comp(path=None, folder=None, file=None, args=[], **kwargs): def rm_path(path, allow_keep=False): """Delete a path.""" + path = os.path.abspath(fixpath(path)) + assert not base_dir.startswith(path), "refusing to delete Coconut itself: " + repr(path) if allow_keep and get_bool_env_var("COCONUT_KEEP_TEST_FILES"): return if os.path.isdir(path): @@ -536,6 +539,7 @@ def add_test_func_names(cls): def spawn_cmd(cmd): """Version of pexpect.spawn that prints the command being run.""" + import pexpect # hide import since not always available print("\n>", cmd) return pexpect.spawn(cmd) diff --git a/coconut/tests/src/cocotest/agnostic/specific.coco b/coconut/tests/src/cocotest/agnostic/specific.coco index cbb1eefbe..e35bb7aad 100644 --- a/coconut/tests/src/cocotest/agnostic/specific.coco +++ b/coconut/tests/src/cocotest/agnostic/specific.coco @@ -164,7 +164,7 @@ def py36_spec_test(tco: bool) -> bool: def py37_spec_test() -> bool: """Tests for any py37+ version.""" - import asyncio, typing + import asyncio, typing, typing_extensions assert py_breakpoint # type: ignore ns: typing.Dict[str, typing.Any] = {} exec("""async def toa(it): @@ -180,7 +180,7 @@ def py37_spec_test() -> bool: assert l == list(range(10)) class HasVarGen[*Ts] # type: ignore assert HasVarGen `issubclass` object - assert typing.Protocol.__module__ == "typing_extensions" + assert typing.Protocol is typing_extensions.Protocol assert_raises((def -> raise ExceptionGroup("derp", [Exception("herp")])), ExceptionGroup) assert_raises((def -> raise BaseExceptionGroup("derp", [BaseException("herp")])), BaseExceptionGroup) return True From 459bf34ad142c467f8f209cdede4f413aba9bb85 Mon Sep 17 00:00:00 2001 From: Evan Hubinger Date: Fri, 17 Nov 2023 02:36:49 -0800 Subject: [PATCH 093/121] More robustification --- coconut/_pyparsing.py | 5 ++++- coconut/compiler/util.py | 2 +- coconut/requirements.py | 5 ++++- coconut/terminal.py | 6 +++++- 4 files changed, 14 insertions(+), 4 deletions(-) diff --git a/coconut/_pyparsing.py b/coconut/_pyparsing.py index 1c7344735..57b99c367 100644 --- a/coconut/_pyparsing.py +++ b/coconut/_pyparsing.py @@ -95,7 +95,10 @@ if CPYPARSING: min_ver = min_versions["cPyparsing"] # inclusive - max_ver = get_next_version(min_versions["cPyparsing"], point_to_increment=len(max_versions["cPyparsing"]) - 1) # exclusive + max_ver = get_next_version( + min_versions["cPyparsing"], + point_to_increment=len(max_versions["cPyparsing"]) - 1, + ) # exclusive else: min_ver = min_versions["pyparsing"] # inclusive max_ver = get_next_version(min_versions["pyparsing"]) # exclusive diff --git a/coconut/compiler/util.py b/coconut/compiler/util.py index 33a861343..f45059fc0 100644 --- a/coconut/compiler/util.py +++ b/coconut/compiler/util.py @@ -889,7 +889,7 @@ def pickle_cache(original, cache_path, include_incremental=True, protocol=pickle with univ_open(cache_path, "wb") as pickle_file: pickle.dump(pickle_info_obj, pickle_file, protocol=protocol) except Exception: - logger.log_exc() + logger.warn_exc() return False else: return True diff --git a/coconut/requirements.py b/coconut/requirements.py index 56b92cee7..05be7b6d4 100644 --- a/coconut/requirements.py +++ b/coconut/requirements.py @@ -130,7 +130,10 @@ def get_req_str(req): max_ver = get_next_version(min_versions[req]) if None in max_ver: assert all(v is None for v in max_ver), "invalid max version " + repr(max_ver) - max_ver = get_next_version(min_versions[req], point_to_increment=len(max_ver) - 1) + max_ver = get_next_version( + min_versions[req], + point_to_increment=len(max_ver) - 1, + ) req_str += ",<" + ver_tuple_to_str(max_ver) return req_str diff --git a/coconut/terminal.py b/coconut/terminal.py index 9564ad982..458a3ae00 100644 --- a/coconut/terminal.py +++ b/coconut/terminal.py @@ -417,7 +417,7 @@ def warn_err(self, warning, force=False): try: raise warning except Exception: - self.print_exc(warning=True) + self.warn_exc() def log_warn(self, *args, **kwargs): """Log a warning.""" @@ -428,6 +428,10 @@ def print_exc(self, err=None, show_tb=None, warning=False): """Properly prints an exception.""" self.print_formatted_error(self.get_error(err, show_tb), warning) + def warn_exc(self, err=None): + """Warn about the current or given exception.""" + self.print_exc(err, warning=True) + def print_exception(self, err_type, err_value, err_tb): """Properly prints the given exception details.""" self.print_formatted_error(format_error(err_value, err_type, err_tb)) From 0ab1785ee7714c0bd8b6b5b1c56a3a4e4532eb24 Mon Sep 17 00:00:00 2001 From: Evan Hubinger Date: Sun, 19 Nov 2023 18:46:02 -0800 Subject: [PATCH 094/121] Fix tests --- Makefile | 14 ++++++++++---- coconut/compiler/util.py | 2 ++ coconut/tests/src/cocotest/agnostic/primary_1.coco | 2 +- 3 files changed, 13 insertions(+), 5 deletions(-) diff --git a/Makefile b/Makefile index c4fe8177d..d9b65f0e9 100644 --- a/Makefile +++ b/Makefile @@ -251,18 +251,24 @@ just-watch-verbose: export COCONUT_USE_COLOR=TRUE just-watch-verbose: coconut ./coconut/tests/src/cocotest/agnostic ./coconut/tests/dest/cocotest --watch --strict --keep-lines --stack-size 4096 --recursion-limit 4096 --verbose --jobs 0 --no-cache -# mini test that just compiles agnostic tests with verbose output +# mini test that just compiles agnostic tests .PHONY: test-mini test-mini: export COCONUT_USE_COLOR=TRUE test-mini: + coconut ./coconut/tests/src/cocotest/agnostic ./coconut/tests/dest/cocotest --force --stack-size 4096 --recursion-limit 4096 + +# same as test-mini but with verbose output +.PHONY: test-mini-verbose +test-mini-verbose: export COCONUT_USE_COLOR=TRUE +test-mini-verbose: coconut ./coconut/tests/src/cocotest/agnostic ./coconut/tests/dest/cocotest --force --verbose --stack-size 4096 --recursion-limit 4096 -# same as test-mini but doesn't overwrite the cache +# same as test-mini-verbose but doesn't overwrite the cache .PHONY: test-mini-cache test-mini-cache: export COCONUT_ALLOW_SAVE_TO_CACHE=FALSE -test-mini-cache: test-mini +test-mini-cache: test-mini-verbose -# same as test-mini but with fully synchronous output and fast failing +# same as test-mini-verbose but with fully synchronous output and fast failing .PHONY: test-mini-sync test-mini-sync: export COCONUT_USE_COLOR=TRUE test-mini-sync: diff --git a/coconut/compiler/util.py b/coconut/compiler/util.py index f45059fc0..19c7c811e 100644 --- a/coconut/compiler/util.py +++ b/coconut/compiler/util.py @@ -1006,6 +1006,8 @@ def load_cache_for(inputstring, codepath): cache_path=cache_path, incremental_info=incremental_info, )) + if incremental_enabled: + logger.warn("Populating initial parsing cache (compilation may take longer than usual)...") else: cache_path = None logger.log("Declined to load cache for {filename!r} ({incremental_info}).".format( diff --git a/coconut/tests/src/cocotest/agnostic/primary_1.coco b/coconut/tests/src/cocotest/agnostic/primary_1.coco index 7418c4e89..bc85179c7 100644 --- a/coconut/tests/src/cocotest/agnostic/primary_1.coco +++ b/coconut/tests/src/cocotest/agnostic/primary_1.coco @@ -952,7 +952,7 @@ def primary_test_1() -> bool: assert (a=1, b=2) == (1, 2) == (a:int=1, b=2) assert (a=1, b: int = 2) == (1, 2) == (a: int=1, b: int=2) assert "_namedtuple_of" in repr((a=1,)) - assert "b=2" in repr <| call$(?, a=1, b=2) + assert "b=2" in repr <| call$(?, 1, b=2) assert lift((,), (.*2), (.**2))(3) == (6, 9) assert_raises(-> (⁻)(1, 2), TypeError) assert -1 == ⁻1 From 3c2878519345d01379344dec761d1c73050fd315 Mon Sep 17 00:00:00 2001 From: Evan Hubinger Date: Sun, 19 Nov 2023 21:25:39 -0800 Subject: [PATCH 095/121] Fix py2 tests --- coconut/tests/src/cocotest/agnostic/primary_2.coco | 6 ------ coconut/tests/src/cocotest/agnostic/specific.coco | 6 ++++++ 2 files changed, 6 insertions(+), 6 deletions(-) diff --git a/coconut/tests/src/cocotest/agnostic/primary_2.coco b/coconut/tests/src/cocotest/agnostic/primary_2.coco index 0bc92d989..435cfbf88 100644 --- a/coconut/tests/src/cocotest/agnostic/primary_2.coco +++ b/coconut/tests/src/cocotest/agnostic/primary_2.coco @@ -381,12 +381,6 @@ def primary_test_2() -> bool: ]: # type: ignore assert list(xs) == list(zip(range(5), range(5))) assert list(xs) == [] - for xs in [ - py_zip((x for x in range(5)), (x for x in range(10))), - py_map((,), (x for x in range(5)), (x for x in range(10))), - ]: # type: ignore - assert list(xs) == list(zip(range(5), range(5))) - assert list(xs) == [] if sys.version_info >= (3,) else list(zip(range(5), range(5))) xs = map((.+1), range(5)) py_xs = py_map((.+1), range(5)) assert list(xs) == list(range(1, 6)) == list(xs) diff --git a/coconut/tests/src/cocotest/agnostic/specific.coco b/coconut/tests/src/cocotest/agnostic/specific.coco index e35bb7aad..82311206e 100644 --- a/coconut/tests/src/cocotest/agnostic/specific.coco +++ b/coconut/tests/src/cocotest/agnostic/specific.coco @@ -41,6 +41,12 @@ def py3_spec_test() -> bool: assert Outer.Inner.f(2) == 2 assert Outer.Inner.f.__name__ == "f" assert Outer.Inner.f.__qualname__.endswith("Outer.Inner.f"), Outer.Inner.f.__qualname__ + for xs in [ + py_zip((x for x in range(5)), (x for x in range(10))), + py_map((,), (x for x in range(5)), (x for x in range(10))), + ]: # type: ignore + assert list(xs) == list(zip(range(5), range(5))) + assert list(xs) == [] if sys.version_info >= (3,) else list(zip(range(5), range(5))) return True From 0110d6169b2b3125b5201a2bd54916dabb619dd0 Mon Sep 17 00:00:00 2001 From: Evan Hubinger Date: Sun, 19 Nov 2023 22:35:26 -0800 Subject: [PATCH 096/121] Improve command running --- Makefile | 5 ++++ coconut/command/util.py | 60 +++++++++++++++++++++++++++++--------- coconut/compiler/util.py | 5 ++++ coconut/constants.py | 7 +++-- coconut/terminal.py | 14 +++++---- coconut/tests/main_test.py | 4 +-- 6 files changed, 70 insertions(+), 25 deletions(-) diff --git a/Makefile b/Makefile index d9b65f0e9..80d436667 100644 --- a/Makefile +++ b/Makefile @@ -131,6 +131,11 @@ test-pypy3: clean pypy3 ./coconut/tests/dest/runner.py pypy3 ./coconut/tests/dest/extras.py +# same as test-univ but reverses any ofs +.PHONY: test-any-of +test-any-of: export COCONUT_REVERSE_ANY_OF=TRUE +test-any-of: test-univ + # same as test-univ but also runs mypy .PHONY: test-mypy-univ test-mypy-univ: export COCONUT_USE_COLOR=TRUE diff --git a/coconut/command/util.py b/coconut/command/util.py index 3750b34bc..a9b1c1883 100644 --- a/coconut/command/util.py +++ b/coconut/command/util.py @@ -24,6 +24,7 @@ import subprocess import shutil import threading +import queue from select import select from contextlib import contextmanager from functools import partial @@ -82,6 +83,7 @@ min_stack_size_kbs, coconut_base_run_args, high_proc_prio, + call_timeout, ) if PY26: @@ -265,28 +267,58 @@ def run_file(path): return runpy.run_path(path, run_name="__main__") -def call_output(cmd, stdin=None, encoding_errors="replace", **kwargs): +def readline_to_queue(file_obj, q): + """Read a line from file_obj and put it in the queue.""" + q.put(file_obj.readline()) + + +def call_output(cmd, stdin=None, encoding_errors="replace", color=None, **kwargs): """Run command and read output.""" - p = subprocess.Popen(cmd, stdout=subprocess.PIPE, stderr=subprocess.PIPE, **kwargs) + p = subprocess.Popen(cmd, stdout=subprocess.PIPE, stderr=subprocess.PIPE, stdin=subprocess.PIPE, **kwargs) + + if stdin is not None: + logger.log_prefix("STDIN < ", stdin.rstrip()) + p.stdin.write(stdin) + + stdout_q = queue.Queue() + stderr_q = queue.Queue() + + stdout_t = stderr_t = None + stdout, stderr, retcode = [], [], None while retcode is None: - if stdin is not None: - logger.log_prefix("STDIN < ", stdin.rstrip()) - raw_out, raw_err = p.communicate(stdin) - stdin = None + if stdout_t is None or not stdout_t.is_alive(): + stdout_t = threading.Thread(target=readline_to_queue, args=(p.stdout, stdout_q)) + stdout_t.start() + if stderr_t is None or not stderr_t.is_alive(): + stderr_t = threading.Thread(target=readline_to_queue, args=(p.stderr, stderr_q)) + stderr_t.start() - out = raw_out.decode(get_encoding(sys.stdout), encoding_errors) if raw_out else "" - if out: - logger.log_stdout(out.rstrip()) - stdout.append(out) + stdout_t.join(timeout=call_timeout) + stderr_t.join(timeout=call_timeout) + try: + raw_out = stdout_q.get(block=False) + except queue.Empty: + raw_out = None + try: + raw_err = stderr_q.get(block=False) + except queue.Empty: + raw_err = None + + out = raw_out.decode(get_encoding(sys.stdout), encoding_errors) if raw_out else "" err = raw_err.decode(get_encoding(sys.stderr), encoding_errors) if raw_err else "" + + if out: + logger.log_stdout(out, color=color, end="") + stdout.append(out) if err: - logger.log(err.rstrip()) - stderr.append(err) + logger.log(err, color=color, end="") + stderr.append(err) retcode = p.poll() - return stdout, stderr, retcode + + return "".join(stdout), "".join(stderr), retcode def run_cmd(cmd, show_output=True, raise_errs=True, **kwargs): @@ -306,7 +338,7 @@ def run_cmd(cmd, show_output=True, raise_errs=True, **kwargs): return subprocess.call(cmd, **kwargs) else: stdout, stderr, retcode = call_output(cmd, **kwargs) - output = "".join(stdout + stderr) + output = stdout + stderr if retcode and raise_errs: raise subprocess.CalledProcessError(retcode, cmd, output=output) return output diff --git a/coconut/compiler/util.py b/coconut/compiler/util.py index 19c7c811e..9b90a03e7 100644 --- a/coconut/compiler/util.py +++ b/coconut/compiler/util.py @@ -132,6 +132,7 @@ save_new_cache_items, cache_validation_info, require_cache_clear_frac, + reverse_any_of, ) from coconut.exceptions import ( CoconutException, @@ -1076,6 +1077,10 @@ def any_of(*exprs, **kwargs): flat_exprs.extend(e.exprs) else: flat_exprs.append(e) + + if reverse_any_of: + flat_exprs.reverse() + return AnyOf(flat_exprs) diff --git a/coconut/constants.py b/coconut/constants.py index a581e5fd6..ca26e3c4a 100644 --- a/coconut/constants.py +++ b/coconut/constants.py @@ -126,6 +126,8 @@ def get_path_env_var(env_var, default): cache_validation_info = DEVELOP +reverse_any_of = get_bool_env_var("COCONUT_REVERSE_ANY_OF", False) + # below constants are experimentally determined to maximize performance use_packrat_parser = True # True also gives us better error messages @@ -135,8 +137,7 @@ def get_path_env_var(env_var, default): use_cache_file = True disable_incremental_for_len = 46080 -# this is disabled by default for now because it doesn't improve performance -# by very much but is very hard to test, so it's hard to be confident in it +# TODO: this is disabled by default until we get test-any-of to pass use_adaptive_any_of = get_bool_env_var("COCONUT_ADAPTIVE_ANY_OF", False) # note that _parseIncremental produces much smaller caches @@ -740,6 +741,8 @@ def get_path_env_var(env_var, default): create_package_retries = 1 +call_timeout = 0.01 + max_orig_lines_in_log_loc = 2 # ----------------------------------------------------------------------------------------------------------------------- diff --git a/coconut/terminal.py b/coconut/terminal.py index 458a3ae00..20f2358fd 100644 --- a/coconut/terminal.py +++ b/coconut/terminal.py @@ -255,10 +255,12 @@ def display( file = file or sys.stdout elif level == "logging": file = file or sys.stderr - color = color or log_color_code + if color is None: + color = log_color_code elif level == "error": file = file or sys.stderr - color = color or error_color_code + if color is None: + color = error_color_code else: raise CoconutInternalException("invalid logging level", level) @@ -316,15 +318,15 @@ def show_error(self, *messages, **kwargs): if not self.quiet: self.display(messages, main_sig, level="error", **kwargs) - def log(self, *messages): + def log(self, *messages, **kwargs): """Logs debug messages if --verbose.""" if self.verbose: - self.printlog(*messages) + self.printlog(*messages, **kwargs) - def log_stdout(self, *messages): + def log_stdout(self, *messages, **kwargs): """Logs debug messages to stdout if --verbose.""" if self.verbose: - self.print(*messages) + self.print(*messages, **kwargs) def log_lambda(self, *msg_funcs): if self.verbose: diff --git a/coconut/tests/main_test.py b/coconut/tests/main_test.py index 3804f1fc8..cd1da15c5 100644 --- a/coconut/tests/main_test.py +++ b/coconut/tests/main_test.py @@ -282,7 +282,7 @@ def call( with using_sys_path(module_dir): stdout, stderr, retcode = call_with_import(module_name, extra_argv) else: - stdout, stderr, retcode = call_output(raw_cmd, **kwargs) + stdout, stderr, retcode = call_output(raw_cmd, color=False, **kwargs) if expect_retcode is not None: assert retcode == expect_retcode, "Return code not as expected ({retcode} != {expect_retcode}) in: {cmd!r}".format( @@ -294,7 +294,6 @@ def call( out = stderr + stdout else: out = stdout + stderr - out = "".join(out) raw_lines = out.splitlines() lines = [] @@ -897,7 +896,6 @@ def test_ipython_extension(self): def test_kernel_installation(self): call(["coconut", "--jupyter"], assert_output=kernel_installation_msg) stdout, stderr, retcode = call_output(["jupyter", "kernelspec", "list"]) - stdout, stderr = "".join(stdout), "".join(stderr) if not stdout: stdout, stderr = stderr, "" assert not retcode and not stderr, stderr From ea531c29458ccb21b5ad153e65fdc898c6fb50aa Mon Sep 17 00:00:00 2001 From: Evan Hubinger Date: Tue, 21 Nov 2023 22:09:08 -0800 Subject: [PATCH 097/121] Improve call_output --- Makefile | 1 + coconut/_pyparsing.py | 28 ++++---- coconut/command/util.py | 71 ++++++++++++------- coconut/compiler/compiler.py | 22 ++++-- coconut/compiler/util.py | 17 ++--- coconut/constants.py | 1 + coconut/terminal.py | 13 ++-- .../tests/src/cocotest/agnostic/specific.coco | 1 + coconut/util.py | 16 +++++ 9 files changed, 111 insertions(+), 59 deletions(-) diff --git a/Makefile b/Makefile index 80d436667..93742e5e7 100644 --- a/Makefile +++ b/Makefile @@ -133,6 +133,7 @@ test-pypy3: clean # same as test-univ but reverses any ofs .PHONY: test-any-of +test-any-of: export COCONUT_ADAPTIVE_ANY_OF=TRUE test-any-of: export COCONUT_REVERSE_ANY_OF=TRUE test-any-of: test-univ diff --git a/coconut/_pyparsing.py b/coconut/_pyparsing.py index 57b99c367..c973208b5 100644 --- a/coconut/_pyparsing.py +++ b/coconut/_pyparsing.py @@ -234,23 +234,11 @@ def enableIncremental(*args, **kwargs): + " (run '{python} -m pip install --upgrade cPyparsing' to fix)".format(python=sys.executable) ) -SUPPORTS_ADAPTIVE = hasattr(MatchFirst, "setAdaptiveMode") -USE_CACHE = SUPPORTS_INCREMENTAL and use_cache_file - -maybe_make_safe = getattr(_pyparsing, "maybe_make_safe", None) - # ----------------------------------------------------------------------------------------------------------------------- # SETUP: # ----------------------------------------------------------------------------------------------------------------------- -if MODERN_PYPARSING: - _trim_arity = _pyparsing.core._trim_arity - _ParseResultsWithOffset = _pyparsing.core._ParseResultsWithOffset -else: - _trim_arity = _pyparsing._trim_arity - _ParseResultsWithOffset = _pyparsing._ParseResultsWithOffset - USE_COMPUTATION_GRAPH = get_bool_env_var( use_computation_graph_env_var, default=( @@ -260,6 +248,22 @@ def enableIncremental(*args, **kwargs): ), ) +SUPPORTS_ADAPTIVE = ( + hasattr(MatchFirst, "setAdaptiveMode") + and USE_COMPUTATION_GRAPH +) + +USE_CACHE = SUPPORTS_INCREMENTAL and use_cache_file + +if MODERN_PYPARSING: + _trim_arity = _pyparsing.core._trim_arity + _ParseResultsWithOffset = _pyparsing.core._ParseResultsWithOffset +else: + _trim_arity = _pyparsing._trim_arity + _ParseResultsWithOffset = _pyparsing._ParseResultsWithOffset + +maybe_make_safe = getattr(_pyparsing, "maybe_make_safe", None) + if enable_pyparsing_warnings: if MODERN_PYPARSING: _pyparsing.enable_all_warnings() diff --git a/coconut/command/util.py b/coconut/command/util.py index a9b1c1883..d996102d6 100644 --- a/coconut/command/util.py +++ b/coconut/command/util.py @@ -269,7 +269,8 @@ def run_file(path): def readline_to_queue(file_obj, q): """Read a line from file_obj and put it in the queue.""" - q.put(file_obj.readline()) + if not is_empty_pipe(file_obj): + q.put(file_obj.readline()) def call_output(cmd, stdin=None, encoding_errors="replace", color=None, **kwargs): @@ -283,40 +284,48 @@ def call_output(cmd, stdin=None, encoding_errors="replace", color=None, **kwargs stdout_q = queue.Queue() stderr_q = queue.Queue() - stdout_t = stderr_t = None + # list for mutability + stdout_t_obj = [None] + stderr_t_obj = [None] stdout, stderr, retcode = [], [], None + checking_stdout = True # alternate between stdout and stderr while retcode is None: - if stdout_t is None or not stdout_t.is_alive(): - stdout_t = threading.Thread(target=readline_to_queue, args=(p.stdout, stdout_q)) - stdout_t.start() - if stderr_t is None or not stderr_t.is_alive(): - stderr_t = threading.Thread(target=readline_to_queue, args=(p.stderr, stderr_q)) - stderr_t.start() + if checking_stdout: + proc_pipe = p.stdout + sys_pipe = sys.stdout + q = stdout_q + t_obj = stdout_t_obj + log_func = logger.log_stdout + out_list = stdout + else: + proc_pipe = p.stderr + sys_pipe = sys.stderr + q = stderr_q + t_obj = stderr_t_obj + log_func = logger.log + out_list = stderr + + if t_obj[0] is None or not t_obj[0].is_alive(): + t_obj[0] = threading.Thread(target=readline_to_queue, args=(proc_pipe, q)) + t_obj[0].daemon = True + t_obj[0].start() - stdout_t.join(timeout=call_timeout) - stderr_t.join(timeout=call_timeout) + t_obj[0].join(timeout=call_timeout) try: - raw_out = stdout_q.get(block=False) + raw_out = q.get(block=False) except queue.Empty: raw_out = None - try: - raw_err = stderr_q.get(block=False) - except queue.Empty: - raw_err = None - out = raw_out.decode(get_encoding(sys.stdout), encoding_errors) if raw_out else "" - err = raw_err.decode(get_encoding(sys.stderr), encoding_errors) if raw_err else "" + out = raw_out.decode(get_encoding(sys_pipe), encoding_errors) if raw_out else "" if out: - logger.log_stdout(out, color=color, end="") - stdout.append(out) - if err: - logger.log(err, color=color, end="") - stderr.append(err) + log_func(out, color=color, end="") + out_list.append(out) retcode = p.poll() + checking_stdout = not checking_stdout return "".join(stdout), "".join(stderr), retcode @@ -431,15 +440,23 @@ def set_mypy_path(): return install_dir -def stdin_readable(): - """Determine whether stdin has any data to read.""" +def is_empty_pipe(pipe): + """Determine if the given pipe file object is empty.""" if not WINDOWS: try: - return bool(select([sys.stdin], [], [], 0)[0]) + return not select.select([pipe], [], [], 0)[0] except Exception: logger.log_exc() - # by default assume not readable - return not isatty(sys.stdin, default=True) + return None + + +def stdin_readable(): + """Determine whether stdin has any data to read.""" + return ( + is_empty_pipe(sys.stdin) is False + # by default assume not readable + or not isatty(sys.stdin, default=True) + ) def set_recursion_limit(limit): diff --git a/coconut/compiler/compiler.py b/coconut/compiler/compiler.py index e82c3b42b..377b1bd7c 100644 --- a/coconut/compiler/compiler.py +++ b/coconut/compiler/compiler.py @@ -88,6 +88,8 @@ in_place_op_funcs, match_first_arg_var, import_existing, + use_adaptive_any_of, + reverse_any_of, ) from coconut.util import ( pickleable_obj, @@ -1085,10 +1087,14 @@ def wrap_error(self, error): def raise_or_wrap_error(self, error): """Raise if USE_COMPUTATION_GRAPH else wrap.""" - if USE_COMPUTATION_GRAPH: - raise error - else: + if ( + not USE_COMPUTATION_GRAPH + or use_adaptive_any_of + or reverse_any_of + ): return self.wrap_error(error) + else: + raise error def type_ignore_comment(self): """Get a "type: ignore" comment.""" @@ -4545,7 +4551,7 @@ def check_strict(self, name, original, loc, tokens=(None,), only_warn=False, alw else: if always_warn: kwargs["extra"] = "remove --strict to downgrade to a warning" - raise self.make_err(CoconutStyleError, message, original, loc, **kwargs) + return self.raise_or_wrap_error(self.make_err(CoconutStyleError, message, original, loc, **kwargs)) elif always_warn: self.syntax_warning(message, original, loc) return tokens[0] @@ -4586,7 +4592,13 @@ def check_py(self, version, name, original, loc, tokens): self.internal_assert(len(tokens) == 1, original, loc, "invalid " + name + " tokens", tokens) version_info = get_target_info(version) if self.target_info < version_info: - raise self.make_err(CoconutTargetError, "found Python " + ".".join(str(v) for v in version_info) + " " + name, original, loc, target=version) + return self.raise_or_wrap_error(self.make_err( + CoconutTargetError, + "found Python " + ".".join(str(v) for v in version_info) + " " + name, + original, + loc, + target=version, + )) else: return tokens[0] diff --git a/coconut/compiler/util.py b/coconut/compiler/util.py index 9b90a03e7..bfd439a19 100644 --- a/coconut/compiler/util.py +++ b/coconut/compiler/util.py @@ -1073,13 +1073,18 @@ def any_of(*exprs, **kwargs): flat_exprs = [] for e in exprs: - if e.__class__ == AnyOf and not hasaction(e): + if ( + # don't merge MatchFirsts when we're reversing + not (reverse_any_of and not use_adaptive) + and e.__class__ == AnyOf + and not hasaction(e) + ): flat_exprs.extend(e.exprs) else: flat_exprs.append(e) if reverse_any_of: - flat_exprs.reverse() + flat_exprs = reversed([trace(e) for e in exprs]) return AnyOf(flat_exprs) @@ -1726,14 +1731,6 @@ def split_leading_trailing_indent(line, max_indents=None): return leading_indent, line, trailing_indent -def split_leading_whitespace(inputstr): - """Split leading whitespace.""" - basestr = inputstr.lstrip() - whitespace = inputstr[:len(inputstr) - len(basestr)] - internal_assert(whitespace + basestr == inputstr, "invalid whitespace split", inputstr) - return whitespace, basestr - - def rem_and_count_indents(inputstr): """Removes and counts the ind_change (opens - closes).""" no_opens = inputstr.replace(openindent, "") diff --git a/coconut/constants.py b/coconut/constants.py index ca26e3c4a..64de7c7f2 100644 --- a/coconut/constants.py +++ b/coconut/constants.py @@ -138,6 +138,7 @@ def get_path_env_var(env_var, default): use_cache_file = True disable_incremental_for_len = 46080 # TODO: this is disabled by default until we get test-any-of to pass +# (and then test-any-of should be added to main_test) use_adaptive_any_of = get_bool_env_var("COCONUT_ADAPTIVE_ANY_OF", False) # note that _parseIncremental produces much smaller caches diff --git a/coconut/terminal.py b/coconut/terminal.py index 20f2358fd..11fb41cf7 100644 --- a/coconut/terminal.py +++ b/coconut/terminal.py @@ -60,6 +60,7 @@ displayable, first_import_time, assert_remove_prefix, + split_trailing_whitespace, ) from coconut.exceptions import ( CoconutWarning, @@ -276,14 +277,16 @@ def display( raw_message = "\n" components = [] - if color: - components.append(ansii_escape + "[" + color + "m") for line in raw_message.splitlines(True): + line, endline = split_trailing_whitespace(line) + if color: + components.append(ansii_escape + "[" + color + "m") if sig: - line = sig + line + components.append(sig) components.append(line) - if color: - components.append(ansii_reset) + if color: + components.append(ansii_reset) + components.append(endline) components.append(end) full_message = "".join(components) diff --git a/coconut/tests/src/cocotest/agnostic/specific.coco b/coconut/tests/src/cocotest/agnostic/specific.coco index 82311206e..57f573d4d 100644 --- a/coconut/tests/src/cocotest/agnostic/specific.coco +++ b/coconut/tests/src/cocotest/agnostic/specific.coco @@ -1,3 +1,4 @@ +import sys from io import StringIO if TYPE_CHECKING: from typing import Any diff --git a/coconut/util.py b/coconut/util.py index 5b2c60b05..15091dd85 100644 --- a/coconut/util.py +++ b/coconut/util.py @@ -298,6 +298,22 @@ def without_keys(inputdict, rem_keys): return {k: v for k, v in inputdict.items() if k not in rem_keys} +def split_leading_whitespace(inputstr): + """Split leading whitespace.""" + basestr = inputstr.lstrip() + whitespace = inputstr[:len(inputstr) - len(basestr)] + assert whitespace + basestr == inputstr, "invalid whitespace split: " + repr(inputstr) + return whitespace, basestr + + +def split_trailing_whitespace(inputstr): + """Split trailing whitespace.""" + basestr = inputstr.rstrip() + whitespace = inputstr[len(basestr):] + assert basestr + whitespace == inputstr, "invalid whitespace split: " + repr(inputstr) + return basestr, whitespace + + # ----------------------------------------------------------------------------------------------------------------------- # VERSIONING: # ----------------------------------------------------------------------------------------------------------------------- From 164fb564395666f2270141b23f31c06c574b3d74 Mon Sep 17 00:00:00 2001 From: Evan Hubinger Date: Thu, 23 Nov 2023 00:30:46 -0800 Subject: [PATCH 098/121] Enable adaptive --- coconut/command/util.py | 133 ++++++++++++++++++++++---------- coconut/compiler/compiler.py | 23 +++--- coconut/compiler/util.py | 56 ++++++++++++-- coconut/constants.py | 17 ++-- coconut/integrations.py | 13 ++-- coconut/root.py | 2 - coconut/tests/__main__.py | 2 + coconut/tests/constants_test.py | 10 ++- coconut/tests/main_test.py | 30 +++++++ 9 files changed, 211 insertions(+), 75 deletions(-) diff --git a/coconut/command/util.py b/coconut/command/util.py index d996102d6..a35c5b326 100644 --- a/coconut/command/util.py +++ b/coconut/command/util.py @@ -52,8 +52,10 @@ ) from coconut.constants import ( WINDOWS, - PY34, + CPYTHON, + PY26, PY32, + PY34, fixpath, base_dir, main_prompt, @@ -90,15 +92,15 @@ import imp else: import runpy +if PY34: + from importlib import reload +else: + from imp import reload try: # just importing readline improves built-in input() import readline # NOQA except ImportError: pass -if PY34: - from importlib import reload -else: - from imp import reload try: import prompt_toolkit @@ -267,65 +269,101 @@ def run_file(path): return runpy.run_path(path, run_name="__main__") +def interrupt_thread(thread, exctype=OSError): + """Attempt to interrupt the given thread.""" + if not CPYTHON: + return False + if thread is None or not thread.is_alive(): + return True + import ctypes + tid = ctypes.c_long(thread.ident) + res = ctypes.pythonapi.PyThreadState_SetAsyncExc( + tid, + ctypes.py_object(exctype), + ) + if res == 0: + return False + elif res == 1: + return True + else: + ctypes.pythonapi.PyThreadState_SetAsyncExc(tid, None) + return False + + def readline_to_queue(file_obj, q): """Read a line from file_obj and put it in the queue.""" if not is_empty_pipe(file_obj): - q.put(file_obj.readline()) + try: + q.put(file_obj.readline()) + except OSError: + pass def call_output(cmd, stdin=None, encoding_errors="replace", color=None, **kwargs): """Run command and read output.""" p = subprocess.Popen(cmd, stdout=subprocess.PIPE, stderr=subprocess.PIPE, stdin=subprocess.PIPE, **kwargs) + stdout_q = queue.Queue() + stderr_q = queue.Queue() + + if WINDOWS or not logger.verbose: + raw_stdout, raw_stderr = p.communicate(stdin) + stdout_q.put(raw_stdout) + stderr_q.put(raw_stderr) + stdin = None + if stdin is not None: logger.log_prefix("STDIN < ", stdin.rstrip()) p.stdin.write(stdin) - stdout_q = queue.Queue() - stderr_q = queue.Queue() - # list for mutability stdout_t_obj = [None] stderr_t_obj = [None] stdout, stderr, retcode = [], [], None checking_stdout = True # alternate between stdout and stderr - while retcode is None: - if checking_stdout: - proc_pipe = p.stdout - sys_pipe = sys.stdout - q = stdout_q - t_obj = stdout_t_obj - log_func = logger.log_stdout - out_list = stdout - else: - proc_pipe = p.stderr - sys_pipe = sys.stderr - q = stderr_q - t_obj = stderr_t_obj - log_func = logger.log - out_list = stderr + try: + while retcode is None or not stdout_q.empty() or not stderr_q.empty(): + if checking_stdout: + proc_pipe = p.stdout + sys_pipe = sys.stdout + q = stdout_q + t_obj = stdout_t_obj + log_func = logger.log_stdout + out_list = stdout + else: + proc_pipe = p.stderr + sys_pipe = sys.stderr + q = stderr_q + t_obj = stderr_t_obj + log_func = logger.log + out_list = stderr - if t_obj[0] is None or not t_obj[0].is_alive(): - t_obj[0] = threading.Thread(target=readline_to_queue, args=(proc_pipe, q)) - t_obj[0].daemon = True - t_obj[0].start() + retcode = p.poll() - t_obj[0].join(timeout=call_timeout) + if retcode is None and t_obj[0] is not False: + if t_obj[0] is None or not t_obj[0].is_alive(): + t_obj[0] = threading.Thread(target=readline_to_queue, args=(proc_pipe, q)) + t_obj[0].daemon = True + t_obj[0].start() - try: - raw_out = q.get(block=False) - except queue.Empty: - raw_out = None + t_obj[0].join(timeout=call_timeout) - out = raw_out.decode(get_encoding(sys_pipe), encoding_errors) if raw_out else "" + try: + raw_out = q.get(block=False) + except queue.Empty: + raw_out = None - if out: - log_func(out, color=color, end="") - out_list.append(out) + out = raw_out.decode(get_encoding(sys_pipe), encoding_errors) if raw_out else "" - retcode = p.poll() - checking_stdout = not checking_stdout + if out: + log_func(out, color=color, end="") + out_list.append(out) + + checking_stdout = not checking_stdout + finally: + interrupt_thread(stdout_t_obj[0]) + interrupt_thread(stderr_t_obj[0]) return "".join(stdout), "".join(stderr), retcode @@ -544,6 +582,21 @@ def get_python_lib(): return fixpath(python_lib) +def import_coconut_header(): + """Import the coconut.__coconut__ header. + This is expensive, so only do it here.""" + try: + from coconut import __coconut__ + return __coconut__ + except ImportError: + # fixes an issue where, when running from the base coconut directory, + # the base coconut directory is treated as a namespace package + if os.path.basename(os.getcwd()) == "coconut": + from coconut.coconut import __coconut__ + return __coconut__ + raise + + # ----------------------------------------------------------------------------------------------------------------------- # CLASSES: # ----------------------------------------------------------------------------------------------------------------------- @@ -695,7 +748,7 @@ def store(self, line): def fix_pickle(self): """Fix pickling of Coconut header objects.""" - from coconut import __coconut__ # this is expensive, so only do it here + __coconut__ = import_coconut_header() for var in self.vars: if not var.startswith("__") and var in dir(__coconut__) and var not in must_use_specific_target_builtins: cur_val = self.vars[var] diff --git a/coconut/compiler/compiler.py b/coconut/compiler/compiler.py index 377b1bd7c..f099f4535 100644 --- a/coconut/compiler/compiler.py +++ b/coconut/compiler/compiler.py @@ -132,6 +132,7 @@ partial_op_item_handle, ) from coconut.compiler.util import ( + ExceptionNode, sys_target, getline, addskip, @@ -888,11 +889,17 @@ def reformat_post_deferred_code_proc(self, snip): """Do post-processing that comes after deferred_code_proc.""" return self.apply_procs(self.reformatprocs[1:], snip, reformatting=True, log=False) - def reformat(self, snip, **kwargs): + def reformat(self, snip, ignore_errors, **kwargs): """Post process a preprocessed snippet.""" - internal_assert("ignore_errors" in kwargs, "reformat() missing required keyword argument: 'ignore_errors'") - with self.complain_on_err(): - return self.apply_procs(self.reformatprocs, snip, reformatting=True, log=False, **kwargs) + with noop_ctx() if ignore_errors else self.complain_on_err(): + return self.apply_procs( + self.reformatprocs, + snip, + reformatting=True, + log=False, + ignore_errors=ignore_errors, + **kwargs, + ) return snip def reformat_locs(self, snip, loc, endpt=None, **kwargs): @@ -1087,12 +1094,10 @@ def wrap_error(self, error): def raise_or_wrap_error(self, error): """Raise if USE_COMPUTATION_GRAPH else wrap.""" - if ( - not USE_COMPUTATION_GRAPH - or use_adaptive_any_of - or reverse_any_of - ): + if not USE_COMPUTATION_GRAPH: return self.wrap_error(error) + elif use_adaptive_any_of or reverse_any_of: + return ExceptionNode(error) else: raise error diff --git a/coconut/compiler/util.py b/coconut/compiler/util.py index bfd439a19..dec2b28df 100644 --- a/coconut/compiler/util.py +++ b/coconut/compiler/util.py @@ -147,10 +147,24 @@ indexable_evaluated_tokens_types = (ParseResults, list, tuple) +def evaluate_all_tokens(all_tokens, is_final, **kwargs): + """Recursively evaluate all the tokens in all_tokens.""" + all_evaluated_toks = [] + for toks in all_tokens: + evaluated_toks = evaluate_tokens(toks, is_final=is_final, **kwargs) + # if we're a final parse, ExceptionNodes will just be raised, but otherwise, if we see any, we need to + # short-circuit the computation and return them, since they imply this parse contains invalid syntax + if not is_final and isinstance(toks, ExceptionNode): + return toks + all_evaluated_toks.append(evaluated_toks) + return all_evaluated_toks + + def evaluate_tokens(tokens, **kwargs): """Evaluate the given tokens in the computation graph. Very performance sensitive.""" - # can't have this be a normal kwarg to make evaluate_tokens a valid parse action + # can't have these be normal kwargs to make evaluate_tokens a valid parse action + is_final = kwargs.pop("is_final", False) evaluated_toklists = kwargs.pop("evaluated_toklists", ()) if DEVELOP: # avoid the overhead of the call if not develop internal_assert(not kwargs, "invalid keyword arguments to evaluate_tokens", kwargs) @@ -168,7 +182,7 @@ def evaluate_tokens(tokens, **kwargs): new_toklist = eval_new_toklist break if new_toklist is None: - new_toklist = [evaluate_tokens(toks, evaluated_toklists=evaluated_toklists) for toks in old_toklist] + new_toklist = evaluate_all_tokens(old_toklist, is_final=is_final, evaluated_toklists=evaluated_toklists) # overwrite evaluated toklists rather than appending, since this # should be all the information we need for evaluating the dictionary evaluated_toklists = ((old_toklist, new_toklist),) @@ -183,7 +197,9 @@ def evaluate_tokens(tokens, **kwargs): for name, occurrences in tokens._ParseResults__tokdict.items(): new_occurrences = [] for value, position in occurrences: - new_value = evaluate_tokens(value, evaluated_toklists=evaluated_toklists) + new_value = evaluate_tokens(value, is_final=is_final, evaluated_toklists=evaluated_toklists) + if not is_final and isinstance(new_value, ExceptionNode): + return new_value new_occurrences.append(_ParseResultsWithOffset(new_value, position)) new_tokdict[name] = new_occurrences new_tokens._ParseResults__tokdict.update(new_tokdict) @@ -217,13 +233,24 @@ def evaluate_tokens(tokens, **kwargs): return tokens elif isinstance(tokens, ComputationNode): - return tokens.evaluate() + result = tokens.evaluate() + if is_final and isinstance(result, ExceptionNode): + raise result.exception + return result elif isinstance(tokens, list): - return [evaluate_tokens(inner_toks, evaluated_toklists=evaluated_toklists) for inner_toks in tokens] + return evaluate_all_tokens(tokens, is_final=is_final, evaluated_toklists=evaluated_toklists) elif isinstance(tokens, tuple): - return tuple(evaluate_tokens(inner_toks, evaluated_toklists=evaluated_toklists) for inner_toks in tokens) + result = evaluate_all_tokens(tokens, is_final=is_final, evaluated_toklists=evaluated_toklists) + if isinstance(result, ExceptionNode): + return result + return tuple(result) + + elif isinstance(tokens, ExceptionNode): + if is_final: + raise tokens.exception + return tokens elif isinstance(tokens, DeferredNode): return tokens @@ -277,6 +304,8 @@ def evaluate(self): evaluated_toks = evaluate_tokens(self.tokens) if logger.tracing: # avoid the overhead of the call if not tracing logger.log_trace(self.name, self.original, self.loc, evaluated_toks, self.tokens) + if isinstance(evaluated_toks, ExceptionNode): + return evaluated_toks # short-circuit if we got an ExceptionNode try: return self.action( self.original, @@ -307,6 +336,7 @@ def __repr__(self): class DeferredNode(object): """A node in the computation graph that has had its evaluation explicitly deferred.""" + __slots__ = ("original", "loc", "tokens") def __init__(self, original, loc, tokens): self.original = original @@ -318,6 +348,16 @@ def evaluate(self): return unpack(self.tokens) +class ExceptionNode(object): + """A node in the computation graph that stores an exception that will be raised upon final evaluation.""" + __slots__ = ("exception",) + + def __init__(self, exception): + if not USE_COMPUTATION_GRAPH: + raise exception + self.exception = exception + + class CombineToNode(Combine): """Modified Combine to work with the computation graph.""" __slots__ = () @@ -377,7 +417,7 @@ def attach(item, action, ignore_no_tokens=None, ignore_one_token=None, ignore_to def final_evaluate_tokens(tokens): """Same as evaluate_tokens but should only be used once a parse is assured.""" clear_packrat_cache() - return evaluate_tokens(tokens) + return evaluate_tokens(tokens, is_final=True) @contextmanager @@ -426,7 +466,7 @@ def defer(item): def unpack(tokens): """Evaluate and unpack the given computation graph.""" logger.log_tag("unpack", tokens) - tokens = evaluate_tokens(tokens) + tokens = final_evaluate_tokens(tokens) if isinstance(tokens, ParseResults) and len(tokens) == 1: tokens = tokens[0] return tokens diff --git a/coconut/constants.py b/coconut/constants.py index 64de7c7f2..718132883 100644 --- a/coconut/constants.py +++ b/coconut/constants.py @@ -71,6 +71,7 @@ def get_path_env_var(env_var, default): WINDOWS = os.name == "nt" PYPY = platform.python_implementation() == "PyPy" CPYTHON = platform.python_implementation() == "CPython" +PY26 = sys.version_info < (2, 7) PY32 = sys.version_info >= (3, 2) PY33 = sys.version_info >= (3, 3) PY34 = sys.version_info >= (3, 4) @@ -126,7 +127,8 @@ def get_path_env_var(env_var, default): cache_validation_info = DEVELOP -reverse_any_of = get_bool_env_var("COCONUT_REVERSE_ANY_OF", False) +reverse_any_of_env_var = "COCONUT_REVERSE_ANY_OF" +reverse_any_of = get_bool_env_var(reverse_any_of_env_var, False) # below constants are experimentally determined to maximize performance @@ -136,10 +138,11 @@ def get_path_env_var(env_var, default): streamline_grammar_for_len = 1536 use_cache_file = True + disable_incremental_for_len = 46080 -# TODO: this is disabled by default until we get test-any-of to pass -# (and then test-any-of should be added to main_test) -use_adaptive_any_of = get_bool_env_var("COCONUT_ADAPTIVE_ANY_OF", False) + +adaptive_any_of_env_var = "COCONUT_ADAPTIVE_ANY_OF" +use_adaptive_any_of = get_bool_env_var(adaptive_any_of_env_var, True) # note that _parseIncremental produces much smaller caches use_incremental_if_available = False @@ -477,6 +480,7 @@ def get_path_env_var(env_var, default): "urllib.parse": ("urllib", (3,)), "pickle": ("cPickle", (3,)), "collections.abc": ("collections", (3, 3)), + "_dummy_thread": ("dummy_thread", (3,)), # ./ in old_name denotes from ... import ... "io.StringIO": ("StringIO./StringIO", (2, 7)), "io.BytesIO": ("cStringIO./StringIO", (2, 7)), @@ -485,7 +489,7 @@ def get_path_env_var(env_var, default): "itertools.zip_longest": ("itertools./izip_longest", (3,)), "math.gcd": ("fractions./gcd", (3, 5)), "time.process_time": ("time./clock", (3, 3)), - "_dummy_thread": ("dummy_thread", (3,)), + "shlex.quote": ("pipes./quote", (3, 3)), # third-party backports "asyncio": ("trollius", (3, 4)), @@ -742,10 +746,11 @@ def get_path_env_var(env_var, default): create_package_retries = 1 -call_timeout = 0.01 +call_timeout = 0.001 max_orig_lines_in_log_loc = 2 + # ----------------------------------------------------------------------------------------------------------------------- # HIGHLIGHTER CONSTANTS: # ----------------------------------------------------------------------------------------------------------------------- diff --git a/coconut/integrations.py b/coconut/integrations.py index 6ca1c377c..e83dd13c2 100644 --- a/coconut/integrations.py +++ b/coconut/integrations.py @@ -49,19 +49,20 @@ def embed(kernel=False, depth=0, **kwargs): def load_ipython_extension(ipython): """Loads Coconut as an IPython extension.""" + # import here to avoid circular dependencies + from coconut import api + from coconut.exceptions import CoconutException + from coconut.terminal import logger + from coconut.command.util import import_coconut_header + # add Coconut built-ins - from coconut import __coconut__ + __coconut__ = import_coconut_header() newvars = {} for var, val in vars(__coconut__).items(): if not var.startswith("__"): newvars[var] = val ipython.push(newvars) - # import here to avoid circular dependencies - from coconut import api - from coconut.exceptions import CoconutException - from coconut.terminal import logger - magic_state = api.get_state() api.setup(state=magic_state, **coconut_kernel_kwargs) api.warm_up(enable_incremental_mode=True, state=magic_state) diff --git a/coconut/root.py b/coconut/root.py index d5c3fe2a9..cfaf43784 100644 --- a/coconut/root.py +++ b/coconut/root.py @@ -358,8 +358,6 @@ def _get_root_header(version="universal"): VERSION_STR = VERSION + (" [" + VERSION_NAME + "]" if VERSION_NAME else "") PY2 = _coconut_sys.version_info < (3,) -PY26 = _coconut_sys.version_info < (2, 7) -PY37 = _coconut_sys.version_info >= (3, 7) # ----------------------------------------------------------------------------------------------------------------------- # SETUP: diff --git a/coconut/tests/__main__.py b/coconut/tests/__main__.py index 1cadb7fa6..649ac82ed 100644 --- a/coconut/tests/__main__.py +++ b/coconut/tests/__main__.py @@ -21,6 +21,7 @@ import sys +from coconut.constants import WINDOWS from coconut.tests.main_test import comp_all # ----------------------------------------------------------------------------------------------------------------------- @@ -48,6 +49,7 @@ def main(args=None): agnostic_target=agnostic_target, expect_retcode=0 if "--mypy" not in args else None, check_errors="--verbose" not in args, + ignore_output=WINDOWS and "--mypy" not in args, ) diff --git a/coconut/tests/constants_test.py b/coconut/tests/constants_test.py index fc32ed6c8..e301e69d0 100644 --- a/coconut/tests/constants_test.py +++ b/coconut/tests/constants_test.py @@ -22,19 +22,21 @@ import sys import os import unittest -if PY26: - import_module = __import__ -else: - from importlib import import_module from coconut import constants from coconut.constants import ( WINDOWS, PYPY, + PY26, PY39, fixpath, ) +if PY26: + import_module = __import__ +else: + from importlib import import_module + # ----------------------------------------------------------------------------------------------------------------------- # UTILITIES: # ----------------------------------------------------------------------------------------------------------------------- diff --git a/coconut/tests/main_test.py b/coconut/tests/main_test.py index cd1da15c5..b8521457a 100644 --- a/coconut/tests/main_test.py +++ b/coconut/tests/main_test.py @@ -40,6 +40,7 @@ from coconut.command.util import ( call_output, reload, + run_cmd, ) from coconut.compiler.util import ( get_psf_target, @@ -50,11 +51,15 @@ IPY, XONSH, MYPY, + PY26, PY35, PY36, PY38, PY39, PY310, + CPYTHON, + adaptive_any_of_env_var, + reverse_any_of_env_var, supported_py2_vers, supported_py3_vers, icoconut_default_kernel_names, @@ -235,6 +240,7 @@ def call( expect_retcode=0, convert_to_import=False, assert_output_only_at_end=None, + ignore_output=False, **kwargs ): """Execute a shell command and assert that no errors were encountered.""" @@ -281,6 +287,9 @@ def call( module_name += ".__main__" with using_sys_path(module_dir): stdout, stderr, retcode = call_with_import(module_name, extra_argv) + elif ignore_output: + retcode = run_cmd(raw_cmd, raise_errs=False, **kwargs) + stdout = stderr = "" else: stdout, stderr, retcode = call_output(raw_cmd, color=False, **kwargs) @@ -543,6 +552,19 @@ def spawn_cmd(cmd): return pexpect.spawn(cmd) +@contextmanager +def using_env_vars(env_vars): + """Run using the given environment variables.""" + old_env = os.environ.copy() + os.environ.update(env_vars) + try: + yield + finally: + for k in env_vars: + del os.environ[k] + os.environ.update(old_env) + + # ----------------------------------------------------------------------------------------------------------------------- # RUNNERS: # ----------------------------------------------------------------------------------------------------------------------- @@ -963,6 +985,14 @@ def test_no_wrap(self): # run fewer tests on Windows so appveyor doesn't time out if not WINDOWS: + if CPYTHON: + def test_any_of(self): + with using_env_vars({ + adaptive_any_of_env_var: "True", + reverse_any_of_env_var: "True", + }): + run() + def test_keep_lines(self): run(["--keep-lines"]) From 57eddf74e60bf258c38b85ea806b171046e3799a Mon Sep 17 00:00:00 2001 From: Evan Hubinger Date: Thu, 23 Nov 2023 02:37:07 -0800 Subject: [PATCH 099/121] Fix errors --- coconut/compiler/header.py | 23 +++++++++-------------- coconut/compiler/util.py | 25 +++++++++++++------------ coconut/root.py | 2 +- 3 files changed, 23 insertions(+), 27 deletions(-) diff --git a/coconut/compiler/header.py b/coconut/compiler/header.py index a81a37f10..8a60ff8cc 100644 --- a/coconut/compiler/header.py +++ b/coconut/compiler/header.py @@ -1012,20 +1012,15 @@ def getheader(which, use_hash, target, no_tco, strict, no_wrap): newline=True, ).format(**format_dict) - if target_info >= (3, 11): - header += _get_root_header("311") - elif target_info >= (3, 9): - header += _get_root_header("39") - if target_info >= (3, 7): - header += _get_root_header("37") - elif target.startswith("3"): - header += _get_root_header("3") - elif target_info >= (2, 7): - header += _get_root_header("27") - elif target.startswith("2"): - header += _get_root_header("2") - else: - header += _get_root_header("universal") + header += _get_root_header( + "311" if target_info >= (3, 11) + else "39" if target_info >= (3, 9) + else "37" if target_info >= (3, 7) + else "3" if target.startswith("3") + else "27" if target_info >= (2, 7) + else "2" if target.startswith("2") + else "universal" + ) header += get_template("header").format(**format_dict) diff --git a/coconut/compiler/util.py b/coconut/compiler/util.py index dec2b28df..207396a2a 100644 --- a/coconut/compiler/util.py +++ b/coconut/compiler/util.py @@ -147,17 +147,17 @@ indexable_evaluated_tokens_types = (ParseResults, list, tuple) -def evaluate_all_tokens(all_tokens, is_final, **kwargs): +def evaluate_all_tokens(all_tokens, **kwargs): """Recursively evaluate all the tokens in all_tokens.""" all_evaluated_toks = [] for toks in all_tokens: - evaluated_toks = evaluate_tokens(toks, is_final=is_final, **kwargs) + evaluated_toks = evaluate_tokens(toks, **kwargs) # if we're a final parse, ExceptionNodes will just be raised, but otherwise, if we see any, we need to # short-circuit the computation and return them, since they imply this parse contains invalid syntax - if not is_final and isinstance(toks, ExceptionNode): - return toks + if isinstance(evaluated_toks, ExceptionNode): + return None, evaluated_toks all_evaluated_toks.append(evaluated_toks) - return all_evaluated_toks + return all_evaluated_toks, None def evaluate_tokens(tokens, **kwargs): @@ -182,7 +182,9 @@ def evaluate_tokens(tokens, **kwargs): new_toklist = eval_new_toklist break if new_toklist is None: - new_toklist = evaluate_all_tokens(old_toklist, is_final=is_final, evaluated_toklists=evaluated_toklists) + new_toklist, exc_node = evaluate_all_tokens(old_toklist, is_final=is_final, evaluated_toklists=evaluated_toklists) + if exc_node is not None: + return exc_node # overwrite evaluated toklists rather than appending, since this # should be all the information we need for evaluating the dictionary evaluated_toklists = ((old_toklist, new_toklist),) @@ -198,7 +200,7 @@ def evaluate_tokens(tokens, **kwargs): new_occurrences = [] for value, position in occurrences: new_value = evaluate_tokens(value, is_final=is_final, evaluated_toklists=evaluated_toklists) - if not is_final and isinstance(new_value, ExceptionNode): + if isinstance(new_value, ExceptionNode): return new_value new_occurrences.append(_ParseResultsWithOffset(new_value, position)) new_tokdict[name] = new_occurrences @@ -239,13 +241,12 @@ def evaluate_tokens(tokens, **kwargs): return result elif isinstance(tokens, list): - return evaluate_all_tokens(tokens, is_final=is_final, evaluated_toklists=evaluated_toklists) + result, exc_node = evaluate_all_tokens(tokens, is_final=is_final, evaluated_toklists=evaluated_toklists) + return result if exc_node is None else exc_node elif isinstance(tokens, tuple): - result = evaluate_all_tokens(tokens, is_final=is_final, evaluated_toklists=evaluated_toklists) - if isinstance(result, ExceptionNode): - return result - return tuple(result) + result, exc_node = evaluate_all_tokens(tokens, is_final=is_final, evaluated_toklists=evaluated_toklists) + return tuple(result) if exc_node is None else exc_node elif isinstance(tokens, ExceptionNode): if is_final: diff --git a/coconut/root.py b/coconut/root.py index cfaf43784..5e71c5f1b 100644 --- a/coconut/root.py +++ b/coconut/root.py @@ -26,7 +26,7 @@ VERSION = "3.0.3" VERSION_NAME = None # False for release, int >= 1 for develop -DEVELOP = 33 +DEVELOP = 34 ALPHA = False # for pre releases rather than post releases assert DEVELOP is False or DEVELOP >= 1, "DEVELOP must be False or an int >= 1" From a3d33ec985aebad31b81ccbf4ba2c82f21711c6d Mon Sep 17 00:00:00 2001 From: Evan Hubinger Date: Thu, 23 Nov 2023 16:50:54 -0800 Subject: [PATCH 100/121] Fix more issues --- coconut/command/command.py | 6 ++++-- coconut/command/util.py | 12 ++++++------ 2 files changed, 10 insertions(+), 8 deletions(-) diff --git a/coconut/command/command.py b/coconut/command/command.py index f5c58e699..95e21d0da 100644 --- a/coconut/command/command.py +++ b/coconut/command/command.py @@ -397,8 +397,10 @@ def execute_args(self, args, interact=True, original_args=None): self.start_jupyter(args.jupyter) elif stdin_readable(): logger.log("Reading piped input from stdin...") - self.execute(self.parse_block(sys.stdin.read())) - got_stdin = True + read_stdin = sys.stdin.read() + if read_stdin: + self.execute(self.parse_block(read_stdin)) + got_stdin = True if args.interact or ( interact and not ( got_stdin diff --git a/coconut/command/util.py b/coconut/command/util.py index a35c5b326..4419c88a7 100644 --- a/coconut/command/util.py +++ b/coconut/command/util.py @@ -482,7 +482,7 @@ def is_empty_pipe(pipe): """Determine if the given pipe file object is empty.""" if not WINDOWS: try: - return not select.select([pipe], [], [], 0)[0] + return not select([pipe], [], [], 0)[0] except Exception: logger.log_exc() return None @@ -490,11 +490,11 @@ def is_empty_pipe(pipe): def stdin_readable(): """Determine whether stdin has any data to read.""" - return ( - is_empty_pipe(sys.stdin) is False - # by default assume not readable - or not isatty(sys.stdin, default=True) - ) + stdin_is_empty = is_empty_pipe(sys.stdin) + if stdin_is_empty is not None: + return stdin_is_empty + # by default assume not readable + return not isatty(sys.stdin, default=True) def set_recursion_limit(limit): From f797edfbdce7e7e79bb4285a1fc44d1917865b2b Mon Sep 17 00:00:00 2001 From: Evan Hubinger Date: Thu, 23 Nov 2023 17:07:35 -0800 Subject: [PATCH 101/121] Prepare for release --- DOCS.md | 8 +++++--- coconut/api.py | 28 +++++++++++++++++----------- coconut/api.pyi | 2 +- coconut/constants.py | 2 +- coconut/root.py | 4 ++-- coconut/tests/main_test.py | 3 ++- 6 files changed, 28 insertions(+), 19 deletions(-) diff --git a/DOCS.md b/DOCS.md index a00f9f696..0bd5b94b4 100644 --- a/DOCS.md +++ b/DOCS.md @@ -4708,13 +4708,15 @@ If _use\_cache\_dir_ is passed, it will turn on or off the usage of a `__coconut Switches the [`breakpoint` built-in](https://www.python.org/dev/peps/pep-0553/) which Coconut makes universally available to use [`coconut.embed`](#coconut-embed) instead of [`pdb.set_trace`](https://docs.python.org/3/library/pdb.html#pdb.set_trace) (or undoes that switch if `on=False`). This function is called automatically when `coconut.api` is imported. -#### `find_and_compile_packages` +#### `find_packages` and `find_and_compile_packages` + +**coconut.api.find_packages**(_where_=`"."`, _exclude_=`()`, _include_=`("*",)`) **coconut.api.find_and_compile_packages**(_where_=`"."`, _exclude_=`()`, _include_=`("*",)`) -Behaves similarly to [`setuptools.find_packages`](https://setuptools.pypa.io/en/latest/userguide/quickstart.html#package-discovery) except that it finds Coconut packages rather than Python packages, and compiles any Coconut packages that it finds in-place. +Both functions behave identically to [`setuptools.find_packages`](https://setuptools.pypa.io/en/latest/userguide/quickstart.html#package-discovery), except that they find Coconut packages rather than Python packages. `find_and_compile_packages` additionally compiles any Coconut packages that it finds in-place. -Note that if you want to use `find_and_compile_packages` in your `setup.py`, you'll need to include `coconut` as a [build-time dependency in your `pyproject.toml`](https://pip.pypa.io/en/stable/reference/build-system/pyproject-toml/#build-time-dependencies). +Note that if you want to use either of these functions in your `setup.py`, you'll need to include `coconut` as a [build-time dependency in your `pyproject.toml`](https://pip.pypa.io/en/stable/reference/build-system/pyproject-toml/#build-time-dependencies). ##### Example diff --git a/coconut/api.py b/coconut/api.py index 562784f64..82f2f0bc5 100644 --- a/coconut/api.py +++ b/coconut/api.py @@ -363,15 +363,7 @@ def get_coconut_encoding(encoding="coconut"): # ----------------------------------------------------------------------------------------------------------------------- class CoconutPackageFinder(PackageFinder, object): - - _coconut_command = None - - @classmethod - def _coconut_compile(cls, path): - """Run the Coconut compiler with the given args.""" - if cls._coconut_command is None: - cls._coconut_command = Command() - return cls._coconut_command.cmd_sys([path], interact=False) + _coconut_compile = None @override @classmethod @@ -380,9 +372,23 @@ def _looks_like_package(cls, path, _package_name=None): os.path.isfile(os.path.join(path, "__init__" + ext)) for ext in code_exts ) - if is_coconut_package: + if is_coconut_package and cls._coconut_compile is not None: cls._coconut_compile(path) return is_coconut_package -find_and_compile_packages = CoconutPackageFinder.find +find_packages = CoconutPackageFinder.find + + +class CoconutPackageCompiler(CoconutPackageFinder): + _coconut_command = None + + @classmethod + def _coconut_compile(cls, path): + """Run the Coconut compiler with the given args.""" + if cls._coconut_command is None: + cls._coconut_command = Command() + return cls._coconut_command.cmd_sys([path], interact=False) + + +find_and_compile_packages = CoconutPackageCompiler.find diff --git a/coconut/api.pyi b/coconut/api.pyi index 27210efa3..850b2eb89 100644 --- a/coconut/api.pyi +++ b/coconut/api.pyi @@ -157,4 +157,4 @@ def get_coconut_encoding(encoding: Text = ...) -> Any: ... -find_and_compile_packages = _find_packages +find_and_compile_packages = find_packages = _find_packages diff --git a/coconut/constants.py b/coconut/constants.py index 718132883..5fd307892 100644 --- a/coconut/constants.py +++ b/coconut/constants.py @@ -1031,7 +1031,7 @@ def get_path_env_var(env_var, default): ("typing_extensions", "py>=38"): (4, 8), ("ipykernel", "py38"): (6,), ("jedi", "py39"): (0, 19), - ("pygments", "py>=39"): (2, 16), + ("pygments", "py>=39"): (2, 17), ("xonsh", "py38"): (0, 14), ("pytest", "py36"): (7,), ("async_generator", "py35"): (1, 10), diff --git a/coconut/root.py b/coconut/root.py index 5e71c5f1b..d24f5a38d 100644 --- a/coconut/root.py +++ b/coconut/root.py @@ -23,10 +23,10 @@ # VERSION: # ----------------------------------------------------------------------------------------------------------------------- -VERSION = "3.0.3" +VERSION = "3.0.4" VERSION_NAME = None # False for release, int >= 1 for develop -DEVELOP = 34 +DEVELOP = False ALPHA = False # for pre releases rather than post releases assert DEVELOP is False or DEVELOP >= 1, "DEVELOP must be False or an int >= 1" diff --git a/coconut/tests/main_test.py b/coconut/tests/main_test.py index b8521457a..3a762c726 100644 --- a/coconut/tests/main_test.py +++ b/coconut/tests/main_test.py @@ -847,7 +847,8 @@ def test_import_hook(self): def test_find_packages(self): with using_pys_in(agnostic_dir): with using_coconut(): - from coconut.api import find_and_compile_packages + from coconut.api import find_packages, find_and_compile_packages + assert find_packages(cocotest_dir) == ["agnostic"] assert find_and_compile_packages(cocotest_dir) == ["agnostic"] def test_runnable(self): From 417ce328c5bd0565f8abc5ae242dad4708d5d8d8 Mon Sep 17 00:00:00 2001 From: Evan Hubinger Date: Thu, 23 Nov 2023 17:24:22 -0800 Subject: [PATCH 102/121] Fix syntax for py2 --- coconut/compiler/compiler.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/coconut/compiler/compiler.py b/coconut/compiler/compiler.py index f099f4535..cdce7ff58 100644 --- a/coconut/compiler/compiler.py +++ b/coconut/compiler/compiler.py @@ -898,7 +898,7 @@ def reformat(self, snip, ignore_errors, **kwargs): reformatting=True, log=False, ignore_errors=ignore_errors, - **kwargs, + **kwargs # no comma for py2 ) return snip From ac16733ddb4c58722aff1685cb3e60a6a2336992 Mon Sep 17 00:00:00 2001 From: Evan Hubinger Date: Thu, 23 Nov 2023 17:47:42 -0800 Subject: [PATCH 103/121] Update to require new cPyparsing --- coconut/constants.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/coconut/constants.py b/coconut/constants.py index 5fd307892..694c394ec 100644 --- a/coconut/constants.py +++ b/coconut/constants.py @@ -1008,7 +1008,7 @@ def get_path_env_var(env_var, default): # min versions are inclusive unpinned_min_versions = { - "cPyparsing": (2, 4, 7, 2, 2, 9), + "cPyparsing": (2, 4, 7, 2, 3, 0), ("pre-commit", "py3"): (3,), ("psutil", "py>=27"): (5,), "jupyter": (1, 0), From 68cdcff40a1b1e0dd76cc0588e6f042f8058366a Mon Sep 17 00:00:00 2001 From: Evan Hubinger Date: Thu, 23 Nov 2023 23:23:50 -0800 Subject: [PATCH 104/121] Clean up docs --- DOCS.md | 2 +- coconut/tests/main_test.py | 9 +++++---- 2 files changed, 6 insertions(+), 5 deletions(-) diff --git a/DOCS.md b/DOCS.md index 0bd5b94b4..44341734f 100644 --- a/DOCS.md +++ b/DOCS.md @@ -4716,7 +4716,7 @@ Switches the [`breakpoint` built-in](https://www.python.org/dev/peps/pep-0553/) Both functions behave identically to [`setuptools.find_packages`](https://setuptools.pypa.io/en/latest/userguide/quickstart.html#package-discovery), except that they find Coconut packages rather than Python packages. `find_and_compile_packages` additionally compiles any Coconut packages that it finds in-place. -Note that if you want to use either of these functions in your `setup.py`, you'll need to include `coconut` as a [build-time dependency in your `pyproject.toml`](https://pip.pypa.io/en/stable/reference/build-system/pyproject-toml/#build-time-dependencies). +Note that if you want to use either of these functions in your `setup.py`, you'll need to include `coconut` as a [build-time dependency in your `pyproject.toml`](https://pip.pypa.io/en/stable/reference/build-system/pyproject-toml/#build-time-dependencies). If you want `setuptools` to package your Coconut files, you'll also need to add `global-include *.coco` to your [`MANIFEST.in`](https://setuptools.pypa.io/en/latest/userguide/miscellaneous.html). ##### Example diff --git a/coconut/tests/main_test.py b/coconut/tests/main_test.py index 3a762c726..bf6c26bf5 100644 --- a/coconut/tests/main_test.py +++ b/coconut/tests/main_test.py @@ -89,6 +89,9 @@ os.environ["PYDEVD_DISABLE_FILE_VALIDATION"] = "1" +# run fewer tests on Windows so appveyor doesn't time out +TEST_ALL = get_bool_env_var("COCONUT_TEST_ALL", not WINDOWS) + # ----------------------------------------------------------------------------------------------------------------------- # CONSTANTS: @@ -984,8 +987,7 @@ def test_no_tco(self): def test_no_wrap(self): run(["--no-wrap"]) - # run fewer tests on Windows so appveyor doesn't time out - if not WINDOWS: + if TEST_ALL: if CPYTHON: def test_any_of(self): with using_env_vars({ @@ -1024,8 +1026,7 @@ def test_trace(self): run(["--jobs", "0", "--trace"], check_errors=False) -# more appveyor timeout prevention -if not WINDOWS: +if TEST_ALL: @add_test_func_names class TestExternal(unittest.TestCase): From 58e2aba24b54f80a0be921164148b7fa1c1aaff5 Mon Sep 17 00:00:00 2001 From: Evan Hubinger Date: Fri, 24 Nov 2023 15:04:53 -0800 Subject: [PATCH 105/121] Fix stdin reading --- coconut/command/util.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/coconut/command/util.py b/coconut/command/util.py index 4419c88a7..7635f2612 100644 --- a/coconut/command/util.py +++ b/coconut/command/util.py @@ -492,7 +492,7 @@ def stdin_readable(): """Determine whether stdin has any data to read.""" stdin_is_empty = is_empty_pipe(sys.stdin) if stdin_is_empty is not None: - return stdin_is_empty + return not stdin_is_empty # by default assume not readable return not isatty(sys.stdin, default=True) From 198387146b81ef28e223d550f13aa2c862a382b4 Mon Sep 17 00:00:00 2001 From: Evan Hubinger Date: Fri, 24 Nov 2023 16:16:35 -0800 Subject: [PATCH 106/121] More fixes --- coconut/command/util.py | 8 ++++++-- coconut/icoconut/root.py | 7 ++++--- coconut/tests/src/cocotest/agnostic/suite.coco | 2 ++ coconut/tests/src/cocotest/agnostic/util.coco | 8 ++++++++ coconut/util.py | 7 +++++++ 5 files changed, 27 insertions(+), 5 deletions(-) diff --git a/coconut/command/util.py b/coconut/command/util.py index 7635f2612..a07d7c1e4 100644 --- a/coconut/command/util.py +++ b/coconut/command/util.py @@ -591,10 +591,14 @@ def import_coconut_header(): except ImportError: # fixes an issue where, when running from the base coconut directory, # the base coconut directory is treated as a namespace package - if os.path.basename(os.getcwd()) == "coconut": + try: from coconut.coconut import __coconut__ + except ImportError: + __coconut__ = None + if __coconut__ is not None: return __coconut__ - raise + else: + raise # the original ImportError, since that's the normal one # ----------------------------------------------------------------------------------------------------------------------- diff --git a/coconut/icoconut/root.py b/coconut/icoconut/root.py index 5d658e28e..0b0cb77f9 100644 --- a/coconut/icoconut/root.py +++ b/coconut/icoconut/root.py @@ -42,9 +42,10 @@ code_exts, conda_build_env_var, coconut_kernel_kwargs, + default_whitespace_chars, ) from coconut.terminal import logger -from coconut.util import override, memoize_with_exceptions +from coconut.util import override, memoize_with_exceptions, replace_all from coconut.compiler import Compiler from coconut.compiler.util import should_indent from coconut.command.util import Runner @@ -160,7 +161,7 @@ def _coconut_compile(self, source, *args, **kwargs): """Version of _compile that checks Coconut code. None means that the code should not be run as is. Any other value means that it can.""" - if source.replace(" ", "").endswith("\n\n"): + if replace_all(source, default_whitespace_chars, "").endswith("\n\n"): return True elif should_indent(source): return None @@ -247,7 +248,7 @@ class CoconutKernel(IPythonKernel, object): "version": VERSION, "mimetype": mimetype, "codemirror_mode": { - "name": "python", + "name": "ipython", "version": py_syntax_version, }, "pygments_lexer": "coconut", diff --git a/coconut/tests/src/cocotest/agnostic/suite.coco b/coconut/tests/src/cocotest/agnostic/suite.coco index 992a0fce1..10cc14a66 100644 --- a/coconut/tests/src/cocotest/agnostic/suite.coco +++ b/coconut/tests/src/cocotest/agnostic/suite.coco @@ -1068,6 +1068,8 @@ forward 2""") == 900 assert x_or_y(x=1) == (1, 1) == x_or_y(y=1) assert DerivedWithMeths().cls_meth() assert DerivedWithMeths().static_meth() + assert Fibs()[100] == 354224848179261915075 + assert tree_depth(Node(Leaf 5, Node(Node(Leaf 10)))) == 3 with process_map.multiple_sequential_calls(): # type: ignore assert process_map(tuple <.. (|>)$(to_sort), qsorts) |> list == [to_sort |> sorted |> tuple] * len(qsorts) diff --git a/coconut/tests/src/cocotest/agnostic/util.coco b/coconut/tests/src/cocotest/agnostic/util.coco index c16ff70f1..c06598be7 100644 --- a/coconut/tests/src/cocotest/agnostic/util.coco +++ b/coconut/tests/src/cocotest/agnostic/util.coco @@ -766,6 +766,14 @@ def depth_2(t): match tree(l=l, r=r) in t: return 1 + max([depth_2(l), depth_2(r)]) +class Tree +data Node(*children) from Tree +data Leaf(elem) from Tree + +def tree_depth(Leaf(_)) = 0 +addpattern def tree_depth(Node(*children)) = # type: ignore + children |> map$(tree_depth) |> max |> (.+1) + # Monads: def base_maybe(x, f) = f(x) if x is not None else None def maybes(*fs) = reduce(base_maybe, fs) diff --git a/coconut/util.py b/coconut/util.py index 15091dd85..ed32f5547 100644 --- a/coconut/util.py +++ b/coconut/util.py @@ -314,6 +314,13 @@ def split_trailing_whitespace(inputstr): return basestr, whitespace +def replace_all(inputstr, all_to_replace, replace_to): + """Replace everything in all_to_replace with replace_to in inputstr.""" + for to_replace in all_to_replace: + inputstr = inputstr.replace(to_replace, replace_to) + return inputstr + + # ----------------------------------------------------------------------------------------------------------------------- # VERSIONING: # ----------------------------------------------------------------------------------------------------------------------- From 6f17751acca5e32471d7632fc9f6ea3b3f58f1dd Mon Sep 17 00:00:00 2001 From: Evan Hubinger Date: Fri, 24 Nov 2023 17:12:25 -0800 Subject: [PATCH 107/121] Add test --- coconut/tests/src/cocotest/agnostic/primary_2.coco | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/coconut/tests/src/cocotest/agnostic/primary_2.coco b/coconut/tests/src/cocotest/agnostic/primary_2.coco index 435cfbf88..f0b3c7e72 100644 --- a/coconut/tests/src/cocotest/agnostic/primary_2.coco +++ b/coconut/tests/src/cocotest/agnostic/primary_2.coco @@ -9,6 +9,9 @@ from importlib import reload # NOQA from .util import assert_raises, typed_eq +operator ! +from math import factorial as (!) + def primary_test_2() -> bool: """Basic no-dependency tests (2/2).""" @@ -403,6 +406,7 @@ def primary_test_2() -> bool: assert ident$(x=?).__name__ == "ident" == ident$(1).__name__ # type: ignore assert collectby(.[0], [(0, 1), (0, 2)], value_func=.[1], reduce_func=(+), reduce_func_init=1) == {0: 4} assert ident$(1, ?) |> type == ident$(1) |> type + assert 10! == 3628800 with process_map.multiple_sequential_calls(): # type: ignore assert map((+), range(3), range(4)$[:-1], strict=True) |> list == [0, 2, 4] == process_map((+), range(3), range(4)$[:-1], strict=True) |> list # type: ignore From db22bcd7ce14d023f110630558146d9782a63541 Mon Sep 17 00:00:00 2001 From: Evan Hubinger Date: Fri, 24 Nov 2023 17:15:37 -0800 Subject: [PATCH 108/121] Fix py2 --- coconut/command/util.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/coconut/command/util.py b/coconut/command/util.py index a07d7c1e4..98fcf6dcd 100644 --- a/coconut/command/util.py +++ b/coconut/command/util.py @@ -24,14 +24,15 @@ import subprocess import shutil import threading -import queue from select import select from contextlib import contextmanager from functools import partial if PY2: import __builtin__ as builtins + import Queue as queue else: import builtins + import queue from coconut.root import _coconut_exec from coconut.terminal import ( From 895a0f1da46d18e36b465186ef14b8567f0a5bbc Mon Sep 17 00:00:00 2001 From: Evan Hubinger Date: Fri, 24 Nov 2023 20:13:43 -0800 Subject: [PATCH 109/121] Fix fancy call_output --- coconut/command/util.py | 22 ++++++++++++++++------ coconut/constants.py | 3 ++- 2 files changed, 18 insertions(+), 7 deletions(-) diff --git a/coconut/command/util.py b/coconut/command/util.py index 98fcf6dcd..cd8750a45 100644 --- a/coconut/command/util.py +++ b/coconut/command/util.py @@ -87,6 +87,7 @@ coconut_base_run_args, high_proc_prio, call_timeout, + use_fancy_call_output, ) if PY26: @@ -293,7 +294,7 @@ def interrupt_thread(thread, exctype=OSError): def readline_to_queue(file_obj, q): """Read a line from file_obj and put it in the queue.""" - if not is_empty_pipe(file_obj): + if not is_empty_pipe(file_obj, False): try: q.put(file_obj.readline()) except OSError: @@ -307,7 +308,7 @@ def call_output(cmd, stdin=None, encoding_errors="replace", color=None, **kwargs stdout_q = queue.Queue() stderr_q = queue.Queue() - if WINDOWS or not logger.verbose: + if use_fancy_call_output: raw_stdout, raw_stderr = p.communicate(stdin) stdout_q.put(raw_stdout) stderr_q.put(raw_stderr) @@ -324,7 +325,13 @@ def call_output(cmd, stdin=None, encoding_errors="replace", color=None, **kwargs stdout, stderr, retcode = [], [], None checking_stdout = True # alternate between stdout and stderr try: - while retcode is None or not stdout_q.empty() or not stderr_q.empty(): + while ( + retcode is None + or not stdout_q.empty() + or not stderr_q.empty() + or not is_empty_pipe(p.stdout, True) + or not is_empty_pipe(p.stderr, True) + ): if checking_stdout: proc_pipe = p.stdout sys_pipe = sys.stdout @@ -342,7 +349,10 @@ def call_output(cmd, stdin=None, encoding_errors="replace", color=None, **kwargs retcode = p.poll() - if retcode is None and t_obj[0] is not False: + if ( + retcode is None + or not is_empty_pipe(proc_pipe, True) + ): if t_obj[0] is None or not t_obj[0].is_alive(): t_obj[0] = threading.Thread(target=readline_to_queue, args=(proc_pipe, q)) t_obj[0].daemon = True @@ -479,14 +489,14 @@ def set_mypy_path(): return install_dir -def is_empty_pipe(pipe): +def is_empty_pipe(pipe, default=None): """Determine if the given pipe file object is empty.""" if not WINDOWS: try: return not select([pipe], [], [], 0)[0] except Exception: logger.log_exc() - return None + return default def stdin_readable(): diff --git a/coconut/constants.py b/coconut/constants.py index 694c394ec..e7761b693 100644 --- a/coconut/constants.py +++ b/coconut/constants.py @@ -746,7 +746,8 @@ def get_path_env_var(env_var, default): create_package_retries = 1 -call_timeout = 0.001 +use_fancy_call_output = get_bool_env_var("COCONUT_FANCY_CALL_OUTPUT", not WINDOWS) +call_timeout = 0.01 max_orig_lines_in_log_loc = 2 From 3651477e013d62aa23d6a4fe7621f8d1485cafff Mon Sep 17 00:00:00 2001 From: Evan Hubinger Date: Fri, 24 Nov 2023 20:20:34 -0800 Subject: [PATCH 110/121] Further fix py2 --- coconut/command/util.py | 2 +- coconut/tests/src/cocotest/agnostic/primary_2.coco | 3 --- coconut/tests/src/cocotest/agnostic/specific.coco | 3 +++ 3 files changed, 4 insertions(+), 4 deletions(-) diff --git a/coconut/command/util.py b/coconut/command/util.py index cd8750a45..b288ac615 100644 --- a/coconut/command/util.py +++ b/coconut/command/util.py @@ -308,7 +308,7 @@ def call_output(cmd, stdin=None, encoding_errors="replace", color=None, **kwargs stdout_q = queue.Queue() stderr_q = queue.Queue() - if use_fancy_call_output: + if not use_fancy_call_output: raw_stdout, raw_stderr = p.communicate(stdin) stdout_q.put(raw_stdout) stderr_q.put(raw_stderr) diff --git a/coconut/tests/src/cocotest/agnostic/primary_2.coco b/coconut/tests/src/cocotest/agnostic/primary_2.coco index f0b3c7e72..2f0802b17 100644 --- a/coconut/tests/src/cocotest/agnostic/primary_2.coco +++ b/coconut/tests/src/cocotest/agnostic/primary_2.coco @@ -385,10 +385,7 @@ def primary_test_2() -> bool: assert list(xs) == list(zip(range(5), range(5))) assert list(xs) == [] xs = map((.+1), range(5)) - py_xs = py_map((.+1), range(5)) assert list(xs) == list(range(1, 6)) == list(xs) - assert list(py_xs) == list(range(1, 6)) - assert list(py_xs) == [] assert count()[:10:2] == range(0, 10, 2) assert count()[10:2] == range(10, 2) some_data = [ diff --git a/coconut/tests/src/cocotest/agnostic/specific.coco b/coconut/tests/src/cocotest/agnostic/specific.coco index 57f573d4d..f72873c04 100644 --- a/coconut/tests/src/cocotest/agnostic/specific.coco +++ b/coconut/tests/src/cocotest/agnostic/specific.coco @@ -48,6 +48,9 @@ def py3_spec_test() -> bool: ]: # type: ignore assert list(xs) == list(zip(range(5), range(5))) assert list(xs) == [] if sys.version_info >= (3,) else list(zip(range(5), range(5))) + py_xs = py_map((.+1), range(5)) + assert list(py_xs) == list(range(1, 6)) + assert list(py_xs) == [] return True From 7a1169f39f1365ce028fb4c26e958a4f15ec8724 Mon Sep 17 00:00:00 2001 From: Evan Hubinger Date: Sat, 25 Nov 2023 15:07:03 -0800 Subject: [PATCH 111/121] Disable fancy call_output --- coconut/constants.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/coconut/constants.py b/coconut/constants.py index e7761b693..560364dc8 100644 --- a/coconut/constants.py +++ b/coconut/constants.py @@ -746,7 +746,7 @@ def get_path_env_var(env_var, default): create_package_retries = 1 -use_fancy_call_output = get_bool_env_var("COCONUT_FANCY_CALL_OUTPUT", not WINDOWS) +use_fancy_call_output = get_bool_env_var("COCONUT_FANCY_CALL_OUTPUT", False) call_timeout = 0.01 max_orig_lines_in_log_loc = 2 From 4c5027558bc6151220a4d5ba33a6625a53cd500d Mon Sep 17 00:00:00 2001 From: Evan Hubinger Date: Sat, 25 Nov 2023 17:17:54 -0800 Subject: [PATCH 112/121] Fix py2 pickling --- coconut/command/util.py | 2 + coconut/compiler/compiler.py | 6 ++- coconut/constants.py | 2 +- coconut/root.py | 52 ++++++++++++++++++- coconut/tests/src/cocotest/agnostic/main.coco | 18 ++++--- .../tests/src/cocotest/agnostic/suite.coco | 5 ++ coconut/tests/src/extras.coco | 4 +- coconut/util.py | 3 ++ 8 files changed, 77 insertions(+), 15 deletions(-) diff --git a/coconut/command/util.py b/coconut/command/util.py index b288ac615..53cb00bfb 100644 --- a/coconut/command/util.py +++ b/coconut/command/util.py @@ -491,6 +491,8 @@ def set_mypy_path(): def is_empty_pipe(pipe, default=None): """Determine if the given pipe file object is empty.""" + if pipe.closed: + return True if not WINDOWS: try: return not select([pipe], [], [], 0)[0] diff --git a/coconut/compiler/compiler.py b/coconut/compiler/compiler.py index cdce7ff58..24307a965 100644 --- a/coconut/compiler/compiler.py +++ b/coconut/compiler/compiler.py @@ -1210,9 +1210,11 @@ def make_err(self, errtype, message, original, loc=0, ln=None, extra=None, refor self.internal_assert(extra is None, original, loc, "make_err cannot include causes with extra") causes = dictset() for cause, _, _ in all_matches(self.parse_err_msg, snippet[loc_in_snip:]): - causes.add(cause) + if cause: + causes.add(cause) for cause, _, _ in all_matches(self.parse_err_msg, snippet[endpt_in_snip:]): - causes.add(cause) + if cause: + causes.add(cause) if causes: extra = "possible cause{s}: {causes}".format( s="s" if len(causes) > 1 else "", diff --git a/coconut/constants.py b/coconut/constants.py index 560364dc8..1f32d3970 100644 --- a/coconut/constants.py +++ b/coconut/constants.py @@ -1009,7 +1009,7 @@ def get_path_env_var(env_var, default): # min versions are inclusive unpinned_min_versions = { - "cPyparsing": (2, 4, 7, 2, 3, 0), + "cPyparsing": (2, 4, 7, 2, 3, 1), ("pre-commit", "py3"): (3,), ("psutil", "py>=27"): (5,), "jupyter": (1, 0), diff --git a/coconut/root.py b/coconut/root.py index d24f5a38d..fca30777b 100644 --- a/coconut/root.py +++ b/coconut/root.py @@ -26,8 +26,8 @@ VERSION = "3.0.4" VERSION_NAME = None # False for release, int >= 1 for develop -DEVELOP = False -ALPHA = False # for pre releases rather than post releases +DEVELOP = 1 +ALPHA = True # for pre releases rather than post releases assert DEVELOP is False or DEVELOP >= 1, "DEVELOP must be False or an int >= 1" assert DEVELOP or not ALPHA, "alpha releases are only for develop" @@ -208,6 +208,54 @@ def _coconut_exec(obj, globals=None, locals=None): if globals is None: globals = _coconut_sys._getframe(1).f_globals exec(obj, globals, locals) +import operator as _coconut_operator +class _coconut_attrgetter(object): + __slots__ = ("attrs",) + def __init__(self, *attrs): + self.attrs = attrs + def __reduce_ex__(self, _): + return self.__reduce__() + def __reduce__(self): + return (self.__class__, self.attrs) + @staticmethod + def _getattr(obj, attr): + for name in attr.split("."): + obj = _coconut.getattr(obj, name) + return obj + def __call__(self, obj): + if len(self.attrs) == 1: + return self._getattr(obj, self.attrs[0]) + return _coconut.tuple(self._getattr(obj, attr) for attr in self.attrs) +_coconut_operator.attrgetter = _coconut_attrgetter +class _coconut_itemgetter(object): + __slots__ = ("items",) + def __init__(self, *items): + self.items = items + def __reduce_ex__(self, _): + return self.__reduce__() + def __reduce__(self): + return (self.__class__, self.items) + def __call__(self, obj): + if len(self.items) == 1: + return obj[self.items[0]] + return _coconut.tuple(obj[item] for item in self.items) +_coconut_operator.itemgetter = _coconut_itemgetter +class _coconut_methodcaller(object): + __slots__ = ("name", "args", "kwargs") + def __init__(self, name, *args, **kwargs): + self.name = name + self.args = args + self.kwargs = kwargs + def __reduce_ex__(self, _): + return self.__reduce__() + def __reduce__(self): + return (self.__class__, (self.name,) + self.args, {"kwargs": self.kwargs}) + def __setstate__(self, setvars): + for k, v in setvars.items(): + _coconut.setattr(self, k, v) + def __call__(self, obj): + return _coconut.getattr(obj, self.name)(*self.args, **self.kwargs) +_coconut_operator.methodcaller = _coconut_methodcaller ''' _non_py37_extras = r'''def _coconut_default_breakpointhook(*args, **kwargs): diff --git a/coconut/tests/src/cocotest/agnostic/main.coco b/coconut/tests/src/cocotest/agnostic/main.coco index 78c0baa5f..97c9d3df7 100644 --- a/coconut/tests/src/cocotest/agnostic/main.coco +++ b/coconut/tests/src/cocotest/agnostic/main.coco @@ -52,9 +52,11 @@ def run_main(outer_MatchError, test_easter_eggs=False) -> bool: print_dot() # .. assert primary_test_1() is True - assert primary_test_2() is True print_dot() # ... + assert primary_test_2() is True + + print_dot() # .... from .specific import ( non_py26_test, non_py32_test, @@ -79,11 +81,11 @@ def run_main(outer_MatchError, test_easter_eggs=False) -> bool: if sys.version_info >= (3, 8): assert py38_spec_test() is True - print_dot() # .... + print_dot() # ..... from .suite import suite_test, tco_test assert suite_test() is True - print_dot() # ..... + print_dot() # ...... assert mypy_test() is True if using_tco: assert hasattr(tco_func, "_coconut_tco_func") @@ -91,7 +93,7 @@ def run_main(outer_MatchError, test_easter_eggs=False) -> bool: if outer_MatchError.__module__ != "__main__": assert package_test(outer_MatchError) is True - print_dot() # ...... + print_dot() # ....... if sys.version_info < (3,): from .py2_test import py2_test assert py2_test() is True @@ -111,21 +113,21 @@ def run_main(outer_MatchError, test_easter_eggs=False) -> bool: from .py311_test import py311_test assert py311_test() is True - print_dot() # ....... + print_dot() # ........ from .target_sys_test import TEST_ASYNCIO, target_sys_test if TEST_ASYNCIO: assert test_asyncio() is True assert target_sys_test() is True - print_dot() # ........ + print_dot() # ......... from .non_strict_test import non_strict_test assert non_strict_test() is True - print_dot() # ......... + print_dot() # .......... from . import tutorial # noQA if test_easter_eggs: - print(".", end="") # .......... + print_dot() # ........... assert easter_egg_test() is True print("\n") diff --git a/coconut/tests/src/cocotest/agnostic/suite.coco b/coconut/tests/src/cocotest/agnostic/suite.coco index 10cc14a66..813fe05b0 100644 --- a/coconut/tests/src/cocotest/agnostic/suite.coco +++ b/coconut/tests/src/cocotest/agnostic/suite.coco @@ -1070,6 +1070,11 @@ forward 2""") == 900 assert DerivedWithMeths().static_meth() assert Fibs()[100] == 354224848179261915075 assert tree_depth(Node(Leaf 5, Node(Node(Leaf 10)))) == 3 + assert pickle_round_trip(.name) <| (name=10) == 10 + assert pickle_round_trip(.[0])([10]) == 10 + assert pickle_round_trip(.loc[0]) <| (loc=[10]) == 10 + assert pickle_round_trip(.method(0)) <| (method=const 10) == 10 + assert pickle_round_trip(.method(x=10)) <| (method=x -> x) == 10 with process_map.multiple_sequential_calls(): # type: ignore assert process_map(tuple <.. (|>)$(to_sort), qsorts) |> list == [to_sort |> sorted |> tuple] * len(qsorts) diff --git a/coconut/tests/src/extras.coco b/coconut/tests/src/extras.coco index bf5ded5f1..c283ab011 100644 --- a/coconut/tests/src/extras.coco +++ b/coconut/tests/src/extras.coco @@ -217,7 +217,7 @@ cannot reassign type variable 'T' (use explicit '\T' syntax if intended) (line 1 ) assert_raises(-> parse("$"), CoconutParseError) - assert_raises(-> parse("@"), CoconutParseError, err_has=("\n ~^", "\n ^")) + assert_raises(-> parse("@"), CoconutParseError) assert_raises(-> parse("range(1,10) |> reduce$(*, initializer = 1000) |> print"), CoconutParseError, err_has=" \\~~~~~~~~~~~~~~~~~~~~~~~^") assert_raises(-> parse("a := b"), CoconutParseError, err_has=" \\~^") assert_raises(-> parse("1 + return"), CoconutParseError, err_has=" \\~~~~^") @@ -362,7 +362,7 @@ else: assert_raises(-> parse("obj."), CoconutStyleError, err_has="getattr") assert_raises(-> parse("def x -> pass, 1"), CoconutStyleError, err_has="statement lambda") assert_raises(-> parse("abc = f'abc'"), CoconutStyleError, err_has="\n ^") - assert_raises(-> parse('f"{f"{f"infinite"}"}"'), CoconutStyleError, err_has=" ^~~~~~~~~~~|") + assert_raises(-> parse('f"{f"{f"infinite"}"}"'), CoconutStyleError, err_has="f-string with no expressions") try: parse(""" import abc diff --git a/coconut/util.py b/coconut/util.py index ed32f5547..b0e04be68 100644 --- a/coconut/util.py +++ b/coconut/util.py @@ -265,6 +265,9 @@ def __missing__(self, key): class dictset(dict, object): """A set implemented using a dictionary to get ordering benefits.""" + def __bool__(self): + return len(self) > 0 # fixes py2 issue + def add(self, item): self[item] = True From 84cc54f132f348a3d435ddbcc81806e23ef2f59e Mon Sep 17 00:00:00 2001 From: Evan Hubinger Date: Sat, 25 Nov 2023 18:03:50 -0800 Subject: [PATCH 113/121] Fix extras test --- coconut/tests/src/extras.coco | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/coconut/tests/src/extras.coco b/coconut/tests/src/extras.coco index c283ab011..034fa30b4 100644 --- a/coconut/tests/src/extras.coco +++ b/coconut/tests/src/extras.coco @@ -218,7 +218,10 @@ cannot reassign type variable 'T' (use explicit '\T' syntax if intended) (line 1 assert_raises(-> parse("$"), CoconutParseError) assert_raises(-> parse("@"), CoconutParseError) - assert_raises(-> parse("range(1,10) |> reduce$(*, initializer = 1000) |> print"), CoconutParseError, err_has=" \\~~~~~~~~~~~~~~~~~~~~~~~^") + assert_raises(-> parse("range(1,10) |> reduce$(*, initializer = 1000) |> print"), CoconutParseError, err_has=( + " \\~~~~~~~~~~~~~~~~~~~~~~~^", + " \\~~~~~~~~~~~~^", + )) assert_raises(-> parse("a := b"), CoconutParseError, err_has=" \\~^") assert_raises(-> parse("1 + return"), CoconutParseError, err_has=" \\~~~~^") assert_raises(-> parse(""" From aa6c431ccd5fc66caaded8d8e9f7e9ac7ef905f0 Mon Sep 17 00:00:00 2001 From: Evan Hubinger Date: Sat, 25 Nov 2023 22:05:27 -0800 Subject: [PATCH 114/121] Fix parsing numbers --- coconut/compiler/grammar.py | 7 +++++-- coconut/compiler/util.py | 9 ++++++--- coconut/constants.py | 8 +++----- coconut/root.py | 2 +- coconut/tests/constants_test.py | 4 ++++ coconut/tests/src/cocotest/agnostic/primary_2.coco | 4 ++++ 6 files changed, 23 insertions(+), 11 deletions(-) diff --git a/coconut/compiler/grammar.py b/coconut/compiler/grammar.py index e545d56f7..958e8afe3 100644 --- a/coconut/compiler/grammar.py +++ b/coconut/compiler/grammar.py @@ -801,7 +801,11 @@ class Grammar(object): | integer + Optional(dot + Optional(integer)) ) sci_e = combine((caseless_literal("e") | fixto(Literal("\u23e8"), "e")) + Optional(plus | neg_minus)) - numitem = ~(Literal("0") + Word(nums + "_", exact=1)) + combine(basenum + Optional(sci_e + integer)) + numitem = combine( + # don't match 0_, 0b_, 0o_, or 0x_ + regex_item(r"(?!0([0-9_]|[box][0-9_]))").suppress() + + basenum + Optional(sci_e + integer) + ) imag_num = combine(numitem + imag_j) maybe_imag_num = combine(numitem + Optional(imag_j)) bin_num = combine(caseless_literal("0b") + Optional(underscore.suppress()) + binint) @@ -812,7 +816,6 @@ class Grammar(object): hex_num, bin_num, oct_num, - use_adaptive=False, ) # make sure that this gets addspaced not condensed so it doesn't produce a SyntaxError num_atom = addspace(number + Optional(condense(dot + unsafe_name))) diff --git a/coconut/compiler/util.py b/coconut/compiler/util.py index 207396a2a..6b7d11885 100644 --- a/coconut/compiler/util.py +++ b/coconut/compiler/util.py @@ -1108,6 +1108,9 @@ def __str__(self): def any_of(*exprs, **kwargs): """Build a MatchAny of the given MatchFirst.""" use_adaptive = kwargs.pop("use_adaptive", use_adaptive_any_of) and SUPPORTS_ADAPTIVE + reverse = reverse_any_of + if DEVELOP: + reverse = kwargs.pop("reverse", reverse) internal_assert(not kwargs, "excess keyword arguments passed to any_of", kwargs) AnyOf = MatchAny if use_adaptive else MatchFirst @@ -1116,7 +1119,7 @@ def any_of(*exprs, **kwargs): for e in exprs: if ( # don't merge MatchFirsts when we're reversing - not (reverse_any_of and not use_adaptive) + not (reverse and not use_adaptive) and e.__class__ == AnyOf and not hasaction(e) ): @@ -1124,7 +1127,7 @@ def any_of(*exprs, **kwargs): else: flat_exprs.append(e) - if reverse_any_of: + if reverse: flat_exprs = reversed([trace(e) for e in exprs]) return AnyOf(flat_exprs) @@ -1228,7 +1231,7 @@ def manage_elem(self, original, loc): raise ParseException(original, loc, self.errmsg, self) for elem in elems: - yield Wrap(elem, manage_elem, include_in_packrat_context=True) + yield Wrap(elem, manage_elem) def disable_outside(item, *elems): diff --git a/coconut/constants.py b/coconut/constants.py index 1f32d3970..9029a8bef 100644 --- a/coconut/constants.py +++ b/coconut/constants.py @@ -109,8 +109,7 @@ def get_path_env_var(env_var, default): # ----------------------------------------------------------------------------------------------------------------------- # set this to False only ever temporarily for ease of debugging -use_fast_pyparsing_reprs = True -assert use_fast_pyparsing_reprs or DEVELOP, "use_fast_pyparsing_reprs should never be disabled on non-develop build" +use_fast_pyparsing_reprs = get_bool_env_var("COCONUT_FAST_PYPARSING_REPRS", True) enable_pyparsing_warnings = DEVELOP warn_on_multiline_regex = False @@ -168,8 +167,7 @@ def get_path_env_var(env_var, default): # ----------------------------------------------------------------------------------------------------------------------- # set this to True only ever temporarily for ease of debugging -embed_on_internal_exc = False -assert not embed_on_internal_exc or DEVELOP, "embed_on_internal_exc should never be enabled on non-develop build" +embed_on_internal_exc = get_bool_env_var("COCONUT_EMBED_ON_INTERNAL_EXC", False) # should be the minimal ref count observed by maybe_copy_elem temp_grammar_item_ref_count = 4 if PY311 else 5 @@ -1009,7 +1007,7 @@ def get_path_env_var(env_var, default): # min versions are inclusive unpinned_min_versions = { - "cPyparsing": (2, 4, 7, 2, 3, 1), + "cPyparsing": (2, 4, 7, 2, 3, 2), ("pre-commit", "py3"): (3,), ("psutil", "py>=27"): (5,), "jupyter": (1, 0), diff --git a/coconut/root.py b/coconut/root.py index fca30777b..5bedf4b40 100644 --- a/coconut/root.py +++ b/coconut/root.py @@ -26,7 +26,7 @@ VERSION = "3.0.4" VERSION_NAME = None # False for release, int >= 1 for develop -DEVELOP = 1 +DEVELOP = 2 ALPHA = True # for pre releases rather than post releases assert DEVELOP is False or DEVELOP >= 1, "DEVELOP must be False or an int >= 1" diff --git a/coconut/tests/constants_test.py b/coconut/tests/constants_test.py index e301e69d0..eb3250b29 100644 --- a/coconut/tests/constants_test.py +++ b/coconut/tests/constants_test.py @@ -78,6 +78,10 @@ def is_importable(name): class TestConstants(unittest.TestCase): + def test_defaults(self): + assert constants.use_fast_pyparsing_reprs + assert not constants.embed_on_internal_exc + def test_fixpath(self): assert os.path.basename(fixpath("CamelCase.py")) == "CamelCase.py" diff --git a/coconut/tests/src/cocotest/agnostic/primary_2.coco b/coconut/tests/src/cocotest/agnostic/primary_2.coco index 2f0802b17..d2679dae3 100644 --- a/coconut/tests/src/cocotest/agnostic/primary_2.coco +++ b/coconut/tests/src/cocotest/agnostic/primary_2.coco @@ -404,6 +404,10 @@ def primary_test_2() -> bool: assert collectby(.[0], [(0, 1), (0, 2)], value_func=.[1], reduce_func=(+), reduce_func_init=1) == {0: 4} assert ident$(1, ?) |> type == ident$(1) |> type assert 10! == 3628800 + assert 0x100 == 256 == 0o400 + assert 0x0 == 0 == 0b0 + x = 10 + assert 0x == 0 == 0 x with process_map.multiple_sequential_calls(): # type: ignore assert map((+), range(3), range(4)$[:-1], strict=True) |> list == [0, 2, 4] == process_map((+), range(3), range(4)$[:-1], strict=True) |> list # type: ignore From 1721293294c2db33335fe4b4579691d4fa3505d2 Mon Sep 17 00:00:00 2001 From: Evan Hubinger Date: Sun, 26 Nov 2023 02:35:20 -0800 Subject: [PATCH 115/121] Further fix number parsing --- coconut/compiler/grammar.py | 17 +++++++---------- coconut/root.py | 2 +- .../tests/src/cocotest/agnostic/primary_2.coco | 2 ++ 3 files changed, 10 insertions(+), 11 deletions(-) diff --git a/coconut/compiler/grammar.py b/coconut/compiler/grammar.py index 958e8afe3..076b9d5ec 100644 --- a/coconut/compiler/grammar.py +++ b/coconut/compiler/grammar.py @@ -801,21 +801,18 @@ class Grammar(object): | integer + Optional(dot + Optional(integer)) ) sci_e = combine((caseless_literal("e") | fixto(Literal("\u23e8"), "e")) + Optional(plus | neg_minus)) - numitem = combine( - # don't match 0_, 0b_, 0o_, or 0x_ - regex_item(r"(?!0([0-9_]|[box][0-9_]))").suppress() - + basenum + Optional(sci_e + integer) - ) + numitem = combine(basenum + Optional(sci_e + integer)) imag_num = combine(numitem + imag_j) maybe_imag_num = combine(numitem + Optional(imag_j)) bin_num = combine(caseless_literal("0b") + Optional(underscore.suppress()) + binint) oct_num = combine(caseless_literal("0o") + Optional(underscore.suppress()) + octint) hex_num = combine(caseless_literal("0x") + Optional(underscore.suppress()) + hexint) - number = any_of( - maybe_imag_num, - hex_num, - bin_num, - oct_num, + number = ( + hex_num + | bin_num + | oct_num + # must come last to avoid matching "0" in "0b" + | maybe_imag_num ) # make sure that this gets addspaced not condensed so it doesn't produce a SyntaxError num_atom = addspace(number + Optional(condense(dot + unsafe_name))) diff --git a/coconut/root.py b/coconut/root.py index 5bedf4b40..fff01f6e2 100644 --- a/coconut/root.py +++ b/coconut/root.py @@ -26,7 +26,7 @@ VERSION = "3.0.4" VERSION_NAME = None # False for release, int >= 1 for develop -DEVELOP = 2 +DEVELOP = 3 ALPHA = True # for pre releases rather than post releases assert DEVELOP is False or DEVELOP >= 1, "DEVELOP must be False or an int >= 1" diff --git a/coconut/tests/src/cocotest/agnostic/primary_2.coco b/coconut/tests/src/cocotest/agnostic/primary_2.coco index d2679dae3..b4e55fb2e 100644 --- a/coconut/tests/src/cocotest/agnostic/primary_2.coco +++ b/coconut/tests/src/cocotest/agnostic/primary_2.coco @@ -408,6 +408,8 @@ def primary_test_2() -> bool: assert 0x0 == 0 == 0b0 x = 10 assert 0x == 0 == 0 x + assert 0xff == 255 == 0x100-1 + assert 11259375 == 0xabcdef with process_map.multiple_sequential_calls(): # type: ignore assert map((+), range(3), range(4)$[:-1], strict=True) |> list == [0, 2, 4] == process_map((+), range(3), range(4)$[:-1], strict=True) |> list # type: ignore From 055b1e5d70980fdbe781e92b442141325a0c8bd9 Mon Sep 17 00:00:00 2001 From: Evan Hubinger Date: Sun, 26 Nov 2023 15:51:01 -0800 Subject: [PATCH 116/121] Fix failing tests --- DOCS.md | 2 +- coconut/compiler/grammar.py | 13 +++++--- coconut/compiler/util.py | 3 +- coconut/root.py | 2 +- coconut/tests/main_test.py | 62 +++++++++++++++++++++-------------- coconut/tests/src/extras.coco | 5 ++- 6 files changed, 54 insertions(+), 33 deletions(-) diff --git a/DOCS.md b/DOCS.md index 44341734f..1355ca8fb 100644 --- a/DOCS.md +++ b/DOCS.md @@ -2139,7 +2139,7 @@ Additionally, if the first argument is not callable, and is instead an `int`, `f Though the first item may be any atom, following arguments are highly restricted, and must be: - variables/attributes (e.g. `a.b`), - literal constants (e.g. `True`), -- number literals (e.g. `1.5`), or +- number literals (e.g. `1.5`) (and no binary, hex, or octal), or - one of the above followed by an exponent (e.g. `a**-5`). For example, `(f .. g) x 1` will work, but `f x [1]`, `f x (1+2)`, and `f "abc"` will not. diff --git a/coconut/compiler/grammar.py b/coconut/compiler/grammar.py index 076b9d5ec..7eaed6226 100644 --- a/coconut/compiler/grammar.py +++ b/coconut/compiler/grammar.py @@ -807,11 +807,15 @@ class Grammar(object): bin_num = combine(caseless_literal("0b") + Optional(underscore.suppress()) + binint) oct_num = combine(caseless_literal("0o") + Optional(underscore.suppress()) + octint) hex_num = combine(caseless_literal("0x") + Optional(underscore.suppress()) + hexint) + non_decimal_num = any_of( + hex_num, + bin_num, + oct_num, + use_adaptive=False, + ) number = ( - hex_num - | bin_num - | oct_num - # must come last to avoid matching "0" in "0b" + non_decimal_num + # must come last | maybe_imag_num ) # make sure that this gets addspaced not condensed so it doesn't produce a SyntaxError @@ -1404,6 +1408,7 @@ class Grammar(object): impl_call_item = condense( disallow_keywords(reserved_vars) + ~any_string + + ~non_decimal_num + atom_item + Optional(power_in_impl_call) ) diff --git a/coconut/compiler/util.py b/coconut/compiler/util.py index 6b7d11885..64a0ff84f 100644 --- a/coconut/compiler/util.py +++ b/coconut/compiler/util.py @@ -912,8 +912,9 @@ def pickle_cache(original, cache_path, include_incremental=True, protocol=pickle internal_assert(lambda: match_any == all_parse_elements[identifier](), "failed to look up match_any by identifier", (match_any, all_parse_elements[identifier]())) if validation_dict is not None: validation_dict[identifier] = match_any.__class__.__name__ + match_any.expr_order.sort(key=lambda i: (-match_any.adaptive_usage[i], i)) all_adaptive_stats[identifier] = (match_any.adaptive_usage, match_any.expr_order) - logger.log("Caching adaptive item:", match_any, "<-", all_adaptive_stats[identifier]) + logger.log("Caching adaptive item:", match_any, all_adaptive_stats[identifier]) logger.log("Saving {num_inc} incremental and {num_adapt} adaptive cache items to {cache_path!r}.".format( num_inc=len(pickleable_cache_items), diff --git a/coconut/root.py b/coconut/root.py index fff01f6e2..439f1e61e 100644 --- a/coconut/root.py +++ b/coconut/root.py @@ -26,7 +26,7 @@ VERSION = "3.0.4" VERSION_NAME = None # False for release, int >= 1 for develop -DEVELOP = 3 +DEVELOP = 4 ALPHA = True # for pre releases rather than post releases assert DEVELOP is False or DEVELOP >= 1, "DEVELOP must be False or an int >= 1" diff --git a/coconut/tests/main_test.py b/coconut/tests/main_test.py index bf6c26bf5..d0b9054fa 100644 --- a/coconut/tests/main_test.py +++ b/coconut/tests/main_test.py @@ -108,10 +108,10 @@ jupyter_timeout = 120 -base = os.path.dirname(os.path.relpath(__file__)) -src = os.path.join(base, "src") -dest = os.path.join(base, "dest") -additional_dest = os.path.join(base, "dest", "additional_dest") +tests_dir = os.path.dirname(os.path.relpath(__file__)) +src = os.path.join(tests_dir, "src") +dest = os.path.join(tests_dir, "dest") +additional_dest = os.path.join(tests_dir, "dest", "additional_dest") src_cache_dir = os.path.join(src, coconut_cache_dir) cocotest_dir = os.path.join(src, "cocotest") @@ -428,29 +428,25 @@ def rm_path(path, allow_keep=False): assert not base_dir.startswith(path), "refusing to delete Coconut itself: " + repr(path) if allow_keep and get_bool_env_var("COCONUT_KEEP_TEST_FILES"): return - if os.path.isdir(path): - try: + try: + if os.path.isdir(path): shutil.rmtree(path) - except OSError: - logger.print_exc() - elif os.path.isfile(path): - os.remove(path) + elif os.path.isfile(path): + os.remove(path) + except OSError: + logger.print_exc() @contextmanager def using_paths(*paths): """Removes paths at the beginning and end.""" for path in paths: - if os.path.exists(path): - rm_path(path) + rm_path(path) try: yield finally: for path in paths: - try: - rm_path(path, allow_keep=True) - except OSError: - logger.print_exc() + rm_path(path, allow_keep=True) @contextmanager @@ -465,10 +461,25 @@ def using_dest(dest=dest, allow_existing=False): try: yield finally: - try: - rm_path(dest, allow_keep=True) - except OSError: - logger.print_exc() + rm_path(dest, allow_keep=True) + + +def clean_caches(): + """Clean out all __coconut_cache__ dirs.""" + for dirpath, dirnames, filenames in os.walk(tests_dir): + for name in dirnames: + if name == coconut_cache_dir: + rm_path(os.path.join(dirpath, name)) + + +@contextmanager +def using_caches(): + """Cleans caches at start and end.""" + clean_caches() + try: + yield + finally: + clean_caches() @contextmanager @@ -990,11 +1001,12 @@ def test_no_wrap(self): if TEST_ALL: if CPYTHON: def test_any_of(self): - with using_env_vars({ - adaptive_any_of_env_var: "True", - reverse_any_of_env_var: "True", - }): - run() + with using_caches(): + with using_env_vars({ + adaptive_any_of_env_var: "True", + reverse_any_of_env_var: "True", + }): + run() def test_keep_lines(self): run(["--keep-lines"]) diff --git a/coconut/tests/src/extras.coco b/coconut/tests/src/extras.coco index 034fa30b4..9e65bf1a2 100644 --- a/coconut/tests/src/extras.coco +++ b/coconut/tests/src/extras.coco @@ -223,7 +223,10 @@ cannot reassign type variable 'T' (use explicit '\T' syntax if intended) (line 1 " \\~~~~~~~~~~~~^", )) assert_raises(-> parse("a := b"), CoconutParseError, err_has=" \\~^") - assert_raises(-> parse("1 + return"), CoconutParseError, err_has=" \\~~~~^") + assert_raises(-> parse("1 + return"), CoconutParseError, err_has=( + " \\~~~^", + " \\~~~~^", + )) assert_raises(-> parse(""" def f() = assert 1 From 9b0ee55c94741c02b8500d863a2755a0005772e1 Mon Sep 17 00:00:00 2001 From: Evan Hubinger Date: Sun, 26 Nov 2023 18:30:21 -0800 Subject: [PATCH 117/121] Fix extras tests --- coconut/tests/src/extras.coco | 45 ++++++++++++++++------------------- 1 file changed, 21 insertions(+), 24 deletions(-) diff --git a/coconut/tests/src/extras.coco b/coconut/tests/src/extras.coco index 9e65bf1a2..e9fe14109 100644 --- a/coconut/tests/src/extras.coco +++ b/coconut/tests/src/extras.coco @@ -27,15 +27,6 @@ from coconut.convenience import ( warm_up, ) -if IPY: - if PY35: - import asyncio - from coconut.icoconut import CoconutKernel # type: ignore - from jupyter_client.session import Session -else: - CoconutKernel = None # type: ignore - Session = object # type: ignore - def assert_raises(c, Exc, not_Exc=None, err_has=None): """Test whether callable c raises an exception of type Exc.""" @@ -83,15 +74,6 @@ def unwrap_future(event_loop, maybe_future): return maybe_future -class FakeSession(Session): - if TYPE_CHECKING: - captured_messages: list[tuple] = [] - else: - captured_messages: list = [] - def send(self, stream, msg_or_type, content, *args, **kwargs): - self.captured_messages.append((msg_or_type, content)) - - def test_setup_none() -> bool: setup(line_numbers=False) @@ -468,6 +450,20 @@ class F: def test_kernel() -> bool: + # hide imports so as to not enable incremental parsing until we want to + if PY35: + import asyncio + from coconut.icoconut import CoconutKernel # type: ignore + from jupyter_client.session import Session + + class FakeSession(Session): + if TYPE_CHECKING: + captured_messages: list[tuple] = [] + else: + captured_messages: list = [] + def send(self, stream, msg_or_type, content, *args, **kwargs): + self.captured_messages.append((msg_or_type, content)) + if PY35: loop = asyncio.new_event_loop() asyncio.set_event_loop(loop) @@ -640,15 +636,16 @@ def test_extras() -> bool: print(".", end="") if not PYPY and PY36: assert test_pandas() is True # . - print(".", end="") - if CoconutKernel is not None: - assert test_kernel() is True # .. print(".") # newline bc we print stuff after this - assert test_setup_none() is True + assert test_setup_none() is True # .. print(".") # ditto - assert test_convenience() is True + assert test_convenience() is True # ... + # everything after here uses incremental parsing, so it must come last print(".", end="") - assert test_incremental() is True # must come last + assert test_incremental() is True # .... + if IPY: + print(".", end="") + assert test_kernel() is True # ..... return True From c14014a1324cf480515bd825883c14498bce4281 Mon Sep 17 00:00:00 2001 From: Evan Hubinger Date: Sun, 26 Nov 2023 20:51:13 -0800 Subject: [PATCH 118/121] Prepare for v3.0.4 --- coconut/constants.py | 2 +- coconut/root.py | 4 ++-- coconut/tests/src/extras.coco | 1 + 3 files changed, 4 insertions(+), 3 deletions(-) diff --git a/coconut/constants.py b/coconut/constants.py index 9029a8bef..a6c276a8e 100644 --- a/coconut/constants.py +++ b/coconut/constants.py @@ -1035,7 +1035,7 @@ def get_path_env_var(env_var, default): ("pytest", "py36"): (7,), ("async_generator", "py35"): (1, 10), ("exceptiongroup", "py37;py<311"): (1,), - ("ipython", "py>=39"): (8, 17), + ("ipython", "py>=39"): (8, 18), "py-spy": (0, 3), } diff --git a/coconut/root.py b/coconut/root.py index 439f1e61e..2d622b4d8 100644 --- a/coconut/root.py +++ b/coconut/root.py @@ -26,8 +26,8 @@ VERSION = "3.0.4" VERSION_NAME = None # False for release, int >= 1 for develop -DEVELOP = 4 -ALPHA = True # for pre releases rather than post releases +DEVELOP = False +ALPHA = False # for pre releases rather than post releases assert DEVELOP is False or DEVELOP >= 1, "DEVELOP must be False or an int >= 1" assert DEVELOP or not ALPHA, "alpha releases are only for develop" diff --git a/coconut/tests/src/extras.coco b/coconut/tests/src/extras.coco index e9fe14109..0d13f39d3 100644 --- a/coconut/tests/src/extras.coco +++ b/coconut/tests/src/extras.coco @@ -235,6 +235,7 @@ def f() = assert_raises(-> parse("return = 1"), CoconutParseError, err_has='invalid use of the keyword "return"') assert_raises(-> parse("if a = b: pass"), CoconutParseError, err_has="misplaced assignment") assert_raises(-> parse("while a == b"), CoconutParseError, err_has="misplaced newline") + assert_raises(-> parse("0xfgf"), CoconutParseError, err_has=" \~~^") try: parse(""" From 019348d01db35a842d99ce90d4be0e9553749a11 Mon Sep 17 00:00:00 2001 From: Evan Hubinger Date: Sun, 26 Nov 2023 23:16:24 -0800 Subject: [PATCH 119/121] Fix test cache management --- coconut/tests/main_test.py | 99 +++++++++++++++++++------------------- 1 file changed, 50 insertions(+), 49 deletions(-) diff --git a/coconut/tests/main_test.py b/coconut/tests/main_test.py index d0b9054fa..558c2a85e 100644 --- a/coconut/tests/main_test.py +++ b/coconut/tests/main_test.py @@ -656,53 +656,54 @@ def run_extras(**kwargs): call_python([os.path.join(dest, "extras.py")], assert_output=True, check_errors=False, stderr_first=True, **kwargs) -def run(args=[], agnostic_target=None, use_run_arg=False, convert_to_import=False, always_sys=False, **kwargs): +def run(args=[], agnostic_target=None, use_run_arg=False, convert_to_import=False, always_sys=False, manage_cache=True, **kwargs): """Compiles and runs tests.""" if agnostic_target is None: agnostic_args = args else: agnostic_args = ["--target", str(agnostic_target)] + args - with using_dest(): - with (using_dest(additional_dest) if "--and" in args else noop_ctx()): - - spec_kwargs = kwargs.copy() - spec_kwargs["always_sys"] = always_sys - if PY2: - comp_2(args, **spec_kwargs) - else: - comp_3(args, **spec_kwargs) - if sys.version_info >= (3, 5): - comp_35(args, **spec_kwargs) - if sys.version_info >= (3, 6): - comp_36(args, **spec_kwargs) - if sys.version_info >= (3, 8): - comp_38(args, **spec_kwargs) - if sys.version_info >= (3, 11): - comp_311(args, **spec_kwargs) - - comp_agnostic(agnostic_args, **kwargs) - comp_sys(args, **kwargs) - # do non-strict at the end so we get the non-strict header - comp_non_strict(args, **kwargs) - - if use_run_arg: - _kwargs = kwargs.copy() - _kwargs["assert_output"] = True - comp_runner(["--run"] + agnostic_args, **_kwargs) - else: - comp_runner(agnostic_args, **kwargs) - run_src(convert_to_import=convert_to_import) # **kwargs are for comp, not run - - if use_run_arg: - _kwargs = kwargs.copy() - _kwargs["assert_output"] = True - _kwargs["check_errors"] = False - _kwargs["stderr_first"] = True - comp_extras(["--run"] + agnostic_args, **_kwargs) - else: - comp_extras(agnostic_args, **kwargs) - run_extras(convert_to_import=convert_to_import) # **kwargs are for comp, not run + with (using_caches() if manage_cache else noop_ctx()): + with using_dest(): + with (using_dest(additional_dest) if "--and" in args else noop_ctx()): + + spec_kwargs = kwargs.copy() + spec_kwargs["always_sys"] = always_sys + if PY2: + comp_2(args, **spec_kwargs) + else: + comp_3(args, **spec_kwargs) + if sys.version_info >= (3, 5): + comp_35(args, **spec_kwargs) + if sys.version_info >= (3, 6): + comp_36(args, **spec_kwargs) + if sys.version_info >= (3, 8): + comp_38(args, **spec_kwargs) + if sys.version_info >= (3, 11): + comp_311(args, **spec_kwargs) + + comp_agnostic(agnostic_args, **kwargs) + comp_sys(args, **kwargs) + # do non-strict at the end so we get the non-strict header + comp_non_strict(args, **kwargs) + + if use_run_arg: + _kwargs = kwargs.copy() + _kwargs["assert_output"] = True + comp_runner(["--run"] + agnostic_args, **_kwargs) + else: + comp_runner(agnostic_args, **kwargs) + run_src(convert_to_import=convert_to_import) # **kwargs are for comp, not run + + if use_run_arg: + _kwargs = kwargs.copy() + _kwargs["assert_output"] = True + _kwargs["check_errors"] = False + _kwargs["stderr_first"] = True + comp_extras(["--run"] + agnostic_args, **_kwargs) + else: + comp_extras(agnostic_args, **kwargs) + run_extras(convert_to_import=convert_to_import) # **kwargs are for comp, not run def comp_all(args=[], agnostic_target=None, **kwargs): @@ -1001,12 +1002,11 @@ def test_no_wrap(self): if TEST_ALL: if CPYTHON: def test_any_of(self): - with using_caches(): - with using_env_vars({ - adaptive_any_of_env_var: "True", - reverse_any_of_env_var: "True", - }): - run() + with using_env_vars({ + adaptive_any_of_env_var: "True", + reverse_any_of_env_var: "True", + }): + run() def test_keep_lines(self): run(["--keep-lines"]) @@ -1026,8 +1026,9 @@ def test_jobs_zero(self): if not PYPY: def test_incremental(self): - run() - run(["--force"]) + with using_caches(): + run(manage_cache=False) + run(["--force"], manage_cache=False) if get_bool_env_var("COCONUT_TEST_VERBOSE"): def test_verbose(self): From dfd40bd0791dd2cb32e5685baffeb3c1ece0caf3 Mon Sep 17 00:00:00 2001 From: Evan Hubinger Date: Mon, 27 Nov 2023 01:15:23 -0800 Subject: [PATCH 120/121] Fix typos --- __coconut__/__init__.pyi | 2 +- coconut/compiler/templates/header.py_template | 6 +++--- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/__coconut__/__init__.pyi b/__coconut__/__init__.pyi index d501ea9f1..70a0646f5 100644 --- a/__coconut__/__init__.pyi +++ b/__coconut__/__init__.pyi @@ -1641,7 +1641,7 @@ def lift(func: _t.Callable[..., _W]) -> _t.Callable[..., _t.Callable[..., _W]]: For a binary function f(x, y) and two unary functions g(z) and h(z), lift works as the S' combinator: lift(f)(g, h)(z) == f(g(z), h(z)) - In general, lift is requivalent to: + In general, lift is equivalent to: def lift(f) = ((*func_args, **func_kwargs) -> (*args, **kwargs) -> f(*(g(*args, **kwargs) for g in func_args), **{lbrace}k: h(*args, **kwargs) for k, h in func_kwargs.items(){rbrace})) diff --git a/coconut/compiler/templates/header.py_template b/coconut/compiler/templates/header.py_template index 71657588f..e7ec5f6f1 100644 --- a/coconut/compiler/templates/header.py_template +++ b/coconut/compiler/templates/header.py_template @@ -1891,7 +1891,7 @@ class lift(_coconut_base_callable): For a binary function f(x, y) and two unary functions g(z) and h(z), lift works as the S' combinator: lift(f)(g, h)(z) == f(g(z), h(z)) - In general, lift is requivalent to: + In general, lift is equivalent to: def lift(f) = ((*func_args, **func_kwargs) -> (*args, **kwargs) -> f(*(g(*args, **kwargs) for g in func_args), **{lbrace}k: h(*args, **kwargs) for k, h in func_kwargs.items(){rbrace})) @@ -1906,10 +1906,10 @@ class lift(_coconut_base_callable): return self def __reduce__(self): return (self.__class__, (self.func,)) - def __call__(self, *func_args, **func_kwargs): - return _coconut_lifted(self.func, *func_args, **func_kwargs) def __repr__(self): return "lift(%r)" % (self.func,) + def __call__(self, *func_args, **func_kwargs): + return _coconut_lifted(self.func, *func_args, **func_kwargs) def all_equal(iterable): """For a given iterable, check whether all elements in that iterable are equal to each other. From d2191eed9fd0a2ad68754a87d18843dcabbf13cf Mon Sep 17 00:00:00 2001 From: Evan Hubinger Date: Mon, 27 Nov 2023 16:22:03 -0800 Subject: [PATCH 121/121] Fix test --- coconut/tests/main_test.py | 1 + 1 file changed, 1 insertion(+) diff --git a/coconut/tests/main_test.py b/coconut/tests/main_test.py index 558c2a85e..2d7bf296e 100644 --- a/coconut/tests/main_test.py +++ b/coconut/tests/main_test.py @@ -174,6 +174,7 @@ "from distutils.version import LooseVersion", ": SyntaxWarning: 'int' object is not ", " assert_raises(", + "Populating initial parsing cache", ) kernel_installation_msg = (