diff --git a/.github/workflows/run-tests.yml b/.github/workflows/run-tests.yml index 2927a0edb..3065514a1 100644 --- a/.github/workflows/run-tests.yml +++ b/.github/workflows/run-tests.yml @@ -7,7 +7,6 @@ jobs: matrix: python-version: - '2.7' - - '3.4' - '3.5' - '3.6' - '3.7' @@ -15,17 +14,19 @@ jobs: - '3.9' - '3.10' - '3.11' + - '3.12' - 'pypy-2.7' - 'pypy-3.6' - 'pypy-3.7' - 'pypy-3.8' - 'pypy-3.9' + - 'pypy-3.10' fail-fast: false name: Python ${{ matrix.python-version }} steps: - uses: actions/checkout@v3 - name: Setup python - uses: MatteoH2O1999/setup-python@v1.3.1 + uses: MatteoH2O1999/setup-python@v2 with: python-version: ${{ matrix.python-version }} cache: pip diff --git a/.gitignore b/.gitignore index 243d558fd..ed62891af 100644 --- a/.gitignore +++ b/.gitignore @@ -138,6 +138,12 @@ pyprover/ bbopt/ coconut-prelude/ index.rst -vprof.json /coconut/icoconut/coconut/ __coconut_cache__/ + +# Profiling +vprof.json +profile.svg +profile.speedscope +runtime_profile.svg +runtime_profile.speedscope diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index df224ace7..2df5155a4 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -1,6 +1,16 @@ repos: +- repo: https://github.com/pre-commit/mirrors-autopep8 + rev: v2.0.4 + hooks: + - id: autopep8 + args: + - --in-place + - --aggressive + - --aggressive + - --experimental + - --ignore=W503,E501,E722,E402,E721 - repo: https://github.com/pre-commit/pre-commit-hooks.git - rev: v4.4.0 + rev: v4.5.0 hooks: - id: check-added-large-files - id: fix-byte-order-marker @@ -24,18 +34,8 @@ repos: args: - --autofix - repo: https://github.com/pycqa/flake8 - rev: 6.0.0 + rev: 6.1.0 hooks: - id: flake8 args: - --ignore=W503,E501,E265,E402,F405,E305,E126 -- repo: https://github.com/pre-commit/mirrors-autopep8 - rev: v2.0.2 - hooks: - - id: autopep8 - args: - - --in-place - - --aggressive - - --aggressive - - --experimental - - --ignore=W503,E501,E722,E402 diff --git a/.readthedocs.yaml b/.readthedocs.yaml index 56e6e605a..fe3e5c3b8 100644 --- a/.readthedocs.yaml +++ b/.readthedocs.yaml @@ -32,4 +32,3 @@ python: path: . extra_requirements: - docs - system_packages: true diff --git a/DOCS.md b/DOCS.md index 9cf16df75..1355ca8fb 100644 --- a/DOCS.md +++ b/DOCS.md @@ -11,9 +11,10 @@ depth: 2 --- ``` + ## Overview -This documentation covers all the features of the [Coconut Programming Language](http://evhub.github.io/coconut/), and is intended as a reference/specification, not a tutorialized introduction. For a full introduction and tutorial of Coconut, see [the tutorial](./HELP.md). +This documentation covers all the features of the [Coconut Programming Language](http://evhub.github.io/coconut/), and is intended as a reference/specification, not a tutorialized introduction. For an introduction to and tutorial of Coconut, see [the tutorial](./HELP.md). Coconut is a variant of [Python](https://www.python.org/) built for **simple, elegant, Pythonic functional programming**. Coconut syntax is a strict superset of the latest Python 3 syntax. Thus, users familiar with Python will already be familiar with most of Coconut. @@ -25,6 +26,7 @@ Thought Coconut syntax is primarily based on that of Python, other languages tha If you want to try Coconut in your browser, check out the [online interpreter](https://cs121-team-panda.github.io/coconut-interpreter). Note, however, that it may be running an outdated version of Coconut. + ## Installation ```{contents} @@ -85,21 +87,15 @@ pip install coconut[opt_dep_1,opt_dep_2] The full list of optional dependencies is: -- `all`: alias for `jupyter,watch,mypy,backports,xonsh` (this is the recommended way to install a feature-complete version of Coconut). +- `all`: alias for everything below (this is the recommended way to install a feature-complete version of Coconut). - `jupyter`/`ipython`: enables use of the `--jupyter` / `--ipython` flag. +- `kernel`: lightweight subset of `jupyter` that only includes the dependencies that are strictly necessary for Coconut's [Jupyter kernel](#kernel). - `watch`: enables use of the `--watch` flag. - `mypy`: enables use of the `--mypy` flag. -- `backports`: installs libraries that backport newer Python features to older versions, which Coconut will automatically use instead of the standard library if the standard library is not available. Specifically: - - Installs [`dataclasses`](https://pypi.org/project/dataclasses/) to backport [`dataclasses`](https://docs.python.org/3/library/dataclasses.html). - - Installs [`typing`](https://pypi.org/project/typing/) to backport [`typing`](https://docs.python.org/3/library/typing.html) ([`typing_extensions`](https://pypi.org/project/typing-extensions/) is always installed for backporting individual `typing` objects). - - Installs [`aenum`](https://pypi.org/project/aenum) to backport [`enum`](https://docs.python.org/3/library/enum.html). - - Installs [`async_generator`](https://github.com/python-trio/async_generator) to backport [`async` generators](https://peps.python.org/pep-0525/) and [`asynccontextmanager`](https://docs.python.org/3/library/contextlib.html#contextlib.asynccontextmanager). - - Installs [`trollius`](https://pypi.python.org/pypi/trollius) to backport [`async`/`await`](https://docs.python.org/3/library/asyncio-task.html) and [`asyncio`](https://docs.python.org/3/library/asyncio.html). - `xonsh`: enables use of Coconut's [`xonsh` support](#xonsh-support). -- `kernel`: lightweight subset of `jupyter` that only includes the dependencies that are strictly necessary for Coconut's [Jupyter kernel](#kernel). -- `tests`: everything necessary to test the Coconut language itself. -- `docs`: everything necessary to build Coconut's documentation. -- `dev`: everything necessary to develop on the Coconut language itself, including all of the dependencies above. +- `numpy`: installs everything necessary for making use of Coconut's [`numpy` integration](#numpy-integration). +- `jupyterlab`: installs everything necessary to use [JupyterLab](https://github.com/jupyterlab/jupyterlab) with Coconut. +- `jupytext`: installs everything necessary to use [Jupytext](https://github.com/mwouts/jupytext) with Coconut. #### Develop Version @@ -111,6 +107,7 @@ which will install the most recent working version from Coconut's [`develop` bra _Note: if you have an existing release version of `coconut` installed, you'll need to `pip uninstall coconut` before installing `coconut-develop`._ + ## Compilation ```{contents} @@ -123,11 +120,11 @@ depth: 1 #### Usage ``` -coconut [-h] [--and source [dest ...]] [-v] [-t version] [-i] [-p] [-a] [-l] [-k] [-w] - [-r] [-n] [-d] [-q] [-s] [--no-tco] [--no-wrap-types] [-c code] [-j processes] - [-f] [--minify] [--jupyter ...] [--mypy ...] [--argv ...] [--tutorial] - [--docs] [--style name] [--history-file path] [--vi-mode] - [--recursion-limit limit] [--stack-size kbs] [--site-install] +coconut [-h] [--and source [dest ...]] [-v] [-t version] [-i] [-p] [-a] [-l] + [--no-line-numbers] [-k] [-w] [-r] [-n] [-d] [-q] [-s] [--no-tco] + [--no-wrap-types] [-c code] [--incremental] [-j processes] [-f] [--minify] + [--jupyter ...] [--mypy ...] [--argv ...] [--tutorial] [--docs] [--style name] + [--vi-mode] [--recursion-limit limit] [--stack-size kbs] [--site-install] [--site-uninstall] [--verbose] [--trace] [--profile] [source] [dest] ``` @@ -149,16 +146,16 @@ dest destination directory for compiled files (defaults to -v, -V, --version print Coconut and Python version information -t version, --target version specify target Python version (defaults to universal) --i, --interact force the interpreter to start (otherwise starts if no other command - is given) (implies --run) +-i, --interact force the interpreter to start (otherwise starts if no other command is + given) (implies --run) -p, --package compile source as part of a package (defaults to only if source is a directory) -a, --standalone, --stand-alone - compile source as standalone files (defaults to only if source is a - single file) + compile source as standalone files (defaults to only if source is a single + file) -l, --line-numbers, --linenumbers - force enable line number comments (--line-numbers are enabled by - default unless --minify is passed) + force enable line number comments (--line-numbers are enabled by default + unless --minify is passed) --no-line-numbers, --nolinenumbers disable line number comments (opposite of --line-numbers) -k, --keep-lines, --keeplines @@ -173,8 +170,8 @@ dest destination directory for compiled files (defaults to -s, --strict enforce code cleanliness standards --no-tco, --notco disable tail call optimization --no-wrap-types, --nowraptypes - disable wrapping type annotations in strings and turn off 'from - __future__ import annotations' behavior + disable wrapping type annotations in strings and turn off 'from __future__ + import annotations' behavior -c code, --code code run Coconut passed in as a string (can also be piped into stdin) -j processes, --jobs processes number of additional processes to use (defaults to 'sys') (0 is no @@ -183,30 +180,32 @@ dest destination directory for compiled files (defaults to haven't changed --minify reduce size of compiled Python --jupyter ..., --ipython ... - run Jupyter/IPython with Coconut as the kernel (remaining args passed - to Jupyter) + run Jupyter/IPython with Coconut as the kernel (remaining args passed to + Jupyter) --mypy ... run MyPy on compiled Python (remaining args passed to MyPy) (implies --package --line-numbers) --argv ..., --args ... - set sys.argv to source plus remaining args for use in the Coconut - script being run + set sys.argv to source plus remaining args for use in the Coconut script + being run --tutorial open Coconut's tutorial in the default web browser --docs, --documentation open Coconut's documentation in the default web browser --style name set Pygments syntax highlighting style (or 'list' to list styles) - (defaults to COCONUT_STYLE environment variable if it exists, - otherwise 'default') ---history-file path set history file (or '' for no file) (can be modified by setting - COCONUT_HOME environment variable) + (defaults to COCONUT_STYLE environment variable if it exists, otherwise + 'default') --vi-mode, --vimode enable vi mode in the interpreter (currently set to False) (can be modified by setting COCONUT_VI_MODE environment variable) --recursion-limit limit, --recursionlimit limit set maximum recursion depth in compiler (defaults to 1920) (when - increasing --recursion-limit, you may also need to increase --stack- - size; setting them to approximately equal values is recommended) + increasing --recursion-limit, you may also need to increase --stack-size; + setting them to approximately equal values is recommended) --stack-size kbs, --stacksize kbs run the compiler in a separate thread with the given stack size in kilobytes +--fail-fast causes the compiler to fail immediately upon encountering a compilation + error rather than attempting to continue compiling other files +--no-cache disables use of Coconut's incremental parsing cache (caches previous + parses to improve recompilation performance for slightly modified files) --site-install, --siteinstall set up coconut.api to be imported on Python start --site-uninstall, --siteuninstall @@ -289,7 +288,7 @@ Finally, while Coconut will try to compile Python-3-specific syntax to its unive - the `nonlocal` keyword, - keyword-only function parameters (use [pattern-matching function definition](#pattern-matching-functions) for universal code), -- `async` and `await` statements (requires a specific target; Coconut will attempt different backports based on the targeted version), +- `async` and `await` statements (requires a specific target; Coconut will attempt different [backports](#backports) based on the targeted version), - `:=` assignment expressions (requires `--target 3.8`), - positional-only function parameters (use [pattern-matching function definition](#pattern-matching-functions) for universal code) (requires `--target 3.8`), - `a[x, *y]` variadic generic syntax (use [type parameter syntax](#type-parameter-syntax) for universal code) (requires `--target 3.11`), and @@ -349,6 +348,21 @@ The style issues which will cause `--strict` to throw an error are: Note that many of the above style issues will still show a warning if `--strict` is not present. +#### Backports + +In addition to the newer Python features that Coconut can backport automatically itself to older Python versions, Coconut will also automatically compile code to make use of a variety of external backports as well. These backports are automatically installed with Coconut if needed and Coconut will automatically use them instead of the standard library if the standard library is not available. These backports are: +- [`typing`](https://pypi.org/project/typing/) for backporting [`typing`](https://docs.python.org/3/library/typing.html). +- [`typing_extensions`](https://pypi.org/project/typing-extensions/) for backporting individual `typing` objects. +- [`backports.functools-lru-cache`](https://pypi.org/project/backports.functools-lru-cache/) for backporting [`functools.lru_cache`](https://docs.python.org/3/library/functools.html#functools.lru_cache). +- [`exceptiongroup`](https://pypi.org/project/exceptiongroup/) for backporting [`ExceptionGroup` and `BaseExceptionGroup`](https://docs.python.org/3/library/exceptions.html#ExceptionGroup). +- [`dataclasses`](https://pypi.org/project/dataclasses/) for backporting [`dataclasses`](https://docs.python.org/3/library/dataclasses.html). +- [`aenum`](https://pypi.org/project/aenum) for backporting [`enum`](https://docs.python.org/3/library/enum.html). +- [`async_generator`](https://github.com/python-trio/async_generator) for backporting [`async` generators](https://peps.python.org/pep-0525/) and [`asynccontextmanager`](https://docs.python.org/3/library/contextlib.html#contextlib.asynccontextmanager). +- [`trollius`](https://pypi.python.org/pypi/trollius) for backporting [`async`/`await`](https://docs.python.org/3/library/asyncio-task.html) and [`asyncio`](https://docs.python.org/3/library/asyncio.html). + +Note that, when distributing compiled Coconut code, if you use any of these backports, you'll need to make sure that the requisite backport module is included as a dependency. + + ## Integrations ```{contents} @@ -405,7 +419,7 @@ Simply installing Coconut should add a `Coconut` kernel to your Jupyter/IPython The Coconut kernel will always compile using the parameters: `--target sys --line-numbers --keep-lines --no-wrap-types`. -Coconut also provides the following api commands: +Coconut also provides the following commands: - `coconut --jupyter notebook` will ensure that the Coconut kernel is available and launch a Jupyter/IPython notebook. - `coconut --jupyter console` will launch a Jupyter/IPython console using the Coconut kernel. @@ -491,6 +505,7 @@ Compilation always uses the same parameters as in the [Coconut Jupyter kernel](# Note that the way that Coconut integrates with `xonsh`, `@()` syntax and the `execx` command will only work with Python code, not Coconut code. Additionally, Coconut will only compile individual commands—Coconut will not touch the `.xonshrc` or any other `.xsh` files. + ## Operators ```{contents} @@ -500,6 +515,7 @@ depth: 1 --- ``` + ### Precedence In order of precedence, highest first, the operators supported in Coconut are: @@ -515,11 +531,11 @@ f x n/a +, - left <<, >> left & left -&: left +&: yes ^ left | left -:: n/a (lazy) -.. n/a +:: yes (lazy) +.. yes a `b` c, left (captures lambda) all custom operators ?? left (short-circuits) @@ -534,7 +550,7 @@ a `b` c, left (captures lambda) not unary and left (short-circuits) or left (short-circuits) -x if c else y, ternary left (short-circuits) +x if c else y, ternary (short-circuits) if c then x else y => right ====================== ========================== @@ -542,6 +558,7 @@ x if c else y, ternary left (short-circuits) For example, since addition has a higher precedence than piping, expressions of the form `x |> y + z` are equivalent to `x |> (y + z)`. + ### Lambdas Coconut provides the simple, clean `=>` operator as an alternative to Python's `lambda` statements. The syntax for the `=>` operator is `(parameters) => expression` (or `parameter => expression` for one-argument lambdas). The operator has the same precedence as the old statement, which means it will often be necessary to surround the lambda in parentheses, and is right-associative. @@ -599,6 +616,7 @@ get_random_number = (=> random.random()) _Note: Nesting implicit lambdas can lead to problems with the scope of the `_` parameter to each lambda. It is recommended that nesting implicit lambdas be avoided._ + ### Partial Application Coconut uses a `$` sign right after a function's name but before the open parenthesis used to call the function to denote partial application. @@ -612,6 +630,8 @@ def new_f(x, *args, **kwargs): return f(*args, **kwargs) ``` +Unlike `functools.partial`, Coconut's partial application will preserve the `__name__` of the wrapped function. + ##### Rationale Partial application, or currying, is a mainstay of functional programming, and for good reason: it allows the dynamic customization of functions to fit the needs of where they are being used. Partial application allows a new function to be created out of an old function with some of its arguments pre-specified. @@ -647,6 +667,7 @@ expnums = map(lambda x: pow(x, 2), range(5)) print(list(expnums)) ``` + ### Pipes Coconut uses pipe operators for pipeline-style function application. All the operators have a precedence in-between function composition pipes and comparisons, and are left-associative. All operators also support in-place versions. The different operators are: @@ -718,6 +739,7 @@ async def do_stuff(some_data): return post_proc(await async_func(some_data)) ``` + ### Function Composition Coconut has three basic function composition operators: `..`, `..>`, and `<..`. Both `..` and `<..` use math-style "backwards" function composition, where the first function is called last, while `..>` uses "forwards" function composition, where the first function is called first. Forwards and backwards function composition pipes cannot be used together in the same expression (unlike normal pipes) and have precedence in-between `None`-coalescing and normal pipes. @@ -763,6 +785,7 @@ fog = lambda *args, **kwargs: f(g(*args, **kwargs)) f_into_g = lambda *args, **kwargs: g(f(*args, **kwargs)) ``` + ### Iterator Slicing Coconut uses a `$` sign right after an iterator before a slice to perform iterator slicing, as in `it$[:5]`. Coconut's iterator slicing works much the same as Python's sequence slicing, and looks much the same as Coconut's partial application, but with brackets instead of parentheses. @@ -781,6 +804,7 @@ map(x => x*2, range(10**100))$[-1] |> print **Python:** _Can't be done without a complicated iterator slicing function and inspection of custom objects. The necessary definitions in Python can be found in the Coconut header._ + ### Iterator Chaining Coconut uses the `::` operator for iterator chaining. Coconut's iterator chaining is done lazily, in that the arguments are not evaluated until they are needed. It has a precedence in-between bitwise or and infix calls. Chains are reiterable (can be iterated over multiple times and get the same result) only when the iterators passed in are reiterable. The in-place operator is `::=`. @@ -814,6 +838,7 @@ def N(n=0) = (n,) :: N(n+1) # no infinite loop because :: is lazy **Python:** _Can't be done without a complicated iterator comprehension in place of the lazy chaining. See the compiled code for the Python syntax._ + ### Infix Functions Coconut allows for infix function calling, where an expression that evaluates to a function is surrounded by backticks and then can have arguments placed in front of or behind it. Infix calling has a precedence in-between chaining and `None`-coalescing, and is left-associative. @@ -854,6 +879,7 @@ def mod(a, b): return a % b print(mod(x, 2)) ``` + ### Custom Operators Coconut allows you to declare your own custom operators with the syntax @@ -924,6 +950,7 @@ print(bool(0)) print(math.log10(100)) ``` + ### None Coalescing Coconut provides `??` as a `None`-coalescing operator, similar to the `??` null-coalescing operator in C# and Swift. Additionally, Coconut implements all of the `None`-aware operators proposed in [PEP 505](https://www.python.org/dev/peps/pep-0505/). @@ -995,6 +1022,7 @@ import functools (lambda result: None if result is None else result.attr[index].method())(could_be_none()) ``` + ### Protocol Intersection Coconut uses the `&:` operator to indicate protocol intersection. That is, for two [`typing.Protocol`s](https://docs.python.org/3/library/typing.html#typing.Protocol) `Protocol1` and `Protocol1`, `Protocol1 &: Protocol2` is equivalent to a `Protocol` that combines the requirements of both `Protocol1` and `Protocol2`. @@ -1050,6 +1078,7 @@ class CanAddAndSub(Protocol, Generic[T, U, V]): raise NotImplementedError ``` + ### Unicode Alternatives Coconut supports Unicode alternatives to many different operator symbols. The Unicode alternatives are relatively straightforward, and chosen to reflect the look and/or meaning of the original symbol. @@ -1105,6 +1134,7 @@ _Note: these are only the default, built-in unicode operators. Coconut supports ⏨ (\u23e8) => "e" (in scientific notation) ``` + ## Keywords ```{contents} @@ -1114,6 +1144,7 @@ depth: 1 --- ``` + ### `match` Coconut provides fully-featured, functional pattern-matching through its `match` statements. @@ -1327,6 +1358,7 @@ _Showcases the use of an iterable search pattern and a view pattern to construct **Python:** _Can't be done without a long series of checks for each `match` statement. See the compiled code for the Python syntax._ + ### `case` Coconut's `case` blocks serve as an extension of Coconut's `match` statement for performing multiple `match` statements against the same value, where only one of them should succeed. Unlike lone `match` statements, only one match statement inside of a `case` block will ever succeed, and thus more general matches should be put below more specific ones. @@ -1390,6 +1422,7 @@ _Example of the `cases` keyword instead._ **Python:** _Can't be done without a long series of checks for each `match` statement. See the compiled code for the Python syntax._ + ### `match for` Coconut supports pattern-matching in for loops, where the pattern is matched against each item in the iterable. The syntax is @@ -1421,6 +1454,7 @@ for user_data in get_data(): print(uid) ``` + ### `data` Coconut's `data` keyword is used to create immutable, algebraic data types, including built-in support for destructuring [pattern-matching](#match) and [`fmap`](#fmap). @@ -1541,31 +1575,33 @@ data namedpt(name `isinstance` str, x `isinstance` int, y `isinstance` int): **Python:** _Can't be done without a series of method definitions for each data type. See the compiled code for the Python syntax._ + ### `where` -Coconut's `where` statement is extremely straightforward. The syntax for a `where` statement is just +Coconut's `where` statement is fairly straightforward. The syntax for a `where` statement is just ``` where: ``` -which just executes `` followed by ``. +which executes `` followed by ``, with the exception that any new variables defined in `` are available _only_ in `` (though they are only mangled, not deleted, such that e.g. lambdas can still capture them). ##### Example **Coconut:** ```coconut -c = a + b where: +result = a + b where: a = 1 b = 2 ``` **Python:** ```coconut_python -a = 1 -b = 2 -c = a + b +_a = 1 +_b = 2 +result = _a + _b ``` + ### `async with for` In modern Python `async` code, such as when using [`contextlib.aclosing`](https://docs.python.org/3/library/contextlib.html#contextlib.aclosing), it is often recommended to use a pattern like @@ -1584,7 +1620,7 @@ This is especially true when using [`trio`](https://github.com/python-trio/trio) Since this pattern can often be quite syntactically cumbersome, Coconut provides the shortcut syntax ``` -async with for aclosing(my_generator()) as values: +async with for value in aclosing(my_generator()): ... ``` which compiles to exactly the pattern above. @@ -1620,6 +1656,7 @@ async with my_generator() as agen: print(value) ``` + ### Handling Keyword/Variable Name Overlap In Coconut, the following keywords are also valid variable names: @@ -1664,6 +1701,7 @@ print(data) x, y = input_list ``` + ## Expressions ```{contents} @@ -1673,6 +1711,7 @@ depth: 1 --- ``` + ### Statement Lambdas The statement lambda syntax is an extension of the [normal lambda syntax](#lambdas) to support statements, not just expressions. @@ -1720,6 +1759,7 @@ g = def (a: int, b: int) -> int => a ** b _Deprecated: if the deprecated `->` is used in place of `=>`, then return type annotations will not be available._ + ### Operator Functions Coconut uses a simple operator function short-hand: surround an operator with parentheses to retrieve its function. Similarly to iterator comprehensions, if the operator function is the only argument to a function, the parentheses of the function call can also serve as the parentheses for the operator function. @@ -1804,6 +1844,7 @@ import operator print(list(map(operator.add, range(0, 5), range(5, 10)))) ``` + ### Implicit Partial Application Coconut supports a number of different syntactical aliases for common partial application use cases. These are: @@ -1817,7 +1858,7 @@ iter$[] => # the equivalent of seq[] for iterators .$[a:b:c] => # the equivalent of .[a:b:c] for iterators ``` -Additionally, `.attr.method(args)`, `.[x][y]`, and `.$[x]$[y]` are also supported. +Additionally, `.attr.method(args)`, `.[x][y]`, `.$[x]$[y]`, and `.method[x]` are also supported. In addition, for every Coconut [operator function](#operator-functions), Coconut supports syntax for implicitly partially applying that operator function as ``` @@ -1851,6 +1892,7 @@ mod(5, 3) (3 * 2) + 1 ``` + ### Enhanced Type Annotation Since Coconut syntax is a superset of the latest Python 3 syntax, it supports [Python 3 function type annotation syntax](https://www.python.org/dev/peps/pep-0484/) and [Python 3.6 variable type annotation syntax](https://www.python.org/dev/peps/pep-0526/). By default, Coconut compiles all type annotations into Python-2-compatible type comments. If you want to keep the type annotations instead, simply pass a `--target` that supports them. @@ -1990,6 +2032,7 @@ class CanAddAndSub(typing.Protocol, typing.Generic[T, U, V]): raise NotImplementedError ``` + ### Multidimensional Array Literal/Concatenation Syntax Coconut supports multidimensional array literal and array [concatenation](https://numpy.org/doc/stable/reference/generated/numpy.concatenate.html)/[stack](https://numpy.org/doc/stable/reference/generated/numpy.stack.html) syntax. @@ -2065,6 +2108,7 @@ _General showcase of how the different concatenation operators work using `numpy **Python:** _The equivalent Python array literals can be seen in the printed representations in each example._ + ### Lazy Lists Coconut supports the creation of lazy lists, where the contents in the list will be treated as an iterator and not evaluated until they are needed. Unlike normal iterators, however, lazy lists can be iterated over multiple times and still return the same result. Lazy lists can be created in Coconut simply by surrounding a comma-separated list of items with `(|` and `|)` (so-called "banana brackets") instead of `[` and `]` for a list or `(` and `)` for a tuple. @@ -2085,6 +2129,7 @@ Lazy lists, where sequences are only evaluated when their contents are requested **Python:** _Can't be done without a complicated iterator comprehension in place of the lazy list. See the compiled code for the Python syntax._ + ### Implicit Function Application and Coefficients Coconut supports implicit function application of the form `f x y`, which is compiled to `f(x, y)` (note: **not** `f(x)(y)` as is common in many languages with automatic currying). @@ -2094,7 +2139,7 @@ Additionally, if the first argument is not callable, and is instead an `int`, `f Though the first item may be any atom, following arguments are highly restricted, and must be: - variables/attributes (e.g. `a.b`), - literal constants (e.g. `True`), -- number literals (e.g. `1.5`), or +- number literals (e.g. `1.5`) (and no binary, hex, or octal), or - one of the above followed by an exponent (e.g. `a**-5`). For example, `(f .. g) x 1` will work, but `f x [1]`, `f x (1+2)`, and `f "abc"` will not. @@ -2142,6 +2187,7 @@ print(p1(5)) quad = 5 * x**2 + 3 * x + 1 ``` + ### Keyword Argument Name Elision When passing in long variable names as keyword arguments of the same name, Coconut supports the syntax @@ -2177,6 +2223,7 @@ main_func( ) ``` + ### Anonymous Namedtuples Coconut supports anonymous [`namedtuple`](https://docs.python.org/3/library/collections.html#collections.namedtuple) literals, such that `(a=1, b=2)` can be used just as `(1, 2)`, but with added names. Anonymous `namedtuple`s are always pickleable. @@ -2215,6 +2262,7 @@ users = [ ] ``` + ### Set Literals Coconut allows an optional `s` to be prepended in front of Python set literals. While in most cases this does nothing, in the case of the empty set it lets Coconut know that it is an empty set and not an empty dictionary. Set literals also support unpacking syntax (e.g. `s{*xs}`). @@ -2233,6 +2281,7 @@ empty_frozen_set = f{} empty_frozen_set = frozenset() ``` + ### Imaginary Literals In addition to Python's `j` or `J` notation for imaginary literals, Coconut also supports `i` or `I`, to make imaginary literals more readable if used in a mathematical context. @@ -2260,6 +2309,7 @@ An imaginary literal yields a complex number with a real part of 0.0. Complex nu print(abs(3 + 4j)) ``` + ### Alternative Ternary Operator Python supports the ternary operator syntax @@ -2296,6 +2346,7 @@ value = ( ) ``` + ## Function Definition ```{contents} @@ -2305,6 +2356,7 @@ depth: 1 --- ``` + ### Tail Call Optimization Coconut will perform automatic [tail call](https://en.wikipedia.org/wiki/Tail_call) optimization and tail recursion elimination on any function that meets the following criteria: @@ -2314,8 +2366,6 @@ Coconut will perform automatic [tail call](https://en.wikipedia.org/wiki/Tail_ca Tail call optimization (though not tail recursion elimination) will work even for 1) mutual recursion and 2) pattern-matching functions split across multiple definitions using [`addpattern`](#addpattern). -If you are encountering a `RuntimeError` due to maximum recursion depth, it is highly recommended that you rewrite your function to meet either the criteria above for tail call optimization, or the corresponding criteria for [`recursive_iterator`](#recursive-iterator), either of which should prevent such errors. - ##### Example **Coconut:** @@ -2370,6 +2420,7 @@ print(foo()) # 2 (!) Because this could have unintended and potentially damaging consequences, Coconut opts to not perform TRE on any function with a lambda or inner function. + ### Assignment Functions Coconut allows for assignment function definition that automatically returns the last line of the function body. An assignment function is constructed by substituting `=` for `:` after the function definition line. Thus, the syntax for assignment function definition is either @@ -2404,6 +2455,7 @@ def binexp(x): return 2**x print(binexp(5)) ``` + ### Pattern-Matching Functions Coconut pattern-matching functions are just normal functions, except where the arguments are patterns to be matched against instead of variables to be assigned to. The syntax for pattern-matching function definition is @@ -2441,6 +2493,7 @@ range(5) |> last_two |> print **Python:** _Can't be done without a long series of checks at the top of the function. See the compiled code for the Python syntax._ + ### `addpattern` Functions Coconut provides the `addpattern def` syntax as a shortcut for the full @@ -2466,6 +2519,7 @@ addpattern def factorial(n) = n * factorial(n - 1) **Python:** _Can't be done without a complicated decorator definition and a long series of checks for each pattern-matching. See the compiled code for the Python syntax._ + ### `copyclosure` Functions Coconut supports the syntax @@ -2517,6 +2571,7 @@ def outer_func(): return funcs ``` + ### Explicit Generators Coconut supports the syntax @@ -2542,6 +2597,7 @@ def empty_it(): yield ``` + ### Dotted Function Definition Coconut allows for function definition using a dotted name to assign a function as a method of an object as specified in [PEP 542](https://www.python.org/dev/peps/pep-0542/). Dotted function definition can be combined with all other types of function definition above. @@ -2561,6 +2617,7 @@ def my_method(self): MyClass.my_method = my_method ``` + ## Statements ```{contents} @@ -2570,6 +2627,7 @@ depth: 1 --- ``` + ### Destructuring Assignment Coconut supports significantly enhanced destructuring assignment, similar to Python's tuple/list destructuring, but much more powerful. The syntax for Coconut's destructuring assignment is @@ -2599,6 +2657,7 @@ print(a, b) **Python:** _Can't be done without a long series of checks in place of the destructuring assignment statement. See the compiled code for the Python syntax._ + ### Type Parameter Syntax Coconut fully supports [Python 3.12 PEP 695](https://peps.python.org/pep-0695/) type parameter syntax on all Python versions. @@ -2682,6 +2741,7 @@ def my_ident[T](x: T) -> T = x **Python:** _Can't be done without a complex definition for the data type. See the compiled code for the Python syntax._ + ### Implicit `pass` Coconut supports the simple `class name(base)` and `data name(args)` as aliases for `class name(base): pass` and `data name(args): pass`. @@ -2699,6 +2759,7 @@ data Node(left, right) from Tree **Python:** _Can't be done without a series of method definitions for each data type. See the compiled code for the Python syntax._ + ### Statement Nesting Coconut supports the nesting of compound statements on the same line. This allows the mixing of `match` and `if` statements together, as well as compound `try` statements. @@ -2727,6 +2788,7 @@ else: print(input_list) ``` + ### `except` Statements Python 3 requires that if multiple exceptions are to be caught, they must be placed inside of parentheses, so as to disallow Python 2's use of a comma instead of `as`. Coconut allows commas in except statements to translate to catching multiple exceptions without the need for parentheses, since, as in Python 3, `as` is always required to bind the exception to a name. @@ -2749,6 +2811,7 @@ except (SyntaxError, ValueError) as err: handle(err) ``` + ### In-line `global` And `nonlocal` Assignment Coconut allows for `global` or `nonlocal` to precede assignment to a list of variables or (augmented) assignment to a variable to make that assignment `global` or `nonlocal`, respectively. @@ -2767,6 +2830,7 @@ global state_a, state_b; state_a, state_b = 10, 100 global state_c; state_c += 1 ``` + ### Code Passthrough Coconut supports the ability to pass arbitrary code through the compiler without being touched, for compatibility with other variants of Python, such as [Cython](http://cython.org/) or [Mython](http://mython.org/). When using Coconut to compile to another variant of Python, make sure you [name your source file properly](#naming-source-files) to ensure the resulting compiled code has the right file extension for the intended usage. @@ -2787,6 +2851,7 @@ cdef f(x): return g(x) ``` + ### Enhanced Parenthetical Continuation Since Coconut syntax is a superset of the latest Python 3 syntax, Coconut supports the same line continuation syntax as Python. That means both backslash line continuation and implied line continuation inside of parentheses, brackets, or braces will all work. @@ -2816,6 +2881,7 @@ with open('/path/to/some/file/you/want/to/read') as file_1: file_2.write(file_1.read()) ``` + ### Assignment Expression Chaining Unlike Python, Coconut allows assignment expressions to be chained, as in `a := b := c`. Note, however, that assignment expressions in general are currently only supported on `--target 3.8` or higher. @@ -2832,6 +2898,7 @@ Unlike Python, Coconut allows assignment expressions to be chained, as in `a := (a := (b := 1)) ``` + ## Built-Ins ```{contents} @@ -2841,6 +2908,7 @@ depth: 2 --- ``` + ### Built-In Function Decorators ```{contents} @@ -2958,6 +3026,8 @@ Coconut provides `functools.lru_cache` as a built-in under the name `memoize` wi Use of `memoize` requires `functools.lru_cache`, which exists in the Python 3 standard library, but under Python 2 will require `pip install backports.functools_lru_cache` to function. Additionally, if on Python 2 and `backports.functools_lru_cache` is present, Coconut will patch `functools` such that `functools.lru_cache = backports.functools_lru_cache.lru_cache`. +Note that, if the function to be memoized is a generator or otherwise returns an iterator, [`recursive_generator`](#recursive_generator) can also be used to achieve a similar effect, the use of which is required for recursive generators. + ##### Python Docs @**memoize**(_user\_function_) @@ -3038,7 +3108,7 @@ def fib(n): **override**(_func_) -Coconut provides the `@override` decorator to allow declaring a method definition in a subclass as an override of some parent class method. When `@override` is used on a method, if a method of the same name does not exist on some parent class, the class definition will raise a `RuntimeError`. +Coconut provides the `@override` decorator to allow declaring a method definition in a subclass as an override of some parent class method. When `@override` is used on a method, if a method of the same name does not exist on some parent class, the class definition will raise a `RuntimeError`. `@override` works with other decorators such as `@classmethod` and `@staticmethod`, but only if `@override` is the outer-most decorator. Additionally, `override` will present to type checkers as [`typing_extensions.override`](https://pypi.org/project/typing-extensions/). @@ -3058,42 +3128,43 @@ class B: **Python:** _Can't be done without a long decorator definition. The full definition of the decorator in Python can be found in the Coconut header._ -#### `recursive_iterator` +#### `recursive_generator` -**recursive\_iterator**(_func_) +**recursive\_generator**(_func_) -Coconut provides a `recursive_iterator` decorator that memoizes any stateless, recursive function that returns an iterator. To use `recursive_iterator` on a function, it must meet the following criteria: +Coconut provides a `recursive_generator` decorator that memoizes and makes [`reiterable`](#reiterable) any generator or other stateless function that returns an iterator. To use `recursive_generator` on a function, it must meet the following criteria: 1. your function either always `return`s an iterator or generates an iterator using `yield`, 2. when called multiple times with arguments that are equal, your function produces the same iterator (your function is stateless), and 3. your function gets called (usually calls itself) multiple times with the same arguments. -If you are encountering a `RuntimeError` due to maximum recursion depth, it is highly recommended that you rewrite your function to meet either the criteria above for `recursive_iterator`, or the corresponding criteria for Coconut's [tail call optimization](#tail-call-optimization), either of which should prevent such errors. - -Furthermore, `recursive_iterator` also allows the resolution of a [nasty segmentation fault in Python's iterator logic that has never been fixed](http://bugs.python.org/issue14010). Specifically, instead of writing +Importantly, `recursive_generator` also allows the resolution of a [nasty segmentation fault in Python's iterator logic that has never been fixed](http://bugs.python.org/issue14010). Specifically, instead of writing ```coconut seq = get_elem() :: seq ``` which will crash due to the aforementioned Python issue, write ```coconut -@recursive_iterator +@recursive_generator def seq() = get_elem() :: seq() ``` which will work just fine. -One pitfall to keep in mind working with `recursive_iterator` is that it shouldn't be used in contexts where the function can potentially be called multiple times with the same iterator object as an input, but with that object not actually corresponding to the same items (e.g. because the first time the object hasn't been iterated over yet and the second time it has been). +One pitfall to keep in mind working with `recursive_generator` is that it shouldn't be used in contexts where the function can potentially be called multiple times with the same iterator object as an input, but with that object not actually corresponding to the same items (e.g. because the first time the object hasn't been iterated over yet and the second time it has been). + +_Deprecated: `recursive_iterator` is available as a deprecated alias for `recursive_generator`. Note that deprecated features are disabled in `--strict` mode._ ##### Example **Coconut:** ```coconut -@recursive_iterator +@recursive_generator def fib() = (1, 1) :: map((+), fib(), fib()$[1:]) ``` **Python:** _Can't be done without a long decorator definition. The full definition of the decorator in Python can be found in the Coconut header._ + ### Built-In Types ```{contents} @@ -3155,6 +3226,10 @@ data Expected[T](result: T? = None, error: BaseException? = None): def __bool__(self) -> bool: return self.error is None def __fmap__[U](self, func: T -> U) -> Expected[U]: + """Maps func over the result if it exists. + + __fmap__ should be used directly only when fmap is not available (e.g. when consuming an Expected in vanilla Python). + """ return self.__class__(func(self.result)) if self else self def and_then[U](self, func: T -> Expected[U]) -> Expected[U]: """Maps a T -> Expected[U] over an Expected[T] to produce an Expected[U]. @@ -3170,23 +3245,43 @@ data Expected[T](result: T? = None, error: BaseException? = None): def map_error(self, func: BaseException -> BaseException) -> Expected[T]: """Maps func over the error if it exists.""" return self if self else self.__class__(error=func(self.error)) + def handle(self, err_type, handler: BaseException -> T) -> Expected[T]: + """Recover from the given err_type by calling handler on the error to determine the result.""" + if not self and isinstance(self.error, err_type): + return self.__class__(handler(self.error)) + return self + def expect_error(self, *err_types: BaseException) -> Expected[T]: + """Raise any errors that do not match the given error types.""" + if not self and not isinstance(self.error, err_types): + raise self.error + return self + def unwrap(self) -> T: + """Unwrap the result or raise the error.""" + if not self: + raise self.error + return self.result def or_else[U](self, func: BaseException -> Expected[U]) -> Expected[T | U]: """Return self if no error, otherwise return the result of evaluating func on the error.""" return self if self else func(self.error) - def result_or[U](self, default: U) -> T | U: - """Return the result if it exists, otherwise return the default.""" - return self.result if self else default def result_or_else[U](self, func: BaseException -> U) -> T | U: """Return the result if it exists, otherwise return the result of evaluating func on the error.""" return self.result if self else func(self.error) - def unwrap(self) -> T: - """Unwrap the result or raise the error.""" - if not self: - raise self.error - return self.result + def result_or[U](self, default: U) -> T | U: + """Return the result if it exists, otherwise return the default. + + Since .result_or() completely silences errors, it is highly recommended that you + call .expect_error() first to explicitly declare what errors you are okay silencing. + """ + return self.result if self else default ``` -`Expected` is primarily used as the return type for [`safe_call`](#safe_call). Generally, the best way to use `Expected` is with [`fmap`](#fmap), which will apply a function to the result if it exists, or otherwise retain the error. If you want to sequence multiple `Expected`-returning operations, `.and_then` should be used instead of `fmap`. +`Expected` is primarily used as the return type for [`safe_call`](#safe_call). + +Generally, the best way to use `Expected` is with [`fmap`](#fmap), which will apply a function to the result if it exists, or otherwise retain the error. If you want to sequence multiple `Expected`-returning operations, `.and_then` should be used instead of `fmap`. To handle specific errors, the following patterns are equivalent: +``` +safe_call(might_raise_IOError).handle(IOError, const 10).unwrap() +safe_call(might_raise_IOError).expect_error(IOError).result_or(10) +``` To match against an `Expected`, just: ``` @@ -3219,6 +3314,7 @@ Additionally, if you are using [view patterns](#match), you might need to raise In some cases where there are multiple Coconut packages installed at the same time, there may be multiple `MatchError`s defined in different packages. Coconut can perform some magic under the hood to make sure that all these `MatchError`s will seamlessly interoperate, but only if all such packages are compiled in [`--package` mode rather than `--standalone` mode](#compilation-modes). + ### Generic Built-In Functions ```{contents} @@ -3334,6 +3430,8 @@ def safe_call(f, /, *args, **kwargs): return Expected(error=err) ``` +To define a function that always returns an `Expected` rather than raising any errors, simply decorate it with `@safe_call$`. + ##### Example **Coconut:** @@ -3491,6 +3589,7 @@ async def load_and_send_data(): return await send_data(proc_data(await load_data_async())) ``` + ### Built-Ins for Working with Iterators ```{contents} @@ -3504,11 +3603,12 @@ depth: 1 Coconut's `map`, `zip`, `filter`, `reversed`, and `enumerate` objects are enhanced versions of their Python equivalents that support: +- The ability to be iterated over multiple times if the underlying iterators can be iterated over multiple times. + - _Note: This can lead to different behavior between Coconut built-ins and Python built-ins. Use `py_` versions if the Python behavior is necessary._ - `reversed` - `repr` - Optimized normal (and iterator) indexing/slicing (`map`, `zip`, `reversed`, and `enumerate` but not `filter`). - `len` (all but `filter`) (though `bool` will still always yield `True`). -- The ability to be iterated over multiple times if the underlying iterators can be iterated over multiple times. - [PEP 618](https://www.python.org/dev/peps/pep-0618) `zip(..., strict=True)` support on all Python versions. - Added `strict=True` support to `map` as well (enforces that iterables are the same length in the multi-iterable case; uses `zip` under the hood such that errors will show up as `zip(..., strict=True)` errors). - Added attributes which subclasses can make use of to get at the original arguments to the object: @@ -3839,16 +3939,16 @@ max_so_far = input_data[0] for x in input_data: if x > max_so_far: max_so_far = x - running_max.append(x) + running_max.append(max_so_far) ``` #### `count` **count**(_start_=`0`, _step_=`1`) -Coconut provides a modified version of `itertools.count` that supports `in`, normal slicing, optimized iterator slicing, the standard `count` and `index` sequence methods, `repr`, and `start`/`step` attributes as a built-in under the name `count`. +Coconut provides a modified version of `itertools.count` that supports `in`, normal slicing, optimized iterator slicing, the standard `count` and `index` sequence methods, `repr`, and `start`/`step` attributes as a built-in under the name `count`. If the _step_ parameter is set to `None`, `count` will behave like `itertools.repeat` instead. -Additionally, if the _step_ parameter is set to `None`, `count` will behave like `itertools.repeat` instead. +Since `count` supports slicing, `count()[...]` can be used as a version of `range` that can in some cases be more readable. In particular, it is easy to accidentally write `range(10, 2)` when you meant `range(0, 10, 2)`, but it is hard to accidentally write `count()[10:2]` when you mean `count()[:10:2]`. ##### Python Docs @@ -4039,56 +4139,6 @@ assert "12345" |> windowsof$(3) |> list == [("1", "2", "3"), ("2", "3", "4"), (" **Python:** _Can't be done without the definition of `windowsof`; see the compiled header for the full definition._ -#### `collectby` - -**collectby**(_key\_func_, _iterable_, _value\_func_=`None`, _reduce\_func_=`None`) - -`collectby(key_func, iterable)` collects the items in `iterable` into a dictionary of lists keyed by `key_func(item)`. - -If `value_func` is passed, `collectby(key_func, iterable, value_func=value_func)` instead collects value_func(item) into each list instead of item. - -If `reduce_func` is passed, `collectby(key_func, iterable, reduce_func=reduce_func)`, instead of collecting the items into lists, reduces over the items of each key with reduce_func, effectively implementing a MapReduce operation. - -`collectby` is effectively equivalent to: -```coconut_python -from collections import defaultdict - -def collectby(key_func, iterable, value_func=ident, reduce_func=None): - collection = defaultdict(list) if reduce_func is None else {} - for item in iterable: - key = key_func(item) - value = value_func(item) - if reduce_func is None: - collection[key].append(value) - else: - old_value = collection.get(key, sentinel) - if old_value is not sentinel: - value = reduce_func(old_value, value) - collection[key] = value - return collection -``` - -`collectby` is similar to [`itertools.groupby`](https://docs.python.org/3/library/itertools.html#itertools.groupby) except that `collectby` aggregates common elements regardless of their order in the input iterable, whereas `groupby` only aggregates common elements that are adjacent in the input iterable. - -##### Example - -**Coconut:** -```coconut -user_balances = ( - balance_data - |> collectby$(.user, value_func=.balance, reduce_func=(+)) -) -``` - -**Python:** -```coconut_python -from collections import defaultdict - -user_balances = defaultdict(int) -for item in balance_data: - user_balances[item.user] += item.balance -``` - #### `all_equal` **all\_equal**(_iterable_) @@ -4118,62 +4168,145 @@ all_equal([1, 1, 1]) all_equal([1, 1, 2]) ``` -#### `parallel_map` +#### `tee` -**parallel\_map**(_function_, *_iterables_, *, _chunksize_=`1`, _strict_=`False`) +**tee**(_iterable_, _n_=`2`) -Coconut provides a parallel version of `map` under the name `parallel_map`. `parallel_map` makes use of multiple processes, and is therefore much faster than `map` for CPU-bound tasks. `parallel_map` never loads the entire input iterator into memory, though it does consume the entire input iterator as soon as a single output is requested. If any exceptions are raised inside of `parallel_map`, a traceback will be printed as soon as they are encountered. +Coconut provides an optimized version of `itertools.tee` as a built-in under the name `tee`. -Because `parallel_map` uses multiple processes for its execution, it is necessary that all of its arguments be pickleable. Only objects defined at the module level, and not lambdas, objects defined inside of a function, or objects defined inside of the interpreter, are pickleable. Furthermore, on Windows, it is necessary that all calls to `parallel_map` occur inside of an `if __name__ == "__main__"` guard. +Though `tee` is not deprecated, [`reiterable`](#reiterable) is generally recommended over `tee`. -`parallel_map` supports a `chunksize` argument, which determines how many items are passed to each process at a time. Larger values of _chunksize_ are recommended when dealing with very long iterables. Additionally, in the multi-iterable case, _strict_ can be set to `True` to ensure that all iterables are the same length. +Custom `tee`/`reiterable` implementations for custom [Containers/Collections](https://docs.python.org/3/library/collections.abc.html) should be put in the `__copy__` method. Note that all [Sequences/Mappings/Sets](https://docs.python.org/3/library/collections.abc.html) are always assumed to be reiterable even without calling `__copy__`. -If multiple sequential calls to `parallel_map` need to be made, it is highly recommended that they be done inside of a `with parallel_map.multiple_sequential_calls():` block, which will cause the different calls to use the same process pool and result in `parallel_map` immediately returning a list rather than a `parallel_map` object. If multiple sequential calls are necessary and the laziness of parallel_map is required, then the `parallel_map` objects should be constructed before the `multiple_sequential_calls` block and then only iterated over once inside the block. +##### Python Docs -`parallel_map.multiple_sequential_calls` also supports a `max_workers` argument to set the number of processes. +**tee**(_iterable, n=2_) -##### Python Docs +Return _n_ independent iterators from a single iterable. Equivalent to: +```coconut_python +def tee(iterable, n=2): + it = iter(iterable) + deques = [collections.deque() for i in range(n)] + def gen(mydeque): + while True: + if not mydeque: # when the local deque is empty + newval = next(it) # fetch a new value and + for d in deques: # load it to all the deques + d.append(newval) + yield mydeque.popleft() + return tuple(gen(d) for d in deques) +``` +Once `tee()` has made a split, the original _iterable_ should not be used anywhere else; otherwise, the _iterable_ could get advanced without the tee objects being informed. -**parallel_map**(_func, \*iterables_, _chunksize_=`1`) +This itertool may require significant auxiliary storage (depending on how much temporary data needs to be stored). In general, if one iterator uses most or all of the data before another iterator starts, it is faster to use `list()` instead of `tee()`. -Equivalent to `map(func, *iterables)` except _func_ is executed asynchronously and several calls to _func_ may be made concurrently. If a call raises an exception, then that exception will be raised when its value is retrieved from the iterator. +##### Example + +**Coconut:** +```coconut +original, temp = tee(original) +sliced = temp$[5:] +``` + +**Python:** +```coconut_python +import itertools +original, temp = itertools.tee(original) +sliced = itertools.islice(temp, 5, None) +``` + +#### `consume` -`parallel_map` chops the iterable into a number of chunks which it submits to the process pool as separate tasks. The (approximate) size of these chunks can be specified by setting _chunksize_ to a positive integer. For very long iterables using a large value for _chunksize_ can make the job complete **much** faster than using the default value of `1`. +**consume**(_iterable_, _keep\_last_=`0`) + +Coconut provides the `consume` function to efficiently exhaust an iterator and thus perform any lazy evaluation contained within it. `consume` takes one optional argument, `keep_last`, that defaults to 0 and specifies how many, if any, items from the end to return as a sequence (`None` will keep all elements). + +Equivalent to: +```coconut +def consume(iterable, keep_last=0): + """Fully exhaust iterable and return the last keep_last elements.""" + return collections.deque(iterable, maxlen=keep_last) # fastest way to exhaust an iterator +``` + +##### Rationale + +In the process of lazily applying operations to iterators, eventually a point is reached where evaluation of the iterator is necessary. To do this efficiently, Coconut provides the `consume` function, which will fully exhaust the iterator given to it. ##### Example **Coconut:** ```coconut -parallel_map(pow$(2), range(100)) |> list |> print +range(10) |> map$((x) => x**2) |> map$(print) |> consume ``` **Python:** ```coconut_python -import functools -from multiprocessing import Pool -with Pool() as pool: - print(list(pool.imap(functools.partial(pow, 2), range(100)))) +collections.deque(map(print, map(lambda x: x**2, range(10))), maxlen=0) ``` -#### `concurrent_map` -**concurrent\_map**(_function_, *_iterables_, *, _chunksize_=`1`, _strict_=`False`) +### Built-Ins for Parallelization + +#### `process_map` and `thread_map` + +##### **process\_map**(_function_, *_iterables_, *, _chunksize_=`1`, _strict_=`False`, _stream_=`False`, _ordered_=`True`) + +Coconut provides a `multiprocessing`-based version of `map` under the name `process_map`. `process_map` makes use of multiple processes, and is therefore much faster than `map` for CPU-bound tasks. If any exceptions are raised inside of `process_map`, a traceback will be printed as soon as they are encountered. Results will be in the same order as the input unless _ordered_=`False`. + +`process_map` never loads the entire input iterator into memory, though by default it does consume the entire input iterator as soon as a single output is requested. Results can be streamed one at a time when iterating by passing _stream_=`True`, however note that _stream_=`True` requires that the resulting iterator only be iterated over inside of a `process_map.multiple_sequential_calls` block (see below). + +Because `process_map` uses multiple processes for its execution, it is necessary that all of its arguments be pickleable. Only objects defined at the module level, and not lambdas, objects defined inside of a function, or objects defined inside of the interpreter, are pickleable. Furthermore, on Windows, it is necessary that all calls to `process_map` occur inside of an `if __name__ == "__main__"` guard. + +`process_map` supports a `chunksize` argument, which determines how many items are passed to each process at a time. Larger values of _chunksize_ are recommended when dealing with very long iterables. Additionally, in the multi-iterable case, _strict_ can be set to `True` to ensure that all iterables are the same length. + +_Deprecated: `parallel_map` is available as a deprecated alias for `process_map`. Note that deprecated features are disabled in `--strict` mode._ + +##### **process\_map\.multiple\_sequential\_calls**(_max\_workers_=`None`) + +If multiple sequential calls to `process_map` need to be made, it is highly recommended that they be done inside of a `with process_map.multiple_sequential_calls():` block, which will cause the different calls to use the same process pool and result in `process_map` immediately returning a list rather than a `process_map` object. If multiple sequential calls are necessary and the laziness of process_map is required, then the `process_map` objects should be constructed before the `multiple_sequential_calls` block and then only iterated over once inside the block. -Coconut provides a concurrent version of [`parallel_map`](#parallel_map) under the name `concurrent_map`. `concurrent_map` behaves identically to `parallel_map` (including support for `concurrent_map.multiple_sequential_calls`) except that it uses multithreading instead of multiprocessing, and is therefore primarily useful only for IO-bound tasks due to CPython's Global Interpreter Lock. +`process_map.multiple_sequential_calls` also supports a _max\_workers_ argument to set the number of processes. If `max_workers=None`, Coconut will pick a suitable _max\_workers_, including reusing worker pools from higher up in the call stack. + +##### **thread\_map**(_function_, *_iterables_, *, _chunksize_=`1`, _strict_=`False`, _stream_=`False`, _ordered_=`True`) + +##### **thread\_map\.multiple\_sequential\_calls**(_max\_workers_=`None`) + +Coconut provides a `multithreading`-based version of [`process_map`](#process_map) under the name `thread_map`. `thread_map` and `thread_map.multiple_sequential_calls` behave identically to `process_map` except that they use multithreading instead of multiprocessing, and are therefore primarily useful only for IO-bound tasks due to CPython's Global Interpreter Lock. + +_Deprecated: `concurrent_map` is available as a deprecated alias for `thread_map`. Note that deprecated features are disabled in `--strict` mode._ ##### Python Docs -**concurrent_map**(_func, \*iterables_, _chunksize_=`1`) +**process_map**(_func, \*iterables_, _chunksize_=`1`) Equivalent to `map(func, *iterables)` except _func_ is executed asynchronously and several calls to _func_ may be made concurrently. If a call raises an exception, then that exception will be raised when its value is retrieved from the iterator. -`concurrent_map` chops the iterable into a number of chunks which it submits to the thread pool as separate tasks. The (approximate) size of these chunks can be specified by setting _chunksize_ to a positive integer. For very long iterables using a large value for _chunksize_ can make the job complete **much** faster than using the default value of `1`. +`process_map` chops the iterable into a number of chunks which it submits to the process pool as separate tasks. The (approximate) size of these chunks can be specified by setting _chunksize_ to a positive integer. For very long iterables using a large value for _chunksize_ can make the job complete **much** faster than using the default value of `1`. -##### Example +**thread_map**(_func, \*iterables_, _chunksize_=`1`) + +Equivalent to `map(func, *iterables)` except _func_ is executed asynchronously and several calls to _func_ may be made concurrently. If a call raises an exception, then that exception will be raised when its value is retrieved from the iterator. + +`thread_map` chops the iterable into a number of chunks which it submits to the thread pool as separate tasks. The (approximate) size of these chunks can be specified by setting _chunksize_ to a positive integer. For very long iterables using a large value for _chunksize_ can make the job complete **much** faster than using the default value of `1`. + +##### Examples **Coconut:** ```coconut -concurrent_map(get_data_for_user, get_all_users()) |> list |> print +process_map(pow$(2), range(100)) |> list |> print +``` + +**Python:** +```coconut_python +import functools +from multiprocessing import Pool +with Pool() as pool: + print(list(pool.imap(functools.partial(pow, 2), range(100)))) +``` + +**Coconut:** +```coconut +thread_map(get_data_for_user, get_all_users()) |> list |> print ``` **Python:** @@ -4184,82 +4317,127 @@ with concurrent.futures.ThreadPoolExecutor() as executor: print(list(executor.map(get_data_for_user, get_all_users()))) ``` -#### `tee` +#### `collectby` and `mapreduce` -**tee**(_iterable_, _n_=`2`) +##### **collectby**(_key\_func_, _iterable_, _value\_func_=`None`, \*, _reduce\_func_=`None`, _collect\_in_=`None`, _reduce\_func\_init_=`...`, _map\_using_=`None`) -Coconut provides an optimized version of `itertools.tee` as a built-in under the name `tee`. +`collectby(key_func, iterable)` collects the items in `iterable` into a dictionary of lists keyed by `key_func(item)`. -Though `tee` is not deprecated, [`reiterable`](#reiterable) is generally recommended over `tee`. +If _value\_func_ is passed, instead collects `value_func(item)` into each list instead of `item`. -Custom `tee`/`reiterable` implementations for custom [Containers/Collections](https://docs.python.org/3/library/collections.abc.html) should be put in the `__copy__` method. Note that all [Sequences/Mappings/Sets](https://docs.python.org/3/library/collections.abc.html) are always assumed to be reiterable even without calling `__copy__`. +If _reduce\_func_ is passed, instead of collecting the items into lists, [`reduce`](#reduce) over the items of each key with _reduce\_func_, effectively implementing a MapReduce operation. If keys are intended to be unique, set `reduce_func=False` (`reduce_func=False` is also the default if _collect\_in_ is passed). If _reduce\_func_ is passed, then _reduce\_func\_init_ may also be passed, and will determine the initial value when reducing with _reduce\_func_. -##### Python Docs +If _collect\_in_ is passed, initializes the collection from _collect\_in_ rather than as a `collections.defaultdict` (if `reduce_func=None`) or an empty `dict` (otherwise). Additionally, _reduce\_func_ defaults to `False` rather than `None` when _collect\_in_ is passed. Useful when you want to collect the results into a `pandas.DataFrame`. -**tee**(_iterable, n=2_) +If _map_using_ is passed, calculates `key_func` and `value_func` by mapping them over the iterable using `map_using` as `map`. Useful with [`process_map`](#process_map)/[`thread_map`](#thread_map). See `.using_threads` and `.using_processes` methods below for simple shortcut methods that make use of `map_using` internally. -Return _n_ independent iterators from a single iterable. Equivalent to: -```coconut_python -def tee(iterable, n=2): - it = iter(iterable) - deques = [collections.deque() for i in range(n)] - def gen(mydeque): - while True: - if not mydeque: # when the local deque is empty - newval = next(it) # fetch a new value and - for d in deques: # load it to all the deques - d.append(newval) - yield mydeque.popleft() - return tuple(gen(d) for d in deques) -``` -Once `tee()` has made a split, the original _iterable_ should not be used anywhere else; otherwise, the _iterable_ could get advanced without the tee objects being informed. +`collectby` is similar to [`itertools.groupby`](https://docs.python.org/3/library/itertools.html#itertools.groupby) except that `collectby` aggregates common elements regardless of their order in the input iterable, whereas `groupby` only aggregates common elements that are adjacent in the input iterable. -This itertool may require significant auxiliary storage (depending on how much temporary data needs to be stored). In general, if one iterator uses most or all of the data before another iterator starts, it is faster to use `list()` instead of `tee()`. +##### **mapreduce**(_key\_value\_func_, _iterable_, \*, _reduce\_func_=`None`, _collect\_in_=`None`, _reduce\_func\_init_=`...`, _map\_using_=`None`) + +`mapreduce(key_value_func, iterable)` functions the same as `collectby`, but allows calculating the keys and values together in one function. _key\_value\_func_ must return a 2-tuple of `(key, value)`. + +##### **collectby.using_threads**(_key\_func_, _iterable_, _value\_func_=`None`, \*, _reduce\_func_=`None`, _collect\_in_=`None`, _reduce\_func\_init_=`...`, _ordered_=`False`, _chunksize_=`1`, _max\_workers_=`None`) + +##### **collectby.using_processes**(_key\_func_, _iterable_, _value\_func_=`None`, \*, _reduce\_func_=`None`, _collect\_in_=`None`, _reduce\_func\_init_=`...`, _ordered_=`False`, _chunksize_=`1`, _max\_workers_=`None`) + +##### **mapreduce.using_threads**(_key\_value\_func_, _iterable_, \*, _reduce\_func_=`None`, _collect\_in_=`None`, _reduce\_func\_init_=`...`, _ordered_=`False`, _chunksize_=`1`, _max\_workers_=`None`) + +##### **mapreduce.using_processes**(_key\_value\_func_, _iterable_, \*, _reduce\_func_=`None`, _collect\_in_=`None`, _reduce\_func\_init_=`...`, _ordered_=`False`, _chunksize_=`1`, _max\_workers_=`None`) + +These shortcut methods call `collectby`/`mapreduce` with `map_using` set to [`process_map`](#process_map)/[`thread_map`](#thread_map), properly managed using the `.multiple_sequential_calls` method and the `stream=True` argument of [`process_map`](#process_map)/[`thread_map`](#thread_map). `reduce_func` will be called as soon as results arrive, and by default in whatever order they arrive in (to enforce the original order, pass _ordered_=`True`). + +To make multiple sequential calls to `collectby.using_threads()`/`mapreduce.using_threads()`, manage them using `thread_map.multiple_sequential_calls()`. Similarly, use `process_map.multiple_sequential_calls()` to manage `.using_processes()`. + +Note that, for very long iterables, it is highly recommended to pass a value other than the default `1` for _chunksize_. + +As an example, `mapreduce.using_processes` is effectively equivalent to: +```coconut +def mapreduce.using_processes(key_value_func, iterable, *, reduce_func=None, ordered=False, chunksize=1, max_workers=None): + with process_map.multiple_sequential_calls(max_workers=max_workers): + return mapreduce( + key_value_func, + iterable, + reduce_func=reduce_func, + map_using=process_map$( + stream=True, + ordered=ordered, + chunksize=chunksize, + ), + ) +``` ##### Example **Coconut:** ```coconut -original, temp = tee(original) -sliced = temp$[5:] +user_balances = ( + balance_data + |> collectby$(.user, value_func=.balance, reduce_func=(+)) +) ``` **Python:** ```coconut_python -import itertools -original, temp = itertools.tee(original) -sliced = itertools.islice(temp, 5, None) +from collections import defaultdict + +user_balances = defaultdict(int) +for item in balance_data: + user_balances[item.user] += item.balance ``` -#### `consume` +#### `async_map` -**consume**(_iterable_, _keep\_last_=`0`) +**async\_map**(_async\_func_, *_iters_, _strict_=`False`) -Coconut provides the `consume` function to efficiently exhaust an iterator and thus perform any lazy evaluation contained within it. `consume` takes one optional argument, `keep_last`, that defaults to 0 and specifies how many, if any, items from the end to return as a sequence (`None` will keep all elements). +`async_map` maps _async\_func_ over _iters_ asynchronously using [`anyio`](https://anyio.readthedocs.io/en/stable/), which must be installed for _async\_func_ to work. _strict_ functions as in [`map`/`zip`](#enhanced-built-ins), enforcing that all the _iters_ must have the same length. Equivalent to: ```coconut -def consume(iterable, keep_last=0): - """Fully exhaust iterable and return the last keep_last elements.""" - return collections.deque(iterable, maxlen=keep_last) # fastest way to exhaust an iterator +async def async_map[T, U]( + async_func: async T -> U, + *iters: T$[], + strict: bool = False +) -> U[]: + """Map async_func over iters asynchronously using anyio.""" + import anyio + results = [] + async def store_func_in_of(i, args): + got = await async_func(*args) + results.extend([None] * (1 + i - len(results))) + results[i] = got + async with anyio.create_task_group() as nursery: + for i, args in enumerate(zip(*iters, strict=strict)): + nursery.start_soon(store_func_in_of, i, args) + return results ``` -##### Rationale - -In the process of lazily applying operations to iterators, eventually a point is reached where evaluation of the iterator is necessary. To do this efficiently, Coconut provides the `consume` function, which will fully exhaust the iterator given to it. - ##### Example **Coconut:** ```coconut -range(10) |> map$((x) => x**2) |> map$(print) |> consume +async def load_pages(urls) = ( + urls + |> async_map$(load_page) + |> await +) ``` **Python:** ```coconut_python -collections.deque(map(print, map(lambda x: x**2, range(10))), maxlen=0) +import anyio + +async def load_pages(urls): + results = [None] * len(urls) + async def proc_url(i, url): + results[i] = await load_page(url) + async with anyio.create_task_group() as nursery: + for i, url in enumerate(urls) + nursery.start_soon(proc_url, i, url) + return results ``` + ### Typing-Specific Built-Ins ```{contents} @@ -4361,6 +4539,7 @@ from coconut.__coconut__ import fmap reveal_type(fmap) ``` + ## Coconut API ```{contents} @@ -4370,6 +4549,7 @@ depth: 2 --- ``` + ### `coconut.embed` **coconut.embed**(_kernel_=`None`, _depth_=`0`, \*\*_kwargs_) @@ -4378,6 +4558,7 @@ If _kernel_=`False` (default), embeds a Coconut Jupyter console initialized from Recommended usage is as a debugging tool, where the code `from coconut import embed; embed()` can be inserted to launch an interactive Coconut shell initialized from that point. + ### Automatic Compilation Automatic compilation lets you simply import Coconut files directly without having to go through a compilation step first. Automatic compilation can be enabled either by importing [`coconut.api`](#coconut-api) before you import anything else, or by running `coconut --site-install`. @@ -4390,6 +4571,7 @@ Automatic compilation is always available in the Coconut interpreter or when usi If using the Coconut interpreter, a `reload` built-in is always provided to easily reload (and thus recompile) imported modules. + ### Coconut Encoding While automatic compilation is the preferred method for dynamically compiling Coconut files, as it caches the compiled code as a `.py` file to prevent recompilation, Coconut also supports a special @@ -4398,6 +4580,7 @@ While automatic compilation is the preferred method for dynamically compiling Co ``` declaration which can be added to `.py` files to have them treated as Coconut files instead. To use such a coding declaration, you'll need to either run `coconut --site-install` or `import coconut.api` at some point before you first attempt to import a file with a `# coding: coconut` declaration. Like automatic compilation, the Coconut encoding is always available from the Coconut interpreter. Compilation always uses the same parameters as in the [Coconut Jupyter kernel](#kernel). + ### `coconut.api` In addition to enabling automatic compilation, `coconut.api` can also be used to call the Coconut compiler from code instead of from the command line. See below for specifications of the different api functions. @@ -4479,18 +4662,24 @@ If _state_ is `False`, the global state object is used. #### `warm_up` -**coconut.api.warm_up**(_force_=`True`, _enable\_incremental\_mode_=`False`, *, _state_=`False`) +**coconut.api.warm_up**(_streamline_=`True`, _enable\_incremental\_mode_=`False`, *, _state_=`False`) -Can optionally be called to warm up the compiler and get it ready for parsing. Passing _force_ will cause the warm up to take longer but will substantially reduce parsing times (by default, this level of warm up is only done when the compiler encounters a large file). Passing _enable\_incremental\_mode_ will enable the compiler's incremental mdoe, where parsing some string, then later parsing a continuation of that string, will yield substantial performance improvements. +Can optionally be called to warm up the compiler and get it ready for parsing. Passing _streamline_ will cause the warm up to take longer but will substantially reduce parsing times (by default, this level of warm up is only done when the compiler encounters a large file). Passing _enable\_incremental\_mode_ will enable the compiler's incremental mdoe, where parsing some string, then later parsing a continuation of that string, will yield substantial performance improvements. #### `cmd` -**coconut.api.cmd**(_args_=`None`, *, _argv_=`None`, _interact_=`False`, _default\_target_=`None`, _state_=`False`) +**coconut.api.cmd**(_args_=`None`, *, _argv_=`None`, _interact_=`False`, _default\_target_=`None`, _default\_jobs_=`None`, _state_=`False`) Executes the given _args_ as if they were fed to `coconut` on the command-line, with the exception that unless _interact_ is true or `-i` is passed, the interpreter will not be started. Additionally, _argv_ can be used to pass in arguments as in `--argv` and _default\_target_ can be used to set the default `--target`. Has the same effect of setting the command-line flags on the given _state_ object as `setup` (with the global `state` object used when _state_ is `False`). +#### `cmd_sys` + +**coconut.api.cmd_sys**(_args_=`None`, *, _argv_=`None`, _interact_=`False`, _default\_target_=`"sys"`, _default\_jobs_=`"0"`, _state_=`False`) + +Same as `coconut.api.cmd` but _default\_target_ is `"sys"` rather than `None` (universal) and _default\_jobs_=`"0"` rather than `None` (`"sys"`). Since `cmd_sys` defaults to not using `multiprocessing`, it is preferred whenever that might be a problem, e.g. [if you're not inside an `if __name__ == "__main__"` block on Windows](https://stackoverflow.com/questions/20360686/compulsory-usage-of-if-name-main-in-windows-while-using-multiprocessi). + #### `coconut_exec` **coconut.api.coconut_exec**(_expression_, _globals_=`None`, _locals_=`None`, _state_=`False`, _keep\_internal\_state_=`None`) @@ -4503,18 +4692,6 @@ Version of [`exec`](https://docs.python.org/3/library/functions.html#exec) which Version of [`eval`](https://docs.python.org/3/library/functions.html#eval) which can evaluate Coconut code. -#### `version` - -**coconut.api.version**(**[**_which_**]**) - -Retrieves a string containing information about the Coconut version. The optional argument _which_ is the type of version information desired. Possible values of _which_ are: - -- `"num"`: the numerical version (the default) -- `"name"`: the version codename -- `"spec"`: the numerical version with the codename attached -- `"tag"`: the version tag used in GitHub and documentation URLs -- `"-v"`: the full string printed by `coconut -v` - #### `auto_compilation` **coconut.api.auto_compilation**(_on_=`True`, _args_=`None`, _use\_cache\_dir_=`None`) @@ -4531,10 +4708,48 @@ If _use\_cache\_dir_ is passed, it will turn on or off the usage of a `__coconut Switches the [`breakpoint` built-in](https://www.python.org/dev/peps/pep-0553/) which Coconut makes universally available to use [`coconut.embed`](#coconut-embed) instead of [`pdb.set_trace`](https://docs.python.org/3/library/pdb.html#pdb.set_trace) (or undoes that switch if `on=False`). This function is called automatically when `coconut.api` is imported. +#### `find_packages` and `find_and_compile_packages` + +**coconut.api.find_packages**(_where_=`"."`, _exclude_=`()`, _include_=`("*",)`) + +**coconut.api.find_and_compile_packages**(_where_=`"."`, _exclude_=`()`, _include_=`("*",)`) + +Both functions behave identically to [`setuptools.find_packages`](https://setuptools.pypa.io/en/latest/userguide/quickstart.html#package-discovery), except that they find Coconut packages rather than Python packages. `find_and_compile_packages` additionally compiles any Coconut packages that it finds in-place. + +Note that if you want to use either of these functions in your `setup.py`, you'll need to include `coconut` as a [build-time dependency in your `pyproject.toml`](https://pip.pypa.io/en/stable/reference/build-system/pyproject-toml/#build-time-dependencies). If you want `setuptools` to package your Coconut files, you'll also need to add `global-include *.coco` to your [`MANIFEST.in`](https://setuptools.pypa.io/en/latest/userguide/miscellaneous.html). + +##### Example + +```coconut_python +# if you put this in your setup.py, your Coconut package will be compiled in-place whenever it is installed + +from setuptools import setup +from coconut.api import find_and_compile_packages + +setup( + name=..., + version=..., + packages=find_and_compile_packages(), +) +``` + +#### `version` + +**coconut.api.version**(**[**_which_**]**) + +Retrieves a string containing information about the Coconut version. The optional argument _which_ is the type of version information desired. Possible values of _which_ are: + +- `"num"`: the numerical version (the default) +- `"name"`: the version codename +- `"spec"`: the numerical version with the codename attached +- `"tag"`: the version tag used in GitHub and documentation URLs +- `"-v"`: the full string printed by `coconut -v` + #### `CoconutException` If an error is encountered in a api function, a `CoconutException` instance may be raised. `coconut.api.CoconutException` is provided to allow catching such errors. + ### `coconut.__coconut__` It is sometimes useful to be able to access Coconut built-ins from pure Python. To accomplish this, Coconut provides `coconut.__coconut__`, which behaves exactly like the `__coconut__.py` header file included when Coconut is compiled in package mode. @@ -4544,5 +4759,5 @@ All Coconut built-ins are accessible from `coconut.__coconut__`. The recommended ##### Example ```coconut_python -from coconut.__coconut__ import parallel_map +from coconut.__coconut__ import process_map ``` diff --git a/FAQ.md b/FAQ.md index 201885b2e..d197f42ed 100644 --- a/FAQ.md +++ b/FAQ.md @@ -34,9 +34,9 @@ Information on every Coconut release is chronicled on the [GitHub releases page] Yes! Coconut compiles the [newest](https://www.python.org/dev/peps/pep-0526/), [fanciest](https://www.python.org/dev/peps/pep-0484/) type annotation syntax into version-independent type comments which can then by checked using Coconut's built-in [MyPy Integration](./DOCS.md#mypy-integration). -### Help! I tried to write a recursive iterator and my Python segfaulted! +### Help! I tried to write a recursive generator and my Python segfaulted! -No problem—just use Coconut's [`recursive_iterator`](./DOCS.md#recursive-iterator) decorator and you should be fine. This is a [known Python issue](http://bugs.python.org/issue14010) but `recursive_iterator` will fix it for you. +No problem—just use Coconut's [`recursive_generator`](./DOCS.md#recursive_generator) decorator and you should be fine. This is a [known Python issue](http://bugs.python.org/issue14010) but `recursive_generator` will fix it for you. ### How do I split an expression across multiple lines in Coconut? diff --git a/Makefile b/Makefile index 99ddd3752..93742e5e7 100644 --- a/Makefile +++ b/Makefile @@ -25,33 +25,37 @@ dev-py3: clean setup-py3 .PHONY: setup setup: - python -m ensurepip + -python -m ensurepip python -m pip install --upgrade setuptools wheel pip pytest_remotedata cython .PHONY: setup-py2 setup-py2: - python2 -m ensurepip + -python2 -m ensurepip python2 -m pip install --upgrade "setuptools<58" wheel pip pytest_remotedata cython .PHONY: setup-py3 setup-py3: - python3 -m ensurepip + -python3 -m ensurepip python3 -m pip install --upgrade setuptools wheel pip pytest_remotedata cython .PHONY: setup-pypy setup-pypy: - pypy -m ensurepip + -pypy -m ensurepip pypy -m pip install --upgrade "setuptools<58" wheel pip pytest_remotedata .PHONY: setup-pypy3 setup-pypy3: - pypy3 -m ensurepip + -pypy3 -m ensurepip pypy3 -m pip install --upgrade setuptools wheel pip pytest_remotedata .PHONY: install install: setup python -m pip install -e .[tests] +.PHONY: install-purepy +install-purepy: setup + python -m pip install --no-deps --upgrade -e . "pyparsing<3" + .PHONY: install-py2 install-py2: setup-py2 python2 -m pip install -e .[tests] @@ -127,11 +131,17 @@ test-pypy3: clean pypy3 ./coconut/tests/dest/runner.py pypy3 ./coconut/tests/dest/extras.py +# same as test-univ but reverses any ofs +.PHONY: test-any-of +test-any-of: export COCONUT_ADAPTIVE_ANY_OF=TRUE +test-any-of: export COCONUT_REVERSE_ANY_OF=TRUE +test-any-of: test-univ + # same as test-univ but also runs mypy .PHONY: test-mypy-univ test-mypy-univ: export COCONUT_USE_COLOR=TRUE test-mypy-univ: clean - python ./coconut/tests --strict --force --keep-lines --mypy --follow-imports silent --ignore-missing-imports --allow-redefinition + python ./coconut/tests --strict --keep-lines --force --mypy --follow-imports silent --ignore-missing-imports --allow-redefinition python ./coconut/tests/dest/runner.py python ./coconut/tests/dest/extras.py @@ -139,7 +149,7 @@ test-mypy-univ: clean .PHONY: test-mypy test-mypy: export COCONUT_USE_COLOR=TRUE test-mypy: clean - python ./coconut/tests --strict --force --target sys --keep-lines --mypy --follow-imports silent --ignore-missing-imports --allow-redefinition + python ./coconut/tests --strict --keep-lines --force --target sys --mypy --follow-imports silent --ignore-missing-imports --allow-redefinition python ./coconut/tests/dest/runner.py python ./coconut/tests/dest/extras.py @@ -147,11 +157,12 @@ test-mypy: clean .PHONY: test-mypy-tests test-mypy-tests: export COCONUT_USE_COLOR=TRUE test-mypy-tests: clean-no-tests - python ./coconut/tests --strict --target sys --keep-lines --mypy --follow-imports silent --ignore-missing-imports --allow-redefinition + python ./coconut/tests --strict --keep-lines --target sys --mypy --follow-imports silent --ignore-missing-imports --allow-redefinition python ./coconut/tests/dest/runner.py python ./coconut/tests/dest/extras.py # same as test-univ but includes verbose output for better debugging +# regex for getting non-timing lines: ^(?!\s*(Time|Packrat|Loaded|Saving|Adaptive|Errorless|Grammar|Failed|Incremental|Pruned)\s)[^\n]*\n* .PHONY: test-verbose test-verbose: export COCONUT_USE_COLOR=TRUE test-verbose: clean @@ -159,7 +170,15 @@ test-verbose: clean python ./coconut/tests/dest/runner.py python ./coconut/tests/dest/extras.py -# same as test-univ but includes verbose output for better debugging and is fully synchronous +# same as test-verbose but doesn't use the incremental cache +.PHONY: test-verbose-no-cache +test-verbose-no-cache: export COCONUT_USE_COLOR=TRUE +test-verbose-no-cache: clean + python ./coconut/tests --strict --keep-lines --force --verbose --no-cache + python ./coconut/tests/dest/runner.py + python ./coconut/tests/dest/extras.py + +# same as test-verbose but is fully synchronous .PHONY: test-verbose-sync test-verbose-sync: export COCONUT_USE_COLOR=TRUE test-verbose-sync: clean @@ -171,7 +190,7 @@ test-verbose-sync: clean .PHONY: test-mypy-verbose test-mypy-verbose: export COCONUT_USE_COLOR=TRUE test-mypy-verbose: clean - python ./coconut/tests --strict --force --target sys --verbose --keep-lines --mypy --follow-imports silent --ignore-missing-imports --allow-redefinition + python ./coconut/tests --strict --keep-lines --force --target sys --verbose --mypy --follow-imports silent --ignore-missing-imports --allow-redefinition python ./coconut/tests/dest/runner.py python ./coconut/tests/dest/extras.py @@ -179,7 +198,7 @@ test-mypy-verbose: clean .PHONY: test-mypy-all test-mypy-all: export COCONUT_USE_COLOR=TRUE test-mypy-all: clean - python ./coconut/tests --strict --force --target sys --keep-lines --mypy --follow-imports silent --ignore-missing-imports --allow-redefinition --check-untyped-defs + python ./coconut/tests --strict --keep-lines --force --target sys --mypy --follow-imports silent --ignore-missing-imports --allow-redefinition --check-untyped-defs python ./coconut/tests/dest/runner.py python ./coconut/tests/dest/extras.py @@ -192,9 +211,14 @@ test-easter-eggs: clean python ./coconut/tests/dest/extras.py # same as test-univ but uses python pyparsing -.PHONY: test-pyparsing -test-pyparsing: export COCONUT_PURE_PYTHON=TRUE -test-pyparsing: test-univ +.PHONY: test-purepy +test-purepy: export COCONUT_PURE_PYTHON=TRUE +test-purepy: test-univ + +# same as test-univ but disables the computation graph +.PHONY: test-no-computation-graph +test-no-computation-graph: export COCONUT_USE_COMPUTATION_GRAPH=FALSE +test-no-computation-graph: test-univ # same as test-univ but uses --minify .PHONY: test-minify @@ -217,30 +241,60 @@ test-no-wrap: clean test-watch: export COCONUT_USE_COLOR=TRUE test-watch: clean python ./coconut/tests --strict --keep-lines --force - coconut ./coconut/tests/src/cocotest/agnostic ./coconut/tests/dest/cocotest --watch --strict --keep-lines --stack-size 4096 --recursion-limit 4096 + make just-watch python ./coconut/tests/dest/runner.py python ./coconut/tests/dest/extras.py -# mini test that just compiles agnostic tests with fully synchronous output +# just watches tests +.PHONY: just-watch +just-watch: export COCONUT_USE_COLOR=TRUE +just-watch: + coconut ./coconut/tests/src/cocotest/agnostic ./coconut/tests/dest/cocotest --watch --strict --keep-lines --stack-size 4096 --recursion-limit 4096 + +# same as just-watch but uses verbose output and is fully sychronous and doesn't use the cache +.PHONY: just-watch-verbose +just-watch-verbose: export COCONUT_USE_COLOR=TRUE +just-watch-verbose: + coconut ./coconut/tests/src/cocotest/agnostic ./coconut/tests/dest/cocotest --watch --strict --keep-lines --stack-size 4096 --recursion-limit 4096 --verbose --jobs 0 --no-cache + +# mini test that just compiles agnostic tests .PHONY: test-mini +test-mini: export COCONUT_USE_COLOR=TRUE test-mini: - coconut ./coconut/tests/src/cocotest/agnostic ./coconut/tests/dest/cocotest --force --jobs 0 --stack-size 4096 --recursion-limit 4096 + coconut ./coconut/tests/src/cocotest/agnostic ./coconut/tests/dest/cocotest --force --stack-size 4096 --recursion-limit 4096 + +# same as test-mini but with verbose output +.PHONY: test-mini-verbose +test-mini-verbose: export COCONUT_USE_COLOR=TRUE +test-mini-verbose: + coconut ./coconut/tests/src/cocotest/agnostic ./coconut/tests/dest/cocotest --force --verbose --stack-size 4096 --recursion-limit 4096 + +# same as test-mini-verbose but doesn't overwrite the cache +.PHONY: test-mini-cache +test-mini-cache: export COCONUT_ALLOW_SAVE_TO_CACHE=FALSE +test-mini-cache: test-mini-verbose + +# same as test-mini-verbose but with fully synchronous output and fast failing +.PHONY: test-mini-sync +test-mini-sync: export COCONUT_USE_COLOR=TRUE +test-mini-sync: + coconut ./coconut/tests/src/cocotest/agnostic ./coconut/tests/dest/cocotest --force --verbose --jobs 0 --fail-fast --stack-size 4096 --recursion-limit 4096 # same as test-univ but debugs crashes .PHONY: test-univ-debug test-univ-debug: export COCONUT_TEST_DEBUG_PYTHON=TRUE test-univ-debug: test-univ -# same as test-mini but debugs crashes +# same as test-mini but debugs crashes, is fully synchronous, and doesn't use verbose output .PHONY: test-mini-debug test-mini-debug: export COCONUT_USE_COLOR=TRUE test-mini-debug: python -X dev -m coconut ./coconut/tests/src/cocotest/agnostic ./coconut/tests/dest/cocotest --strict --keep-lines --force --jobs 0 --stack-size 4096 --recursion-limit 4096 # same as test-mini-debug but uses vanilla pyparsing -.PHONY: test-mini-debug-pyparsing -test-mini-debug-pyparsing: export COCONUT_PURE_PYTHON=TRUE -test-mini-debug-pyparsing: test-mini-debug +.PHONY: test-mini-debug-purepy +test-mini-debug-purepy: export COCONUT_PURE_PYTHON=TRUE +test-mini-debug-purepy: test-mini-debug .PHONY: debug-test-crash debug-test-crash: @@ -266,16 +320,16 @@ clean-no-tests: .PHONY: clean clean: clean-no-tests rm -rf ./coconut/tests/dest + -find . -name "__coconut_cache__" -type d -prune -exec rm -rf '{}' + + -powershell -Command "get-childitem -Include __coconut_cache__ -Recurse -force | Remove-Item -Force -Recurse" .PHONY: wipe wipe: clean rm -rf ./coconut/tests/dest vprof.json profile.log *.egg-info - -find . -name "__pycache__" -delete - -C:/GnuWin32/bin/find.exe . -name "__pycache__" -delete - -find . -name "__coconut_cache__" -delete - -C:/GnuWin32/bin/find.exe . -name "__coconut_cache__" -delete + -find . -name "__pycache__" -type d -prune -exec rm -rf '{}' + + -powershell -Command "get-childitem -Include __pycache__ -Recurse -force | Remove-Item -Force -Recurse" -find . -name "*.pyc" -delete - -C:/GnuWin32/bin/find.exe . -name "*.pyc" -delete + -powershell -Command "get-childitem -Include *.pyc -Recurse -force | Remove-Item -Force -Recurse" -python -m coconut --site-uninstall -python3 -m coconut --site-uninstall -python2 -m coconut --site-uninstall @@ -302,20 +356,45 @@ upload: wipe dev just-upload check-reqs: python ./coconut/requirements.py -.PHONY: profile-parser -profile-parser: export COCONUT_USE_COLOR=TRUE -profile-parser: export COCONUT_PURE_PYTHON=TRUE -profile-parser: - coconut ./coconut/tests/src/cocotest/agnostic ./coconut/tests/dest/cocotest --force --profile --verbose --stack-size 4096 --recursion-limit 4096 2>&1 | tee ./profile.log - -.PHONY: profile-time -profile-time: - vprof -c h "./coconut ./coconut/tests/src/cocotest/agnostic ./coconut/tests/dest/cocotest --force --stack-size 4096 --recursion-limit 4096" --output-file ./vprof.json - -.PHONY: profile-memory -profile-memory: - vprof -c m "./coconut ./coconut/tests/src/cocotest/agnostic ./coconut/tests/dest/cocotest --force --stack-size 4096 --recursion-limit 4096" --output-file ./vprof.json - -.PHONY: view-profile -view-profile: +.PHONY: profile +profile: export COCONUT_USE_COLOR=TRUE +profile: + coconut ./coconut/tests/src/cocotest/agnostic/util.coco ./coconut/tests/dest/cocotest --force --jobs 0 --profile --verbose --stack-size 4096 --recursion-limit 4096 2>&1 | tee ./profile.log + +.PHONY: open-speedscope +open-speedscope: + npm install -g speedscope + speedscope ./profile.speedscope + +.PHONY: pyspy +pyspy: + py-spy record -o profile.speedscope --format speedscope --subprocesses --rate 75 -- python -m coconut ./coconut/tests/src/cocotest/agnostic ./coconut/tests/dest/cocotest --force + make open-speedscope + +.PHONY: pyspy-purepy +pyspy-purepy: export COCONUT_PURE_PYTHON=TRUE +pyspy-purepy: pyspy + +.PHONY: pyspy-native +pyspy-native: + py-spy record -o profile.speedscope --format speedscope --native --rate 75 -- python -m coconut ./coconut/tests/src/cocotest/agnostic ./coconut/tests/dest/cocotest --force --jobs 0 + make open-speedscope + +.PHONY: pyspy-runtime +pyspy-runtime: + py-spy record -o runtime_profile.speedscope --format speedscope -- python ./coconut/tests/dest/runner.py + speedscope ./runtime_profile.speedscope + +.PHONY: vprof-time +vprof-time: + vprof -c h "./coconut ./coconut/tests/src/cocotest/agnostic ./coconut/tests/dest/cocotest --force --jobs 0 --stack-size 4096 --recursion-limit 4096" --output-file ./vprof.json + make view-vprof + +.PHONY: vprof-memory +vprof-memory: + vprof -c m "./coconut ./coconut/tests/src/cocotest/agnostic ./coconut/tests/dest/cocotest --force --jobs 0 --stack-size 4096 --recursion-limit 4096" --output-file ./vprof.json + make view-vprof + +.PHONY: view-vprof +view-vprof: vprof --input-file ./vprof.json diff --git a/__coconut__/__init__.pyi b/__coconut__/__init__.pyi index d4b4ff4a6..70a0646f5 100644 --- a/__coconut__/__init__.pyi +++ b/__coconut__/__init__.pyi @@ -207,15 +207,14 @@ dropwhile = _coconut.itertools.dropwhile tee = _coconut.itertools.tee starmap = _coconut.itertools.starmap cartesian_product = _coconut.itertools.product -multiset = _coconut.collections.Counter +_coconut_partial = _coconut.functools.partial _coconut_tee = tee _coconut_starmap = starmap _coconut_cartesian_product = cartesian_product -_coconut_multiset = multiset -parallel_map = concurrent_map = _coconut_map = map +process_map = thread_map = parallel_map = concurrent_map = _coconut_map = map TYPE_CHECKING = _t.TYPE_CHECKING @@ -255,46 +254,50 @@ def _coconut_tco(func: _Tfunc) -> _Tfunc: # any changes here should also be made to safe_call and call_or_coefficient below @_t.overload def call( - _func: _t.Callable[[_T], _U], - _x: _T, + _func: _t.Callable[[], _U], ) -> _U: ... @_t.overload def call( - _func: _t.Callable[[_T, _U], _V], - _x: _T, - _y: _U, -) -> _V: ... -@_t.overload -def call( - _func: _t.Callable[[_T, _U, _V], _W], - _x: _T, - _y: _U, - _z: _V, -) -> _W: ... -@_t.overload -def call( - _func: _t.Callable[_t.Concatenate[_T, _P], _U], + _func: _t.Callable[[_T], _U], _x: _T, - *args: _t.Any, - **kwargs: _t.Any, ) -> _U: ... @_t.overload def call( - _func: _t.Callable[_t.Concatenate[_T, _U, _P], _V], + _func: _t.Callable[[_T, _U], _V], _x: _T, _y: _U, - *args: _t.Any, - **kwargs: _t.Any, ) -> _V: ... @_t.overload def call( - _func: _t.Callable[_t.Concatenate[_T, _U, _V, _P], _W], + _func: _t.Callable[[_T, _U, _V], _W], _x: _T, _y: _U, _z: _V, - *args: _t.Any, - **kwargs: _t.Any, ) -> _W: ... +# @_t.overload +# def call( +# _func: _t.Callable[_t.Concatenate[_T, _P], _U], +# _x: _T, +# *args: _t.Any, +# **kwargs: _t.Any, +# ) -> _U: ... +# @_t.overload +# def call( +# _func: _t.Callable[_t.Concatenate[_T, _U, _P], _V], +# _x: _T, +# _y: _U, +# *args: _t.Any, +# **kwargs: _t.Any, +# ) -> _V: ... +# @_t.overload +# def call( +# _func: _t.Callable[_t.Concatenate[_T, _U, _V, _P], _W], +# _x: _T, +# _y: _U, +# _z: _V, +# *args: _t.Any, +# **kwargs: _t.Any, +# ) -> _W: ... @_t.overload def call( _func: _t.Callable[..., _T], @@ -325,6 +328,10 @@ class Expected(_BaseExpected[_T]): def __bool__(self) -> bool: return self.error is None def __fmap__[U](self, func: T -> U) -> Expected[U]: + """Maps func over the result if it exists. + + __fmap__ should be used directly only when fmap is not available (e.g. when consuming an Expected in vanilla Python). + """ return self.__class__(func(self.result)) if self else self def and_then[U](self, func: T -> Expected[U]) -> Expected[U]: """Maps a T -> Expected[U] over an Expected[T] to produce an Expected[U]. @@ -340,20 +347,34 @@ class Expected(_BaseExpected[_T]): def map_error(self, func: BaseException -> BaseException) -> Expected[T]: """Maps func over the error if it exists.""" return self if self else self.__class__(error=func(self.error)) + def handle(self, err_type, handler: BaseException -> T) -> Expected[T]: + """Recover from the given err_type by calling handler on the error to determine the result.""" + if not self and isinstance(self.error, err_type): + return self.__class__(handler(self.error)) + return self + def expect_error(self, *err_types: BaseException) -> Expected[T]: + """Raise any errors that do not match the given error types.""" + if not self and not isinstance(self.error, err_types): + raise self.error + return self + def unwrap(self) -> T: + """Unwrap the result or raise the error.""" + if not self: + raise self.error + return self.result def or_else[U](self, func: BaseException -> Expected[U]) -> Expected[T | U]: """Return self if no error, otherwise return the result of evaluating func on the error.""" return self if self else func(self.error) - def result_or[U](self, default: U) -> T | U: - """Return the result if it exists, otherwise return the default.""" - return self.result if self else default def result_or_else[U](self, func: BaseException -> U) -> T | U: """Return the result if it exists, otherwise return the result of evaluating func on the error.""" return self.result if self else func(self.error) - def unwrap(self) -> T: - """Unwrap the result or raise the error.""" - if not self: - raise self.error - return self.result + def result_or[U](self, default: U) -> T | U: + """Return the result if it exists, otherwise return the default. + + Since .result_or() completely silences errors, it is highly recommended that you + call .expect_error() first to explicitly declare what errors you are okay silencing. + """ + return self.result if self else default ''' __slots__ = () _coconut_is_data = True @@ -404,17 +425,27 @@ class Expected(_BaseExpected[_T]): def map_error(self, func: _t.Callable[[BaseException], BaseException]) -> Expected[_T]: """Maps func over the error if it exists.""" ... + def handle(self, err_type: _t.Type[BaseException], handler: _t.Callable[[BaseException], _T]) -> Expected[_T]: + """Recover from the given err_type by calling handler on the error to determine the result.""" + ... + def expect_error(self, *err_types: BaseException) -> Expected[_T]: + """Raise any errors that do not match the given error types.""" + ... + def unwrap(self) -> _T: + """Unwrap the result or raise the error.""" + ... def or_else(self, func: _t.Callable[[BaseException], Expected[_U]]) -> Expected[_T | _U]: """Return self if no error, otherwise return the result of evaluating func on the error.""" ... - def result_or(self, default: _U) -> _T | _U: - """Return the result if it exists, otherwise return the default.""" - ... def result_or_else(self, func: _t.Callable[[BaseException], _U]) -> _T | _U: """Return the result if it exists, otherwise return the result of evaluating func on the error.""" ... - def unwrap(self) -> _T: - """Unwrap the result or raise the error.""" + def result_or(self, default: _U) -> _T | _U: + """Return the result if it exists, otherwise return the default. + + Since .result_or() completely silences errors, it is highly recommended that you + call .expect_error() first to explicitly declare what errors you are okay silencing. + """ ... _coconut_Expected = Expected @@ -423,46 +454,50 @@ _coconut_Expected = Expected # should match call above but with Expected @_t.overload def safe_call( - _func: _t.Callable[[_T], _U], - _x: _T, + _func: _t.Callable[[], _U], ) -> Expected[_U]: ... @_t.overload def safe_call( - _func: _t.Callable[[_T, _U], _V], - _x: _T, - _y: _U, -) -> Expected[_V]: ... -@_t.overload -def safe_call( - _func: _t.Callable[[_T, _U, _V], _W], - _x: _T, - _y: _U, - _z: _V, -) -> Expected[_W]: ... -@_t.overload -def safe_call( - _func: _t.Callable[_t.Concatenate[_T, _P], _U], + _func: _t.Callable[[_T], _U], _x: _T, - *args: _t.Any, - **kwargs: _t.Any, ) -> Expected[_U]: ... @_t.overload def safe_call( - _func: _t.Callable[_t.Concatenate[_T, _U, _P], _V], + _func: _t.Callable[[_T, _U], _V], _x: _T, _y: _U, - *args: _t.Any, - **kwargs: _t.Any, ) -> Expected[_V]: ... @_t.overload def safe_call( - _func: _t.Callable[_t.Concatenate[_T, _U, _V, _P], _W], + _func: _t.Callable[[_T, _U, _V], _W], _x: _T, _y: _U, _z: _V, - *args: _t.Any, - **kwargs: _t.Any, ) -> Expected[_W]: ... +# @_t.overload +# def safe_call( +# _func: _t.Callable[_t.Concatenate[_T, _P], _U], +# _x: _T, +# *args: _t.Any, +# **kwargs: _t.Any, +# ) -> Expected[_U]: ... +# @_t.overload +# def safe_call( +# _func: _t.Callable[_t.Concatenate[_T, _U, _P], _V], +# _x: _T, +# _y: _U, +# *args: _t.Any, +# **kwargs: _t.Any, +# ) -> Expected[_V]: ... +# @_t.overload +# def safe_call( +# _func: _t.Callable[_t.Concatenate[_T, _U, _V, _P], _W], +# _x: _T, +# _y: _U, +# _z: _V, +# *args: _t.Any, +# **kwargs: _t.Any, +# ) -> Expected[_W]: ... @_t.overload def safe_call( _func: _t.Callable[..., _T], @@ -482,46 +517,50 @@ def safe_call( ... -# based on call above +# based on call above@_t.overload @_t.overload def _coconut_call_or_coefficient( - _func: _t.Callable[[_T], _U], - _x: _T, + _func: _t.Callable[[], _U], ) -> _U: ... @_t.overload def _coconut_call_or_coefficient( - _func: _t.Callable[[_T, _U], _V], - _x: _T, - _y: _U, -) -> _V: ... -@_t.overload -def _coconut_call_or_coefficient( - _func: _t.Callable[[_T, _U, _V], _W], - _x: _T, - _y: _U, - _z: _V, -) -> _W: ... -@_t.overload -def _coconut_call_or_coefficient( - _func: _t.Callable[_t.Concatenate[_T, _P], _U], + _func: _t.Callable[[_T], _U], _x: _T, - *args: _t.Any, ) -> _U: ... @_t.overload def _coconut_call_or_coefficient( - _func: _t.Callable[_t.Concatenate[_T, _U, _P], _V], + _func: _t.Callable[[_T, _U], _V], _x: _T, _y: _U, - *args: _t.Any, ) -> _V: ... @_t.overload def _coconut_call_or_coefficient( - _func: _t.Callable[_t.Concatenate[_T, _U, _V, _P], _W], + _func: _t.Callable[[_T, _U, _V], _W], _x: _T, _y: _U, _z: _V, - *args: _t.Any, ) -> _W: ... +# @_t.overload +# def _coconut_call_or_coefficient( +# _func: _t.Callable[_t.Concatenate[_T, _P], _U], +# _x: _T, +# *args: _t.Any, +# ) -> _U: ... +# @_t.overload +# def _coconut_call_or_coefficient( +# _func: _t.Callable[_t.Concatenate[_T, _U, _P], _V], +# _x: _T, +# _y: _U, +# *args: _t.Any, +# ) -> _V: ... +# @_t.overload +# def _coconut_call_or_coefficient( +# _func: _t.Callable[_t.Concatenate[_T, _U, _V, _P], _W], +# _x: _T, +# _y: _U, +# _z: _V, +# *args: _t.Any, +# ) -> _W: ... @_t.overload def _coconut_call_or_coefficient( _func: _t.Callable[..., _T], @@ -534,9 +573,10 @@ def _coconut_call_or_coefficient( ) -> _T: ... -def recursive_iterator(func: _T_iter_func) -> _T_iter_func: +def recursive_generator(func: _T_iter_func) -> _T_iter_func: """Decorator that memoizes a recursive function that returns an iterator (e.g. a recursive generator).""" return func +recursive_iterator = recursive_generator # if sys.version_info >= (3, 12): @@ -590,7 +630,7 @@ def addpattern( *add_funcs: _Callable, allow_any_func: bool=False, ) -> _t.Callable[..., _t.Any]: - """Decorator to add a new case to a pattern-matching function (where the new case is checked last). + """Decorator to add new cases to a pattern-matching function (where the new case is checked last). Pass allow_any_func=True to allow any object as the base_func rather than just pattern-matching functions. If add_funcs are passed, addpattern(base_func, add_func) is equivalent to addpattern(base_func)(add_func). @@ -605,7 +645,8 @@ def _coconut_mark_as_match(func: _Tfunc) -> _Tfunc: return func -class _coconut_partial(_t.Generic[_T]): +class _coconut_complex_partial(_t.Generic[_T]): + func: _t.Callable[..., _T] = ... args: _Tuple = ... required_nargs: int = ... keywords: _t.Dict[_t.Text, _t.Any] = ... @@ -619,6 +660,7 @@ class _coconut_partial(_t.Generic[_T]): **kwargs: _t.Any, ) -> None: ... def __call__(self, *args: _t.Any, **kwargs: _t.Any) -> _T: ... + __name__: str | None = ... @_t.overload @@ -640,10 +682,16 @@ def _coconut_iter_getitem( ... +def _coconut_attritemgetter( + attr: _t.Optional[_t.Text], + *is_iter_and_items: _t.Tuple[_t.Tuple[bool, _t.Any], ...], +) -> _t.Callable[[_t.Any], _t.Any]: ... + + def _coconut_base_compose( func: _t.Callable[[_T], _t.Any], *func_infos: _t.Tuple[_Callable, int, bool], - ) -> _t.Callable[[_T], _t.Any]: ... +) -> _t.Callable[[_T], _t.Any]: ... def and_then( @@ -1185,6 +1233,22 @@ def reiterable(iterable: _t.Iterable[_T]) -> _t.Iterable[_T]: _coconut_reiterable = reiterable +@_t.overload +def async_map( + async_func: _t.Callable[[_T], _t.Awaitable[_U]], + iter: _t.Iterable[_T], + strict: bool = False, +) -> _t.Awaitable[_t.List[_U]]: ... +@_t.overload +def async_map( + async_func: _t.Callable[..., _t.Awaitable[_U]], + *iters: _t.Iterable, + strict: bool = False, +) -> _t.Awaitable[_t.List[_U]]: + """Map async_func over iters asynchronously using anyio.""" + ... + + def multi_enumerate(iterable: _Iterable) -> _t.Iterable[_t.Tuple[_t.Tuple[int, ...], _t.Any]]: """Enumerate an iterable of iterables. Works like enumerate, but indexes through inner iterables and produces a tuple index representing the index @@ -1268,6 +1332,7 @@ class groupsof(_t.Generic[_T]): cls, n: _SupportsIndex, iterable: _t.Iterable[_T], + fillvalue: _T = ..., ) -> groupsof[_T]: ... def __iter__(self) -> _t.Iterator[_t.Tuple[_T, ...]]: ... def __hash__(self) -> int: ... @@ -1287,8 +1352,8 @@ class windowsof(_t.Generic[_T]): cls, size: _SupportsIndex, iterable: _t.Iterable[_T], - fillvalue: _T=..., - step: _SupportsIndex=1, + fillvalue: _T = ..., + step: _SupportsIndex = 1, ) -> windowsof[_T]: ... def __iter__(self) -> _t.Iterator[_t.Tuple[_T, ...]]: ... def __hash__(self) -> int: ... @@ -1304,7 +1369,7 @@ class flatten(_t.Iterable[_T]): def __new__( cls, iterable: _t.Iterable[_t.Iterable[_T]], - levels: _t.Optional[_SupportsIndex]=1, + levels: _t.Optional[_SupportsIndex] = 1, ) -> flatten[_T]: ... def __iter__(self) -> _t.Iterator[_T]: ... @@ -1345,6 +1410,17 @@ def consume( ... +class multiset(_t.Generic[_T], _coconut.collections.Counter[_T]): + def add(self, item: _T) -> None: ... + def discard(self, item: _T) -> None: ... + def remove(self, item: _T) -> None: ... + def isdisjoint(self, other: _coconut.collections.Counter[_T]) -> bool: ... + def __xor__(self, other: _coconut.collections.Counter[_T]) -> multiset[_T]: ... + def count(self, item: _T) -> int: ... + def __fmap__(self, func: _t.Callable[[_T], _U]) -> multiset[_U]: ... +_coconut_multiset = multiset + + class _FMappable(_t.Protocol[_Tfunc_contra, _Tco]): def __fmap__(self, func: _Tfunc_contra) -> _Tco: ... @@ -1560,12 +1636,12 @@ def lift(func: _t.Callable[[_T, _U], _W]) -> _coconut_lifted_2[_T, _U, _W]: ... def lift(func: _t.Callable[[_T, _U, _V], _W]) -> _coconut_lifted_3[_T, _U, _V, _W]: ... @_t.overload def lift(func: _t.Callable[..., _W]) -> _t.Callable[..., _t.Callable[..., _W]]: - """Lifts a function up so that all of its arguments are functions. + """Lift a function up so that all of its arguments are functions. For a binary function f(x, y) and two unary functions g(z) and h(z), lift works as the S' combinator: lift(f)(g, h)(z) == f(g(z), h(z)) - In general, lift is requivalent to: + In general, lift is equivalent to: def lift(f) = ((*func_args, **func_kwargs) -> (*args, **kwargs) -> f(*(g(*args, **kwargs) for g in func_args), **{lbrace}k: h(*args, **kwargs) for k, h in func_kwargs.items(){rbrace})) @@ -1587,22 +1663,119 @@ def all_equal(iterable: _Iterable) -> bool: def collectby( key_func: _t.Callable[[_T], _U], iterable: _t.Iterable[_T], + *, + map_using: _t.Callable | None = None, ) -> _t.DefaultDict[_U, _t.List[_T]]: ... @_t.overload def collectby( key_func: _t.Callable[[_T], _U], iterable: _t.Iterable[_T], + *, reduce_func: _t.Callable[[_T, _T], _V], -) -> _t.DefaultDict[_U, _V]: + reduce_func_init: _T = ..., + map_using: _t.Callable | None = None, +) -> _t.Dict[_U, _V]: ... +@_t.overload +def collectby( + key_func: _t.Callable[[_T], _U], + iterable: _t.Iterable[_T], + value_func: _t.Callable[[_T], _W], + *, + map_using: _t.Callable | None = None, +) -> _t.DefaultDict[_U, _t.List[_W]]: ... +@_t.overload +def collectby( + key_func: _t.Callable[[_T], _U], + iterable: _t.Iterable[_T], + value_func: _t.Callable[[_T], _W], + *, + reduce_func: _t.Callable[[_W, _W], _V], + reduce_func_init: _W = ..., + map_using: _t.Callable | None = None, +) -> _t.Dict[_U, _V]: ... +@_t.overload +def collectby( + key_func: _t.Callable[[_T], _U], + iterable: _t.Iterable[_T], + *, + reduce_func: _t.Callable[[_T, _T], _V], + reduce_func_init: _T = ..., + map_using: _t.Callable | None = None, +) -> _t.Dict[_U, _V]: ... +@_t.overload +def collectby( + key_func: _t.Callable[[_U], _t.Any], + iterable: _t.Iterable[_U], + value_func: _t.Callable[[_U], _t.Any] | None = None, + *, + collect_in: _T, + reduce_func: _t.Callable | None | _t.Literal[False] = None, + reduce_func_init: _t.Any = ..., + map_using: _t.Callable | None = None, +) -> _T: """Collect the items in iterable into a dictionary of lists keyed by key_func(item). - if value_func is passed, collect value_func(item) into each list instead of item. + If value_func is passed, collect value_func(item) into each list instead of item. If reduce_func is passed, instead of collecting the items into lists, reduce over - the items of each key with reduce_func, effectively implementing a MapReduce operation. + the items for each key with reduce_func, effectively implementing a MapReduce operation. + + If map_using is passed, calculate key_func and value_func by mapping them over + the iterable using map_using as map. Useful with process_map/thread_map. + """ + ... + +collectby.using_processes = collectby.using_threads = collectby # type: ignore + + +@_t.overload +def mapreduce( + key_value_func: _t.Callable[[_T], _t.Tuple[_U, _W]], + iterable: _t.Iterable[_T], + *, + map_using: _t.Callable | None = None, +) -> _t.DefaultDict[_U, _t.List[_W]]: ... +@_t.overload +def mapreduce( + key_value_func: _t.Callable[[_T], _t.Tuple[_U, _W]], + iterable: _t.Iterable[_T], + *, + reduce_func: _t.Callable[[_W, _W], _V], + reduce_func_init: _W = ..., + map_using: _t.Callable | None = None, +) -> _t.Dict[_U, _V]: ... +@_t.overload +def mapreduce( + key_value_func: _t.Callable[[_T], _t.Tuple[_U, _W]], + iterable: _t.Iterable[_T], + *, + reduce_func: _t.Callable[[_X, _W], _V], + reduce_func_init: _X = ..., + map_using: _t.Callable | None = None, +) -> _t.Dict[_U, _V]: ... +@_t.overload +def mapreduce( + key_value_func: _t.Callable[[_U], _t.Tuple[_t.Any, _t.Any]], + iterable: _t.Iterable[_U], + *, + collect_in: _T, + reduce_func: _t.Callable | None | _t.Literal[False] = None, + reduce_func_init: _t.Any = ..., + map_using: _t.Callable | None = None, +) -> _T: + """Map key_value_func over iterable, then collect the values into a dictionary of lists keyed by the keys. + + If reduce_func is passed, instead of collecting the values into lists, reduce over + the values for each key with reduce_func, effectively implementing a MapReduce operation. + + If map_using is passed, calculate key_value_func by mapping them over + the iterable using map_using as map. Useful with process_map/thread_map. """ ... +mapreduce.using_processes = mapreduce.using_threads = mapreduce # type: ignore +_coconut_mapreduce = mapreduce + @_t.overload def _namedtuple_of(**kwargs: _t.Dict[_t.Text, _T]) -> _t.Tuple[_T, ...]: ... diff --git a/_coconut/__init__.pyi b/_coconut/__init__.pyi index c00dfdcb1..31d9fd411 100644 --- a/_coconut/__init__.pyi +++ b/_coconut/__init__.pyi @@ -29,6 +29,7 @@ import traceback as _traceback import weakref as _weakref import multiprocessing as _multiprocessing import pickle as _pickle +import inspect as _inspect from multiprocessing import dummy as _multiprocessing_dummy if sys.version_info >= (3,): @@ -86,6 +87,8 @@ contextlib = _contextlib traceback = _traceback weakref = _weakref multiprocessing = _multiprocessing +inspect = _inspect + multiprocessing_dummy = _multiprocessing_dummy copyreg = _copyreg diff --git a/coconut/__coconut__.pyi b/coconut/__coconut__.pyi index 45d413ea3..e56d0e55e 100644 --- a/coconut/__coconut__.pyi +++ b/coconut/__coconut__.pyi @@ -1,2 +1,2 @@ from __coconut__ import * -from __coconut__ import _coconut_tail_call, _coconut_tco, _coconut_call_set_names, _coconut_handle_cls_kwargs, _coconut_handle_cls_stargs, _namedtuple_of, _coconut, _coconut_Expected, _coconut_MatchError, _coconut_SupportsAdd, _coconut_SupportsMinus, _coconut_SupportsMul, _coconut_SupportsPow, _coconut_SupportsTruediv, _coconut_SupportsFloordiv, _coconut_SupportsMod, _coconut_SupportsAnd, _coconut_SupportsXor, _coconut_SupportsOr, _coconut_SupportsLshift, _coconut_SupportsRshift, _coconut_SupportsMatmul, _coconut_SupportsInv, _coconut_Expected, _coconut_MatchError, _coconut_iter_getitem, _coconut_base_compose, _coconut_forward_compose, _coconut_back_compose, _coconut_forward_star_compose, _coconut_back_star_compose, _coconut_forward_dubstar_compose, _coconut_back_dubstar_compose, _coconut_pipe, _coconut_star_pipe, _coconut_dubstar_pipe, _coconut_back_pipe, _coconut_back_star_pipe, _coconut_back_dubstar_pipe, _coconut_none_pipe, _coconut_none_star_pipe, _coconut_none_dubstar_pipe, _coconut_bool_and, _coconut_bool_or, _coconut_none_coalesce, _coconut_minus, _coconut_map, _coconut_partial, _coconut_get_function_match_error, _coconut_base_pattern_func, _coconut_addpattern, _coconut_sentinel, _coconut_assert, _coconut_raise, _coconut_mark_as_match, _coconut_reiterable, _coconut_self_match_types, _coconut_dict_merge, _coconut_exec, _coconut_comma_op, _coconut_multi_dim_arr, _coconut_mk_anon_namedtuple, _coconut_matmul, _coconut_py_str, _coconut_flatten, _coconut_multiset, _coconut_back_none_pipe, _coconut_back_none_star_pipe, _coconut_back_none_dubstar_pipe, _coconut_forward_none_compose, _coconut_back_none_compose, _coconut_forward_none_star_compose, _coconut_back_none_star_compose, _coconut_forward_none_dubstar_compose, _coconut_back_none_dubstar_compose, _coconut_call_or_coefficient, _coconut_in, _coconut_not_in +from __coconut__ import _coconut_tail_call, _coconut_tco, _coconut_call_set_names, _coconut_handle_cls_kwargs, _coconut_handle_cls_stargs, _namedtuple_of, _coconut, _coconut_Expected, _coconut_MatchError, _coconut_SupportsAdd, _coconut_SupportsMinus, _coconut_SupportsMul, _coconut_SupportsPow, _coconut_SupportsTruediv, _coconut_SupportsFloordiv, _coconut_SupportsMod, _coconut_SupportsAnd, _coconut_SupportsXor, _coconut_SupportsOr, _coconut_SupportsLshift, _coconut_SupportsRshift, _coconut_SupportsMatmul, _coconut_SupportsInv, _coconut_Expected, _coconut_MatchError, _coconut_iter_getitem, _coconut_base_compose, _coconut_forward_compose, _coconut_back_compose, _coconut_forward_star_compose, _coconut_back_star_compose, _coconut_forward_dubstar_compose, _coconut_back_dubstar_compose, _coconut_pipe, _coconut_star_pipe, _coconut_dubstar_pipe, _coconut_back_pipe, _coconut_back_star_pipe, _coconut_back_dubstar_pipe, _coconut_none_pipe, _coconut_none_star_pipe, _coconut_none_dubstar_pipe, _coconut_bool_and, _coconut_bool_or, _coconut_none_coalesce, _coconut_minus, _coconut_map, _coconut_partial, _coconut_complex_partial, _coconut_get_function_match_error, _coconut_base_pattern_func, _coconut_addpattern, _coconut_sentinel, _coconut_assert, _coconut_raise, _coconut_mark_as_match, _coconut_reiterable, _coconut_self_match_types, _coconut_dict_merge, _coconut_exec, _coconut_comma_op, _coconut_multi_dim_arr, _coconut_mk_anon_namedtuple, _coconut_matmul, _coconut_py_str, _coconut_flatten, _coconut_multiset, _coconut_back_none_pipe, _coconut_back_none_star_pipe, _coconut_back_none_dubstar_pipe, _coconut_forward_none_compose, _coconut_back_none_compose, _coconut_forward_none_star_compose, _coconut_back_none_star_compose, _coconut_forward_none_dubstar_compose, _coconut_back_none_dubstar_compose, _coconut_call_or_coefficient, _coconut_in, _coconut_not_in, _coconut_attritemgetter diff --git a/coconut/_pyparsing.py b/coconut/_pyparsing.py index 936b8b6a7..c973208b5 100644 --- a/coconut/_pyparsing.py +++ b/coconut/_pyparsing.py @@ -20,15 +20,16 @@ from coconut.root import * # NOQA import os +import re import sys import traceback -import functools -import inspect from warnings import warn from collections import defaultdict +from itertools import permutations +from functools import wraps +from pprint import pprint from coconut.constants import ( - PYPY, PURE_PYTHON, use_fast_pyparsing_reprs, use_packrat_parser, @@ -36,15 +37,18 @@ default_whitespace_chars, varchars, min_versions, + max_versions, pure_python_env_var, enable_pyparsing_warnings, use_left_recursion_if_available, get_bool_env_var, use_computation_graph_env_var, use_incremental_if_available, - incremental_cache_size, + default_incremental_cache_size, never_clear_incremental_cache, warn_on_multiline_regex, + num_displayed_timing_items, + use_cache_file, ) from coconut.util import get_clock_time # NOQA from coconut.util import ( @@ -63,7 +67,7 @@ from cPyparsing import * # NOQA from cPyparsing import __version__ - PYPARSING_PACKAGE = "cPyparsing" + CPYPARSING = True PYPARSING_INFO = "Cython cPyparsing v" + __version__ except ImportError: @@ -73,13 +77,13 @@ from pyparsing import * # NOQA from pyparsing import __version__ - PYPARSING_PACKAGE = "pyparsing" + CPYPARSING = False PYPARSING_INFO = "Python pyparsing v" + __version__ except ImportError: traceback.print_exc() __version__ = None - PYPARSING_PACKAGE = "cPyparsing" + CPYPARSING = True PYPARSING_INFO = None @@ -87,32 +91,53 @@ # VERSIONING: # ----------------------------------------------------------------------------------------------------------------------- -min_ver = min(min_versions["pyparsing"], min_versions["cPyparsing"][:3]) # inclusive -max_ver = get_next_version(max(min_versions["pyparsing"], min_versions["cPyparsing"][:3])) # exclusive -cur_ver = None if __version__ is None else ver_str_to_tuple(__version__) +PYPARSING_PACKAGE = "cPyparsing" if CPYPARSING else "pyparsing" + +if CPYPARSING: + min_ver = min_versions["cPyparsing"] # inclusive + max_ver = get_next_version( + min_versions["cPyparsing"], + point_to_increment=len(max_versions["cPyparsing"]) - 1, + ) # exclusive +else: + min_ver = min_versions["pyparsing"] # inclusive + max_ver = get_next_version(min_versions["pyparsing"]) # exclusive -min_ver_str = ver_tuple_to_str(min_ver) -max_ver_str = ver_tuple_to_str(max_ver) +cur_ver = None if __version__ is None else ver_str_to_tuple(__version__) if cur_ver is None or cur_ver < min_ver: raise ImportError( - "This version of Coconut requires pyparsing/cPyparsing version >= " + min_ver_str - + ("; got " + PYPARSING_INFO if PYPARSING_INFO is not None else "") - + " (run '{python} -m pip install --upgrade {package}' to fix)".format(python=sys.executable, package=PYPARSING_PACKAGE), + ( + "This version of Coconut requires {package} version >= {min_ver}" + + ("; got " + PYPARSING_INFO if PYPARSING_INFO is not None else "") + + " (run '{python} -m pip install --upgrade {package}' to fix)" + ).format( + python=sys.executable, + package=PYPARSING_PACKAGE, + min_ver=ver_tuple_to_str(min_ver), + ) ) elif cur_ver >= max_ver: warn( - "This version of Coconut was built for pyparsing/cPyparsing versions < " + max_ver_str - + ("; got " + PYPARSING_INFO if PYPARSING_INFO is not None else "") - + " (run '{python} -m pip install {package}<{max_ver}' to fix)".format(python=sys.executable, package=PYPARSING_PACKAGE, max_ver=max_ver_str), + ( + "This version of Coconut was built for {package} versions < {max_ver}" + + ("; got " + PYPARSING_INFO if PYPARSING_INFO is not None else "") + + " (run '{python} -m pip install {package}<{max_ver}' to fix)" + ).format( + python=sys.executable, + package=PYPARSING_PACKAGE, + max_ver=ver_tuple_to_str(max_ver), + ) ) MODERN_PYPARSING = cur_ver >= (3,) if MODERN_PYPARSING: warn( - "This version of Coconut is not built for pyparsing v3; some syntax features WILL NOT WORK" - + " (run either '{python} -m pip install cPyparsing<{max_ver}' or '{python} -m pip install pyparsing<{max_ver}' to fix)".format(python=sys.executable, max_ver=max_ver_str), + "This version of Coconut is not built for pyparsing v3; some syntax features WILL NOT WORK (run either '{python} -m pip install cPyparsing<{max_ver}' or '{python} -m pip install pyparsing<{max_ver}' to fix)".format( + python=sys.executable, + max_ver=ver_tuple_to_str(max_ver), + ) ) @@ -120,41 +145,79 @@ # OVERRIDES: # ----------------------------------------------------------------------------------------------------------------------- -if PYPARSING_PACKAGE != "cPyparsing": - if not MODERN_PYPARSING: - HIT, MISS = 0, 1 - - def _parseCache(self, instring, loc, doActions=True, callPreParse=True): - # [CPYPARSING] include packrat_context - lookup = (self, instring, loc, callPreParse, doActions, tuple(self.packrat_context)) - with ParserElement.packrat_cache_lock: - cache = ParserElement.packrat_cache - value = cache.get(lookup) - if value is cache.not_in_cache: - ParserElement.packrat_cache_stats[MISS] += 1 - try: - value = self._parseNoCache(instring, loc, doActions, callPreParse) - except ParseBaseException as pe: - # cache a copy of the exception, without the traceback - cache.set(lookup, pe.__class__(*pe.args)) - raise - else: - cache.set(lookup, (value[0], value[1].copy())) - return value - else: - ParserElement.packrat_cache_stats[HIT] += 1 - if isinstance(value, Exception): - raise value - return value[0], value[1].copy() - ParserElement.packrat_context = [] - ParserElement._parseCache = _parseCache - -elif not hasattr(ParserElement, "packrat_context"): - raise ImportError( +if MODERN_PYPARSING: + SUPPORTS_PACKRAT_CONTEXT = False + +elif CPYPARSING: + assert hasattr(ParserElement, "packrat_context"), ( "This version of Coconut requires cPyparsing>=" + ver_tuple_to_str(min_versions["cPyparsing"]) + "; got cPyparsing==" + __version__ + " (run '{python} -m pip install --upgrade cPyparsing' to fix)".format(python=sys.executable), ) + SUPPORTS_PACKRAT_CONTEXT = True + +else: + SUPPORTS_PACKRAT_CONTEXT = True + HIT, MISS = 0, 1 + + def _parseCache(self, instring, loc, doActions=True, callPreParse=True): + # [CPYPARSING] HIT, MISS are constants + # [CPYPARSING] include packrat_context, merge callPreParse and doActions + lookup = (self, instring, loc, callPreParse | doActions << 1, ParserElement.packrat_context) + with ParserElement.packrat_cache_lock: + cache = ParserElement.packrat_cache + value = cache.get(lookup) + if value is cache.not_in_cache: + ParserElement.packrat_cache_stats[MISS] += 1 + try: + value = self._parseNoCache(instring, loc, doActions, callPreParse) + except ParseBaseException as pe: + # cache a copy of the exception, without the traceback + cache.set(lookup, pe.__class__(*pe.args)) + raise + else: + cache.set(lookup, (value[0], value[1].copy())) + return value + else: + ParserElement.packrat_cache_stats[HIT] += 1 + if isinstance(value, Exception): + raise value + return value[0], value[1].copy() + + ParserElement._parseCache = _parseCache + + # [CPYPARSING] fix append + def append(self, other): + if (self.parseAction + or self.resultsName is not None + or self.debug): + return self.__class__([self, other]) + elif (other.__class__ == self.__class__ + and not other.parseAction + and other.resultsName is None + and not other.debug): + self.exprs += other.exprs + self.strRepr = None + self.saveAsList |= other.saveAsList + if isinstance(self, And): + self.mayReturnEmpty &= other.mayReturnEmpty + else: + self.mayReturnEmpty |= other.mayReturnEmpty + self.mayIndexError |= other.mayIndexError + else: + self.exprs.append(other) + self.strRepr = None + if isinstance(self, And): + self.mayReturnEmpty &= other.mayReturnEmpty + else: + self.mayReturnEmpty |= other.mayReturnEmpty + self.mayIndexError |= other.mayIndexError + self.saveAsList |= other.saveAsList + return self + ParseExpression.append = append + +if SUPPORTS_PACKRAT_CONTEXT: + ParserElement.packrat_context = frozenset() if hasattr(ParserElement, "enableIncremental"): SUPPORTS_INCREMENTAL = sys.version_info >= (3, 8) # avoids stack overflows on py<=37 @@ -176,6 +239,22 @@ def enableIncremental(*args, **kwargs): # SETUP: # ----------------------------------------------------------------------------------------------------------------------- +USE_COMPUTATION_GRAPH = get_bool_env_var( + use_computation_graph_env_var, + default=( + not MODERN_PYPARSING # not yet supported + # commented out to minimize memory footprint when running tests: + # and not PYPY # experimentally determined + ), +) + +SUPPORTS_ADAPTIVE = ( + hasattr(MatchFirst, "setAdaptiveMode") + and USE_COMPUTATION_GRAPH +) + +USE_CACHE = SUPPORTS_INCREMENTAL and use_cache_file + if MODERN_PYPARSING: _trim_arity = _pyparsing.core._trim_arity _ParseResultsWithOffset = _pyparsing.core._ParseResultsWithOffset @@ -183,13 +262,7 @@ def enableIncremental(*args, **kwargs): _trim_arity = _pyparsing._trim_arity _ParseResultsWithOffset = _pyparsing._ParseResultsWithOffset -USE_COMPUTATION_GRAPH = get_bool_env_var( - use_computation_graph_env_var, - default=( - not MODERN_PYPARSING # not yet supported - and not PYPY # experimentally determined - ), -) +maybe_make_safe = getattr(_pyparsing, "maybe_make_safe", None) if enable_pyparsing_warnings: if MODERN_PYPARSING: @@ -202,7 +275,7 @@ def enableIncremental(*args, **kwargs): if MODERN_PYPARSING and use_left_recursion_if_available: ParserElement.enable_left_recursion() elif SUPPORTS_INCREMENTAL and use_incremental_if_available: - ParserElement.enableIncremental(incremental_cache_size, still_reset_cache=not never_clear_incremental_cache) + ParserElement.enableIncremental(default_incremental_cache_size, still_reset_cache=not never_clear_incremental_cache) elif use_packrat_parser: ParserElement.enablePackrat(packrat_cache_size) @@ -210,18 +283,24 @@ def enableIncremental(*args, **kwargs): Keyword.setDefaultKeywordChars(varchars) +if SUPPORTS_INCREMENTAL: + all_parse_elements = ParserElement.collectParseElements() +else: + all_parse_elements = None + # ----------------------------------------------------------------------------------------------------------------------- # MISSING OBJECTS: # ----------------------------------------------------------------------------------------------------------------------- -if not hasattr(_pyparsing, "python_quoted_string"): - import re as _re +python_quoted_string = getattr(_pyparsing, "python_quoted_string", None) +if python_quoted_string is None: python_quoted_string = _pyparsing.Combine( - (_pyparsing.Regex(r'"""(?:[^"\\]|""(?!")|"(?!"")|\\.)*', flags=_re.MULTILINE) + '"""').setName("multiline double quoted string") - ^ (_pyparsing.Regex(r"'''(?:[^'\\]|''(?!')|'(?!'')|\\.)*", flags=_re.MULTILINE) + "'''").setName("multiline single quoted string") - ^ (_pyparsing.Regex(r'"(?:[^"\n\r\\]|(?:\\")|(?:\\(?:[^x]|x[0-9a-fA-F]+)))*') + '"').setName("double quoted string") - ^ (_pyparsing.Regex(r"'(?:[^'\n\r\\]|(?:\\')|(?:\\(?:[^x]|x[0-9a-fA-F]+)))*") + "'").setName("single quoted string") + # multiline strings must come first + (_pyparsing.Regex(r'"""(?:[^"\\]|""(?!")|"(?!"")|\\.)*', flags=re.MULTILINE) + '"""').setName("multiline double quoted string") + | (_pyparsing.Regex(r"'''(?:[^'\\]|''(?!')|'(?!'')|\\.)*", flags=re.MULTILINE) + "'''").setName("multiline single quoted string") + | (_pyparsing.Regex(r'"(?:[^"\n\r\\]|(?:\\")|(?:\\(?:[^x]|x[0-9a-fA-F]+)))*') + '"').setName("double quoted string") + | (_pyparsing.Regex(r"'(?:[^'\n\r\\]|(?:\\')|(?:\\(?:[^x]|x[0-9a-fA-F]+)))*") + "'").setName("single quoted string") ).setName("Python quoted string") _pyparsing.python_quoted_string = python_quoted_string @@ -230,10 +309,14 @@ def enableIncremental(*args, **kwargs): # FAST REPRS: # ----------------------------------------------------------------------------------------------------------------------- -if PY2: - def fast_repr(cls): +if DEVELOP: + def fast_repr(self): """A very simple, fast __repr__/__str__ implementation.""" - return "<" + cls.__name__ + ">" + return getattr(self, "name", self.__class__.__name__) +elif PY2: + def fast_repr(self): + """A very simple, fast __repr__/__str__ implementation.""" + return "<" + self.__class__.__name__ + ">" else: fast_repr = object.__repr__ @@ -247,8 +330,8 @@ def set_fast_pyparsing_reprs(): try: if issubclass(obj, ParserElement): _old_pyparsing_reprs.append((obj, (obj.__repr__, obj.__str__))) - obj.__repr__ = functools.partial(fast_repr, obj) - obj.__str__ = functools.partial(fast_repr, obj) + obj.__repr__ = fast_repr + obj.__str__ = fast_repr except TypeError: pass @@ -258,6 +341,7 @@ def unset_fast_pyparsing_reprs(): for obj, (repr_method, str_method) in _old_pyparsing_reprs: obj.__repr__ = repr_method obj.__str__ = str_method + _old_pyparsing_reprs[:] = [] if use_fast_pyparsing_reprs: @@ -278,66 +362,15 @@ class _timing_sentinel(object): def add_timing_to_method(cls, method_name, method): """Add timing collection to the given method. It's a monstrosity, but it's only used for profiling.""" - from coconut.terminal import internal_assert # hide to avoid circular import - - args, varargs, keywords, defaults = inspect.getargspec(method) - internal_assert(args[:1] == ["self"], "cannot add timing to method", method_name) - - if not defaults: - defaults = [] - num_undefaulted_args = len(args) - len(defaults) - def_args = [] - call_args = [] - fix_arg_defaults = [] - defaults_dict = {} - for i, arg in enumerate(args): - if i >= num_undefaulted_args: - default = defaults[i - num_undefaulted_args] - def_args.append(arg + "=_timing_sentinel") - defaults_dict[arg] = default - fix_arg_defaults.append( - """ - if {arg} is _timing_sentinel: - {arg} = _exec_dict["defaults_dict"]["{arg}"] -""".strip("\n").format( - arg=arg, - ), - ) - else: - def_args.append(arg) - call_args.append(arg) - if varargs: - def_args.append("*" + varargs) - call_args.append("*" + varargs) - if keywords: - def_args.append("**" + keywords) - call_args.append("**" + keywords) - - new_method_name = "new_" + method_name + "_func" - _exec_dict = globals().copy() - _exec_dict.update(locals()) - new_method_code = """ -def {new_method_name}({def_args}): -{fix_arg_defaults} - - _all_args = (lambda *args, **kwargs: args + tuple(kwargs.values()))({call_args}) - _exec_dict["internal_assert"](not any(_arg is _timing_sentinel for _arg in _all_args), "error handling arguments in timed method {new_method_name}({def_args}); got", _all_args) - - _start_time = _exec_dict["get_clock_time"]() - try: - return _exec_dict["method"]({call_args}) - finally: - _timing_info[0][str(self)] += _exec_dict["get_clock_time"]() - _start_time -{new_method_name}._timed = True - """.format( - fix_arg_defaults="\n".join(fix_arg_defaults), - new_method_name=new_method_name, - def_args=", ".join(def_args), - call_args=", ".join(call_args), - ) - exec(new_method_code, _exec_dict) - - setattr(cls, method_name, _exec_dict[new_method_name]) + @wraps(method) + def new_method(self, *args, **kwargs): + start_time = get_clock_time() + try: + return method(self, *args, **kwargs) + finally: + _timing_info[0][ascii(self)] += get_clock_time() - start_time + new_method._timed = True + setattr(cls, method_name, new_method) return True @@ -384,6 +417,7 @@ def collect_timing_info(): added_timing |= add_timing_to_method(obj, attr_name, attr) if added_timing: logger.log("\tadded timing to", obj) + return _timing_info def print_timing_info(): @@ -398,6 +432,131 @@ def print_timing_info(): num=len(_timing_info[0]), ), ) - sorted_timing_info = sorted(_timing_info[0].items(), key=lambda kv: kv[1]) + sorted_timing_info = sorted(_timing_info[0].items(), key=lambda kv: kv[1])[-num_displayed_timing_items:] for method_name, total_time in sorted_timing_info: print("{method_name}:\t{total_time}".format(method_name=method_name, total_time=total_time)) + + +_profiled_MatchFirst_objs = {} + + +def add_profiling_to_MatchFirsts(): + """Add profiling to MatchFirst objects to look for possible reorderings.""" + + @wraps(MatchFirst.parseImpl) + def new_parseImpl(self, instring, loc, doActions=True): + if id(self) not in _profiled_MatchFirst_objs: + _profiled_MatchFirst_objs[id(self)] = self + self.expr_usage_stats = [] + self.expr_timing_stats = [] + while len(self.expr_usage_stats) < len(self.exprs): + self.expr_usage_stats.append(0) + self.expr_timing_stats.append([]) + maxExcLoc = -1 + maxException = None + for i, e in enumerate(self.exprs): + try: + start_time = get_clock_time() + try: + ret = e._parse(instring, loc, doActions) + finally: + self.expr_timing_stats[i].append(get_clock_time() - start_time) + self.expr_usage_stats[i] += 1 + return ret + except _pyparsing.ParseException as err: + if err.loc > maxExcLoc: + maxException = err + maxExcLoc = err.loc + except IndexError: + if len(instring) > maxExcLoc: + maxException = _pyparsing.ParseException(instring, len(instring), e.errmsg, self) + maxExcLoc = len(instring) + else: + if maxException is not None: + maxException.msg = self.errmsg + raise maxException + else: + raise _pyparsing.ParseException(instring, loc, "no defined alternatives to match", self) + + _pyparsing.MatchFirst.parseImpl = new_parseImpl + return _profiled_MatchFirst_objs + + +def time_for_ordering(expr_usage_stats, expr_timing_aves): + """Get the total time for a given MatchFirst ordering.""" + total_time = 0 + for i, n in enumerate(expr_usage_stats): + total_time += n * sum(expr_timing_aves[:i + 1]) + return total_time + + +def find_best_ordering(obj, num_perms_to_eval=None): + """Get the best ordering of the MatchFirst.""" + if num_perms_to_eval is None: + num_perms_to_eval = True if len(obj.exprs) <= 10 else 100000 + best_ordering = None + best_time = float("inf") + stats_zip = tuple(zip(obj.expr_usage_stats, obj.expr_timing_aves, obj.exprs)) + if num_perms_to_eval is True: + perms_to_eval = permutations(stats_zip) + else: + perms_to_eval = [ + stats_zip, + sorted(stats_zip, key=lambda u_t_e: (-u_t_e[0], u_t_e[1])), + sorted(stats_zip, key=lambda u_t_e: (u_t_e[1], -u_t_e[0])), + ] + if num_perms_to_eval: + max_usage = max(obj.expr_usage_stats) + max_time = max(obj.expr_timing_aves) + for i in range(1, num_perms_to_eval): + a = i / num_perms_to_eval + perms_to_eval.append(sorted( + stats_zip, + key=lambda u_t_e: + -a * u_t_e[0] / max_usage + + (1 - a) * u_t_e[1] / max_time, + )) + for perm in perms_to_eval: + perm_expr_usage_stats, perm_expr_timing_aves = zip(*[(usage, timing) for usage, timing, expr in perm]) + perm_time = time_for_ordering(perm_expr_usage_stats, perm_expr_timing_aves) + if perm_time < best_time: + best_time = perm_time + best_ordering = [(obj.exprs.index(expr), parse_expr_repr(expr)) for usage, timing, expr in perm] + return best_ordering, best_time + + +def naive_timing_improvement(obj): + """Get the expected timing improvement for a better MatchFirst ordering.""" + _, best_time = find_best_ordering(obj, num_perms_to_eval=False) + return time_for_ordering(obj.expr_usage_stats, obj.expr_timing_aves) - best_time + + +def parse_expr_repr(obj): + """Get a clean repr of a parse expression for displaying.""" + return ascii(getattr(obj, "name", None) or obj) + + +def print_poorly_ordered_MatchFirsts(): + """Print poorly ordered MatchFirsts.""" + for obj in _profiled_MatchFirst_objs.values(): + obj.expr_timing_aves = [sum(ts) / len(ts) if ts else 0 for ts in obj.expr_timing_stats] + obj.naive_timing_improvement = naive_timing_improvement(obj) + most_improveable = sorted(_profiled_MatchFirst_objs.values(), key=lambda obj: obj.naive_timing_improvement)[-num_displayed_timing_items:] + for obj in most_improveable: + print("\n" + parse_expr_repr(obj) + " (" + str(obj.naive_timing_improvement) + "):") + pprint(list(zip(map(parse_expr_repr, obj.exprs), obj.expr_usage_stats, obj.expr_timing_aves))) + best_ordering, best_time = find_best_ordering(obj) + print("\tbest (" + str(best_time) + "):") + pprint(best_ordering) + + +def start_profiling(): + """Do all the setup to begin profiling.""" + add_profiling_to_MatchFirsts() + collect_timing_info() + + +def print_profiling_results(): + """Print all profiling results.""" + print_timing_info() + print_poorly_ordered_MatchFirsts() diff --git a/coconut/api.py b/coconut/api.py index c8a8bb995..82f2f0bc5 100644 --- a/coconut/api.py +++ b/coconut/api.py @@ -23,15 +23,17 @@ import os.path import codecs from functools import partial +from setuptools import PackageFinder try: from encodings import utf_8 except ImportError: utf_8 = None from coconut.root import _coconut_exec +from coconut.util import override from coconut.integrations import embed from coconut.exceptions import CoconutException -from coconut.command import Command +from coconut.command.command import Command from coconut.command.cli import cli_version from coconut.command.util import proc_run_args from coconut.compiler import Compiler @@ -42,7 +44,6 @@ coconut_kernel_kwargs, default_use_cache_dir, coconut_cache_dir, - coconut_run_kwargs, ) # ----------------------------------------------------------------------------------------------------------------------- @@ -68,9 +69,16 @@ def get_state(state=None): def cmd(cmd_args, **kwargs): """Process command-line arguments.""" state = kwargs.pop("state", False) + cmd_func = kwargs.pop("_cmd_func", "cmd") if isinstance(cmd_args, (str, bytes)): cmd_args = cmd_args.split() - return get_state(state).cmd(cmd_args, **kwargs) + return getattr(get_state(state), cmd_func)(cmd_args, **kwargs) + + +def cmd_sys(*args, **kwargs): + """Same as api.cmd() but defaults to --target sys.""" + kwargs["_cmd_func"] = "cmd_sys" + return cmd(*args, **kwargs) VERSIONS = { @@ -214,7 +222,7 @@ def cmd(self, *args): """Run the Coconut compiler with the given args.""" if self.command is None: self.command = Command() - return self.command.cmd(list(args) + self.args, interact=False, **coconut_run_kwargs) + return self.command.cmd_sys(list(args) + self.args, interact=False) def compile(self, path, package): """Compile a path to a file or package.""" @@ -315,6 +323,7 @@ def compile_coconut(cls, source): cls.coconut_compiler = Compiler(**coconut_kernel_kwargs) return cls.coconut_compiler.parse_sys(source) + @override @classmethod def decode(cls, input_bytes, errors="strict"): """Decode and compile the given Coconut source bytes.""" @@ -347,3 +356,39 @@ def get_coconut_encoding(encoding="coconut"): codecs.register(get_coconut_encoding) + + +# ----------------------------------------------------------------------------------------------------------------------- +# SETUPTOOLS: +# ----------------------------------------------------------------------------------------------------------------------- + +class CoconutPackageFinder(PackageFinder, object): + _coconut_compile = None + + @override + @classmethod + def _looks_like_package(cls, path, _package_name=None): + is_coconut_package = any( + os.path.isfile(os.path.join(path, "__init__" + ext)) + for ext in code_exts + ) + if is_coconut_package and cls._coconut_compile is not None: + cls._coconut_compile(path) + return is_coconut_package + + +find_packages = CoconutPackageFinder.find + + +class CoconutPackageCompiler(CoconutPackageFinder): + _coconut_command = None + + @classmethod + def _coconut_compile(cls, path): + """Run the Coconut compiler with the given args.""" + if cls._coconut_command is None: + cls._coconut_command = Command() + return cls._coconut_command.cmd_sys([path], interact=False) + + +find_and_compile_packages = CoconutPackageCompiler.find diff --git a/coconut/api.pyi b/coconut/api.pyi index 97f6fbf80..850b2eb89 100644 --- a/coconut/api.pyi +++ b/coconut/api.pyi @@ -21,6 +21,8 @@ from typing import ( Text, ) +from setuptools import find_packages as _find_packages + from coconut.command.command import Command class CoconutException(Exception): @@ -46,10 +48,13 @@ def cmd( argv: Iterable[Text] | None = None, interact: bool = False, default_target: Text | None = None, + default_jobs: Text | None = None, ) -> None: """Process command-line arguments.""" ... +cmd_sys = cmd + VERSIONS: Dict[Text, Text] = ... @@ -80,7 +85,7 @@ def setup( def warm_up( - force: bool = False, + streamline: bool = False, enable_incremental_mode: bool = False, *, state: Optional[Command] = ..., @@ -150,3 +155,6 @@ def auto_compilation( def get_coconut_encoding(encoding: Text = ...) -> Any: """Get a CodecInfo for the given Coconut encoding.""" ... + + +find_and_compile_packages = find_packages = _find_packages diff --git a/coconut/command/cli.py b/coconut/command/cli.py index 5087e52d0..5ea28e199 100644 --- a/coconut/command/cli.py +++ b/coconut/command/cli.py @@ -31,10 +31,8 @@ default_style, vi_mode_env_var, prompt_vi_mode, - prompt_histfile, - home_env_var, py_version_str, - default_jobs, + base_default_jobs, ) # ----------------------------------------------------------------------------------------------------------------------- @@ -189,7 +187,7 @@ "-j", "--jobs", metavar="processes", type=str, - help="number of additional processes to use (defaults to " + ascii(default_jobs) + ") (0 is no additional processes; 'sys' uses machine default)", + help="number of additional processes to use (defaults to " + ascii(base_default_jobs) + ") (0 is no additional processes; 'sys' uses machine default)", ) arguments.add_argument( @@ -245,13 +243,6 @@ + style_env_var + " environment variable if it exists, otherwise '" + default_style + "')", ) -arguments.add_argument( - "--history-file", - metavar="path", - type=str, - help="set history file (or '' for no file) (currently set to " + ascii(prompt_histfile) + ") (can be modified by setting " + home_env_var + " environment variable)", -) - arguments.add_argument( "--vi-mode", "--vimode", action="store_true", @@ -272,6 +263,18 @@ help="run the compiler in a separate thread with the given stack size in kilobytes", ) +arguments.add_argument( + "--fail-fast", + action="store_true", + help="causes the compiler to fail immediately upon encountering a compilation error rather than attempting to continue compiling other files", +) + +arguments.add_argument( + "--no-cache", + action="store_true", + help="disables use of Coconut's incremental parsing cache (caches previous parses to improve recompilation performance for slightly modified files)", +) + arguments.add_argument( "--site-install", "--siteinstall", action="store_true", diff --git a/coconut/command/command.py b/coconut/command/command.py index fee072a41..95e21d0da 100644 --- a/coconut/command/command.py +++ b/coconut/command/command.py @@ -28,9 +28,10 @@ from subprocess import CalledProcessError from coconut._pyparsing import ( + USE_CACHE, unset_fast_pyparsing_reprs, - collect_timing_info, - print_timing_info, + start_profiling, + print_profiling_results, ) from coconut.compiler import Compiler @@ -44,7 +45,6 @@ internal_assert, ) from coconut.constants import ( - PY32, PY35, fixpath, code_exts, @@ -67,11 +67,11 @@ coconut_pth_file, error_color_code, jupyter_console_commands, - default_jobs, + base_default_jobs, create_package_retries, default_use_cache_dir, coconut_cache_dir, - coconut_run_kwargs, + coconut_sys_kwargs, interpreter_uses_incremental, ) from coconut.util import ( @@ -79,6 +79,7 @@ ver_tuple_to_str, install_custom_kernel, get_clock_time, + ensure_dir, first_import_time, ) from coconut.command.util import ( @@ -102,13 +103,13 @@ invert_mypy_arg, run_with_stack_size, proc_run_args, + get_python_lib, ) from coconut.compiler.util import ( should_indent, get_target_info_smart, ) from coconut.compiler.header import gethash -from coconut.compiler.grammar import set_grammar_names from coconut.command.cli import arguments, cli_version # ----------------------------------------------------------------------------------------------------------------------- @@ -129,15 +130,10 @@ class Command(object): mypy_args = None # corresponds to --mypy flag argv_args = None # corresponds to --argv flag stack_size = 0 # corresponds to --stack-size flag + use_cache = USE_CACHE # corresponds to --no-cache flag + fail_fast = False # corresponds to --fail-fast flag - _prompt = None - - @property - def prompt(self): - """Delay creation of a Prompt() until it's needed.""" - if self._prompt is None: - self._prompt = Prompt() - return self._prompt + prompt = Prompt() def start(self, run=False): """Endpoint for coconut and coconut-run.""" @@ -168,15 +164,21 @@ def start(self, run=False): dest = os.path.join(os.path.dirname(source), coconut_cache_dir) else: dest = os.path.join(source, coconut_cache_dir) - self.cmd(args, argv=argv, use_dest=dest, **coconut_run_kwargs) + self.cmd_sys(args, argv=argv, use_dest=dest) else: self.cmd() + def cmd_sys(self, *args, **in_kwargs): + """Same as .cmd(), but uses defaults from coconut_sys_kwargs.""" + out_kwargs = coconut_sys_kwargs.copy() + out_kwargs.update(in_kwargs) + return self.cmd(*args, **out_kwargs) + # new external parameters should be updated in api.pyi and DOCS - def cmd(self, args=None, argv=None, interact=True, default_target=None, use_dest=None): + def cmd(self, args=None, argv=None, interact=True, default_target=None, default_jobs=None, use_dest=None): """Process command-line arguments.""" result = None - with self.handling_exceptions(): + with self.handling_exceptions(exit_on_error=True): if args is None: parsed_args = arguments.parse_args() else: @@ -187,13 +189,14 @@ def cmd(self, args=None, argv=None, interact=True, default_target=None, use_dest parsed_args.argv = argv if parsed_args.target is None: parsed_args.target = default_target + if parsed_args.jobs is None: + parsed_args.jobs = default_jobs if use_dest is not None and not parsed_args.no_write: internal_assert(parsed_args.dest is None, "coconut-run got passed a dest", parsed_args) parsed_args.dest = use_dest self.exit_code = 0 self.stack_size = parsed_args.stack_size result = self.run_with_stack_size(self.execute_args, parsed_args, interact, original_args=args) - self.exit_on_error() return result def run_with_stack_size(self, func, *args, **kwargs): @@ -239,12 +242,10 @@ def execute_args(self, args, interact=True, original_args=None): verbose=args.verbose, tracing=args.trace, ) - if args.verbose or args.trace or args.profile: - set_grammar_names() if args.trace or args.profile: unset_fast_pyparsing_reprs() if args.profile: - collect_timing_info() + start_profiling() logger.enable_colors() logger.log(cli_version) @@ -274,14 +275,15 @@ def execute_args(self, args, interact=True, original_args=None): self.set_jobs(args.jobs, args.profile) if args.recursion_limit is not None: set_recursion_limit(args.recursion_limit) + self.fail_fast = args.fail_fast self.display = args.display self.prompt.vi_mode = args.vi_mode if args.style is not None: self.prompt.set_style(args.style) - if args.history_file is not None: - self.prompt.set_history_file(args.history_file) if args.argv is not None: self.argv_args = list(args.argv) + if args.no_cache: + self.use_cache = False # execute non-compilation tasks if args.docs: @@ -312,13 +314,26 @@ def execute_args(self, args, interact=True, original_args=None): no_tco=args.no_tco, no_wrap=args.no_wrap_types, ) - if args.watch: - self.comp.warm_up(enable_incremental_mode=True) + self.comp.warm_up( + streamline=( + not self.using_jobs + and (args.watch or args.profile) + ), + enable_incremental_mode=( + not self.using_jobs + and args.watch + ), + set_debug_names=( + args.verbose + or args.trace + or args.profile + ), + ) # process mypy args and print timing info (must come after compiler setup) if args.mypy is not None: self.set_mypy_args(args.mypy) - logger.log("Grammar init time: " + str(self.comp.grammar_init_time) + " secs / Total init time: " + str(get_clock_time() - first_import_time) + " secs") + logger.log_compiler_stats(self.comp) # do compilation, keeping track of compiled filepaths filepaths = [] @@ -352,7 +367,10 @@ def execute_args(self, args, interact=True, original_args=None): self.disable_jobs() # do compilation - with self.running_jobs(exit_on_error=not args.watch): + with self.running_jobs(exit_on_error=not ( + args.watch + or args.profile + )): for source, dest, package in src_dest_package_triples: filepaths += self.compile_path(source, dest, package, run=args.run or args.interact, force=args.force) self.run_mypy(filepaths) @@ -379,8 +397,10 @@ def execute_args(self, args, interact=True, original_args=None): self.start_jupyter(args.jupyter) elif stdin_readable(): logger.log("Reading piped input from stdin...") - self.execute(self.parse_block(sys.stdin.read())) - got_stdin = True + read_stdin = sys.stdin.read() + if read_stdin: + self.execute(self.parse_block(read_stdin)) + got_stdin = True if args.interact or ( interact and not ( got_stdin @@ -400,7 +420,7 @@ def execute_args(self, args, interact=True, original_args=None): # src_dest_package_triples is always available here self.watch(src_dest_package_triples, args.run, args.force) if args.profile: - print_timing_info() + print_profiling_results() # make sure to return inside handling_exceptions to ensure filepaths is available return filepaths @@ -466,8 +486,10 @@ def register_exit_code(self, code=1, errmsg=None, err=None): self.exit_code = code or self.exit_code @contextmanager - def handling_exceptions(self): + def handling_exceptions(self, exit_on_error=None, on_keyboard_interrupt=None): """Perform proper exception handling.""" + if exit_on_error is None: + exit_on_error = self.fail_fast try: if self.using_jobs: with handling_broken_process_pool(): @@ -476,17 +498,23 @@ def handling_exceptions(self): yield except SystemExit as err: self.register_exit_code(err.code) + # make sure we don't catch GeneratorExit below + except GeneratorExit: + raise except BaseException as err: - if isinstance(err, GeneratorExit): - raise - elif isinstance(err, CoconutException): + if isinstance(err, CoconutException): logger.print_exc() - elif not isinstance(err, KeyboardInterrupt): + elif isinstance(err, KeyboardInterrupt): + if on_keyboard_interrupt is not None: + on_keyboard_interrupt() + else: logger.print_exc() logger.printerr(report_this_text) self.register_exit_code(err=err) + if exit_on_error: + self.exit_on_error() - def compile_path(self, path, write=True, package=True, **kwargs): + def compile_path(self, path, write=True, package=True, handling_exceptions_kwargs={}, **kwargs): """Compile a path and return paths to compiled files.""" if not isinstance(write, bool): write = fixpath(write) @@ -494,11 +522,11 @@ def compile_path(self, path, write=True, package=True, **kwargs): destpath = self.compile_file(path, write, package, **kwargs) return [destpath] if destpath is not None else [] elif os.path.isdir(path): - return self.compile_folder(path, write, package, **kwargs) + return self.compile_folder(path, write, package, handling_exceptions_kwargs=handling_exceptions_kwargs, **kwargs) else: raise CoconutException("could not find source path", path) - def compile_folder(self, directory, write=True, package=True, **kwargs): + def compile_folder(self, directory, write=True, package=True, handling_exceptions_kwargs={}, **kwargs): """Compile a directory and return paths to compiled files.""" if not isinstance(write, bool) and os.path.isfile(write): raise CoconutException("destination path cannot point to a file when compiling a directory") @@ -510,7 +538,7 @@ def compile_folder(self, directory, write=True, package=True, **kwargs): writedir = os.path.join(write, os.path.relpath(dirpath, directory)) for filename in filenames: if os.path.splitext(filename)[1] in code_exts: - with self.handling_exceptions(): + with self.handling_exceptions(**handling_exceptions_kwargs): destpath = self.compile_file(os.path.join(dirpath, filename), writedir, package, **kwargs) if destpath is not None: filepaths.append(destpath) @@ -565,8 +593,7 @@ def compile(self, codepath, destpath=None, package=False, run=False, force=False if destpath is not None: destpath = fixpath(destpath) destdir = os.path.dirname(destpath) - if not os.path.exists(destdir): - os.makedirs(destdir) + ensure_dir(destdir, logger=logger) if package is True: package_level = self.get_package_level(codepath) if package_level == 0: @@ -599,10 +626,14 @@ def callback(compiled): else: self.execute_file(destpath, argv_source_path=codepath) + parse_kwargs = dict( + codepath=codepath, + use_cache=self.use_cache, + ) if package is True: - self.submit_comp_job(codepath, callback, "parse_package", code, package_level=package_level, filename=os.path.basename(codepath)) + self.submit_comp_job(codepath, callback, "parse_package", code, package_level=package_level, **parse_kwargs) elif package is False: - self.submit_comp_job(codepath, callback, "parse_file", code, filename=os.path.basename(codepath)) + self.submit_comp_job(codepath, callback, "parse_file", code, **parse_kwargs) else: raise CoconutInternalException("invalid value for package", package) @@ -627,7 +658,6 @@ def get_package_level(self, codepath): logger.warn("missing __init__" + code_exts[0] + " in package", check_dir, extra="remove --strict to dismiss") package_level = 0 return package_level - return 0 def create_package(self, dirpath, retries_left=create_package_retries): """Set up a package directory.""" @@ -688,7 +718,7 @@ def disable_jobs(self): def get_max_workers(self): """Get the max_workers to use for creating ProcessPoolExecutor.""" - jobs = self.jobs if self.jobs is not None else default_jobs + jobs = self.jobs if self.jobs is not None else base_default_jobs if jobs == "sys": return None else: @@ -703,7 +733,7 @@ def using_jobs(self): @contextmanager def running_jobs(self, exit_on_error=True): """Initialize multiprocessing.""" - with self.handling_exceptions(): + with self.handling_exceptions(exit_on_error=exit_on_error): if self.using_jobs: from concurrent.futures import ProcessPoolExecutor try: @@ -713,8 +743,6 @@ def running_jobs(self, exit_on_error=True): self.executor = None else: yield - if exit_on_error: - self.exit_on_error() def has_hash_of(self, destpath, code, package_level): """Determine if a file has the hash of the code.""" @@ -810,9 +838,10 @@ def execute(self, compiled=None, path=None, use_eval=False, allow_show=True): if path is None: # header is not included if not self.mypy: no_str_code = self.comp.remove_strs(compiled) - result = mypy_builtin_regex.search(no_str_code) - if result: - logger.warn("found mypy-only built-in " + repr(result.group(0)) + "; pass --mypy to use mypy-only built-ins at the interpreter") + if no_str_code is not None: + result = mypy_builtin_regex.search(no_str_code) + if result: + logger.warn("found mypy-only built-in " + repr(result.group(0)) + "; pass --mypy to use mypy-only built-ins at the interpreter") else: # header is included compiled = rem_encoding(compiled) @@ -1064,17 +1093,30 @@ def watch(self, src_dest_package_triples, run=False, force=False): logger.show() logger.show_tabulated("Watching", showpath(src), "(press Ctrl-C to end)...") + interrupted = [False] # in list to allow modification + + def interrupt(): + interrupted[0] = True + def recompile(path, src, dest, package): path = fixpath(path) if os.path.isfile(path) and os.path.splitext(path)[1] in code_exts: - with self.handling_exceptions(): + with self.handling_exceptions(on_keyboard_interrupt=interrupt): if dest is True or dest is None: writedir = dest else: # correct the compilation path based on the relative position of path to src dirpath = os.path.dirname(path) writedir = os.path.join(dest, os.path.relpath(dirpath, src)) - filepaths = self.compile_path(path, writedir, package, run=run, force=force, show_unchanged=False) + filepaths = self.compile_path( + path, + writedir, + package, + run=run, + force=force, + show_unchanged=False, + handling_exceptions_kwargs=dict(on_keyboard_interrupt=interrupt), + ) self.run_mypy(filepaths) observer = Observer() @@ -1087,37 +1129,28 @@ def recompile(path, src, dest, package): with self.running_jobs(): observer.start() try: - while True: + while not interrupted[0]: time.sleep(watch_interval) for wcher in watchers: wcher.keep_watching() except KeyboardInterrupt: - logger.show_sig("Got KeyboardInterrupt; stopping watcher.") + interrupt() finally: + if interrupted[0]: + logger.show_sig("Got KeyboardInterrupt; stopping watcher.") observer.stop() observer.join() - def get_python_lib(self): - """Get current Python lib location.""" - # these are expensive, so should only be imported here - if PY32: - from sysconfig import get_path - python_lib = get_path("purelib") - else: - from distutils import sysconfig - python_lib = sysconfig.get_python_lib() - return fixpath(python_lib) - def site_install(self): """Add Coconut's pth file to site-packages.""" - python_lib = self.get_python_lib() + python_lib = get_python_lib() shutil.copy(coconut_pth_file, python_lib) logger.show_sig("Added %s to %s" % (os.path.basename(coconut_pth_file), python_lib)) def site_uninstall(self): """Remove Coconut's pth file from site-packages.""" - python_lib = self.get_python_lib() + python_lib = get_python_lib() pth_file = os.path.join(python_lib, os.path.basename(coconut_pth_file)) if os.path.isfile(pth_file): diff --git a/coconut/command/command.pyi b/coconut/command/command.pyi index 3f1d4ba40..f69b9ec2b 100644 --- a/coconut/command/command.pyi +++ b/coconut/command/command.pyi @@ -15,7 +15,10 @@ Description: MyPy stub file for command.py. # MAIN: # ----------------------------------------------------------------------------------------------------------------------- +from typing import Callable + class Command: """Coconut command-line interface.""" - ... + cmd: Callable + cmd_sys: Callable diff --git a/coconut/command/util.py b/coconut/command/util.py index 57d2872d6..53cb00bfb 100644 --- a/coconut/command/util.py +++ b/coconut/command/util.py @@ -29,8 +29,10 @@ from functools import partial if PY2: import __builtin__ as builtins + import Queue as queue else: import builtins + import queue from coconut.root import _coconut_exec from coconut.terminal import ( @@ -51,8 +53,10 @@ ) from coconut.constants import ( WINDOWS, - PY34, + CPYTHON, + PY26, PY32, + PY34, fixpath, base_dir, main_prompt, @@ -81,21 +85,24 @@ kilobyte, min_stack_size_kbs, coconut_base_run_args, + high_proc_prio, + call_timeout, + use_fancy_call_output, ) if PY26: import imp else: import runpy +if PY34: + from importlib import reload +else: + from imp import reload try: # just importing readline improves built-in input() import readline # NOQA except ImportError: pass -if PY34: - from importlib import reload -else: - from imp import reload try: import prompt_toolkit @@ -130,6 +137,11 @@ ), ) prompt_toolkit = None +try: + import psutil +except ImportError: + psutil = None + # ----------------------------------------------------------------------------------------------------------------------- # UTILITIES: @@ -222,9 +234,7 @@ def handling_broken_process_pool(): def kill_children(): """Terminate all child processes.""" - try: - import psutil - except ImportError: + if psutil is None: logger.warn( "missing psutil; --jobs may not properly terminate", extra="run '{python} -m pip install psutil' to fix".format(python=sys.executable), @@ -261,28 +271,112 @@ def run_file(path): return runpy.run_path(path, run_name="__main__") -def call_output(cmd, stdin=None, encoding_errors="replace", **kwargs): +def interrupt_thread(thread, exctype=OSError): + """Attempt to interrupt the given thread.""" + if not CPYTHON: + return False + if thread is None or not thread.is_alive(): + return True + import ctypes + tid = ctypes.c_long(thread.ident) + res = ctypes.pythonapi.PyThreadState_SetAsyncExc( + tid, + ctypes.py_object(exctype), + ) + if res == 0: + return False + elif res == 1: + return True + else: + ctypes.pythonapi.PyThreadState_SetAsyncExc(tid, None) + return False + + +def readline_to_queue(file_obj, q): + """Read a line from file_obj and put it in the queue.""" + if not is_empty_pipe(file_obj, False): + try: + q.put(file_obj.readline()) + except OSError: + pass + + +def call_output(cmd, stdin=None, encoding_errors="replace", color=None, **kwargs): """Run command and read output.""" - p = subprocess.Popen(cmd, stdout=subprocess.PIPE, stderr=subprocess.PIPE, **kwargs) - stdout, stderr, retcode = [], [], None - while retcode is None: - if stdin is not None: - logger.log_prefix("STDIN < ", stdin.rstrip()) - raw_out, raw_err = p.communicate(stdin) + p = subprocess.Popen(cmd, stdout=subprocess.PIPE, stderr=subprocess.PIPE, stdin=subprocess.PIPE, **kwargs) + + stdout_q = queue.Queue() + stderr_q = queue.Queue() + + if not use_fancy_call_output: + raw_stdout, raw_stderr = p.communicate(stdin) + stdout_q.put(raw_stdout) + stderr_q.put(raw_stderr) stdin = None - out = raw_out.decode(get_encoding(sys.stdout), encoding_errors) if raw_out else "" - if out: - logger.log_stdout(out.rstrip()) - stdout.append(out) + if stdin is not None: + logger.log_prefix("STDIN < ", stdin.rstrip()) + p.stdin.write(stdin) - err = raw_err.decode(get_encoding(sys.stderr), encoding_errors) if raw_err else "" - if err: - logger.log(err.rstrip()) - stderr.append(err) + # list for mutability + stdout_t_obj = [None] + stderr_t_obj = [None] - retcode = p.poll() - return stdout, stderr, retcode + stdout, stderr, retcode = [], [], None + checking_stdout = True # alternate between stdout and stderr + try: + while ( + retcode is None + or not stdout_q.empty() + or not stderr_q.empty() + or not is_empty_pipe(p.stdout, True) + or not is_empty_pipe(p.stderr, True) + ): + if checking_stdout: + proc_pipe = p.stdout + sys_pipe = sys.stdout + q = stdout_q + t_obj = stdout_t_obj + log_func = logger.log_stdout + out_list = stdout + else: + proc_pipe = p.stderr + sys_pipe = sys.stderr + q = stderr_q + t_obj = stderr_t_obj + log_func = logger.log + out_list = stderr + + retcode = p.poll() + + if ( + retcode is None + or not is_empty_pipe(proc_pipe, True) + ): + if t_obj[0] is None or not t_obj[0].is_alive(): + t_obj[0] = threading.Thread(target=readline_to_queue, args=(proc_pipe, q)) + t_obj[0].daemon = True + t_obj[0].start() + + t_obj[0].join(timeout=call_timeout) + + try: + raw_out = q.get(block=False) + except queue.Empty: + raw_out = None + + out = raw_out.decode(get_encoding(sys_pipe), encoding_errors) if raw_out else "" + + if out: + log_func(out, color=color, end="") + out_list.append(out) + + checking_stdout = not checking_stdout + finally: + interrupt_thread(stdout_t_obj[0]) + interrupt_thread(stderr_t_obj[0]) + + return "".join(stdout), "".join(stderr), retcode def run_cmd(cmd, show_output=True, raise_errs=True, **kwargs): @@ -302,7 +396,7 @@ def run_cmd(cmd, show_output=True, raise_errs=True, **kwargs): return subprocess.call(cmd, **kwargs) else: stdout, stderr, retcode = call_output(cmd, **kwargs) - output = "".join(stdout + stderr) + output = stdout + stderr if retcode and raise_errs: raise subprocess.CalledProcessError(retcode, cmd, output=output) return output @@ -395,13 +489,23 @@ def set_mypy_path(): return install_dir -def stdin_readable(): - """Determine whether stdin has any data to read.""" +def is_empty_pipe(pipe, default=None): + """Determine if the given pipe file object is empty.""" + if pipe.closed: + return True if not WINDOWS: try: - return bool(select([sys.stdin], [], [], 0)[0]) + return not select([pipe], [], [], 0)[0] except Exception: logger.log_exc() + return default + + +def stdin_readable(): + """Determine whether stdin has any data to read.""" + stdin_is_empty = is_empty_pipe(sys.stdin) + if stdin_is_empty is not None: + return not stdin_is_empty # by default assume not readable return not isatty(sys.stdin, default=True) @@ -479,6 +583,37 @@ def proc_run_args(args=()): return args +def get_python_lib(): + """Get current Python lib location.""" + # these are expensive, so should only be imported here + if PY32: + from sysconfig import get_path + python_lib = get_path("purelib") + else: + from distutils import sysconfig + python_lib = sysconfig.get_python_lib() + return fixpath(python_lib) + + +def import_coconut_header(): + """Import the coconut.__coconut__ header. + This is expensive, so only do it here.""" + try: + from coconut import __coconut__ + return __coconut__ + except ImportError: + # fixes an issue where, when running from the base coconut directory, + # the base coconut directory is treated as a namespace package + try: + from coconut.coconut import __coconut__ + except ImportError: + __coconut__ = None + if __coconut__ is not None: + return __coconut__ + else: + raise # the original ImportError, since that's the normal one + + # ----------------------------------------------------------------------------------------------------------------------- # CLASSES: # ----------------------------------------------------------------------------------------------------------------------- @@ -495,14 +630,24 @@ class Prompt(object): session = None style = None runner = None + lexer = None + suggester = None if prompt_use_suggester else False - def __init__(self, use_suggester=prompt_use_suggester): + def __init__(self, setup_now=False): """Set up the prompt.""" if prompt_toolkit is not None: self.set_style(os.getenv(style_env_var, default_style)) self.set_history_file(prompt_histfile) + if setup_now: + self.setup() + + def setup(self): + """Actually initialize the underlying Prompt. + We do this lazily since it's expensive.""" + if self.lexer is None: self.lexer = PygmentsLexer(CoconutLexer) - self.suggester = AutoSuggestFromHistory() if use_suggester else None + if self.suggester is None: + self.suggester = AutoSuggestFromHistory() def set_style(self, style): """Set pygments syntax highlighting style.""" @@ -555,6 +700,7 @@ def input(self, more=False): def prompt(self, msg): """Get input using prompt_toolkit.""" + self.setup() try: # prompt_toolkit v2 if self.session is None: @@ -619,7 +765,7 @@ def store(self, line): def fix_pickle(self): """Fix pickling of Coconut header objects.""" - from coconut import __coconut__ # this is expensive, so only do it here + __coconut__ = import_coconut_header() for var in self.vars: if not var.startswith("__") and var in dir(__coconut__) and var not in must_use_specific_target_builtins: cur_val = self.vars[var] @@ -698,6 +844,19 @@ def was_run_code(self, get_all=True): return self.stored[-1] +def highten_process(): + """Set the current process to high priority.""" + if high_proc_prio and psutil is not None: + try: + p = psutil.Process() + if WINDOWS: + p.nice(psutil.HIGH_PRIORITY_CLASS) + else: + p.nice(-10) + except Exception: + logger.log_exc() + + class multiprocess_wrapper(pickleable_obj): """Wrapper for a method that needs to be multiprocessed.""" __slots__ = ("base", "method", "stack_size", "rec_limit", "logger", "argv") @@ -717,6 +876,7 @@ def __reduce__(self): def __call__(self, *args, **kwargs): """Call the method.""" + highten_process() sys.setrecursionlimit(self.rec_limit) logger.copy_from(self.logger) sys.argv = self.argv diff --git a/coconut/compiler/compiler.py b/coconut/compiler/compiler.py index 9abf5dd15..24307a965 100644 --- a/coconut/compiler/compiler.py +++ b/coconut/compiler/compiler.py @@ -30,6 +30,7 @@ from coconut.root import * # NOQA import sys +import os import re from contextlib import contextmanager from functools import partial, wraps @@ -38,6 +39,7 @@ from coconut._pyparsing import ( USE_COMPUTATION_GRAPH, + USE_CACHE, ParseBaseException, ParseResults, col as getcol, @@ -86,6 +88,8 @@ in_place_op_funcs, match_first_arg_var, import_existing, + use_adaptive_any_of, + reverse_any_of, ) from coconut.util import ( pickleable_obj, @@ -128,6 +132,7 @@ partial_op_item_handle, ) from coconut.compiler.util import ( + ExceptionNode, sys_target, getline, addskip, @@ -150,7 +155,6 @@ append_it, interleaved_join, handle_indentation, - Wrap, tuple_str_of, join_args, parse_where, @@ -171,6 +175,10 @@ get_psf_target, move_loc_to_non_whitespace, move_endpt_to_non_whitespace, + load_cache_for, + pickle_cache, + handle_and_manage, + sub_all, ) from coconut.compiler.header import ( minify_header, @@ -450,7 +458,7 @@ def call_decorators(decorators, func_name): class Compiler(Grammar, pickleable_obj): """The Coconut compiler.""" lock = Lock() - current_compiler = [None] # list for mutability + current_compiler = None preprocs = [ lambda self: self.prepare, @@ -463,6 +471,7 @@ class Compiler(Grammar, pickleable_obj): reformatprocs = [ # deferred_code_proc must come first lambda self: self.deferred_code_proc, + lambda self: partial(self.base_passthrough_repl, wrap_char=early_passthrough_wrapper), lambda self: self.reind_proc, lambda self: self.endline_repl, lambda self: partial(self.base_passthrough_repl, wrap_char="\\"), @@ -594,6 +603,8 @@ def reset(self, keep_state=False, filename=None): @contextmanager def inner_environment(self, ln=None): """Set up compiler to evaluate inner expressions.""" + if ln is None: + ln = self.outer_ln outer_ln, self.outer_ln = self.outer_ln, ln line_numbers, self.line_numbers = self.line_numbers, False keep_lines, self.keep_lines = self.keep_lines, False @@ -626,6 +637,15 @@ def current_parsing_context(self, name, default=None): else: return default + @contextmanager + def add_to_parsing_context(self, name, obj): + """Add the given object to the parsing context for the given name.""" + self.parsing_context[name].append(obj) + try: + yield + finally: + self.parsing_context[name].pop() + @contextmanager def disable_checks(self): """Run the block without checking names or strict errors.""" @@ -646,6 +666,8 @@ def post_transform(self, grammar, text): def get_temp_var(self, base_name="temp", loc=None): """Get a unique temporary variable name.""" + if isinstance(base_name, tuple): + base_name = "_".join(base_name) if loc is None: key = None else: @@ -672,7 +694,7 @@ def method(cls, method_name, is_action=None, **kwargs): @wraps(cls_method) def method(original, loc, tokens): - self_method = getattr(cls.current_compiler[0], method_name) + self_method = getattr(cls.current_compiler, method_name) if kwargs: self_method = partial(self_method, **kwargs) if trim_arity: @@ -692,38 +714,63 @@ def method(original, loc, tokens): def bind(cls): """Binds reference objects to the proper parse actions.""" # handle parsing_context for class definitions - new_classdef = attach(cls.classdef_ref, cls.method("classdef_handle")) - cls.classdef <<= Wrap(new_classdef, cls.method("class_manage"), greedy=True) - - new_datadef = attach(cls.datadef_ref, cls.method("datadef_handle")) - cls.datadef <<= Wrap(new_datadef, cls.method("class_manage"), greedy=True) - - new_match_datadef = attach(cls.match_datadef_ref, cls.method("match_datadef_handle")) - cls.match_datadef <<= Wrap(new_match_datadef, cls.method("class_manage"), greedy=True) + cls.classdef <<= handle_and_manage( + cls.classdef_ref, + cls.method("classdef_handle"), + cls.method("class_manage"), + ) + cls.datadef <<= handle_and_manage( + cls.datadef_ref, + cls.method("datadef_handle"), + cls.method("class_manage"), + ) + cls.match_datadef <<= handle_and_manage( + cls.match_datadef_ref, + cls.method("match_datadef_handle"), + cls.method("class_manage"), + ) # handle parsing_context for function definitions - new_stmt_lambdef = attach(cls.stmt_lambdef_ref, cls.method("stmt_lambdef_handle")) - cls.stmt_lambdef <<= Wrap(new_stmt_lambdef, cls.method("func_manage"), greedy=True) - - new_decoratable_normal_funcdef_stmt = attach( + cls.stmt_lambdef <<= handle_and_manage( + cls.stmt_lambdef_ref, + cls.method("stmt_lambdef_handle"), + cls.method("func_manage"), + ) + cls.decoratable_normal_funcdef_stmt <<= handle_and_manage( cls.decoratable_normal_funcdef_stmt_ref, cls.method("decoratable_funcdef_stmt_handle"), + cls.method("func_manage"), ) - cls.decoratable_normal_funcdef_stmt <<= Wrap(new_decoratable_normal_funcdef_stmt, cls.method("func_manage"), greedy=True) - - new_decoratable_async_funcdef_stmt = attach( + cls.decoratable_async_funcdef_stmt <<= handle_and_manage( cls.decoratable_async_funcdef_stmt_ref, cls.method("decoratable_funcdef_stmt_handle", is_async=True), + cls.method("func_manage"), ) - cls.decoratable_async_funcdef_stmt <<= Wrap(new_decoratable_async_funcdef_stmt, cls.method("func_manage"), greedy=True) # handle parsing_context for type aliases - new_type_alias_stmt = attach(cls.type_alias_stmt_ref, cls.method("type_alias_stmt_handle")) - cls.type_alias_stmt <<= Wrap(new_type_alias_stmt, cls.method("type_alias_stmt_manage"), greedy=True) + cls.type_alias_stmt <<= handle_and_manage( + cls.type_alias_stmt_ref, + cls.method("type_alias_stmt_handle"), + cls.method("type_alias_stmt_manage"), + ) + + # handle parsing_context for where statements + cls.where_stmt <<= handle_and_manage( + cls.where_stmt_ref, + cls.method("where_stmt_handle"), + cls.method("where_stmt_manage"), + ) + cls.implicit_return_where <<= handle_and_manage( + cls.implicit_return_where_ref, + cls.method("where_stmt_handle"), + cls.method("where_stmt_manage"), + ) # greedy handlers (we need to know about them even if suppressed and/or they use the parsing_context) cls.comment <<= attach(cls.comment_tokens, cls.method("comment_handle"), greedy=True) cls.type_param <<= attach(cls.type_param_ref, cls.method("type_param_handle"), greedy=True) + cls.where_item <<= attach(cls.where_item_ref, cls.method("where_item_handle"), greedy=True) + cls.implicit_return_where_item <<= attach(cls.implicit_return_where_item_ref, cls.method("where_item_handle"), greedy=True) # name handlers cls.refname <<= attach(cls.name_ref, cls.method("name_handle")) @@ -842,11 +889,17 @@ def reformat_post_deferred_code_proc(self, snip): """Do post-processing that comes after deferred_code_proc.""" return self.apply_procs(self.reformatprocs[1:], snip, reformatting=True, log=False) - def reformat(self, snip, **kwargs): + def reformat(self, snip, ignore_errors, **kwargs): """Post process a preprocessed snippet.""" - internal_assert("ignore_errors" in kwargs, "reformat() missing required keyword argument: 'ignore_errors'") - with self.complain_on_err(): - return self.apply_procs(self.reformatprocs, snip, reformatting=True, log=False, **kwargs) + with noop_ctx() if ignore_errors else self.complain_on_err(): + return self.apply_procs( + self.reformatprocs, + snip, + reformatting=True, + log=False, + ignore_errors=ignore_errors, + **kwargs # no comma for py2 + ) return snip def reformat_locs(self, snip, loc, endpt=None, **kwargs): @@ -862,10 +915,14 @@ def reformat_locs(self, snip, loc, endpt=None, **kwargs): if endpt is None: return new_snip, new_loc - new_endpt = move_endpt_to_non_whitespace( - new_snip, - len(self.reformat(snip[:endpt], **kwargs)), + new_endpt = clip( + move_endpt_to_non_whitespace( + new_snip, + len(self.reformat(snip[:endpt], **kwargs)), + ), + min=new_loc, ) + return new_snip, new_loc, new_endpt def reformat_without_adding_code_before(self, code, **kwargs): @@ -874,6 +931,14 @@ def reformat_without_adding_code_before(self, code, **kwargs): reformatted_code = self.reformat(code, put_code_to_add_before_in=got_code_to_add_before, **kwargs) return reformatted_code, tuple(got_code_to_add_before.keys()), got_code_to_add_before.values() + def extract_deferred_code(self, code): + """Extract the code to be added before in code.""" + got_code_to_add_before = {} + procd_out = self.deferred_code_proc(code, put_code_to_add_before_in=got_code_to_add_before) + added_names = tuple(got_code_to_add_before.keys()) + add_code_before = "\n".join(got_code_to_add_before.values()) + return procd_out, added_names, add_code_before + def literal_eval(self, code): """Version of ast.literal_eval that reformats first.""" return literal_eval(self.reformat(code, ignore_errors=False)) @@ -928,12 +993,14 @@ def complain_on_err(self): except CoconutException as err: complain(err) - def remove_strs(self, inputstring, inner_environment=True): - """Remove strings/comments from the given input.""" - with self.complain_on_err(): + def remove_strs(self, inputstring, inner_environment=True, **kwargs): + """Remove strings/comments from the given input if possible.""" + try: with (self.inner_environment() if inner_environment else noop_ctx()): - return self.str_proc(inputstring) - return inputstring + return self.str_proc(inputstring, **kwargs) + except Exception: + logger.log_exc() + return None def get_matcher(self, original, loc, check_var, name_list=None): """Get a Matcher object.""" @@ -1004,6 +1071,9 @@ def wrap_passthrough(self, text, multiline=True, early=False): if not multiline: text = text.lstrip() if early: + # early passthroughs can be nested, so un-nest them + while early_passthrough_wrapper in text: + text = self.base_passthrough_repl(text, wrap_char=early_passthrough_wrapper) out = early_passthrough_wrapper elif multiline: out = "\\" @@ -1024,10 +1094,12 @@ def wrap_error(self, error): def raise_or_wrap_error(self, error): """Raise if USE_COMPUTATION_GRAPH else wrap.""" - if USE_COMPUTATION_GRAPH: - raise error - else: + if not USE_COMPUTATION_GRAPH: return self.wrap_error(error) + elif use_adaptive_any_of or reverse_any_of: + return ExceptionNode(error) + else: + raise error def type_ignore_comment(self): """Get a "type: ignore" comment.""" @@ -1114,7 +1186,10 @@ def make_err(self, errtype, message, original, loc=0, ln=None, extra=None, refor # get line number if ln is None: - ln = self.outer_ln or self.adjust(lineno(loc, original)) + if self.outer_ln is None: + ln = self.adjust(lineno(loc, original)) + else: + ln = self.outer_ln # get line indices for the error locs original_lines = tuple(logical_lines(original, True)) @@ -1135,9 +1210,11 @@ def make_err(self, errtype, message, original, loc=0, ln=None, extra=None, refor self.internal_assert(extra is None, original, loc, "make_err cannot include causes with extra") causes = dictset() for cause, _, _ in all_matches(self.parse_err_msg, snippet[loc_in_snip:]): - causes.add(cause) + if cause: + causes.add(cause) for cause, _, _ in all_matches(self.parse_err_msg, snippet[endpt_in_snip:]): - causes.add(cause) + if cause: + causes.add(cause) if causes: extra = "possible cause{s}: {causes}".format( s="s" if len(causes) > 1 else "", @@ -1156,10 +1233,10 @@ def make_err(self, errtype, message, original, loc=0, ln=None, extra=None, refor kwargs["extra"] = extra return errtype(message, snippet, loc_in_snip, ln, endpoint=endpt_in_snip, filename=self.filename, **kwargs) - def make_syntax_err(self, err, original): + def make_syntax_err(self, err, original, after_parsing=False): """Make a CoconutSyntaxError from a CoconutDeferredSyntaxError.""" msg, loc = err.args - return self.make_err(CoconutSyntaxError, msg, original, loc) + return self.make_err(CoconutSyntaxError, msg, original, loc, endpoint=not after_parsing) def make_parse_err(self, err, msg=None, include_ln=True, **kwargs): """Make a CoconutParseError from a ParseBaseException.""" @@ -1191,34 +1268,41 @@ def inner_parse_eval( """Parse eval code in an inner environment.""" if parser is None: parser = self.eval_parser - with self.inner_environment(ln=self.adjust(lineno(loc, original))): - self.streamline(parser, inputstring) + outer_ln = self.outer_ln + if outer_ln is None: + outer_ln = self.adjust(lineno(loc, original)) + with self.inner_environment(ln=outer_ln): + self.streamline(parser, inputstring, inner=True) pre_procd = self.pre(inputstring, **preargs) parsed = parse(parser, pre_procd) return self.post(parsed, **postargs) @contextmanager - def parsing(self, keep_state=False, filename=None): + def parsing(self, keep_state=False, codepath=None): """Acquire the lock and reset the parser.""" + filename = None if codepath is None else os.path.basename(codepath) with self.lock: self.reset(keep_state, filename) - self.current_compiler[0] = self + Compiler.current_compiler = self yield - def streamline(self, grammar, inputstring="", force=False): + def streamline(self, grammar, inputstring=None, force=False, inner=False): """Streamline the given grammar for the given inputstring.""" - if force or (streamline_grammar_for_len is not None and len(inputstring) >= streamline_grammar_for_len): + input_len = 0 if inputstring is None else len(inputstring) + if force or (streamline_grammar_for_len is not None and input_len > streamline_grammar_for_len): start_time = get_clock_time() prep_grammar(grammar, streamline=True) logger.log_lambda( - lambda: "Streamlined {grammar} in {time} seconds (streamlined due to receiving input of length {length}).".format( + lambda: "Streamlined {grammar} in {time} seconds{info}.".format( grammar=get_name(grammar), time=get_clock_time() - start_time, - length=len(inputstring), + info="" if inputstring is None else " (streamlined due to receiving input of length {length})".format( + length=input_len, + ), ), ) - else: - logger.log("No streamlining done for input of length {length}.".format(length=len(inputstring))) + elif inputstring is not None and not inner: + logger.log("No streamlining done for input of length {length}.".format(length=input_len)) def run_final_checks(self, original, keep_state=False): """Run post-parsing checks to raise any necessary errors/warnings.""" @@ -1233,30 +1317,54 @@ def run_final_checks(self, original, keep_state=False): "found unused import " + repr(self.reformat(name, ignore_errors=True)) + " (add '# NOQA' to suppress)", original, loc, + endpoint=False, ) - def parse(self, inputstring, parser, preargs, postargs, streamline=True, keep_state=False, filename=None): + def parse( + self, + inputstring, + parser, + preargs, + postargs, + streamline=True, + keep_state=False, + codepath=None, + use_cache=None, + ): """Use the parser to parse the inputstring with appropriate setup and teardown.""" - with self.parsing(keep_state, filename): + if use_cache is None: + use_cache = USE_CACHE + use_cache = use_cache and codepath is not None + with self.parsing(keep_state, codepath): if streamline: self.streamline(parser, inputstring) - with logger.gather_parsing_stats(): - pre_procd = None - try: - pre_procd = self.pre(inputstring, keep_state=keep_state, **preargs) - parsed = parse(parser, pre_procd, inner=False) - out = self.post(parsed, keep_state=keep_state, **postargs) - except ParseBaseException as err: - raise self.make_parse_err(err) - except CoconutDeferredSyntaxError as err: - internal_assert(pre_procd is not None, "invalid deferred syntax error in pre-processing", err) - raise self.make_syntax_err(err, pre_procd) - # RuntimeError, not RecursionError, for Python < 3.5 - except RuntimeError as err: - raise CoconutException( - str(err), extra="try again with --recursion-limit greater than the current " - + str(sys.getrecursionlimit()) + " (you may also need to increase --stack-size)", - ) + # unpickling must happen after streamlining and must occur in the + # compiler so that it happens in the same process as compilation + if use_cache: + cache_path, incremental_enabled = load_cache_for(inputstring, codepath) + else: + cache_path = None + pre_procd = parsed = None + try: + with logger.gather_parsing_stats(): + try: + pre_procd = self.pre(inputstring, keep_state=keep_state, **preargs) + parsed = parse(parser, pre_procd, inner=False) + out = self.post(parsed, keep_state=keep_state, **postargs) + except ParseBaseException as err: + raise self.make_parse_err(err) + except CoconutDeferredSyntaxError as err: + internal_assert(pre_procd is not None, "invalid deferred syntax error in pre-processing", err) + raise self.make_syntax_err(err, pre_procd, after_parsing=parsed is not None) + # RuntimeError, not RecursionError, for Python < 3.5 + except RuntimeError as err: + raise CoconutException( + str(err), extra="try again with --recursion-limit greater than the current " + + str(sys.getrecursionlimit()) + " (you may also need to increase --stack-size)", + ) + finally: + if cache_path is not None and pre_procd is not None: + pickle_cache(pre_procd, cache_path, include_incremental=incremental_enabled) self.run_final_checks(pre_procd, keep_state) return out @@ -2233,7 +2341,7 @@ def proc_funcdef(self, original, loc, decorators, funcdef, is_async, in_method, undotted_name = None if func_name is not None and "." in func_name: undotted_name = func_name.rsplit(".", 1)[-1] - def_name = self.get_temp_var("dotted_" + undotted_name, loc) + def_name = self.get_temp_var(("dotted", undotted_name), loc) # detect pattern-matching functions is_match_func = func_paramdef == match_func_paramdef @@ -2263,10 +2371,7 @@ def proc_funcdef(self, original, loc, decorators, funcdef, is_async, in_method, # modify function definition to use def_name if def_name != func_name: - def_stmt_pre_lparen, def_stmt_post_lparen = def_stmt.split("(", 1) - def_stmt_def, def_stmt_name = def_stmt_pre_lparen.rsplit(" ", 1) - def_stmt_name = def_stmt_name.replace(func_name, def_name) - def_stmt = def_stmt_def + " " + def_stmt_name + "(" + def_stmt_post_lparen + def_stmt = compile_regex(r"\b" + re.escape(func_name) + r"\b").sub(def_name, def_stmt) # detect generators is_gen = self.detect_is_gen(raw_lines) @@ -2481,6 +2586,14 @@ def {mock_var}({mock_paramdef}): internal_assert(not decorators, "unhandled decorators", decorators) return "".join(out) + def modify_add_code_before(self, add_code_before_names, code_modifier): + """Apply code_modifier to all the code corresponding to add_code_before_names.""" + for name in add_code_before_names: + self.add_code_before[name] = code_modifier(self.add_code_before[name]) + replacement = self.add_code_before_replacements.get(name) + if replacement is not None: + self.add_code_before_replacements[name] = code_modifier(replacement) + def add_code_before_marker_with_replacement(self, replacement, add_code_before, add_spaces=True, ignore_names=None): """Add code before a marker that will later be replaced.""" # temp_marker will be set back later, but needs to be a unique name until then for add_code_before @@ -2511,9 +2624,6 @@ def deferred_code_proc(self, inputstring, add_code_at_start=False, ignore_names= for raw_line in inputstring.splitlines(True): bef_ind, line, aft_ind = split_leading_trailing_indent(raw_line) - # handle early passthroughs - line = self.base_passthrough_repl(line, wrap_char=early_passthrough_wrapper, **kwargs) - # look for deferred errors while errwrapper in raw_line: pre_err_line, err_line = raw_line.split(errwrapper, 1) @@ -2650,7 +2760,7 @@ def pipe_item_split(self, tokens, loc): - (expr,) for expression - (func, pos_args, kwd_args) for partial - (name, args) for attr/method - - (op, args)+ for itemgetter + - (attr, [(op, args)]) for itemgetter - (op, arg) for right op partial """ # list implies artificial tokens, which must be expr @@ -2665,8 +2775,12 @@ def pipe_item_split(self, tokens, loc): name, args = attrgetter_atom_split(tokens) return "attrgetter", (name, args) elif "itemgetter" in tokens: - internal_assert(len(tokens) >= 2, "invalid itemgetter pipe item tokens", tokens) - return "itemgetter", tokens + if len(tokens) == 1: + attr = None + ops_and_args, = tokens + else: + attr, ops_and_args = tokens + return "itemgetter", (attr, ops_and_args) elif "op partial" in tokens: inner_toks, = tokens if "left partial" in inner_toks: @@ -2699,7 +2813,7 @@ def pipe_handle(self, original, loc, tokens, **kwargs): return expr elif name == "partial": self.internal_assert(len(split_item) == 3, original, loc) - return "_coconut.functools.partial(" + join_args(split_item) + ")" + return "_coconut_partial(" + join_args(split_item) + ")" elif name == "attrgetter": return attrgetter_atom_handle(loc, item) elif name == "itemgetter": @@ -2756,12 +2870,13 @@ def pipe_handle(self, original, loc, tokens, **kwargs): elif name == "itemgetter": if stars: raise CoconutDeferredSyntaxError("cannot star pipe into item getting", loc) - self.internal_assert(len(split_item) % 2 == 0, original, loc, "invalid itemgetter pipe tokens", split_item) - out = subexpr - for i in range(0, len(split_item), 2): - op, args = split_item[i:i + 2] + attr, ops_and_args = split_item + out = "(" + subexpr + ")" + if attr is not None: + out += "." + attr + for op, args in ops_and_args: if op == "[": - fmtstr = "({x})[{args}]" + fmtstr = "{x}[{args}]" elif op == "$[": fmtstr = "_coconut_iter_getitem({x}, ({args}))" else: @@ -2792,14 +2907,14 @@ def item_handle(self, original, loc, tokens): out += trailer elif len(trailer) == 1: if trailer[0] == "$[]": - out = "_coconut.functools.partial(_coconut_iter_getitem, " + out + ")" + out = "_coconut_partial(_coconut_iter_getitem, " + out + ")" elif trailer[0] == "$": - out = "_coconut.functools.partial(_coconut.functools.partial, " + out + ")" + out = "_coconut_partial(_coconut_partial, " + out + ")" elif trailer[0] == "[]": - out = "_coconut.functools.partial(_coconut.operator.getitem, " + out + ")" + out = "_coconut_partial(_coconut.operator.getitem, " + out + ")" elif trailer[0] == ".": self.strict_err_or_warn("'obj.' as a shorthand for 'getattr$(obj)' is deprecated (just use the getattr partial)", original, loc) - out = "_coconut.functools.partial(_coconut.getattr, " + out + ")" + out = "_coconut_partial(_coconut.getattr, " + out + ")" elif trailer[0] == "type:[]": out = "_coconut.typing.Sequence[" + out + "]" elif trailer[0] == "type:$[]": @@ -2832,18 +2947,23 @@ def item_handle(self, original, loc, tokens): args = trailer[1][1:-1] if not args: raise CoconutDeferredSyntaxError("a partial application argument is required", loc) - out = "_coconut.functools.partial(" + out + ", " + args + ")" + out = "_coconut_partial(" + out + ", " + args + ")" elif trailer[0] == "$[": out = "_coconut_iter_getitem(" + out + ", " + trailer[1] + ")" elif trailer[0] == "$(?": pos_args, star_args, base_kwd_args, dubstar_args = self.split_function_call(trailer[1], loc) + has_question_mark = False + needs_complex_partial = False argdict_pairs = [] + last_pos_i = -1 for i, arg in enumerate(pos_args): if arg == "?": has_question_mark = True else: + if last_pos_i != i - 1: + needs_complex_partial = True argdict_pairs.append(str(i) + ": " + arg) pos_kwargs = [] @@ -2851,25 +2971,36 @@ def item_handle(self, original, loc, tokens): for i, arg in enumerate(base_kwd_args): if arg.endswith("=?"): has_question_mark = True + needs_complex_partial = True pos_kwargs.append(arg[:-2]) else: kwd_args.append(arg) - extra_args_str = join_args(star_args, kwd_args, dubstar_args) if not has_question_mark: raise CoconutInternalException("no question mark in question mark partial", trailer[1]) - elif argdict_pairs or pos_kwargs or extra_args_str: + + if needs_complex_partial: + extra_args_str = join_args(star_args, kwd_args, dubstar_args) + if argdict_pairs or pos_kwargs or extra_args_str: + out = ( + "_coconut_complex_partial(" + + out + + ", {" + ", ".join(argdict_pairs) + "}" + + ", " + str(len(pos_args)) + + ", " + tuple_str_of(pos_kwargs, add_quotes=True) + + (", " if extra_args_str else "") + extra_args_str + + ")" + ) + else: + raise CoconutDeferredSyntaxError("a non-? partial application argument is required", loc) + else: out = ( "_coconut_partial(" + out - + ", {" + ", ".join(argdict_pairs) + "}" - + ", " + str(len(pos_args)) - + ", " + tuple_str_of(pos_kwargs, add_quotes=True) - + (", " if extra_args_str else "") + extra_args_str + + ", " + + join_args([arg for arg in pos_args if arg != "?"], star_args, kwd_args, dubstar_args) + ")" ) - else: - raise CoconutDeferredSyntaxError("a non-? partial application argument is required", loc) else: raise CoconutInternalException("invalid special trailer", trailer[0]) @@ -3883,8 +4014,8 @@ def type_param_handle(self, original, loc, tokens): kwargs = "" if bound_op is not None: self.internal_assert(bound_op_type in ("bound", "constraint"), original, loc, "invalid type_param bound_op", bound_op) - # # uncomment this line whenever mypy adds support for infer_variance in TypeVar - # # (and remove the warning about it in the DOCS) + # uncomment this line whenever mypy adds support for infer_variance in TypeVar + # (and remove the warning about it in the DOCS) # kwargs = ", infer_variance=True" if bound_op == "<=": self.strict_err_or_warn( @@ -3912,7 +4043,7 @@ def type_param_handle(self, original, loc, tokens): else: if name in typevar_info["all_typevars"]: raise CoconutDeferredSyntaxError("type variable {name!r} already defined".format(name=name), loc) - temp_name = self.get_temp_var("typevar_" + name, name_loc) + temp_name = self.get_temp_var(("typevar", name), name_loc) typevar_info["all_typevars"][name] = temp_name typevar_info["new_typevars"].append((TypeVarFunc, temp_name)) typevar_info["typevar_locs"][name] = name_loc @@ -3945,17 +4076,13 @@ def get_generic_for_typevars(self): @contextmanager def type_alias_stmt_manage(self, item=None, original=None, loc=None): """Manage the typevars parsing context.""" - typevars_stack = self.parsing_context["typevars"] prev_typevar_info = self.current_parsing_context("typevars") - typevars_stack.append({ + with self.add_to_parsing_context("typevars", { "all_typevars": {} if prev_typevar_info is None else prev_typevar_info["all_typevars"].copy(), "new_typevars": [], "typevar_locs": {}, - }) - try: + }): yield - finally: - typevars_stack.pop() def type_alias_stmt_handle(self, tokens): """Handle type alias statements.""" @@ -3973,6 +4100,51 @@ def type_alias_stmt_handle(self, tokens): self.wrap_typedef(typedef, for_py_typedef=False), ]) + def where_item_handle(self, tokens): + """Manage where items.""" + where_context = self.current_parsing_context("where") + internal_assert(not where_context["assigns"], "invalid where_context", where_context) + where_context["assigns"] = set() + return tokens + + @contextmanager + def where_stmt_manage(self, item, original, loc): + """Manage where statements.""" + with self.add_to_parsing_context("where", { + "assigns": None, + }): + yield + + def where_stmt_handle(self, loc, tokens): + """Process where statements.""" + main_stmt, body_stmts = tokens + + where_assigns = self.current_parsing_context("where")["assigns"] + internal_assert(lambda: where_assigns is not None, "missing where_assigns") + + where_init = "".join(body_stmts) + where_final = main_stmt + "\n" + out = where_init + where_final + if not where_assigns: + return out + + name_regexes = { + name: compile_regex(r"\b" + name + r"\b") + for name in where_assigns + } + name_replacements = { + name: self.get_temp_var(("where", name), loc) + for name in where_assigns + } + + where_init = self.deferred_code_proc(where_init) + where_final = self.deferred_code_proc(where_final) + out = where_init + where_final + + out = sub_all(out, name_regexes, name_replacements) + + return self.wrap_passthrough(out, early=True) + def with_stmt_handle(self, tokens): """Process with statements.""" withs, body = tokens @@ -4386,7 +4558,7 @@ def check_strict(self, name, original, loc, tokens=(None,), only_warn=False, alw else: if always_warn: kwargs["extra"] = "remove --strict to downgrade to a warning" - raise self.make_err(CoconutStyleError, message, original, loc, **kwargs) + return self.raise_or_wrap_error(self.make_err(CoconutStyleError, message, original, loc, **kwargs)) elif always_warn: self.syntax_warning(message, original, loc) return tokens[0] @@ -4427,7 +4599,13 @@ def check_py(self, version, name, original, loc, tokens): self.internal_assert(len(tokens) == 1, original, loc, "invalid " + name + " tokens", tokens) version_info = get_target_info(version) if self.target_info < version_info: - raise self.make_err(CoconutTargetError, "found Python " + ".".join(str(v) for v in version_info) + " " + name, original, loc, target=version) + return self.raise_or_wrap_error(self.make_err( + CoconutTargetError, + "found Python " + ".".join(str(v) for v in version_info) + " " + name, + original, + loc, + target=version, + )) else: return tokens[0] @@ -4486,14 +4664,21 @@ def name_handle(self, original, loc, tokens, assign=False, classname=False): else: escaped = False + if self.disable_name_check: + return name + + if assign: + where_context = self.current_parsing_context("where") + if where_context is not None: + where_assigns = where_context["assigns"] + if where_assigns is not None: + where_assigns.add(name) + if classname: cls_context = self.current_parsing_context("class") self.internal_assert(cls_context is not None, original, loc, "found classname outside of class", tokens) cls_context["name"] = name - if self.disable_name_check: - return name - # raise_or_wrap_error for all errors here to make sure we don't # raise spurious errors if not using the computation graph @@ -4662,10 +4847,12 @@ def parse_xonsh(self, inputstring, **kwargs): """Parse xonsh code.""" return self.parse(inputstring, self.xonsh_parser, {"strip": True}, {"header": "none", "initial": "none"}, streamline=False, **kwargs) - def warm_up(self, force=False, enable_incremental_mode=False): + def warm_up(self, streamline=False, enable_incremental_mode=False, set_debug_names=False): """Warm up the compiler by streamlining the file_parser.""" - self.streamline(self.file_parser, force=force) - self.streamline(self.eval_parser, force=force) + if set_debug_names: + self.set_grammar_names() + self.streamline(self.file_parser, force=streamline) + self.streamline(self.eval_parser, force=streamline) if enable_incremental_mode: enable_incremental_parsing() diff --git a/coconut/compiler/grammar.py b/coconut/compiler/grammar.py index bbd3bd902..7eaed6226 100644 --- a/coconut/compiler/grammar.py +++ b/coconut/compiler/grammar.py @@ -116,6 +116,9 @@ compile_regex, always_match, caseless_literal, + using_fast_grammar_methods, + disambiguate_literal, + any_of, ) @@ -437,22 +440,24 @@ def subscriptgroup_handle(tokens): def itemgetter_handle(tokens): """Process implicit itemgetter partials.""" - if len(tokens) == 2: - op, args = tokens + if len(tokens) == 1: + attr = None + ops_and_args, = tokens + else: + attr, ops_and_args = tokens + if attr is None and len(ops_and_args) == 1: + (op, args), = ops_and_args if op == "[": return "_coconut.operator.itemgetter((" + args + "))" elif op == "$[": - return "_coconut.functools.partial(_coconut_iter_getitem, index=(" + args + "))" + return "_coconut_partial(_coconut_iter_getitem, index=(" + args + "))" else: raise CoconutInternalException("invalid implicit itemgetter type", op) - elif len(tokens) > 2: - internal_assert(len(tokens) % 2 == 0, "invalid itemgetter composition tokens", tokens) - itemgetters = [] - for i in range(0, len(tokens), 2): - itemgetters.append(itemgetter_handle(tokens[i:i + 2])) - return "_coconut_forward_compose(" + ", ".join(itemgetters) + ")" else: - raise CoconutInternalException("invalid implicit itemgetter tokens", tokens) + return "_coconut_attritemgetter({attr}, {is_iter_and_items})".format( + attr=repr(attr), + is_iter_and_items=", ".join("({is_iter}, ({item}))".format(is_iter=op == "$[", item=args) for op, args in ops_and_args), + ) def class_suite_handle(tokens): @@ -518,12 +523,6 @@ def join_match_funcdef(tokens): ) -def where_handle(tokens): - """Process where statements.""" - final_stmt, init_stmts = tokens - return "".join(init_stmts) + final_stmt + "\n" - - def kwd_err_msg_handle(tokens): """Handle keyword parse error messages.""" kwd, = tokens @@ -544,10 +543,10 @@ def partial_op_item_handle(tokens): tok_grp, = tokens if "left partial" in tok_grp: arg, op = tok_grp - return "_coconut.functools.partial(" + op + ", " + arg + ")" + return "_coconut_partial(" + op + ", " + arg + ")" elif "right partial" in tok_grp: op, arg = tok_grp - return "_coconut_partial(" + op + ", {1: " + arg + "}, 2, ())" + return "_coconut_complex_partial(" + op + ", {1: " + arg + "}, 2, ())" else: raise CoconutInternalException("invalid operator function implicit partial token group", tok_grp) @@ -618,1931 +617,2028 @@ def typedef_op_item_handle(loc, tokens): class Grammar(object): """Coconut grammar specification.""" grammar_init_time = get_clock_time() + with using_fast_grammar_methods(): + + comma = Literal(",") + dubstar = Literal("**") + star = disambiguate_literal("*", ["**"]) + at = Literal("@") + arrow = Literal("->") | fixto(Literal("\u2192"), "->") + unsafe_fat_arrow = Literal("=>") | fixto(Literal("\u21d2"), "=>") + colon_eq = Literal(":=") + unsafe_dubcolon = Literal("::") + unsafe_colon = Literal(":") + colon = disambiguate_literal(":", ["::", ":="]) + lt_colon = Literal("<:") + semicolon = Literal(";") | invalid_syntax("\u037e", "invalid Greek question mark instead of semicolon", greedy=True) + multisemicolon = combine(OneOrMore(semicolon)) + eq = Literal("==") + equals = disambiguate_literal("=", ["==", "=>"]) + lbrack = Literal("[") + rbrack = Literal("]") + lbrace = Literal("{") + rbrace = Literal("}") + lbanana = disambiguate_literal("(|", ["(|)", "(|>", "(|*", "(|?"]) + rbanana = Literal("|)") + lparen = ~lbanana + Literal("(") + rparen = Literal(")") + unsafe_dot = Literal(".") + dot = disambiguate_literal(".", [".."]) + plus = Literal("+") + minus = disambiguate_literal("-", ["->"]) + dubslash = Literal("//") + slash = disambiguate_literal("/", ["//"]) + pipe = Literal("|>") | fixto(Literal("\u21a6"), "|>") + star_pipe = Literal("|*>") | fixto(Literal("*\u21a6"), "|*>") + dubstar_pipe = Literal("|**>") | fixto(Literal("**\u21a6"), "|**>") + back_pipe = Literal("<|") | fixto(Literal("\u21a4"), "<|") + back_star_pipe = Literal("<*|") | ~Literal("\u21a4**") + fixto(Literal("\u21a4*"), "<*|") + back_dubstar_pipe = Literal("<**|") | fixto(Literal("\u21a4**"), "<**|") + none_pipe = Literal("|?>") | fixto(Literal("?\u21a6"), "|?>") + none_star_pipe = ( + Literal("|?*>") + | fixto(Literal("?*\u21a6"), "|?*>") + | invalid_syntax("|*?>", "Coconut's None-aware forward multi-arg pipe is '|?*>', not '|*?>'") + ) + none_dubstar_pipe = ( + Literal("|?**>") + | fixto(Literal("?**\u21a6"), "|?**>") + | invalid_syntax("|**?>", "Coconut's None-aware forward keyword pipe is '|?**>', not '|**?>'") + ) + back_none_pipe = Literal("", "..*", "..?"]) + | fixto(disambiguate_literal("\u2218", ["\u2218>", "\u2218*", "\u2218?"]), "..") + ) + comp_pipe = Literal("..>") | fixto(Literal("\u2218>"), "..>") + comp_back_pipe = Literal("<..") | fixto(Literal("<\u2218"), "<..") + comp_star_pipe = Literal("..*>") | fixto(Literal("\u2218*>"), "..*>") + comp_back_star_pipe = Literal("<*..") | fixto(Literal("<*\u2218"), "<*..") + comp_dubstar_pipe = Literal("..**>") | fixto(Literal("\u2218**>"), "..**>") + comp_back_dubstar_pipe = Literal("<**..") | fixto(Literal("<**\u2218"), "<**..") + comp_none_pipe = Literal("..?>") | fixto(Literal("\u2218?>"), "..?>") + comp_back_none_pipe = Literal("") + | fixto(Literal("\u2218?*>"), "..?*>") + | invalid_syntax("..*?>", "Coconut's None-aware forward multi-arg composition pipe is '..?*>', not '..*?>'") + ) + comp_back_none_star_pipe = ( + Literal("<*?..") + | fixto(Literal("<*?\u2218"), "<*?..") + | invalid_syntax("") + | fixto(Literal("\u2218?**>"), "..?**>") + | invalid_syntax("..**?>", "Coconut's None-aware forward keyword composition pipe is '..?**>', not '..**?>'") + ) + comp_back_none_dubstar_pipe = ( + Literal("<**?..") + | fixto(Literal("<**?\u2218"), "<**?..") + | invalid_syntax("", "|*"]) + | fixto(Literal("\u222a"), "|") + ) + bar = disambiguate_literal("|", ["|)"]) | invalid_syntax("\xa6", "invalid broken bar character", greedy=True) + percent = Literal("%") + dollar = Literal("$") + lshift = Literal("<<") | fixto(Literal("\xab"), "<<") + rshift = Literal(">>") | fixto(Literal("\xbb"), ">>") + tilde = Literal("~") + underscore = Literal("_") + pound = Literal("#") + unsafe_backtick = Literal("`") + dubbackslash = Literal("\\\\") + backslash = disambiguate_literal("\\", ["\\\\"]) + dubquestion = Literal("??") + questionmark = disambiguate_literal("?", ["??"]) + bang = disambiguate_literal("!", ["!="]) + + kwds = keydefaultdict(partial(base_keyword, explicit_prefix=colon)) + keyword = kwds.__getitem__ + + except_star_kwd = combine(keyword("except") + star) + kwds["except"] = ~except_star_kwd + keyword("except") + kwds["lambda"] = keyword("lambda") | fixto(keyword("\u03bb"), "lambda") + kwds["operator"] = base_keyword("operator", explicit_prefix=colon, require_whitespace=True) + + ellipsis = Forward() + ellipsis_tokens = Literal("...") | fixto(Literal("\u2026"), "...") + + lt = ( + disambiguate_literal("<", ["<<", "<=", "<|", "<..", "<*", "<:"]) + | fixto(Literal("\u228a"), "<") + ) + gt = ( + disambiguate_literal(">", [">>", ">="]) + | fixto(Literal("\u228b"), ">") + ) + le = Literal("<=") | fixto(Literal("\u2264") | Literal("\u2286"), "<=") + ge = Literal(">=") | fixto(Literal("\u2265") | Literal("\u2287"), ">=") + ne = Literal("!=") | fixto(Literal("\xac=") | Literal("\u2260"), "!=") + + mul_star = star | fixto(Literal("\xd7"), "*") + exp_dubstar = dubstar | fixto(Literal("\u2191"), "**") + neg_minus = ( + minus + | fixto(Literal("\u207b"), "-") + ) + sub_minus = ( + minus + | invalid_syntax("\u207b", "U+207b is only for negation, not subtraction") + ) + div_slash = slash | fixto(Literal("\xf7") + ~slash, "/") + div_dubslash = dubslash | fixto(combine(Literal("\xf7") + slash), "//") + matrix_at = at + + test = Forward() + test_no_chain, dubcolon = disable_inside(test, unsafe_dubcolon) + test_no_infix, backtick = disable_inside(test, unsafe_backtick) + + base_name_regex = r"" + for no_kwd in keyword_vars + const_vars: + base_name_regex += r"(?!" + no_kwd + r"\b)" + # we disallow ['"{] after to not match the "b" in b"" or the "s" in s{} + base_name_regex += r"(?![0-9])\w+\b(?![{" + strwrapper + r"])" + base_name = regex_item(base_name_regex) + + refname = Forward() + setname = Forward() + classname = Forward() + name_ref = combine(Optional(backslash) + base_name) + unsafe_name = combine(Optional(backslash.suppress()) + base_name) + + # use unsafe_name for dotted components since name should only be used for base names + dotted_refname = condense(refname + ZeroOrMore(dot + unsafe_name)) + dotted_setname = condense(setname + ZeroOrMore(dot + unsafe_name)) + unsafe_dotted_name = condense(unsafe_name + ZeroOrMore(dot + unsafe_name)) + must_be_dotted_name = condense(refname + OneOrMore(dot + unsafe_name)) + + integer = combine(Word(nums) + ZeroOrMore(underscore.suppress() + Word(nums))) + binint = combine(Word("01") + ZeroOrMore(underscore.suppress() + Word("01"))) + octint = combine(Word("01234567") + ZeroOrMore(underscore.suppress() + Word("01234567"))) + hexint = combine(Word(hexnums) + ZeroOrMore(underscore.suppress() + Word(hexnums))) + + imag_j = caseless_literal("j") | fixto(caseless_literal("i", suppress=True), "j") + basenum = combine( + Optional(integer) + dot + integer + | integer + Optional(dot + Optional(integer)) + ) + sci_e = combine((caseless_literal("e") | fixto(Literal("\u23e8"), "e")) + Optional(plus | neg_minus)) + numitem = combine(basenum + Optional(sci_e + integer)) + imag_num = combine(numitem + imag_j) + maybe_imag_num = combine(numitem + Optional(imag_j)) + bin_num = combine(caseless_literal("0b") + Optional(underscore.suppress()) + binint) + oct_num = combine(caseless_literal("0o") + Optional(underscore.suppress()) + octint) + hex_num = combine(caseless_literal("0x") + Optional(underscore.suppress()) + hexint) + non_decimal_num = any_of( + hex_num, + bin_num, + oct_num, + use_adaptive=False, + ) + number = ( + non_decimal_num + # must come last + | maybe_imag_num + ) + # make sure that this gets addspaced not condensed so it doesn't produce a SyntaxError + num_atom = addspace(number + Optional(condense(dot + unsafe_name))) + + moduledoc_item = Forward() + unwrap = Literal(unwrapper) + comment = Forward() + comment_tokens = combine(pound + integer + unwrap) + string_item = ( + combine(Literal(strwrapper) + integer + unwrap) + | invalid_syntax(("\u201c", "\u201d", "\u2018", "\u2019"), "invalid unicode quotation mark; strings must use \" or '", greedy=True) + ) - comma = Literal(",") - dubstar = Literal("**") - star = ~dubstar + Literal("*") - at = Literal("@") - arrow = Literal("->") | fixto(Literal("\u2192"), "->") - unsafe_fat_arrow = Literal("=>") | fixto(Literal("\u21d2"), "=>") - colon_eq = Literal(":=") - unsafe_dubcolon = Literal("::") - unsafe_colon = Literal(":") - colon = ~unsafe_dubcolon + ~colon_eq + unsafe_colon - lt_colon = Literal("<:") - semicolon = Literal(";") | invalid_syntax("\u037e", "invalid Greek question mark instead of semicolon", greedy=True) - multisemicolon = combine(OneOrMore(semicolon)) - eq = Literal("==") - equals = ~eq + ~Literal("=>") + Literal("=") - lbrack = Literal("[") - rbrack = Literal("]") - lbrace = Literal("{") - rbrace = Literal("}") - lbanana = ~Literal("(|)") + ~Literal("(|>") + ~Literal("(|*") + ~Literal("(|?") + Literal("(|") - rbanana = Literal("|)") - lparen = ~lbanana + Literal("(") - rparen = Literal(")") - unsafe_dot = Literal(".") - dot = ~Literal("..") + unsafe_dot - plus = Literal("+") - minus = ~Literal("->") + Literal("-") - dubslash = Literal("//") - slash = ~dubslash + Literal("/") - pipe = Literal("|>") | fixto(Literal("\u21a6"), "|>") - star_pipe = Literal("|*>") | fixto(Literal("*\u21a6"), "|*>") - dubstar_pipe = Literal("|**>") | fixto(Literal("**\u21a6"), "|**>") - back_pipe = Literal("<|") | fixto(Literal("\u21a4"), "<|") - back_star_pipe = Literal("<*|") | ~Literal("\u21a4**") + fixto(Literal("\u21a4*"), "<*|") - back_dubstar_pipe = Literal("<**|") | fixto(Literal("\u21a4**"), "<**|") - none_pipe = Literal("|?>") | fixto(Literal("?\u21a6"), "|?>") - none_star_pipe = ( - Literal("|?*>") - | fixto(Literal("?*\u21a6"), "|?*>") - | invalid_syntax("|*?>", "Coconut's None-aware forward multi-arg pipe is '|?*>', not '|*?>'") - ) - none_dubstar_pipe = ( - Literal("|?**>") - | fixto(Literal("?**\u21a6"), "|?**>") - | invalid_syntax("|**?>", "Coconut's None-aware forward keyword pipe is '|?**>', not '|**?>'") - ) - back_none_pipe = Literal("") + ~Literal("..*") + ~Literal("..?") + Literal("..") - | ~Literal("\u2218>") + ~Literal("\u2218*") + ~Literal("\u2218?") + fixto(Literal("\u2218"), "..") - ) - comp_pipe = Literal("..>") | fixto(Literal("\u2218>"), "..>") - comp_back_pipe = Literal("<..") | fixto(Literal("<\u2218"), "<..") - comp_star_pipe = Literal("..*>") | fixto(Literal("\u2218*>"), "..*>") - comp_back_star_pipe = Literal("<*..") | fixto(Literal("<*\u2218"), "<*..") - comp_dubstar_pipe = Literal("..**>") | fixto(Literal("\u2218**>"), "..**>") - comp_back_dubstar_pipe = Literal("<**..") | fixto(Literal("<**\u2218"), "<**..") - comp_none_pipe = Literal("..?>") | fixto(Literal("\u2218?>"), "..?>") - comp_back_none_pipe = Literal("") - | fixto(Literal("\u2218?*>"), "..?*>") - | invalid_syntax("..*?>", "Coconut's None-aware forward multi-arg composition pipe is '..?*>', not '..*?>'") - ) - comp_back_none_star_pipe = ( - Literal("<*?..") - | fixto(Literal("<*?\u2218"), "<*?..") - | invalid_syntax("") - | fixto(Literal("\u2218?**>"), "..?**>") - | invalid_syntax("..**?>", "Coconut's None-aware forward keyword composition pipe is '..?**>', not '..**?>'") - ) - comp_back_none_dubstar_pipe = ( - Literal("<**?..") - | fixto(Literal("<**?\u2218"), "<**?..") - | invalid_syntax("") + ~Literal("|*") + Literal("|") | fixto(Literal("\u222a"), "|") - bar = ~rbanana + unsafe_bar | invalid_syntax("\xa6", "invalid broken bar character", greedy=True) - percent = Literal("%") - dollar = Literal("$") - lshift = Literal("<<") | fixto(Literal("\xab"), "<<") - rshift = Literal(">>") | fixto(Literal("\xbb"), ">>") - tilde = Literal("~") - underscore = Literal("_") - pound = Literal("#") - unsafe_backtick = Literal("`") - dubbackslash = Literal("\\\\") - backslash = ~dubbackslash + Literal("\\") - dubquestion = Literal("??") - questionmark = ~dubquestion + Literal("?") - bang = ~Literal("!=") + Literal("!") - - kwds = keydefaultdict(partial(base_keyword, explicit_prefix=colon)) - keyword = kwds.__getitem__ - - except_star_kwd = combine(keyword("except") + star) - kwds["except"] = ~except_star_kwd + keyword("except") - kwds["lambda"] = keyword("lambda") | fixto(keyword("\u03bb"), "lambda") - kwds["operator"] = base_keyword("operator", explicit_prefix=colon, require_whitespace=True) - - ellipsis = Forward() - ellipsis_tokens = Literal("...") | fixto(Literal("\u2026"), "...") - - lt = ( - ~Literal("<<") - + ~Literal("<=") - + ~Literal("<|") - + ~Literal("<..") - + ~Literal("<*") - + ~Literal("<:") - + Literal("<") - | fixto(Literal("\u228a"), "<") - ) - gt = ( - ~Literal(">>") - + ~Literal(">=") - + Literal(">") - | fixto(Literal("\u228b"), ">") - ) - le = Literal("<=") | fixto(Literal("\u2264") | Literal("\u2286"), "<=") - ge = Literal(">=") | fixto(Literal("\u2265") | Literal("\u2287"), ">=") - ne = Literal("!=") | fixto(Literal("\xac=") | Literal("\u2260"), "!=") - - mul_star = star | fixto(Literal("\xd7"), "*") - exp_dubstar = dubstar | fixto(Literal("\u2191"), "**") - neg_minus = ( - minus - | fixto(Literal("\u207b"), "-") - ) - sub_minus = ( - minus - | invalid_syntax("\u207b", "U+207b is only for negation, not subtraction") - ) - div_slash = slash | fixto(Literal("\xf7") + ~slash, "/") - div_dubslash = dubslash | fixto(combine(Literal("\xf7") + slash), "//") - matrix_at = at - - test = Forward() - test_no_chain, dubcolon = disable_inside(test, unsafe_dubcolon) - test_no_infix, backtick = disable_inside(test, unsafe_backtick) - - base_name_regex = r"" - for no_kwd in keyword_vars + const_vars: - base_name_regex += r"(?!" + no_kwd + r"\b)" - # we disallow ['"{] after to not match the "b" in b"" or the "s" in s{} - base_name_regex += r"(?![0-9])\w+\b(?![{" + strwrapper + r"])" - base_name = regex_item(base_name_regex) - - refname = Forward() - setname = Forward() - classname = Forward() - name_ref = combine(Optional(backslash) + base_name) - unsafe_name = combine(Optional(backslash.suppress()) + base_name) - - # use unsafe_name for dotted components since name should only be used for base names - dotted_refname = condense(refname + ZeroOrMore(dot + unsafe_name)) - dotted_setname = condense(setname + ZeroOrMore(dot + unsafe_name)) - unsafe_dotted_name = condense(unsafe_name + ZeroOrMore(dot + unsafe_name)) - must_be_dotted_name = condense(refname + OneOrMore(dot + unsafe_name)) - - integer = combine(Word(nums) + ZeroOrMore(underscore.suppress() + Word(nums))) - binint = combine(Word("01") + ZeroOrMore(underscore.suppress() + Word("01"))) - octint = combine(Word("01234567") + ZeroOrMore(underscore.suppress() + Word("01234567"))) - hexint = combine(Word(hexnums) + ZeroOrMore(underscore.suppress() + Word(hexnums))) - - imag_j = caseless_literal("j") | fixto(caseless_literal("i", suppress=True), "j") - basenum = combine( - integer + dot + Optional(integer) - | Optional(integer) + dot + integer - ) | integer - sci_e = combine((caseless_literal("e") | fixto(Literal("\u23e8"), "e")) + Optional(plus | neg_minus)) - numitem = ~(Literal("0") + Word(nums + "_", exact=1)) + combine(basenum + Optional(sci_e + integer)) - imag_num = combine(numitem + imag_j) - bin_num = combine(caseless_literal("0b") + Optional(underscore.suppress()) + binint) - oct_num = combine(caseless_literal("0o") + Optional(underscore.suppress()) + octint) - hex_num = combine(caseless_literal("0x") + Optional(underscore.suppress()) + hexint) - number = ( - bin_num - | oct_num - | hex_num - | imag_num - | numitem - ) - # make sure that this gets addspaced not condensed so it doesn't produce a SyntaxError - num_atom = addspace(number + Optional(condense(dot + unsafe_name))) - - moduledoc_item = Forward() - unwrap = Literal(unwrapper) - comment = Forward() - comment_tokens = combine(pound + integer + unwrap) - string_item = ( - combine(Literal(strwrapper) + integer + unwrap) - | invalid_syntax(("\u201c", "\u201d", "\u2018", "\u2019"), "invalid unicode quotation mark; strings must use \" or '", greedy=True) - ) - - xonsh_command = Forward() - passthrough_item = combine((backslash | Literal(early_passthrough_wrapper)) + integer + unwrap) | xonsh_command - passthrough_block = combine(fixto(dubbackslash, "\\") + integer + unwrap) - - endline = Forward() - endline_ref = condense(OneOrMore(Literal("\n"))) - lineitem = ZeroOrMore(comment) + endline - newline = condense(OneOrMore(lineitem)) - # rparen handles simple stmts ending parenthesized stmt lambdas - end_simple_stmt_item = FollowedBy(semicolon | newline | rparen) - - start_marker = StringStart() - moduledoc_marker = condense(ZeroOrMore(lineitem) - Optional(moduledoc_item)) - end_marker = StringEnd() - indent = Literal(openindent) - dedent = Literal(closeindent) - - u_string = Forward() - f_string = Forward() - - bit_b = caseless_literal("b") - raw_r = caseless_literal("r") - unicode_u = caseless_literal("u", suppress=True) - format_f = caseless_literal("f", suppress=True) - - string = combine(Optional(raw_r) + string_item) - # Python 2 only supports br"..." not rb"..." - b_string = combine((bit_b + Optional(raw_r) | fixto(raw_r + bit_b, "br")) + string_item) - # ur"..."/ru"..." strings are not suppored in Python 3 - u_string_ref = combine(unicode_u + string_item) - f_string_tokens = combine((format_f + Optional(raw_r) | raw_r + format_f) + string_item) - nonbf_string = string | u_string - nonb_string = nonbf_string | f_string - any_string = nonb_string | b_string - moduledoc = any_string + newline - docstring = condense(moduledoc) - - pipe_augassign = ( - combine(pipe + equals) - | combine(star_pipe + equals) - | combine(dubstar_pipe + equals) - | combine(back_pipe + equals) - | combine(back_star_pipe + equals) - | combine(back_dubstar_pipe + equals) - | combine(none_pipe + equals) - | combine(none_star_pipe + equals) - | combine(none_dubstar_pipe + equals) - | combine(back_none_pipe + equals) - | combine(back_none_star_pipe + equals) - | combine(back_none_dubstar_pipe + equals) - ) - augassign = ( - pipe_augassign - | combine(comp_pipe + equals) - | combine(dotdot + equals) - | combine(comp_back_pipe + equals) - | combine(comp_star_pipe + equals) - | combine(comp_back_star_pipe + equals) - | combine(comp_dubstar_pipe + equals) - | combine(comp_back_dubstar_pipe + equals) - | combine(comp_none_pipe + equals) - | combine(comp_back_none_pipe + equals) - | combine(comp_none_star_pipe + equals) - | combine(comp_back_none_star_pipe + equals) - | combine(comp_none_dubstar_pipe + equals) - | combine(comp_back_none_dubstar_pipe + equals) - | combine(unsafe_dubcolon + equals) - | combine(div_dubslash + equals) - | combine(div_slash + equals) - | combine(exp_dubstar + equals) - | combine(mul_star + equals) - | combine(plus + equals) - | combine(sub_minus + equals) - | combine(percent + equals) - | combine(amp + equals) - | combine(bar + equals) - | combine(caret + equals) - | combine(lshift + equals) - | combine(rshift + equals) - | combine(matrix_at + equals) - | combine(dubquestion + equals) - ) + xonsh_command = Forward() + passthrough_item = combine((Literal(early_passthrough_wrapper) | backslash) + integer + unwrap) | xonsh_command + passthrough_block = combine(fixto(dubbackslash, "\\") + integer + unwrap) + + endline = Forward() + endline_ref = condense(OneOrMore(Literal("\n"))) + lineitem = ZeroOrMore(comment) + endline + newline = condense(OneOrMore(lineitem)) + # rparen handles simple stmts ending parenthesized stmt lambdas + end_simple_stmt_item = FollowedBy(newline | semicolon | rparen) + + start_marker = StringStart() + moduledoc_marker = condense(ZeroOrMore(lineitem) - Optional(moduledoc_item)) + end_marker = StringEnd() + indent = Literal(openindent) + dedent = Literal(closeindent) + + u_string = Forward() + f_string = Forward() + + bit_b = caseless_literal("b") + raw_r = caseless_literal("r") + unicode_u = caseless_literal("u", suppress=True) + format_f = caseless_literal("f", suppress=True) + + string = combine(Optional(raw_r) + string_item) + # Python 2 only supports br"..." not rb"..." + b_string = combine((bit_b + Optional(raw_r) | fixto(raw_r + bit_b, "br")) + string_item) + # ur"..."/ru"..." strings are not suppored in Python 3 + u_string_ref = combine(unicode_u + string_item) + f_string_tokens = combine((format_f + Optional(raw_r) | raw_r + format_f) + string_item) + nonbf_string = string | u_string + nonb_string = nonbf_string | f_string + any_string = nonb_string | b_string + moduledoc = any_string + newline + docstring = condense(moduledoc) + + pipe_augassign = any_of( + combine(pipe + equals), + combine(star_pipe + equals), + combine(dubstar_pipe + equals), + combine(back_pipe + equals), + combine(back_star_pipe + equals), + combine(back_dubstar_pipe + equals), + combine(none_pipe + equals), + combine(none_star_pipe + equals), + combine(none_dubstar_pipe + equals), + combine(back_none_pipe + equals), + combine(back_none_star_pipe + equals), + combine(back_none_dubstar_pipe + equals), + use_adaptive=False, + ) + augassign = any_of( + combine(plus + equals), + combine(sub_minus + equals), + combine(bar + equals), + combine(amp + equals), + combine(mul_star + equals), + combine(div_slash + equals), + combine(div_dubslash + equals), + combine(percent + equals), + combine(lshift + equals), + combine(rshift + equals), + combine(matrix_at + equals), + combine(exp_dubstar + equals), + combine(caret + equals), + combine(dubquestion + equals), + combine(comp_pipe + equals), + combine(comp_back_pipe + equals), + combine(comp_star_pipe + equals), + combine(comp_back_star_pipe + equals), + combine(comp_dubstar_pipe + equals), + combine(comp_back_dubstar_pipe + equals), + combine(comp_none_pipe + equals), + combine(comp_back_none_pipe + equals), + combine(comp_none_star_pipe + equals), + combine(comp_back_none_star_pipe + equals), + combine(comp_none_dubstar_pipe + equals), + combine(comp_back_none_dubstar_pipe + equals), + combine(unsafe_dubcolon + equals), + combine(dotdot + equals), + pipe_augassign, + use_adaptive=False, + ) - comp_op = ( - le | ge | ne | lt | gt | eq - | addspace(keyword("not") + keyword("in")) - | keyword("in") - | addspace(keyword("is") + keyword("not")) - | keyword("is") - ) + comp_op = ( + eq + | ne + | keyword("in") + | lt + | gt + | le + | ge + | addspace(keyword("not") + keyword("in")) + # is not must come before is + | addspace(keyword("is") + keyword("not")) + | keyword("is") + ) - atom_item = Forward() - expr = Forward() - star_expr = Forward() - dubstar_expr = Forward() - comp_for = Forward() - test_no_cond = Forward() - infix_op = Forward() - namedexpr_test = Forward() - # for namedexpr locations only supported in Python 3.10 - new_namedexpr_test = Forward() - lambdef = Forward() - - typedef = Forward() - typedef_default = Forward() - unsafe_typedef_default = Forward() - typedef_test = Forward() - typedef_tuple = Forward() - typedef_ellipsis = Forward() - typedef_op_item = Forward() - - negable_atom_item = condense(Optional(neg_minus) + atom_item) - - testlist = itemlist(test, comma, suppress_trailing=False) - testlist_has_comma = addspace(OneOrMore(condense(test + comma)) + Optional(test)) - new_namedexpr_testlist_has_comma = addspace(OneOrMore(condense(new_namedexpr_test + comma)) + Optional(test)) - - testlist_star_expr = Forward() - testlist_star_expr_ref = tokenlist(Group(test) | star_expr, comma, suppress=False) - testlist_star_namedexpr = Forward() - testlist_star_namedexpr_tokens = tokenlist(Group(namedexpr_test) | star_expr, comma, suppress=False) - # for testlist_star_expr locations only supported in Python 3.9 - new_testlist_star_expr = Forward() - new_testlist_star_expr_ref = testlist_star_expr - - yield_from = Forward() - dict_comp = Forward() - dict_literal = Forward() - yield_classic = addspace(keyword("yield") + Optional(new_testlist_star_expr)) - yield_from_ref = keyword("yield").suppress() + keyword("from").suppress() + test - yield_expr = yield_from | yield_classic - dict_comp_ref = lbrace.suppress() + ( - test + colon.suppress() + test + comp_for - | invalid_syntax(dubstar_expr + comp_for, "dict unpacking cannot be used in dict comprehension") - ) + rbrace.suppress() - dict_literal_ref = ( - lbrace.suppress() - + Optional(tokenlist( - Group(test + colon + test) - | dubstar_expr, - comma, - )) - + rbrace.suppress() - ) - test_expr = yield_expr | testlist_star_expr - - base_op_item = ( - # must go dubstar then star then no star - fixto(dubstar_pipe, "_coconut_dubstar_pipe") - | fixto(back_dubstar_pipe, "_coconut_back_dubstar_pipe") - | fixto(none_dubstar_pipe, "_coconut_none_dubstar_pipe") - | fixto(back_none_dubstar_pipe, "_coconut_back_none_dubstar_pipe") - | fixto(star_pipe, "_coconut_star_pipe") - | fixto(back_star_pipe, "_coconut_back_star_pipe") - | fixto(none_star_pipe, "_coconut_none_star_pipe") - | fixto(back_none_star_pipe, "_coconut_back_none_star_pipe") - | fixto(pipe, "_coconut_pipe") - | fixto(back_pipe, "_coconut_back_pipe") - | fixto(none_pipe, "_coconut_none_pipe") - | fixto(back_none_pipe, "_coconut_back_none_pipe") - - # must go dubstar then star then no star - | fixto(comp_dubstar_pipe, "_coconut_forward_dubstar_compose") - | fixto(comp_back_dubstar_pipe, "_coconut_back_dubstar_compose") - | fixto(comp_none_dubstar_pipe, "_coconut_forward_none_dubstar_compose") - | fixto(comp_back_none_dubstar_pipe, "_coconut_back_none_dubstar_compose") - | fixto(comp_star_pipe, "_coconut_forward_star_compose") - | fixto(comp_back_star_pipe, "_coconut_back_star_compose") - | fixto(comp_none_star_pipe, "_coconut_forward_none_star_compose") - | fixto(comp_back_none_star_pipe, "_coconut_back_none_star_compose") - | fixto(comp_pipe, "_coconut_forward_compose") - | fixto(dotdot | comp_back_pipe, "_coconut_back_compose") - | fixto(comp_none_pipe, "_coconut_forward_none_compose") - | fixto(comp_back_none_pipe, "_coconut_back_none_compose") - - # neg_minus must come after minus - | fixto(minus, "_coconut_minus") - | fixto(neg_minus, "_coconut.operator.neg") - - | fixto(keyword("assert"), "_coconut_assert") - | fixto(keyword("raise"), "_coconut_raise") - | fixto(keyword("and"), "_coconut_bool_and") - | fixto(keyword("or"), "_coconut_bool_or") - | fixto(comma, "_coconut_comma_op") - | fixto(dubquestion, "_coconut_none_coalesce") - | fixto(dot, "_coconut.getattr") - | fixto(unsafe_dubcolon, "_coconut.itertools.chain") - | fixto(dollar, "_coconut.functools.partial") - | fixto(exp_dubstar, "_coconut.operator.pow") - | fixto(mul_star, "_coconut.operator.mul") - | fixto(div_dubslash, "_coconut.operator.floordiv") - | fixto(div_slash, "_coconut.operator.truediv") - | fixto(percent, "_coconut.operator.mod") - | fixto(plus, "_coconut.operator.add") - | fixto(amp, "_coconut.operator.and_") - | fixto(caret, "_coconut.operator.xor") - | fixto(unsafe_bar, "_coconut.operator.or_") - | fixto(lshift, "_coconut.operator.lshift") - | fixto(rshift, "_coconut.operator.rshift") - | fixto(lt, "_coconut.operator.lt") - | fixto(gt, "_coconut.operator.gt") - | fixto(eq, "_coconut.operator.eq") - | fixto(le, "_coconut.operator.le") - | fixto(ge, "_coconut.operator.ge") - | fixto(ne, "_coconut.operator.ne") - | fixto(tilde, "_coconut.operator.inv") - | fixto(matrix_at, "_coconut_matmul") - | fixto(keyword("is") + keyword("not"), "_coconut.operator.is_not") - | fixto(keyword("not") + keyword("in"), "_coconut_not_in") - - # must come after is not / not in - | fixto(keyword("not"), "_coconut.operator.not_") - | fixto(keyword("is"), "_coconut.operator.is_") - | fixto(keyword("in"), "_coconut_in") - ) - partialable_op = base_op_item | infix_op - partial_op_item_tokens = ( - labeled_group(dot.suppress() + partialable_op + test_no_infix, "right partial") - | labeled_group(test_no_infix + partialable_op + dot.suppress(), "left partial") - ) - partial_op_item = attach(partial_op_item_tokens, partial_op_item_handle) - op_item = ( - typedef_op_item - | partial_op_item - | base_op_item - ) + atom_item = Forward() + expr = Forward() + star_expr = Forward() + dubstar_expr = Forward() + comp_for = Forward() + test_no_cond = Forward() + infix_op = Forward() + namedexpr_test = Forward() + # for namedexpr locations only supported in Python 3.10 + new_namedexpr_test = Forward() + lambdef = Forward() + + typedef = Forward() + typedef_default = Forward() + unsafe_typedef_default = Forward() + typedef_test = Forward() + typedef_tuple = Forward() + typedef_ellipsis = Forward() + typedef_op_item = Forward() + + negable_atom_item = condense(Optional(neg_minus) + atom_item) + + testlist = itemlist(test, comma, suppress_trailing=False) + testlist_has_comma = addspace(OneOrMore(condense(test + comma)) + Optional(test)) + new_namedexpr_testlist_has_comma = addspace(OneOrMore(condense(new_namedexpr_test + comma)) + Optional(test)) + + testlist_star_expr = Forward() + testlist_star_expr_ref = tokenlist(Group(test) | star_expr, comma, suppress=False) + testlist_star_namedexpr = Forward() + testlist_star_namedexpr_tokens = tokenlist(Group(namedexpr_test) | star_expr, comma, suppress=False) + # for testlist_star_expr locations only supported in Python 3.9 + new_testlist_star_expr = Forward() + new_testlist_star_expr_ref = testlist_star_expr + + yield_from = Forward() + dict_comp = Forward() + dict_literal = Forward() + yield_classic = addspace(keyword("yield") + Optional(new_testlist_star_expr)) + yield_from_ref = keyword("yield").suppress() + keyword("from").suppress() + test + yield_expr = ( + # yield_from must come first + yield_from + | yield_classic + ) + dict_comp_ref = lbrace.suppress() + ( + test + colon.suppress() + test + comp_for + | invalid_syntax(dubstar_expr + comp_for, "dict unpacking cannot be used in dict comprehension") + ) + rbrace.suppress() + dict_literal_ref = ( + lbrace.suppress() + + Optional(tokenlist( + Group(test + colon + test) + | dubstar_expr, + comma, + )) + + rbrace.suppress() + ) + test_expr = testlist_star_expr | yield_expr + + base_op_item = ( + # pipes must come first, and must go dubstar then star then no star + fixto(dubstar_pipe, "_coconut_dubstar_pipe") + | fixto(back_dubstar_pipe, "_coconut_back_dubstar_pipe") + | fixto(none_dubstar_pipe, "_coconut_none_dubstar_pipe") + | fixto(back_none_dubstar_pipe, "_coconut_back_none_dubstar_pipe") + | fixto(star_pipe, "_coconut_star_pipe") + | fixto(back_star_pipe, "_coconut_back_star_pipe") + | fixto(none_star_pipe, "_coconut_none_star_pipe") + | fixto(back_none_star_pipe, "_coconut_back_none_star_pipe") + | fixto(pipe, "_coconut_pipe") + | fixto(back_pipe, "_coconut_back_pipe") + | fixto(none_pipe, "_coconut_none_pipe") + | fixto(back_none_pipe, "_coconut_back_none_pipe") + + # comp pipes must come early, and must go dubstar then star then no star + | fixto(comp_dubstar_pipe, "_coconut_forward_dubstar_compose") + | fixto(comp_back_dubstar_pipe, "_coconut_back_dubstar_compose") + | fixto(comp_none_dubstar_pipe, "_coconut_forward_none_dubstar_compose") + | fixto(comp_back_none_dubstar_pipe, "_coconut_back_none_dubstar_compose") + | fixto(comp_star_pipe, "_coconut_forward_star_compose") + | fixto(comp_back_star_pipe, "_coconut_back_star_compose") + | fixto(comp_none_star_pipe, "_coconut_forward_none_star_compose") + | fixto(comp_back_none_star_pipe, "_coconut_back_none_star_compose") + | fixto(comp_pipe, "_coconut_forward_compose") + | fixto(comp_none_pipe, "_coconut_forward_none_compose") + | fixto(comp_back_none_pipe, "_coconut_back_none_compose") + # dotdot must come last + | fixto(dotdot | comp_back_pipe, "_coconut_back_compose") + + | fixto(plus, "_coconut.operator.add") + | fixto(mul_star, "_coconut.operator.mul") + | fixto(div_slash, "_coconut.operator.truediv") + | fixto(unsafe_bar, "_coconut.operator.or_") + | fixto(amp, "_coconut.operator.and_") + | fixto(lshift, "_coconut.operator.lshift") + | fixto(rshift, "_coconut.operator.rshift") + | fixto(lt, "_coconut.operator.lt") + | fixto(gt, "_coconut.operator.gt") + | fixto(eq, "_coconut.operator.eq") + | fixto(le, "_coconut.operator.le") + | fixto(ge, "_coconut.operator.ge") + | fixto(ne, "_coconut.operator.ne") + | fixto(matrix_at, "_coconut_matmul") + | fixto(div_dubslash, "_coconut.operator.floordiv") + | fixto(caret, "_coconut.operator.xor") + | fixto(percent, "_coconut.operator.mod") + | fixto(exp_dubstar, "_coconut.operator.pow") + | fixto(tilde, "_coconut.operator.inv") + | fixto(dot, "_coconut.getattr") + | fixto(comma, "_coconut_comma_op") + | fixto(keyword("and"), "_coconut_bool_and") + | fixto(keyword("or"), "_coconut_bool_or") + | fixto(dubquestion, "_coconut_none_coalesce") + | fixto(unsafe_dubcolon, "_coconut.itertools.chain") + | fixto(dollar, "_coconut_partial") + | fixto(keyword("assert"), "_coconut_assert") + | fixto(keyword("raise"), "_coconut_raise") + | fixto(keyword("is") + keyword("not"), "_coconut.operator.is_not") + | fixto(keyword("not") + keyword("in"), "_coconut_not_in") + + # neg_minus must come after minus + | fixto(minus, "_coconut_minus") + | fixto(neg_minus, "_coconut.operator.neg") + + # must come after is not / not in + | fixto(keyword("not"), "_coconut.operator.not_") + | fixto(keyword("is"), "_coconut.operator.is_") + | fixto(keyword("in"), "_coconut_in") + ) + partialable_op = base_op_item | infix_op + partial_op_item_tokens = ( + labeled_group(dot.suppress() + partialable_op + test_no_infix, "right partial") + | labeled_group(test_no_infix + partialable_op + dot.suppress(), "left partial") + ) + partial_op_item = attach(partial_op_item_tokens, partial_op_item_handle) + op_item = ( + # partial_op_item must come first, then typedef_op_item must come after base_op_item + partial_op_item + | typedef_op_item + | base_op_item + ) - partial_op_atom_tokens = lparen.suppress() + partial_op_item_tokens + rparen.suppress() - - # we include (var)arg_comma to ensure the pattern matches the whole arg - arg_comma = comma | fixto(FollowedBy(rparen), "") - setarg_comma = arg_comma | fixto(FollowedBy(colon), "") - typedef_ref = setname + colon.suppress() + typedef_test + arg_comma - default = condense(equals + test) - unsafe_typedef_default_ref = setname + colon.suppress() + typedef_test + Optional(default) - typedef_default_ref = unsafe_typedef_default_ref + arg_comma - tfpdef = typedef | condense(setname + arg_comma) - tfpdef_default = typedef_default | condense(setname + Optional(default) + arg_comma) - - star_sep_arg = Forward() - star_sep_arg_ref = condense(star + arg_comma) - star_sep_setarg = Forward() - star_sep_setarg_ref = condense(star + setarg_comma) - - slash_sep_arg = Forward() - slash_sep_arg_ref = condense(slash + arg_comma) - slash_sep_setarg = Forward() - slash_sep_setarg_ref = condense(slash + setarg_comma) - - just_star = star + rparen - just_slash = slash + rparen - just_op = just_star | just_slash - - match = Forward() - args_list = ( - ~just_op - + addspace( - ZeroOrMore( - condense( - # everything here must end with arg_comma - (star | dubstar) + tfpdef - | star_sep_arg - | slash_sep_arg - | tfpdef_default + partial_op_atom_tokens = lparen.suppress() + partial_op_item_tokens + rparen.suppress() + + # we include (var)arg_comma to ensure the pattern matches the whole arg + arg_comma = comma | fixto(FollowedBy(rparen), "") + setarg_comma = arg_comma | fixto(FollowedBy(colon), "") + typedef_ref = setname + colon.suppress() + typedef_test + arg_comma + default = condense(equals + test) + unsafe_typedef_default_ref = setname + colon.suppress() + typedef_test + Optional(default) + typedef_default_ref = unsafe_typedef_default_ref + arg_comma + tfpdef = condense(setname + arg_comma) | typedef + tfpdef_default = condense(setname + Optional(default) + arg_comma) | typedef_default + + star_sep_arg = Forward() + star_sep_arg_ref = condense(star + arg_comma) + star_sep_setarg = Forward() + star_sep_setarg_ref = condense(star + setarg_comma) + + slash_sep_arg = Forward() + slash_sep_arg_ref = condense(slash + arg_comma) + slash_sep_setarg = Forward() + slash_sep_setarg_ref = condense(slash + setarg_comma) + + just_star = star + rparen + just_slash = slash + rparen + just_op = just_star | just_slash + + match = Forward() + args_list = ( + ~just_op + + addspace( + ZeroOrMore( + condense( + # everything here must end with arg_comma + tfpdef_default + | (star | dubstar) + tfpdef + | star_sep_arg + | slash_sep_arg + ) ) ) ) - ) - parameters = condense(lparen + args_list + rparen) - set_args_list = ( - ~just_op - + addspace( - ZeroOrMore( - condense( - # everything here must end with setarg_comma - (star | dubstar) + setname + setarg_comma - | star_sep_setarg - | slash_sep_setarg - | setname + Optional(default) + setarg_comma + parameters = condense(lparen + args_list + rparen) + set_args_list = ( + ~just_op + + addspace( + ZeroOrMore( + condense( + # everything here must end with setarg_comma + setname + Optional(default) + setarg_comma + | (star | dubstar) + setname + setarg_comma + | star_sep_setarg + | slash_sep_setarg + ) ) ) ) - ) - match_args_list = Group(Optional( - tokenlist( - Group( - (star | dubstar) + match - | star # not star_sep because pattern-matching can handle star separators on any Python version - | slash # not slash_sep as above - | match + Optional(equals.suppress() + test) - ), - comma, + match_args_list = Group(Optional( + tokenlist( + Group( + (star | dubstar) + match + | star # not star_sep because pattern-matching can handle star separators on any Python version + | slash # not slash_sep as above + | match + Optional(equals.suppress() + test) + ), + comma, + ) + )) + + call_item = ( + unsafe_name + default + # ellipsis must come before namedexpr_test + | ellipsis_tokens + equals.suppress() + refname + | namedexpr_test + | star + test + | dubstar + test + ) + function_call_tokens = lparen.suppress() + ( + # everything here must end with rparen + rparen.suppress() + | tokenlist(Group(call_item), comma) + rparen.suppress() + | Group(attach(addspace(test + comp_for), add_parens_handle)) + rparen.suppress() + | Group(op_item) + rparen.suppress() + ) + function_call = Forward() + questionmark_call_tokens = Group( + tokenlist( + Group( + questionmark + | unsafe_name + condense(equals + questionmark) + | call_item + ), + comma, + ) + ) + methodcaller_args = ( + itemlist(condense(call_item), comma) + | op_item ) - )) - call_item = ( - dubstar + test - | star + test - | unsafe_name + default - | ellipsis_tokens + equals.suppress() + refname - | namedexpr_test - ) - function_call_tokens = lparen.suppress() + ( - # everything here must end with rparen - rparen.suppress() - | tokenlist(Group(call_item), comma) + rparen.suppress() - | Group(attach(addspace(test + comp_for), add_parens_handle)) + rparen.suppress() - | Group(op_item) + rparen.suppress() - ) - function_call = Forward() - questionmark_call_tokens = Group( - tokenlist( + subscript_star = Forward() + subscript_star_ref = star + slicetest = Optional(test_no_chain) + sliceop = condense(unsafe_colon + slicetest) + subscript = condense( + slicetest + sliceop + Optional(sliceop) + | Optional(subscript_star) + test + ) + subscriptlist = itemlist(subscript, comma, suppress_trailing=False) | new_namedexpr_test + + slicetestgroup = Optional(test_no_chain, default="") + sliceopgroup = unsafe_colon.suppress() + slicetestgroup + subscriptgroup = attach( + slicetestgroup + sliceopgroup + Optional(sliceopgroup) + | test, + subscriptgroup_handle, + ) + subscriptgrouplist = itemlist(subscriptgroup, comma) + + anon_namedtuple = Forward() + maybe_typedef = Optional(colon.suppress() + typedef_test) + anon_namedtuple_ref = tokenlist( Group( - questionmark - | unsafe_name + condense(equals + questionmark) - | call_item + unsafe_name + maybe_typedef + equals.suppress() + test + | ellipsis_tokens + maybe_typedef + equals.suppress() + refname ), comma, ) - ) - methodcaller_args = ( - itemlist(condense(call_item), comma) - | op_item - ) - - subscript_star = Forward() - subscript_star_ref = star - slicetest = Optional(test_no_chain) - sliceop = condense(unsafe_colon + slicetest) - subscript = condense( - slicetest + sliceop + Optional(sliceop) - | Optional(subscript_star) + test - ) - subscriptlist = itemlist(subscript, comma, suppress_trailing=False) | new_namedexpr_test - - slicetestgroup = Optional(test_no_chain, default="") - sliceopgroup = unsafe_colon.suppress() + slicetestgroup - subscriptgroup = attach( - slicetestgroup + sliceopgroup + Optional(sliceopgroup) - | test, - subscriptgroup_handle, - ) - subscriptgrouplist = itemlist(subscriptgroup, comma) - - anon_namedtuple = Forward() - maybe_typedef = Optional(colon.suppress() + typedef_test) - anon_namedtuple_ref = tokenlist( - Group( - unsafe_name + maybe_typedef + equals.suppress() + test - | ellipsis_tokens + maybe_typedef + equals.suppress() + refname - ), - comma, - ) - comprehension_expr = ( - addspace(namedexpr_test + comp_for) - | invalid_syntax(star_expr + comp_for, "iterable unpacking cannot be used in comprehension") - ) - paren_atom = condense( - lparen + ( + comprehension_expr = ( + addspace(namedexpr_test + comp_for) + | invalid_syntax(star_expr + comp_for, "iterable unpacking cannot be used in comprehension") + ) + paren_atom = condense(lparen + any_of( # everything here must end with rparen - rparen - | yield_expr + rparen - | comprehension_expr + rparen - | testlist_star_namedexpr + rparen - | op_item + rparen - | anon_namedtuple + rparen - ) | ( - lparen.suppress() - + typedef_tuple - + rparen.suppress() + rparen, + testlist_star_namedexpr + rparen, + comprehension_expr + rparen, + op_item + rparen, + yield_expr + rparen, + anon_namedtuple + rparen, + typedef_tuple + rparen, + )) + + list_expr = Forward() + list_expr_ref = testlist_star_namedexpr_tokens + array_literal = attach( + lbrack.suppress() + OneOrMore( + multisemicolon + | attach(comprehension_expr, add_bracks_handle) + | namedexpr_test + ~comma + | list_expr + ) + rbrack.suppress(), + array_literal_handle, + ) + list_item = ( + lbrack.suppress() + list_expr + rbrack.suppress() + | condense(lbrack + Optional(comprehension_expr) + rbrack) + # array_literal must come last + | array_literal ) - ) - list_expr = Forward() - list_expr_ref = testlist_star_namedexpr_tokens - array_literal = attach( - lbrack.suppress() + OneOrMore( - multisemicolon - | attach(comprehension_expr, add_bracks_handle) - | namedexpr_test + ~comma - | list_expr - ) + rbrack.suppress(), - array_literal_handle, - ) - list_item = ( - condense(lbrack + Optional(comprehension_expr) + rbrack) - | lbrack.suppress() + list_expr + rbrack.suppress() - | array_literal - ) + string_atom = Forward() + string_atom_ref = OneOrMore(nonb_string) | OneOrMore(b_string) + fixed_len_string_tokens = OneOrMore(nonbf_string) | OneOrMore(b_string) + f_string_atom = Forward() + f_string_atom_ref = ZeroOrMore(nonbf_string) + f_string + ZeroOrMore(nonb_string) + + keyword_atom = any_keyword_in(const_vars) + passthrough_atom = addspace(OneOrMore(passthrough_item)) + + set_literal = Forward() + set_letter_literal = Forward() + set_s = caseless_literal("s") + set_f = caseless_literal("f") + set_m = caseless_literal("m") + set_letter = set_s | set_f | set_m + setmaker = Group( + (new_namedexpr_test + FollowedBy(rbrace))("test") + | (new_namedexpr_testlist_has_comma + FollowedBy(rbrace))("list") + | addspace(new_namedexpr_test + comp_for + FollowedBy(rbrace))("comp") + | (testlist_star_namedexpr + FollowedBy(rbrace))("testlist_star_expr") + ) + set_literal_ref = lbrace.suppress() + setmaker + rbrace.suppress() + set_letter_literal_ref = combine(set_letter + lbrace.suppress()) + Optional(setmaker) + rbrace.suppress() + + lazy_items = Optional(tokenlist(test, comma)) + lazy_list = attach(lbanana.suppress() + lazy_items + rbanana.suppress(), lazy_list_handle) + + known_atom = ( + keyword_atom + | string_atom + | num_atom + | list_item + | dict_literal + | dict_comp + | set_literal + | set_letter_literal + | lazy_list + # typedef ellipsis must come before ellipsis + | typedef_ellipsis + | ellipsis + ) + atom = ( + # known_atom must come before name to properly parse string prefixes + known_atom + | refname + | paren_atom + | passthrough_atom + ) - string_atom = Forward() - string_atom_ref = OneOrMore(nonb_string) | OneOrMore(b_string) - fixed_len_string_tokens = OneOrMore(nonbf_string) | OneOrMore(b_string) - f_string_atom = Forward() - f_string_atom_ref = ZeroOrMore(nonbf_string) + f_string + ZeroOrMore(nonb_string) - - keyword_atom = any_keyword_in(const_vars) - passthrough_atom = addspace(OneOrMore(passthrough_item)) - - set_literal = Forward() - set_letter_literal = Forward() - set_s = caseless_literal("s") - set_f = caseless_literal("f") - set_m = caseless_literal("m") - set_letter = set_s | set_f | set_m - setmaker = Group( - (new_namedexpr_test + FollowedBy(rbrace))("test") - | (new_namedexpr_testlist_has_comma + FollowedBy(rbrace))("list") - | addspace(new_namedexpr_test + comp_for + FollowedBy(rbrace))("comp") - | (testlist_star_namedexpr + FollowedBy(rbrace))("testlist_star_expr") - ) - set_literal_ref = lbrace.suppress() + setmaker + rbrace.suppress() - set_letter_literal_ref = combine(set_letter + lbrace.suppress()) + Optional(setmaker) + rbrace.suppress() - - lazy_items = Optional(tokenlist(test, comma)) - lazy_list = attach(lbanana.suppress() + lazy_items + rbanana.suppress(), lazy_list_handle) - - known_atom = ( - keyword_atom - | string_atom - | num_atom - | list_item - | dict_comp - | dict_literal - | set_literal - | set_letter_literal - | lazy_list - | typedef_ellipsis - | ellipsis - ) - atom = ( - # known_atom must come before name to properly parse string prefixes - known_atom - | refname - | paren_atom - | passthrough_atom - ) + typedef_trailer = Forward() + typedef_or_expr = Forward() - typedef_trailer = Forward() - typedef_or_expr = Forward() + simple_trailer = any_of( + condense(dot + unsafe_name), + condense(lbrack + subscriptlist + rbrack), + use_adaptive=False, + ) + call_trailer = ( + function_call + | invalid_syntax(dollar + questionmark, "'?' must come before '$' in None-coalescing partial application") + | Group(dollar + ~lparen + ~lbrack) # keep $ for item_handle + ) + known_trailer = typedef_trailer | ( + Group(condense(dollar + lbrack) + subscriptgroup + rbrack.suppress()) # $[ + | Group(condense(dollar + lbrack + rbrack)) # $[] + | Group(condense(lbrack + rbrack)) # [] + | Group(questionmark) # ? + | Group(dot + ~unsafe_name + ~lbrack + ~dot) # . + ) + ~questionmark + partial_trailer = ( + Group(fixto(dollar, "$(") + function_call) # $( + | Group(fixto(dollar + lparen, "$(?") + questionmark_call_tokens) + rparen.suppress() # $(? + ) + ~questionmark + partial_trailer_tokens = Group(dollar.suppress() + function_call_tokens) + + no_call_trailer = simple_trailer | partial_trailer | known_trailer + + no_partial_complex_trailer = call_trailer | known_trailer + no_partial_trailer = simple_trailer | no_partial_complex_trailer + + complex_trailer = no_partial_complex_trailer | partial_trailer + trailer = simple_trailer | complex_trailer + + attrgetter_atom_tokens = dot.suppress() + unsafe_dotted_name + Optional( + lparen + Optional(methodcaller_args) + rparen.suppress() + ) + attrgetter_atom = attach(attrgetter_atom_tokens, attrgetter_atom_handle) + + itemgetter_atom_tokens = ( + dot.suppress() + + Optional(unsafe_dotted_name) + + Group(OneOrMore(Group( + condense(Optional(dollar) + lbrack) + + subscriptgrouplist + + rbrack.suppress() + ))) + ) + itemgetter_atom = attach(itemgetter_atom_tokens, itemgetter_handle) + + implicit_partial_atom = ( + # itemgetter must come before attrgetter + itemgetter_atom + | attrgetter_atom + | fixto(dot + lbrack + rbrack, "_coconut.operator.getitem") + | fixto(dot + dollar + lbrack + rbrack, "_coconut_iter_getitem") + ) - simple_trailer = ( - condense(dot + unsafe_name) - | condense(lbrack + subscriptlist + rbrack) - ) - call_trailer = ( - function_call - | invalid_syntax(dollar + questionmark, "'?' must come before '$' in None-coalescing partial application") - | Group(dollar + ~lparen + ~lbrack) # keep $ for item_handle - ) - known_trailer = typedef_trailer | ( - Group(condense(dollar + lbrack) + subscriptgroup + rbrack.suppress()) # $[ - | Group(condense(dollar + lbrack + rbrack)) # $[] - | Group(condense(lbrack + rbrack)) # [] - | Group(questionmark) # ? - | Group(dot + ~unsafe_name + ~lbrack + ~dot) # . - ) + ~questionmark - partial_trailer = ( - Group(fixto(dollar, "$(") + function_call) # $( - | Group(fixto(dollar + lparen, "$(?") + questionmark_call_tokens) + rparen.suppress() # $(? - ) + ~questionmark - partial_trailer_tokens = Group(dollar.suppress() + function_call_tokens) - - no_call_trailer = simple_trailer | partial_trailer | known_trailer - - no_partial_complex_trailer = call_trailer | known_trailer - no_partial_trailer = simple_trailer | no_partial_complex_trailer - - complex_trailer = no_partial_complex_trailer | partial_trailer - trailer = simple_trailer | complex_trailer - - attrgetter_atom_tokens = dot.suppress() + unsafe_dotted_name + Optional( - lparen + Optional(methodcaller_args) + rparen.suppress() - ) - attrgetter_atom = attach(attrgetter_atom_tokens, attrgetter_atom_handle) - itemgetter_atom_tokens = dot.suppress() + OneOrMore(condense(Optional(dollar) + lbrack) + subscriptgrouplist + rbrack.suppress()) - itemgetter_atom = attach(itemgetter_atom_tokens, itemgetter_handle) - implicit_partial_atom = ( - attrgetter_atom - | itemgetter_atom - | fixto(dot + lbrack + rbrack, "_coconut.operator.getitem") - | fixto(dot + dollar + lbrack + rbrack, "_coconut_iter_getitem") - ) + trailer_atom = Forward() + trailer_atom_ref = atom + ZeroOrMore(trailer) + atom_item <<= ( + trailer_atom + | implicit_partial_atom + ) - trailer_atom = Forward() - trailer_atom_ref = atom + ZeroOrMore(trailer) - atom_item <<= ( - trailer_atom - | implicit_partial_atom - ) + no_partial_trailer_atom = Forward() + no_partial_trailer_atom_ref = atom + ZeroOrMore(no_partial_trailer) + partial_atom_tokens = no_partial_trailer_atom + partial_trailer_tokens - no_partial_trailer_atom = Forward() - no_partial_trailer_atom_ref = atom + ZeroOrMore(no_partial_trailer) - partial_atom_tokens = no_partial_trailer_atom + partial_trailer_tokens + simple_assign = Forward() + simple_assign_ref = maybeparens( + lparen, + (setname | passthrough_atom) + + ZeroOrMore(ZeroOrMore(complex_trailer) + OneOrMore(simple_trailer)), + rparen, + ) + simple_assignlist = maybeparens(lparen, itemlist(simple_assign, comma, suppress_trailing=False), rparen) + + assignlist = Forward() + star_assign_item = Forward() + base_assign_item = condense( + simple_assign + | lparen + assignlist + rparen + | lbrack + assignlist + rbrack + ) + star_assign_item_ref = condense(star + base_assign_item) + assign_item = base_assign_item | star_assign_item + assignlist <<= itemlist(assign_item, comma, suppress_trailing=False) + + typed_assign_stmt = Forward() + typed_assign_stmt_ref = simple_assign + colon.suppress() + typedef_test + Optional(equals.suppress() + test_expr) + basic_stmt = addspace(ZeroOrMore(assignlist + equals) + test_expr) + + type_param = Forward() + type_param_bound_op = lt_colon | colon | le + type_var_name = stores_loc_item + setname + type_param_constraint = lparen.suppress() + Group(tokenlist(typedef_test, comma, require_sep=True)) + rparen.suppress() + type_param_ref = ( + # constraint must come before test + (type_var_name + type_param_bound_op + type_param_constraint)("TypeVar constraint") + | (type_var_name + Optional(type_param_bound_op + typedef_test))("TypeVar") + | (star.suppress() + type_var_name)("TypeVarTuple") + | (dubstar.suppress() + type_var_name)("ParamSpec") + ) + type_params = Group(lbrack.suppress() + tokenlist(type_param, comma) + rbrack.suppress()) + + type_alias_stmt = Forward() + type_alias_stmt_ref = keyword("type").suppress() + setname + Optional(type_params) + equals.suppress() + typedef_test + + await_expr = Forward() + await_expr_ref = keyword("await").suppress() + atom_item + await_item = await_expr | atom_item + + factor = Forward() + unary = neg_minus | plus | tilde + + power = condense(exp_dubstar + ZeroOrMore(unary) + await_item) + power_in_impl_call = Forward() + + impl_call_arg = condense(( + disallow_keywords(reserved_vars) + dotted_refname + | number + | keyword_atom + ) + Optional(power_in_impl_call)) + impl_call_item = condense( + disallow_keywords(reserved_vars) + + ~any_string + + ~non_decimal_num + + atom_item + + Optional(power_in_impl_call) + ) + impl_call = Forward() + # we need to disable this inside the xonsh parser + impl_call_ref = Forward() + unsafe_impl_call_ref = ( + impl_call_item + OneOrMore(impl_call_arg) + ) - simple_assign = Forward() - simple_assign_ref = maybeparens( - lparen, - (setname | passthrough_atom) - + ZeroOrMore(ZeroOrMore(complex_trailer) + OneOrMore(simple_trailer)), - rparen, - ) - simple_assignlist = maybeparens(lparen, itemlist(simple_assign, comma, suppress_trailing=False), rparen) - - assignlist = Forward() - star_assign_item = Forward() - base_assign_item = condense( - simple_assign - | lparen + assignlist + rparen - | lbrack + assignlist + rbrack - ) - star_assign_item_ref = condense(star + base_assign_item) - assign_item = star_assign_item | base_assign_item - assignlist <<= itemlist(assign_item, comma, suppress_trailing=False) - - typed_assign_stmt = Forward() - typed_assign_stmt_ref = simple_assign + colon.suppress() + typedef_test + Optional(equals.suppress() + test_expr) - basic_stmt = addspace(ZeroOrMore(assignlist + equals) + test_expr) - - type_param = Forward() - type_param_bound_op = lt_colon | colon | le - type_var_name = stores_loc_item + setname - type_param_constraint = lparen.suppress() + Group(tokenlist(typedef_test, comma, require_sep=True)) + rparen.suppress() - type_param_ref = ( - (type_var_name + type_param_bound_op + type_param_constraint)("TypeVar constraint") - | (type_var_name + Optional(type_param_bound_op + typedef_test))("TypeVar") - | (star.suppress() + type_var_name)("TypeVarTuple") - | (dubstar.suppress() + type_var_name)("ParamSpec") - ) - type_params = Group(lbrack.suppress() + tokenlist(type_param, comma) + rbrack.suppress()) - - type_alias_stmt = Forward() - type_alias_stmt_ref = keyword("type").suppress() + setname + Optional(type_params) + equals.suppress() + typedef_test - - await_expr = Forward() - await_expr_ref = keyword("await").suppress() + atom_item - await_item = await_expr | atom_item - - factor = Forward() - unary = plus | neg_minus | tilde - - power = condense(exp_dubstar + ZeroOrMore(unary) + await_item) - power_in_impl_call = Forward() - - impl_call_arg = condense(( - keyword_atom - | number - | disallow_keywords(reserved_vars) + dotted_refname - ) + Optional(power_in_impl_call)) - impl_call_item = condense( - disallow_keywords(reserved_vars) - + ~any_string - + atom_item - + Optional(power_in_impl_call) - ) - impl_call = Forward() - # we need to disable this inside the xonsh parser - impl_call_ref = Forward() - unsafe_impl_call_ref = ( - impl_call_item + OneOrMore(impl_call_arg) - ) + factor <<= condense( + ZeroOrMore(unary) + ( + impl_call + | await_item + Optional(power) + ) + ) - factor <<= condense( - ZeroOrMore(unary) + ( - impl_call - | await_item + Optional(power) + mulop = any_of( + mul_star, + div_slash, + div_dubslash, + percent, + matrix_at, + ) + addop = any_of(plus, sub_minus) + shift = any_of(lshift, rshift) + + term = Forward() + term_ref = tokenlist(factor, mulop, allow_trailing=False, suppress=False) + + # we condense all of these down, since Python handles the precedence, not Coconut + # arith_expr = exprlist(term, addop) + # shift_expr = exprlist(arith_expr, shift) + # and_expr = exprlist(shift_expr, amp) + and_expr = exprlist( + term, + any_of( + addop, + shift, + amp, + ), ) - ) - mulop = mul_star | div_slash | div_dubslash | percent | matrix_at - addop = plus | sub_minus - shift = lshift | rshift - - term = Forward() - term_ref = tokenlist(factor, mulop, allow_trailing=False, suppress=False) - - # we condense all of these down, since Python handles the precedence, not Coconut - # arith_expr = exprlist(term, addop) - # shift_expr = exprlist(arith_expr, shift) - # and_expr = exprlist(shift_expr, amp) - and_expr = exprlist( - term, - addop - | shift - | amp, - ) + protocol_intersect_expr = Forward() + protocol_intersect_expr_ref = tokenlist(and_expr, amp_colon, allow_trailing=False) - protocol_intersect_expr = Forward() - protocol_intersect_expr_ref = tokenlist(and_expr, amp_colon, allow_trailing=False) + xor_expr = exprlist(protocol_intersect_expr, caret) - xor_expr = exprlist(protocol_intersect_expr, caret) + or_expr = typedef_or_expr | exprlist(xor_expr, bar) - or_expr = typedef_or_expr | exprlist(xor_expr, bar) + chain_expr = attach(tokenlist(or_expr, dubcolon, allow_trailing=False), chain_handle) - chain_expr = attach(tokenlist(or_expr, dubcolon, allow_trailing=False), chain_handle) + compose_expr = attach( + tokenlist( + chain_expr, + dotdot + Optional(invalid_syntax(lambdef, "lambdas only allowed after composition pipe operators '..>' and '<..', not '..' (replace '..' with '<..' to fix)")), + allow_trailing=False, + ), compose_expr_handle, + ) - compose_expr = attach( - tokenlist( - chain_expr, - dotdot + Optional(invalid_syntax(lambdef, "lambdas only allowed after composition pipe operators '..>' and '<..', not '..' (replace '..' with '<..' to fix)")), - allow_trailing=False, - ), compose_expr_handle, - ) + infix_op <<= backtick.suppress() + test_no_infix + backtick.suppress() + infix_expr = Forward() + infix_item = attach( + Group(Optional(compose_expr)) + + OneOrMore( + infix_op + Group(Optional( + # lambdef must come first + lambdef | compose_expr + )) + ), + infix_handle, + ) + infix_expr <<= ( + compose_expr + ~backtick + | infix_item + ) - infix_op <<= backtick.suppress() + test_no_infix + backtick.suppress() - infix_expr = Forward() - infix_item = attach( - Group(Optional(compose_expr)) - + OneOrMore( - infix_op + Group(Optional(lambdef | compose_expr)) - ), - infix_handle, - ) - infix_expr <<= ( - compose_expr + ~backtick - | infix_item - ) + none_coalesce_expr = attach(tokenlist(infix_expr, dubquestion, allow_trailing=False), none_coalesce_handle) + + comp_pipe_op = any_of( + comp_pipe, + comp_star_pipe, + comp_back_pipe, + comp_back_star_pipe, + comp_dubstar_pipe, + comp_back_dubstar_pipe, + comp_none_dubstar_pipe, + comp_back_none_dubstar_pipe, + comp_none_star_pipe, + comp_back_none_star_pipe, + comp_none_pipe, + comp_back_none_pipe, + use_adaptive=False, + ) + comp_pipe_item = attach( + OneOrMore(none_coalesce_expr + comp_pipe_op) + ( + # lambdef must come first + lambdef | none_coalesce_expr + ), + comp_pipe_handle, + ) + comp_pipe_expr = ( + none_coalesce_expr + ~comp_pipe_op + | comp_pipe_item + ) - none_coalesce_expr = attach(tokenlist(infix_expr, dubquestion, allow_trailing=False), none_coalesce_handle) - - comp_pipe_op = ( - comp_pipe - | comp_star_pipe - | comp_back_pipe - | comp_back_star_pipe - | comp_dubstar_pipe - | comp_back_dubstar_pipe - | comp_none_dubstar_pipe - | comp_back_none_dubstar_pipe - | comp_none_star_pipe - | comp_back_none_star_pipe - | comp_none_pipe - | comp_back_none_pipe - ) - comp_pipe_item = attach( - OneOrMore(none_coalesce_expr + comp_pipe_op) + (lambdef | none_coalesce_expr), - comp_pipe_handle, - ) - comp_pipe_expr = ( - comp_pipe_item - | none_coalesce_expr - ) + pipe_op = any_of( + pipe, + star_pipe, + dubstar_pipe, + back_pipe, + back_star_pipe, + back_dubstar_pipe, + none_pipe, + none_star_pipe, + none_dubstar_pipe, + back_none_pipe, + back_none_star_pipe, + back_none_dubstar_pipe, + use_adaptive=False, + ) + pipe_item = ( + # we need the pipe_op since any of the atoms could otherwise be the start of an expression + labeled_group(keyword("await"), "await") + pipe_op + | labeled_group(partial_atom_tokens, "partial") + pipe_op + # itemgetter must come before attrgetter + | labeled_group(itemgetter_atom_tokens, "itemgetter") + pipe_op + | labeled_group(attrgetter_atom_tokens, "attrgetter") + pipe_op + | labeled_group(partial_op_atom_tokens, "op partial") + pipe_op + # expr must come at end + | labeled_group(comp_pipe_expr, "expr") + pipe_op + ) + pipe_augassign_item = ( + # should match pipe_item but with pipe_op -> end_simple_stmt_item and no expr + labeled_group(keyword("await"), "await") + end_simple_stmt_item + | labeled_group(partial_atom_tokens, "partial") + end_simple_stmt_item + | labeled_group(itemgetter_atom_tokens, "itemgetter") + end_simple_stmt_item + | labeled_group(attrgetter_atom_tokens, "attrgetter") + end_simple_stmt_item + | labeled_group(partial_op_atom_tokens, "op partial") + end_simple_stmt_item + ) + last_pipe_item = Group( + lambdef("expr") + # we need longest here because there's no following pipe_op we can use as above + | longest( + keyword("await")("await"), + itemgetter_atom_tokens("itemgetter"), + attrgetter_atom_tokens("attrgetter"), + partial_atom_tokens("partial"), + partial_op_atom_tokens("op partial"), + comp_pipe_expr("expr"), + ) + ) + normal_pipe_expr = Forward() + normal_pipe_expr_tokens = OneOrMore(pipe_item) + last_pipe_item - pipe_op = ( - pipe - | star_pipe - | dubstar_pipe - | back_pipe - | back_star_pipe - | back_dubstar_pipe - | none_pipe - | none_star_pipe - | none_dubstar_pipe - | back_none_pipe - | back_none_star_pipe - | back_none_dubstar_pipe - ) - pipe_item = ( - # we need the pipe_op since any of the atoms could otherwise be the start of an expression - labeled_group(keyword("await"), "await") + pipe_op - | labeled_group(attrgetter_atom_tokens, "attrgetter") + pipe_op - | labeled_group(itemgetter_atom_tokens, "itemgetter") + pipe_op - | labeled_group(partial_atom_tokens, "partial") + pipe_op - | labeled_group(partial_op_atom_tokens, "op partial") + pipe_op - # expr must come at end - | labeled_group(comp_pipe_expr, "expr") + pipe_op - ) - pipe_augassign_item = ( - # should match pipe_item but with pipe_op -> end_simple_stmt_item and no expr - labeled_group(keyword("await"), "await") + end_simple_stmt_item - | labeled_group(attrgetter_atom_tokens, "attrgetter") + end_simple_stmt_item - | labeled_group(itemgetter_atom_tokens, "itemgetter") + end_simple_stmt_item - | labeled_group(partial_atom_tokens, "partial") + end_simple_stmt_item - | labeled_group(partial_op_atom_tokens, "op partial") + end_simple_stmt_item - ) - last_pipe_item = Group( - lambdef("expr") - # we need longest here because there's no following pipe_op we can use as above - | longest( - keyword("await")("await"), - attrgetter_atom_tokens("attrgetter"), - itemgetter_atom_tokens("itemgetter"), - partial_atom_tokens("partial"), - partial_op_atom_tokens("op partial"), - comp_pipe_expr("expr"), + pipe_expr = ( + comp_pipe_expr + ~pipe_op + | normal_pipe_expr ) - ) - normal_pipe_expr = Forward() - normal_pipe_expr_tokens = OneOrMore(pipe_item) + last_pipe_item - pipe_expr = ( - comp_pipe_expr + ~pipe_op - | normal_pipe_expr - ) + expr <<= pipe_expr + + # though 3.9 allows tests in the grammar here, they still raise a SyntaxError later + star_expr <<= Group(star + expr) + dubstar_expr <<= Group(dubstar + expr) + + comparison = exprlist(expr, comp_op) + not_test = addspace(ZeroOrMore(keyword("not")) + comparison) + # we condense "and" and "or" into one, since Python handles the precedence, not Coconut + # and_test = exprlist(not_test, keyword("and")) + # test_item = exprlist(and_test, keyword("or")) + test_item = exprlist(not_test, keyword("and") | keyword("or")) + + simple_stmt_item = Forward() + unsafe_simple_stmt_item = Forward() + simple_stmt = Forward() + stmt = Forward() + suite = Forward() + nocolon_suite = Forward() + base_suite = Forward() + + fat_arrow = Forward() + lambda_arrow = Forward() + unsafe_lambda_arrow = any_of(fat_arrow, arrow) + + keyword_lambdef_params = maybeparens(lparen, set_args_list, rparen) + arrow_lambdef_params = lparen.suppress() + set_args_list + rparen.suppress() | setname + + keyword_lambdef = Forward() + keyword_lambdef_ref = addspace(keyword("lambda") + condense(keyword_lambdef_params + colon)) + arrow_lambdef = attach(arrow_lambdef_params + lambda_arrow.suppress(), lambdef_handle) + implicit_lambdef = fixto(lambda_arrow, "lambda _=None:") + lambdef_base = any_of( + arrow_lambdef, + implicit_lambdef, + keyword_lambdef, + ) - expr <<= pipe_expr - - # though 3.9 allows tests in the grammar here, they still raise a SyntaxError later - star_expr <<= Group(star + expr) - dubstar_expr <<= Group(dubstar + expr) - - comparison = exprlist(expr, comp_op) - not_test = addspace(ZeroOrMore(keyword("not")) + comparison) - # we condense "and" and "or" into one, since Python handles the precedence, not Coconut - # and_test = exprlist(not_test, keyword("and")) - # test_item = exprlist(and_test, keyword("or")) - test_item = exprlist(not_test, keyword("and") | keyword("or")) - - simple_stmt_item = Forward() - unsafe_simple_stmt_item = Forward() - simple_stmt = Forward() - stmt = Forward() - suite = Forward() - nocolon_suite = Forward() - base_suite = Forward() - - fat_arrow = Forward() - lambda_arrow = Forward() - unsafe_lambda_arrow = fat_arrow | arrow - - keyword_lambdef_params = maybeparens(lparen, set_args_list, rparen) - arrow_lambdef_params = lparen.suppress() + set_args_list + rparen.suppress() | setname - - keyword_lambdef = Forward() - keyword_lambdef_ref = addspace(keyword("lambda") + condense(keyword_lambdef_params + colon)) - arrow_lambdef = attach(arrow_lambdef_params + lambda_arrow.suppress(), lambdef_handle) - implicit_lambdef = fixto(lambda_arrow, "lambda _=None:") - lambdef_base = keyword_lambdef | arrow_lambdef | implicit_lambdef - - stmt_lambdef = Forward() - match_guard = Optional(keyword("if").suppress() + namedexpr_test) - closing_stmt = longest(new_testlist_star_expr("tests"), unsafe_simple_stmt_item) - stmt_lambdef_match_params = Group(lparen.suppress() + match_args_list + match_guard + rparen.suppress()) - stmt_lambdef_params = Optional( - attach(setname, add_parens_handle) - | parameters - | stmt_lambdef_match_params, - default="(_=None)", - ) - stmt_lambdef_body = Group( - Group(OneOrMore(simple_stmt_item + semicolon.suppress())) + Optional(closing_stmt) - | Group(ZeroOrMore(simple_stmt_item + semicolon.suppress())) + closing_stmt, - ) + stmt_lambdef = Forward() + match_guard = Optional(keyword("if").suppress() + namedexpr_test) + closing_stmt = longest(new_testlist_star_expr("tests"), unsafe_simple_stmt_item) + stmt_lambdef_match_params = Group(lparen.suppress() + match_args_list + match_guard + rparen.suppress()) + stmt_lambdef_params = Optional( + attach(setname, add_parens_handle) + | parameters + | stmt_lambdef_match_params, + default="(_=None)", + ) + stmt_lambdef_body = Group( + Group(OneOrMore(simple_stmt_item + semicolon.suppress())) + Optional(closing_stmt) + | Group(ZeroOrMore(simple_stmt_item + semicolon.suppress())) + closing_stmt, + ) - no_fat_arrow_stmt_lambdef_body, _fat_arrow = disable_inside(stmt_lambdef_body, unsafe_fat_arrow) - fat_arrow <<= _fat_arrow - stmt_lambdef_suite = ( - arrow.suppress() + no_fat_arrow_stmt_lambdef_body + ~fat_arrow - | Optional(arrow.suppress() + typedef_test) + fat_arrow.suppress() + stmt_lambdef_body - ) + no_fat_arrow_stmt_lambdef_body, _fat_arrow = disable_inside(stmt_lambdef_body, unsafe_fat_arrow) + fat_arrow <<= _fat_arrow + stmt_lambdef_suite = ( + arrow.suppress() + no_fat_arrow_stmt_lambdef_body + ~fat_arrow + | Optional(arrow.suppress() + typedef_test) + fat_arrow.suppress() + stmt_lambdef_body + ) - general_stmt_lambdef = ( - Group(any_len_perm( - keyword("async"), - keyword("copyclosure"), - )) + keyword("def").suppress() - + stmt_lambdef_params - + stmt_lambdef_suite - ) - match_stmt_lambdef = ( - Group(any_len_perm( - keyword("match").suppress(), - keyword("async"), - keyword("copyclosure"), - )) + keyword("def").suppress() - + stmt_lambdef_match_params - + stmt_lambdef_suite - ) - stmt_lambdef_ref = trace( - general_stmt_lambdef - | match_stmt_lambdef - ) + ( - fixto(FollowedBy(comma), ",") - | fixto(always_match, "") - ) + general_stmt_lambdef = ( + Group(any_len_perm( + keyword("async"), + keyword("copyclosure"), + )) + keyword("def").suppress() + + stmt_lambdef_params + + stmt_lambdef_suite + ) + match_stmt_lambdef = ( + Group(any_len_perm( + keyword("match").suppress(), + keyword("async"), + keyword("copyclosure"), + )) + keyword("def").suppress() + + stmt_lambdef_match_params + + stmt_lambdef_suite + ) + stmt_lambdef_ref = trace( + general_stmt_lambdef + | match_stmt_lambdef + ) + ( + fixto(FollowedBy(comma), ",") + | fixto(always_match, "") + ) - lambdef <<= addspace(lambdef_base + test) | stmt_lambdef - lambdef_no_cond = addspace(lambdef_base + test_no_cond) + lambdef <<= addspace(lambdef_base + test) | stmt_lambdef + lambdef_no_cond = addspace(lambdef_base + test_no_cond) - typedef_callable_arg = Group( - test("arg") - | (dubstar.suppress() + refname)("paramspec") - ) - typedef_callable_params = Optional(Group( - labeled_group(maybeparens(lparen, ellipsis_tokens, rparen), "ellipsis") - | lparen.suppress() + Optional(tokenlist(typedef_callable_arg, comma)) + rparen.suppress() - | labeled_group(negable_atom_item, "arg") - )) - unsafe_typedef_callable = attach( - Optional(keyword("async"), default="") - + typedef_callable_params - + arrow.suppress() - + typedef_test, - typedef_callable_handle, - ) + typedef_callable_arg = Group( + test("arg") + | (dubstar.suppress() + refname)("paramspec") + ) + typedef_callable_params = Optional(Group( + labeled_group(maybeparens(lparen, ellipsis_tokens, rparen), "ellipsis") + | lparen.suppress() + Optional(tokenlist(typedef_callable_arg, comma)) + rparen.suppress() + | labeled_group(negable_atom_item, "arg") + )) + unsafe_typedef_callable = attach( + Optional(keyword("async"), default="") + + typedef_callable_params + + arrow.suppress() + + typedef_test, + typedef_callable_handle, + ) - unsafe_typedef_trailer = ( # use special type signifier for item_handle - Group(fixto(lbrack + rbrack, "type:[]")) - | Group(fixto(dollar + lbrack + rbrack, "type:$[]")) - | Group(fixto(questionmark + ~questionmark, "type:?")) - ) + unsafe_typedef_trailer = ( # use special type signifier for item_handle + Group(fixto(lbrack + rbrack, "type:[]")) + | Group(fixto(dollar + lbrack + rbrack, "type:$[]")) + | Group(fixto(questionmark + ~questionmark, "type:?")) + ) - unsafe_typedef_or_expr = Forward() - unsafe_typedef_or_expr_ref = tokenlist(xor_expr, bar, allow_trailing=False, at_least_two=True) + unsafe_typedef_or_expr = Forward() + unsafe_typedef_or_expr_ref = tokenlist(xor_expr, bar, allow_trailing=False, at_least_two=True) - unsafe_typedef_tuple = Forward() - # should mimic testlist_star_namedexpr but with require_sep=True - unsafe_typedef_tuple_ref = tokenlist(Group(namedexpr_test) | star_expr, fixto(semicolon, ","), suppress=False, require_sep=True) + unsafe_typedef_tuple = Forward() + # should mimic testlist_star_namedexpr but with require_sep=True + unsafe_typedef_tuple_ref = tokenlist(Group(namedexpr_test) | star_expr, fixto(semicolon, ","), suppress=False, require_sep=True) - unsafe_typedef_ellipsis = ellipsis_tokens + unsafe_typedef_ellipsis = ellipsis_tokens - unsafe_typedef_op_item = attach(base_op_item, typedef_op_item_handle) + unsafe_typedef_op_item = attach(base_op_item, typedef_op_item_handle) - unsafe_typedef_test, typedef_callable, _typedef_trailer, _typedef_or_expr, _typedef_tuple, _typedef_ellipsis, _typedef_op_item = disable_outside( - test, - unsafe_typedef_callable, - unsafe_typedef_trailer, - unsafe_typedef_or_expr, - unsafe_typedef_tuple, - unsafe_typedef_ellipsis, - unsafe_typedef_op_item, - ) - typedef_trailer <<= _typedef_trailer - typedef_or_expr <<= _typedef_or_expr - typedef_tuple <<= _typedef_tuple - typedef_ellipsis <<= _typedef_ellipsis - typedef_op_item <<= _typedef_op_item - - _typedef_test, _lambda_arrow = disable_inside( - unsafe_typedef_test, - unsafe_lambda_arrow, - ) - typedef_test <<= _typedef_test - lambda_arrow <<= _lambda_arrow - - alt_ternary_expr = attach(keyword("if").suppress() + test_item + keyword("then").suppress() + test_item + keyword("else").suppress() + test, alt_ternary_handle) - test <<= ( - typedef_callable - | lambdef - | alt_ternary_expr - | addspace(test_item + Optional(keyword("if") + test_item + keyword("else") + test)) # must come last since it includes plain test_item - ) - test_no_cond <<= lambdef_no_cond | test_item + unsafe_typedef_test, typedef_callable, _typedef_trailer, _typedef_or_expr, _typedef_tuple, _typedef_ellipsis, _typedef_op_item = disable_outside( + test, + unsafe_typedef_callable, + unsafe_typedef_trailer, + unsafe_typedef_or_expr, + unsafe_typedef_tuple, + unsafe_typedef_ellipsis, + unsafe_typedef_op_item, + ) + typedef_trailer <<= _typedef_trailer + typedef_or_expr <<= _typedef_or_expr + typedef_tuple <<= _typedef_tuple + typedef_ellipsis <<= _typedef_ellipsis + typedef_op_item <<= _typedef_op_item + + _typedef_test, _lambda_arrow = disable_inside( + unsafe_typedef_test, + unsafe_lambda_arrow, + ) + typedef_test <<= _typedef_test + lambda_arrow <<= _lambda_arrow + + alt_ternary_expr = attach(keyword("if").suppress() + test_item + keyword("then").suppress() + test_item + keyword("else").suppress() + test, alt_ternary_handle) + test <<= ( + typedef_callable + | lambdef + # must come near end since it includes plain test_item + | addspace(test_item + Optional(keyword("if") + test_item + keyword("else") + test)) + | alt_ternary_expr + ) + test_no_cond <<= lambdef_no_cond | test_item - namedexpr = Forward() - namedexpr_ref = addspace( - setname + colon_eq + ( + namedexpr = Forward() + namedexpr_ref = addspace( + setname + colon_eq + ( + test + ~colon_eq + | attach(namedexpr, add_parens_handle) + ) + ) + namedexpr_test <<= ( test + ~colon_eq - | attach(namedexpr, add_parens_handle) + | namedexpr ) - ) - namedexpr_test <<= ( - test + ~colon_eq - | namedexpr - ) - - new_namedexpr = Forward() - new_namedexpr_ref = namedexpr_ref - new_namedexpr_test <<= ( - test + ~colon_eq - | new_namedexpr - ) - - classdef = Forward() - decorators = Forward() - classlist = Group( - Optional(function_call_tokens) - + ~equals, # don't match class destructuring assignment - ) - class_suite = suite | attach(newline, class_suite_handle) - classdef_ref = ( - Optional(decorators, default="") - + keyword("class").suppress() - + classname - + Optional(type_params, default=()) - + classlist - + class_suite - ) - async_comp_for = Forward() - comp_iter = Forward() - comp_it_item = ( - invalid_syntax(maybeparens(lparen, namedexpr, rparen), "PEP 572 disallows assignment expressions in comprehension iterable expressions") - | test_item - ) - base_comp_for = addspace(keyword("for") + assignlist + keyword("in") + comp_it_item + Optional(comp_iter)) - async_comp_for_ref = addspace(keyword("async") + base_comp_for) - comp_for <<= async_comp_for | base_comp_for - comp_if = addspace(keyword("if") + test_no_cond + Optional(comp_iter)) - comp_iter <<= comp_for | comp_if - - return_stmt = addspace(keyword("return") - Optional(new_testlist_star_expr)) - - complex_raise_stmt = Forward() - pass_stmt = keyword("pass") - break_stmt = keyword("break") - continue_stmt = keyword("continue") - simple_raise_stmt = addspace(keyword("raise") + Optional(test)) - complex_raise_stmt_ref = keyword("raise").suppress() + test + keyword("from").suppress() - test - raise_stmt = complex_raise_stmt | simple_raise_stmt - flow_stmt = ( - return_stmt - | raise_stmt - | break_stmt - | yield_expr - | continue_stmt - ) - - imp_name = ( - # maybeparens allows for using custom operator names here - maybeparens(lparen, setname, rparen) - | passthrough_item - ) - unsafe_imp_name = ( - # should match imp_name except with unsafe_name instead of setname - maybeparens(lparen, unsafe_name, rparen) - | passthrough_item - ) - dotted_imp_name = ( - dotted_setname - | passthrough_item - ) - unsafe_dotted_imp_name = ( - # should match dotted_imp_name except with unsafe_dotted_name - unsafe_dotted_name - | passthrough_item - ) - imp_as = keyword("as").suppress() - imp_name - import_item = Group( - unsafe_dotted_imp_name + imp_as - | dotted_imp_name - ) - from_import_item = Group( - unsafe_imp_name + imp_as - | imp_name - ) + new_namedexpr = Forward() + new_namedexpr_ref = namedexpr_ref + new_namedexpr_test <<= ( + test + ~colon_eq + | new_namedexpr + ) - import_names = Group( - maybeparens(lparen, tokenlist(import_item, comma), rparen) - | star - ) - from_import_names = Group( - maybeparens(lparen, tokenlist(from_import_item, comma), rparen) - | star - ) - basic_import = keyword("import").suppress() - import_names - import_from_name = condense( - ZeroOrMore(unsafe_dot) + unsafe_dotted_name - | OneOrMore(unsafe_dot) - | star - ) - from_import = ( - keyword("from").suppress() - - import_from_name - - keyword("import").suppress() - from_import_names - ) - import_stmt = Forward() - import_stmt_ref = from_import | basic_import + classdef = Forward() + decorators = Forward() + classlist = Group( + Optional(function_call_tokens) + + ~equals, # don't match class destructuring assignment + ) + class_suite = suite | attach(newline, class_suite_handle) + classdef_ref = ( + Optional(decorators, default="") + + keyword("class").suppress() + + classname + + Optional(type_params, default=()) + + classlist + + class_suite + ) - augassign_stmt = Forward() - augassign_rhs = ( - labeled_group(pipe_augassign + pipe_augassign_item, "pipe") - | labeled_group(augassign + test_expr, "simple") - ) - augassign_stmt_ref = simple_assign + augassign_rhs + async_comp_for = Forward() + comp_iter = Forward() + comp_it_item = ( + invalid_syntax(maybeparens(lparen, namedexpr, rparen), "PEP 572 disallows assignment expressions in comprehension iterable expressions") + | test_item + ) + base_comp_for = addspace(keyword("for") + assignlist + keyword("in") + comp_it_item + Optional(comp_iter)) + async_comp_for_ref = addspace(keyword("async") + base_comp_for) + comp_for <<= base_comp_for | async_comp_for + comp_if = addspace(keyword("if") + test_no_cond + Optional(comp_iter)) + comp_iter <<= any_of(comp_for, comp_if) + + return_stmt = addspace(keyword("return") - Optional(new_testlist_star_expr)) + + complex_raise_stmt = Forward() + pass_stmt = keyword("pass") + break_stmt = keyword("break") + continue_stmt = keyword("continue") + simple_raise_stmt = addspace(keyword("raise") + Optional(test)) + ~keyword("from") + complex_raise_stmt_ref = keyword("raise").suppress() + test + keyword("from").suppress() - test + + imp_name = ( + # maybeparens allows for using custom operator names here + maybeparens(lparen, setname, rparen) + | passthrough_item + ) + unsafe_imp_name = ( + # should match imp_name except with unsafe_name instead of setname + maybeparens(lparen, unsafe_name, rparen) + | passthrough_item + ) + dotted_imp_name = ( + dotted_setname + | passthrough_item + ) + unsafe_dotted_imp_name = ( + # should match dotted_imp_name except with unsafe_dotted_name + unsafe_dotted_name + | passthrough_item + ) + imp_as = keyword("as").suppress() - imp_name + import_item = Group( + unsafe_dotted_imp_name + imp_as + | dotted_imp_name + ) + from_import_item = Group( + unsafe_imp_name + imp_as + | imp_name + ) - simple_kwd_assign = attach( - maybeparens(lparen, itemlist(setname, comma), rparen) + Optional(equals.suppress() - test_expr), - simple_kwd_assign_handle, - ) - kwd_augassign = Forward() - kwd_augassign_ref = setname + augassign_rhs - kwd_assign = ( - kwd_augassign - | simple_kwd_assign - ) - global_stmt = addspace(keyword("global") - kwd_assign) - nonlocal_stmt = Forward() - nonlocal_stmt_ref = addspace(keyword("nonlocal") - kwd_assign) - - del_stmt = addspace(keyword("del") - simple_assignlist) - - matchlist_data_item = Group(Optional(star | Optional(dot) + unsafe_name + equals) + match) - matchlist_data = Group(Optional(tokenlist(matchlist_data_item, comma))) - - match_check_equals = Forward() - match_check_equals_ref = equals - - match_dotted_name_const = Forward() - complex_number = condense(Optional(neg_minus) + number + (plus | sub_minus) + Optional(neg_minus) + imag_num) - match_const = condense( - (eq | match_check_equals).suppress() + negable_atom_item - | string_atom - | complex_number - | Optional(neg_minus) + number - | match_dotted_name_const - ) - empty_const = fixto( - lparen + rparen - | lbrack + rbrack - | set_letter + lbrace + rbrace, - "()", - ) + import_names = Group( + maybeparens(lparen, tokenlist(import_item, comma), rparen) + | star + ) + from_import_names = Group( + maybeparens(lparen, tokenlist(from_import_item, comma), rparen) + | star + ) + basic_import = keyword("import").suppress() - import_names + import_from_name = condense( + ZeroOrMore(unsafe_dot) + unsafe_dotted_name + | OneOrMore(unsafe_dot) + | star + ) + from_import = ( + keyword("from").suppress() + - import_from_name + - keyword("import").suppress() - from_import_names + ) + import_stmt = Forward() + import_stmt_ref = from_import | basic_import + + augassign_stmt = Forward() + augassign_rhs = ( + # pipe_augassign must come first + labeled_group(pipe_augassign + pipe_augassign_item, "pipe") + | labeled_group(augassign + test_expr, "simple") + ) + augassign_stmt_ref = simple_assign + augassign_rhs - match_pair = Group(match_const + colon.suppress() + match) - matchlist_dict = Group(Optional(tokenlist(match_pair, comma))) - set_star = star.suppress() + (keyword(wildcard) | empty_const) + simple_kwd_assign = attach( + maybeparens(lparen, itemlist(setname, comma), rparen) + Optional(equals.suppress() + test_expr), + simple_kwd_assign_handle, + ) + kwd_augassign = Forward() + kwd_augassign_ref = setname + augassign_rhs + kwd_assign = ( + kwd_augassign + | simple_kwd_assign + ) + global_stmt = addspace(keyword("global") + kwd_assign) + nonlocal_stmt = Forward() + nonlocal_stmt_ref = addspace(keyword("nonlocal") + kwd_assign) + + del_stmt = addspace(keyword("del") - simple_assignlist) + + matchlist_data_item = Group(Optional(star | Optional(dot) + unsafe_name + equals) + match) + matchlist_data = Group(Optional(tokenlist(matchlist_data_item, comma))) + + match_check_equals = Forward() + match_check_equals_ref = equals + + match_dotted_name_const = Forward() + complex_number = condense(Optional(neg_minus) + number + (plus | sub_minus) + Optional(neg_minus) + imag_num) + match_const = condense( + (eq | match_check_equals).suppress() + negable_atom_item + | string_atom + | complex_number + | Optional(neg_minus) + number + | match_dotted_name_const + ) + empty_const = fixto( + lparen + rparen + | lbrack + rbrack + | set_letter + lbrace + rbrace, + "()", + ) - matchlist_tuple_items = ( - match + OneOrMore(comma.suppress() + match) + Optional(comma.suppress()) - | match + comma.suppress() - ) - matchlist_tuple = Group(Optional(matchlist_tuple_items)) - matchlist_list = Group(Optional(tokenlist(match, comma))) - match_list = Group(lbrack + matchlist_list + rbrack.suppress()) - match_tuple = Group(lparen + matchlist_tuple + rparen.suppress()) - match_lazy = Group(lbanana + matchlist_list + rbanana.suppress()) - - interior_name_match = labeled_group(setname, "var") - match_string = interleaved_tokenlist( - # f_string_atom must come first - f_string_atom("f_string") | fixed_len_string_tokens("string"), - interior_name_match("capture"), - plus, - at_least_two=True, - )("string_sequence") - sequence_match = interleaved_tokenlist( - (match_list | match_tuple)("literal"), - interior_name_match("capture"), - plus, - )("sequence") - iter_match = interleaved_tokenlist( - (match_list | match_tuple | match_lazy)("literal"), - interior_name_match("capture"), - unsafe_dubcolon, - at_least_two=True, - )("iter") - matchlist_star = interleaved_tokenlist( - star.suppress() + match("capture"), - match("elem"), - comma, - allow_trailing=True, - ) - star_match = ( - lbrack.suppress() + matchlist_star + rbrack.suppress() - | lparen.suppress() + matchlist_star + rparen.suppress() - )("star") - - base_match = Group( - (negable_atom_item + arrow.suppress() + match)("view") - | match_string - | match_const("const") - | (keyword_atom | keyword("is").suppress() + negable_atom_item)("is") - | (keyword("in").suppress() + negable_atom_item)("in") - | iter_match - | match_lazy("lazy") - | sequence_match - | star_match - | (lparen.suppress() + match + rparen.suppress())("paren") - | (lbrace.suppress() + matchlist_dict + Optional(dubstar.suppress() + (setname | condense(lbrace + rbrace)) + Optional(comma.suppress())) + rbrace.suppress())("dict") - | ( - Group(Optional(set_letter)) - + lbrace.suppress() - + ( - Group(tokenlist(match_const, comma, allow_trailing=False)) + Optional(comma.suppress() + set_star + Optional(comma.suppress())) - | Group(always_match) + set_star + Optional(comma.suppress()) - | Group(Optional(tokenlist(match_const, comma))) - ) + rbrace.suppress() - )("set") - | (keyword("data").suppress() + dotted_refname + lparen.suppress() + matchlist_data + rparen.suppress())("data") - | (keyword("class").suppress() + dotted_refname + lparen.suppress() + matchlist_data + rparen.suppress())("class") - | (dotted_refname + lparen.suppress() + matchlist_data + rparen.suppress())("data_or_class") - | Optional(keyword("as").suppress()) + setname("var") - ) + match_pair = Group(match_const + colon.suppress() + match) + matchlist_dict = Group(Optional(tokenlist(match_pair, comma))) + set_star = star.suppress() + (keyword(wildcard) | empty_const) - matchlist_isinstance = base_match + OneOrMore(keyword("is").suppress() + negable_atom_item) - isinstance_match = labeled_group(matchlist_isinstance, "isinstance_is") | base_match + matchlist_tuple_items = ( + match + OneOrMore(comma.suppress() + match) + Optional(comma.suppress()) + | match + comma.suppress() + ) + matchlist_tuple = Group(Optional(matchlist_tuple_items)) + matchlist_list = Group(Optional(tokenlist(match, comma))) + match_list = Group(lbrack + matchlist_list + rbrack.suppress()) + match_tuple = Group(lparen + matchlist_tuple + rparen.suppress()) + match_lazy = Group(lbanana + matchlist_list + rbanana.suppress()) + + interior_name_match = labeled_group(setname, "var") + match_string = interleaved_tokenlist( + # f_string_atom must come first + f_string_atom("f_string") | fixed_len_string_tokens("string"), + interior_name_match("capture"), + plus, + at_least_two=True, + )("string_sequence") + sequence_match = interleaved_tokenlist( + (match_list | match_tuple)("literal"), + interior_name_match("capture"), + plus, + )("sequence") + iter_match = interleaved_tokenlist( + (match_list | match_tuple | match_lazy)("literal"), + interior_name_match("capture"), + unsafe_dubcolon, + at_least_two=True, + )("iter") + matchlist_star = interleaved_tokenlist( + star.suppress() + match("capture"), + match("elem"), + comma, + allow_trailing=True, + ) + star_match = ( + lbrack.suppress() + matchlist_star + rbrack.suppress() + | lparen.suppress() + matchlist_star + rparen.suppress() + )("star") + + base_match = Group( + (negable_atom_item + arrow.suppress() + match)("view") + | match_string + | match_const("const") + | (keyword_atom | keyword("is").suppress() + negable_atom_item)("is") + | (keyword("in").suppress() + negable_atom_item)("in") + | iter_match + | match_lazy("lazy") + | sequence_match + | star_match + | (lparen.suppress() + match + rparen.suppress())("paren") + | (lbrace.suppress() + matchlist_dict + Optional(dubstar.suppress() + (setname | condense(lbrace + rbrace)) + Optional(comma.suppress())) + rbrace.suppress())("dict") + | ( + Group(Optional(set_letter)) + + lbrace.suppress() + + ( + Group(tokenlist(match_const, comma, allow_trailing=False)) + Optional(comma.suppress() + set_star + Optional(comma.suppress())) + | Group(always_match) + set_star + Optional(comma.suppress()) + | Group(Optional(tokenlist(match_const, comma))) + ) + rbrace.suppress() + )("set") + | (keyword("data").suppress() + dotted_refname + lparen.suppress() + matchlist_data + rparen.suppress())("data") + | (keyword("class").suppress() + dotted_refname + lparen.suppress() + matchlist_data + rparen.suppress())("class") + | (dotted_refname + lparen.suppress() + matchlist_data + rparen.suppress())("data_or_class") + | Optional(keyword("as").suppress()) + setname("var"), + ) - matchlist_bar_or = isinstance_match + OneOrMore(bar.suppress() + isinstance_match) - bar_or_match = labeled_group(matchlist_bar_or, "or") | isinstance_match + matchlist_isinstance = base_match + OneOrMore(keyword("is").suppress() + negable_atom_item) + isinstance_match = ( + labeled_group(matchlist_isinstance, "isinstance_is") + | base_match + ) - matchlist_infix = bar_or_match + OneOrMore(Group(infix_op + Optional(negable_atom_item))) - infix_match = labeled_group(matchlist_infix, "infix") | bar_or_match + matchlist_bar_or = isinstance_match + OneOrMore(bar.suppress() + isinstance_match) + bar_or_match = ( + labeled_group(matchlist_bar_or, "or") + | isinstance_match + ) - matchlist_as = infix_match + OneOrMore(keyword("as").suppress() + setname) - as_match = labeled_group(matchlist_as, "as") | infix_match + matchlist_infix = bar_or_match + OneOrMore(Group(infix_op + Optional(negable_atom_item))) + infix_match = ( + labeled_group(matchlist_infix, "infix") + | bar_or_match + ) - matchlist_and = as_match + OneOrMore(keyword("and").suppress() + as_match) - and_match = labeled_group(matchlist_and, "and") | as_match + matchlist_as = infix_match + OneOrMore(keyword("as").suppress() + setname) + as_match = ( + labeled_group(matchlist_as, "as") + | infix_match + ) - matchlist_kwd_or = and_match + OneOrMore(keyword("or").suppress() + and_match) - kwd_or_match = labeled_group(matchlist_kwd_or, "or") | and_match + matchlist_and = as_match + OneOrMore(keyword("and").suppress() + as_match) + and_match = ( + labeled_group(matchlist_and, "and") + | as_match + ) - match <<= kwd_or_match + matchlist_kwd_or = and_match + OneOrMore(keyword("or").suppress() + and_match) + kwd_or_match = ( + labeled_group(matchlist_kwd_or, "or") + | and_match + ) - many_match = ( - labeled_group(matchlist_star, "star") - | labeled_group(matchlist_tuple_items, "implicit_tuple") - | match - ) + match <<= kwd_or_match - else_stmt = condense(keyword("else") - suite) - full_suite = colon.suppress() - Group((newline.suppress() - indent.suppress() - OneOrMore(stmt) - dedent.suppress()) | simple_stmt) - full_match = Forward() - full_match_ref = ( - keyword("match").suppress() - + many_match - + addspace(Optional(keyword("not")) + keyword("in")) - + testlist_star_namedexpr - + match_guard - # avoid match match-case blocks - + ~FollowedBy(colon + newline + indent + keyword("case")) - - full_suite - ) - match_stmt = condense(full_match - Optional(else_stmt)) - - destructuring_stmt = Forward() - base_destructuring_stmt = Optional(keyword("match").suppress()) + many_match + equals.suppress() + test_expr - destructuring_stmt_ref, match_dotted_name_const_ref = disable_inside(base_destructuring_stmt, must_be_dotted_name + ~lparen) - - # both syntaxes here must be kept the same except for the keywords - case_match_co_syntax = Group( - (keyword("match") | keyword("case")).suppress() - + stores_loc_item - + many_match - + Optional(keyword("if").suppress() + namedexpr_test) - - full_suite - ) - cases_stmt_co_syntax = ( - (keyword("cases") | keyword("case")) + testlist_star_namedexpr + colon.suppress() + newline.suppress() - + indent.suppress() + Group(OneOrMore(case_match_co_syntax)) - + dedent.suppress() + Optional(keyword("else").suppress() + suite) - ) - case_match_py_syntax = Group( - keyword("case").suppress() - + stores_loc_item - + many_match - + Optional(keyword("if").suppress() + namedexpr_test) - - full_suite - ) - cases_stmt_py_syntax = ( - keyword("match") + testlist_star_namedexpr + colon.suppress() + newline.suppress() - + indent.suppress() + Group(OneOrMore(case_match_py_syntax)) - + dedent.suppress() + Optional(keyword("else").suppress() - suite) - ) - cases_stmt = Forward() - cases_stmt_ref = cases_stmt_co_syntax | cases_stmt_py_syntax + many_match = ( + labeled_group(matchlist_star, "star") + | labeled_group(matchlist_tuple_items, "implicit_tuple") + | match + ) - assert_stmt = addspace( - keyword("assert") - - ( - lparen.suppress() + testlist + rparen.suppress() + end_simple_stmt_item - | testlist + else_stmt = condense(keyword("else") - suite) + full_suite = colon.suppress() - Group((newline.suppress() - indent.suppress() - OneOrMore(stmt) - dedent.suppress()) | simple_stmt) + full_match = Forward() + full_match_ref = ( + keyword("match").suppress() + + many_match + + addspace(Optional(keyword("not")) + keyword("in")) + + testlist_star_namedexpr + + match_guard + # avoid match match-case blocks + + ~FollowedBy(colon + newline + indent + keyword("case")) + + full_suite ) - ) - if_stmt = condense( - addspace(keyword("if") + condense(namedexpr_test + suite)) - - ZeroOrMore(addspace(keyword("elif") - condense(namedexpr_test - suite))) - - Optional(else_stmt) - ) - while_stmt = addspace(keyword("while") - condense(namedexpr_test - suite - Optional(else_stmt))) + match_stmt = condense(full_match - Optional(else_stmt)) + + destructuring_stmt = Forward() + base_destructuring_stmt = Optional(keyword("match").suppress()) + many_match + equals.suppress() + test_expr + destructuring_stmt_ref, match_dotted_name_const_ref = disable_inside(base_destructuring_stmt, must_be_dotted_name + ~lparen) + + # both syntaxes here must be kept the same except for the keywords + case_match_co_syntax = Group( + (keyword("match") | keyword("case")).suppress() + + stores_loc_item + + many_match + + Optional(keyword("if").suppress() + namedexpr_test) + - full_suite + ) + cases_stmt_co_syntax = ( + (keyword("cases") | keyword("case")) + testlist_star_namedexpr + colon.suppress() + newline.suppress() + + indent.suppress() + Group(OneOrMore(case_match_co_syntax)) + + dedent.suppress() + Optional(keyword("else").suppress() + suite) + ) + case_match_py_syntax = Group( + keyword("case").suppress() + + stores_loc_item + + many_match + + Optional(keyword("if").suppress() + namedexpr_test) + - full_suite + ) + cases_stmt_py_syntax = ( + keyword("match") + testlist_star_namedexpr + colon.suppress() + newline.suppress() + + indent.suppress() + Group(OneOrMore(case_match_py_syntax)) + + dedent.suppress() + Optional(keyword("else").suppress() - suite) + ) + cases_stmt = Forward() + cases_stmt_ref = cases_stmt_co_syntax | cases_stmt_py_syntax + + assert_stmt = addspace( + keyword("assert") + - ( + lparen.suppress() + testlist + rparen.suppress() + end_simple_stmt_item + | testlist + ) + ) + if_stmt = condense( + addspace(keyword("if") + condense(namedexpr_test + suite)) + - ZeroOrMore(addspace(keyword("elif") - condense(namedexpr_test - suite))) + - Optional(else_stmt) + ) + while_stmt = addspace(keyword("while") - condense(namedexpr_test - suite - Optional(else_stmt))) - for_stmt = addspace(keyword("for") + assignlist + keyword("in") - condense(new_testlist_star_expr - suite - Optional(else_stmt))) + for_stmt = addspace(keyword("for") + assignlist + keyword("in") - condense(new_testlist_star_expr - suite - Optional(else_stmt))) - suite_with_else_tokens = colon.suppress() + condense(nocolon_suite + Optional(else_stmt)) + suite_with_else_tokens = colon.suppress() + condense(nocolon_suite + Optional(else_stmt)) - base_match_for_stmt = Forward() - base_match_for_stmt_ref = ( - keyword("for").suppress() - + many_match - + keyword("in").suppress() - - new_testlist_star_expr - - suite_with_else_tokens - ) - match_for_stmt = Optional(keyword("match").suppress()) + base_match_for_stmt + base_match_for_stmt = Forward() + base_match_for_stmt_ref = ( + keyword("for").suppress() + + many_match + + keyword("in").suppress() + - new_testlist_star_expr + - suite_with_else_tokens + ) + match_for_stmt = Optional(keyword("match").suppress()) + base_match_for_stmt + any_for_stmt = ( + # match must come after + for_stmt + | match_for_stmt + ) - except_item = ( - testlist_has_comma("list") - | test("test") - ) - Optional( - keyword("as").suppress() - setname - ) - except_clause = attach(keyword("except") + except_item, except_handle) - except_star_clause = Forward() - except_star_clause_ref = attach(except_star_kwd + except_item, except_handle) - try_stmt = condense( - keyword("try") - suite + ( - keyword("finally") - suite - | ( - OneOrMore(except_clause - suite) - Optional(keyword("except") - suite) - | keyword("except") - suite - | OneOrMore(except_star_clause - suite) - ) - Optional(else_stmt) - Optional(keyword("finally") - suite) + except_item = ( + testlist_has_comma("list") + | test("test") + ) - Optional( + keyword("as").suppress() - setname + ) + except_clause = attach(keyword("except") + except_item, except_handle) + except_star_clause = Forward() + except_star_clause_ref = attach(except_star_kwd + except_item, except_handle) + try_stmt = condense( + keyword("try") - suite + ( + keyword("finally") - suite + | ( + OneOrMore(except_clause - suite) - Optional(keyword("except") - suite) + | keyword("except") - suite + | OneOrMore(except_star_clause - suite) + ) - Optional(else_stmt) - Optional(keyword("finally") - suite) + ) ) - ) - with_item = addspace(test + Optional(keyword("as") + base_assign_item)) - with_item_list = Group(maybeparens(lparen, tokenlist(with_item, comma), rparen)) - with_stmt_ref = keyword("with").suppress() - with_item_list - suite - with_stmt = Forward() - - funcname_typeparams = Forward() - funcname_typeparams_ref = dotted_setname + Optional(type_params) - name_funcdef = condense(funcname_typeparams + parameters) - op_tfpdef = unsafe_typedef_default | condense(setname + Optional(default)) - op_funcdef_arg = setname | condense(lparen.suppress() + op_tfpdef + rparen.suppress()) - op_funcdef_name = unsafe_backtick.suppress() + funcname_typeparams + unsafe_backtick.suppress() - op_funcdef = attach( - Group(Optional(op_funcdef_arg)) - + op_funcdef_name - + Group(Optional(op_funcdef_arg)), - op_funcdef_handle, - ) + with_item = addspace(test + Optional(keyword("as") + base_assign_item)) + with_item_list = Group(maybeparens(lparen, tokenlist(with_item, comma), rparen)) + with_stmt_ref = keyword("with").suppress() + with_item_list + suite + with_stmt = Forward() + + funcname_typeparams = Forward() + funcname_typeparams_ref = dotted_setname + Optional(type_params) + name_funcdef = condense(funcname_typeparams + parameters) + op_tfpdef = unsafe_typedef_default | condense(setname + Optional(default)) + op_funcdef_arg = setname | condense(lparen.suppress() + op_tfpdef + rparen.suppress()) + op_funcdef_name = unsafe_backtick.suppress() + funcname_typeparams + unsafe_backtick.suppress() + op_funcdef = attach( + Group(Optional(op_funcdef_arg)) + + op_funcdef_name + + Group(Optional(op_funcdef_arg)), + op_funcdef_handle, + ) - return_typedef = Forward() - return_typedef_ref = arrow.suppress() + typedef_test - end_func_colon = return_typedef + colon.suppress() | colon - base_funcdef = op_funcdef | name_funcdef - funcdef = addspace(keyword("def") + condense(base_funcdef + end_func_colon + nocolon_suite)) + return_typedef = Forward() + return_typedef_ref = arrow.suppress() + typedef_test + end_func_colon = return_typedef + colon.suppress() | colon + base_funcdef = name_funcdef | op_funcdef + funcdef = addspace(keyword("def") + condense(base_funcdef + end_func_colon + nocolon_suite)) - name_match_funcdef = Forward() - op_match_funcdef = Forward() - op_match_funcdef_arg = Group(Optional( - Group( + name_match_funcdef = Forward() + op_match_funcdef = Forward() + op_match_funcdef_arg = Group(Optional( + Group( + ( + lparen.suppress() + + match + + Optional(equals.suppress() + test) + + rparen.suppress() + ) | interior_name_match + ) + )) + name_match_funcdef_ref = keyword("def").suppress() + funcname_typeparams + lparen.suppress() + match_args_list + match_guard + rparen.suppress() + op_match_funcdef_ref = keyword("def").suppress() + op_match_funcdef_arg + op_funcdef_name + op_match_funcdef_arg + match_guard + base_match_funcdef = name_match_funcdef | op_match_funcdef + func_suite = ( ( - lparen.suppress() - + match - + Optional(equals.suppress() + test) - + rparen.suppress() - ) | interior_name_match - ) - )) - name_match_funcdef_ref = keyword("def").suppress() + funcname_typeparams + lparen.suppress() + match_args_list + match_guard + rparen.suppress() - op_match_funcdef_ref = keyword("def").suppress() + op_match_funcdef_arg + op_funcdef_name + op_match_funcdef_arg + match_guard - base_match_funcdef = op_match_funcdef | name_match_funcdef - func_suite = ( - attach(simple_stmt, make_suite_handle) - | ( - newline.suppress() - - indent.suppress() - - Optional(docstring) - - attach(condense(OneOrMore(stmt)), make_suite_handle) - - dedent.suppress() + newline.suppress() + - indent.suppress() + - Optional(docstring) + - attach(condense(OneOrMore(stmt)), make_suite_handle) + - dedent.suppress() + ) + | attach(simple_stmt, make_suite_handle) ) - ) - def_match_funcdef = attach( - base_match_funcdef - + end_func_colon - - func_suite, - join_match_funcdef, - ) - match_def_modifiers = any_len_perm( - keyword("match").suppress(), - # addpattern is detected later - keyword("addpattern"), - ) - match_funcdef = addspace(match_def_modifiers + def_match_funcdef) - - where_stmt = attach( - unsafe_simple_stmt_item - + keyword("where").suppress() - - full_suite, - where_handle, - ) - - implicit_return = ( - invalid_syntax(return_stmt, "expected expression but got return statement") - | attach(new_testlist_star_expr, implicit_return_handle) - ) - implicit_return_where = attach( - implicit_return - + keyword("where").suppress() - - full_suite, - where_handle, - ) - implicit_return_stmt = ( - condense(implicit_return + newline) - | implicit_return_where - ) - math_funcdef_body = condense(ZeroOrMore(~(implicit_return_stmt + dedent) + stmt) - implicit_return_stmt) - math_funcdef_suite = ( - attach(implicit_return_stmt, make_suite_handle) - | condense(newline - indent - math_funcdef_body - dedent) - ) - end_func_equals = return_typedef + equals.suppress() | fixto(equals, ":") - math_funcdef = attach( - condense(addspace(keyword("def") + base_funcdef) + end_func_equals) - math_funcdef_suite, - math_funcdef_handle, - ) - math_match_funcdef = addspace( - match_def_modifiers - + attach( + def_match_funcdef = attach( base_match_funcdef - + end_func_equals - + ( - attach(implicit_return_stmt, make_suite_handle) - | ( - newline.suppress() - indent.suppress() - + Optional(docstring) - + attach(math_funcdef_body, make_suite_handle) - + dedent.suppress() - ) - ), + + end_func_colon + - func_suite, join_match_funcdef, ) - ) - - async_stmt = Forward() - async_with_for_stmt = Forward() - async_with_for_stmt_ref = ( - labeled_group( - (keyword("async") + keyword("with") + keyword("for")).suppress() - + assignlist + keyword("in").suppress() - - test - - suite_with_else_tokens, - "normal", - ) - | labeled_group( - (any_len_perm( - keyword("match"), - required=(keyword("async"), keyword("with")), - ) + keyword("for")).suppress() - + many_match + keyword("in").suppress() - - test - - suite_with_else_tokens, - "match", - ) - ) - async_stmt_ref = addspace( - keyword("async") + (with_stmt | for_stmt | match_for_stmt) # handles async [match] for - | keyword("match").suppress() + keyword("async") + base_match_for_stmt # handles match async for - | async_with_for_stmt - ) - - async_funcdef = keyword("async").suppress() + (funcdef | math_funcdef) - async_match_funcdef = addspace( - any_len_perm( + match_def_modifiers = any_len_perm( keyword("match").suppress(), # addpattern is detected later keyword("addpattern"), - required=(keyword("async").suppress(),), - ) + (def_match_funcdef | math_match_funcdef), - ) + ) + match_funcdef = addspace(match_def_modifiers + def_match_funcdef) - async_keyword_normal_funcdef = Group( - any_len_perm_at_least_one( - keyword("yield"), - keyword("copyclosure"), - required=(keyword("async").suppress(),), - ) - ) + (funcdef | math_funcdef) - async_keyword_match_funcdef = Group( - any_len_perm_at_least_one( - keyword("yield"), - keyword("copyclosure"), - keyword("match").suppress(), - # addpattern is detected later - keyword("addpattern"), - required=(keyword("async").suppress(),), + where_suite = keyword("where").suppress() - full_suite + + where_stmt = Forward() + where_item = Forward() + where_item_ref = unsafe_simple_stmt_item + where_stmt_ref = where_item + where_suite + + implicit_return = ( + invalid_syntax(return_stmt, "expected expression but got return statement") + | attach(new_testlist_star_expr, implicit_return_handle) + ) + implicit_return_where = Forward() + implicit_return_where_item = Forward() + implicit_return_where_item_ref = implicit_return + implicit_return_where_ref = implicit_return_where_item + where_suite + implicit_return_stmt = ( + condense(implicit_return + newline) + | implicit_return_where ) - ) + (def_match_funcdef | math_match_funcdef) - async_keyword_funcdef = Forward() - async_keyword_funcdef_ref = async_keyword_normal_funcdef | async_keyword_match_funcdef - async_funcdef_stmt = ( - async_funcdef - | async_match_funcdef - | async_keyword_funcdef - ) + math_funcdef_body = condense(ZeroOrMore(~(implicit_return_stmt + dedent) + stmt) - implicit_return_stmt) + math_funcdef_suite = ( + condense(newline - indent - math_funcdef_body - dedent) + | attach(implicit_return_stmt, make_suite_handle) + ) + end_func_equals = return_typedef + equals.suppress() | fixto(equals, ":") + math_funcdef = attach( + condense(addspace(keyword("def") + base_funcdef) + end_func_equals) - math_funcdef_suite, + math_funcdef_handle, + ) + math_match_funcdef = addspace( + match_def_modifiers + + attach( + base_match_funcdef + + end_func_equals + - ( + attach(implicit_return_stmt, make_suite_handle) + | ( + newline.suppress() + - indent.suppress() + - Optional(docstring) + - attach(math_funcdef_body, make_suite_handle) + - dedent.suppress() + ) + ), + join_match_funcdef, + ) + ) - keyword_normal_funcdef = Group( - any_len_perm_at_least_one( - keyword("yield"), - keyword("copyclosure"), + async_stmt = Forward() + async_with_for_stmt = Forward() + async_with_for_stmt_ref = ( + labeled_group( + (keyword("async") + keyword("with") + keyword("for")).suppress() + + assignlist + keyword("in").suppress() + - test + - suite_with_else_tokens, + "normal", + ) + | labeled_group( + (any_len_perm( + keyword("match"), + required=(keyword("async"), keyword("with")), + ) + keyword("for")).suppress() + + many_match + keyword("in").suppress() + - test + - suite_with_else_tokens, + "match", + ) ) - ) + (funcdef | math_funcdef) - keyword_match_funcdef = Group( - any_len_perm_at_least_one( - keyword("yield"), - keyword("copyclosure"), - keyword("match").suppress(), - # addpattern is detected later - keyword("addpattern"), + async_stmt_ref = addspace( + keyword("async") + (with_stmt | any_for_stmt) # handles async [match] for + | keyword("match").suppress() + keyword("async") + base_match_for_stmt # handles match async for + | async_with_for_stmt ) - ) + (def_match_funcdef | math_match_funcdef) - keyword_funcdef = Forward() - keyword_funcdef_ref = keyword_normal_funcdef | keyword_match_funcdef - normal_funcdef_stmt = ( - funcdef - | math_funcdef - | math_match_funcdef - | match_funcdef - | keyword_funcdef - ) + async_funcdef = keyword("async").suppress() + (funcdef | math_funcdef) + async_match_funcdef = addspace( + any_len_perm( + keyword("match").suppress(), + # addpattern is detected later + keyword("addpattern"), + required=(keyword("async").suppress(),), + ) + (def_match_funcdef | math_match_funcdef), + ) - datadef = Forward() - data_args = Group(Optional( - lparen.suppress() + ZeroOrMore(Group( - # everything here must end with arg_comma - (unsafe_name + arg_comma.suppress())("name") - | (unsafe_name + equals.suppress() + test + arg_comma.suppress())("default") - | (star.suppress() + unsafe_name + arg_comma.suppress())("star") - | (unsafe_name + colon.suppress() + typedef_test + equals.suppress() + test + arg_comma.suppress())("type default") - | (unsafe_name + colon.suppress() + typedef_test + arg_comma.suppress())("type") - )) + rparen.suppress() - )) - data_inherit = Optional(keyword("from").suppress() + testlist) - data_suite = Group( - colon.suppress() - ( - (newline.suppress() + indent.suppress() + Optional(docstring) + Group(OneOrMore(stmt)) - dedent.suppress())("complex") - | (newline.suppress() + indent.suppress() + docstring - dedent.suppress() | docstring)("docstring") - | simple_stmt("simple") - ) | newline("empty") - ) - datadef_ref = ( - Optional(decorators, default="") - + keyword("data").suppress() - + classname - + Optional(type_params, default=()) - + data_args - + data_inherit - + data_suite - ) + async_keyword_normal_funcdef = Group( + any_len_perm_at_least_one( + keyword("yield"), + keyword("copyclosure"), + required=(keyword("async").suppress(),), + ) + ) + (funcdef | math_funcdef) + async_keyword_match_funcdef = Group( + any_len_perm_at_least_one( + keyword("yield"), + keyword("copyclosure"), + keyword("match").suppress(), + # addpattern is detected later + keyword("addpattern"), + required=(keyword("async").suppress(),), + ) + ) + (def_match_funcdef | math_match_funcdef) + async_keyword_funcdef = Forward() + async_keyword_funcdef_ref = async_keyword_normal_funcdef | async_keyword_match_funcdef + + async_funcdef_stmt = ( + # match funcdefs must come after normal + async_funcdef + | async_match_funcdef + | async_keyword_funcdef + ) - match_datadef = Forward() - match_data_args = lparen.suppress() + Group( - match_args_list + match_guard - ) + rparen.suppress() - # we don't support type_params here since we don't support types - match_datadef_ref = ( - Optional(decorators, default="") - + Optional(keyword("match").suppress()) - + keyword("data").suppress() - + classname - + match_data_args - + data_inherit - + data_suite - ) + keyword_normal_funcdef = Group( + any_len_perm_at_least_one( + keyword("yield"), + keyword("copyclosure"), + ) + ) + (funcdef | math_funcdef) + keyword_match_funcdef = Group( + any_len_perm_at_least_one( + keyword("yield"), + keyword("copyclosure"), + keyword("match").suppress(), + # addpattern is detected later + keyword("addpattern"), + ) + ) + (def_match_funcdef | math_match_funcdef) + keyword_funcdef = Forward() + keyword_funcdef_ref = keyword_normal_funcdef | keyword_match_funcdef + + normal_funcdef_stmt = ( + # match funcdefs must come after normal + funcdef + | math_funcdef + | match_funcdef + | math_match_funcdef + | keyword_funcdef + ) - simple_decorator = condense(dotted_refname + Optional(function_call) + newline)("simple") - complex_decorator = condense(namedexpr_test + newline)("complex") - decorators_ref = OneOrMore( - at.suppress() - - Group( - simple_decorator - | complex_decorator + datadef = Forward() + data_args = Group(Optional( + lparen.suppress() + ZeroOrMore(Group( + # everything here must end with arg_comma + (unsafe_name + arg_comma.suppress())("name") + | (unsafe_name + equals.suppress() + test + arg_comma.suppress())("default") + | (star.suppress() + unsafe_name + arg_comma.suppress())("star") + | (unsafe_name + colon.suppress() + typedef_test + equals.suppress() + test + arg_comma.suppress())("type default") + | (unsafe_name + colon.suppress() + typedef_test + arg_comma.suppress())("type") + )) + rparen.suppress() + )) + data_inherit = Optional(keyword("from").suppress() + testlist) + data_suite = Group( + colon.suppress() - ( + (newline.suppress() + indent.suppress() + Optional(docstring) + Group(OneOrMore(stmt)) - dedent.suppress())("complex") + | (newline.suppress() + indent.suppress() + docstring - dedent.suppress() | docstring)("docstring") + | simple_stmt("simple") + ) | newline("empty") + ) + datadef_ref = ( + Optional(decorators, default="") + + keyword("data").suppress() + + classname + + Optional(type_params, default=()) + + data_args + + data_inherit + + data_suite ) - ) - decoratable_normal_funcdef_stmt = Forward() - decoratable_normal_funcdef_stmt_ref = Optional(decorators) + normal_funcdef_stmt + match_datadef = Forward() + match_data_args = lparen.suppress() + Group( + match_args_list + match_guard + ) + rparen.suppress() + # we don't support type_params here since we don't support types + match_datadef_ref = ( + Optional(decorators, default="") + + Optional(keyword("match").suppress()) + + keyword("data").suppress() + + classname + + match_data_args + + data_inherit + + data_suite + ) - decoratable_async_funcdef_stmt = Forward() - decoratable_async_funcdef_stmt_ref = Optional(decorators) + async_funcdef_stmt + simple_decorator = condense(dotted_refname + Optional(function_call) + newline)("simple") + complex_decorator = condense(namedexpr_test + newline)("complex") + decorators_ref = OneOrMore( + at.suppress() + + Group( + simple_decorator + | complex_decorator + ) + ) - decoratable_func_stmt = decoratable_normal_funcdef_stmt | decoratable_async_funcdef_stmt + decoratable_normal_funcdef_stmt = Forward() + decoratable_normal_funcdef_stmt_ref = Optional(decorators) + normal_funcdef_stmt - # decorators are integrated into the definitions of each item here - decoratable_class_stmt = classdef | datadef | match_datadef + decoratable_async_funcdef_stmt = Forward() + decoratable_async_funcdef_stmt_ref = Optional(decorators) + async_funcdef_stmt - passthrough_stmt = condense(passthrough_block - (base_suite | newline)) + decoratable_func_stmt = any_of( + decoratable_normal_funcdef_stmt, + decoratable_async_funcdef_stmt, + ) + decoratable_data_stmt = ( + # match must come after + datadef + | match_datadef + ) - simple_compound_stmt = ( - if_stmt - | try_stmt - | match_stmt - | passthrough_stmt - ) - compound_stmt = ( - decoratable_class_stmt - | decoratable_func_stmt - | for_stmt - | while_stmt - | with_stmt - | async_stmt - | match_for_stmt - | simple_compound_stmt - | where_stmt - ) - endline_semicolon = Forward() - endline_semicolon_ref = semicolon.suppress() + newline - keyword_stmt = ( - flow_stmt - | import_stmt - | assert_stmt - | pass_stmt - | del_stmt - | global_stmt - | nonlocal_stmt - ) - special_stmt = ( - keyword_stmt - | augassign_stmt - | typed_assign_stmt - | type_alias_stmt - ) - unsafe_simple_stmt_item <<= special_stmt | longest(basic_stmt, destructuring_stmt) - simple_stmt_item <<= ( - special_stmt - | basic_stmt + end_simple_stmt_item - | destructuring_stmt + end_simple_stmt_item - ) - simple_stmt <<= condense( - simple_stmt_item - + ZeroOrMore(fixto(semicolon, "\n") + simple_stmt_item) - + (newline | endline_semicolon) - ) - anything_stmt = Forward() - stmt <<= final( - compound_stmt - | simple_stmt - # must be after destructuring due to ambiguity - | cases_stmt - # at the very end as a fallback case for the anything parser - | anything_stmt - ) - base_suite <<= condense(newline + indent - OneOrMore(stmt) - dedent) - simple_suite = attach(stmt, make_suite_handle) - nocolon_suite <<= base_suite | simple_suite - suite <<= condense(colon + nocolon_suite) - line = newline | stmt - - single_input = condense(Optional(line) - ZeroOrMore(newline)) - file_input = condense(moduledoc_marker - ZeroOrMore(line)) - eval_input = condense(testlist - ZeroOrMore(newline)) - - single_parser = start_marker - single_input - end_marker - file_parser = start_marker - file_input - end_marker - eval_parser = start_marker - eval_input - end_marker - some_eval_parser = start_marker + eval_input - - parens = originalTextFor(nestedExpr("(", ")", ignoreExpr=None)) - brackets = originalTextFor(nestedExpr("[", "]", ignoreExpr=None)) - braces = originalTextFor(nestedExpr("{", "}", ignoreExpr=None)) - - unsafe_anything_stmt = originalTextFor(regex_item("[^\n]+\n+")) - unsafe_xonsh_command = originalTextFor( - (Optional(at) + dollar | bang) - + ~(lparen + rparen | lbrack + rbrack | lbrace + rbrace) - + (parens | brackets | braces | unsafe_name) - ) - unsafe_xonsh_parser, _impl_call_ref = disable_inside( - single_parser, - unsafe_impl_call_ref, - ) - impl_call_ref <<= _impl_call_ref - xonsh_parser, _anything_stmt, _xonsh_command = disable_outside( - unsafe_xonsh_parser, - unsafe_anything_stmt, - unsafe_xonsh_command, - ) - anything_stmt <<= _anything_stmt - xonsh_command <<= _xonsh_command + passthrough_stmt = condense(passthrough_block - (base_suite | newline)) + + compound_stmt = any_of( + # decorators should be integrated into the definitions of any items that need them + if_stmt, + decoratable_func_stmt, + classdef, + while_stmt, + try_stmt, + with_stmt, + any_for_stmt, + async_stmt, + decoratable_data_stmt, + match_stmt, + passthrough_stmt, + where_stmt, + ) + + flow_stmt = any_of( + return_stmt, + simple_raise_stmt, + break_stmt, + continue_stmt, + yield_expr, + complex_raise_stmt, + ) + keyword_stmt = any_of( + flow_stmt, + import_stmt, + assert_stmt, + pass_stmt, + del_stmt, + global_stmt, + nonlocal_stmt, + ) + special_stmt = any_of( + keyword_stmt, + augassign_stmt, + typed_assign_stmt, + type_alias_stmt, + ) + unsafe_simple_stmt_item <<= special_stmt | longest(basic_stmt, destructuring_stmt) + simple_stmt_item <<= ( + # destructuring stmt must come after basic + special_stmt + | basic_stmt + end_simple_stmt_item + | destructuring_stmt + end_simple_stmt_item + ) + endline_semicolon = Forward() + endline_semicolon_ref = semicolon.suppress() + newline + simple_stmt <<= condense( + simple_stmt_item + + ZeroOrMore(fixto(semicolon, "\n") + simple_stmt_item) + + (newline | endline_semicolon) + ) + + anything_stmt = Forward() + stmt <<= final( + compound_stmt + | simple_stmt # includes destructuring + # must be after destructuring due to ambiguity + | cases_stmt + # at the very end as a fallback case for the anything parser + | anything_stmt + ) + + base_suite <<= condense(newline + indent - OneOrMore(stmt) - dedent) + simple_suite = attach(stmt, make_suite_handle) + nocolon_suite <<= base_suite | simple_suite + suite <<= condense(colon + nocolon_suite) + + line = newline | stmt + + single_input = condense(Optional(line) - ZeroOrMore(newline)) + file_input = condense(moduledoc_marker - ZeroOrMore(line)) + eval_input = condense(testlist - ZeroOrMore(newline)) + + single_parser = start_marker - single_input - end_marker + file_parser = start_marker - file_input - end_marker + eval_parser = start_marker - eval_input - end_marker + some_eval_parser = start_marker + eval_input + + parens = originalTextFor(nestedExpr("(", ")", ignoreExpr=None)) + brackets = originalTextFor(nestedExpr("[", "]", ignoreExpr=None)) + braces = originalTextFor(nestedExpr("{", "}", ignoreExpr=None)) + + unsafe_anything_stmt = originalTextFor(regex_item("[^\n]+\n+")) + unsafe_xonsh_command = originalTextFor( + (Optional(at) + dollar | bang) + + ~(lparen + rparen | lbrack + rbrack | lbrace + rbrace) + + any_of( + parens, + brackets, + braces, + unsafe_name, + ) + ) + unsafe_xonsh_parser, _impl_call_ref = disable_inside( + single_parser, + unsafe_impl_call_ref, + ) + impl_call_ref <<= _impl_call_ref + xonsh_parser, _anything_stmt, _xonsh_command = disable_outside( + unsafe_xonsh_parser, + unsafe_anything_stmt, + unsafe_xonsh_command, + ) + anything_stmt <<= _anything_stmt + xonsh_command <<= _xonsh_command # end: MAIN GRAMMAR # ----------------------------------------------------------------------------------------------------------------------- # EXTRA GRAMMAR: # ----------------------------------------------------------------------------------------------------------------------- - # we don't need to include opens/closes here because those are explicitly disallowed - existing_operator_regex = compile_regex(r"([.;\\]|([+-=@%^&|*:,/<>~]|\*\*|//|>>|<<)=?|!=|" + r"|".join(new_operators) + r")$") + # we don't need to include opens/closes here because those are explicitly disallowed + existing_operator_regex = compile_regex(r"([.;\\]|([+-=@%^&|*:,/<>~]|\*\*|//|>>|<<)=?|!=|" + r"|".join(new_operators) + r")$") - whitespace_regex = compile_regex(r"\s") + whitespace_regex = compile_regex(r"\s") - def_regex = compile_regex(r"\b((async|addpattern|copyclosure)\s+)*def\b") - yield_regex = compile_regex(r"\byield(?!\s+_coconut\.asyncio\.From)\b") - yield_from_regex = compile_regex(r"\byield\s+from\b") + def_regex = compile_regex(r"\b((async|addpattern|copyclosure)\s+)*def\b") + yield_regex = compile_regex(r"\byield(?!\s+_coconut\.asyncio\.From)\b") + yield_from_regex = compile_regex(r"\byield\s+from\b") - tco_disable_regex = compile_regex(r"\b(try\b|(async\s+)?(with\b|for\b)|while\b)") - return_regex = compile_regex(r"\breturn\b") + tco_disable_regex = compile_regex(r"\b(try\b|(async\s+)?(with\b|for\b)|while\b)") + return_regex = compile_regex(r"\breturn\b") - noqa_regex = compile_regex(r"\b[Nn][Oo][Qq][Aa]\b") + noqa_regex = compile_regex(r"\b[Nn][Oo][Qq][Aa]\b") - just_non_none_atom = start_marker + ~keyword("None") + known_atom + end_marker + just_non_none_atom = start_marker + ~keyword("None") + known_atom + end_marker - original_function_call_tokens = ( - lparen.suppress() + rparen.suppress() - # we need to keep the parens here, since f(x for x in y) is fine but tail_call(f, x for x in y) is not - | condense(lparen + originalTextFor(test + comp_for) + rparen) - | attach(parens, strip_parens_handle) - ) + original_function_call_tokens = ( + lparen.suppress() + rparen.suppress() + # we need to keep the parens here, since f(x for x in y) is fine but tail_call(f, x for x in y) is not + | condense(lparen + originalTextFor(test + comp_for) + rparen) + | attach(parens, strip_parens_handle) + ) - tre_func_name = Forward() - tre_return = ( - start_marker - + keyword("return").suppress() - + maybeparens( - lparen, - tre_func_name + original_function_call_tokens, - rparen, - ) + end_marker - ) + tre_func_name = Forward() + tre_return = ( + start_marker + + keyword("return").suppress() + + maybeparens( + lparen, + tre_func_name + original_function_call_tokens, + rparen, + ) + end_marker + ) - tco_return = attach( - start_marker - + keyword("return").suppress() - + maybeparens( - lparen, - disallow_keywords(untcoable_funcs, with_suffix="(") - + condense( - (unsafe_name | parens | brackets | braces | string_atom) - + ZeroOrMore( - dot + unsafe_name - | brackets - # don't match the last set of parentheses - | parens + ~end_marker + ~rparen - ), - ) - + original_function_call_tokens, - rparen, - ) + end_marker, - tco_return_handle, - # this is the root in what it's used for, so might as well evaluate greedily - greedy=True, - ) + tco_return = attach( + start_marker + + keyword("return").suppress() + + maybeparens( + lparen, + disallow_keywords(untcoable_funcs, with_suffix="(") + + condense( + any_of( + unsafe_name, + parens, + string_atom, + brackets, + braces, + ) + + ZeroOrMore(any_of( + dot + unsafe_name, + brackets, + # don't match the last set of parentheses + parens + ~end_marker + ~rparen, + )), + ) + + original_function_call_tokens, + rparen, + ) + end_marker, + tco_return_handle, + # this is the root in what it's used for, so might as well evaluate greedily + greedy=True, + ) - rest_of_lambda = Forward() - lambdas = keyword("lambda") - rest_of_lambda - colon - rest_of_lambda <<= ZeroOrMore( - # handle anything that could capture colon - parens - | brackets - | braces - | lambdas - | ~colon + any_char - ) - rest_of_tfpdef = originalTextFor( - ZeroOrMore( - # handle anything that could capture comma, rparen, or equals + rest_of_lambda = Forward() + lambdas = keyword("lambda") - rest_of_lambda - colon + rest_of_lambda <<= ZeroOrMore( + # handle anything that could capture colon parens | brackets | braces | lambdas - | ~comma + ~rparen + ~equals + any_char + | ~colon + any_char + ) + rest_of_tfpdef = originalTextFor( + ZeroOrMore( + # handle anything that could capture comma, rparen, or equals + parens + | brackets + | braces + | lambdas + | ~comma + ~rparen + ~equals + any_char + ) + ) + tfpdef_tokens = unsafe_name - Optional(colon - rest_of_tfpdef).suppress() + tfpdef_default_tokens = tfpdef_tokens - Optional(equals - rest_of_tfpdef) + type_comment = Optional( + comment_tokens + | passthrough_item + ).suppress() + parameters_tokens = Group( + Optional(tokenlist( + Group( + tfpdef_default_tokens + | star - Optional(tfpdef_tokens) + | dubstar - tfpdef_tokens + | slash + ) + type_comment, + comma + type_comment, + )) ) - ) - tfpdef_tokens = unsafe_name - Optional(colon - rest_of_tfpdef).suppress() - tfpdef_default_tokens = tfpdef_tokens - Optional(equals - rest_of_tfpdef) - type_comment = Optional( - comment_tokens - | passthrough_item - ).suppress() - parameters_tokens = Group( - Optional(tokenlist( - Group( - dubstar - tfpdef_tokens - | star - Optional(tfpdef_tokens) - | slash - | tfpdef_default_tokens - ) + type_comment, - comma + type_comment, - )) - ) - split_func = ( - start_marker - - keyword("def").suppress() - - unsafe_dotted_name - - lparen.suppress() - parameters_tokens - rparen.suppress() - ) + split_func = ( + start_marker + - keyword("def").suppress() + - unsafe_dotted_name + - Optional(brackets).suppress() + - lparen.suppress() + - parameters_tokens + - rparen.suppress() + ) - stores_scope = boundary + ( - keyword("lambda") - # match comprehensions but not for loops - | ~indent + ~dedent + any_char + keyword("for") + unsafe_name + keyword("in") - ) + stores_scope = boundary + ( + keyword("lambda") + # match comprehensions but not for loops + | ~indent + ~dedent + any_char + keyword("for") + unsafe_name + keyword("in") + ) - just_a_string = start_marker + string_atom + end_marker + just_a_string = start_marker + string_atom + end_marker - end_of_line = end_marker | Literal("\n") | pound + end_of_line = end_marker | Literal("\n") | pound - unsafe_equals = Literal("=") + unsafe_equals = Literal("=") - kwd_err_msg = attach(any_keyword_in(keyword_vars + reserved_vars), kwd_err_msg_handle) - parse_err_msg = ( - start_marker + ( - fixto(end_of_line, "misplaced newline (maybe missing ':')") - | fixto(Optional(keyword("if") + skip_to_in_line(unsafe_equals)) + equals, "misplaced assignment (maybe should be '==')") - | kwd_err_msg - ) - | fixto( - questionmark - + ~dollar - + ~lparen - + ~lbrack - + ~dot, - "misplaced '?' (naked '?' is only supported inside partial application arguments)", + kwd_err_msg = attach(any_keyword_in(keyword_vars + reserved_vars), kwd_err_msg_handle) + parse_err_msg = ( + start_marker + ( + fixto(end_of_line, "misplaced newline (maybe missing ':')") + | fixto(Optional(keyword("if") + skip_to_in_line(unsafe_equals)) + equals, "misplaced assignment (maybe should be '==')") + | kwd_err_msg + ) + | fixto( + questionmark + + ~dollar + + ~lparen + + ~lbrack + + ~dot, + "misplaced '?' (naked '?' is only supported inside partial application arguments)", + ) ) - ) - end_f_str_expr = combine(start_marker + (bang | colon | rbrace)) + end_f_str_expr = combine(start_marker + (rbrace | colon | bang)) - string_start = start_marker + python_quoted_string + string_start = start_marker + python_quoted_string - no_unquoted_newlines = start_marker + ZeroOrMore(python_quoted_string | ~Literal("\n") + any_char) + end_marker + no_unquoted_newlines = start_marker + ZeroOrMore(python_quoted_string | ~Literal("\n") + any_char) + end_marker - operator_stmt = ( - start_marker - + keyword("operator").suppress() - + restOfLine - ) + operator_stmt = ( + start_marker + + keyword("operator").suppress() + + restOfLine + ) - unsafe_import_from_name = condense(ZeroOrMore(unsafe_dot) + unsafe_dotted_name | OneOrMore(unsafe_dot)) - from_import_operator = ( - start_marker - + keyword("from").suppress() - + unsafe_import_from_name - + keyword("import").suppress() - + keyword("operator").suppress() - + restOfLine - ) + unsafe_import_from_name = condense(ZeroOrMore(unsafe_dot) + unsafe_dotted_name | OneOrMore(unsafe_dot)) + from_import_operator = ( + start_marker + + keyword("from").suppress() + + unsafe_import_from_name + + keyword("import").suppress() + + keyword("operator").suppress() + + restOfLine + ) # end: EXTRA GRAMMAR # ----------------------------------------------------------------------------------------------------------------------- @@ -2561,12 +2657,12 @@ def add_to_grammar_init_time(cls): finally: cls.grammar_init_time += get_clock_time() - start_time - -def set_grammar_names(): - """Set names of grammar elements to their variable names.""" - for varname, val in vars(Grammar).items(): - if isinstance(val, ParserElement): - val.setName(varname) + @staticmethod + def set_grammar_names(): + """Set names of grammar elements to their variable names.""" + for varname, val in vars(Grammar).items(): + if isinstance(val, ParserElement): + val.setName(varname) # end: TRACING diff --git a/coconut/compiler/header.py b/coconut/compiler/header.py index 3be75c8fd..8a60ff8cc 100644 --- a/coconut/compiler/header.py +++ b/coconut/compiler/header.py @@ -44,6 +44,7 @@ univ_open, get_target_info, assert_remove_prefix, + memoize, ) from coconut.compiler.util import ( split_comment, @@ -96,6 +97,7 @@ def minify_header(compiled): template_dir = os.path.join(os.path.dirname(os.path.abspath(__file__)), "templates") +@memoize() def get_template(template): """Read the given template file.""" with univ_open(os.path.join(template_dir, template) + template_ext, "r") as template_file: @@ -123,7 +125,16 @@ def prepare(code, indent=0, **kwargs): return _indent(code, by=indent, strip=True, **kwargs) -def base_pycondition(target, ver, if_lt=None, if_ge=None, indent=None, newline=False, initial_newline=False, fallback=""): +def base_pycondition( + target, + ver, + if_lt=None, + if_ge=None, + indent=None, + newline=False, + initial_newline=False, + fallback="", +): """Produce code that depends on the Python version for the given target.""" internal_assert(isinstance(ver, tuple), "invalid pycondition version") internal_assert(if_lt or if_ge, "either if_lt or if_ge must be specified") @@ -177,6 +188,52 @@ def base_pycondition(target, ver, if_lt=None, if_ge=None, indent=None, newline=F return out +def def_in_exec(name, code, needs_vars={}, decorator=None): + """Get code that runs code in an exec and extracts name.""" + return ''' +_coconut_{name}_ns = {lbrace}"_coconut": _coconut{needs_vars}{rbrace} +_coconut_exec({code}, _coconut_{name}_ns) +{name} = {open_decorator}_coconut_{name}_ns["{name}"]{close_decorator} + '''.format( + lbrace="{", + rbrace="}", + name=name, + code=repr(code.strip()), + needs_vars=( + ", " + ", ".join( + repr(var_in_def) + ": " + var_out_def + for var_in_def, var_out_def in needs_vars.items() + ) + if needs_vars else "" + ), + open_decorator=decorator + "(" if decorator is not None else "", + close_decorator=")" if decorator is not None else "", + ) + + +def base_async_def( + target, + func_name, + async_def, + no_async_def, + needs_vars={}, + decorator=None, + **kwargs # no comma; breaks on <=3.5 +): + """Build up a universal async function definition.""" + target_info = get_target_info(target) + if target_info >= (3, 5): + out = async_def + else: + out = base_pycondition( + target, + (3, 5), + if_ge=def_in_exec(func_name, async_def, needs_vars=needs_vars, decorator=decorator), + if_lt=no_async_def, + ) + return prepare(out, **kwargs) + + def make_py_str(str_contents, target, after_py_str_defined=False): """Get code that effectively wraps the given code in py_str.""" return ( @@ -207,6 +264,7 @@ def process_header_args(which, use_hash, target, no_tco, strict, no_wrap): """Create the dictionary passed to str.format in the header.""" target_info = get_target_info(target) pycondition = partial(base_pycondition, target) + async_def = partial(base_async_def, target) format_dict = dict( COMMENT=COMMENT, @@ -230,6 +288,9 @@ def process_header_args(which, use_hash, target, no_tco, strict, no_wrap): comma_object="" if target.startswith("3") else ", object", comma_slash=", /" if target_info >= (3, 8) else "", report_this_text=report_this_text, + from_None=" from None" if target.startswith("3") else "", + process_="process_" if target_info >= (3, 13) else "", + numpy_modules=tuple_str_of(numpy_modules, add_quotes=True), pandas_numpy_modules=tuple_str_of(pandas_numpy_modules, add_quotes=True), jax_numpy_modules=tuple_str_of(jax_numpy_modules, add_quotes=True), @@ -300,31 +361,39 @@ def process_header_args(which, use_hash, target, no_tco, strict, no_wrap): ), # disabled mocks must have different docstrings so the # interpreter can tell them apart from the real thing - def_prepattern=( - r'''def prepattern(base_func, **kwargs): + def_aliases=prepare( + r''' +def prepattern(base_func, **kwargs): """DEPRECATED: use addpattern instead.""" def pattern_prepender(func): return addpattern(func, base_func, **kwargs) - return pattern_prepender''' - if not strict else - r'''def prepattern(*args, **kwargs): - """Deprecated Coconut built-in 'prepattern' disabled by --strict compilation; use 'addpattern' instead.""" - raise _coconut.NameError("deprecated Coconut built-in 'prepattern' disabled by --strict compilation; use 'addpattern' instead")''' - ), - def_datamaker=( - r'''def datamaker(data_type): + return pattern_prepender +def datamaker(data_type): """DEPRECATED: use makedata instead.""" - return _coconut.functools.partial(makedata, data_type)''' + return _coconut_partial(makedata, data_type) +of, parallel_map, concurrent_map, recursive_iterator = call, process_map, thread_map, recursive_generator + ''' if not strict else - r'''def datamaker(*args, **kwargs): + r''' +def prepattern(*args, **kwargs): + """Deprecated Coconut built-in 'prepattern' disabled by --strict compilation; use 'addpattern' instead.""" + raise _coconut.NameError("deprecated Coconut built-in 'prepattern' disabled by --strict compilation; use 'addpattern' instead") +def datamaker(*args, **kwargs): """Deprecated Coconut built-in 'datamaker' disabled by --strict compilation; use 'makedata' instead.""" - raise _coconut.NameError("deprecated Coconut built-in 'datamaker' disabled by --strict compilation; use 'makedata' instead")''' - ), - of_is_call=( - "of = call" if not strict else - r'''def of(*args, **kwargs): + raise _coconut.NameError("deprecated Coconut built-in 'datamaker' disabled by --strict compilation; use 'makedata' instead") +def of(*args, **kwargs): """Deprecated Coconut built-in 'of' disabled by --strict compilation; use 'call' instead.""" - raise _coconut.NameError("deprecated Coconut built-in 'of' disabled by --strict compilation; use 'call' instead")''' + raise _coconut.NameError("deprecated Coconut built-in 'of' disabled by --strict compilation; use 'call' instead") +def parallel_map(*args, **kwargs): + """Deprecated Coconut built-in 'parallel_map' disabled by --strict compilation; use 'process_map' instead.""" + raise _coconut.NameError("deprecated Coconut built-in 'parallel_map' disabled by --strict compilation; use 'process_map' instead") +def concurrent_map(*args, **kwargs): + """Deprecated Coconut built-in 'concurrent_map' disabled by --strict compilation; use 'thread_map' instead.""" + raise _coconut.NameError("deprecated Coconut built-in 'concurrent_map' disabled by --strict compilation; use 'thread_map' instead") +def recursive_iterator(*args, **kwargs): + """Deprecated Coconut built-in 'recursive_iterator' disabled by --strict compilation; use 'recursive_generator' instead.""" + raise _coconut.NameError("deprecated Coconut built-in 'recursive_iterator' disabled by --strict compilation; use 'recursive_generator' instead") + ''' ), return_method_of_self=pycondition( (3,), @@ -490,8 +559,9 @@ def __bool__(self): indent=1, newline=True, ), - def_async_compose_call=prepare( - r''' + def_async_compose_call=async_def( + "__call__", + async_def=r''' async def __call__(self, *args, **kwargs): arg = await self._coconut_func(*args, **kwargs) for f, await_f in self._coconut_func_infos: @@ -499,34 +569,23 @@ async def __call__(self, *args, **kwargs): if await_f: arg = await arg return arg - ''' if target_info >= (3, 5) else - pycondition( - (3, 5), - if_ge=r''' -_coconut_call_ns = {"_coconut": _coconut} -_coconut_exec("""async def __call__(self, *args, **kwargs): - arg = await self._coconut_func(*args, **kwargs) - for f, await_f in self._coconut_func_infos: - arg = f(arg) - if await_f: - arg = await arg - return arg""", _coconut_call_ns) -__call__ = _coconut_call_ns["__call__"] - ''', - if_lt=pycondition( - (3, 4), - if_ge=r''' -_coconut_call_ns = {"_coconut": _coconut} -_coconut_exec("""def __call__(self, *args, **kwargs): + ''', + no_async_def=pycondition( + (3, 4), + if_ge=def_in_exec( + "__call__", + r''' +def __call__(self, *args, **kwargs): arg = yield from self._coconut_func(*args, **kwargs) for f, await_f in self._coconut_func_infos: arg = f(arg) if await_f: arg = yield from arg - raise _coconut.StopIteration(arg)""", _coconut_call_ns) -__call__ = _coconut.asyncio.coroutine(_coconut_call_ns["__call__"]) + raise _coconut.StopIteration(arg) ''', - if_lt=''' + decorator="_coconut.asyncio.coroutine", + ), + if_lt=''' @_coconut.asyncio.coroutine def __call__(self, *args, **kwargs): arg = yield _coconut.asyncio.From(self._coconut_func(*args, **kwargs)) @@ -536,7 +595,6 @@ def __call__(self, *args, **kwargs): arg = yield _coconut.asyncio.From(arg) raise _coconut.asyncio.Return(arg) ''', - ), ), indent=1 ), @@ -545,26 +603,20 @@ def __call__(self, *args, **kwargs): tco_comma="_coconut_tail_call, _coconut_tco, " if not no_tco else "", call_set_names_comma="_coconut_call_set_names, " if target_info < (3, 6) else "", handle_cls_args_comma="_coconut_handle_cls_kwargs, _coconut_handle_cls_stargs, " if not target.startswith("3") else "", - async_def_anext=prepare( - r''' + async_def_anext=async_def( + "__anext__", + async_def=r''' async def __anext__(self): return self.func(await self.aiter.__anext__()) - ''' if target_info >= (3, 5) else - pycondition( - (3, 5), - if_ge=r''' -_coconut_anext_ns = {"_coconut": _coconut} -_coconut_exec("""async def __anext__(self): - return self.func(await self.aiter.__anext__())""", _coconut_anext_ns) -__anext__ = _coconut_anext_ns["__anext__"] - ''', - if_lt=r''' -_coconut_anext_ns = {"_coconut": _coconut} -_coconut_exec("""def __anext__(self): + ''', + no_async_def=def_in_exec( + "__anext__", + r''' +def __anext__(self): result = yield from self.aiter.__anext__() - return self.func(result)""", _coconut_anext_ns) -__anext__ = _coconut.asyncio.coroutine(_coconut_anext_ns["__anext__"]) + return self.func(result) ''', + decorator="_coconut.asyncio.coroutine", ), indent=1, ), @@ -586,7 +638,7 @@ async def __anext__(self): # (extra_format_dict is to keep indentation levels matching) extra_format_dict = dict( # when anything is added to this list it must also be added to *both* __coconut__ stub files - underscore_imports="{tco_comma}{call_set_names_comma}{handle_cls_args_comma}_namedtuple_of, _coconut, _coconut_Expected, _coconut_MatchError, _coconut_SupportsAdd, _coconut_SupportsMinus, _coconut_SupportsMul, _coconut_SupportsPow, _coconut_SupportsTruediv, _coconut_SupportsFloordiv, _coconut_SupportsMod, _coconut_SupportsAnd, _coconut_SupportsXor, _coconut_SupportsOr, _coconut_SupportsLshift, _coconut_SupportsRshift, _coconut_SupportsMatmul, _coconut_SupportsInv, _coconut_iter_getitem, _coconut_base_compose, _coconut_forward_compose, _coconut_back_compose, _coconut_forward_star_compose, _coconut_back_star_compose, _coconut_forward_dubstar_compose, _coconut_back_dubstar_compose, _coconut_pipe, _coconut_star_pipe, _coconut_dubstar_pipe, _coconut_back_pipe, _coconut_back_star_pipe, _coconut_back_dubstar_pipe, _coconut_none_pipe, _coconut_none_star_pipe, _coconut_none_dubstar_pipe, _coconut_bool_and, _coconut_bool_or, _coconut_none_coalesce, _coconut_minus, _coconut_map, _coconut_partial, _coconut_get_function_match_error, _coconut_base_pattern_func, _coconut_addpattern, _coconut_sentinel, _coconut_assert, _coconut_raise, _coconut_mark_as_match, _coconut_reiterable, _coconut_self_match_types, _coconut_dict_merge, _coconut_exec, _coconut_comma_op, _coconut_multi_dim_arr, _coconut_mk_anon_namedtuple, _coconut_matmul, _coconut_py_str, _coconut_flatten, _coconut_multiset, _coconut_back_none_pipe, _coconut_back_none_star_pipe, _coconut_back_none_dubstar_pipe, _coconut_forward_none_compose, _coconut_back_none_compose, _coconut_forward_none_star_compose, _coconut_back_none_star_compose, _coconut_forward_none_dubstar_compose, _coconut_back_none_dubstar_compose, _coconut_call_or_coefficient, _coconut_in, _coconut_not_in".format(**format_dict), + underscore_imports="{tco_comma}{call_set_names_comma}{handle_cls_args_comma}_namedtuple_of, _coconut, _coconut_Expected, _coconut_MatchError, _coconut_SupportsAdd, _coconut_SupportsMinus, _coconut_SupportsMul, _coconut_SupportsPow, _coconut_SupportsTruediv, _coconut_SupportsFloordiv, _coconut_SupportsMod, _coconut_SupportsAnd, _coconut_SupportsXor, _coconut_SupportsOr, _coconut_SupportsLshift, _coconut_SupportsRshift, _coconut_SupportsMatmul, _coconut_SupportsInv, _coconut_iter_getitem, _coconut_base_compose, _coconut_forward_compose, _coconut_back_compose, _coconut_forward_star_compose, _coconut_back_star_compose, _coconut_forward_dubstar_compose, _coconut_back_dubstar_compose, _coconut_pipe, _coconut_star_pipe, _coconut_dubstar_pipe, _coconut_back_pipe, _coconut_back_star_pipe, _coconut_back_dubstar_pipe, _coconut_none_pipe, _coconut_none_star_pipe, _coconut_none_dubstar_pipe, _coconut_bool_and, _coconut_bool_or, _coconut_none_coalesce, _coconut_minus, _coconut_map, _coconut_partial, _coconut_complex_partial, _coconut_get_function_match_error, _coconut_base_pattern_func, _coconut_addpattern, _coconut_sentinel, _coconut_assert, _coconut_raise, _coconut_mark_as_match, _coconut_reiterable, _coconut_self_match_types, _coconut_dict_merge, _coconut_exec, _coconut_comma_op, _coconut_multi_dim_arr, _coconut_mk_anon_namedtuple, _coconut_matmul, _coconut_py_str, _coconut_flatten, _coconut_multiset, _coconut_back_none_pipe, _coconut_back_none_star_pipe, _coconut_back_none_dubstar_pipe, _coconut_forward_none_compose, _coconut_back_none_compose, _coconut_forward_none_star_compose, _coconut_back_none_star_compose, _coconut_forward_none_dubstar_compose, _coconut_back_none_dubstar_compose, _coconut_call_or_coefficient, _coconut_in, _coconut_not_in, _coconut_attritemgetter".format(**format_dict), import_typing=pycondition( (3, 5), if_ge=''' @@ -705,10 +757,10 @@ def Return(self, obj): ), class_amap=pycondition( (3, 3), - if_lt=r''' + if_lt=''' _coconut_amap = None ''', - if_ge=r''' + if_ge=''' class _coconut_amap(_coconut_baseclass): __slots__ = ("func", "aiter") def __init__(self, func, aiter): @@ -776,6 +828,31 @@ def __neg__(self): '''.format(**format_dict), indent=1, ), + def_async_map=async_def( + "async_map", + async_def=''' +async def async_map(async_func, *iters, strict=False): + """Map async_func over iters asynchronously using anyio.""" + import anyio + results = [] + async def store_func_in_of(i, args): + got = await async_func(*args) + results.extend([None] * (1 + i - _coconut.len(results))) + results[i] = got + async with anyio.create_task_group() as nursery: + for i, args in _coconut.enumerate({_coconut_}zip(*iters, strict=strict)): + nursery.start_soon(store_func_in_of, i, args) + return results + '''.format(**format_dict), + no_async_def=''' +def async_map(*args, **kwargs): + """async_map not available on Python < 3.5""" + raise _coconut.NameError("async_map not available on Python < 3.5") + ''', + needs_vars={ + "{_coconut_}zip".format(**format_dict): "zip", + }, + ), ) format_dict.update(extra_format_dict) @@ -787,6 +864,7 @@ def __neg__(self): # ----------------------------------------------------------------------------------------------------------------------- +@memoize() def getheader(which, use_hash, target, no_tco, strict, no_wrap): """Generate the specified header. @@ -934,18 +1012,15 @@ def getheader(which, use_hash, target, no_tco, strict, no_wrap): newline=True, ).format(**format_dict) - if target_info >= (3, 9): - header += _get_root_header("39") - if target_info >= (3, 7): - header += _get_root_header("37") - elif target.startswith("3"): - header += _get_root_header("3") - elif target_info >= (2, 7): - header += _get_root_header("27") - elif target.startswith("2"): - header += _get_root_header("2") - else: - header += _get_root_header("universal") + header += _get_root_header( + "311" if target_info >= (3, 11) + else "39" if target_info >= (3, 9) + else "37" if target_info >= (3, 7) + else "3" if target.startswith("3") + else "27" if target_info >= (2, 7) + else "2" if target.startswith("2") + else "universal" + ) header += get_template("header").format(**format_dict) diff --git a/coconut/compiler/templates/header.py_template b/coconut/compiler/templates/header.py_template index 845f6265b..e7ec5f6f1 100644 --- a/coconut/compiler/templates/header.py_template +++ b/coconut/compiler/templates/header.py_template @@ -13,13 +13,13 @@ def _coconut_super(type=None, object_or_type=None): try: cls = frame.f_locals["__class__"] except _coconut.AttributeError: - raise _coconut.RuntimeError("super(): __class__ cell not found") + raise _coconut.RuntimeError("super(): __class__ cell not found"){from_None} self = frame.f_locals[frame.f_code.co_varnames[0]] return _coconut_py_super(cls, self) return _coconut_py_super(type, object_or_type) {set_super} class _coconut{object}:{COMMENT.EVERYTHING_HERE_MUST_BE_COPIED_TO_STUB_FILE} - import collections, copy, functools, types, itertools, operator, threading, os, warnings, contextlib, traceback, weakref, multiprocessing + import collections, copy, functools, types, itertools, operator, threading, os, warnings, contextlib, traceback, weakref, multiprocessing, inspect from multiprocessing import dummy as multiprocessing_dummy {maybe_bind_lru_cache}{import_copyreg} {import_asyncio} @@ -61,6 +61,11 @@ class _coconut{object}:{COMMENT.EVERYTHING_HERE_MUST_BE_COPIED_TO_STUB_FILE} fmappables = list, tuple, dict, set, frozenset abc.Sequence.register(collections.deque) Ellipsis, NotImplemented, NotImplementedError, Exception, AttributeError, ImportError, IndexError, KeyError, NameError, TypeError, ValueError, StopIteration, RuntimeError, all, any, bool, bytes, callable, classmethod, complex, dict, enumerate, filter, float, frozenset, getattr, hasattr, hash, id, int, isinstance, issubclass, iter, len, list, locals, globals, map, min, max, next, object, property, range, reversed, set, setattr, slice, str, sum, super, tuple, type, vars, zip, repr, print{comma_bytearray} = Ellipsis, NotImplemented, NotImplementedError, Exception, AttributeError, ImportError, IndexError, KeyError, NameError, TypeError, ValueError, StopIteration, RuntimeError, all, any, bool, bytes, callable, classmethod, complex, dict, enumerate, filter, float, frozenset, getattr, hasattr, hash, id, int, isinstance, issubclass, iter, len, list, locals, globals, map, min, max, next, object, property, range, reversed, set, setattr, slice, str, sum, {lstatic}super{rstatic}, tuple, type, vars, zip, {lstatic}repr{rstatic}, {lstatic}print{rstatic}{comma_bytearray} +@_coconut.functools.wraps(_coconut.functools.partial) +def _coconut_partial(_coconut_func, *args, **kwargs): + partial_func = _coconut.functools.partial(_coconut_func, *args, **kwargs) + partial_func.__name__ = _coconut.getattr(_coconut_func, "__name__", None) + return partial_func def _coconut_handle_cls_kwargs(**kwargs): """Some code taken from six under the terms of its MIT license.""" metaclass = kwargs.pop("metaclass", None) @@ -103,6 +108,12 @@ class _coconut_baseclass{object}: if getitem is None: raise _coconut.NotImplementedError return getitem(index) +class _coconut_base_callable(_coconut_baseclass): + __slots__ = () + def __get__(self, obj, objtype=None): + if obj is None: + return self +{return_method_of_self} class _coconut_Sentinel(_coconut_baseclass): __slots__ = () def __reduce__(self): @@ -150,7 +161,7 @@ class _coconut_tail_call(_coconut_baseclass): self.kwargs = kwargs def __reduce__(self): return (self.__class__, (self.func, self.args, self.kwargs)) -_coconut_tco_func_dict = {empty_dict} +_coconut_tco_func_dict = _coconut.weakref.WeakValueDictionary() def _coconut_tco(func): @_coconut.functools.wraps(func) def tail_call_optimized_func(*args, **kwargs): @@ -159,16 +170,14 @@ def _coconut_tco(func): if _coconut.isinstance(call_func, _coconut_base_pattern_func): call_func = call_func._coconut_tco_func elif _coconut.isinstance(call_func, _coconut.types.MethodType): - wkref = _coconut_tco_func_dict.get(_coconut.id(call_func.__func__)) - wkref_func = None if wkref is None else wkref() + wkref_func = _coconut_tco_func_dict.get(_coconut.id(call_func.__func__)) if wkref_func is call_func.__func__: if call_func.__self__ is None: call_func = call_func._coconut_tco_func else: - call_func = _coconut.functools.partial(call_func._coconut_tco_func, call_func.__self__) + call_func = _coconut_partial(call_func._coconut_tco_func, call_func.__self__) else: - wkref = _coconut_tco_func_dict.get(_coconut.id(call_func)) - wkref_func = None if wkref is None else wkref() + wkref_func = _coconut_tco_func_dict.get(_coconut.id(call_func)) if wkref_func is call_func: call_func = call_func._coconut_tco_func result = call_func(*args, **kwargs) # use 'coconut --no-tco' to clean up your traceback @@ -179,12 +188,12 @@ def _coconut_tco(func): tail_call_optimized_func.__module__ = _coconut.getattr(func, "__module__", None) tail_call_optimized_func.__name__ = _coconut.getattr(func, "__name__", None) tail_call_optimized_func.__qualname__ = _coconut.getattr(func, "__qualname__", None) - _coconut_tco_func_dict[_coconut.id(tail_call_optimized_func)] = _coconut.weakref.ref(tail_call_optimized_func) + _coconut_tco_func_dict[_coconut.id(tail_call_optimized_func)] = tail_call_optimized_func return tail_call_optimized_func @_coconut.functools.wraps(_coconut.itertools.tee) def tee(iterable, n=2): if n < 0: - raise ValueError("tee: n cannot be negative") + raise _coconut.ValueError("tee: n cannot be negative") elif n == 0: return () elif n == 1: @@ -205,16 +214,14 @@ def tee(iterable, n=2): return _coconut.tuple(existing_copies) return _coconut.itertools.tee(iterable, n) class _coconut_has_iter(_coconut_baseclass): - __slots__ = ("lock", "iter") + __slots__ = ("iter",) def __new__(cls, iterable): self = _coconut.super(_coconut_has_iter, cls).__new__(cls) - self.lock = _coconut.threading.Lock() self.iter = iterable return self def get_new_iter(self): """Tee the underlying iterator.""" - with self.lock: - self.iter = {_coconut_}reiterable(self.iter) + self.iter = {_coconut_}reiterable(self.iter) return self.iter def __fmap__(self, func): return {_coconut_}map(func, self) @@ -227,8 +234,7 @@ class reiterable(_coconut_has_iter): return _coconut.super({_coconut_}reiterable, cls).__new__(cls, iterable) def get_new_iter(self): """Tee the underlying iterator.""" - with self.lock: - self.iter, new_iter = {_coconut_}tee(self.iter) + self.iter, new_iter = {_coconut_}tee(self.iter) return new_iter def __iter__(self): return _coconut.iter(self.get_new_iter()) @@ -354,7 +360,26 @@ def _coconut_iter_getitem(iterable, index): return () iterable = _coconut.itertools.islice(iterable, 0, n) return _coconut.tuple(iterable)[i::step] -class _coconut_compostion_baseclass(_coconut_baseclass):{COMMENT.no_slots_to_allow_update_wrapper}{COMMENT.must_use_coconut_attrs_to_avoid_interacting_with_update_wrapper} +class _coconut_attritemgetter(_coconut_base_callable): + __slots__ = ("attr", "is_iter_and_items") + def __init__(self, attr, *is_iter_and_items): + self.attr = attr + self.is_iter_and_items = is_iter_and_items + def __call__(self, obj): + out = obj + if self.attr is not None: + out = _coconut.getattr(out, self.attr) + for is_iter, item in self.is_iter_and_items: + if is_iter: + out = _coconut_iter_getitem(out, item) + else: + out = out[item] + return out + def __repr__(self): + return "." + (self.attr or "") + "".join(("$" if is_iter else "") + "[" + _coconut.repr(item) + "]" for is_iter, item in self.is_iter_and_items) + def __reduce__(self): + return (self.__class__, (self.attr,) + self.is_iter_and_items) +class _coconut_compostion_baseclass(_coconut_base_callable):{COMMENT.no_slots_to_allow_update_wrapper}{COMMENT.must_use_coconut_attrs_to_avoid_interacting_with_update_wrapper} def __init__(self, func, *func_infos): try: _coconut.functools.update_wrapper(self, func) @@ -376,10 +401,6 @@ class _coconut_compostion_baseclass(_coconut_baseclass):{COMMENT.no_slots_to_all self._coconut_func_infos = _coconut.tuple(self._coconut_func_infos) def __reduce__(self): return (self.__class__, (self._coconut_func,) + self._coconut_func_infos) - def __get__(self, obj, objtype=None): - if obj is None: - return self -{return_method_of_self} class _coconut_base_compose(_coconut_compostion_baseclass): __slots__ = () def __call__(self, *args, **kwargs): @@ -648,14 +669,13 @@ class flatten(_coconut_has_iter):{COMMENT.cant_implement_len_else_list_calls_bec return self def get_new_iter(self): """Tee the underlying iterator.""" - with self.lock: - if not self._made_reit: - for i in _coconut.reversed(_coconut.range(0 if self.levels is None else self.levels + 1)): - mapper = {_coconut_}reiterable - for _ in _coconut.range(i): - mapper = _coconut.functools.partial({_coconut_}map, mapper) - self.iter = mapper(self.iter) - self._made_reit = True + if not self._made_reit: + for i in _coconut.reversed(_coconut.range(0 if self.levels is None else self.levels + 1)): + mapper = {_coconut_}reiterable + for _ in _coconut.range(i): + mapper = _coconut.functools.partial({_coconut_}map, mapper) + self.iter = mapper(self.iter) + self._made_reit = True return self.iter def __iter__(self): if self.levels is None: @@ -707,10 +727,10 @@ class flatten(_coconut_has_iter):{COMMENT.cant_implement_len_else_list_calls_bec return ind + it.index(elem) except _coconut.ValueError: ind += _coconut.len(it) - raise ValueError("%r not in %r" % (elem, self)) + raise _coconut.ValueError("%r not in %r" % (elem, self)) def __fmap__(self, func): if self.levels == 1: - return self.__class__({_coconut_}map(_coconut.functools.partial({_coconut_}map, func), self.get_new_iter())) + return self.__class__({_coconut_}map(_coconut_partial({_coconut_}map, func), self.get_new_iter())) return {_coconut_}map(func, self) class cartesian_product(_coconut_baseclass): __slots__ = ("iters", "repeat") @@ -812,10 +832,10 @@ class map(_coconut_baseclass, _coconut.map): self.iters = _coconut.tuple({_coconut_}reiterable(it) for it in self.iters) return self.__class__(self.func, *self.iters) def __iter__(self): - return _coconut.iter(_coconut.map(self.func, *self.iters)) + return _coconut.map(self.func, *self.iters) def __fmap__(self, func): return self.__class__(_coconut_forward_compose(self.func, func), *self.iters) -class _coconut_parallel_concurrent_map_func_wrapper(_coconut_baseclass): +class _coconut_parallel_map_func_wrapper(_coconut_baseclass): __slots__ = ("map_cls", "func", "star") def __init__(self, map_cls, func, star): self.map_cls = map_cls @@ -824,10 +844,10 @@ class _coconut_parallel_concurrent_map_func_wrapper(_coconut_baseclass): def __reduce__(self): return (self.__class__, (self.map_cls, self.func, self.star)) def __call__(self, *args, **kwargs): - self.map_cls.get_pool_stack().append(None) + self.map_cls._get_pool_stack().append(None) try: if self.star: - assert _coconut.len(args) == 1, "internal parallel/concurrent map error {report_this_text}" + assert _coconut.len(args) == 1, "internal process_map/thread_map error {report_this_text}" return self.func(*args[0], **kwargs) else: return self.func(*args, **kwargs) @@ -836,73 +856,96 @@ class _coconut_parallel_concurrent_map_func_wrapper(_coconut_baseclass): _coconut.traceback.print_exc() raise finally: - assert self.map_cls.get_pool_stack().pop() is None, "internal parallel/concurrent map error {report_this_text}" -class _coconut_base_parallel_concurrent_map(map): - __slots__ = ("result", "chunksize", "strict") + assert self.map_cls._get_pool_stack().pop() is None, "internal process_map/thread_map error {report_this_text}" +class _coconut_base_parallel_map(map): + __slots__ = ("result", "chunksize", "strict", "stream", "ordered") @classmethod - def get_pool_stack(cls): - return cls.threadlocal_ns.__dict__.setdefault("pool_stack", [None]) + def _get_pool_stack(cls): + return cls._threadlocal_ns.__dict__.setdefault("pool_stack", [None]) def __new__(cls, function, *iterables, **kwargs): - self = _coconut.super(_coconut_base_parallel_concurrent_map, cls).__new__(cls, function, *iterables) + self = _coconut.super(_coconut_base_parallel_map, cls).__new__(cls, function, *iterables) self.result = None self.chunksize = kwargs.pop("chunksize", 1) self.strict = kwargs.pop("strict", False) + self.stream = kwargs.pop("stream", False) + self.ordered = kwargs.pop("ordered", True) if kwargs: raise _coconut.TypeError(cls.__name__ + "() got unexpected keyword arguments " + _coconut.repr(kwargs)) - if cls.get_pool_stack()[-1] is not None: - return self.get_list() + if not self.stream and cls._get_pool_stack()[-1] is not None: + return self.to_tuple() return self + def __reduce__(self): + return (self.__class__, (self.func,) + self.iters, {lbrace}"chunksize": self.chunksize, "strict": self.strict, "stream": self.stream, "ordered": self.ordered{rbrace}) @classmethod @_coconut.contextlib.contextmanager def multiple_sequential_calls(cls, max_workers=None): """Context manager that causes nested calls to use the same pool.""" - if cls.get_pool_stack()[-1] is None: - cls.get_pool_stack()[-1] = cls.make_pool(max_workers) + if cls._get_pool_stack()[-1] is None: + cls._get_pool_stack()[-1] = cls._make_pool(max_workers) + try: + yield + finally: + cls._get_pool_stack()[-1].terminate() + cls._get_pool_stack()[-1] = None + elif max_workers is not None: + self.map_cls._get_pool_stack().append(cls._make_pool(max_workers)) try: yield finally: - cls.get_pool_stack()[-1].terminate() - cls.get_pool_stack()[-1] = None + cls._get_pool_stack()[-1].terminate() + cls._get_pool_stack().pop() else: yield - def get_list(self): + def _execute_map(self): + map_func = self._get_pool_stack()[-1].imap if self.ordered else self._get_pool_stack()[-1].imap_unordered + if _coconut.len(self.iters) == 1: + return map_func(_coconut_parallel_map_func_wrapper(self.__class__, self.func, False), self.iters[0], self.chunksize) + elif self.strict: + return map_func(_coconut_parallel_map_func_wrapper(self.__class__, self.func, True), {_coconut_}zip(*self.iters, strict=True), self.chunksize) + else: + return map_func(_coconut_parallel_map_func_wrapper(self.__class__, self.func, True), _coconut.zip(*self.iters), self.chunksize) + def to_tuple(self): + """Execute the map operation and return the results as a tuple.""" if self.result is None: with self.multiple_sequential_calls(): - if _coconut.len(self.iters) == 1: - self.result = _coconut.list(self.get_pool_stack()[-1].imap(_coconut_parallel_concurrent_map_func_wrapper(self.__class__, self.func, False), self.iters[0], self.chunksize)) - elif self.strict: - self.result = _coconut.list(self.get_pool_stack()[-1].imap(_coconut_parallel_concurrent_map_func_wrapper(self.__class__, self.func, True), {_coconut_}zip(*self.iters, strict=True), self.chunksize)) - else: - self.result = _coconut.list(self.get_pool_stack()[-1].imap(_coconut_parallel_concurrent_map_func_wrapper(self.__class__, self.func, True), _coconut.zip(*self.iters), self.chunksize)) + self.result = _coconut.tuple(self._execute_map()) self.func = {_coconut_}ident self.iters = (self.result,) return self.result + def to_stream(self): + """Stream the map operation, yielding results one at a time.""" + if self._get_pool_stack()[-1] is None: + raise _coconut.RuntimeError("cannot stream outside of " + cls.__name__ + ".multiple_sequential_calls context") + return self._execute_map() def __iter__(self): - return _coconut.iter(self.get_list()) -class parallel_map(_coconut_base_parallel_concurrent_map): + if self.stream: + return self.to_stream() + else: + return _coconut.iter(self.to_tuple()){COMMENT.have_to_to_tuple_so_finishes_before_return_else_cant_manage_context} +class process_map(_coconut_base_parallel_map): """Multi-process implementation of map. Requires arguments to be pickleable. For multiple sequential calls, use: - with parallel_map.multiple_sequential_calls(): + with process_map.multiple_sequential_calls(): ... """ __slots__ = () - threadlocal_ns = _coconut.threading.local() + _threadlocal_ns = _coconut.threading.local() @staticmethod - def make_pool(max_workers=None): + def _make_pool(max_workers=None): return _coconut.multiprocessing.Pool(max_workers) -class concurrent_map(_coconut_base_parallel_concurrent_map): +class thread_map(_coconut_base_parallel_map): """Multi-thread implementation of map. For multiple sequential calls, use: - with concurrent_map.multiple_sequential_calls(): + with thread_map.multiple_sequential_calls(): ... """ __slots__ = () - threadlocal_ns = _coconut.threading.local() + _threadlocal_ns = _coconut.threading.local() @staticmethod - def make_pool(max_workers=None): - return _coconut.multiprocessing_dummy.Pool(_coconut.multiprocessing.cpu_count() * 5 if max_workers is None else max_workers) + def _make_pool(max_workers=None): + return _coconut.multiprocessing_dummy.Pool(_coconut.multiprocessing.{process_}cpu_count() * 5 if max_workers is None else max_workers) class zip(_coconut_baseclass, _coconut.zip): __slots__ = ("iters", "strict") __doc__ = getattr(_coconut.zip, "__doc__", "") @@ -1278,53 +1321,40 @@ class groupsof(_coconut_has_iter): return (self.__class__, (self.group_size, self.iter)) def __copy__(self): return self.__class__(self.group_size, self.get_new_iter()) -class recursive_iterator(_coconut_baseclass): - """Decorator that memoizes a recursive function that returns an iterator (e.g. a recursive generator).""" - __slots__ = ("func", "reit_store", "backup_reit_store") +class recursive_generator(_coconut_base_callable): + """Decorator that memoizes a generator (or any function that returns an iterator). + Particularly useful for recursive generators, which may require recursive_generator to function properly.""" + __slots__ = ("func", "reit_store") def __init__(self, func): self.func = func self.reit_store = {empty_dict} - self.backup_reit_store = [] def __call__(self, *args, **kwargs): - key = (args, _coconut.frozenset(kwargs.items())) - use_backup = False + key = (0, args, _coconut.frozenset(kwargs.items())) try: _coconut.hash(key) - except _coconut.Exception: + except _coconut.TypeError: try: - key = _coconut.pickle.dumps(key, -1) + key = (1, _coconut.pickle.dumps(key, -1)) except _coconut.Exception: - use_backup = True - if use_backup: - for k, v in self.backup_reit_store: - if k == key: - return reit + raise _coconut.TypeError("recursive_generator() requires function arguments to be hashable or pickleable"){from_None} + reit = self.reit_store.get(key) + if reit is None: reit = {_coconut_}reiterable(self.func(*args, **kwargs)) - self.backup_reit_store.append([key, reit]) - return reit - else: - reit = self.reit_store.get(key) - if reit is None: - reit = {_coconut_}reiterable(self.func(*args, **kwargs)) - self.reit_store[key] = reit - return reit + self.reit_store[key] = reit + return reit def __repr__(self): - return "recursive_iterator(%r)" % (self.func,) + return "recursive_generator(%r)" % (self.func,) def __reduce__(self): return (self.__class__, (self.func,)) - def __get__(self, obj, objtype=None): - if obj is None: - return self -{return_method_of_self} class _coconut_FunctionMatchErrorContext(_coconut_baseclass): __slots__ = ("exc_class", "taken") - threadlocal_ns = _coconut.threading.local() + _threadlocal_ns = _coconut.threading.local() def __init__(self, exc_class): self.exc_class = exc_class self.taken = False @classmethod def get_contexts(cls): - return cls.threadlocal_ns.__dict__.setdefault("contexts", []) + return cls._threadlocal_ns.__dict__.setdefault("contexts", []) def __enter__(self): self.get_contexts().append(self) def __exit__(self, type, value, traceback): @@ -1340,7 +1370,7 @@ def _coconut_get_function_match_error(): return {_coconut_}MatchError ctx.taken = True return ctx.exc_class -class _coconut_base_pattern_func(_coconut_baseclass):{COMMENT.no_slots_to_allow_func_attrs} +class _coconut_base_pattern_func(_coconut_base_callable):{COMMENT.no_slots_to_allow_func_attrs} _coconut_is_match = True def __init__(self, *funcs): self.FunctionMatchError = _coconut.type(_coconut_py_str("MatchError"), ({_coconut_}MatchError,), {empty_py_dict}) @@ -1378,15 +1408,11 @@ class _coconut_base_pattern_func(_coconut_baseclass):{COMMENT.no_slots_to_allow_ return "addpattern(%r)(*%r)" % (self.patterns[0], self.patterns[1:]) def __reduce__(self): return (self.__class__, _coconut.tuple(self.patterns)) - def __get__(self, obj, objtype=None): - if obj is None: - return self -{return_method_of_self} def _coconut_mark_as_match(base_func):{COMMENT._coconut_is_match_is_used_above_and_in_main_coco} base_func._coconut_is_match = True return base_func def addpattern(base_func, *add_funcs, **kwargs): - """Decorator to add a new case to a pattern-matching function (where the new case is checked last). + """Decorator to add new cases to a pattern-matching function (where the new case is checked last). Pass allow_any_func=True to allow any object as the base_func rather than just pattern-matching functions. If add_funcs are passed, addpattern(base_func, add_func) is equivalent to addpattern(base_func)(add_func). @@ -1398,11 +1424,10 @@ def addpattern(base_func, *add_funcs, **kwargs): raise _coconut.TypeError("addpattern() got unexpected keyword arguments " + _coconut.repr(kwargs)) if add_funcs: return _coconut_base_pattern_func(base_func, *add_funcs) - return _coconut.functools.partial(_coconut_base_pattern_func, base_func) + return _coconut_partial(_coconut_base_pattern_func, base_func) _coconut_addpattern = addpattern -{def_prepattern} -class _coconut_partial(_coconut_baseclass): - __slots__ = ("func", "_argdict", "_arglen", "_pos_kwargs", "_stargs", "keywords") +class _coconut_complex_partial(_coconut_base_callable): + __slots__ = ("func", "_argdict", "_arglen", "_pos_kwargs", "_stargs", "keywords", "__name__") def __init__(self, _coconut_func, _coconut_argdict, _coconut_arglen, _coconut_pos_kwargs, *args, **kwargs): self.func = _coconut_func self._argdict = _coconut_argdict @@ -1410,6 +1435,7 @@ class _coconut_partial(_coconut_baseclass): self._pos_kwargs = _coconut_pos_kwargs self._stargs = args self.keywords = kwargs + self.__name__ = _coconut.getattr(_coconut_func, "__name__", None) def __reduce__(self): return (self.__class__, (self.func, self._argdict, self._arglen, self._pos_kwargs) + self._stargs, {lbrace}"keywords": self.keywords{rbrace}) @property @@ -1494,22 +1520,21 @@ class multiset(_coconut.collections.Counter{comma_object}): def add(self, item): """Add an element to a multiset.""" self[item] += 1 - def discard(self, item): - """Remove an element from a multiset if it is a member.""" - item_count = self[item] - if item_count > 0: - self[item] = item_count - 1 - if item_count - 1 <= 0: - del self[item] - def remove(self, item): + def remove(self, item, **kwargs): """Remove an element from a multiset; it must be a member.""" + allow_missing = kwargs.pop("allow_missing", False) + if kwargs: + raise _coconut.TypeError("multiset.remove() got unexpected keyword arguments " + _coconut.repr(kwargs)) item_count = self[item] if item_count > 0: self[item] = item_count - 1 if item_count - 1 <= 0: del self[item] - else: + elif not allow_missing: raise _coconut.KeyError(item) + def discard(self, item): + """Remove an element from a multiset if it is a member.""" + return self.remove(item, allow_missing=True) def isdisjoint(self, other): """Return True if two multisets have a null intersection.""" return not self & other @@ -1549,7 +1574,6 @@ def makedata(data_type, *args, **kwargs): if kwargs: raise _coconut.TypeError("makedata() got unexpected keyword arguments " + _coconut.repr(kwargs)) return _coconut_base_makedata(data_type, args, fallback_to_init=fallback_to_init) -{def_datamaker} {class_amap} def fmap(func, obj, **kwargs): """fmap(func, obj) creates a copy of obj with func applied to its contents. @@ -1619,6 +1643,12 @@ class override(_coconut_baseclass): def __init__(self, func): self.func = func def __get__(self, obj, objtype=None): + self_func_get = _coconut.getattr(self.func, "__get__", None) + if self_func_get is not None: + if objtype is None: + return self_func_get(obj) + else: + return self_func_get(obj, objtype) if obj is None: return self.func {return_method_of_self_func} @@ -1663,7 +1693,6 @@ def call(_coconut_f{comma_slash}, *args, **kwargs): def call(f, /, *args, **kwargs) = f(*args, **kwargs). """ return _coconut_f(*args, **kwargs) -{of_is_call} def safe_call(_coconut_f{comma_slash}, *args, **kwargs): """safe_call is a version of call that catches any Exceptions and returns an Expected containing either the result or the error. @@ -1688,6 +1717,10 @@ class Expected(_coconut.collections.namedtuple("Expected", ("result", "error")){ def __bool__(self) -> bool: return self.error is None def __fmap__[U](self, func: T -> U) -> Expected[U]: + """Maps func over the result if it exists. + + __fmap__ should be used directly only when fmap is not available (e.g. when consuming an Expected in vanilla Python). + """ return self.__class__(func(self.result)) if self else self def and_then[U](self, func: T -> Expected[U]) -> Expected[U]: """Maps a T -> Expected[U] over an Expected[T] to produce an Expected[U]. @@ -1703,20 +1736,34 @@ class Expected(_coconut.collections.namedtuple("Expected", ("result", "error")){ def map_error(self, func: BaseException -> BaseException) -> Expected[T]: """Maps func over the error if it exists.""" return self if self else self.__class__(error=func(self.error)) + def handle(self, err_type, handler: BaseException -> T) -> Expected[T]: + """Recover from the given err_type by calling handler on the error to determine the result.""" + if not self and isinstance(self.error, err_type): + return self.__class__(handler(self.error)) + return self + def expect_error(self, *err_types: BaseException) -> Expected[T]: + """Raise any errors that do not match the given error types.""" + if not self and not isinstance(self.error, err_types): + raise self.error + return self + def unwrap(self) -> T: + """Unwrap the result or raise the error.""" + if not self: + raise self.error + return self.result def or_else[U](self, func: BaseException -> Expected[U]) -> Expected[T | U]: """Return self if no error, otherwise return the result of evaluating func on the error.""" return self if self else func(self.error) - def result_or[U](self, default: U) -> T | U: - """Return the result if it exists, otherwise return the default.""" - return self.result if self else default def result_or_else[U](self, func: BaseException -> U) -> T | U: """Return the result if it exists, otherwise return the result of evaluating func on the error.""" return self.result if self else func(self.error) - def unwrap(self) -> T: - """Unwrap the result or raise the error.""" - if not self: - raise self.error - return self.result + def result_or[U](self, default: U) -> T | U: + """Return the result if it exists, otherwise return the default. + + Since .result_or() completely silences errors, it is highly recommended that you + call .expect_error() first to explicitly declare what errors you are okay silencing. + """ + return self.result if self else default ''' __slots__ = () {is_data_var} = True @@ -1760,6 +1807,21 @@ class Expected(_coconut.collections.namedtuple("Expected", ("result", "error")){ def map_error(self, func): """Maps func over the error if it exists.""" return self if self else self.__class__(error=func(self.error)) + def handle(self, err_type, handler): + """Recover from the given err_type by calling handler on the error to determine the result.""" + if not self and _coconut.isinstance(self.error, err_type): + return self.__class__(handler(self.error)) + return self + def expect_error(self, *err_types): + """Raise any errors that do not match the given error types.""" + if not self and not _coconut.isinstance(self.error, err_types): + raise self.error + return self + def unwrap(self): + """Unwrap the result or raise the error.""" + if not self: + raise self.error + return self.result def or_else(self, func): """Return self if no error, otherwise return the result of evaluating func on the error.""" if self: @@ -1768,18 +1830,17 @@ class Expected(_coconut.collections.namedtuple("Expected", ("result", "error")){ if not _coconut.isinstance(got, {_coconut_}Expected): raise _coconut.TypeError("Expected.or_else() requires a function that returns an Expected") return got - def result_or(self, default): - """Return the result if it exists, otherwise return the default.""" - return self.result if self else default def result_or_else(self, func): """Return the result if it exists, otherwise return the result of evaluating func on the error.""" return self.result if self else func(self.error) - def unwrap(self): - """Unwrap the result or raise the error.""" - if not self: - raise self.error - return self.result -class flip(_coconut_baseclass): + def result_or(self, default): + """Return the result if it exists, otherwise return the default. + + Since .result_or() completely silences errors, it is highly recommended that you + call .expect_error() first to explicitly declare what errors you are okay silencing. + """ + return self.result if self else default +class flip(_coconut_base_callable): """Given a function, return a new function with inverse argument order. If nargs is passed, only the first nargs arguments are reversed.""" __slots__ = ("func", "nargs") @@ -1801,7 +1862,7 @@ class flip(_coconut_baseclass): return self.func(*(args[self.nargs-1::-1] + args[self.nargs:]), **kwargs) def __repr__(self): return "flip(%r%s)" % (self.func, "" if self.nargs is None else ", " + _coconut.repr(self.nargs)) -class const(_coconut_baseclass): +class const(_coconut_base_callable): """Create a function that, whatever its arguments, just returns the given value.""" __slots__ = ("value",) def __init__(self, value): @@ -1812,7 +1873,7 @@ class const(_coconut_baseclass): return self.value def __repr__(self): return "const(%s)" % (_coconut.repr(self.value),) -class _coconut_lifted(_coconut_baseclass): +class _coconut_lifted(_coconut_base_callable): __slots__ = ("func", "func_args", "func_kwargs") def __init__(self, _coconut_func, *func_args, **func_kwargs): self.func = _coconut_func @@ -1824,13 +1885,13 @@ class _coconut_lifted(_coconut_baseclass): return self.func(*(g(*args, **kwargs) for g in self.func_args), **_coconut_py_dict((k, h(*args, **kwargs)) for k, h in self.func_kwargs.items())) def __repr__(self): return "lift(%r)(%s%s)" % (self.func, ", ".join(_coconut.repr(g) for g in self.func_args), ", ".join(k + "=" + _coconut.repr(h) for k, h in self.func_kwargs.items())) -class lift(_coconut_baseclass): - """Lifts a function up so that all of its arguments are functions. +class lift(_coconut_base_callable): + """Lift a function up so that all of its arguments are functions. For a binary function f(x, y) and two unary functions g(z) and h(z), lift works as the S' combinator: lift(f)(g, h)(z) == f(g(z), h(z)) - In general, lift is requivalent to: + In general, lift is equivalent to: def lift(f) = ((*func_args, **func_kwargs) -> (*args, **kwargs) -> f(*(g(*args, **kwargs) for g in func_args), **{lbrace}k: h(*args, **kwargs) for k, h in func_kwargs.items(){rbrace})) @@ -1845,10 +1906,10 @@ class lift(_coconut_baseclass): return self def __reduce__(self): return (self.__class__, (self.func,)) - def __call__(self, *func_args, **func_kwargs): - return _coconut_lifted(self.func, *func_args, **func_kwargs) def __repr__(self): return "lift(%r)" % (self.func,) + def __call__(self, *func_args, **func_kwargs): + return _coconut_lifted(self.func, *func_args, **func_kwargs) def all_equal(iterable): """For a given iterable, check whether all elements in that iterable are equal to each other. @@ -1866,27 +1927,56 @@ def all_equal(iterable): elif first_item != item: return False return True -def collectby(key_func, iterable, value_func=None, reduce_func=None): - """Collect the items in iterable into a dictionary of lists keyed by key_func(item). +def mapreduce(key_value_func, iterable, **kwargs): + """Map key_value_func over iterable, then collect the values into a dictionary of lists keyed by the keys. - if value_func is passed, collect value_func(item) into each list instead of item. + If reduce_func is passed, instead of collecting the values into lists, reduce over + the values for each key with reduce_func, effectively implementing a MapReduce operation. - If reduce_func is passed, instead of collecting the items into lists, reduce over - the items of each key with reduce_func, effectively implementing a MapReduce operation. + If collect_in is passed, initialize the collection from . """ - collection = _coconut.collections.defaultdict(_coconut.list) if reduce_func is None else {empty_dict} - for item in iterable: - key = key_func(item) - if value_func is not None: - item = value_func(item) + collect_in = kwargs.pop("collect_in", None) + reduce_func = kwargs.pop("reduce_func", None if collect_in is None else False) + reduce_func_init = kwargs.pop("reduce_func_init", _coconut_sentinel) + if reduce_func_init is not _coconut_sentinel and not reduce_func: + raise _coconut.TypeError("reduce_func_init requires reduce_func") + map_using = kwargs.pop("map_using", _coconut.map) + if kwargs: + raise _coconut.TypeError("mapreduce()/collectby() got unexpected keyword arguments " + _coconut.repr(kwargs)) + collection = collect_in if collect_in is not None else _coconut.collections.defaultdict(_coconut.list) if reduce_func is None else {empty_dict} + for key, val in map_using(key_value_func, iterable): if reduce_func is None: - collection[key].append(item) + collection[key].append(val) else: - old_item = collection.get(key, _coconut_sentinel) - if old_item is not _coconut_sentinel: - item = reduce_func(old_item, item) - collection[key] = item + old_val = collection.get(key, reduce_func_init) + if old_val is not _coconut_sentinel: + if reduce_func is False: + raise _coconut.ValueError("mapreduce()/collectby() got duplicate key " + repr(key) + " with reduce_func=False") + val = reduce_func(old_val, val) + collection[key] = val return collection +def _coconut_parallel_mapreduce(mapreduce_func, map_cls, *args, **kwargs): + if "map_using" in kwargs: + raise _coconut.TypeError("redundant map_using argument to process/thread mapreduce/collectby") + kwargs["map_using"] = _coconut.functools.partial(map_cls, stream=True, ordered=kwargs.pop("ordered", False), chunksize=kwargs.pop("chunksize", 1)) + with map_cls.multiple_sequential_calls(max_workers=kwargs.pop("max_workers", None)): + return mapreduce_func(*args, **kwargs) +mapreduce.using_processes = _coconut_partial(_coconut_parallel_mapreduce, mapreduce, process_map) +mapreduce.using_threads = _coconut_partial(_coconut_parallel_mapreduce, mapreduce, thread_map) +def collectby(key_func, iterable, value_func=None, **kwargs): + """Collect the items in iterable into a dictionary of lists keyed by key_func(item). + + If value_func is passed, collect value_func(item) into each list instead of item. + + If reduce_func is passed, instead of collecting the items into lists, reduce over + the items for each key with reduce_func, effectively implementing a MapReduce operation. + + If map_using is passed, calculate key_func and value_func by mapping them over + the iterable using map_using as map. Useful with process_map/thread_map. + """ + return {_coconut_}mapreduce(_coconut_lifted(_coconut_comma_op, key_func, {_coconut_}ident if value_func is None else value_func), iterable, **kwargs) +collectby.using_processes = _coconut_partial(_coconut_parallel_mapreduce, collectby, process_map) +collectby.using_threads = _coconut_partial(_coconut_parallel_mapreduce, collectby, thread_map) def _namedtuple_of(**kwargs): """Construct an anonymous namedtuple of the given keyword arguments.""" {namedtuple_of_implementation} @@ -1969,7 +2059,7 @@ class _coconut_SupportsAdd(_coconut.typing.Protocol): raise NotImplementedError(...) """ def __add__(self, other): - raise NotImplementedError("Protocol methods cannot be called at runtime ((+) in a typing context is a Protocol)") + raise _coconut.NotImplementedError("Protocol methods cannot be called at runtime ((+) in a typing context is a Protocol)") class _coconut_SupportsMinus(_coconut.typing.Protocol): """Coconut (-) Protocol. Equivalent to: @@ -1980,9 +2070,9 @@ class _coconut_SupportsMinus(_coconut.typing.Protocol): raise NotImplementedError """ def __sub__(self, other): - raise NotImplementedError("Protocol methods cannot be called at runtime ((-) in a typing context is a Protocol)") + raise _coconut.NotImplementedError("Protocol methods cannot be called at runtime ((-) in a typing context is a Protocol)") def __neg__(self): - raise NotImplementedError("Protocol methods cannot be called at runtime ((-) in a typing context is a Protocol)") + raise _coconut.NotImplementedError("Protocol methods cannot be called at runtime ((-) in a typing context is a Protocol)") class _coconut_SupportsMul(_coconut.typing.Protocol): """Coconut (*) Protocol. Equivalent to: @@ -1991,7 +2081,7 @@ class _coconut_SupportsMul(_coconut.typing.Protocol): raise NotImplementedError(...) """ def __mul__(self, other): - raise NotImplementedError("Protocol methods cannot be called at runtime ((*) in a typing context is a Protocol)") + raise _coconut.NotImplementedError("Protocol methods cannot be called at runtime ((*) in a typing context is a Protocol)") class _coconut_SupportsPow(_coconut.typing.Protocol): """Coconut (**) Protocol. Equivalent to: @@ -2000,7 +2090,7 @@ class _coconut_SupportsPow(_coconut.typing.Protocol): raise NotImplementedError(...) """ def __pow__(self, other): - raise NotImplementedError("Protocol methods cannot be called at runtime ((**) in a typing context is a Protocol)") + raise _coconut.NotImplementedError("Protocol methods cannot be called at runtime ((**) in a typing context is a Protocol)") class _coconut_SupportsTruediv(_coconut.typing.Protocol): """Coconut (/) Protocol. Equivalent to: @@ -2009,7 +2099,7 @@ class _coconut_SupportsTruediv(_coconut.typing.Protocol): raise NotImplementedError(...) """ def __truediv__(self, other): - raise NotImplementedError("Protocol methods cannot be called at runtime ((/) in a typing context is a Protocol)") + raise _coconut.NotImplementedError("Protocol methods cannot be called at runtime ((/) in a typing context is a Protocol)") class _coconut_SupportsFloordiv(_coconut.typing.Protocol): """Coconut (//) Protocol. Equivalent to: @@ -2018,7 +2108,7 @@ class _coconut_SupportsFloordiv(_coconut.typing.Protocol): raise NotImplementedError(...) """ def __floordiv__(self, other): - raise NotImplementedError("Protocol methods cannot be called at runtime ((//) in a typing context is a Protocol)") + raise _coconut.NotImplementedError("Protocol methods cannot be called at runtime ((//) in a typing context is a Protocol)") class _coconut_SupportsMod(_coconut.typing.Protocol): """Coconut (%) Protocol. Equivalent to: @@ -2027,7 +2117,7 @@ class _coconut_SupportsMod(_coconut.typing.Protocol): raise NotImplementedError(...) """ def __mod__(self, other): - raise NotImplementedError("Protocol methods cannot be called at runtime ((%) in a typing context is a Protocol)") + raise _coconut.NotImplementedError("Protocol methods cannot be called at runtime ((%) in a typing context is a Protocol)") class _coconut_SupportsAnd(_coconut.typing.Protocol): """Coconut (&) Protocol. Equivalent to: @@ -2036,7 +2126,7 @@ class _coconut_SupportsAnd(_coconut.typing.Protocol): raise NotImplementedError(...) """ def __and__(self, other): - raise NotImplementedError("Protocol methods cannot be called at runtime ((&) in a typing context is a Protocol)") + raise _coconut.NotImplementedError("Protocol methods cannot be called at runtime ((&) in a typing context is a Protocol)") class _coconut_SupportsXor(_coconut.typing.Protocol): """Coconut (^) Protocol. Equivalent to: @@ -2045,7 +2135,7 @@ class _coconut_SupportsXor(_coconut.typing.Protocol): raise NotImplementedError(...) """ def __xor__(self, other): - raise NotImplementedError("Protocol methods cannot be called at runtime ((^) in a typing context is a Protocol)") + raise _coconut.NotImplementedError("Protocol methods cannot be called at runtime ((^) in a typing context is a Protocol)") class _coconut_SupportsOr(_coconut.typing.Protocol): """Coconut (|) Protocol. Equivalent to: @@ -2054,7 +2144,7 @@ class _coconut_SupportsOr(_coconut.typing.Protocol): raise NotImplementedError(...) """ def __or__(self, other): - raise NotImplementedError("Protocol methods cannot be called at runtime ((|) in a typing context is a Protocol)") + raise _coconut.NotImplementedError("Protocol methods cannot be called at runtime ((|) in a typing context is a Protocol)") class _coconut_SupportsLshift(_coconut.typing.Protocol): """Coconut (<<) Protocol. Equivalent to: @@ -2063,7 +2153,7 @@ class _coconut_SupportsLshift(_coconut.typing.Protocol): raise NotImplementedError(...) """ def __lshift__(self, other): - raise NotImplementedError("Protocol methods cannot be called at runtime ((<<) in a typing context is a Protocol)") + raise _coconut.NotImplementedError("Protocol methods cannot be called at runtime ((<<) in a typing context is a Protocol)") class _coconut_SupportsRshift(_coconut.typing.Protocol): """Coconut (>>) Protocol. Equivalent to: @@ -2072,7 +2162,7 @@ class _coconut_SupportsRshift(_coconut.typing.Protocol): raise NotImplementedError(...) """ def __rshift__(self, other): - raise NotImplementedError("Protocol methods cannot be called at runtime ((>>) in a typing context is a Protocol)") + raise _coconut.NotImplementedError("Protocol methods cannot be called at runtime ((>>) in a typing context is a Protocol)") class _coconut_SupportsMatmul(_coconut.typing.Protocol): """Coconut (@) Protocol. Equivalent to: @@ -2081,7 +2171,7 @@ class _coconut_SupportsMatmul(_coconut.typing.Protocol): raise NotImplementedError(...) """ def __matmul__(self, other): - raise NotImplementedError("Protocol methods cannot be called at runtime ((@) in a typing context is a Protocol)") + raise _coconut.NotImplementedError("Protocol methods cannot be called at runtime ((@) in a typing context is a Protocol)") class _coconut_SupportsInv(_coconut.typing.Protocol): """Coconut (~) Protocol. Equivalent to: @@ -2090,6 +2180,8 @@ class _coconut_SupportsInv(_coconut.typing.Protocol): raise NotImplementedError(...) """ def __invert__(self): - raise NotImplementedError("Protocol methods cannot be called at runtime ((~) in a typing context is a Protocol)") + raise _coconut.NotImplementedError("Protocol methods cannot be called at runtime ((~) in a typing context is a Protocol)") +{def_async_map} +{def_aliases} _coconut_self_match_types = {self_match_types} -_coconut_Expected, _coconut_MatchError, _coconut_cartesian_product, _coconut_count, _coconut_cycle, _coconut_enumerate, _coconut_flatten, _coconut_filter, _coconut_groupsof, _coconut_ident, _coconut_lift, _coconut_map, _coconut_multiset, _coconut_range, _coconut_reiterable, _coconut_reversed, _coconut_scan, _coconut_starmap, _coconut_tee, _coconut_windowsof, _coconut_zip, _coconut_zip_longest, TYPE_CHECKING, reduce, takewhile, dropwhile = Expected, MatchError, cartesian_product, count, cycle, enumerate, flatten, filter, groupsof, ident, lift, map, multiset, range, reiterable, reversed, scan, starmap, tee, windowsof, zip, zip_longest, False, _coconut.functools.reduce, _coconut.itertools.takewhile, _coconut.itertools.dropwhile{COMMENT.anything_added_here_should_be_copied_to_stub_file} +_coconut_Expected, _coconut_MatchError, _coconut_cartesian_product, _coconut_count, _coconut_cycle, _coconut_enumerate, _coconut_flatten, _coconut_filter, _coconut_groupsof, _coconut_ident, _coconut_lift, _coconut_map, _coconut_mapreduce, _coconut_multiset, _coconut_range, _coconut_reiterable, _coconut_reversed, _coconut_scan, _coconut_starmap, _coconut_tee, _coconut_windowsof, _coconut_zip, _coconut_zip_longest, TYPE_CHECKING, reduce, takewhile, dropwhile = Expected, MatchError, cartesian_product, count, cycle, enumerate, flatten, filter, groupsof, ident, lift, map, mapreduce, multiset, range, reiterable, reversed, scan, starmap, tee, windowsof, zip, zip_longest, False, _coconut.functools.reduce, _coconut.itertools.takewhile, _coconut.itertools.dropwhile{COMMENT.anything_added_here_should_be_copied_to_stub_file} diff --git a/coconut/compiler/util.py b/coconut/compiler/util.py index 7605cecae..64a0ff84f 100644 --- a/coconut/compiler/util.py +++ b/coconut/compiler/util.py @@ -28,20 +28,31 @@ from coconut.root import * # NOQA import sys +import os import re import ast import inspect import __future__ import itertools +import weakref import datetime as dt from functools import partial, reduce from collections import defaultdict from contextlib import contextmanager from pprint import pformat, pprint +if sys.version_info >= (3,): + import pickle +else: + import cPickle as pickle + from coconut._pyparsing import ( + CPYPARSING, + MODERN_PYPARSING, USE_COMPUTATION_GRAPH, SUPPORTS_INCREMENTAL, + SUPPORTS_ADAPTIVE, + SUPPORTS_PACKRAT_CONTEXT, replaceWith, ZeroOrMore, OneOrMore, @@ -59,9 +70,13 @@ CaselessLiteral, Group, ParserElement, + MatchFirst, + And, _trim_arity, _ParseResultsWithOffset, + all_parse_elements, line as _line, + __version__ as pyparsing_version, ) from coconut.integrations import embed @@ -70,6 +85,9 @@ get_name, get_target_info, memoize, + univ_open, + ensure_dir, + get_clock_time, ) from coconut.terminal import ( logger, @@ -91,7 +109,6 @@ specific_targets, pseudo_targets, reserved_vars, - use_packrat_parser, packrat_cache_size, temp_grammar_item_ref_count, indchars, @@ -99,9 +116,23 @@ non_syntactic_newline, allow_explicit_keyword_vars, reserved_prefix, - incremental_cache_size, + incremental_mode_cache_size, + default_incremental_cache_size, repeatedly_clear_incremental_cache, py_vers_with_eols, + unwrapper, + incremental_cache_limit, + incremental_mode_cache_successes, + adaptive_reparse_usage_weight, + use_adaptive_any_of, + disable_incremental_for_len, + coconut_cache_dir, + use_adaptive_if_available, + use_fast_pyparsing_reprs, + save_new_cache_items, + cache_validation_info, + require_cache_clear_frac, + reverse_any_of, ) from coconut.exceptions import ( CoconutException, @@ -116,10 +147,24 @@ indexable_evaluated_tokens_types = (ParseResults, list, tuple) +def evaluate_all_tokens(all_tokens, **kwargs): + """Recursively evaluate all the tokens in all_tokens.""" + all_evaluated_toks = [] + for toks in all_tokens: + evaluated_toks = evaluate_tokens(toks, **kwargs) + # if we're a final parse, ExceptionNodes will just be raised, but otherwise, if we see any, we need to + # short-circuit the computation and return them, since they imply this parse contains invalid syntax + if isinstance(evaluated_toks, ExceptionNode): + return None, evaluated_toks + all_evaluated_toks.append(evaluated_toks) + return all_evaluated_toks, None + + def evaluate_tokens(tokens, **kwargs): """Evaluate the given tokens in the computation graph. Very performance sensitive.""" - # can't have this be a normal kwarg to make evaluate_tokens a valid parse action + # can't have these be normal kwargs to make evaluate_tokens a valid parse action + is_final = kwargs.pop("is_final", False) evaluated_toklists = kwargs.pop("evaluated_toklists", ()) if DEVELOP: # avoid the overhead of the call if not develop internal_assert(not kwargs, "invalid keyword arguments to evaluate_tokens", kwargs) @@ -137,7 +182,9 @@ def evaluate_tokens(tokens, **kwargs): new_toklist = eval_new_toklist break if new_toklist is None: - new_toklist = [evaluate_tokens(toks, evaluated_toklists=evaluated_toklists) for toks in old_toklist] + new_toklist, exc_node = evaluate_all_tokens(old_toklist, is_final=is_final, evaluated_toklists=evaluated_toklists) + if exc_node is not None: + return exc_node # overwrite evaluated toklists rather than appending, since this # should be all the information we need for evaluating the dictionary evaluated_toklists = ((old_toklist, new_toklist),) @@ -152,7 +199,9 @@ def evaluate_tokens(tokens, **kwargs): for name, occurrences in tokens._ParseResults__tokdict.items(): new_occurrences = [] for value, position in occurrences: - new_value = evaluate_tokens(value, evaluated_toklists=evaluated_toklists) + new_value = evaluate_tokens(value, is_final=is_final, evaluated_toklists=evaluated_toklists) + if isinstance(new_value, ExceptionNode): + return new_value new_occurrences.append(_ParseResultsWithOffset(new_value, position)) new_tokdict[name] = new_occurrences new_tokens._ParseResults__tokdict.update(new_tokdict) @@ -186,13 +235,23 @@ def evaluate_tokens(tokens, **kwargs): return tokens elif isinstance(tokens, ComputationNode): - return tokens.evaluate() + result = tokens.evaluate() + if is_final and isinstance(result, ExceptionNode): + raise result.exception + return result elif isinstance(tokens, list): - return [evaluate_tokens(inner_toks, evaluated_toklists=evaluated_toklists) for inner_toks in tokens] + result, exc_node = evaluate_all_tokens(tokens, is_final=is_final, evaluated_toklists=evaluated_toklists) + return result if exc_node is None else exc_node elif isinstance(tokens, tuple): - return tuple(evaluate_tokens(inner_toks, evaluated_toklists=evaluated_toklists) for inner_toks in tokens) + result, exc_node = evaluate_all_tokens(tokens, is_final=is_final, evaluated_toklists=evaluated_toklists) + return tuple(result) if exc_node is None else exc_node + + elif isinstance(tokens, ExceptionNode): + if is_final: + raise tokens.exception + return tokens elif isinstance(tokens, DeferredNode): return tokens @@ -203,7 +262,7 @@ def evaluate_tokens(tokens, **kwargs): class ComputationNode(object): """A single node in the computation graph.""" - __slots__ = ("action", "original", "loc", "tokens") + (("been_called",) if DEVELOP else ()) + __slots__ = ("action", "original", "loc", "tokens") pprinting = False def __new__(cls, action, original, loc, tokens, ignore_no_tokens=False, ignore_one_token=False, greedy=False, trim_arity=True): @@ -225,8 +284,6 @@ def __new__(cls, action, original, loc, tokens, ignore_no_tokens=False, ignore_o self.original = original self.loc = loc self.tokens = tokens - if DEVELOP: - self.been_called = False if greedy: return self.evaluate() else: @@ -242,12 +299,14 @@ def name(self): def evaluate(self): """Get the result of evaluating the computation graph at this node. Very performance sensitive.""" - if DEVELOP: # avoid the overhead of the call if not develop - internal_assert(not self.been_called, "inefficient reevaluation of action " + self.name + " with tokens", self.tokens) - self.been_called = True + # note that this should never cache, since if a greedy Wrap that doesn't add to the packrat context + # hits the cache, it'll get the same ComputationNode object, but since it's greedy that object needs + # to actually be reevaluated evaluated_toks = evaluate_tokens(self.tokens) if logger.tracing: # avoid the overhead of the call if not tracing logger.log_trace(self.name, self.original, self.loc, evaluated_toks, self.tokens) + if isinstance(evaluated_toks, ExceptionNode): + return evaluated_toks # short-circuit if we got an ExceptionNode try: return self.action( self.original, @@ -278,6 +337,7 @@ def __repr__(self): class DeferredNode(object): """A node in the computation graph that has had its evaluation explicitly deferred.""" + __slots__ = ("original", "loc", "tokens") def __init__(self, original, loc, tokens): self.original = original @@ -289,6 +349,16 @@ def evaluate(self): return unpack(self.tokens) +class ExceptionNode(object): + """A node in the computation graph that stores an exception that will be raised upon final evaluation.""" + __slots__ = ("exception",) + + def __init__(self, exception): + if not USE_COMPUTATION_GRAPH: + raise exception + self.exception = exception + + class CombineToNode(Combine): """Modified Combine to work with the computation graph.""" __slots__ = () @@ -315,10 +385,8 @@ def postParse(self, original, loc, tokens): def add_action(item, action, make_copy=None): """Add a parse action to the given item.""" if make_copy is None: - item_ref_count = sys.getrefcount(item) if CPYTHON and not on_new_python else float("inf") - internal_assert(lambda: item_ref_count >= temp_grammar_item_ref_count, "add_action got item with too low ref count", (item, type(item), item_ref_count)) - make_copy = item_ref_count > temp_grammar_item_ref_count - if make_copy: + item = maybe_copy_elem(item, "attach") + elif make_copy: item = item.copy() return item.addParseAction(action) @@ -347,31 +415,45 @@ def attach(item, action, ignore_no_tokens=None, ignore_one_token=None, ignore_to return add_action(item, action, make_copy) -def should_clear_cache(): - """Determine if we should be clearing the packrat cache.""" - return ( - use_packrat_parser - and ( - not ParserElement._incrementalEnabled - or ( - ParserElement._incrementalWithResets - and repeatedly_clear_incremental_cache - ) - ) - ) - - def final_evaluate_tokens(tokens): """Same as evaluate_tokens but should only be used once a parse is assured.""" - # don't clear the cache in incremental mode - if should_clear_cache(): - # clear cache without resetting stats - ParserElement.packrat_cache.clear() - return evaluate_tokens(tokens) + clear_packrat_cache() + return evaluate_tokens(tokens, is_final=True) + + +@contextmanager +def adaptive_manager(item, original, loc, reparse=False): + """Manage the use of MatchFirst.setAdaptiveMode.""" + if reparse: + cleared_cache = clear_packrat_cache() + if cleared_cache is not True: + item.include_in_packrat_context = True + MatchFirst.setAdaptiveMode(False, usage_weight=adaptive_reparse_usage_weight) + try: + yield + finally: + MatchFirst.setAdaptiveMode(False, usage_weight=1) + if cleared_cache is not True: + item.include_in_packrat_context = False + else: + MatchFirst.setAdaptiveMode(True) + try: + yield + except Exception as exc: + if DEVELOP: + logger.log("reparsing due to:", exc) + logger.record_stat("adaptive", False) + else: + if DEVELOP: + logger.record_stat("adaptive", True) + finally: + MatchFirst.setAdaptiveMode(False) def final(item): """Collapse the computation graph upon parsing the given item.""" + if SUPPORTS_ADAPTIVE and use_adaptive_if_available: + item = Wrap(item, adaptive_manager, greedy=True) # evaluate_tokens expects a computation graph, so we just call add_action directly return add_action(trace(item), final_evaluate_tokens) @@ -385,17 +467,22 @@ def defer(item): def unpack(tokens): """Evaluate and unpack the given computation graph.""" logger.log_tag("unpack", tokens) - tokens = evaluate_tokens(tokens) + tokens = final_evaluate_tokens(tokens) if isinstance(tokens, ParseResults) and len(tokens) == 1: tokens = tokens[0] return tokens +def in_incremental_mode(): + """Determine if we are using incremental parsing mode.""" + return ParserElement._incrementalEnabled and not ParserElement._incrementalWithResets + + def force_reset_packrat_cache(): """Forcibly reset the packrat cache and all packrat stats.""" if ParserElement._incrementalEnabled: ParserElement._incrementalEnabled = False - ParserElement.enableIncremental(incremental_cache_size, still_reset_cache=ParserElement._incrementalWithResets) + ParserElement.enableIncremental(incremental_mode_cache_size if in_incremental_mode() else default_incremental_cache_size, still_reset_cache=False) else: ParserElement._packratEnabled = False ParserElement.enablePackrat(packrat_cache_size) @@ -404,7 +491,9 @@ def force_reset_packrat_cache(): @contextmanager def parsing_context(inner_parse=True): """Context to manage the packrat cache across parse calls.""" - if inner_parse and should_clear_cache(): + if not inner_parse: + yield + elif should_clear_cache(): # store old packrat cache old_cache = ParserElement.packrat_cache old_cache_stats = ParserElement.packrat_cache_stats[:] @@ -418,7 +507,8 @@ def parsing_context(inner_parse=True): if logger.verbose: ParserElement.packrat_cache_stats[0] += old_cache_stats[0] ParserElement.packrat_cache_stats[1] += old_cache_stats[1] - elif inner_parse and ParserElement._incrementalWithResets: + # if we shouldn't clear the cache, but we're using incrementalWithResets, then do this to avoid clearing it + elif ParserElement._incrementalWithResets: incrementalWithResets, ParserElement._incrementalWithResets = ParserElement._incrementalWithResets, False try: yield @@ -493,79 +583,10 @@ def transform(grammar, text, inner=True): return result -# ----------------------------------------------------------------------------------------------------------------------- -# PARSING INTROSPECTION: -# ----------------------------------------------------------------------------------------------------------------------- - - -def get_func_closure(func): - """Get variables in func's closure.""" - if PY2: - varnames = func.func_code.co_freevars - cells = func.func_closure - else: - varnames = func.__code__.co_freevars - cells = func.__closure__ - return {v: c.cell_contents for v, c in zip(varnames, cells)} - - -def get_pyparsing_cache(): - """Extract the underlying pyparsing packrat cache.""" - packrat_cache = ParserElement.packrat_cache - if isinstance(packrat_cache, dict): # if enablePackrat is never called - return packrat_cache - elif hasattr(packrat_cache, "cache"): # cPyparsing adds this - return packrat_cache.cache - else: # on pyparsing we have to do this - try: - # this is sketchy, so errors should only be complained - return get_func_closure(packrat_cache.get.__func__)["cache"] - except Exception as err: - complain(err) - return {} - - -def add_to_cache(new_cache_items): - """Add the given items directly to the pyparsing packrat cache.""" - packrat_cache = ParserElement.packrat_cache - for lookup, value in new_cache_items: - packrat_cache.set(lookup, value) - - -def get_cache_items_for(original): - """Get items from the pyparsing cache filtered to only from parsing original.""" - cache = get_pyparsing_cache() - for lookup, value in cache.items(): - got_orig = lookup[1] - if got_orig == original: - yield lookup, value - - -def get_highest_parse_loc(original): - """Get the highest observed parse location.""" - # find the highest observed parse location - highest_loc = 0 - for item, _ in get_cache_items_for(original): - loc = item[2] - if loc > highest_loc: - highest_loc = loc - return highest_loc - - -def enable_incremental_parsing(force=False): - """Enable incremental parsing mode where prefix parses are reused.""" - if SUPPORTS_INCREMENTAL or force: - try: - ParserElement.enableIncremental(incremental_cache_size, still_reset_cache=False) - except ImportError as err: - raise CoconutException(str(err)) - else: - logger.log("Incremental parsing mode enabled.") - - # ----------------------------------------------------------------------------------------------------------------------- # TARGETS: # ----------------------------------------------------------------------------------------------------------------------- + on_new_python = False raw_sys_target = str(sys.version_info[0]) + str(sys.version_info[1]) @@ -640,19 +661,491 @@ def get_target_info_smart(target, mode="lowest"): raise CoconutInternalException("unknown get_target_info_smart mode", mode) +# ----------------------------------------------------------------------------------------------------------------------- +# PARSING INTROSPECTION: +# ----------------------------------------------------------------------------------------------------------------------- + +def maybe_copy_elem(item, name): + """Copy the given grammar element if it's referenced somewhere else.""" + item_ref_count = sys.getrefcount(item) if CPYTHON and not on_new_python else float("inf") + internal_assert(lambda: item_ref_count >= temp_grammar_item_ref_count, "add_action got item with too low ref count", (item, type(item), item_ref_count)) + if item_ref_count <= temp_grammar_item_ref_count: + if DEVELOP: + logger.record_stat("maybe_copy_" + name, False) + return item + else: + if DEVELOP: + logger.record_stat("maybe_copy_" + name, True) + return item.copy() + + +def hasaction(elem): + """Determine if the given grammar element has any actions associated with it.""" + return ( + MODERN_PYPARSING + or elem.parseAction + or elem.resultsName is not None + or elem.debug + ) + + +@contextmanager +def using_fast_grammar_methods(): + """Enables grammar methods that modify their operands when they aren't referenced elsewhere.""" + if MODERN_PYPARSING: + yield + return + + def fast_add(self, other): + if hasaction(self): + return old_add(self, other) + self = maybe_copy_elem(self, "add") + self += other + return self + old_add, And.__add__ = And.__add__, fast_add + + def fast_or(self, other): + if hasaction(self): + return old_or(self, other) + self = maybe_copy_elem(self, "or") + self |= other + return self + old_or, MatchFirst.__or__ = MatchFirst.__or__, fast_or + + try: + yield + finally: + And.__add__ = old_add + MatchFirst.__or__ = old_or + + +def get_func_closure(func): + """Get variables in func's closure.""" + if PY2: + varnames = func.func_code.co_freevars + cells = func.func_closure + else: + varnames = func.__code__.co_freevars + cells = func.__closure__ + return {v: c.cell_contents for v, c in zip(varnames, cells)} + + +def get_pyparsing_cache(): + """Extract the underlying pyparsing packrat cache.""" + packrat_cache = ParserElement.packrat_cache + if isinstance(packrat_cache, dict): # if enablePackrat is never called + return packrat_cache + elif CPYPARSING: + return packrat_cache.cache # cPyparsing adds this + else: # on pyparsing we have to do this + try: + # this is sketchy, so errors should only be complained + # use .set instead of .get for the sake of MODERN_PYPARSING + return get_func_closure(packrat_cache.set.__func__)["cache"] + except Exception as err: + complain(err) + return {} + + +def should_clear_cache(force=False): + """Determine if we should be clearing the packrat cache.""" + if force: + return True + elif not ParserElement._packratEnabled: + return False + elif SUPPORTS_INCREMENTAL and ParserElement._incrementalEnabled: + if not in_incremental_mode(): + return repeatedly_clear_incremental_cache + if ( + incremental_cache_limit is not None + and len(ParserElement.packrat_cache) > incremental_cache_limit + ): + return "smart" + return False + else: + return True + + +def add_packrat_cache_items(new_items, clear_first=False): + """Add the given items to the packrat cache.""" + if clear_first: + ParserElement.packrat_cache.clear() + if new_items: + if PY2 or not CPYPARSING: + for lookup, value in new_items: + ParserElement.packrat_cache.set(lookup, value) + else: + ParserElement.packrat_cache.update(new_items) + + +def execute_clear_strat(clear_cache): + """Clear PyParsing cache using clear_cache.""" + orig_cache_len = None + if clear_cache is True: + # clear cache without resetting stats + ParserElement.packrat_cache.clear() + elif clear_cache == "smart": + orig_cache_len = execute_clear_strat("useless") + cleared_frac = (orig_cache_len - len(get_pyparsing_cache())) / orig_cache_len + if cleared_frac < require_cache_clear_frac: + logger.log("Packrat cache pruning using 'useless' strat failed; falling back to 'second half' strat.") + execute_clear_strat("second half") + else: + internal_assert(CPYPARSING, "unsupported clear_cache strategy", clear_cache) + cache = get_pyparsing_cache() + orig_cache_len = len(cache) + if clear_cache == "useless": + keys_to_del = [] + for lookup, value in cache.items(): + if not value[-1][0]: + keys_to_del.append(lookup) + for del_key in keys_to_del: + del cache[del_key] + elif clear_cache == "second half": + all_keys = tuple(cache.keys()) + for del_key in all_keys[len(all_keys) // 2: len(all_keys)]: + del cache[del_key] + else: + raise CoconutInternalException("invalid clear_cache strategy", clear_cache) + return orig_cache_len + + +def clear_packrat_cache(force=False): + """Clear the packrat cache if applicable. + Very performance-sensitive for incremental parsing mode.""" + clear_cache = should_clear_cache(force=force) + if clear_cache: + if DEVELOP: + start_time = get_clock_time() + orig_cache_len = execute_clear_strat(clear_cache) + if DEVELOP and orig_cache_len is not None: + logger.log("Pruned packrat cache from {orig_len} items to {new_len} items using {strat!r} strategy ({time} secs).".format( + orig_len=orig_cache_len, + new_len=len(get_pyparsing_cache()), + strat=clear_cache, + time=get_clock_time() - start_time, + )) + return clear_cache + + +def get_cache_items_for(original, only_useful=False, exclude_stale=True): + """Get items from the pyparsing cache filtered to only be from parsing original.""" + cache = get_pyparsing_cache() + for lookup, value in cache.items(): + got_orig = lookup[1] + internal_assert(lambda: isinstance(got_orig, (bytes, str)), "failed to look up original in pyparsing cache item", (lookup, value)) + if ParserElement._incrementalEnabled: + (is_useful,) = value[-1] + if only_useful and not is_useful: + continue + if exclude_stale and is_useful >= 2: + continue + if got_orig == original: + yield lookup, value + + +def get_highest_parse_loc(original): + """Get the highest observed parse location.""" + # find the highest observed parse location + highest_loc = 0 + for lookup, _ in get_cache_items_for(original): + loc = lookup[2] + if loc > highest_loc: + highest_loc = loc + return highest_loc + + +def enable_incremental_parsing(): + """Enable incremental parsing mode where prefix/suffix parses are reused.""" + if not SUPPORTS_INCREMENTAL: + return False + if in_incremental_mode(): # incremental mode is already enabled + return True + ParserElement._incrementalEnabled = False + try: + ParserElement.enableIncremental(incremental_mode_cache_size, still_reset_cache=False, cache_successes=incremental_mode_cache_successes) + except ImportError as err: + raise CoconutException(str(err)) + logger.log("Incremental parsing mode enabled.") + return True + + +def pickle_cache(original, cache_path, include_incremental=True, protocol=pickle.HIGHEST_PROTOCOL): + """Pickle the pyparsing cache for original to cache_path.""" + internal_assert(all_parse_elements is not None, "pickle_cache requires cPyparsing") + if not save_new_cache_items: + logger.log("Skipping saving cache items due to environment variable.") + return + + validation_dict = {} if cache_validation_info else None + + pickleable_cache_items = [] + if ParserElement._incrementalEnabled and include_incremental: + # note that exclude_stale is fine here because that means it was never used, + # since _parseIncremental sets usefullness to True when a cache item is used + for lookup, value in get_cache_items_for(original, only_useful=True): + if incremental_mode_cache_size is not None and len(pickleable_cache_items) > incremental_mode_cache_size: + logger.log( + "Got too large incremental cache: " + + str(len(get_pyparsing_cache())) + " > " + str(incremental_mode_cache_size) + ) + break + if len(pickleable_cache_items) >= incremental_cache_limit: + break + loc = lookup[2] + # only include cache items that aren't at the start or end, since those + # are the only ones that parseIncremental will reuse + if 0 < loc < len(original) - 1: + elem = lookup[0] + identifier = elem.parse_element_index + internal_assert(lambda: elem == all_parse_elements[identifier](), "failed to look up parse element by identifier", (elem, all_parse_elements[identifier]())) + if validation_dict is not None: + validation_dict[identifier] = elem.__class__.__name__ + pickleable_lookup = (identifier,) + lookup[1:] + pickleable_cache_items.append((pickleable_lookup, value)) + + all_adaptive_stats = {} + for wkref in MatchAny.all_match_anys: + match_any = wkref() + if match_any is not None and match_any.adaptive_usage is not None: + identifier = match_any.parse_element_index + internal_assert(lambda: match_any == all_parse_elements[identifier](), "failed to look up match_any by identifier", (match_any, all_parse_elements[identifier]())) + if validation_dict is not None: + validation_dict[identifier] = match_any.__class__.__name__ + match_any.expr_order.sort(key=lambda i: (-match_any.adaptive_usage[i], i)) + all_adaptive_stats[identifier] = (match_any.adaptive_usage, match_any.expr_order) + logger.log("Caching adaptive item:", match_any, all_adaptive_stats[identifier]) + + logger.log("Saving {num_inc} incremental and {num_adapt} adaptive cache items to {cache_path!r}.".format( + num_inc=len(pickleable_cache_items), + num_adapt=len(all_adaptive_stats), + cache_path=cache_path, + )) + pickle_info_obj = { + "VERSION": VERSION, + "pyparsing_version": pyparsing_version, + "validation_dict": validation_dict, + "pickleable_cache_items": pickleable_cache_items, + "all_adaptive_stats": all_adaptive_stats, + } + try: + with univ_open(cache_path, "wb") as pickle_file: + pickle.dump(pickle_info_obj, pickle_file, protocol=protocol) + except Exception: + logger.warn_exc() + return False + else: + return True + finally: + # clear the packrat cache when we're done so we don't interfere with anything else happening in this process + clear_packrat_cache(force=True) + + +def unpickle_cache(cache_path): + """Unpickle and load the given incremental cache file.""" + internal_assert(all_parse_elements is not None, "unpickle_cache requires cPyparsing") + + if not os.path.exists(cache_path): + return False + try: + with univ_open(cache_path, "rb") as pickle_file: + pickle_info_obj = pickle.load(pickle_file) + except Exception: + logger.log_exc() + return False + if ( + pickle_info_obj["VERSION"] != VERSION + or pickle_info_obj["pyparsing_version"] != pyparsing_version + ): + return False + + validation_dict = pickle_info_obj["validation_dict"] + if ParserElement._incrementalEnabled: + pickleable_cache_items = pickle_info_obj["pickleable_cache_items"] + else: + pickleable_cache_items = [] + all_adaptive_stats = pickle_info_obj["all_adaptive_stats"] + + for identifier, (adaptive_usage, expr_order) in all_adaptive_stats.items(): + if identifier < len(all_parse_elements): + maybe_elem = all_parse_elements[identifier]() + if maybe_elem is not None: + if validation_dict is not None: + internal_assert(maybe_elem.__class__.__name__ == validation_dict[identifier], "adaptive cache pickle-unpickle inconsistency", (maybe_elem, validation_dict[identifier])) + maybe_elem.adaptive_usage = adaptive_usage + maybe_elem.expr_order = expr_order + + max_cache_size = min( + incremental_mode_cache_size or float("inf"), + incremental_cache_limit or float("inf"), + ) + if max_cache_size != float("inf"): + pickleable_cache_items = pickleable_cache_items[-max_cache_size:] + + new_cache_items = [] + for pickleable_lookup, value in pickleable_cache_items: + identifier = pickleable_lookup[0] + if identifier < len(all_parse_elements): + maybe_elem = all_parse_elements[identifier]() + if maybe_elem is not None: + if validation_dict is not None: + internal_assert(maybe_elem.__class__.__name__ == validation_dict[identifier], "incremental cache pickle-unpickle inconsistency", (maybe_elem, validation_dict[identifier])) + lookup = (maybe_elem,) + pickleable_lookup[1:] + usefullness = value[-1][0] + internal_assert(usefullness, "loaded useless cache item", (lookup, value)) + stale_value = value[:-1] + ([usefullness + 1],) + new_cache_items.append((lookup, stale_value)) + add_packrat_cache_items(new_cache_items) + + num_inc = len(pickleable_cache_items) + num_adapt = len(all_adaptive_stats) + return num_inc, num_adapt + + +def load_cache_for(inputstring, codepath): + """Load cache_path (for the given inputstring and filename).""" + if not SUPPORTS_INCREMENTAL: + raise CoconutException("the parsing cache requires cPyparsing (run '{python} -m pip install --upgrade cPyparsing' to fix)".format(python=sys.executable)) + filename = os.path.basename(codepath) + + if in_incremental_mode(): + incremental_enabled = True + incremental_info = "using incremental parsing mode since it was already enabled" + elif len(inputstring) < disable_incremental_for_len: + incremental_enabled = enable_incremental_parsing() + if incremental_enabled: + incremental_info = "incremental parsing mode enabled due to len == {input_len} < {max_len}".format( + input_len=len(inputstring), + max_len=disable_incremental_for_len, + ) + else: + incremental_info = "failed to enable incremental parsing mode" + else: + incremental_enabled = False + incremental_info = "not using incremental parsing mode due to len == {input_len} >= {max_len}".format( + input_len=len(inputstring), + max_len=disable_incremental_for_len, + ) + + if ( + # only load the cache if we're using anything that makes use of it + incremental_enabled + or use_adaptive_any_of + or use_adaptive_if_available + ): + cache_path = get_cache_path(codepath) + did_load_cache = unpickle_cache(cache_path) + if did_load_cache: + num_inc, num_adapt = did_load_cache + logger.log("Loaded {num_inc} incremental and {num_adapt} adaptive cache items for {filename!r} ({incremental_info}).".format( + num_inc=num_inc, + num_adapt=num_adapt, + filename=filename, + incremental_info=incremental_info, + )) + else: + logger.log("Failed to load cache for {filename!r} from {cache_path!r} ({incremental_info}).".format( + filename=filename, + cache_path=cache_path, + incremental_info=incremental_info, + )) + if incremental_enabled: + logger.warn("Populating initial parsing cache (compilation may take longer than usual)...") + else: + cache_path = None + logger.log("Declined to load cache for {filename!r} ({incremental_info}).".format( + filename=filename, + incremental_info=incremental_info, + )) + + return cache_path, incremental_enabled + + +def get_cache_path(codepath): + """Get the cache filename to use for the given codepath.""" + code_dir, code_fname = os.path.split(codepath) + + cache_dir = os.path.join(code_dir, coconut_cache_dir) + ensure_dir(cache_dir, logger=logger) + + pickle_fname = code_fname + ".pkl" + return os.path.join(cache_dir, pickle_fname) + + # ----------------------------------------------------------------------------------------------------------------------- # PARSE ELEMENTS: # ----------------------------------------------------------------------------------------------------------------------- +class MatchAny(MatchFirst): + """Version of MatchFirst that always uses adaptive parsing.""" + all_match_anys = [] + + def __init__(self, *args, **kwargs): + super(MatchAny, self).__init__(*args, **kwargs) + self.all_match_anys.append(weakref.ref(self)) + + def __or__(self, other): + if isinstance(other, MatchAny): + self = maybe_copy_elem(self, "any_or") + return self.append(other) + else: + return MatchFirst([self, other]) + + @override + def copy(self): + self = super(MatchAny, self).copy() + self.all_match_anys.append(weakref.ref(self)) + return self + + if not use_fast_pyparsing_reprs: + def __str__(self): + return self.__class__.__name__ + ":" + super(MatchAny, self).__str__() + + +if SUPPORTS_ADAPTIVE: + MatchAny.setAdaptiveMode(True) + + +def any_of(*exprs, **kwargs): + """Build a MatchAny of the given MatchFirst.""" + use_adaptive = kwargs.pop("use_adaptive", use_adaptive_any_of) and SUPPORTS_ADAPTIVE + reverse = reverse_any_of + if DEVELOP: + reverse = kwargs.pop("reverse", reverse) + internal_assert(not kwargs, "excess keyword arguments passed to any_of", kwargs) + + AnyOf = MatchAny if use_adaptive else MatchFirst + + flat_exprs = [] + for e in exprs: + if ( + # don't merge MatchFirsts when we're reversing + not (reverse and not use_adaptive) + and e.__class__ == AnyOf + and not hasaction(e) + ): + flat_exprs.extend(e.exprs) + else: + flat_exprs.append(e) + + if reverse: + flat_exprs = reversed([trace(e) for e in exprs]) + + return AnyOf(flat_exprs) + + class Wrap(ParseElementEnhance): """PyParsing token that wraps the given item in the given context manager.""" + global_instance_counter = 0 inside = False def __init__(self, item, wrapper, greedy=False, include_in_packrat_context=False): super(Wrap, self).__init__(item) self.wrapper = wrapper self.greedy = greedy - self.include_in_packrat_context = include_in_packrat_context and hasattr(ParserElement, "packrat_context") + self.include_in_packrat_context = SUPPORTS_PACKRAT_CONTEXT and include_in_packrat_context + self.identifier = Wrap.global_instance_counter + Wrap.global_instance_counter += 1 @property def wrapped_name(self): @@ -666,12 +1159,15 @@ def wrapped_context(self): and unwrapped parses. Only supported natively on cPyparsing.""" was_inside, self.inside = self.inside, True if self.include_in_packrat_context: - ParserElement.packrat_context.append(self.wrapper) + old_packrat_context = ParserElement.packrat_context + new_packrat_context = old_packrat_context | frozenset((self.identifier,)) + ParserElement.packrat_context = new_packrat_context try: yield finally: if self.include_in_packrat_context: - ParserElement.packrat_context.pop() + internal_assert(ParserElement.packrat_context == new_packrat_context, "invalid popped Wrap identifier", self.identifier) + ParserElement.packrat_context = old_packrat_context self.inside = was_inside @override @@ -680,11 +1176,17 @@ def parseImpl(self, original, loc, *args, **kwargs): if logger.tracing: # avoid the overhead of the call if not tracing logger.log_trace(self.wrapped_name, original, loc) with logger.indent_tracing(): - with self.wrapper(self, original, loc): - with self.wrapped_context(): - parse_loc, tokens = super(Wrap, self).parseImpl(original, loc, *args, **kwargs) - if self.greedy: - tokens = evaluate_tokens(tokens) + reparse = False + parse_loc = None + while parse_loc is None: # lets wrapper catch errors to trigger a reparse + with self.wrapper(self, original, loc, **(dict(reparse=True) if reparse else {})): + with self.wrapped_context(): + parse_loc, tokens = super(Wrap, self).parseImpl(original, loc, *args, **kwargs) + if self.greedy: + tokens = evaluate_tokens(tokens) + if reparse and parse_loc is None: + raise CoconutInternalException("illegal double reparse in", self) + reparse = True if logger.tracing: # avoid the overhead of the call if not tracing logger.log_trace(self.wrapped_name, original, loc, tokens) return parse_loc, tokens @@ -696,6 +1198,12 @@ def __repr__(self): return self.wrapped_name +def handle_and_manage(item, handler, manager): + """Attach a handler and a manager to the given parse item.""" + new_item = attach(item, handler) + return Wrap(new_item, manager, greedy=True) + + def disable_inside(item, *elems, **kwargs): """Prevent elems from matching inside of item. @@ -724,7 +1232,7 @@ def manage_elem(self, original, loc): raise ParseException(original, loc, self.errmsg, self) for elem in elems: - yield Wrap(elem, manage_elem, include_in_packrat_context=True) + yield Wrap(elem, manage_elem) def disable_outside(item, *elems): @@ -771,6 +1279,7 @@ def longest(*args): return matcher +@memoize(64) def compile_regex(regex, options=None): """Compiles the given regex to support unicode.""" if options is None: @@ -780,9 +1289,6 @@ def compile_regex(regex, options=None): return re.compile(regex, options) -memoized_compile_regex = memoize(64)(compile_regex) - - def regex_item(regex, options=None): """pyparsing.Regex except it always uses unicode.""" if options is None: @@ -924,6 +1430,14 @@ def disallow_keywords(kwds, with_suffix=""): return regex_item(r"(?!" + "|".join(to_disallow) + r")").suppress() +def disambiguate_literal(literal, not_literals): + """Get an item that matchesl literal and not any of not_literals.""" + return regex_item( + r"(?!" + "|".join(re.escape(s) for s in not_literals) + ")" + + re.escape(literal) + ) + + def any_keyword_in(kwds): """Match any of the given keywords.""" return regex_item(r"|".join(k + r"\b" for k in kwds)) @@ -1262,14 +1776,6 @@ def split_leading_trailing_indent(line, max_indents=None): return leading_indent, line, trailing_indent -def split_leading_whitespace(inputstr): - """Split leading whitespace.""" - basestr = inputstr.lstrip() - whitespace = inputstr[:len(inputstr) - len(basestr)] - internal_assert(whitespace + basestr == inputstr, "invalid whitespace split", inputstr) - return whitespace, basestr - - def rem_and_count_indents(inputstr): """Removes and counts the ind_change (opens - closes).""" no_opens = inputstr.replace(openindent, "") @@ -1476,9 +1982,9 @@ def move_loc_to_non_whitespace(original, loc, backwards=False): original, loc, chars_to_move_forwards={ - default_whitespace_chars: not backwards, # for loc, move backwards on newlines/indents, which we can do safely without removing anything from the error indchars: False, + default_whitespace_chars: not backwards, }, ) @@ -1489,12 +1995,21 @@ def move_endpt_to_non_whitespace(original, loc, backwards=False): original, loc, chars_to_move_forwards={ - default_whitespace_chars: not backwards, # for endpt, ignore newlines/indents to avoid introducing unnecessary lines into the error + default_whitespace_chars: not backwards, + # always move forwards on unwrapper to ensure we don't cut wrapped objects in the middle + unwrapper: True, }, ) +def sub_all(inputstr, regexes, replacements): + """Sub all regexes for replacements in inputstr.""" + for key, regex in regexes.items(): + inputstr = regex.sub(lambda match: replacements[key], inputstr) + return inputstr + + # ----------------------------------------------------------------------------------------------------------------------- # PYTEST: # ----------------------------------------------------------------------------------------------------------------------- diff --git a/coconut/constants.py b/coconut/constants.py index 38f1c671d..a6c276a8e 100644 --- a/coconut/constants.py +++ b/coconut/constants.py @@ -50,6 +50,11 @@ def get_bool_env_var(env_var, default=False): return default +def get_path_env_var(env_var, default): + """Get a path from an environment variable.""" + return fixpath(os.getenv(env_var, default)) + + # ----------------------------------------------------------------------------------------------------------------------- # VERSION CONSTANTS: # ----------------------------------------------------------------------------------------------------------------------- @@ -66,6 +71,7 @@ def get_bool_env_var(env_var, default=False): WINDOWS = os.name == "nt" PYPY = platform.python_implementation() == "PyPy" CPYTHON = platform.python_implementation() == "CPython" +PY26 = sys.version_info < (2, 7) PY32 = sys.version_info >= (3, 2) PY33 = sys.version_info >= (3, 3) PY34 = sys.version_info >= (3, 4) @@ -76,17 +82,19 @@ def get_bool_env_var(env_var, default=False): PY39 = sys.version_info >= (3, 9) PY310 = sys.version_info >= (3, 10) PY311 = sys.version_info >= (3, 11) +PY312 = sys.version_info >= (3, 12) IPY = ( - ((PY2 and not PY26) or PY35) + PY35 and (PY37 or not PYPY) and not (PYPY and WINDOWS) - and not (PY2 and WINDOWS) and sys.version_info[:2] != (3, 7) ) MYPY = ( - PY37 + PY38 and not WINDOWS and not PYPY + # disabled until MyPy supports PEP 695 + and not PY312 ) XONSH = ( PY35 @@ -101,8 +109,7 @@ def get_bool_env_var(env_var, default=False): # ----------------------------------------------------------------------------------------------------------------------- # set this to False only ever temporarily for ease of debugging -use_fast_pyparsing_reprs = True -assert use_fast_pyparsing_reprs or DEVELOP, "use_fast_pyparsing_reprs should never be disabled on non-develop build" +use_fast_pyparsing_reprs = get_bool_env_var("COCONUT_FAST_PYPARSING_REPRS", True) enable_pyparsing_warnings = DEVELOP warn_on_multiline_regex = False @@ -113,20 +120,46 @@ def get_bool_env_var(env_var, default=False): use_computation_graph_env_var = "COCONUT_USE_COMPUTATION_GRAPH" -# below constants are experimentally determined to maximize performance +num_displayed_timing_items = 100 + +save_new_cache_items = get_bool_env_var("COCONUT_ALLOW_SAVE_TO_CACHE", True) + +cache_validation_info = DEVELOP -streamline_grammar_for_len = 4000 +reverse_any_of_env_var = "COCONUT_REVERSE_ANY_OF" +reverse_any_of = get_bool_env_var(reverse_any_of_env_var, False) + +# below constants are experimentally determined to maximize performance use_packrat_parser = True # True also gives us better error messages packrat_cache_size = None # only works because final() clears the cache +streamline_grammar_for_len = 1536 + +use_cache_file = True + +disable_incremental_for_len = 46080 + +adaptive_any_of_env_var = "COCONUT_ADAPTIVE_ANY_OF" +use_adaptive_any_of = get_bool_env_var(adaptive_any_of_env_var, True) + # note that _parseIncremental produces much smaller caches -use_incremental_if_available = True -incremental_cache_size = None +use_incremental_if_available = False + +use_adaptive_if_available = False # currently broken +adaptive_reparse_usage_weight = 10 + # these only apply to use_incremental_if_available, not compiler.util.enable_incremental_parsing() +default_incremental_cache_size = None repeatedly_clear_incremental_cache = True never_clear_incremental_cache = False +# this is what gets used in compiler.util.enable_incremental_parsing() +incremental_mode_cache_size = None +incremental_cache_limit = 2097152 # clear cache when it gets this large +incremental_mode_cache_successes = False +require_cache_clear_frac = 0.3125 # require that at least this much of the cache must be cleared on each cache clear + use_left_recursion_if_available = False # ----------------------------------------------------------------------------------------------------------------------- @@ -134,11 +167,10 @@ def get_bool_env_var(env_var, default=False): # ----------------------------------------------------------------------------------------------------------------------- # set this to True only ever temporarily for ease of debugging -embed_on_internal_exc = False -assert not embed_on_internal_exc or DEVELOP, "embed_on_internal_exc should never be enabled on non-develop build" +embed_on_internal_exc = get_bool_env_var("COCONUT_EMBED_ON_INTERNAL_EXC", False) -# should be the minimal ref count observed by attach -temp_grammar_item_ref_count = 3 if PY311 else 5 +# should be the minimal ref count observed by maybe_copy_elem +temp_grammar_item_ref_count = 4 if PY311 else 5 minimum_recursion_limit = 128 # shouldn't be raised any higher to avoid stack overflows @@ -446,6 +478,7 @@ def get_bool_env_var(env_var, default=False): "urllib.parse": ("urllib", (3,)), "pickle": ("cPickle", (3,)), "collections.abc": ("collections", (3, 3)), + "_dummy_thread": ("dummy_thread", (3,)), # ./ in old_name denotes from ... import ... "io.StringIO": ("StringIO./StringIO", (2, 7)), "io.BytesIO": ("cStringIO./StringIO", (2, 7)), @@ -454,8 +487,7 @@ def get_bool_env_var(env_var, default=False): "itertools.zip_longest": ("itertools./izip_longest", (3,)), "math.gcd": ("fractions./gcd", (3, 5)), "time.process_time": ("time./clock", (3, 3)), - # # _dummy_thread was removed in Python 3.9, so this no longer works - # "_dummy_thread": ("dummy_thread", (3,)), + "shlex.quote": ("pipes./quote", (3, 3)), # third-party backports "asyncio": ("trollius", (3, 4)), @@ -571,7 +603,9 @@ def get_bool_env_var(env_var, default=False): '__file__', '__annotations__', '__debug__', - # # don't include builtins that aren't always made available by Coconut: + # we treat these as coconut_exceptions so the highlighter will always know about them: + # 'ExceptionGroup', 'BaseExceptionGroup', + # don't include builtins that aren't always made available by Coconut: # 'BlockingIOError', 'ChildProcessError', 'ConnectionError', # 'BrokenPipeError', 'ConnectionAbortedError', 'ConnectionRefusedError', # 'ConnectionResetError', 'FileExistsError', 'FileNotFoundError', @@ -604,14 +638,17 @@ def get_bool_env_var(env_var, default=False): force_verbose_logger = get_bool_env_var("COCONUT_FORCE_VERBOSE", False) -coconut_home = fixpath(os.getenv(home_env_var, "~")) +coconut_home = get_path_env_var(home_env_var, "~") use_color = get_bool_env_var("COCONUT_USE_COLOR", None) error_color_code = "31" log_color_code = "93" default_style = "default" -prompt_histfile = os.path.join(coconut_home, ".coconut_history") +prompt_histfile = get_path_env_var( + "COCONUT_HISTORY_FILE", + os.path.join(coconut_home, ".coconut_history"), +) prompt_multiline = False prompt_vi_mode = get_bool_env_var(vi_mode_env_var, False) prompt_wrap_lines = True @@ -653,7 +690,7 @@ def get_bool_env_var(env_var, default=False): # always use atomic --xxx=yyy rather than --xxx yyy # and don't include --run, --quiet, or --target as they're added separately coconut_base_run_args = ("--keep-lines",) -coconut_run_kwargs = dict(default_target="sys") # passed to Command.cmd +coconut_sys_kwargs = dict(default_target="sys", default_jobs="0") # passed to Command.cmd default_mypy_args = ( "--pretty", @@ -685,7 +722,9 @@ def get_bool_env_var(env_var, default=False): kilobyte = 1024 min_stack_size_kbs = 160 -default_jobs = "sys" if not PY26 else 0 +base_default_jobs = "sys" if not PY26 else 0 + +high_proc_prio = True mypy_install_arg = "install" jupyter_install_arg = "install" @@ -705,6 +744,12 @@ def get_bool_env_var(env_var, default=False): create_package_retries = 1 +use_fancy_call_output = get_bool_env_var("COCONUT_FANCY_CALL_OUTPUT", False) +call_timeout = 0.01 + +max_orig_lines_in_log_loc = 2 + + # ----------------------------------------------------------------------------------------------------------------------- # HIGHLIGHTER CONSTANTS: # ----------------------------------------------------------------------------------------------------------------------- @@ -729,10 +774,10 @@ def get_bool_env_var(env_var, default=False): "count", "makedata", "consume", - "parallel_map", + "process_map", + "thread_map", "addpattern", - "recursive_iterator", - "concurrent_map", + "recursive_generator", "fmap", "starmap", "reiterable", @@ -750,6 +795,7 @@ def get_bool_env_var(env_var, default=False): "lift", "all_equal", "collectby", + "mapreduce", "multi_enumerate", "cartesian_product", "multiset", @@ -757,6 +803,7 @@ def get_bool_env_var(env_var, default=False): "windowsof", "and_then", "and_then_await", + "async_map", "py_chr", "py_dict", "py_hex", @@ -790,8 +837,11 @@ def get_bool_env_var(env_var, default=False): coconut_exceptions = ( "MatchError", + "ExceptionGroup", + "BaseExceptionGroup", ) +highlight_builtins = coconut_specific_builtins + interp_only_builtins all_builtins = frozenset(python_builtins + coconut_specific_builtins + coconut_exceptions) magic_methods = ( @@ -870,7 +920,16 @@ def get_bool_env_var(env_var, default=False): ("pygments", "py>=39"), ("typing_extensions", "py<36"), ("typing_extensions", "py==36"), - ("typing_extensions", "py>=37"), + ("typing_extensions", "py==37"), + ("typing_extensions", "py>=38"), + ("trollius", "py<3;cpy"), + ("aenum", "py<34"), + ("dataclasses", "py==36"), + ("typing", "py<35"), + ("async_generator", "py35"), + ("exceptiongroup", "py37;py<311"), + ("anyio", "py36"), + "setuptools", ), "cpython": ( "cPyparsing", @@ -882,7 +941,8 @@ def get_bool_env_var(env_var, default=False): ("ipython", "py<3"), ("ipython", "py3;py<37"), ("ipython", "py==37"), - ("ipython", "py38"), + ("ipython", "py==38"), + ("ipython", "py>=39"), ("ipykernel", "py<3"), ("ipykernel", "py3;py<38"), ("ipykernel", "py38"), @@ -898,9 +958,13 @@ def get_bool_env_var(env_var, default=False): ("jupyter-console", "py<35"), ("jupyter-console", "py>=35;py<37"), ("jupyter-console", "py37"), + "papermill", + ), + "jupyterlab": ( ("jupyterlab", "py35"), + ), + "jupytext": ( ("jupytext", "py3"), - "papermill", ), "mypy": ( "mypy[python2]", @@ -915,17 +979,11 @@ def get_bool_env_var(env_var, default=False): ("xonsh", "py>=36;py<38"), ("xonsh", "py38"), ), - "backports": ( - ("trollius", "py<3;cpy"), - ("aenum", "py<34"), - ("dataclasses", "py==36"), - ("typing", "py<35"), - ("async_generator", "py35"), - ), "dev": ( ("pre-commit", "py3"), "requests", "vprof", + "py-spy", ), "docs": ( "sphinx", @@ -934,19 +992,22 @@ def get_bool_env_var(env_var, default=False): "myst-parser", "pydata-sphinx-theme", ), + "numpy": ( + ("numpy", "py<3;cpy"), + ("numpy", "py34;py<39"), + ("numpy", "py39"), + ("pandas", "py36"), + ), "tests": ( ("pytest", "py<36"), ("pytest", "py36"), "pexpect", - ("numpy", "py34"), - ("numpy", "py<3;cpy"), - ("pandas", "py36"), ), } # min versions are inclusive -min_versions = { - "cPyparsing": (2, 4, 7, 2, 2, 1), +unpinned_min_versions = { + "cPyparsing": (2, 4, 7, 2, 3, 2), ("pre-commit", "py3"): (3,), ("psutil", "py>=27"): (5,), "jupyter": (1, 0), @@ -957,30 +1018,37 @@ def get_bool_env_var(env_var, default=False): "pexpect": (4,), ("trollius", "py<3;cpy"): (2, 2), "requests": (2, 31), - ("numpy", "py34"): (1,), - ("numpy", "py<3;cpy"): (1,), + ("numpy", "py39"): (1, 26), ("dataclasses", "py==36"): (0, 8), ("aenum", "py<34"): (3, 1, 15), - "pydata-sphinx-theme": (0, 13), + "pydata-sphinx-theme": (0, 14), "myst-parser": (2,), "sphinx": (7,), - "mypy[python2]": (1, 4), + "mypy[python2]": (1, 7), ("jupyter-console", "py37"): (6, 6), ("typing", "py<35"): (3, 10), - ("typing_extensions", "py>=37"): (4, 7), - ("ipython", "py38"): (8,), + ("typing_extensions", "py>=38"): (4, 8), ("ipykernel", "py38"): (6,), ("jedi", "py39"): (0, 19), - ("pygments", "py>=39"): (2, 15), + ("pygments", "py>=39"): (2, 17), ("xonsh", "py38"): (0, 14), ("pytest", "py36"): (7,), ("async_generator", "py35"): (1, 10), + ("exceptiongroup", "py37;py<311"): (1,), + ("ipython", "py>=39"): (8, 18), + "py-spy": (0, 3), +} - # pinned reqs: (must be added to pinned_reqs below) - - # don't upgrade these; they breaks on Python 3.7 +pinned_min_versions = { + # don't upgrade these; they break on Python 3.9 + ("numpy", "py34;py<39"): (1, 18), + # don't upgrade these; they break on Python 3.8 + ("ipython", "py==38"): (8, 12), + # don't upgrade these; they break on Python 3.7 ("ipython", "py==37"): (7, 34), - # don't upgrade these; they breaks on Python 3.6 + ("typing_extensions", "py==37"): (4, 7), + # don't upgrade these; they break on Python 3.6 + ("anyio", "py36"): (3,), ("xonsh", "py>=36;py<38"): (0, 11), ("pandas", "py36"): (1,), ("jupyter-client", "py36"): (7, 1, 2), @@ -1003,6 +1071,7 @@ def get_bool_env_var(env_var, default=False): # don't upgrade this; it breaks on Python 3.4 ("pygments", "py<39"): (2, 3), # don't upgrade these; they break on Python 2 + "setuptools": (44,), ("jupyter-client", "py<35"): (5, 3), ("pywinpty", "py<3;windows"): (0, 5), ("jupyter-console", "py<35"): (5, 2), @@ -1011,42 +1080,16 @@ def get_bool_env_var(env_var, default=False): ("prompt_toolkit", "py<3"): (1,), "watchdog": (0, 10), "papermill": (1, 2), + ("numpy", "py<3;cpy"): (1, 16), # don't upgrade this; it breaks with old IPython versions ("jedi", "py<39"): (0, 17), # Coconut requires pyparsing 2 "pyparsing": (2, 4, 7), } -# should match the reqs with comments above -pinned_reqs = ( - ("ipython", "py==37"), - ("xonsh", "py>=36;py<38"), - ("pandas", "py36"), - ("jupyter-client", "py36"), - ("typing_extensions", "py==36"), - ("jupyter-client", "py<35"), - ("ipykernel", "py3;py<38"), - ("ipython", "py3;py<37"), - ("jupyter-console", "py>=35;py<37"), - ("jupyter-client", "py==35"), - ("jupytext", "py3"), - ("jupyterlab", "py35"), - ("xonsh", "py<36"), - ("typing_extensions", "py<36"), - ("prompt_toolkit", "py>=3"), - ("pytest", "py<36"), - "vprof", - ("pygments", "py<39"), - ("pywinpty", "py<3;windows"), - ("jupyter-console", "py<35"), - ("ipython", "py<3"), - ("ipykernel", "py<3"), - ("prompt_toolkit", "py<3"), - "watchdog", - "papermill", - ("jedi", "py<39"), - "pyparsing", -) +min_versions = {} +min_versions.update(pinned_min_versions) +min_versions.update(unpinned_min_versions) # max versions are exclusive; None implies that the max version should # be generated by incrementing the min version; multiple Nones implies @@ -1055,7 +1098,7 @@ def get_bool_env_var(env_var, default=False): max_versions = { ("jupyter-client", "py==35"): _, "pyparsing": _, - "cPyparsing": (_, _, _), + "cPyparsing": (_, _, _, _, _,), ("prompt_toolkit", "py<3"): _, ("jedi", "py<39"): _, ("pywinpty", "py<3;windows"): _, @@ -1089,6 +1132,7 @@ def get_bool_env_var(env_var, default=False): "Programming Language :: Python :: 3.9", "Programming Language :: Python :: 3.10", "Programming Language :: Python :: 3.11", + "Programming Language :: Python :: 3.12", "Programming Language :: Other", "Programming Language :: Other Scripting Engines", "Programming Language :: Python :: Implementation :: CPython", @@ -1118,6 +1162,7 @@ def get_bool_env_var(env_var, default=False): "recursion", "call", "recursive", + "recursive_iterator", "infix", "function", "composition", @@ -1140,6 +1185,7 @@ def get_bool_env_var(env_var, default=False): "datamaker", "prepattern", "iterator", + "generator", "none", "coalesce", "coalescing", diff --git a/coconut/exceptions.py b/coconut/exceptions.py index 61319e4ed..341ef3831 100644 --- a/coconut/exceptions.py +++ b/coconut/exceptions.py @@ -180,11 +180,14 @@ def message(self, message, source, point, ln, extra=None, endpoint=None, filenam point_ind = clip(point_ind, 0, len(lines[0])) endpoint_ind = clip(endpoint_ind, 0, len(lines[-1])) + max_line_len = max(len(line) for line in lines) + message += "\n" + " " * (taberrfmt + point_ind) if point_ind >= len(lines[0]): - message += "|\n" + message += "|" else: - message += "/" + "~" * (len(lines[0]) - point_ind - 1) + "\n" + message += "/" + "~" * (len(lines[0]) - point_ind - 1) + message += "~" * (max_line_len - len(lines[0])) + "\n" for line in lines: message += "\n" + " " * taberrfmt + line message += ( diff --git a/coconut/highlighter.py b/coconut/highlighter.py index aef74f588..a12686a06 100644 --- a/coconut/highlighter.py +++ b/coconut/highlighter.py @@ -25,8 +25,7 @@ from pygments.util import shebang_matches from coconut.constants import ( - coconut_specific_builtins, - interp_only_builtins, + highlight_builtins, new_operators, tabideal, default_encoding, @@ -94,7 +93,7 @@ class CoconutLexer(Python3Lexer): (words(reserved_vars, prefix=r"(? " + ", ".join(new_versions + ["(" + v + ")" for v in same_versions]) ) - if req in pinned_reqs: + if req in pinned_min_versions: pinned_updates.append(update_str) elif new_versions: new_updates.append(update_str) diff --git a/coconut/root.py b/coconut/root.py index 32cd33428..2d622b4d8 100644 --- a/coconut/root.py +++ b/coconut/root.py @@ -23,7 +23,7 @@ # VERSION: # ----------------------------------------------------------------------------------------------------------------------- -VERSION = "3.0.3" +VERSION = "3.0.4" VERSION_NAME = None # False for release, int >= 1 for develop DEVELOP = False @@ -45,6 +45,16 @@ def _indent(code, by=1, tabsize=4, strip=False, newline=False, initial_newline=F ) + ("\n" if newline else "") +def _get_target_info(target): + """Return target information as a version tuple.""" + if not target or target == "universal": + return () + elif len(target) == 1: + return (int(target),) + else: + return (int(target[0]), int(target[1:])) + + # ----------------------------------------------------------------------------------------------------------------------- # HEADER: # ----------------------------------------------------------------------------------------------------------------------- @@ -198,6 +208,54 @@ def _coconut_exec(obj, globals=None, locals=None): if globals is None: globals = _coconut_sys._getframe(1).f_globals exec(obj, globals, locals) +import operator as _coconut_operator +class _coconut_attrgetter(object): + __slots__ = ("attrs",) + def __init__(self, *attrs): + self.attrs = attrs + def __reduce_ex__(self, _): + return self.__reduce__() + def __reduce__(self): + return (self.__class__, self.attrs) + @staticmethod + def _getattr(obj, attr): + for name in attr.split("."): + obj = _coconut.getattr(obj, name) + return obj + def __call__(self, obj): + if len(self.attrs) == 1: + return self._getattr(obj, self.attrs[0]) + return _coconut.tuple(self._getattr(obj, attr) for attr in self.attrs) +_coconut_operator.attrgetter = _coconut_attrgetter +class _coconut_itemgetter(object): + __slots__ = ("items",) + def __init__(self, *items): + self.items = items + def __reduce_ex__(self, _): + return self.__reduce__() + def __reduce__(self): + return (self.__class__, self.items) + def __call__(self, obj): + if len(self.items) == 1: + return obj[self.items[0]] + return _coconut.tuple(obj[item] for item in self.items) +_coconut_operator.itemgetter = _coconut_itemgetter +class _coconut_methodcaller(object): + __slots__ = ("name", "args", "kwargs") + def __init__(self, name, *args, **kwargs): + self.name = name + self.args = args + self.kwargs = kwargs + def __reduce_ex__(self, _): + return self.__reduce__() + def __reduce__(self): + return (self.__class__, (self.name,) + self.args, {"kwargs": self.kwargs}) + def __setstate__(self, setvars): + for k, v in setvars.items(): + _coconut.setattr(self, k, v) + def __call__(self, obj): + return _coconut.getattr(obj, self.name)(*self.args, **self.kwargs) +_coconut_operator.methodcaller = _coconut_methodcaller ''' _non_py37_extras = r'''def _coconut_default_breakpointhook(*args, **kwargs): @@ -264,15 +322,24 @@ def _coconut_reduce_partial(self): _coconut_copy_reg.pickle(_coconut_functools.partial, _coconut_reduce_partial) ''' +_py3_before_py311_extras = '''try: + from exceptiongroup import ExceptionGroup, BaseExceptionGroup +except ImportError: + class you_need_to_install_exceptiongroup(object): + __slots__ = () + ExceptionGroup = BaseExceptionGroup = you_need_to_install_exceptiongroup() +''' + # whenever new versions are added here, header.py must be updated to use them ROOT_HEADER_VERSIONS = ( "universal", "2", - "3", "27", + "3", "37", "39", + "311", ) @@ -284,6 +351,7 @@ def _get_root_header(version="universal"): ''' + _indent(_get_root_header("2")) + '''else: ''' + _indent(_get_root_header("3")) + version_info = _get_target_info(version) header = "" if version.startswith("3"): @@ -293,7 +361,7 @@ def _get_root_header(version="universal"): # if a new assignment is added below, a new builtins import should be added alongside it header += _base_py2_header - if version in ("37", "39"): + if version_info >= (3, 7): header += r'''py_breakpoint = breakpoint ''' elif version == "3": @@ -311,7 +379,7 @@ def _get_root_header(version="universal"): header += r'''if _coconut_sys.version_info < (3, 7): ''' + _indent(_below_py37_extras) + r'''elif _coconut_sys.version_info < (3, 9): ''' + _indent(_py37_py38_extras) - elif version == "37": + elif (3, 7) <= version_info < (3, 9): header += r'''if _coconut_sys.version_info < (3, 9): ''' + _indent(_py37_py38_extras) elif version.startswith("2"): @@ -320,7 +388,11 @@ def _get_root_header(version="universal"): dict.items = _coconut_OrderedDict.viewitems ''' else: - assert version == "39", version + assert version_info >= (3, 9), version + + if (3,) <= version_info < (3, 11): + header += r'''if _coconut_sys.version_info < (3, 11): +''' + _indent(_py3_before_py311_extras) return header @@ -334,8 +406,6 @@ def _get_root_header(version="universal"): VERSION_STR = VERSION + (" [" + VERSION_NAME + "]" if VERSION_NAME else "") PY2 = _coconut_sys.version_info < (3,) -PY26 = _coconut_sys.version_info < (2, 7) -PY37 = _coconut_sys.version_info >= (3, 7) # ----------------------------------------------------------------------------------------------------------------------- # SETUP: diff --git a/coconut/terminal.py b/coconut/terminal.py index 30db0ecf4..11fb41cf7 100644 --- a/coconut/terminal.py +++ b/coconut/terminal.py @@ -36,6 +36,7 @@ lineno, col, ParserElement, + maybe_make_safe, ) from coconut.root import _indent @@ -51,11 +52,15 @@ log_color_code, ansii_escape, force_verbose_logger, + max_orig_lines_in_log_loc, ) from coconut.util import ( get_clock_time, get_name, displayable, + first_import_time, + assert_remove_prefix, + split_trailing_whitespace, ) from coconut.exceptions import ( CoconutWarning, @@ -96,22 +101,24 @@ def format_error(err_value, err_type=None, err_trace=None): return "".join(traceback.format_exception(err_type, err_value, err_trace)).strip() -def complain(error): +def complain(error_or_msg, *args, **kwargs): """Raises in develop; warns in release.""" - if callable(error): + if callable(error_or_msg): if DEVELOP: - error = error() + error_or_msg = error_or_msg() else: return - if not isinstance(error, BaseException) or (not isinstance(error, CoconutInternalException) and isinstance(error, CoconutException)): - error = CoconutInternalException(str(error)) + if not isinstance(error_or_msg, BaseException) or (not isinstance(error_or_msg, CoconutInternalException) and isinstance(error_or_msg, CoconutException)): + error_or_msg = CoconutInternalException(str(error_or_msg), *args, **kwargs) + else: + internal_assert(not args and not kwargs, "if error_or_msg is an error, args and kwargs must be empty, not", (args, kwargs)) if not DEVELOP: - logger.warn_err(error) + logger.warn_err(error_or_msg) elif embed_on_internal_exc: - logger.warn_err(error) + logger.warn_err(error_or_msg) embed(depth=1) else: - raise error + raise error_or_msg def internal_assert(condition, message=None, item=None, extra=None, exc_maker=None): @@ -125,6 +132,8 @@ def internal_assert(condition, message=None, item=None, extra=None, exc_maker=No item = condition elif callable(message): message = message() + # ensure the item is pickleable so that the exception can be transferred back across processes + item = str(item) if callable(extra): extra = extra() if exc_maker is None: @@ -180,14 +189,17 @@ def logging(self): class Logger(object): """Container object for various logger functions and variables.""" force_verbose = force_verbose_logger + colors_enabled = False + verbose = force_verbose quiet = False path = None name = None - colors_enabled = False tracing = False trace_ind = 0 + recorded_stats = defaultdict(lambda: [0, 0]) + def __init__(self, other=None): """Create a logger, optionally from another logger.""" if other is not None: @@ -225,6 +237,7 @@ def setup(self, quiet=None, verbose=None, tracing=None): self.verbose = verbose if tracing is not None: self.tracing = tracing + ParserElement.verbose_stacktrace = self.verbose def display( self, @@ -243,10 +256,12 @@ def display( file = file or sys.stdout elif level == "logging": file = file or sys.stderr - color = color or log_color_code + if color is None: + color = log_color_code elif level == "error": file = file or sys.stderr - color = color or error_color_code + if color is None: + color = error_color_code else: raise CoconutInternalException("invalid logging level", level) @@ -262,14 +277,16 @@ def display( raw_message = "\n" components = [] - if color: - components.append(ansii_escape + "[" + color + "m") for line in raw_message.splitlines(True): + line, endline = split_trailing_whitespace(line) + if color: + components.append(ansii_escape + "[" + color + "m") if sig: - line = sig + line + components.append(sig) components.append(line) - if color: - components.append(ansii_reset) + if color: + components.append(ansii_reset) + components.append(endline) components.append(end) full_message = "".join(components) @@ -304,15 +321,15 @@ def show_error(self, *messages, **kwargs): if not self.quiet: self.display(messages, main_sig, level="error", **kwargs) - def log(self, *messages): + def log(self, *messages, **kwargs): """Logs debug messages if --verbose.""" if self.verbose: - self.printlog(*messages) + self.printlog(*messages, **kwargs) - def log_stdout(self, *messages): + def log_stdout(self, *messages, **kwargs): """Logs debug messages to stdout if --verbose.""" if self.verbose: - self.print(*messages) + self.print(*messages, **kwargs) def log_lambda(self, *msg_funcs): if self.verbose: @@ -350,9 +367,18 @@ def log_vars(self, message, variables, rem_vars=("self",)): def log_loc(self, name, original, loc): """Log a location in source code.""" - if self.verbose: + if self.tracing: if isinstance(loc, int): - self.printlog("in error construction:", str(name), "=", repr(original[:loc]), "|", repr(original[loc:])) + pre_loc_orig, post_loc_orig = original[:loc], original[loc:] + if pre_loc_orig.count("\n") > max_orig_lines_in_log_loc: + pre_loc_orig_repr = "... " + repr(pre_loc_orig.rsplit("\n", 1)[-1]) + else: + pre_loc_orig_repr = repr(pre_loc_orig) + if post_loc_orig.count("\n") > max_orig_lines_in_log_loc: + post_loc_orig_repr = repr(post_loc_orig.split("\n", 1)[0]) + " ..." + else: + post_loc_orig_repr = repr(post_loc_orig) + self.printlog("in error construction:", str(name), "=", pre_loc_orig_repr, "|", post_loc_orig_repr) else: self.printlog("in error construction:", str(name), "=", repr(loc)) @@ -396,12 +422,21 @@ def warn_err(self, warning, force=False): try: raise warning except Exception: - self.print_exc(warning=True) + self.warn_exc() + + def log_warn(self, *args, **kwargs): + """Log a warning.""" + if self.verbose: + return self.warn(*args, **kwargs) def print_exc(self, err=None, show_tb=None, warning=False): """Properly prints an exception.""" self.print_formatted_error(self.get_error(err, show_tb), warning) + def warn_exc(self, err=None): + """Warn about the current or given exception.""" + self.print_exc(err, warning=True) + def print_exception(self, err_type, err_value, err_tb): """Properly prints the given exception details.""" self.print_formatted_error(format_error(err_value, err_type, err_tb)) @@ -449,17 +484,17 @@ def print_trace(self, *args): trace = " ".join(str(arg) for arg in args) self.printlog(_indent(trace, self.trace_ind)) - def log_tag(self, tag, code, multiline=False, force=False): + def log_tag(self, tag, block, multiline=False, force=False): """Logs a tagged message if tracing.""" if self.tracing or force: assert not (not DEVELOP and force), tag - if callable(code): - code = code() + if callable(block): + block = block() tagstr = "[" + str(tag) + "]" if multiline: - self.print_trace(tagstr + "\n" + displayable(code)) + self.print_trace(tagstr + "\n" + displayable(block)) else: - self.print_trace(tagstr, ascii(code)) + self.print_trace(tagstr, ascii(block)) def log_trace(self, expr, original, loc, item=None, extra=None): """Formats and displays a trace if tracing.""" @@ -510,10 +545,15 @@ def trace(self, item): item.debug = True return item + def record_stat(self, stat_name, stat_bool): + """Record the given boolean statistic for the given stat_name.""" + self.recorded_stats[stat_name][stat_bool] += 1 + @contextmanager def gather_parsing_stats(self): """Times parsing if --verbose.""" if self.verbose: + self.recorded_stats.pop("adaptive", None) start_time = get_clock_time() try: yield @@ -526,9 +566,25 @@ def gather_parsing_stats(self): # reset stats after printing if in incremental mode if ParserElement._incrementalEnabled: ParserElement.packrat_cache_stats[:] = [0] * len(ParserElement.packrat_cache_stats) + if "adaptive" in self.recorded_stats: + failures, successes = self.recorded_stats["adaptive"] + self.printlog("\tAdaptive parsing stats:", successes, "successes;", failures, "failures") + if maybe_make_safe is not None: + hits, misses = maybe_make_safe.stats + self.printlog("\tErrorless parsing stats:", hits, "errorless;", misses, "with errors") else: yield + def log_compiler_stats(self, comp): + """Log stats for the given compiler.""" + if self.verbose: + self.log("Grammar init time: " + str(comp.grammar_init_time) + " secs / Total init time: " + str(get_clock_time() - first_import_time) + " secs") + for stat_name, (no_copy, yes_copy) in self.recorded_stats.items(): + if not stat_name.startswith("maybe_copy_"): + continue + name = assert_remove_prefix(stat_name, "maybe_copy_") + self.printlog("\tGrammar copying stats (" + name + "):", no_copy, "not copied;", yes_copy, "copied") + total_block_time = defaultdict(int) @contextmanager diff --git a/coconut/tests/__main__.py b/coconut/tests/__main__.py index 1cadb7fa6..649ac82ed 100644 --- a/coconut/tests/__main__.py +++ b/coconut/tests/__main__.py @@ -21,6 +21,7 @@ import sys +from coconut.constants import WINDOWS from coconut.tests.main_test import comp_all # ----------------------------------------------------------------------------------------------------------------------- @@ -48,6 +49,7 @@ def main(args=None): agnostic_target=agnostic_target, expect_retcode=0 if "--mypy" not in args else None, check_errors="--verbose" not in args, + ignore_output=WINDOWS and "--mypy" not in args, ) diff --git a/coconut/tests/constants_test.py b/coconut/tests/constants_test.py index 2df0da3ba..eb3250b29 100644 --- a/coconut/tests/constants_test.py +++ b/coconut/tests/constants_test.py @@ -22,18 +22,21 @@ import sys import os import unittest -if PY26: - import_module = __import__ -else: - from importlib import import_module from coconut import constants from coconut.constants import ( WINDOWS, PYPY, + PY26, + PY39, fixpath, ) +if PY26: + import_module = __import__ +else: + from importlib import import_module + # ----------------------------------------------------------------------------------------------------------------------- # UTILITIES: # ----------------------------------------------------------------------------------------------------------------------- @@ -75,6 +78,10 @@ def is_importable(name): class TestConstants(unittest.TestCase): + def test_defaults(self): + assert constants.use_fast_pyparsing_reprs + assert not constants.embed_on_internal_exc + def test_fixpath(self): assert os.path.basename(fixpath("CamelCase.py")) == "CamelCase.py" @@ -100,6 +107,8 @@ def test_imports(self): or PYPY and old_imp in ("trollius", "aenum") # don't test typing_extensions, async_generator or old_imp.startswith(("typing_extensions", "async_generator")) + # don't test _dummy_thread on Py3.9 + or PY39 and new_imp == "_dummy_thread" ): pass elif sys.version_info >= ver_cutoff: @@ -108,8 +117,8 @@ def test_imports(self): assert is_importable(old_imp), "Failed to import " + old_imp def test_reqs(self): - assert set(constants.pinned_reqs) <= set(constants.min_versions), "found old pinned requirement" - assert set(constants.max_versions) <= set(constants.pinned_reqs) | set(("cPyparsing",)), "found unlisted constrained but unpinned requirements" + assert not set(constants.unpinned_min_versions) & set(constants.pinned_min_versions), "found pinned and unpinned requirements" + assert set(constants.max_versions) <= set(constants.pinned_min_versions) | set(("cPyparsing",)), "found unlisted constrained but unpinned requirements" for maxed_ver in constants.max_versions: assert isinstance(maxed_ver, tuple) or maxed_ver in ("pyparsing", "cPyparsing"), "maxed versions must be tagged to a specific Python version" diff --git a/coconut/tests/main_test.py b/coconut/tests/main_test.py index d30e9b793..2d7bf296e 100644 --- a/coconut/tests/main_test.py +++ b/coconut/tests/main_test.py @@ -31,7 +31,6 @@ import imp import pytest -import pexpect from coconut.util import noop_ctx, get_target_info from coconut.terminal import ( @@ -41,6 +40,7 @@ from coconut.command.util import ( call_output, reload, + run_cmd, ) from coconut.compiler.util import ( get_psf_target, @@ -51,11 +51,15 @@ IPY, XONSH, MYPY, + PY26, PY35, PY36, PY38, PY39, PY310, + CPYTHON, + adaptive_any_of_env_var, + reverse_any_of_env_var, supported_py2_vers, supported_py3_vers, icoconut_default_kernel_names, @@ -64,6 +68,8 @@ get_bool_env_var, coconut_cache_dir, default_use_cache_dir, + base_dir, + fixpath, ) from coconut.api import ( @@ -83,6 +89,9 @@ os.environ["PYDEVD_DISABLE_FILE_VALIDATION"] = "1" +# run fewer tests on Windows so appveyor doesn't time out +TEST_ALL = get_bool_env_var("COCONUT_TEST_ALL", not WINDOWS) + # ----------------------------------------------------------------------------------------------------------------------- # CONSTANTS: @@ -91,15 +100,22 @@ default_recursion_limit = "6144" default_stack_size = "6144" +default_jobs = ( + # fix EOMs on GitHub actions + "0" if PYPY + else None +) jupyter_timeout = 120 -base = os.path.dirname(os.path.relpath(__file__)) -src = os.path.join(base, "src") -dest = os.path.join(base, "dest") -additional_dest = os.path.join(base, "dest", "additional_dest") +tests_dir = os.path.dirname(os.path.relpath(__file__)) +src = os.path.join(tests_dir, "src") +dest = os.path.join(tests_dir, "dest") +additional_dest = os.path.join(tests_dir, "dest", "additional_dest") src_cache_dir = os.path.join(src, coconut_cache_dir) +cocotest_dir = os.path.join(src, "cocotest") +agnostic_dir = os.path.join(cocotest_dir, "agnostic") runnable_coco = os.path.join(src, "runnable.coco") runnable_py = os.path.join(src, "runnable.py") @@ -128,6 +144,11 @@ "*** glibc detected ***", "INTERNAL ERROR", ) +ignore_error_lines_with = ( + # ignore SyntaxWarnings containing assert_raises + "assert_raises(", + " raise ", +) mypy_snip = "a: str = count()[0]" mypy_snip_err_2 = '''error: Incompatible types in assignment (expression has type\n"int", variable has type "unicode")''' @@ -153,6 +174,7 @@ "from distutils.version import LooseVersion", ": SyntaxWarning: 'int' object is not ", " assert_raises(", + "Populating initial parsing cache", ) kernel_installation_msg = ( @@ -222,6 +244,7 @@ def call( expect_retcode=0, convert_to_import=False, assert_output_only_at_end=None, + ignore_output=False, **kwargs ): """Execute a shell command and assert that no errors were encountered.""" @@ -268,8 +291,11 @@ def call( module_name += ".__main__" with using_sys_path(module_dir): stdout, stderr, retcode = call_with_import(module_name, extra_argv) + elif ignore_output: + retcode = run_cmd(raw_cmd, raise_errs=False, **kwargs) + stdout = stderr = "" else: - stdout, stderr, retcode = call_output(raw_cmd, **kwargs) + stdout, stderr, retcode = call_output(raw_cmd, color=False, **kwargs) if expect_retcode is not None: assert retcode == expect_retcode, "Return code not as expected ({retcode} != {expect_retcode}) in: {cmd!r}".format( @@ -281,7 +307,6 @@ def call( out = stderr + stdout else: out = stdout + stderr - out = "".join(out) raw_lines = out.splitlines() lines = [] @@ -331,8 +356,7 @@ def call( for line in lines: for errstr in always_err_strs: assert errstr not in line, "{errstr!r} in {line!r}".format(errstr=errstr, line=line) - # ignore SyntaxWarnings containing assert_raises - if check_errors and "assert_raises(" not in line: + if check_errors and not any(ignore in line for ignore in ignore_error_lines_with): assert "Traceback (most recent call last):" not in line, "Traceback in " + repr(line) assert "Exception" not in line, "Exception in " + repr(line) assert "Error" not in line, "Error in " + repr(line) @@ -371,6 +395,8 @@ def call_coconut(args, **kwargs): args = ["--recursion-limit", default_recursion_limit] + args if default_stack_size is not None and "--stack-size" not in args: args = ["--stack-size", default_stack_size] + args + if default_jobs is not None and "--jobs" not in args: + args = ["--jobs", default_jobs] + args if "--mypy" in args and "check_mypy" not in kwargs: kwargs["check_mypy"] = True if PY26: @@ -399,31 +425,29 @@ def comp(path=None, folder=None, file=None, args=[], **kwargs): def rm_path(path, allow_keep=False): """Delete a path.""" + path = os.path.abspath(fixpath(path)) + assert not base_dir.startswith(path), "refusing to delete Coconut itself: " + repr(path) if allow_keep and get_bool_env_var("COCONUT_KEEP_TEST_FILES"): return - if os.path.isdir(path): - try: + try: + if os.path.isdir(path): shutil.rmtree(path) - except OSError: - logger.print_exc() - elif os.path.isfile(path): - os.remove(path) + elif os.path.isfile(path): + os.remove(path) + except OSError: + logger.print_exc() @contextmanager def using_paths(*paths): """Removes paths at the beginning and end.""" for path in paths: - if os.path.exists(path): - rm_path(path) + rm_path(path) try: yield finally: for path in paths: - try: - rm_path(path, allow_keep=True) - except OSError: - logger.print_exc() + rm_path(path, allow_keep=True) @contextmanager @@ -438,10 +462,25 @@ def using_dest(dest=dest, allow_existing=False): try: yield finally: - try: - rm_path(dest, allow_keep=True) - except OSError: - logger.print_exc() + rm_path(dest, allow_keep=True) + + +def clean_caches(): + """Clean out all __coconut_cache__ dirs.""" + for dirpath, dirnames, filenames in os.walk(tests_dir): + for name in dirnames: + if name == coconut_cache_dir: + rm_path(os.path.join(dirpath, name)) + + +@contextmanager +def using_caches(): + """Cleans caches at start and end.""" + clean_caches() + try: + yield + finally: + clean_caches() @contextmanager @@ -461,6 +500,26 @@ def using_coconut(fresh_logger=True, fresh_api=False): logger.copy_from(saved_logger) +def remove_pys_in(dirpath): + removed_pys = 0 + for fname in os.listdir(dirpath): + if fname.endswith(".py"): + rm_path(os.path.join(dirpath, fname)) + removed_pys += 1 + return removed_pys + + +@contextmanager +def using_pys_in(dirpath): + """Remove *.py in dirpath at start and finish.""" + remove_pys_in(dirpath) + try: + yield + finally: + removed_pys = remove_pys_in(dirpath) + assert removed_pys > 0, os.listdir(dirpath) + + @contextmanager def using_sys_path(path, prepend=False): """Adds a path to sys.path.""" @@ -503,10 +562,24 @@ def add_test_func_names(cls): def spawn_cmd(cmd): """Version of pexpect.spawn that prints the command being run.""" + import pexpect # hide import since not always available print("\n>", cmd) return pexpect.spawn(cmd) +@contextmanager +def using_env_vars(env_vars): + """Run using the given environment variables.""" + old_env = os.environ.copy() + os.environ.update(env_vars) + try: + yield + finally: + for k in env_vars: + del os.environ[k] + os.environ.update(old_env) + + # ----------------------------------------------------------------------------------------------------------------------- # RUNNERS: # ----------------------------------------------------------------------------------------------------------------------- @@ -558,6 +631,11 @@ def comp_38(args=[], always_sys=False, **kwargs): comp(path="cocotest", folder="target_38", args=["--target", "38" if not always_sys else "sys"] + args, **kwargs) +def comp_311(args=[], always_sys=False, **kwargs): + """Compiles target_311.""" + comp(path="cocotest", folder="target_311", args=["--target", "311" if not always_sys else "sys"] + args, **kwargs) + + def comp_sys(args=[], **kwargs): """Compiles target_sys.""" comp(path="cocotest", folder="target_sys", args=["--target", "sys"] + args, **kwargs) @@ -579,50 +657,54 @@ def run_extras(**kwargs): call_python([os.path.join(dest, "extras.py")], assert_output=True, check_errors=False, stderr_first=True, **kwargs) -def run(args=[], agnostic_target=None, use_run_arg=False, convert_to_import=False, always_sys=False, **kwargs): +def run(args=[], agnostic_target=None, use_run_arg=False, convert_to_import=False, always_sys=False, manage_cache=True, **kwargs): """Compiles and runs tests.""" if agnostic_target is None: agnostic_args = args else: agnostic_args = ["--target", str(agnostic_target)] + args - with using_dest(): - with (using_dest(additional_dest) if "--and" in args else noop_ctx()): - - spec_kwargs = kwargs.copy() - spec_kwargs["always_sys"] = always_sys - if PY2: - comp_2(args, **spec_kwargs) - else: - comp_3(args, **spec_kwargs) - if sys.version_info >= (3, 5): - comp_35(args, **spec_kwargs) - if sys.version_info >= (3, 6): - comp_36(args, **spec_kwargs) - if sys.version_info >= (3, 8): - comp_38(args, **spec_kwargs) - - comp_agnostic(agnostic_args, **kwargs) - comp_sys(args, **kwargs) - comp_non_strict(args, **kwargs) - - if use_run_arg: - _kwargs = kwargs.copy() - _kwargs["assert_output"] = True - comp_runner(["--run"] + agnostic_args, **_kwargs) - else: - comp_runner(agnostic_args, **kwargs) - run_src(convert_to_import=convert_to_import) # **kwargs are for comp, not run - - if use_run_arg: - _kwargs = kwargs.copy() - _kwargs["assert_output"] = True - _kwargs["check_errors"] = False - _kwargs["stderr_first"] = True - comp_extras(["--run"] + agnostic_args, **_kwargs) - else: - comp_extras(agnostic_args, **kwargs) - run_extras(convert_to_import=convert_to_import) # **kwargs are for comp, not run + with (using_caches() if manage_cache else noop_ctx()): + with using_dest(): + with (using_dest(additional_dest) if "--and" in args else noop_ctx()): + + spec_kwargs = kwargs.copy() + spec_kwargs["always_sys"] = always_sys + if PY2: + comp_2(args, **spec_kwargs) + else: + comp_3(args, **spec_kwargs) + if sys.version_info >= (3, 5): + comp_35(args, **spec_kwargs) + if sys.version_info >= (3, 6): + comp_36(args, **spec_kwargs) + if sys.version_info >= (3, 8): + comp_38(args, **spec_kwargs) + if sys.version_info >= (3, 11): + comp_311(args, **spec_kwargs) + + comp_agnostic(agnostic_args, **kwargs) + comp_sys(args, **kwargs) + # do non-strict at the end so we get the non-strict header + comp_non_strict(args, **kwargs) + + if use_run_arg: + _kwargs = kwargs.copy() + _kwargs["assert_output"] = True + comp_runner(["--run"] + agnostic_args, **_kwargs) + else: + comp_runner(agnostic_args, **kwargs) + run_src(convert_to_import=convert_to_import) # **kwargs are for comp, not run + + if use_run_arg: + _kwargs = kwargs.copy() + _kwargs["assert_output"] = True + _kwargs["check_errors"] = False + _kwargs["stderr_first"] = True + comp_extras(["--run"] + agnostic_args, **_kwargs) + else: + comp_extras(agnostic_args, **kwargs) + run_extras(convert_to_import=convert_to_import) # **kwargs are for comp, not run def comp_all(args=[], agnostic_target=None, **kwargs): @@ -637,18 +719,20 @@ def comp_all(args=[], agnostic_target=None, **kwargs): except Exception: pass + comp_agnostic(agnostic_args, **kwargs) + comp_runner(agnostic_args, **kwargs) + comp_extras(agnostic_args, **kwargs) + comp_2(args, **kwargs) comp_3(args, **kwargs) comp_35(args, **kwargs) comp_36(args, **kwargs) comp_38(args, **kwargs) + comp_311(args, **kwargs) comp_sys(args, **kwargs) + # do non-strict at the end so we get the non-strict header comp_non_strict(args, **kwargs) - comp_agnostic(agnostic_args, **kwargs) - comp_runner(agnostic_args, **kwargs) - comp_extras(agnostic_args, **kwargs) - def comp_pyston(args=[], **kwargs): """Compiles evhub/pyston.""" @@ -776,6 +860,13 @@ def test_import_hook(self): reload(runnable) assert runnable.success == "" + def test_find_packages(self): + with using_pys_in(agnostic_dir): + with using_coconut(): + from coconut.api import find_packages, find_and_compile_packages + assert find_packages(cocotest_dir) == ["agnostic"] + assert find_and_compile_packages(cocotest_dir) == ["agnostic"] + def test_runnable(self): run_runnable() @@ -808,6 +899,8 @@ def test_xontrib(self): p.sendline('echo f"{$ENV_VAR}"; echo f"{$ENV_VAR}"') p.expect("ABC") p.expect("ABC") + p.sendline('len("""1\n3\n5""")\n') + p.expect("5") if not PYPY or PY39: if PY36: p.sendline("echo 123;; 123") @@ -842,7 +935,6 @@ def test_ipython_extension(self): def test_kernel_installation(self): call(["coconut", "--jupyter"], assert_output=kernel_installation_msg) stdout, stderr, retcode = call_output(["jupyter", "kernelspec", "list"]) - stdout, stderr = "".join(stdout), "".join(stderr) if not stdout: stdout, stderr = stderr, "" assert not retcode and not stderr, stderr @@ -904,8 +996,19 @@ def test_package(self): def test_no_tco(self): run(["--no-tco"]) - # run fewer tests on Windows so appveyor doesn't time out - if not WINDOWS: + if PY35: + def test_no_wrap(self): + run(["--no-wrap"]) + + if TEST_ALL: + if CPYTHON: + def test_any_of(self): + with using_env_vars({ + adaptive_any_of_env_var: "True", + reverse_any_of_env_var: "True", + }): + run() + def test_keep_lines(self): run(["--keep-lines"]) @@ -915,9 +1018,18 @@ def test_strict(self): def test_and(self): run(["--and"]) # src and dest built by comp - if PY35: - def test_no_wrap(self): - run(["--no-wrap"]) + def test_run_arg(self): + run(use_run_arg=True) + + if not PYPY and not PY26: + def test_jobs_zero(self): + run(["--jobs", "0"]) + + if not PYPY: + def test_incremental(self): + with using_caches(): + run(manage_cache=False) + run(["--force"], manage_cache=False) if get_bool_env_var("COCONUT_TEST_VERBOSE"): def test_verbose(self): @@ -927,19 +1039,8 @@ def test_verbose(self): def test_trace(self): run(["--jobs", "0", "--trace"], check_errors=False) - # avoids a strange, unreproducable failure on appveyor - if not (WINDOWS and sys.version_info[:2] == (3, 8)): - def test_run_arg(self): - run(use_run_arg=True) - - # not WINDOWS is for appveyor timeout prevention - if not WINDOWS and not PYPY and not PY26: - def test_jobs_zero(self): - run(["--jobs", "0"]) - -# more appveyor timeout prevention -if not WINDOWS: +if TEST_ALL: @add_test_func_names class TestExternal(unittest.TestCase): diff --git a/coconut/tests/src/cocotest/agnostic/main.coco b/coconut/tests/src/cocotest/agnostic/main.coco index 2e5402122..97c9d3df7 100644 --- a/coconut/tests/src/cocotest/agnostic/main.coco +++ b/coconut/tests/src/cocotest/agnostic/main.coco @@ -1,6 +1,8 @@ import sys -from .primary import assert_raises, primary_test +from .util import assert_raises +from .primary_1 import primary_test_1 +from .primary_2 import primary_test_2 def test_asyncio() -> bool: @@ -49,9 +51,12 @@ def run_main(outer_MatchError, test_easter_eggs=False) -> bool: using_tco = "_coconut_tco" in globals() or "_coconut_tco" in locals() print_dot() # .. - assert primary_test() is True + assert primary_test_1() is True print_dot() # ... + assert primary_test_2() is True + + print_dot() # .... from .specific import ( non_py26_test, non_py32_test, @@ -76,11 +81,11 @@ def run_main(outer_MatchError, test_easter_eggs=False) -> bool: if sys.version_info >= (3, 8): assert py38_spec_test() is True - print_dot() # .... + print_dot() # ..... from .suite import suite_test, tco_test assert suite_test() is True - print_dot() # ..... + print_dot() # ...... assert mypy_test() is True if using_tco: assert hasattr(tco_func, "_coconut_tco_func") @@ -88,7 +93,7 @@ def run_main(outer_MatchError, test_easter_eggs=False) -> bool: if outer_MatchError.__module__ != "__main__": assert package_test(outer_MatchError) is True - print_dot() # ...... + print_dot() # ....... if sys.version_info < (3,): from .py2_test import py2_test assert py2_test() is True @@ -104,22 +109,25 @@ def run_main(outer_MatchError, test_easter_eggs=False) -> bool: if sys.version_info >= (3, 8): from .py38_test import py38_test assert py38_test() is True + if sys.version_info >= (3, 11): + from .py311_test import py311_test + assert py311_test() is True - print_dot() # ....... + print_dot() # ........ from .target_sys_test import TEST_ASYNCIO, target_sys_test if TEST_ASYNCIO: assert test_asyncio() is True assert target_sys_test() is True - print_dot() # ........ + print_dot() # ......... from .non_strict_test import non_strict_test assert non_strict_test() is True - print_dot() # ......... + print_dot() # .......... from . import tutorial # noQA if test_easter_eggs: - print(".", end="") # .......... + print_dot() # ........... assert easter_egg_test() is True print("\n") diff --git a/coconut/tests/src/cocotest/agnostic/primary.coco b/coconut/tests/src/cocotest/agnostic/primary_1.coco similarity index 73% rename from coconut/tests/src/cocotest/agnostic/primary.coco rename to coconut/tests/src/cocotest/agnostic/primary_1.coco index b4db19453..bc85179c7 100644 --- a/coconut/tests/src/cocotest/agnostic/primary.coco +++ b/coconut/tests/src/cocotest/agnostic/primary_1.coco @@ -1,7 +1,6 @@ import itertools import collections import collections.abc -import weakref import platform from copy import copy @@ -12,11 +11,11 @@ from importlib import reload # NOQA if platform.python_implementation() == "CPython": # fixes weird aenum issue on pypy from enum import Enum # noqa -from .util import assert_raises, typed_eq +from .util import assert_raises -def primary_test() -> bool: - """Basic no-dependency tests.""" +def primary_test_1() -> bool: + """Basic no-dependency tests (1/2).""" # must come at start so that local sys binding is correct import sys import queue as q, builtins, email.mime.base @@ -212,19 +211,12 @@ def primary_test() -> bool: assert map((-), range(5)).iters[0] |> tuple == range(5) |> tuple == zip(range(5), range(6)).iters[0] |> tuple # type: ignore assert repr(zip((0,1), (1,2))) == "zip((0, 1), (1, 2))" assert repr(map((-), range(5))).startswith("map(") # type: ignore - assert repr(parallel_map((-), range(5))).startswith("parallel_map(") # type: ignore - assert parallel_map((-), range(5)) |> tuple == (0, -1, -2, -3, -4) == parallel_map(map$((-)), (range(5),))$[0] |> tuple # type: ignore - assert parallel_map(zip, (range(2),), (range(2),)) |> map$(tuple) |> tuple == (((0,0), (1,1)),) # type: ignore - assert parallel_map(zip_longest$(fillvalue=10), (range(1),), (range(2),)) |> map$(tuple) |> tuple == (((0,0), (10,1)),) # type: ignore + assert repr(thread_map((-), range(5))).startswith("thread_map(") # type: ignore + with thread_map.multiple_sequential_calls(max_workers=4): # type: ignore + assert thread_map((-), range(5), stream=True) |> tuple == (0, -1, -2, -3, -4) == thread_map(map$((-)), (range(5),))$[0] |> tuple # type: ignore + assert thread_map(zip, (range(2),), (range(2),)) |> map$(tuple) |> tuple == (((0,0), (1,1)),) # type: ignore assert (range(0, 5), range(5, 10)) |*> map$(+) |> tuple == (5, 7, 9, 11, 13) - assert parallel_map((*)$(2)..(+)$(1), range(5)) |> tuple == (2, 4, 6, 8, 10) - assert parallel_map((+), range(5), range(5), chunksize=2) |> list == map((*)$(2), range(5)) |> list == concurrent_map((+), range(5), range(5), chunksize=2) |> list # type: ignore - assert repr(concurrent_map((-), range(5))).startswith("concurrent_map(") # type: ignore - with concurrent_map.multiple_sequential_calls(max_workers=4): # type: ignore - assert concurrent_map((-), range(5)) |> tuple == (0, -1, -2, -3, -4) == concurrent_map(map$((-)), (range(5),))$[0] |> tuple # type: ignore - assert concurrent_map(zip, (range(2),), (range(2),)) |> map$(tuple) |> tuple == (((0,0), (1,1)),) # type: ignore - assert (range(0, 5), range(5, 10)) |*> map$(+) |> tuple == (5, 7, 9, 11, 13) - assert concurrent_map((*)$(2)..(+)$(1), range(5)) |> tuple == (2, 4, 6, 8, 10) + assert thread_map((*)$(2)..(+)$(1), range(5)) |> tuple == (2, 4, 6, 8, 10) assert 0 in range(1) assert range(1).count(0) == 1 assert 2 in range(5) @@ -320,7 +312,6 @@ def primary_test() -> bool: assert pow$(?, 2)(3) == 9 assert [] |> reduce$((+), ?, ()) == () assert pow$(?, 2) |> repr == "$(?, 2)" - assert parallel_map(pow$(?, 2), range(10)) |> tuple == (0, 1, 4, 9, 16, 25, 36, 49, 64, 81) assert pow$(?, 2).args == (None, 2) assert range(20) |> filter$((x) -> x < 5) |> reversed |> tuple == (4,3,2,1,0) == (0,1,2,3,4,5,6,7,8,9) |> filter$((x) -> x < 5) |> reversed |> tuple # type: ignore assert (range(10) |> map$((x) -> x) |> reversed) `isinstance` map # type: ignore @@ -572,8 +563,9 @@ def primary_test() -> bool: a = A() f = 10 def a.f(x) = x # type: ignore + def a.a(x) = x # type: ignore assert f == 10 - assert a.f 1 == 1 + assert a.f 1 == 1 == a.a 1 def f(x, y) = (x, y) # type: ignore assert f 1 2 == (1, 2) def f(0) = 'a' # type: ignore @@ -623,15 +615,20 @@ def primary_test() -> bool: it3 = iter(it2) item3 = next(it3) assert item3 != item2 - for map_func in (parallel_map, concurrent_map): + for map_func in (process_map, thread_map): m1 = map_func((+)$(1), range(5)) assert m1 `isinstance` map_func with map_func.multiple_sequential_calls(): # type: ignore m2 = map_func((+)$(1), range(5)) - assert m2 `isinstance` list + assert m2 `isinstance` tuple assert m1.result is None - assert m2 == [1, 2, 3, 4, 5] == list(m1) - assert m1.result == [1, 2, 3, 4, 5] == list(m1) + assert m2 == (1, 2, 3, 4, 5) == tuple(m1) + m3 = tuple(map_func((.+1), range(5), stream=True)) + assert m3 == (1, 2, 3, 4, 5) + m4 = set(map_func((.+1), range(5), ordered=False)) + m5 = set(map_func((.+1), range(5), ordered=False, stream=True)) + assert m4 == {1, 2, 3, 4, 5} == m5 + assert m1.result == (1, 2, 3, 4, 5) == tuple(m1) for it in ((), [], (||)): assert_raises(-> it$[0], IndexError) assert_raises(-> it$[-1], IndexError) @@ -944,18 +941,18 @@ def primary_test() -> bool: assert 1 `(,)` 2 == (1, 2) == (,) 1 2 assert (-1+.)(2) == 1 ==-1 = -1 - assert collectby((def -> assert False), [], (def (x,y) -> assert False)) == {} - assert collectby(ident, range(5)) == {0: [0], 1: [1], 2: [2], 3: [3], 4: [4]} - assert collectby(.[1], zip(range(5), reversed(range(5)))) == {0: [(4, 0)], 1: [(3, 1)], 2: [(2, 2)], 3: [(1, 3)], 4: [(0, 4)]} - assert collectby(ident, range(5) :: range(5)) == {0: [0, 0], 1: [1, 1], 2: [2, 2], 3: [3, 3], 4: [4, 4]} - assert collectby(ident, range(5) :: range(5), reduce_func=(+)) == {0: 0, 1: 2, 2: 4, 3: 6, 4: 8} == collectby(ident, range(5) :: range(5), ident, (+)) + assert collectby((def -> assert False), [], (def (x,y) -> assert False)) == {} == collectby((def -> assert False), [], (def (x,y) -> assert False), map_using=map) + assert collectby(ident, range(5)) == {0: [0], 1: [1], 2: [2], 3: [3], 4: [4]} == collectby(ident, range(5), map_using=map) + assert collectby(.[1], zip(range(5), reversed(range(5)))) == {0: [(4, 0)], 1: [(3, 1)], 2: [(2, 2)], 3: [(1, 3)], 4: [(0, 4)]} == collectby(.[1], zip(range(5), reversed(range(5))), map_using=map) + assert collectby(ident, range(5) :: range(5)) == {0: [0, 0], 1: [1, 1], 2: [2, 2], 3: [3, 3], 4: [4, 4]} == collectby(ident, range(5) :: range(5), map_using=map) + assert collectby(ident, range(5) :: range(5), reduce_func=(+)) == {0: 0, 1: 2, 2: 4, 3: 6, 4: 8} == collectby(ident, range(5) :: range(5), reduce_func=(+), map_using=map) def dub(xs) = xs :: xs - assert collectby(.[0], dub <| zip(range(5), reversed(range(5))), value_func=.[1], reduce_func=(+)) == {0: 8, 1: 6, 2: 4, 3: 2, 4: 0} + assert collectby(.[0], dub <| zip(range(5), reversed(range(5))), value_func=.[1], reduce_func=(+)) == {0: 8, 1: 6, 2: 4, 3: 2, 4: 0} == mapreduce(ident, dub <| zip(range(5), reversed(range(5))), reduce_func=(+)) assert int(1e9) in range(2**31-1) assert (a=1, b=2) == (1, 2) == (a:int=1, b=2) assert (a=1, b: int = 2) == (1, 2) == (a: int=1, b: int=2) assert "_namedtuple_of" in repr((a=1,)) - assert "b=2" in repr <| call$(?, a=1, b=2) + assert "b=2" in repr <| call$(?, 1, b=2) assert lift((,), (.*2), (.**2))(3) == (6, 9) assert_raises(-> (⁻)(1, 2), TypeError) assert -1 == ⁻1 @@ -1144,10 +1141,6 @@ def primary_test() -> bool: def __call__(self) = super().__call__() HasSuper assert HasSuper5.HasHasSuper.HasSuper()() == 10 == HasSuper6().get_HasSuper()()() - assert parallel_map((.+(10,)), [ - (a=1, b=2), - (x=3, y=4), - ]) |> list == [(1, 2, 10), (3, 4, 10)] assert f"{'a' + 'b'}" == "ab" int_str_tup: (int; str) = (1, "a") key = "abc" @@ -1298,350 +1291,18 @@ def primary_test() -> bool: assert err is some_err assert Expected(error=TypeError()).map_error(const some_err) == Expected(error=some_err) assert Expected(10).map_error(const some_err) == Expected(10) + assert repr(process_map((-), range(5))).startswith("process_map(") # type: ignore - recit = ([1,2,3] :: recit) |> map$(.+1) - assert tee(recit) - rawit = (_ for _ in (0, 1)) - t1, t2 = tee(rawit) - t1a, t1b = tee(t1) - assert (list(t1a), list(t1b), list(t2)) == ([0, 1], [0, 1], [0, 1]) - assert m{1, 3, 1}[1] == 2 - assert m{1, 2} |> repr in ("multiset({1: 1, 2: 1})", "multiset({2: 1, 1: 1})") - m = m{} - m.add(1) - m.add(1) - m.add(2) - assert m == m{1, 1, 2} - assert m != m{1, 2} - m.discard(2) - m.discard(2) - assert m == m{1, 1} - assert m != m{1} - m.remove(1) - assert m == m{1} - m.remove(1) - assert m == m{} - assert_raises(-> m.remove(1), KeyError) - assert 1 not in m - assert 2 not in m - assert m{1, 2}.isdisjoint(m{3, 4}) - assert not m{1, 2}.isdisjoint(m{2, 3}) - assert m{1, 2} ^ m{2, 3} `typed_eq` m{1, 3} - m = m{1, 2} - m ^= m{2, 3} - assert m `typed_eq` m{1, 3} - assert m{1, 1} ^ m{1} `typed_eq` m{1} - assert multiset((1, 2)) == m{1, 2} == multiset(m{1, 2}) - assert multiset({1: 2, 2: 1}) == m{1, 1, 2} - assert m{} `isinstance` multiset - assert m{} `isinstance` collections.abc.Set - assert m{} `isinstance` collections.abc.MutableSet - assert True `isinstance` bool - class HasBool: - def __bool__(self) = False - assert not HasBool() - assert m{1}.count(2) == 0 - assert m{1, 1}.count(1) == 2 - bad_m = m{} - bad_m[1] = -1 - assert_raises(-> bad_m.count(1), ValueError) - assert len(m{1, 1}) == 1 - assert m{1, 1}.total() == 2 == m{1, 2}.total() - weird_m = m{1, 2} - weird_m[3] = 0 - assert weird_m == m{1, 2} - assert not (weird_m != m{1, 2}) - assert m{} <= m{} < m{1, 2} < m{1, 1, 2} <= m{1, 1, 2} - assert m{1, 1, 2} >= m{1, 1, 2} > m{1, 2} > m{} >= m{} - assert m{1} != {1:1, 2:0} - assert not (m{1} == {1:1, 2:0}) - assert s{1, 2, *(2, 3, 4), *(4, 5)} == s{1, 2, 3, 4, 5} - assert m{1, 2, *(2, 3, 4), *(4, 5)} == m{1, 2, 2, 3, 4, 4, 5} - assert {*(1, 2)} == {1, 2} - assert cycle(range(3))[:5] |> list == [0, 1, 2, 0, 1] == cycle(range(3)) |> iter |> .$[:5] |> list - assert cycle(range(2), 2)[:5] |> list == [0, 1, 0, 1] == cycle(range(2), 2) |> iter |> .$[:5] |> list - assert 2 in cycle(range(3)) - assert reversed(cycle(range(2), 2)) |> list == [1, 0, 1, 0] - assert cycle((_ for _ in range(2)), 2) |> list == [0, 1, 0, 1] - assert cycle(range(3)).count(0) == float("inf") - assert cycle(range(3), 3).index(2) == 2 - assert zip(range(2), range(2))[0] == (0, 0) == enumerate(range(2))[0] - assert reversed([0,1,3])[0] == 3 - assert cycle((), 0) |> list == [] - assert "1234" |> windowsof$(2) |> map$("".join) |> list == ["12", "23", "34"] - assert len(windowsof(2, "1234")) == 3 - assert windowsof(3, "12345", None) |> map$("".join) |> list == ["123", "234", "345"] - assert len(windowsof(3, "12345", None)) == 3 - assert windowsof(3, "1") |> list == [] == windowsof(2, "1", step=2) |> list - assert len(windowsof(2, "1")) == 0 == len(windowsof(2, "1", step=2)) - assert windowsof(2, "1", None) |> list == [("1", None)] == windowsof(2, "1", None, 2) |> list - assert len(windowsof(2, "1", None)) == 1 == len(windowsof(2, "1", None, 2)) - assert windowsof(2, "1234", step=2) |> map$("".join) |> list == ["12", "34"] == windowsof(2, "1234", fillvalue=None, step=2) |> map$("".join) |> list - assert len(windowsof(2, "1234", step=2)) == 2 == len(windowsof(2, "1234", fillvalue=None, step=2)) - assert repr(windowsof(2, "1234", None, 3)) == "windowsof(2, '1234', fillvalue=None, step=3)" - assert lift(,)((+), (*))(2, 3) == (5, 6) - assert "abac" |> windowsof$(2) |> filter$(addpattern( - (def (("a", b) if b != "b") -> True), - (def ((_, _)) -> False), - )) |> list == [("a", "c")] - assert "[A], [B]" |> windowsof$(3) |> map$(addpattern( - (def (("[","A","]")) -> "A"), - (def (("[","B","]")) -> "B"), - (def ((_,_,_)) -> None), - )) |> filter$((.is None) ..> (not)) |> list == ["A", "B"] - assert windowsof(3, "abcdefg", step=3) |> map$("".join) |> list == ["abc", "def"] - assert windowsof(3, "abcdefg", step=3) |> len == 2 - assert windowsof(3, "abcdefg", step=3, fillvalue="") |> map$("".join) |> list == ["abc", "def", "g"] - assert windowsof(3, "abcdefg", step=3, fillvalue="") |> len == 3 - assert windowsof(3, "abcdefg", step=2, fillvalue="") |> map$("".join) |> list == ["abc", "cde", "efg", "g"] - assert windowsof(3, "abcdefg", step=2, fillvalue="") |> len == 4 - assert windowsof(5, "abc", step=2, fillvalue="") |> map$("".join) |> list == ["abc"] - assert windowsof(5, "abc", step=2, fillvalue="") |> len == 1 - assert groupsof(2, "123", fillvalue="") |> map$("".join) |> list == ["12", "3"] - assert groupsof(2, "123", fillvalue="") |> len == 2 - assert groupsof(2, "123", fillvalue="") |> repr == "groupsof(2, '123', fillvalue='')" - assert flip((,), 0)(1, 2) == (1, 2) - assert flatten([1, 2, [3, 4]], 0) == [1, 2, [3, 4]] - assert flatten([1, 2, [3, 4]], None) |> list == [1, 2, 3, 4] - assert flatten([1, 2, [3, 4]], None) |> reversed |> list == [4, 3, 2, 1] - assert_raises(-> flatten([1, 2, [3, 4]]) |> list, TypeError) - assert flatten([[[1,2]], [[3], [4]]], 2) |> list == [1, 2, 3, 4] - assert flatten([[[1,2]], [[3], [4]]], 2) |> reversed |> list == [4, 3, 2, 1] - assert_raises(-> map((+), range(3), range(4), strict=True) |> list, ValueError) - assert map((+), range(3), range(4)$[:-1], strict=True) |> list == [0, 2, 4] == parallel_map((+), range(3), range(4)$[:-1], strict=True) |> list - assert range(3) |> map$((.+1), strict=True) |> list == [1, 2, 3] == range(3) |> parallel_map$((.+1), strict=True) |> list - assert cartesian_product((1, 2), (3, 4), repeat=0) |> list == [()] - assert (a=1, b=2)[1] == 2 - obj = object() - assert_raises((def -> obj.abc = 123), AttributeError) # type: ignore - hardref = map((.+1), [1,2,3]) - assert weakref.ref(hardref)() |> list == [2, 3, 4] - my_match_err = MatchError("my match error", 123) - assert parallel_map(ident, [my_match_err]) |> list |> str == str([my_match_err]) - # repeat the same thing again now that my_match_err.str has been called - assert parallel_map(ident, [my_match_err]) |> list |> str == str([my_match_err]) - match data tuple(1, 2) in (1, 2, 3): - assert False - data TestDefaultMatching(x="x default", y="y default") - TestDefaultMatching(got_x) = TestDefaultMatching(1) - assert got_x == 1 - TestDefaultMatching(y=got_y) = TestDefaultMatching(y=10) - assert got_y == 10 - TestDefaultMatching() = TestDefaultMatching() - data HasStar(x, y, *zs) - HasStar(x, *ys) = HasStar(1, 2, 3, 4) - assert x == 1 - assert ys == (2, 3, 4) - HasStar(x, y, z) = HasStar(1, 2, 3) - assert (x, y, z) == (1, 2, 3) - HasStar(5, y=10) = HasStar(5, 10) - HasStar(1, 2, 3, zs=(3,)) = HasStar(1, 2, 3) - HasStar(x=1, y=2) = HasStar(1, 2) - match HasStar(x) in HasStar(1, 2): - assert False - match HasStar(x, y) in HasStar(1, 2, 3): - assert False - data HasStarAndDef(x, y="y", *zs) - HasStarAndDef(1, "y") = HasStarAndDef(1) - HasStarAndDef(1) = HasStarAndDef(1) - HasStarAndDef(x=1) = HasStarAndDef(1) - HasStarAndDef(1, 2, 3, zs=(3,)) = HasStarAndDef(1, 2, 3) - HasStarAndDef(1, y=2) = HasStarAndDef(1, 2) - match HasStarAndDef(x, y) in HasStarAndDef(1, 2, 3): - assert False - - assert (.+1) kwargs) <**?| None is None - assert ((**kwargs) -> kwargs) <**?| {"a": 1, "b": 2} == {"a": 1, "b": 2} - assert (<**?|)((**kwargs) -> kwargs, None) is None - assert (<**?|)((**kwargs) -> kwargs, {"a": 1, "b": 2}) == {"a": 1, "b": 2} - optx = (**kwargs) -> kwargs - optx <**?|= None - assert optx is None - optx = (**kwargs) -> kwargs - optx <**?|= {"a": 1, "b": 2} - assert optx == {"a": 1, "b": 2} - - assert `const None ..?> (.+1)` is None is (..?>)(const None, (.+1))() - assert `(.+1) (.+1)` == 6 == (..?>)(const 5, (.+1))() - assert `(.+1) (+)` is None is (..?*>)(const None, (+))() - assert `(+) <*?.. const None` is None is (<*?..)((+), const None)() - assert `const((5, 2)) ..?*> (+)` == 7 == (..?*>)(const((5, 2)), (+))() - assert `(+) <*?.. const((5, 2))` == 7 == (<*?..)((+), const((5, 2)))() - assert `const None ..?**> (**kwargs) -> kwargs` is None is (..?**>)(const None, (**kwargs) -> kwargs)() - assert `((**kwargs) -> kwargs) <**?.. const None` is None is (<**?..)((**kwargs) -> kwargs, const None)() - assert `const({"a": 1}) ..?**> (**kwargs) -> kwargs` == {"a": 1} == (..?**>)(const({"a": 1}), (**kwargs) -> kwargs)() - assert `((**kwargs) -> kwargs) <**?.. const({"a": 1})` == {"a": 1} == (<**?..)((**kwargs) -> kwargs, const({"a": 1}))() - optx = const None - optx ..?>= (.+1) - optx ..?*>= (+) - optx ..?**>= (,) - assert optx() is None - optx = (.+1) - optx five (two + three), TypeError) - assert_raises(-> 5 (10), TypeError) - assert_raises(-> 5 [0], TypeError) - assert five ** 2 two == 50 - assert 2i x == 20i - some_str = "some" - assert_raises(-> some_str five, TypeError) - assert (not in)("a", "bcd") - assert not (not in)("a", "abc") - assert ("a" not in .)("bcd") - assert (. not in "abc")("d") - assert (is not)(1, True) - assert not (is not)(False, False) - assert (True is not .)(1) - assert (. is not True)(1) - a_dict = {} - a_dict[1] = 1 - a_dict[3] = 2 - a_dict[2] = 3 - assert a_dict |> str == "{1: 1, 3: 2, 2: 3}" == a_dict |> repr, a_dict - assert a_dict.keys() |> tuple == (1, 3, 2) - assert not a_dict.keys() `isinstance` list - assert not a_dict.values() `isinstance` list - assert not a_dict.items() `isinstance` list - assert len(a_dict.keys()) == len(a_dict.values()) == len(a_dict.items()) == 3 - assert {1: 1, 3: 2, 2: 3}.keys() |> tuple == (1, 3, 2) - assert {**{1: 0, 3: 0}, 2: 0}.keys() |> tuple == (1, 3, 2) == {**dict([(1, 1), (3, 2), (2, 3)])}.keys() |> tuple - assert a_dict == {1: 1, 2: 3, 3: 2} - assert {1: 1} |> str == "{1: 1}" == {1: 1} |> repr - assert py_dict `issubclass` dict - assert py_dict() `isinstance` dict - assert {5:0, 3:0, **{2:0, 6:0}, 8:0}.keys() |> tuple == (5, 3, 2, 6, 8) - a_multiset = m{1,1,2} - assert not a_multiset.keys() `isinstance` list - assert not a_multiset.values() `isinstance` list - assert not a_multiset.items() `isinstance` list - assert len(a_multiset.keys()) == len(a_multiset.values()) == len(a_multiset.items()) == 2 - assert (in)(1, [1, 2]) - assert not (1 not in .)([1, 2]) - assert not (in)([[]], []) - assert ("{a}" . .)("format")(a=1) == "1" - a_dict = {"a": 1, "b": 2} - a_dict |= {"a": 10, "c": 20} - assert a_dict == {"a": 10, "b": 2, "c": 20} == {"a": 1, "b": 2} | {"a": 10, "c": 20} - assert ["abc" ; "def"] == ['abc', 'def'] - assert ["abc" ;; "def"] == [['abc'], ['def']] - assert {"a":0, "b":1}$[0] == "a" - assert (|0, NotImplemented, 2|)$[1] is NotImplemented - assert m{1, 1, 2} |> fmap$(.+1) == m{2, 2, 3} - assert (+) ..> ((*) ..> (/)) == (+) ..> (*) ..> (/) == ((+) ..> (*)) ..> (/) - def f(x, y=1) = x, y # type: ignore - f.is_f = True # type: ignore - assert (f ..*> (+)).is_f # type: ignore - really_long_var = 10 - assert (...=really_long_var) == (10,) - assert (...=really_long_var, abc="abc") == (10, "abc") - assert (abc="abc", ...=really_long_var) == ("abc", 10) - assert (...=really_long_var).really_long_var == 10 - n = [0] - assert n[0] == 0 - assert_raises(-> m{{1:2,2:3}}, TypeError) - assert_raises((def -> from typing import blah), ImportError) # NOQA - assert type(m{1, 2}) is multiset - assert multiset(collections.Counter({1: 1, 2: 1})) `typed_eq` m{1, 2} - assert +m{-1, 1} `typed_eq` m{-1, 1} - assert -m{-1, 1} `typed_eq` m{} - assert m{1, 1, 2} + m{1, 3} `typed_eq` m{1, 1, 1, 2, 3} - assert m{1, 1, 2} | m{1, 3} `typed_eq` m{1, 1, 2, 3} - assert m{1, 1, 2} & m{1, 3} `typed_eq` m{1} - assert m{1, 1, 2} - m{1, 3} `typed_eq` m{1, 2} - assert (.+1) `and_then` (.*2) `and_then_await` (./3) |> repr == "$(?, 1) `and_then` $(?, 2) `and_then_await` $(?, 3)" - assert 5.5⏨3 == 5.5 * 10**3 - assert (x => x)(5) == 5 == (def x => x)(5) - assert (=> _)(5) == 5 == (def => _)(5) - assert ((x, y) => (x, y))(1, 2) == (1, 2) == (def (x, y) => (x, y))(1, 2) - assert (def (int(x)) => x)(5) == 5 == (def (int -> x) => x)("5") - assert (def (x: int) -> int => x)(5) == 5 == (def (int(x)) -> int => x)(5) - assert (x ⇒ x)(5) == 5 == (def x ⇒ x)(5) - assert f"a: { "abc" }" == "a: abc" == f'a: { 'abc' }' - assert f"1 + {"a" + "b"} + 2 + {"c" + "d"}" == "1 + ab + 2 + cd" == f'1 + {'a' + 'b'} + 2 + {'c' + 'd'}' - assert f"{"a" + "b"} + c + {"d" + "e"}" == "ab + c + de" == f'{'a' + 'b'} + c + {'d' + 'e'}' - assert f"""{""" -"""}""" == """ -""" == f"""{''' -'''}""" - assert f"""{( - )}""" == "()" == f'''{( - )}''' - assert f"{'\n'.join(["", ""])}" == "\n" - assert f"""{f'''{f'{f"{1+1}"}'}'''}""" == "2" == f"{f"{f"{f"{f"{f"{1+1}"}"}"}"}"}" - assert f"___{ - 1 -}___" == '___1___' == f"___{( - 1 -)}___" + with process_map.multiple_sequential_calls(): # type: ignore + assert process_map((-), range(5)) |> tuple == (0, -1, -2, -3, -4) == process_map(map$((-)), (range(5),))$[0] |> tuple # type: ignore + assert process_map(zip, (range(2),), (range(2),)) |> map$(tuple) |> tuple == (((0,0), (1,1)),) # type: ignore + assert process_map(zip_longest$(fillvalue=10), (range(1),), (range(2),)) |> map$(tuple) |> tuple == (((0,0), (10,1)),) # type: ignore + assert (range(0, 5), range(5, 10)) |*> map$(+) |> tuple == (5, 7, 9, 11, 13) + assert process_map((*)$(2)..(+)$(1), range(5)) |> tuple == (2, 4, 6, 8, 10) + assert process_map((+), range(5), range(5), chunksize=2) |> list == map((*)$(2), range(5)) |> list == thread_map((+), range(5), range(5), chunksize=2) |> list # type: ignore + assert process_map(pow$(?, 2), range(10)) |> tuple == (0, 1, 4, 9, 16, 25, 36, 49, 64, 81) + assert process_map((.+(10,)), [ + (a=1, b=2), + (x=3, y=4), + ]) |> list == [(1, 2, 10), (3, 4, 10)] return True diff --git a/coconut/tests/src/cocotest/agnostic/primary_2.coco b/coconut/tests/src/cocotest/agnostic/primary_2.coco new file mode 100644 index 000000000..b4e55fb2e --- /dev/null +++ b/coconut/tests/src/cocotest/agnostic/primary_2.coco @@ -0,0 +1,422 @@ +import collections +import collections.abc +import weakref +import sys + +if TYPE_CHECKING or sys.version_info >= (3, 5): + from typing import Any, Iterable +from importlib import reload # NOQA + +from .util import assert_raises, typed_eq + +operator ! +from math import factorial as (!) + + +def primary_test_2() -> bool: + """Basic no-dependency tests (2/2).""" + recit: Iterable[int] = ([1,2,3] :: recit) |> map$(.+1) + assert tee(recit) + rawit = (_ for _ in (0, 1)) + t1, t2 = tee(rawit) + t1a, t1b = tee(t1) + assert (list(t1a), list(t1b), list(t2)) == ([0, 1], [0, 1], [0, 1]) + assert m{1, 3, 1}[1] == 2 + assert m{1, 2} |> repr in ("multiset({1: 1, 2: 1})", "multiset({2: 1, 1: 1})") + m: multiset = m{} + m.add(1) + m.add(1) + m.add(2) + assert m == m{1, 1, 2} + assert m != m{1, 2} + m.discard(2) + m.discard(2) + assert m == m{1, 1} + assert m != m{1} + m.remove(1) + assert m == m{1} + m.remove(1) + assert m == m{} + assert_raises(-> m.remove(1), KeyError) + assert 1 not in m + assert 2 not in m + assert m{1, 2}.isdisjoint(m{3, 4}) + assert not m{1, 2}.isdisjoint(m{2, 3}) + assert m{1, 2} ^ m{2, 3} `typed_eq` m{1, 3} + m = m{1, 2} + m ^= m{2, 3} + assert m `typed_eq` m{1, 3} + assert m{1, 1} ^ m{1} `typed_eq` m{1} + assert multiset((1, 2)) == m{1, 2} == multiset(m{1, 2}) + assert multiset({1: 2, 2: 1}) == m{1, 1, 2} + assert m{} `isinstance` multiset + assert m{} `isinstance` collections.abc.Set + assert m{} `isinstance` collections.abc.MutableSet + assert True `isinstance` bool + class HasBool: + def __bool__(self) = False + assert not HasBool() + assert m{1}.count(2) == 0 + assert m{1, 1}.count(1) == 2 + bad_m: multiset = m{} + bad_m[1] = -1 + assert_raises(-> bad_m.count(1), ValueError) + assert len(m{1, 1}) == 1 + assert m{1, 1}.total() == 2 == m{1, 2}.total() + weird_m = m{1, 2} + weird_m[3] = 0 + assert weird_m == m{1, 2} + assert not (weird_m != m{1, 2}) + assert m{} <= m{} < m{1, 2} < m{1, 1, 2} <= m{1, 1, 2} + assert m{1, 1, 2} >= m{1, 1, 2} > m{1, 2} > m{} >= m{} + assert m{1} != {1:1, 2:0} + assert not (m{1} == {1:1, 2:0}) + assert s{1, 2, *(2, 3, 4), *(4, 5)} == s{1, 2, 3, 4, 5} + assert m{1, 2, *(2, 3, 4), *(4, 5)} == m{1, 2, 2, 3, 4, 4, 5} + assert {*(1, 2)} == {1, 2} + assert cycle(range(3))[:5] |> list == [0, 1, 2, 0, 1] == cycle(range(3)) |> iter |> .$[:5] |> list + assert cycle(range(2), 2)[:5] |> list == [0, 1, 0, 1] == cycle(range(2), 2) |> iter |> .$[:5] |> list + assert 2 in cycle(range(3)) + assert reversed(cycle(range(2), 2)) |> list == [1, 0, 1, 0] + assert cycle((_ for _ in range(2)), 2) |> list == [0, 1, 0, 1] + assert cycle(range(3)).count(0) == float("inf") + assert cycle(range(3), 3).index(2) == 2 + assert zip(range(2), range(2))[0] == (0, 0) == enumerate(range(2))[0] # type: ignore + assert reversed([0,1,3])[0] == 3 # type: ignore + assert cycle((), 0) |> list == [] + assert "1234" |> windowsof$(2) |> map$("".join) |> list == ["12", "23", "34"] + assert len(windowsof(2, "1234")) == 3 + assert windowsof(3, "12345", None) |> map$("".join) |> list == ["123", "234", "345"] # type: ignore + assert len(windowsof(3, "12345", None)) == 3 + assert windowsof(3, "1") |> list == [] == windowsof(2, "1", step=2) |> list + assert len(windowsof(2, "1")) == 0 == len(windowsof(2, "1", step=2)) + assert windowsof(2, "1", None) |> list == [("1", None)] == windowsof(2, "1", None, 2) |> list + assert len(windowsof(2, "1", None)) == 1 == len(windowsof(2, "1", None, 2)) + assert windowsof(2, "1234", step=2) |> map$("".join) |> list == ["12", "34"] == windowsof(2, "1234", fillvalue=None, step=2) |> map$("".join) |> list # type: ignore + assert len(windowsof(2, "1234", step=2)) == 2 == len(windowsof(2, "1234", fillvalue=None, step=2)) + assert repr(windowsof(2, "1234", None, 3)) == "windowsof(2, '1234', fillvalue=None, step=3)" + assert lift(,)((+), (*))(2, 3) == (5, 6) + assert "abac" |> windowsof$(2) |> filter$(addpattern( + (def (("a", b) if b != "b") -> True), + (def ((_, _)) -> False), + )) |> list == [("a", "c")] + assert "[A], [B]" |> windowsof$(3) |> map$(addpattern( + (def (("[","A","]")) -> "A"), + (def (("[","B","]")) -> "B"), + (def ((_,_,_)) -> None), + )) |> filter$((.is None) ..> (not)) |> list == ["A", "B"] + assert windowsof(3, "abcdefg", step=3) |> map$("".join) |> list == ["abc", "def"] + assert windowsof(3, "abcdefg", step=3) |> len == 2 + assert windowsof(3, "abcdefg", step=3, fillvalue="") |> map$("".join) |> list == ["abc", "def", "g"] + assert windowsof(3, "abcdefg", step=3, fillvalue="") |> len == 3 + assert windowsof(3, "abcdefg", step=2, fillvalue="") |> map$("".join) |> list == ["abc", "cde", "efg", "g"] + assert windowsof(3, "abcdefg", step=2, fillvalue="") |> len == 4 + assert windowsof(5, "abc", step=2, fillvalue="") |> map$("".join) |> list == ["abc"] + assert windowsof(5, "abc", step=2, fillvalue="") |> len == 1 + assert groupsof(2, "123", fillvalue="") |> map$("".join) |> list == ["12", "3"] + assert groupsof(2, "123", fillvalue="") |> len == 2 + assert groupsof(2, "123", fillvalue="") |> repr == "groupsof(2, '123', fillvalue='')" + assert flip((,), 0)(1, 2) == (1, 2) + assert flatten([1, 2, [3, 4]], 0) == [1, 2, [3, 4]] # type: ignore + assert flatten([1, 2, [3, 4]], None) |> list == [1, 2, 3, 4] # type: ignore + assert flatten([1, 2, [3, 4]], None) |> reversed |> list == [4, 3, 2, 1] # type: ignore + assert_raises(-> flatten([1, 2, [3, 4]]) |> list, TypeError) # type: ignore + assert flatten([[[1,2]], [[3], [4]]], 2) |> list == [1, 2, 3, 4] + assert flatten([[[1,2]], [[3], [4]]], 2) |> reversed |> list == [4, 3, 2, 1] + assert_raises(-> map((+), range(3), range(4), strict=True) |> list, ValueError) # type: ignore + assert cartesian_product((1, 2), (3, 4), repeat=0) |> list == [()] + assert (a=1, b=2)[1] == 2 + obj = object() + assert_raises((def -> obj.abc = 123), AttributeError) # type: ignore + hardref = map((.+1), [1,2,3]) + assert weakref.ref(hardref)() |> list == [2, 3, 4] # type: ignore + match data tuple(1, 2) in (1, 2, 3): + assert False + data TestDefaultMatching(x="x default", y="y default") + TestDefaultMatching(got_x) = TestDefaultMatching(1) + assert got_x == 1 + TestDefaultMatching(y=got_y) = TestDefaultMatching(y=10) + assert got_y == 10 + TestDefaultMatching() = TestDefaultMatching() + data HasStar(x, y, *zs) + HasStar(x, *ys) = HasStar(1, 2, 3, 4) + assert x == 1 + assert ys == (2, 3, 4) + HasStar(x, y, z) = HasStar(1, 2, 3) + assert (x, y, z) == (1, 2, 3) + HasStar(5, y=10) = HasStar(5, 10) + HasStar(1, 2, 3, zs=(3,)) = HasStar(1, 2, 3) + HasStar(x=1, y=2) = HasStar(1, 2) + match HasStar(x) in HasStar(1, 2): + assert False + match HasStar(x, y) in HasStar(1, 2, 3): + assert False + data HasStarAndDef(x, y="y", *zs) + HasStarAndDef(1, "y") = HasStarAndDef(1) + HasStarAndDef(1) = HasStarAndDef(1) + HasStarAndDef(x=1) = HasStarAndDef(1) + HasStarAndDef(1, 2, 3, zs=(3,)) = HasStarAndDef(1, 2, 3) + HasStarAndDef(1, y=2) = HasStarAndDef(1, 2) + match HasStarAndDef(x, y) in HasStarAndDef(1, 2, 3): + assert False + + assert (.+1) kwargs) <**?| None is None # type: ignore + assert ((**kwargs) -> kwargs) <**?| {"a": 1, "b": 2} == {"a": 1, "b": 2} # type: ignore + assert (<**?|)((**kwargs) -> kwargs, None) is None # type: ignore + assert (<**?|)((**kwargs) -> kwargs, {"a": 1, "b": 2}) == {"a": 1, "b": 2} # type: ignore + optx = (**kwargs) -> kwargs + optx <**?|= None + assert optx is None + optx = (**kwargs) -> kwargs + optx <**?|= {"a": 1, "b": 2} + assert optx == {"a": 1, "b": 2} + + assert `const None ..?> (.+1)` is None is (..?>)(const None, (.+1))() # type: ignore + assert `(.+1) (.+1)` == 6 == (..?>)(const 5, (.+1))() # type: ignore + assert `(.+1) (+)` is None is (..?*>)(const None, (+))() # type: ignore + assert `(+) <*?.. const None` is None is (<*?..)((+), const None)() # type: ignore + assert `const((5, 2)) ..?*> (+)` == 7 == (..?*>)(const((5, 2)), (+))() # type: ignore + assert `(+) <*?.. const((5, 2))` == 7 == (<*?..)((+), const((5, 2)))() # type: ignore + assert `const None ..?**> (**kwargs) -> kwargs` is None is (..?**>)(const None, (**kwargs) -> kwargs)() # type: ignore + assert `((**kwargs) -> kwargs) <**?.. const None` is None is (<**?..)((**kwargs) -> kwargs, const None)() # type: ignore + assert `const({"a": 1}) ..?**> (**kwargs) -> kwargs` == {"a": 1} == (..?**>)(const({"a": 1}), (**kwargs) -> kwargs)() # type: ignore + assert `((**kwargs) -> kwargs) <**?.. const({"a": 1})` == {"a": 1} == (<**?..)((**kwargs) -> kwargs, const({"a": 1}))() # type: ignore + optx = const None + optx ..?>= (.+1) + optx ..?*>= (+) + optx ..?**>= (,) + assert optx() is None + optx = (.+1) + optx five (two + three), TypeError) # type: ignore + assert_raises(-> 5 (10), TypeError) # type: ignore + assert_raises(-> 5 [0], TypeError) # type: ignore + assert five ** 2 two == 50 + assert 2i x == 20i + some_str = "some" + assert_raises(-> some_str five, TypeError) + assert (not in)("a", "bcd") + assert not (not in)("a", "abc") + assert ("a" not in .)("bcd") + assert (. not in "abc")("d") + assert (is not)(1, True) + assert not (is not)(False, False) + assert (True is not .)(1) + assert (. is not True)(1) + a_dict = {} + a_dict[1] = 1 + a_dict[3] = 2 + a_dict[2] = 3 + assert a_dict |> str == "{1: 1, 3: 2, 2: 3}" == a_dict |> repr, a_dict + assert a_dict.keys() |> tuple == (1, 3, 2) + assert not a_dict.keys() `isinstance` list + assert not a_dict.values() `isinstance` list + assert not a_dict.items() `isinstance` list + assert len(a_dict.keys()) == len(a_dict.values()) == len(a_dict.items()) == 3 + assert {1: 1, 3: 2, 2: 3}.keys() |> tuple == (1, 3, 2) + assert {**{1: 0, 3: 0}, 2: 0}.keys() |> tuple == (1, 3, 2) == {**dict([(1, 1), (3, 2), (2, 3)])}.keys() |> tuple + assert a_dict == {1: 1, 2: 3, 3: 2} + assert {1: 1} |> str == "{1: 1}" == {1: 1} |> repr + assert py_dict `issubclass` dict + assert py_dict() `isinstance` dict + assert {5:0, 3:0, **{2:0, 6:0}, 8:0}.keys() |> tuple == (5, 3, 2, 6, 8) + a_multiset = m{1,1,2} + assert not a_multiset.keys() `isinstance` list + assert not a_multiset.values() `isinstance` list + assert not a_multiset.items() `isinstance` list + assert len(a_multiset.keys()) == len(a_multiset.values()) == len(a_multiset.items()) == 2 + assert (in)(1, [1, 2]) + assert not (1 not in .)([1, 2]) + assert not (in)([[]], []) + assert ("{a}" . .)("format")(a=1) == "1" + a_dict = {"a": 1, "b": 2} + a_dict |= {"a": 10, "c": 20} + assert a_dict == {"a": 10, "b": 2, "c": 20} == {"a": 1, "b": 2} | {"a": 10, "c": 20} + assert ["abc" ; "def"] == ['abc', 'def'] + assert ["abc" ;; "def"] == [['abc'], ['def']] + assert {"a":0, "b":1}$[0] == "a" + assert (|0, NotImplemented, 2|)$[1] is NotImplemented + assert m{1, 1, 2} |> fmap$(.+1) == m{2, 2, 3} + assert (+) ..> ((*) ..> (/)) == (+) ..> (*) ..> (/) == ((+) ..> (*)) ..> (/) # type: ignore + def f(x, y=1) = x, y # type: ignore + f.is_f = True # type: ignore + assert (f ..*> (+)).is_f # type: ignore + really_long_var = 10 + assert (...=really_long_var) == (10,) + assert (...=really_long_var, abc="abc") == (10, "abc") + assert (abc="abc", ...=really_long_var) == ("abc", 10) + assert (...=really_long_var).really_long_var == 10 # type: ignore + n = [0] + assert n[0] == 0 + assert_raises(-> m{{1:2,2:3}}, TypeError) + assert_raises((def -> from typing import blah), ImportError) # NOQA + assert type(m{1, 2}) is multiset + assert multiset(collections.Counter({1: 1, 2: 1})) `typed_eq` m{1, 2} + assert +m{-1, 1} `typed_eq` m{-1, 1} + assert -m{-1, 1} `typed_eq` m{} + assert m{1, 1, 2} + m{1, 3} `typed_eq` m{1, 1, 1, 2, 3} + assert m{1, 1, 2} | m{1, 3} `typed_eq` m{1, 1, 2, 3} + assert m{1, 1, 2} & m{1, 3} `typed_eq` m{1} + assert m{1, 1, 2} - m{1, 3} `typed_eq` m{1, 2} + assert (.+1) `and_then` (.*2) `and_then_await` (./3) |> repr == "$(?, 1) `and_then` $(?, 2) `and_then_await` $(?, 3)" + assert 5.5⏨3 == 5.5 * 10**3 + assert (x => x)(5) == 5 == (def x => x)(5) + assert (=> _)(5) == 5 == (def => _)(5) # type: ignore + assert ((x, y) => (x, y))(1, 2) == (1, 2) == (def (x, y) => (x, y))(1, 2) + assert (def (int(x)) => x)(5) == 5 == (def (int -> x) => x)("5") + assert (def (x: int) -> int => x)(5) == 5 == (def (int(x)) -> int => x)(5) + assert (x ⇒ x)(5) == 5 == (def x ⇒ x)(5) + assert f"a: { "abc" }" == "a: abc" == f'a: { 'abc' }' + assert f"1 + {"a" + "b"} + 2 + {"c" + "d"}" == "1 + ab + 2 + cd" == f'1 + {'a' + 'b'} + 2 + {'c' + 'd'}' + assert f"{"a" + "b"} + c + {"d" + "e"}" == "ab + c + de" == f'{'a' + 'b'} + c + {'d' + 'e'}' + assert f"""{""" +"""}""" == """ +""" == f"""{''' +'''}""" + assert f"""{( + )}""" == "()" == f'''{( + )}''' + assert f"{'\n'.join(["", ""])}" == "\n" + assert f"""{f'''{f'{f"{1+1}"}'}'''}""" == "2" == f"{f"{f"{f"{f"{f"{1+1}"}"}"}"}"}" + assert f"___{ + 1 +}___" == '___1___' == f"___{( + 1 +)}___" + x = 10 + assert x == 5 where: + x = 5 + assert x == 10, x + def nested() = f where: + f = def -> g where: + def g() = x where: + x = 5 + assert nested()()() == 5 + class HasPartial: + def f(self, x) = (self, x) + g = f$(?, 1) + has_partial = HasPartial() + assert has_partial.g() == (has_partial, 1) + xs = zip([1, 2], [3, 4]) + py_xs = py_zip([1, 2], [3, 4]) + assert list(xs) == [(1, 3), (2, 4)] == list(xs) + assert list(py_xs) == [(1, 3), (2, 4)] + assert list(py_xs) == [] if sys.version_info >= (3,) else [(1, 3), (2, 4)] + xs = map((+), [1, 2], [3, 4]) + py_xs = py_map((+), [1, 2], [3, 4]) + assert list(xs) == [4, 6] == list(xs) + assert list(py_xs) == [4, 6] + assert list(py_xs) == [] if sys.version_info >= (3,) else [4, 6] + for xs in [ + zip((x for x in range(5)), (x for x in range(10))), + map((,), (x for x in range(5)), (x for x in range(10))), + ]: # type: ignore + assert list(xs) == list(zip(range(5), range(5))) + assert list(xs) == [] + xs = map((.+1), range(5)) + assert list(xs) == list(range(1, 6)) == list(xs) + assert count()[:10:2] == range(0, 10, 2) + assert count()[10:2] == range(10, 2) + some_data = [ + (name="a", val="123"), + (name="b", val="567"), + ] + for mapreducer in ( + mapreduce.using_processes$(lift(,)(.name, .val)), # type: ignore + mapreduce.using_threads$(lift(,)(.name, .val)), # type: ignore + collectby.using_processes$(.name, value_func=.val), # type: ignore + collectby.using_threads$(.name, value_func=.val), # type: ignore + ): + assert some_data |> mapreducer == {"a": ["123"], "b": ["567"]} + assert_raises(-> collectby(.[0], [(0, 1), (0, 2)], reduce_func=False), ValueError) # type: ignore + assert ident$(x=?).__name__ == "ident" == ident$(1).__name__ # type: ignore + assert collectby(.[0], [(0, 1), (0, 2)], value_func=.[1], reduce_func=(+), reduce_func_init=1) == {0: 4} + assert ident$(1, ?) |> type == ident$(1) |> type + assert 10! == 3628800 + assert 0x100 == 256 == 0o400 + assert 0x0 == 0 == 0b0 + x = 10 + assert 0x == 0 == 0 x + assert 0xff == 255 == 0x100-1 + assert 11259375 == 0xabcdef + + with process_map.multiple_sequential_calls(): # type: ignore + assert map((+), range(3), range(4)$[:-1], strict=True) |> list == [0, 2, 4] == process_map((+), range(3), range(4)$[:-1], strict=True) |> list # type: ignore + assert range(3) |> map$((.+1), strict=True) |> list == [1, 2, 3] == range(3) |> process_map$((.+1), strict=True) |> list # type: ignore + my_match_err = MatchError("my match error", 123) + assert process_map(ident, [my_match_err]) |> list |> str == str([my_match_err]) # type: ignore + # repeat the same thing again now that my_match_err.str has been called + assert process_map(ident, [my_match_err]) |> list |> str == str([my_match_err]) # type: ignore + + return True diff --git a/coconut/tests/src/cocotest/agnostic/specific.coco b/coconut/tests/src/cocotest/agnostic/specific.coco index 1a3b8ba6f..f72873c04 100644 --- a/coconut/tests/src/cocotest/agnostic/specific.coco +++ b/coconut/tests/src/cocotest/agnostic/specific.coco @@ -1,8 +1,9 @@ +import sys from io import StringIO if TYPE_CHECKING: from typing import Any -from .util import mod # NOQA +from .util import mod, assert_raises # NOQA def non_py26_test() -> bool: @@ -41,6 +42,15 @@ def py3_spec_test() -> bool: assert Outer.Inner.f(2) == 2 assert Outer.Inner.f.__name__ == "f" assert Outer.Inner.f.__qualname__.endswith("Outer.Inner.f"), Outer.Inner.f.__qualname__ + for xs in [ + py_zip((x for x in range(5)), (x for x in range(10))), + py_map((,), (x for x in range(5)), (x for x in range(10))), + ]: # type: ignore + assert list(xs) == list(zip(range(5), range(5))) + assert list(xs) == [] if sys.version_info >= (3,) else list(zip(range(5), range(5))) + py_xs = py_map((.+1), range(5)) + assert list(py_xs) == list(range(1, 6)) + assert list(py_xs) == [] return True @@ -164,7 +174,7 @@ def py36_spec_test(tco: bool) -> bool: def py37_spec_test() -> bool: """Tests for any py37+ version.""" - import asyncio, typing + import asyncio, typing, typing_extensions assert py_breakpoint # type: ignore ns: typing.Dict[str, typing.Any] = {} exec("""async def toa(it): @@ -180,7 +190,9 @@ def py37_spec_test() -> bool: assert l == list(range(10)) class HasVarGen[*Ts] # type: ignore assert HasVarGen `issubclass` object - assert typing.Protocol.__module__ == "typing_extensions" + assert typing.Protocol is typing_extensions.Protocol + assert_raises((def -> raise ExceptionGroup("derp", [Exception("herp")])), ExceptionGroup) + assert_raises((def -> raise BaseExceptionGroup("derp", [BaseException("herp")])), BaseExceptionGroup) return True diff --git a/coconut/tests/src/cocotest/agnostic/suite.coco b/coconut/tests/src/cocotest/agnostic/suite.coco index 7e0440630..813fe05b0 100644 --- a/coconut/tests/src/cocotest/agnostic/suite.coco +++ b/coconut/tests/src/cocotest/agnostic/suite.coco @@ -44,12 +44,12 @@ def suite_test() -> bool: def test_sqplus1_plus1sq(sqplus1, plus1sq, parallel=True): assert sqplus1(3) == 10 == (plus1..square)(3), sqplus1 if parallel: - assert parallel_map(sqplus1, range(3)) |> tuple == (1, 2, 5), sqplus1 # type: ignore + assert process_map(sqplus1, range(3)) |> tuple == (1, 2, 5), sqplus1 # type: ignore assert 3 `plus1sq` == 16, plus1sq assert 3 `sqplus1` == 10, sqplus1 test_sqplus1_plus1sq(sqplus1_1, plus1sq_1) test_sqplus1_plus1sq(sqplus1_2, plus1sq_2, parallel=False) - with parallel_map.multiple_sequential_calls(max_workers=2): # type: ignore + with process_map.multiple_sequential_calls(max_workers=2): # type: ignore test_sqplus1_plus1sq(sqplus1_3, plus1sq_3) test_sqplus1_plus1sq(sqplus1_4, plus1sq_4) test_sqplus1_plus1sq(sqplus1_5, plus1sq_5) @@ -67,7 +67,6 @@ def suite_test() -> bool: to_sort = rand_list(10) assert to_sort |> qsort |> tuple == to_sort |> sorted |> tuple, qsort # type: ignore to_sort = rand_list(10) - assert parallel_map(tuple <.. (|>)$(to_sort), qsorts) |> list == [to_sort |> sorted |> tuple] * len(qsorts) assert repeat(3)$[2] == 3 == repeat_(3)$[2] assert sum_(repeat(1)$[:5]) == 5 == sum_(repeat_(1)$[:5]) assert (sum_(takewhile((x)-> x<5, N())) @@ -279,7 +278,6 @@ def suite_test() -> bool: assert fibs()$[:10] |> list == [1,1,2,3,5,8,13,21,34,55] == fibs_()$[:10] |> list assert fibs() |> takewhile$((i) -> i < 4000000 ) |> filter$((i) -> i % 2 == 0 ) |> sum == 4613732 == fibs_() |> takewhile$((i) -> i < 4000000 ) |> filter$((i) -> i % 2 == 0 ) |> sum # type: ignore assert loop([1,2])$[:4] |> list == [1, 2] * 2 - assert parallel_map(list .. .$[:2] .. loop, ([1], [2]))$[:2] |> tuple == ([1, 1], [2, 2]) assert nest("a") |> .$[1] |> .$[1] |> .$[0] == "a" assert (def -> mod)()(5, 3) == 2 assert sieve((2, 3, 4, 5)) |> list == [2, 3, 5] @@ -732,7 +730,7 @@ def suite_test() -> bool: only_match_if(1) -> _ = 1 match only_match_if(1) -> _ in 2: assert False - only_match_int -> _ = 1 + only_match_int -> _ = 10 match only_match_int -> _ in "abc": assert False only_match_abc -> _ = "abc" @@ -748,7 +746,6 @@ def suite_test() -> bool: class inh_A() `isinstance` clsA `isinstance` object = inh_A() for maxdiff in (maxdiff1, maxdiff2, maxdiff3, maxdiff_): assert maxdiff([7,1,4,5]) == 4, "failed for " + repr(maxdiff) - assert all(r == 4 for r in parallel_map(call$(?, [7,1,4,5]), [maxdiff1, maxdiff2, maxdiff3])) assert ret_ret_func(1) == ((), {"func": 1}) == ret_args_kwargs$(func=1)() # type: ignore assert ret_args_kwargs$(?, func=2)(1) == ((1,), {"func": 2}) assert lift(ret_args_kwargs)(ident, plus1, times2, sq=square)(3) == ((3, 4, 6), {"sq": 9}) @@ -1053,6 +1050,36 @@ forward 2""") == 900 assert ret_args_kwargs(123, ...=really_long_var, abc="abc") == ((123,), {"really_long_var": 10, "abc": "abc"}) == ret_args_kwargs$(123, ...=really_long_var, abc="abc")() assert "Coconut version of typing" in typing.__doc__ numlist: NumList = [1, 2.3, 5] + assert hasloc([[1, 2]]).loc[0][1] == 2 == hasloc([[1, 2]]) |> .loc[0][1] + locgetter = .loc[0][1] + assert hasloc([[1, 2]]) |> locgetter == 2 == (hasloc([[1, 2]]) |> .loc[0])[1] + haslocobj = hasloc([[1, 2]]) + haslocobj |>= .loc[0][1] + assert haslocobj == 2 + assert hasloc([[1, 2]]).iloc$[0]$[1] == 2 == hasloc([[1, 2]]) |> .iloc$[0]$[1] + locgetter = .iloc$[0]$[1] + assert hasloc([[1, 2]]) |> locgetter == 2 == (hasloc([[1, 2]]) |> .iloc$[0])$[1] + haslocobj = hasloc([[1, 2]]) + haslocobj |>= .iloc$[0]$[1] + assert haslocobj == 2 + assert safe_raise_exc(IOError).error `isinstance` IOError + assert safe_raise_exc(IOError).handle(IOError, const 10).unwrap() == 10 == safe_raise_exc(IOError).expect_error(IOError).result_or(10) + assert pickle_round_trip(ident$(1))() == 1 == pickle_round_trip(ident$(x=?))(1) + assert x_or_y(x=1) == (1, 1) == x_or_y(y=1) + assert DerivedWithMeths().cls_meth() + assert DerivedWithMeths().static_meth() + assert Fibs()[100] == 354224848179261915075 + assert tree_depth(Node(Leaf 5, Node(Node(Leaf 10)))) == 3 + assert pickle_round_trip(.name) <| (name=10) == 10 + assert pickle_round_trip(.[0])([10]) == 10 + assert pickle_round_trip(.loc[0]) <| (loc=[10]) == 10 + assert pickle_round_trip(.method(0)) <| (method=const 10) == 10 + assert pickle_round_trip(.method(x=10)) <| (method=x -> x) == 10 + + with process_map.multiple_sequential_calls(): # type: ignore + assert process_map(tuple <.. (|>)$(to_sort), qsorts) |> list == [to_sort |> sorted |> tuple] * len(qsorts) + assert process_map(list .. .$[:2] .. loop, ([1], [2]))$[:2] |> tuple == ([1, 1], [2, 2]) + assert all(r == 4 for r in process_map(call$(?, [7,1,4,5]), [maxdiff1, maxdiff2, maxdiff3])) # must come at end assert fibs_calls[0] == 1 diff --git a/coconut/tests/src/cocotest/agnostic/util.coco b/coconut/tests/src/cocotest/agnostic/util.coco index 427245454..c06598be7 100644 --- a/coconut/tests/src/cocotest/agnostic/util.coco +++ b/coconut/tests/src/cocotest/agnostic/util.coco @@ -1,6 +1,7 @@ # Imports: import sys import random +import pickle import operator # NOQA from contextlib import contextmanager from functools import wraps @@ -45,6 +46,12 @@ except NameError, TypeError: def x `typed_eq` y = (type(x), x) == (type(y), y) +def pickle_round_trip(obj) = ( + obj + |> pickle.dumps + |> pickle.loads +) + # Old functions: old_fmap = fmap$(starmap_over_mappings=True) @@ -127,7 +134,7 @@ product = reduce$((*), ?, 1) product_ = reduce$(*) def zipwith(f, *args) = map((items) -> f(*items), zip(*args)) def zipwith_(f, *args) = starmap$(f)..zip <*| args -zipsum = zip ..> map$(sum) +zipsum = zip ..> map$(sum) # type: ignore ident_ = (x) -> x @ ident .. ident # type: ignore def plus1_(x: int) -> int = x + 1 @@ -207,10 +214,12 @@ operator !! # bool operator lol lols = [-1] -match def lol = "lol" where: +match def lol = lols[0] += 1 -addpattern def (s) lol = s + "ol" where: # type: ignore + "lol" +addpattern def (s) lol = # type: ignore lols[0] += 1 + s + "ol" lol operator *** @@ -415,7 +424,7 @@ def partition(items, pivot, lprefix=[], rprefix=[]): return partition(tail, pivot, lprefix, [head]::rprefix) match []::_: return lprefix, rprefix -partition_ = recursive_iterator(partition) +partition_ = recursive_generator(partition) def myreduce(func, items): match [first]::tail1 in items: @@ -757,6 +766,14 @@ def depth_2(t): match tree(l=l, r=r) in t: return 1 + max([depth_2(l), depth_2(r)]) +class Tree +data Node(*children) from Tree +data Leaf(elem) from Tree + +def tree_depth(Leaf(_)) = 0 +addpattern def tree_depth(Node(*children)) = # type: ignore + children |> map$(tree_depth) |> max |> (.+1) + # Monads: def base_maybe(x, f) = f(x) if x is not None else None def maybes(*fs) = reduce(base_maybe, fs) @@ -872,6 +889,20 @@ class inh_inh_A(inh_A): @override def true(self) = False +class BaseWithMeths: + @classmethod + def cls_meth(cls) = False + @staticmethod + def static_meth() = False + +class DerivedWithMeths(BaseWithMeths): + @override + @classmethod + def cls_meth(cls) = True + @override + @staticmethod + def static_meth() = True + class MyExc(Exception): def __init__(self, m): super().__init__(m) @@ -922,7 +953,7 @@ def grid_map(func, gridsample): def parallel_grid_map(func, gridsample): """Map a function over every point in a grid in parallel.""" - return gridsample |> parallel_map$(parallel_map$(func)) + return gridsample |> process_map$(process_map$(func)) def grid_trim(gridsample, xmax, ymax): """Convert a grid to a list of lists up to xmax and ymax.""" @@ -953,7 +984,7 @@ def still_ident(x) = @prepattern(ident, allow_any_func=True) def not_ident(x) = "bar" -# Pattern-matching functions with guards +# Pattern-matching functions def pattern_abs(x if x < 0) = -x addpattern def pattern_abs(x) = x # type: ignore @@ -961,23 +992,25 @@ addpattern def pattern_abs(x) = x # type: ignore def `pattern_abs_` (x) if x < 0 = -x addpattern def `pattern_abs_` (x) = x # type: ignore +def x_or_y(x and y) = (x, y) + # Recursive iterator -@recursive_iterator +@recursive_generator def fibs() = fibs_calls[0] += 1 (1, 1) :: map((+), fibs(), fibs()$[1:]) fibs_calls = [0] -@recursive_iterator +@recursive_generator def fibs_() = map((+), (1, 1) :: fibs_(), (0, 0) :: fibs_()$[1:]) # use separate name for base func for pickle def _loop(it) = it :: loop(it) -loop = recursive_iterator(_loop) +loop = recursive_generator(_loop) -@recursive_iterator +@recursive_generator def nest(x) = (|x, nest(x)|) # Sieve Example @@ -1012,6 +1045,10 @@ def minus(a, b) = b - a def raise_exc(): raise Exception("raise_exc") +@safe_call$ +def safe_raise_exc(exc_cls = Exception): + raise exc_cls() + def does_raise_exc(func): try: return func() @@ -1057,6 +1094,25 @@ class unrepresentable: def __repr__(self): raise Fail("unrepresentable") +class hasloc: + def __init__(self, arr): + self.arr = arr + class Loc: + def __init__(inner, outer): + inner.outer = outer + def __getitem__(inner, item) = + inner.outer.arr[item] + @property + def loc(self) = self.Loc(self) + class ILoc: + def __init__(inner, outer): + inner.outer = outer + def __iter_getitem__(inner, item) = + inner.outer.arr$[item] + @property + def iloc(self) = self.ILoc(self) + + # Typing if TYPE_CHECKING or sys.version_info >= (3, 5): # test from typing import *, but that doesn't actually get us @@ -1273,7 +1329,7 @@ def fib(n if n < 2) = n @memoize() # type: ignore addpattern def fib(n) = fib(n-1) + fib(n-2) # type: ignore -@recursive_iterator +@recursive_generator def Fibs() = (0, 1) :: map((+), Fibs(), Fibs()$[1:]) fib_ = reiterable(Fibs())$[] @@ -1332,11 +1388,11 @@ class descriptor_test: lam = self -> self comp = tuplify .. ident - @recursive_iterator + @recursive_generator def N(self, i=0) = [(self, i)] :: self.N(i+1) - @recursive_iterator + @recursive_generator match def N_(self, *, i=0) = [(self, i)] :: self.N_(i=i+1) diff --git a/coconut/tests/src/cocotest/non_strict/non_strict_test.coco b/coconut/tests/src/cocotest/non_strict/non_strict_test.coco index 5550ee1f5..a21b8a155 100644 --- a/coconut/tests/src/cocotest/non_strict/non_strict_test.coco +++ b/coconut/tests/src/cocotest/non_strict/non_strict_test.coco @@ -89,6 +89,10 @@ def non_strict_test() -> bool: assert f"a" r"b" fr"c" rf"d" == "abcd" assert "a" fr"b" == "ab" == "a" rf"b" assert f"{f"{f"infinite"}"}" + " " + f"{f"nesting!!!"}" == "infinite nesting!!!" + assert range(100) |> parallel_map$(.**2) |> list |> .$[-1] == 9801 + @recursive_iterator + def fib() = (1, 1) :: map((+), fib(), fib()$[1:]) + assert fib()$[:5] |> list == [1, 1, 2, 3, 5] return True if __name__ == "__main__": diff --git a/coconut/tests/src/cocotest/target_311/py311_test.coco b/coconut/tests/src/cocotest/target_311/py311_test.coco new file mode 100644 index 000000000..a2c655815 --- /dev/null +++ b/coconut/tests/src/cocotest/target_311/py311_test.coco @@ -0,0 +1,10 @@ +def py311_test() -> bool: + """Performs Python-3.11-specific tests.""" + multi_err = ExceptionGroup("herp", [ValueError("a"), ValueError("b")]) + got_err = None + try: + raise multi_err + except* ValueError as err: + got_err = err + assert repr(got_err) == repr(multi_err), (got_err, multi_err) + return True diff --git a/coconut/tests/src/cocotest/target_36/py36_test.coco b/coconut/tests/src/cocotest/target_36/py36_test.coco index c7645db71..2d7afee34 100644 --- a/coconut/tests/src/cocotest/target_36/py36_test.coco +++ b/coconut/tests/src/cocotest/target_36/py36_test.coco @@ -14,6 +14,14 @@ def py36_test() -> bool: funcs.append(async copyclosure def -> x) return funcs async def await_all(xs) = [await x for x in xs] + async def aplus1(x) = x + 1 + async def async_mapreduce(func, iterable, **kwargs) = ( + iterable + |> async_map$(func) + |> await + |> mapreduce$(ident, **kwargs) + ) + async def atest(): assert ( outer_func() @@ -21,7 +29,51 @@ def py36_test() -> bool: |> map$(call) |> await_all |> await + ) == range(5) |> list == ( + outer_func() + |> await + |> async_map$(call) + |> await + ) + assert ( + range(5) + |> map$(./10) + |> reversed + |> async_map$(lift(asyncio.sleep)(ident, result=ident)) + |> await + |> reversed + |> map$(.*10) + |> list ) == range(5) |> list + assert ( + {"a": 0, "b": 1} + |> .items() + |> async_mapreduce$( + (async def ((k, v)) => + (key=k, value=await aplus1(v))), + collect_in={"c": 0}, + ) + |> await + ) == {"a": 1, "b": 2, "c": 0} + assert ( + [0, 2, 0] + |> async_mapreduce$( + (async def x => + (key=x, value=await aplus1(x))), + reduce_func=(+), + ) + |> await + ) == {0: 2, 2: 3} + assert ( + [0, 2, 0] + |> async_mapreduce$( + (async def x => + (key=x, value=await aplus1(x))), + reduce_func=(+), + reduce_func_init=10, + ) + |> await + ) == {0: 12, 2: 13} loop.run_until_complete(atest()) loop.close() diff --git a/coconut/tests/src/cocotest/target_sys/target_sys_test.coco b/coconut/tests/src/cocotest/target_sys/target_sys_test.coco index 012c4a6eb..c65bc4125 100644 --- a/coconut/tests/src/cocotest/target_sys/target_sys_test.coco +++ b/coconut/tests/src/cocotest/target_sys/target_sys_test.coco @@ -47,14 +47,14 @@ def asyncio_test() -> bool: def toa(f) = async def (*args, **kwargs) -> f(*args, **kwargs) async def async_map_0(args): - return parallel_map(args[0], *args[1:]) - async def async_map_1(args) = parallel_map(args[0], *args[1:]) - async def async_map_2([func] + iters) = parallel_map(func, *iters) - async match def async_map_3([func] + iters) = parallel_map(func, *iters) - match async def async_map_4([func] + iters) = parallel_map(func, *iters) + return thread_map(args[0], *args[1:]) + async def async_map_1(args) = thread_map(args[0], *args[1:]) + async def async_map_2([func] + iters) = thread_map(func, *iters) + async match def async_map_3([func] + iters) = thread_map(func, *iters) + match async def async_map_4([func] + iters) = thread_map(func, *iters) async def async_map_test() = - for async_map in (async_map_0, async_map_1, async_map_2, async_map_3, async_map_4): - assert (await ((pow$(2), range(5)) |> async_map)) |> tuple == (1, 2, 4, 8, 16) + for async_map_ in (async_map_0, async_map_1, async_map_2, async_map_3, async_map_4): + assert (await ((pow$(2), range(5)) |> async_map_)) |> tuple == (1, 2, 4, 8, 16) True async def aplus(x) = y -> x + y diff --git a/coconut/tests/src/extras.coco b/coconut/tests/src/extras.coco index 854903912..0d13f39d3 100644 --- a/coconut/tests/src/extras.coco +++ b/coconut/tests/src/extras.coco @@ -27,15 +27,6 @@ from coconut.convenience import ( warm_up, ) -if IPY: - if PY35: - import asyncio - from coconut.icoconut import CoconutKernel # type: ignore - from jupyter_client.session import Session -else: - CoconutKernel = None # type: ignore - Session = object # type: ignore - def assert_raises(c, Exc, not_Exc=None, err_has=None): """Test whether callable c raises an exception of type Exc.""" @@ -83,15 +74,6 @@ def unwrap_future(event_loop, maybe_future): return maybe_future -class FakeSession(Session): - if TYPE_CHECKING: - captured_messages: list[tuple] = [] - else: - captured_messages: list = [] - def send(self, stream, msg_or_type, content, *args, **kwargs): - self.captured_messages.append((msg_or_type, content)) - - def test_setup_none() -> bool: setup(line_numbers=False) @@ -217,10 +199,16 @@ cannot reassign type variable 'T' (use explicit '\T' syntax if intended) (line 1 ) assert_raises(-> parse("$"), CoconutParseError) - assert_raises(-> parse("@"), CoconutParseError, err_has=("\n ~^", "\n ^")) - assert_raises(-> parse("range(1,10) |> reduce$(*, initializer = 1000) |> print"), CoconutParseError, err_has=" \\~~~~~~~~~~~~~~~~~~~~~~~^") + assert_raises(-> parse("@"), CoconutParseError) + assert_raises(-> parse("range(1,10) |> reduce$(*, initializer = 1000) |> print"), CoconutParseError, err_has=( + " \\~~~~~~~~~~~~~~~~~~~~~~~^", + " \\~~~~~~~~~~~~^", + )) assert_raises(-> parse("a := b"), CoconutParseError, err_has=" \\~^") - assert_raises(-> parse("1 + return"), CoconutParseError, err_has=" \\~~~~^") + assert_raises(-> parse("1 + return"), CoconutParseError, err_has=( + " \\~~~^", + " \\~~~~^", + )) assert_raises(-> parse(""" def f() = assert 1 @@ -241,12 +229,13 @@ def f() = assert_raises(-> parse('"a" 10'), CoconutParseError, err_has=" \\~~~^") assert_raises(-> parse("A. ."), CoconutParseError, err_has=" \\~~^") assert_raises(-> parse('''f"""{ -}"""'''), CoconutSyntaxError, err_has=(" ~~~~|", "\n ^~~/")) +}"""'''), CoconutSyntaxError, err_has="parsing failed for format string expression") assert_raises(-> parse("f([] {})"), CoconutParseError, err_has=" \\~~~~^") assert_raises(-> parse("return = 1"), CoconutParseError, err_has='invalid use of the keyword "return"') assert_raises(-> parse("if a = b: pass"), CoconutParseError, err_has="misplaced assignment") assert_raises(-> parse("while a == b"), CoconutParseError, err_has="misplaced newline") + assert_raises(-> parse("0xfgf"), CoconutParseError, err_has=" \~~^") try: parse(""" @@ -277,6 +266,21 @@ def gam_eps_rate(bitarr) = ( else: assert False + try: + parse(""" +def f(x=1, y) = x, y + +class A + +def g(x) = x + """.strip()) + except CoconutSyntaxError as err: + err_str = str(err) + assert "non-default arguments must come first" in err_str, err_str + assert "class A" not in err_str, err_str + else: + assert False + assert parse("def f(x):\n ${var}", "xonsh") == "def f(x):\n ${var}\n" assert "data ABC" not in parse("data ABC:\n ${var}", "xonsh") @@ -325,7 +329,10 @@ line 6''') assert_raises(-> parse("a=1;"), CoconutStyleError, err_has="\n ^") assert_raises(-> parse("class derp(object)"), CoconutStyleError) assert_raises(-> parse("def f(a.b) = True"), CoconutStyleError, err_has="\n ^") - assert_raises(-> parse("match def kwd_only_x_is_int_def_0(*, x is int = 0) = x"), CoconutStyleError, err_has="\n ^~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~|") + assert_raises(-> parse("match def kwd_only_x_is_int_def_0(*, x is int = 0) = x"), CoconutStyleError, err_has=( + "\n ^~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~|", + "\n ^~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~/", + )) try: parse(""" try: @@ -344,6 +351,17 @@ else: assert_raises(-> parse("obj."), CoconutStyleError, err_has="getattr") assert_raises(-> parse("def x -> pass, 1"), CoconutStyleError, err_has="statement lambda") assert_raises(-> parse("abc = f'abc'"), CoconutStyleError, err_has="\n ^") + assert_raises(-> parse('f"{f"{f"infinite"}"}"'), CoconutStyleError, err_has="f-string with no expressions") + try: + parse(""" +import abc +1 +2 +3 + """.strip()) + except CoconutStyleError as err: + assert str(err) == """found unused import 'abc' (add '# NOQA' to suppress) (remove --strict to downgrade to a warning) (line 1) + import abc""" setup(line_numbers=False, strict=True, target="sys") assert_raises(-> parse("await f x"), CoconutParseError, err_has='invalid use of the keyword "await"') @@ -433,6 +451,20 @@ class F: def test_kernel() -> bool: + # hide imports so as to not enable incremental parsing until we want to + if PY35: + import asyncio + from coconut.icoconut import CoconutKernel # type: ignore + from jupyter_client.session import Session + + class FakeSession(Session): + if TYPE_CHECKING: + captured_messages: list[tuple] = [] + else: + captured_messages: list = [] + def send(self, stream, msg_or_type, content, *args, **kwargs): + self.captured_messages.append((msg_or_type, content)) + if PY35: loop = asyncio.new_event_loop() asyncio.set_event_loop(loop) @@ -441,11 +473,12 @@ def test_kernel() -> bool: k = CoconutKernel() fake_session = FakeSession() + assert k.shell is not None k.shell.displayhook.session = fake_session exec_result = k.do_execute("derp = pow$(?, 2)", False, True, {"two": "(+)(1, 1)"}, True) |> unwrap_future$(loop) - assert exec_result["status"] == "ok" - assert exec_result["user_expressions"]["two"]["data"]["text/plain"] == "2" + assert exec_result["status"] == "ok", exec_result + assert exec_result["user_expressions"]["two"]["data"]["text/plain"] == "2", exec_result assert k.do_execute("operator ++", False, True, {}, True) |> unwrap_future$(loop) |> .["status"] == "ok" assert k.do_execute("(++) = 1", False, True, {}, True) |> unwrap_future$(loop) |> .["status"] == "ok" @@ -590,6 +623,11 @@ def test_pandas() -> bool: ], dtype=object) # type: ignore d4 = d1 |> fmap$(def r -> r["nums2"] = r["nums"]*2; r) assert (d4["nums"] * 2 == d4["nums2"]).all() + df = pd.DataFrame({"123": [1, 2, 3]}) + mapreduce(ident, [("456", [4, 5, 6])], collect_in=df) + assert df["456"] |> list == [4, 5, 6] + mapreduce(ident, [("789", [7, 8, 9])], collect_in=df, reduce_func=False) + assert df["789"] |> list == [7, 8, 9] return True @@ -599,15 +637,16 @@ def test_extras() -> bool: print(".", end="") if not PYPY and PY36: assert test_pandas() is True # . - print(".", end="") - if CoconutKernel is not None: - assert test_kernel() is True # .. print(".") # newline bc we print stuff after this - assert test_setup_none() is True + assert test_setup_none() is True # .. print(".") # ditto - assert test_convenience() is True + assert test_convenience() is True # ... + # everything after here uses incremental parsing, so it must come last print(".", end="") - assert test_incremental() is True # must come last + assert test_incremental() is True # .... + if IPY: + print(".", end="") + assert test_kernel() is True # ..... return True diff --git a/coconut/util.py b/coconut/util.py index 69e0e2f3c..b0e04be68 100644 --- a/coconut/util.py +++ b/coconut/util.py @@ -30,6 +30,7 @@ from types import MethodType from contextlib import contextmanager from collections import defaultdict +from functools import partial if sys.version_info >= (3, 2): from functools import lru_cache @@ -39,6 +40,7 @@ except ImportError: lru_cache = None +from coconut.root import _get_target_info from coconut.constants import ( fixpath, default_encoding, @@ -90,6 +92,20 @@ def __reduce_ex__(self, _): return self.__reduce__() +class const(pickleable_obj): + """Implementaiton of Coconut's const for use within Coconut.""" + __slots__ = ("value",) + + def __init__(self, value): + self.value = value + + def __reduce__(self): + return (self.__class__, (self.value,)) + + def __call__(self, *args, **kwargs): + return self.value + + class override(pickleable_obj): """Implementation of Coconut's @override for use within Coconut.""" __slots__ = ("func",) @@ -105,6 +121,11 @@ def __init__(self, func): self.func = func def __get__(self, obj, objtype=None): + if hasattr(self.func, "__get__"): + if objtype is None: + return self.func.__get__(obj) + else: + return self.func.__get__(obj, objtype) if obj is None: return self.func if PY2: @@ -244,21 +265,73 @@ def __missing__(self, key): class dictset(dict, object): """A set implemented using a dictionary to get ordering benefits.""" + def __bool__(self): + return len(self) > 0 # fixes py2 issue + def add(self, item): self[item] = True -def assert_remove_prefix(inputstr, prefix): +def assert_remove_prefix(inputstr, prefix, allow_no_prefix=False): """Remove prefix asserting that inputstr starts with it.""" - assert inputstr.startswith(prefix), inputstr + if not allow_no_prefix: + assert inputstr.startswith(prefix), inputstr + elif not inputstr.startswith(prefix): + return inputstr return inputstr[len(prefix):] +remove_prefix = partial(assert_remove_prefix, allow_no_prefix=True) + + +def ensure_dir(dirpath, logger=None): + """Ensure that a directory exists.""" + if not os.path.exists(dirpath): + try: + os.makedirs(dirpath) + except OSError: + if logger is not None: + logger.log_exc() + return False + return True + + +def without_keys(inputdict, rem_keys): + """Get a copy of inputdict without rem_keys.""" + return {k: v for k, v in inputdict.items() if k not in rem_keys} + + +def split_leading_whitespace(inputstr): + """Split leading whitespace.""" + basestr = inputstr.lstrip() + whitespace = inputstr[:len(inputstr) - len(basestr)] + assert whitespace + basestr == inputstr, "invalid whitespace split: " + repr(inputstr) + return whitespace, basestr + + +def split_trailing_whitespace(inputstr): + """Split trailing whitespace.""" + basestr = inputstr.rstrip() + whitespace = inputstr[len(basestr):] + assert basestr + whitespace == inputstr, "invalid whitespace split: " + repr(inputstr) + return basestr, whitespace + + +def replace_all(inputstr, all_to_replace, replace_to): + """Replace everything in all_to_replace with replace_to in inputstr.""" + for to_replace in all_to_replace: + inputstr = inputstr.replace(to_replace, replace_to) + return inputstr + + # ----------------------------------------------------------------------------------------------------------------------- # VERSIONING: # ----------------------------------------------------------------------------------------------------------------------- +get_target_info = _get_target_info + + def ver_tuple_to_str(req_ver): """Converts a requirement version tuple into a version string.""" return ".".join(str(x) for x in req_ver) @@ -281,16 +354,6 @@ def get_next_version(req_ver, point_to_increment=-1): return req_ver[:point_to_increment] + (req_ver[point_to_increment] + 1,) -def get_target_info(target): - """Return target information as a version tuple.""" - if not target: - return () - elif len(target) == 1: - return (int(target),) - else: - return (int(target[0]), int(target[1:])) - - def get_displayable_target(target): """Get a displayable version of the target.""" try: @@ -327,8 +390,7 @@ def install_custom_kernel(executable=None, logger=None): kernel_dest = fixpath(os.path.join(sys.exec_prefix, icoconut_custom_kernel_install_loc)) try: make_custom_kernel(executable) - if not os.path.exists(kernel_dest): - os.makedirs(kernel_dest) + ensure_dir(kernel_dest) shutil.copy(kernel_source, kernel_dest) except OSError: existing_kernel = os.path.join(kernel_dest, "kernel.json")