diff --git a/.cirrus-DISABLED.yml b/.cirrus-DISABLED.yml deleted file mode 100644 index f20835cb6cac2a..00000000000000 --- a/.cirrus-DISABLED.yml +++ /dev/null @@ -1,29 +0,0 @@ -# gh-91960: Job disabled since Python is out of free credit (September 2023): -# https://discuss.python.org/t/freebsd-gets-a-new-cirrus-ci-github-action-job-and-a-new-buildbot/33122/26 - -freebsd_task: - freebsd_instance: - matrix: - - image: freebsd-13-2-release-amd64 - # Turn off TCP and UDP blackhole. It is not enabled by default in FreeBSD, - # but it is in the FreeBSD GCE images as used by Cirrus-CI. It causes even - # local local connections to fail with ETIMEDOUT instead of ECONNREFUSED. - # For more information see https://reviews.freebsd.org/D41751 and - # https://github.com/cirruslabs/cirrus-ci-docs/issues/483. - sysctl_script: - - sysctl net.inet.tcp.blackhole=0 - - sysctl net.inet.udp.blackhole=0 - configure_script: - - mkdir build - - cd build - - ../configure --with-pydebug - build_script: - - cd build - - make -j$(sysctl -n hw.ncpu) - pythoninfo_script: - - cd build - - make pythoninfo - test_script: - - cd build - # dtrace fails to build on FreeBSD - see gh-73263 - - make buildbottest TESTOPTS="-j0 -x test_dtrace --timeout=600" diff --git a/.github/CODEOWNERS b/.github/CODEOWNERS index db28c2a231ae04..7933d319550576 100644 --- a/.github/CODEOWNERS +++ b/.github/CODEOWNERS @@ -21,6 +21,7 @@ configure* @erlend-aasland @corona10 **/*context* @1st1 **/*genobject* @markshannon **/*hamt* @1st1 +**/*jit* @brandtbucher Objects/set* @rhettinger Objects/dict* @methane @markshannon Objects/typevarobject.c @JelleZijlstra @@ -37,11 +38,37 @@ Python/ast_opt.c @isidentical Python/bytecodes.c @markshannon @gvanrossum Python/optimizer*.c @markshannon @gvanrossum Lib/test/test_patma.py @brandtbucher -Lib/test/test_peepholer.py @brandtbucher Lib/test/test_type_*.py @JelleZijlstra Lib/test/test_capi/test_misc.py @markshannon @gvanrossum Tools/c-analyzer/ @ericsnowcurrently +# dbm +**/*dbm* @corona10 @erlend-aasland @serhiy-storchaka + +# runtime state/lifecycle +**/*pylifecycle* @ericsnowcurrently +**/*pystate* @ericsnowcurrently +**/*preconfig* @ericsnowcurrently +**/*initconfig* @ericsnowcurrently +**/*pathconfig* @ericsnowcurrently +**/*sysmodule* @ericsnowcurrently +**/*bltinmodule* @ericsnowcurrently +**/*gil* @ericsnowcurrently +Include/internal/pycore_runtime.h @ericsnowcurrently +Include/internal/pycore_interp.h @ericsnowcurrently +Include/internal/pycore_tstate.h @ericsnowcurrently +Include/internal/pycore_*_state.h @ericsnowcurrently +Include/internal/pycore_*_init.h @ericsnowcurrently +Include/internal/pycore_atexit.h @ericsnowcurrently +Include/internal/pycore_freelist.h @ericsnowcurrently +Include/internal/pycore_global_objects.h @ericsnowcurrently +Include/internal/pycore_obmalloc.h @ericsnowcurrently +Include/internal/pycore_pymem.h @ericsnowcurrently +Modules/main.c @ericsnowcurrently +Programs/_bootstrap_python.c @ericsnowcurrently +Programs/python.c @ericsnowcurrently +Tools/build/generate_global_objects.py @ericsnowcurrently + # Exceptions Lib/traceback.py @iritkatriel Lib/test/test_except*.py @iritkatriel @@ -50,13 +77,13 @@ Objects/exceptions.c @iritkatriel Python/traceback.c @iritkatriel # Hashing -**/*hashlib* @tiran -**/*pyhash* @tiran -**/*sha* @tiran -**/*md5* @tiran -**/*blake* @tiran -/Modules/_blake2/** @tiran -/Modules/_sha3/** @tiran +**/*hashlib* @gpshead @tiran +**/*pyhash* @gpshead @tiran +**/sha* @gpshead @tiran +Modules/md5* @gpshead @tiran +**/*blake* @gpshead @tiran +Modules/_blake2/** @gpshead @tiran +Modules/_hacl/** @gpshead # logging **/*logging* @vsajip @@ -76,7 +103,20 @@ Python/traceback.c @iritkatriel # Import (including importlib). **/*import* @brettcannon @ericsnowcurrently @ncoghlan @warsaw /Python/import.c @kumaraditya303 -**/*importlib/resources/* @jaraco @warsaw @FFY00 +Python/dynload_*.c @ericsnowcurrently +**/*freeze* @ericsnowcurrently +**/*frozen* @ericsnowcurrently +**/*modsupport* @ericsnowcurrently +**/*modulefinder* @ericsnowcurrently +**/*moduleobject* @ericsnowcurrently +**/*multiphase* @ericsnowcurrently +**/*pkgutil* @ericsnowcurrently +**/*pythonrun* @ericsnowcurrently +**/*runpy* @ericsnowcurrently +**/*singlephase* @ericsnowcurrently +Lib/test/test_module/ @ericsnowcurrently +Doc/c-api/module.rst @ericsnowcurrently +**/*importlib/resources/* @jaraco @warsaw @FFY00 **/importlib/metadata/* @jaraco @warsaw # Dates and times @@ -120,6 +160,9 @@ Lib/ast.py @isidentical /Lib/unittest/mock.py @cjw296 /Lib/test/test_unittest/testmock/* @cjw296 +# multiprocessing +**/*multiprocessing* @gpshead + # SQLite 3 **/*sqlite* @berkerpeksag @erlend-aasland @@ -157,6 +200,8 @@ Doc/c-api/stable.rst @encukou **/*dataclasses* @ericvsmith +**/*ensurepip* @pfmoore @pradyunsg + **/*idlelib* @terryjreedy **/*typing* @JelleZijlstra @AlexWaygood @@ -185,11 +230,13 @@ Doc/c-api/stable.rst @encukou **/*zipfile/_path/* @jaraco # Argument Clinic -/Tools/clinic/** @erlend-aasland @AlexWaygood -/Lib/test/test_clinic.py @erlend-aasland @AlexWaygood +/Tools/clinic/** @erlend-aasland +/Lib/test/test_clinic.py @erlend-aasland Doc/howto/clinic.rst @erlend-aasland # Subinterpreters +**/*interpreteridobject.* @ericsnowcurrently +**/*crossinterp* @ericsnowcurrently Lib/test/support/interpreters/ @ericsnowcurrently Modules/_xx*interp*module.c @ericsnowcurrently Lib/test/test_interpreters/ @ericsnowcurrently diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml index cfb36c8c32e18d..0a2f6da50ed8a0 100644 --- a/.github/workflows/build.yml +++ b/.github/workflows/build.yml @@ -128,11 +128,11 @@ jobs: if: needs.check_source.outputs.run_tests == 'true' steps: - uses: actions/checkout@v4 - - uses: actions/setup-python@v4 + - uses: actions/setup-python@v5 with: python-version: '3.x' - name: Restore config.cache - uses: actions/cache@v3 + uses: actions/cache@v4 with: path: config.cache key: ${{ github.job }}-${{ runner.os }}-${{ needs.check_source.outputs.config_hash }}-${{ env.pythonLocation }} @@ -142,6 +142,8 @@ jobs: run: echo "PATH=/usr/lib/ccache:$PATH" >> $GITHUB_ENV - name: Configure ccache action uses: hendrikmuhs/ccache-action@v1.2 + with: + save: false - name: Check Autoconf and aclocal versions run: | grep "Generated by GNU Autoconf 2.71" configure @@ -187,13 +189,13 @@ jobs: if: needs.check_source.outputs.run_tests == 'true' uses: ./.github/workflows/reusable-windows.yml - build_windows_free_threaded: - name: 'Windows (free-threaded)' + build_windows_free_threading: + name: 'Windows (free-threading)' needs: check_source if: needs.check_source.outputs.run_tests == 'true' uses: ./.github/workflows/reusable-windows.yml with: - free-threaded: true + free-threading: true build_macos: name: 'macOS' @@ -203,14 +205,14 @@ jobs: with: config_hash: ${{ needs.check_source.outputs.config_hash }} - build_macos_free_threaded: - name: 'macOS (free-threaded)' + build_macos_free_threading: + name: 'macOS (free-threading)' needs: check_source if: needs.check_source.outputs.run_tests == 'true' uses: ./.github/workflows/reusable-macos.yml with: config_hash: ${{ needs.check_source.outputs.config_hash }} - free-threaded: true + free-threading: true build_ubuntu: name: 'Ubuntu' @@ -225,8 +227,8 @@ jobs: --with-pydebug \ --with-openssl=$OPENSSL_DIR - build_ubuntu_free_threaded: - name: 'Ubuntu (free-threaded)' + build_ubuntu_free_threading: + name: 'Ubuntu (free-threading)' needs: check_source if: needs.check_source.outputs.run_tests == 'true' uses: ./.github/workflows/reusable-ubuntu.yml @@ -248,7 +250,7 @@ jobs: strategy: fail-fast: false matrix: - openssl_ver: [1.1.1w, 3.0.11, 3.1.3] + openssl_ver: [1.1.1w, 3.0.13, 3.1.5, 3.2.1] env: OPENSSL_VER: ${{ matrix.openssl_ver }} MULTISSL_DIR: ${{ github.workspace }}/multissl @@ -257,7 +259,7 @@ jobs: steps: - uses: actions/checkout@v4 - name: Restore config.cache - uses: actions/cache@v3 + uses: actions/cache@v4 with: path: config.cache key: ${{ github.job }}-${{ runner.os }}-${{ needs.check_source.outputs.config_hash }} @@ -272,7 +274,7 @@ jobs: echo "LD_LIBRARY_PATH=${GITHUB_WORKSPACE}/multissl/openssl/${OPENSSL_VER}/lib" >> $GITHUB_ENV - name: 'Restore OpenSSL build' id: cache-openssl - uses: actions/cache@v3 + uses: actions/cache@v4 with: path: ./multissl/openssl/${{ env.OPENSSL_VER }} key: ${{ runner.os }}-multissl-openssl-${{ env.OPENSSL_VER }} @@ -284,6 +286,8 @@ jobs: echo "PATH=/usr/lib/ccache:$PATH" >> $GITHUB_ENV - name: Configure ccache action uses: hendrikmuhs/ccache-action@v1.2 + with: + save: false - name: Configure CPython run: ./configure --config-cache --with-pydebug --with-openssl=$OPENSSL_DIR - name: Build CPython @@ -300,7 +304,7 @@ jobs: needs: check_source if: needs.check_source.outputs.run_tests == 'true' && needs.check_source.outputs.run_hypothesis == 'true' env: - OPENSSL_VER: 3.0.11 + OPENSSL_VER: 3.0.13 PYTHONSTRICTEXTENSIONBUILD: 1 steps: - uses: actions/checkout@v4 @@ -315,7 +319,7 @@ jobs: echo "LD_LIBRARY_PATH=${GITHUB_WORKSPACE}/multissl/openssl/${OPENSSL_VER}/lib" >> $GITHUB_ENV - name: 'Restore OpenSSL build' id: cache-openssl - uses: actions/cache@v3 + uses: actions/cache@v4 with: path: ./multissl/openssl/${{ env.OPENSSL_VER }} key: ${{ runner.os }}-multissl-openssl-${{ env.OPENSSL_VER }} @@ -327,6 +331,8 @@ jobs: echo "PATH=/usr/lib/ccache:$PATH" >> $GITHUB_ENV - name: Configure ccache action uses: hendrikmuhs/ccache-action@v1.2 + with: + save: false - name: Setup directory envs for out-of-tree builds run: | echo "CPYTHON_RO_SRCDIR=$(realpath -m ${GITHUB_WORKSPACE}/../cpython-ro-srcdir)" >> $GITHUB_ENV @@ -336,7 +342,7 @@ jobs: - name: Bind mount sources read-only run: sudo mount --bind -o ro $GITHUB_WORKSPACE $CPYTHON_RO_SRCDIR - name: Restore config.cache - uses: actions/cache@v3 + uses: actions/cache@v4 with: path: ${{ env.CPYTHON_BUILDDIR }}/config.cache key: ${{ github.job }}-${{ runner.os }}-${{ needs.check_source.outputs.config_hash }} @@ -369,7 +375,7 @@ jobs: ./python -m venv $VENV_LOC && $VENV_PYTHON -m pip install -r ${GITHUB_WORKSPACE}/Tools/requirements-hypothesis.txt - name: 'Restore Hypothesis database' id: cache-hypothesis-database - uses: actions/cache@v3 + uses: actions/cache@v4 with: path: ./hypothesis key: hypothesis-database-${{ github.head_ref || github.run_id }} @@ -395,7 +401,7 @@ jobs: -x test_subprocess \ -x test_signal \ -x test_sysconfig - - uses: actions/upload-artifact@v3 + - uses: actions/upload-artifact@v4 if: always() with: name: hypothesis-example-db @@ -409,13 +415,13 @@ jobs: needs: check_source if: needs.check_source.outputs.run_tests == 'true' env: - OPENSSL_VER: 3.0.11 + OPENSSL_VER: 3.0.13 PYTHONSTRICTEXTENSIONBUILD: 1 ASAN_OPTIONS: detect_leaks=0:allocator_may_return_null=1:handle_segv=0 steps: - uses: actions/checkout@v4 - name: Restore config.cache - uses: actions/cache@v3 + uses: actions/cache@v4 with: path: config.cache key: ${{ github.job }}-${{ runner.os }}-${{ needs.check_source.outputs.config_hash }} @@ -434,7 +440,7 @@ jobs: echo "LD_LIBRARY_PATH=${GITHUB_WORKSPACE}/multissl/openssl/${OPENSSL_VER}/lib" >> $GITHUB_ENV - name: 'Restore OpenSSL build' id: cache-openssl - uses: actions/cache@v3 + uses: actions/cache@v4 with: path: ./multissl/openssl/${{ env.OPENSSL_VER }} key: ${{ runner.os }}-multissl-openssl-${{ env.OPENSSL_VER }} @@ -446,6 +452,9 @@ jobs: echo "PATH=/usr/lib/ccache:$PATH" >> $GITHUB_ENV - name: Configure ccache action uses: hendrikmuhs/ccache-action@v1.2 + with: + save: ${{ github.event_name == 'push' }} + max-size: "200M" - name: Configure CPython run: ./configure --config-cache --with-address-sanitizer --without-pymalloc - name: Build CPython @@ -483,14 +492,14 @@ jobs: output-sarif: true sanitizer: ${{ matrix.sanitizer }} - name: Upload crash - uses: actions/upload-artifact@v3 + uses: actions/upload-artifact@v4 if: failure() && steps.build.outcome == 'success' with: name: ${{ matrix.sanitizer }}-artifacts path: ./out/artifacts - name: Upload SARIF if: always() && steps.build.outcome == 'success' - uses: github/codeql-action/upload-sarif@v2 + uses: github/codeql-action/upload-sarif@v3 with: sarif_file: cifuzz-sarif/results.sarif checkout_path: cifuzz-sarif @@ -504,12 +513,12 @@ jobs: - check-docs - check_generated_files - build_macos - - build_macos_free_threaded + - build_macos_free_threading - build_ubuntu - - build_ubuntu_free_threaded + - build_ubuntu_free_threading - build_ubuntu_ssltests - build_windows - - build_windows_free_threaded + - build_windows_free_threading - test_hypothesis - build_asan - cifuzz @@ -537,12 +546,12 @@ jobs: && ' check_generated_files, build_macos, - build_macos_free_threaded, + build_macos_free_threading, build_ubuntu, - build_ubuntu_free_threaded, + build_ubuntu_free_threading, build_ubuntu_ssltests, build_windows, - build_windows_free_threaded, + build_windows_free_threading, build_asan, ' || '' diff --git a/.github/workflows/build_msi.yml b/.github/workflows/build_msi.yml index 29282dffa37ec0..65d32c734e7745 100644 --- a/.github/workflows/build_msi.yml +++ b/.github/workflows/build_msi.yml @@ -32,6 +32,8 @@ jobs: strategy: matrix: type: [x86, x64, arm64] + env: + IncludeFreethreaded: true steps: - uses: actions/checkout@v4 - name: Build CPython installer diff --git a/.github/workflows/jit.yml b/.github/workflows/jit.yml new file mode 100644 index 00000000000000..69648d87947ad6 --- /dev/null +++ b/.github/workflows/jit.yml @@ -0,0 +1,127 @@ +name: JIT +on: + pull_request: + paths: + - '**jit**' + - 'Python/bytecodes.c' + push: + paths: + - '**jit**' + - 'Python/bytecodes.c' + workflow_dispatch: + +concurrency: + group: ${{ github.workflow }}-${{ github.head_ref || github.run_id }} + cancel-in-progress: true + +jobs: + jit: + name: ${{ matrix.target }} (${{ matrix.debug && 'Debug' || 'Release' }}) + runs-on: ${{ matrix.runner }} + timeout-minutes: 60 + strategy: + fail-fast: false + matrix: + target: + - i686-pc-windows-msvc/msvc + - x86_64-pc-windows-msvc/msvc + - x86_64-apple-darwin/clang + - aarch64-apple-darwin/clang + - x86_64-unknown-linux-gnu/gcc + - x86_64-unknown-linux-gnu/clang + - aarch64-unknown-linux-gnu/gcc + - aarch64-unknown-linux-gnu/clang + debug: + - true + - false + llvm: + - 16 + include: + - target: i686-pc-windows-msvc/msvc + architecture: Win32 + runner: windows-latest + compiler: msvc + - target: x86_64-pc-windows-msvc/msvc + architecture: x64 + runner: windows-latest + compiler: msvc + - target: x86_64-apple-darwin/clang + architecture: x86_64 + runner: macos-13 + compiler: clang + - target: aarch64-apple-darwin/clang + architecture: aarch64 + runner: macos-14 + compiler: clang + - target: x86_64-unknown-linux-gnu/gcc + architecture: x86_64 + runner: ubuntu-latest + compiler: gcc + - target: x86_64-unknown-linux-gnu/clang + architecture: x86_64 + runner: ubuntu-latest + compiler: clang + - target: aarch64-unknown-linux-gnu/gcc + architecture: aarch64 + runner: ubuntu-latest + compiler: gcc + # These fail because of emulation, not because of the JIT: + exclude: test_unix_events test_init test_process_pool test_shutdown test_multiprocessing_fork test_cmd_line test_faulthandler test_os test_perf_profiler test_posix test_signal test_socket test_subprocess test_threading test_venv + - target: aarch64-unknown-linux-gnu/clang + architecture: aarch64 + runner: ubuntu-latest + compiler: clang + # These fail because of emulation, not because of the JIT: + exclude: test_unix_events test_init test_process_pool test_shutdown test_multiprocessing_fork test_cmd_line test_faulthandler test_os test_perf_profiler test_posix test_signal test_socket test_subprocess test_threading test_venv + env: + CC: ${{ matrix.compiler }} + steps: + - uses: actions/checkout@v4 + - uses: actions/setup-python@v5 + with: + python-version: '3.11' + + - name: Windows + if: runner.os == 'Windows' + run: | + choco install llvm --allow-downgrade --no-progress --version ${{ matrix.llvm }} + ./PCbuild/build.bat --experimental-jit ${{ matrix.debug && '-d' || '--pgo' }} -p ${{ matrix.architecture }} + ./PCbuild/rt.bat ${{ matrix.debug && '-d' }} -p ${{ matrix.architecture }} -q --exclude ${{ matrix.exclude }} --multiprocess 0 --timeout 3600 --verbose2 --verbose3 + + - name: macOS + if: runner.os == 'macOS' + run: | + brew install llvm@${{ matrix.llvm }} + export SDKROOT="$(xcrun --show-sdk-path)" + ./configure --enable-experimental-jit ${{ matrix.debug && '--with-pydebug' || '--enable-optimizations --with-lto' }} + make all --jobs 4 + ./python.exe -m test --exclude ${{ matrix.exclude }} --multiprocess 0 --timeout 3600 --verbose2 --verbose3 + + - name: Native Linux + if: runner.os == 'Linux' && matrix.architecture == 'x86_64' + run: | + sudo bash -c "$(wget -O - https://apt.llvm.org/llvm.sh)" ./llvm.sh ${{ matrix.llvm }} + export PATH="$(llvm-config-${{ matrix.llvm }} --bindir):$PATH" + ./configure --enable-experimental-jit ${{ matrix.debug && '--with-pydebug' || '--enable-optimizations --with-lto' }} + make all --jobs 4 + ./python -m test --exclude ${{ matrix.exclude }} --multiprocess 0 --timeout 3600 --verbose2 --verbose3 + + - name: Emulated Linux + if: runner.os == 'Linux' && matrix.architecture != 'x86_64' + run: | + sudo bash -c "$(wget -O - https://apt.llvm.org/llvm.sh)" ./llvm.sh ${{ matrix.llvm }} + export PATH="$(llvm-config-${{ matrix.llvm }} --bindir):$PATH" + ./configure --prefix="$(pwd)/../build" + make install --jobs 4 + make clean --jobs 4 + export HOST=${{ matrix.architecture }}-linux-gnu + sudo apt install --yes "gcc-$HOST" qemu-user + ${{ !matrix.debug && matrix.compiler == 'clang' && './configure --enable-optimizations' || '' }} + ${{ !matrix.debug && matrix.compiler == 'clang' && 'make profile-run-stamp --jobs 4' || '' }} + export CC="${{ matrix.compiler == 'clang' && 'clang --target=$HOST' || '$HOST-gcc' }}" + export CPP="$CC --preprocess" + export HOSTRUNNER=qemu-${{ matrix.architecture }} + export QEMU_LD_PREFIX="/usr/$HOST" + ./configure --enable-experimental-jit ${{ matrix.debug && '--with-pydebug' || '--enable-optimizations --with-lto' }} --build=x86_64-linux-gnu --host="$HOST" --with-build-python=../build/bin/python3 --with-pkg-config=no ac_cv_buggy_getaddrinfo=no ac_cv_file__dev_ptc=no ac_cv_file__dev_ptmx=yes + make all --jobs 4 + ./python -m test --exclude ${{ matrix.exclude }} --multiprocess 0 --timeout 3600 --verbose2 --verbose3 diff --git a/.github/workflows/lint.yml b/.github/workflows/lint.yml index 6c1c29a58cf4fc..4a70ec6205a05b 100644 --- a/.github/workflows/lint.yml +++ b/.github/workflows/lint.yml @@ -20,7 +20,7 @@ jobs: steps: - uses: actions/checkout@v4 - - uses: actions/setup-python@v4 + - uses: actions/setup-python@v5 with: python-version: "3.x" - uses: pre-commit/action@v3.0.0 diff --git a/.github/workflows/mypy.yml b/.github/workflows/mypy.yml index 792903a90a4880..b766785de405d2 100644 --- a/.github/workflows/mypy.yml +++ b/.github/workflows/mypy.yml @@ -12,6 +12,7 @@ on: - "Tools/build/generate_sbom.py" - "Tools/cases_generator/**" - "Tools/clinic/**" + - "Tools/jit/**" - "Tools/peg_generator/**" - "Tools/requirements-dev.txt" - "Tools/wasm/**" @@ -38,6 +39,7 @@ jobs: "Tools/build/", "Tools/cases_generator", "Tools/clinic", + "Tools/jit", "Tools/peg_generator", "Tools/wasm", ] @@ -46,7 +48,7 @@ jobs: timeout-minutes: 10 steps: - uses: actions/checkout@v4 - - uses: actions/setup-python@v4 + - uses: actions/setup-python@v5 with: python-version: "3.11" cache: pip diff --git a/.github/workflows/require-pr-label.yml b/.github/workflows/require-pr-label.yml index 080204bcfd3b94..ff5cbdf3eda749 100644 --- a/.github/workflows/require-pr-label.yml +++ b/.github/workflows/require-pr-label.yml @@ -11,6 +11,7 @@ permissions: jobs: label: name: DO-NOT-MERGE / unresolved review + if: github.repository_owner == 'python' runs-on: ubuntu-latest timeout-minutes: 10 diff --git a/.github/workflows/reusable-docs.yml b/.github/workflows/reusable-docs.yml index 1c4fa4239c1e34..cea8f93d67b29c 100644 --- a/.github/workflows/reusable-docs.yml +++ b/.github/workflows/reusable-docs.yml @@ -41,7 +41,7 @@ jobs: git fetch origin ${{ env.refspec_base }} --shallow-since="${DATE}" \ --no-tags --prune --no-recurse-submodules - name: 'Set up Python' - uses: actions/setup-python@v4 + uses: actions/setup-python@v5 with: python-version: '3' cache: 'pip' @@ -72,7 +72,7 @@ jobs: steps: - uses: actions/checkout@v4 - name: 'Set up Python' - uses: actions/setup-python@v4 + uses: actions/setup-python@v5 with: python-version: '3.11' # known to work with Sphinx 4.2 cache: 'pip' @@ -89,7 +89,7 @@ jobs: timeout-minutes: 60 steps: - uses: actions/checkout@v4 - - uses: actions/cache@v3 + - uses: actions/cache@v4 with: path: ~/.cache/pip key: ubuntu-doc-${{ hashFiles('Doc/requirements.txt') }} diff --git a/.github/workflows/reusable-macos.yml b/.github/workflows/reusable-macos.yml index 22f46d18e1b43a..cad619b78ce5f2 100644 --- a/.github/workflows/reusable-macos.yml +++ b/.github/workflows/reusable-macos.yml @@ -4,7 +4,7 @@ on: config_hash: required: true type: string - free-threaded: + free-threading: required: false type: boolean default: false @@ -12,20 +12,27 @@ on: jobs: build_macos: name: 'build and test' - runs-on: macos-latest timeout-minutes: 60 env: HOMEBREW_NO_ANALYTICS: 1 HOMEBREW_NO_AUTO_UPDATE: 1 HOMEBREW_NO_INSTALL_CLEANUP: 1 PYTHONSTRICTEXTENSIONBUILD: 1 + strategy: + fail-fast: false + matrix: + os: [ + "macos-14", # M1 + "macos-13", # Intel + ] + runs-on: ${{ matrix.os }} steps: - uses: actions/checkout@v4 - name: Restore config.cache - uses: actions/cache@v3 + uses: actions/cache@v4 with: path: config.cache - key: ${{ github.job }}-${{ runner.os }}-${{ inputs.config_hash }} + key: ${{ github.job }}-${{ matrix.os }}-${{ inputs.config_hash }} - name: Install Homebrew dependencies run: brew install pkg-config openssl@3.0 xz gdbm tcl-tk - name: Configure CPython @@ -35,7 +42,7 @@ jobs: ./configure \ --config-cache \ --with-pydebug \ - ${{ inputs.free-threaded && '--disable-gil' || '' }} \ + ${{ inputs.free-threading && '--disable-gil' || '' }} \ --prefix=/opt/python-dev \ --with-openssl="$(brew --prefix openssl@3.0)" - name: Build CPython diff --git a/.github/workflows/reusable-ubuntu.yml b/.github/workflows/reusable-ubuntu.yml index 819b45bda7f980..0cbad57f0c6572 100644 --- a/.github/workflows/reusable-ubuntu.yml +++ b/.github/workflows/reusable-ubuntu.yml @@ -14,7 +14,7 @@ jobs: timeout-minutes: 60 runs-on: ubuntu-20.04 env: - OPENSSL_VER: 3.0.11 + OPENSSL_VER: 3.0.13 PYTHONSTRICTEXTENSIONBUILD: 1 steps: - uses: actions/checkout@v4 @@ -29,7 +29,7 @@ jobs: echo "LD_LIBRARY_PATH=${GITHUB_WORKSPACE}/multissl/openssl/${OPENSSL_VER}/lib" >> $GITHUB_ENV - name: 'Restore OpenSSL build' id: cache-openssl - uses: actions/cache@v3 + uses: actions/cache@v4 with: path: ./multissl/openssl/${{ env.OPENSSL_VER }} key: ${{ runner.os }}-multissl-openssl-${{ env.OPENSSL_VER }} @@ -41,6 +41,9 @@ jobs: echo "PATH=/usr/lib/ccache:$PATH" >> $GITHUB_ENV - name: Configure ccache action uses: hendrikmuhs/ccache-action@v1.2 + with: + save: ${{ github.event_name == 'push' }} + max-size: "200M" - name: Setup directory envs for out-of-tree builds run: | echo "CPYTHON_RO_SRCDIR=$(realpath -m ${GITHUB_WORKSPACE}/../cpython-ro-srcdir)" >> $GITHUB_ENV @@ -50,7 +53,7 @@ jobs: - name: Bind mount sources read-only run: sudo mount --bind -o ro $GITHUB_WORKSPACE $CPYTHON_RO_SRCDIR - name: Restore config.cache - uses: actions/cache@v3 + uses: actions/cache@v4 with: path: ${{ env.CPYTHON_BUILDDIR }}/config.cache key: ${{ github.job }}-${{ runner.os }}-${{ inputs.config_hash }} diff --git a/.github/workflows/reusable-windows.yml b/.github/workflows/reusable-windows.yml index 47a3c10d2ca4c1..c0209e0e1c92e9 100644 --- a/.github/workflows/reusable-windows.yml +++ b/.github/workflows/reusable-windows.yml @@ -1,7 +1,7 @@ on: workflow_call: inputs: - free-threaded: + free-threading: required: false type: boolean default: false @@ -16,11 +16,11 @@ jobs: steps: - uses: actions/checkout@v4 - name: Build CPython - run: .\PCbuild\build.bat -e -d -v -p Win32 ${{ inputs.free-threaded && '--disable-gil' || '' }} + run: .\PCbuild\build.bat -e -d -v -p Win32 ${{ inputs.free-threading && '--disable-gil' || '' }} - name: Display build info run: .\python.bat -m test.pythoninfo - name: Tests - run: .\PCbuild\rt.bat -p Win32 -d -q --fast-ci + run: .\PCbuild\rt.bat -p Win32 -d -q --fast-ci ${{ inputs.free-threading && '--disable-gil' || '' }} build_win_amd64: name: 'build and test (x64)' @@ -33,11 +33,11 @@ jobs: - name: Register MSVC problem matcher run: echo "::add-matcher::.github/problem-matchers/msvc.json" - name: Build CPython - run: .\PCbuild\build.bat -e -d -v -p x64 ${{ inputs.free-threaded && '--disable-gil' || '' }} + run: .\PCbuild\build.bat -e -d -v -p x64 ${{ inputs.free-threading && '--disable-gil' || '' }} - name: Display build info run: .\python.bat -m test.pythoninfo - name: Tests - run: .\PCbuild\rt.bat -p x64 -d -q --fast-ci + run: .\PCbuild\rt.bat -p x64 -d -q --fast-ci ${{ inputs.free-threading && '--disable-gil' || '' }} build_win_arm64: name: 'build (arm64)' @@ -50,4 +50,4 @@ jobs: - name: Register MSVC problem matcher run: echo "::add-matcher::.github/problem-matchers/msvc.json" - name: Build CPython - run: .\PCbuild\build.bat -e -d -v -p arm64 ${{ inputs.free-threaded && '--disable-gil' || '' }} + run: .\PCbuild\build.bat -e -d -v -p arm64 ${{ inputs.free-threading && '--disable-gil' || '' }} diff --git a/.github/workflows/stale.yml b/.github/workflows/stale.yml index 94676f5ee5fffc..07608fe91b4dbe 100644 --- a/.github/workflows/stale.yml +++ b/.github/workflows/stale.yml @@ -16,7 +16,7 @@ jobs: steps: - name: "Check PRs" - uses: actions/stale@v8 + uses: actions/stale@v9 with: repo-token: ${{ secrets.GITHUB_TOKEN }} stale-pr-message: 'This PR is stale because it has been open for 30 days with no activity.' diff --git a/.github/workflows/verify-ensurepip-wheels.yml b/.github/workflows/verify-ensurepip-wheels.yml index 4a545037bf6e2b..83b007f1c9c2ef 100644 --- a/.github/workflows/verify-ensurepip-wheels.yml +++ b/.github/workflows/verify-ensurepip-wheels.yml @@ -26,7 +26,7 @@ jobs: timeout-minutes: 10 steps: - uses: actions/checkout@v4 - - uses: actions/setup-python@v4 + - uses: actions/setup-python@v5 with: python-version: '3' - name: Compare checksum of bundled wheels to the ones published on PyPI diff --git a/.gitignore b/.gitignore index c424a894c2a6e0..6ed7197e3ab626 100644 --- a/.gitignore +++ b/.gitignore @@ -126,6 +126,7 @@ Tools/unicode/data/ # hendrikmuhs/ccache-action@v1 /.ccache /cross-build/ +/jit_stencils.h /platform /profile-clean-stamp /profile-run-stamp @@ -158,5 +159,5 @@ Python/frozen_modules/MANIFEST /python !/Python/ -# main branch only: ABI files are not checked/maintained +# main branch only: ABI files are not checked/maintained. Doc/data/python*.abi diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index 9bd9c59a1ddc74..69d85238985150 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -1,6 +1,6 @@ repos: - repo: https://github.com/astral-sh/ruff-pre-commit - rev: v0.1.7 + rev: v0.2.0 hooks: - id: ruff name: Run Ruff on Lib/test/ @@ -28,7 +28,7 @@ repos: hooks: - id: sphinx-lint args: [--enable=default-role] - files: ^Doc/|^Misc/NEWS.d/next/ + files: ^Doc/|^Misc/NEWS.d/ - repo: meta hooks: diff --git a/Doc/Makefile b/Doc/Makefile index 7af56e965e1be4..38fd60f2ae01d1 100644 --- a/Doc/Makefile +++ b/Doc/Makefile @@ -19,8 +19,12 @@ SPHINXERRORHANDLING = -W PAPEROPT_a4 = -D latex_elements.papersize=a4paper PAPEROPT_letter = -D latex_elements.papersize=letterpaper -ALLSPHINXOPTS = -b $(BUILDER) -d build/doctrees $(PAPEROPT_$(PAPER)) -j $(JOBS) \ - $(SPHINXOPTS) $(SPHINXERRORHANDLING) . build/$(BUILDER) $(SOURCES) +ALLSPHINXOPTS = -b $(BUILDER) \ + -d build/doctrees \ + -j $(JOBS) \ + $(PAPEROPT_$(PAPER)) \ + $(SPHINXOPTS) $(SPHINXERRORHANDLING) \ + . build/$(BUILDER) $(SOURCES) .PHONY: help help: @@ -142,7 +146,7 @@ htmlview: html .PHONY: htmllive htmllive: SPHINXBUILD = $(VENVDIR)/bin/sphinx-autobuild -htmllive: SPHINXOPTS = --re-ignore="/venv/" +htmllive: SPHINXOPTS = --re-ignore="/venv/" --open-browser --delay 0 htmllive: html .PHONY: clean diff --git a/Doc/c-api/code.rst b/Doc/c-api/code.rst index 5082b0cb6ad3f3..11c12e685fcace 100644 --- a/Doc/c-api/code.rst +++ b/Doc/c-api/code.rst @@ -22,12 +22,13 @@ bound into a function. .. c:var:: PyTypeObject PyCode_Type This is an instance of :c:type:`PyTypeObject` representing the Python - :class:`code` type. + :ref:`code object `. .. c:function:: int PyCode_Check(PyObject *co) - Return true if *co* is a :class:`code` object. This function always succeeds. + Return true if *co* is a :ref:`code object `. + This function always succeeds. .. c:function:: int PyCode_GetNumFree(PyCodeObject *co) diff --git a/Doc/c-api/complex.rst b/Doc/c-api/complex.rst index e3fd001c599c80..5a0474869071d9 100644 --- a/Doc/c-api/complex.rst +++ b/Doc/c-api/complex.rst @@ -117,11 +117,29 @@ Complex Numbers as Python Objects Return the real part of *op* as a C :c:expr:`double`. + If *op* is not a Python complex number object but has a + :meth:`~object.__complex__` method, this method will first be called to + convert *op* to a Python complex number object. If :meth:`!__complex__` is + not defined then it falls back to call :c:func:`PyFloat_AsDouble` and + returns its result. Upon failure, this method returns ``-1.0``, so one + should call :c:func:`PyErr_Occurred` to check for errors. + + .. versionchanged:: 3.13 + Use :meth:`~object.__complex__` if available. .. c:function:: double PyComplex_ImagAsDouble(PyObject *op) Return the imaginary part of *op* as a C :c:expr:`double`. + If *op* is not a Python complex number object but has a + :meth:`~object.__complex__` method, this method will first be called to + convert *op* to a Python complex number object. If :meth:`!__complex__` is + not defined then it falls back to call :c:func:`PyFloat_AsDouble` and + returns ``0.0`` on success. Upon failure, this method returns ``-1.0``, so + one should call :c:func:`PyErr_Occurred` to check for errors. + + .. versionchanged:: 3.13 + Use :meth:`~object.__complex__` if available. .. c:function:: Py_complex PyComplex_AsCComplex(PyObject *op) diff --git a/Doc/c-api/conversion.rst b/Doc/c-api/conversion.rst index c5350123dfdfdc..4aaf3905e81c8a 100644 --- a/Doc/c-api/conversion.rst +++ b/Doc/c-api/conversion.rst @@ -48,6 +48,42 @@ The return value (*rv*) for these functions should be interpreted as follows: The following functions provide locale-independent string to number conversions. +.. c:function:: unsigned long PyOS_strtoul(const char *str, char **ptr, int base) + + Convert the initial part of the string in ``str`` to an :c:expr:`unsigned + long` value according to the given ``base``, which must be between ``2`` and + ``36`` inclusive, or be the special value ``0``. + + Leading white space and case of characters are ignored. If ``base`` is zero + it looks for a leading ``0b``, ``0o`` or ``0x`` to tell which base. If + these are absent it defaults to ``10``. Base must be 0 or between 2 and 36 + (inclusive). If ``ptr`` is non-``NULL`` it will contain a pointer to the + end of the scan. + + If the converted value falls out of range of corresponding return type, + range error occurs (:c:data:`errno` is set to :c:macro:`!ERANGE`) and + :c:macro:`!ULONG_MAX` is returned. If no conversion can be performed, ``0`` + is returned. + + See also the Unix man page :manpage:`strtoul(3)`. + + .. versionadded:: 3.2 + + +.. c:function:: long PyOS_strtol(const char *str, char **ptr, int base) + + Convert the initial part of the string in ``str`` to an :c:expr:`long` value + according to the given ``base``, which must be between ``2`` and ``36`` + inclusive, or be the special value ``0``. + + Same as :c:func:`PyOS_strtoul`, but return a :c:expr:`long` value instead + and :c:macro:`LONG_MAX` on overflows. + + See also the Unix man page :manpage:`strtol(3)`. + + .. versionadded:: 3.2 + + .. c:function:: double PyOS_string_to_double(const char *s, char **endptr, PyObject *overflow_exception) Convert a string ``s`` to a :c:expr:`double`, raising a Python diff --git a/Doc/c-api/dict.rst b/Doc/c-api/dict.rst index 8471c98d044872..03f3d28187bfe9 100644 --- a/Doc/c-api/dict.rst +++ b/Doc/c-api/dict.rst @@ -174,6 +174,26 @@ Dictionary Objects .. versionadded:: 3.4 +.. c:function:: int PyDict_SetDefaultRef(PyObject *p, PyObject *key, PyObject *default_value, PyObject **result) + + Inserts *default_value* into the dictionary *p* with a key of *key* if the + key is not already present in the dictionary. If *result* is not ``NULL``, + then *\*result* is set to a :term:`strong reference` to either + *default_value*, if the key was not present, or the existing value, if *key* + was already present in the dictionary. + Returns ``1`` if the key was present and *default_value* was not inserted, + or ``0`` if the key was not present and *default_value* was inserted. + On failure, returns ``-1``, sets an exception, and sets ``*result`` + to ``NULL``. + + For clarity: if you have a strong reference to *default_value* before + calling this function, then after it returns, you hold a strong reference + to both *default_value* and *\*result* (if it's not ``NULL``). + These may refer to the same object: in that case you hold two separate + references to it. + .. versionadded:: 3.13 + + .. c:function:: int PyDict_Pop(PyObject *p, PyObject *key, PyObject **result) Remove *key* from dictionary *p* and optionally return the removed value. diff --git a/Doc/c-api/exceptions.rst b/Doc/c-api/exceptions.rst index 284a9c71e420da..c7e3cd9463e5d7 100644 --- a/Doc/c-api/exceptions.rst +++ b/Doc/c-api/exceptions.rst @@ -440,7 +440,7 @@ Querying the error indicator .. c:function:: PyObject *PyErr_GetRaisedException(void) Return the exception currently being raised, clearing the error indicator at - the same time. + the same time. Return ``NULL`` if the error indicator is not set. This function is used by code that needs to catch exceptions, or code that needs to save and restore the error indicator temporarily. diff --git a/Doc/c-api/file.rst b/Doc/c-api/file.rst index b36c800e00444a..d3a78c588454e8 100644 --- a/Doc/c-api/file.rst +++ b/Doc/c-api/file.rst @@ -65,8 +65,14 @@ the :mod:`io` APIs instead. Overrides the normal behavior of :func:`io.open_code` to pass its parameter through the provided handler. - The handler is a function of type :c:expr:`PyObject *(\*)(PyObject *path, - void *userData)`, where *path* is guaranteed to be :c:type:`PyUnicodeObject`. + The *handler* is a function of type: + + .. c:namespace:: NULL + .. c:type:: PyObject * (*Py_OpenCodeHookFunction)(PyObject *, void *) + + Equivalent of :c:expr:`PyObject *(\*)(PyObject *path, + void *userData)`, where *path* is guaranteed to be + :c:type:`PyUnicodeObject`. The *userData* pointer is passed into the hook function. Since hook functions may be called from different runtimes, this pointer should not diff --git a/Doc/c-api/import.rst b/Doc/c-api/import.rst index 51c20b202f091c..7c74e9e88678dc 100644 --- a/Doc/c-api/import.rst +++ b/Doc/c-api/import.rst @@ -13,20 +13,8 @@ Importing Modules single: __all__ (package variable) single: modules (in module sys) - This is a simplified interface to :c:func:`PyImport_ImportModuleEx` below, - leaving the *globals* and *locals* arguments set to ``NULL`` and *level* set - to 0. When the *name* - argument contains a dot (when it specifies a submodule of a package), the - *fromlist* argument is set to the list ``['*']`` so that the return value is the - named module rather than the top-level package containing it as would otherwise - be the case. (Unfortunately, this has an additional side effect when *name* in - fact specifies a subpackage instead of a submodule: the submodules specified in - the package's ``__all__`` variable are loaded.) Return a new reference to the - imported module, or ``NULL`` with an exception set on failure. A failing - import of a module doesn't leave the module in :data:`sys.modules`. - - This function always uses absolute imports. - + This is a wrapper around :c:func:`PyImport_Import()` which takes a + :c:expr:`const char *` as an argument instead of a :c:expr:`PyObject *`. .. c:function:: PyObject* PyImport_ImportModuleNoBlock(const char *name) diff --git a/Doc/c-api/intro.rst b/Doc/c-api/intro.rst index 4dbca92b18b5cd..dcda1071a58f35 100644 --- a/Doc/c-api/intro.rst +++ b/Doc/c-api/intro.rst @@ -148,7 +148,7 @@ complete listing. worse performances (due to increased code size for example). The compiler is usually smarter than the developer for the cost/benefit analysis. - If Python is :ref:`built in debug mode ` (if the ``Py_DEBUG`` + If Python is :ref:`built in debug mode ` (if the :c:macro:`Py_DEBUG` macro is defined), the :c:macro:`Py_ALWAYS_INLINE` macro does nothing. It must be specified before the function return type. Usage:: @@ -812,12 +812,14 @@ available that support tracing of reference counts, debugging the memory allocator, or low-level profiling of the main interpreter loop. Only the most frequently used builds will be described in the remainder of this section. -Compiling the interpreter with the :c:macro:`Py_DEBUG` macro defined produces +.. c:macro:: Py_DEBUG + +Compiling the interpreter with the :c:macro:`!Py_DEBUG` macro defined produces what is generally meant by :ref:`a debug build of Python `. -:c:macro:`Py_DEBUG` is enabled in the Unix build by adding +:c:macro:`!Py_DEBUG` is enabled in the Unix build by adding :option:`--with-pydebug` to the :file:`./configure` command. It is also implied by the presence of the -not-Python-specific :c:macro:`_DEBUG` macro. When :c:macro:`Py_DEBUG` is enabled +not-Python-specific :c:macro:`!_DEBUG` macro. When :c:macro:`!Py_DEBUG` is enabled in the Unix build, compiler optimization is disabled. In addition to the reference count debugging described below, extra checks are @@ -832,4 +834,3 @@ after every statement run by the interpreter.) Please refer to :file:`Misc/SpecialBuilds.txt` in the Python source distribution for more detailed information. - diff --git a/Doc/c-api/list.rst b/Doc/c-api/list.rst index c8b64bad702f50..53eb54d3e1021a 100644 --- a/Doc/c-api/list.rst +++ b/Doc/c-api/list.rst @@ -56,13 +56,21 @@ List Objects Similar to :c:func:`PyList_Size`, but without error checking. -.. c:function:: PyObject* PyList_GetItem(PyObject *list, Py_ssize_t index) +.. c:function:: PyObject* PyList_GetItemRef(PyObject *list, Py_ssize_t index) Return the object at position *index* in the list pointed to by *list*. The position must be non-negative; indexing from the end of the list is not - supported. If *index* is out of bounds (<0 or >=len(list)), + supported. If *index* is out of bounds (:code:`<0 or >=len(list)`), return ``NULL`` and set an :exc:`IndexError` exception. + .. versionadded:: 3.13 + + +.. c:function:: PyObject* PyList_GetItem(PyObject *list, Py_ssize_t index) + + Like :c:func:`PyList_GetItemRef`, but returns a + :term:`borrowed reference` instead of a :term:`strong reference`. + .. c:function:: PyObject* PyList_GET_ITEM(PyObject *list, Py_ssize_t i) diff --git a/Doc/c-api/memory.rst b/Doc/c-api/memory.rst index 1f392e55078e77..c05282ffc59521 100644 --- a/Doc/c-api/memory.rst +++ b/Doc/c-api/memory.rst @@ -267,14 +267,14 @@ The following type-oriented macros are provided for convenience. Note that .. c:macro:: PyMem_New(TYPE, n) Same as :c:func:`PyMem_Malloc`, but allocates ``(n * sizeof(TYPE))`` bytes of - memory. Returns a pointer cast to :c:expr:`TYPE*`. The memory will not have + memory. Returns a pointer cast to ``TYPE*``. The memory will not have been initialized in any way. .. c:macro:: PyMem_Resize(p, TYPE, n) Same as :c:func:`PyMem_Realloc`, but the memory block is resized to ``(n * - sizeof(TYPE))`` bytes. Returns a pointer cast to :c:expr:`TYPE*`. On return, + sizeof(TYPE))`` bytes. Returns a pointer cast to ``TYPE*``. On return, *p* will be a pointer to the new memory area, or ``NULL`` in the event of failure. diff --git a/Doc/c-api/memoryview.rst b/Doc/c-api/memoryview.rst index 2aa43318e7a455..f6038032805259 100644 --- a/Doc/c-api/memoryview.rst +++ b/Doc/c-api/memoryview.rst @@ -20,6 +20,17 @@ any other object. read/write, otherwise it may be either read-only or read/write at the discretion of the exporter. + +.. c:macro:: PyBUF_READ + + Flag to request a readonly buffer. + + +.. c:macro:: PyBUF_WRITE + + Flag to request a writable buffer. + + .. c:function:: PyObject *PyMemoryView_FromMemory(char *mem, Py_ssize_t size, int flags) Create a memoryview object using *mem* as the underlying buffer. @@ -41,6 +52,8 @@ any other object. original memory. Otherwise, a copy is made and the memoryview points to a new bytes object. + *buffertype* can be one of :c:macro:`PyBUF_READ` or :c:macro:`PyBUF_WRITE`. + .. c:function:: int PyMemoryView_Check(PyObject *obj) diff --git a/Doc/c-api/object.rst b/Doc/c-api/object.rst index a4e3e74861a315..12476412799a4f 100644 --- a/Doc/c-api/object.rst +++ b/Doc/c-api/object.rst @@ -19,6 +19,14 @@ Object Protocol to NotImplemented and return it). +.. c:macro:: Py_PRINT_RAW + + Flag to be used with multiple functions that print the object (like + :c:func:`PyObject_Print` and :c:func:`PyFile_WriteObject`). + If passed, these function would use the :func:`str` of the object + instead of the :func:`repr`. + + .. c:function:: int PyObject_Print(PyObject *o, FILE *fp, int flags) Print an object *o*, on file *fp*. Returns ``-1`` on error. The flags argument @@ -47,9 +55,8 @@ Object Protocol .. c:function:: int PyObject_HasAttr(PyObject *o, PyObject *attr_name) - Returns ``1`` if *o* has the attribute *attr_name*, and ``0`` otherwise. This - is equivalent to the Python expression ``hasattr(o, attr_name)``. This function - always succeeds. + Returns ``1`` if *o* has the attribute *attr_name*, and ``0`` otherwise. + This function always succeeds. .. note:: @@ -222,12 +229,8 @@ Object Protocol .. c:function:: int PyObject_RichCompareBool(PyObject *o1, PyObject *o2, int opid) Compare the values of *o1* and *o2* using the operation specified by *opid*, - which must be one of :c:macro:`Py_LT`, :c:macro:`Py_LE`, :c:macro:`Py_EQ`, - :c:macro:`Py_NE`, :c:macro:`Py_GT`, or :c:macro:`Py_GE`, corresponding to ``<``, - ``<=``, ``==``, ``!=``, ``>``, or ``>=`` respectively. Returns ``-1`` on error, - ``0`` if the result is false, ``1`` otherwise. This is the equivalent of the - Python expression ``o1 op o2``, where ``op`` is the operator corresponding to - *opid*. + like :c:func:`PyObject_RichCompare`, but returns ``-1`` on error, ``0`` if + the result is false, ``1`` otherwise. .. note:: If *o1* and *o2* are the same object, :c:func:`PyObject_RichCompareBool` diff --git a/Doc/c-api/stable.rst b/Doc/c-api/stable.rst index 63a100a6f26f24..5b9e43874c7f2b 100644 --- a/Doc/c-api/stable.rst +++ b/Doc/c-api/stable.rst @@ -16,7 +16,7 @@ CPython's Application Binary Interface (ABI) is forward- and backwards-compatible across a minor release (if these are compiled the same way; see :ref:`stable-abi-platform` below). So, code compiled for Python 3.10.0 will work on 3.10.8 and vice versa, -but will need to be compiled separately for 3.9.x and 3.10.x. +but will need to be compiled separately for 3.9.x and 3.11.x. There are two tiers of C API with different stability expectations: diff --git a/Doc/c-api/structures.rst b/Doc/c-api/structures.rst index 7d82f7839dfcd7..0032da9659636c 100644 --- a/Doc/c-api/structures.rst +++ b/Doc/c-api/structures.rst @@ -399,6 +399,40 @@ definition with the same method name. slot. This is helpful because calls to PyCFunctions are optimized more than wrapper object calls. +.. c:function:: PyObject * PyCMethod_New(PyMethodDef *ml, PyObject *self, PyObject *module, PyTypeObject *cls) + + Turn *ml* into a Python :term:`callable` object. + The caller must ensure that *ml* outlives the :term:`callable`. + Typically, *ml* is defined as a static variable. + + The *self* parameter will be passed as the *self* argument + to the C function in ``ml->ml_meth`` when invoked. + *self* can be ``NULL``. + + The :term:`callable` object's ``__module__`` attribute + can be set from the given *module* argument. + *module* should be a Python string, + which will be used as name of the module the function is defined in. + If unavailable, it can be set to :const:`None` or ``NULL``. + + .. seealso:: :attr:`function.__module__` + + The *cls* parameter will be passed as the *defining_class* + argument to the C function. + Must be set if :c:macro:`METH_METHOD` is set on ``ml->ml_flags``. + + .. versionadded:: 3.9 + + +.. c:function:: PyObject * PyCFunction_NewEx(PyMethodDef *ml, PyObject *self, PyObject *module) + + Equivalent to ``PyCMethod_New(ml, self, module, NULL)``. + + +.. c:function:: PyObject * PyCFunction_New(PyMethodDef *ml, PyObject *self) + + Equivalent to ``PyCMethod_New(ml, self, NULL, NULL)``. + Accessing attributes of extension types --------------------------------------- @@ -517,11 +551,11 @@ The following flags can be used with :c:member:`PyMemberDef.flags`: from ``PyObject``. Can only be used as part of :c:member:`Py_tp_members ` - :c:type:`slot ` when creating a class using negative + :c:type:`slot ` when creating a class using negative :c:member:`~PyType_Spec.basicsize`. It is mandatory in that case. - This flag is only used in :c:type:`PyTypeSlot`. + This flag is only used in :c:type:`PyType_Slot`. When setting :c:member:`~PyTypeObject.tp_members` during class creation, Python clears it and sets :c:member:`PyMemberDef.offset` to the offset from the ``PyObject`` struct. @@ -659,7 +693,8 @@ Defining Getters and Setters .. c:member:: setter set - Optional C function to set or delete the attribute, if omitted the attribute is readonly. + Optional C function to set or delete the attribute. + If ``NULL``, the attribute is read-only. .. c:member:: const char* doc @@ -669,18 +704,18 @@ Defining Getters and Setters Optional function pointer, providing additional data for getter and setter. - The ``get`` function takes one :c:expr:`PyObject*` parameter (the - instance) and a function pointer (the associated ``closure``):: +.. c:type:: PyObject *(*getter)(PyObject *, void *) - typedef PyObject *(*getter)(PyObject *, void *); + The ``get`` function takes one :c:expr:`PyObject*` parameter (the + instance) and a function pointer (the associated ``closure``): It should return a new reference on success or ``NULL`` with a set exception on failure. - ``set`` functions take two :c:expr:`PyObject*` parameters (the instance and - the value to be set) and a function pointer (the associated ``closure``):: +.. c:type:: int (*setter)(PyObject *, PyObject *, void *) - typedef int (*setter)(PyObject *, PyObject *, void *); + ``set`` functions take two :c:expr:`PyObject*` parameters (the instance and + the value to be set) and a function pointer (the associated ``closure``): In case the attribute should be deleted the second parameter is ``NULL``. Should return ``0`` on success or ``-1`` with a set exception on failure. diff --git a/Doc/conf.py b/Doc/conf.py index dc09b0b51ca84c..c2d57696aeeaa3 100644 --- a/Doc/conf.py +++ b/Doc/conf.py @@ -6,7 +6,9 @@ # The contents of this file are pickled, so don't put values in the namespace # that aren't pickleable (module imports are okay, they're removed automatically). -import sys, os, time +import os +import sys +import time sys.path.append(os.path.abspath('tools/extensions')) sys.path.append(os.path.abspath('includes')) @@ -55,7 +57,7 @@ # General substitutions. project = 'Python' -copyright = '2001-%s, Python Software Foundation' % time.strftime('%Y') +copyright = f"2001-{time.strftime('%Y')}, Python Software Foundation" # We look for the Include/patchlevel.h file in the current Python source tree # and replace the values accordingly. @@ -74,6 +76,10 @@ # Minimum version of sphinx required needs_sphinx = '4.2' +# Create table of contents entries for domain objects (e.g. functions, classes, +# attributes, etc.). Default is True. +toc_object_entries = False + # Ignore any .rst files in the includes/ directory; # they're embedded in pages but not rendered individually. # Ignore any .rst files in the venv/ directory. @@ -85,22 +91,32 @@ nitpick_ignore = [ # Standard C functions ('c:func', 'calloc'), + ('c:func', 'ctime'), ('c:func', 'dlopen'), ('c:func', 'exec'), ('c:func', 'fcntl'), ('c:func', 'fork'), ('c:func', 'free'), + ('c:func', 'gettimeofday'), ('c:func', 'gmtime'), + ('c:func', 'grantpt'), + ('c:func', 'localeconv'), ('c:func', 'localtime'), ('c:func', 'main'), ('c:func', 'malloc'), + ('c:func', 'mktime'), + ('c:func', 'posix_openpt'), ('c:func', 'printf'), + ('c:func', 'ptsname'), + ('c:func', 'ptsname_r'), ('c:func', 'realloc'), ('c:func', 'snprintf'), ('c:func', 'sprintf'), ('c:func', 'stat'), + ('c:func', 'strftime'), ('c:func', 'system'), ('c:func', 'time'), + ('c:func', 'unlockpt'), ('c:func', 'vsnprintf'), # Standard C types ('c:type', 'FILE'), @@ -288,6 +304,9 @@ 'root_include_title': False # We use the version switcher instead. } +if os.getenv("READTHEDOCS"): + html_theme_options["hosted_on"] = 'Read the Docs' + # Override stylesheet fingerprinting for Windows CHM htmlhelp to fix GH-91207 # https://github.com/python/cpython/issues/91207 if any('htmlhelp' in arg for arg in sys.argv): @@ -296,7 +315,7 @@ print("It may be removed in the future\n") # Short title used e.g. for HTML tags. -html_short_title = '%s Documentation' % release +html_short_title = f'{release} Documentation' # Deployment preview information # (See .readthedocs.yml and https://docs.readthedocs.io/en/stable/reference/environment-variables.html) @@ -345,12 +364,9 @@ latex_engine = 'xelatex' -# Get LaTeX to handle Unicode correctly latex_elements = { -} - -# Additional stuff for the LaTeX preamble. -latex_elements['preamble'] = r''' + # For the LaTeX preamble. + 'preamble': r''' \authoraddress{ \sphinxstrong{Python Software Foundation}\\ Email: \sphinxemail{docs@python.org} @@ -358,13 +374,12 @@ \let\Verbatim=\OriginalVerbatim \let\endVerbatim=\endOriginalVerbatim \setcounter{tocdepth}{2} -''' - -# The paper size ('letter' or 'a4'). -latex_elements['papersize'] = 'a4' - -# The font size ('10pt', '11pt' or '12pt'). -latex_elements['pointsize'] = '10pt' +''', + # The paper size ('letter' or 'a4'). + 'papersize': 'a4', + # The font size ('10pt', '11pt' or '12pt'). + 'pointsize': '10pt', +} # Grouping the document tree into LaTeX files. List of tuples # (source start file, target name, title, author, document class [howto/manual]). @@ -427,9 +442,9 @@ # Regexes to find C items in the source files. coverage_c_regexes = { - 'cfunction': (r'^PyAPI_FUNC\(.*\)\s+([^_][\w_]+)'), - 'data': (r'^PyAPI_DATA\(.*\)\s+([^_][\w_]+)'), - 'macro': (r'^#define ([^_][\w_]+)\(.*\)[\s|\\]'), + 'cfunction': r'^PyAPI_FUNC\(.*\)\s+([^_][\w_]+)', + 'data': r'^PyAPI_DATA\(.*\)\s+([^_][\w_]+)', + 'macro': r'^#define ([^_][\w_]+)\(.*\)[\s|\\]', } # The coverage checker will ignore all C items whose names match these regexes diff --git a/Doc/copyright.rst b/Doc/copyright.rst index 9b71683155eebe..8629ed1fc38009 100644 --- a/Doc/copyright.rst +++ b/Doc/copyright.rst @@ -4,7 +4,7 @@ Copyright Python and this documentation is: -Copyright © 2001-2023 Python Software Foundation. All rights reserved. +Copyright © 2001-2024 Python Software Foundation. All rights reserved. Copyright © 2000 BeOpen.com. All rights reserved. diff --git a/Doc/data/refcounts.dat b/Doc/data/refcounts.dat index ef9ac1617a284b..62a96146d605ff 100644 --- a/Doc/data/refcounts.dat +++ b/Doc/data/refcounts.dat @@ -402,6 +402,21 @@ PyContextVar_Reset:int::: PyContextVar_Reset:PyObject*:var:0: PyContextVar_Reset:PyObject*:token:-1: +PyCFunction_New:PyObject*::+1: +PyCFunction_New:PyMethodDef*:ml:: +PyCFunction_New:PyObject*:self:+1: + +PyCFunction_NewEx:PyObject*::+1: +PyCFunction_NewEx:PyMethodDef*:ml:: +PyCFunction_NewEx:PyObject*:self:+1: +PyCFunction_NewEx:PyObject*:module:+1: + +PyCMethod_New:PyObject*::+1: +PyCMethod_New:PyMethodDef*:ml:: +PyCMethod_New:PyObject*:self:+1: +PyCMethod_New:PyObject*:module:+1: +PyCMethod_New:PyObject*:cls:+1: + PyDate_Check:int::: PyDate_Check:PyObject*:ob:0: @@ -1118,6 +1133,10 @@ PyList_GetItem:PyObject*::0: PyList_GetItem:PyObject*:list:0: PyList_GetItem:Py_ssize_t:index:: +PyList_GetItemRef:PyObject*::+1: +PyList_GetItemRef:PyObject*:list:0: +PyList_GetItemRef:Py_ssize_t:index:: + PyList_GetSlice:PyObject*::+1: PyList_GetSlice:PyObject*:list:0: PyList_GetSlice:Py_ssize_t:low:: @@ -1589,6 +1608,13 @@ PyObject_Call:PyObject*:callable_object:0: PyObject_Call:PyObject*:args:0: PyObject_Call:PyObject*:kw:0: +PyObject_CallNoArgs:PyObject*::+1: +PyObject_CallNoArgs:PyObject*:callable_object:0: + +PyObject_CallOneArg:PyObject*::+1: +PyObject_CallOneArg:PyObject*:callable_object:0: +PyObject_CallOneArg:PyObject*:arg:0: + PyObject_CallFunction:PyObject*::+1: PyObject_CallFunction:PyObject*:callable_object:0: PyObject_CallFunction:const char*:format:: diff --git a/Doc/data/stable_abi.dat b/Doc/data/stable_abi.dat index 811b1bd84d2417..def1903204add7 100644 --- a/Doc/data/stable_abi.dat +++ b/Doc/data/stable_abi.dat @@ -220,6 +220,7 @@ var,PyExc_GeneratorExit,3.2,, var,PyExc_IOError,3.2,, var,PyExc_ImportError,3.2,, var,PyExc_ImportWarning,3.2,, +var,PyExc_IncompleteInputError,3.13,, var,PyExc_IndentationError,3.2,, var,PyExc_IndexError,3.2,, var,PyExc_InterruptedError,3.7,, @@ -335,6 +336,7 @@ var,PyListRevIter_Type,3.2,, function,PyList_Append,3.2,, function,PyList_AsTuple,3.2,, function,PyList_GetItem,3.2,, +function,PyList_GetItemRef,3.13,, function,PyList_GetSlice,3.2,, function,PyList_Insert,3.2,, function,PyList_New,3.2,, diff --git a/Doc/faq/design.rst b/Doc/faq/design.rst index ae02c443e5938b..300e1b6cc40a58 100644 --- a/Doc/faq/design.rst +++ b/Doc/faq/design.rst @@ -451,7 +451,7 @@ on the key and a per-process seed; for example, ``'Python'`` could hash to to ``1142331976``. The hash code is then used to calculate a location in an internal array where the value will be stored. Assuming that you're storing keys that all have different hash values, this means that dictionaries take -constant time -- O(1), in Big-O notation -- to retrieve a key. +constant time -- *O*\ (1), in Big-O notation -- to retrieve a key. Why must dictionary keys be immutable? diff --git a/Doc/faq/library.rst b/Doc/faq/library.rst index 476a43d9c288f1..e2f8004c7e3aea 100644 --- a/Doc/faq/library.rst +++ b/Doc/faq/library.rst @@ -405,22 +405,37 @@ lists. When in doubt, use a mutex! Can't we get rid of the Global Interpreter Lock? ------------------------------------------------ -.. XXX link to dbeazley's talk about GIL? - The :term:`global interpreter lock` (GIL) is often seen as a hindrance to Python's deployment on high-end multiprocessor server machines, because a multi-threaded Python program effectively only uses one CPU, due to the insistence that (almost) all Python code can only run while the GIL is held. -Back in the days of Python 1.5, Greg Stein actually implemented a comprehensive +With the approval of :pep:`703` work is now underway to remove the GIL from the +CPython implementation of Python. Initially it will be implemented as an +optional compiler flag when building the interpreter, and so separate +builds will be available with and without the GIL. Long-term, the hope is +to settle on a single build, once the performance implications of removing the +GIL are fully understood. Python 3.13 is likely to be the first release +containing this work, although it may not be completely functional in this +release. + +The current work to remove the GIL is based on a +`fork of Python 3.9 with the GIL removed <https://github.com/colesbury/nogil>`_ +by Sam Gross. +Prior to that, +in the days of Python 1.5, Greg Stein actually implemented a comprehensive patch set (the "free threading" patches) that removed the GIL and replaced it -with fine-grained locking. Adam Olsen recently did a similar experiment +with fine-grained locking. Adam Olsen did a similar experiment in his `python-safethread <https://code.google.com/archive/p/python-safethread>`_ -project. Unfortunately, both experiments exhibited a sharp drop in single-thread +project. Unfortunately, both of these earlier experiments exhibited a sharp +drop in single-thread performance (at least 30% slower), due to the amount of fine-grained locking -necessary to compensate for the removal of the GIL. +necessary to compensate for the removal of the GIL. The Python 3.9 fork +is the first attempt at removing the GIL with an acceptable performance +impact. -This doesn't mean that you can't make good use of Python on multi-CPU machines! +The presence of the GIL in current Python releases +doesn't mean that you can't make good use of Python on multi-CPU machines! You just have to be creative with dividing the work up between multiple *processes* rather than multiple *threads*. The :class:`~concurrent.futures.ProcessPoolExecutor` class in the new @@ -434,22 +449,13 @@ thread of execution is in the C code and allow other threads to get some work done. Some standard library modules such as :mod:`zlib` and :mod:`hashlib` already do this. -It has been suggested that the GIL should be a per-interpreter-state lock rather -than truly global; interpreters then wouldn't be able to share objects. -Unfortunately, this isn't likely to happen either. It would be a tremendous -amount of work, because many object implementations currently have global state. -For example, small integers and short strings are cached; these caches would -have to be moved to the interpreter state. Other object types have their own -free list; these free lists would have to be moved to the interpreter state. -And so on. - -And I doubt that it can even be done in finite time, because the same problem -exists for 3rd party extensions. It is likely that 3rd party extensions are -being written at a faster rate than you can convert them to store all their -global state in the interpreter state. - -And finally, once you have multiple interpreters not sharing any state, what -have you gained over running each interpreter in a separate process? +An alternative approach to reducing the impact of the GIL is +to make the GIL a per-interpreter-state lock rather than truly global. +This was :ref:`first implemented in Python 3.12 <whatsnew312-pep684>` and is +available in the C API. A Python interface to it is expected in Python 3.13. +The main limitation to it at the moment is likely to be 3rd party extension +modules, since these must be written with multiple interpreters in mind in +order to be usable, so many older extension modules will not be usable. Input and Output diff --git a/Doc/glossary.rst b/Doc/glossary.rst index 601443d5aade94..f656e32514c717 100644 --- a/Doc/glossary.rst +++ b/Doc/glossary.rst @@ -341,7 +341,7 @@ Glossary docstring A string literal which appears as the first expression in a class, function or module. While ignored when the suite is executed, it is - recognized by the compiler and put into the :attr:`__doc__` attribute + recognized by the compiler and put into the :attr:`!__doc__` attribute of the enclosing class, function or module. Since it is available via introspection, it is the canonical place for documentation of the object. @@ -742,7 +742,7 @@ Glossary list A built-in Python :term:`sequence`. Despite its name it is more akin to an array in other languages than to a linked list since access to - elements is O(1). + elements is *O*\ (1). list comprehension A compact way to process all or part of the elements in a sequence and @@ -1104,10 +1104,12 @@ Glossary The :class:`collections.abc.Sequence` abstract base class defines a much richer interface that goes beyond just :meth:`~object.__getitem__` and :meth:`~object.__len__`, adding - :meth:`count`, :meth:`index`, :meth:`~object.__contains__`, and + :meth:`!count`, :meth:`!index`, :meth:`~object.__contains__`, and :meth:`~object.__reversed__`. Types that implement this expanded interface can be registered explicitly using - :func:`~abc.ABCMeta.register`. + :func:`~abc.ABCMeta.register`. For more documentation on sequence + methods generally, see + :ref:`Common Sequence Operations <typesseq-common>`. set comprehension A compact way to process all or part of the elements in an iterable and diff --git a/Doc/howto/descriptor.rst b/Doc/howto/descriptor.rst index 87274a5133d1cf..75346f2c7618c2 100644 --- a/Doc/howto/descriptor.rst +++ b/Doc/howto/descriptor.rst @@ -1250,7 +1250,7 @@ instance:: <function D.f at 0x00C45070> >>> d.f.__self__ - <__main__.D object at 0x1012e1f98> + <__main__.D object at 0x00B18C90> If you have ever wondered where *self* comes from in regular methods or where *cls* comes from in class methods, this is it! diff --git a/Doc/howto/enum.rst b/Doc/howto/enum.rst index 1e9ac9b6761b64..30be15230fc088 100644 --- a/Doc/howto/enum.rst +++ b/Doc/howto/enum.rst @@ -497,13 +497,30 @@ the :meth:`~Enum.__repr__` omits the inherited class' name. For example:: >>> Creature.DOG <Creature.DOG: size='medium', legs=4> -Use the :func:`!dataclass` argument ``repr=False`` +Use the :func:`~dataclasses.dataclass` argument ``repr=False`` to use the standard :func:`repr`. .. versionchanged:: 3.12 Only the dataclass fields are shown in the value area, not the dataclass' name. +.. note:: + + Adding :func:`~dataclasses.dataclass` decorator to :class:`Enum` + and its subclasses is not supported. It will not raise any errors, + but it will produce very strange results at runtime, such as members + being equal to each other:: + + >>> @dataclass # don't do this: it does not make any sense + ... class Color(Enum): + ... RED = 1 + ... BLUE = 2 + ... + >>> Color.RED is Color.BLUE + False + >>> Color.RED == Color.BLUE # problem is here: they should not be equal + True + Pickling -------- diff --git a/Doc/howto/isolating-extensions.rst b/Doc/howto/isolating-extensions.rst index 835c0afad7c6c8..e35855deedbe5f 100644 --- a/Doc/howto/isolating-extensions.rst +++ b/Doc/howto/isolating-extensions.rst @@ -337,7 +337,7 @@ That is, heap types should: - Have the :c:macro:`Py_TPFLAGS_HAVE_GC` flag. - Define a traverse function using ``Py_tp_traverse``, which - visits the type (e.g. using :c:expr:`Py_VISIT(Py_TYPE(self))`). + visits the type (e.g. using ``Py_VISIT(Py_TYPE(self))``). Please refer to the the documentation of :c:macro:`Py_TPFLAGS_HAVE_GC` and :c:member:`~PyTypeObject.tp_traverse` @@ -482,7 +482,7 @@ The largest roadblock is getting *the class a method was defined in*, or that method's "defining class" for short. The defining class can have a reference to the module it is part of. -Do not confuse the defining class with :c:expr:`Py_TYPE(self)`. If the method +Do not confuse the defining class with ``Py_TYPE(self)``. If the method is called on a *subclass* of your type, ``Py_TYPE(self)`` will refer to that subclass, which may be defined in different module than yours. diff --git a/Doc/howto/logging-cookbook.rst b/Doc/howto/logging-cookbook.rst index 588f5a0a53ded0..80147e31fcbae1 100644 --- a/Doc/howto/logging-cookbook.rst +++ b/Doc/howto/logging-cookbook.rst @@ -332,10 +332,10 @@ Suppose you configure logging with the following JSON: } } -This configuration does *almost* what we want, except that ``sys.stdout`` would -show messages of severity ``ERROR`` and above as well as ``INFO`` and -``WARNING`` messages. To prevent this, we can set up a filter which excludes -those messages and add it to the relevant handler. This can be configured by +This configuration does *almost* what we want, except that ``sys.stdout`` would show messages +of severity ``ERROR`` and only events of this severity and higher will be tracked +as well as ``INFO`` and ``WARNING`` messages. To prevent this, we can set up a filter which +excludes those messages and add it to the relevant handler. This can be configured by adding a ``filters`` section parallel to ``formatters`` and ``handlers``: .. code-block:: json @@ -1933,30 +1933,28 @@ This dictionary is passed to :func:`~config.dictConfig` to put the configuration LOGGING = { 'version': 1, - 'disable_existing_loggers': True, + 'disable_existing_loggers': False, 'formatters': { 'verbose': { - 'format': '%(levelname)s %(asctime)s %(module)s %(process)d %(thread)d %(message)s' + 'format': '{levelname} {asctime} {module} {process:d} {thread:d} {message}', + 'style': '{', }, 'simple': { - 'format': '%(levelname)s %(message)s' + 'format': '{levelname} {message}', + 'style': '{', }, }, 'filters': { 'special': { '()': 'project.logging.SpecialFilter', 'foo': 'bar', - } + }, }, 'handlers': { - 'null': { - 'level':'DEBUG', - 'class':'django.utils.log.NullHandler', - }, - 'console':{ - 'level':'DEBUG', - 'class':'logging.StreamHandler', - 'formatter': 'simple' + 'console': { + 'level': 'INFO', + 'class': 'logging.StreamHandler', + 'formatter': 'simple', }, 'mail_admins': { 'level': 'ERROR', @@ -1966,9 +1964,8 @@ This dictionary is passed to :func:`~config.dictConfig` to put the configuration }, 'loggers': { 'django': { - 'handlers':['null'], + 'handlers': ['console'], 'propagate': True, - 'level':'INFO', }, 'django.request': { 'handlers': ['mail_admins'], diff --git a/Doc/howto/logging.rst b/Doc/howto/logging.rst index 7330cf675baa36..347330e98dd00c 100644 --- a/Doc/howto/logging.rst +++ b/Doc/howto/logging.rst @@ -89,9 +89,8 @@ described below (in increasing order of severity): | | itself may be unable to continue running. | +--------------+---------------------------------------------+ -The default level is ``WARNING``, which means that only events of this level -and above will be tracked, unless the logging package is configured to do -otherwise. +The default level is ``WARNING``, which means that only events of this severity and higher +will be tracked, unless the logging package is configured to do otherwise. Events that are tracked can be handled in different ways. The simplest way of handling tracked events is to print them to the console. Another common way @@ -521,7 +520,7 @@ custom handlers) are the following configuration methods: * The :meth:`~Handler.setLevel` method, just as in logger objects, specifies the lowest severity that will be dispatched to the appropriate destination. Why - are there two :func:`setLevel` methods? The level set in the logger + are there two :meth:`~Handler.setLevel` methods? The level set in the logger determines which severity of messages it will pass to its handlers. The level set in each handler determines which messages that handler will send on. @@ -775,29 +774,29 @@ What happens if no configuration is provided If no logging configuration is provided, it is possible to have a situation where a logging event needs to be output, but no handlers can be found to -output the event. The behaviour of the logging package in these -circumstances is dependent on the Python version. +output the event. -For versions of Python prior to 3.2, the behaviour is as follows: +The event is output using a 'handler of last resort', stored in +:data:`lastResort`. This internal handler is not associated with any +logger, and acts like a :class:`~logging.StreamHandler` which writes the +event description message to the current value of ``sys.stderr`` (therefore +respecting any redirections which may be in effect). No formatting is +done on the message - just the bare event description message is printed. +The handler's level is set to ``WARNING``, so all events at this and +greater severities will be output. -* If *logging.raiseExceptions* is ``False`` (production mode), the event is - silently dropped. +.. versionchanged:: 3.2 -* If *logging.raiseExceptions* is ``True`` (development mode), a message - 'No handlers could be found for logger X.Y.Z' is printed once. + For versions of Python prior to 3.2, the behaviour is as follows: -In Python 3.2 and later, the behaviour is as follows: + * If :data:`raiseExceptions` is ``False`` (production mode), the event is + silently dropped. -* The event is output using a 'handler of last resort', stored in - ``logging.lastResort``. This internal handler is not associated with any - logger, and acts like a :class:`~logging.StreamHandler` which writes the - event description message to the current value of ``sys.stderr`` (therefore - respecting any redirections which may be in effect). No formatting is - done on the message - just the bare event description message is printed. - The handler's level is set to ``WARNING``, so all events at this and - greater severities will be output. + * If :data:`raiseExceptions` is ``True`` (development mode), a message + 'No handlers could be found for logger X.Y.Z' is printed once. -To obtain the pre-3.2 behaviour, ``logging.lastResort`` can be set to ``None``. + To obtain the pre-3.2 behaviour, + :data:`lastResort` can be set to ``None``. .. _library-config: @@ -999,7 +998,7 @@ Logged messages are formatted for presentation through instances of the use with the % operator and a dictionary. For formatting multiple messages in a batch, instances of -:class:`~handlers.BufferingFormatter` can be used. In addition to the format +:class:`BufferingFormatter` can be used. In addition to the format string (which is applied to each message in the batch), there is provision for header and trailer format strings. @@ -1035,7 +1034,8 @@ checks to see if a module-level variable, :data:`raiseExceptions`, is set. If set, a traceback is printed to :data:`sys.stderr`. If not set, the exception is swallowed. -.. note:: The default value of :data:`raiseExceptions` is ``True``. This is +.. note:: + The default value of :data:`raiseExceptions` is ``True``. This is because during development, you typically want to be notified of any exceptions that occur. It's advised that you set :data:`raiseExceptions` to ``False`` for production usage. @@ -1073,7 +1073,7 @@ You can write code like this:: expensive_func2()) so that if the logger's threshold is set above ``DEBUG``, the calls to -:func:`expensive_func1` and :func:`expensive_func2` are never made. +``expensive_func1`` and ``expensive_func2`` are never made. .. note:: In some cases, :meth:`~Logger.isEnabledFor` can itself be more expensive than you'd like (e.g. for deeply nested loggers where an explicit diff --git a/Doc/howto/urllib2.rst b/Doc/howto/urllib2.rst index 570435d48866d3..7f54a410881514 100644 --- a/Doc/howto/urllib2.rst +++ b/Doc/howto/urllib2.rst @@ -392,16 +392,16 @@ info and geturl =============== The response returned by urlopen (or the :exc:`~urllib.error.HTTPError` instance) has two -useful methods :meth:`info` and :meth:`geturl` and is defined in the module -:mod:`urllib.response`.. +useful methods :meth:`!info` and :meth:`!geturl` and is defined in the module +:mod:`urllib.response`. -**geturl** - this returns the real URL of the page fetched. This is useful -because ``urlopen`` (or the opener object used) may have followed a -redirect. The URL of the page fetched may not be the same as the URL requested. +* **geturl** - this returns the real URL of the page fetched. This is useful + because ``urlopen`` (or the opener object used) may have followed a + redirect. The URL of the page fetched may not be the same as the URL requested. -**info** - this returns a dictionary-like object that describes the page -fetched, particularly the headers sent by the server. It is currently an -:class:`http.client.HTTPMessage` instance. +* **info** - this returns a dictionary-like object that describes the page + fetched, particularly the headers sent by the server. It is currently an + :class:`http.client.HTTPMessage` instance. Typical headers include 'Content-length', 'Content-type', and so on. See the `Quick Reference to HTTP Headers <https://jkorpela.fi/http.html>`_ @@ -507,7 +507,7 @@ than the URL you pass to .add_password() will also match. :: In the above example we only supplied our ``HTTPBasicAuthHandler`` to ``build_opener``. By default openers have the handlers for normal situations - -- ``ProxyHandler`` (if a proxy setting such as an :envvar:`http_proxy` + -- ``ProxyHandler`` (if a proxy setting such as an :envvar:`!http_proxy` environment variable is set), ``UnknownHandler``, ``HTTPHandler``, ``HTTPDefaultErrorHandler``, ``HTTPRedirectHandler``, ``FTPHandler``, ``FileHandler``, ``DataHandler``, ``HTTPErrorProcessor``. diff --git a/Doc/library/__future__.rst b/Doc/library/__future__.rst index d261e4a4f338a5..762f8b4695b3dd 100644 --- a/Doc/library/__future__.rst +++ b/Doc/library/__future__.rst @@ -8,20 +8,68 @@ -------------- -:mod:`__future__` is a real module, and serves three purposes: +Imports of the form ``from __future__ import feature`` are called +:ref:`future statements <future>`. These are special-cased by the Python compiler +to allow the use of new Python features in modules containing the future statement +before the release in which the feature becomes standard. + +While these future statements are given additional special meaning by the +Python compiler, they are still executed like any other import statement and +the :mod:`__future__` exists and is handled by the import system the same way +any other Python module would be. This design serves three purposes: * To avoid confusing existing tools that analyze import statements and expect to find the modules they're importing. -* To ensure that :ref:`future statements <future>` run under releases prior to - 2.1 at least yield runtime exceptions (the import of :mod:`__future__` will - fail, because there was no module of that name prior to 2.1). - * To document when incompatible changes were introduced, and when they will be --- or were --- made mandatory. This is a form of executable documentation, and can be inspected programmatically via importing :mod:`__future__` and examining its contents. +* To ensure that :ref:`future statements <future>` run under releases prior to + Python 2.1 at least yield runtime exceptions (the import of :mod:`__future__` + will fail, because there was no module of that name prior to 2.1). + +Module Contents +--------------- + +No feature description will ever be deleted from :mod:`__future__`. Since its +introduction in Python 2.1 the following features have found their way into the +language using this mechanism: + ++------------------+-------------+--------------+---------------------------------------------+ +| feature | optional in | mandatory in | effect | ++==================+=============+==============+=============================================+ +| nested_scopes | 2.1.0b1 | 2.2 | :pep:`227`: | +| | | | *Statically Nested Scopes* | ++------------------+-------------+--------------+---------------------------------------------+ +| generators | 2.2.0a1 | 2.3 | :pep:`255`: | +| | | | *Simple Generators* | ++------------------+-------------+--------------+---------------------------------------------+ +| division | 2.2.0a2 | 3.0 | :pep:`238`: | +| | | | *Changing the Division Operator* | ++------------------+-------------+--------------+---------------------------------------------+ +| absolute_import | 2.5.0a1 | 3.0 | :pep:`328`: | +| | | | *Imports: Multi-Line and Absolute/Relative* | ++------------------+-------------+--------------+---------------------------------------------+ +| with_statement | 2.5.0a1 | 2.6 | :pep:`343`: | +| | | | *The "with" Statement* | ++------------------+-------------+--------------+---------------------------------------------+ +| print_function | 2.6.0a2 | 3.0 | :pep:`3105`: | +| | | | *Make print a function* | ++------------------+-------------+--------------+---------------------------------------------+ +| unicode_literals | 2.6.0a2 | 3.0 | :pep:`3112`: | +| | | | *Bytes literals in Python 3000* | ++------------------+-------------+--------------+---------------------------------------------+ +| generator_stop | 3.5.0b1 | 3.7 | :pep:`479`: | +| | | | *StopIteration handling inside generators* | ++------------------+-------------+--------------+---------------------------------------------+ +| annotations | 3.7.0b1 | TBD [1]_ | :pep:`563`: | +| | | | *Postponed evaluation of annotations* | ++------------------+-------------+--------------+---------------------------------------------+ + +.. XXX Adding a new entry? Remember to update simple_stmts.rst, too. + .. _future-classes: .. class:: _Feature @@ -65,43 +113,6 @@ dynamically compiled code. This flag is stored in the :attr:`_Feature.compiler_flag` attribute on :class:`_Feature` instances. -No feature description will ever be deleted from :mod:`__future__`. Since its -introduction in Python 2.1 the following features have found their way into the -language using this mechanism: - -+------------------+-------------+--------------+---------------------------------------------+ -| feature | optional in | mandatory in | effect | -+==================+=============+==============+=============================================+ -| nested_scopes | 2.1.0b1 | 2.2 | :pep:`227`: | -| | | | *Statically Nested Scopes* | -+------------------+-------------+--------------+---------------------------------------------+ -| generators | 2.2.0a1 | 2.3 | :pep:`255`: | -| | | | *Simple Generators* | -+------------------+-------------+--------------+---------------------------------------------+ -| division | 2.2.0a2 | 3.0 | :pep:`238`: | -| | | | *Changing the Division Operator* | -+------------------+-------------+--------------+---------------------------------------------+ -| absolute_import | 2.5.0a1 | 3.0 | :pep:`328`: | -| | | | *Imports: Multi-Line and Absolute/Relative* | -+------------------+-------------+--------------+---------------------------------------------+ -| with_statement | 2.5.0a1 | 2.6 | :pep:`343`: | -| | | | *The "with" Statement* | -+------------------+-------------+--------------+---------------------------------------------+ -| print_function | 2.6.0a2 | 3.0 | :pep:`3105`: | -| | | | *Make print a function* | -+------------------+-------------+--------------+---------------------------------------------+ -| unicode_literals | 2.6.0a2 | 3.0 | :pep:`3112`: | -| | | | *Bytes literals in Python 3000* | -+------------------+-------------+--------------+---------------------------------------------+ -| generator_stop | 3.5.0b1 | 3.7 | :pep:`479`: | -| | | | *StopIteration handling inside generators* | -+------------------+-------------+--------------+---------------------------------------------+ -| annotations | 3.7.0b1 | TBD [1]_ | :pep:`563`: | -| | | | *Postponed evaluation of annotations* | -+------------------+-------------+--------------+---------------------------------------------+ - -.. XXX Adding a new entry? Remember to update simple_stmts.rst, too. - .. [1] ``from __future__ import annotations`` was previously scheduled to become mandatory in Python 3.10, but the Python Steering Council @@ -115,3 +126,6 @@ language using this mechanism: :ref:`future` How the compiler treats future imports. + + :pep:`236` - Back to the __future__ + The original proposal for the __future__ mechanism. diff --git a/Doc/library/argparse.rst b/Doc/library/argparse.rst index fbffa71d200735..952643a46416d2 100644 --- a/Doc/library/argparse.rst +++ b/Doc/library/argparse.rst @@ -777,6 +777,8 @@ The add_argument() method * dest_ - The name of the attribute to be added to the object returned by :meth:`parse_args`. + * deprecated_ - Whether or not use of the argument is deprecated. + The following sections describe how each of these are used. @@ -1439,6 +1441,34 @@ behavior:: >>> parser.parse_args('--foo XXX'.split()) Namespace(bar='XXX') + +.. _deprecated: + +deprecated +^^^^^^^^^^ + +During a project's lifetime, some arguments may need to be removed from the +command line. Before removing them, you should inform +your users that the arguments are deprecated and will be removed. +The ``deprecated`` keyword argument of +:meth:`~ArgumentParser.add_argument`, which defaults to ``False``, +specifies if the argument is deprecated and will be removed +in the future. +For arguments, if ``deprecated`` is ``True``, then a warning will be +printed to standard error when the argument is used:: + + >>> import argparse + >>> parser = argparse.ArgumentParser(prog='snake.py') + >>> parser.add_argument('--legs', default=0, type=int, deprecated=True) + >>> parser.parse_args([]) + Namespace(legs=0) + >>> parser.parse_args(['--legs', '4']) # doctest: +SKIP + snake.py: warning: option '--legs' is deprecated + Namespace(legs=4) + +.. versionchanged:: 3.13 + + Action classes ^^^^^^^^^^^^^^ @@ -1842,7 +1872,8 @@ Sub-commands {foo,bar} additional help - Furthermore, ``add_parser`` supports an additional ``aliases`` argument, + Furthermore, :meth:`~_SubParsersAction.add_parser` supports an additional + *aliases* argument, which allows multiple strings to refer to the same subparser. This example, like ``svn``, aliases ``co`` as a shorthand for ``checkout``:: @@ -1853,6 +1884,20 @@ Sub-commands >>> parser.parse_args(['co', 'bar']) Namespace(foo='bar') + :meth:`~_SubParsersAction.add_parser` supports also an additional + *deprecated* argument, which allows to deprecate the subparser. + + >>> import argparse + >>> parser = argparse.ArgumentParser(prog='chicken.py') + >>> subparsers = parser.add_subparsers() + >>> run = subparsers.add_parser('run') + >>> fly = subparsers.add_parser('fly', deprecated=True) + >>> parser.parse_args(['fly']) # doctest: +SKIP + chicken.py: warning: command 'fly' is deprecated + Namespace() + + .. versionadded:: 3.13 + One particularly effective way of handling sub-commands is to combine the use of the :meth:`add_subparsers` method with calls to :meth:`set_defaults` so that each subparser knows which Python function it should execute. For @@ -1936,8 +1981,8 @@ FileType objects >>> parser.parse_args(['-']) Namespace(infile=<_io.TextIOWrapper name='<stdin>' encoding='UTF-8'>) - .. versionadded:: 3.4 - The *encodings* and *errors* keyword arguments. + .. versionchanged:: 3.4 + Added the *encodings* and *errors* parameters. Argument groups diff --git a/Doc/library/array.rst b/Doc/library/array.rst index ad622627724217..a0e8bb20a098fd 100644 --- a/Doc/library/array.rst +++ b/Doc/library/array.rst @@ -79,14 +79,16 @@ The module defines the following type: .. class:: array(typecode[, initializer]) A new array whose items are restricted by *typecode*, and initialized - from the optional *initializer* value, which must be a list, a - :term:`bytes-like object`, or iterable over elements of the - appropriate type. + from the optional *initializer* value, which must be a :class:`bytes` + or :class:`bytearray` object, a Unicode string, or iterable over elements + of the appropriate type. - If given a list or string, the initializer is passed to the new array's - :meth:`fromlist`, :meth:`frombytes`, or :meth:`fromunicode` method (see below) - to add initial items to the array. Otherwise, the iterable initializer is - passed to the :meth:`extend` method. + If given a :class:`bytes` or :class:`bytearray` object, the initializer + is passed to the new array's :meth:`frombytes` method; + if given a Unicode string, the initializer is passed to the + :meth:`fromunicode` method; + otherwise, the initializer's iterator is passed to the :meth:`extend` method + to add initial items to the array. Array objects support the ordinary sequence operations of indexing, slicing, concatenation, and multiplication. When using slice assignment, the assigned @@ -152,10 +154,11 @@ The module defines the following type: must be the right type to be appended to the array. - .. method:: frombytes(s) + .. method:: frombytes(buffer) - Appends items from the string, interpreting the string as an array of machine - values (as if it had been read from a file using the :meth:`fromfile` method). + Appends items from the :term:`bytes-like object`, interpreting + its content as an array of machine values (as if it had been read + from a file using the :meth:`fromfile` method). .. versionadded:: 3.2 :meth:`!fromstring` is renamed to :meth:`frombytes` for clarity. @@ -177,7 +180,7 @@ The module defines the following type: .. method:: fromunicode(s) - Extends this array with data from the given unicode string. + Extends this array with data from the given Unicode string. The array must have type code ``'u'`` or ``'w'``; otherwise a :exc:`ValueError` is raised. Use ``array.frombytes(unicodestring.encode(enc))`` to append Unicode data to an array of some other type. @@ -239,24 +242,27 @@ The module defines the following type: .. method:: tounicode() - Convert the array to a unicode string. The array must have a type ``'u'`` or ``'w'``; + Convert the array to a Unicode string. The array must have a type ``'u'`` or ``'w'``; otherwise a :exc:`ValueError` is raised. Use ``array.tobytes().decode(enc)`` to - obtain a unicode string from an array of some other type. + obtain a Unicode string from an array of some other type. -When an array object is printed or converted to a string, it is represented as -``array(typecode, initializer)``. The *initializer* is omitted if the array is -empty, otherwise it is a string if the *typecode* is ``'u'`` or ``'w'``, -otherwise it is a list of numbers. -The string is guaranteed to be able to be converted back to an +The string representation of array objects has the form +``array(typecode, initializer)``. +The *initializer* is omitted if the array is empty, otherwise it is +a Unicode string if the *typecode* is ``'u'`` or ``'w'``, otherwise it is +a list of numbers. +The string representation is guaranteed to be able to be converted back to an array with the same type and value using :func:`eval`, so long as the :class:`~array.array` class has been imported using ``from array import array``. +Variables ``inf`` and ``nan`` must also be defined if it contains +corresponding floating point values. Examples:: array('l') array('w', 'hello \u2641') array('l', [1, 2, 3, 4, 5]) - array('d', [1.0, 2.0, 3.14]) + array('d', [1.0, 2.0, 3.14, -inf, nan]) .. seealso:: diff --git a/Doc/library/asyncio-policy.rst b/Doc/library/asyncio-policy.rst index 0d7821e608ec98..346b740a8f757a 100644 --- a/Doc/library/asyncio-policy.rst +++ b/Doc/library/asyncio-policy.rst @@ -237,7 +237,7 @@ implementation used by the asyncio event loop: It works reliably even when the asyncio event loop is run in a non-main OS thread. - There is no noticeable overhead when handling a big number of children (*O(1)* each + There is no noticeable overhead when handling a big number of children (*O*\ (1) each time a child terminates), but starting a thread per process requires extra memory. This watcher is used by default. @@ -257,7 +257,7 @@ implementation used by the asyncio event loop: watcher is installed. The solution is safe but it has a significant overhead when - handling a big number of processes (*O(n)* each time a + handling a big number of processes (*O*\ (*n*) each time a :py:data:`SIGCHLD` is received). .. versionadded:: 3.8 @@ -273,7 +273,7 @@ implementation used by the asyncio event loop: The watcher avoids disrupting other code spawning processes by polling every process explicitly on a :py:data:`SIGCHLD` signal. - This solution is as safe as :class:`MultiLoopChildWatcher` and has the same *O(N)* + This solution is as safe as :class:`MultiLoopChildWatcher` and has the same *O*\ (*n*) complexity but requires a running event loop in the main thread to work. .. deprecated:: 3.12 @@ -285,7 +285,7 @@ implementation used by the asyncio event loop: processes and waiting for their termination. There is no noticeable overhead when handling a big number of - children (*O(1)* each time a child terminates). + children (*O*\ (1) each time a child terminates). This solution requires a running event loop in the main thread to work, as :class:`SafeChildWatcher`. diff --git a/Doc/library/asyncio-stream.rst b/Doc/library/asyncio-stream.rst index 0736e783bbc8c8..3427da1b43caef 100644 --- a/Doc/library/asyncio-stream.rst +++ b/Doc/library/asyncio-stream.rst @@ -77,8 +77,8 @@ and work with streams: .. versionchanged:: 3.7 Added the *ssl_handshake_timeout* parameter. - .. versionadded:: 3.8 - Added *happy_eyeballs_delay* and *interleave* parameters. + .. versionchanged:: 3.8 + Added the *happy_eyeballs_delay* and *interleave* parameters. .. versionchanged:: 3.10 Removed the *loop* parameter. diff --git a/Doc/library/asyncio-sync.rst b/Doc/library/asyncio-sync.rst index 05bdf5488af143..3cf8e2737e85dc 100644 --- a/Doc/library/asyncio-sync.rst +++ b/Doc/library/asyncio-sync.rst @@ -216,8 +216,8 @@ Condition .. method:: notify(n=1) - Wake up at most *n* tasks (1 by default) waiting on this - condition. The method is no-op if no tasks are waiting. + Wake up *n* tasks (1 by default) waiting on this + condition. If fewer than *n* tasks are waiting they are all awakened. The lock must be acquired before this method is called and released shortly after. If called with an *unlocked* lock @@ -257,12 +257,18 @@ Condition Once awakened, the Condition re-acquires its lock and this method returns ``True``. + Note that a task *may* return from this call spuriously, + which is why the caller should always re-check the state + and be prepared to :meth:`wait` again. For this reason, you may + prefer to use :meth:`wait_for` instead. + .. coroutinemethod:: wait_for(predicate) Wait until a predicate becomes *true*. The predicate must be a callable which result will be - interpreted as a boolean value. The final value is the + interpreted as a boolean value. The method will repeatedly + :meth:`wait` until the predicate evaluates to *true*. The final value is the return value. diff --git a/Doc/library/asyncio-task.rst b/Doc/library/asyncio-task.rst index 797065c8ccf894..24bd36e6431b4f 100644 --- a/Doc/library/asyncio-task.rst +++ b/Doc/library/asyncio-task.rst @@ -828,23 +828,22 @@ Waiting Primitives *return_when* indicates when this function should return. It must be one of the following constants: - .. tabularcolumns:: |l|L| - - +-----------------------------+----------------------------------------+ - | Constant | Description | - +=============================+========================================+ - | :const:`FIRST_COMPLETED` | The function will return when any | - | | future finishes or is cancelled. | - +-----------------------------+----------------------------------------+ - | :const:`FIRST_EXCEPTION` | The function will return when any | - | | future finishes by raising an | - | | exception. If no future raises an | - | | exception then it is equivalent to | - | | :const:`ALL_COMPLETED`. | - +-----------------------------+----------------------------------------+ - | :const:`ALL_COMPLETED` | The function will return when all | - | | futures finish or are cancelled. | - +-----------------------------+----------------------------------------+ + .. list-table:: + :header-rows: 1 + + * - Constant + - Description + + * - .. data:: FIRST_COMPLETED + - The function will return when any future finishes or is cancelled. + + * - .. data:: FIRST_EXCEPTION + - The function will return when any future finishes by raising an + exception. If no future raises an exception + then it is equivalent to :const:`ALL_COMPLETED`. + + * - .. data:: ALL_COMPLETED + - The function will return when all futures finish or are cancelled. Unlike :func:`~asyncio.wait_for`, ``wait()`` does not cancel the futures when a timeout occurs. diff --git a/Doc/library/atexit.rst b/Doc/library/atexit.rst index 3dbef69580d9b3..43a8bd2d7cd133 100644 --- a/Doc/library/atexit.rst +++ b/Doc/library/atexit.rst @@ -4,8 +4,8 @@ .. module:: atexit :synopsis: Register and execute cleanup functions. -.. moduleauthor:: Skip Montanaro <skip@pobox.com> -.. sectionauthor:: Skip Montanaro <skip@pobox.com> +.. moduleauthor:: Skip Montanaro <skip.montanaro@gmail.com> +.. sectionauthor:: Skip Montanaro <skip.montanaro@gmail.com> -------------- diff --git a/Doc/library/bdb.rst b/Doc/library/bdb.rst index 4ce5c9bcde38ff..52f0ca7c013482 100644 --- a/Doc/library/bdb.rst +++ b/Doc/library/bdb.rst @@ -132,8 +132,8 @@ The :mod:`bdb` module also defines two classes: frame is considered to originate in a certain module is determined by the ``__name__`` in the frame globals. - .. versionadded:: 3.1 - The *skip* argument. + .. versionchanged:: 3.1 + Added the *skip* parameter. The following methods of :class:`Bdb` normally don't need to be overridden. diff --git a/Doc/library/bisect.rst b/Doc/library/bisect.rst index 8022c596f0af97..31c79b91061591 100644 --- a/Doc/library/bisect.rst +++ b/Doc/library/bisect.rst @@ -19,9 +19,9 @@ linear searches or frequent resorting. The module is called :mod:`bisect` because it uses a basic bisection algorithm to do its work. Unlike other bisection tools that search for a specific value, the functions in this module are designed to locate an -insertion point. Accordingly, the functions never call an :meth:`__eq__` +insertion point. Accordingly, the functions never call an :meth:`~object.__eq__` method to determine whether a value has been found. Instead, the -functions only call the :meth:`__lt__` method and will return an insertion +functions only call the :meth:`~object.__lt__` method and will return an insertion point between values in an array. .. _bisect functions: @@ -73,13 +73,13 @@ The following functions are provided: Insert *x* in *a* in sorted order. This function first runs :py:func:`~bisect.bisect_left` to locate an insertion point. - Next, it runs the :meth:`insert` method on *a* to insert *x* at the + Next, it runs the :meth:`!insert` method on *a* to insert *x* at the appropriate position to maintain sort order. To support inserting records in a table, the *key* function (if any) is applied to *x* for the search step but not for the insertion step. - Keep in mind that the ``O(log n)`` search is dominated by the slow O(n) + Keep in mind that the *O*\ (log *n*) search is dominated by the slow *O*\ (*n*) insertion step. .. versionchanged:: 3.10 @@ -93,13 +93,13 @@ The following functions are provided: entries of *x*. This function first runs :py:func:`~bisect.bisect_right` to locate an insertion point. - Next, it runs the :meth:`insert` method on *a* to insert *x* at the + Next, it runs the :meth:`!insert` method on *a* to insert *x* at the appropriate position to maintain sort order. To support inserting records in a table, the *key* function (if any) is applied to *x* for the search step but not for the insertion step. - Keep in mind that the ``O(log n)`` search is dominated by the slow O(n) + Keep in mind that the *O*\ (log *n*) search is dominated by the slow *O*\ (*n*) insertion step. .. versionchanged:: 3.10 @@ -115,7 +115,7 @@ thoughts in mind: * Bisection is effective for searching ranges of values. For locating specific values, dictionaries are more performant. -* The *insort()* functions are ``O(n)`` because the logarithmic search step +* The *insort()* functions are *O*\ (*n*) because the logarithmic search step is dominated by the linear time insertion step. * The search functions are stateless and discard key function results after diff --git a/Doc/library/calendar.rst b/Doc/library/calendar.rst index 157a7537f97dc6..e699a7284ac802 100644 --- a/Doc/library/calendar.rst +++ b/Doc/library/calendar.rst @@ -196,6 +196,13 @@ interpreted as prescribed by the ISO 8601 standard. Year 0 is 1 BC, year -1 is output (defaulting to the system default encoding). + .. method:: formatmonthname(theyear, themonth, withyear=True) + + Return a month name as an HTML table row. If *withyear* is true the year + will be included in the row, otherwise just the month name will be + used. + + :class:`!HTMLCalendar` has the following attributes you can override to customize the CSS classes used by the calendar: @@ -289,7 +296,7 @@ interpreted as prescribed by the ISO 8601 standard. Year 0 is 1 BC, year -1 is .. note:: - The constructor, :meth:`formatweekday` and :meth:`formatmonthname` methods + The constructor, :meth:`!formatweekday` and :meth:`!formatmonthname` methods of these two classes temporarily change the ``LC_TIME`` locale to the given *locale*. Because the current locale is a process-wide setting, they are not thread-safe. @@ -358,7 +365,7 @@ For simple text calendars this module provides the following functions. .. function:: month(theyear, themonth, w=0, l=0) - Returns a month's calendar in a multi-line string using the :meth:`formatmonth` + Returns a month's calendar in a multi-line string using the :meth:`~TextCalendar.formatmonth` of the :class:`TextCalendar` class. @@ -370,7 +377,7 @@ For simple text calendars this module provides the following functions. .. function:: calendar(year, w=2, l=1, c=6, m=3) Returns a 3-column calendar for an entire year as a multi-line string using - the :meth:`formatyear` of the :class:`TextCalendar` class. + the :meth:`~TextCalendar.formatyear` of the :class:`TextCalendar` class. .. function:: timegm(tuple) @@ -505,7 +512,7 @@ to interactively print a calendar. python -m calendar [-h] [-L LOCALE] [-e ENCODING] [-t {text,html}] [-w WIDTH] [-l LINES] [-s SPACING] [-m MONTHS] [-c CSS] - [year] [month] + [-f FIRST_WEEKDAY] [year] [month] For example, to print a calendar for the year 2000: @@ -579,10 +586,17 @@ The following options are accepted: or as an HTML document. +.. option:: --first-weekday FIRST_WEEKDAY, -f FIRST_WEEKDAY + + The weekday to start each week. + Must be a number between 0 (Monday) and 6 (Sunday). + Defaults to 0. + + .. versionadded:: 3.13 + .. option:: year The year to print the calendar for. - Must be a number between 1 and 9999. Defaults to the current year. diff --git a/Doc/library/cmd.rst b/Doc/library/cmd.rst index a79882ed1cca4f..39ef4b481478d1 100644 --- a/Doc/library/cmd.rst +++ b/Doc/library/cmd.rst @@ -76,27 +76,30 @@ A :class:`Cmd` instance has the following methods: single: ! (exclamation); in a command interpreter An interpreter instance will recognize a command name ``foo`` if and only if it - has a method :meth:`do_foo`. As a special case, a line beginning with the + has a method :meth:`!do_foo`. As a special case, a line beginning with the character ``'?'`` is dispatched to the method :meth:`do_help`. As another special case, a line beginning with the character ``'!'`` is dispatched to the - method :meth:`do_shell` (if such a method is defined). + method :meth:`!do_shell` (if such a method is defined). This method will return when the :meth:`postcmd` method returns a true value. The *stop* argument to :meth:`postcmd` is the return value from the command's corresponding :meth:`!do_\*` method. If completion is enabled, completing commands will be done automatically, and - completing of commands args is done by calling :meth:`complete_foo` with + completing of commands args is done by calling :meth:`!complete_foo` with arguments *text*, *line*, *begidx*, and *endidx*. *text* is the string prefix we are attempting to match: all returned matches must begin with it. *line* is the current input line with leading whitespace removed, *begidx* and *endidx* are the beginning and ending indexes of the prefix text, which could be used to provide different completion depending upon which position the argument is in. - All subclasses of :class:`Cmd` inherit a predefined :meth:`do_help`. This + +.. method:: Cmd.do_help(arg) + + All subclasses of :class:`Cmd` inherit a predefined :meth:`!do_help`. This method, called with an argument ``'bar'``, invokes the corresponding method - :meth:`help_bar`, and if that is not present, prints the docstring of - :meth:`do_bar`, if available. With no argument, :meth:`do_help` lists all + :meth:`!help_bar`, and if that is not present, prints the docstring of + :meth:`!do_bar`, if available. With no argument, :meth:`!do_help` lists all available help topics (that is, all commands with corresponding :meth:`!help_\*` methods or commands that have docstrings), and also lists any undocumented commands. @@ -229,8 +232,8 @@ Instances of :class:`Cmd` subclasses have some public instance variables: .. attribute:: Cmd.use_rawinput A flag, defaulting to true. If true, :meth:`cmdloop` uses :func:`input` to - display a prompt and read the next command; if false, :meth:`sys.stdout.write` - and :meth:`sys.stdin.readline` are used. (This means that by importing + display a prompt and read the next command; if false, :data:`sys.stdout.write() <sys.stdout>` + and :data:`sys.stdin.readline() <sys.stdin>` are used. (This means that by importing :mod:`readline`, on systems that support it, the interpreter will automatically support :program:`Emacs`\ -like line editing and command-history keystrokes.) @@ -249,14 +252,14 @@ This section presents a simple example of how to build a shell around a few of the commands in the :mod:`turtle` module. Basic turtle commands such as :meth:`~turtle.forward` are added to a -:class:`Cmd` subclass with method named :meth:`do_forward`. The argument is +:class:`Cmd` subclass with method named :meth:`!do_forward`. The argument is converted to a number and dispatched to the turtle module. The docstring is used in the help utility provided by the shell. The example also includes a basic record and playback facility implemented with the :meth:`~Cmd.precmd` method which is responsible for converting the input to -lowercase and writing the commands to a file. The :meth:`do_playback` method -reads the file and adds the recorded commands to the :attr:`cmdqueue` for +lowercase and writing the commands to a file. The :meth:`!do_playback` method +reads the file and adds the recorded commands to the :attr:`~Cmd.cmdqueue` for immediate playback:: import cmd, sys diff --git a/Doc/library/collections.abc.rst b/Doc/library/collections.abc.rst index 582bb18f752bd5..7bcaba60c6ddbd 100644 --- a/Doc/library/collections.abc.rst +++ b/Doc/library/collections.abc.rst @@ -136,8 +136,8 @@ ABC Inherits from Abstract Methods Mi :class:`Collection` ``__len__`` ``index``, and ``count`` :class:`MutableSequence` :class:`Sequence` ``__getitem__``, Inherited :class:`Sequence` methods and - ``__setitem__``, ``append``, ``reverse``, ``extend``, ``pop``, - ``__delitem__``, ``remove``, and ``__iadd__`` + ``__setitem__``, ``append``, ``clear``, ``reverse``, ``extend``, + ``__delitem__``, ``pop``, ``remove``, and ``__iadd__`` ``__len__``, ``insert`` diff --git a/Doc/library/collections.rst b/Doc/library/collections.rst index 233b2c6a771f4a..c246173c1bbf53 100644 --- a/Doc/library/collections.rst +++ b/Doc/library/collections.rst @@ -458,10 +458,10 @@ or subtracting from an empty counter. Deques are a generalization of stacks and queues (the name is pronounced "deck" and is short for "double-ended queue"). Deques support thread-safe, memory efficient appends and pops from either side of the deque with approximately the - same O(1) performance in either direction. + same *O*\ (1) performance in either direction. Though :class:`list` objects support similar operations, they are optimized for - fast fixed-length operations and incur O(n) memory movement costs for + fast fixed-length operations and incur *O*\ (*n*) memory movement costs for ``pop(0)`` and ``insert(0, v)`` operations which change both the size and position of the underlying data representation. @@ -585,7 +585,7 @@ or subtracting from an empty counter. In addition to the above, deques support iteration, pickling, ``len(d)``, ``reversed(d)``, ``copy.copy(d)``, ``copy.deepcopy(d)``, membership testing with the :keyword:`in` operator, and subscript references such as ``d[0]`` to access -the first element. Indexed access is O(1) at both ends but slows to O(n) in +the first element. Indexed access is *O*\ (1) at both ends but slows to *O*\ (*n*) in the middle. For fast random access, use lists instead. Starting in version 3.5, deques support ``__add__()``, ``__mul__()``, diff --git a/Doc/library/concurrent.futures.rst b/Doc/library/concurrent.futures.rst index deefb8606ead84..d3c7a40aa9d390 100644 --- a/Doc/library/concurrent.futures.rst +++ b/Doc/library/concurrent.futures.rst @@ -39,14 +39,14 @@ Executor Objects future = executor.submit(pow, 323, 1235) print(future.result()) - .. method:: map(func, *iterables, timeout=None, chunksize=1) + .. method:: map(fn, *iterables, timeout=None, chunksize=1) - Similar to :func:`map(func, *iterables) <map>` except: + Similar to :func:`map(fn, *iterables) <map>` except: * the *iterables* are collected immediately rather than lazily; - * *func* is executed asynchronously and several calls to - *func* may be made concurrently. + * *fn* is executed asynchronously and several calls to + *fn* may be made concurrently. The returned iterator raises a :exc:`TimeoutError` if :meth:`~iterator.__next__` is called and the result isn't available @@ -54,7 +54,7 @@ Executor Objects *timeout* can be an int or a float. If *timeout* is not specified or ``None``, there is no limit to the wait time. - If a *func* call raises an exception, then that exception will be + If a *fn* call raises an exception, then that exception will be raised when its value is retrieved from the iterator. When using :class:`ProcessPoolExecutor`, this method chops *iterables* @@ -171,8 +171,8 @@ And:: should be higher than the number of workers for :class:`ProcessPoolExecutor`. - .. versionadded:: 3.6 - The *thread_name_prefix* argument was added to allow users to + .. versionchanged:: 3.6 + Added the *thread_name_prefix* parameter to allow users to control the :class:`threading.Thread` names for worker threads created by the pool for easier debugging. @@ -275,7 +275,8 @@ to a :class:`ProcessPoolExecutor` will result in deadlock. .. versionchanged:: 3.3 When one of the worker processes terminates abruptly, a - :exc:`BrokenProcessPool` error is now raised. Previously, behaviour + :exc:`~concurrent.futures.process.BrokenProcessPool` error is now raised. + Previously, behaviour was undefined but operations on the executor or its futures would often freeze or deadlock. @@ -493,23 +494,22 @@ Module Functions *return_when* indicates when this function should return. It must be one of the following constants: - .. tabularcolumns:: |l|L| - - +-----------------------------+----------------------------------------+ - | Constant | Description | - +=============================+========================================+ - | :const:`FIRST_COMPLETED` | The function will return when any | - | | future finishes or is cancelled. | - +-----------------------------+----------------------------------------+ - | :const:`FIRST_EXCEPTION` | The function will return when any | - | | future finishes by raising an | - | | exception. If no future raises an | - | | exception then it is equivalent to | - | | :const:`ALL_COMPLETED`. | - +-----------------------------+----------------------------------------+ - | :const:`ALL_COMPLETED` | The function will return when all | - | | futures finish or are cancelled. | - +-----------------------------+----------------------------------------+ + .. list-table:: + :header-rows: 1 + + * - Constant + - Description + + * - .. data:: FIRST_COMPLETED + - The function will return when any future finishes or is cancelled. + + * - .. data:: FIRST_EXCEPTION + - The function will return when any future finishes by raising an + exception. If no future raises an exception + then it is equivalent to :const:`ALL_COMPLETED`. + + * - .. data:: ALL_COMPLETED + - The function will return when all futures finish or are cancelled. .. function:: as_completed(fs, timeout=None) @@ -570,7 +570,8 @@ Exception classes .. exception:: BrokenThreadPool Derived from :exc:`~concurrent.futures.BrokenExecutor`, this exception - class is raised when one of the workers of a :class:`ThreadPoolExecutor` + class is raised when one of the workers + of a :class:`~concurrent.futures.ThreadPoolExecutor` has failed initializing. .. versionadded:: 3.7 @@ -581,7 +582,8 @@ Exception classes Derived from :exc:`~concurrent.futures.BrokenExecutor` (formerly :exc:`RuntimeError`), this exception class is raised when one of the - workers of a :class:`ProcessPoolExecutor` has terminated in a non-clean + workers of a :class:`~concurrent.futures.ProcessPoolExecutor` + has terminated in a non-clean fashion (for example, if it was killed from the outside). .. versionadded:: 3.3 diff --git a/Doc/library/configparser.rst b/Doc/library/configparser.rst index 12eee47613d186..18e5bc20f3f690 100644 --- a/Doc/library/configparser.rst +++ b/Doc/library/configparser.rst @@ -208,7 +208,7 @@ converters and customize the provided ones. [1]_ Fallback Values --------------- -As with a dictionary, you can use a section's :meth:`get` method to +As with a dictionary, you can use a section's :meth:`~ConfigParser.get` method to provide fallback values: .. doctest:: @@ -232,7 +232,7 @@ even if we specify a fallback: >>> topsecret.get('CompressionLevel', '3') '9' -One more thing to be aware of is that the parser-level :meth:`get` method +One more thing to be aware of is that the parser-level :meth:`~ConfigParser.get` method provides a custom, more complex interface, maintained for backwards compatibility. When using this method, a fallback value can be provided via the ``fallback`` keyword-only argument: @@ -271,7 +271,7 @@ out. Values can also span multiple lines, as long as they are indented deeper than the first line of the value. Depending on the parser's mode, blank lines may be treated as parts of multiline values or ignored. -By default, a valid section name can be any string that does not contain '\\n' or ']'. +By default, a valid section name can be any string that does not contain '\\n'. To change this, see :attr:`ConfigParser.SECTCRE`. Configuration files may include comments, prefixed by specific @@ -481,7 +481,7 @@ historical background and it's very likely that you will want to customize some of the features. The most common way to change the way a specific config parser works is to use -the :meth:`__init__` options: +the :meth:`!__init__` options: * *defaults*, default value: ``None`` @@ -491,7 +491,7 @@ the :meth:`__init__` options: the documented default. Hint: if you want to specify default values for a specific section, use - :meth:`read_dict` before you read the actual file. + :meth:`~ConfigParser.read_dict` before you read the actual file. * *dict_type*, default value: :class:`dict` @@ -635,8 +635,8 @@ the :meth:`__init__` options: * *strict*, default value: ``True`` When set to ``True``, the parser will not allow for any section or option - duplicates while reading from a single source (using :meth:`read_file`, - :meth:`read_string` or :meth:`read_dict`). It is recommended to use strict + duplicates while reading from a single source (using :meth:`~ConfigParser.read_file`, + :meth:`~ConfigParser.read_string` or :meth:`~ConfigParser.read_dict`). It is recommended to use strict parsers in new applications. .. versionchanged:: 3.2 @@ -697,7 +697,7 @@ the :meth:`__init__` options: desirable, users may define them in a subclass or pass a dictionary where each key is a name of the converter and each value is a callable implementing said conversion. For instance, passing ``{'decimal': decimal.Decimal}`` would add - :meth:`getdecimal` on both the parser object and all section proxies. In + :meth:`!getdecimal` on both the parser object and all section proxies. In other words, it will be possible to write both ``parser_instance.getdecimal('section', 'key', fallback=0)`` and ``parser_instance['section'].getdecimal('key', 0)``. @@ -1045,14 +1045,14 @@ ConfigParser Objects config.read(['site.cfg', os.path.expanduser('~/.myapp.cfg')], encoding='cp1250') - .. versionadded:: 3.2 - The *encoding* parameter. Previously, all files were read using the - default encoding for :func:`open`. + .. versionchanged:: 3.2 + Added the *encoding* parameter. + Previously, all files were read using the default encoding for :func:`open`. - .. versionadded:: 3.6.1 + .. versionchanged:: 3.6.1 The *filenames* parameter accepts a :term:`path-like object`. - .. versionadded:: 3.7 + .. versionchanged:: 3.7 The *filenames* parameter accepts a :class:`bytes` object. @@ -1062,11 +1062,11 @@ ConfigParser Objects yielding Unicode strings (for example files opened in text mode). Optional argument *source* specifies the name of the file being read. If - not given and *f* has a :attr:`name` attribute, that is used for + not given and *f* has a :attr:`!name` attribute, that is used for *source*; the default is ``'<???>'``. .. versionadded:: 3.2 - Replaces :meth:`readfp`. + Replaces :meth:`!readfp`. .. method:: read_string(string, source='<string>') @@ -1214,7 +1214,7 @@ ConfigParser Objects .. data:: MAX_INTERPOLATION_DEPTH - The maximum depth for recursive interpolation for :meth:`get` when the *raw* + The maximum depth for recursive interpolation for :meth:`~configparser.ConfigParser.get` when the *raw* parameter is false. This is relevant only when the default *interpolation* is used. @@ -1287,13 +1287,13 @@ Exceptions .. exception:: DuplicateSectionError - Exception raised if :meth:`add_section` is called with the name of a section + Exception raised if :meth:`~ConfigParser.add_section` is called with the name of a section that is already present or in strict parsers when a section if found more than once in a single input file, string or dictionary. - .. versionadded:: 3.2 - Optional ``source`` and ``lineno`` attributes and arguments to - :meth:`__init__` were added. + .. versionchanged:: 3.2 + Added the optional *source* and *lineno* attributes and parameters to + :meth:`!__init__`. .. exception:: DuplicateOptionError @@ -1345,9 +1345,9 @@ Exceptions Exception raised when errors occur attempting to parse a file. -.. versionchanged:: 3.12 - The ``filename`` attribute and :meth:`__init__` constructor argument were - removed. They have been available using the name ``source`` since 3.2. + .. versionchanged:: 3.12 + The ``filename`` attribute and :meth:`!__init__` constructor argument were + removed. They have been available using the name ``source`` since 3.2. .. rubric:: Footnotes diff --git a/Doc/library/contextlib.rst b/Doc/library/contextlib.rst index aab319cbe7405e..73e53aec9cbf1c 100644 --- a/Doc/library/contextlib.rst +++ b/Doc/library/contextlib.rst @@ -182,6 +182,14 @@ Functions and classes provided: without needing to explicitly close ``page``. Even if an error occurs, ``page.close()`` will be called when the :keyword:`with` block is exited. + .. note:: + + Most types managing resources support the :term:`context manager` protocol, + which closes *thing* on leaving the :keyword:`with` statement. + As such, :func:`!closing` is most useful for third party types that don't + support context managers. + This example is purely for illustration purposes, + as :func:`~urllib.request.urlopen` would normally be used in a context manager. .. function:: aclosing(thing) diff --git a/Doc/library/contextvars.rst b/Doc/library/contextvars.rst index 0ac2f3d85749b7..647832447de946 100644 --- a/Doc/library/contextvars.rst +++ b/Doc/library/contextvars.rst @@ -131,7 +131,7 @@ Manual Context Management ctx: Context = copy_context() print(list(ctx.items())) - The function has an O(1) complexity, i.e. works equally fast for + The function has an *O*\ (1) complexity, i.e. works equally fast for contexts with a few context variables and for contexts that have a lot of them. diff --git a/Doc/library/csv.rst b/Doc/library/csv.rst index 7a5589e68b3052..4ee7820585d3a2 100644 --- a/Doc/library/csv.rst +++ b/Doc/library/csv.rst @@ -4,7 +4,7 @@ .. module:: csv :synopsis: Write and read tabular data to and from delimited files. -.. sectionauthor:: Skip Montanaro <skip@pobox.com> +.. sectionauthor:: Skip Montanaro <skip.montanaro@gmail.com> **Source code:** :source:`Lib/csv.py` @@ -88,7 +88,7 @@ The :mod:`csv` module defines the following functions: Return a writer object responsible for converting the user's data into delimited strings on the given file-like object. *csvfile* can be any object with a - :func:`write` method. If *csvfile* is a file object, it should be opened with + :meth:`~io.TextIOBase.write` method. If *csvfile* is a file object, it should be opened with ``newline=''`` [1]_. An optional *dialect* parameter can be given which is used to define a set of parameters specific to a particular CSV dialect. It may be an instance of a subclass of the @@ -197,10 +197,10 @@ The :mod:`csv` module defines the following classes: Create an object which operates like a regular writer but maps dictionaries onto output rows. The *fieldnames* parameter is a :mod:`sequence <collections.abc>` of keys that identify the order in which values in the - dictionary passed to the :meth:`writerow` method are written to file + dictionary passed to the :meth:`~csvwriter.writerow` method are written to file *f*. The optional *restval* parameter specifies the value to be written if the dictionary is missing a key in *fieldnames*. If the - dictionary passed to the :meth:`writerow` method contains a key not found in + dictionary passed to the :meth:`~csvwriter.writerow` method contains a key not found in *fieldnames*, the optional *extrasaction* parameter indicates what action to take. If it is set to ``'raise'``, the default value, a :exc:`ValueError` @@ -244,7 +244,6 @@ The :mod:`csv` module defines the following classes: with open('students.csv', 'w', newline='') as csvfile: writer = csv.writer(csvfile, dialect='unix') - ^^^^^^^^^^^^^^ .. class:: excel() @@ -351,6 +350,8 @@ The :mod:`csv` module defines the following constants: Instructs :class:`reader` objects to interpret an empty (unquoted) field as None and to otherwise behave as :data:`QUOTE_ALL`. + .. versionadded:: 3.12 + .. data:: QUOTE_STRINGS Instructs :class:`writer` objects to always place quotes around fields @@ -360,6 +361,8 @@ The :mod:`csv` module defines the following constants: Instructs :class:`reader` objects to interpret an empty (unquoted) string as ``None`` and to otherwise behave as :data:`QUOTE_NONNUMERIC`. + .. versionadded:: 3.12 + The :mod:`csv` module defines the following exception: @@ -374,8 +377,8 @@ Dialects and Formatting Parameters To make it easier to specify the format of input and output records, specific formatting parameters are grouped together into dialects. A dialect is a -subclass of the :class:`Dialect` class having a set of specific methods and a -single :meth:`validate` method. When creating :class:`reader` or +subclass of the :class:`Dialect` class containing various attributes +describing the format of the CSV file. When creating :class:`reader` or :class:`writer` objects, the programmer can specify a string or a subclass of the :class:`Dialect` class as the dialect parameter. In addition to, or instead of, the *dialect* parameter, the programmer can also specify individual @@ -492,9 +495,9 @@ DictReader objects have the following public attribute: Writer Objects -------------- -:class:`Writer` objects (:class:`DictWriter` instances and objects returned by +:class:`writer` objects (:class:`DictWriter` instances and objects returned by the :func:`writer` function) have the following public methods. A *row* must be -an iterable of strings or numbers for :class:`Writer` objects and a dictionary +an iterable of strings or numbers for :class:`writer` objects and a dictionary mapping fieldnames to strings or numbers (by passing them through :func:`str` first) for :class:`DictWriter` objects. Note that complex numbers are written out surrounded by parens. This may cause some problems for other programs which diff --git a/Doc/library/dataclasses.rst b/Doc/library/dataclasses.rst index bbbbcb00d8fef8..4ada69d63abada 100644 --- a/Doc/library/dataclasses.rst +++ b/Doc/library/dataclasses.rst @@ -141,7 +141,7 @@ Module contents then :func:`dataclass` *may* add an implicit :meth:`~object.__hash__` method. Although not recommended, you can force :func:`dataclass` to create a :meth:`~object.__hash__` method with ``unsafe_hash=True``. This might be the case - if your class is logically immutable but can nonetheless be mutated. + if your class is logically immutable but can still be mutated. This is a specialized use case and should be considered carefully. Here are the rules governing implicit creation of a :meth:`~object.__hash__` @@ -536,10 +536,10 @@ class :meth:`~object.__init__` methods. If the base class has an :meth:`~object. that has to be called, it is common to call this method in a :meth:`__post_init__` method:: - @dataclass class Rectangle: - height: float - width: float + def __init__(self, height, width): + self.height = height + self.width = width @dataclass class Square(Rectangle): diff --git a/Doc/library/datetime.rst b/Doc/library/datetime.rst index 686b37754368c1..930af6cbbe9e8d 100644 --- a/Doc/library/datetime.rst +++ b/Doc/library/datetime.rst @@ -14,7 +14,7 @@ .. XXX what order should the types be discussed in? -The :mod:`datetime` module supplies classes for manipulating dates and times. +The :mod:`!datetime` module supplies classes for manipulating dates and times. While date and time arithmetic is supported, the focus of the implementation is on efficient attribute extraction for output formatting and manipulation. @@ -70,7 +70,7 @@ These :class:`tzinfo` objects capture information about the offset from UTC time, the time zone name, and whether daylight saving time is in effect. Only one concrete :class:`tzinfo` class, the :class:`timezone` class, is -supplied by the :mod:`datetime` module. The :class:`timezone` class can +supplied by the :mod:`!datetime` module. The :class:`timezone` class can represent simple timezones with fixed offsets from UTC, such as UTC itself or North American EST and EDT timezones. Supporting timezones at deeper levels of detail is up to the application. The rules for time adjustment across the @@ -80,7 +80,7 @@ standard suitable for every application aside from UTC. Constants --------- -The :mod:`datetime` module exports the following constants: +The :mod:`!datetime` module exports the following constants: .. data:: MINYEAR @@ -130,8 +130,8 @@ Available Types .. class:: timedelta :noindex: - A duration expressing the difference between two :class:`date`, :class:`.time`, - or :class:`.datetime` instances to microsecond resolution. + A duration expressing the difference between two :class:`.datetime` + or :class:`date` instances to microsecond resolution. .. class:: tzinfo @@ -203,7 +203,7 @@ objects. -------------------------- A :class:`timedelta` object represents a duration, the difference between two -dates or times. +:class:`.datetime` or :class:`date` instances. .. class:: timedelta(days=0, seconds=0, microseconds=0, milliseconds=0, minutes=0, hours=0, weeks=0) @@ -400,30 +400,7 @@ objects (see below). the :func:`divmod` function. True division and multiplication of a :class:`timedelta` object by a :class:`float` object are now supported. - -Comparisons of :class:`timedelta` objects are supported, with some caveats. - -The comparisons ``==`` or ``!=`` *always* return a :class:`bool`, no matter -the type of the compared object:: - - >>> from datetime import timedelta - >>> delta1 = timedelta(seconds=57) - >>> delta2 = timedelta(hours=25, seconds=2) - >>> delta2 != delta1 - True - >>> delta2 == 5 - False - -For all other comparisons (such as ``<`` and ``>``), when a :class:`timedelta` -object is compared to an object of a different type, :exc:`TypeError` -is raised:: - - >>> delta2 > delta1 - True - >>> delta2 > 5 - Traceback (most recent call last): - File "<stdin>", line 1, in <module> - TypeError: '>' not supported between instances of 'datetime.timedelta' and 'int' +:class:`timedelta` objects support equality and order comparisons. In Boolean contexts, a :class:`timedelta` object is considered to be true if and only if it isn't equal to ``timedelta(0)``. @@ -536,7 +513,15 @@ Other constructors, all class methods: .. classmethod:: date.fromisoformat(date_string) Return a :class:`date` corresponding to a *date_string* given in any valid - ISO 8601 format, except ordinal dates (e.g. ``YYYY-DDD``):: + ISO 8601 format, with the following exceptions: + + 1. Reduced precision dates are not currently supported (``YYYY-MM``, + ``YYYY``). + 2. Extended date representations are not currently supported + (``±YYYYYY-MM-DD``). + 3. Ordinal dates are not currently supported (``YYYY-OOO``). + + Examples:: >>> from datetime import date >>> date.fromisoformat('2019-12-04') @@ -606,8 +591,13 @@ Supported operations: +-------------------------------+----------------------------------------------+ | ``timedelta = date1 - date2`` | \(3) | +-------------------------------+----------------------------------------------+ -| ``date1 < date2`` | *date1* is considered less than *date2* when | -| | *date1* precedes *date2* in time. (4) | +| | ``date1 == date2`` | Equality comparison. (4) | +| | ``date1 != date2`` | | ++-------------------------------+----------------------------------------------+ +| | ``date1 < date2`` | Order comparison. (5) | +| | ``date1 > date2`` | | +| | ``date1 <= date2`` | | +| | ``date1 >= date2`` | | +-------------------------------+----------------------------------------------+ Notes: @@ -627,15 +617,12 @@ Notes: timedelta.microseconds are 0, and date2 + timedelta == date1 after. (4) + :class:`date` objects are equal if they represent the same date. + +(5) + *date1* is considered less than *date2* when *date1* precedes *date2* in time. In other words, ``date1 < date2`` if and only if ``date1.toordinal() < - date2.toordinal()``. Date comparison raises :exc:`TypeError` if - the other comparand isn't also a :class:`date` object. However, - ``NotImplemented`` is returned instead if the other comparand has a - :meth:`timetuple` attribute. This hook gives other kinds of date objects a - chance at implementing mixed-type comparison. If not, when a :class:`date` - object is compared to an object of a different type, :exc:`TypeError` is raised - unless the comparison is ``==`` or ``!=``. The latter cases return - :const:`False` or :const:`True`, respectively. + date2.toordinal()``. In Boolean contexts, all :class:`date` objects are considered to be true. @@ -859,8 +846,8 @@ Constructor: If an argument outside those ranges is given, :exc:`ValueError` is raised. - .. versionadded:: 3.6 - Added the ``fold`` argument. + .. versionchanged:: 3.6 + Added the *fold* parameter. Other constructors, all class methods: @@ -1017,8 +1004,12 @@ Other constructors, all class methods: 1. Time zone offsets may have fractional seconds. 2. The ``T`` separator may be replaced by any single unicode character. - 3. Ordinal dates are not currently supported. - 4. Fractional hours and minutes are not supported. + 3. Fractional hours and minutes are not supported. + 4. Reduced precision dates are not currently supported (``YYYY-MM``, + ``YYYY``). + 5. Extended date representations are not currently supported + (``±YYYYYY-MM-DD``). + 6. Ordinal dates are not currently supported (``YYYY-OOO``). Examples:: @@ -1158,8 +1149,13 @@ Supported operations: +---------------------------------------+--------------------------------+ | ``timedelta = datetime1 - datetime2`` | \(3) | +---------------------------------------+--------------------------------+ -| ``datetime1 < datetime2`` | Compares :class:`.datetime` to | -| | :class:`.datetime`. (4) | +| | ``datetime1 == datetime2`` | Equality comparison. (4) | +| | ``datetime1 != datetime2`` | | ++---------------------------------------+--------------------------------+ +| | ``datetime1 < datetime2`` | Order comparison. (5) | +| | ``datetime1 > datetime2`` | | +| | ``datetime1 <= datetime2`` | | +| | ``datetime1 >= datetime2`` | | +---------------------------------------+--------------------------------+ (1) @@ -1187,39 +1183,40 @@ Supported operations: are done in this case. If both are aware and have different :attr:`~.datetime.tzinfo` attributes, ``a-b`` acts - as if *a* and *b* were first converted to naive UTC datetimes first. The + as if *a* and *b* were first converted to naive UTC datetimes. The result is ``(a.replace(tzinfo=None) - a.utcoffset()) - (b.replace(tzinfo=None) - b.utcoffset())`` except that the implementation never overflows. (4) - *datetime1* is considered less than *datetime2* when *datetime1* precedes - *datetime2* in time. + :class:`.datetime` objects are equal if they represent the same date + and time, taking into account the time zone. - If one comparand is naive and the other is aware, :exc:`TypeError` - is raised if an order comparison is attempted. For equality - comparisons, naive instances are never equal to aware instances. + Naive and aware :class:`!datetime` objects are never equal. + :class:`!datetime` objects are never equal to :class:`date` objects + that are not also :class:`!datetime` instances, even if they represent + the same date. - If both comparands are aware, and have the same :attr:`~.datetime.tzinfo` attribute, the - common :attr:`~.datetime.tzinfo` attribute is ignored and the base datetimes are - compared. If both comparands are aware and have different :attr:`~.datetime.tzinfo` - attributes, the comparands are first adjusted by subtracting their UTC - offsets (obtained from ``self.utcoffset()``). + If both comparands are aware and have different :attr:`~.datetime.tzinfo` + attributes, the comparison acts as comparands were first converted to UTC + datetimes except that the implementation never overflows. + :class:`!datetime` instances in a repeated interval are never equal to + :class:`!datetime` instances in other time zone. - .. versionchanged:: 3.3 - Equality comparisons between aware and naive :class:`.datetime` - instances don't raise :exc:`TypeError`. +(5) + *datetime1* is considered less than *datetime2* when *datetime1* precedes + *datetime2* in time, taking into account the time zone. - .. note:: + Order comparison between naive and aware :class:`.datetime` objects, + as well as a :class:`!datetime` object and a :class:`!date` object + that is not also a :class:`!datetime` instance, raises :exc:`TypeError`. - In order to stop comparison from falling back to the default scheme of comparing - object addresses, datetime comparison normally raises :exc:`TypeError` if the - other comparand isn't also a :class:`.datetime` object. However, - ``NotImplemented`` is returned instead if the other comparand has a - :meth:`timetuple` attribute. This hook gives other kinds of date objects a - chance at implementing mixed-type comparison. If not, when a :class:`.datetime` - object is compared to an object of a different type, :exc:`TypeError` is raised - unless the comparison is ``==`` or ``!=``. The latter cases return - :const:`False` or :const:`True`, respectively. + If both comparands are aware and have different :attr:`~.datetime.tzinfo` + attributes, the comparison acts as comparands were first converted to UTC + datetimes except that the implementation never overflows. + +.. versionchanged:: 3.3 + Equality comparisons between aware and naive :class:`.datetime` + instances don't raise :exc:`TypeError`. Instance methods: @@ -1255,11 +1252,11 @@ Instance methods: ``tzinfo=None`` can be specified to create a naive datetime from an aware datetime with no conversion of date and time data. - :class:`datetime` objects are also supported by generic function + :class:`.datetime` objects are also supported by generic function :func:`copy.replace`. - .. versionadded:: 3.6 - Added the ``fold`` argument. + .. versionchanged:: 3.6 + Added the *fold* parameter. .. method:: datetime.astimezone(tz=None) @@ -1347,22 +1344,22 @@ Instance methods: where ``yday = d.toordinal() - date(d.year, 1, 1).toordinal() + 1`` is the day number within the current year starting with ``1`` for January - 1st. The :attr:`tm_isdst` flag of the result is set according to the + 1st. The :attr:`~time.struct_time.tm_isdst` flag of the result is set according to the :meth:`dst` method: :attr:`.tzinfo` is ``None`` or :meth:`dst` returns - ``None``, :attr:`tm_isdst` is set to ``-1``; else if :meth:`dst` returns a - non-zero value, :attr:`tm_isdst` is set to ``1``; else :attr:`tm_isdst` is + ``None``, :attr:`!tm_isdst` is set to ``-1``; else if :meth:`dst` returns a + non-zero value, :attr:`!tm_isdst` is set to ``1``; else :attr:`!tm_isdst` is set to ``0``. .. method:: datetime.utctimetuple() If :class:`.datetime` instance *d* is naive, this is the same as - ``d.timetuple()`` except that :attr:`tm_isdst` is forced to 0 regardless of what + ``d.timetuple()`` except that :attr:`~.time.struct_time.tm_isdst` is forced to 0 regardless of what ``d.dst()`` returns. DST is never in effect for a UTC time. If *d* is aware, *d* is normalized to UTC time, by subtracting ``d.utcoffset()``, and a :class:`time.struct_time` for the - normalized time is returned. :attr:`tm_isdst` is forced to 0. Note + normalized time is returned. :attr:`!tm_isdst` is forced to 0. Note that an :exc:`OverflowError` may be raised if *d*.year was ``MINYEAR`` or ``MAXYEAR`` and UTC adjustment spills over a year boundary. @@ -1502,8 +1499,8 @@ Instance methods: >>> dt.isoformat(timespec='microseconds') '2015-01-01T12:30:59.000000' - .. versionadded:: 3.6 - Added the *timespec* argument. + .. versionchanged:: 3.6 + Added the *timespec* parameter. .. method:: datetime.__str__() @@ -1550,7 +1547,7 @@ Instance methods: Examples of Usage: :class:`.datetime` ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ -Examples of working with :class:`~datetime.datetime` objects: +Examples of working with :class:`.datetime` objects: .. doctest:: @@ -1678,7 +1675,7 @@ Usage of ``KabulTz`` from above:: :class:`.time` Objects ---------------------- -A :class:`time` object represents a (local) time of day, independent of any particular +A :class:`.time` object represents a (local) time of day, independent of any particular day, and subject to adjustment via a :class:`tzinfo` object. .. class:: time(hour=0, minute=0, second=0, microsecond=0, tzinfo=None, *, fold=0) @@ -1754,24 +1751,21 @@ Instance attributes (read-only): .. versionadded:: 3.6 -:class:`.time` objects support comparison of :class:`.time` to :class:`.time`, -where *a* is considered less -than *b* when *a* precedes *b* in time. If one comparand is naive and the other -is aware, :exc:`TypeError` is raised if an order comparison is attempted. For equality -comparisons, naive instances are never equal to aware instances. +:class:`.time` objects support equality and order comparisons, +where *a* is considered less than *b* when *a* precedes *b* in time. + +Naive and aware :class:`!time` objects are never equal. +Order comparison between naive and aware :class:`!time` objects raises +:exc:`TypeError`. If both comparands are aware, and have -the same :attr:`~time.tzinfo` attribute, the common :attr:`~time.tzinfo` attribute is +the same :attr:`~.time.tzinfo` attribute, the common :attr:`!tzinfo` attribute is ignored and the base times are compared. If both comparands are aware and -have different :attr:`~time.tzinfo` attributes, the comparands are first adjusted by -subtracting their UTC offsets (obtained from ``self.utcoffset()``). In order -to stop mixed-type comparisons from falling back to the default comparison by -object address, when a :class:`.time` object is compared to an object of a -different type, :exc:`TypeError` is raised unless the comparison is ``==`` or -``!=``. The latter cases return :const:`False` or :const:`True`, respectively. +have different :attr:`!tzinfo` attributes, the comparands are first adjusted by +subtracting their UTC offsets (obtained from ``self.utcoffset()``). .. versionchanged:: 3.3 - Equality comparisons between aware and naive :class:`~datetime.time` instances + Equality comparisons between aware and naive :class:`.time` instances don't raise :exc:`TypeError`. In Boolean contexts, a :class:`.time` object is always considered to be true. @@ -1799,6 +1793,8 @@ Other constructor: Examples:: + .. doctest:: + >>> from datetime import time >>> time.fromisoformat('04:23:01') datetime.time(4, 23, 1) @@ -1808,7 +1804,7 @@ Other constructor: datetime.time(4, 23, 1) >>> time.fromisoformat('04:23:01.000384') datetime.time(4, 23, 1, 384) - >>> time.fromisoformat('04:23:01,000') + >>> time.fromisoformat('04:23:01,000384') datetime.time(4, 23, 1, 384) >>> time.fromisoformat('04:23:01+04:00') datetime.time(4, 23, 1, tzinfo=datetime.timezone(datetime.timedelta(seconds=14400))) @@ -1834,11 +1830,11 @@ Instance methods: ``tzinfo=None`` can be specified to create a naive :class:`.time` from an aware :class:`.time`, without conversion of the time data. - :class:`time` objects are also supported by generic function + :class:`.time` objects are also supported by generic function :func:`copy.replace`. - .. versionadded:: 3.6 - Added the ``fold`` argument. + .. versionchanged:: 3.6 + Added the *fold* parameter. .. method:: time.isoformat(timespec='auto') @@ -1881,8 +1877,8 @@ Instance methods: >>> dt.isoformat(timespec='auto') '12:34:56' - .. versionadded:: 3.6 - Added the *timespec* argument. + .. versionchanged:: 3.6 + Added the *timespec* parameter. .. method:: time.__str__() @@ -1979,7 +1975,7 @@ Examples of working with a :class:`.time` object:: You need to derive a concrete subclass, and (at least) supply implementations of the standard :class:`tzinfo` methods needed by the - :class:`.datetime` methods you use. The :mod:`datetime` module provides + :class:`.datetime` methods you use. The :mod:`!datetime` module provides :class:`timezone`, a simple concrete subclass of :class:`tzinfo` which can represent timezones with fixed offset from UTC such as UTC itself or North American EST and EDT. @@ -1992,7 +1988,7 @@ Examples of working with a :class:`.time` object:: A concrete subclass of :class:`tzinfo` may need to implement the following methods. Exactly which methods are needed depends on the uses made of aware - :mod:`datetime` objects. If in doubt, simply implement all of them. + :mod:`!datetime` objects. If in doubt, simply implement all of them. .. method:: tzinfo.utcoffset(dt) @@ -2033,7 +2029,7 @@ Examples of working with a :class:`.time` object:: already been added to the UTC offset returned by :meth:`utcoffset`, so there's no need to consult :meth:`dst` unless you're interested in obtaining DST info separately. For example, :meth:`datetime.timetuple` calls its :attr:`~.datetime.tzinfo` - attribute's :meth:`dst` method to determine how the :attr:`tm_isdst` flag + attribute's :meth:`dst` method to determine how the :attr:`~time.struct_time.tm_isdst` flag should be set, and :meth:`tzinfo.fromutc` calls :meth:`dst` to account for DST changes when crossing time zones. @@ -2049,7 +2045,7 @@ Examples of working with a :class:`.time` object:: relies on this, but cannot detect violations; it's the programmer's responsibility to ensure it. If a :class:`tzinfo` subclass cannot guarantee this, it may be able to override the default implementation of - :meth:`tzinfo.fromutc` to work correctly with :meth:`astimezone` regardless. + :meth:`tzinfo.fromutc` to work correctly with :meth:`~.datetime.astimezone` regardless. Most implementations of :meth:`dst` will probably look like one of these two:: @@ -2078,7 +2074,7 @@ Examples of working with a :class:`.time` object:: .. method:: tzinfo.tzname(dt) Return the time zone name corresponding to the :class:`.datetime` object *dt*, as - a string. Nothing about string names is defined by the :mod:`datetime` module, + a string. Nothing about string names is defined by the :mod:`!datetime` module, and there's no requirement that it mean anything in particular. For example, "GMT", "UTC", "-500", "-5:00", "EDT", "US/Eastern", "America/New York" are all valid replies. Return ``None`` if a string name isn't known. Note that this is @@ -2126,7 +2122,7 @@ There is one more :class:`tzinfo` method that a subclass may wish to override: different years. An example of a time zone the default :meth:`fromutc` implementation may not handle correctly in all cases is one where the standard offset (from UTC) depends on the specific date and time passed, which can happen - for political reasons. The default implementations of :meth:`astimezone` and + for political reasons. The default implementations of :meth:`~.datetime.astimezone` and :meth:`fromutc` may not produce the result you want if the result is one of the hours straddling the moment the standard offset changes. @@ -2192,10 +2188,10 @@ hour that can't be spelled unambiguously in local wall time: the last hour of daylight time. In Eastern, that's times of the form 5:MM UTC on the day daylight time ends. The local wall clock leaps from 1:59 (daylight time) back to 1:00 (standard time) again. Local times of the form 1:MM are ambiguous. -:meth:`astimezone` mimics the local clock's behavior by mapping two adjacent UTC +:meth:`~.datetime.astimezone` mimics the local clock's behavior by mapping two adjacent UTC hours into the same local hour then. In the Eastern example, UTC times of the form 5:MM and 6:MM both map to 1:MM when converted to Eastern, but earlier times -have the :attr:`~datetime.fold` attribute set to 0 and the later times have it set to 1. +have the :attr:`~.datetime.fold` attribute set to 0 and the later times have it set to 1. For example, at the Fall back transition of 2016, we get:: >>> u0 = datetime(2016, 11, 6, 4, tzinfo=timezone.utc) @@ -2210,10 +2206,10 @@ For example, at the Fall back transition of 2016, we get:: 07:00:00 UTC = 02:00:00 EST 0 Note that the :class:`.datetime` instances that differ only by the value of the -:attr:`~datetime.fold` attribute are considered equal in comparisons. +:attr:`~.datetime.fold` attribute are considered equal in comparisons. Applications that can't bear wall-time ambiguities should explicitly check the -value of the :attr:`~datetime.fold` attribute or avoid using hybrid +value of the :attr:`~.datetime.fold` attribute or avoid using hybrid :class:`tzinfo` subclasses; there are no ambiguities when using :class:`timezone`, or any other fixed-offset :class:`tzinfo` subclass (such as a class representing only EST (fixed offset -5 hours), or only EDT (fixed offset -4 hours)). @@ -2221,7 +2217,7 @@ only EST (fixed offset -5 hours), or only EDT (fixed offset -4 hours)). .. seealso:: :mod:`zoneinfo` - The :mod:`datetime` module has a basic :class:`timezone` class (for + The :mod:`!datetime` module has a basic :class:`timezone` class (for handling arbitrary fixed offsets from UTC) and its :attr:`timezone.utc` attribute (a UTC timezone instance). @@ -2239,7 +2235,7 @@ only EST (fixed offset -5 hours), or only EDT (fixed offset -4 hours)). .. _datetime-timezone: :class:`timezone` Objects --------------------------- +------------------------- The :class:`timezone` class is a subclass of :class:`tzinfo`, each instance of which represents a timezone defined by a fixed offset from @@ -2314,8 +2310,8 @@ Class attributes: .. _strftime-strptime-behavior: -:meth:`strftime` and :meth:`strptime` Behavior ----------------------------------------------- +:meth:`~.datetime.strftime` and :meth:`~.datetime.strptime` Behavior +-------------------------------------------------------------------- :class:`date`, :class:`.datetime`, and :class:`.time` objects all support a ``strftime(format)`` method, to create a string representing the time under the @@ -2325,8 +2321,8 @@ Conversely, the :meth:`datetime.strptime` class method creates a :class:`.datetime` object from a string representing a date and time and a corresponding format string. -The table below provides a high-level comparison of :meth:`strftime` -versus :meth:`strptime`: +The table below provides a high-level comparison of :meth:`~.datetime.strftime` +versus :meth:`~.datetime.strptime`: +----------------+--------------------------------------------------------+------------------------------------------------------------------------------+ | | ``strftime`` | ``strptime`` | @@ -2343,8 +2339,8 @@ versus :meth:`strptime`: .. _format-codes: -:meth:`strftime` and :meth:`strptime` Format Codes -^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ +:meth:`~.datetime.strftime` and :meth:`~.datetime.strptime` Format Codes +^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ These methods accept format codes that can be used to parse and format dates:: @@ -2483,13 +2479,13 @@ convenience. These parameters all correspond to ISO 8601 date values. | | naive). | -03:07:12.345216 | | +-----------+--------------------------------+------------------------+-------+ -These may not be available on all platforms when used with the :meth:`strftime` +These may not be available on all platforms when used with the :meth:`~.datetime.strftime` method. The ISO 8601 year and ISO 8601 week directives are not interchangeable -with the year and week number directives above. Calling :meth:`strptime` with +with the year and week number directives above. Calling :meth:`~.datetime.strptime` with incomplete or ambiguous ISO 8601 directives will raise a :exc:`ValueError`. The full set of format codes supported varies across platforms, because Python -calls the platform C library's :func:`strftime` function, and platform +calls the platform C library's :c:func:`strftime` function, and platform variations are common. To see the full set of format codes supported on your platform, consult the :manpage:`strftime(3)` documentation. There are also differences between platforms in handling of unsupported format specifiers. @@ -2505,9 +2501,9 @@ Technical Detail Broadly speaking, ``d.strftime(fmt)`` acts like the :mod:`time` module's ``time.strftime(fmt, d.timetuple())`` although not all objects support a -:meth:`timetuple` method. +:meth:`~date.timetuple` method. -For the :meth:`datetime.strptime` class method, the default value is +For the :meth:`.datetime.strptime` class method, the default value is ``1900-01-01T00:00:00.000``: any components not specified in the format string will be pulled from the default value. [#]_ @@ -2520,7 +2516,7 @@ information, which are supported in ``datetime.strptime`` but are discarded by ``time.strptime``. For :class:`.time` objects, the format codes for year, month, and day should not -be used, as :class:`time` objects have no such values. If they're used anyway, +be used, as :class:`!time` objects have no such values. If they're used anyway, ``1900`` is substituted for the year, and ``1`` for the month and day. For :class:`date` objects, the format codes for hours, minutes, seconds, and @@ -2542,27 +2538,27 @@ Notes: contain non-ASCII characters. (2) - The :meth:`strptime` method can parse years in the full [1, 9999] range, but + The :meth:`~.datetime.strptime` method can parse years in the full [1, 9999] range, but years < 1000 must be zero-filled to 4-digit width. .. versionchanged:: 3.2 - In previous versions, :meth:`strftime` method was restricted to + In previous versions, :meth:`~.datetime.strftime` method was restricted to years >= 1900. .. versionchanged:: 3.3 - In version 3.2, :meth:`strftime` method was restricted to + In version 3.2, :meth:`~.datetime.strftime` method was restricted to years >= 1000. (3) - When used with the :meth:`strptime` method, the ``%p`` directive only affects + When used with the :meth:`~.datetime.strptime` method, the ``%p`` directive only affects the output hour field if the ``%I`` directive is used to parse the hour. (4) - Unlike the :mod:`time` module, the :mod:`datetime` module does not support + Unlike the :mod:`time` module, the :mod:`!datetime` module does not support leap seconds. (5) - When used with the :meth:`strptime` method, the ``%f`` directive + When used with the :meth:`~.datetime.strptime` method, the ``%f`` directive accepts from one to six digits and zero pads on the right. ``%f`` is an extension to the set of format characters in the C standard (but implemented separately in datetime objects, and therefore always @@ -2575,7 +2571,7 @@ Notes: For an aware object: ``%z`` - :meth:`utcoffset` is transformed into a string of the form + :meth:`~.datetime.utcoffset` is transformed into a string of the form ``±HHMM[SS[.ffffff]]``, where ``HH`` is a 2-digit string giving the number of UTC offset hours, ``MM`` is a 2-digit string giving the number of UTC offset minutes, ``SS`` is a 2-digit string giving the number of UTC offset @@ -2583,14 +2579,14 @@ Notes: offset microseconds. The ``ffffff`` part is omitted when the offset is a whole number of seconds and both the ``ffffff`` and the ``SS`` part is omitted when the offset is a whole number of minutes. For example, if - :meth:`utcoffset` returns ``timedelta(hours=-3, minutes=-30)``, ``%z`` is + :meth:`~.datetime.utcoffset` returns ``timedelta(hours=-3, minutes=-30)``, ``%z`` is replaced with the string ``'-0330'``. .. versionchanged:: 3.7 The UTC offset is not restricted to a whole number of minutes. .. versionchanged:: 3.7 - When the ``%z`` directive is provided to the :meth:`strptime` method, + When the ``%z`` directive is provided to the :meth:`~.datetime.strptime` method, the UTC offsets can have a colon as a separator between hours, minutes and seconds. For example, ``'+01:00:00'`` will be parsed as an offset of one hour. @@ -2601,11 +2597,11 @@ Notes: hours, minutes and seconds. ``%Z`` - In :meth:`strftime`, ``%Z`` is replaced by an empty string if - :meth:`tzname` returns ``None``; otherwise ``%Z`` is replaced by the + In :meth:`~.datetime.strftime`, ``%Z`` is replaced by an empty string if + :meth:`~.datetime.tzname` returns ``None``; otherwise ``%Z`` is replaced by the returned value, which must be a string. - :meth:`strptime` only accepts certain values for ``%Z``: + :meth:`~.datetime.strptime` only accepts certain values for ``%Z``: 1. any value in ``time.tzname`` for your machine's locale 2. the hard-coded values ``UTC`` and ``GMT`` @@ -2615,23 +2611,23 @@ Notes: invalid values. .. versionchanged:: 3.2 - When the ``%z`` directive is provided to the :meth:`strptime` method, an + When the ``%z`` directive is provided to the :meth:`~.datetime.strptime` method, an aware :class:`.datetime` object will be produced. The ``tzinfo`` of the result will be set to a :class:`timezone` instance. (7) - When used with the :meth:`strptime` method, ``%U`` and ``%W`` are only used + When used with the :meth:`~.datetime.strptime` method, ``%U`` and ``%W`` are only used in calculations when the day of the week and the calendar year (``%Y``) are specified. (8) Similar to ``%U`` and ``%W``, ``%V`` is only used in calculations when the day of the week and the ISO year (``%G``) are specified in a - :meth:`strptime` format string. Also note that ``%G`` and ``%Y`` are not + :meth:`~.datetime.strptime` format string. Also note that ``%G`` and ``%Y`` are not interchangeable. (9) - When used with the :meth:`strptime` method, the leading zero is optional + When used with the :meth:`~.datetime.strptime` method, the leading zero is optional for formats ``%d``, ``%m``, ``%H``, ``%I``, ``%M``, ``%S``, ``%j``, ``%U``, ``%W``, and ``%V``. Format ``%y`` does require a leading zero. diff --git a/Doc/library/dbm.rst b/Doc/library/dbm.rst index 766847b971b645..9bb5e5f8950956 100644 --- a/Doc/library/dbm.rst +++ b/Doc/library/dbm.rst @@ -28,70 +28,85 @@ the Oracle Berkeley DB. available --- :mod:`dbm.gnu`, :mod:`dbm.ndbm` or :mod:`dbm.dumb` --- should be used to open a given file. - Returns one of the following values: ``None`` if the file can't be opened - because it's unreadable or doesn't exist; the empty string (``''``) if the - file's format can't be guessed; or a string containing the required module - name, such as ``'dbm.ndbm'`` or ``'dbm.gnu'``. + Return one of the following values: -.. versionchanged:: 3.11 - Accepts :term:`path-like object` for filename. + * ``None`` if the file can't be opened because it's unreadable or doesn't exist + * the empty string (``''``) if the file's format can't be guessed + * a string containing the required module name, such as ``'dbm.ndbm'`` or ``'dbm.gnu'`` -.. function:: open(file, flag='r', mode=0o666) + .. versionchanged:: 3.11 + *filename* accepts a :term:`path-like object`. - Open the database file *file* and return a corresponding object. +.. Substitutions for the open() flag param docs; + all submodules use the same text. - If the database file already exists, the :func:`whichdb` function is used to - determine its type and the appropriate module is used; if it does not exist, - the first module listed above that can be imported is used. +.. |flag_r| replace:: + Open existing database for reading only. - The optional *flag* argument can be: +.. |flag_w| replace:: + Open existing database for reading and writing. - +---------+-------------------------------------------+ - | Value | Meaning | - +=========+===========================================+ - | ``'r'`` | Open existing database for reading only | - | | (default) | - +---------+-------------------------------------------+ - | ``'w'`` | Open existing database for reading and | - | | writing | - +---------+-------------------------------------------+ - | ``'c'`` | Open database for reading and writing, | - | | creating it if it doesn't exist | - +---------+-------------------------------------------+ - | ``'n'`` | Always create a new, empty database, open | - | | for reading and writing | - +---------+-------------------------------------------+ +.. |flag_c| replace:: + Open database for reading and writing, creating it if it doesn't exist. - The optional *mode* argument is the Unix mode of the file, used only when the - database has to be created. It defaults to octal ``0o666`` (and will be - modified by the prevailing umask). +.. |flag_n| replace:: + Always create a new, empty database, open for reading and writing. +.. |mode_param_doc| replace:: + The Unix file access mode of the file (default: octal ``0o666``), + used only when the database has to be created. -The object returned by :func:`.open` supports the same basic functionality as -dictionaries; keys and their corresponding values can be stored, retrieved, and -deleted, and the :keyword:`in` operator and the :meth:`keys` method are -available, as well as :meth:`get` and :meth:`setdefault`. +.. |incompat_note| replace:: + The file formats created by :mod:`dbm.gnu` and :mod:`dbm.ndbm` are incompatible + and can not be used interchangeably. -.. versionchanged:: 3.2 - :meth:`get` and :meth:`setdefault` are now available in all database modules. +.. function:: open(file, flag='r', mode=0o666) -.. versionchanged:: 3.8 - Deleting a key from a read-only database raises database module specific error - instead of :exc:`KeyError`. + Open a database and return the corresponding database object. + + :param file: + The database file to open. + + If the database file already exists, the :func:`whichdb` function is used to + determine its type and the appropriate module is used; if it does not exist, + the first submodule listed above that can be imported is used. + :type file: :term:`path-like object` + + :param str flag: + * ``'r'`` (default): |flag_r| + * ``'w'``: |flag_w| + * ``'c'``: |flag_c| + * ``'n'``: |flag_n| -.. versionchanged:: 3.11 - Accepts :term:`path-like object` for file. + :param int mode: + |mode_param_doc| -Key and values are always stored as bytes. This means that when + .. versionchanged:: 3.11 + *file* accepts a :term:`path-like object`. + +The object returned by :func:`~dbm.open` supports the same basic functionality as a +:class:`dict`; keys and their corresponding values can be stored, retrieved, and +deleted, and the :keyword:`in` operator and the :meth:`!keys` method are +available, as well as :meth:`!get` and :meth:`!setdefault` methods. + +Key and values are always stored as :class:`bytes`. This means that when strings are used they are implicitly converted to the default encoding before being stored. These objects also support being used in a :keyword:`with` statement, which will automatically close them when done. +.. versionchanged:: 3.2 + :meth:`!get` and :meth:`!setdefault` methods are now available for all + :mod:`dbm` backends. + .. versionchanged:: 3.4 Added native support for the context management protocol to the objects - returned by :func:`.open`. + returned by :func:`~dbm.open`. + +.. versionchanged:: 3.8 + Deleting a key from a read-only database raises a database module specific exception + instead of :exc:`KeyError`. The following example records some hostnames and a corresponding title, and then prints out the contents of the database:: @@ -130,27 +145,26 @@ then prints out the contents of the database:: The individual submodules are described in the following sections. -:mod:`dbm.gnu` --- GNU's reinterpretation of dbm ------------------------------------------------- +:mod:`dbm.gnu` --- GNU database manager +--------------------------------------- .. module:: dbm.gnu :platform: Unix - :synopsis: GNU's reinterpretation of dbm. + :synopsis: GNU database manager **Source code:** :source:`Lib/dbm/gnu.py` -------------- -This module is quite similar to the :mod:`dbm` module, but uses the GNU library -``gdbm`` instead to provide some additional functionality. Please note that the -file formats created by :mod:`dbm.gnu` and :mod:`dbm.ndbm` are incompatible. +The :mod:`dbm.gnu` module provides an interface to the :abbr:`GDBM (GNU dbm)` +library, similar to the :mod:`dbm.ndbm` module, but with additional +functionality like crash tolerance. + +:class:`!gdbm` objects behave similar to :term:`mappings <mapping>`, +except that keys and values are always converted to :class:`bytes` before storing, +and the :meth:`!items` and :meth:`!values` methods are not supported. -The :mod:`dbm.gnu` module provides an interface to the GNU DBM library. -``dbm.gnu.gdbm`` objects behave like mappings (dictionaries), except that keys and -values are always converted to bytes before storing. Printing a ``gdbm`` -object doesn't print the -keys and values, and the :meth:`items` and :meth:`values` methods are not -supported. +.. note:: |incompat_note| .. exception:: error @@ -158,62 +172,52 @@ supported. raised for general mapping errors like specifying an incorrect key. -.. function:: open(filename[, flag[, mode]]) - - Open a ``gdbm`` database and return a :class:`gdbm` object. The *filename* - argument is the name of the database file. - - The optional *flag* argument can be: - - +---------+-------------------------------------------+ - | Value | Meaning | - +=========+===========================================+ - | ``'r'`` | Open existing database for reading only | - | | (default) | - +---------+-------------------------------------------+ - | ``'w'`` | Open existing database for reading and | - | | writing | - +---------+-------------------------------------------+ - | ``'c'`` | Open database for reading and writing, | - | | creating it if it doesn't exist | - +---------+-------------------------------------------+ - | ``'n'`` | Always create a new, empty database, open | - | | for reading and writing | - +---------+-------------------------------------------+ - - The following additional characters may be appended to the flag to control - how the database is opened: - - +---------+--------------------------------------------+ - | Value | Meaning | - +=========+============================================+ - | ``'f'`` | Open the database in fast mode. Writes | - | | to the database will not be synchronized. | - +---------+--------------------------------------------+ - | ``'s'`` | Synchronized mode. This will cause changes | - | | to the database to be immediately written | - | | to the file. | - +---------+--------------------------------------------+ - | ``'u'`` | Do not lock database. | - +---------+--------------------------------------------+ - - Not all flags are valid for all versions of ``gdbm``. The module constant - :const:`open_flags` is a string of supported flag characters. The exception - :exc:`error` is raised if an invalid flag is specified. - - The optional *mode* argument is the Unix mode of the file, used only when the - database has to be created. It defaults to octal ``0o666``. - - In addition to the dictionary-like methods, ``gdbm`` objects have the - following methods: +.. function:: open(filename, flag="r", mode=0o666, /) + + Open a GDBM database and return a :class:`!gdbm` object. + + :param filename: + The database file to open. + :type filename: :term:`path-like object` + + :param str flag: + * ``'r'`` (default): |flag_r| + * ``'w'``: |flag_w| + * ``'c'``: |flag_c| + * ``'n'``: |flag_n| + + The following additional characters may be appended + to control how the database is opened: + + * ``'f'``: Open the database in fast mode. + Writes to the database will not be synchronized. + * ``'s'``: Synchronized mode. + Changes to the database will be written immediately to the file. + * ``'u'``: Do not lock database. + + Not all flags are valid for all versions of GDBM. + See the :data:`open_flags` member for a list of supported flag characters. + + :param int mode: + |mode_param_doc| + + :raises error: + If an invalid *flag* argument is passed. .. versionchanged:: 3.11 - Accepts :term:`path-like object` for filename. + *filename* accepts a :term:`path-like object`. + + .. data:: open_flags + + A string of characters the *flag* parameter of :meth:`~dbm.gnu.open` supports. + + In addition to the dictionary-like methods, :class:`gdbm` objects have the + following methods and attributes: .. method:: gdbm.firstkey() It's possible to loop over every key in the database using this method and the - :meth:`nextkey` method. The traversal is ordered by ``gdbm``'s internal + :meth:`nextkey` method. The traversal is ordered by GDBM's internal hash values, and won't be sorted by the key values. This method returns the starting key. @@ -231,7 +235,7 @@ supported. .. method:: gdbm.reorganize() If you have carried out a lot of deletions and would like to shrink the space - used by the ``gdbm`` file, this routine will reorganize the database. ``gdbm`` + used by the GDBM file, this routine will reorganize the database. :class:`!gdbm` objects will not shorten the length of a database file except by using this reorganization; otherwise, deleted file space will be kept and reused as new (key, value) pairs are added. @@ -243,34 +247,43 @@ supported. .. method:: gdbm.close() - Close the ``gdbm`` database. + Close the GDBM database. .. method:: gdbm.clear() - Remove all items from the ``gdbm`` database. + Remove all items from the GDBM database. .. versionadded:: 3.13 -:mod:`dbm.ndbm` --- Interface based on ndbm -------------------------------------------- +:mod:`dbm.ndbm` --- New Database Manager +---------------------------------------- .. module:: dbm.ndbm :platform: Unix - :synopsis: The standard "database" interface, based on ndbm. + :synopsis: The New Database Manager **Source code:** :source:`Lib/dbm/ndbm.py` -------------- -The :mod:`dbm.ndbm` module provides an interface to the Unix "(n)dbm" library. -Dbm objects behave like mappings (dictionaries), except that keys and values are -always stored as bytes. Printing a ``dbm`` object doesn't print the keys and -values, and the :meth:`items` and :meth:`values` methods are not supported. +The :mod:`dbm.ndbm` module provides an interface to the +:abbr:`NDBM (New Database Manager)` library. +:class:`!ndbm` objects behave similar to :term:`mappings <mapping>`, +except that keys and values are always stored as :class:`bytes`, +and the :meth:`!items` and :meth:`!values` methods are not supported. + +This module can be used with the "classic" NDBM interface or the +:abbr:`GDBM (GNU dbm)` compatibility interface. + +.. note:: |incompat_note| -This module can be used with the "classic" ndbm interface or the GNU GDBM -compatibility interface. On Unix, the :program:`configure` script will attempt -to locate the appropriate header file to simplify building this module. +.. warning:: + + The NDBM library shipped as part of macOS has an undocumented limitation on the + size of values, which can result in corrupted database files + when storing values larger than this limit. Reading such corrupted files can + result in a hard crash (segmentation fault). .. exception:: error @@ -280,37 +293,28 @@ to locate the appropriate header file to simplify building this module. .. data:: library - Name of the ``ndbm`` implementation library used. + Name of the NDBM implementation library used. -.. function:: open(filename[, flag[, mode]]) +.. function:: open(filename, flag="r", mode=0o666, /) - Open a dbm database and return a ``ndbm`` object. The *filename* argument is the - name of the database file (without the :file:`.dir` or :file:`.pag` extensions). + Open an NDBM database and return an :class:`!ndbm` object. - The optional *flag* argument must be one of these values: + :param filename: + The basename of the database file + (without the :file:`.dir` or :file:`.pag` extensions). + :type filename: :term:`path-like object` - +---------+-------------------------------------------+ - | Value | Meaning | - +=========+===========================================+ - | ``'r'`` | Open existing database for reading only | - | | (default) | - +---------+-------------------------------------------+ - | ``'w'`` | Open existing database for reading and | - | | writing | - +---------+-------------------------------------------+ - | ``'c'`` | Open database for reading and writing, | - | | creating it if it doesn't exist | - +---------+-------------------------------------------+ - | ``'n'`` | Always create a new, empty database, open | - | | for reading and writing | - +---------+-------------------------------------------+ + :param str flag: + * ``'r'`` (default): |flag_r| + * ``'w'``: |flag_w| + * ``'c'``: |flag_c| + * ``'n'``: |flag_n| - The optional *mode* argument is the Unix mode of the file, used only when the - database has to be created. It defaults to octal ``0o666`` (and will be - modified by the prevailing umask). + :param int mode: + |mode_param_doc| - In addition to the dictionary-like methods, ``ndbm`` objects + In addition to the dictionary-like methods, :class:`!ndbm` objects provide the following method: .. versionchanged:: 3.11 @@ -318,11 +322,11 @@ to locate the appropriate header file to simplify building this module. .. method:: ndbm.close() - Close the ``ndbm`` database. + Close the NDBM database. .. method:: ndbm.clear() - Remove all items from the ``ndbm`` database. + Remove all items from the NDBM database. .. versionadded:: 3.13 @@ -346,13 +350,14 @@ to locate the appropriate header file to simplify building this module. -------------- -The :mod:`dbm.dumb` module provides a persistent dictionary-like interface which -is written entirely in Python. Unlike other modules such as :mod:`dbm.gnu` no -external library is required. As with other persistent mappings, the keys and -values are always stored as bytes. - -The module defines the following: +The :mod:`dbm.dumb` module provides a persistent :class:`dict`-like +interface which is written entirely in Python. +Unlike other :mod:`dbm` backends, such as :mod:`dbm.gnu`, no +external library is required. +As with other :mod:`dbm` backends, +the keys and values are always stored as :class:`bytes`. +The :mod:`!dbm.dumb` module defines the following: .. exception:: error @@ -360,34 +365,29 @@ The module defines the following: raised for general mapping errors like specifying an incorrect key. -.. function:: open(filename[, flag[, mode]]) +.. function:: open(filename, flag="c", mode=0o666) - Open a ``dumbdbm`` database and return a dumbdbm object. The *filename* argument is - the basename of the database file (without any specific extensions). When a - dumbdbm database is created, files with :file:`.dat` and :file:`.dir` extensions - are created. + Open a :mod:`!dbm.dumb` database. + The returned database object behaves similar to a :term:`mapping`, + in addition to providing :meth:`~dumbdbm.sync` and :meth:`~dumbdbm.close` + methods. - The optional *flag* argument can be: + :param filename: + The basename of the database file (without extensions). + A new database creates the following files: - +---------+-------------------------------------------+ - | Value | Meaning | - +=========+===========================================+ - | ``'r'`` | Open existing database for reading only | - | | (default) | - +---------+-------------------------------------------+ - | ``'w'`` | Open existing database for reading and | - | | writing | - +---------+-------------------------------------------+ - | ``'c'`` | Open database for reading and writing, | - | | creating it if it doesn't exist | - +---------+-------------------------------------------+ - | ``'n'`` | Always create a new, empty database, open | - | | for reading and writing | - +---------+-------------------------------------------+ + - :file:`{filename}.dat` + - :file:`{filename}.dir` + :type database: :term:`path-like object` - The optional *mode* argument is the Unix mode of the file, used only when the - database has to be created. It defaults to octal ``0o666`` (and will be modified - by the prevailing umask). + :param str flag: + * ``'r'``: |flag_r| + * ``'w'``: |flag_w| + * ``'c'`` (default): |flag_c| + * ``'n'``: |flag_n| + + :param int mode: + |mode_param_doc| .. warning:: It is possible to crash the Python interpreter when loading a database @@ -395,20 +395,18 @@ The module defines the following: Python's AST compiler. .. versionchanged:: 3.5 - :func:`.open` always creates a new database when the flag has the value - ``'n'``. + :func:`~dbm.dumb.open` always creates a new database when *flag* is ``'n'``. .. versionchanged:: 3.8 - A database opened with flags ``'r'`` is now read-only. Opening with - flags ``'r'`` and ``'w'`` no longer creates a database if it does not - exist. + A database opened read-only if *flag* is ``'r'``. + A database is not created if it does not exist if *flag* is ``'r'`` or ``'w'``. .. versionchanged:: 3.11 - Accepts :term:`path-like object` for filename. + *filename* accepts a :term:`path-like object`. In addition to the methods provided by the - :class:`collections.abc.MutableMapping` class, :class:`dumbdbm` objects - provide the following methods: + :class:`collections.abc.MutableMapping` class, + the following methods are provided: .. method:: dumbdbm.sync() @@ -417,5 +415,5 @@ The module defines the following: .. method:: dumbdbm.close() - Close the ``dumbdbm`` database. + Close the database. diff --git a/Doc/library/difflib.rst b/Doc/library/difflib.rst index 9abf19557f989c..d45e46448207a4 100644 --- a/Doc/library/difflib.rst +++ b/Doc/library/difflib.rst @@ -52,8 +52,8 @@ diffs. For comparing directories and files, see also, the :mod:`filecmp` module. the purpose of sequence matching. This heuristic can be turned off by setting the ``autojunk`` argument to ``False`` when creating the :class:`SequenceMatcher`. - .. versionadded:: 3.2 - The *autojunk* parameter. + .. versionchanged:: 3.2 + Added the *autojunk* parameter. .. class:: Differ @@ -383,8 +383,8 @@ The :class:`SequenceMatcher` class has this constructor: The optional argument *autojunk* can be used to disable the automatic junk heuristic. - .. versionadded:: 3.2 - The *autojunk* parameter. + .. versionchanged:: 3.2 + Added the *autojunk* parameter. SequenceMatcher objects get three data attributes: *bjunk* is the set of elements of *b* for which *isjunk* is ``True``; *bpopular* is the set of diff --git a/Doc/library/dis.rst b/Doc/library/dis.rst index 5647021d6a9ba6..e654760fb91c65 100644 --- a/Doc/library/dis.rst +++ b/Doc/library/dis.rst @@ -342,17 +342,18 @@ operation is being performed, so the intermediate analysis object isn't useful: .. function:: findlinestarts(code) - This generator function uses the ``co_lines`` method - of the code object *code* to find the offsets which are starts of + This generator function uses the :meth:`~codeobject.co_lines` method + of the :ref:`code object <code-objects>` *code* to find the offsets which + are starts of lines in the source code. They are generated as ``(offset, lineno)`` pairs. .. versionchanged:: 3.6 Line numbers can be decreasing. Before, they were always increasing. .. versionchanged:: 3.10 - The :pep:`626` ``co_lines`` method is used instead of the + The :pep:`626` :meth:`~codeobject.co_lines` method is used instead of the :attr:`~codeobject.co_firstlineno` and :attr:`~codeobject.co_lnotab` - attributes of the code object. + attributes of the :ref:`code object <code-objects>`. .. versionchanged:: 3.13 Line numbers can be ``None`` for bytecode that does not map to source lines. @@ -545,8 +546,8 @@ operations on it as if it was a Python list. The top of the stack corresponds to .. opcode:: END_FOR - Removes the top two values from the stack. - Equivalent to ``POP_TOP``; ``POP_TOP``. + Removes the top-of-stack item. + Equivalent to ``POP_TOP``. Used to clean up at the end of loops, hence the name. .. versionadded:: 3.12 @@ -575,7 +576,7 @@ operations on it as if it was a Python list. The top of the stack corresponds to Swap the top of the stack with the i-th element:: - STACK[-i], STACK[-1] = stack[-1], STACK[-i] + STACK[-i], STACK[-1] = STACK[-1], STACK[-i] .. versionadded:: 3.11 @@ -1214,9 +1215,10 @@ iterations of the loop. ``super(cls, self).method()``, ``super(cls, self).attr``). It pops three values from the stack (from top of stack down): - - ``self``: the first argument to the current method - - ``cls``: the class within which the current method was defined - - the global ``super`` + + * ``self``: the first argument to the current method + * ``cls``: the class within which the current method was defined + * the global ``super`` With respect to its argument, it works similarly to :opcode:`LOAD_ATTR`, except that ``namei`` is shifted left by 2 bits instead of 1. diff --git a/Doc/library/doctest.rst b/Doc/library/doctest.rst index 8c28e4478bb70e..1bfcd69f72df2e 100644 --- a/Doc/library/doctest.rst +++ b/Doc/library/doctest.rst @@ -134,7 +134,7 @@ That's all you need to know to start making productive use of :mod:`doctest`! Jump in. The following sections provide full details. Note that there are many examples of doctests in the standard Python test suite and libraries. Especially useful examples can be found in the standard test file -:file:`Lib/test/test_doctest.py`. +:file:`Lib/test/test_doctest/test_doctest.py`. .. _doctest-simple-testmod: @@ -280,7 +280,7 @@ searched. Objects imported into the module are not searched. In addition, there are cases when you want tests to be part of a module but not part of the help text, which requires that the tests not be included in the docstring. Doctest looks for a module-level variable called ``__test__`` and uses it to locate other -tests. If ``M.__test__`` exists and is truthy, it must be a dict, and each +tests. If ``M.__test__`` exists, it must be a dict, and each entry maps a (string) name to a function object, class object, or string. Function and class object docstrings found from ``M.__test__`` are searched, and strings are treated as if they were docstrings. In output, a key ``K`` in @@ -944,8 +944,8 @@ and :ref:`doctest-simple-testfile`. (or module :mod:`__main__` if *m* is not supplied or is ``None``), starting with ``m.__doc__``. - Also test examples reachable from dict ``m.__test__``, if it exists and is not - ``None``. ``m.__test__`` maps names (strings) to functions, classes and + Also test examples reachable from dict ``m.__test__``, if it exists. + ``m.__test__`` maps names (strings) to functions, classes and strings; function and class docstrings are searched for examples; strings are searched directly, as if they were docstrings. diff --git a/Doc/library/email.message.rst b/Doc/library/email.message.rst index f58d93da6ed687..adea067e082615 100644 --- a/Doc/library/email.message.rst +++ b/Doc/library/email.message.rst @@ -40,9 +40,9 @@ over the object tree. The :class:`EmailMessage` dictionary-like interface is indexed by the header names, which must be ASCII values. The values of the dictionary are strings with some extra methods. Headers are stored and returned in case-preserving -form, but field names are matched case-insensitively. Unlike a real dict, -there is an ordering to the keys, and there can be duplicate keys. Additional -methods are provided for working with headers that have duplicate keys. +form, but field names are matched case-insensitively. The keys are ordered, +but unlike a real dict, there can be duplicates. Addtional methods are +provided for working with headers that have duplicate keys. The *payload* is either a string or bytes object, in the case of simple message objects, or a list of :class:`EmailMessage` objects, for MIME container diff --git a/Doc/library/email.mime.rst b/Doc/library/email.mime.rst index d7c0d203d191f8..dc0dd3b9eebde6 100644 --- a/Doc/library/email.mime.rst +++ b/Doc/library/email.mime.rst @@ -28,7 +28,7 @@ make things easier. Here are the classes: -.. currentmodule:: email.mime.base +.. module:: email.mime.base .. class:: MIMEBase(_maintype, _subtype, *, policy=compat32, **_params) @@ -58,7 +58,7 @@ Here are the classes: Added *policy* keyword-only parameter. -.. currentmodule:: email.mime.nonmultipart +.. module:: email.mime.nonmultipart .. class:: MIMENonMultipart() @@ -72,7 +72,7 @@ Here are the classes: is called, a :exc:`~email.errors.MultipartConversionError` exception is raised. -.. currentmodule:: email.mime.multipart +.. module:: email.mime.multipart .. class:: MIMEMultipart(_subtype='mixed', boundary=None, _subparts=None, \ *, policy=compat32, **_params) @@ -104,7 +104,7 @@ Here are the classes: .. versionchanged:: 3.6 Added *policy* keyword-only parameter. -.. currentmodule:: email.mime.application +.. module:: email.mime.application .. class:: MIMEApplication(_data, _subtype='octet-stream', \ _encoder=email.encoders.encode_base64, \ @@ -135,7 +135,7 @@ Here are the classes: .. versionchanged:: 3.6 Added *policy* keyword-only parameter. -.. currentmodule:: email.mime.audio +.. module:: email.mime.audio .. class:: MIMEAudio(_audiodata, _subtype=None, \ _encoder=email.encoders.encode_base64, \ @@ -169,7 +169,7 @@ Here are the classes: .. versionchanged:: 3.6 Added *policy* keyword-only parameter. -.. currentmodule:: email.mime.image +.. module:: email.mime.image .. class:: MIMEImage(_imagedata, _subtype=None, \ _encoder=email.encoders.encode_base64, \ @@ -205,7 +205,7 @@ Here are the classes: .. versionchanged:: 3.6 Added *policy* keyword-only parameter. -.. currentmodule:: email.mime.message +.. module:: email.mime.message .. class:: MIMEMessage(_msg, _subtype='rfc822', *, policy=compat32) @@ -225,7 +225,7 @@ Here are the classes: .. versionchanged:: 3.6 Added *policy* keyword-only parameter. -.. currentmodule:: email.mime.text +.. module:: email.mime.text .. class:: MIMEText(_text, _subtype='plain', _charset=None, *, policy=compat32) diff --git a/Doc/library/email.policy.rst b/Doc/library/email.policy.rst index fd47dd0dc5df36..f4777bb2462138 100644 --- a/Doc/library/email.policy.rst +++ b/Doc/library/email.policy.rst @@ -219,7 +219,6 @@ added matters. To illustrate:: Default: :const:`False`. .. versionadded:: 3.5 - The *mangle_from_* parameter. .. attribute:: message_factory diff --git a/Doc/library/enum.rst b/Doc/library/enum.rst index 20222bfb3611ab..30d80ce8d488cc 100644 --- a/Doc/library/enum.rst +++ b/Doc/library/enum.rst @@ -286,6 +286,19 @@ Data Types appropriate value will be chosen for you. See :class:`auto` for the details. + .. attribute:: Enum._name_ + + Name of the member. + + .. attribute:: Enum._value_ + + Value of the member, can be set in :meth:`~object.__new__`. + + .. attribute:: Enum._order_ + + No longer used, kept for backward compatibility. + (class attribute, removed during class creation). + .. attribute:: Enum._ignore_ ``_ignore_`` is only used during creation and is removed from the @@ -337,6 +350,17 @@ Data Types >>> PowersOfThree.SECOND.value 9 + .. method:: Enum.__init__(self, \*args, \**kwds) + + By default, does nothing. If multiple values are given in the member + assignment, those values become separate arguments to ``__init__``; e.g. + + >>> from enum import Enum + >>> class Weekday(Enum): + ... MONDAY = 1, 'Mon' + + ``Weekday.__init__()`` would be called as ``Weekday.__init__(self, 1, 'Mon')`` + .. method:: Enum.__init_subclass__(cls, \**kwds) A *classmethod* that is used to further configure subsequent subclasses. @@ -364,6 +388,18 @@ Data Types >>> Build('deBUG') <Build.DEBUG: 'debug'> + .. method:: Enum.__new__(cls, \*args, \**kwds) + + By default, doesn't exist. If specified, either in the enum class + definition or in a mixin class (such as ``int``), all values given + in the member assignment will be passed; e.g. + + >>> from enum import Enum + >>> class MyIntEnum(Enum): + ... SEVENTEEN = '1a', 16 + + results in the call ``int('1a', 16)`` and a value of ``17`` for the member. + .. method:: Enum.__repr__(self) Returns the string used for *repr()* calls. By default, returns the @@ -477,9 +513,9 @@ Data Types .. class:: Flag - *Flag* members support the bitwise operators ``&`` (*AND*), ``|`` (*OR*), - ``^`` (*XOR*), and ``~`` (*INVERT*); the results of those operators are members - of the enumeration. + ``Flag`` is the same as :class:`Enum`, but its members support the bitwise + operators ``&`` (*AND*), ``|`` (*OR*), ``^`` (*XOR*), and ``~`` (*INVERT*); + the results of those operators are members of the enumeration. .. method:: __contains__(self, value) @@ -511,9 +547,7 @@ Data Types >>> list(purple) [<Color.RED: 1>, <Color.BLUE: 4>] - .. versionchanged:: 3.11 - - Aliases are no longer returned during iteration. + .. versionadded:: 3.11 .. method:: __len__(self): @@ -802,8 +836,8 @@ Supported ``_sunder_`` names - :attr:`~Enum._ignore_` -- a list of names, either as a :class:`list` or a :class:`str`, that will not be transformed into members, and will be removed from the final class -- :attr:`~Enum._order_` -- used in Python 2/3 code to ensure member order is - consistent (class attribute, removed during class creation) +- :attr:`~Enum._order_` -- no longer used, kept for backward + compatibility (class attribute, removed during class creation) - :meth:`~Enum._generate_next_value_` -- used to get an appropriate value for an enum member; may be overridden @@ -838,7 +872,7 @@ Utilities and Decorators * ``FIRST = auto()`` will work (auto() is replaced with ``1``); * ``SECOND = auto(), -2`` will work (auto is replaced with ``2``, so ``2, -2`` is - used to create the ``SECOND`` enum member; + used to create the ``SECOND`` enum member; * ``THREE = [auto(), -3]`` will *not* work (``<auto instance>, -3`` is used to create the ``THREE`` enum member) diff --git a/Doc/library/exceptions.rst b/Doc/library/exceptions.rst index f821776c286133..3191315049ad5a 100644 --- a/Doc/library/exceptions.rst +++ b/Doc/library/exceptions.rst @@ -16,7 +16,7 @@ equivalent, even if they have the same name. .. index:: pair: statement; raise -The built-in exceptions listed below can be generated by the interpreter or +The built-in exceptions listed in this chapter can be generated by the interpreter or built-in functions. Except where mentioned, they have an "associated value" indicating the detailed cause of the error. This may be a string or a tuple of several items of information (e.g., an error code and a string explaining the diff --git a/Doc/library/fcntl.rst b/Doc/library/fcntl.rst index 309ad652d4af34..13ad2dd7da5090 100644 --- a/Doc/library/fcntl.rst +++ b/Doc/library/fcntl.rst @@ -31,26 +31,26 @@ descriptor. raise an :exc:`OSError`. .. versionchanged:: 3.8 - The fcntl module now contains ``F_ADD_SEALS``, ``F_GET_SEALS``, and + The :mod:`!fcntl` module now contains ``F_ADD_SEALS``, ``F_GET_SEALS``, and ``F_SEAL_*`` constants for sealing of :func:`os.memfd_create` file descriptors. .. versionchanged:: 3.9 - On macOS, the fcntl module exposes the ``F_GETPATH`` constant, which obtains - the path of a file from a file descriptor. - On Linux(>=3.15), the fcntl module exposes the ``F_OFD_GETLK``, ``F_OFD_SETLK`` - and ``F_OFD_SETLKW`` constants, which are used when working with open file - description locks. + On macOS, the :mod:`!fcntl` module exposes the ``F_GETPATH`` constant, + which obtains the path of a file from a file descriptor. + On Linux(>=3.15), the :mod:`!fcntl` module exposes the ``F_OFD_GETLK``, + ``F_OFD_SETLK`` and ``F_OFD_SETLKW`` constants, which are used when working + with open file description locks. .. versionchanged:: 3.10 - On Linux >= 2.6.11, the fcntl module exposes the ``F_GETPIPE_SZ`` and + On Linux >= 2.6.11, the :mod:`!fcntl` module exposes the ``F_GETPIPE_SZ`` and ``F_SETPIPE_SZ`` constants, which allow to check and modify a pipe's size respectively. .. versionchanged:: 3.11 - On FreeBSD, the fcntl module exposes the ``F_DUP2FD`` and ``F_DUP2FD_CLOEXEC`` - constants, which allow to duplicate a file descriptor, the latter setting - ``FD_CLOEXEC`` flag in addition. + On FreeBSD, the :mod:`!fcntl` module exposes the ``F_DUP2FD`` and + ``F_DUP2FD_CLOEXEC`` constants, which allow to duplicate a file descriptor, + the latter setting ``FD_CLOEXEC`` flag in addition. .. versionchanged:: 3.12 On Linux >= 4.5, the :mod:`fcntl` module exposes the ``FICLONE`` and @@ -58,6 +58,27 @@ descriptor. another file by reflinking on some filesystems (e.g., btrfs, OCFS2, and XFS). This behavior is commonly referred to as "copy-on-write". +.. versionchanged:: 3.13 + On Linux >= 2.6.32, the :mod:`!fcntl` module exposes the + ``F_GETOWN_EX``, ``F_SETOWN_EX``, ``F_OWNER_TID``, ``F_OWNER_PID``, ``F_OWNER_PGRP`` constants, which allow to direct I/O availability signals + to a specific thread, process, or process group. + On Linux >= 4.13, the :mod:`!fcntl` module exposes the + ``F_GET_RW_HINT``, ``F_SET_RW_HINT``, ``F_GET_FILE_RW_HINT``, + ``F_SET_FILE_RW_HINT``, and ``RWH_WRITE_LIFE_*`` constants, which allow + to inform the kernel about the relative expected lifetime of writes on + a given inode or via a particular open file description. + On Linux >= 5.1 and NetBSD, the :mod:`!fcntl` module exposes the + ``F_SEAL_FUTURE_WRITE`` constant for use with ``F_ADD_SEALS`` and + ``F_GET_SEALS`` operations. + On FreeBSD, the :mod:`!fcntl` module exposes the ``F_READAHEAD``, ``F_ISUNIONSTACK``, and ``F_KINFO`` constants. + On macOS and FreeBSD, the :mod:`!fcntl` module exposes the ``F_RDAHEAD`` + constant. + On NetBSD and AIX, the :mod:`!fcntl` module exposes the ``F_CLOSEM`` + constant. + On NetBSD, the :mod:`!fcntl` module exposes the ``F_MAXFD`` constant. + On macOS and NetBSD, the :mod:`!fcntl` module exposes the ``F_GETNOSIGPIPE`` + and ``F_SETNOSIGPIPE`` constant. + The module defines the following functions: diff --git a/Doc/library/fnmatch.rst b/Doc/library/fnmatch.rst index aed8991d44772f..7cddecd5e80887 100644 --- a/Doc/library/fnmatch.rst +++ b/Doc/library/fnmatch.rst @@ -50,10 +50,10 @@ Also note that :func:`functools.lru_cache` with the *maxsize* of 32768 is used t cache the compiled regex patterns in the following functions: :func:`fnmatch`, :func:`fnmatchcase`, :func:`.filter`. -.. function:: fnmatch(filename, pattern) +.. function:: fnmatch(name, pat) - Test whether the *filename* string matches the *pattern* string, returning - :const:`True` or :const:`False`. Both parameters are case-normalized + Test whether the filename string *name* matches the pattern string *pat*, + returning ``True`` or ``False``. Both parameters are case-normalized using :func:`os.path.normcase`. :func:`fnmatchcase` can be used to perform a case-sensitive comparison, regardless of whether that's standard for the operating system. @@ -69,22 +69,24 @@ cache the compiled regex patterns in the following functions: :func:`fnmatch`, print(file) -.. function:: fnmatchcase(filename, pattern) +.. function:: fnmatchcase(name, pat) - Test whether *filename* matches *pattern*, returning :const:`True` or - :const:`False`; the comparison is case-sensitive and does not apply - :func:`os.path.normcase`. + Test whether the filename string *name* matches the pattern string *pat*, + returning ``True`` or ``False``; + the comparison is case-sensitive and does not apply :func:`os.path.normcase`. -.. function:: filter(names, pattern) +.. function:: filter(names, pat) - Construct a list from those elements of the iterable *names* that match *pattern*. It is the same as - ``[n for n in names if fnmatch(n, pattern)]``, but implemented more efficiently. + Construct a list from those elements of the :term:`iterable` *names* + that match pattern *pat*. + It is the same as ``[n for n in names if fnmatch(n, pat)]``, + but implemented more efficiently. -.. function:: translate(pattern) +.. function:: translate(pat) - Return the shell-style *pattern* converted to a regular expression for + Return the shell-style pattern *pat* converted to a regular expression for using with :func:`re.match`. Example: diff --git a/Doc/library/ftplib.rst b/Doc/library/ftplib.rst index d1fe6414ea020c..2f98a272c297ae 100644 --- a/Doc/library/ftplib.rst +++ b/Doc/library/ftplib.rst @@ -45,19 +45,73 @@ Here's a sample session using the :mod:`ftplib` module:: '221 Goodbye.' -The module defines the following items: +.. _ftplib-reference: -.. class:: FTP(host='', user='', passwd='', acct='', timeout=None, source_address=None, *, encoding='utf-8') +Reference +--------- - Return a new instance of the :class:`FTP` class. When *host* is given, the - method call ``connect(host)`` is made. When *user* is given, additionally - the method call ``login(user, passwd, acct)`` is made (where *passwd* and - *acct* default to the empty string when not given). The optional *timeout* - parameter specifies a timeout in seconds for blocking operations like the - connection attempt (if is not specified, the global default timeout setting - will be used). *source_address* is a 2-tuple ``(host, port)`` for the socket - to bind to as its source address before connecting. The *encoding* parameter - specifies the encoding for directories and filenames. +.. _ftp-objects: + +FTP objects +^^^^^^^^^^^ + +.. Use substitutions for some param docs so we don't need to repeat them + in multiple places. + +.. |param_doc_user| replace:: + The username to log in with (default: ``'anonymous'``). + +.. |param_doc_passwd| replace:: + The password to use when logging in. + If not given, and if *passwd* is the empty string or ``"-"``, + a password will be automatically generated. + +.. Ideally, we'd like to use the :rfc: directive, but Sphinx will not allow it. + +.. |param_doc_acct| replace:: + Account information to be used for the ``ACCT`` FTP command. + Few systems implement this. + See `RFC-959 <https://datatracker.ietf.org/doc/html/rfc959.html>`__ + for more details. + +.. |param_doc_source_address| replace:: + A 2-tuple ``(host, port)`` for the socket to bind to as its + source address before connecting. + +.. |param_doc_encoding| replace:: + The encoding for directories and filenames (default: ``'utf-8'``). + +.. class:: FTP(host='', user='', passwd='', acct='', timeout=None, \ + source_address=None, *, encoding='utf-8') + + Return a new instance of the :class:`FTP` class. + + :param str host: + The hostname to connect to. + If given, :code:`connect(host)` is implicitly called by the constructor. + + :param str user: + |param_doc_user| + If given, :code:`login(host, passwd, acct)` is implicitly called + by the constructor. + + :param str passwd: + |param_doc_passwd| + + :param str acct: + |param_doc_acct| + + :param timeout: + A timeout in seconds for blocking operations like :meth:`connect` + (default: the global default timeout setting). + :type timeout: int | None + + :param source_address: + |param_doc_source_address| + :type source_address: tuple | None + + :param str encoding: + |param_doc_encoding| The :class:`FTP` class supports the :keyword:`with` statement, e.g.: @@ -85,376 +139,460 @@ The module defines the following items: The *encoding* parameter was added, and the default was changed from Latin-1 to UTF-8 to follow :rfc:`2640`. -.. class:: FTP_TLS(host='', user='', passwd='', acct='', *, context=None, - timeout=None, source_address=None, encoding='utf-8') + Several :class:`!FTP` methods are available in two flavors: + one for handling text files and another for binary files. + The methods are named for the command which is used followed by + ``lines`` for the text version or ``binary`` for the binary version. - A :class:`FTP` subclass which adds TLS support to FTP as described in - :rfc:`4217`. - Connect as usual to port 21 implicitly securing the FTP control connection - before authenticating. Securing the data connection requires the user to - explicitly ask for it by calling the :meth:`prot_p` method. *context* - is a :class:`ssl.SSLContext` object which allows bundling SSL configuration - options, certificates and private keys into a single (potentially - long-lived) structure. Please read :ref:`ssl-security` for best practices. + :class:`FTP` instances have the following methods: - .. versionadded:: 3.2 + .. method:: FTP.set_debuglevel(level) - .. versionchanged:: 3.3 - *source_address* parameter was added. + Set the instance's debugging level as an :class:`int`. + This controls the amount of debugging output printed. + The debug levels are: - .. versionchanged:: 3.4 - The class now supports hostname check with - :attr:`ssl.SSLContext.check_hostname` and *Server Name Indication* (see - :const:`ssl.HAS_SNI`). + * ``0`` (default): No debug output. + * ``1``: Produce a moderate amount of debug output, + generally a single line per request. + * ``2`` or higher: Produce the maximum amount of debugging output, + logging each line sent and received on the control connection. - .. versionchanged:: 3.9 - If the *timeout* parameter is set to be zero, it will raise a - :class:`ValueError` to prevent the creation of a non-blocking socket. - The *encoding* parameter was added, and the default was changed from - Latin-1 to UTF-8 to follow :rfc:`2640`. + .. method:: FTP.connect(host='', port=0, timeout=None, source_address=None) - .. versionchanged:: 3.12 - The deprecated *keyfile* and *certfile* parameters have been removed. + Connect to the given host and port. + This function should be called only once for each instance; + it should not be called if a *host* argument was given + when the :class:`FTP` instance was created. + All other :class:`!FTP` methods can only be called + after a connection has successfully been made. - Here's a sample session using the :class:`FTP_TLS` class:: + :param str host: + The host to connect to. - >>> ftps = FTP_TLS('ftp.pureftpd.org') - >>> ftps.login() - '230 Anonymous user logged in' - >>> ftps.prot_p() - '200 Data protection level set to "private"' - >>> ftps.nlst() - ['6jack', 'OpenBSD', 'antilink', 'blogbench', 'bsdcam', 'clockspeed', 'djbdns-jedi', 'docs', 'eaccelerator-jedi', 'favicon.ico', 'francotone', 'fugu', 'ignore', 'libpuzzle', 'metalog', 'minidentd', 'misc', 'mysql-udf-global-user-variables', 'php-jenkins-hash', 'php-skein-hash', 'php-webdav', 'phpaudit', 'phpbench', 'pincaster', 'ping', 'posto', 'pub', 'public', 'public_keys', 'pure-ftpd', 'qscan', 'qtc', 'sharedance', 'skycache', 'sound', 'tmp', 'ucarp'] + :param int port: + The TCP port to connect to (default: ``21``, + as specified by the FTP protocol specification). + It is rarely needed to specify a different port number. + :param timeout: + A timeout in seconds for the connection attempt + (default: the global default timeout setting). + :type timeout: int | None -.. exception:: error_reply + :param source_address: + |param_doc_source_address| + :type source_address: tuple | None - Exception raised when an unexpected reply is received from the server. + .. audit-event:: ftplib.connect self,host,port ftplib.FTP.connect + .. versionchanged:: 3.3 + *source_address* parameter was added. -.. exception:: error_temp - Exception raised when an error code signifying a temporary error (response - codes in the range 400--499) is received. + .. method:: FTP.getwelcome() + Return the welcome message sent by the server in reply to the initial + connection. (This message sometimes contains disclaimers or help information + that may be relevant to the user.) -.. exception:: error_perm - Exception raised when an error code signifying a permanent error (response - codes in the range 500--599) is received. + .. method:: FTP.login(user='anonymous', passwd='', acct='') + Log on to the connected FTP server. + This function should be called only once for each instance, + after a connection has been established; + it should not be called if the *host* and *user* arguments were given + when the :class:`FTP` instance was created. + Most FTP commands are only allowed after the client has logged in. -.. exception:: error_proto + :param str user: + |param_doc_user| - Exception raised when a reply is received from the server that does not fit - the response specifications of the File Transfer Protocol, i.e. begin with a - digit in the range 1--5. + :param str passwd: + |param_doc_passwd| + :param str acct: + |param_doc_acct| -.. data:: all_errors - The set of all exceptions (as a tuple) that methods of :class:`FTP` - instances may raise as a result of problems with the FTP connection (as - opposed to programming errors made by the caller). This set includes the - four exceptions listed above as well as :exc:`OSError` and :exc:`EOFError`. + .. method:: FTP.abort() + Abort a file transfer that is in progress. Using this does not always work, but + it's worth a try. -.. seealso:: - Module :mod:`netrc` - Parser for the :file:`.netrc` file format. The file :file:`.netrc` is - typically used by FTP clients to load user authentication information - before prompting the user. + .. method:: FTP.sendcmd(cmd) + Send a simple command string to the server and return the response string. -.. _ftp-objects: + .. audit-event:: ftplib.sendcmd self,cmd ftplib.FTP.sendcmd -FTP Objects ------------ -Several methods are available in two flavors: one for handling text files and -another for binary files. These are named for the command which is used -followed by ``lines`` for the text version or ``binary`` for the binary version. + .. method:: FTP.voidcmd(cmd) -:class:`FTP` instances have the following methods: + Send a simple command string to the server and handle the response. Return + nothing if a response code corresponding to success (codes in the range + 200--299) is received. Raise :exc:`error_reply` otherwise. + .. audit-event:: ftplib.sendcmd self,cmd ftplib.FTP.voidcmd -.. method:: FTP.set_debuglevel(level) - Set the instance's debugging level. This controls the amount of debugging - output printed. The default, ``0``, produces no debugging output. A value of - ``1`` produces a moderate amount of debugging output, generally a single line - per request. A value of ``2`` or higher produces the maximum amount of - debugging output, logging each line sent and received on the control connection. + .. method:: FTP.retrbinary(cmd, callback, blocksize=8192, rest=None) + Retrieve a file in binary transfer mode. -.. method:: FTP.connect(host='', port=0, timeout=None, source_address=None) + :param str cmd: + An appropriate ``STOR`` command: :samp:`"STOR {filename}"`. - Connect to the given host and port. The default port number is ``21``, as - specified by the FTP protocol specification. It is rarely needed to specify a - different port number. This function should be called only once for each - instance; it should not be called at all if a host was given when the instance - was created. All other methods can only be used after a connection has been - made. - The optional *timeout* parameter specifies a timeout in seconds for the - connection attempt. If no *timeout* is passed, the global default timeout - setting will be used. - *source_address* is a 2-tuple ``(host, port)`` for the socket to bind to as - its source address before connecting. + :param callback: + A single parameter callable that is called + for each block of data received, + with its single argument being the data as :class:`bytes`. + :type callback: :term:`callable` - .. audit-event:: ftplib.connect self,host,port ftplib.FTP.connect + :param int blocksize: + The maximum chunk size to read on the low-level + :class:`~socket.socket` object created to do the actual transfer. + This also corresponds to the largest size of data + that will be passed to *callback*. + Defaults to ``8192``. - .. versionchanged:: 3.3 - *source_address* parameter was added. + :param int rest: + A ``REST`` command to be sent to the server. + See the documentation for the *rest* parameter of the :meth:`transfercmd` method. -.. method:: FTP.getwelcome() + .. method:: FTP.retrlines(cmd, callback=None) - Return the welcome message sent by the server in reply to the initial - connection. (This message sometimes contains disclaimers or help information - that may be relevant to the user.) + Retrieve a file or directory listing in the encoding specified by the + *encoding* parameter at initialization. + *cmd* should be an appropriate ``RETR`` command (see :meth:`retrbinary`) or + a command such as ``LIST`` or ``NLST`` (usually just the string ``'LIST'``). + ``LIST`` retrieves a list of files and information about those files. + ``NLST`` retrieves a list of file names. + The *callback* function is called for each line with a string argument + containing the line with the trailing CRLF stripped. The default *callback* + prints the line to :data:`sys.stdout`. -.. method:: FTP.login(user='anonymous', passwd='', acct='') + .. method:: FTP.set_pasv(val) - Log in as the given *user*. The *passwd* and *acct* parameters are optional and - default to the empty string. If no *user* is specified, it defaults to - ``'anonymous'``. If *user* is ``'anonymous'``, the default *passwd* is - ``'anonymous@'``. This function should be called only once for each instance, - after a connection has been established; it should not be called at all if a - host and user were given when the instance was created. Most FTP commands are - only allowed after the client has logged in. The *acct* parameter supplies - "accounting information"; few systems implement this. + Enable "passive" mode if *val* is true, otherwise disable passive mode. + Passive mode is on by default. -.. method:: FTP.abort() + .. method:: FTP.storbinary(cmd, fp, blocksize=8192, callback=None, rest=None) - Abort a file transfer that is in progress. Using this does not always work, but - it's worth a try. + Store a file in binary transfer mode. + :param str cmd: + An appropriate ``STOR`` command: :samp:`"STOR {filename}"`. -.. method:: FTP.sendcmd(cmd) + :param fp: + A file object (opened in binary mode) which is read until EOF, + using its :meth:`~io.RawIOBase.read` method in blocks of size *blocksize* + to provide the data to be stored. + :type fp: :term:`file object` - Send a simple command string to the server and return the response string. + :param int blocksize: + The read block size. + Defaults to ``8192``. - .. audit-event:: ftplib.sendcmd self,cmd ftplib.FTP.sendcmd + :param callback: + A single parameter callable that is called + for each block of data sent, + with its single argument being the data as :class:`bytes`. + :type callback: :term:`callable` + :param int rest: + A ``REST`` command to be sent to the server. + See the documentation for the *rest* parameter of the :meth:`transfercmd` method. -.. method:: FTP.voidcmd(cmd) + .. versionchanged:: 3.2 + The *rest* parameter was added. - Send a simple command string to the server and handle the response. Return - nothing if a response code corresponding to success (codes in the range - 200--299) is received. Raise :exc:`error_reply` otherwise. - .. audit-event:: ftplib.sendcmd self,cmd ftplib.FTP.voidcmd + .. method:: FTP.storlines(cmd, fp, callback=None) + Store a file in line mode. *cmd* should be an appropriate + ``STOR`` command (see :meth:`storbinary`). Lines are read until EOF from the + :term:`file object` *fp* (opened in binary mode) using its :meth:`~io.IOBase.readline` + method to provide the data to be stored. *callback* is an optional single + parameter callable that is called on each line after it is sent. -.. method:: FTP.retrbinary(cmd, callback, blocksize=8192, rest=None) - Retrieve a file in binary transfer mode. *cmd* should be an appropriate - ``RETR`` command: ``'RETR filename'``. The *callback* function is called for - each block of data received, with a single bytes argument giving the data - block. The optional *blocksize* argument specifies the maximum chunk size to - read on the low-level socket object created to do the actual transfer (which - will also be the largest size of the data blocks passed to *callback*). A - reasonable default is chosen. *rest* means the same thing as in the - :meth:`transfercmd` method. + .. method:: FTP.transfercmd(cmd, rest=None) + Initiate a transfer over the data connection. If the transfer is active, send an + ``EPRT`` or ``PORT`` command and the transfer command specified by *cmd*, and + accept the connection. If the server is passive, send an ``EPSV`` or ``PASV`` + command, connect to it, and start the transfer command. Either way, return the + socket for the connection. -.. method:: FTP.retrlines(cmd, callback=None) + If optional *rest* is given, a ``REST`` command is sent to the server, passing + *rest* as an argument. *rest* is usually a byte offset into the requested file, + telling the server to restart sending the file's bytes at the requested offset, + skipping over the initial bytes. Note however that the :meth:`transfercmd` + method converts *rest* to a string with the *encoding* parameter specified + at initialization, but no check is performed on the string's contents. If the + server does not recognize the ``REST`` command, an :exc:`error_reply` exception + will be raised. If this happens, simply call :meth:`transfercmd` without a + *rest* argument. - Retrieve a file or directory listing in the encoding specified by the - *encoding* parameter at initialization. - *cmd* should be an appropriate ``RETR`` command (see :meth:`retrbinary`) or - a command such as ``LIST`` or ``NLST`` (usually just the string ``'LIST'``). - ``LIST`` retrieves a list of files and information about those files. - ``NLST`` retrieves a list of file names. - The *callback* function is called for each line with a string argument - containing the line with the trailing CRLF stripped. The default *callback* - prints the line to ``sys.stdout``. + .. method:: FTP.ntransfercmd(cmd, rest=None) -.. method:: FTP.set_pasv(val) + Like :meth:`transfercmd`, but returns a tuple of the data connection and the + expected size of the data. If the expected size could not be computed, ``None`` + will be returned as the expected size. *cmd* and *rest* means the same thing as + in :meth:`transfercmd`. - Enable "passive" mode if *val* is true, otherwise disable passive mode. - Passive mode is on by default. + .. method:: FTP.mlsd(path="", facts=[]) -.. method:: FTP.storbinary(cmd, fp, blocksize=8192, callback=None, rest=None) + List a directory in a standardized format by using ``MLSD`` command + (:rfc:`3659`). If *path* is omitted the current directory is assumed. + *facts* is a list of strings representing the type of information desired + (e.g. ``["type", "size", "perm"]``). Return a generator object yielding a + tuple of two elements for every file found in path. First element is the + file name, the second one is a dictionary containing facts about the file + name. Content of this dictionary might be limited by the *facts* argument + but server is not guaranteed to return all requested facts. - Store a file in binary transfer mode. *cmd* should be an appropriate - ``STOR`` command: ``"STOR filename"``. *fp* is a :term:`file object` - (opened in binary mode) which is read until EOF using its :meth:`~io.IOBase.read` - method in blocks of size *blocksize* to provide the data to be stored. - The *blocksize* argument defaults to 8192. *callback* is an optional single - parameter callable that is called on each block of data after it is sent. - *rest* means the same thing as in the :meth:`transfercmd` method. + .. versionadded:: 3.3 - .. versionchanged:: 3.2 - *rest* parameter added. + .. method:: FTP.nlst(argument[, ...]) + + Return a list of file names as returned by the ``NLST`` command. The + optional *argument* is a directory to list (default is the current server + directory). Multiple arguments can be used to pass non-standard options to + the ``NLST`` command. -.. method:: FTP.storlines(cmd, fp, callback=None) + .. note:: If your server supports the command, :meth:`mlsd` offers a better API. - Store a file in line mode. *cmd* should be an appropriate - ``STOR`` command (see :meth:`storbinary`). Lines are read until EOF from the - :term:`file object` *fp* (opened in binary mode) using its :meth:`~io.IOBase.readline` - method to provide the data to be stored. *callback* is an optional single - parameter callable that is called on each line after it is sent. + .. method:: FTP.dir(argument[, ...]) -.. method:: FTP.transfercmd(cmd, rest=None) + Produce a directory listing as returned by the ``LIST`` command, printing it to + standard output. The optional *argument* is a directory to list (default is the + current server directory). Multiple arguments can be used to pass non-standard + options to the ``LIST`` command. If the last argument is a function, it is used + as a *callback* function as for :meth:`retrlines`; the default prints to + :data:`sys.stdout`. This method returns ``None``. - Initiate a transfer over the data connection. If the transfer is active, send an - ``EPRT`` or ``PORT`` command and the transfer command specified by *cmd*, and - accept the connection. If the server is passive, send an ``EPSV`` or ``PASV`` - command, connect to it, and start the transfer command. Either way, return the - socket for the connection. + .. note:: If your server supports the command, :meth:`mlsd` offers a better API. - If optional *rest* is given, a ``REST`` command is sent to the server, passing - *rest* as an argument. *rest* is usually a byte offset into the requested file, - telling the server to restart sending the file's bytes at the requested offset, - skipping over the initial bytes. Note however that the :meth:`transfercmd` - method converts *rest* to a string with the *encoding* parameter specified - at initialization, but no check is performed on the string's contents. If the - server does not recognize the ``REST`` command, an :exc:`error_reply` exception - will be raised. If this happens, simply call :meth:`transfercmd` without a - *rest* argument. + .. method:: FTP.rename(fromname, toname) -.. method:: FTP.ntransfercmd(cmd, rest=None) + Rename file *fromname* on the server to *toname*. - Like :meth:`transfercmd`, but returns a tuple of the data connection and the - expected size of the data. If the expected size could not be computed, ``None`` - will be returned as the expected size. *cmd* and *rest* means the same thing as - in :meth:`transfercmd`. + .. method:: FTP.delete(filename) -.. method:: FTP.mlsd(path="", facts=[]) + Remove the file named *filename* from the server. If successful, returns the + text of the response, otherwise raises :exc:`error_perm` on permission errors or + :exc:`error_reply` on other errors. - List a directory in a standardized format by using ``MLSD`` command - (:rfc:`3659`). If *path* is omitted the current directory is assumed. - *facts* is a list of strings representing the type of information desired - (e.g. ``["type", "size", "perm"]``). Return a generator object yielding a - tuple of two elements for every file found in path. First element is the - file name, the second one is a dictionary containing facts about the file - name. Content of this dictionary might be limited by the *facts* argument - but server is not guaranteed to return all requested facts. - .. versionadded:: 3.3 + .. method:: FTP.cwd(pathname) + Set the current directory on the server. -.. method:: FTP.nlst(argument[, ...]) - Return a list of file names as returned by the ``NLST`` command. The - optional *argument* is a directory to list (default is the current server - directory). Multiple arguments can be used to pass non-standard options to - the ``NLST`` command. + .. method:: FTP.mkd(pathname) - .. note:: If your server supports the command, :meth:`mlsd` offers a better API. + Create a new directory on the server. -.. method:: FTP.dir(argument[, ...]) + .. method:: FTP.pwd() - Produce a directory listing as returned by the ``LIST`` command, printing it to - standard output. The optional *argument* is a directory to list (default is the - current server directory). Multiple arguments can be used to pass non-standard - options to the ``LIST`` command. If the last argument is a function, it is used - as a *callback* function as for :meth:`retrlines`; the default prints to - ``sys.stdout``. This method returns ``None``. + Return the pathname of the current directory on the server. - .. note:: If your server supports the command, :meth:`mlsd` offers a better API. + .. method:: FTP.rmd(dirname) -.. method:: FTP.rename(fromname, toname) + Remove the directory named *dirname* on the server. - Rename file *fromname* on the server to *toname*. + .. method:: FTP.size(filename) -.. method:: FTP.delete(filename) + Request the size of the file named *filename* on the server. On success, the + size of the file is returned as an integer, otherwise ``None`` is returned. + Note that the ``SIZE`` command is not standardized, but is supported by many + common server implementations. - Remove the file named *filename* from the server. If successful, returns the - text of the response, otherwise raises :exc:`error_perm` on permission errors or - :exc:`error_reply` on other errors. + .. method:: FTP.quit() -.. method:: FTP.cwd(pathname) + Send a ``QUIT`` command to the server and close the connection. This is the + "polite" way to close a connection, but it may raise an exception if the server + responds with an error to the ``QUIT`` command. This implies a call to the + :meth:`close` method which renders the :class:`FTP` instance useless for + subsequent calls (see below). - Set the current directory on the server. + .. method:: FTP.close() -.. method:: FTP.mkd(pathname) + Close the connection unilaterally. This should not be applied to an already + closed connection such as after a successful call to :meth:`~FTP.quit`. + After this call the :class:`FTP` instance should not be used any more (after + a call to :meth:`close` or :meth:`~FTP.quit` you cannot reopen the + connection by issuing another :meth:`login` method). - Create a new directory on the server. +FTP_TLS objects +^^^^^^^^^^^^^^^ -.. method:: FTP.pwd() +.. class:: FTP_TLS(host='', user='', passwd='', acct='', *, context=None, \ + timeout=None, source_address=None, encoding='utf-8') - Return the pathname of the current directory on the server. + An :class:`FTP` subclass which adds TLS support to FTP as described in + :rfc:`4217`. + Connect to port 21 implicitly securing the FTP control connection + before authenticating. + .. note:: + The user must explicitly secure the data connection + by calling the :meth:`prot_p` method. -.. method:: FTP.rmd(dirname) + :param str host: + The hostname to connect to. + If given, :code:`connect(host)` is implicitly called by the constructor. - Remove the directory named *dirname* on the server. + :param str user: + |param_doc_user| + If given, :code:`login(host, passwd, acct)` is implicitly called + by the constructor. + :param str passwd: + |param_doc_passwd| -.. method:: FTP.size(filename) + :param str acct: + |param_doc_acct| - Request the size of the file named *filename* on the server. On success, the - size of the file is returned as an integer, otherwise ``None`` is returned. - Note that the ``SIZE`` command is not standardized, but is supported by many - common server implementations. + :param context: + An SSL context object which allows bundling SSL configuration options, + certificates and private keys into a single, potentially long-lived, + structure. + Please read :ref:`ssl-security` for best practices. + :type context: :class:`ssl.SSLContext` + :param timeout: + A timeout in seconds for blocking operations like :meth:`~FTP.connect` + (default: the global default timeout setting). + :type timeout: int | None -.. method:: FTP.quit() + :param source_address: + |param_doc_source_address| + :type source_address: tuple | None - Send a ``QUIT`` command to the server and close the connection. This is the - "polite" way to close a connection, but it may raise an exception if the server - responds with an error to the ``QUIT`` command. This implies a call to the - :meth:`close` method which renders the :class:`FTP` instance useless for - subsequent calls (see below). + :param str encoding: + |param_doc_encoding| + .. versionadded:: 3.2 -.. method:: FTP.close() + .. versionchanged:: 3.3 + Added the *source_address* parameter. - Close the connection unilaterally. This should not be applied to an already - closed connection such as after a successful call to :meth:`~FTP.quit`. - After this call the :class:`FTP` instance should not be used any more (after - a call to :meth:`close` or :meth:`~FTP.quit` you cannot reopen the - connection by issuing another :meth:`login` method). + .. versionchanged:: 3.4 + The class now supports hostname check with + :attr:`ssl.SSLContext.check_hostname` and *Server Name Indication* (see + :const:`ssl.HAS_SNI`). + .. versionchanged:: 3.9 + If the *timeout* parameter is set to be zero, it will raise a + :class:`ValueError` to prevent the creation of a non-blocking socket. + The *encoding* parameter was added, and the default was changed from + Latin-1 to UTF-8 to follow :rfc:`2640`. -FTP_TLS Objects ---------------- + .. versionchanged:: 3.12 + The deprecated *keyfile* and *certfile* parameters have been removed. -:class:`FTP_TLS` class inherits from :class:`FTP`, defining these additional objects: + Here's a sample session using the :class:`FTP_TLS` class:: -.. attribute:: FTP_TLS.ssl_version + >>> ftps = FTP_TLS('ftp.pureftpd.org') + >>> ftps.login() + '230 Anonymous user logged in' + >>> ftps.prot_p() + '200 Data protection level set to "private"' + >>> ftps.nlst() + ['6jack', 'OpenBSD', 'antilink', 'blogbench', 'bsdcam', 'clockspeed', 'djbdns-jedi', 'docs', 'eaccelerator-jedi', 'favicon.ico', 'francotone', 'fugu', 'ignore', 'libpuzzle', 'metalog', 'minidentd', 'misc', 'mysql-udf-global-user-variables', 'php-jenkins-hash', 'php-skein-hash', 'php-webdav', 'phpaudit', 'phpbench', 'pincaster', 'ping', 'posto', 'pub', 'public', 'public_keys', 'pure-ftpd', 'qscan', 'qtc', 'sharedance', 'skycache', 'sound', 'tmp', 'ucarp'] - The SSL version to use (defaults to :data:`ssl.PROTOCOL_SSLv23`). + :class:`!FTP_TLS` class inherits from :class:`FTP`, + defining these additional methods and attributes: -.. method:: FTP_TLS.auth() + .. attribute:: FTP_TLS.ssl_version - Set up a secure control connection by using TLS or SSL, depending on what - is specified in the :attr:`ssl_version` attribute. + The SSL version to use (defaults to :data:`ssl.PROTOCOL_SSLv23`). - .. versionchanged:: 3.4 - The method now supports hostname check with - :attr:`ssl.SSLContext.check_hostname` and *Server Name Indication* (see - :const:`ssl.HAS_SNI`). + .. method:: FTP_TLS.auth() + + Set up a secure control connection by using TLS or SSL, depending on what + is specified in the :attr:`ssl_version` attribute. -.. method:: FTP_TLS.ccc() + .. versionchanged:: 3.4 + The method now supports hostname check with + :attr:`ssl.SSLContext.check_hostname` and *Server Name Indication* (see + :const:`ssl.HAS_SNI`). - Revert control channel back to plaintext. This can be useful to take - advantage of firewalls that know how to handle NAT with non-secure FTP - without opening fixed ports. + .. method:: FTP_TLS.ccc() - .. versionadded:: 3.3 + Revert control channel back to plaintext. This can be useful to take + advantage of firewalls that know how to handle NAT with non-secure FTP + without opening fixed ports. -.. method:: FTP_TLS.prot_p() + .. versionadded:: 3.3 - Set up secure data connection. + .. method:: FTP_TLS.prot_p() -.. method:: FTP_TLS.prot_c() + Set up secure data connection. + + .. method:: FTP_TLS.prot_c() + + Set up clear text data connection. + + +Module variables +^^^^^^^^^^^^^^^^ + +.. exception:: error_reply + + Exception raised when an unexpected reply is received from the server. - Set up clear text data connection. + +.. exception:: error_temp + + Exception raised when an error code signifying a temporary error (response + codes in the range 400--499) is received. + + +.. exception:: error_perm + + Exception raised when an error code signifying a permanent error (response + codes in the range 500--599) is received. + + +.. exception:: error_proto + + Exception raised when a reply is received from the server that does not fit + the response specifications of the File Transfer Protocol, i.e. begin with a + digit in the range 1--5. + + +.. data:: all_errors + + The set of all exceptions (as a tuple) that methods of :class:`FTP` + instances may raise as a result of problems with the FTP connection (as + opposed to programming errors made by the caller). This set includes the + four exceptions listed above as well as :exc:`OSError` and :exc:`EOFError`. + + +.. seealso:: + + Module :mod:`netrc` + Parser for the :file:`.netrc` file format. The file :file:`.netrc` is + typically used by FTP clients to load user authentication information + before prompting the user. diff --git a/Doc/library/functions.rst b/Doc/library/functions.rst index c731b6fd333275..27fce5aa0f1a63 100644 --- a/Doc/library/functions.rst +++ b/Doc/library/functions.rst @@ -668,16 +668,15 @@ are always available. They are listed here in alphabetical order. sign: "+" | "-" infinity: "Infinity" | "inf" nan: "nan" - digitpart: `!digit` (["_"] `!digit`)* + digit: <a Unicode decimal digit, i.e. characters in Unicode general category Nd> + digitpart: `digit` (["_"] `digit`)* number: [`digitpart`] "." `digitpart` | `digitpart` ["."] exponent: ("e" | "E") ["+" | "-"] `digitpart` floatnumber: number [`exponent`] floatvalue: [`sign`] (`floatnumber` | `infinity` | `nan`) - Here ``digit`` is a Unicode decimal digit (character in the Unicode general - category ``Nd``). Case is not significant, so, for example, "inf", "Inf", - "INFINITY", and "iNfINity" are all acceptable spellings for positive - infinity. + Case is not significant, so, for example, "inf", "Inf", "INFINITY", and + "iNfINity" are all acceptable spellings for positive infinity. Otherwise, if the argument is an integer or a floating point number, a floating point number with the same value (within Python's floating point @@ -1074,8 +1073,8 @@ are always available. They are listed here in alphabetical order. such as ``sorted(iterable, key=keyfunc, reverse=True)[0]`` and ``heapq.nlargest(1, iterable, key=keyfunc)``. - .. versionadded:: 3.4 - The *default* keyword-only argument. + .. versionchanged:: 3.4 + Added the *default* keyword-only parameter. .. versionchanged:: 3.8 The *key* can be ``None``. @@ -1112,8 +1111,8 @@ are always available. They are listed here in alphabetical order. such as ``sorted(iterable, key=keyfunc)[0]`` and ``heapq.nsmallest(1, iterable, key=keyfunc)``. - .. versionadded:: 3.4 - The *default* keyword-only argument. + .. versionchanged:: 3.4 + Added the *default* keyword-only parameter. .. versionchanged:: 3.8 The *key* can be ``None``. @@ -1800,6 +1799,13 @@ are always available. They are listed here in alphabetical order. the second argument is a type, ``issubclass(type2, type)`` must be true (this is useful for classmethods). + When called directly within an ordinary method of a class, both arguments may + be omitted ("zero-argument :func:`!super`"). In this case, *type* will be the + enclosing class, and *obj* will be the first argument of the immediately + enclosing function (typically ``self``). (This means that zero-argument + :func:`!super` will not work as expected within nested functions, including + generator expressions, which implicitly create nested functions.) + There are two typical use cases for *super*. In a class hierarchy with single inheritance, *super* can be used to refer to parent classes without naming them explicitly, thus making the code more maintainable. This use diff --git a/Doc/library/functools.rst b/Doc/library/functools.rst index 69ec1eb3ecd89d..20fcbe76c36985 100644 --- a/Doc/library/functools.rst +++ b/Doc/library/functools.rst @@ -194,7 +194,7 @@ The :mod:`functools` module defines the following functions: In contrast, the tuple arguments ``('answer', Decimal(42))`` and ``('answer', Fraction(42))`` are treated as equivalent. - The wrapped function is instrumented with a :func:`cache_parameters` + The wrapped function is instrumented with a :func:`!cache_parameters` function that returns a new :class:`dict` showing the values for *maxsize* and *typed*. This is for information purposes only. Mutating the values has no effect. @@ -275,8 +275,8 @@ The :mod:`functools` module defines the following functions: .. versionchanged:: 3.8 Added the *user_function* option. - .. versionadded:: 3.9 - Added the function :func:`cache_parameters` + .. versionchanged:: 3.9 + Added the function :func:`!cache_parameters` .. decorator:: total_ordering @@ -742,7 +742,7 @@ have three read-only attributes: called. :class:`partial` objects are like :class:`function` objects in that they are -callable, weak referencable, and can have attributes. There are some important +callable, weak referenceable, and can have attributes. There are some important differences. For instance, the :attr:`~definition.__name__` and :attr:`__doc__` attributes are not created automatically. Also, :class:`partial` objects defined in classes behave like static methods and do not transform into bound methods diff --git a/Doc/library/gc.rst b/Doc/library/gc.rst index 82277aa52aee01..e36a71af2b64ab 100644 --- a/Doc/library/gc.rst +++ b/Doc/library/gc.rst @@ -96,7 +96,7 @@ The :mod:`gc` module provides the following functions: .. versionadded:: 3.4 -.. function:: set_threshold(threshold0[, threshold1[, threshold2]]) +.. function:: set_threshold(threshold0, [threshold1, [threshold2]]) Set the garbage collection thresholds (the collection frequency). Setting *threshold0* to zero disables collection. diff --git a/Doc/library/glob.rst b/Doc/library/glob.rst index 6e4f72c19ff4c9..19a0bbba8966ba 100644 --- a/Doc/library/glob.rst +++ b/Doc/library/glob.rst @@ -147,8 +147,9 @@ The :mod:`glob` module defines the following functions: .. seealso:: - :meth:`pathlib.PurePath.match` and :meth:`pathlib.Path.glob` methods, - which call this function to implement pattern matching and globbing. + :meth:`pathlib.PurePath.full_match` and :meth:`pathlib.Path.glob` + methods, which call this function to implement pattern matching and + globbing. .. versionadded:: 3.13 diff --git a/Doc/library/gzip.rst b/Doc/library/gzip.rst index 50cde09fa10a9d..79be215a766045 100644 --- a/Doc/library/gzip.rst +++ b/Doc/library/gzip.rst @@ -61,7 +61,7 @@ The module defines the following items: .. exception:: BadGzipFile - An exception raised for invalid gzip files. It inherits :exc:`OSError`. + An exception raised for invalid gzip files. It inherits from :exc:`OSError`. :exc:`EOFError` and :exc:`zlib.error` can also be raised for invalid gzip files. @@ -287,4 +287,3 @@ Command line options .. option:: -h, --help Show the help message. - diff --git a/Doc/library/heapq.rst b/Doc/library/heapq.rst index 8b00f7b27959b6..ddbada13bddf5b 100644 --- a/Doc/library/heapq.rst +++ b/Doc/library/heapq.rst @@ -270,7 +270,7 @@ winner. The simplest algorithmic way to remove it and find the "next" winner is to move some loser (let's say cell 30 in the diagram above) into the 0 position, and then percolate this new 0 down the tree, exchanging values, until the invariant is re-established. This is clearly logarithmic on the total number of -items in the tree. By iterating over all items, you get an O(n log n) sort. +items in the tree. By iterating over all items, you get an *O*\ (*n* log *n*) sort. A nice feature of this sort is that you can efficiently insert new items while the sort is going on, provided that the inserted items are not "better" than the diff --git a/Doc/library/http.client.rst b/Doc/library/http.client.rst index c46314fc5e253b..7e4502064f22a1 100644 --- a/Doc/library/http.client.rst +++ b/Doc/library/http.client.rst @@ -92,7 +92,7 @@ The module provides the following classes: .. versionchanged:: 3.4.3 This class now performs all the necessary certificate and hostname checks by default. To revert to the previous, unverified, behavior - :func:`ssl._create_unverified_context` can be passed to the *context* + :func:`!ssl._create_unverified_context` can be passed to the *context* parameter. .. versionchanged:: 3.8 @@ -103,7 +103,7 @@ The module provides the following classes: .. versionchanged:: 3.10 This class now sends an ALPN extension with protocol indicator ``http/1.1`` when no *context* is given. Custom *context* should set - ALPN protocols with :meth:`~ssl.SSLContext.set_alpn_protocol`. + ALPN protocols with :meth:`~ssl.SSLContext.set_alpn_protocols`. .. versionchanged:: 3.12 The deprecated *key_file*, *cert_file* and *check_hostname* parameters @@ -124,7 +124,7 @@ This module provides the following function: .. function:: parse_headers(fp) Parse the headers from a file pointer *fp* representing a HTTP - request/response. The file has to be a :class:`BufferedIOBase` reader + request/response. The file has to be a :class:`~io.BufferedIOBase` reader (i.e. not text) and must provide a valid :rfc:`2822` style header. This function returns an instance of :class:`http.client.HTTPMessage` @@ -311,7 +311,7 @@ HTTPConnection Objects :class:`str` or bytes-like object that is not also a file as the body representation. - .. versionadded:: 3.2 + .. versionchanged:: 3.2 *body* can now be an iterable. .. versionchanged:: 3.6 @@ -416,7 +416,7 @@ HTTPConnection Objects .. versionadded:: 3.7 -As an alternative to using the :meth:`request` method described above, you can +As an alternative to using the :meth:`~HTTPConnection.request` method described above, you can also send your request step by step, by using the four functions below. @@ -461,9 +461,8 @@ also send your request step by step, by using the four functions below. This is to avoid premature termination of the read of the request by the target server due to malformed encoding. - .. versionadded:: 3.6 - Chunked encoding support. The *encode_chunked* parameter was - added. + .. versionchanged:: 3.6 + Added chunked encoding support and the *encode_chunked* parameter. .. method:: HTTPConnection.send(data) @@ -648,6 +647,8 @@ method attribute. Here is an example session that uses the ``PUT`` method:: HTTPMessage Objects ------------------- +.. class:: HTTPMessage(email.message.Message) + An :class:`http.client.HTTPMessage` instance holds the headers from an HTTP response. It is implemented using the :class:`email.message.Message` class. diff --git a/Doc/library/http.server.rst b/Doc/library/http.server.rst index 64bddd23f82933..bc59d3d17912fd 100644 --- a/Doc/library/http.server.rst +++ b/Doc/library/http.server.rst @@ -328,8 +328,8 @@ provides three different variants: or the current directory if *directory* is not provided, directly mapping the directory structure to HTTP requests. - .. versionadded:: 3.7 - The *directory* parameter. + .. versionchanged:: 3.7 + Added the *directory* parameter. .. versionchanged:: 3.9 The *directory* parameter accepts a :term:`path-like object`. @@ -438,11 +438,11 @@ to bind to localhost only:: python -m http.server --bind 127.0.0.1 -.. versionadded:: 3.4 - ``--bind`` argument was introduced. +.. versionchanged:: 3.4 + Added the ``--bind`` option. -.. versionadded:: 3.8 - ``--bind`` argument enhanced to support IPv6 +.. versionchanged:: 3.8 + Support IPv6 in the ``--bind`` option. By default, the server uses the current directory. The option ``-d/--directory`` specifies a directory to which it should serve the files. For example, @@ -450,8 +450,8 @@ the following command uses a specific directory:: python -m http.server --directory /tmp/ -.. versionadded:: 3.7 - ``--directory`` argument was introduced. +.. versionchanged:: 3.7 + Added the ``--directory`` option. By default, the server is conformant to HTTP/1.0. The option ``-p/--protocol`` specifies the HTTP version to which the server is conformant. For example, the @@ -459,8 +459,8 @@ following command runs an HTTP/1.1 conformant server:: python -m http.server --protocol HTTP/1.1 -.. versionadded:: 3.11 - ``--protocol`` argument was introduced. +.. versionchanged:: 3.11 + Added the ``--protocol`` option. .. class:: CGIHTTPRequestHandler(request, client_address, server) @@ -537,5 +537,5 @@ default :class:`BaseHTTPRequestHandler` ``.log_message`` implementation. This could allow remote clients connecting to your server to send nefarious control codes to your terminal. -.. versionadded:: 3.12 +.. versionchanged:: 3.12 Control characters are scrubbed in stderr logs. diff --git a/Doc/library/idle.rst b/Doc/library/idle.rst index e710d0bacf3fee..249dc0ea6ba735 100644 --- a/Doc/library/idle.rst +++ b/Doc/library/idle.rst @@ -18,8 +18,6 @@ IDLE is Python's Integrated Development and Learning Environment. IDLE has the following features: -* coded in 100% pure Python, using the :mod:`tkinter` GUI toolkit - * cross-platform: works mostly the same on Windows, Unix, and macOS * Python shell window (interactive interpreter) with colorizing @@ -422,41 +420,34 @@ and that other files do not. Run Python code with the Run menu. Key bindings ^^^^^^^^^^^^ -In this section, 'C' refers to the :kbd:`Control` key on Windows and Unix and -the :kbd:`Command` key on macOS. - -* :kbd:`Backspace` deletes to the left; :kbd:`Del` deletes to the right - -* :kbd:`C-Backspace` delete word left; :kbd:`C-Del` delete word to the right - -* Arrow keys and :kbd:`Page Up`/:kbd:`Page Down` to move around - -* :kbd:`C-LeftArrow` and :kbd:`C-RightArrow` moves by words +The IDLE insertion cursor is a thin vertical bar between character +positions. When characters are entered, the insertion cursor and +everything to its right moves right one character and +the new character is entered in the new space. -* :kbd:`Home`/:kbd:`End` go to begin/end of line +Several non-character keys move the cursor and possibly +delete characters. Deletion does not puts text on the clipboard, +but IDLE has an undo list. Wherever this doc discusses keys, +'C' refers to the :kbd:`Control` key on Windows and +Unix and the :kbd:`Command` key on macOS. (And all such dicussions +assume that the keys have not been re-bound to something else.) -* :kbd:`C-Home`/:kbd:`C-End` go to begin/end of file +* Arrow keys move the cursor one character or line. -* Some useful Emacs bindings are inherited from Tcl/Tk: +* :kbd:`C-LeftArrow` and :kbd:`C-RightArrow` moves left or right one word. - * :kbd:`C-a` beginning of line +* :kbd:`Home` and :kbd:`End` go to the beginning or end of the line. - * :kbd:`C-e` end of line +* :kbd:`Page Up` and :kbd:`Page Down` go up or down one screen. - * :kbd:`C-k` kill line (but doesn't put it in clipboard) +* :kbd:`C-Home` and :kbd:`C-End` go to beginning or end of the file. - * :kbd:`C-l` center window around the insertion point +* :kbd:`Backspace` and :kbd:`Del` (or :kbd:`C-d`) delete the previous + or next character. - * :kbd:`C-b` go backward one character without deleting (usually you can - also use the cursor key for this) +* :kbd:`C-Backspace` and :kbd:`C-Del` delete one word left or right. - * :kbd:`C-f` go forward one character without deleting (usually you can - also use the cursor key for this) - - * :kbd:`C-p` go up one line (usually you can also use the cursor key for - this) - - * :kbd:`C-d` delete next character +* :kbd:`C-k` deletes ('kills') everything to the right. Standard keybindings (like :kbd:`C-c` to copy and :kbd:`C-v` to paste) may work. Keybindings are selected in the Configure IDLE dialog. @@ -611,23 +602,18 @@ when one requests a restart on the Shell menu, or when one runs code in an editor window. The editing features described in previous subsections work when entering -code interactively. IDLE's Shell window also responds to the following keys. - -* :kbd:`C-c` interrupts executing command - -* :kbd:`C-d` sends end-of-file; closes window if typed at a ``>>>`` prompt - -* :kbd:`Alt-/` (Expand word) is also useful to reduce typing +code interactively. IDLE's Shell window also responds to the following: - Command history +* :kbd:`C-c` attemps to interrupt statement execution (but may fail). - * :kbd:`Alt-p` retrieves previous command matching what you have typed. On - macOS use :kbd:`C-p`. +* :kbd:`C-d` closes Shell if typed at a ``>>>`` prompt. - * :kbd:`Alt-n` retrieves next. On macOS use :kbd:`C-n`. +* :kbd:`Alt-p` and :kbd:`Alt-n` (:kbd:`C-p` and :kbd:`C-n` on macOS) + retrieve to the current prompt the previous or next previously + entered statement that matches anything already typed. - * :kbd:`Return` while the cursor is on any previous command - retrieves that command +* :kbd:`Return` while the cursor is on any previous statement + appends the latter to anything already typed at the prompt. Text colors ^^^^^^^^^^^ diff --git a/Doc/library/imaplib.rst b/Doc/library/imaplib.rst index 1f774e64b0eae3..d5c868def3b64f 100644 --- a/Doc/library/imaplib.rst +++ b/Doc/library/imaplib.rst @@ -531,7 +531,7 @@ An :class:`IMAP4` instance has the following methods: allowed creation of such tags, and popular IMAP servers, such as Gmail, accept and produce such flags. There are non-Python programs which also create such tags. Although it is an RFC violation and IMAP clients and - servers are supposed to be strict, imaplib nonetheless continues to allow + servers are supposed to be strict, imaplib still continues to allow such tags to be created for backward compatibility reasons, and as of Python 3.6, handles them if they are sent from the server, since this improves real-world compatibility. diff --git a/Doc/library/io.rst b/Doc/library/io.rst index 6736aa9ee2b0ef..8eb531aa4ea248 100644 --- a/Doc/library/io.rst +++ b/Doc/library/io.rst @@ -466,7 +466,7 @@ I/O Base Classes .. class:: RawIOBase - Base class for raw binary streams. It inherits :class:`IOBase`. + Base class for raw binary streams. It inherits from :class:`IOBase`. Raw binary streams typically provide low-level access to an underlying OS device or API, and do not try to encapsulate it in high-level primitives @@ -519,7 +519,7 @@ I/O Base Classes .. class:: BufferedIOBase Base class for binary streams that support some kind of buffering. - It inherits :class:`IOBase`. + It inherits from :class:`IOBase`. The main difference with :class:`RawIOBase` is that methods :meth:`read`, :meth:`readinto` and :meth:`write` will try (respectively) to read as much @@ -633,7 +633,7 @@ Raw File I/O .. class:: FileIO(name, mode='r', closefd=True, opener=None) A raw binary stream representing an OS-level file containing bytes data. It - inherits :class:`RawIOBase`. + inherits from :class:`RawIOBase`. The *name* can be one of two things: @@ -696,7 +696,7 @@ than raw I/O does. .. class:: BytesIO(initial_bytes=b'') - A binary stream using an in-memory bytes buffer. It inherits + A binary stream using an in-memory bytes buffer. It inherits from :class:`BufferedIOBase`. The buffer is discarded when the :meth:`~IOBase.close` method is called. @@ -745,7 +745,7 @@ than raw I/O does. .. class:: BufferedReader(raw, buffer_size=DEFAULT_BUFFER_SIZE) A buffered binary stream providing higher-level access to a readable, non - seekable :class:`RawIOBase` raw binary stream. It inherits + seekable :class:`RawIOBase` raw binary stream. It inherits from :class:`BufferedIOBase`. When reading data from this object, a larger amount of data may be @@ -783,7 +783,7 @@ than raw I/O does. .. class:: BufferedWriter(raw, buffer_size=DEFAULT_BUFFER_SIZE) A buffered binary stream providing higher-level access to a writeable, non - seekable :class:`RawIOBase` raw binary stream. It inherits + seekable :class:`RawIOBase` raw binary stream. It inherits from :class:`BufferedIOBase`. When writing to this object, data is normally placed into an internal @@ -818,7 +818,7 @@ than raw I/O does. .. class:: BufferedRandom(raw, buffer_size=DEFAULT_BUFFER_SIZE) A buffered binary stream providing higher-level access to a seekable - :class:`RawIOBase` raw binary stream. It inherits :class:`BufferedReader` + :class:`RawIOBase` raw binary stream. It inherits from :class:`BufferedReader` and :class:`BufferedWriter`. The constructor creates a reader and writer for a seekable raw stream, given @@ -834,7 +834,7 @@ than raw I/O does. A buffered binary stream providing higher-level access to two non seekable :class:`RawIOBase` raw binary streams---one readable, the other writeable. - It inherits :class:`BufferedIOBase`. + It inherits from :class:`BufferedIOBase`. *reader* and *writer* are :class:`RawIOBase` objects that are readable and writeable respectively. If the *buffer_size* is omitted it defaults to @@ -857,7 +857,7 @@ Text I/O .. class:: TextIOBase Base class for text streams. This class provides a character and line based - interface to stream I/O. It inherits :class:`IOBase`. + interface to stream I/O. It inherits from :class:`IOBase`. :class:`TextIOBase` provides or overrides these data attributes and methods in addition to those from :class:`IOBase`: @@ -946,7 +946,7 @@ Text I/O line_buffering=False, write_through=False) A buffered text stream providing higher-level access to a - :class:`BufferedIOBase` buffered binary stream. It inherits + :class:`BufferedIOBase` buffered binary stream. It inherits from :class:`TextIOBase`. *encoding* gives the name of the encoding that the stream will be decoded or @@ -1073,7 +1073,7 @@ Text I/O .. class:: StringIO(initial_value='', newline='\n') - A text stream using an in-memory text buffer. It inherits + A text stream using an in-memory text buffer. It inherits from :class:`TextIOBase`. The text buffer is discarded when the :meth:`~IOBase.close` method is @@ -1124,7 +1124,7 @@ Text I/O .. class:: IncrementalNewlineDecoder A helper codec that decodes newlines for :term:`universal newlines` mode. - It inherits :class:`codecs.IncrementalDecoder`. + It inherits from :class:`codecs.IncrementalDecoder`. Performance diff --git a/Doc/library/itertools.rst b/Doc/library/itertools.rst index c016fb76bfd0a0..338a5f9615aae3 100644 --- a/Doc/library/itertools.rst +++ b/Doc/library/itertools.rst @@ -803,11 +803,11 @@ which incur interpreter overhead. import random def take(n, iterable): - "Return first n items of the iterable as a list" + "Return first n items of the iterable as a list." return list(islice(iterable, n)) def prepend(value, iterable): - "Prepend a single value in front of an iterable" + "Prepend a single value in front of an iterable." # prepend(1, [2, 3, 4]) --> 1 2 3 4 return chain([value], iterable) @@ -825,15 +825,15 @@ which incur interpreter overhead. return starmap(func, repeat(args, times)) def flatten(list_of_lists): - "Flatten one level of nesting" + "Flatten one level of nesting." return chain.from_iterable(list_of_lists) def ncycles(iterable, n): - "Returns the sequence elements n times" + "Returns the sequence elements n times." return chain.from_iterable(repeat(tuple(iterable), n)) def tail(n, iterable): - "Return an iterator over the last n items" + "Return an iterator over the last n items." # tail(3, 'ABCDEFG') --> E F G return iter(collections.deque(iterable, maxlen=n)) @@ -848,7 +848,7 @@ which incur interpreter overhead. next(islice(iterator, n, n), None) def nth(iterable, n, default=None): - "Returns the nth item or a default value" + "Returns the nth item or a default value." return next(islice(iterable, n, None), default) def quantify(iterable, pred=bool): @@ -856,7 +856,7 @@ which incur interpreter overhead. return sum(map(pred, iterable)) def all_equal(iterable): - "Returns True if all the elements are equal to each other" + "Returns True if all the elements are equal to each other." g = groupby(iterable) return next(g, True) and not next(g, False) @@ -873,6 +873,30 @@ which incur interpreter overhead. # first_true([a,b], x, f) --> a if f(a) else b if f(b) else x return next(filter(pred, iterable), default) + def unique_everseen(iterable, key=None): + "List unique elements, preserving order. Remember all elements ever seen." + # unique_everseen('AAAABBBCCDAABBB') --> A B C D + # unique_everseen('ABBcCAD', str.casefold) --> A B c D + seen = set() + if key is None: + for element in filterfalse(seen.__contains__, iterable): + seen.add(element) + yield element + else: + for element in iterable: + k = key(element) + if k not in seen: + seen.add(k) + yield element + + def unique_justseen(iterable, key=None): + "List unique elements, preserving order. Remember only the element just seen." + # unique_justseen('AAAABBBCCDAABBB') --> A B C D A B + # unique_justseen('ABBcCAD', str.casefold) --> A B c A D + if key is None: + return map(operator.itemgetter(0), groupby(iterable)) + return map(next, map(operator.itemgetter(1), groupby(iterable, key))) + def iter_index(iterable, value, start=0, stop=None): "Return indices where a value occurs in a sequence or iterable." # iter_index('AABCADEAF', 'A') --> 0 1 4 7 @@ -893,31 +917,17 @@ which incur interpreter overhead. except ValueError: pass - def iter_except(func, exception, first=None): - """ Call a function repeatedly until an exception is raised. - - Converts a call-until-exception interface to an iterator interface. - Like builtins.iter(func, sentinel) but uses an exception instead - of a sentinel to end the loop. - - Examples: - iter_except(functools.partial(heappop, h), IndexError) # priority queue iterator - iter_except(d.popitem, KeyError) # non-blocking dict iterator - iter_except(d.popleft, IndexError) # non-blocking deque iterator - iter_except(q.get_nowait, Queue.Empty) # loop over a producer Queue - iter_except(s.pop, KeyError) # non-blocking set iterator - - """ - try: - if first is not None: - yield first() # For database APIs needing an initial cast to db.first() - while True: - yield func() - except exception: - pass + def sliding_window(iterable, n): + "Collect data into overlapping fixed-length chunks or blocks." + # sliding_window('ABCDEFG', 4) --> ABCD BCDE CDEF DEFG + it = iter(iterable) + window = collections.deque(islice(it, n-1), maxlen=n) + for x in it: + window.append(x) + yield tuple(window) def grouper(iterable, n, *, incomplete='fill', fillvalue=None): - "Collect data into non-overlapping fixed-length chunks or blocks" + "Collect data into non-overlapping fixed-length chunks or blocks." # grouper('ABCDEFG', 3, fillvalue='x') --> ABC DEF Gxx # grouper('ABCDEFG', 3, incomplete='strict') --> ABC DEF ValueError # grouper('ABCDEFG', 3, incomplete='ignore') --> ABC DEF @@ -932,16 +942,9 @@ which incur interpreter overhead. case _: raise ValueError('Expected fill, strict, or ignore') - def sliding_window(iterable, n): - # sliding_window('ABCDEFG', 4) --> ABCD BCDE CDEF DEFG - it = iter(iterable) - window = collections.deque(islice(it, n-1), maxlen=n) - for x in it: - window.append(x) - yield tuple(window) - def roundrobin(*iterables): - "roundrobin('ABC', 'D', 'EF') --> A D E B F C" + "Visit input iterables in a cycle until each is exhausted." + # roundrobin('ABC', 'D', 'EF') --> A D E B F C # Recipe credited to George Sakkis num_active = len(iterables) nexts = cycle(iter(it).__next__ for it in iterables) @@ -964,11 +967,43 @@ which incur interpreter overhead. return filterfalse(pred, t1), filter(pred, t2) def subslices(seq): - "Return all contiguous non-empty subslices of a sequence" + "Return all contiguous non-empty subslices of a sequence." # subslices('ABCD') --> A AB ABC ABCD B BC BCD C CD D slices = starmap(slice, combinations(range(len(seq) + 1), 2)) return map(operator.getitem, repeat(seq), slices) + def iter_except(func, exception, first=None): + """ Call a function repeatedly until an exception is raised. + + Converts a call-until-exception interface to an iterator interface. + Like builtins.iter(func, sentinel) but uses an exception instead + of a sentinel to end the loop. + + Priority queue iterator: + iter_except(functools.partial(heappop, h), IndexError) + + Non-blocking dictionary iterator: + iter_except(d.popitem, KeyError) + + Non-blocking deque iterator: + iter_except(d.popleft, IndexError) + + Non-blocking iterator over a producer Queue: + iter_except(q.get_nowait, Queue.Empty) + + Non-blocking set iterator: + iter_except(s.pop, KeyError) + + """ + try: + if first is not None: + # For database APIs needing an initial call to db.first() + yield first() + while True: + yield func() + except exception: + pass + def before_and_after(predicate, it): """ Variant of takewhile() that allows complete access to the remainder of the iterator. @@ -980,12 +1015,12 @@ which incur interpreter overhead. >>> ''.join(remainder) # takewhile() would lose the 'd' 'dEfGhI' - Note that the first iterator must be fully - consumed before the second iterator can - generate valid results. + Note that the true iterator must be fully consumed + before the remainder iterator can generate valid results. """ it = iter(it) transition = [] + def true_iterator(): for elem in it: if predicate(elem): @@ -993,41 +1028,8 @@ which incur interpreter overhead. else: transition.append(elem) return - def remainder_iterator(): - yield from transition - yield from it - return true_iterator(), remainder_iterator() - def unique_everseen(iterable, key=None): - "List unique elements, preserving order. Remember all elements ever seen." - # unique_everseen('AAAABBBCCDAABBB') --> A B C D - # unique_everseen('ABBcCAD', str.lower) --> A B c D - seen = set() - if key is None: - for element in filterfalse(seen.__contains__, iterable): - seen.add(element) - yield element - # For order preserving deduplication, - # a faster but non-lazy solution is: - # yield from dict.fromkeys(iterable) - else: - for element in iterable: - k = key(element) - if k not in seen: - seen.add(k) - yield element - # For use cases that allow the last matching element to be returned, - # a faster but non-lazy solution is: - # t1, t2 = tee(iterable) - # yield from dict(zip(map(key, t1), t2)).values() - - def unique_justseen(iterable, key=None): - "List unique elements, preserving order. Remember only the element just seen." - # unique_justseen('AAAABBBCCDAABBB') --> A B C D A B - # unique_justseen('ABBcCAD', str.lower) --> A B c A D - if key is None: - return map(operator.itemgetter(0), groupby(iterable)) - return map(next, map(operator.itemgetter(1), groupby(iterable, key))) + return true_iterator(), chain(transition, it) The following recipes have a more mathematical flavor: @@ -1148,7 +1150,7 @@ The following recipes have a more mathematical flavor: # https://mathworld.wolfram.com/TotientFunction.html # totient(12) --> 4 because len([1, 5, 7, 11]) == 4 for p in unique_justseen(factor(n)): - n = n // p * (p - 1) + n -= n // p return n @@ -1562,16 +1564,16 @@ The following recipes have a more mathematical flavor: >>> list(unique_everseen('AAAABBBCCDAABBB')) ['A', 'B', 'C', 'D'] - >>> list(unique_everseen('ABBCcAD', str.lower)) + >>> list(unique_everseen('ABBCcAD', str.casefold)) ['A', 'B', 'C', 'D'] - >>> list(unique_everseen('ABBcCAD', str.lower)) + >>> list(unique_everseen('ABBcCAD', str.casefold)) ['A', 'B', 'c', 'D'] >>> list(unique_justseen('AAAABBBCCDAABBB')) ['A', 'B', 'C', 'D', 'A', 'B'] - >>> list(unique_justseen('ABBCcAD', str.lower)) + >>> list(unique_justseen('ABBCcAD', str.casefold)) ['A', 'B', 'C', 'A', 'D'] - >>> list(unique_justseen('ABBcCAD', str.lower)) + >>> list(unique_justseen('ABBcCAD', str.casefold)) ['A', 'B', 'c', 'A', 'D'] >>> d = dict(a=1, b=2, c=3) diff --git a/Doc/library/kde_example.png b/Doc/library/kde_example.png index f4504895699974..4c26f26292faa5 100644 Binary files a/Doc/library/kde_example.png and b/Doc/library/kde_example.png differ diff --git a/Doc/library/locale.rst b/Doc/library/locale.rst index a7201199191215..414979524e57b6 100644 --- a/Doc/library/locale.rst +++ b/Doc/library/locale.rst @@ -18,7 +18,7 @@ know all the specifics of each country where the software is executed. .. index:: pair: module; _locale -The :mod:`locale` module is implemented on top of the :mod:`_locale` module, +The :mod:`locale` module is implemented on top of the :mod:`!_locale` module, which in turn uses an ANSI C locale implementation if available. The :mod:`locale` module defines the following exception and functions: @@ -192,7 +192,13 @@ The :mod:`locale` module defines the following exception and functions: Get a format string for :func:`time.strftime` to represent time in the am/pm format. - .. data:: DAY_1 ... DAY_7 + .. data:: DAY_1 + DAY_2 + DAY_3 + DAY_4 + DAY_5 + DAY_6 + DAY_7 Get the name of the n-th day of the week. @@ -202,15 +208,43 @@ The :mod:`locale` module defines the following exception and functions: international convention (ISO 8601) that Monday is the first day of the week. - .. data:: ABDAY_1 ... ABDAY_7 + .. data:: ABDAY_1 + ABDAY_2 + ABDAY_3 + ABDAY_4 + ABDAY_5 + ABDAY_6 + ABDAY_7 Get the abbreviated name of the n-th day of the week. - .. data:: MON_1 ... MON_12 + .. data:: MON_1 + MON_2 + MON_3 + MON_4 + MON_5 + MON_6 + MON_7 + MON_8 + MON_9 + MON_10 + MON_11 + MON_12 Get the name of the n-th month. - .. data:: ABMON_1 ... ABMON_12 + .. data:: ABMON_1 + ABMON_2 + ABMON_3 + ABMON_4 + ABMON_5 + ABMON_6 + ABMON_7 + ABMON_8 + ABMON_9 + ABMON_10 + ABMON_11 + ABMON_12 Get the abbreviated name of the n-th month. @@ -229,14 +263,14 @@ The :mod:`locale` module defines the following exception and functions: .. data:: NOEXPR - Get a regular expression that can be used with the regex(3) function to + Get a regular expression that can be used with the ``regex(3)`` function to recognize a negative response to a yes/no question. .. note:: The regular expressions for :const:`YESEXPR` and :const:`NOEXPR` use syntax suitable for the - :c:func:`regex` function from the C library, which might + ``regex`` function from the C library, which might differ from the syntax used in :mod:`re`. .. data:: CRNCYSTR @@ -581,9 +615,9 @@ the locale is ``C``). When Python code uses the :mod:`locale` module to change the locale, this also affects the embedding application. If the embedding application doesn't want -this to happen, it should remove the :mod:`_locale` extension module (which does +this to happen, it should remove the :mod:`!_locale` extension module (which does all the work) from the table of built-in modules in the :file:`config.c` file, -and make sure that the :mod:`_locale` module is not accessible as a shared +and make sure that the :mod:`!_locale` module is not accessible as a shared library. @@ -597,17 +631,18 @@ Access to message catalogs .. function:: dcgettext(domain, msg, category) .. function:: textdomain(domain) .. function:: bindtextdomain(domain, dir) +.. function:: bind_textdomain_codeset(domain, codeset) The locale module exposes the C library's gettext interface on systems that -provide this interface. It consists of the functions :func:`!gettext`, -:func:`!dgettext`, :func:`!dcgettext`, :func:`!textdomain`, :func:`!bindtextdomain`, -and :func:`!bind_textdomain_codeset`. These are similar to the same functions in +provide this interface. It consists of the functions :func:`gettext`, +:func:`dgettext`, :func:`dcgettext`, :func:`textdomain`, :func:`bindtextdomain`, +and :func:`bind_textdomain_codeset`. These are similar to the same functions in the :mod:`gettext` module, but use the C library's binary format for message catalogs, and the C library's search algorithms for locating message catalogs. Python applications should normally find no need to invoke these functions, and should use :mod:`gettext` instead. A known exception to this rule are applications that link with additional C libraries which internally invoke -:c:func:`gettext` or :c:func:`dcgettext`. For these applications, it may be +C functions ``gettext`` or ``dcgettext``. For these applications, it may be necessary to bind the text domain, so that the libraries can properly locate their message catalogs. diff --git a/Doc/library/logging.config.rst b/Doc/library/logging.config.rst index 85a53e6aa7a78b..13850c91446da5 100644 --- a/Doc/library/logging.config.rst +++ b/Doc/library/logging.config.rst @@ -93,8 +93,8 @@ in :mod:`logging` itself) and defining handlers which are declared either in :param fname: A filename, or a file-like object, or an instance derived from :class:`~configparser.RawConfigParser`. If a - ``RawConfigParser``-derived instance is passed, it is used as - is. Otherwise, a :class:`~configparser.Configparser` is + :class:`!RawConfigParser`-derived instance is passed, it is used as + is. Otherwise, a :class:`~configparser.ConfigParser` is instantiated, and the configuration read by it from the object passed in ``fname``. If that has a :meth:`readline` method, it is assumed to be a file-like object and read using @@ -103,7 +103,7 @@ in :mod:`logging` itself) and defining handlers which are declared either in :meth:`~configparser.ConfigParser.read`. - :param defaults: Defaults to be passed to the ConfigParser can be specified + :param defaults: Defaults to be passed to the :class:`!ConfigParser` can be specified in this argument. :param disable_existing_loggers: If specified as ``False``, loggers which @@ -127,8 +127,8 @@ in :mod:`logging` itself) and defining handlers which are declared either in application (e.g. based on command-line parameters or other aspects of the runtime environment) before being passed to ``fileConfig``. - .. versionadded:: 3.10 - The *encoding* parameter is added. + .. versionchanged:: 3.10 + Added the *encoding* parameter. .. versionchanged:: 3.12 An exception will be thrown if the provided file diff --git a/Doc/library/logging.handlers.rst b/Doc/library/logging.handlers.rst index 2dd4bd081b0429..2fe9370333beaf 100644 --- a/Doc/library/logging.handlers.rst +++ b/Doc/library/logging.handlers.rst @@ -871,8 +871,8 @@ supports sending logging messages to an email address via SMTP. A timeout can be specified for communication with the SMTP server using the *timeout* argument. - .. versionadded:: 3.3 - The *timeout* argument was added. + .. versionchanged:: 3.3 + Added the *timeout* parameter. .. method:: emit(record) diff --git a/Doc/library/logging.rst b/Doc/library/logging.rst index acdeb88a546261..39eb41ce1f1670 100644 --- a/Doc/library/logging.rst +++ b/Doc/library/logging.rst @@ -531,12 +531,12 @@ subclasses. However, the :meth:`!__init__` method in subclasses needs to call This method should be called from handlers when an exception is encountered during an :meth:`emit` call. If the module-level attribute - ``raiseExceptions`` is ``False``, exceptions get silently ignored. This is + :data:`raiseExceptions` is ``False``, exceptions get silently ignored. This is what is mostly wanted for a logging system - most users will not care about errors in the logging system, they are more interested in application errors. You could, however, replace this with a custom handler if you wish. The specified record is the one which was being processed when the exception - occurred. (The default value of ``raiseExceptions`` is ``True``, as that is + occurred. (The default value of :data:`raiseExceptions` is ``True``, as that is more useful during development). @@ -615,14 +615,14 @@ Formatter Objects ``logging.Formatter('%(ip)s %(message)s', defaults={"ip": None})`` :type defaults: dict[str, Any] - .. versionadded:: 3.2 - The *style* parameter. + .. versionchanged:: 3.2 + Added the *style* parameter. - .. versionadded:: 3.8 - The *validate* parameter. + .. versionchanged:: 3.8 + Added the *validate* parameter. - .. versionadded:: 3.10 - The *defaults* parameter. + .. versionchanged:: 3.10 + Added the *defaults* parameter. .. method:: format(record) @@ -1494,6 +1494,18 @@ Module-Level Attributes .. versionadded:: 3.2 +.. attribute:: raiseExceptions + + Used to see if exceptions during handling should be propagated. + + Default: ``True``. + + If :data:`raiseExceptions` is ``False``, + exceptions get silently ignored. This is what is mostly wanted + for a logging system - most users will not care about errors in + the logging system, they are more interested in application errors. + + Integration with the warnings module ------------------------------------ diff --git a/Doc/library/mailbox.rst b/Doc/library/mailbox.rst index c98496d1fff993..a613548c9e518e 100644 --- a/Doc/library/mailbox.rst +++ b/Doc/library/mailbox.rst @@ -364,6 +364,9 @@ Supported mailbox formats are Maildir, mbox, MH, Babyl, and MMDF. The :attr:`!colon` attribute may also be set on a per-instance basis. + .. versionchanged:: 3.13 + :class:`Maildir` now ignores files with a leading dot. + :class:`!Maildir` instances have all of the methods of :class:`Mailbox` in addition to the following: @@ -641,6 +644,10 @@ Supported mailbox formats are Maildir, mbox, MH, Babyl, and MMDF. :class:`!MH` instances have all of the methods of :class:`Mailbox` in addition to the following: + .. versionchanged:: 3.13 + + Supported folders that don't contain a :file:`.mh_sequences` file. + .. method:: list_folders() @@ -1129,8 +1136,8 @@ When a :class:`!MaildirMessage` instance is created based upon a leading "From " or trailing newline. For convenience, *time_* may be specified and will be formatted appropriately and appended to *from_*. If *time_* is specified, it should be a :class:`time.struct_time` instance, a - tuple suitable for passing to :meth:`time.strftime`, or ``True`` (to use - :meth:`time.gmtime`). + tuple suitable for passing to :func:`time.strftime`, or ``True`` (to use + :func:`time.gmtime`). .. method:: get_flags() @@ -1501,8 +1508,8 @@ When a :class:`!BabylMessage` instance is created based upon an leading "From " or trailing newline. For convenience, *time_* may be specified and will be formatted appropriately and appended to *from_*. If *time_* is specified, it should be a :class:`time.struct_time` instance, a - tuple suitable for passing to :meth:`time.strftime`, or ``True`` (to use - :meth:`time.gmtime`). + tuple suitable for passing to :func:`time.strftime`, or ``True`` (to use + :func:`time.gmtime`). .. method:: get_flags() diff --git a/Doc/library/marshal.rst b/Doc/library/marshal.rst index 0556f19699dc15..c6a006b7b4028a 100644 --- a/Doc/library/marshal.rst +++ b/Doc/library/marshal.rst @@ -23,7 +23,11 @@ transfer of Python objects through RPC calls, see the modules :mod:`pickle` and :mod:`shelve`. The :mod:`marshal` module exists mainly to support reading and writing the "pseudo-compiled" code for Python modules of :file:`.pyc` files. Therefore, the Python maintainers reserve the right to modify the marshal format -in backward incompatible ways should the need arise. If you're serializing and +in backward incompatible ways should the need arise. +The format of code objects is not compatible between Python versions, +even if the version of the format is the same. +De-serializing a code object in the incorrect Python version has undefined behavior. +If you're serializing and de-serializing Python objects, use the :mod:`pickle` module instead -- the performance is comparable, version independence is guaranteed, and pickle supports a substantially wider range of objects than marshal. @@ -40,7 +44,8 @@ Not all Python object types are supported; in general, only objects whose value is independent from a particular invocation of Python can be written and read by this module. The following types are supported: booleans, integers, floating point numbers, complex numbers, strings, bytes, bytearrays, tuples, lists, sets, -frozensets, dictionaries, and code objects, where it should be understood that +frozensets, dictionaries, and code objects (if *allow_code* is true), +where it should be understood that tuples, lists, sets, frozensets and dictionaries are only supported as long as the values contained therein are themselves supported. The singletons :const:`None`, :const:`Ellipsis` and :exc:`StopIteration` can also be @@ -54,7 +59,7 @@ bytes-like objects. The module defines these functions: -.. function:: dump(value, file[, version]) +.. function:: dump(value, file, version=version, /, *, allow_code=True) Write the value on the open file. The value must be a supported type. The file must be a writeable :term:`binary file`. @@ -62,19 +67,24 @@ The module defines these functions: If the value has (or contains an object that has) an unsupported type, a :exc:`ValueError` exception is raised --- but garbage data will also be written to the file. The object will not be properly read back by :func:`load`. + :ref:`Code objects <code-objects>` are only supported if *allow_code* is true. The *version* argument indicates the data format that ``dump`` should use (see below). .. audit-event:: marshal.dumps value,version marshal.dump + .. versionchanged:: 3.13 + Added the *allow_code* parameter. -.. function:: load(file) + +.. function:: load(file, /, *, allow_code=True) Read one value from the open file and return it. If no valid value is read (e.g. because the data has a different Python version's incompatible marshal - format), raise :exc:`EOFError`, :exc:`ValueError` or :exc:`TypeError`. The - file must be a readable :term:`binary file`. + format), raise :exc:`EOFError`, :exc:`ValueError` or :exc:`TypeError`. + :ref:`Code objects <code-objects>` are only supported if *allow_code* is true. + The file must be a readable :term:`binary file`. .. audit-event:: marshal.load "" marshal.load @@ -88,24 +98,32 @@ The module defines these functions: This call used to raise a ``code.__new__`` audit event for each code object. Now it raises a single ``marshal.load`` event for the entire load operation. + .. versionchanged:: 3.13 + Added the *allow_code* parameter. + -.. function:: dumps(value[, version]) +.. function:: dumps(value, version=version, /, *, allow_code=True) Return the bytes object that would be written to a file by ``dump(value, file)``. The value must be a supported type. Raise a :exc:`ValueError` exception if value has (or contains an object that has) an unsupported type. + :ref:`Code objects <code-objects>` are only supported if *allow_code* is true. The *version* argument indicates the data format that ``dumps`` should use (see below). .. audit-event:: marshal.dumps value,version marshal.dump + .. versionchanged:: 3.13 + Added the *allow_code* parameter. -.. function:: loads(bytes) + +.. function:: loads(bytes, /, *, allow_code=True) Convert the :term:`bytes-like object` to a value. If no valid value is found, raise - :exc:`EOFError`, :exc:`ValueError` or :exc:`TypeError`. Extra bytes in the - input are ignored. + :exc:`EOFError`, :exc:`ValueError` or :exc:`TypeError`. + :ref:`Code objects <code-objects>` are only supported if *allow_code* is true. + Extra bytes in the input are ignored. .. audit-event:: marshal.loads bytes marshal.load @@ -114,6 +132,9 @@ The module defines these functions: This call used to raise a ``code.__new__`` audit event for each code object. Now it raises a single ``marshal.loads`` event for the entire load operation. + .. versionchanged:: 3.13 + Added the *allow_code* parameter. + In addition, the following constants are defined: diff --git a/Doc/library/mmap.rst b/Doc/library/mmap.rst index 3fa69e717329e4..758721433f77de 100644 --- a/Doc/library/mmap.rst +++ b/Doc/library/mmap.rst @@ -48,7 +48,7 @@ update the underlying file. To map anonymous memory, -1 should be passed as the fileno along with the length. -.. class:: mmap(fileno, length, tagname=None, access=ACCESS_DEFAULT[, offset]) +.. class:: mmap(fileno, length, tagname=None, access=ACCESS_DEFAULT, offset=0) **(Windows version)** Maps *length* bytes from the file specified by the file handle *fileno*, and creates a mmap object. If *length* is larger @@ -62,8 +62,8 @@ To map anonymous memory, -1 should be passed as the fileno along with the length the same file. If you specify the name of an existing tag, that tag is opened, otherwise a new tag of this name is created. If this parameter is omitted or ``None``, the mapping is created without a name. Avoiding the - use of the tag parameter will assist in keeping your code portable between - Unix and Windows. + use of the *tagname* parameter will assist in keeping your code portable + between Unix and Windows. *offset* may be specified as a non-negative integer offset. mmap references will be relative to the offset from the beginning of the file. *offset* @@ -71,7 +71,8 @@ To map anonymous memory, -1 should be passed as the fileno along with the length .. audit-event:: mmap.__new__ fileno,length,access,offset mmap.mmap -.. class:: mmap(fileno, length, flags=MAP_SHARED, prot=PROT_WRITE|PROT_READ, access=ACCESS_DEFAULT[, offset]) +.. class:: mmap(fileno, length, flags=MAP_SHARED, prot=PROT_WRITE|PROT_READ, \ + access=ACCESS_DEFAULT, offset=0, *, trackfd=True) :noindex: **(Unix version)** Maps *length* bytes from the file specified by the file @@ -102,10 +103,20 @@ To map anonymous memory, -1 should be passed as the fileno along with the length defaults to 0. *offset* must be a multiple of :const:`ALLOCATIONGRANULARITY` which is equal to :const:`PAGESIZE` on Unix systems. + If *trackfd* is ``False``, the file descriptor specified by *fileno* will + not be duplicated, and the resulting :class:`!mmap` object will not + be associated with the map's underlying file. + This means that the :meth:`~mmap.mmap.size` and :meth:`~mmap.mmap.resize` + methods will fail. + This mode is useful to limit the number of open file descriptors. + To ensure validity of the created memory mapping the file specified by the descriptor *fileno* is internally automatically synchronized with the physical backing store on macOS. + .. versionchanged:: 3.13 + The *trackfd* parameter was added. + This example shows a simple way of using :class:`~mmap.mmap`:: import mmap @@ -254,9 +265,12 @@ To map anonymous memory, -1 should be passed as the fileno along with the length .. method:: resize(newsize) - Resizes the map and the underlying file, if any. If the mmap was created - with :const:`ACCESS_READ` or :const:`ACCESS_COPY`, resizing the map will - raise a :exc:`TypeError` exception. + Resizes the map and the underlying file, if any. + + Resizing a map created with *access* of :const:`ACCESS_READ` or + :const:`ACCESS_COPY`, will raise a :exc:`TypeError` exception. + Resizing a map created with with *trackfd* set to ``False``, + will raise a :exc:`ValueError` exception. **On Windows**: Resizing the map will raise an :exc:`OSError` if there are other maps against the same named file. Resizing an anonymous map (ie against the @@ -372,14 +386,25 @@ MAP_* Constants .. data:: MAP_SHARED MAP_PRIVATE - MAP_DENYWRITE - MAP_EXECUTABLE + MAP_32BIT + MAP_ALIGNED_SUPER MAP_ANON MAP_ANONYMOUS + MAP_CONCEAL + MAP_DENYWRITE + MAP_EXECUTABLE + MAP_HASSEMAPHORE + MAP_JIT + MAP_NOCACHE + MAP_NOEXTEND + MAP_NORESERVE MAP_POPULATE + MAP_RESILIENT_CODESIGN + MAP_RESILIENT_MEDIA MAP_STACK - MAP_ALIGNED_SUPER - MAP_CONCEAL + MAP_TPRO + MAP_TRANSLATED_ALLOW_EXECUTE + MAP_UNIX03 These are the various flags that can be passed to :meth:`mmap.mmap`. :data:`MAP_ALIGNED_SUPER` is only available at FreeBSD and :data:`MAP_CONCEAL` is only available at OpenBSD. Note @@ -392,5 +417,12 @@ MAP_* Constants Added :data:`MAP_STACK` constant. .. versionadded:: 3.12 - Added :data:`MAP_ALIGNED_SUPER` constant. - Added :data:`MAP_CONCEAL` constant. + Added :data:`MAP_ALIGNED_SUPER` and :data:`MAP_CONCEAL` constants. + + .. versionadded:: 3.13 + Added :data:`MAP_32BIT`, :data:`MAP_HASSEMAPHORE`, :data:`MAP_JIT`, + :data:`MAP_NOCACHE`, :data:`MAP_NOEXTEND`, :data:`MAP_NORESERVE`, + :data:`MAP_RESILIENT_CODESIGN`, :data:`MAP_RESILIENT_MEDIA`, + :data:`MAP_TPRO`, :data:`MAP_TRANSLATED_ALLOW_EXECUTE`, and + :data:`MAP_UNIX03` constants. + diff --git a/Doc/library/msvcrt.rst b/Doc/library/msvcrt.rst index 0b059e746c61af..2a6d980ab78a60 100644 --- a/Doc/library/msvcrt.rst +++ b/Doc/library/msvcrt.rst @@ -75,10 +75,14 @@ File Operations .. function:: open_osfhandle(handle, flags) Create a C runtime file descriptor from the file handle *handle*. The *flags* - parameter should be a bitwise OR of :const:`os.O_APPEND`, :const:`os.O_RDONLY`, - and :const:`os.O_TEXT`. The returned file descriptor may be used as a parameter + parameter should be a bitwise OR of :const:`os.O_APPEND`, + :const:`os.O_RDONLY`, :const:`os.O_TEXT` and :const:`os.O_NOINHERIT`. + The returned file descriptor may be used as a parameter to :func:`os.fdopen` to create a file object. + The file descriptor is inheritable by default. Pass :const:`os.O_NOINHERIT` + flag to make it non inheritable. + .. audit-event:: msvcrt.open_osfhandle handle,flags msvcrt.open_osfhandle diff --git a/Doc/library/multiprocessing.rst b/Doc/library/multiprocessing.rst index 789a84b02d59d2..b104a6483b70e6 100644 --- a/Doc/library/multiprocessing.rst +++ b/Doc/library/multiprocessing.rst @@ -56,7 +56,7 @@ will print to standard output :: The :class:`Process` class -~~~~~~~~~~~~~~~~~~~~~~~~~~ +^^^^^^^^^^^^^^^^^^^^^^^^^^ In :mod:`multiprocessing`, processes are spawned by creating a :class:`Process` object and then calling its :meth:`~Process.start` method. :class:`Process` @@ -102,7 +102,7 @@ necessary, see :ref:`multiprocessing-programming`. .. _multiprocessing-start-methods: Contexts and start methods -~~~~~~~~~~~~~~~~~~~~~~~~~~ +^^^^^^^^^^^^^^^^^^^^^^^^^^ Depending on the platform, :mod:`multiprocessing` supports three ways to start a process. These *start methods* are @@ -231,7 +231,7 @@ library user. Exchanging objects between processes -~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ +^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ :mod:`multiprocessing` supports two types of communication channel between processes: @@ -283,7 +283,7 @@ processes: Synchronization between processes -~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ +^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ :mod:`multiprocessing` contains equivalents of all the synchronization primitives from :mod:`threading`. For instance one can use a lock to ensure @@ -309,7 +309,7 @@ mixed up. Sharing state between processes -~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ +^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ As mentioned above, when doing concurrent programming it is usually best to avoid using shared state as far as possible. This is particularly true when @@ -399,7 +399,7 @@ However, if you really do need to use some shared data then Using a pool of workers -~~~~~~~~~~~~~~~~~~~~~~~ +^^^^^^^^^^^^^^^^^^^^^^^ The :class:`~multiprocessing.pool.Pool` class represents a pool of worker processes. It has methods which allows tasks to be offloaded to the worker @@ -490,7 +490,7 @@ The :mod:`multiprocessing` package mostly replicates the API of the :class:`Process` and exceptions -~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ +^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ .. class:: Process(group=None, target=None, name=None, args=(), kwargs={}, \ *, daemon=None) @@ -724,7 +724,7 @@ The :mod:`multiprocessing` package mostly replicates the API of the Raised by methods with a timeout when the timeout expires. Pipes and Queues -~~~~~~~~~~~~~~~~ +^^^^^^^^^^^^^^^^ When using multiple processes, one generally uses message passing for communication between processes and avoids having to use any synchronization @@ -981,7 +981,7 @@ For an example of the usage of queues for interprocess communication see Miscellaneous -~~~~~~~~~~~~~ +^^^^^^^^^^^^^ .. function:: active_children() @@ -1150,7 +1150,7 @@ Miscellaneous Connection Objects -~~~~~~~~~~~~~~~~~~ +^^^^^^^^^^^^^^^^^^ .. currentmodule:: multiprocessing.connection @@ -1292,7 +1292,7 @@ For example: Synchronization primitives -~~~~~~~~~~~~~~~~~~~~~~~~~~ +^^^^^^^^^^^^^^^^^^^^^^^^^^ .. currentmodule:: multiprocessing @@ -1481,7 +1481,7 @@ object -- see :ref:`multiprocessing-managers`. Shared :mod:`ctypes` Objects -~~~~~~~~~~~~~~~~~~~~~~~~~~~~ +^^^^^^^^^^^^^^^^^^^^^^^^^^^^ It is possible to create shared objects using shared memory which can be inherited by child processes. @@ -1543,7 +1543,7 @@ inherited by child processes. The :mod:`multiprocessing.sharedctypes` module ->>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>> +"""""""""""""""""""""""""""""""""""""""""""""" .. module:: multiprocessing.sharedctypes :synopsis: Allocate ctypes objects from shared memory. @@ -1709,7 +1709,7 @@ The results printed are :: .. _multiprocessing-managers: Managers -~~~~~~~~ +^^^^^^^^ Managers provide a way to create data which can be shared between different processes, including sharing over a network between processes running on @@ -1954,7 +1954,7 @@ their parent process exits. The manager classes are defined in the Customized managers ->>>>>>>>>>>>>>>>>>> +""""""""""""""""""" To create one's own manager, one creates a subclass of :class:`BaseManager` and uses the :meth:`~BaseManager.register` classmethod to register new types or @@ -1981,7 +1981,7 @@ callables with the manager class. For example:: Using a remote manager ->>>>>>>>>>>>>>>>>>>>>> +"""""""""""""""""""""" It is possible to run a manager server on one machine and have clients use it from other machines (assuming that the firewalls involved allow it). @@ -2044,7 +2044,7 @@ client to access it remotely:: .. _multiprocessing-proxy_objects: Proxy Objects -~~~~~~~~~~~~~ +^^^^^^^^^^^^^ A proxy is an object which *refers* to a shared object which lives (presumably) in a different process. The shared object is said to be the *referent* of the @@ -2196,7 +2196,7 @@ demonstrates a level of control over the synchronization. Cleanup ->>>>>>> +""""""" A proxy object uses a weakref callback so that when it gets garbage collected it deregisters itself from the manager which owns its referent. @@ -2206,7 +2206,7 @@ any proxies referring to it. Process Pools -~~~~~~~~~~~~~ +^^^^^^^^^^^^^ .. module:: multiprocessing.pool :synopsis: Create pools of processes. @@ -2442,7 +2442,7 @@ The following example demonstrates the use of a pool:: .. _multiprocessing-listeners-clients: Listeners and Clients -~~~~~~~~~~~~~~~~~~~~~ +^^^^^^^^^^^^^^^^^^^^^ .. module:: multiprocessing.connection :synopsis: API for dealing with sockets. @@ -2665,7 +2665,7 @@ wait for messages from multiple processes at once:: .. _multiprocessing-address-formats: Address Formats ->>>>>>>>>>>>>>> +""""""""""""""" * An ``'AF_INET'`` address is a tuple of the form ``(hostname, port)`` where *hostname* is a string and *port* is an integer. @@ -2685,7 +2685,7 @@ an ``'AF_PIPE'`` address rather than an ``'AF_UNIX'`` address. .. _multiprocessing-auth-keys: Authentication keys -~~~~~~~~~~~~~~~~~~~ +^^^^^^^^^^^^^^^^^^^ When one uses :meth:`Connection.recv <Connection.recv>`, the data received is automatically @@ -2711,7 +2711,7 @@ Suitable authentication keys can also be generated by using :func:`os.urandom`. Logging -~~~~~~~ +^^^^^^^ Some support for logging is available. Note, however, that the :mod:`logging` package does not use process shared locks so it is possible (depending on the @@ -2759,7 +2759,7 @@ For a full table of logging levels, see the :mod:`logging` module. The :mod:`multiprocessing.dummy` module -~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ +^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ .. module:: multiprocessing.dummy :synopsis: Dumb wrapper around threading. @@ -2818,7 +2818,7 @@ There are certain guidelines and idioms which should be adhered to when using All start methods -~~~~~~~~~~~~~~~~~ +^^^^^^^^^^^^^^^^^ The following applies to all start methods. @@ -2977,7 +2977,7 @@ Beware of replacing :data:`sys.stdin` with a "file like object" For more information, see :issue:`5155`, :issue:`5313` and :issue:`5331` The *spawn* and *forkserver* start methods -~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ +^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ There are a few extra restriction which don't apply to the *fork* start method. diff --git a/Doc/library/multiprocessing.shared_memory.rst b/Doc/library/multiprocessing.shared_memory.rst index 671130d9b29fc0..933fd07d62418a 100644 --- a/Doc/library/multiprocessing.shared_memory.rst +++ b/Doc/library/multiprocessing.shared_memory.rst @@ -20,10 +20,10 @@ and management of shared memory to be accessed by one or more processes on a multicore or symmetric multiprocessor (SMP) machine. To assist with the life-cycle management of shared memory especially across distinct processes, a :class:`~multiprocessing.managers.BaseManager` subclass, -:class:`SharedMemoryManager`, is also provided in the -``multiprocessing.managers`` module. +:class:`~multiprocessing.managers.SharedMemoryManager`, is also provided in the +:mod:`multiprocessing.managers` module. -In this module, shared memory refers to "System V style" shared memory blocks +In this module, shared memory refers to "POSIX style" shared memory blocks (though is not necessarily implemented explicitly as such) and does not refer to "distributed shared memory". This style of shared memory permits distinct processes to potentially read and write to a common (or shared) region of @@ -38,7 +38,8 @@ copying of data. .. class:: SharedMemory(name=None, create=False, size=0, *, track=True) - Creates a new shared memory block or attaches to an existing shared + Create an instance of the :class:`!SharedMemory` class for either + creating a new shared memory block or attaching to an existing shared memory block. Each shared memory block is assigned a unique name. In this way, one process can create a shared memory block with a particular name and a different process can attach to that same shared @@ -47,58 +48,65 @@ copying of data. As a resource for sharing data across processes, shared memory blocks may outlive the original process that created them. When one process no longer needs access to a shared memory block that might still be - needed by other processes, the :meth:`close()` method should be called. + needed by other processes, the :meth:`close` method should be called. When a shared memory block is no longer needed by any process, the - :meth:`unlink()` method should be called to ensure proper cleanup. - - *name* is the unique name for the requested shared memory, specified as - a string. When creating a new shared memory block, if ``None`` (the - default) is supplied for the name, a novel name will be generated. - - *create* controls whether a new shared memory block is created (``True``) - or an existing shared memory block is attached (``False``). - - *size* specifies the requested number of bytes when creating a new shared - memory block. Because some platforms choose to allocate chunks of memory - based upon that platform's memory page size, the exact size of the shared - memory block may be larger or equal to the size requested. When attaching - to an existing shared memory block, the ``size`` parameter is ignored. - - *track*, when enabled, registers the shared memory block with a resource - tracker process on platforms where the OS does not do this automatically. - The resource tracker ensures proper cleanup of the shared memory even - if all other processes with access to the memory exit without doing so. - Python processes created from a common ancestor using :mod:`multiprocessing` - facilities share a single resource tracker process, and the lifetime of - shared memory segments is handled automatically among these processes. - Python processes created in any other way will receive their own - resource tracker when accessing shared memory with *track* enabled. - This will cause the shared memory to be deleted by the resource tracker - of the first process that terminates. - To avoid this issue, users of :mod:`subprocess` or standalone Python - processes should set *track* to ``False`` when there is already another - process in place that does the bookkeeping. - *track* is ignored on Windows, which has its own tracking and - automatically deletes shared memory when all handles to it have been closed. - - .. versionchanged:: 3.13 Added *track* parameter. + :meth:`unlink` method should be called to ensure proper cleanup. + + :param name: + The unique name for the requested shared memory, specified as a string. + When creating a new shared memory block, if ``None`` (the default) + is supplied for the name, a novel name will be generated. + :type name: str | None + + :param bool create: + Control whether a new shared memory block is created (``True``) + or an existing shared memory block is attached (``False``). + + :param int size: + The requested number of bytes when creating a new shared memory block. + Because some platforms choose to allocate chunks of memory + based upon that platform's memory page size, the exact size of the shared + memory block may be larger or equal to the size requested. + When attaching to an existing shared memory block, + the *size* parameter is ignored. + + :param bool track: + When ``True``, register the shared memory block with a resource + tracker process on platforms where the OS does not do this automatically. + The resource tracker ensures proper cleanup of the shared memory even + if all other processes with access to the memory exit without doing so. + Python processes created from a common ancestor using :mod:`multiprocessing` + facilities share a single resource tracker process, and the lifetime of + shared memory segments is handled automatically among these processes. + Python processes created in any other way will receive their own + resource tracker when accessing shared memory with *track* enabled. + This will cause the shared memory to be deleted by the resource tracker + of the first process that terminates. + To avoid this issue, users of :mod:`subprocess` or standalone Python + processes should set *track* to ``False`` when there is already another + process in place that does the bookkeeping. + *track* is ignored on Windows, which has its own tracking and + automatically deletes shared memory when all handles to it have been closed. + + .. versionchanged:: 3.13 + Added the *track* parameter. .. method:: close() - Closes the file descriptor/handle to the shared memory from this - instance. :meth:`close()` should be called once access to the shared + Close the file descriptor/handle to the shared memory from this + instance. :meth:`close` should be called once access to the shared memory block from this instance is no longer needed. Depending on operating system, the underlying memory may or may not be freed even if all handles to it have been closed. To ensure proper cleanup, - use the :meth:`unlink()` method. + use the :meth:`unlink` method. .. method:: unlink() - Deletes the underlying shared memory block. This should be called only + Delete the underlying shared memory block. This should be called only once per shared memory block regardless of the number of handles to it, even in other processes. - :meth:`unlink()` and :meth:`close()` can be called in any order, but - trying to access data inside a shared memory block after :meth:`unlink()` + :meth:`unlink` and :meth:`close` can be called in any order, but + trying to access data inside a shared memory block after :meth:`unlink` may result in memory access errors, depending on platform. This method has no effect on Windows, where the only way to delete a @@ -145,7 +153,7 @@ instances:: The following example demonstrates a practical use of the :class:`SharedMemory` class with `NumPy arrays <https://numpy.org/>`_, accessing the -same ``numpy.ndarray`` from two distinct Python shells: +same :class:`!numpy.ndarray` from two distinct Python shells: .. doctest:: :options: +SKIP @@ -197,43 +205,43 @@ same ``numpy.ndarray`` from two distinct Python shells: .. class:: SharedMemoryManager([address[, authkey]]) :module: multiprocessing.managers - A subclass of :class:`~multiprocessing.managers.BaseManager` which can be + A subclass of :class:`multiprocessing.managers.BaseManager` which can be used for the management of shared memory blocks across processes. A call to :meth:`~multiprocessing.managers.BaseManager.start` on a - :class:`SharedMemoryManager` instance causes a new process to be started. + :class:`!SharedMemoryManager` instance causes a new process to be started. This new process's sole purpose is to manage the life cycle of all shared memory blocks created through it. To trigger the release of all shared memory blocks managed by that process, call - :meth:`~multiprocessing.managers.BaseManager.shutdown()` on the instance. - This triggers a :meth:`SharedMemory.unlink()` call on all of the - :class:`SharedMemory` objects managed by that process and then - stops the process itself. By creating ``SharedMemory`` instances - through a ``SharedMemoryManager``, we avoid the need to manually track + :meth:`~multiprocessing.managers.BaseManager.shutdown` on the instance. + This triggers a :meth:`~multiprocessing.shared_memory.SharedMemory.unlink` call + on all of the :class:`SharedMemory` objects managed by that process and then + stops the process itself. By creating :class:`!SharedMemory` instances + through a :class:`!SharedMemoryManager`, we avoid the need to manually track and trigger the freeing of shared memory resources. This class provides methods for creating and returning :class:`SharedMemory` instances and for creating a list-like object (:class:`ShareableList`) backed by shared memory. - Refer to :class:`multiprocessing.managers.BaseManager` for a description + Refer to :class:`~multiprocessing.managers.BaseManager` for a description of the inherited *address* and *authkey* optional input arguments and how - they may be used to connect to an existing ``SharedMemoryManager`` service + they may be used to connect to an existing :class:`!SharedMemoryManager` service from other processes. .. method:: SharedMemory(size) Create and return a new :class:`SharedMemory` object with the - specified ``size`` in bytes. + specified *size* in bytes. .. method:: ShareableList(sequence) Create and return a new :class:`ShareableList` object, initialized - by the values from the input ``sequence``. + by the values from the input *sequence*. The following example demonstrates the basic mechanisms of a -:class:`SharedMemoryManager`: +:class:`~multiprocessing.managers.SharedMemoryManager`: .. doctest:: :options: +SKIP @@ -251,9 +259,9 @@ The following example demonstrates the basic mechanisms of a >>> smm.shutdown() # Calls unlink() on sl, raw_shm, and another_sl The following example depicts a potentially more convenient pattern for using -:class:`SharedMemoryManager` objects via the :keyword:`with` statement to -ensure that all shared memory blocks are released after they are no longer -needed: +:class:`~multiprocessing.managers.SharedMemoryManager` objects via the +:keyword:`with` statement to ensure that all shared memory blocks are released +after they are no longer needed: .. doctest:: :options: +SKIP @@ -269,38 +277,46 @@ needed: ... p2.join() # Wait for all work to complete in both processes ... total_result = sum(sl) # Consolidate the partial results now in sl -When using a :class:`SharedMemoryManager` in a :keyword:`with` statement, the -shared memory blocks created using that manager are all released when the -:keyword:`with` statement's code block finishes execution. +When using a :class:`~multiprocessing.managers.SharedMemoryManager` +in a :keyword:`with` statement, the shared memory blocks created using that +manager are all released when the :keyword:`!with` statement's code block +finishes execution. -.. class:: ShareableList(sequence=None, \*, name=None) +.. class:: ShareableList(sequence=None, *, name=None) - Provides a mutable list-like object where all values stored within are - stored in a shared memory block. This constrains storable values to - only the ``int`` (signed 64-bit), ``float``, ``bool``, ``str`` (less - than 10M bytes each when encoded as utf-8), ``bytes`` (less than 10M - bytes each), and ``None`` built-in data types. It also notably - differs from the built-in ``list`` type in that these lists can not - change their overall length (i.e. no append, insert, etc.) and do not - support the dynamic creation of new :class:`ShareableList` instances + Provide a mutable list-like object where all values stored within are + stored in a shared memory block. + This constrains storable values to the following built-in data types: + + * :class:`int` (signed 64-bit) + * :class:`float` + * :class:`bool` + * :class:`str` (less than 10M bytes each when encoded as UTF-8) + * :class:`bytes` (less than 10M bytes each) + * ``None`` + + It also notably differs from the built-in :class:`list` type + in that these lists can not change their overall length + (i.e. no :meth:`!append`, :meth:`!insert`, etc.) and do not + support the dynamic creation of new :class:`!ShareableList` instances via slicing. - *sequence* is used in populating a new ``ShareableList`` full of values. + *sequence* is used in populating a new :class:`!ShareableList` full of values. Set to ``None`` to instead attach to an already existing - ``ShareableList`` by its unique shared memory name. + :class:`!ShareableList` by its unique shared memory name. *name* is the unique name for the requested shared memory, as described in the definition for :class:`SharedMemory`. When attaching to an - existing ``ShareableList``, specify its shared memory block's unique - name while leaving ``sequence`` set to ``None``. + existing :class:`!ShareableList`, specify its shared memory block's unique + name while leaving *sequence* set to ``None``. .. note:: A known issue exists for :class:`bytes` and :class:`str` values. If they end with ``\x00`` nul bytes or characters, those may be *silently stripped* when fetching them by index from the - :class:`ShareableList`. This ``.rstrip(b'\x00')`` behavior is + :class:`!ShareableList`. This ``.rstrip(b'\x00')`` behavior is considered a bug and may go away in the future. See :gh:`106939`. For applications where rstripping of trailing nulls is a problem, @@ -326,12 +342,12 @@ shared memory blocks created using that manager are all released when the .. method:: count(value) - Returns the number of occurrences of ``value``. + Return the number of occurrences of *value*. .. method:: index(value) - Returns first index position of ``value``. Raises :exc:`ValueError` if - ``value`` is not present. + Return first index position of *value*. + Raise :exc:`ValueError` if *value* is not present. .. attribute:: format @@ -391,8 +407,8 @@ behind it: >>> c.shm.close() >>> c.shm.unlink() -The following examples demonstrates that ``ShareableList`` -(and underlying ``SharedMemory``) objects +The following examples demonstrates that :class:`ShareableList` +(and underlying :class:`SharedMemory`) objects can be pickled and unpickled if needed. Note, that it will still be the same shared object. This happens, because the deserialized object has diff --git a/Doc/library/os.path.rst b/Doc/library/os.path.rst index 95933f56d50542..34bc76b231de92 100644 --- a/Doc/library/os.path.rst +++ b/Doc/library/os.path.rst @@ -239,12 +239,16 @@ the :mod:`glob` module.) .. function:: isabs(path) Return ``True`` if *path* is an absolute pathname. On Unix, that means it - begins with a slash, on Windows that it begins with a (back)slash after chopping - off a potential drive letter. + begins with a slash, on Windows that it begins with two (back)slashes, or a + drive letter, colon, and (back)slash together. .. versionchanged:: 3.6 Accepts a :term:`path-like object`. + .. versionchanged:: 3.13 + On Windows, returns ``False`` if the given path starts with exactly one + (back)slash. + .. function:: isfile(path) @@ -322,6 +326,28 @@ the :mod:`glob` module.) .. versionadded:: 3.12 +.. function:: isreserved(path) + + Return ``True`` if *path* is a reserved pathname on the current system. + + On Windows, reserved filenames include those that end with a space or dot; + those that contain colons (i.e. file streams such as "name:stream"), + wildcard characters (i.e. ``'*?"<>'``), pipe, or ASCII control characters; + as well as DOS device names such as "NUL", "CON", "CONIN$", "CONOUT$", + "AUX", "PRN", "COM1", and "LPT1". + + .. note:: + + This function approximates rules for reserved paths on most Windows + systems. These rules change over time in various Windows releases. + This function may be updated in future Python releases as changes to + the rules become broadly available. + + .. availability:: Windows. + + .. versionadded:: 3.13 + + .. function:: join(path, *paths) Join one or more path segments intelligently. The return value is the diff --git a/Doc/library/os.rst b/Doc/library/os.rst index 1138cc1f249ee7..cc9f3e75a80c51 100644 --- a/Doc/library/os.rst +++ b/Doc/library/os.rst @@ -1001,11 +1001,14 @@ as internal buffering of data. .. audit-event:: os.chmod path,mode,dir_fd os.fchmod - .. availability:: Unix. + .. availability:: Unix, Windows. The function is limited on Emscripten and WASI, see :ref:`wasm-availability` for more information. + .. versionchanged:: 3.13 + Added support on Windows. + .. function:: fchown(fd, uid, gid) @@ -1119,6 +1122,20 @@ as internal buffering of data. .. versionchanged:: 3.12 Added support for pipes on Windows. + +.. function:: grantpt(fd, /) + + Grant access to the slave pseudo-terminal device associated with the + master pseudo-terminal device to which the file descriptor *fd* refers. + The file descriptor *fd* is not closed upon failure. + + Calls the C standard library function :c:func:`grantpt`. + + .. availability:: Unix, not Emscripten, not WASI. + + .. versionadded:: 3.13 + + .. function:: isatty(fd, /) Return ``True`` if the file descriptor *fd* is open and connected to a @@ -1251,8 +1268,8 @@ as internal buffering of data. :meth:`~file.read` and :meth:`~file.write` methods (and many more). To wrap a file descriptor in a file object, use :func:`fdopen`. - .. versionadded:: 3.3 - The *dir_fd* argument. + .. versionchanged:: 3.3 + Added the *dir_fd* parameter. .. versionchanged:: 3.5 If the system call is interrupted and the signal handler does not raise an @@ -1426,6 +1443,23 @@ or `the MSDN <https://msdn.microsoft.com/en-us/library/z0kc8e3z.aspx>`_ on Windo .. versionadded:: 3.3 +.. function:: posix_openpt(oflag, /) + + Open and return a file descriptor for a master pseudo-terminal device. + + Calls the C standard library function :c:func:`posix_openpt`. The *oflag* + argument is used to set file status flags and file access modes as + specified in the manual page of :c:func:`posix_openpt` of your system. + + The returned file descriptor is :ref:`non-inheritable <fd_inheritance>`. + If the value :data:`O_CLOEXEC` is available on the system, it is added to + *oflag*. + + .. availability:: Unix, not Emscripten, not WASI. + + .. versionadded:: 3.13 + + .. function:: preadv(fd, buffers, offset, flags=0, /) Read from a file descriptor *fd* at a position of *offset* into mutable @@ -1483,6 +1517,21 @@ or `the MSDN <https://msdn.microsoft.com/en-us/library/z0kc8e3z.aspx>`_ on Windo .. versionadded:: 3.7 +.. function:: ptsname(fd, /) + + Return the name of the slave pseudo-terminal device associated with the + master pseudo-terminal device to which the file descriptor *fd* refers. + The file descriptor *fd* is not closed upon failure. + + Calls the reentrant C standard library function :c:func:`ptsname_r` if + it is available; otherwise, the C standard library function + :c:func:`ptsname`, which is not guaranteed to be thread-safe, is called. + + .. availability:: Unix, not Emscripten, not WASI. + + .. versionadded:: 3.13 + + .. function:: pwrite(fd, str, offset, /) Write the bytestring in *str* to file descriptor *fd* at position of @@ -1735,6 +1784,19 @@ or `the MSDN <https://msdn.microsoft.com/en-us/library/z0kc8e3z.aspx>`_ on Windo .. availability:: Unix. +.. function:: unlockpt(fd, /) + + Unlock the slave pseudo-terminal device associated with the master + pseudo-terminal device to which the file descriptor *fd* refers. + The file descriptor *fd* is not closed upon failure. + + Calls the C standard library function :c:func:`unlockpt`. + + .. availability:: Unix, not Emscripten, not WASI. + + .. versionadded:: 3.13 + + .. function:: write(fd, str, /) Write the bytestring in *str* to file descriptor *fd*. @@ -1988,7 +2050,7 @@ features: .. audit-event:: os.chdir path os.chdir - .. versionadded:: 3.3 + .. versionchanged:: 3.3 Added support for specifying *path* as a file descriptor on some platforms. @@ -2020,8 +2082,8 @@ features: .. availability:: Unix, not Emscripten, not WASI. - .. versionadded:: 3.3 - The *follow_symlinks* argument. + .. versionchanged:: 3.3 + Added the *follow_symlinks* parameter. .. versionchanged:: 3.6 Accepts a :term:`path-like object`. @@ -2077,7 +2139,8 @@ features: Accepts a :term:`path-like object`. .. versionchanged:: 3.13 - Added support for the *follow_symlinks* argument on Windows. + Added support for a file descriptor and the *follow_symlinks* argument + on Windows. .. function:: chown(path, uid, gid, *, dir_fd=None, follow_symlinks=True) @@ -2206,8 +2269,8 @@ features: .. versionchanged:: 3.2 Added Windows support. - .. versionadded:: 3.3 - Added the *src_dir_fd*, *dst_dir_fd*, and *follow_symlinks* arguments. + .. versionchanged:: 3.3 + Added the *src_dir_fd*, *dst_dir_fd*, and *follow_symlinks* parameters. .. versionchanged:: 3.6 Accepts a :term:`path-like object` for *src* and *dst*. @@ -2371,8 +2434,8 @@ features: .. audit-event:: os.mkdir path,mode,dir_fd os.mkdir - .. versionadded:: 3.3 - The *dir_fd* argument. + .. versionchanged:: 3.3 + Added the *dir_fd* parameter. .. versionchanged:: 3.6 Accepts a :term:`path-like object`. @@ -2405,8 +2468,8 @@ features: .. audit-event:: os.mkdir path,mode,dir_fd os.makedirs - .. versionadded:: 3.2 - The *exist_ok* parameter. + .. versionchanged:: 3.2 + Added the *exist_ok* parameter. .. versionchanged:: 3.4.1 @@ -2439,8 +2502,8 @@ features: .. availability:: Unix, not Emscripten, not WASI. - .. versionadded:: 3.3 - The *dir_fd* argument. + .. versionchanged:: 3.3 + Added the *dir_fd* parameter. .. versionchanged:: 3.6 Accepts a :term:`path-like object`. @@ -2461,8 +2524,8 @@ features: .. availability:: Unix, not Emscripten, not WASI. - .. versionadded:: 3.3 - The *dir_fd* argument. + .. versionchanged:: 3.3 + Added the *dir_fd* parameter. .. versionchanged:: 3.6 Accepts a :term:`path-like object`. @@ -2542,8 +2605,8 @@ features: .. versionchanged:: 3.2 Added support for Windows 6.0 (Vista) symbolic links. - .. versionadded:: 3.3 - The *dir_fd* argument. + .. versionchanged:: 3.3 + Added the *dir_fd* parameter. .. versionchanged:: 3.6 Accepts a :term:`path-like object` on Unix. @@ -2573,8 +2636,8 @@ features: .. audit-event:: os.remove path,dir_fd os.remove - .. versionadded:: 3.3 - The *dir_fd* argument. + .. versionchanged:: 3.3 + Added the *dir_fd* parameter. .. versionchanged:: 3.6 Accepts a :term:`path-like object`. @@ -2624,8 +2687,8 @@ features: .. audit-event:: os.rename src,dst,src_dir_fd,dst_dir_fd os.rename - .. versionadded:: 3.3 - The *src_dir_fd* and *dst_dir_fd* arguments. + .. versionchanged:: 3.3 + Added the *src_dir_fd* and *dst_dir_fd* parameters. .. versionchanged:: 3.6 Accepts a :term:`path-like object` for *src* and *dst*. @@ -2680,8 +2743,8 @@ features: .. audit-event:: os.rmdir path,dir_fd os.rmdir - .. versionadded:: 3.3 - The *dir_fd* parameter. + .. versionchanged:: 3.3 + Added the *dir_fd* parameter. .. versionchanged:: 3.6 Accepts a :term:`path-like object`. @@ -2755,7 +2818,7 @@ features: .. versionadded:: 3.5 - .. versionadded:: 3.6 + .. versionchanged:: 3.6 Added support for the :term:`context manager` protocol and the :func:`~scandir.close()` method. If a :func:`scandir` iterator is neither exhausted nor explicitly closed a :exc:`ResourceWarning` will be emitted @@ -2969,9 +3032,9 @@ features: :func:`fstat` and :func:`lstat` functions. - .. versionadded:: 3.3 - Added the *dir_fd* and *follow_symlinks* arguments, specifying a file - descriptor instead of a path. + .. versionchanged:: 3.3 + Added the *dir_fd* and *follow_symlinks* parameters, + specifying a file descriptor instead of a path. .. versionchanged:: 3.6 Accepts a :term:`path-like object`. @@ -3052,16 +3115,22 @@ features: Time of most recent access expressed in nanoseconds as an integer. + .. versionadded: 3.3 + .. attribute:: st_mtime_ns Time of most recent content modification expressed in nanoseconds as an integer. + .. versionadded: 3.3 + .. attribute:: st_ctime_ns Time of most recent metadata change expressed in nanoseconds as an integer. + .. versionadded: 3.3 + .. versionchanged:: 3.12 ``st_ctime_ns`` is deprecated on Windows. Use ``st_birthtime_ns`` for the file creation time. In the future, ``st_ctime`` will contain @@ -3162,6 +3231,8 @@ features: See the :const:`!FILE_ATTRIBUTE_* <stat.FILE_ATTRIBUTE_ARCHIVE>` constants in the :mod:`stat` module. + .. versionadded:: 3.5 + .. attribute:: st_reparse_tag When :attr:`st_file_attributes` has the :const:`~stat.FILE_ATTRIBUTE_REPARSE_POINT` @@ -3182,13 +3253,6 @@ features: some implementations. For compatibility with older Python versions, accessing :class:`stat_result` as a tuple always returns integers. - .. versionadded:: 3.3 - Added the :attr:`st_atime_ns`, :attr:`st_mtime_ns`, and - :attr:`st_ctime_ns` members. - - .. versionadded:: 3.5 - Added the :attr:`st_file_attributes` member on Windows. - .. versionchanged:: 3.5 Windows now returns the file index as :attr:`st_ino` when available. @@ -3253,7 +3317,7 @@ features: .. versionchanged:: 3.2 The :const:`ST_RDONLY` and :const:`ST_NOSUID` constants were added. - .. versionadded:: 3.3 + .. versionchanged:: 3.3 Added support for specifying *path* as an open file descriptor. .. versionchanged:: 3.4 @@ -3265,8 +3329,8 @@ features: .. versionchanged:: 3.6 Accepts a :term:`path-like object`. - .. versionadded:: 3.7 - Added :attr:`f_fsid`. + .. versionchanged:: 3.7 + Added the :attr:`f_fsid` attribute. .. data:: supports_dir_fd @@ -3390,8 +3454,8 @@ features: .. versionchanged:: 3.2 Added support for Windows 6.0 (Vista) symbolic links. - .. versionadded:: 3.3 - Added the *dir_fd* argument, and now allow *target_is_directory* + .. versionchanged:: 3.3 + Added the *dir_fd* parameter, and now allow *target_is_directory* on non-Windows platforms. .. versionchanged:: 3.6 @@ -3439,8 +3503,8 @@ features: .. audit-event:: os.remove path,dir_fd os.unlink - .. versionadded:: 3.3 - The *dir_fd* parameter. + .. versionchanged:: 3.3 + Added the *dir_fd* parameter. .. versionchanged:: 3.6 Accepts a :term:`path-like object`. @@ -3478,7 +3542,7 @@ features: .. audit-event:: os.utime path,times,ns,dir_fd os.utime - .. versionadded:: 3.3 + .. versionchanged:: 3.3 Added support for specifying *path* as an open file descriptor, and the *dir_fd*, *follow_symlinks*, and *ns* parameters. @@ -4213,7 +4277,7 @@ to be ignored. .. availability:: Unix, Windows, not Emscripten, not WASI. - .. versionadded:: 3.3 + .. versionchanged:: 3.3 Added support for specifying *path* as an open file descriptor for :func:`execve`. @@ -4467,8 +4531,8 @@ written in Python, such as a mail server's external command delivery program. .. availability:: Unix, Windows, not Emscripten, not WASI. - .. versionadded:: 3.2 - Windows support. + .. versionchanged:: 3.2 + Added Windows support. .. function:: killpg(pgid, sig, /) @@ -4984,6 +5048,9 @@ written in Python, such as a mail server's external command delivery program. .. versionadded:: 3.3 + .. versionchanged:: 3.13 + This function is now available on macOS as well. + .. function:: waitpid(pid, options, /) diff --git a/Doc/library/pathlib.rst b/Doc/library/pathlib.rst index 60791725c2323d..f94b6fb3805684 100644 --- a/Doc/library/pathlib.rst +++ b/Doc/library/pathlib.rst @@ -485,19 +485,6 @@ Pure paths provide the following methods and properties: 'c:/windows' -.. method:: PurePath.as_uri() - - Represent the path as a ``file`` URI. :exc:`ValueError` is raised if - the path isn't absolute. - - >>> p = PurePosixPath('/etc/passwd') - >>> p.as_uri() - 'file:///etc/passwd' - >>> p = PureWindowsPath('c:/Windows') - >>> p.as_uri() - 'file:///c:/Windows' - - .. method:: PurePath.is_absolute() Return whether the path is absolute or not. A path is considered absolute @@ -528,6 +515,13 @@ Pure paths provide the following methods and properties: >>> p.is_relative_to('/usr') False + This method is string-based; it neither accesses the filesystem nor treats + "``..``" segments specially. The following code is equivalent: + + >>> u = PurePath('/usr') + >>> u == p or u in p.parents + False + .. versionadded:: 3.9 .. deprecated-removed:: 3.12 3.14 @@ -541,14 +535,13 @@ Pure paths provide the following methods and properties: reserved under Windows, ``False`` otherwise. With :class:`PurePosixPath`, ``False`` is always returned. - >>> PureWindowsPath('nul').is_reserved() - True - >>> PurePosixPath('nul').is_reserved() - False - - File system calls on reserved paths can fail mysteriously or have - unintended effects. + .. versionchanged:: 3.13 + Windows path names that contain a colon, or end with a dot or a space, + are considered reserved. UNC paths may be reserved. + .. deprecated-removed:: 3.13 3.15 + This method is deprecated; use :func:`os.path.isreserved` to detect + reserved paths on Windows. .. method:: PurePath.joinpath(*pathsegments) @@ -565,55 +558,55 @@ Pure paths provide the following methods and properties: PureWindowsPath('c:/Program Files') -.. method:: PurePath.match(pattern, *, case_sensitive=None) +.. method:: PurePath.full_match(pattern, *, case_sensitive=None) Match this path against the provided glob-style pattern. Return ``True`` - if matching is successful, ``False`` otherwise. - - If *pattern* is relative, the path can be either relative or absolute, - and matching is done from the right:: + if matching is successful, ``False`` otherwise. For example:: - >>> PurePath('a/b.py').match('*.py') + >>> PurePath('a/b.py').full_match('a/*.py') True - >>> PurePath('/a/b/c.py').match('b/*.py') - True - >>> PurePath('/a/b/c.py').match('a/*.py') + >>> PurePath('a/b.py').full_match('*.py') False + >>> PurePath('/a/b/c.py').full_match('/a/**') + True + >>> PurePath('/a/b/c.py').full_match('**/*.py') + True - If *pattern* is absolute, the path must be absolute, and the whole path - must match:: + As with other methods, case-sensitivity follows platform defaults:: - >>> PurePath('/a.py').match('/*.py') - True - >>> PurePath('a/b.py').match('/*.py') + >>> PurePosixPath('b.py').full_match('*.PY') False + >>> PureWindowsPath('b.py').full_match('*.PY') + True - The *pattern* may be another path object; this speeds up matching the same - pattern against multiple files:: + Set *case_sensitive* to ``True`` or ``False`` to override this behaviour. - >>> pattern = PurePath('*.py') - >>> PurePath('a/b.py').match(pattern) - True + .. versionadded:: 3.13 - .. versionchanged:: 3.12 - Accepts an object implementing the :class:`os.PathLike` interface. - As with other methods, case-sensitivity follows platform defaults:: +.. method:: PurePath.match(pattern, *, case_sensitive=None) - >>> PurePosixPath('b.py').match('*.PY') - False - >>> PureWindowsPath('b.py').match('*.PY') + Match this path against the provided non-recursive glob-style pattern. + Return ``True`` if matching is successful, ``False`` otherwise. + + This method is similar to :meth:`~PurePath.full_match`, but empty patterns + aren't allowed (:exc:`ValueError` is raised), the recursive wildcard + "``**``" isn't supported (it acts like non-recursive "``*``"), and if a + relative pattern is provided, then matching is done from the right:: + + >>> PurePath('a/b.py').match('*.py') + True + >>> PurePath('/a/b/c.py').match('b/*.py') True + >>> PurePath('/a/b/c.py').match('a/*.py') + False - Set *case_sensitive* to ``True`` or ``False`` to override this behaviour. + .. versionchanged:: 3.12 + The *pattern* parameter accepts a :term:`path-like object`. .. versionchanged:: 3.12 The *case_sensitive* parameter was added. - .. versionchanged:: 3.13 - Support for the recursive wildcard "``**``" was added. In previous - versions, it acted like the non-recursive wildcard "``*``". - .. method:: PurePath.relative_to(other, walk_up=False) @@ -813,6 +806,67 @@ bugs or failures in your application):: UnsupportedOperation: cannot instantiate 'WindowsPath' on your system +File URIs +^^^^^^^^^ + +Concrete path objects can be created from, and represented as, 'file' URIs +conforming to :rfc:`8089`. + +.. note:: + + File URIs are not portable across machines with different + :ref:`filesystem encodings <filesystem-encoding>`. + +.. classmethod:: Path.from_uri(uri) + + Return a new path object from parsing a 'file' URI. For example:: + + >>> p = Path.from_uri('file:///etc/hosts') + PosixPath('/etc/hosts') + + On Windows, DOS device and UNC paths may be parsed from URIs:: + + >>> p = Path.from_uri('file:///c:/windows') + WindowsPath('c:/windows') + >>> p = Path.from_uri('file://server/share') + WindowsPath('//server/share') + + Several variant forms are supported:: + + >>> p = Path.from_uri('file:////server/share') + WindowsPath('//server/share') + >>> p = Path.from_uri('file://///server/share') + WindowsPath('//server/share') + >>> p = Path.from_uri('file:c:/windows') + WindowsPath('c:/windows') + >>> p = Path.from_uri('file:/c|/windows') + WindowsPath('c:/windows') + + :exc:`ValueError` is raised if the URI does not start with ``file:``, or + the parsed path isn't absolute. + + .. versionadded:: 3.13 + + +.. method:: Path.as_uri() + + Represent the path as a 'file' URI. :exc:`ValueError` is raised if + the path isn't absolute. + + .. code-block:: pycon + + >>> p = PosixPath('/etc/passwd') + >>> p.as_uri() + 'file:///etc/passwd' + >>> p = WindowsPath('c:/Windows') + >>> p.as_uri() + 'file:///c:/Windows' + + For historical reasons, this method is also available from + :class:`PurePath` objects. However, its use of :func:`os.fsencode` makes + it strictly impure. + + Methods ^^^^^^^ @@ -853,42 +907,6 @@ call fails (for example because the path doesn't exist). .. versionadded:: 3.5 -.. classmethod:: Path.from_uri(uri) - - Return a new path object from parsing a 'file' URI conforming to - :rfc:`8089`. For example:: - - >>> p = Path.from_uri('file:///etc/hosts') - PosixPath('/etc/hosts') - - On Windows, DOS device and UNC paths may be parsed from URIs:: - - >>> p = Path.from_uri('file:///c:/windows') - WindowsPath('c:/windows') - >>> p = Path.from_uri('file://server/share') - WindowsPath('//server/share') - - Several variant forms are supported:: - - >>> p = Path.from_uri('file:////server/share') - WindowsPath('//server/share') - >>> p = Path.from_uri('file://///server/share') - WindowsPath('//server/share') - >>> p = Path.from_uri('file:c:/windows') - WindowsPath('c:/windows') - >>> p = Path.from_uri('file:/c|/windows') - WindowsPath('c:/windows') - - :exc:`ValueError` is raised if the URI does not start with ``file:``, or - the parsed path isn't absolute. - - :func:`os.fsdecode` is used to decode percent-escaped byte sequences, and - so file URIs are not portable across machines with different - :ref:`filesystem encodings <filesystem-encoding>`. - - .. versionadded:: 3.13 - - .. method:: Path.stat(*, follow_symlinks=True) Return a :class:`os.stat_result` object containing information about this path, like :func:`os.stat`. @@ -993,6 +1011,10 @@ call fails (for example because the path doesn't exist). Set *follow_symlinks* to ``True`` or ``False`` to improve performance of recursive globbing. + This method calls :meth:`Path.is_dir` on the top-level directory and + propagates any :exc:`OSError` exception that is raised. Subsequent + :exc:`OSError` exceptions from scanning directories are suppressed. + By default, or when the *case_sensitive* keyword-only argument is set to ``None``, this method matches paths using platform-specific casing rules: typically, case-sensitive on POSIX, and case-insensitive on Windows. @@ -1016,9 +1038,11 @@ call fails (for example because the path doesn't exist). The *follow_symlinks* parameter was added. .. versionchanged:: 3.13 - Emits :exc:`FutureWarning` if the pattern ends with "``**``". In a - future Python release, patterns with this ending will match both files - and directories. Add a trailing slash to match only directories. + Return files and directories if *pattern* ends with "``**``". In + previous versions, only directories were returned. + + .. versionchanged:: 3.13 + The *pattern* parameter accepts a :term:`path-like object`. .. method:: Path.group(*, follow_symlinks=True) @@ -1280,9 +1304,9 @@ call fails (for example because the path doesn't exist). If *exist_ok* is false (the default), :exc:`FileExistsError` is raised if the target directory already exists. - If *exist_ok* is true, :exc:`FileExistsError` exceptions will be - ignored (same behavior as the POSIX ``mkdir -p`` command), but only if the - last path component is not an existing non-directory file. + If *exist_ok* is true, :exc:`FileExistsError` will not be raised unless the given + path already exists in the file system and is not a directory (same + behavior as the POSIX ``mkdir -p`` command). .. versionchanged:: 3.5 The *exist_ok* parameter was added. @@ -1482,6 +1506,9 @@ call fails (for example because the path doesn't exist). .. versionchanged:: 3.13 The *follow_symlinks* parameter was added. + .. versionchanged:: 3.13 + The *pattern* parameter accepts a :term:`path-like object`. + .. method:: Path.rmdir() Remove this directory. The directory must be empty. @@ -1510,9 +1537,13 @@ call fails (for example because the path doesn't exist). .. method:: Path.symlink_to(target, target_is_directory=False) - Make this path a symbolic link to *target*. Under Windows, - *target_is_directory* must be true (default ``False``) if the link's target - is a directory. Under POSIX, *target_is_directory*'s value is ignored. + Make this path a symbolic link pointing to *target*. + + On Windows, a symlink represents either a file or a directory, and does not + morph to the target dynamically. If the target is present, the type of the + symlink will be created to match. Otherwise, the symlink will be created + as a directory if *target_is_directory* is ``True`` or a file symlink (the + default) otherwise. On non-Windows platforms, *target_is_directory* is ignored. :: diff --git a/Doc/library/pdb.rst b/Doc/library/pdb.rst index 2495dcf50bb17f..cb17acfb367619 100644 --- a/Doc/library/pdb.rst +++ b/Doc/library/pdb.rst @@ -48,7 +48,7 @@ at the location you want to break into the debugger, and then run the program. You can then step through the code following this statement, and continue running without the debugger using the :pdbcmd:`continue` command. -.. versionadded:: 3.7 +.. versionchanged:: 3.7 The built-in :func:`breakpoint()`, when called with defaults, can be used instead of ``import pdb; pdb.set_trace()``. @@ -86,12 +86,12 @@ after normal exit of the program), pdb will restart the program. Automatic restarting preserves pdb's state (such as breakpoints) and in most cases is more useful than quitting the debugger upon program's exit. -.. versionadded:: 3.2 - ``-c`` option is introduced to execute commands as if given - in a :file:`.pdbrc` file, see :ref:`debugger-commands`. +.. versionchanged:: 3.2 + Added the ``-c`` option to execute commands as if given + in a :file:`.pdbrc` file; see :ref:`debugger-commands`. -.. versionadded:: 3.7 - ``-m`` option is introduced to execute modules similar to the way +.. versionchanged:: 3.7 + Added the ``-m`` option to execute modules similar to the way ``python -m`` does. As with a script, the debugger will pause execution just before the first line of the module. @@ -209,12 +209,12 @@ access further features, you have to do this yourself: .. audit-event:: pdb.Pdb "" pdb.Pdb - .. versionadded:: 3.1 - The *skip* argument. + .. versionchanged:: 3.1 + Added the *skip* parameter. - .. versionadded:: 3.2 - The *nosigint* argument. Previously, a SIGINT handler was never set by - Pdb. + .. versionchanged:: 3.2 + Added the *nosigint* parameter. + Previously, a SIGINT handler was never set by Pdb. .. versionchanged:: 3.6 The *readrc* argument. @@ -467,8 +467,8 @@ can be overridden by the local file. raised or propagated is indicated by ``>>``, if it differs from the current line. - .. versionadded:: 3.2 - The ``>>`` marker. + .. versionchanged:: 3.2 + Added the ``>>`` marker. .. pdbcommand:: ll | longlist @@ -583,8 +583,8 @@ can be overridden by the local file. .. versionadded:: 3.2 - .. versionadded:: 3.13 - ``exit()`` and ``quit()`` can be used to exit :pdbcmd:`interact` + .. versionchanged:: 3.13 + ``exit()`` and ``quit()`` can be used to exit the :pdbcmd:`interact` command. .. versionchanged:: 3.13 diff --git a/Doc/library/pickle.rst b/Doc/library/pickle.rst index 93387fb0b45038..acada092afb679 100644 --- a/Doc/library/pickle.rst +++ b/Doc/library/pickle.rst @@ -272,13 +272,13 @@ The :mod:`pickle` module defines three exceptions: .. exception:: PickleError - Common base class for the other pickling exceptions. It inherits + Common base class for the other pickling exceptions. It inherits from :exc:`Exception`. .. exception:: PicklingError Error raised when an unpicklable object is encountered by :class:`Pickler`. - It inherits :exc:`PickleError`. + It inherits from :exc:`PickleError`. Refer to :ref:`pickle-picklable` to learn what kinds of objects can be pickled. @@ -286,7 +286,7 @@ The :mod:`pickle` module defines three exceptions: .. exception:: UnpicklingError Error raised when there is a problem unpickling an object, such as a data - corruption or a security violation. It inherits :exc:`PickleError`. + corruption or a security violation. It inherits from :exc:`PickleError`. Note that other exceptions may also be raised during unpickling, including (but not necessarily limited to) AttributeError, EOFError, ImportError, and @@ -345,6 +345,10 @@ The :mod:`pickle` module exports three classes, :class:`Pickler`, See :ref:`pickle-persistent` for details and examples of uses. + .. versionchanged:: 3.13 + Add the default implementation of this method in the C implementation + of :class:`!Pickler`. + .. attribute:: dispatch_table A pickler object's dispatch table is a registry of *reduction @@ -352,7 +356,7 @@ The :mod:`pickle` module exports three classes, :class:`Pickler`, :func:`copyreg.pickle`. It is a mapping whose keys are classes and whose values are reduction functions. A reduction function takes a single argument of the associated class and should - conform to the same interface as a :meth:`__reduce__` + conform to the same interface as a :meth:`~object.__reduce__` method. By default, a pickler object will not have a @@ -372,7 +376,7 @@ The :mod:`pickle` module exports three classes, :class:`Pickler`, Special reducer that can be defined in :class:`Pickler` subclasses. This method has priority over any reducer in the :attr:`dispatch_table`. It - should conform to the same interface as a :meth:`__reduce__` method, and + should conform to the same interface as a :meth:`~object.__reduce__` method, and can optionally return ``NotImplemented`` to fallback on :attr:`dispatch_table`-registered reducers to pickle ``obj``. @@ -446,6 +450,10 @@ The :mod:`pickle` module exports three classes, :class:`Pickler`, See :ref:`pickle-persistent` for details and examples of uses. + .. versionchanged:: 3.13 + Add the default implementation of this method in the C implementation + of :class:`!Unpickler`. + .. method:: find_class(module, name) Import *module* if necessary and return the object called *name* from it, @@ -508,7 +516,7 @@ The following types can be pickled: * classes accessible from the top level of a module; -* instances of such classes whose the result of calling :meth:`__getstate__` +* instances of such classes whose the result of calling :meth:`~object.__getstate__` is picklable (see section :ref:`pickle-inst` for details). Attempts to pickle unpicklable objects will raise the :exc:`PicklingError` @@ -544,7 +552,7 @@ purpose, so you can fix bugs in a class or add methods to the class and still load objects that were created with an earlier version of the class. If you plan to have long-lived objects that will see many versions of a class, it may be worthwhile to put a version number in the objects so that suitable -conversions can be made by the class's :meth:`__setstate__` method. +conversions can be made by the class's :meth:`~object.__setstate__` method. .. _pickle-inst: @@ -559,7 +567,7 @@ customize, and control how class instances are pickled and unpickled. In most cases, no additional code is needed to make instances picklable. By default, pickle will retrieve the class and the attributes of an instance via -introspection. When a class instance is unpickled, its :meth:`__init__` method +introspection. When a class instance is unpickled, its :meth:`~object.__init__` method is usually *not* invoked. The default behaviour first creates an uninitialized instance and then restores the saved attributes. The following code shows an implementation of this behaviour:: @@ -650,30 +658,30 @@ methods: Refer to the section :ref:`pickle-state` for more information about how to use -the methods :meth:`__getstate__` and :meth:`__setstate__`. +the methods :meth:`~object.__getstate__` and :meth:`~object.__setstate__`. .. note:: - At unpickling time, some methods like :meth:`__getattr__`, - :meth:`__getattribute__`, or :meth:`__setattr__` may be called upon the + At unpickling time, some methods like :meth:`~object.__getattr__`, + :meth:`~object.__getattribute__`, or :meth:`~object.__setattr__` may be called upon the instance. In case those methods rely on some internal invariant being - true, the type should implement :meth:`__new__` to establish such an - invariant, as :meth:`__init__` is not called when unpickling an + true, the type should implement :meth:`~object.__new__` to establish such an + invariant, as :meth:`~object.__init__` is not called when unpickling an instance. .. index:: pair: copy; protocol As we shall see, pickle does not use directly the methods described above. In fact, these methods are part of the copy protocol which implements the -:meth:`__reduce__` special method. The copy protocol provides a unified +:meth:`~object.__reduce__` special method. The copy protocol provides a unified interface for retrieving the data necessary for pickling and copying objects. [#]_ -Although powerful, implementing :meth:`__reduce__` directly in your classes is +Although powerful, implementing :meth:`~object.__reduce__` directly in your classes is error prone. For this reason, class designers should use the high-level -interface (i.e., :meth:`__getnewargs_ex__`, :meth:`__getstate__` and -:meth:`__setstate__`) whenever possible. We will show, however, cases where -using :meth:`__reduce__` is the only option or leads to more efficient pickling +interface (i.e., :meth:`~object.__getnewargs_ex__`, :meth:`~object.__getstate__` and +:meth:`~object.__setstate__`) whenever possible. We will show, however, cases where +using :meth:`!__reduce__` is the only option or leads to more efficient pickling or both. .. method:: object.__reduce__() @@ -708,8 +716,9 @@ or both. These items will be appended to the object either using ``obj.append(item)`` or, in batch, using ``obj.extend(list_of_items)``. This is primarily used for list subclasses, but may be used by other - classes as long as they have :meth:`append` and :meth:`extend` methods with - the appropriate signature. (Whether :meth:`append` or :meth:`extend` is + classes as long as they have + :ref:`append and extend methods <typesseq-common>` with + the appropriate signature. (Whether :meth:`!append` or :meth:`!extend` is used depends on which pickle protocol version is used as well as the number of items to append, so both must be supported.) @@ -785,8 +794,8 @@ any other code which depends on pickling, then one can create a pickler with a private dispatch table. The global dispatch table managed by the :mod:`copyreg` module is -available as :data:`copyreg.dispatch_table`. Therefore, one may -choose to use a modified copy of :data:`copyreg.dispatch_table` as a +available as :data:`!copyreg.dispatch_table`. Therefore, one may +choose to use a modified copy of :data:`!copyreg.dispatch_table` as a private dispatch table. For example :: @@ -825,12 +834,12 @@ Handling Stateful Objects single: __setstate__() (copy protocol) Here's an example that shows how to modify pickling behavior for a class. -The :class:`TextReader` class opens a text file, and returns the line number and +The :class:`!TextReader` class below opens a text file, and returns the line number and line contents each time its :meth:`!readline` method is called. If a -:class:`TextReader` instance is pickled, all attributes *except* the file object +:class:`!TextReader` instance is pickled, all attributes *except* the file object member are saved. When the instance is unpickled, the file is reopened, and -reading resumes from the last location. The :meth:`__setstate__` and -:meth:`__getstate__` methods are used to implement this behavior. :: +reading resumes from the last location. The :meth:`!__setstate__` and +:meth:`!__getstate__` methods are used to implement this behavior. :: class TextReader: """Print and number lines in a text file.""" @@ -895,7 +904,7 @@ functions and classes. For those cases, it is possible to subclass from the :class:`Pickler` class and implement a :meth:`~Pickler.reducer_override` method. This method can return an -arbitrary reduction tuple (see :meth:`__reduce__`). It can alternatively return +arbitrary reduction tuple (see :meth:`~object.__reduce__`). It can alternatively return ``NotImplemented`` to fallback to the traditional behavior. If both the :attr:`~Pickler.dispatch_table` and @@ -963,7 +972,7 @@ provided by pickle protocol 5 and higher. Provider API ^^^^^^^^^^^^ -The large data objects to be pickled must implement a :meth:`__reduce_ex__` +The large data objects to be pickled must implement a :meth:`~object.__reduce_ex__` method specialized for protocol 5 and higher, which returns a :class:`PickleBuffer` instance (instead of e.g. a :class:`bytes` object) for any large data. diff --git a/Doc/library/pickletools.rst b/Doc/library/pickletools.rst index 41930f8cbe8412..9739207a224431 100644 --- a/Doc/library/pickletools.rst +++ b/Doc/library/pickletools.rst @@ -94,8 +94,8 @@ Programmatic Interface a short description. The value of *annotate* is used as a hint for the column where annotation should start. - .. versionadded:: 3.2 - The *annotate* argument. + .. versionchanged:: 3.2 + Added the *annotate* parameter. .. function:: genops(pickle) diff --git a/Doc/library/plistlib.rst b/Doc/library/plistlib.rst index 732ef3536863cc..7416ca2650bab4 100644 --- a/Doc/library/plistlib.rst +++ b/Doc/library/plistlib.rst @@ -27,7 +27,7 @@ top level object is a dictionary. To write out and to parse a plist file, use the :func:`dump` and :func:`load` functions. -To work with plist data in bytes objects, use :func:`dumps` +To work with plist data in bytes or string objects, use :func:`dumps` and :func:`loads`. Values can be strings, integers, floats, booleans, tuples, lists, dictionaries @@ -52,7 +52,7 @@ or :class:`datetime.datetime` objects. This module defines the following functions: -.. function:: load(fp, *, fmt=None, dict_type=dict) +.. function:: load(fp, *, fmt=None, dict_type=dict, aware_datetime=False) Read a plist file. *fp* should be a readable and binary file object. Return the unpacked root object (which usually is a @@ -69,6 +69,10 @@ This module defines the following functions: The *dict_type* is the type used for dictionaries that are read from the plist file. + When *aware_datetime* is true, fields with type ``datetime.datetime`` will + be created as :ref:`aware object <datetime-naive-aware>`, with + :attr:`!tzinfo` as :attr:`datetime.UTC`. + XML data for the :data:`FMT_XML` format is parsed using the Expat parser from :mod:`xml.parsers.expat` -- see its documentation for possible exceptions on ill-formed XML. Unknown elements will simply be ignored @@ -79,16 +83,21 @@ This module defines the following functions: .. versionadded:: 3.4 + .. versionchanged:: 3.13 + The keyword-only parameter *aware_datetime* has been added. + -.. function:: loads(data, *, fmt=None, dict_type=dict) +.. function:: loads(data, *, fmt=None, dict_type=dict, aware_datetime=False) - Load a plist from a bytes object. See :func:`load` for an explanation of - the keyword arguments. + Load a plist from a bytes or string object. See :func:`load` for an + explanation of the keyword arguments. .. versionadded:: 3.4 + .. versionchanged:: 3.13 + *data* can be a string when *fmt* equals :data:`FMT_XML`. -.. function:: dump(value, fp, *, fmt=FMT_XML, sort_keys=True, skipkeys=False) +.. function:: dump(value, fp, *, fmt=FMT_XML, sort_keys=True, skipkeys=False, aware_datetime=False) Write *value* to a plist file. *Fp* should be a writable, binary file object. @@ -107,6 +116,10 @@ This module defines the following functions: When *skipkeys* is false (the default) the function raises :exc:`TypeError` when a key of a dictionary is not a string, otherwise such keys are skipped. + When *aware_datetime* is true and any field with type ``datetime.datetime`` + is set as a :ref:`aware object <datetime-naive-aware>`, it will convert to + UTC timezone before writing it. + A :exc:`TypeError` will be raised if the object is of an unsupported type or a container that contains objects of unsupported types. @@ -115,8 +128,11 @@ This module defines the following functions: .. versionadded:: 3.4 + .. versionchanged:: 3.13 + The keyword-only parameter *aware_datetime* has been added. + -.. function:: dumps(value, *, fmt=FMT_XML, sort_keys=True, skipkeys=False) +.. function:: dumps(value, *, fmt=FMT_XML, sort_keys=True, skipkeys=False, aware_datetime=False) Return *value* as a plist-formatted bytes object. See the documentation for :func:`dump` for an explanation of the keyword diff --git a/Doc/library/pyclbr.rst b/Doc/library/pyclbr.rst index 1c40ba4838ca75..1e9876849b02f3 100644 --- a/Doc/library/pyclbr.rst +++ b/Doc/library/pyclbr.rst @@ -58,106 +58,115 @@ of these classes. Function Objects ---------------- -Class :class:`Function` instances describe functions defined by def -statements. They have the following attributes: +.. class:: Function -.. attribute:: Function.file + Class :class:`!Function` instances describe functions defined by def + statements. They have the following attributes: - Name of the file in which the function is defined. + .. attribute:: file -.. attribute:: Function.module + Name of the file in which the function is defined. - The name of the module defining the function described. + .. attribute:: module -.. attribute:: Function.name + The name of the module defining the function described. - The name of the function. + .. attribute:: name -.. attribute:: Function.lineno + The name of the function. - The line number in the file where the definition starts. + .. attribute:: lineno -.. attribute:: Function.parent + The line number in the file where the definition starts. - For top-level functions, None. For nested functions, the parent. - .. versionadded:: 3.7 + .. attribute:: parent + For top-level functions, ``None``. For nested functions, the parent. -.. attribute:: Function.children + .. versionadded:: 3.7 - A dictionary mapping names to descriptors for nested functions and - classes. - .. versionadded:: 3.7 + .. attribute:: children + A :class:`dictionary <dict>` mapping names to descriptors for nested functions and + classes. -.. attribute:: Function.is_async + .. versionadded:: 3.7 - ``True`` for functions that are defined with the ``async`` prefix, ``False`` otherwise. - .. versionadded:: 3.10 + .. attribute:: is_async + + ``True`` for functions that are defined with the + :keyword:`async <async def>` prefix, ``False`` otherwise. + + .. versionadded:: 3.10 .. _pyclbr-class-objects: Class Objects ------------- -Class :class:`Class` instances describe classes defined by class -statements. They have the same attributes as Functions and two more. + +.. class:: Class + + Class :class:`!Class` instances describe classes defined by class + statements. They have the same attributes as :class:`Functions <Function>` + and two more. -.. attribute:: Class.file + .. attribute:: file - Name of the file in which the class is defined. + Name of the file in which the class is defined. -.. attribute:: Class.module + .. attribute:: module - The name of the module defining the class described. + The name of the module defining the class described. -.. attribute:: Class.name + .. attribute:: name - The name of the class. + The name of the class. -.. attribute:: Class.lineno + .. attribute:: lineno - The line number in the file where the definition starts. + The line number in the file where the definition starts. -.. attribute:: Class.parent + .. attribute:: parent - For top-level classes, None. For nested classes, the parent. + For top-level classes, None. For nested classes, the parent. - .. versionadded:: 3.7 + .. versionadded:: 3.7 -.. attribute:: Class.children + .. attribute:: children - A dictionary mapping names to descriptors for nested functions and - classes. + A dictionary mapping names to descriptors for nested functions and + classes. - .. versionadded:: 3.7 + .. versionadded:: 3.7 -.. attribute:: Class.super + .. attribute:: super - A list of :class:`Class` objects which describe the immediate base - classes of the class being described. Classes which are named as - superclasses but which are not discoverable by :func:`readmodule_ex` - are listed as a string with the class name instead of as - :class:`Class` objects. + A list of :class:`!Class` objects which describe the immediate base + classes of the class being described. Classes which are named as + superclasses but which are not discoverable by :func:`readmodule_ex` + are listed as a string with the class name instead of as + :class:`!Class` objects. -.. attribute:: Class.methods + .. attribute:: methods - A dictionary mapping method names to line numbers. This can be - derived from the newer children dictionary, but remains for - back-compatibility. + A :class:`dictionary <dict>` mapping method names to line numbers. + This can be derived from the newer :attr:`children` dictionary, + but remains for + back-compatibility. diff --git a/Doc/library/queue.rst b/Doc/library/queue.rst index b2b787c5a8260c..1421fc2e552f0e 100644 --- a/Doc/library/queue.rst +++ b/Doc/library/queue.rst @@ -93,6 +93,14 @@ The :mod:`queue` module defines the following classes and exceptions: on a :class:`Queue` object which is full. +.. exception:: ShutDown + + Exception raised when :meth:`~Queue.put` or :meth:`~Queue.get` is called on + a :class:`Queue` object which has been shut down. + + .. versionadded:: 3.13 + + .. _queueobjects: Queue Objects @@ -135,6 +143,8 @@ provide the public methods described below. immediately available, else raise the :exc:`Full` exception (*timeout* is ignored in that case). + Raises :exc:`ShutDown` if the queue has been shut down. + .. method:: Queue.put_nowait(item) @@ -155,6 +165,9 @@ provide the public methods described below. an uninterruptible wait on an underlying lock. This means that no exceptions can occur, and in particular a SIGINT will not trigger a :exc:`KeyboardInterrupt`. + Raises :exc:`ShutDown` if the queue has been shut down and is empty, or if + the queue has been shut down immediately. + .. method:: Queue.get_nowait() @@ -177,6 +190,8 @@ fully processed by daemon consumer threads. Raises a :exc:`ValueError` if called more times than there were items placed in the queue. + Raises :exc:`ShutDown` if the queue has been shut down immediately. + .. method:: Queue.join() @@ -187,6 +202,8 @@ fully processed by daemon consumer threads. indicate that the item was retrieved and all work on it is complete. When the count of unfinished tasks drops to zero, :meth:`join` unblocks. + Raises :exc:`ShutDown` if the queue has been shut down immediately. + Example of how to wait for enqueued tasks to be completed:: @@ -214,6 +231,27 @@ Example of how to wait for enqueued tasks to be completed:: print('All work completed') +Terminating queues +^^^^^^^^^^^^^^^^^^ + +:class:`Queue` objects can be made to prevent further interaction by shutting +them down. + +.. method:: Queue.shutdown(immediate=False) + + Shut down the queue, making :meth:`~Queue.get` and :meth:`~Queue.put` raise + :exc:`ShutDown`. + + By default, :meth:`~Queue.get` on a shut down queue will only raise once the + queue is empty. Set *immediate* to true to make :meth:`~Queue.get` raise + immediately instead. + + All blocked callers of :meth:`~Queue.put` will be unblocked. If *immediate* + is true, also unblock callers of :meth:`~Queue.get` and :meth:`~Queue.join`. + + .. versionadded:: 3.13 + + SimpleQueue Objects ------------------- diff --git a/Doc/library/random.rst b/Doc/library/random.rst index feaf260caf3568..d0ced2416c9578 100644 --- a/Doc/library/random.rst +++ b/Doc/library/random.rst @@ -34,10 +34,8 @@ instance of the :class:`random.Random` class. You can instantiate your own instances of :class:`Random` to get generators that don't share state. Class :class:`Random` can also be subclassed if you want to use a different -basic generator of your own devising: in that case, override the :meth:`~Random.random`, -:meth:`~Random.seed`, :meth:`~Random.getstate`, and :meth:`~Random.setstate` methods. -Optionally, a new generator can supply a :meth:`~Random.getrandbits` method --- this -allows :meth:`randrange` to produce selections over an arbitrarily large range. +basic generator of your own devising: see the documentation on that class for +more details. The :mod:`random` module also provides the :class:`SystemRandom` class which uses the system function :func:`os.urandom` to generate random numbers @@ -88,7 +86,7 @@ Bookkeeping functions .. versionchanged:: 3.11 The *seed* must be one of the following types: - *NoneType*, :class:`int`, :class:`float`, :class:`str`, + ``None``, :class:`int`, :class:`float`, :class:`str`, :class:`bytes`, or :class:`bytearray`. .. function:: getstate() @@ -412,6 +410,37 @@ Alternative Generator ``None``, :class:`int`, :class:`float`, :class:`str`, :class:`bytes`, or :class:`bytearray`. + Subclasses of :class:`!Random` should override the following methods if they + wish to make use of a different basic generator: + + .. method:: Random.seed(a=None, version=2) + + Override this method in subclasses to customise the :meth:`~random.seed` + behaviour of :class:`!Random` instances. + + .. method:: Random.getstate() + + Override this method in subclasses to customise the :meth:`~random.getstate` + behaviour of :class:`!Random` instances. + + .. method:: Random.setstate(state) + + Override this method in subclasses to customise the :meth:`~random.setstate` + behaviour of :class:`!Random` instances. + + .. method:: Random.random() + + Override this method in subclasses to customise the :meth:`~random.random` + behaviour of :class:`!Random` instances. + + Optionally, a custom generator subclass can also supply the following method: + + .. method:: Random.getrandbits(k) + + Override this method in subclasses to customise the + :meth:`~random.getrandbits` behaviour of :class:`!Random` instances. + + .. class:: SystemRandom([seed]) Class that uses the :func:`os.urandom` function for generating random numbers @@ -445,30 +474,30 @@ Examples Basic examples:: - >>> random() # Random float: 0.0 <= x < 1.0 + >>> random() # Random float: 0.0 <= x < 1.0 0.37444887175646646 - >>> uniform(2.5, 10.0) # Random float: 2.5 <= x <= 10.0 + >>> uniform(2.5, 10.0) # Random float: 2.5 <= x <= 10.0 3.1800146073117523 - >>> expovariate(1 / 5) # Interval between arrivals averaging 5 seconds + >>> expovariate(1 / 5) # Interval between arrivals averaging 5 seconds 5.148957571865031 - >>> randrange(10) # Integer from 0 to 9 inclusive + >>> randrange(10) # Integer from 0 to 9 inclusive 7 - >>> randrange(0, 101, 2) # Even integer from 0 to 100 inclusive + >>> randrange(0, 101, 2) # Even integer from 0 to 100 inclusive 26 - >>> choice(['win', 'lose', 'draw']) # Single random element from a sequence + >>> choice(['win', 'lose', 'draw']) # Single random element from a sequence 'draw' >>> deck = 'ace two three four'.split() - >>> shuffle(deck) # Shuffle a list + >>> shuffle(deck) # Shuffle a list >>> deck ['four', 'two', 'ace', 'three'] - >>> sample([10, 20, 30, 40, 50], k=4) # Four samples without replacement + >>> sample([10, 20, 30, 40, 50], k=4) # Four samples without replacement [40, 10, 50, 30] Simulations:: @@ -572,14 +601,14 @@ Simulation of arrival times and service deliveries for a multiserver queue:: including simulation, sampling, shuffling, and cross-validation. `Economics Simulation - <https://nbviewer.jupyter.org/url/norvig.com/ipython/Economics.ipynb>`_ + <https://nbviewer.org/url/norvig.com/ipython/Economics.ipynb>`_ a simulation of a marketplace by `Peter Norvig <https://norvig.com/bio.html>`_ that shows effective use of many of the tools and distributions provided by this module (gauss, uniform, sample, betavariate, choice, triangular, and randrange). `A Concrete Introduction to Probability (using Python) - <https://nbviewer.jupyter.org/url/norvig.com/ipython/Probability.ipynb>`_ + <https://nbviewer.org/url/norvig.com/ipython/Probability.ipynb>`_ a tutorial by `Peter Norvig <https://norvig.com/bio.html>`_ covering the basics of probability theory, how to write simulations, and how to perform data analysis using Python. diff --git a/Doc/library/re.rst b/Doc/library/re.rst index 302f7224de4a7a..0a8c88b50cdeec 100644 --- a/Doc/library/re.rst +++ b/Doc/library/re.rst @@ -17,7 +17,7 @@ those found in Perl. Both patterns and strings to be searched can be Unicode strings (:class:`str`) as well as 8-bit strings (:class:`bytes`). However, Unicode strings and 8-bit strings cannot be mixed: -that is, you cannot match a Unicode string with a byte pattern or +that is, you cannot match a Unicode string with a bytes pattern or vice-versa; similarly, when asking for a substitution, the replacement string must be of the same type as both the pattern and the search string. @@ -257,8 +257,7 @@ The special characters are: .. index:: single: \ (backslash); in regular expressions * Character classes such as ``\w`` or ``\S`` (defined below) are also accepted - inside a set, although the characters they match depends on whether - :const:`ASCII` or :const:`LOCALE` mode is in force. + inside a set, although the characters they match depend on the flags_ used. .. index:: single: ^ (caret); in regular expressions @@ -326,18 +325,24 @@ The special characters are: currently supported extensions. ``(?aiLmsux)`` - (One or more letters from the set ``'a'``, ``'i'``, ``'L'``, ``'m'``, - ``'s'``, ``'u'``, ``'x'``.) The group matches the empty string; the - letters set the corresponding flags: :const:`re.A` (ASCII-only matching), - :const:`re.I` (ignore case), :const:`re.L` (locale dependent), - :const:`re.M` (multi-line), :const:`re.S` (dot matches all), - :const:`re.U` (Unicode matching), and :const:`re.X` (verbose), - for the entire regular expression. + (One or more letters from the set + ``'a'``, ``'i'``, ``'L'``, ``'m'``, ``'s'``, ``'u'``, ``'x'``.) + The group matches the empty string; + the letters set the corresponding flags for the entire regular expression: + + * :const:`re.A` (ASCII-only matching) + * :const:`re.I` (ignore case) + * :const:`re.L` (locale dependent) + * :const:`re.M` (multi-line) + * :const:`re.S` (dot matches all) + * :const:`re.U` (Unicode matching) + * :const:`re.X` (verbose) + (The flags are described in :ref:`contents-of-module-re`.) This is useful if you wish to include the flags as part of the regular expression, instead of passing a *flag* argument to the - :func:`re.compile` function. Flags should be used first in the - expression string. + :func:`re.compile` function. + Flags should be used first in the expression string. .. versionchanged:: 3.11 This construction can only be used at the start of the expression. @@ -351,14 +356,20 @@ The special characters are: pattern. ``(?aiLmsux-imsx:...)`` - (Zero or more letters from the set ``'a'``, ``'i'``, ``'L'``, ``'m'``, - ``'s'``, ``'u'``, ``'x'``, optionally followed by ``'-'`` followed by + (Zero or more letters from the set + ``'a'``, ``'i'``, ``'L'``, ``'m'``, ``'s'``, ``'u'``, ``'x'``, + optionally followed by ``'-'`` followed by one or more letters from the ``'i'``, ``'m'``, ``'s'``, ``'x'``.) - The letters set or remove the corresponding flags: - :const:`re.A` (ASCII-only matching), :const:`re.I` (ignore case), - :const:`re.L` (locale dependent), :const:`re.M` (multi-line), - :const:`re.S` (dot matches all), :const:`re.U` (Unicode matching), - and :const:`re.X` (verbose), for the part of the expression. + The letters set or remove the corresponding flags for the part of the expression: + + * :const:`re.A` (ASCII-only matching) + * :const:`re.I` (ignore case) + * :const:`re.L` (locale dependent) + * :const:`re.M` (multi-line) + * :const:`re.S` (dot matches all) + * :const:`re.U` (Unicode matching) + * :const:`re.X` (verbose) + (The flags are described in :ref:`contents-of-module-re`.) The letters ``'a'``, ``'L'`` and ``'u'`` are mutually exclusive when used @@ -366,7 +377,7 @@ The special characters are: when one of them appears in an inline group, it overrides the matching mode in the enclosing group. In Unicode patterns ``(?a:...)`` switches to ASCII-only matching, and ``(?u:...)`` switches to Unicode matching - (default). In byte pattern ``(?L:...)`` switches to locale depending + (default). In bytes patterns ``(?L:...)`` switches to locale dependent matching, and ``(?a:...)`` switches to ASCII-only matching (default). This override is only in effect for the narrow inline group, and the original matching mode is restored outside of the group. @@ -529,47 +540,61 @@ character ``'$'``. ``\b`` Matches the empty string, but only at the beginning or end of a word. - A word is defined as a sequence of word characters. Note that formally, - ``\b`` is defined as the boundary between a ``\w`` and a ``\W`` character - (or vice versa), or between ``\w`` and the beginning/end of the string. - This means that ``r'\bfoo\b'`` matches ``'foo'``, ``'foo.'``, ``'(foo)'``, - ``'bar foo baz'`` but not ``'foobar'`` or ``'foo3'``. - - By default Unicode alphanumerics are the ones used in Unicode patterns, but - this can be changed by using the :const:`ASCII` flag. Word boundaries are - determined by the current locale if the :const:`LOCALE` flag is used. - Inside a character range, ``\b`` represents the backspace character, for - compatibility with Python's string literals. + A word is defined as a sequence of word characters. + Note that formally, ``\b`` is defined as the boundary + between a ``\w`` and a ``\W`` character (or vice versa), + or between ``\w`` and the beginning or end of the string. + This means that ``r'\bat\b'`` matches ``'at'``, ``'at.'``, ``'(at)'``, + and ``'as at ay'`` but not ``'attempt'`` or ``'atlas'``. + + The default word characters in Unicode (str) patterns + are Unicode alphanumerics and the underscore, + but this can be changed by using the :py:const:`~re.ASCII` flag. + Word boundaries are determined by the current locale + if the :py:const:`~re.LOCALE` flag is used. + + .. note:: + + Inside a character range, ``\b`` represents the backspace character, + for compatibility with Python's string literals. .. index:: single: \B; in regular expressions ``\B`` - Matches the empty string, but only when it is *not* at the beginning or end - of a word. This means that ``r'py\B'`` matches ``'python'``, ``'py3'``, - ``'py2'``, but not ``'py'``, ``'py.'``, or ``'py!'``. - ``\B`` is just the opposite of ``\b``, so word characters in Unicode - patterns are Unicode alphanumerics or the underscore, although this can - be changed by using the :const:`ASCII` flag. Word boundaries are - determined by the current locale if the :const:`LOCALE` flag is used. + Matches the empty string, + but only when it is *not* at the beginning or end of a word. + This means that ``r'at\B'`` matches ``'athens'``, ``'atom'``, + ``'attorney'``, but not ``'at'``, ``'at.'``, or ``'at!'``. + ``\B`` is the opposite of ``\b``, + so word characters in Unicode (str) patterns + are Unicode alphanumerics or the underscore, + although this can be changed by using the :py:const:`~re.ASCII` flag. + Word boundaries are determined by the current locale + if the :py:const:`~re.LOCALE` flag is used. .. index:: single: \d; in regular expressions ``\d`` For Unicode (str) patterns: - Matches any Unicode decimal digit (that is, any character in - Unicode character category [Nd]). This includes ``[0-9]``, and - also many other digit characters. If the :const:`ASCII` flag is - used only ``[0-9]`` is matched. + Matches any Unicode decimal digit + (that is, any character in Unicode character category `[Nd]`__). + This includes ``[0-9]``, and also many other digit characters. + + Matches ``[0-9]`` if the :py:const:`~re.ASCII` flag is used. + + __ https://www.unicode.org/versions/Unicode15.0.0/ch04.pdf#G134153 For 8-bit (bytes) patterns: - Matches any decimal digit; this is equivalent to ``[0-9]``. + Matches any decimal digit in the ASCII character set; + this is equivalent to ``[0-9]``. .. index:: single: \D; in regular expressions ``\D`` - Matches any character which is not a decimal digit. This is - the opposite of ``\d``. If the :const:`ASCII` flag is used this - becomes the equivalent of ``[^0-9]``. + Matches any character which is not a decimal digit. + This is the opposite of ``\d``. + + Matches ``[^0-9]`` if the :py:const:`~re.ASCII` flag is used. .. index:: single: \s; in regular expressions @@ -578,8 +603,9 @@ character ``'$'``. Matches Unicode whitespace characters (which includes ``[ \t\n\r\f\v]``, and also many other characters, for example the non-breaking spaces mandated by typography rules in many - languages). If the :const:`ASCII` flag is used, only - ``[ \t\n\r\f\v]`` is matched. + languages). + + Matches ``[ \t\n\r\f\v]`` if the :py:const:`~re.ASCII` flag is used. For 8-bit (bytes) patterns: Matches characters considered whitespace in the ASCII character set; @@ -589,30 +615,39 @@ character ``'$'``. ``\S`` Matches any character which is not a whitespace character. This is - the opposite of ``\s``. If the :const:`ASCII` flag is used this - becomes the equivalent of ``[^ \t\n\r\f\v]``. + the opposite of ``\s``. + + Matches ``[^ \t\n\r\f\v]`` if the :py:const:`~re.ASCII` flag is used. .. index:: single: \w; in regular expressions ``\w`` For Unicode (str) patterns: - Matches Unicode word characters; this includes alphanumeric characters (as defined by :meth:`str.isalnum`) + Matches Unicode word characters; + this includes all Unicode alphanumeric characters + (as defined by :py:meth:`str.isalnum`), as well as the underscore (``_``). - If the :const:`ASCII` flag is used, only ``[a-zA-Z0-9_]`` is matched. + + Matches ``[a-zA-Z0-9_]`` if the :py:const:`~re.ASCII` flag is used. For 8-bit (bytes) patterns: Matches characters considered alphanumeric in the ASCII character set; - this is equivalent to ``[a-zA-Z0-9_]``. If the :const:`LOCALE` flag is - used, matches characters considered alphanumeric in the current locale - and the underscore. + this is equivalent to ``[a-zA-Z0-9_]``. + If the :py:const:`~re.LOCALE` flag is used, + matches characters considered alphanumeric in the current locale and the underscore. .. index:: single: \W; in regular expressions ``\W`` - Matches any character which is not a word character. This is - the opposite of ``\w``. If the :const:`ASCII` flag is used this - becomes the equivalent of ``[^a-zA-Z0-9_]``. If the :const:`LOCALE` flag is - used, matches characters which are neither alphanumeric in the current locale + Matches any character which is not a word character. + This is the opposite of ``\w``. + By default, matches non-underscore (``_``) characters + for which :py:meth:`str.isalnum` returns ``False``. + + Matches ``[^a-zA-Z0-9_]`` if the :py:const:`~re.ASCII` flag is used. + + If the :py:const:`~re.LOCALE` flag is used, + matches characters which are neither alphanumeric in the current locale nor the underscore. .. index:: single: \Z; in regular expressions @@ -644,9 +679,11 @@ string literals are also accepted by the regular expression parser:: (Note that ``\b`` is used to represent word boundaries, and means "backspace" only inside character classes.) -``'\u'``, ``'\U'``, and ``'\N'`` escape sequences are only recognized in Unicode -patterns. In bytes patterns they are errors. Unknown escapes of ASCII -letters are reserved for future use and treated as errors. +``'\u'``, ``'\U'``, and ``'\N'`` escape sequences are +only recognized in Unicode (str) patterns. +In bytes patterns they are errors. +Unknown escapes of ASCII letters are reserved +for future use and treated as errors. Octal escapes are included in a limited form. If the first digit is a 0, or if there are three octal digits, it is considered an octal escape. Otherwise, it is @@ -694,30 +731,37 @@ Flags Make ``\w``, ``\W``, ``\b``, ``\B``, ``\d``, ``\D``, ``\s`` and ``\S`` perform ASCII-only matching instead of full Unicode matching. This is only - meaningful for Unicode patterns, and is ignored for byte patterns. + meaningful for Unicode (str) patterns, and is ignored for bytes patterns. + Corresponds to the inline flag ``(?a)``. - Note that for backward compatibility, the :const:`re.U` flag still - exists (as well as its synonym :const:`re.UNICODE` and its embedded - counterpart ``(?u)``), but these are redundant in Python 3 since - matches are Unicode by default for strings (and Unicode matching - isn't allowed for bytes). + .. note:: + + The :py:const:`~re.U` flag still exists for backward compatibility, + but is redundant in Python 3 since + matches are Unicode by default for ``str`` patterns, + and Unicode matching isn't allowed for bytes patterns. + :py:const:`~re.UNICODE` and the inline flag ``(?u)`` are similarly redundant. .. data:: DEBUG Display debug information about compiled expression. + No corresponding inline flag. .. data:: I IGNORECASE - Perform case-insensitive matching; expressions like ``[A-Z]`` will also - match lowercase letters. Full Unicode matching (such as ``Ü`` matching - ``ü``) also works unless the :const:`re.ASCII` flag is used to disable - non-ASCII matches. The current locale does not change the effect of this - flag unless the :const:`re.LOCALE` flag is also used. + Perform case-insensitive matching; + expressions like ``[A-Z]`` will also match lowercase letters. + Full Unicode matching (such as ``Ü`` matching ``ü``) + also works unless the :py:const:`~re.ASCII` flag + is used to disable non-ASCII matches. + The current locale does not change the effect of this flag + unless the :py:const:`~re.LOCALE` flag is also used. + Corresponds to the inline flag ``(?i)``. Note that when the Unicode patterns ``[a-z]`` or ``[A-Z]`` are used in @@ -725,29 +769,35 @@ Flags letters and 4 additional non-ASCII letters: 'İ' (U+0130, Latin capital letter I with dot above), 'ı' (U+0131, Latin small letter dotless i), 'ſ' (U+017F, Latin small letter long s) and 'K' (U+212A, Kelvin sign). - If the :const:`ASCII` flag is used, only letters 'a' to 'z' + If the :py:const:`~re.ASCII` flag is used, only letters 'a' to 'z' and 'A' to 'Z' are matched. .. data:: L LOCALE Make ``\w``, ``\W``, ``\b``, ``\B`` and case-insensitive matching - dependent on the current locale. This flag can be used only with bytes - patterns. The use of this flag is discouraged as the locale mechanism - is very unreliable, it only handles one "culture" at a time, and it only - works with 8-bit locales. Unicode matching is already enabled by default - in Python 3 for Unicode (str) patterns, and it is able to handle different - locales/languages. + dependent on the current locale. + This flag can be used only with bytes patterns. + Corresponds to the inline flag ``(?L)``. + .. warning:: + + This flag is discouraged; consider Unicode matching instead. + The locale mechanism is very unreliable + as it only handles one "culture" at a time + and only works with 8-bit locales. + Unicode matching is enabled by default for Unicode (str) patterns + and it is able to handle different locales and languages. + .. versionchanged:: 3.6 - :const:`re.LOCALE` can be used only with bytes patterns and is - not compatible with :const:`re.ASCII`. + :py:const:`~re.LOCALE` can be used only with bytes patterns + and is not compatible with :py:const:`~re.ASCII`. .. versionchanged:: 3.7 - Compiled regular expression objects with the :const:`re.LOCALE` flag no - longer depend on the locale at compile time. Only the locale at - matching time affects the result of matching. + Compiled regular expression objects with the :py:const:`~re.LOCALE` flag + no longer depend on the locale at compile time. + Only the locale at matching time affects the result of matching. .. data:: M @@ -759,6 +809,7 @@ Flags end of each line (immediately preceding each newline). By default, ``'^'`` matches only at the beginning of the string, and ``'$'`` only at the end of the string and immediately before the newline (if any) at the end of the string. + Corresponds to the inline flag ``(?m)``. .. data:: NOFLAG @@ -778,19 +829,19 @@ Flags Make the ``'.'`` special character match any character at all, including a newline; without this flag, ``'.'`` will match anything *except* a newline. + Corresponds to the inline flag ``(?s)``. .. data:: U UNICODE - In Python 2, this flag made :ref:`special sequences <re-special-sequences>` - include Unicode characters in matches. Since Python 3, Unicode characters - are matched by default. - - See :const:`A` for restricting matching on ASCII characters instead. + In Python 3, Unicode characters are matched by default + for ``str`` patterns. + This flag is therefore redundant with **no effect** + and is only kept for backward compatibility. - This flag is only kept for backward compatibility. + See :py:const:`~re.ASCII` to restrict matching to ASCII characters instead. .. data:: X VERBOSE @@ -829,8 +880,8 @@ Functions below. The expression's behaviour can be modified by specifying a *flags* value. - Values can be any of the following variables, combined using bitwise OR (the - ``|`` operator). + Values can be any of the `flags`_ variables, combined using bitwise OR + (the ``|`` operator). The sequence :: @@ -914,6 +965,8 @@ Functions Empty matches for the pattern split the string only when not adjacent to a previous empty match. + .. code:: pycon + >>> re.split(r'\b', 'Words, words, words.') ['', 'Words', ', ', 'words', ', ', 'words', '.'] >>> re.split(r'\W*', '...words...') @@ -1237,7 +1290,7 @@ Regular Expression Objects The regex matching flags. This is a combination of the flags given to :func:`.compile`, any ``(?...)`` inline flags in the pattern, and implicit - flags such as :data:`UNICODE` if the pattern is a Unicode string. + flags such as :py:const:`~re.UNICODE` if the pattern is a Unicode string. .. attribute:: Pattern.groups diff --git a/Doc/library/readline.rst b/Doc/library/readline.rst index 1adafcaa02eab9..54c6d9f3b32b1a 100644 --- a/Doc/library/readline.rst +++ b/Doc/library/readline.rst @@ -5,7 +5,7 @@ :platform: Unix :synopsis: GNU readline support for Python. -.. sectionauthor:: Skip Montanaro <skip@pobox.com> +.. sectionauthor:: Skip Montanaro <skip.montanaro@gmail.com> -------------- diff --git a/Doc/library/runpy.rst b/Doc/library/runpy.rst index 406b080b7be30f..f2cb595f495f6b 100644 --- a/Doc/library/runpy.rst +++ b/Doc/library/runpy.rst @@ -32,7 +32,7 @@ The :mod:`runpy` module provides two functions: .. index:: pair: module; __main__ - Execute the code of the specified module and return the resulting module + Execute the code of the specified module and return the resulting module's globals dictionary. The module's code is first located using the standard import mechanism (refer to :pep:`302` for details) and then executed in a fresh module namespace. @@ -44,16 +44,16 @@ The :mod:`runpy` module provides two functions: returned. The optional dictionary argument *init_globals* may be used to pre-populate - the module's globals dictionary before the code is executed. The supplied - dictionary will not be modified. If any of the special global variables - below are defined in the supplied dictionary, those definitions are + the module's globals dictionary before the code is executed. + *init_globals* will not be modified. If any of the special global variables + below are defined in *init_globals*, those definitions are overridden by :func:`run_module`. The special global variables ``__name__``, ``__spec__``, ``__file__``, ``__cached__``, ``__loader__`` and ``__package__`` are set in the globals - dictionary before the module code is executed (Note that this is a + dictionary before the module code is executed. (Note that this is a minimal set of variables - other variables may be set implicitly as an - interpreter implementation detail). + interpreter implementation detail.) ``__name__`` is set to *run_name* if this optional argument is not :const:`None`, to ``mod_name + '.__main__'`` if the named module is a @@ -61,7 +61,7 @@ The :mod:`runpy` module provides two functions: ``__spec__`` will be set appropriately for the *actually* imported module (that is, ``__spec__.name`` will always be *mod_name* or - ``mod_name + '.__main__``, never *run_name*). + ``mod_name + '.__main__'``, never *run_name*). ``__file__``, ``__cached__``, ``__loader__`` and ``__package__`` are :ref:`set as normal <import-mod-attrs>` based on the module spec. @@ -104,11 +104,11 @@ The :mod:`runpy` module provides two functions: pair: module; __main__ Execute the code at the named filesystem location and return the resulting - module globals dictionary. As with a script name supplied to the CPython - command line, the supplied path may refer to a Python source file, a + module's globals dictionary. As with a script name supplied to the CPython + command line, *file_path* may refer to a Python source file, a compiled bytecode file or a valid :data:`sys.path` entry containing a :mod:`__main__` module - (e.g. a zipfile containing a top-level ``__main__.py`` file). + (e.g. a zipfile containing a top-level :file:`__main__.py` file). For a simple script, the specified code is simply executed in a fresh module namespace. For a valid :data:`sys.path` entry (typically a zipfile or @@ -119,26 +119,26 @@ The :mod:`runpy` module provides two functions: there is no such module at the specified location. The optional dictionary argument *init_globals* may be used to pre-populate - the module's globals dictionary before the code is executed. The supplied - dictionary will not be modified. If any of the special global variables - below are defined in the supplied dictionary, those definitions are + the module's globals dictionary before the code is executed. + *init_globals* will not be modified. If any of the special global variables + below are defined in *init_globals*, those definitions are overridden by :func:`run_path`. The special global variables ``__name__``, ``__spec__``, ``__file__``, ``__cached__``, ``__loader__`` and ``__package__`` are set in the globals - dictionary before the module code is executed (Note that this is a + dictionary before the module code is executed. (Note that this is a minimal set of variables - other variables may be set implicitly as an - interpreter implementation detail). + interpreter implementation detail.) ``__name__`` is set to *run_name* if this optional argument is not :const:`None` and to ``'<run_path>'`` otherwise. - If the supplied path directly references a script file (whether as source - or as precompiled byte code), then ``__file__`` will be set to the - supplied path, and ``__spec__``, ``__cached__``, ``__loader__`` and + If *file_path* directly references a script file (whether as source + or as precompiled byte code), then ``__file__`` will be set to + *file_path*, and ``__spec__``, ``__cached__``, ``__loader__`` and ``__package__`` will all be set to :const:`None`. - If the supplied path is a reference to a valid :data:`sys.path` entry, then + If *file_path* is a reference to a valid :data:`sys.path` entry, then ``__spec__`` will be set appropriately for the imported :mod:`__main__` module (that is, ``__spec__.name`` will always be ``__main__``). ``__file__``, ``__cached__``, ``__loader__`` and ``__package__`` will be @@ -146,7 +146,7 @@ The :mod:`runpy` module provides two functions: A number of alterations are also made to the :mod:`sys` module. Firstly, :data:`sys.path` may be altered as described above. ``sys.argv[0]`` is updated - with the value of ``path_name`` and ``sys.modules[__name__]`` is updated + with the value of *file_path* and ``sys.modules[__name__]`` is updated with a temporary module object for the module being executed. All modifications to items in :mod:`sys` are reverted before the function returns. diff --git a/Doc/library/select.rst b/Doc/library/select.rst index c2941e628d9d78..a0058046d0ce4c 100644 --- a/Doc/library/select.rst +++ b/Doc/library/select.rst @@ -185,8 +185,8 @@ The module defines the following: ----------------------------- Solaris and derivatives have ``/dev/poll``. While :c:func:`!select` is -O(highest file descriptor) and :c:func:`!poll` is O(number of file -descriptors), ``/dev/poll`` is O(active file descriptors). +*O*\ (*highest file descriptor*) and :c:func:`!poll` is *O*\ (*number of file +descriptors*), ``/dev/poll`` is *O*\ (*active file descriptors*). ``/dev/poll`` behaviour is very close to the standard :c:func:`!poll` object. @@ -381,8 +381,8 @@ scalability for network servers that service many, many clients at the same time. :c:func:`!poll` scales better because the system call only requires listing the file descriptors of interest, while :c:func:`!select` builds a bitmap, turns on bits for the fds of interest, and then afterward the whole bitmap has to be -linearly scanned again. :c:func:`!select` is O(highest file descriptor), while -:c:func:`!poll` is O(number of file descriptors). +linearly scanned again. :c:func:`!select` is *O*\ (*highest file descriptor*), while +:c:func:`!poll` is *O*\ (*number of file descriptors*). .. method:: poll.register(fd[, eventmask]) diff --git a/Doc/library/shelve.rst b/Doc/library/shelve.rst index 88802d717d7383..95c54991887022 100644 --- a/Doc/library/shelve.rst +++ b/Doc/library/shelve.rst @@ -113,6 +113,9 @@ Restrictions differs across Unix versions and requires knowledge about the database implementation used. +* On macOS :mod:`dbm.ndbm` can silently corrupt the database file on updates, + which can cause hard crashes when trying to read from the database. + .. class:: Shelf(dict, protocol=None, writeback=False, keyencoding='utf-8') diff --git a/Doc/library/shutil.rst b/Doc/library/shutil.rst index f61ef8b0ecc7ba..ff8c9a189ab3de 100644 --- a/Doc/library/shutil.rst +++ b/Doc/library/shutil.rst @@ -289,8 +289,8 @@ Directory and files operations copy the file more efficiently. See :ref:`shutil-platform-dependent-efficient-copy-operations` section. - .. versionadded:: 3.8 - The *dirs_exist_ok* parameter. + .. versionchanged:: 3.8 + Added the *dirs_exist_ok* parameter. .. function:: rmtree(path, ignore_errors=False, onerror=None, *, onexc=None, dir_fd=None) @@ -360,21 +360,24 @@ Directory and files operations .. function:: move(src, dst, copy_function=copy2) - Recursively move a file or directory (*src*) to another location (*dst*) - and return the destination. + Recursively move a file or directory (*src*) to another location and return + the destination. + + If *dst* is an existing directory or a symlink to a directory, then *src* + is moved inside that directory. The destination path in that directory must + not already exist. - If the destination is an existing directory, then *src* is moved inside that - directory. If the destination already exists but is not a directory, it may - be overwritten depending on :func:`os.rename` semantics. + If *dst* already exists but is not a directory, it may be overwritten + depending on :func:`os.rename` semantics. If the destination is on the current filesystem, then :func:`os.rename` is - used. Otherwise, *src* is copied to *dst* using *copy_function* and then - removed. In case of symlinks, a new symlink pointing to the target of *src* - will be created in or as *dst* and *src* will be removed. + used. Otherwise, *src* is copied to the destination using *copy_function* + and then removed. In case of symlinks, a new symlink pointing to the target + of *src* will be created as the destination and *src* will be removed. - If *copy_function* is given, it must be a callable that takes two arguments - *src* and *dst*, and will be used to copy *src* to *dst* if - :func:`os.rename` cannot be used. If the source is a directory, + If *copy_function* is given, it must be a callable that takes two arguments, + *src* and the destination, and will be used to copy *src* to the destination + if :func:`os.rename` cannot be used. If the source is a directory, :func:`copytree` is called, passing it the *copy_function*. The default *copy_function* is :func:`copy2`. Using :func:`~shutil.copy` as the *copy_function* allows the move to succeed when it is not possible to also @@ -586,7 +589,9 @@ provided. They rely on the :mod:`zipfile` and :mod:`tarfile` modules. Create an archive file (such as zip or tar) and return its name. *base_name* is the name of the file to create, including the path, minus - any format-specific extension. *format* is the archive format: one of + any format-specific extension. + + *format* is the archive format: one of "zip" (if the :mod:`zlib` module is available), "tar", "gztar" (if the :mod:`zlib` module is available), "bztar" (if the :mod:`bz2` module is available), or "xztar" (if the :mod:`lzma` module is available). diff --git a/Doc/library/signal.rst b/Doc/library/signal.rst index 7ee5ece8859825..85a073aad233ac 100644 --- a/Doc/library/signal.rst +++ b/Doc/library/signal.rst @@ -157,6 +157,8 @@ The variables defined in the :mod:`signal` module are: Alias to :data:`SIGCHLD`. + .. availability:: not macOS. + .. data:: SIGCONT Continue the process if it is currently stopped diff --git a/Doc/library/socketserver.rst b/Doc/library/socketserver.rst index 5fd213fa613c8d..864b1dadb78562 100644 --- a/Doc/library/socketserver.rst +++ b/Doc/library/socketserver.rst @@ -494,7 +494,7 @@ This is the server side:: def handle(self): # self.request is the TCP socket connected to the client self.data = self.request.recv(1024).strip() - print("{} wrote:".format(self.client_address[0])) + print("Received from {}:".format(self.client_address[0])) print(self.data) # just send back the same data, but upper-cased self.request.sendall(self.data.upper()) @@ -525,8 +525,9 @@ objects that simplify communication by providing the standard file interface):: The difference is that the ``readline()`` call in the second handler will call ``recv()`` multiple times until it encounters a newline character, while the -single ``recv()`` call in the first handler will just return what has been sent -from the client in one ``sendall()`` call. +single ``recv()`` call in the first handler will just return what has been +received so far from the client's ``sendall()`` call (typically all of it, but +this is not guaranteed by the TCP protocol). This is the client side:: diff --git a/Doc/library/sqlite3.rst b/Doc/library/sqlite3.rst index 6dbb34a84a4c40..87d5ef1e42ca3a 100644 --- a/Doc/library/sqlite3.rst +++ b/Doc/library/sqlite3.rst @@ -343,17 +343,17 @@ Module functions .. audit-event:: sqlite3.connect database sqlite3.connect .. audit-event:: sqlite3.connect/handle connection_handle sqlite3.connect - .. versionadded:: 3.4 - The *uri* parameter. + .. versionchanged:: 3.4 + Added the *uri* parameter. .. versionchanged:: 3.7 *database* can now also be a :term:`path-like object`, not only a string. - .. versionadded:: 3.10 - The ``sqlite3.connect/handle`` auditing event. + .. versionchanged:: 3.10 + Added the ``sqlite3.connect/handle`` auditing event. - .. versionadded:: 3.12 - The *autocommit* parameter. + .. versionchanged:: 3.12 + Added the *autocommit* parameter. .. versionchanged:: 3.13 Positional use of the parameters *timeout*, *detect_types*, @@ -747,8 +747,8 @@ Connection objects `deterministic <https://sqlite.org/deterministic.html>`_, which allows SQLite to perform additional optimizations. - .. versionadded:: 3.8 - The *deterministic* parameter. + .. versionchanged:: 3.8 + Added the *deterministic* parameter. Example: @@ -1132,17 +1132,24 @@ Connection objects .. versionchanged:: 3.10 Added the ``sqlite3.load_extension`` auditing event. - .. versionadded:: 3.12 - The *entrypoint* parameter. + .. versionchanged:: 3.12 + Added the *entrypoint* parameter. .. _Loading an Extension: https://www.sqlite.org/loadext.html#loading_an_extension_ - .. method:: iterdump + .. method:: iterdump(*, filter=None) Return an :term:`iterator` to dump the database as SQL source code. Useful when saving an in-memory database for later restoration. Similar to the ``.dump`` command in the :program:`sqlite3` shell. + :param filter: + + An optional ``LIKE`` pattern for database objects to dump, e.g. ``prefix_%``. + If ``None`` (the default), all database objects will be included. + + :type filter: str | None + Example: .. testcode:: @@ -1158,6 +1165,8 @@ Connection objects :ref:`sqlite3-howto-encoding` + .. versionchanged:: 3.13 + Added the *filter* parameter. .. method:: backup(target, *, pages=-1, progress=None, name="main", sleep=0.250) @@ -1762,10 +1771,10 @@ Row objects Blob objects ^^^^^^^^^^^^ -.. versionadded:: 3.11 - .. class:: Blob + .. versionadded:: 3.11 + A :class:`Blob` instance is a :term:`file-like object` that can read and write data in an SQLite :abbr:`BLOB (Binary Large OBject)`. Call :func:`len(blob) <len>` to get the size (number of bytes) of the blob. diff --git a/Doc/library/ssl.rst b/Doc/library/ssl.rst index e8709b516ae07a..f9648fa6744bdc 100644 --- a/Doc/library/ssl.rst +++ b/Doc/library/ssl.rst @@ -2574,12 +2574,8 @@ provided. :exc:`SSLWantReadError` if it needs more data than the incoming BIO has available. - - There is no module-level ``wrap_bio()`` call like there is for - :meth:`~SSLContext.wrap_socket`. An :class:`SSLObject` is always created - via an :class:`SSLContext`. - .. versionchanged:: 3.7 - :class:`SSLObject` instances must to created with + :class:`SSLObject` instances must be created with :meth:`~SSLContext.wrap_bio`. In earlier versions, it was possible to create instances directly. This was never documented or officially supported. diff --git a/Doc/library/stat.rst b/Doc/library/stat.rst index 77538514598a50..c941d5557e31b5 100644 --- a/Doc/library/stat.rst +++ b/Doc/library/stat.rst @@ -350,6 +350,12 @@ The following flags can also be used in the *mode* argument of :func:`os.chmod`: The following flags can be used in the *flags* argument of :func:`os.chflags`: +.. data:: UF_SETTABLE + + All user settable flags. + + .. versionadded: 3.13 + .. data:: UF_NODUMP Do not dump the file. @@ -374,10 +380,44 @@ The following flags can be used in the *flags* argument of :func:`os.chflags`: The file is stored compressed (macOS 10.6+). +.. data:: UF_TRACKED + + Used for handling document IDs (macOS) + + .. versionadded: 3.13 + +.. data:: UF_DATAVAULT + + The file needs an entitlement for reading or writing (macOS 10.13+) + + .. versionadded: 3.13 + .. data:: UF_HIDDEN The file should not be displayed in a GUI (macOS 10.5+). +.. data:: SF_SETTABLE + + All super-user changeable flags + + .. versionadded: 3.13 + +.. data:: SF_SUPPORTED + + All super-user supported flags + + .. availability:: macOS + + .. versionadded: 3.13 + +.. data:: SF_SYNTHETIC + + All super-user read-only synthetic flags + + .. availability:: macOS + + .. versionadded: 3.13 + .. data:: SF_ARCHIVED The file may be archived. @@ -390,6 +430,12 @@ The following flags can be used in the *flags* argument of :func:`os.chflags`: The file may only be appended to. +.. data:: SF_RESTRICTED + + The file needs an entitlement to write to (macOS 10.13+) + + .. versionadded: 3.13 + .. data:: SF_NOUNLINK The file may not be renamed or deleted. @@ -398,6 +444,18 @@ The following flags can be used in the *flags* argument of :func:`os.chflags`: The file is a snapshot file. +.. data:: SF_FIRMLINK + + The file is a firmlink (macOS 10.15+) + + .. versionadded: 3.13 + +.. data:: SF_DATALESS + + The file is a dataless object (macOS 10.15+) + + .. versionadded: 3.13 + See the \*BSD or macOS systems man page :manpage:`chflags(2)` for more information. On Windows, the following file attribute constants are available for use when diff --git a/Doc/library/statistics.rst b/Doc/library/statistics.rst index 5c8ad3a7dd7380..0417b3f38a9807 100644 --- a/Doc/library/statistics.rst +++ b/Doc/library/statistics.rst @@ -1026,19 +1026,16 @@ probability that the Python room will stay within its capacity limits? >>> round(NormalDist(mu=n*p, sigma=sqrt(n*p*q)).cdf(k + 0.5), 4) 0.8402 - >>> # Solution using the cumulative binomial distribution + >>> # Exact solution using the cumulative binomial distribution >>> from math import comb, fsum >>> round(fsum(comb(n, r) * p**r * q**(n-r) for r in range(k+1)), 4) 0.8402 >>> # Approximation using a simulation - >>> from random import seed, choices + >>> from random import seed, binomialvariate >>> seed(8675309) - >>> def trial(): - ... return choices(('Python', 'Ruby'), (p, q), k=n).count('Python') - ... - >>> mean(trial() <= k for i in range(10_000)) - 0.8398 + >>> mean(binomialvariate(n, p) <= k for i in range(10_000)) + 0.8406 Naive bayesian classifier @@ -1107,17 +1104,15 @@ from a fixed number of discrete samples. The basic idea is to smooth the data using `a kernel function such as a normal distribution, triangular distribution, or uniform distribution <https://en.wikipedia.org/wiki/Kernel_(statistics)#Kernel_functions_in_common_use>`_. -The degree of smoothing is controlled by a single -parameter, ``h``, representing the variance of the kernel function. +The degree of smoothing is controlled by a scaling parameter, ``h``, +which is called the *bandwidth*. .. testcode:: - import math - def kde_normal(sample, h): "Create a continuous probability density function from a sample." - # Smooth the sample with a normal distribution of variance h. - kernel_h = NormalDist(0.0, math.sqrt(h)).pdf + # Smooth the sample with a normal distribution kernel scaled by h. + kernel_h = NormalDist(0.0, h).pdf n = len(sample) def pdf(x): return sum(kernel_h(x - x_i) for x_i in sample) / n @@ -1131,7 +1126,7 @@ a probability density function estimated from a small sample: .. doctest:: >>> sample = [-2.1, -1.3, -0.4, 1.9, 5.1, 6.2] - >>> f_hat = kde_normal(sample, h=2.25) + >>> f_hat = kde_normal(sample, h=1.5) >>> xarr = [i/100 for i in range(-750, 1100)] >>> yarr = [f_hat(x) for x in xarr] diff --git a/Doc/library/stdtypes.rst b/Doc/library/stdtypes.rst index 9028ff5c134fa9..1a4c12590c1018 100644 --- a/Doc/library/stdtypes.rst +++ b/Doc/library/stdtypes.rst @@ -1528,7 +1528,7 @@ between them will be implicitly converted to a single string literal. That is, ``("spam " "eggs") == "spam eggs"``. See :ref:`strings` for more about the various forms of string literal, -including supported escape sequences, and the ``r`` ("raw") prefix that +including supported :ref:`escape sequences <escape-sequences>`, and the ``r`` ("raw") prefix that disables most escape sequence processing. Strings may also be created from other objects using the :class:`str` diff --git a/Doc/library/string.rst b/Doc/library/string.rst index 262b785bbcbfc1..1867678b2077fc 100644 --- a/Doc/library/string.rst +++ b/Doc/library/string.rst @@ -208,13 +208,13 @@ The grammar for a replacement field is as follows: .. productionlist:: format-string replacement_field: "{" [`field_name`] ["!" `conversion`] [":" `format_spec`] "}" - field_name: arg_name ("." `attribute_name` | "[" `element_index` "]")* - arg_name: [`identifier` | `digit`+] - attribute_name: `identifier` - element_index: `digit`+ | `index_string` + field_name: `arg_name` ("." `attribute_name` | "[" `element_index` "]")* + arg_name: [`~python-grammar:identifier` | `~python-grammar:digit`+] + attribute_name: `~python-grammar:identifier` + element_index: `~python-grammar:digit`+ | `index_string` index_string: <any source character except "]"> + conversion: "r" | "s" | "a" - format_spec: <described in the next section> + format_spec: `format-spec:format_spec` In less formal terms, the replacement field can start with a *field_name* that specifies the object whose value is to be formatted and inserted @@ -316,9 +316,9 @@ The general form of a *standard format specifier* is: fill: <any character> align: "<" | ">" | "=" | "^" sign: "+" | "-" | " " - width: `digit`+ + width: `~python-grammar:digit`+ grouping_option: "_" | "," - precision: `digit`+ + precision: `~python-grammar:digit`+ type: "b" | "c" | "d" | "e" | "E" | "f" | "F" | "g" | "G" | "n" | "o" | "s" | "x" | "X" | "%" If a valid *align* value is specified, it can be preceded by a *fill* diff --git a/Doc/library/subprocess.rst b/Doc/library/subprocess.rst index d6b892a7ed957d..f63ca73b3ec067 100644 --- a/Doc/library/subprocess.rst +++ b/Doc/library/subprocess.rst @@ -308,10 +308,10 @@ default values. The arguments that are most commonly needed are: If text mode is not used, *stdin*, *stdout* and *stderr* will be opened as binary streams. No encoding or line ending conversion is performed. - .. versionadded:: 3.6 - Added *encoding* and *errors* parameters. + .. versionchanged:: 3.6 + Added the *encoding* and *errors* parameters. - .. versionadded:: 3.7 + .. versionchanged:: 3.7 Added the *text* parameter as an alias for *universal_newlines*. .. note:: @@ -664,7 +664,8 @@ functions. If given, *startupinfo* will be a :class:`STARTUPINFO` object, which is passed to the underlying ``CreateProcess`` function. - *creationflags*, if given, can be one or more of the following flags: + + If given, *creationflags*, can be one or more of the following flags: * :data:`CREATE_NEW_CONSOLE` * :data:`CREATE_NEW_PROCESS_GROUP` @@ -684,8 +685,8 @@ functions. is only changed on platforms that support this (only Linux at this time of writing). Other platforms will ignore this parameter. - .. versionadded:: 3.10 - The ``pipesize`` parameter was added. + .. versionchanged:: 3.10 + Added the *pipesize* parameter. Popen objects are supported as context managers via the :keyword:`with` statement: on exit, standard file descriptors are closed, and the process is waited for. @@ -1461,8 +1462,8 @@ Return code handling translates as follows:: print("There were some errors") -Replacing functions from the :mod:`popen2` module -^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ +Replacing functions from the :mod:`!popen2` module +^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ .. note:: @@ -1538,8 +1539,8 @@ handling consistency are valid for these functions. as it did in Python 3.3.3 and earlier. exitcode has the same value as :attr:`~Popen.returncode`. - .. versionadded:: 3.11 - Added *encoding* and *errors* arguments. + .. versionchanged:: 3.11 + Added the *encoding* and *errors* parameters. .. function:: getoutput(cmd, *, encoding=None, errors=None) @@ -1556,8 +1557,8 @@ handling consistency are valid for these functions. .. versionchanged:: 3.3.4 Windows support added - .. versionadded:: 3.11 - Added *encoding* and *errors* arguments. + .. versionchanged:: 3.11 + Added the *encoding* and *errors* parameters. Notes diff --git a/Doc/library/symtable.rst b/Doc/library/symtable.rst index 46159dcef940e7..47568387f9a7ce 100644 --- a/Doc/library/symtable.rst +++ b/Doc/library/symtable.rst @@ -97,7 +97,7 @@ Examining Symbol Tables .. class:: Function - A namespace for a function or method. This class inherits + A namespace for a function or method. This class inherits from :class:`SymbolTable`. .. method:: get_parameters() @@ -123,7 +123,7 @@ Examining Symbol Tables .. class:: Class - A namespace of a class. This class inherits :class:`SymbolTable`. + A namespace of a class. This class inherits from :class:`SymbolTable`. .. method:: get_methods() diff --git a/Doc/library/sys.monitoring.rst b/Doc/library/sys.monitoring.rst index 762581b7eda7f1..4980227c60b21e 100644 --- a/Doc/library/sys.monitoring.rst +++ b/Doc/library/sys.monitoring.rst @@ -75,9 +75,6 @@ following IDs are pre-defined to make co-operation of tools easier:: sys.monitoring.PROFILER_ID = 2 sys.monitoring.OPTIMIZER_ID = 5 -There is no obligation to set an ID, nor is there anything preventing a tool -from using an ID even it is already in use. -However, tools are encouraged to use a unique ID and respect other tools. Events ------ diff --git a/Doc/library/sys.rst b/Doc/library/sys.rst index aaf79205d44282..a97a369b77b88a 100644 --- a/Doc/library/sys.rst +++ b/Doc/library/sys.rst @@ -1268,10 +1268,13 @@ always available. .. versionchanged:: 3.4 :term:`Module specs <module spec>` were introduced in Python 3.4, by - :pep:`451`. Earlier versions of Python looked for a method called - :meth:`!find_module`. - This is still called as a fallback if a :data:`meta_path` entry doesn't - have a :meth:`~importlib.abc.MetaPathFinder.find_spec` method. + :pep:`451`. + + .. versionchanged:: 3.12 + + Removed the fallback that looked for a :meth:`!find_module` method + if a :data:`meta_path` entry didn't have a + :meth:`~importlib.abc.MetaPathFinder.find_spec` method. .. data:: modules @@ -1290,7 +1293,10 @@ always available. The list of the original command line arguments passed to the Python executable. - See also :data:`sys.argv`. + The elements of :data:`sys.orig_argv` are the arguments to the Python interpreter, + while the elements of :data:`sys.argv` are the arguments to the user's program. + Arguments consumed by the interpreter itself will be present in :data:`sys.orig_argv` + and missing from :data:`sys.argv`. .. versionadded:: 3.10 @@ -1652,7 +1658,7 @@ always available. ``'opcode'`` event type added; :attr:`~frame.f_trace_lines` and :attr:`~frame.f_trace_opcodes` attributes added to frames -.. function:: set_asyncgen_hooks(firstiter, finalizer) +.. function:: set_asyncgen_hooks([firstiter] [, finalizer]) Accepts two optional keyword arguments which are callables that accept an :term:`asynchronous generator iterator` as an argument. The *firstiter* @@ -1744,9 +1750,17 @@ always available. .. availability:: Windows. + .. note:: + Changing the filesystem encoding after Python startup is risky because + the old fsencoding or paths encoded by the old fsencoding may be cached + somewhere. Use :envvar:`PYTHONLEGACYWINDOWSFSENCODING` instead. + .. versionadded:: 3.6 See :pep:`529` for more details. + .. deprecated-removed:: 3.13 3.16 + Use :envvar:`PYTHONLEGACYWINDOWSFSENCODING` instead. + .. data:: stdin stdout stderr diff --git a/Doc/library/tarfile.rst b/Doc/library/tarfile.rst index f4e83d64bb1580..2134293a0bb0de 100644 --- a/Doc/library/tarfile.rst +++ b/Doc/library/tarfile.rst @@ -116,10 +116,12 @@ Some facts and figures: ``'filemode|[compression]'``. :func:`tarfile.open` will return a :class:`TarFile` object that processes its data as a stream of blocks. No random seeking will be done on the file. If given, *fileobj* may be any object that has a - :meth:`~io.TextIOBase.read` or :meth:`~io.TextIOBase.write` method (depending on the *mode*). *bufsize* - specifies the blocksize and defaults to ``20 * 512`` bytes. Use this variant - in combination with e.g. ``sys.stdin``, a socket :term:`file object` or a tape - device. However, such a :class:`TarFile` object is limited in that it does + :meth:`~io.RawIOBase.read` or :meth:`~io.RawIOBase.write` method + (depending on the *mode*) that works with bytes. + *bufsize* specifies the blocksize and defaults to ``20 * 512`` bytes. + Use this variant in combination with e.g. ``sys.stdin.buffer``, a socket + :term:`file object` or a tape device. + However, such a :class:`TarFile` object is limited in that it does not allow random access, see :ref:`tar-examples`. The currently possible modes: @@ -671,6 +673,7 @@ be finalized; only the internally used file object will be closed. See the .. attribute:: TarFile.pax_headers + :type: dict A dictionary containing key-value pairs of pax global headers. @@ -835,6 +838,41 @@ A ``TarInfo`` object has the following public data attributes: :meth:`~TarFile.extractall`, causing extraction to skip applying this attribute. +.. attribute:: TarInfo.chksum + :type: int + + Header checksum. + + +.. attribute:: TarInfo.devmajor + :type: int + + Device major number. + + +.. attribute:: TarInfo.devminor + :type: int + + Device minor number. + + +.. attribute:: TarInfo.offset + :type: int + + The tar header starts here. + + +.. attribute:: TarInfo.offset_data + :type: int + + The file's data starts here. + + +.. attribute:: TarInfo.sparse + + Sparse member information. + + .. attribute:: TarInfo.pax_headers :type: dict diff --git a/Doc/library/test.rst b/Doc/library/test.rst index 9173db07fd0071..7d28f625345726 100644 --- a/Doc/library/test.rst +++ b/Doc/library/test.rst @@ -143,7 +143,7 @@ guidelines to be followed: arg = (1, 2, 3) When using this pattern, remember that all classes that inherit from - :class:`unittest.TestCase` are run as tests. The :class:`Mixin` class in the example above + :class:`unittest.TestCase` are run as tests. The :class:`!TestFuncAcceptsSequencesMixin` class in the example above does not have any data and so can't be run by itself, thus it does not inherit from :class:`unittest.TestCase`. @@ -324,9 +324,9 @@ The :mod:`test.support` module defines the following constants: .. data:: Py_DEBUG - True if Python is built with the :c:macro:`Py_DEBUG` macro defined: if - Python is :ref:`built in debug mode <debug-build>` - (:option:`./configure --with-pydebug <--with-pydebug>`). + True if Python was built with the :c:macro:`Py_DEBUG` macro + defined, that is, if + Python was :ref:`built in debug mode <debug-build>`. .. versionadded:: 3.12 diff --git a/Doc/library/threading.rst b/Doc/library/threading.rst index b85b7f008d1594..5fbf9379b8202c 100644 --- a/Doc/library/threading.rst +++ b/Doc/library/threading.rst @@ -534,9 +534,10 @@ All methods are executed atomically. lock, subsequent attempts to acquire it block, until it is released; any thread may release it. - Note that ``Lock`` is actually a factory function which returns an instance - of the most efficient version of the concrete Lock class that is supported - by the platform. + .. versionchanged:: 3.13 + ``Lock`` is now a class. In earlier Pythons, ``Lock`` was a factory + function which returned an instance of the underlying private lock + type. .. method:: acquire(blocking=True, timeout=-1) diff --git a/Doc/library/time.rst b/Doc/library/time.rst index 577600881676b3..2782a961363666 100644 --- a/Doc/library/time.rst +++ b/Doc/library/time.rst @@ -840,6 +840,15 @@ These constants are used as parameters for :func:`clock_getres` and .. versionadded:: 3.3 +.. data:: CLOCK_MONOTONIC_RAW_APPROX + + Similar to :data:`CLOCK_MONOTONIC_RAW`, but reads a value cached by + the system at context switch and hence has less accuracy. + + .. availability:: macOS >= 10.12. + + .. versionadded:: 3.13 + .. data:: CLOCK_PROCESS_CPUTIME_ID @@ -899,6 +908,15 @@ These constants are used as parameters for :func:`clock_getres` and .. versionadded:: 3.8 +.. data:: CLOCK_UPTIME_RAW_APPROX + + Like :data:`CLOCK_UPTIME_RAW`, but the value is cached by the system + at context switches and therefore has less accuracy. + + .. availability:: macOS >= 10.12. + + .. versionadded:: 3.13 + The following constant is the only parameter that can be sent to :func:`clock_settime`. diff --git a/Doc/library/tkinter.ttk.rst b/Doc/library/tkinter.ttk.rst index 1609dc2ce9218e..bd0d8b3799a0f1 100644 --- a/Doc/library/tkinter.ttk.rst +++ b/Doc/library/tkinter.ttk.rst @@ -1118,7 +1118,7 @@ ttk.Treeview as the item identifier; *iid* must not already exist in the tree. Otherwise, a new unique identifier is generated. - See `Item Options`_ for the list of available points. + See `Item Options`_ for the list of available options. .. method:: item(item, option=None, **kw) diff --git a/Doc/library/tty.rst b/Doc/library/tty.rst index 20ba7d7e0a45b3..ed63561c40de24 100644 --- a/Doc/library/tty.rst +++ b/Doc/library/tty.rst @@ -35,8 +35,15 @@ The :mod:`tty` module defines the following functions: Convert the tty attribute list *mode*, which is a list like the one returned by :func:`termios.tcgetattr`, to that of a tty in cbreak mode. + This clears the ``ECHO`` and ``ICANON`` local mode flags in *mode* as well + as setting the minimum input to 1 byte with no delay. + .. versionadded:: 3.12 + .. versionchanged:: 3.12.2 + The ``ICRNL`` flag is no longer cleared. This matches Linux and macOS + ``stty cbreak`` behavior and what :func:`setcbreak` historically did. + .. function:: setraw(fd, when=termios.TCSAFLUSH) @@ -56,9 +63,17 @@ The :mod:`tty` module defines the following functions: :func:`termios.tcsetattr`. The return value of :func:`termios.tcgetattr` is saved before setting *fd* to cbreak mode; this value is returned. + This clears the ``ECHO`` and ``ICANON`` local mode flags as well as setting + the minimum input to 1 byte with no delay. + .. versionchanged:: 3.12 The return value is now the original tty attributes, instead of None. + .. versionchanged:: 3.12.2 + The ``ICRNL`` flag is no longer cleared. This restores the behavior + of Python 3.11 and earlier as well as matching what Linux, macOS, & BSDs + describe in their ``stty(1)`` man pages regarding cbreak mode. + .. seealso:: diff --git a/Doc/library/types.rst b/Doc/library/types.rst index 8ce67cf77253c3..c8c981024c1aeb 100644 --- a/Doc/library/types.rst +++ b/Doc/library/types.rst @@ -398,6 +398,10 @@ Standard names are defined for the following types: data members which use standard conversion functions; it has the same purpose as the :class:`property` type, but for classes defined in extension modules. + In addition, when a class is defined with a :attr:`~object.__slots__` attribute, then for + each slot, an instance of :class:`!MemberDescriptorType` will be added as an attribute + on the class. This allows the slot to appear in the class's :attr:`~object.__dict__`. + .. impl-detail:: In other implementations of Python, this type may be identical to diff --git a/Doc/library/unittest.mock.rst b/Doc/library/unittest.mock.rst index f1cc482c5cfe2a..eca20b94ec8e74 100644 --- a/Doc/library/unittest.mock.rst +++ b/Doc/library/unittest.mock.rst @@ -2009,8 +2009,8 @@ Mocking Magic Methods ~~~~~~~~~~~~~~~~~~~~~ :class:`Mock` supports mocking the Python protocol methods, also known as -"magic methods". This allows mock objects to replace containers or other -objects that implement Python protocols. +:term:`"magic methods" <magic method>`. This allows mock objects to replace +containers or other objects that implement Python protocols. Because magic methods are looked up differently from normal methods [#]_, this support has been specially implemented. This means that only specific magic @@ -2108,8 +2108,8 @@ There are two ``MagicMock`` variants: :class:`MagicMock` and :class:`NonCallable .. class:: MagicMock(*args, **kw) ``MagicMock`` is a subclass of :class:`Mock` with default implementations - of most of the magic methods. You can use ``MagicMock`` without having to - configure the magic methods yourself. + of most of the :term:`magic methods <magic method>`. You can use + ``MagicMock`` without having to configure the magic methods yourself. The constructor parameters have the same meaning as for :class:`Mock`. diff --git a/Doc/library/unittest.rst b/Doc/library/unittest.rst index 70b4c84c05f818..e6140ac70eb87a 100644 --- a/Doc/library/unittest.rst +++ b/Doc/library/unittest.rst @@ -2196,8 +2196,8 @@ Loading and running tests .. versionadded:: 3.2 - .. versionadded:: 3.12 - Added *durations* keyword argument. + .. versionchanged:: 3.12 + Added the *durations* keyword parameter. .. data:: defaultTestLoader @@ -2290,7 +2290,7 @@ Loading and running tests The *testRunner* argument can either be a test runner class or an already created instance of it. By default ``main`` calls :func:`sys.exit` with an exit code indicating success (0) or failure (1) of the tests run. - An exit code of 5 indicates that no tests were run. + An exit code of 5 indicates that no tests were run or skipped. The *testLoader* argument has to be a :class:`TestLoader` instance, and defaults to :data:`defaultTestLoader`. diff --git a/Doc/library/urllib.parse.rst b/Doc/library/urllib.parse.rst index 53e5f0395715d7..3c898c3e826304 100644 --- a/Doc/library/urllib.parse.rst +++ b/Doc/library/urllib.parse.rst @@ -729,8 +729,8 @@ task isn't already covered by the URL parsing functions above. .. versionchanged:: 3.2 *query* supports bytes and string objects. - .. versionadded:: 3.5 - *quote_via* parameter. + .. versionchanged:: 3.5 + Added the *quote_via* parameter. .. seealso:: diff --git a/Doc/library/urllib.request.rst b/Doc/library/urllib.request.rst index 0e18db73280a63..affdce144cd5fc 100644 --- a/Doc/library/urllib.request.rst +++ b/Doc/library/urllib.request.rst @@ -78,7 +78,7 @@ The :mod:`urllib.request` module defines the following functions: :class:`UnknownHandler` to ensure this never happens). In addition, if proxy settings are detected (for example, when a ``*_proxy`` - environment variable like :envvar:`http_proxy` is set), + environment variable like :envvar:`!http_proxy` is set), :class:`ProxyHandler` is default installed and makes sure the requests are handled through the proxy. @@ -113,7 +113,7 @@ The :mod:`urllib.request` module defines the following functions: .. versionchanged:: 3.10 HTTPS connection now send an ALPN extension with protocol indicator ``http/1.1`` when no *context* is given. Custom *context* should set - ALPN protocols with :meth:`~ssl.SSLContext.set_alpn_protocol`. + ALPN protocols with :meth:`~ssl.SSLContext.set_alpn_protocols`. .. versionchanged:: 3.13 Remove *cafile*, *capath* and *cadefault* parameters: use the *context* @@ -618,25 +618,25 @@ OpenerDirector Objects the actual HTTP code, for example :meth:`http_error_404` would handle HTTP 404 errors. - * :meth:`<protocol>_open` --- signal that the handler knows how to open *protocol* + * :meth:`!<protocol>_open` --- signal that the handler knows how to open *protocol* URLs. See |protocol_open|_ for more information. - * :meth:`http_error_\<type\>` --- signal that the handler knows how to handle HTTP + * :meth:`!http_error_\<type\>` --- signal that the handler knows how to handle HTTP errors with HTTP error code *type*. See |http_error_nnn|_ for more information. - * :meth:`<protocol>_error` --- signal that the handler knows how to handle errors + * :meth:`!<protocol>_error` --- signal that the handler knows how to handle errors from (non-\ ``http``) *protocol*. - * :meth:`<protocol>_request` --- signal that the handler knows how to pre-process + * :meth:`!<protocol>_request` --- signal that the handler knows how to pre-process *protocol* requests. See |protocol_request|_ for more information. - * :meth:`<protocol>_response` --- signal that the handler knows how to + * :meth:`!<protocol>_response` --- signal that the handler knows how to post-process *protocol* responses. See |protocol_response|_ for more information. @@ -663,7 +663,7 @@ OpenerDirector Objects Handle an error of the given protocol. This will call the registered error handlers for the given protocol with the given arguments (which are protocol specific). The HTTP protocol is a special case which uses the HTTP response - code to determine the specific error handler; refer to the :meth:`http_error_\<type\>` + code to determine the specific error handler; refer to the :meth:`!http_error_\<type\>` methods of the handler classes. Return values and exceptions raised are the same as those of :func:`urlopen`. @@ -673,25 +673,25 @@ OpenerDirector objects open URLs in three stages: The order in which these methods are called within each stage is determined by sorting the handler instances. -#. Every handler with a method named like :meth:`<protocol>_request` has that +#. Every handler with a method named like :meth:`!<protocol>_request` has that method called to pre-process the request. -#. Handlers with a method named like :meth:`<protocol>_open` are called to handle +#. Handlers with a method named like :meth:`!<protocol>_open` are called to handle the request. This stage ends when a handler either returns a non-\ :const:`None` value (ie. a response), or raises an exception (usually :exc:`~urllib.error.URLError`). Exceptions are allowed to propagate. In fact, the above algorithm is first tried for methods named - :meth:`default_open`. If all such methods return :const:`None`, the algorithm - is repeated for methods named like :meth:`<protocol>_open`. If all such methods + :meth:`~BaseHandler.default_open`. If all such methods return :const:`None`, the algorithm + is repeated for methods named like :meth:`!<protocol>_open`. If all such methods return :const:`None`, the algorithm is repeated for methods named - :meth:`unknown_open`. + :meth:`~BaseHandler.unknown_open`. Note that the implementation of these methods may involve calls of the parent :class:`OpenerDirector` instance's :meth:`~OpenerDirector.open` and :meth:`~OpenerDirector.error` methods. -#. Every handler with a method named like :meth:`<protocol>_response` has that +#. Every handler with a method named like :meth:`!<protocol>_response` has that method called to post-process the response. @@ -740,7 +740,7 @@ The following attribute and methods should only be used by classes derived from the return value of the :meth:`~OpenerDirector.open` method of :class:`OpenerDirector`, or ``None``. It should raise :exc:`~urllib.error.URLError`, unless a truly exceptional thing happens (for example, :exc:`MemoryError` should not be mapped to - :exc:`URLError`). + :exc:`~urllib.error.URLError`). This method will be called before any protocol-specific open method. @@ -753,7 +753,7 @@ The following attribute and methods should only be used by classes derived from define it if they want to handle URLs with the given protocol. This method, if defined, will be called by the parent :class:`OpenerDirector`. - Return values should be the same as for :meth:`default_open`. + Return values should be the same as for :meth:`~BaseHandler.default_open`. .. method:: BaseHandler.unknown_open(req) @@ -793,7 +793,7 @@ The following attribute and methods should only be used by classes derived from Subclasses should override this method to handle specific HTTP errors. Arguments, return values and exceptions raised should be the same as for - :meth:`http_error_default`. + :meth:`~BaseHandler.http_error_default`. .. _protocol_request: @@ -833,7 +833,7 @@ HTTPRedirectHandler Objects is the case, :exc:`~urllib.error.HTTPError` is raised. See :rfc:`2616` for details of the precise meanings of the various redirection codes. - An :class:`HTTPError` exception raised as a security consideration if the + An :exc:`~urllib.error.HTTPError` exception raised as a security consideration if the HTTPRedirectHandler is presented with a redirected URL which is not an HTTP, HTTPS or FTP URL. @@ -910,7 +910,7 @@ ProxyHandler Objects .. method:: ProxyHandler.<protocol>_open(request) :noindex: - The :class:`ProxyHandler` will have a method :meth:`<protocol>_open` for every + The :class:`ProxyHandler` will have a method :meth:`!<protocol>_open` for every *protocol* which has a proxy in the *proxies* dictionary given in the constructor. The method will modify requests to go through the proxy, by calling ``request.set_proxy()``, and call the next handler in the chain to @@ -1166,7 +1166,7 @@ HTTPErrorProcessor Objects For 200 error codes, the response object is returned immediately. For non-200 error codes, this simply passes the job on to the - :meth:`http_error_\<type\>` handler methods, via :meth:`OpenerDirector.error`. + :meth:`!http_error_\<type\>` handler methods, via :meth:`OpenerDirector.error`. Eventually, :class:`HTTPDefaultErrorHandler` will raise an :exc:`~urllib.error.HTTPError` if no other handler handles the error. @@ -1273,7 +1273,7 @@ Use of Basic HTTP Authentication:: :func:`build_opener` provides many handlers by default, including a :class:`ProxyHandler`. By default, :class:`ProxyHandler` uses the environment variables named ``<scheme>_proxy``, where ``<scheme>`` is the URL scheme -involved. For example, the :envvar:`http_proxy` environment variable is read to +involved. For example, the :envvar:`!http_proxy` environment variable is read to obtain the HTTP proxy's URL. This example replaces the default :class:`ProxyHandler` with one that uses @@ -1368,7 +1368,7 @@ some point in the future. points to a local file, the object will not be copied unless filename is supplied. Return a tuple ``(filename, headers)`` where *filename* is the local file name under which the object can be found, and *headers* is whatever - the :meth:`info` method of the object returned by :func:`urlopen` returned (for + the :meth:`!info` method of the object returned by :func:`urlopen` returned (for a remote object). Exceptions are the same as for :func:`urlopen`. The second argument, if present, specifies the file location to copy to (if @@ -1393,7 +1393,7 @@ some point in the future. :mimetype:`application/x-www-form-urlencoded` format; see the :func:`urllib.parse.urlencode` function. - :func:`urlretrieve` will raise :exc:`ContentTooShortError` when it detects that + :func:`urlretrieve` will raise :exc:`~urllib.error.ContentTooShortError` when it detects that the amount of data available was less than the expected amount (which is the size reported by a *Content-Length* header). This can occur, for example, when the download is interrupted. @@ -1402,8 +1402,8 @@ some point in the future. urlretrieve reads more data, but if less data is available, it raises the exception. - You can still retrieve the downloaded data in this case, it is stored in the - :attr:`content` attribute of the exception instance. + You can still retrieve the downloaded data in this case, it is stored in the + :attr:`!content` attribute of the exception instance. If no *Content-Length* header was supplied, urlretrieve can not check the size of the data it has downloaded, and just returns it. In this case you just have @@ -1497,7 +1497,7 @@ some point in the future. authentication is performed. For the 30x response codes, recursion is bounded by the value of the *maxtries* attribute, which defaults to 10. - For all other response codes, the method :meth:`http_error_default` is called + For all other response codes, the method :meth:`~BaseHandler.http_error_default` is called which you can override in subclasses to handle the error appropriately. .. note:: diff --git a/Doc/library/urllib.robotparser.rst b/Doc/library/urllib.robotparser.rst index f063e463753e0b..b5a49d9c592387 100644 --- a/Doc/library/urllib.robotparser.rst +++ b/Doc/library/urllib.robotparser.rst @@ -5,7 +5,7 @@ :synopsis: Load a robots.txt file and answer questions about fetchability of other URLs. -.. sectionauthor:: Skip Montanaro <skip@pobox.com> +.. sectionauthor:: Skip Montanaro <skip.montanaro@gmail.com> **Source code:** :source:`Lib/urllib/robotparser.py` diff --git a/Doc/library/venv.rst b/Doc/library/venv.rst index da8942c554dea1..aa18873f223a6b 100644 --- a/Doc/library/venv.rst +++ b/Doc/library/venv.rst @@ -201,13 +201,13 @@ creation according to their needs, the :class:`EnvBuilder` class. .. versionchanged:: 3.4 Added the ``with_pip`` parameter - .. versionadded:: 3.6 + .. versionchanged:: 3.6 Added the ``prompt`` parameter - .. versionadded:: 3.9 + .. versionchanged:: 3.9 Added the ``upgrade_deps`` parameter - .. versionadded:: 3.13 + .. versionchanged:: 3.13 Added the ``scm_ignore_files`` parameter Creators of third-party virtual environment tools will be free to use the diff --git a/Doc/library/warnings.rst b/Doc/library/warnings.rst index a9c469707e8227..500398636e11ae 100644 --- a/Doc/library/warnings.rst +++ b/Doc/library/warnings.rst @@ -396,7 +396,7 @@ Available Functions ------------------- -.. function:: warn(message, category=None, stacklevel=1, source=None, \*, skip_file_prefixes=None) +.. function:: warn(message, category=None, stacklevel=1, source=None, *, skip_file_prefixes=None) Issue a warning, or maybe ignore it or raise an exception. The *category* argument, if given, must be a :ref:`warning category class <warning-categories>`; it diff --git a/Doc/library/xml.etree.elementtree.rst b/Doc/library/xml.etree.elementtree.rst index 57cfbb8d92244b..75a7915c15240d 100644 --- a/Doc/library/xml.etree.elementtree.rst +++ b/Doc/library/xml.etree.elementtree.rst @@ -625,6 +625,8 @@ Functions target. Returns an :term:`iterator` providing ``(event, elem)`` pairs; it has a ``root`` attribute that references the root element of the resulting XML tree once *source* is fully read. + The iterator has the :meth:`!close` method that closes the internal + file object if *source* is a filename. Note that while :func:`iterparse` builds the tree incrementally, it issues blocking reads on *source* (or the file it names). As such, it's unsuitable @@ -647,6 +649,9 @@ Functions .. versionchanged:: 3.8 The ``comment`` and ``pi`` events were added. + .. versionchanged:: 3.13 + Added the :meth:`!close` method. + .. function:: parse(source, parser=None) @@ -664,7 +669,7 @@ Functions given. Returns an element instance, representing a processing instruction. Note that :class:`XMLParser` skips over processing instructions - in the input instead of creating comment objects for them. An + in the input instead of creating PI objects for them. An :class:`ElementTree` will only contain processing instruction nodes if they have been inserted into to the tree using one of the :class:`Element` methods. @@ -705,11 +710,11 @@ Functions meaning as in :meth:`ElementTree.write`. Returns an (optionally) encoded string containing the XML data. - .. versionadded:: 3.4 - The *short_empty_elements* parameter. + .. versionchanged:: 3.4 + Added the *short_empty_elements* parameter. - .. versionadded:: 3.8 - The *xml_declaration* and *default_namespace* parameters. + .. versionchanged:: 3.8 + Added the *xml_declaration* and *default_namespace* parameters. .. versionchanged:: 3.8 The :func:`tostring` function now preserves the attribute order @@ -732,11 +737,11 @@ Functions .. versionadded:: 3.2 - .. versionadded:: 3.4 - The *short_empty_elements* parameter. + .. versionchanged:: 3.4 + Added the *short_empty_elements* parameter. - .. versionadded:: 3.8 - The *xml_declaration* and *default_namespace* parameters. + .. versionchanged:: 3.8 + Added the *xml_declaration* and *default_namespace* parameters. .. versionchanged:: 3.8 The :func:`tostringlist` function now preserves the attribute order @@ -858,8 +863,8 @@ Functions this is a Unicode string. If the loader fails, it can return None or raise an exception. - .. versionadded:: 3.9 - The *base_url* and *max_depth* parameters. + .. versionchanged:: 3.9 + Added the *base_url* and *max_depth* parameters. .. _elementtree-element-objects: @@ -1189,8 +1194,8 @@ ElementTree Objects :term:`file object`; make sure you do not try to write a string to a binary stream and vice versa. - .. versionadded:: 3.4 - The *short_empty_elements* parameter. + .. versionchanged:: 3.4 + Added the *short_empty_elements* parameter. .. versionchanged:: 3.8 The :meth:`write` method now preserves the attribute order specified @@ -1302,8 +1307,8 @@ TreeBuilder Objects .. method:: pi(target, text) - Creates a comment with the given *target* name and *text*. If - ``insert_pis`` is true, this will also add it to the tree. + Creates a process instruction with the given *target* name and *text*. + If ``insert_pis`` is true, this will also add it to the tree. .. versionadded:: 3.8 diff --git a/Doc/library/xml.sax.utils.rst b/Doc/library/xml.sax.utils.rst index e57e76dcac7820..3a524c9c0d5a9f 100644 --- a/Doc/library/xml.sax.utils.rst +++ b/Doc/library/xml.sax.utils.rst @@ -71,8 +71,8 @@ or as base classes. content: if ``False`` (the default) they are emitted as a pair of start/end tags, if set to ``True`` they are emitted as a single self-closed tag. - .. versionadded:: 3.2 - The *short_empty_elements* parameter. + .. versionchanged:: 3.2 + Added the *short_empty_elements* parameter. .. class:: XMLFilterBase(base) diff --git a/Doc/library/zipapp.rst b/Doc/library/zipapp.rst index 104afca23a20b4..c8a059bdb1cb93 100644 --- a/Doc/library/zipapp.rst +++ b/Doc/library/zipapp.rst @@ -171,8 +171,8 @@ The module defines two convenience functions: passed to the ``zipfile.ZipFile`` class, and must supply the methods needed by that class. - .. versionadded:: 3.7 - Added the *filter* and *compressed* arguments. + .. versionchanged:: 3.7 + Added the *filter* and *compressed* parameters. .. function:: get_interpreter(archive) diff --git a/Doc/library/zipfile.rst b/Doc/library/zipfile.rst index a77e49a7643826..b6f881fd2dfd70 100644 --- a/Doc/library/zipfile.rst +++ b/Doc/library/zipfile.rst @@ -79,6 +79,11 @@ The module defines the following items: of the last modification to the file; the fields are described in section :ref:`zipinfo-objects`. + .. versionchanged:: 3.13 + A public :attr:`!compress_level` attribute has been added to expose the + formerly protected :attr:`!_compresslevel`. The older protected name + continues to work as a property for backwards compatibility. + .. function:: is_zipfile(filename) Returns ``True`` if *filename* is a valid ZIP file based on its magic number, @@ -213,7 +218,7 @@ ZipFile Objects That flag takes precedence over *metadata_encoding*, which is a Python-specific extension. - .. versionadded:: 3.2 + .. versionchanged:: 3.2 Added the ability to use :class:`ZipFile` as a context manager. .. versionchanged:: 3.3 @@ -236,8 +241,8 @@ ZipFile Objects .. versionchanged:: 3.7 Add the *compresslevel* parameter. - .. versionadded:: 3.8 - The *strict_timestamps* keyword-only argument + .. versionchanged:: 3.8 + The *strict_timestamps* keyword-only parameter. .. versionchanged:: 3.11 Added support for specifying member name encoding for reading @@ -643,8 +648,8 @@ The :class:`PyZipFile` constructor takes the same parameters as the .. class:: PyZipFile(file, mode='r', compression=ZIP_STORED, allowZip64=True, \ optimize=-1) - .. versionadded:: 3.2 - The *optimize* parameter. + .. versionchanged:: 3.2 + Added the *optimize* parameter. .. versionchanged:: 3.4 ZIP64 extensions are enabled by default. @@ -699,8 +704,8 @@ The :class:`PyZipFile` constructor takes the same parameters as the test/bogus/__init__.pyc # Subpackage directory test/bogus/myfile.pyc # Submodule test.bogus.myfile - .. versionadded:: 3.4 - The *filterfunc* parameter. + .. versionchanged:: 3.4 + Added the *filterfunc* parameter. .. versionchanged:: 3.6.2 The *pathname* parameter accepts a :term:`path-like object`. @@ -744,8 +749,8 @@ file: .. versionchanged:: 3.6.2 The *filename* parameter accepts a :term:`path-like object`. - .. versionadded:: 3.8 - The *strict_timestamps* keyword-only argument + .. versionchanged:: 3.8 + Added the *strict_timestamps* keyword-only parameter. Instances have the following methods and attributes: diff --git a/Doc/license.rst b/Doc/license.rst index 8aad93062a5a88..9fc0ff7161a591 100644 --- a/Doc/license.rst +++ b/Doc/license.rst @@ -100,7 +100,7 @@ PSF LICENSE AGREEMENT FOR PYTHON |release| analyze, test, perform and/or display publicly, prepare derivative works, distribute, and otherwise use Python |release| alone or in any derivative version, provided, however, that PSF's License Agreement and PSF's notice of - copyright, i.e., "Copyright © 2001-2023 Python Software Foundation; All Rights + copyright, i.e., "Copyright © 2001-2024 Python Software Foundation; All Rights Reserved" are retained in Python |release| alone or in any derivative version prepared by Licensee. @@ -1066,3 +1066,32 @@ AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. + + +asyncio +---------- + +Parts of the :mod:`asyncio` module are incorporated from +`uvloop 0.16 <https://github.com/MagicStack/uvloop/tree/v0.16.0>`_, +which is distributed under the MIT license:: + + Copyright (c) 2015-2021 MagicStack Inc. http://magic.io + + Permission is hereby granted, free of charge, to any person obtaining + a copy of this software and associated documentation files (the + "Software"), to deal in the Software without restriction, including + without limitation the rights to use, copy, modify, merge, publish, + distribute, sublicense, and/or sell copies of the Software, and to + permit persons to whom the Software is furnished to do so, subject to + the following conditions: + + The above copyright notice and this permission notice shall be + included in all copies or substantial portions of the Software. + + THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, + EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF + MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND + NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE + LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION + OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION + WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. diff --git a/Doc/reference/compound_stmts.rst b/Doc/reference/compound_stmts.rst index 7a735095bdecb2..374404bf33abbe 100644 --- a/Doc/reference/compound_stmts.rst +++ b/Doc/reference/compound_stmts.rst @@ -1362,12 +1362,15 @@ access the local variables of the function containing the def. See section :pep:`526` - Syntax for Variable Annotations Ability to type hint variable declarations, including class - variables and instance variables + variables and instance variables. :pep:`563` - Postponed Evaluation of Annotations Support for forward references within annotations by preserving annotations in a string form at runtime instead of eager evaluation. + :pep:`318` - Decorators for Functions and Methods + Function and method decorators were introduced. + Class decorators were introduced in :pep:`3129`. .. _class: diff --git a/Doc/reference/datamodel.rst b/Doc/reference/datamodel.rst index b3af5c6298d02d..885ee825c12296 100644 --- a/Doc/reference/datamodel.rst +++ b/Doc/reference/datamodel.rst @@ -1219,8 +1219,8 @@ If a code object represents a function, the first item in :attr:`~codeobject.co_consts` is the documentation string of the function, or ``None`` if undefined. -The :meth:`!co_positions` method -~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ +Methods on code objects +~~~~~~~~~~~~~~~~~~~~~~~ .. method:: codeobject.co_positions() @@ -1255,6 +1255,41 @@ The :meth:`!co_positions` method :option:`-X` ``no_debug_ranges`` command line flag or the :envvar:`PYTHONNODEBUGRANGES` environment variable can be used. +.. method:: codeobject.co_lines() + + Returns an iterator that yields information about successive ranges of + :term:`bytecode`\s. Each item yielded is a ``(start, end, lineno)`` + :class:`tuple`: + + * ``start`` (an :class:`int`) represents the offset (inclusive) of the start + of the :term:`bytecode` range + * ``end`` (an :class:`int`) represents the offset (exclusive) of the end of + the :term:`bytecode` range + * ``lineno`` is an :class:`int` representing the line number of the + :term:`bytecode` range, or ``None`` if the bytecodes in the given range + have no line number + + The items yielded will have the following properties: + + * The first range yielded will have a ``start`` of 0. + * The ``(start, end)`` ranges will be non-decreasing and consecutive. That + is, for any pair of :class:`tuple`\s, the ``start`` of the second will be + equal to the ``end`` of the first. + * No range will be backwards: ``end >= start`` for all triples. + * The last :class:`tuple` yielded will have ``end`` equal to the size of the + :term:`bytecode`. + + Zero-width ranges, where ``start == end``, are allowed. Zero-width ranges + are used for lines that are present in the source code, but have been + eliminated by the :term:`bytecode` compiler. + + .. versionadded:: 3.10 + + .. seealso:: + + :pep:`626` - Precise line numbers for debugging and other tools. + The PEP that introduced the :meth:`!co_lines` method. + .. _frame-objects: @@ -1494,7 +1529,7 @@ Class method objects A class method object, like a static method object, is a wrapper around another object that alters the way in which that object is retrieved from classes and class instances. The behaviour of class method objects upon such retrieval is -described above, under "User-defined methods". Class method objects are created +described above, under :ref:`"instance methods" <instance-methods>`. Class method objects are created by the built-in :func:`classmethod` constructor. @@ -1841,7 +1876,7 @@ Basic customization This is intended to provide protection against a denial-of-service caused by carefully chosen inputs that exploit the worst case performance of a - dict insertion, O(n\ :sup:`2`) complexity. See + dict insertion, *O*\ (*n*\ :sup:`2`) complexity. See http://ocert.org/advisories/ocert-2011-003.html for details. Changing hash values affects the iteration order of sets. @@ -1953,8 +1988,8 @@ access (use of, assignment to, or deletion of ``x.name``) for class instances. .. method:: object.__dir__(self) - Called when :func:`dir` is called on the object. A sequence must be - returned. :func:`dir` converts the returned sequence to a list and sorts it. + Called when :func:`dir` is called on the object. An iterable must be + returned. :func:`dir` converts the returned iterable to a list and sorts it. Customizing module attribute access @@ -1974,7 +2009,7 @@ not found on a module object through the normal lookup, i.e. the module ``__dict__`` before raising an :exc:`AttributeError`. If found, it is called with the attribute name and the result is returned. -The ``__dir__`` function should accept no arguments, and return a sequence of +The ``__dir__`` function should accept no arguments, and return an iterable of strings that represents the names accessible on module. If present, this function overrides the standard :func:`dir` search on a module. @@ -2273,7 +2308,7 @@ class defining the method. this method is implicitly converted to a class method. Keyword arguments which are given to a new class are passed to - the parent's class ``__init_subclass__``. For compatibility with + the parent class's ``__init_subclass__``. For compatibility with other classes using ``__init_subclass__``, one should take out the needed keyword arguments and pass the others over to the base class, as in:: diff --git a/Doc/reference/expressions.rst b/Doc/reference/expressions.rst index 3f6d5bfafee9d1..50e0f97a6534af 100644 --- a/Doc/reference/expressions.rst +++ b/Doc/reference/expressions.rst @@ -14,7 +14,7 @@ be used to describe syntax, not lexical analysis. When (one alternative of) a syntax rule has the form .. productionlist:: python-grammar - name: `othername` + name: othername and no semantics are given, the semantics of this form of ``name`` are the same as for ``othername``. @@ -422,7 +422,8 @@ Yield expressions .. productionlist:: python-grammar yield_atom: "(" `yield_expression` ")" - yield_expression: "yield" [`expression_list` | "from" `expression`] + yield_from: "yield" "from" `expression` + yield_expression: "yield" `expression_list` | `yield_from` The yield expression is used when defining a :term:`generator` function or an :term:`asynchronous generator` function and @@ -1000,7 +1001,7 @@ but does not affect the semantics. The primary must evaluate to a callable object (user-defined functions, built-in functions, methods of built-in objects, class objects, methods of class -instances, and all objects having a :meth:`__call__` method are callable). All +instances, and all objects having a :meth:`~object.__call__` method are callable). All argument expressions are evaluated before the call is attempted. Please refer to section :ref:`function` for the syntax of formal :term:`parameter` lists. @@ -1158,7 +1159,7 @@ a class instance: pair: instance; call single: __call__() (object method) - The class must define a :meth:`__call__` method; the effect is then the same as + The class must define a :meth:`~object.__call__` method; the effect is then the same as if that method was called. @@ -1210,7 +1211,7 @@ Raising ``0.0`` to a negative power results in a :exc:`ZeroDivisionError`. Raising a negative number to a fractional power results in a :class:`complex` number. (In earlier versions it raised a :exc:`ValueError`.) -This operation can be customized using the special :meth:`__pow__` method. +This operation can be customized using the special :meth:`~object.__pow__` method. .. _unary: @@ -1233,7 +1234,7 @@ All unary arithmetic and bitwise operations have the same priority: single: - (minus); unary operator The unary ``-`` (minus) operator yields the negation of its numeric argument; the -operation can be overridden with the :meth:`__neg__` special method. +operation can be overridden with the :meth:`~object.__neg__` special method. .. index:: single: plus @@ -1241,7 +1242,7 @@ operation can be overridden with the :meth:`__neg__` special method. single: + (plus); unary operator The unary ``+`` (plus) operator yields its numeric argument unchanged; the -operation can be overridden with the :meth:`__pos__` special method. +operation can be overridden with the :meth:`~object.__pos__` special method. .. index:: single: inversion @@ -1250,7 +1251,7 @@ operation can be overridden with the :meth:`__pos__` special method. The unary ``~`` (invert) operator yields the bitwise inversion of its integer argument. The bitwise inversion of ``x`` is defined as ``-(x+1)``. It only applies to integral numbers or to custom objects that override the -:meth:`__invert__` special method. +:meth:`~object.__invert__` special method. @@ -1288,8 +1289,8 @@ the other must be a sequence. In the former case, the numbers are converted to a common type and then multiplied together. In the latter case, sequence repetition is performed; a negative repetition factor yields an empty sequence. -This operation can be customized using the special :meth:`__mul__` and -:meth:`__rmul__` methods. +This operation can be customized using the special :meth:`~object.__mul__` and +:meth:`~object.__rmul__` methods. .. index:: single: matrix multiplication @@ -1313,8 +1314,8 @@ integer; the result is that of mathematical division with the 'floor' function applied to the result. Division by zero raises the :exc:`ZeroDivisionError` exception. -This operation can be customized using the special :meth:`__truediv__` and -:meth:`__floordiv__` methods. +This operation can be customized using the special :meth:`~object.__truediv__` and +:meth:`~object.__floordiv__` methods. .. index:: single: modulo @@ -1339,7 +1340,7 @@ also overloaded by string objects to perform old-style string formatting (also known as interpolation). The syntax for string formatting is described in the Python Library Reference, section :ref:`old-string-formatting`. -The *modulo* operation can be customized using the special :meth:`__mod__` method. +The *modulo* operation can be customized using the special :meth:`~object.__mod__` method. The floor division operator, the modulo operator, and the :func:`divmod` function are not defined for complex numbers. Instead, convert to a floating @@ -1355,8 +1356,8 @@ must either both be numbers or both be sequences of the same type. In the former case, the numbers are converted to a common type and then added together. In the latter case, the sequences are concatenated. -This operation can be customized using the special :meth:`__add__` and -:meth:`__radd__` methods. +This operation can be customized using the special :meth:`~object.__add__` and +:meth:`~object.__radd__` methods. .. index:: single: subtraction @@ -1366,7 +1367,7 @@ This operation can be customized using the special :meth:`__add__` and The ``-`` (subtraction) operator yields the difference of its arguments. The numeric arguments are first converted to a common type. -This operation can be customized using the special :meth:`__sub__` method. +This operation can be customized using the special :meth:`~object.__sub__` method. .. _shifting: @@ -1387,8 +1388,8 @@ The shifting operations have lower priority than the arithmetic operations: These operators accept integers as arguments. They shift the first argument to the left or right by the number of bits given by the second argument. -This operation can be customized using the special :meth:`__lshift__` and -:meth:`__rshift__` methods. +This operation can be customized using the special :meth:`~object.__lshift__` and +:meth:`~object.__rshift__` methods. .. index:: pair: exception; ValueError @@ -1415,8 +1416,8 @@ Each of the three bitwise operations has a different priority level: pair: operator; & (ampersand) The ``&`` operator yields the bitwise AND of its arguments, which must be -integers or one of them must be a custom object overriding :meth:`__and__` or -:meth:`__rand__` special methods. +integers or one of them must be a custom object overriding :meth:`~object.__and__` or +:meth:`~object.__rand__` special methods. .. index:: pair: bitwise; xor @@ -1424,8 +1425,8 @@ integers or one of them must be a custom object overriding :meth:`__and__` or pair: operator; ^ (caret) The ``^`` operator yields the bitwise XOR (exclusive OR) of its arguments, which -must be integers or one of them must be a custom object overriding :meth:`__xor__` or -:meth:`__rxor__` special methods. +must be integers or one of them must be a custom object overriding :meth:`~object.__xor__` or +:meth:`~object.__rxor__` special methods. .. index:: pair: bitwise; or @@ -1433,8 +1434,8 @@ must be integers or one of them must be a custom object overriding :meth:`__xor_ pair: operator; | (vertical bar) The ``|`` operator yields the bitwise (inclusive) OR of its arguments, which -must be integers or one of them must be a custom object overriding :meth:`__or__` or -:meth:`__ror__` special methods. +must be integers or one of them must be a custom object overriding :meth:`~object.__or__` or +:meth:`~object.__ror__` special methods. .. _comparisons: @@ -1501,7 +1502,7 @@ comparison implementation. Because all types are (direct or indirect) subtypes of :class:`object`, they inherit the default comparison behavior from :class:`object`. Types can customize their comparison behavior by implementing -:dfn:`rich comparison methods` like :meth:`__lt__`, described in +:dfn:`rich comparison methods` like :meth:`~object.__lt__`, described in :ref:`customization`. The default behavior for equality comparison (``==`` and ``!=``) is based on @@ -1665,12 +1666,12 @@ substring of *y*. An equivalent test is ``y.find(x) != -1``. Empty strings are always considered to be a substring of any other string, so ``"" in "abc"`` will return ``True``. -For user-defined classes which define the :meth:`__contains__` method, ``x in +For user-defined classes which define the :meth:`~object.__contains__` method, ``x in y`` returns ``True`` if ``y.__contains__(x)`` returns a true value, and ``False`` otherwise. -For user-defined classes which do not define :meth:`__contains__` but do define -:meth:`__iter__`, ``x in y`` is ``True`` if some value ``z``, for which the +For user-defined classes which do not define :meth:`~object.__contains__` but do define +:meth:`~object.__iter__`, ``x in y`` is ``True`` if some value ``z``, for which the expression ``x is z or x == z`` is true, is produced while iterating over ``y``. If an exception is raised during the iteration, it is as if :keyword:`in` raised that exception. @@ -1889,8 +1890,9 @@ the unpacking. .. index:: pair: trailing; comma -The trailing comma is required only to create a single tuple (a.k.a. a -*singleton*); it is optional in all other cases. A single expression without a +A trailing comma is required only to create a one-item tuple, +such as ``1,``; it is optional in all other cases. +A single expression without a trailing comma doesn't create a tuple, but rather yields the value of that expression. (To create an empty tuple, use an empty pair of parentheses: ``()``.) diff --git a/Doc/reference/import.rst b/Doc/reference/import.rst index a7beeea29b4556..f8c9724114da9e 100644 --- a/Doc/reference/import.rst +++ b/Doc/reference/import.rst @@ -327,14 +327,15 @@ modules, and one that knows how to import modules from an :term:`import path` finders replaced :meth:`!find_module`, which is now deprecated. While it will continue to work without change, the import machinery will try it only if the finder does not implement - ``find_spec()``. + :meth:`~importlib.abc.MetaPathFinder.find_spec`. .. versionchanged:: 3.10 Use of :meth:`!find_module` by the import system now raises :exc:`ImportWarning`. .. versionchanged:: 3.12 - ``find_module()`` has been removed. Use :meth:`find_spec` instead. + :meth:`!find_module` has been removed. + Use :meth:`~importlib.abc.MetaPathFinder.find_spec` instead. Loading @@ -812,7 +813,7 @@ attributes on package objects are also used. These provide additional ways that the import machinery can be customized. :data:`sys.path` contains a list of strings providing search locations for -modules and packages. It is initialized from the :data:`PYTHONPATH` +modules and packages. It is initialized from the :envvar:`PYTHONPATH` environment variable and various other installation- and implementation-specific defaults. Entries in :data:`sys.path` can name directories on the file system, zip files, and potentially other "locations" diff --git a/Doc/requirements.txt b/Doc/requirements.txt index 04334fd5a464d4..118e6c322b4be2 100644 --- a/Doc/requirements.txt +++ b/Doc/requirements.txt @@ -6,8 +6,7 @@ # Sphinx version is pinned so that new versions that introduce new warnings # won't suddenly cause build failures. Updating the version is fine as long # as no warnings are raised by doing so. -# PR #104777: Sphinx 6.2 no longer uses imghdr, removed in Python 3.13. -sphinx==6.2.1 +sphinx~=7.2.0 blurb diff --git a/Doc/tools/.nitignore b/Doc/tools/.nitignore index 8b0fc13832d51a..9db02c5c3c73c9 100644 --- a/Doc/tools/.nitignore +++ b/Doc/tools/.nitignore @@ -4,18 +4,13 @@ Doc/c-api/descriptor.rst Doc/c-api/exceptions.rst -Doc/c-api/file.rst Doc/c-api/float.rst Doc/c-api/gcsupport.rst Doc/c-api/init.rst Doc/c-api/init_config.rst Doc/c-api/intro.rst -Doc/c-api/memory.rst -Doc/c-api/memoryview.rst Doc/c-api/module.rst -Doc/c-api/object.rst Doc/c-api/stable.rst -Doc/c-api/structures.rst Doc/c-api/sys.rst Doc/c-api/type.rst Doc/c-api/typeobj.rst @@ -23,59 +18,39 @@ Doc/extending/extending.rst Doc/glossary.rst Doc/howto/descriptor.rst Doc/howto/enum.rst -Doc/howto/isolating-extensions.rst -Doc/howto/logging.rst -Doc/howto/urllib2.rst Doc/library/ast.rst Doc/library/asyncio-extending.rst Doc/library/asyncio-policy.rst Doc/library/asyncio-subprocess.rst -Doc/library/asyncio-task.rst Doc/library/bdb.rst -Doc/library/bisect.rst -Doc/library/calendar.rst -Doc/library/cmd.rst Doc/library/collections.rst -Doc/library/concurrent.futures.rst -Doc/library/configparser.rst -Doc/library/csv.rst -Doc/library/datetime.rst Doc/library/dbm.rst Doc/library/decimal.rst Doc/library/email.charset.rst Doc/library/email.compat32-message.rst Doc/library/email.errors.rst -Doc/library/email.mime.rst Doc/library/email.parser.rst Doc/library/email.policy.rst -Doc/library/enum.rst Doc/library/exceptions.rst Doc/library/faulthandler.rst Doc/library/fcntl.rst -Doc/library/ftplib.rst Doc/library/functools.rst -Doc/library/http.client.rst Doc/library/http.cookiejar.rst Doc/library/http.server.rst Doc/library/importlib.rst -Doc/library/locale.rst Doc/library/logging.config.rst Doc/library/logging.handlers.rst Doc/library/lzma.rst Doc/library/mmap.rst Doc/library/multiprocessing.rst -Doc/library/multiprocessing.shared_memory.rst Doc/library/optparse.rst Doc/library/os.rst -Doc/library/pickle.rst Doc/library/pickletools.rst Doc/library/platform.rst Doc/library/plistlib.rst Doc/library/profile.rst -Doc/library/pyclbr.rst Doc/library/pydoc.rst Doc/library/pyexpat.rst -Doc/library/random.rst Doc/library/readline.rst Doc/library/resource.rst Doc/library/select.rst @@ -84,7 +59,6 @@ Doc/library/smtplib.rst Doc/library/socket.rst Doc/library/ssl.rst Doc/library/stdtypes.rst -Doc/library/string.rst Doc/library/subprocess.rst Doc/library/termios.rst Doc/library/test.rst @@ -107,13 +81,10 @@ Doc/library/xmlrpc.server.rst Doc/library/zlib.rst Doc/reference/compound_stmts.rst Doc/reference/datamodel.rst -Doc/reference/expressions.rst -Doc/reference/import.rst Doc/tutorial/datastructures.rst Doc/using/windows.rst Doc/whatsnew/2.0.rst Doc/whatsnew/2.1.rst -Doc/whatsnew/2.2.rst Doc/whatsnew/2.4.rst Doc/whatsnew/2.5.rst Doc/whatsnew/2.6.rst diff --git a/Doc/tools/extensions/c_annotations.py b/Doc/tools/extensions/c_annotations.py index 42c2f10e0be260..a8b6d8995e3f40 100644 --- a/Doc/tools/extensions/c_annotations.py +++ b/Doc/tools/extensions/c_annotations.py @@ -42,7 +42,7 @@ } -# Monkeypatch nodes.Node.findall for forwards compatability +# Monkeypatch nodes.Node.findall for forwards compatibility # This patch can be dropped when the minimum Sphinx version is 4.4.0 # or the minimum Docutils version is 0.18.1. if docutils.__version_info__ < (0, 18, 1): @@ -126,7 +126,8 @@ def add_annotations(self, app, doctree): f"Object type mismatch in limited API annotation " f"for {name}: {record['role']!r} != {objtype!r}") stable_added = record['added'] - message = sphinx_gettext(' Part of the ') + message = sphinx_gettext('Part of the') + message = message.center(len(message) + 2) emph_node = nodes.emphasis(message, message, classes=['stableabi']) ref_node = addnodes.pending_xref( @@ -139,27 +140,27 @@ def add_annotations(self, app, doctree): ref_node += nodes.Text(sphinx_gettext('Stable ABI')) emph_node += ref_node if struct_abi_kind == 'opaque': - emph_node += nodes.Text(sphinx_gettext(' (as an opaque struct)')) + emph_node += nodes.Text(' ' + sphinx_gettext('(as an opaque struct)')) elif struct_abi_kind == 'full-abi': - emph_node += nodes.Text(sphinx_gettext(' (including all members)')) + emph_node += nodes.Text(' ' + sphinx_gettext('(including all members)')) if record['ifdef_note']: emph_node += nodes.Text(' ' + record['ifdef_note']) if stable_added == '3.2': # Stable ABI was introduced in 3.2. pass else: - emph_node += nodes.Text(sphinx_gettext(' since version %s') % stable_added) + emph_node += nodes.Text(' ' + sphinx_gettext('since version %s') % stable_added) emph_node += nodes.Text('.') if struct_abi_kind == 'members': emph_node += nodes.Text( - sphinx_gettext(' (Only some members are part of the stable ABI.)')) + ' ' + sphinx_gettext('(Only some members are part of the stable ABI.)')) node.insert(0, emph_node) # Unstable API annotation. if name.startswith('PyUnstable'): warn_node = nodes.admonition( classes=['unstable-c-api', 'warning']) - message = sphinx_gettext('This is ') + message = sphinx_gettext('This is') + ' ' emph_node = nodes.emphasis(message, message) ref_node = addnodes.pending_xref( 'Unstable API', refdomain="std", diff --git a/Doc/tools/extensions/pyspecific.py b/Doc/tools/extensions/pyspecific.py index 31c2544caf601c..cd441836f62bde 100644 --- a/Doc/tools/extensions/pyspecific.py +++ b/Doc/tools/extensions/pyspecific.py @@ -48,6 +48,11 @@ Body.enum.converters['lowerroman'] = \ Body.enum.converters['upperroman'] = lambda x: None +# monkey-patch the productionlist directive to allow hyphens in group names +# https://github.com/sphinx-doc/sphinx/issues/11854 +from sphinx.domains import std + +std.token_re = re.compile(r'`((~?[\w-]*:)?\w+)`') # Support for marking up and linking to bugs.python.org issues diff --git a/Doc/tools/templates/dummy.html b/Doc/tools/templates/dummy.html index 3a0acab8836b11..49c2a71a5e40cf 100644 --- a/Doc/tools/templates/dummy.html +++ b/Doc/tools/templates/dummy.html @@ -9,14 +9,14 @@ In extensions/c_annotations.py: -{% trans %} Part of the {% endtrans %} +{% trans %}Part of the{% endtrans %} {% trans %}Limited API{% endtrans %} {% trans %}Stable ABI{% endtrans %} -{% trans %} (as an opaque struct){% endtrans %} -{% trans %} (including all members){% endtrans %} -{% trans %} since version %s{% endtrans %} -{% trans %} (Only some members are part of the stable ABI.){% endtrans %} -{% trans %}This is {% endtrans %} +{% trans %}(as an opaque struct){% endtrans %} +{% trans %}(including all members){% endtrans %} +{% trans %}since version %s{% endtrans %} +{% trans %}(Only some members are part of the stable ABI.){% endtrans %} +{% trans %}This is{% endtrans %} {% trans %}Unstable API{% endtrans %} {% trans %}. It may change without warning in minor releases.{% endtrans %} {% trans %}Return value: Always NULL.{% endtrans %} diff --git a/Doc/tutorial/appendix.rst b/Doc/tutorial/appendix.rst index 588591fcdb726f..4bea0d8a49ce20 100644 --- a/Doc/tutorial/appendix.rst +++ b/Doc/tutorial/appendix.rst @@ -20,7 +20,7 @@ In interactive mode, it then returns to the primary prompt; when input came from a file, it exits with a nonzero exit status after printing the stack trace. (Exceptions handled by an :keyword:`except` clause in a :keyword:`try` statement are not errors in this context.) Some errors are unconditionally fatal and -cause an exit with a nonzero exit; this applies to internal inconsistencies and +cause an exit with a nonzero exit status; this applies to internal inconsistencies and some cases of running out of memory. All error messages are written to the standard error stream; normal output from executed commands is written to standard output. diff --git a/Doc/tutorial/classes.rst b/Doc/tutorial/classes.rst index 3bf138ca225ee5..d1c303ef037027 100644 --- a/Doc/tutorial/classes.rst +++ b/Doc/tutorial/classes.rst @@ -386,12 +386,11 @@ general, calling a method with a list of *n* arguments is equivalent to calling the corresponding function with an argument list that is created by inserting the method's instance object before the first argument. -If you still don't understand how methods work, a look at the implementation can -perhaps clarify matters. When a non-data attribute of an instance is -referenced, the instance's class is searched. If the name denotes a valid class -attribute that is a function object, a method object is created by packing -(pointers to) the instance object and the function object just found together in -an abstract object: this is the method object. When the method object is called +In general, methods work as follows. When a non-data attribute +of an instance is referenced, the instance's class is searched. +If the name denotes a valid class attribute that is a function object, +references to both the instance object and the function object +are packed into a method object. When the method object is called with an argument list, a new argument list is constructed from the instance object and the argument list, and the function object is called with this new argument list. diff --git a/Doc/tutorial/datastructures.rst b/Doc/tutorial/datastructures.rst index 87614d082a1d4e..de2827461e2f24 100644 --- a/Doc/tutorial/datastructures.rst +++ b/Doc/tutorial/datastructures.rst @@ -48,10 +48,9 @@ objects: :noindex: Remove the item at the given position in the list, and return it. If no index - is specified, ``a.pop()`` removes and returns the last item in the list. (The - square brackets around the *i* in the method signature denote that the parameter - is optional, not that you should type square brackets at that position. You - will see this notation frequently in the Python Library Reference.) + is specified, ``a.pop()`` removes and returns the last item in the list. + It raises an :exc:`IndexError` if the list is empty or the index is + outside the list range. .. method:: list.clear() diff --git a/Doc/using/cmdline.rst b/Doc/using/cmdline.rst index e032a1971bc6d6..53c95ca1a05c9b 100644 --- a/Doc/using/cmdline.rst +++ b/Doc/using/cmdline.rst @@ -369,7 +369,7 @@ Miscellaneous options Hash randomization is intended to provide protection against a denial-of-service caused by carefully chosen inputs that exploit the worst - case performance of a dict construction, O(n\ :sup:`2`) complexity. See + case performance of a dict construction, *O*\ (*n*\ :sup:`2`) complexity. See http://ocert.org/advisories/ocert-2011-003.html for details. :envvar:`PYTHONHASHSEED` allows you to set a fixed value for the hash @@ -623,7 +623,7 @@ Setting the environment variable ``TERM`` to ``dumb`` will disable color. If the environment variable ``FORCE_COLOR`` is set, then color will be enabled regardless of the value of TERM. This is useful on CI systems which -aren’t terminals but can none-the-less display ANSI escape sequences. +aren’t terminals but can still display ANSI escape sequences. If the environment variable ``NO_COLOR`` is set, Python will disable all color in the output. This takes precedence over ``FORCE_COLOR``. @@ -1139,6 +1139,14 @@ conflict. .. versionadded:: 3.13 +.. envvar:: PYTHON_HISTORY + + This environment variable can be used to set the location of a + ``.python_history`` file (by default, it is ``.python_history`` in the + user's home directory). + + .. versionadded:: 3.13 + Debug-mode variables ~~~~~~~~~~~~~~~~~~~~ diff --git a/Doc/using/configure.rst b/Doc/using/configure.rst index cb7eda42fe3fad..aab9469b44828a 100644 --- a/Doc/using/configure.rst +++ b/Doc/using/configure.rst @@ -745,6 +745,13 @@ Debug options .. versionadded:: 3.6 +.. option:: --with-thread-sanitizer + + Enable ThreadSanitizer data race detector, ``tsan`` + (default is no). + + .. versionadded:: 3.13 + Linker options -------------- diff --git a/Doc/using/win_install_freethreaded.png b/Doc/using/win_install_freethreaded.png new file mode 100644 index 00000000000000..0aa01c1df6e051 Binary files /dev/null and b/Doc/using/win_install_freethreaded.png differ diff --git a/Doc/using/windows.rst b/Doc/using/windows.rst index 598bf3ca9bcc04..2a0e7b4b06f586 100644 --- a/Doc/using/windows.rst +++ b/Doc/using/windows.rst @@ -307,6 +307,46 @@ settings and replace any that have been removed or modified. "Uninstall" will remove Python entirely, with the exception of the :ref:`launcher`, which has its own entry in Programs and Features. +.. _install-freethreaded-windows: + +Installing Free-threaded Binaries +--------------------------------- + +.. versionadded:: 3.13 (Experimental) + +.. note:: + + Everything described in this section is considered experimental, + and should be expected to change in future releases. + +To install pre-built binaries with free-threading enabled (see :pep:`703`), you +should select "Customize installation". The second page of options includes the +"Download free-threaded binaries" checkbox. + +.. image:: win_install_freethreaded.png + +Selecting this option will download and install additional binaries to the same +location as the main Python install. The main executable is called +``python3.13t.exe``, and other binaries either receive a ``t`` suffix or a full +ABI suffix. Python source files and bundled third-party dependencies are shared +with the main install. + +The free-threaded version is registered as a regular Python install with the +tag ``3.13t`` (with a ``-32`` or ``-arm64`` suffix as normal for those +platforms). This allows tools to discover it, and for the :ref:`launcher` to +support ``py.exe -3.13t``. Note that the launcher will interpret ``py.exe -3`` +(or a ``python3`` shebang) as "the latest 3.x install", which will prefer the +free-threaded binaries over the regular ones, while ``py.exe -3.13`` will not. +If you use the short style of option, you may prefer to not install the +free-threaded binaries at this time. + +To specify the install option at the command line, use +``Include_freethreaded=1``. See :ref:`install-layout-option` for instructions on +pre-emptively downloading the additional binaries for offline install. The +options to include debug symbols and binaries also apply to the free-threaded +builds. + +Free-threaded binaries are also available :ref:`on nuget.org <windows-nuget>`. .. _windows-store: @@ -450,9 +490,29 @@ automatically use the headers and import libraries in your build. The package information pages on nuget.org are `www.nuget.org/packages/python <https://www.nuget.org/packages/python>`_ -for the 64-bit version and `www.nuget.org/packages/pythonx86 -<https://www.nuget.org/packages/pythonx86>`_ for the 32-bit version. +for the 64-bit version, `www.nuget.org/packages/pythonx86 +<https://www.nuget.org/packages/pythonx86>`_ for the 32-bit version, and +`www.nuget.org/packages/pythonarm64 +<https://www.nuget.org/packages/pythonarm64>`_ for the ARM64 version + +Free-threaded packages +---------------------- + +.. versionadded:: 3.13 (Experimental) + +.. note:: + Everything described in this section is considered experimental, + and should be expected to change in future releases. + +Packages containing free-threaded binaries are named +`python-freethreaded <https://www.nuget.org/packages/python-freethreaded>`_ +for the 64-bit version, `pythonx86-freethreaded +<https://www.nuget.org/packages/pythonx86-freethreaded>`_ for the 32-bit +version, and `pythonarm64-freethreaded +<https://www.nuget.org/packages/pythonarm64-freethreaded>`_ for the ARM64 +version. These packages contain both the ``python3.13t.exe`` and +``python.exe`` entry points, both of which run free threaded. .. _windows-embeddable: diff --git a/Doc/whatsnew/2.0.rst b/Doc/whatsnew/2.0.rst index 6d6e51006d5bd8..af8171487fbcfa 100644 --- a/Doc/whatsnew/2.0.rst +++ b/Doc/whatsnew/2.0.rst @@ -130,7 +130,7 @@ Guidelines": Read the rest of :pep:`1` for the details of the PEP editorial process, style, and format. PEPs are kept in the Python CVS tree on SourceForge, though they're not part of the Python 2.0 distribution, and are also available in HTML form from -https://peps.python.org/. As of September 2000, there are 25 PEPS, ranging +https://peps.python.org/. As of September 2000, there are 25 PEPs, ranging from :pep:`201`, "Lockstep Iteration", to PEP 225, "Elementwise/Objectwise Operators". @@ -1030,7 +1030,7 @@ Module changes Lots of improvements and bugfixes were made to Python's extensive standard library; some of the affected modules include :mod:`readline`, -:mod:`ConfigParser`, :mod:`!cgi`, :mod:`calendar`, :mod:`posix`, :mod:`readline`, +:mod:`ConfigParser <configparser>`, :mod:`!cgi`, :mod:`calendar`, :mod:`posix`, :mod:`readline`, :mod:`xmllib`, :mod:`!aifc`, :mod:`!chunk`, :mod:`wave`, :mod:`random`, :mod:`shelve`, and :mod:`!nntplib`. Consult the CVS logs for the exact patch-by-patch details. @@ -1039,12 +1039,12 @@ is an implementation of the Secure Socket Layer, which encrypts the data being sent over a socket. When compiling Python, you can edit :file:`Modules/Setup` to include SSL support, which adds an additional function to the :mod:`socket` module: ``socket.ssl(socket, keyfile, certfile)``, which takes a socket -object and returns an SSL socket. The :mod:`httplib` and :mod:`urllib` modules +object and returns an SSL socket. The :mod:`httplib <http>` and :mod:`urllib` modules were also changed to support ``https://`` URLs, though no one has implemented FTP or SMTP over SSL. -The :mod:`httplib` module has been rewritten by Greg Stein to support HTTP/1.1. -Backward compatibility with the 1.5 version of :mod:`httplib` is provided, +The :mod:`httplib <http>` module has been rewritten by Greg Stein to support HTTP/1.1. +Backward compatibility with the 1.5 version of :mod:`!httplib` is provided, though using HTTP/1.1 features such as pipelining will require rewriting code to use a different set of interfaces. @@ -1095,8 +1095,8 @@ module. GNU gettext message catalog library. (Integrated by Barry Warsaw, from separate contributions by Martin von Löwis, Peter Funk, and James Henstridge.) -* :mod:`linuxaudiodev`: Support for the :file:`/dev/audio` device on Linux, a - twin to the existing :mod:`sunaudiodev` module. (Contributed by Peter Bosch, +* :mod:`!linuxaudiodev`: Support for the :file:`/dev/audio` device on Linux, a + twin to the existing :mod:`!sunaudiodev` module. (Contributed by Peter Bosch, with fixes by Jeremy Hylton.) * :mod:`mmap`: An interface to memory-mapped files on both Windows and Unix. A @@ -1108,7 +1108,7 @@ module. * :mod:`pyexpat`: An interface to the Expat XML parser. (Contributed by Paul Prescod.) -* :mod:`robotparser`: Parse a :file:`robots.txt` file, which is used for writing +* :mod:`robotparser <urllib.robotparser>`: Parse a :file:`robots.txt` file, which is used for writing web spiders that politely avoid certain areas of a web site. The parser accepts the contents of a :file:`robots.txt` file, builds a set of rules from it, and can then answer questions about the fetchability of a given URL. (Contributed @@ -1129,18 +1129,18 @@ module. :file:`Tools/idle/BrowserControl.py`, and adapted for the standard library by Fred.) -* :mod:`_winreg`: An interface to the Windows registry. :mod:`_winreg` is an +* :mod:`_winreg <winreg>`: An interface to the Windows registry. :mod:`!_winreg` is an adaptation of functions that have been part of PythonWin since 1995, but has now been added to the core distribution, and enhanced to support Unicode. - :mod:`_winreg` was written by Bill Tutt and Mark Hammond. + :mod:`!_winreg` was written by Bill Tutt and Mark Hammond. * :mod:`zipfile`: A module for reading and writing ZIP-format archives. These are archives produced by :program:`PKZIP` on DOS/Windows or :program:`zip` on Unix, not to be confused with :program:`gzip`\ -format files (which are supported by the :mod:`gzip` module) (Contributed by James C. Ahlstrom.) -* :mod:`imputil`: A module that provides a simpler way for writing customized - import hooks, in comparison to the existing :mod:`ihooks` module. (Implemented +* :mod:`!imputil`: A module that provides a simpler way for writing customized + import hooks, in comparison to the existing :mod:`!ihooks` module. (Implemented by Greg Stein, with much discussion on python-dev along the way.) .. ====================================================================== diff --git a/Doc/whatsnew/2.2.rst b/Doc/whatsnew/2.2.rst index 6dfe79cef00987..e6c13f957b8d54 100644 --- a/Doc/whatsnew/2.2.rst +++ b/Doc/whatsnew/2.2.rst @@ -53,9 +53,9 @@ A long time ago I wrote a web page listing flaws in Python's design. One of the most significant flaws was that it's impossible to subclass Python types implemented in C. In particular, it's not possible to subclass built-in types, so you can't just subclass, say, lists in order to add a single useful method to -them. The :mod:`UserList` module provides a class that supports all of the +them. The :mod:`!UserList` module provides a class that supports all of the methods of lists and that can be subclassed further, but there's lots of C code -that expects a regular Python list and won't accept a :class:`UserList` +that expects a regular Python list and won't accept a :class:`~collections.UserList` instance. Python 2.2 fixes this, and in the process adds some exciting new capabilities. @@ -69,7 +69,7 @@ A brief summary: * It's also possible to automatically call methods on accessing or setting an instance attribute by using a new mechanism called :dfn:`properties`. Many uses - of :meth:`__getattr__` can be rewritten to use properties instead, making the + of :meth:`~object.__getattr__` can be rewritten to use properties instead, making the resulting code simpler and faster. As a small side benefit, attributes can now have docstrings, too. @@ -120,7 +120,7 @@ added so if no built-in type is suitable, you can just subclass This means that :keyword:`class` statements that don't have any base classes are always classic classes in Python 2.2. (Actually you can also change this by -setting a module-level variable named :attr:`__metaclass__` --- see :pep:`253` +setting a module-level variable named :attr:`!__metaclass__` --- see :pep:`253` for the details --- but it's easier to just subclass :class:`object`.) The type objects for the built-in types are available as built-ins, named using @@ -134,8 +134,8 @@ type objects that behave as factories when called. :: 123 To make the set of types complete, new type objects such as :func:`dict` and -:func:`file` have been added. Here's a more interesting example, adding a -:meth:`lock` method to file objects:: +:func:`!file` have been added. Here's a more interesting example, adding a +:meth:`!lock` method to file objects:: class LockableFile(file): def lock (self, operation, length=0, start=0, whence=0): @@ -143,10 +143,10 @@ To make the set of types complete, new type objects such as :func:`dict` and return fcntl.lockf(self.fileno(), operation, length, start, whence) -The now-obsolete :mod:`posixfile` module contained a class that emulated all of -a file object's methods and also added a :meth:`lock` method, but this class +The now-obsolete :mod:`!posixfile` module contained a class that emulated all of +a file object's methods and also added a :meth:`!lock` method, but this class couldn't be passed to internal functions that expected a built-in file, -something which is possible with our new :class:`LockableFile`. +something which is possible with our new :class:`!LockableFile`. Descriptors @@ -154,11 +154,11 @@ Descriptors In previous versions of Python, there was no consistent way to discover what attributes and methods were supported by an object. There were some informal -conventions, such as defining :attr:`__members__` and :attr:`__methods__` +conventions, such as defining :attr:`!__members__` and :attr:`!__methods__` attributes that were lists of names, but often the author of an extension type or a class wouldn't bother to define them. You could fall back on inspecting the :attr:`~object.__dict__` of an object, but when class inheritance or an arbitrary -:meth:`__getattr__` hook were in use this could still be inaccurate. +:meth:`!__getattr__` hook were in use this could still be inaccurate. The one big idea underlying the new class model is that an API for describing the attributes of an object using :dfn:`descriptors` has been formalized. @@ -171,7 +171,7 @@ attributes of their own: * :attr:`~definition.__name__` is the attribute's name. -* :attr:`__doc__` is the attribute's docstring. +* :attr:`!__doc__` is the attribute's docstring. * ``__get__(object)`` is a method that retrieves the attribute value from *object*. @@ -186,7 +186,7 @@ are:: descriptor = obj.__class__.x descriptor.__get__(obj) -For methods, :meth:`descriptor.__get__` returns a temporary object that's +For methods, :meth:`!descriptor.__get__` returns a temporary object that's callable, and wraps up the instance and the method to be called on it. This is also why static methods and class methods are now possible; they have descriptors that wrap up just the method, or the method and the class. As a @@ -204,7 +204,7 @@ methods are defined like this:: ... g = classmethod(g) -The :func:`staticmethod` function takes the function :func:`f`, and returns it +The :func:`staticmethod` function takes the function :func:`!f`, and returns it wrapped up in a descriptor so it can be stored in the class object. You might expect there to be special syntax for creating such methods (``def static f``, ``defstatic f()``, or something like that) but no such syntax has been defined @@ -232,10 +232,10 @@ like this:: f = eiffelmethod(f, pre_f, post_f) -Note that a person using the new :func:`eiffelmethod` doesn't have to understand +Note that a person using the new :func:`!eiffelmethod` doesn't have to understand anything about descriptors. This is why I think the new features don't increase the basic complexity of the language. There will be a few wizards who need to -know about it in order to write :func:`eiffelmethod` or the ZODB or whatever, +know about it in order to write :func:`!eiffelmethod` or the ZODB or whatever, but most users will just write code on top of the resulting libraries and ignore the implementation details. @@ -263,10 +263,10 @@ from :pep:`253` by Guido van Rossum):: The lookup rule for classic classes is simple but not very smart; the base classes are searched depth-first, going from left to right. A reference to -:meth:`D.save` will search the classes :class:`D`, :class:`B`, and then -:class:`A`, where :meth:`save` would be found and returned. :meth:`C.save` -would never be found at all. This is bad, because if :class:`C`'s :meth:`save` -method is saving some internal state specific to :class:`C`, not calling it will +:meth:`!D.save` will search the classes :class:`!D`, :class:`!B`, and then +:class:`!A`, where :meth:`!save` would be found and returned. :meth:`!C.save` +would never be found at all. This is bad, because if :class:`!C`'s :meth:`!save` +method is saving some internal state specific to :class:`!C`, not calling it will result in that state never getting saved. New-style classes follow a different algorithm that's a bit more complicated to @@ -276,22 +276,22 @@ produces more useful results for really complicated inheritance graphs.) #. List all the base classes, following the classic lookup rule and include a class multiple times if it's visited repeatedly. In the above example, the list - of visited classes is [:class:`D`, :class:`B`, :class:`A`, :class:`C`, - :class:`A`]. + of visited classes is [:class:`!D`, :class:`!B`, :class:`!A`, :class:`!C`, + :class:`!A`]. #. Scan the list for duplicated classes. If any are found, remove all but one occurrence, leaving the *last* one in the list. In the above example, the list - becomes [:class:`D`, :class:`B`, :class:`C`, :class:`A`] after dropping + becomes [:class:`!D`, :class:`!B`, :class:`!C`, :class:`!A`] after dropping duplicates. -Following this rule, referring to :meth:`D.save` will return :meth:`C.save`, +Following this rule, referring to :meth:`!D.save` will return :meth:`!C.save`, which is the behaviour we're after. This lookup rule is the same as the one followed by Common Lisp. A new built-in function, :func:`super`, provides a way to get at a class's superclasses without having to reimplement Python's algorithm. The most commonly used form will be ``super(class, obj)``, which returns a bound superclass object (not the actual class object). This form will be used in methods to call a method in the superclass; for example, -:class:`D`'s :meth:`save` method would look like this:: +:class:`!D`'s :meth:`!save` method would look like this:: class D (B,C): def save (self): @@ -309,7 +309,7 @@ Attribute Access ---------------- A fair number of sophisticated Python classes define hooks for attribute access -using :meth:`__getattr__`; most commonly this is done for convenience, to make +using :meth:`~object.__getattr__`; most commonly this is done for convenience, to make code more readable by automatically mapping an attribute access such as ``obj.parent`` into a method call such as ``obj.get_parent``. Python 2.2 adds some new ways of controlling attribute access. @@ -321,22 +321,22 @@ instance's dictionary. New-style classes also support a new method, ``__getattribute__(attr_name)``. The difference between the two methods is -that :meth:`__getattribute__` is *always* called whenever any attribute is -accessed, while the old :meth:`__getattr__` is only called if ``foo`` isn't +that :meth:`~object.__getattribute__` is *always* called whenever any attribute is +accessed, while the old :meth:`~object.__getattr__` is only called if ``foo`` isn't found in the instance's dictionary. However, Python 2.2's support for :dfn:`properties` will often be a simpler way -to trap attribute references. Writing a :meth:`__getattr__` method is +to trap attribute references. Writing a :meth:`!__getattr__` method is complicated because to avoid recursion you can't use regular attribute accesses inside them, and instead have to mess around with the contents of -:attr:`~object.__dict__`. :meth:`__getattr__` methods also end up being called by Python -when it checks for other methods such as :meth:`__repr__` or :meth:`__coerce__`, +:attr:`~object.__dict__`. :meth:`~object.__getattr__` methods also end up being called by Python +when it checks for other methods such as :meth:`~object.__repr__` or :meth:`!__coerce__`, and so have to be written with this in mind. Finally, calling a function on every attribute access results in a sizable performance loss. :class:`property` is a new built-in type that packages up three functions that get, set, or delete an attribute, and a docstring. For example, if you want to -define a :attr:`size` attribute that's computed, but also settable, you could +define a :attr:`!size` attribute that's computed, but also settable, you could write:: class C(object): @@ -355,9 +355,9 @@ write:: "Storage size of this instance") That is certainly clearer and easier to write than a pair of -:meth:`__getattr__`/:meth:`__setattr__` methods that check for the :attr:`size` +:meth:`!__getattr__`/:meth:`!__setattr__` methods that check for the :attr:`!size` attribute and handle it specially while retrieving all other attributes from the -instance's :attr:`~object.__dict__`. Accesses to :attr:`size` are also the only ones +instance's :attr:`~object.__dict__`. Accesses to :attr:`!size` are also the only ones which have to perform the work of calling a function, so references to other attributes run at their usual speed. @@ -447,7 +447,7 @@ an iterator for the object *obj*, while ``iter(C, sentinel)`` returns an iterator that will invoke the callable object *C* until it returns *sentinel* to signal that the iterator is done. -Python classes can define an :meth:`__iter__` method, which should create and +Python classes can define an :meth:`!__iter__` method, which should create and return a new iterator for the object; if the object is its own iterator, this method can just return ``self``. In particular, iterators will usually be their own iterators. Extension types implemented in C can implement a :c:member:`~PyTypeObject.tp_iter` @@ -478,7 +478,7 @@ there are no more values to be returned, calling :meth:`next` should raise the In 2.2, Python's :keyword:`for` statement no longer expects a sequence; it expects something for which :func:`iter` will return an iterator. For backward compatibility and convenience, an iterator is automatically constructed for -sequences that don't implement :meth:`__iter__` or a :c:member:`~PyTypeObject.tp_iter` slot, so +sequences that don't implement :meth:`!__iter__` or a :c:member:`~PyTypeObject.tp_iter` slot, so ``for i in [1,2,3]`` will still work. Wherever the Python interpreter loops over a sequence, it's been changed to use the iterator protocol. This means you can do things like this:: @@ -510,8 +510,8 @@ Iterator support has been added to some of Python's basic types. Calling Oct 10 That's just the default behaviour. If you want to iterate over keys, values, or -key/value pairs, you can explicitly call the :meth:`iterkeys`, -:meth:`itervalues`, or :meth:`iteritems` methods to get an appropriate iterator. +key/value pairs, you can explicitly call the :meth:`!iterkeys`, +:meth:`!itervalues`, or :meth:`!iteritems` methods to get an appropriate iterator. In a minor related change, the :keyword:`in` operator now works on dictionaries, so ``key in dict`` is now equivalent to ``dict.has_key(key)``. @@ -580,7 +580,7 @@ allowed inside the :keyword:`!try` block of a :keyword:`try`...\ :keyword:`finally` statement; read :pep:`255` for a full explanation of the interaction between :keyword:`!yield` and exceptions.) -Here's a sample usage of the :func:`generate_ints` generator:: +Here's a sample usage of the :func:`!generate_ints` generator:: >>> gen = generate_ints(3) >>> gen @@ -641,7 +641,7 @@ like:: sentence := "Store it in the neighboring harbor" if (i := find("or", sentence)) > 5 then write(i) -In Icon the :func:`find` function returns the indexes at which the substring +In Icon the :func:`!find` function returns the indexes at which the substring "or" is found: 3, 23, 33. In the :keyword:`if` statement, ``i`` is first assigned a value of 3, but 3 is less than 5, so the comparison fails, and Icon retries it with the second value of 23. 23 is greater than 5, so the comparison @@ -671,7 +671,7 @@ PEP 237: Unifying Long Integers and Integers In recent versions, the distinction between regular integers, which are 32-bit values on most machines, and long integers, which can be of arbitrary size, was becoming an annoyance. For example, on platforms that support files larger than -``2**32`` bytes, the :meth:`tell` method of file objects has to return a long +``2**32`` bytes, the :meth:`!tell` method of file objects has to return a long integer. However, there were various bits of Python that expected plain integers and would raise an error if a long integer was provided instead. For example, in Python 1.5, only regular integers could be used as a slice index, and @@ -752,7 +752,7 @@ Here are the changes 2.2 introduces: 0.5. Without the ``__future__`` statement, ``/`` still means classic division. The default meaning of ``/`` will not change until Python 3.0. -* Classes can define methods called :meth:`__truediv__` and :meth:`__floordiv__` +* Classes can define methods called :meth:`~object.__truediv__` and :meth:`~object.__floordiv__` to overload the two division operators. At the C level, there are also slots in the :c:type:`PyNumberMethods` structure so extension types can define the two operators. @@ -785,17 +785,17 @@ support.) When built to use UCS-4 (a "wide Python"), the interpreter can natively handle Unicode characters from U+000000 to U+110000, so the range of legal values for -the :func:`unichr` function is expanded accordingly. Using an interpreter +the :func:`!unichr` function is expanded accordingly. Using an interpreter compiled to use UCS-2 (a "narrow Python"), values greater than 65535 will still -cause :func:`unichr` to raise a :exc:`ValueError` exception. This is all +cause :func:`!unichr` to raise a :exc:`ValueError` exception. This is all described in :pep:`261`, "Support for 'wide' Unicode characters"; consult it for further details. Another change is simpler to explain. Since their introduction, Unicode strings -have supported an :meth:`encode` method to convert the string to a selected +have supported an :meth:`!encode` method to convert the string to a selected encoding such as UTF-8 or Latin-1. A symmetric ``decode([*encoding*])`` method has been added to 8-bit strings (though not to Unicode strings) in 2.2. -:meth:`decode` assumes that the string is in the specified encoding and decodes +:meth:`!decode` assumes that the string is in the specified encoding and decodes it, returning whatever is returned by the codec. Using this new feature, codecs have been added for tasks not directly related to @@ -819,10 +819,10 @@ encoding, and compression with the :mod:`zlib` module:: >>> "sheesh".encode('rot-13') 'furrfu' -To convert a class instance to Unicode, a :meth:`__unicode__` method can be -defined by a class, analogous to :meth:`__str__`. +To convert a class instance to Unicode, a :meth:`!__unicode__` method can be +defined by a class, analogous to :meth:`!__str__`. -:meth:`encode`, :meth:`decode`, and :meth:`__unicode__` were implemented by +:meth:`!encode`, :meth:`!decode`, and :meth:`!__unicode__` were implemented by Marc-André Lemburg. The changes to support using UCS-4 internally were implemented by Fredrik Lundh and Martin von Löwis. @@ -859,7 +859,7 @@ doesn't work:: return g(value-1) + 1 ... -The function :func:`g` will always raise a :exc:`NameError` exception, because +The function :func:`!g` will always raise a :exc:`NameError` exception, because the binding of the name ``g`` isn't in either its local namespace or in the module-level namespace. This isn't much of a problem in practice (how often do you recursively define interior functions like this?), but this also made using @@ -915,7 +915,7 @@ To make the preceding explanation a bit clearer, here's an example:: Line 4 containing the ``exec`` statement is a syntax error, since ``exec`` would define a new local variable named ``x`` whose value should -be accessed by :func:`g`. +be accessed by :func:`!g`. This shouldn't be much of a limitation, since ``exec`` is rarely used in most Python code (and when it is used, it's often a sign of a poor design @@ -933,7 +933,7 @@ anyway). New and Improved Modules ======================== -* The :mod:`xmlrpclib` module was contributed to the standard library by Fredrik +* The :mod:`xmlrpclib <xmlrpc.client>` module was contributed to the standard library by Fredrik Lundh, providing support for writing XML-RPC clients. XML-RPC is a simple remote procedure call protocol built on top of HTTP and XML. For example, the following snippet retrieves a list of RSS channels from the O'Reilly Network, @@ -956,7 +956,7 @@ New and Improved Modules # 'description': 'A utility which converts HTML to XSL FO.', # 'title': 'html2fo 0.3 (Default)'}, ... ] - The :mod:`SimpleXMLRPCServer` module makes it easy to create straightforward + The :mod:`SimpleXMLRPCServer <xmlrpc.server>` module makes it easy to create straightforward XML-RPC servers. See http://xmlrpc.scripting.com/ for more information about XML-RPC. * The new :mod:`hmac` module implements the HMAC algorithm described by @@ -964,9 +964,9 @@ New and Improved Modules * Several functions that originally returned lengthy tuples now return pseudo-sequences that still behave like tuples but also have mnemonic attributes such - as memberst_mtime or :attr:`tm_year`. The enhanced functions include - :func:`stat`, :func:`fstat`, :func:`statvfs`, and :func:`fstatvfs` in the - :mod:`os` module, and :func:`localtime`, :func:`gmtime`, and :func:`strptime` in + as :attr:`!memberst_mtime` or :attr:`~time.struct_time.tm_year`. The enhanced functions include + :func:`~os.stat`, :func:`~os.fstat`, :func:`~os.statvfs`, and :func:`~os.fstatvfs` in the + :mod:`os` module, and :func:`~time.localtime`, :func:`~time.gmtime`, and :func:`~time.strptime` in the :mod:`time` module. For example, to obtain a file's size using the old tuples, you'd end up writing @@ -999,7 +999,7 @@ New and Improved Modules underlying the :mod:`re` module. For example, the :func:`re.sub` and :func:`re.split` functions have been rewritten in C. Another contributed patch speeds up certain Unicode character ranges by a factor of two, and a new - :meth:`finditer` method that returns an iterator over all the non-overlapping + :meth:`~re.finditer` method that returns an iterator over all the non-overlapping matches in a given string. (SRE is maintained by Fredrik Lundh. The BIGCHARSET patch was contributed by Martin von Löwis.) @@ -1012,33 +1012,33 @@ New and Improved Modules new extensions: the NAMESPACE extension defined in :rfc:`2342`, SORT, GETACL and SETACL. (Contributed by Anthony Baxter and Michel Pelletier.) -* The :mod:`rfc822` module's parsing of email addresses is now compliant with +* The :mod:`!rfc822` module's parsing of email addresses is now compliant with :rfc:`2822`, an update to :rfc:`822`. (The module's name is *not* going to be changed to ``rfc2822``.) A new package, :mod:`email`, has also been added for parsing and generating e-mail messages. (Contributed by Barry Warsaw, and arising out of his work on Mailman.) -* The :mod:`difflib` module now contains a new :class:`Differ` class for +* The :mod:`difflib` module now contains a new :class:`!Differ` class for producing human-readable lists of changes (a "delta") between two sequences of - lines of text. There are also two generator functions, :func:`ndiff` and - :func:`restore`, which respectively return a delta from two sequences, or one of + lines of text. There are also two generator functions, :func:`!ndiff` and + :func:`!restore`, which respectively return a delta from two sequences, or one of the original sequences from a delta. (Grunt work contributed by David Goodger, from ndiff.py code by Tim Peters who then did the generatorization.) -* New constants :const:`ascii_letters`, :const:`ascii_lowercase`, and - :const:`ascii_uppercase` were added to the :mod:`string` module. There were - several modules in the standard library that used :const:`string.letters` to +* New constants :const:`!ascii_letters`, :const:`!ascii_lowercase`, and + :const:`!ascii_uppercase` were added to the :mod:`string` module. There were + several modules in the standard library that used :const:`!string.letters` to mean the ranges A-Za-z, but that assumption is incorrect when locales are in - use, because :const:`string.letters` varies depending on the set of legal + use, because :const:`!string.letters` varies depending on the set of legal characters defined by the current locale. The buggy modules have all been fixed - to use :const:`ascii_letters` instead. (Reported by an unknown person; fixed by + to use :const:`!ascii_letters` instead. (Reported by an unknown person; fixed by Fred L. Drake, Jr.) * The :mod:`mimetypes` module now makes it easier to use alternative MIME-type - databases by the addition of a :class:`MimeTypes` class, which takes a list of + databases by the addition of a :class:`~mimetypes.MimeTypes` class, which takes a list of filenames to be parsed. (Contributed by Fred L. Drake, Jr.) -* A :class:`Timer` class was added to the :mod:`threading` module that allows +* A :class:`~threading.Timer` class was added to the :mod:`threading` module that allows scheduling an activity to happen at some future time. (Contributed by Itamar Shtull-Trauring.) @@ -1114,7 +1114,7 @@ code, none of the changes described here will affect you very much. * Two new wrapper functions, :c:func:`PyOS_snprintf` and :c:func:`PyOS_vsnprintf` were added to provide cross-platform implementations for the relatively new :c:func:`snprintf` and :c:func:`vsnprintf` C lib APIs. In contrast to the standard - :c:func:`sprintf` and :c:func:`vsprintf` functions, the Python versions check the + :c:func:`sprintf` and :c:func:`!vsprintf` functions, the Python versions check the bounds of the buffer used to protect against buffer overruns. (Contributed by M.-A. Lemburg.) @@ -1212,12 +1212,12 @@ Some of the more notable changes are: * The :file:`Tools/scripts/ftpmirror.py` script now parses a :file:`.netrc` file, if you have one. (Contributed by Mike Romberg.) -* Some features of the object returned by the :func:`xrange` function are now +* Some features of the object returned by the :func:`!xrange` function are now deprecated, and trigger warnings when they're accessed; they'll disappear in - Python 2.3. :class:`xrange` objects tried to pretend they were full sequence + Python 2.3. :class:`!xrange` objects tried to pretend they were full sequence types by supporting slicing, sequence multiplication, and the :keyword:`in` operator, but these features were rarely used and therefore buggy. The - :meth:`tolist` method and the :attr:`start`, :attr:`stop`, and :attr:`step` + :meth:`!tolist` method and the :attr:`!start`, :attr:`!stop`, and :attr:`!step` attributes are also being deprecated. At the C level, the fourth argument to the :c:func:`!PyRange_New` function, ``repeat``, has also been deprecated. diff --git a/Doc/whatsnew/2.3.rst b/Doc/whatsnew/2.3.rst index 8ebcbfaf248551..37cd41add8132c 100644 --- a/Doc/whatsnew/2.3.rst +++ b/Doc/whatsnew/2.3.rst @@ -1196,7 +1196,7 @@ Optimizations * Multiplication of large long integers is now much faster thanks to an implementation of Karatsuba multiplication, an algorithm that scales better than - the O(n\*n) required for the grade-school multiplication algorithm. (Original + the *O*\ (*n*\ :sup:`2`) required for the grade-school multiplication algorithm. (Original patch by Christopher A. Craig, and significantly reworked by Tim Peters.) * The ``SET_LINENO`` opcode is now gone. This may provide a small speed @@ -1308,7 +1308,7 @@ complete list of changes, or look through the CVS logs for all the details. partially sorted order such that, for every index *k*, ``heap[k] <= heap[2*k+1]`` and ``heap[k] <= heap[2*k+2]``. This makes it quick to remove the smallest item, and inserting a new item while maintaining the heap property is - O(lg n). (See https://xlinux.nist.gov/dads//HTML/priorityque.html for more + *O*\ (log *n*). (See https://xlinux.nist.gov/dads//HTML/priorityque.html for more information about the priority queue data structure.) The :mod:`heapq` module provides :func:`~heapq.heappush` and :func:`~heapq.heappop` functions diff --git a/Doc/whatsnew/2.4.rst b/Doc/whatsnew/2.4.rst index bc748dd44f5f8e..7e235d4370edaa 100644 --- a/Doc/whatsnew/2.4.rst +++ b/Doc/whatsnew/2.4.rst @@ -387,13 +387,13 @@ The standard library provides a number of ways to execute a subprocess, offering different features and different levels of complexity. ``os.system(command)`` is easy to use, but slow (it runs a shell process which executes the command) and dangerous (you have to be careful about escaping -the shell's metacharacters). The :mod:`popen2` module offers classes that can +the shell's metacharacters). The :mod:`!popen2` module offers classes that can capture standard output and standard error from the subprocess, but the naming is confusing. The :mod:`subprocess` module cleans this up, providing a unified interface that offers all the features you might need. -Instead of :mod:`popen2`'s collection of classes, :mod:`subprocess` contains a -single class called :class:`Popen` whose constructor supports a number of +Instead of :mod:`!popen2`'s collection of classes, :mod:`subprocess` contains a +single class called :class:`subprocess.Popen` whose constructor supports a number of different keyword arguments. :: class Popen(args, bufsize=0, executable=None, @@ -995,7 +995,7 @@ fixes. Here's a partial list of the most notable changes, sorted alphabetically by module name. Consult the :file:`Misc/NEWS` file in the source tree for a more complete list of changes, or look through the CVS logs for all the details. -* The :mod:`asyncore` module's :func:`loop` function now has a *count* parameter +* The :mod:`!asyncore` module's :func:`!loop` function now has a *count* parameter that lets you perform a limited number of passes through the polling loop. The default is still to loop forever. @@ -1052,9 +1052,9 @@ complete list of changes, or look through the CVS logs for all the details. advantage of :class:`collections.deque` for improved performance. (Contributed by Raymond Hettinger.) -* The :mod:`ConfigParser` classes have been enhanced slightly. The :meth:`read` +* The :mod:`ConfigParser <configparser>` classes have been enhanced slightly. The :meth:`~configparser.ConfigParser.read` method now returns a list of the files that were successfully parsed, and the - :meth:`set` method raises :exc:`TypeError` if passed a *value* argument that + :meth:`~configparser.ConfigParser.set` method raises :exc:`TypeError` if passed a *value* argument that isn't a string. (Contributed by John Belmonte and David Goodger.) * The :mod:`curses` module now supports the ncurses extension @@ -1081,7 +1081,7 @@ complete list of changes, or look through the CVS logs for all the details. :func:`nsmallest` that use heaps to find the N largest or smallest values in a dataset without the expense of a full sort. (Contributed by Raymond Hettinger.) -* The :mod:`httplib` module now contains constants for HTTP status codes defined +* The :mod:`httplib <http>` module now contains constants for HTTP status codes defined in various HTTP-related RFC documents. Constants have names such as :const:`OK`, :const:`CREATED`, :const:`CONTINUE`, and :const:`MOVED_PERMANENTLY`; use pydoc to get a full list. (Contributed by @@ -1218,10 +1218,10 @@ complete list of changes, or look through the CVS logs for all the details. now include the string ``'%default'``, which will be replaced by the option's default value. (Contributed by Greg Ward.) -* The long-term plan is to deprecate the :mod:`rfc822` module in some future +* The long-term plan is to deprecate the :mod:`!rfc822` module in some future Python release in favor of the :mod:`email` package. To this end, the - :func:`email.Utils.formatdate` function has been changed to make it usable as a - replacement for :func:`rfc822.formatdate`. You may want to write new e-mail + :func:`email.Utils.formatdate <email.utils.formatdate>` function has been changed to make it usable as a + replacement for :func:`!rfc822.formatdate`. You may want to write new e-mail processing code with this in mind. (Change implemented by Anthony Baxter.) * A new ``urandom(n)`` function was added to the :mod:`os` module, returning @@ -1308,7 +1308,7 @@ complete list of changes, or look through the CVS logs for all the details. sockets, and regular expression pattern objects. (Contributed by Raymond Hettinger.) -* The :mod:`xmlrpclib` module now supports a multi-call extension for +* The :mod:`xmlrpclib <xmlrpc.client>` module now supports a multi-call extension for transmitting multiple XML-RPC calls in a single HTTP operation. (Contributed by Brian Quinlan.) @@ -1323,8 +1323,8 @@ complete list of changes, or look through the CVS logs for all the details. cookielib --------- -The :mod:`cookielib` library supports client-side handling for HTTP cookies, -mirroring the :mod:`Cookie` module's server-side cookie support. Cookies are +The :mod:`cookielib <http.cookiejar>` library supports client-side handling for HTTP cookies, +mirroring the :mod:`Cookie <http.cookies>` module's server-side cookie support. Cookies are stored in cookie jars; the library transparently stores cookies offered by the web server in the cookie jar, and fetches the cookie from the jar when connecting to the server. As in web browsers, policy objects control whether @@ -1335,7 +1335,7 @@ are provided: one that stores cookies in the Netscape format so applications can use the Mozilla or Lynx cookie files, and one that stores cookies in the same format as the Perl libwww library. -:mod:`urllib2` has been changed to interact with :mod:`cookielib`: +:mod:`urllib2 <urllib.request>` has been changed to interact with :mod:`cookielib <http.cookiejar>`: :class:`HTTPCookieProcessor` manages a cookie jar that is used when accessing URLs. @@ -1529,7 +1529,7 @@ code: will now always be unequal, and relative comparisons (``<``, ``>``) will raise a :exc:`TypeError`. -* :func:`dircache.listdir` now passes exceptions to the caller instead of +* :func:`!dircache.listdir` now passes exceptions to the caller instead of returning empty lists. * :func:`LexicalHandler.startDTD` used to receive the public and system IDs in diff --git a/Doc/whatsnew/2.5.rst b/Doc/whatsnew/2.5.rst index 627c918dd6d8b4..2ae26e7a106a0b 100644 --- a/Doc/whatsnew/2.5.rst +++ b/Doc/whatsnew/2.5.rst @@ -1478,8 +1478,8 @@ complete list of changes, or look through the SVN logs for all the details. .. Patch 790710 -* The :mod:`pickle` and :mod:`cPickle` modules no longer accept a return value - of ``None`` from the :meth:`__reduce__` method; the method must return a tuple +* The :mod:`pickle` and :mod:`!cPickle` modules no longer accept a return value + of ``None`` from the :meth:`~object.__reduce__` method; the method must return a tuple of arguments instead. The ability to return ``None`` was deprecated in Python 2.4, so this completes the removal of the feature. @@ -1519,7 +1519,7 @@ complete list of changes, or look through the SVN logs for all the details. .. Patch #1472854 -* The :mod:`SimpleXMLRPCServer` and :mod:`DocXMLRPCServer` classes now have a +* The :mod:`SimpleXMLRPCServer <xmlrpc.server>` and :mod:`DocXMLRPCServer <xmlrpc.server>` classes now have a :attr:`rpc_paths` attribute that constrains XML-RPC operations to a limited set of URL paths; the default is to allow only ``'/'`` and ``'/RPC2'``. Setting :attr:`rpc_paths` to ``None`` or an empty tuple disables this path checking. @@ -1650,9 +1650,9 @@ complete list of changes, or look through the SVN logs for all the details. .. Patch #754022 -* The :mod:`xmlrpclib` module now supports returning :class:`~datetime.datetime` objects - for the XML-RPC date type. Supply ``use_datetime=True`` to the :func:`loads` - function or the :class:`Unmarshaller` class to enable this feature. (Contributed +* The :mod:`xmlrpclib <xmlrpc.client>` module now supports returning :class:`~datetime.datetime` objects + for the XML-RPC date type. Supply ``use_datetime=True`` to the :func:`~xmlrpc.client.loads` + function or the :class:`!Unmarshaller` class to enable this feature. (Contributed by Skip Montanaro.) .. Patch 1120353 @@ -1680,7 +1680,7 @@ The ctypes package The :mod:`ctypes` package, written by Thomas Heller, has been added to the standard library. :mod:`ctypes` lets you call arbitrary functions in shared -libraries or DLLs. Long-time users may remember the :mod:`dl` module, which +libraries or DLLs. Long-time users may remember the :mod:`!dl` module, which provides functions for loading shared libraries and calling functions in them. The :mod:`ctypes` package is much fancier. @@ -1877,12 +1877,12 @@ The hashlib package ------------------- A new :mod:`hashlib` module, written by Gregory P. Smith, has been added to -replace the :mod:`md5` and :mod:`sha` modules. :mod:`hashlib` adds support for +replace the :mod:`!md5` and :mod:`!sha` modules. :mod:`hashlib` adds support for additional secure hashes (SHA-224, SHA-256, SHA-384, and SHA-512). When available, the module uses OpenSSL for fast platform optimized implementations of algorithms. -The old :mod:`md5` and :mod:`sha` modules still exist as wrappers around hashlib +The old :mod:`!md5` and :mod:`!sha` modules still exist as wrappers around hashlib to preserve backwards compatibility. The new module's interface is very close to that of the old modules, but not identical. The most significant difference is that the constructor functions for creating new hashing objects are named @@ -2253,12 +2253,12 @@ code: appeared. In Python 2.5, the argument must be exactly one %char specifier with no surrounding text. -* Library: The :mod:`pickle` and :mod:`cPickle` modules no longer accept a - return value of ``None`` from the :meth:`__reduce__` method; the method must +* Library: The :mod:`pickle` and :mod:`!cPickle` modules no longer accept a + return value of ``None`` from the :meth:`~object.__reduce__` method; the method must return a tuple of arguments instead. The modules also no longer accept the deprecated *bin* keyword parameter. -* Library: The :mod:`SimpleXMLRPCServer` and :mod:`DocXMLRPCServer` classes now +* Library: The :mod:`SimpleXMLRPCServer <xmlrpc.server>` and :mod:`DocXMLRPCServer <xmlrpc.server>` classes now have a :attr:`rpc_paths` attribute that constrains XML-RPC operations to a limited set of URL paths; the default is to allow only ``'/'`` and ``'/RPC2'``. Setting :attr:`rpc_paths` to ``None`` or an empty tuple disables this path diff --git a/Doc/whatsnew/2.6.rst b/Doc/whatsnew/2.6.rst index e8c1709c42abac..7d3769a22286e2 100644 --- a/Doc/whatsnew/2.6.rst +++ b/Doc/whatsnew/2.6.rst @@ -4,8 +4,6 @@ What's New in Python 2.6 **************************** -.. XXX add trademark info for Apple, Microsoft, SourceForge. - :Author: A.M. Kuchling (amk at amk.ca) .. $Id$ @@ -128,7 +126,7 @@ and to C extension code as :c:data:`!Py_Py3kWarningFlag`. The 3\ *xxx* series of PEPs, which contains proposals for Python 3.0. :pep:`3000` describes the development process for Python 3.0. Start with :pep:`3100` that describes the general goals for Python - 3.0, and then explore the higher-numbered PEPS that propose + 3.0, and then explore the higher-numbered PEPs that propose specific features. @@ -1051,8 +1049,6 @@ the :mod:`io` module: sockets, but Python 2.6 hasn't restructured its file and socket objects in this way. - .. XXX should 2.6 register them in io.py? - * :class:`BufferedIOBase` is an abstract base class that buffers data in memory to reduce the number of system calls used, making I/O processing more efficient. @@ -1086,7 +1082,7 @@ the :mod:`io` module: (In Python 2.6, :class:`io.StringIO` is implemented in pure Python, so it's pretty slow. You should therefore stick with the - existing :mod:`StringIO` module or :mod:`cStringIO` for now. At some + existing :mod:`!StringIO` module or :mod:`!cStringIO` for now. At some point Python 3.0's :mod:`io` module will be rewritten into C for speed, and perhaps the C implementation will be backported to the 2.x releases.) @@ -1133,8 +1129,6 @@ while an external caller could be modifying the contents, so there's a corresponding ``PyBuffer_Release(Py_buffer *view)`` to indicate that the external caller is done. -.. XXX PyObject_GetBuffer not documented in c-api - The *flags* argument to :c:func:`PyObject_GetBuffer` specifies constraints upon the memory returned. Some examples are: @@ -1789,7 +1783,7 @@ changes, sorted alphabetically by module name. Consult the :file:`Misc/NEWS` file in the source tree for a more complete list of changes, or look through the Subversion logs for all the details. -* The :mod:`asyncore` and :mod:`asynchat` modules are +* The :mod:`!asyncore` and :mod:`!asynchat` modules are being actively maintained again, and a number of patches and bugfixes were applied. (Maintained by Josiah Carlson; see :issue:`1736190` for one patch.) @@ -1813,7 +1807,7 @@ changes, or look through the Subversion logs for all the details. Nubis; :issue:`1817`.) The :func:`parse_qs` and :func:`parse_qsl` functions have been - relocated from the :mod:`!cgi` module to the :mod:`urlparse` module. + relocated from the :mod:`!cgi` module to the :mod:`urlparse <urllib.parse>` module. The versions still available in the :mod:`!cgi` module will trigger :exc:`PendingDeprecationWarning` messages in 2.6 (:issue:`600362`). @@ -1901,8 +1895,8 @@ changes, or look through the Subversion logs for all the details. (Contributed by Raymond Hettinger.) -* The :mod:`Cookie` module's :class:`Morsel` objects now support an - :attr:`httponly` attribute. In some browsers. cookies with this attribute +* The :mod:`Cookie <http.cookies>` module's :class:`~http.cookies.Morsel` objects now support an + :attr:`~http.cookies.Morsel.httponly` attribute. In some browsers. cookies with this attribute set cannot be accessed or manipulated by JavaScript code. (Contributed by Arvin Schnell; :issue:`1638033`.) @@ -1993,8 +1987,8 @@ changes, or look through the Subversion logs for all the details. (Contributed by Raymond Hettinger.) * An optional ``timeout`` parameter, specifying a timeout measured in - seconds, was added to the :class:`httplib.HTTPConnection` and - :class:`HTTPSConnection` class constructors. (Added by Facundo + seconds, was added to the :class:`httplib.HTTPConnection <http.client.HTTPConnection>` and + :class:`HTTPSConnection <http.client.HTTPSConnection>` class constructors. (Added by Facundo Batista.) * Most of the :mod:`inspect` module's functions, such as @@ -2377,10 +2371,10 @@ changes, or look through the Subversion logs for all the details. ``socket(socket.AF_INET, ...)`` may be all that's required to make your code work with IPv6. -* The base classes in the :mod:`SocketServer` module now support - calling a :meth:`handle_timeout` method after a span of inactivity - specified by the server's :attr:`timeout` attribute. (Contributed - by Michael Pomraning.) The :meth:`serve_forever` method +* The base classes in the :mod:`SocketServer <socketserver>` module now support + calling a :meth:`~socketserver.BaseServer.handle_timeout` method after a span of inactivity + specified by the server's :attr:`~socketserver.BaseServer.timeout` attribute. (Contributed + by Michael Pomraning.) The :meth:`~socketserver.BaseServer.serve_forever` method now takes an optional poll interval measured in seconds, controlling how often the server will check for a shutdown request. (Contributed by Pedro Werneck and Jeffrey Yasskin; @@ -2484,9 +2478,9 @@ changes, or look through the Subversion logs for all the details. ``with tempfile.NamedTemporaryFile() as tmp: ...``. (Contributed by Alexander Belopolsky; :issue:`2021`.) -* The :mod:`test.test_support` module gained a number +* The :mod:`test.test_support <test.support>` module gained a number of context managers useful for writing tests. - :func:`EnvironmentVarGuard` is a + :func:`~test.support.os_helper.EnvironmentVarGuard` is a context manager that temporarily changes environment variables and automatically restores them to their old values. @@ -2583,9 +2577,9 @@ changes, or look through the Subversion logs for all the details. (:issue:`1513695`) * An optional ``timeout`` parameter was added to the - :func:`urllib.urlopen` function and the + :func:`urllib.urlopen <urllib.request.urlopen>` function and the :class:`urllib.ftpwrapper` class constructor, as well as the - :func:`urllib2.urlopen` function. The parameter specifies a timeout + :func:`urllib2.urlopen <urllib.request.urlopen>` function. The parameter specifies a timeout measured in seconds. For example:: >>> u = urllib2.urlopen("http://slow.example.com", @@ -2610,7 +2604,7 @@ changes, or look through the Subversion logs for all the details. intended for testing purposes that lets you temporarily modify the warning filters and then restore their original values (:issue:`3781`). -* The XML-RPC :class:`SimpleXMLRPCServer` and :class:`DocXMLRPCServer` +* The XML-RPC :class:`SimpleXMLRPCServer <xmlrpc.server>` and :class:`DocXMLRPCServer <xmlrpc.server>` classes can now be prevented from immediately opening and binding to their socket by passing ``False`` as the *bind_and_activate* constructor parameter. This can be used to modify the instance's @@ -2627,11 +2621,11 @@ changes, or look through the Subversion logs for all the details. information. (Contributed by Alan McIntyre as part of his project for Google's Summer of Code 2007.) -* The :mod:`xmlrpclib` module no longer automatically converts +* The :mod:`xmlrpclib <xmlrpc.client>` module no longer automatically converts :class:`datetime.date` and :class:`datetime.time` to the - :class:`xmlrpclib.DateTime` type; the conversion semantics were + :class:`xmlrpclib.DateTime <xmlrpc.client.DateTime>` type; the conversion semantics were not necessarily correct for all applications. Code using - :mod:`xmlrpclib` should convert :class:`date` and :class:`~datetime.time` + :mod:`!xmlrpclib` should convert :class:`date` and :class:`~datetime.time` instances. (:issue:`1330538`) The code can also handle dates before 1900 (contributed by Ralf Schmitt; :issue:`2014`) and 64-bit integers represented by using ``<i8>`` in XML-RPC responses @@ -2916,8 +2910,8 @@ Deprecations and Removals * Changes to the :class:`Exception` interface as dictated by :pep:`352` continue to be made. For 2.6, - the :attr:`message` attribute is being deprecated in favor of the - :attr:`args` attribute. + the :attr:`!message` attribute is being deprecated in favor of the + :attr:`~BaseException.args` attribute. * (3.0-warning mode) Python 3.0 will feature a reorganized standard library that will drop many outdated modules and rename others. @@ -2925,51 +2919,51 @@ Deprecations and Removals when they are imported. The list of deprecated modules is: - :mod:`audiodev`, - :mod:`bgenlocations`, - :mod:`buildtools`, - :mod:`bundlebuilder`, - :mod:`Canvas`, - :mod:`compiler`, - :mod:`dircache`, - :mod:`dl`, - :mod:`fpformat`, - :mod:`gensuitemodule`, - :mod:`ihooks`, - :mod:`imageop`, - :mod:`imgfile`, - :mod:`linuxaudiodev`, - :mod:`mhlib`, - :mod:`mimetools`, - :mod:`multifile`, - :mod:`new`, - :mod:`pure`, - :mod:`statvfs`, - :mod:`sunaudiodev`, - :mod:`test.testall`, and - :mod:`toaiff`. - -* The :mod:`gopherlib` module has been removed. - -* The :mod:`MimeWriter` module and :mod:`mimify` module + :mod:`!audiodev`, + :mod:`!bgenlocations`, + :mod:`!buildtools`, + :mod:`!bundlebuilder`, + :mod:`!Canvas`, + :mod:`!compiler`, + :mod:`!dircache`, + :mod:`!dl`, + :mod:`!fpformat`, + :mod:`!gensuitemodule`, + :mod:`!ihooks`, + :mod:`!imageop`, + :mod:`!imgfile`, + :mod:`!linuxaudiodev`, + :mod:`!mhlib`, + :mod:`!mimetools`, + :mod:`!multifile`, + :mod:`!new`, + :mod:`!pure`, + :mod:`!statvfs`, + :mod:`!sunaudiodev`, + :mod:`!test.testall`, and + :mod:`!toaiff`. + +* The :mod:`!gopherlib` module has been removed. + +* The :mod:`!MimeWriter` module and :mod:`!mimify` module have been deprecated; use the :mod:`email` package instead. -* The :mod:`md5` module has been deprecated; use the :mod:`hashlib` module +* The :mod:`!md5` module has been deprecated; use the :mod:`hashlib` module instead. -* The :mod:`posixfile` module has been deprecated; :func:`fcntl.lockf` +* The :mod:`!posixfile` module has been deprecated; :func:`fcntl.lockf` provides better locking. -* The :mod:`popen2` module has been deprecated; use the :mod:`subprocess` +* The :mod:`!popen2` module has been deprecated; use the :mod:`subprocess` module. -* The :mod:`rgbimg` module has been removed. +* The :mod:`!rgbimg` module has been removed. -* The :mod:`sets` module has been deprecated; it's better to +* The :mod:`!sets` module has been deprecated; it's better to use the built-in :class:`set` and :class:`frozenset` types. -* The :mod:`sha` module has been deprecated; use the :mod:`hashlib` module +* The :mod:`!sha` module has been deprecated; use the :mod:`hashlib` module instead. @@ -3110,8 +3104,8 @@ Port-Specific Changes: Windows * The :mod:`msvcrt` module now supports both the normal and wide char variants of the console I/O - API. The :func:`getwch` function reads a keypress and returns a Unicode - value, as does the :func:`getwche` function. The :func:`putwch` function + API. The :func:`~msvcrt.getwch` function reads a keypress and returns a Unicode + value, as does the :func:`~msvcrt.getwche` function. The :func:`~msvcrt.putwch` function takes a Unicode character and writes it to the console. (Contributed by Christian Heimes.) @@ -3120,24 +3114,24 @@ Port-Specific Changes: Windows directory path. (Contributed by Josiah Carlson; :issue:`957650`.) * The :mod:`socket` module's socket objects now have an - :meth:`ioctl` method that provides a limited interface to the + :meth:`~socket.socket.ioctl` method that provides a limited interface to the :c:func:`WSAIoctl` system interface. -* The :mod:`_winreg` module now has a function, - :func:`ExpandEnvironmentStrings`, +* The :mod:`_winreg <winreg>` module now has a function, + :func:`~winreg.ExpandEnvironmentStrings`, that expands environment variable references such as ``%NAME%`` in an input string. The handle objects provided by this module now support the context protocol, so they can be used in :keyword:`with` statements. (Contributed by Christian Heimes.) - :mod:`_winreg` also has better support for x64 systems, - exposing the :func:`DisableReflectionKey`, :func:`EnableReflectionKey`, - and :func:`QueryReflectionKey` functions, which enable and disable + :mod:`_winreg <winreg>` also has better support for x64 systems, + exposing the :func:`~winreg.DisableReflectionKey`, :func:`~winreg.EnableReflectionKey`, + and :func:`~winreg.QueryReflectionKey` functions, which enable and disable registry reflection for 32-bit processes running on 64-bit systems. (:issue:`1753245`) -* The :mod:`!msilib` module's :class:`Record` object - gained :meth:`GetInteger` and :meth:`GetString` methods that +* The :mod:`!msilib` module's :class:`!Record` object + gained :meth:`!GetInteger` and :meth:`!GetString` methods that return field values as an integer or a string. (Contributed by Floris Bruynooghe; :issue:`2125`.) @@ -3151,49 +3145,49 @@ Port-Specific Changes: Mac OS X :option:`!--with-framework-name=` option to the :program:`configure` script. -* The :mod:`macfs` module has been removed. This in turn required the - :func:`macostools.touched` function to be removed because it depended on the - :mod:`macfs` module. (:issue:`1490190`) +* The :mod:`!macfs` module has been removed. This in turn required the + :func:`!macostools.touched` function to be removed because it depended on the + :mod:`!macfs` module. (:issue:`1490190`) * Many other Mac OS modules have been deprecated and will be removed in Python 3.0: - :mod:`_builtinSuites`, - :mod:`aepack`, - :mod:`aetools`, - :mod:`aetypes`, - :mod:`applesingle`, - :mod:`appletrawmain`, - :mod:`appletrunner`, - :mod:`argvemulator`, - :mod:`Audio_mac`, - :mod:`autoGIL`, - :mod:`Carbon`, - :mod:`cfmfile`, - :mod:`CodeWarrior`, - :mod:`ColorPicker`, - :mod:`EasyDialogs`, - :mod:`Explorer`, - :mod:`Finder`, - :mod:`FrameWork`, - :mod:`findertools`, - :mod:`ic`, - :mod:`icglue`, - :mod:`icopen`, - :mod:`macerrors`, - :mod:`MacOS`, - :mod:`macfs`, - :mod:`macostools`, - :mod:`macresource`, - :mod:`MiniAEFrame`, - :mod:`Nav`, - :mod:`Netscape`, - :mod:`OSATerminology`, - :mod:`pimp`, - :mod:`PixMapWrapper`, - :mod:`StdSuites`, - :mod:`SystemEvents`, - :mod:`Terminal`, and - :mod:`terminalcommand`. + :mod:`!_builtinSuites`, + :mod:`!aepack`, + :mod:`!aetools`, + :mod:`!aetypes`, + :mod:`!applesingle`, + :mod:`!appletrawmain`, + :mod:`!appletrunner`, + :mod:`!argvemulator`, + :mod:`!Audio_mac`, + :mod:`!autoGIL`, + :mod:`!Carbon`, + :mod:`!cfmfile`, + :mod:`!CodeWarrior`, + :mod:`!ColorPicker`, + :mod:`!EasyDialogs`, + :mod:`!Explorer`, + :mod:`!Finder`, + :mod:`!FrameWork`, + :mod:`!findertools`, + :mod:`!ic`, + :mod:`!icglue`, + :mod:`!icopen`, + :mod:`!macerrors`, + :mod:`!MacOS`, + :mod:`!macfs`, + :mod:`!macostools`, + :mod:`!macresource`, + :mod:`!MiniAEFrame`, + :mod:`!Nav`, + :mod:`!Netscape`, + :mod:`!OSATerminology`, + :mod:`!pimp`, + :mod:`!PixMapWrapper`, + :mod:`!StdSuites`, + :mod:`!SystemEvents`, + :mod:`!Terminal`, and + :mod:`!terminalcommand`. .. ====================================================================== @@ -3202,29 +3196,29 @@ Port-Specific Changes: IRIX A number of old IRIX-specific modules were deprecated and will be removed in Python 3.0: -:mod:`al` and :mod:`AL`, -:mod:`cd`, -:mod:`cddb`, -:mod:`cdplayer`, -:mod:`CL` and :mod:`cl`, -:mod:`DEVICE`, -:mod:`ERRNO`, -:mod:`FILE`, -:mod:`FL` and :mod:`fl`, -:mod:`flp`, -:mod:`fm`, -:mod:`GET`, -:mod:`GLWS`, -:mod:`GL` and :mod:`gl`, -:mod:`IN`, -:mod:`IOCTL`, -:mod:`jpeg`, -:mod:`panelparser`, -:mod:`readcd`, -:mod:`SV` and :mod:`sv`, -:mod:`torgb`, -:mod:`videoreader`, and -:mod:`WAIT`. +:mod:`!al` and :mod:`!AL`, +:mod:`!cd`, +:mod:`!cddb`, +:mod:`!cdplayer`, +:mod:`!CL` and :mod:`!cl`, +:mod:`!DEVICE`, +:mod:`!ERRNO`, +:mod:`!FILE`, +:mod:`!FL` and :mod:`!fl`, +:mod:`!flp`, +:mod:`!fm`, +:mod:`!GET`, +:mod:`!GLWS`, +:mod:`!GL` and :mod:`!gl`, +:mod:`!IN`, +:mod:`!IOCTL`, +:mod:`!jpeg`, +:mod:`!panelparser`, +:mod:`!readcd`, +:mod:`!SV` and :mod:`!sv`, +:mod:`!torgb`, +:mod:`!videoreader`, and +:mod:`!WAIT`. .. ====================================================================== @@ -3280,11 +3274,11 @@ that may require changes to your code: :exc:`StandardError` but now it is, through :exc:`IOError`. (Implemented by Gregory P. Smith; :issue:`1706815`.) -* The :mod:`xmlrpclib` module no longer automatically converts +* The :mod:`xmlrpclib <xmlrpc.client>` module no longer automatically converts :class:`datetime.date` and :class:`datetime.time` to the - :class:`xmlrpclib.DateTime` type; the conversion semantics were + :class:`xmlrpclib.DateTime <xmlrpc.client.DateTime>` type; the conversion semantics were not necessarily correct for all applications. Code using - :mod:`xmlrpclib` should convert :class:`date` and :class:`~datetime.time` + :mod:`!xmlrpclib` should convert :class:`date` and :class:`~datetime.time` instances. (:issue:`1330538`) * (3.0-warning mode) The :class:`Exception` class now warns diff --git a/Doc/whatsnew/2.7.rst b/Doc/whatsnew/2.7.rst index 5af700bd5d3506..ada05aa22b46f6 100644 --- a/Doc/whatsnew/2.7.rst +++ b/Doc/whatsnew/2.7.rst @@ -282,12 +282,12 @@ How does the :class:`~collections.OrderedDict` work? It maintains a doubly linked list of keys, appending new keys to the list as they're inserted. A secondary dictionary maps keys to their corresponding list node, so deletion doesn't have to traverse the entire linked list and therefore -remains O(1). +remains *O*\ (1). The standard library now supports use of ordered dictionaries in several modules. -* The :mod:`ConfigParser` module uses them by default, meaning that +* The :mod:`ConfigParser <configparser>` module uses them by default, meaning that configuration files can now be read, modified, and then written back in their original order. @@ -915,7 +915,7 @@ used with the :option:`-W` switch, separated by commas. (Contributed by Brian Curtin; :issue:`7301`.) For example, the following setting will print warnings every time -they occur, but turn warnings from the :mod:`Cookie` module into an +they occur, but turn warnings from the :mod:`Cookie <http.cookies>` module into an error. (The exact syntax for setting an environment variable varies across operating systems and shells.) @@ -1012,12 +1012,12 @@ Several performance enhancements have been added: scan. This is sometimes faster by a factor of 10. (Added by Florent Xicluna; :issue:`7462` and :issue:`7622`.) -* The :mod:`pickle` and :mod:`cPickle` modules now automatically +* The :mod:`pickle` and :mod:`!cPickle` modules now automatically intern the strings used for attribute names, reducing memory usage of the objects resulting from unpickling. (Contributed by Jake McGuire; :issue:`5084`.) -* The :mod:`cPickle` module now special-cases dictionaries, +* The :mod:`!cPickle` module now special-cases dictionaries, nearly halving the time required to pickle them. (Contributed by Collin Winter; :issue:`5670`.) @@ -1134,7 +1134,7 @@ changes, or look through the Subversion logs for all the details. another type that isn't a :class:`Mapping`. (Fixed by Daniel Stutzbach; :issue:`8729`.) -* Constructors for the parsing classes in the :mod:`ConfigParser` module now +* Constructors for the parsing classes in the :mod:`ConfigParser <configparser>` module now take an *allow_no_value* parameter, defaulting to false; if true, options without values will be allowed. For example:: @@ -1163,7 +1163,7 @@ changes, or look through the Subversion logs for all the details. statement, has been deprecated, because the :keyword:`!with` statement now supports multiple context managers. -* The :mod:`cookielib` module now ignores cookies that have an invalid +* The :mod:`cookielib <http.cookiejar>` module now ignores cookies that have an invalid version field, one that doesn't contain an integer value. (Fixed by John J. Lee; :issue:`3924`.) @@ -1306,18 +1306,18 @@ changes, or look through the Subversion logs for all the details. ``('md5', 'sha1', 'sha224', 'sha256', 'sha384', 'sha512')``. (Contributed by Carl Chenet; :issue:`7418`.) -* The default :class:`~httplib.HTTPResponse` class used by the :mod:`httplib` module now +* The default :class:`~http.client.HTTPResponse` class used by the :mod:`httplib <http>` module now supports buffering, resulting in much faster reading of HTTP responses. (Contributed by Kristján Valur Jónsson; :issue:`4879`.) - The :class:`~httplib.HTTPConnection` and :class:`~httplib.HTTPSConnection` classes + The :class:`~http.client.HTTPConnection` and :class:`~http.client.HTTPSConnection` classes now support a *source_address* parameter, a ``(host, port)`` 2-tuple giving the source address that will be used for the connection. (Contributed by Eldon Ziegler; :issue:`3972`.) -* The :mod:`ihooks` module now supports relative imports. Note that - :mod:`ihooks` is an older module for customizing imports, - superseded by the :mod:`imputil` module added in Python 2.0. +* The :mod:`!ihooks` module now supports relative imports. Note that + :mod:`!ihooks` is an older module for customizing imports, + superseded by the :mod:`!imputil` module added in Python 2.0. (Relative import support added by Neil Schemenauer.) .. revision 75423 @@ -1518,16 +1518,16 @@ changes, or look through the Subversion logs for all the details. the :class:`bytearray` and :class:`memoryview` objects. (Implemented by Antoine Pitrou; :issue:`8104`.) -* The :mod:`SocketServer` module's :class:`~SocketServer.TCPServer` class now +* The :mod:`SocketServer <socketserver>` module's :class:`~socketserver.TCPServer` class now supports socket timeouts and disabling the Nagle algorithm. - The :attr:`~SocketServer.TCPServer.disable_nagle_algorithm` class attribute + The :attr:`!disable_nagle_algorithm` class attribute defaults to ``False``; if overridden to be true, new request connections will have the TCP_NODELAY option set to prevent buffering many small sends into a single TCP packet. - The :attr:`~SocketServer.BaseServer.timeout` class attribute can hold + The :attr:`~socketserver.BaseServer.timeout` class attribute can hold a timeout in seconds that will be applied to the request socket; if - no request is received within that time, :meth:`~SocketServer.BaseServer.handle_timeout` - will be called and :meth:`~SocketServer.BaseServer.handle_request` will return. + no request is received within that time, :meth:`~socketserver.BaseServer.handle_timeout` + will be called and :meth:`~socketserver.BaseServer.handle_request` will return. (Contributed by Kristján Valur Jónsson; :issue:`6192` and :issue:`6267`.) * Updated module: the :mod:`sqlite3` module has been updated to @@ -1648,7 +1648,7 @@ changes, or look through the Subversion logs for all the details. and has been updated to version 5.2.0 (updated by Florent Xicluna; :issue:`8024`). -* The :mod:`urlparse` module's :func:`~urlparse.urlsplit` now handles +* The :mod:`urlparse <urllib.parse>` module's :func:`~urllib.parse.urlsplit` now handles unknown URL schemes in a fashion compliant with :rfc:`3986`: if the URL is of the form ``"<something>://..."``, the text before the ``://`` is treated as the scheme, even if it's a made-up scheme that @@ -1675,7 +1675,7 @@ changes, or look through the Subversion logs for all the details. (Python 2.7 actually produces slightly different output, since it returns a named tuple instead of a standard tuple.) - The :mod:`urlparse` module also supports IPv6 literal addresses as defined by + The :mod:`urlparse <urllib.parse>` module also supports IPv6 literal addresses as defined by :rfc:`2732` (contributed by Senthil Kumaran; :issue:`2987`). .. doctest:: @@ -1697,8 +1697,8 @@ changes, or look through the Subversion logs for all the details. or comment (which looks like ``<!-- comment -->``). (Patch by Neil Muller; :issue:`2746`.) -* The XML-RPC client and server, provided by the :mod:`xmlrpclib` and - :mod:`SimpleXMLRPCServer` modules, have improved performance by +* The XML-RPC client and server, provided by the :mod:`xmlrpclib <xmlrpc.client>` and + :mod:`SimpleXMLRPCServer <xmlrpc.server>` modules, have improved performance by supporting HTTP/1.1 keep-alive and by optionally using gzip encoding to compress the XML being exchanged. The gzip compression is controlled by the :attr:`encode_threshold` attribute of @@ -2130,7 +2130,7 @@ Changes to Python's build process and to the C API include: only the filename, function name, and first line number are required. This is useful for extension modules that are attempting to construct a more useful traceback stack. Previously such - extensions needed to call :c:func:`PyCode_New`, which had many + extensions needed to call :c:func:`!PyCode_New`, which had many more arguments. (Added by Jeffrey Yasskin.) * New function: :c:func:`PyErr_NewExceptionWithDoc` creates a new @@ -2334,11 +2334,11 @@ Port-Specific Changes: Windows and :data:`LIBRARIES_ASSEMBLY_NAME_PREFIX`. (Contributed by David Cournapeau; :issue:`4365`.) -* The :mod:`_winreg` module for accessing the registry now implements - the :func:`~_winreg.CreateKeyEx` and :func:`~_winreg.DeleteKeyEx` +* The :mod:`_winreg <winreg>` module for accessing the registry now implements + the :func:`~winreg.CreateKeyEx` and :func:`~winreg.DeleteKeyEx` functions, extended versions of previously supported functions that - take several extra arguments. The :func:`~_winreg.DisableReflectionKey`, - :func:`~_winreg.EnableReflectionKey`, and :func:`~_winreg.QueryReflectionKey` + take several extra arguments. The :func:`~winreg.DisableReflectionKey`, + :func:`~winreg.EnableReflectionKey`, and :func:`~winreg.QueryReflectionKey` were also tested and documented. (Implemented by Brian Curtin: :issue:`7347`.) @@ -2508,7 +2508,7 @@ In the standard library: which raises an exception if there's an error. (Changed by Lars Gustäbel; :issue:`7357`.) -* The :mod:`urlparse` module's :func:`~urlparse.urlsplit` now handles +* The :mod:`urlparse <urllib.parse>` module's :func:`~urllib.parse.urlsplit` now handles unknown URL schemes in a fashion compliant with :rfc:`3986`: if the URL is of the form ``"<something>://..."``, the text before the ``://`` is treated as the scheme, even if it's a made-up scheme that @@ -2711,8 +2711,8 @@ and :ref:`setuptools-index`. PEP 476: Enabling certificate verification by default for stdlib http clients ----------------------------------------------------------------------------- -:pep:`476` updated :mod:`httplib` and modules which use it, such as -:mod:`urllib2` and :mod:`xmlrpclib`, to now verify that the server +:pep:`476` updated :mod:`httplib <http>` and modules which use it, such as +:mod:`urllib2 <urllib.request>` and :mod:`xmlrpclib`, to now verify that the server presents a certificate which is signed by a Certificate Authority in the platform trust store and whose hostname matches the hostname being requested by default, significantly improving security for many applications. This diff --git a/Doc/whatsnew/3.0.rst b/Doc/whatsnew/3.0.rst index 89e12062abaddd..888e6279754fc2 100644 --- a/Doc/whatsnew/3.0.rst +++ b/Doc/whatsnew/3.0.rst @@ -337,7 +337,7 @@ changed. (However, the standard library remains ASCII-only with the exception of contributor names in comments.) -* The :mod:`StringIO` and :mod:`cStringIO` modules are gone. Instead, +* The :mod:`!StringIO` and :mod:`!cStringIO` modules are gone. Instead, import the :mod:`io` module and use :class:`io.StringIO` or :class:`io.BytesIO` for text and data respectively. @@ -555,15 +555,15 @@ very extensive changes to the standard library. :pep:`3108` is the reference for the major changes to the library. Here's a capsule review: -* Many old modules were removed. Some, like :mod:`gopherlib` (no - longer used) and :mod:`md5` (replaced by :mod:`hashlib`), were +* Many old modules were removed. Some, like :mod:`!gopherlib` (no + longer used) and :mod:`!md5` (replaced by :mod:`hashlib`), were already deprecated by :pep:`4`. Others were removed as a result of the removal of support for various platforms such as Irix, BeOS and Mac OS 9 (see :pep:`11`). Some modules were also selected for removal in Python 3.0 due to lack of use or because a better replacement exists. See :pep:`3108` for an exhaustive list. -* The :mod:`bsddb3` package was removed because its presence in the +* The :mod:`!bsddb3` package was removed because its presence in the core standard library has proved over time to be a particular burden for the core developers due to testing instability and Berkeley DB's release schedule. However, the package is alive and well, @@ -588,45 +588,45 @@ review: * A common pattern in Python 2.x is to have one version of a module implemented in pure Python, with an optional accelerated version implemented as a C extension; for example, :mod:`pickle` and - :mod:`cPickle`. This places the burden of importing the accelerated + :mod:`!cPickle`. This places the burden of importing the accelerated version and falling back on the pure Python version on each user of these modules. In Python 3.0, the accelerated versions are considered implementation details of the pure Python versions. Users should always import the standard version, which attempts to import the accelerated version and falls back to the pure Python - version. The :mod:`pickle` / :mod:`cPickle` pair received this + version. The :mod:`pickle` / :mod:`!cPickle` pair received this treatment. The :mod:`profile` module is on the list for 3.1. The - :mod:`StringIO` module has been turned into a class in the :mod:`io` + :mod:`!StringIO` module has been turned into a class in the :mod:`io` module. * Some related modules have been grouped into packages, and usually the submodule names have been simplified. The resulting new packages are: - * :mod:`dbm` (:mod:`anydbm`, :mod:`dbhash`, :mod:`dbm`, - :mod:`dumbdbm`, :mod:`gdbm`, :mod:`whichdb`). + * :mod:`dbm` (:mod:`!anydbm`, :mod:`!dbhash`, :mod:`!dbm`, + :mod:`!dumbdbm`, :mod:`!gdbm`, :mod:`!whichdb`). - * :mod:`html` (:mod:`HTMLParser`, :mod:`htmlentitydefs`). + * :mod:`html` (:mod:`!HTMLParser`, :mod:`!htmlentitydefs`). - * :mod:`http` (:mod:`httplib`, :mod:`BaseHTTPServer`, - :mod:`CGIHTTPServer`, :mod:`SimpleHTTPServer`, :mod:`Cookie`, - :mod:`cookielib`). + * :mod:`http` (:mod:`!httplib`, :mod:`!BaseHTTPServer`, + :mod:`!CGIHTTPServer`, :mod:`!SimpleHTTPServer`, :mod:`!Cookie`, + :mod:`!cookielib`). * :mod:`tkinter` (all :mod:`Tkinter`-related modules except :mod:`turtle`). The target audience of :mod:`turtle` doesn't really care about :mod:`tkinter`. Also note that as of Python 2.6, the functionality of :mod:`turtle` has been greatly enhanced. - * :mod:`urllib` (:mod:`urllib`, :mod:`urllib2`, :mod:`urlparse`, - :mod:`robotparse`). + * :mod:`urllib` (:mod:`!urllib`, :mod:`!urllib2`, :mod:`!urlparse`, + :mod:`!robotparse`). - * :mod:`xmlrpc` (:mod:`xmlrpclib`, :mod:`DocXMLRPCServer`, - :mod:`SimpleXMLRPCServer`). + * :mod:`xmlrpc` (:mod:`!xmlrpclib`, :mod:`!DocXMLRPCServer`, + :mod:`!SimpleXMLRPCServer`). Some other changes to standard library modules, not covered by :pep:`3108`: -* Killed :mod:`sets`. Use the built-in :func:`set` class. +* Killed :mod:`!sets`. Use the built-in :func:`set` class. * Cleanup of the :mod:`sys` module: removed :func:`sys.exitfunc`, :func:`sys.exc_clear`, :data:`sys.exc_type`, :data:`sys.exc_value`, @@ -642,13 +642,13 @@ Some other changes to standard library modules, not covered by * Cleanup of the :mod:`operator` module: removed :func:`sequenceIncludes` and :func:`isCallable`. -* Cleanup of the :mod:`thread` module: :func:`acquire_lock` and - :func:`release_lock` are gone; use :func:`acquire` and - :func:`release` instead. +* Cleanup of the :mod:`!thread` module: :func:`!acquire_lock` and + :func:`!release_lock` are gone; use :meth:`~threading.Lock.acquire` and + :meth:`~threading.Lock.release` instead. * Cleanup of the :mod:`random` module: removed the :func:`jumpahead` API. -* The :mod:`new` module is gone. +* The :mod:`!new` module is gone. * The functions :func:`os.tmpnam`, :func:`os.tempnam` and :func:`os.tmpfile` have been removed in favor of the :mod:`tempfile` diff --git a/Doc/whatsnew/3.10.rst b/Doc/whatsnew/3.10.rst index 2da90b7ed55744..7dc06e9af694d9 100644 --- a/Doc/whatsnew/3.10.rst +++ b/Doc/whatsnew/3.10.rst @@ -402,9 +402,11 @@ Tracing events, with the correct line number, are generated for all lines of cod The :attr:`~frame.f_lineno` attribute of frame objects will always contain the expected line number. -The :attr:`~codeobject.co_lnotab` attribute of code objects is deprecated and +The :attr:`~codeobject.co_lnotab` attribute of +:ref:`code objects <code-objects>` is deprecated and will be removed in 3.12. -Code that needs to convert from offset to line number should use the new ``co_lines()`` method instead. +Code that needs to convert from offset to line number should use the new +:meth:`~codeobject.co_lines` method instead. PEP 634: Structural Pattern Matching ------------------------------------ @@ -1278,7 +1280,7 @@ Add negative indexing support to :attr:`PurePath.parents (Contributed by Yaroslav Pankovych in :issue:`21041`.) Add :meth:`Path.hardlink_to <pathlib.Path.hardlink_to>` method that -supersedes :meth:`~pathlib.Path.link_to`. The new method has the same argument +supersedes :meth:`!link_to`. The new method has the same argument order as :meth:`~pathlib.Path.symlink_to`. (Contributed by Barney Gale in :issue:`39950`.) @@ -1314,8 +1316,8 @@ pyclbr ------ Add an ``end_lineno`` attribute to the ``Function`` and ``Class`` -objects in the tree returned by :func:`pyclbr.readline` and -:func:`pyclbr.readline_ex`. It matches the existing (start) ``lineno``. +objects in the tree returned by :func:`pyclbr.readmodule` and +:func:`pyclbr.readmodule_ex`. It matches the existing (start) ``lineno``. (Contributed by Aviral Srivastava in :issue:`38307`.) shelve @@ -1740,7 +1742,7 @@ Deprecated (Contributed by Jelle Zijlstra in :gh:`87889`.) -* :meth:`pathlib.Path.link_to` is deprecated and slated for removal in +* :meth:`!pathlib.Path.link_to` is deprecated and slated for removal in Python 3.12. Use :meth:`pathlib.Path.hardlink_to` instead. (Contributed by Barney Gale in :issue:`39950`.) @@ -1771,7 +1773,7 @@ Deprecated * NPN features like :meth:`ssl.SSLSocket.selected_npn_protocol` and :meth:`ssl.SSLContext.set_npn_protocols` are replaced by ALPN. -* The threading debug (:envvar:`PYTHONTHREADDEBUG` environment variable) is +* The threading debug (:envvar:`!PYTHONTHREADDEBUG` environment variable) is deprecated in Python 3.10 and will be removed in Python 3.12. This feature requires a :ref:`debug build of Python <debug-build>`. (Contributed by Victor Stinner in :issue:`44584`.) diff --git a/Doc/whatsnew/3.11.rst b/Doc/whatsnew/3.11.rst index 8db133b90a7a4b..4f4c1de8d8d596 100644 --- a/Doc/whatsnew/3.11.rst +++ b/Doc/whatsnew/3.11.rst @@ -672,7 +672,7 @@ enum * Changed :meth:`Enum.__format__() <enum.Enum.__format__>` (the default for :func:`format`, :meth:`str.format` and :term:`f-string`\s) to always produce - the same result as :meth:`Enum.__str__()`: for enums inheriting from + the same result as :meth:`Enum.__str__() <enum.Enum.__str__>`: for enums inheriting from :class:`~enum.ReprEnum` it will be the member's value; for all other enums it will be the enum and member name (e.g. ``Color.RED``). @@ -1604,7 +1604,7 @@ raw, adaptive bytecode containing quickened data. New opcodes ----------- -* :opcode:`ASYNC_GEN_WRAP`, :opcode:`RETURN_GENERATOR` and :opcode:`SEND`, +* :opcode:`!ASYNC_GEN_WRAP`, :opcode:`RETURN_GENERATOR` and :opcode:`SEND`, used in generators and co-routines. * :opcode:`COPY_FREE_VARS`, @@ -1615,7 +1615,7 @@ New opcodes * :opcode:`MAKE_CELL`, to create :ref:`cell-objects`. -* :opcode:`CHECK_EG_MATCH` and :opcode:`PREP_RERAISE_STAR`, +* :opcode:`CHECK_EG_MATCH` and :opcode:`!PREP_RERAISE_STAR`, to handle the :ref:`new exception groups and except* <whatsnew311-pep654>` added in :pep:`654`. @@ -1630,38 +1630,38 @@ New opcodes Replaced opcodes ---------------- -+------------------------------------+-----------------------------------+-----------------------------------------+ -| Replaced Opcode(s) | New Opcode(s) | Notes | -+====================================+===================================+=========================================+ -| | :opcode:`!BINARY_*` | :opcode:`BINARY_OP` | Replaced all numeric binary/in-place | -| | :opcode:`!INPLACE_*` | | opcodes with a single opcode | -+------------------------------------+-----------------------------------+-----------------------------------------+ -| | :opcode:`!CALL_FUNCTION` | | :opcode:`CALL` | Decouples argument shifting for methods | -| | :opcode:`!CALL_FUNCTION_KW` | | :opcode:`KW_NAMES` | from handling of keyword arguments; | -| | :opcode:`!CALL_METHOD` | | :opcode:`PRECALL` | allows better specialization of calls | -| | | :opcode:`PUSH_NULL` | | -+------------------------------------+-----------------------------------+-----------------------------------------+ -| | :opcode:`!DUP_TOP` | | :opcode:`COPY` | Stack manipulation instructions | -| | :opcode:`!DUP_TOP_TWO` | | :opcode:`SWAP` | | -| | :opcode:`!ROT_TWO` | | | -| | :opcode:`!ROT_THREE` | | | -| | :opcode:`!ROT_FOUR` | | | -| | :opcode:`!ROT_N` | | | -+------------------------------------+-----------------------------------+-----------------------------------------+ -| | :opcode:`!JUMP_IF_NOT_EXC_MATCH` | | :opcode:`CHECK_EXC_MATCH` | Now performs check but doesn't jump | -+------------------------------------+-----------------------------------+-----------------------------------------+ -| | :opcode:`!JUMP_ABSOLUTE` | | :opcode:`JUMP_BACKWARD` | See [#bytecode-jump]_; | -| | :opcode:`!POP_JUMP_IF_FALSE` | | :opcode:`POP_JUMP_BACKWARD_IF_* | ``TRUE``, ``FALSE``, | -| | :opcode:`!POP_JUMP_IF_TRUE` | <POP_JUMP_BACKWARD_IF_TRUE>` | ``NONE`` and ``NOT_NONE`` variants | -| | | :opcode:`POP_JUMP_FORWARD_IF_* | for each direction | -| | <POP_JUMP_FORWARD_IF_TRUE>` | | -+------------------------------------+-----------------------------------+-----------------------------------------+ -| | :opcode:`!SETUP_WITH` | :opcode:`BEFORE_WITH` | :keyword:`with` block setup | -| | :opcode:`!SETUP_ASYNC_WITH` | | | -+------------------------------------+-----------------------------------+-----------------------------------------+ ++------------------------------------+------------------------------------+-----------------------------------------+ +| Replaced Opcode(s) | New Opcode(s) | Notes | ++====================================+====================================+=========================================+ +| | :opcode:`!BINARY_*` | :opcode:`BINARY_OP` | Replaced all numeric binary/in-place | +| | :opcode:`!INPLACE_*` | | opcodes with a single opcode | ++------------------------------------+------------------------------------+-----------------------------------------+ +| | :opcode:`!CALL_FUNCTION` | | :opcode:`CALL` | Decouples argument shifting for methods | +| | :opcode:`!CALL_FUNCTION_KW` | | :opcode:`!KW_NAMES` | from handling of keyword arguments; | +| | :opcode:`!CALL_METHOD` | | :opcode:`!PRECALL` | allows better specialization of calls | +| | | :opcode:`PUSH_NULL` | | ++------------------------------------+------------------------------------+-----------------------------------------+ +| | :opcode:`!DUP_TOP` | | :opcode:`COPY` | Stack manipulation instructions | +| | :opcode:`!DUP_TOP_TWO` | | :opcode:`SWAP` | | +| | :opcode:`!ROT_TWO` | | | +| | :opcode:`!ROT_THREE` | | | +| | :opcode:`!ROT_FOUR` | | | +| | :opcode:`!ROT_N` | | | ++------------------------------------+------------------------------------+-----------------------------------------+ +| | :opcode:`!JUMP_IF_NOT_EXC_MATCH` | | :opcode:`CHECK_EXC_MATCH` | Now performs check but doesn't jump | ++------------------------------------+------------------------------------+-----------------------------------------+ +| | :opcode:`!JUMP_ABSOLUTE` | | :opcode:`JUMP_BACKWARD` | See [#bytecode-jump]_; | +| | :opcode:`!POP_JUMP_IF_FALSE` | | :opcode:`!POP_JUMP_BACKWARD_IF_*`| ``TRUE``, ``FALSE``, | +| | :opcode:`!POP_JUMP_IF_TRUE` | | :opcode:`!POP_JUMP_FORWARD_IF_*` | ``NONE`` and ``NOT_NONE`` variants | +| | | for each direction | +| | | | ++------------------------------------+------------------------------------+-----------------------------------------+ +| | :opcode:`!SETUP_WITH` | :opcode:`BEFORE_WITH` | :keyword:`with` block setup | +| | :opcode:`!SETUP_ASYNC_WITH` | | | ++------------------------------------+------------------------------------+-----------------------------------------+ .. [#bytecode-jump] All jump opcodes are now relative, including the - existing :opcode:`JUMP_IF_TRUE_OR_POP` and :opcode:`JUMP_IF_FALSE_OR_POP`. + existing :opcode:`!JUMP_IF_TRUE_OR_POP` and :opcode:`!JUMP_IF_FALSE_OR_POP`. The argument is now an offset from the current instruction rather than an absolute location. @@ -1747,7 +1747,7 @@ Modules (Contributed by Brett Cannon in :issue:`47061` and Victor Stinner in :gh:`68966`.) -* The :mod:`asynchat`, :mod:`asyncore` and :mod:`smtpd` modules have been +* The :mod:`!asynchat`, :mod:`!asyncore` and :mod:`!smtpd` modules have been deprecated since at least Python 3.6. Their documentation and deprecation warnings have now been updated to note they will be removed in Python 3.12. (Contributed by Hugo van Kemenade in :issue:`47022`.) @@ -1773,7 +1773,7 @@ Standard Library * the :class:`!configparser.SafeConfigParser` class * the :attr:`!configparser.ParsingError.filename` property - * the :meth:`configparser.RawConfigParser.readfp` method + * the :meth:`!configparser.RawConfigParser.readfp` method (Contributed by Hugo van Kemenade in :issue:`45173`.) @@ -1789,13 +1789,13 @@ Standard Library and will be removed in a future Python version, due to not supporting resources located within package subdirectories: - * :func:`importlib.resources.contents` - * :func:`importlib.resources.is_resource` - * :func:`importlib.resources.open_binary` - * :func:`importlib.resources.open_text` - * :func:`importlib.resources.read_binary` - * :func:`importlib.resources.read_text` - * :func:`importlib.resources.path` + * :func:`!importlib.resources.contents` + * :func:`!importlib.resources.is_resource` + * :func:`!importlib.resources.open_binary` + * :func:`!importlib.resources.open_text` + * :func:`!importlib.resources.read_binary` + * :func:`!importlib.resources.read_text` + * :func:`!importlib.resources.path` * The :func:`locale.getdefaultlocale` function is deprecated and will be removed in Python 3.15. Use :func:`locale.setlocale`, @@ -1803,7 +1803,7 @@ Standard Library :func:`locale.getlocale` functions instead. (Contributed by Victor Stinner in :gh:`90817`.) -* The :func:`locale.resetlocale` function is deprecated and will be +* The :func:`!locale.resetlocale` function is deprecated and will be removed in Python 3.13. Use ``locale.setlocale(locale.LC_ALL, "")`` instead. (Contributed by Victor Stinner in :gh:`90817`.) @@ -1860,7 +1860,7 @@ Standard Library (Contributed by Erlend E. Aasland in :issue:`5846`.) -* :meth:`~!unittest.TestProgram.usageExit` is marked deprecated, to be removed +* :meth:`!unittest.TestProgram.usageExit` is marked deprecated, to be removed in 3.13. (Contributed by Carlos Damázio in :gh:`67048`.) @@ -1877,8 +1877,8 @@ and will be removed in Python 3.12. C APIs pending removal are :ref:`listed separately <whatsnew311-c-api-pending-removal>`. -* The :mod:`asynchat` module -* The :mod:`asyncore` module +* The :mod:`!asynchat` module +* The :mod:`!asyncore` module * The :ref:`entire distutils package <distutils-deprecated>` * The :mod:`!imp` module * The :class:`typing.io <typing.IO>` namespace @@ -1902,10 +1902,10 @@ C APIs pending removal are * :func:`!importlib.util.set_package_wrapper` * :class:`!pkgutil.ImpImporter` * :class:`!pkgutil.ImpLoader` -* :meth:`pathlib.Path.link_to` +* :meth:`!pathlib.Path.link_to` * :func:`!sqlite3.enable_shared_cache` * :func:`!sqlite3.OptimizedUnicode` -* :envvar:`PYTHONTHREADDEBUG` environment variable +* :envvar:`!PYTHONTHREADDEBUG` environment variable * The following deprecated aliases in :mod:`unittest`: ============================ =============================== =============== @@ -1967,7 +1967,7 @@ Removed C APIs are :ref:`listed separately <whatsnew311-c-api-removed>`. (Contributed by Victor Stinner in :issue:`45085`.) -* Removed the :mod:`distutils` ``bdist_msi`` command deprecated in Python 3.9. +* Removed the :mod:`!distutils` ``bdist_msi`` command deprecated in Python 3.9. Use ``bdist_wheel`` (wheel packages) instead. (Contributed by Hugo van Kemenade in :issue:`45124`.) @@ -2007,7 +2007,7 @@ Removed C APIs are :ref:`listed separately <whatsnew311-c-api-removed>`. because it was not used and added by mistake in previous versions. (Contributed by Nikita Sobolev in :issue:`46483`.) -* Removed the :class:`!MailmanProxy` class in the :mod:`smtpd` module, +* Removed the :class:`!MailmanProxy` class in the :mod:`!smtpd` module, as it is unusable without the external :mod:`!mailman` package. (Contributed by Donghee Na in :issue:`35800`.) @@ -2295,7 +2295,7 @@ Porting to Python 3.11 as its second parameter, instead of ``PyFrameObject*``. See :pep:`523` for more details of how to use this function pointer type. -* :c:func:`PyCode_New` and :c:func:`PyCode_NewWithPosOnlyArgs` now take +* :c:func:`!PyCode_New` and :c:func:`!PyCode_NewWithPosOnlyArgs` now take an additional ``exception_table`` argument. Using these functions should be avoided, if at all possible. To get a custom code object: create a code object using the compiler, @@ -2402,7 +2402,7 @@ Porting to Python 3.11 been included directly, consider including ``Python.h`` instead. (Contributed by Victor Stinner in :issue:`35134`.) -* The :c:func:`PyUnicode_CHECK_INTERNED` macro has been excluded from the +* The :c:func:`!PyUnicode_CHECK_INTERNED` macro has been excluded from the limited C API. It was never usable there, because it used internal structures which are not available in the limited C API. (Contributed by Victor Stinner in :issue:`46007`.) @@ -2465,7 +2465,7 @@ Porting to Python 3.11 Debuggers that accessed the :attr:`~frame.f_locals` directly *must* call :c:func:`PyFrame_GetLocals` instead. They no longer need to call - :c:func:`PyFrame_FastToLocalsWithError` or :c:func:`PyFrame_LocalsToFast`, + :c:func:`!PyFrame_FastToLocalsWithError` or :c:func:`!PyFrame_LocalsToFast`, in fact they should not call those functions. The necessary updating of the frame is now managed by the virtual machine. @@ -2604,8 +2604,8 @@ and will be removed in Python 3.12. * :c:func:`!PyUnicode_GET_DATA_SIZE` * :c:func:`!PyUnicode_GET_SIZE` * :c:func:`!PyUnicode_GetSize` -* :c:func:`PyUnicode_IS_COMPACT` -* :c:func:`PyUnicode_IS_READY` +* :c:func:`!PyUnicode_IS_COMPACT` +* :c:func:`!PyUnicode_IS_READY` * :c:func:`PyUnicode_READY` * :c:func:`!PyUnicode_WSTR_LENGTH` * :c:func:`!_PyUnicode_AsUnicode` @@ -2660,7 +2660,7 @@ Removed (Contributed by Victor Stinner in :issue:`45474`.) * Exclude :c:func:`PyWeakref_GET_OBJECT` from the limited C API. It never - worked since the :c:type:`PyWeakReference` structure is opaque in the + worked since the :c:type:`!PyWeakReference` structure is opaque in the limited C API. (Contributed by Victor Stinner in :issue:`35134`.) @@ -2701,4 +2701,30 @@ Removed (Contributed by Inada Naoki in :issue:`44029`.) +Notable changes in 3.11.4 +========================= + +tarfile +------- + +* The extraction methods in :mod:`tarfile`, and :func:`shutil.unpack_archive`, + have a new a *filter* argument that allows limiting tar features than may be + surprising or dangerous, such as creating files outside the destination + directory. + See :ref:`tarfile-extraction-filter` for details. + In Python 3.12, use without the *filter* argument will show a + :exc:`DeprecationWarning`. + In Python 3.14, the default will switch to ``'data'``. + (Contributed by Petr Viktorin in :pep:`706`.) + + +Notable changes in 3.11.5 +========================= + +OpenSSL +------- + +* Windows builds and macOS installers from python.org now use OpenSSL 3.0. + + .. _libb2: https://www.blake2.net/ diff --git a/Doc/whatsnew/3.12.rst b/Doc/whatsnew/3.12.rst index 8551b35438e2c3..100312a5940b79 100644 --- a/Doc/whatsnew/3.12.rst +++ b/Doc/whatsnew/3.12.rst @@ -343,7 +343,9 @@ cores. This is currently only available through the C-API, though a Python API is :pep:`anticipated for 3.13 <554>`. Use the new :c:func:`Py_NewInterpreterFromConfig` function to -create an interpreter with its own GIL:: +create an interpreter with its own GIL: + +.. code-block:: c PyInterpreterConfig config = { .check_multi_interp_extensions = 1, @@ -688,7 +690,7 @@ csv * Add :const:`csv.QUOTE_NOTNULL` and :const:`csv.QUOTE_STRINGS` flags to provide finer grained control of ``None`` and empty strings by - :class:`csv.writer` objects. + :class:`~csv.reader` and :class:`~csv.writer` objects. dis --- @@ -1372,6 +1374,18 @@ APIs: * :meth:`!unittest.TestProgram.usageExit` (:gh:`67048`) * :class:`!webbrowser.MacOSX` (:gh:`86421`) * :class:`classmethod` descriptor chaining (:gh:`89519`) +* :mod:`importlib.resources` deprecated methods: + + * ``contents()`` + * ``is_resource()`` + * ``open_binary()`` + * ``open_text()`` + * ``path()`` + * ``read_binary()`` + * ``read_text()`` + + Use :func:`importlib.resources.files()` instead. Refer to `importlib-resources: Migrating from Legacy + <https://importlib-resources.readthedocs.io/en/latest/using.html#migrating-from-legacy>`_ (:gh:`106531`) Pending Removal in Python 3.14 ------------------------------ diff --git a/Doc/whatsnew/3.13.rst b/Doc/whatsnew/3.13.rst index 7dc02dacdc68f7..de79bd979aff80 100644 --- a/Doc/whatsnew/3.13.rst +++ b/Doc/whatsnew/3.13.rst @@ -81,6 +81,13 @@ Important deprecations, removals or restrictions: * Python 3.13 and later have two years of full support, followed by three years of security fixes. +Interpreter improvements: + +* A basic :ref:`JIT compiler <whatsnew313-jit-compiler>` was added. + It is currently disabled by default (though we may turn it on later). + Performance improvements are modest -- we expect to be improving this + over the next few releases. + New Features ============ @@ -94,6 +101,17 @@ Improved Error Messages variables. See also :ref:`using-on-controlling-color`. (Contributed by Pablo Galindo Salgado in :gh:`112730`.) +* When an incorrect keyword argument is passed to a function, the error message + now potentially suggests the correct keyword argument. + (Contributed by Pablo Galindo Salgado and Shantanu Jain in :gh:`107944`.) + + >>> "better error messages!".split(max_split=1) + Traceback (most recent call last): + File "<stdin>", line 1, in <module> + "better error messages!".split(max_split=1) + ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~^^^^^^^^^^^^^ + TypeError: split() got an unexpected keyword argument 'max_split'. Did you mean 'maxsplit'? + Other Language Changes ====================== @@ -127,11 +145,21 @@ Other Language Changes is rejected when the global is used in the :keyword:`else` block. (Contributed by Irit Katriel in :gh:`111123`.) +* Many functions now emit a warning if a boolean value is passed as + a file descriptor argument. + This can help catch some errors earlier. + (Contributed by Serhiy Storchaka in :gh:`82626`.) + * Added a new environment variable :envvar:`PYTHON_FROZEN_MODULES`. It determines whether or not frozen modules are ignored by the import machinery, equivalent of the :option:`-X frozen_modules <-X>` command-line option. (Contributed by Yilei Yang in :gh:`111374`.) +* The new :envvar:`PYTHON_HISTORY` environment variable can be used to change + the location of a ``.python_history`` file. + (Contributed by Levi Sabah, Zackery Spytz and Hugo van Kemenade in + :gh:`73965`.) + New Modules =========== @@ -141,13 +169,14 @@ New Modules Improved Modules ================ -ast ---- +argparse +-------- -* :func:`ast.parse` now accepts an optional argument ``optimize`` - which is passed on to the :func:`compile` built-in. This makes it - possible to obtain an optimized ``AST``. - (Contributed by Irit Katriel in :gh:`108113`). +* Add parameter *deprecated* in methods + :meth:`~argparse.ArgumentParser.add_argument` and :meth:`!add_parser` + which allows to deprecate command-line options, positional arguments and + subcommands. + (Contributed by Serhiy Storchaka in :gh:`83648`). array ----- @@ -156,6 +185,14 @@ array It can be used instead of ``'u'`` type code, which is deprecated. (Contributed by Inada Naoki in :gh:`80480`.) +ast +--- + +* :func:`ast.parse` now accepts an optional argument ``optimize`` + which is passed on to the :func:`compile` built-in. This makes it + possible to obtain an optimized ``AST``. + (Contributed by Irit Katriel in :gh:`108113`.) + asyncio ------- @@ -175,6 +212,13 @@ copy any user classes which define the :meth:`!__replace__` method. (Contributed by Serhiy Storchaka in :gh:`108751`.) +dbm +--- + +* Add :meth:`dbm.gnu.gdbm.clear` and :meth:`dbm.ndbm.ndbm.clear` methods that remove all items + from the database. + (Contributed by Donghee Na in :gh:`107122`.) + dis --- @@ -184,13 +228,6 @@ dis the ``show_offsets`` parameter. (Contributed by Irit Katriel in :gh:`112137`.) -dbm ---- - -* Add :meth:`dbm.gnu.gdbm.clear` and :meth:`dbm.ndbm.ndbm.clear` methods that remove all items - from the database. - (Contributed by Donghee Na in :gh:`107122`.) - doctest ------- @@ -208,7 +245,7 @@ email parameter to these two functions: use ``strict=False`` to get the old behavior, accept malformed inputs. ``getattr(email.utils, 'supports_strict_parsing', False)`` can be use to - check if the *strict* paramater is available. + check if the *strict* parameter is available. (Contributed by Thomas Dwyer and Victor Stinner for :gh:`102988` to improve the CVE-2023-27043 fix.) @@ -218,7 +255,7 @@ fractions * Formatting for objects of type :class:`fractions.Fraction` now supports the standard format specification mini-language rules for fill, alignment, sign handling, minimum width and grouping. (Contributed by Mark Dickinson - in :gh:`111320`) + in :gh:`111320`.) glob ---- @@ -242,13 +279,24 @@ ipaddress * Add the :attr:`ipaddress.IPv4Address.ipv6_mapped` property, which returns the IPv4-mapped IPv6 address. (Contributed by Charles Machalow in :gh:`109466`.) +marshal +------- + +* Add the *allow_code* parameter in module functions. + Passing ``allow_code=False`` prevents serialization and de-serialization of + code objects which are incompatible between Python versions. + (Contributed by Serhiy Storchaka in :gh:`113626`.) + mmap ---- * The :class:`mmap.mmap` class now has an :meth:`~mmap.mmap.seekable` method - that can be used where it requires a file-like object with seekable and - the :meth:`~mmap.mmap.seek` method return the new absolute position. + that can be used when a seekable file-like object is required. + The :meth:`~mmap.mmap.seek` method now returns the new absolute position. (Contributed by Donghee Na and Sylvie Liberman in :gh:`111835`.) +* :class:`mmap.mmap` now has a *trackfd* parameter on Unix; if it is ``False``, + the file descriptor specified by *fileno* will not be duplicated. + (Contributed by Zackery Spytz and Petr Viktorin in :gh:`78502`.) opcode ------ @@ -281,13 +329,17 @@ os the new environment variable :envvar:`PYTHON_CPU_COUNT` or the new command-line option :option:`-X cpu_count <-X>`. This option is useful for users who need to limit CPU resources of a container system without having to modify the container (application code). - (Contributed by Donghee Na in :gh:`109595`) + (Contributed by Donghee Na in :gh:`109595`.) * Add support of :func:`os.lchmod` and the *follow_symlinks* argument in :func:`os.chmod` on Windows. Note that the default value of *follow_symlinks* in :func:`!os.lchmod` is ``False`` on Windows. - (Contributed by Serhiy Storchaka in :gh:`59616`) + (Contributed by Serhiy Storchaka in :gh:`59616`.) + +* Add support of :func:`os.fchmod` and a file descriptor + in :func:`os.chmod` on Windows. + (Contributed by Serhiy Storchaka in :gh:`113191`.) * :func:`os.posix_spawn` now accepts ``env=None``, which makes the newly spawned process use the current process environment. @@ -298,6 +350,16 @@ os :c:func:`!posix_spawn_file_actions_addclosefrom_np`. (Contributed by Jakub Kulik in :gh:`113117`.) +os.path +------- + +* Add :func:`os.path.isreserved` to check if a path is reserved on the current + system. This function is only available on Windows. + (Contributed by Barney Gale in :gh:`88569`.) +* On Windows, :func:`os.path.isabs` no longer considers paths starting with + exactly one (back)slash to be absolute. + (Contributed by Barney Gale and Jon Foster in :gh:`44626`.) + pathlib ------- @@ -306,10 +368,11 @@ pathlib (Contributed by Barney Gale in :gh:`89812`.) * Add :meth:`pathlib.Path.from_uri`, a new constructor to create a :class:`pathlib.Path` - object from a 'file' URI (``file:/``). + object from a 'file' URI (``file://``). (Contributed by Barney Gale in :gh:`107465`.) -* Add support for recursive wildcards in :meth:`pathlib.PurePath.match`. +* Add :meth:`pathlib.PurePath.full_match` for matching paths with + shell-style wildcards, including the recursive wildcard "``**``". (Contributed by Barney Gale in :gh:`73435`.) * Add *follow_symlinks* keyword-only argument to :meth:`pathlib.Path.glob`, @@ -319,6 +382,11 @@ pathlib (Contributed by Barney Gale in :gh:`77609` and :gh:`105793`, and Kamil Turek in :gh:`107962`). +* Return files and directories from :meth:`pathlib.Path.glob` and + :meth:`~pathlib.Path.rglob` when given a pattern that ends with "``**``". In + earlier versions, only directories were returned. + (Contributed by Barney Gale in :gh:`70303`). + pdb --- @@ -326,7 +394,7 @@ pdb the new ``exceptions [exc_number]`` command for Pdb. (Contributed by Matthias Bussonnier in :gh:`106676`.) -* Expressions/Statements whose prefix is a pdb command are now correctly +* Expressions/statements whose prefix is a pdb command are now correctly identified and executed. (Contributed by Tian Gao in :gh:`108464`.) @@ -335,6 +403,13 @@ pdb command line option or :envvar:`PYTHONSAFEPATH` environment variable). (Contributed by Tian Gao and Christian Walther in :gh:`111762`.) +queue +----- + +* Add :meth:`queue.Queue.shutdown` (along with :exc:`queue.ShutDown`) for queue + termination. + (Contributed by Laurie Opperman and Yves Duprat in :gh:`104750`.) + re -- * Rename :exc:`!re.error` to :exc:`re.PatternError` for improved clarity. @@ -347,6 +422,10 @@ sqlite3 object is not :meth:`closed <sqlite3.Connection.close>` explicitly. (Contributed by Erlend E. Aasland in :gh:`105539`.) +* Add *filter* keyword-only parameter to :meth:`sqlite3.Connection.iterdump` + for filtering database objects to dump. + (Contributed by Mariusz Felisiak in :gh:`91602`.) + subprocess ---------- @@ -378,6 +457,15 @@ tkinter :meth:`!tk_busy_current`, and :meth:`!tk_busy_status`. (Contributed by Miguel, klappnase and Serhiy Storchaka in :gh:`72684`.) +* The :mod:`tkinter` widget method :meth:`!wm_attributes` now accepts + the attribute name without the minus prefix to get window attributes, + e.g. ``w.wm_attributes('alpha')`` and allows to specify attributes and + values to set as keyword arguments, e.g. ``w.wm_attributes(alpha=0.5)``. + Add new optional keyword-only parameter *return_python_dict*: calling + ``w.wm_attributes(return_python_dict=True)`` returns the attributes as + a dict instead of a tuple. + (Contributed by Serhiy Storchaka in :gh:`43457`.) + * Add support of the "vsapi" element type in the :meth:`~tkinter.ttk.Style.element_create` method of :class:`tkinter.ttk.Style`. @@ -429,6 +517,14 @@ warnings warning may also be emitted when a decorated function or class is used at runtime. See :pep:`702`. (Contributed by Jelle Zijlstra in :gh:`104003`.) +xml.etree.ElementTree +--------------------- + +* Add the :meth:`!close` method for the iterator returned by + :func:`~xml.etree.ElementTree.iterparse` for explicit cleaning up. + (Contributed by Serhiy Storchaka in :gh:`69893`.) + + Optimizations ============= @@ -441,6 +537,46 @@ Optimizations FreeBSD and Solaris. See the ``subprocess`` section above for details. (Contributed by Jakub Kulik in :gh:`113117`.) +.. _whatsnew313-jit-compiler: + +Experimental JIT Compiler +========================= + +When CPython is configured using the ``--enable-experimental-jit`` option, +a just-in-time compiler is added which can speed up some Python programs. + +The internal architecture is roughly as follows. + +* We start with specialized *Tier 1 bytecode*. + See :ref:`What's new in 3.11 <whatsnew311-pep659>` for details. + +* When the Tier 1 bytecode gets hot enough, it gets translated + to a new, purely internal *Tier 2 IR*, a.k.a. micro-ops ("uops"). + +* The Tier 2 IR uses the same stack-based VM as Tier 1, but the + instruction format is better suited to translation to machine code. + +* We have several optimization passes for Tier 2 IR, which are applied + before it is interpreted or translated to machine code. + +* There is a Tier 2 interpreter, but it is mostly intended for debugging + the earlier stages of the optimization pipeline. If the JIT is not + enabled, the Tier 2 interpreter can be invoked by passing Python the + ``-X uops`` option or by setting the ``PYTHON_UOPS`` environment + variable to ``1``. + +* When the ``--enable-experimental-jit`` option is used, the optimized + Tier 2 IR is translated to machine code, which is then executed. + This does not require additional runtime options. + +* The machine code translation process uses an architecture called + *copy-and-patch*. It has no runtime dependencies, but there is a new + build-time dependency on LLVM. + +(JIT by Brandt Bucher, inspired by a paper by Haoran Xu and Fredrik Kjolstad. +Tier 2 IR by Mark Shannon and Guido van Rossum. +Tier 2 optimizer by Ken Jin.) + Deprecated ========== @@ -456,24 +592,69 @@ Deprecated Replace ``ctypes.ARRAY(item_type, size)`` with ``item_type * size``. (Contributed by Victor Stinner in :gh:`105733`.) +* :mod:`decimal`: Deprecate non-standard format specifier "N" for + :class:`decimal.Decimal`. + It was not documented and only supported in the C implementation. + (Contributed by Serhiy Storchaka in :gh:`89902`.) + +* :mod:`dis`: The ``dis.HAVE_ARGUMENT`` separator is deprecated. Check + membership in :data:`~dis.hasarg` instead. + (Contributed by Irit Katriel in :gh:`109319`.) + * :mod:`getopt` and :mod:`optparse` modules: They are now - :term:`soft deprecated`: the :mod:`argparse` should be used for new projects. + :term:`soft deprecated`: the :mod:`argparse` module should be used for new projects. Previously, the :mod:`optparse` module was already deprecated, its removal was not scheduled, and no warnings was emitted: so there is no change in practice. (Contributed by Victor Stinner in :gh:`106535`.) +* :mod:`gettext`: Emit deprecation warning for non-integer numbers in + :mod:`gettext` functions and methods that consider plural forms even if the + translation was not found. + (Contributed by Serhiy Storchaka in :gh:`88434`.) + * :mod:`http.server`: :class:`http.server.CGIHTTPRequestHandler` now emits a - :exc:`DeprecationWarning` as it will be removed in 3.15. Process based CGI - http servers have been out of favor for a very long time. This code was + :exc:`DeprecationWarning` as it will be removed in 3.15. Process-based CGI + HTTP servers have been out of favor for a very long time. This code was outdated, unmaintained, and rarely used. It has a high potential for both security and functionality bugs. This includes removal of the ``--cgi`` flag to the ``python -m http.server`` command line in 3.15. -* :mod:`traceback`: +* :mod:`pathlib`: + :meth:`pathlib.PurePath.is_reserved` is deprecated and scheduled for + removal in Python 3.15. Use :func:`os.path.isreserved` to detect reserved + paths on Windows. + +* :mod:`pydoc`: Deprecate undocumented :func:`!pydoc.ispackage` function. + (Contributed by Zackery Spytz in :gh:`64020`.) - * The field *exc_type* of :class:`traceback.TracebackException` is - deprecated. Use *exc_type_str* instead. +* :mod:`sqlite3`: Passing more than one positional argument to + :func:`sqlite3.connect` and the :class:`sqlite3.Connection` constructor is + deprecated. The remaining parameters will become keyword-only in Python 3.15. + + Deprecate passing name, number of arguments, and the callable as keyword + arguments for the following :class:`sqlite3.Connection` APIs: + + * :meth:`~sqlite3.Connection.create_function` + * :meth:`~sqlite3.Connection.create_aggregate` + + Deprecate passing the callback callable by keyword for the following + :class:`sqlite3.Connection` APIs: + + * :meth:`~sqlite3.Connection.set_authorizer` + * :meth:`~sqlite3.Connection.set_progress_handler` + * :meth:`~sqlite3.Connection.set_trace_callback` + + The affected parameters will become positional-only in Python 3.15. + + (Contributed by Erlend E. Aasland in :gh:`107948` and :gh:`108278`.) + +* :mod:`sys`: :func:`sys._enablelegacywindowsfsencoding` function. + Replace it with the :envvar:`PYTHONLEGACYWINDOWSFSENCODING` environment variable. + (Contributed by Inada Naoki in :gh:`73427`.) + +* :mod:`traceback`: The field *exc_type* of :class:`traceback.TracebackException` + is deprecated. Use *exc_type_str* instead. * :mod:`typing`: @@ -509,39 +690,6 @@ Deprecated They will be removed in Python 3.15. (Contributed by Victor Stinner in :gh:`105096`.) -* Passing more than one positional argument to :func:`sqlite3.connect` and the - :class:`sqlite3.Connection` constructor is deprecated. The remaining - parameters will become keyword-only in Python 3.15. - - Deprecate passing name, number of arguments, and the callable as keyword - arguments, for the following :class:`sqlite3.Connection` APIs: - - * :meth:`~sqlite3.Connection.create_function` - * :meth:`~sqlite3.Connection.create_aggregate` - - Deprecate passing the callback callable by keyword for the following - :class:`sqlite3.Connection` APIs: - - * :meth:`~sqlite3.Connection.set_authorizer` - * :meth:`~sqlite3.Connection.set_progress_handler` - * :meth:`~sqlite3.Connection.set_trace_callback` - - The affected parameters will become positional-only in Python 3.15. - - (Contributed by Erlend E. Aasland in :gh:`107948` and :gh:`108278`.) - -* The ``dis.HAVE_ARGUMENT`` separator is deprecated. Check membership - in :data:`~dis.hasarg` instead. - (Contributed by Irit Katriel in :gh:`109319`.) - -* Deprecate non-standard format specifier "N" for :class:`decimal.Decimal`. - It was not documented and only supported in the C implementation. - (Contributed by Serhiy Storchaka in :gh:`89902`.) - -* Emit deprecation warning for non-integer numbers in :mod:`gettext` functions - and methods that consider plural forms even if the translation was not found. - (Contributed by Serhiy Storchaka in :gh:`88434`.) - * Calling :meth:`frame.clear` on a suspended frame raises :exc:`RuntimeError` (as has always been the case for an executing frame). (Contributed by Irit Katriel in :gh:`79932`.) @@ -613,11 +761,11 @@ Pending Removal in Python 3.14 :func:`~multiprocessing.set_start_method` APIs to explicitly specify when your code *requires* ``'fork'``. See :ref:`multiprocessing-start-methods`. -* :mod:`pathlib`: :meth:`~pathlib.PurePath.is_relative_to`, +* :mod:`pathlib`: :meth:`~pathlib.PurePath.is_relative_to` and :meth:`~pathlib.PurePath.relative_to`: passing additional arguments is deprecated. -* :func:`pkgutil.find_loader` and :func:`pkgutil.get_loader` +* :mod:`pkgutil`: :func:`~pkgutil.find_loader` and :func:`~pkgutil.get_loader` now raise :exc:`DeprecationWarning`; use :func:`importlib.util.find_spec` instead. (Contributed by Nikita Sobolev in :gh:`97850`.) @@ -674,6 +822,18 @@ Pending Removal in Python 3.15 :func:`locale.getlocale()` instead. (Contributed by Hugo van Kemenade in :gh:`111187`.) +* :mod:`pathlib`: + :meth:`pathlib.PurePath.is_reserved` is deprecated and scheduled for + removal in Python 3.15. Use :func:`os.path.isreserved` to detect reserved + paths on Windows. + +* :mod:`threading`: + Passing any arguments to :func:`threading.RLock` is now deprecated. + C version allows any numbers of args and kwargs, + but they are just ignored. Python version does not allow any arguments. + All arguments will be removed from :func:`threading.RLock` in Python 3.15. + (Contributed by Nikita Sobolev in :gh:`102029`.) + * :class:`typing.NamedTuple`: * The undocumented keyword argument syntax for creating NamedTuple classes @@ -699,12 +859,6 @@ Pending Removal in Python 3.15 They will be removed in Python 3.15. (Contributed by Victor Stinner in :gh:`105096`.) -* Passing any arguments to :func:`threading.RLock` is now deprecated. - C version allows any numbers of args and kwargs, - but they are just ignored. Python version does not allow any arguments. - All arguments will be removed from :func:`threading.RLock` in Python 3.15. - (Contributed by Nikita Sobolev in :gh:`102029`.) - Pending Removal in Python 3.16 ------------------------------ @@ -751,6 +905,9 @@ although there is currently no date scheduled for their removal. :data:`calendar.FEBRUARY`. (Contributed by Prince Roshan in :gh:`103636`.) +* :attr:`codeobject.co_lnotab`: use the :meth:`codeobject.co_lines` method + instead. + * :mod:`datetime`: * :meth:`~datetime.datetime.utcnow`: @@ -786,11 +943,13 @@ although there is currently no date scheduled for their removal. underscore. (Contributed by Serhiy Storchaka in :gh:`91760`.) +* :mod:`!sre_compile`, :mod:`!sre_constants` and :mod:`!sre_parse` modules. + * :mod:`ssl` options and protocols: * :class:`ssl.SSLContext` without protocol argument is deprecated. * :class:`ssl.SSLContext`: :meth:`~ssl.SSLContext.set_npn_protocols` and - :meth:`!~ssl.SSLContext.selected_npn_protocol` are deprecated: use ALPN + :meth:`!selected_npn_protocol` are deprecated: use ALPN instead. * ``ssl.OP_NO_SSL*`` options * ``ssl.OP_NO_TLS*`` options @@ -803,12 +962,6 @@ although there is currently no date scheduled for their removal. * ``ssl.TLSVersion.TLSv1`` * ``ssl.TLSVersion.TLSv1_1`` -* :mod:`!sre_compile`, :mod:`!sre_constants` and :mod:`!sre_parse` modules. - -* :attr:`~codeobject.co_lnotab`: use the ``co_lines`` attribute instead. - -* :class:`typing.Text` (:gh:`92332`). - * :func:`sysconfig.is_python_build` *check_home* parameter is deprecated and ignored. @@ -823,14 +976,10 @@ although there is currently no date scheduled for their removal. * :meth:`!threading.currentThread`: use :meth:`threading.current_thread`. * :meth:`!threading.activeCount`: use :meth:`threading.active_count`. -* :class:`unittest.IsolatedAsyncioTestCase`: it is deprecated to return a value - that is not None from a test case. - -* :mod:`urllib.request`: :class:`~urllib.request.URLopener` and - :class:`~urllib.request.FancyURLopener` style of invoking requests is - deprecated. Use newer :func:`~urllib.request.urlopen` functions and methods. +* :class:`typing.Text` (:gh:`92332`). -* :func:`!urllib.parse.to_bytes`. +* :class:`unittest.IsolatedAsyncioTestCase`: it is deprecated to return a value + that is not ``None`` from a test case. * :mod:`urllib.parse` deprecated functions: :func:`~urllib.parse.urlparse` instead @@ -844,6 +993,11 @@ although there is currently no date scheduled for their removal. * ``splittype()`` * ``splituser()`` * ``splitvalue()`` + * ``to_bytes()`` + +* :mod:`urllib.request`: :class:`~urllib.request.URLopener` and + :class:`~urllib.request.FancyURLopener` style of invoking requests is + deprecated. Use newer :func:`~urllib.request.urlopen` functions and methods. * :mod:`wsgiref`: ``SimpleHandler.stdout.write()`` should not do partial writes. @@ -1139,10 +1293,10 @@ Changes in the Python API * Functions :c:func:`PyDict_GetItem`, :c:func:`PyDict_GetItemString`, :c:func:`PyMapping_HasKey`, :c:func:`PyMapping_HasKeyString`, :c:func:`PyObject_HasAttr`, :c:func:`PyObject_HasAttrString`, and - :c:func:`PySys_GetObject`, which clear all errors occurred during calling - the function, report now them using :func:`sys.unraisablehook`. - You can consider to replace these functions with other functions as - recomended in the documentation. + :c:func:`PySys_GetObject`, which clear all errors which occurred when calling + them, now report them using :func:`sys.unraisablehook`. + You may replace them with other functions as + recommended in the documentation. (Contributed by Serhiy Storchaka in :gh:`106672`.) * An :exc:`OSError` is now raised by :func:`getpass.getuser` for any failure to @@ -1151,7 +1305,7 @@ Changes in the Python API * The :mod:`threading` module now expects the :mod:`!_thread` module to have an ``_is_main_interpreter`` attribute. It is a function with no - arguments that returns ``True`` if the current interpreter is the + arguments that return ``True`` if the current interpreter is the main interpreter. Any library or application that provides a custom ``_thread`` module @@ -1159,6 +1313,14 @@ Changes in the Python API other "private" attributes. (See :gh:`112826`.) +* :class:`mailbox.Maildir` now ignores files with a leading dot. + (Contributed by Zackery Spytz in :gh:`65559`.) + +* :meth:`pathlib.Path.glob` and :meth:`~pathlib.Path.rglob` now return both + files and directories if a pattern that ends with "``**``" is given, rather + than directories only. Users may add a trailing slash to match only + directories. + Build Changes ============= @@ -1171,7 +1333,7 @@ Build Changes (Contributed by Erlend Aasland in :gh:`105875`.) * Python built with :file:`configure` :option:`--with-trace-refs` (tracing - references) is now ABI compatible with Python release build and + references) is now ABI compatible with the Python release build and :ref:`debug build <debug-build>`. (Contributed by Victor Stinner in :gh:`108634`.) @@ -1184,6 +1346,12 @@ Build Changes :ref:`limited C API <limited-c-api>`. (Contributed by Victor Stinner in :gh:`85283`.) +* ``wasm32-wasi`` is now a tier 2 platform. + (Contributed by Brett Cannon in :gh:`115192`.) + +* ``wasm32-emscripten`` is no longer a supported platform. + (Contributed by Brett Cannon in :gh:`115192`.) + C API Changes ============= @@ -1198,7 +1366,7 @@ New Features (Contributed by Inada Naoki in :gh:`104922`.) * The *keywords* parameter of :c:func:`PyArg_ParseTupleAndKeywords` and - :c:func:`PyArg_VaParseTupleAndKeywords` has now type :c:expr:`char * const *` + :c:func:`PyArg_VaParseTupleAndKeywords` now has type :c:expr:`char * const *` in C and :c:expr:`const char * const *` in C++, instead of :c:expr:`char **`. It makes these functions compatible with arguments of type :c:expr:`const char * const *`, :c:expr:`const char **` or @@ -1255,18 +1423,28 @@ New Features always steals a reference to the value. (Contributed by Serhiy Storchaka in :gh:`86493`.) -* Added :c:func:`PyDict_GetItemRef` and :c:func:`PyDict_GetItemStringRef` +* Add :c:func:`PyDict_GetItemRef` and :c:func:`PyDict_GetItemStringRef` functions: similar to :c:func:`PyDict_GetItemWithError` but returning a :term:`strong reference` instead of a :term:`borrowed reference`. Moreover, these functions return -1 on error and so checking ``PyErr_Occurred()`` is not needed. (Contributed by Victor Stinner in :gh:`106004`.) -* Added :c:func:`PyDict_ContainsString` function: same as +* Added :c:func:`PyDict_SetDefaultRef`, which is similar to + :c:func:`PyDict_SetDefault` but returns a :term:`strong reference` instead of + a :term:`borrowed reference`. This function returns ``-1`` on error, ``0`` on + insertion, and ``1`` if the key was already present in the dictionary. + (Contributed by Sam Gross in :gh:`112066`.) + +* Add :c:func:`PyDict_ContainsString` function: same as :c:func:`PyDict_Contains`, but *key* is specified as a :c:expr:`const char*` UTF-8 encoded bytes string, rather than a :c:expr:`PyObject*`. (Contributed by Victor Stinner in :gh:`108314`.) +* Added :c:func:`PyList_GetItemRef` function: similar to + :c:func:`PyList_GetItem` but returns a :term:`strong reference` instead of + a :term:`borrowed reference`. + * Add :c:func:`Py_IsFinalizing` function: check if the main Python interpreter is :term:`shutting down <interpreter shutdown>`. (Contributed by Victor Stinner in :gh:`108014`.) @@ -1320,7 +1498,7 @@ New Features (Contributed by Victor Stinner in :gh:`85283`.) * Add :c:func:`PyErr_FormatUnraisable` function: similar to - :c:func:`PyErr_WriteUnraisable`, but allow to customize the warning mesage. + :c:func:`PyErr_WriteUnraisable`, but allow customizing the warning message. (Contributed by Serhiy Storchaka in :gh:`108082`.) * Add :c:func:`PyList_Extend` and :c:func:`PyList_Clear` functions: similar to @@ -1330,7 +1508,7 @@ New Features * Add :c:func:`PyDict_Pop` and :c:func:`PyDict_PopString` functions: remove a key from a dictionary and optionally return the removed value. This is similar to :meth:`dict.pop`, but without the default value and not raising - :exc:`KeyError` if the key missing. + :exc:`KeyError` if the key is missing. (Contributed by Stefan Behnel and Victor Stinner in :gh:`111262`.) * Add :c:func:`Py_HashPointer` function to hash a pointer. @@ -1443,7 +1621,7 @@ Removed ------- * Removed chained :class:`classmethod` descriptors (introduced in - :issue:`19072`). This can no longer be used to wrap other descriptors + :gh:`63272`). This can no longer be used to wrap other descriptors such as :class:`property`. The core design of this feature was flawed and caused a number of downstream problems. To "pass-through" a :class:`classmethod`, consider using the :attr:`!__wrapped__` @@ -1457,14 +1635,14 @@ Removed add ``cc @vstinner`` to the issue to notify Victor Stinner. (Contributed by Victor Stinner in :gh:`106320`.) -* Remove functions deprecated in Python 3.9. +* Remove functions deprecated in Python 3.9: * ``PyEval_CallObject()``, ``PyEval_CallObjectWithKeywords()``: use :c:func:`PyObject_CallNoArgs` or :c:func:`PyObject_Call` instead. Warning: :c:func:`PyObject_Call` positional arguments must be a - :class:`tuple` and must not be *NULL*, keyword arguments must be a - :class:`dict` or *NULL*, whereas removed functions checked arguments type - and accepted *NULL* positional and keyword arguments. + :class:`tuple` and must not be ``NULL``, keyword arguments must be a + :class:`dict` or ``NULL``, whereas removed functions checked arguments type + and accepted ``NULL`` positional and keyword arguments. To replace ``PyEval_CallObjectWithKeywords(func, NULL, kwargs)`` with :c:func:`PyObject_Call`, pass an empty tuple as positional arguments using :c:func:`PyTuple_New(0) <PyTuple_New>`. diff --git a/Doc/whatsnew/3.2.rst b/Doc/whatsnew/3.2.rst index 5ef76968e9d86b..9834bc03dc4b74 100644 --- a/Doc/whatsnew/3.2.rst +++ b/Doc/whatsnew/3.2.rst @@ -183,7 +183,7 @@ PEP 391: Dictionary Based Configuration for Logging The :mod:`logging` module provided two kinds of configuration, one style with function calls for each option or another style driven by an external file saved -in a :mod:`ConfigParser` format. Those options did not provide the flexibility +in a :mod:`configparser` format. Those options did not provide the flexibility to create configurations from JSON or YAML files, nor did they support incremental configuration, which is needed for specifying logger options from a command line. @@ -1858,12 +1858,12 @@ structure. asyncore -------- -:class:`asyncore.dispatcher` now provides a -:meth:`~asyncore.dispatcher.handle_accepted()` method +:class:`!asyncore.dispatcher` now provides a +:meth:`!handle_accepted()` method returning a ``(sock, addr)`` pair which is called when a connection has actually been established with a new remote endpoint. This is supposed to be used as a -replacement for old :meth:`~asyncore.dispatcher.handle_accept()` and avoids -the user to call :meth:`~asyncore.dispatcher.accept()` directly. +replacement for old :meth:`!handle_accept()` and avoids +the user to call :meth:`!accept()` directly. (Contributed by Giampaolo Rodolà; :issue:`6706`.) @@ -2134,7 +2134,7 @@ configparser The :mod:`configparser` module was modified to improve usability and predictability of the default parser and its supported INI syntax. The old -:class:`ConfigParser` class was removed in favor of :class:`SafeConfigParser` +:class:`!ConfigParser` class was removed in favor of :class:`!SafeConfigParser` which has in turn been renamed to :class:`~configparser.ConfigParser`. Support for inline comments is now turned off by default and section or option duplicates are not allowed in a single configuration source. @@ -2414,7 +2414,7 @@ when one operand is much larger than the other (patch by Andress Bennetts in (:issue:`1569291` by Alexander Belopolsky). The :class:`BaseHTTPRequestHandler` has more efficient buffering (:issue:`3709` by Andrew Schaaf). The :func:`operator.attrgetter` function has been sped-up (:issue:`10160` by -Christos Georgiou). And :class:`ConfigParser` loads multi-line arguments a bit +Christos Georgiou). And :class:`~configparser.ConfigParser` loads multi-line arguments a bit faster (:issue:`7113` by Łukasz Langa). @@ -2614,8 +2614,8 @@ This section lists previously described changes and other bugfixes that may require changes to your code: * The :mod:`configparser` module has a number of clean-ups. The major change is - to replace the old :class:`ConfigParser` class with long-standing preferred - alternative :class:`SafeConfigParser`. In addition there are a number of + to replace the old :class:`!ConfigParser` class with long-standing preferred + alternative :class:`!SafeConfigParser`. In addition there are a number of smaller incompatibilities: * The interpolation syntax is now validated on @@ -2737,8 +2737,8 @@ require changes to your code: thread-state aware APIs (such as :c:func:`PyEval_SaveThread` and :c:func:`PyEval_RestoreThread`) should be used instead. -* Due to security risks, :func:`asyncore.handle_accept` has been deprecated, and - a new function, :func:`asyncore.handle_accepted`, was added to replace it. +* Due to security risks, :func:`!asyncore.handle_accept` has been deprecated, and + a new function, :func:`!asyncore.handle_accepted`, was added to replace it. (Contributed by Giampaolo Rodola in :issue:`6706`.) diff --git a/Doc/whatsnew/3.3.rst b/Doc/whatsnew/3.3.rst index 5674bc7f359b72..29b4034e32821c 100644 --- a/Doc/whatsnew/3.3.rst +++ b/Doc/whatsnew/3.3.rst @@ -174,7 +174,7 @@ Features b or c are now hashable. (Contributed by Antoine Pitrou in :issue:`13411`.) * Arbitrary slicing of any 1-D arrays type is supported. For example, it - is now possible to reverse a memoryview in O(1) by using a negative step. + is now possible to reverse a memoryview in *O*\ (1) by using a negative step. API changes ----------- @@ -1052,7 +1052,7 @@ their ``__init__`` method (for example, file objects) or in their crypt ----- -Addition of salt and modular crypt format (hashing method) and the :func:`~!crypt.mksalt` +Addition of salt and modular crypt format (hashing method) and the :func:`!mksalt` function to the :mod:`!crypt` module. (:issue:`10924`) @@ -1845,7 +1845,7 @@ signal smtpd ----- -The :mod:`smtpd` module now supports :rfc:`5321` (extended SMTP) and :rfc:`1870` +The :mod:`!smtpd` module now supports :rfc:`5321` (extended SMTP) and :rfc:`1870` (size extension). Per the standard, these extensions are enabled if and only if the client initiates the session with an ``EHLO`` command. diff --git a/Doc/whatsnew/3.4.rst b/Doc/whatsnew/3.4.rst index 72d12461d8f730..e07eda985d9bad 100644 --- a/Doc/whatsnew/3.4.rst +++ b/Doc/whatsnew/3.4.rst @@ -605,15 +605,15 @@ Using ``ABC`` as a base class has essentially the same effect as specifying aifc ---- -The :meth:`~!aifc.aifc.getparams` method now returns a namedtuple rather than a +The :meth:`!getparams` method now returns a namedtuple rather than a plain tuple. (Contributed by Claudiu Popa in :issue:`17818`.) :func:`!aifc.open` now supports the context management protocol: when used in a -:keyword:`with` block, the :meth:`~!aifc.aifc.close` method of the returned +:keyword:`with` block, the :meth:`!close` method of the returned object will be called automatically at the end of the block. (Contributed by Serhiy Storchacha in :issue:`16486`.) -The :meth:`~!aifc.aifc.writeframesraw` and :meth:`~!aifc.aifc.writeframes` +The :meth:`!writeframesraw` and :meth:`!writeframes` methods now accept any :term:`bytes-like object`. (Contributed by Serhiy Storchaka in :issue:`8311`.) @@ -632,7 +632,7 @@ audioop :mod:`!audioop` now supports 24-bit samples. (Contributed by Serhiy Storchaka in :issue:`12866`.) -New :func:`~!audioop.byteswap` function converts big-endian samples to +New :func:`!byteswap` function converts big-endian samples to little-endian and vice versa. (Contributed by Serhiy Storchaka in :issue:`19641`.) @@ -1369,9 +1369,9 @@ error. (Contributed by Atsuo Ishimoto and Hynek Schlawack in smtpd ----- -The :class:`~smtpd.SMTPServer` and :class:`~smtpd.SMTPChannel` classes now +The :class:`!SMTPServer` and :class:`!SMTPChannel` classes now accept a *map* keyword argument which, if specified, is passed in to -:class:`asynchat.async_chat` as its *map* argument. This allows an application +:class:`!asynchat.async_chat` as its *map* argument. This allows an application to avoid affecting the global socket map. (Contributed by Vinay Sajip in :issue:`11959`.) @@ -1528,7 +1528,7 @@ work on Windows. This change was actually inadvertently made in 3.3.4. sunau ----- -The :meth:`~!sunau.getparams` method now returns a namedtuple rather than a +The :meth:`!getparams` method now returns a namedtuple rather than a plain tuple. (Contributed by Claudiu Popa in :issue:`18901`.) :meth:`!sunau.open` now supports the context management protocol: when used in a @@ -1540,8 +1540,8 @@ in :issue:`18878`.) support for writing 24 sample using the module. (Contributed by Serhiy Storchaka in :issue:`19261`.) -The :meth:`~!sunau.AU_write.writeframesraw` and -:meth:`~!sunau.AU_write.writeframes` methods now accept any :term:`bytes-like +The :meth:`!writeframesraw` and +:meth:`!writeframes` methods now accept any :term:`bytes-like object`. (Contributed by Serhiy Storchaka in :issue:`8311`.) @@ -2370,7 +2370,7 @@ Changes in the Python API :issue:`18011`.) Note: this change was also inadvertently applied in Python 3.3.3. -* The :attr:`~cgi.FieldStorage.file` attribute is now automatically closed when +* The :attr:`!file` attribute is now automatically closed when the creating :class:`!cgi.FieldStorage` instance is garbage collected. If you were pulling the file object out separately from the :class:`!cgi.FieldStorage` instance and not keeping the instance alive, then you should either store the diff --git a/Doc/whatsnew/3.5.rst b/Doc/whatsnew/3.5.rst index a32866094ffeb5..5c2ec230441b42 100644 --- a/Doc/whatsnew/3.5.rst +++ b/Doc/whatsnew/3.5.rst @@ -878,7 +878,7 @@ size of decompressed data. (Contributed by Nikolaus Rath in :issue:`15955`.) cgi --- -The :class:`~cgi.FieldStorage` class now supports the :term:`context manager` +The :class:`!FieldStorage` class now supports the :term:`context manager` protocol. (Contributed by Berker Peksag in :issue:`20289`.) @@ -1252,7 +1252,7 @@ Oberkirch in :issue:`21800`.) imghdr ------ -The :func:`~!imghdr.what` function now recognizes the +The :func:`!what` function now recognizes the `OpenEXR <https://www.openexr.com>`_ format (contributed by Martin Vignali and Claudiu Popa in :issue:`20295`), and the `WebP <https://en.wikipedia.org/wiki/WebP>`_ format @@ -1663,34 +1663,34 @@ during debugging, instead of integer "magic numbers". smtpd ----- -Both the :class:`~smtpd.SMTPServer` and :class:`~smtpd.SMTPChannel` classes now +Both the :class:`!SMTPServer` and :class:`!SMTPChannel` classes now accept a *decode_data* keyword argument to determine if the ``DATA`` portion of the SMTP transaction is decoded using the ``"utf-8"`` codec or is instead provided to the -:meth:`SMTPServer.process_message() <smtpd.SMTPServer.process_message>` +:meth:`!SMTPServer.process_message()` method as a byte string. The default is ``True`` for backward compatibility reasons, but will change to ``False`` in Python 3.6. If *decode_data* is set to ``False``, the ``process_message`` method must be prepared to accept keyword arguments. (Contributed by Maciej Szulik in :issue:`19662`.) -The :class:`~smtpd.SMTPServer` class now advertises the ``8BITMIME`` extension +The :class:`!SMTPServer` class now advertises the ``8BITMIME`` extension (:rfc:`6152`) if *decode_data* has been set ``True``. If the client specifies ``BODY=8BITMIME`` on the ``MAIL`` command, it is passed to -:meth:`SMTPServer.process_message() <smtpd.SMTPServer.process_message>` +:meth:`!SMTPServer.process_message()` via the *mail_options* keyword. (Contributed by Milan Oberkirch and R. David Murray in :issue:`21795`.) -The :class:`~smtpd.SMTPServer` class now also supports the ``SMTPUTF8`` +The :class:`!SMTPServer` class now also supports the ``SMTPUTF8`` extension (:rfc:`6531`: Internationalized Email). If the client specified ``SMTPUTF8 BODY=8BITMIME`` on the ``MAIL`` command, they are passed to -:meth:`SMTPServer.process_message() <smtpd.SMTPServer.process_message>` +:meth:`!SMTPServer.process_message()` via the *mail_options* keyword. It is the responsibility of the ``process_message`` method to correctly handle the ``SMTPUTF8`` data. (Contributed by Milan Oberkirch in :issue:`21725`.) It is now possible to provide, directly or via name resolution, IPv6 -addresses in the :class:`~smtpd.SMTPServer` constructor, and have it +addresses in the :class:`!SMTPServer` constructor, and have it successfully connect. (Contributed by Milan Oberkirch in :issue:`14758`.) @@ -1714,7 +1714,7 @@ support :rfc:`6531` (SMTPUTF8). sndhdr ------ -The :func:`~sndhdr.what` and :func:`~sndhdr.whathdr` functions now return +The :func:`!what` and :func:`!whathdr` functions now return a :func:`~collections.namedtuple`. (Contributed by Claudiu Popa in :issue:`18615`.) @@ -2296,9 +2296,9 @@ slated for removal in Python 3.6. The :func:`asyncio.async` function is deprecated in favor of :func:`~asyncio.ensure_future`. -The :mod:`smtpd` module has in the past always decoded the DATA portion of +The :mod:`!smtpd` module has in the past always decoded the DATA portion of email messages using the ``utf-8`` codec. This can now be controlled by the -new *decode_data* keyword to :class:`~smtpd.SMTPServer`. The default value is +new *decode_data* keyword to :class:`!SMTPServer`. The default value is ``True``, but this default is deprecated. Specify the *decode_data* keyword with an appropriate value to avoid the deprecation warning. @@ -2418,7 +2418,7 @@ Changes in the Python API (Contributed by Victor Stinner in :issue:`21205`.) * The deprecated "strict" mode and argument of :class:`~html.parser.HTMLParser`, - :meth:`HTMLParser.error`, and the :exc:`HTMLParserError` exception have been + :meth:`!HTMLParser.error`, and the :exc:`!HTMLParserError` exception have been removed. (Contributed by Ezio Melotti in :issue:`15114`.) The *convert_charrefs* argument of :class:`~html.parser.HTMLParser` is now ``True`` by default. (Contributed by Berker Peksag in :issue:`21047`.) diff --git a/Doc/whatsnew/3.6.rst b/Doc/whatsnew/3.6.rst index 5a3cea0ec87cb2..11e1d73232a96d 100644 --- a/Doc/whatsnew/3.6.rst +++ b/Doc/whatsnew/3.6.rst @@ -1961,14 +1961,14 @@ Deprecated Python modules, functions and methods asynchat ~~~~~~~~ -The :mod:`asynchat` has been deprecated in favor of :mod:`asyncio`. +The :mod:`!asynchat` has been deprecated in favor of :mod:`asyncio`. (Contributed by Mariatta in :issue:`25002`.) asyncore ~~~~~~~~ -The :mod:`asyncore` has been deprecated in favor of :mod:`asyncio`. +The :mod:`!asyncore` has been deprecated in favor of :mod:`asyncio`. (Contributed by Mariatta in :issue:`25002`.) @@ -2189,7 +2189,7 @@ Changes in the Python API :mod:`calendar`, :mod:`!cgi`, :mod:`csv`, :mod:`~xml.etree.ElementTree`, :mod:`enum`, :mod:`fileinput`, :mod:`ftplib`, :mod:`logging`, :mod:`mailbox`, - :mod:`mimetypes`, :mod:`optparse`, :mod:`plistlib`, :mod:`smtpd`, + :mod:`mimetypes`, :mod:`optparse`, :mod:`plistlib`, :mod:`!smtpd`, :mod:`subprocess`, :mod:`tarfile`, :mod:`threading` and :mod:`wave`. This means they will export new symbols when ``import *`` is used. @@ -2219,11 +2219,11 @@ Changes in the Python API an error (e.g. ``EBADF``) was reported by the underlying system call. (Contributed by Martin Panter in :issue:`26685`.) -* The *decode_data* argument for the :class:`smtpd.SMTPChannel` and - :class:`smtpd.SMTPServer` constructors is now ``False`` by default. +* The *decode_data* argument for the :class:`!smtpd.SMTPChannel` and + :class:`!smtpd.SMTPServer` constructors is now ``False`` by default. This means that the argument passed to - :meth:`~smtpd.SMTPServer.process_message` is now a bytes object by - default, and ``process_message()`` will be passed keyword arguments. + :meth:`!process_message` is now a bytes object by + default, and :meth:`!process_message` will be passed keyword arguments. Code that has already been updated in accordance with the deprecation warning generated by 3.5 will not be affected. diff --git a/Doc/whatsnew/3.7.rst b/Doc/whatsnew/3.7.rst index 616e51571388a8..402b15a277e53d 100644 --- a/Doc/whatsnew/3.7.rst +++ b/Doc/whatsnew/3.7.rst @@ -851,7 +851,7 @@ crypt The :mod:`!crypt` module now supports the Blowfish hashing method. (Contributed by Serhiy Storchaka in :issue:`31664`.) -The :func:`~!crypt.mksalt` function now allows specifying the number of rounds +The :func:`!mksalt` function now allows specifying the number of rounds for hashing. (Contributed by Serhiy Storchaka in :issue:`31702`.) @@ -2004,15 +2004,15 @@ importlib --------- Methods -:meth:`MetaPathFinder.find_module() <!importlib.abc.MetaPathFinder.find_module>` +:meth:`!MetaPathFinder.find_module()` (replaced by :meth:`MetaPathFinder.find_spec() <importlib.abc.MetaPathFinder.find_spec>`) and -:meth:`PathEntryFinder.find_loader() <!importlib.abc.PathEntryFinder.find_loader>` +:meth:`!PathEntryFinder.find_loader()` (replaced by :meth:`PathEntryFinder.find_spec() <importlib.abc.PathEntryFinder.find_spec>`) both deprecated in Python 3.4 now emit :exc:`DeprecationWarning`. -(Contributed by Matthias Bussonnier in :issue:`29576`) +(Contributed by Matthias Bussonnier in :issue:`29576`.) The :class:`importlib.abc.ResourceLoader` ABC has been deprecated in favour of :class:`importlib.abc.ResourceReader`. @@ -2304,9 +2304,9 @@ Changes in the Python API * The :attr:`struct.Struct.format` type is now :class:`str` instead of :class:`bytes`. (Contributed by Victor Stinner in :issue:`21071`.) -* :func:`~cgi.parse_multipart` now accepts the *encoding* and *errors* +* :func:`!cgi.parse_multipart` now accepts the *encoding* and *errors* arguments and returns the same results as - :class:`~FieldStorage`: for non-file fields, the value associated to a key + :class:`!FieldStorage`: for non-file fields, the value associated to a key is a list of strings, not bytes. (Contributed by Pierre Quentel in :issue:`29979`.) diff --git a/Doc/whatsnew/3.8.rst b/Doc/whatsnew/3.8.rst index e4dcb9bf872e28..b041e592d61ed1 100644 --- a/Doc/whatsnew/3.8.rst +++ b/Doc/whatsnew/3.8.rst @@ -754,8 +754,8 @@ datetime -------- Added new alternate constructors :meth:`datetime.date.fromisocalendar` and -:meth:`datetime.datetime.fromisocalendar`, which construct :class:`date` and -:class:`datetime` objects respectively from ISO year, week number, and weekday; +:meth:`datetime.datetime.fromisocalendar`, which construct :class:`~datetime.date` and +:class:`~datetime.datetime` objects respectively from ISO year, week number, and weekday; these are the inverse of each class's ``isocalendar`` method. (Contributed by Paul Ganssle in :issue:`36004`.) @@ -1086,7 +1086,7 @@ pathlib contain characters unrepresentable at the OS level. (Contributed by Serhiy Storchaka in :issue:`33721`.) -Added :meth:`pathlib.Path.link_to()` which creates a hard link pointing +Added :meth:`!pathlib.Path.link_to()` which creates a hard link pointing to a path. (Contributed by Joannah Nanjekye in :issue:`26978`) Note that ``link_to`` was deprecated in 3.10 and removed in 3.12 in @@ -1623,8 +1623,8 @@ Build and C API Changes allocation or deallocation may need to be adjusted. (Contributed by Eddie Elizondo in :issue:`35810`.) -* The new function :c:func:`PyCode_NewWithPosOnlyArgs` allows to create - code objects like :c:func:`PyCode_New`, but with an extra *posonlyargcount* +* The new function :c:func:`!PyCode_NewWithPosOnlyArgs` allows to create + code objects like :c:func:`!PyCode_New`, but with an extra *posonlyargcount* parameter for indicating the number of positional-only arguments. (Contributed by Pablo Galindo in :issue:`37221`.) diff --git a/Doc/whatsnew/3.9.rst b/Doc/whatsnew/3.9.rst index cb2482ee48d7fa..f7ad4372325ccb 100644 --- a/Doc/whatsnew/3.9.rst +++ b/Doc/whatsnew/3.9.rst @@ -585,7 +585,7 @@ queue. nntplib ------- -:class:`~!nntplib.NNTP` and :class:`~!nntplib.NNTP_SSL` now raise a :class:`ValueError` +:class:`!NNTP` and :class:`!NNTP_SSL` now raise a :class:`ValueError` if the given timeout for their constructor is zero to prevent the creation of a non-blocking socket. (Contributed by Donghee Na in :issue:`39259`.) @@ -931,7 +931,7 @@ Deprecated * Passing ``None`` as the first argument to the :func:`shlex.split` function has been deprecated. (Contributed by Zackery Spytz in :issue:`33262`.) -* :func:`smtpd.MailmanProxy` is now deprecated as it is unusable without +* :func:`!smtpd.MailmanProxy` is now deprecated as it is unusable without an external module, ``mailman``. (Contributed by Samuel Colvin in :issue:`35800`.) * The :mod:`!lib2to3` module now emits a :exc:`PendingDeprecationWarning`. diff --git a/Grammar/python.gram b/Grammar/python.gram index c9269b058bc7f6..174b4dbb6f7842 100644 --- a/Grammar/python.gram +++ b/Grammar/python.gram @@ -968,6 +968,8 @@ for_if_clause[comprehension_ty]: CHECK_VERSION(comprehension_ty, 6, "Async comprehensions are", _PyAST_comprehension(a, b, c, 1, p->arena)) } | 'for' a=star_targets 'in' ~ b=disjunction c[asdl_expr_seq*]=('if' z=disjunction { z })* { _PyAST_comprehension(a, b, c, 0, p->arena) } + | 'async'? 'for' (bitwise_or (',' bitwise_or)* [',']) !'in' { + RAISE_SYNTAX_ERROR("'in' expected after for-loop variables") } | invalid_for_target listcomp[expr_ty]: diff --git a/Include/cpython/code.h b/Include/cpython/code.h index cf715c55a2b3b8..1f47d99fb60443 100644 --- a/Include/cpython/code.h +++ b/Include/cpython/code.h @@ -208,6 +208,8 @@ struct PyCodeObject _PyCode_DEF(1); #define CO_FUTURE_GENERATOR_STOP 0x800000 #define CO_FUTURE_ANNOTATIONS 0x1000000 +#define CO_NO_MONITORING_EVENTS 0x2000000 + /* This should be defined if a future statement modifies the syntax. For example, when a keyword is added. */ diff --git a/Include/cpython/dictobject.h b/Include/cpython/dictobject.h index 944965fb9e5351..35b6a822a0dfff 100644 --- a/Include/cpython/dictobject.h +++ b/Include/cpython/dictobject.h @@ -17,6 +17,9 @@ typedef struct { /* Dictionary version: globally unique, value change each time the dictionary is modified */ #ifdef Py_BUILD_CORE + /* Bits 0-7 are for dict watchers. + * Bits 8-11 are for the watched mutation counter (used by tier2 optimization) + * The remaining bits (12-63) are the actual version tag. */ uint64_t ma_version_tag; #else Py_DEPRECATED(3.12) uint64_t ma_version_tag; @@ -38,6 +41,16 @@ PyAPI_FUNC(PyObject *) _PyDict_GetItemStringWithError(PyObject *, const char *); PyAPI_FUNC(PyObject *) PyDict_SetDefault( PyObject *mp, PyObject *key, PyObject *defaultobj); +// Inserts `key` with a value `default_value`, if `key` is not already present +// in the dictionary. If `result` is not NULL, then the value associated +// with `key` is returned in `*result` (either the existing value, or the now +// inserted `default_value`). +// Returns: +// -1 on error +// 0 if `key` was not present and `default_value` was inserted +// 1 if `key` was present and `default_value` was not inserted +PyAPI_FUNC(int) PyDict_SetDefaultRef(PyObject *mp, PyObject *key, PyObject *default_value, PyObject **result); + /* Get the number of items of a dictionary. */ static inline Py_ssize_t PyDict_GET_SIZE(PyObject *op) { PyDictObject *mp; diff --git a/Include/cpython/listobject.h b/Include/cpython/listobject.h index 8ade1b164681f9..49f5e8d6d1a0d6 100644 --- a/Include/cpython/listobject.h +++ b/Include/cpython/listobject.h @@ -29,7 +29,11 @@ typedef struct { static inline Py_ssize_t PyList_GET_SIZE(PyObject *op) { PyListObject *list = _PyList_CAST(op); +#ifdef Py_GIL_DISABLED + return _Py_atomic_load_ssize_relaxed(&(_PyVarObject_CAST(list)->ob_size)); +#else return Py_SIZE(list); +#endif } #define PyList_GET_SIZE(op) PyList_GET_SIZE(_PyObject_CAST(op)) diff --git a/Include/cpython/object.h b/Include/cpython/object.h index 762e8a3b86ee1e..7512bb70c760fd 100644 --- a/Include/cpython/object.h +++ b/Include/cpython/object.h @@ -4,6 +4,7 @@ PyAPI_FUNC(void) _Py_NewReference(PyObject *op); PyAPI_FUNC(void) _Py_NewReferenceNoTotal(PyObject *op); +PyAPI_FUNC(void) _Py_ResurrectReference(PyObject *op); #ifdef Py_REF_DEBUG /* These are useful as debugging aids when chasing down refleaks. */ @@ -39,6 +40,10 @@ typedef struct _Py_Identifier { // Index in PyInterpreterState.unicode.ids.array. It is process-wide // unique and must be initialized to -1. Py_ssize_t index; + // Hidden PyMutex struct for non free-threaded build. + struct { + uint8_t v; + } mutex; } _Py_Identifier; #ifndef Py_BUILD_CORE @@ -224,6 +229,7 @@ struct _typeobject { /* bitset of which type-watchers care about this type */ unsigned char tp_watched; + uint16_t tp_versions_used; }; /* This struct is used by the specializer diff --git a/Include/cpython/optimizer.h b/Include/cpython/optimizer.h index d521eac79d1b97..5a9ccaea3b2209 100644 --- a/Include/cpython/optimizer.h +++ b/Include/cpython/optimizer.h @@ -29,18 +29,28 @@ typedef struct { _PyExecutorLinkListNode links; } _PyVMData; +typedef struct { + uint16_t opcode; + uint16_t oparg; + uint32_t target; + uint64_t operand; // A cache entry +} _PyUOpInstruction; + typedef struct _PyExecutorObject { PyObject_VAR_HEAD - /* WARNING: execute consumes a reference to self. This is necessary to allow executors to tail call into each other. */ - _Py_CODEUNIT *(*execute)(struct _PyExecutorObject *self, struct _PyInterpreterFrame *frame, PyObject **stack_pointer); _PyVMData vm_data; /* Used by the VM, but opaque to the optimizer */ - /* Data needed by the executor goes here, but is opaque to the VM */ + void *jit_code; + size_t jit_size; + _PyUOpInstruction trace[1]; } _PyExecutorObject; typedef struct _PyOptimizerObject _PyOptimizerObject; /* Should return > 0 if a new executor is created. O if no executor is produced and < 0 if an error occurred. */ -typedef int (*optimize_func)(_PyOptimizerObject* self, PyCodeObject *code, _Py_CODEUNIT *instr, _PyExecutorObject **, int curr_stackentries); +typedef int (*optimize_func)( + _PyOptimizerObject* self, struct _PyInterpreterFrame *frame, + _Py_CODEUNIT *instr, _PyExecutorObject **exec_ptr, + int curr_stackentries); typedef struct _PyOptimizerObject { PyObject_HEAD @@ -52,6 +62,12 @@ typedef struct _PyOptimizerObject { /* Data needed by the optimizer goes here, but is opaque to the VM */ } _PyOptimizerObject; +/** Test support **/ +typedef struct { + _PyOptimizerObject base; + int64_t count; +} _PyCounterOptimizerObject; + PyAPI_FUNC(int) PyUnstable_Replace_Executor(PyCodeObject *code, _Py_CODEUNIT *instr, _PyExecutorObject *executor); PyAPI_FUNC(void) PyUnstable_SetOptimizer(_PyOptimizerObject* optimizer); @@ -61,7 +77,7 @@ PyAPI_FUNC(_PyOptimizerObject *) PyUnstable_GetOptimizer(void); PyAPI_FUNC(_PyExecutorObject *) PyUnstable_GetExecutor(PyCodeObject *code, int offset); int -_PyOptimizer_BackEdge(struct _PyInterpreterFrame *frame, _Py_CODEUNIT *src, _Py_CODEUNIT *dest, PyObject **stack_pointer); +_PyOptimizer_Optimize(struct _PyInterpreterFrame *frame, _Py_CODEUNIT *start, PyObject **stack_pointer); extern _PyOptimizerObject _PyOptimizer_Default; @@ -81,6 +97,9 @@ PyAPI_FUNC(PyObject *)PyUnstable_Optimizer_NewUOpOptimizer(void); /* Minimum of 16 additional executions before retry */ #define MINIMUM_TIER2_BACKOFF 4 +#define _Py_MAX_ALLOWED_BUILTINS_MODIFICATIONS 3 +#define _Py_MAX_ALLOWED_GLOBALS_MODIFICATIONS 6 + #ifdef __cplusplus } #endif diff --git a/Include/cpython/pystate.h b/Include/cpython/pystate.h index 56172d231c44f4..9bc8758e72bd8f 100644 --- a/Include/cpython/pystate.h +++ b/Include/cpython/pystate.h @@ -102,7 +102,7 @@ struct _ts { #endif int _whence; - /* Thread state (_Py_THREAD_ATTACHED, _Py_THREAD_DETACHED, _Py_THREAD_GC). + /* Thread state (_Py_THREAD_ATTACHED, _Py_THREAD_DETACHED, _Py_THREAD_SUSPENDED). See Include/internal/pycore_pystate.h for more details. */ int state; @@ -217,15 +217,24 @@ struct _ts { #ifdef Py_DEBUG // A debug build is likely built with low optimization level which implies // higher stack memory usage than a release build: use a lower limit. -# define Py_C_RECURSION_LIMIT 500 +# if defined(__wasi__) + // Based on wasmtime 16. +# define Py_C_RECURSION_LIMIT 150 +# else +# define Py_C_RECURSION_LIMIT 500 +# endif #elif defined(__wasi__) - // WASI has limited call stack. Python's recursion limit depends on code - // layout, optimization, and WASI runtime. Wasmtime can handle about 700 - // recursions, sometimes less. 500 is a more conservative limit. + // Based on wasmtime 16. # define Py_C_RECURSION_LIMIT 500 +#elif defined(__s390x__) +# define Py_C_RECURSION_LIMIT 800 +#elif defined(_WIN32) +# define Py_C_RECURSION_LIMIT 3000 +#elif defined(_Py_ADDRESS_SANITIZER) +# define Py_C_RECURSION_LIMIT 4000 #else // This value is duplicated in Lib/test/support/__init__.py -# define Py_C_RECURSION_LIMIT 1500 +# define Py_C_RECURSION_LIMIT 10000 #endif diff --git a/Include/cpython/pystats.h b/Include/cpython/pystats.h index ba67eefef3e37a..bf0cfe4cb695b4 100644 --- a/Include/cpython/pystats.h +++ b/Include/cpython/pystats.h @@ -122,11 +122,25 @@ typedef struct _optimization_stats { uint64_t optimized_trace_length_hist[_Py_UOP_HIST_SIZE]; } OptimizationStats; +typedef struct _rare_event_stats { + /* Setting an object's class, obj.__class__ = ... */ + uint64_t set_class; + /* Setting the bases of a class, cls.__bases__ = ... */ + uint64_t set_bases; + /* Setting the PEP 523 frame eval function, _PyInterpreterState_SetFrameEvalFunc() */ + uint64_t set_eval_frame_func; + /* Modifying the builtins, __builtins__.__dict__[var] = ... */ + uint64_t builtin_dict; + /* Modifying a function, e.g. func.__defaults__ = ..., etc. */ + uint64_t func_modification; +} RareEventStats; + typedef struct _stats { OpcodeStats opcode_stats[256]; CallStats call_stats; ObjectStats object_stats; OptimizationStats optimization_stats; + RareEventStats rare_event_stats; GCStats *gc_stats; } PyStats; diff --git a/Include/internal/mimalloc/mimalloc/internal.h b/Include/internal/mimalloc/mimalloc/internal.h index f076bc6a40f977..8af841cfdffc01 100644 --- a/Include/internal/mimalloc/mimalloc/internal.h +++ b/Include/internal/mimalloc/mimalloc/internal.h @@ -23,23 +23,6 @@ terms of the MIT license. A copy of the license can be found in the file #define mi_trace_message(...) #endif -#define MI_CACHE_LINE 64 -#if defined(_MSC_VER) -#pragma warning(disable:4127) // suppress constant conditional warning (due to MI_SECURE paths) -#pragma warning(disable:26812) // unscoped enum warning -#define mi_decl_noinline __declspec(noinline) -#define mi_decl_thread __declspec(thread) -#define mi_decl_cache_align __declspec(align(MI_CACHE_LINE)) -#elif (defined(__GNUC__) && (__GNUC__ >= 3)) || defined(__clang__) // includes clang and icc -#define mi_decl_noinline __attribute__((noinline)) -#define mi_decl_thread __thread -#define mi_decl_cache_align __attribute__((aligned(MI_CACHE_LINE))) -#else -#define mi_decl_noinline -#define mi_decl_thread __thread // hope for the best :-) -#define mi_decl_cache_align -#endif - #if defined(__EMSCRIPTEN__) && !defined(__wasi__) #define __wasi__ #endif @@ -85,6 +68,7 @@ mi_threadid_t _mi_thread_id(void) mi_attr_noexcept; mi_heap_t* _mi_heap_main_get(void); // statically allocated main backing heap void _mi_thread_done(mi_heap_t* heap); void _mi_thread_data_collect(void); +void _mi_tld_init(mi_tld_t* tld, mi_heap_t* bheap); // os.c void _mi_os_init(void); // called from process init @@ -130,11 +114,14 @@ void _mi_segment_map_allocated_at(const mi_segment_t* segment); void _mi_segment_map_freed_at(const mi_segment_t* segment); // "segment.c" +extern mi_abandoned_pool_t _mi_abandoned_default; // global abandoned pool mi_page_t* _mi_segment_page_alloc(mi_heap_t* heap, size_t block_size, size_t page_alignment, mi_segments_tld_t* tld, mi_os_tld_t* os_tld); void _mi_segment_page_free(mi_page_t* page, bool force, mi_segments_tld_t* tld); void _mi_segment_page_abandon(mi_page_t* page, mi_segments_tld_t* tld); bool _mi_segment_try_reclaim_abandoned( mi_heap_t* heap, bool try_all, mi_segments_tld_t* tld); void _mi_segment_thread_collect(mi_segments_tld_t* tld); +bool _mi_abandoned_pool_visit_blocks(mi_abandoned_pool_t* pool, uint8_t page_tag, bool visit_blocks, mi_block_visit_fun* visitor, void* arg); + #if MI_HUGE_PAGE_ABANDON void _mi_segment_huge_page_free(mi_segment_t* segment, mi_page_t* page, mi_block_t* block); @@ -144,7 +131,7 @@ void _mi_segment_huge_page_reset(mi_segment_t* segment, mi_page_t* page, m uint8_t* _mi_segment_page_start(const mi_segment_t* segment, const mi_page_t* page, size_t* page_size); // page start for any page void _mi_abandoned_reclaim_all(mi_heap_t* heap, mi_segments_tld_t* tld); -void _mi_abandoned_await_readers(void); +void _mi_abandoned_await_readers(mi_abandoned_pool_t *pool); void _mi_abandoned_collect(mi_heap_t* heap, bool force, mi_segments_tld_t* tld); // "page.c" @@ -170,11 +157,14 @@ size_t _mi_bin_size(uint8_t bin); // for stats uint8_t _mi_bin(size_t size); // for stats // "heap.c" +void _mi_heap_init_ex(mi_heap_t* heap, mi_tld_t* tld, mi_arena_id_t arena_id, bool no_reclaim, uint8_t tag); void _mi_heap_destroy_pages(mi_heap_t* heap); void _mi_heap_collect_abandon(mi_heap_t* heap); void _mi_heap_set_default_direct(mi_heap_t* heap); bool _mi_heap_memid_is_suitable(mi_heap_t* heap, mi_memid_t memid); void _mi_heap_unsafe_destroy_all(void); +void _mi_heap_area_init(mi_heap_area_t* area, mi_page_t* page); +bool _mi_heap_area_visit_blocks(const mi_heap_area_t* area, mi_page_t *page, mi_block_visit_fun* visitor, void* arg); // "stats.c" void _mi_stats_done(mi_stats_t* stats); diff --git a/Include/internal/mimalloc/mimalloc/types.h b/Include/internal/mimalloc/mimalloc/types.h index 7616f37e4b978f..b8cae24507fc5e 100644 --- a/Include/internal/mimalloc/mimalloc/types.h +++ b/Include/internal/mimalloc/mimalloc/types.h @@ -33,6 +33,23 @@ terms of the MIT license. A copy of the license can be found in the file #define MI_MAX_ALIGN_SIZE 16 // sizeof(max_align_t) #endif +#define MI_CACHE_LINE 64 +#if defined(_MSC_VER) +#pragma warning(disable:4127) // suppress constant conditional warning (due to MI_SECURE paths) +#pragma warning(disable:26812) // unscoped enum warning +#define mi_decl_noinline __declspec(noinline) +#define mi_decl_thread __declspec(thread) +#define mi_decl_cache_align __declspec(align(MI_CACHE_LINE)) +#elif (defined(__GNUC__) && (__GNUC__ >= 3)) || defined(__clang__) // includes clang and icc +#define mi_decl_noinline __attribute__((noinline)) +#define mi_decl_thread __thread +#define mi_decl_cache_align __attribute__((aligned(MI_CACHE_LINE))) +#else +#define mi_decl_noinline +#define mi_decl_thread __thread // hope for the best :-) +#define mi_decl_cache_align +#endif + // ------------------------------------------------------ // Variants // ------------------------------------------------------ @@ -294,6 +311,7 @@ typedef struct mi_page_s { uint32_t slice_offset; // distance from the actual page data slice (0 if a page) uint8_t is_committed : 1; // `true` if the page virtual memory is committed uint8_t is_zero_init : 1; // `true` if the page was initially zero initialized + uint8_t tag : 4; // tag from the owning heap // layout like this to optimize access in `mi_malloc` and `mi_free` uint16_t capacity; // number of blocks committed, must be the first field, see `segment.c:page_clear` @@ -445,6 +463,28 @@ typedef struct mi_segment_s { mi_slice_t slices[MI_SLICES_PER_SEGMENT+1]; // one more for huge blocks with large alignment } mi_segment_t; +typedef uintptr_t mi_tagged_segment_t; + +// Segments unowned by any thread are put in a shared pool +typedef struct mi_abandoned_pool_s { + // This is a list of visited abandoned pages that were full at the time. + // this list migrates to `abandoned` when that becomes NULL. The use of + // this list reduces contention and the rate at which segments are visited. + mi_decl_cache_align _Atomic(mi_segment_t*) abandoned_visited; // = NULL + + // The abandoned page list (tagged as it supports pop) + mi_decl_cache_align _Atomic(mi_tagged_segment_t) abandoned; // = NULL + + // Maintain these for debug purposes (these counts may be a bit off) + mi_decl_cache_align _Atomic(size_t) abandoned_count; + mi_decl_cache_align _Atomic(size_t) abandoned_visited_count; + + // We also maintain a count of current readers of the abandoned list + // in order to prevent resetting/decommitting segment memory if it might + // still be read. + mi_decl_cache_align _Atomic(size_t) abandoned_readers; // = 0 +} mi_abandoned_pool_t; + // ------------------------------------------------------ // Heaps @@ -512,6 +552,7 @@ struct mi_heap_s { size_t page_retired_max; // largest retired index into the `pages` array. mi_heap_t* next; // list of heaps per thread bool no_reclaim; // `true` if this heap should not reclaim abandoned pages + uint8_t tag; // custom identifier for this heap }; @@ -654,6 +695,7 @@ typedef struct mi_segments_tld_s { size_t peak_size; // peak size of all segments mi_stats_t* stats; // points to tld stats mi_os_tld_t* os; // points to os stats + mi_abandoned_pool_t* abandoned; // pool of abandoned segments } mi_segments_tld_t; // Thread local data diff --git a/Include/internal/pycore_ast_state.h b/Include/internal/pycore_ast_state.h index 6ffd30aca7b11b..f1b1786264803b 100644 --- a/Include/internal/pycore_ast_state.h +++ b/Include/internal/pycore_ast_state.h @@ -16,8 +16,6 @@ extern "C" { struct ast_state { _PyOnceFlag once; int finalized; - int recursion_depth; - int recursion_limit; PyObject *AST_type; PyObject *Add_singleton; PyObject *Add_type; diff --git a/Include/internal/pycore_brc.h b/Include/internal/pycore_brc.h new file mode 100644 index 00000000000000..3453d83b57ca97 --- /dev/null +++ b/Include/internal/pycore_brc.h @@ -0,0 +1,74 @@ +#ifndef Py_INTERNAL_BRC_H +#define Py_INTERNAL_BRC_H + +#include <stdint.h> +#include "pycore_llist.h" // struct llist_node +#include "pycore_lock.h" // PyMutex +#include "pycore_object_stack.h" // _PyObjectStack + +#ifdef __cplusplus +extern "C" { +#endif + +#ifndef Py_BUILD_CORE +# error "this header requires Py_BUILD_CORE define" +#endif + +#ifdef Py_GIL_DISABLED + +// Prime number to avoid correlations with memory addresses. +#define _Py_BRC_NUM_BUCKETS 257 + +// Hash table bucket +struct _brc_bucket { + // Mutex protects both the bucket and thread state queues in this bucket. + PyMutex mutex; + + // Linked list of _PyThreadStateImpl objects hashed to this bucket. + struct llist_node root; +}; + +// Per-interpreter biased reference counting state +struct _brc_state { + // Hash table of thread states by thread-id. Thread states within a bucket + // are chained using a doubly-linked list. + struct _brc_bucket table[_Py_BRC_NUM_BUCKETS]; +}; + +// Per-thread biased reference counting state +struct _brc_thread_state { + // Linked-list of thread states per hash bucket + struct llist_node bucket_node; + + // Thread-id as determined by _PyThread_Id() + uintptr_t tid; + + // Objects with refcounts to be merged (protected by bucket mutex) + _PyObjectStack objects_to_merge; + + // Local stack of objects to be merged (not accessed by other threads) + _PyObjectStack local_objects_to_merge; +}; + +// Initialize/finalize the per-thread biased reference counting state +void _Py_brc_init_thread(PyThreadState *tstate); +void _Py_brc_remove_thread(PyThreadState *tstate); + +// Initialize per-interpreter state +void _Py_brc_init_state(PyInterpreterState *interp); + +void _Py_brc_after_fork(PyInterpreterState *interp); + +// Enqueues an object to be merged by it's owning thread (tid). This +// steals a reference to the object. +void _Py_brc_queue_object(PyObject *ob); + +// Merge the refcounts of queued objects for the current thread. +void _Py_brc_merge_refcounts(PyThreadState *tstate); + +#endif /* Py_GIL_DISABLED */ + +#ifdef __cplusplus +} +#endif +#endif /* !Py_INTERNAL_BRC_H */ diff --git a/Include/internal/pycore_ceval.h b/Include/internal/pycore_ceval.h index a357bfa3a26064..b158fc9ff5ebc1 100644 --- a/Include/internal/pycore_ceval.h +++ b/Include/internal/pycore_ceval.h @@ -205,6 +205,8 @@ void _PyEval_FrameClearAndPop(PyThreadState *tstate, _PyInterpreterFrame *frame) #define _PY_CALLS_TO_DO_BIT 2 #define _PY_ASYNC_EXCEPTION_BIT 3 #define _PY_GC_SCHEDULED_BIT 4 +#define _PY_EVAL_PLEASE_STOP_BIT 5 +#define _PY_EVAL_EXPLICIT_MERGE_BIT 6 /* Reserve a few bits for future use */ #define _PY_EVAL_EVENTS_BITS 8 diff --git a/Include/internal/pycore_code.h b/Include/internal/pycore_code.h index 73df6c3568ffe0..fdd5918228455d 100644 --- a/Include/internal/pycore_code.h +++ b/Include/internal/pycore_code.h @@ -295,6 +295,7 @@ extern int _PyStaticCode_Init(PyCodeObject *co); _Py_stats->optimization_stats.name[bucket]++; \ } \ } while (0) +#define RARE_EVENT_STAT_INC(name) do { if (_Py_stats) _Py_stats->rare_event_stats.name++; } while (0) // Export for '_opcode' shared extension PyAPI_FUNC(PyObject*) _Py_GetSpecializationStats(void); @@ -313,6 +314,7 @@ PyAPI_FUNC(PyObject*) _Py_GetSpecializationStats(void); #define UOP_STAT_INC(opname, name) ((void)0) #define OPT_UNSUPPORTED_OPCODE(opname) ((void)0) #define OPT_HIST(length, name) ((void)0) +#define RARE_EVENT_STAT_INC(name) ((void)0) #endif // !Py_STATS // Utility functions for reading/writing 32/64-bit values in the inline caches. diff --git a/Include/internal/pycore_condvar.h b/Include/internal/pycore_condvar.h index 34c21aaad43197..ee9533484e8048 100644 --- a/Include/internal/pycore_condvar.h +++ b/Include/internal/pycore_condvar.h @@ -35,14 +35,14 @@ #include <windows.h> // CRITICAL_SECTION /* options */ -/* non-emulated condition variables are provided for those that want - * to target Windows Vista. Modify this macro to enable them. +/* emulated condition variables are provided for those that want + * to target Windows XP or earlier. Modify this macro to enable them. */ #ifndef _PY_EMULATED_WIN_CV -#define _PY_EMULATED_WIN_CV 1 /* use emulated condition variables */ +#define _PY_EMULATED_WIN_CV 0 /* use non-emulated condition variables */ #endif -/* fall back to emulation if not targeting Vista */ +/* fall back to emulation if targeting earlier than Vista */ #if !defined NTDDI_VISTA || NTDDI_VERSION < NTDDI_VISTA #undef _PY_EMULATED_WIN_CV #define _PY_EMULATED_WIN_CV 1 @@ -77,7 +77,7 @@ typedef struct _PyCOND_T #else /* !_PY_EMULATED_WIN_CV */ -/* Use native Win7 primitives if build target is Win7 or higher */ +/* Use native Windows primitives if build target is Vista or higher */ /* SRWLOCK is faster and better than CriticalSection */ typedef SRWLOCK PyMUTEX_T; diff --git a/Include/internal/pycore_context.h b/Include/internal/pycore_context.h index ec884e9e0f55a9..ae5c47f195eb7f 100644 --- a/Include/internal/pycore_context.h +++ b/Include/internal/pycore_context.h @@ -5,6 +5,7 @@ # error "this header requires Py_BUILD_CORE define" #endif +#include "pycore_freelist.h" // _PyFreeListState #include "pycore_hamt.h" // PyHamtObject @@ -13,7 +14,6 @@ extern PyTypeObject _PyContextTokenMissing_Type; /* runtime lifecycle */ PyStatus _PyContext_Init(PyInterpreterState *); -void _PyContext_Fini(PyInterpreterState *); /* other API */ @@ -22,23 +22,6 @@ typedef struct { PyObject_HEAD } _PyContextTokenMissing; -#ifndef WITH_FREELISTS -// without freelists -# define PyContext_MAXFREELIST 0 -#endif - -#ifndef PyContext_MAXFREELIST -# define PyContext_MAXFREELIST 255 -#endif - -struct _Py_context_state { -#if PyContext_MAXFREELIST > 0 - // List of free PyContext objects - PyContext *freelist; - int numfree; -#endif -}; - struct _pycontextobject { PyObject_HEAD PyContext *ctx_prev; diff --git a/Include/internal/pycore_critical_section.h b/Include/internal/pycore_critical_section.h index bf2bbfffc38bd0..38ed8cd69804ba 100644 --- a/Include/internal/pycore_critical_section.h +++ b/Include/internal/pycore_critical_section.h @@ -104,12 +104,37 @@ extern "C" { # define Py_END_CRITICAL_SECTION2() \ _PyCriticalSection2_End(&_cs2); \ } + +// Asserts that the mutex is locked. The mutex must be held by the +// top-most critical section otherwise there's the possibility +// that the mutex would be swalled out in some code paths. +#define _Py_CRITICAL_SECTION_ASSERT_MUTEX_LOCKED(mutex) \ + _PyCriticalSection_AssertHeld(mutex) + +// Asserts that the mutex for the given object is locked. The mutex must +// be held by the top-most critical section otherwise there's the +// possibility that the mutex would be swalled out in some code paths. +#ifdef Py_DEBUG + +#define _Py_CRITICAL_SECTION_ASSERT_OBJECT_LOCKED(op) \ + if (Py_REFCNT(op) != 1) { \ + _Py_CRITICAL_SECTION_ASSERT_MUTEX_LOCKED(&_PyObject_CAST(op)->ob_mutex); \ + } + +#else /* Py_DEBUG */ + +#define _Py_CRITICAL_SECTION_ASSERT_OBJECT_LOCKED(op) + +#endif /* Py_DEBUG */ + #else /* !Py_GIL_DISABLED */ // The critical section APIs are no-ops with the GIL. # define Py_BEGIN_CRITICAL_SECTION(op) # define Py_END_CRITICAL_SECTION() # define Py_BEGIN_CRITICAL_SECTION2(a, b) # define Py_END_CRITICAL_SECTION2() +# define _Py_CRITICAL_SECTION_ASSERT_MUTEX_LOCKED(mutex) +# define _Py_CRITICAL_SECTION_ASSERT_OBJECT_LOCKED(op) #endif /* !Py_GIL_DISABLED */ typedef struct { @@ -236,6 +261,27 @@ _PyCriticalSection2_End(_PyCriticalSection2 *c) PyAPI_FUNC(void) _PyCriticalSection_SuspendAll(PyThreadState *tstate); +#ifdef Py_GIL_DISABLED + +static inline void +_PyCriticalSection_AssertHeld(PyMutex *mutex) { +#ifdef Py_DEBUG + PyThreadState *tstate = _PyThreadState_GET(); + uintptr_t prev = tstate->critical_section; + if (prev & _Py_CRITICAL_SECTION_TWO_MUTEXES) { + _PyCriticalSection2 *cs = (_PyCriticalSection2 *)(prev & ~_Py_CRITICAL_SECTION_MASK); + assert(cs != NULL && (cs->base.mutex == mutex || cs->mutex2 == mutex)); + } + else { + _PyCriticalSection *cs = (_PyCriticalSection *)(tstate->critical_section & ~_Py_CRITICAL_SECTION_MASK); + assert(cs != NULL && cs->mutex == mutex); + } + +#endif +} + +#endif + #ifdef __cplusplus } #endif diff --git a/Include/internal/pycore_dict.h b/Include/internal/pycore_dict.h index d96870e9197bbf..233da058f464d1 100644 --- a/Include/internal/pycore_dict.h +++ b/Include/internal/pycore_dict.h @@ -9,6 +9,7 @@ extern "C" { # error "this header requires Py_BUILD_CORE define" #endif +#include "pycore_freelist.h" // _PyFreeListState #include "pycore_identifier.h" // _Py_Identifier #include "pycore_object.h" // PyDictOrValues @@ -69,7 +70,7 @@ extern PyObject* _PyDictView_Intersect(PyObject* self, PyObject *other); /* runtime lifecycle */ -extern void _PyDict_Fini(PyInterpreterState *interp); +extern void _PyDict_Fini(PyInterpreterState *state); /* other API */ @@ -206,11 +207,17 @@ static inline PyDictUnicodeEntry* DK_UNICODE_ENTRIES(PyDictKeysObject *dk) { #define DK_IS_UNICODE(dk) ((dk)->dk_kind != DICT_KEYS_GENERAL) -#define DICT_VERSION_INCREMENT (1 << DICT_MAX_WATCHERS) -#define DICT_VERSION_MASK (DICT_VERSION_INCREMENT - 1) +#define DICT_VERSION_INCREMENT (1 << (DICT_MAX_WATCHERS + DICT_WATCHED_MUTATION_BITS)) +#define DICT_WATCHER_MASK ((1 << DICT_MAX_WATCHERS) - 1) +#ifdef Py_GIL_DISABLED +#define DICT_NEXT_VERSION(INTERP) \ + (_Py_atomic_add_uint64(&(INTERP)->dict_state.global_version, DICT_VERSION_INCREMENT) + DICT_VERSION_INCREMENT) + +#else #define DICT_NEXT_VERSION(INTERP) \ ((INTERP)->dict_state.global_version += DICT_VERSION_INCREMENT) +#endif void _PyDict_SendEvent(int watcher_bits, @@ -227,7 +234,7 @@ _PyDict_NotifyEvent(PyInterpreterState *interp, PyObject *value) { assert(Py_REFCNT((PyObject*)mp) > 0); - int watcher_bits = mp->ma_version_tag & DICT_VERSION_MASK; + int watcher_bits = mp->ma_version_tag & DICT_WATCHER_MASK; if (watcher_bits) { _PyDict_SendEvent(watcher_bits, event, mp, key, value); return DICT_NEXT_VERSION(interp) | watcher_bits; diff --git a/Include/internal/pycore_dict_state.h b/Include/internal/pycore_dict_state.h index ece0f10ca25170..1a44755c7a01a3 100644 --- a/Include/internal/pycore_dict_state.h +++ b/Include/internal/pycore_dict_state.h @@ -8,17 +8,8 @@ extern "C" { # error "this header requires Py_BUILD_CORE define" #endif - -#ifndef WITH_FREELISTS -// without freelists -# define PyDict_MAXFREELIST 0 -#endif - -#ifndef PyDict_MAXFREELIST -# define PyDict_MAXFREELIST 80 -#endif - #define DICT_MAX_WATCHERS 8 +#define DICT_WATCHED_MUTATION_BITS 4 struct _Py_dict_state { /*Global counter used to set ma_version_tag field of dictionary. @@ -26,15 +17,6 @@ struct _Py_dict_state { * time that a dictionary is modified. */ uint64_t global_version; uint32_t next_keys_version; - -#if PyDict_MAXFREELIST > 0 - /* Dictionary reuse scheme to save calls to malloc and free */ - PyDictObject *free_list[PyDict_MAXFREELIST]; - PyDictKeysObject *keys_free_list[PyDict_MAXFREELIST]; - int numfree; - int keys_numfree; -#endif - PyDict_WatchCallback watchers[DICT_MAX_WATCHERS]; }; diff --git a/Include/internal/pycore_floatobject.h b/Include/internal/pycore_floatobject.h index 4e5474841bc25d..3767df5506d43f 100644 --- a/Include/internal/pycore_floatobject.h +++ b/Include/internal/pycore_floatobject.h @@ -8,14 +8,13 @@ extern "C" { # error "this header requires Py_BUILD_CORE define" #endif - +#include "pycore_freelist.h" // _PyFreeListState #include "pycore_unicodeobject.h" // _PyUnicodeWriter /* runtime lifecycle */ extern void _PyFloat_InitState(PyInterpreterState *); extern PyStatus _PyFloat_InitTypes(PyInterpreterState *); -extern void _PyFloat_Fini(PyInterpreterState *); extern void _PyFloat_FiniType(PyInterpreterState *); @@ -33,24 +32,7 @@ struct _Py_float_runtime_state { }; -#ifndef WITH_FREELISTS -// without freelists -# define PyFloat_MAXFREELIST 0 -#endif - -#ifndef PyFloat_MAXFREELIST -# define PyFloat_MAXFREELIST 100 -#endif -struct _Py_float_state { -#if PyFloat_MAXFREELIST > 0 - /* Special free list - free_list is a singly-linked list of available PyFloatObjects, - linked via abuse of their ob_type members. */ - int numfree; - PyFloatObject *free_list; -#endif -}; void _PyFloat_ExactDealloc(PyObject *op); diff --git a/Include/internal/pycore_freelist.h b/Include/internal/pycore_freelist.h new file mode 100644 index 00000000000000..1bc551914794f0 --- /dev/null +++ b/Include/internal/pycore_freelist.h @@ -0,0 +1,141 @@ +#ifndef Py_INTERNAL_FREELIST_H +#define Py_INTERNAL_FREELIST_H +#ifdef __cplusplus +extern "C" { +#endif + +#ifndef Py_BUILD_CORE +# error "this header requires Py_BUILD_CORE define" +#endif + +// PyTuple_MAXSAVESIZE - largest tuple to save on free list +// PyTuple_MAXFREELIST - maximum number of tuples of each size to save + +#ifdef WITH_FREELISTS +// with freelists +# define PyTuple_MAXSAVESIZE 20 +# define PyTuple_NFREELISTS PyTuple_MAXSAVESIZE +# define PyTuple_MAXFREELIST 2000 +# define PyList_MAXFREELIST 80 +# define PyDict_MAXFREELIST 80 +# define PyFloat_MAXFREELIST 100 +# define PyContext_MAXFREELIST 255 +# define _PyAsyncGen_MAXFREELIST 80 +# define _PyObjectStackChunk_MAXFREELIST 4 +#else +# define PyTuple_NFREELISTS 0 +# define PyTuple_MAXFREELIST 0 +# define PyList_MAXFREELIST 0 +# define PyDict_MAXFREELIST 0 +# define PyFloat_MAXFREELIST 0 +# define PyContext_MAXFREELIST 0 +# define _PyAsyncGen_MAXFREELIST 0 +# define _PyObjectStackChunk_MAXFREELIST 0 +#endif + +struct _Py_list_state { +#ifdef WITH_FREELISTS + PyListObject *free_list[PyList_MAXFREELIST]; + int numfree; +#endif +}; + +struct _Py_tuple_state { +#if WITH_FREELISTS + /* There is one freelist for each size from 1 to PyTuple_MAXSAVESIZE. + The empty tuple is handled separately. + + Each tuple stored in the array is the head of the linked list + (and the next available tuple) for that size. The actual tuple + object is used as the linked list node, with its first item + (ob_item[0]) pointing to the next node (i.e. the previous head). + Each linked list is initially NULL. */ + PyTupleObject *free_list[PyTuple_NFREELISTS]; + int numfree[PyTuple_NFREELISTS]; +#else + char _unused; // Empty structs are not allowed. +#endif +}; + +struct _Py_float_state { +#ifdef WITH_FREELISTS + /* Special free list + free_list is a singly-linked list of available PyFloatObjects, + linked via abuse of their ob_type members. */ + int numfree; + PyFloatObject *free_list; +#endif +}; + +struct _Py_dict_freelist { +#ifdef WITH_FREELISTS + /* Dictionary reuse scheme to save calls to malloc and free */ + PyDictObject *free_list[PyDict_MAXFREELIST]; + PyDictKeysObject *keys_free_list[PyDict_MAXFREELIST]; + int numfree; + int keys_numfree; +#endif +}; + +struct _Py_slice_state { +#ifdef WITH_FREELISTS + /* Using a cache is very effective since typically only a single slice is + created and then deleted again. */ + PySliceObject *slice_cache; +#endif +}; + +struct _Py_context_state { +#ifdef WITH_FREELISTS + // List of free PyContext objects + PyContext *freelist; + int numfree; +#endif +}; + +struct _Py_async_gen_state { +#ifdef WITH_FREELISTS + /* Freelists boost performance 6-10%; they also reduce memory + fragmentation, as _PyAsyncGenWrappedValue and PyAsyncGenASend + are short-living objects that are instantiated for every + __anext__() call. */ + struct _PyAsyncGenWrappedValue* value_freelist[_PyAsyncGen_MAXFREELIST]; + int value_numfree; + + struct PyAsyncGenASend* asend_freelist[_PyAsyncGen_MAXFREELIST]; + int asend_numfree; +#endif +}; + +struct _PyObjectStackChunk; + +struct _Py_object_stack_state { + struct _PyObjectStackChunk *free_list; + Py_ssize_t numfree; +}; + +typedef struct _Py_freelist_state { + struct _Py_float_state floats; + struct _Py_tuple_state tuples; + struct _Py_list_state lists; + struct _Py_dict_freelist dicts; + struct _Py_slice_state slices; + struct _Py_context_state contexts; + struct _Py_async_gen_state async_gens; + struct _Py_object_stack_state object_stacks; +} _PyFreeListState; + +extern void _PyObject_ClearFreeLists(_PyFreeListState *state, int is_finalization); +extern void _PyTuple_ClearFreeList(_PyFreeListState *state, int is_finalization); +extern void _PyFloat_ClearFreeList(_PyFreeListState *state, int is_finalization); +extern void _PyList_ClearFreeList(_PyFreeListState *state, int is_finalization); +extern void _PySlice_ClearFreeList(_PyFreeListState *state, int is_finalization); +extern void _PyDict_ClearFreeList(_PyFreeListState *state, int is_finalization); +extern void _PyAsyncGen_ClearFreeLists(_PyFreeListState *state, int is_finalization); +extern void _PyContext_ClearFreeList(_PyFreeListState *state, int is_finalization); +extern void _PyObjectStackChunk_ClearFreeList(_PyFreeListState *state, int is_finalization); + +#ifdef __cplusplus +} +#endif +#endif /* !Py_INTERNAL_FREELIST_H */ diff --git a/Include/internal/pycore_gc.h b/Include/internal/pycore_gc.h index 2d33aa76d78229..582a16bf5218ce 100644 --- a/Include/internal/pycore_gc.h +++ b/Include/internal/pycore_gc.h @@ -8,6 +8,8 @@ extern "C" { # error "this header requires Py_BUILD_CORE define" #endif +#include "pycore_freelist.h" // _PyFreeListState + /* GC information is stored BEFORE the object structure. */ typedef struct { // Pointer to next object in the list. @@ -35,10 +37,23 @@ static inline PyObject* _Py_FROM_GC(PyGC_Head *gc) { } +/* Bit flags for ob_gc_bits (in Py_GIL_DISABLED builds) */ +#ifdef Py_GIL_DISABLED +# define _PyGC_BITS_TRACKED (1) +# define _PyGC_BITS_FINALIZED (2) +# define _PyGC_BITS_UNREACHABLE (4) +# define _PyGC_BITS_FROZEN (8) +# define _PyGC_BITS_SHARED (16) +#endif + /* True if the object is currently tracked by the GC. */ static inline int _PyObject_GC_IS_TRACKED(PyObject *op) { +#ifdef Py_GIL_DISABLED + return (op->ob_gc_bits & _PyGC_BITS_TRACKED) != 0; +#else PyGC_Head *gc = _Py_AS_GC(op); return (gc->_gc_next != 0); +#endif } #define _PyObject_GC_IS_TRACKED(op) _PyObject_GC_IS_TRACKED(_Py_CAST(PyObject*, op)) @@ -54,6 +69,22 @@ static inline int _PyObject_GC_MAY_BE_TRACKED(PyObject *obj) { return 1; } +#ifdef Py_GIL_DISABLED + +/* True if an object is shared between multiple threads and + * needs special purpose when freeing to do the possibility + * of in-flight lock-free reads occuring */ +static inline int _PyObject_GC_IS_SHARED(PyObject *op) { + return (op->ob_gc_bits & _PyGC_BITS_SHARED) != 0; +} +#define _PyObject_GC_IS_SHARED(op) _PyObject_GC_IS_SHARED(_Py_CAST(PyObject*, op)) + +static inline void _PyObject_GC_SET_SHARED(PyObject *op) { + op->ob_gc_bits |= _PyGC_BITS_SHARED; +} +#define _PyObject_GC_SET_SHARED(op) _PyObject_GC_SET_SHARED(_Py_CAST(PyObject*, op)) + +#endif /* Bit flags for _gc_prev */ /* Bit 0 is set when tp_finalize is called */ @@ -64,6 +95,26 @@ static inline int _PyObject_GC_MAY_BE_TRACKED(PyObject *obj) { #define _PyGC_PREV_SHIFT (2) #define _PyGC_PREV_MASK (((uintptr_t) -1) << _PyGC_PREV_SHIFT) +/* set for debugging information */ +#define _PyGC_DEBUG_STATS (1<<0) /* print collection statistics */ +#define _PyGC_DEBUG_COLLECTABLE (1<<1) /* print collectable objects */ +#define _PyGC_DEBUG_UNCOLLECTABLE (1<<2) /* print uncollectable objects */ +#define _PyGC_DEBUG_SAVEALL (1<<5) /* save all garbage in gc.garbage */ +#define _PyGC_DEBUG_LEAK _PyGC_DEBUG_COLLECTABLE | \ + _PyGC_DEBUG_UNCOLLECTABLE | \ + _PyGC_DEBUG_SAVEALL + +typedef enum { + // GC was triggered by heap allocation + _Py_GC_REASON_HEAP, + + // GC was called during shutdown + _Py_GC_REASON_SHUTDOWN, + + // GC was called by gc.collect() or PyGC_Collect() + _Py_GC_REASON_MANUAL +} _PyGC_Reason; + // Lowest bit of _gc_next is used for flags only in GC. // But it is always 0 for normal code. static inline PyGC_Head* _PyGCHead_NEXT(PyGC_Head *gc) { @@ -85,20 +136,29 @@ static inline void _PyGCHead_SET_PREV(PyGC_Head *gc, PyGC_Head *prev) { gc->_gc_prev = ((gc->_gc_prev & ~_PyGC_PREV_MASK) | uprev); } -static inline int _PyGCHead_FINALIZED(PyGC_Head *gc) { - return ((gc->_gc_prev & _PyGC_PREV_MASK_FINALIZED) != 0); -} -static inline void _PyGCHead_SET_FINALIZED(PyGC_Head *gc) { - gc->_gc_prev |= _PyGC_PREV_MASK_FINALIZED; -} - static inline int _PyGC_FINALIZED(PyObject *op) { +#ifdef Py_GIL_DISABLED + return (op->ob_gc_bits & _PyGC_BITS_FINALIZED) != 0; +#else PyGC_Head *gc = _Py_AS_GC(op); - return _PyGCHead_FINALIZED(gc); + return ((gc->_gc_prev & _PyGC_PREV_MASK_FINALIZED) != 0); +#endif } static inline void _PyGC_SET_FINALIZED(PyObject *op) { +#ifdef Py_GIL_DISABLED + op->ob_gc_bits |= _PyGC_BITS_FINALIZED; +#else PyGC_Head *gc = _Py_AS_GC(op); - _PyGCHead_SET_FINALIZED(gc); + gc->_gc_prev |= _PyGC_PREV_MASK_FINALIZED; +#endif +} +static inline void _PyGC_CLEAR_FINALIZED(PyObject *op) { +#ifdef Py_GIL_DISABLED + op->ob_gc_bits &= ~_PyGC_BITS_FINALIZED; +#else + PyGC_Head *gc = _Py_AS_GC(op); + gc->_gc_prev &= ~_PyGC_PREV_MASK_FINALIZED; +#endif } @@ -203,16 +263,22 @@ struct _gc_runtime_state { extern void _PyGC_InitState(struct _gc_runtime_state *); +extern Py_ssize_t _PyGC_Collect(PyThreadState *tstate, int generation, + _PyGC_Reason reason); extern Py_ssize_t _PyGC_CollectNoFail(PyThreadState *tstate); +/* Freeze objects tracked by the GC and ignore them in future collections. */ +extern void _PyGC_Freeze(PyInterpreterState *interp); +/* Unfreezes objects placing them in the oldest generation */ +extern void _PyGC_Unfreeze(PyInterpreterState *interp); +/* Number of frozen objects */ +extern Py_ssize_t _PyGC_GetFreezeCount(PyInterpreterState *interp); + +extern PyObject *_PyGC_GetObjects(PyInterpreterState *interp, Py_ssize_t generation); +extern PyObject *_PyGC_GetReferrers(PyInterpreterState *interp, PyObject *objs); // Functions to clear types free lists -extern void _PyTuple_ClearFreeList(PyInterpreterState *interp); -extern void _PyFloat_ClearFreeList(PyInterpreterState *interp); -extern void _PyList_ClearFreeList(PyInterpreterState *interp); -extern void _PyDict_ClearFreeList(PyInterpreterState *interp); -extern void _PyAsyncGen_ClearFreeLists(PyInterpreterState *interp); -extern void _PyContext_ClearFreeList(PyInterpreterState *interp); +extern void _PyGC_ClearAllFreeLists(PyInterpreterState *interp); extern void _Py_ScheduleGC(PyInterpreterState *interp); extern void _Py_RunGC(PyThreadState *tstate); diff --git a/Include/internal/pycore_genobject.h b/Include/internal/pycore_genobject.h index cf58a2750a31f9..b2aa017598409f 100644 --- a/Include/internal/pycore_genobject.h +++ b/Include/internal/pycore_genobject.h @@ -8,6 +8,8 @@ extern "C" { # error "this header requires Py_BUILD_CORE define" #endif +#include "pycore_freelist.h" + extern PyObject *_PyGen_yf(PyGenObject *); extern void _PyGen_Finalize(PyObject *self); @@ -24,37 +26,6 @@ extern PyTypeObject _PyCoroWrapper_Type; extern PyTypeObject _PyAsyncGenWrappedValue_Type; extern PyTypeObject _PyAsyncGenAThrow_Type; -/* runtime lifecycle */ - -extern void _PyAsyncGen_Fini(PyInterpreterState *); - - -/* other API */ - -#ifndef WITH_FREELISTS -// without freelists -# define _PyAsyncGen_MAXFREELIST 0 -#endif - -#ifndef _PyAsyncGen_MAXFREELIST -# define _PyAsyncGen_MAXFREELIST 80 -#endif - -struct _Py_async_gen_state { -#if _PyAsyncGen_MAXFREELIST > 0 - /* Freelists boost performance 6-10%; they also reduce memory - fragmentation, as _PyAsyncGenWrappedValue and PyAsyncGenASend - are short-living objects that are instantiated for every - __anext__() call. */ - struct _PyAsyncGenWrappedValue* value_freelist[_PyAsyncGen_MAXFREELIST]; - int value_numfree; - - struct PyAsyncGenASend* asend_freelist[_PyAsyncGen_MAXFREELIST]; - int asend_numfree; -#endif -}; - - #ifdef __cplusplus } #endif diff --git a/Include/internal/pycore_global_objects_fini_generated.h b/Include/internal/pycore_global_objects_fini_generated.h index 89ec8cbbbcd649..932738c3049882 100644 --- a/Include/internal/pycore_global_objects_fini_generated.h +++ b/Include/internal/pycore_global_objects_fini_generated.h @@ -787,8 +787,11 @@ _PyStaticObjects_CheckRefcnt(PyInterpreterState *interp) { _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(after_in_child)); _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(after_in_parent)); _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(aggregate_class)); + _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(alias)); + _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(allow_code)); _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(append)); _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(argdefs)); + _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(args)); _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(arguments)); _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(argv)); _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(as_integer_ratio)); @@ -873,6 +876,7 @@ _PyStaticObjects_CheckRefcnt(PyInterpreterState *interp) { _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(d)); _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(data)); _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(database)); + _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(day)); _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(decode)); _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(decoder)); _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(default)); @@ -912,6 +916,8 @@ _PyStaticObjects_CheckRefcnt(PyInterpreterState *interp) { _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(errors)); _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(event)); _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(eventmask)); + _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(exc_type)); + _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(exc_value)); _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(excepthook)); _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(exception)); _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(existing_file_name)); @@ -934,12 +940,14 @@ _PyStaticObjects_CheckRefcnt(PyInterpreterState *interp) { _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(fileno)); _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(filepath)); _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(fillvalue)); + _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(filter)); _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(filters)); _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(final)); _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(find_class)); _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(fix_imports)); _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(flags)); _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(flush)); + _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(fold)); _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(follow_symlinks)); _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(format)); _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(from_param)); @@ -970,6 +978,7 @@ _PyStaticObjects_CheckRefcnt(PyInterpreterState *interp) { _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(headers)); _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(hi)); _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(hook)); + _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(hour)); _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(id)); _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(ident)); _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(identity_hint)); @@ -1015,6 +1024,7 @@ _PyStaticObjects_CheckRefcnt(PyInterpreterState *interp) { _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(kw)); _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(kw1)); _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(kw2)); + _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(kwdefaults)); _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(lambda)); _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(last)); _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(last_exc)); @@ -1043,6 +1053,7 @@ _PyStaticObjects_CheckRefcnt(PyInterpreterState *interp) { _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(max_length)); _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(maxdigits)); _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(maxevents)); + _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(maxlen)); _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(maxmem)); _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(maxsplit)); _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(maxvalue)); @@ -1052,11 +1063,14 @@ _PyStaticObjects_CheckRefcnt(PyInterpreterState *interp) { _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(metaclass)); _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(metadata)); _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(method)); + _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(microsecond)); + _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(minute)); _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(mod)); _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(mode)); _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(module)); _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(module_globals)); _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(modules)); + _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(month)); _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(mro)); _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(msg)); _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(mycmp)); @@ -1161,6 +1175,7 @@ _PyStaticObjects_CheckRefcnt(PyInterpreterState *interp) { _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(salt)); _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(sched_priority)); _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(scheduler)); + _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(second)); _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(seek)); _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(seekable)); _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(selectors)); @@ -1227,6 +1242,7 @@ _PyStaticObjects_CheckRefcnt(PyInterpreterState *interp) { _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(timetuple)); _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(top)); _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(trace_callback)); + _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(traceback)); _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(trailers)); _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(translate)); _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(true)); @@ -1236,6 +1252,7 @@ _PyStaticObjects_CheckRefcnt(PyInterpreterState *interp) { _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(type)); _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(type_params)); _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(tz)); + _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(tzinfo)); _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(tzname)); _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(uid)); _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(unlink)); diff --git a/Include/internal/pycore_global_strings.h b/Include/internal/pycore_global_strings.h index 62c3ee3ae2a0bd..da62b4f0a951ff 100644 --- a/Include/internal/pycore_global_strings.h +++ b/Include/internal/pycore_global_strings.h @@ -276,8 +276,11 @@ struct _Py_global_strings { STRUCT_FOR_ID(after_in_child) STRUCT_FOR_ID(after_in_parent) STRUCT_FOR_ID(aggregate_class) + STRUCT_FOR_ID(alias) + STRUCT_FOR_ID(allow_code) STRUCT_FOR_ID(append) STRUCT_FOR_ID(argdefs) + STRUCT_FOR_ID(args) STRUCT_FOR_ID(arguments) STRUCT_FOR_ID(argv) STRUCT_FOR_ID(as_integer_ratio) @@ -362,6 +365,7 @@ struct _Py_global_strings { STRUCT_FOR_ID(d) STRUCT_FOR_ID(data) STRUCT_FOR_ID(database) + STRUCT_FOR_ID(day) STRUCT_FOR_ID(decode) STRUCT_FOR_ID(decoder) STRUCT_FOR_ID(default) @@ -401,6 +405,8 @@ struct _Py_global_strings { STRUCT_FOR_ID(errors) STRUCT_FOR_ID(event) STRUCT_FOR_ID(eventmask) + STRUCT_FOR_ID(exc_type) + STRUCT_FOR_ID(exc_value) STRUCT_FOR_ID(excepthook) STRUCT_FOR_ID(exception) STRUCT_FOR_ID(existing_file_name) @@ -423,12 +429,14 @@ struct _Py_global_strings { STRUCT_FOR_ID(fileno) STRUCT_FOR_ID(filepath) STRUCT_FOR_ID(fillvalue) + STRUCT_FOR_ID(filter) STRUCT_FOR_ID(filters) STRUCT_FOR_ID(final) STRUCT_FOR_ID(find_class) STRUCT_FOR_ID(fix_imports) STRUCT_FOR_ID(flags) STRUCT_FOR_ID(flush) + STRUCT_FOR_ID(fold) STRUCT_FOR_ID(follow_symlinks) STRUCT_FOR_ID(format) STRUCT_FOR_ID(from_param) @@ -459,6 +467,7 @@ struct _Py_global_strings { STRUCT_FOR_ID(headers) STRUCT_FOR_ID(hi) STRUCT_FOR_ID(hook) + STRUCT_FOR_ID(hour) STRUCT_FOR_ID(id) STRUCT_FOR_ID(ident) STRUCT_FOR_ID(identity_hint) @@ -504,6 +513,7 @@ struct _Py_global_strings { STRUCT_FOR_ID(kw) STRUCT_FOR_ID(kw1) STRUCT_FOR_ID(kw2) + STRUCT_FOR_ID(kwdefaults) STRUCT_FOR_ID(lambda) STRUCT_FOR_ID(last) STRUCT_FOR_ID(last_exc) @@ -532,6 +542,7 @@ struct _Py_global_strings { STRUCT_FOR_ID(max_length) STRUCT_FOR_ID(maxdigits) STRUCT_FOR_ID(maxevents) + STRUCT_FOR_ID(maxlen) STRUCT_FOR_ID(maxmem) STRUCT_FOR_ID(maxsplit) STRUCT_FOR_ID(maxvalue) @@ -541,11 +552,14 @@ struct _Py_global_strings { STRUCT_FOR_ID(metaclass) STRUCT_FOR_ID(metadata) STRUCT_FOR_ID(method) + STRUCT_FOR_ID(microsecond) + STRUCT_FOR_ID(minute) STRUCT_FOR_ID(mod) STRUCT_FOR_ID(mode) STRUCT_FOR_ID(module) STRUCT_FOR_ID(module_globals) STRUCT_FOR_ID(modules) + STRUCT_FOR_ID(month) STRUCT_FOR_ID(mro) STRUCT_FOR_ID(msg) STRUCT_FOR_ID(mycmp) @@ -650,6 +664,7 @@ struct _Py_global_strings { STRUCT_FOR_ID(salt) STRUCT_FOR_ID(sched_priority) STRUCT_FOR_ID(scheduler) + STRUCT_FOR_ID(second) STRUCT_FOR_ID(seek) STRUCT_FOR_ID(seekable) STRUCT_FOR_ID(selectors) @@ -716,6 +731,7 @@ struct _Py_global_strings { STRUCT_FOR_ID(timetuple) STRUCT_FOR_ID(top) STRUCT_FOR_ID(trace_callback) + STRUCT_FOR_ID(traceback) STRUCT_FOR_ID(trailers) STRUCT_FOR_ID(translate) STRUCT_FOR_ID(true) @@ -725,6 +741,7 @@ struct _Py_global_strings { STRUCT_FOR_ID(type) STRUCT_FOR_ID(type_params) STRUCT_FOR_ID(tz) + STRUCT_FOR_ID(tzinfo) STRUCT_FOR_ID(tzname) STRUCT_FOR_ID(uid) STRUCT_FOR_ID(unlink) diff --git a/Include/internal/pycore_interp.h b/Include/internal/pycore_interp.h index 04d7a6a615e370..31d88071e19d0c 100644 --- a/Include/internal/pycore_interp.h +++ b/Include/internal/pycore_interp.h @@ -20,6 +20,7 @@ extern "C" { #include "pycore_dtoa.h" // struct _dtoa_state #include "pycore_exceptions.h" // struct _Py_exc_state #include "pycore_floatobject.h" // struct _Py_float_state +#include "pycore_freelist.h" // struct _Py_freelist_state #include "pycore_function.h" // FUNC_MAX_WATCHERS #include "pycore_gc.h" // struct _gc_runtime_state #include "pycore_genobject.h" // struct _Py_async_gen_state @@ -27,6 +28,7 @@ extern "C" { #include "pycore_import.h" // struct _import_state #include "pycore_instruments.h" // _PY_MONITORING_EVENTS #include "pycore_list.h" // struct _Py_list_state +#include "pycore_mimalloc.h" // struct _mimalloc_interp_state #include "pycore_object_state.h" // struct _py_object_state #include "pycore_obmalloc.h" // struct _obmalloc_state #include "pycore_tstate.h" // _PyThreadStateImpl @@ -40,9 +42,39 @@ struct _Py_long_state { int max_str_digits; }; +// Support for stop-the-world events. This exists in both the PyRuntime struct +// for global pauses and in each PyInterpreterState for per-interpreter pauses. +struct _stoptheworld_state { + PyMutex mutex; // Serializes stop-the-world attempts. + + // NOTE: The below fields are protected by HEAD_LOCK(runtime), not by the + // above mutex. + bool requested; // Set when a pause is requested. + bool world_stopped; // Set when the world is stopped. + bool is_global; // Set when contained in PyRuntime struct. + + PyEvent stop_event; // Set when thread_countdown reaches zero. + Py_ssize_t thread_countdown; // Number of threads that must pause. + + PyThreadState *requester; // Thread that requested the pause (may be NULL). +}; /* cross-interpreter data registry */ +/* Tracks some rare events per-interpreter, used by the optimizer to turn on/off + specific optimizations. */ +typedef struct _rare_events { + /* Setting an object's class, obj.__class__ = ... */ + uint8_t set_class; + /* Setting the bases of a class, cls.__bases__ = ... */ + uint8_t set_bases; + /* Setting the PEP 523 frame eval function, _PyInterpreterState_SetFrameEvalFunc() */ + uint8_t set_eval_frame_func; + /* Modifying the builtins, __builtins__.__dict__[var] = ... */ + uint8_t builtin_dict; + /* Modifying a function, e.g. func.__defaults__ = ..., etc. */ + uint8_t func_modification; +} _rare_events; /* interpreter state */ @@ -165,8 +197,24 @@ struct _is { struct _warnings_runtime_state warnings; struct atexit_state atexit; + struct _stoptheworld_state stoptheworld; - struct _obmalloc_state obmalloc; +#if defined(Py_GIL_DISABLED) + struct _mimalloc_interp_state mimalloc; + struct _brc_state brc; // biased reference counting state +#endif + + // Per-interpreter state for the obmalloc allocator. For the main + // interpreter and for all interpreters that don't have their + // own obmalloc state, this points to the static structure in + // obmalloc.c obmalloc_state_main. For other interpreters, it is + // heap allocated by _PyMem_init_obmalloc() and freed when the + // interpreter structure is freed. In the case of a heap allocated + // obmalloc state, it is not safe to hold on to or use memory after + // the interpreter is freed. The obmalloc state corresponding to + // that allocated memory is gone. See free_obmalloc_arenas() for + // more comments. + struct _obmalloc_state *obmalloc; PyObject *audit_hooks; PyType_WatchCallback type_watchers[TYPE_MAX_WATCHERS]; @@ -174,21 +222,16 @@ struct _is { // One bit is set for each non-NULL entry in code_watchers uint8_t active_code_watchers; +#if !defined(Py_GIL_DISABLED) + struct _Py_freelist_state freelist_state; +#endif struct _py_object_state object_state; struct _Py_unicode_state unicode; - struct _Py_float_state float_state; struct _Py_long_state long_state; struct _dtoa_state dtoa; struct _py_func_state func_state; - /* Using a cache is very effective since typically only a single slice is - created and then deleted again. */ - PySliceObject *slice_cache; - struct _Py_tuple_state tuple; - struct _Py_list_state list; struct _Py_dict_state dict_state; - struct _Py_async_gen_state async_gen; - struct _Py_context_state context; struct _Py_exc_state exc_state; struct ast_state ast; @@ -199,6 +242,8 @@ struct _is { uint16_t optimizer_resume_threshold; uint16_t optimizer_backedge_threshold; uint32_t next_func_version; + _rare_events rare_events; + PyDict_WatchCallback builtins_dict_watcher; _Py_GlobalMonitors monitors; bool sys_profile_initialized; @@ -329,6 +374,19 @@ PyAPI_FUNC(PyStatus) _PyInterpreterState_New( PyInterpreterState **pinterp); +#define RARE_EVENT_INTERP_INC(interp, name) \ + do { \ + /* saturating add */ \ + if (interp->rare_events.name < UINT8_MAX) interp->rare_events.name++; \ + RARE_EVENT_STAT_INC(name); \ + } while (0); \ + +#define RARE_EVENT_INC(name) \ + do { \ + PyInterpreterState *interp = PyInterpreterState_Get(); \ + RARE_EVENT_INTERP_INC(interp, name); \ + } while (0); \ + #ifdef __cplusplus } #endif diff --git a/Include/internal/pycore_jit.h b/Include/internal/pycore_jit.h new file mode 100644 index 00000000000000..0b71eb6f758ac6 --- /dev/null +++ b/Include/internal/pycore_jit.h @@ -0,0 +1,25 @@ +#ifndef Py_INTERNAL_JIT_H +#define Py_INTERNAL_JIT_H + +#ifdef __cplusplus +extern "C" { +#endif + +#ifndef Py_BUILD_CORE +# error "this header requires Py_BUILD_CORE define" +#endif + +#ifdef _Py_JIT + +typedef _Py_CODEUNIT *(*jit_func)(_PyInterpreterFrame *frame, PyObject **stack_pointer, PyThreadState *tstate); + +int _PyJIT_Compile(_PyExecutorObject *executor, _PyUOpInstruction *trace, size_t length); +void _PyJIT_Free(_PyExecutorObject *executor); + +#endif // _Py_JIT + +#ifdef __cplusplus +} +#endif + +#endif // !Py_INTERNAL_JIT_H diff --git a/Include/internal/pycore_list.h b/Include/internal/pycore_list.h index 55d67b32bc8a63..50dc13c4da4487 100644 --- a/Include/internal/pycore_list.h +++ b/Include/internal/pycore_list.h @@ -8,46 +8,23 @@ extern "C" { # error "this header requires Py_BUILD_CORE define" #endif +#include "pycore_freelist.h" // _PyFreeListState extern PyObject* _PyList_Extend(PyListObject *, PyObject *); extern void _PyList_DebugMallocStats(FILE *out); - -/* runtime lifecycle */ - -extern void _PyList_Fini(PyInterpreterState *); - - -/* other API */ - -#ifndef WITH_FREELISTS -// without freelists -# define PyList_MAXFREELIST 0 -#endif - -/* Empty list reuse scheme to save calls to malloc and free */ -#ifndef PyList_MAXFREELIST -# define PyList_MAXFREELIST 80 -#endif - -struct _Py_list_state { -#if PyList_MAXFREELIST > 0 - PyListObject *free_list[PyList_MAXFREELIST]; - int numfree; -#endif -}; - #define _PyList_ITEMS(op) _Py_RVALUE(_PyList_CAST(op)->ob_item) extern int _PyList_AppendTakeRefListResize(PyListObject *self, PyObject *newitem); +// In free-threaded build: self should be locked by the caller, if it should be thread-safe. static inline int _PyList_AppendTakeRef(PyListObject *self, PyObject *newitem) { assert(self != NULL && newitem != NULL); assert(PyList_Check(self)); - Py_ssize_t len = PyList_GET_SIZE(self); + Py_ssize_t len = Py_SIZE(self); Py_ssize_t allocated = self->allocated; assert((size_t)len + 1 < PY_SSIZE_T_MAX); if (allocated > len) { diff --git a/Include/internal/pycore_llist.h b/Include/internal/pycore_llist.h index 5fd261da05fa5d..f629902fda9ff1 100644 --- a/Include/internal/pycore_llist.h +++ b/Include/internal/pycore_llist.h @@ -37,8 +37,7 @@ struct llist_node { }; // Get the struct containing a node. -#define llist_data(node, type, member) \ - (type*)((char*)node - offsetof(type, member)) +#define llist_data(node, type, member) (_Py_CONTAINER_OF(node, type, member)) // Iterate over a list. #define llist_for_each(node, head) \ diff --git a/Include/internal/pycore_mimalloc.h b/Include/internal/pycore_mimalloc.h index c29dc82a42762a..1e7ed5a4ca62e2 100644 --- a/Include/internal/pycore_mimalloc.h +++ b/Include/internal/pycore_mimalloc.h @@ -9,11 +9,43 @@ # error "pycore_mimalloc.h must be included before mimalloc.h" #endif +typedef enum { + _Py_MIMALLOC_HEAP_MEM = 0, // PyMem_Malloc() and friends + _Py_MIMALLOC_HEAP_OBJECT = 1, // non-GC objects + _Py_MIMALLOC_HEAP_GC = 2, // GC objects without pre-header + _Py_MIMALLOC_HEAP_GC_PRE = 3, // GC objects with pre-header + _Py_MIMALLOC_HEAP_COUNT +} _Py_mimalloc_heap_id; + #include "pycore_pymem.h" + +#ifdef WITH_MIMALLOC #define MI_DEBUG_UNINIT PYMEM_CLEANBYTE #define MI_DEBUG_FREED PYMEM_DEADBYTE #define MI_DEBUG_PADDING PYMEM_FORBIDDENBYTE +#ifdef Py_DEBUG +# define MI_DEBUG 1 +#else +# define MI_DEBUG 0 +#endif #include "mimalloc.h" +#include "mimalloc/types.h" +#include "mimalloc/internal.h" +#endif + +#ifdef Py_GIL_DISABLED +struct _mimalloc_interp_state { + // When exiting, threads place any segments with live blocks in this + // shared pool for other threads to claim and reuse. + mi_abandoned_pool_t abandoned_pool; +}; + +struct _mimalloc_thread_state { + mi_heap_t *current_object_heap; + mi_heap_t heaps[_Py_MIMALLOC_HEAP_COUNT]; + mi_tld_t tld; +}; +#endif #endif // Py_INTERNAL_MIMALLOC_H diff --git a/Include/internal/pycore_object.h b/Include/internal/pycore_object.h index f413b8451e5ab4..34a83ea228e8b1 100644 --- a/Include/internal/pycore_object.h +++ b/Include/internal/pycore_object.h @@ -178,7 +178,7 @@ _Py_DECREF_SPECIALIZED(PyObject *op, const destructor destruct) } _Py_DECREF_STAT_INC(); #ifdef Py_REF_DEBUG - _Py_DEC_REFTOTAL(_PyInterpreterState_GET()); + _Py_DEC_REFTOTAL(PyInterpreterState_Get()); #endif if (--op->ob_refcnt != 0) { assert(op->ob_refcnt > 0); @@ -199,7 +199,7 @@ _Py_DECREF_NO_DEALLOC(PyObject *op) } _Py_DECREF_STAT_INC(); #ifdef Py_REF_DEBUG - _Py_DEC_REFTOTAL(_PyInterpreterState_GET()); + _Py_DEC_REFTOTAL(PyInterpreterState_Get()); #endif op->ob_refcnt--; #ifdef Py_DEBUG @@ -315,7 +315,9 @@ static inline void _PyObject_GC_TRACK( _PyObject_ASSERT_FROM(op, !_PyObject_GC_IS_TRACKED(op), "object already tracked by the garbage collector", filename, lineno, __func__); - +#ifdef Py_GIL_DISABLED + op->ob_gc_bits |= _PyGC_BITS_TRACKED; +#else PyGC_Head *gc = _Py_AS_GC(op); _PyObject_ASSERT_FROM(op, (gc->_gc_prev & _PyGC_PREV_MASK_COLLECTING) == 0, @@ -329,6 +331,7 @@ static inline void _PyObject_GC_TRACK( _PyGCHead_SET_PREV(gc, last); _PyGCHead_SET_NEXT(gc, generation0); generation0->_gc_prev = (uintptr_t)gc; +#endif } /* Tell the GC to stop tracking this object. @@ -352,6 +355,9 @@ static inline void _PyObject_GC_UNTRACK( "object not tracked by the garbage collector", filename, lineno, __func__); +#ifdef Py_GIL_DISABLED + op->ob_gc_bits &= ~_PyGC_BITS_TRACKED; +#else PyGC_Head *gc = _Py_AS_GC(op); PyGC_Head *prev = _PyGCHead_PREV(gc); PyGC_Head *next = _PyGCHead_NEXT(gc); @@ -359,6 +365,7 @@ static inline void _PyObject_GC_UNTRACK( _PyGCHead_SET_PREV(next, prev); gc->_gc_next = 0; gc->_gc_prev &= _PyGC_PREV_MASK_FINALIZED; +#endif } // Macros to accept any type for the parameter, and to automatically pass @@ -376,6 +383,142 @@ static inline void _PyObject_GC_UNTRACK( _PyObject_GC_UNTRACK(__FILE__, __LINE__, _PyObject_CAST(op)) #endif +#ifdef Py_GIL_DISABLED + +/* Tries to increment an object's reference count + * + * This is a specialized version of _Py_TryIncref that only succeeds if the + * object is immortal or local to this thread. It does not handle the case + * where the reference count modification requires an atomic operation. This + * allows call sites to specialize for the immortal/local case. + */ +static inline int +_Py_TryIncrefFast(PyObject *op) { + uint32_t local = _Py_atomic_load_uint32_relaxed(&op->ob_ref_local); + local += 1; + if (local == 0) { + // immortal + return 1; + } + if (_Py_IsOwnedByCurrentThread(op)) { + _Py_INCREF_STAT_INC(); + _Py_atomic_store_uint32_relaxed(&op->ob_ref_local, local); +#ifdef Py_REF_DEBUG + _Py_IncRefTotal(_PyInterpreterState_GET()); +#endif + return 1; + } + return 0; +} + +static inline int +_Py_TryIncRefShared(PyObject *op) +{ + Py_ssize_t shared = _Py_atomic_load_ssize_relaxed(&op->ob_ref_shared); + for (;;) { + // If the shared refcount is zero and the object is either merged + // or may not have weak references, then we cannot incref it. + if (shared == 0 || shared == _Py_REF_MERGED) { + return 0; + } + + if (_Py_atomic_compare_exchange_ssize( + &op->ob_ref_shared, + &shared, + shared + (1 << _Py_REF_SHARED_SHIFT))) { +#ifdef Py_REF_DEBUG + _Py_IncRefTotal(_PyInterpreterState_GET()); +#endif + _Py_INCREF_STAT_INC(); + return 1; + } + } +} + +/* Tries to incref the object op and ensures that *src still points to it. */ +static inline int +_Py_TryIncref(PyObject **src, PyObject *op) +{ + if (_Py_TryIncrefFast(op)) { + return 1; + } + if (!_Py_TryIncRefShared(op)) { + return 0; + } + if (op != _Py_atomic_load_ptr(src)) { + Py_DECREF(op); + return 0; + } + return 1; +} + +/* Loads and increfs an object from ptr, which may contain a NULL value. + Safe with concurrent (atomic) updates to ptr. + NOTE: The writer must set maybe-weakref on the stored object! */ +static inline PyObject * +_Py_XGetRef(PyObject **ptr) +{ + for (;;) { + PyObject *value = _Py_atomic_load_ptr(ptr); + if (value == NULL) { + return value; + } + if (_Py_TryIncref(ptr, value)) { + return value; + } + } +} + +/* Attempts to loads and increfs an object from ptr. Returns NULL + on failure, which may be due to a NULL value or a concurrent update. */ +static inline PyObject * +_Py_TryXGetRef(PyObject **ptr) +{ + PyObject *value = _Py_atomic_load_ptr(ptr); + if (value == NULL) { + return value; + } + if (_Py_TryIncref(ptr, value)) { + return value; + } + return NULL; +} + +/* Like Py_NewRef but also optimistically sets _Py_REF_MAYBE_WEAKREF + on objects owned by a different thread. */ +static inline PyObject * +_Py_NewRefWithLock(PyObject *op) +{ + if (_Py_TryIncrefFast(op)) { + return op; + } + _Py_INCREF_STAT_INC(); + for (;;) { + Py_ssize_t shared = _Py_atomic_load_ssize_relaxed(&op->ob_ref_shared); + Py_ssize_t new_shared = shared + (1 << _Py_REF_SHARED_SHIFT); + if ((shared & _Py_REF_SHARED_FLAG_MASK) == 0) { + new_shared |= _Py_REF_MAYBE_WEAKREF; + } + if (_Py_atomic_compare_exchange_ssize( + &op->ob_ref_shared, + &shared, + new_shared)) { + return op; + } + } +} + +static inline PyObject * +_Py_XNewRefWithLock(PyObject *obj) +{ + if (obj == NULL) { + return NULL; + } + return _Py_NewRefWithLock(obj); +} + +#endif + #ifdef Py_REF_DEBUG extern void _PyInterpreterState_FinalizeRefTotal(PyInterpreterState *); extern void _Py_FinalizeRefTotal(_PyRuntimeState *); @@ -450,8 +593,12 @@ _PyObject_IS_GC(PyObject *obj) static inline size_t _PyType_PreHeaderSize(PyTypeObject *tp) { - return _PyType_IS_GC(tp) * sizeof(PyGC_Head) + - _PyType_HasFeature(tp, Py_TPFLAGS_PREHEADER) * 2 * sizeof(PyObject *); + return ( +#ifndef Py_GIL_DISABLED + _PyType_IS_GC(tp) * sizeof(PyGC_Head) + +#endif + _PyType_HasFeature(tp, Py_TPFLAGS_PREHEADER) * 2 * sizeof(PyObject *) + ); } void _PyObject_GC_Link(PyObject *op); @@ -481,6 +628,14 @@ extern int _PyObject_StoreInstanceAttribute(PyObject *obj, PyDictValues *values, PyObject * _PyObject_GetInstanceAttribute(PyObject *obj, PyDictValues *values, PyObject *name); +#ifdef Py_GIL_DISABLED +# define MANAGED_DICT_OFFSET (((Py_ssize_t)sizeof(PyObject *))*-1) +# define MANAGED_WEAKREF_OFFSET (((Py_ssize_t)sizeof(PyObject *))*-2) +#else +# define MANAGED_DICT_OFFSET (((Py_ssize_t)sizeof(PyObject *))*-3) +# define MANAGED_WEAKREF_OFFSET (((Py_ssize_t)sizeof(PyObject *))*-4) +#endif + typedef union { PyObject *dict; /* Use a char* to generate a warning if directly assigning a PyDictValues */ @@ -491,7 +646,7 @@ static inline PyDictOrValues * _PyObject_DictOrValuesPointer(PyObject *obj) { assert(Py_TYPE(obj)->tp_flags & Py_TPFLAGS_MANAGED_DICT); - return ((PyDictOrValues *)obj)-3; + return (PyDictOrValues *)((char *)obj + MANAGED_DICT_OFFSET); } static inline int @@ -520,8 +675,6 @@ _PyDictOrValues_SetValues(PyDictOrValues *ptr, PyDictValues *values) ptr->values = ((char *)values) - 1; } -#define MANAGED_WEAKREF_OFFSET (((Py_ssize_t)sizeof(PyObject *))*-4) - extern PyObject ** _PyObject_ComputedDictPointer(PyObject *); extern void _PyObject_FreeInstanceAttributes(PyObject *obj); extern int _PyObject_IsInstanceDictEmpty(PyObject *); diff --git a/Include/internal/pycore_object_alloc.h b/Include/internal/pycore_object_alloc.h new file mode 100644 index 00000000000000..8cc7a444bc93e7 --- /dev/null +++ b/Include/internal/pycore_object_alloc.h @@ -0,0 +1,71 @@ +#ifndef Py_INTERNAL_OBJECT_ALLOC_H +#define Py_INTERNAL_OBJECT_ALLOC_H + +#include "pycore_object.h" // _PyType_HasFeature() +#include "pycore_pystate.h" // _PyThreadState_GET() +#include "pycore_tstate.h" // _PyThreadStateImpl + +#ifdef __cplusplus +extern "C" { +#endif + +#ifndef Py_BUILD_CORE +# error "this header requires Py_BUILD_CORE define" +#endif + +#ifdef Py_GIL_DISABLED +static inline mi_heap_t * +_PyObject_GetAllocationHeap(_PyThreadStateImpl *tstate, PyTypeObject *tp) +{ + struct _mimalloc_thread_state *m = &tstate->mimalloc; + if (_PyType_HasFeature(tp, Py_TPFLAGS_PREHEADER)) { + return &m->heaps[_Py_MIMALLOC_HEAP_GC_PRE]; + } + else if (_PyType_IS_GC(tp)) { + return &m->heaps[_Py_MIMALLOC_HEAP_GC]; + } + else { + return &m->heaps[_Py_MIMALLOC_HEAP_OBJECT]; + } +} +#endif + +// Sets the heap used for PyObject_Malloc(), PyObject_Realloc(), etc. calls in +// Py_GIL_DISABLED builds. We use different heaps depending on if the object +// supports GC and if it has a pre-header. We smuggle the choice of heap +// through the _mimalloc_thread_state. In the default build, this simply +// calls PyObject_Malloc(). +static inline void * +_PyObject_MallocWithType(PyTypeObject *tp, size_t size) +{ +#ifdef Py_GIL_DISABLED + _PyThreadStateImpl *tstate = (_PyThreadStateImpl *)_PyThreadState_GET(); + struct _mimalloc_thread_state *m = &tstate->mimalloc; + m->current_object_heap = _PyObject_GetAllocationHeap(tstate, tp); +#endif + void *mem = PyObject_Malloc(size); +#ifdef Py_GIL_DISABLED + m->current_object_heap = &m->heaps[_Py_MIMALLOC_HEAP_OBJECT]; +#endif + return mem; +} + +static inline void * +_PyObject_ReallocWithType(PyTypeObject *tp, void *ptr, size_t size) +{ +#ifdef Py_GIL_DISABLED + _PyThreadStateImpl *tstate = (_PyThreadStateImpl *)_PyThreadState_GET(); + struct _mimalloc_thread_state *m = &tstate->mimalloc; + m->current_object_heap = _PyObject_GetAllocationHeap(tstate, tp); +#endif + void *mem = PyObject_Realloc(ptr, size); +#ifdef Py_GIL_DISABLED + m->current_object_heap = &m->heaps[_Py_MIMALLOC_HEAP_OBJECT]; +#endif + return mem; +} + +#ifdef __cplusplus +} +#endif +#endif // !Py_INTERNAL_OBJECT_ALLOC_H diff --git a/Include/internal/pycore_object_stack.h b/Include/internal/pycore_object_stack.h new file mode 100644 index 00000000000000..fc130b1e9920b4 --- /dev/null +++ b/Include/internal/pycore_object_stack.h @@ -0,0 +1,87 @@ +#ifndef Py_INTERNAL_OBJECT_STACK_H +#define Py_INTERNAL_OBJECT_STACK_H + +#include "pycore_freelist.h" // _PyFreeListState + +#ifdef __cplusplus +extern "C" { +#endif + +#ifndef Py_BUILD_CORE +# error "this header requires Py_BUILD_CORE define" +#endif + +// _PyObjectStack is a stack of Python objects implemented as a linked list of +// fixed size buffers. + +// Chosen so that _PyObjectStackChunk is a power-of-two size. +#define _Py_OBJECT_STACK_CHUNK_SIZE 254 + +typedef struct _PyObjectStackChunk { + struct _PyObjectStackChunk *prev; + Py_ssize_t n; + PyObject *objs[_Py_OBJECT_STACK_CHUNK_SIZE]; +} _PyObjectStackChunk; + +typedef struct _PyObjectStack { + _PyObjectStackChunk *head; +} _PyObjectStack; + + +extern _PyObjectStackChunk * +_PyObjectStackChunk_New(void); + +extern void +_PyObjectStackChunk_Free(_PyObjectStackChunk *); + +// Push an item onto the stack. Return -1 on allocation failure, 0 on success. +static inline int +_PyObjectStack_Push(_PyObjectStack *stack, PyObject *obj) +{ + _PyObjectStackChunk *buf = stack->head; + if (buf == NULL || buf->n == _Py_OBJECT_STACK_CHUNK_SIZE) { + buf = _PyObjectStackChunk_New(); + if (buf == NULL) { + return -1; + } + buf->prev = stack->head; + buf->n = 0; + stack->head = buf; + } + + assert(buf->n >= 0 && buf->n < _Py_OBJECT_STACK_CHUNK_SIZE); + buf->objs[buf->n] = obj; + buf->n++; + return 0; +} + +// Pop the top item from the stack. Return NULL if the stack is empty. +static inline PyObject * +_PyObjectStack_Pop(_PyObjectStack *stack) +{ + _PyObjectStackChunk *buf = stack->head; + if (buf == NULL) { + return NULL; + } + assert(buf->n > 0 && buf->n <= _Py_OBJECT_STACK_CHUNK_SIZE); + buf->n--; + PyObject *obj = buf->objs[buf->n]; + if (buf->n == 0) { + stack->head = buf->prev; + _PyObjectStackChunk_Free(buf); + } + return obj; +} + +// Merge src into dst, leaving src empty +extern void +_PyObjectStack_Merge(_PyObjectStack *dst, _PyObjectStack *src); + +// Remove all items from the stack +extern void +_PyObjectStack_Clear(_PyObjectStack *stack); + +#ifdef __cplusplus +} +#endif +#endif // !Py_INTERNAL_OBJECT_STACK_H diff --git a/Include/internal/pycore_obmalloc.h b/Include/internal/pycore_obmalloc.h index 17572dba65487d..9140d8f08f0af1 100644 --- a/Include/internal/pycore_obmalloc.h +++ b/Include/internal/pycore_obmalloc.h @@ -686,6 +686,8 @@ extern Py_ssize_t _Py_GetGlobalAllocatedBlocks(void); _Py_GetGlobalAllocatedBlocks() extern Py_ssize_t _PyInterpreterState_GetAllocatedBlocks(PyInterpreterState *); extern void _PyInterpreterState_FinalizeAllocatedBlocks(PyInterpreterState *); +extern int _PyMem_init_obmalloc(PyInterpreterState *interp); +extern bool _PyMem_obmalloc_state_on_heap(PyInterpreterState *interp); #ifdef WITH_PYMALLOC diff --git a/Include/internal/pycore_obmalloc_init.h b/Include/internal/pycore_obmalloc_init.h index 8ee72ff2d4126f..e6811b7aeca73c 100644 --- a/Include/internal/pycore_obmalloc_init.h +++ b/Include/internal/pycore_obmalloc_init.h @@ -59,13 +59,6 @@ extern "C" { .dump_debug_stats = -1, \ } -#define _obmalloc_state_INIT(obmalloc) \ - { \ - .pools = { \ - .used = _obmalloc_pools_INIT(obmalloc.pools), \ - }, \ - } - #ifdef __cplusplus } diff --git a/Include/internal/pycore_opcode_metadata.h b/Include/internal/pycore_opcode_metadata.h index 7d39e4bc03099c..75d7f44025328e 100644 --- a/Include/internal/pycore_opcode_metadata.h +++ b/Include/internal/pycore_opcode_metadata.h @@ -176,7 +176,7 @@ int _PyOpcode_num_popped(int opcode, int oparg) { case END_ASYNC_FOR: return 2; case END_FOR: - return 2; + return 1; case END_SEND: return 2; case ENTER_EXECUTOR: @@ -647,7 +647,7 @@ int _PyOpcode_num_pushed(int opcode, int oparg) { case INSTRUMENTED_CALL_KW: return 0; case INSTRUMENTED_END_FOR: - return 0; + return 1; case INSTRUMENTED_END_SEND: return 1; case INSTRUMENTED_FOR_ITER: @@ -909,6 +909,8 @@ enum InstructionFormat { #define HAS_DEOPT_FLAG (128) #define HAS_ERROR_FLAG (256) #define HAS_ESCAPES_FLAG (512) +#define HAS_PURE_FLAG (1024) +#define HAS_PASSTHROUGH_FLAG (2048) #define OPCODE_HAS_ARG(OP) (_PyOpcode_opcode_metadata[OP].flags & (HAS_ARG_FLAG)) #define OPCODE_HAS_CONST(OP) (_PyOpcode_opcode_metadata[OP].flags & (HAS_CONST_FLAG)) #define OPCODE_HAS_NAME(OP) (_PyOpcode_opcode_metadata[OP].flags & (HAS_NAME_FLAG)) @@ -919,6 +921,8 @@ enum InstructionFormat { #define OPCODE_HAS_DEOPT(OP) (_PyOpcode_opcode_metadata[OP].flags & (HAS_DEOPT_FLAG)) #define OPCODE_HAS_ERROR(OP) (_PyOpcode_opcode_metadata[OP].flags & (HAS_ERROR_FLAG)) #define OPCODE_HAS_ESCAPES(OP) (_PyOpcode_opcode_metadata[OP].flags & (HAS_ESCAPES_FLAG)) +#define OPCODE_HAS_PURE(OP) (_PyOpcode_opcode_metadata[OP].flags & (HAS_PURE_FLAG)) +#define OPCODE_HAS_PASSTHROUGH(OP) (_PyOpcode_opcode_metadata[OP].flags & (HAS_PASSTHROUGH_FLAG)) #define OPARG_FULL 0 #define OPARG_CACHE_1 1 @@ -996,7 +1000,7 @@ const struct opcode_metadata _PyOpcode_opcode_metadata[268] = { [COMPARE_OP_STR] = { true, INSTR_FMT_IBC, HAS_ARG_FLAG | HAS_DEOPT_FLAG | HAS_ESCAPES_FLAG }, [CONTAINS_OP] = { true, INSTR_FMT_IB, HAS_ARG_FLAG | HAS_ERROR_FLAG | HAS_ESCAPES_FLAG }, [CONVERT_VALUE] = { true, INSTR_FMT_IB, HAS_ARG_FLAG | HAS_ERROR_FLAG }, - [COPY] = { true, INSTR_FMT_IB, HAS_ARG_FLAG }, + [COPY] = { true, INSTR_FMT_IB, HAS_ARG_FLAG | HAS_PURE_FLAG }, [COPY_FREE_VARS] = { true, INSTR_FMT_IB, HAS_ARG_FLAG }, [DELETE_ATTR] = { true, INSTR_FMT_IB, HAS_ARG_FLAG | HAS_NAME_FLAG | HAS_ERROR_FLAG | HAS_ESCAPES_FLAG }, [DELETE_DEREF] = { true, INSTR_FMT_IB, HAS_ARG_FLAG | HAS_FREE_FLAG | HAS_ERROR_FLAG | HAS_ESCAPES_FLAG }, @@ -1007,9 +1011,9 @@ const struct opcode_metadata _PyOpcode_opcode_metadata[268] = { [DICT_MERGE] = { true, INSTR_FMT_IB, HAS_ARG_FLAG | HAS_ERROR_FLAG | HAS_ESCAPES_FLAG }, [DICT_UPDATE] = { true, INSTR_FMT_IB, HAS_ARG_FLAG | HAS_ERROR_FLAG | HAS_ESCAPES_FLAG }, [END_ASYNC_FOR] = { true, INSTR_FMT_IX, HAS_ERROR_FLAG | HAS_ESCAPES_FLAG }, - [END_FOR] = { true, INSTR_FMT_IX, 0 }, - [END_SEND] = { true, INSTR_FMT_IX, 0 }, - [ENTER_EXECUTOR] = { true, INSTR_FMT_IB, HAS_ARG_FLAG | HAS_EVAL_BREAK_FLAG | HAS_ERROR_FLAG | HAS_ESCAPES_FLAG }, + [END_FOR] = { true, INSTR_FMT_IX, HAS_PURE_FLAG }, + [END_SEND] = { true, INSTR_FMT_IX, HAS_PURE_FLAG }, + [ENTER_EXECUTOR] = { true, INSTR_FMT_IB, HAS_ARG_FLAG | HAS_EVAL_BREAK_FLAG }, [EXIT_INIT_CHECK] = { true, INSTR_FMT_IX, HAS_ERROR_FLAG | HAS_ESCAPES_FLAG }, [EXTENDED_ARG] = { true, INSTR_FMT_IB, HAS_ARG_FLAG }, [FORMAT_SIMPLE] = { true, INSTR_FMT_IX, HAS_ERROR_FLAG | HAS_ESCAPES_FLAG }, @@ -1067,9 +1071,9 @@ const struct opcode_metadata _PyOpcode_opcode_metadata[268] = { [LOAD_ATTR_SLOT] = { true, INSTR_FMT_IBC00000000, HAS_ARG_FLAG | HAS_DEOPT_FLAG }, [LOAD_ATTR_WITH_HINT] = { true, INSTR_FMT_IBC00000000, HAS_ARG_FLAG | HAS_NAME_FLAG | HAS_DEOPT_FLAG | HAS_ESCAPES_FLAG }, [LOAD_BUILD_CLASS] = { true, INSTR_FMT_IX, HAS_ERROR_FLAG | HAS_ESCAPES_FLAG }, - [LOAD_CONST] = { true, INSTR_FMT_IB, HAS_ARG_FLAG | HAS_CONST_FLAG }, + [LOAD_CONST] = { true, INSTR_FMT_IB, HAS_ARG_FLAG | HAS_CONST_FLAG | HAS_PURE_FLAG }, [LOAD_DEREF] = { true, INSTR_FMT_IB, HAS_ARG_FLAG | HAS_FREE_FLAG | HAS_ERROR_FLAG | HAS_ESCAPES_FLAG }, - [LOAD_FAST] = { true, INSTR_FMT_IB, HAS_ARG_FLAG | HAS_LOCAL_FLAG }, + [LOAD_FAST] = { true, INSTR_FMT_IB, HAS_ARG_FLAG | HAS_LOCAL_FLAG | HAS_PURE_FLAG }, [LOAD_FAST_AND_CLEAR] = { true, INSTR_FMT_IB, HAS_ARG_FLAG | HAS_LOCAL_FLAG }, [LOAD_FAST_CHECK] = { true, INSTR_FMT_IB, HAS_ARG_FLAG | HAS_LOCAL_FLAG | HAS_ERROR_FLAG }, [LOAD_FAST_LOAD_FAST] = { true, INSTR_FMT_IB, HAS_ARG_FLAG | HAS_LOCAL_FLAG }, @@ -1096,9 +1100,9 @@ const struct opcode_metadata _PyOpcode_opcode_metadata[268] = { [POP_JUMP_IF_NONE] = { true, INSTR_FMT_IBC, HAS_ARG_FLAG | HAS_JUMP_FLAG }, [POP_JUMP_IF_NOT_NONE] = { true, INSTR_FMT_IBC, HAS_ARG_FLAG | HAS_JUMP_FLAG }, [POP_JUMP_IF_TRUE] = { true, INSTR_FMT_IBC, HAS_ARG_FLAG | HAS_JUMP_FLAG }, - [POP_TOP] = { true, INSTR_FMT_IX, 0 }, + [POP_TOP] = { true, INSTR_FMT_IX, HAS_PURE_FLAG }, [PUSH_EXC_INFO] = { true, INSTR_FMT_IX, 0 }, - [PUSH_NULL] = { true, INSTR_FMT_IX, 0 }, + [PUSH_NULL] = { true, INSTR_FMT_IX, HAS_PURE_FLAG }, [RAISE_VARARGS] = { true, INSTR_FMT_IB, HAS_ARG_FLAG | HAS_ERROR_FLAG | HAS_ESCAPES_FLAG }, [RERAISE] = { true, INSTR_FMT_IB, HAS_ARG_FLAG | HAS_ERROR_FLAG | HAS_ESCAPES_FLAG }, [RESERVED] = { true, INSTR_FMT_IX, HAS_ESCAPES_FLAG }, @@ -1127,7 +1131,7 @@ const struct opcode_metadata _PyOpcode_opcode_metadata[268] = { [STORE_SUBSCR] = { true, INSTR_FMT_IXC, HAS_ERROR_FLAG | HAS_ESCAPES_FLAG }, [STORE_SUBSCR_DICT] = { true, INSTR_FMT_IXC, HAS_DEOPT_FLAG | HAS_ERROR_FLAG | HAS_ESCAPES_FLAG }, [STORE_SUBSCR_LIST_INT] = { true, INSTR_FMT_IXC, HAS_DEOPT_FLAG | HAS_ESCAPES_FLAG }, - [SWAP] = { true, INSTR_FMT_IB, HAS_ARG_FLAG }, + [SWAP] = { true, INSTR_FMT_IB, HAS_ARG_FLAG | HAS_PURE_FLAG }, [TO_BOOL] = { true, INSTR_FMT_IXC00, HAS_ERROR_FLAG | HAS_ESCAPES_FLAG }, [TO_BOOL_ALWAYS_TRUE] = { true, INSTR_FMT_IXC00, HAS_DEOPT_FLAG }, [TO_BOOL_BOOL] = { true, INSTR_FMT_IXC00, HAS_DEOPT_FLAG }, @@ -1137,7 +1141,7 @@ const struct opcode_metadata _PyOpcode_opcode_metadata[268] = { [TO_BOOL_STR] = { true, INSTR_FMT_IXC00, HAS_DEOPT_FLAG }, [UNARY_INVERT] = { true, INSTR_FMT_IX, HAS_ERROR_FLAG | HAS_ESCAPES_FLAG }, [UNARY_NEGATIVE] = { true, INSTR_FMT_IX, HAS_ERROR_FLAG | HAS_ESCAPES_FLAG }, - [UNARY_NOT] = { true, INSTR_FMT_IX, 0 }, + [UNARY_NOT] = { true, INSTR_FMT_IX, HAS_PURE_FLAG }, [UNPACK_EX] = { true, INSTR_FMT_IB, HAS_ARG_FLAG | HAS_ERROR_FLAG | HAS_ESCAPES_FLAG }, [UNPACK_SEQUENCE] = { true, INSTR_FMT_IBC, HAS_ARG_FLAG | HAS_ERROR_FLAG | HAS_ESCAPES_FLAG }, [UNPACK_SEQUENCE_LIST] = { true, INSTR_FMT_IBC, HAS_ARG_FLAG | HAS_DEOPT_FLAG }, @@ -1147,7 +1151,7 @@ const struct opcode_metadata _PyOpcode_opcode_metadata[268] = { [YIELD_VALUE] = { true, INSTR_FMT_IB, HAS_ARG_FLAG | HAS_ESCAPES_FLAG }, [JUMP] = { true, -1, HAS_ARG_FLAG | HAS_JUMP_FLAG | HAS_EVAL_BREAK_FLAG | HAS_ERROR_FLAG | HAS_ESCAPES_FLAG }, [JUMP_NO_INTERRUPT] = { true, -1, HAS_ARG_FLAG | HAS_JUMP_FLAG }, - [LOAD_CLOSURE] = { true, -1, HAS_ARG_FLAG | HAS_LOCAL_FLAG }, + [LOAD_CLOSURE] = { true, -1, HAS_ARG_FLAG | HAS_LOCAL_FLAG | HAS_PURE_FLAG }, [LOAD_METHOD] = { true, -1, HAS_ARG_FLAG | HAS_NAME_FLAG | HAS_ERROR_FLAG | HAS_ESCAPES_FLAG }, [LOAD_SUPER_METHOD] = { true, -1, HAS_ARG_FLAG | HAS_NAME_FLAG | HAS_ERROR_FLAG | HAS_ESCAPES_FLAG }, [LOAD_ZERO_SUPER_ATTR] = { true, -1, HAS_ARG_FLAG | HAS_NAME_FLAG | HAS_ERROR_FLAG | HAS_ESCAPES_FLAG }, @@ -1228,7 +1232,7 @@ _PyOpcode_macro_expansion[256] = { [DELETE_SUBSCR] = { .nuops = 1, .uops = { { _DELETE_SUBSCR, 0, 0 } } }, [DICT_MERGE] = { .nuops = 1, .uops = { { _DICT_MERGE, 0, 0 } } }, [DICT_UPDATE] = { .nuops = 1, .uops = { { _DICT_UPDATE, 0, 0 } } }, - [END_FOR] = { .nuops = 2, .uops = { { _POP_TOP, 0, 0 }, { _POP_TOP, 0, 0 } } }, + [END_FOR] = { .nuops = 1, .uops = { { _POP_TOP, 0, 0 } } }, [END_SEND] = { .nuops = 1, .uops = { { _END_SEND, 0, 0 } } }, [EXIT_INIT_CHECK] = { .nuops = 1, .uops = { { _EXIT_INIT_CHECK, 0, 0 } } }, [FORMAT_SIMPLE] = { .nuops = 1, .uops = { { _FORMAT_SIMPLE, 0, 0 } } }, diff --git a/Include/internal/pycore_optimizer.h b/Include/internal/pycore_optimizer.h index b052460b44b791..e21412fc815540 100644 --- a/Include/internal/pycore_optimizer.h +++ b/Include/internal/pycore_optimizer.h @@ -8,10 +8,9 @@ extern "C" { # error "this header requires Py_BUILD_CORE define" #endif -#include "pycore_uops.h" // _PyUOpInstruction - -int _Py_uop_analyze_and_optimize(PyCodeObject *code, - _PyUOpInstruction *trace, int trace_len, int curr_stackentries); +int _Py_uop_analyze_and_optimize(_PyInterpreterFrame *frame, + _PyUOpInstruction *trace, int trace_len, int curr_stackentries, + _PyBloomFilter *dependencies); extern PyTypeObject _PyCounterExecutor_Type; extern PyTypeObject _PyCounterOptimizer_Type; diff --git a/Include/internal/pycore_pymem.h b/Include/internal/pycore_pymem.h index 8631ca34a5e616..1a72d07b50b738 100644 --- a/Include/internal/pycore_pymem.h +++ b/Include/internal/pycore_pymem.h @@ -44,6 +44,7 @@ struct _pymem_allocators { debug_alloc_api_t mem; debug_alloc_api_t obj; } debug; + int is_debug_enabled; PyObjectArenaAllocator obj_arena; }; @@ -64,7 +65,7 @@ extern int _PyMem_SetDefaultAllocator( - PYMEM_FORBIDDENBYTE: untouchable bytes at each end of a block Byte patterns 0xCB, 0xDB and 0xFB have been replaced with 0xCD, 0xDD and - 0xFD to use the same values than Windows CRT debug malloc() and free(). + 0xFD to use the same values as Windows CRT debug malloc() and free(). If modified, _PyMem_IsPtrFreed() should be updated as well. */ #define PYMEM_CLEANBYTE 0xCD #define PYMEM_DEADBYTE 0xDD @@ -106,6 +107,8 @@ extern int _PyMem_GetAllocatorName( PYMEM_ALLOCATOR_NOT_SET does nothing. */ extern int _PyMem_SetupAllocators(PyMemAllocatorName allocator); +/* Is the debug allocator enabled? */ +extern int _PyMem_DebugEnabled(void); #ifdef __cplusplus } diff --git a/Include/internal/pycore_pymem_init.h b/Include/internal/pycore_pymem_init.h index 360fb9218a9cda..96c49ed7338d6d 100644 --- a/Include/internal/pycore_pymem_init.h +++ b/Include/internal/pycore_pymem_init.h @@ -70,6 +70,7 @@ extern void _PyMem_ArenaFree(void *, void *, size_t); PYDBGMEM_ALLOC(runtime), \ PYDBGOBJ_ALLOC(runtime), \ } +# define _pymem_is_debug_enabled_INIT 1 #else # define _pymem_allocators_standard_INIT(runtime) \ { \ @@ -77,6 +78,7 @@ extern void _PyMem_ArenaFree(void *, void *, size_t); PYMEM_ALLOC, \ PYOBJ_ALLOC, \ } +# define _pymem_is_debug_enabled_INIT 0 #endif #define _pymem_allocators_debug_INIT \ diff --git a/Include/internal/pycore_pystate.h b/Include/internal/pycore_pystate.h index c031a38cd6bfa3..289ef28f0dd9a9 100644 --- a/Include/internal/pycore_pystate.h +++ b/Include/internal/pycore_pystate.h @@ -8,7 +8,9 @@ extern "C" { # error "this header requires Py_BUILD_CORE define" #endif +#include "pycore_freelist.h" // _PyFreeListState #include "pycore_runtime.h" // _PyRuntime +#include "pycore_tstate.h" // _PyThreadStateImpl // Values for PyThreadState.state. A thread must be in the "attached" state @@ -19,23 +21,27 @@ extern "C" { // interpreter at the same time. Only the "bound" thread may perform the // transitions between "attached" and "detached" on its own PyThreadState. // -// The "gc" state is used to implement stop-the-world pauses, such as for -// cyclic garbage collection. It is only used in `--disable-gil` builds. It is -// similar to the "detached" state, but only the thread performing a -// stop-the-world pause may transition threads between the "detached" and "gc" -// states. A thread trying to "attach" from the "gc" state will block until -// it is transitioned back to "detached" when the stop-the-world pause is -// complete. +// The "suspended" state is used to implement stop-the-world pauses, such as +// for cyclic garbage collection. It is only used in `--disable-gil` builds. +// The "suspended" state is similar to the "detached" state in that in both +// states the thread is not allowed to call most Python APIs. However, unlike +// the "detached" state, a thread may not transition itself out from the +// "suspended" state. Only the thread performing a stop-the-world pause may +// transition a thread from the "suspended" state back to the "detached" state. // // State transition diagram: // // (bound thread) (stop-the-world thread) -// [attached] <-> [detached] <-> [gc] +// [attached] <-> [detached] <-> [suspended] +// | ^ +// +---------------------------->---------------------------+ +// (bound thread) // -// See `_PyThreadState_Attach()` and `_PyThreadState_Detach()`. +// The (bound thread) and (stop-the-world thread) labels indicate which thread +// is allowed to perform the transition. #define _Py_THREAD_DETACHED 0 #define _Py_THREAD_ATTACHED 1 -#define _Py_THREAD_GC 2 +#define _Py_THREAD_SUSPENDED 2 /* Check if the current thread is the main thread. @@ -138,13 +144,36 @@ _PyThreadState_GET(void) // // High-level code should generally call PyEval_RestoreThread() instead, which // calls this function. -void _PyThreadState_Attach(PyThreadState *tstate); +extern void _PyThreadState_Attach(PyThreadState *tstate); // Detaches the current thread from the interpreter. // // High-level code should generally call PyEval_SaveThread() instead, which // calls this function. -void _PyThreadState_Detach(PyThreadState *tstate); +extern void _PyThreadState_Detach(PyThreadState *tstate); + +// Detaches the current thread to the "suspended" state if a stop-the-world +// pause is in progress. +// +// If there is no stop-the-world pause in progress, then the thread switches +// to the "detached" state. +extern void _PyThreadState_Suspend(PyThreadState *tstate); + +// Perform a stop-the-world pause for all threads in the all interpreters. +// +// Threads in the "attached" state are paused and transitioned to the "GC" +// state. Threads in the "detached" state switch to the "GC" state, preventing +// them from reattaching until the stop-the-world pause is complete. +// +// NOTE: This is a no-op outside of Py_GIL_DISABLED builds. +extern void _PyEval_StopTheWorldAll(_PyRuntimeState *runtime); +extern void _PyEval_StartTheWorldAll(_PyRuntimeState *runtime); + +// Perform a stop-the-world pause for threads in the specified interpreter. +// +// NOTE: This is a no-op outside of Py_GIL_DISABLED builds. +extern void _PyEval_StopTheWorld(PyInterpreterState *interp); +extern void _PyEval_StartTheWorld(PyInterpreterState *interp); static inline void @@ -187,6 +216,7 @@ extern PyThreadState * _PyThreadState_New( int whence); extern void _PyThreadState_Bind(PyThreadState *tstate); extern void _PyThreadState_DeleteExcept(PyThreadState *tstate); +extern void _PyThreadState_ClearMimallocHeaps(PyThreadState *tstate); // Export for '_testinternalcapi' shared extension PyAPI_FUNC(PyObject*) _PyThreadState_GetDict(PyThreadState *tstate); @@ -238,6 +268,20 @@ PyAPI_FUNC(const PyConfig*) _Py_GetConfig(void); // See also PyInterpreterState_Get() and _PyInterpreterState_GET(). extern PyInterpreterState* _PyGILState_GetInterpreterStateUnsafe(void); +static inline _PyFreeListState* _PyFreeListState_GET(void) +{ + PyThreadState *tstate = _PyThreadState_GET(); +#ifdef Py_DEBUG + _Py_EnsureTstateNotNULL(tstate); +#endif + +#ifdef Py_GIL_DISABLED + return &((_PyThreadStateImpl*)tstate)->freelist_state; +#else + return &tstate->interp->freelist_state; +#endif +} + #ifdef __cplusplus } #endif diff --git a/Include/internal/pycore_pythread.h b/Include/internal/pycore_pythread.h index 9c9a09f60f3441..265299d7574838 100644 --- a/Include/internal/pycore_pythread.h +++ b/Include/internal/pycore_pythread.h @@ -9,6 +9,7 @@ extern "C" { #endif #include "dynamic_annotations.h" // _Py_ANNOTATE_PURE_HAPPENS_BEFORE_MUTEX +#include "pycore_llist.h" // struct llist_node // Get _POSIX_THREADS and _POSIX_SEMAPHORES macros if available #if (defined(HAVE_UNISTD_H) && !defined(_POSIX_THREADS) \ @@ -75,14 +76,22 @@ struct _pythread_runtime_state { struct py_stub_tls_entry tls_entries[PTHREAD_KEYS_MAX]; } stubs; #endif + + // Linked list of ThreadHandleObjects + struct llist_node handles; }; +#define _pythread_RUNTIME_INIT(pythread) \ + { \ + .handles = LLIST_INIT(pythread.handles), \ + } #ifdef HAVE_FORK /* Private function to reinitialize a lock at fork in the child process. Reset the lock to the unlocked state. Return 0 on success, return -1 on error. */ extern int _PyThread_at_fork_reinit(PyThread_type_lock *lock); +extern void _PyThread_AfterFork(struct _pythread_runtime_state *state); #endif /* HAVE_FORK */ @@ -143,12 +152,6 @@ PyAPI_FUNC(int) PyThread_join_thread(PyThread_handle_t); */ PyAPI_FUNC(int) PyThread_detach_thread(PyThread_handle_t); -/* - * Obtain the new thread ident and handle in a forked child process. - */ -PyAPI_FUNC(void) PyThread_update_thread_after_fork(PyThread_ident_t* ident, - PyThread_handle_t* handle); - #ifdef __cplusplus } #endif diff --git a/Include/internal/pycore_runtime.h b/Include/internal/pycore_runtime.h index e3348296ea61b7..7c705d1224f915 100644 --- a/Include/internal/pycore_runtime.h +++ b/Include/internal/pycore_runtime.h @@ -227,6 +227,13 @@ typedef struct pyruntimestate { struct _faulthandler_runtime_state faulthandler; struct _tracemalloc_runtime_state tracemalloc; + // The rwmutex is used to prevent overlapping global and per-interpreter + // stop-the-world events. Global stop-the-world events lock the mutex + // exclusively (as a "writer"), while per-interpreter stop-the-world events + // lock it non-exclusively (as "readers"). + _PyRWMutex stoptheworld_mutex; + struct _stoptheworld_state stoptheworld; + PyPreConfig preconfig; // Audit values must be preserved when Py_Initialize()/Py_Finalize() @@ -261,7 +268,7 @@ typedef struct pyruntimestate { a pointer type. */ - /* PyInterpreterState.interpreters.main */ + /* _PyRuntimeState.interpreters.main */ PyInterpreterState _main_interpreter; #if defined(__EMSCRIPTEN__) && defined(PY_CALL_TRAMPOLINE) diff --git a/Include/internal/pycore_runtime_init.h b/Include/internal/pycore_runtime_init.h index d324a94278839c..571a7d612c94e2 100644 --- a/Include/internal/pycore_runtime_init.h +++ b/Include/internal/pycore_runtime_init.h @@ -16,6 +16,7 @@ extern "C" { #include "pycore_parser.h" // _parser_runtime_state_INIT #include "pycore_pyhash.h" // pyhash_state_INIT #include "pycore_pymem_init.h" // _pymem_allocators_standard_INIT +#include "pycore_pythread.h" // _pythread_RUNTIME_INIT #include "pycore_runtime_init_generated.h" // _Py_bytes_characters_INIT #include "pycore_signal.h" // _signals_RUNTIME_INIT #include "pycore_tracemalloc.h" // _tracemalloc_runtime_state_INIT @@ -86,9 +87,11 @@ extern PyTypeObject _PyExc_MemoryError; .standard = _pymem_allocators_standard_INIT(runtime), \ .debug = _pymem_allocators_debug_INIT, \ .obj_arena = _pymem_allocators_obj_arena_INIT, \ + .is_debug_enabled = _pymem_is_debug_enabled_INIT, \ }, \ .obmalloc = _obmalloc_global_state_INIT, \ .pyhash_state = pyhash_state_INIT, \ + .threads = _pythread_RUNTIME_INIT(runtime.threads), \ .signals = _signals_RUNTIME_INIT, \ .interpreters = { \ /* This prevents interpreters from getting created \ @@ -115,6 +118,9 @@ extern PyTypeObject _PyExc_MemoryError; }, \ .faulthandler = _faulthandler_runtime_state_INIT, \ .tracemalloc = _tracemalloc_runtime_state_INIT, \ + .stoptheworld = { \ + .is_global = 1, \ + }, \ .float_state = { \ .float_format = _py_float_format_unknown, \ .double_format = _py_float_format_unknown, \ @@ -151,7 +157,6 @@ extern PyTypeObject _PyExc_MemoryError; { \ .id_refcount = -1, \ .imports = IMPORTS_INIT, \ - .obmalloc = _obmalloc_state_INIT(INTERP.obmalloc), \ .ceval = { \ .recursion_limit = Py_DEFAULT_RECURSION_LIMIT, \ }, \ diff --git a/Include/internal/pycore_runtime_init_generated.h b/Include/internal/pycore_runtime_init_generated.h index 1defa39f816e78..68fbbcb4378e17 100644 --- a/Include/internal/pycore_runtime_init_generated.h +++ b/Include/internal/pycore_runtime_init_generated.h @@ -785,8 +785,11 @@ extern "C" { INIT_ID(after_in_child), \ INIT_ID(after_in_parent), \ INIT_ID(aggregate_class), \ + INIT_ID(alias), \ + INIT_ID(allow_code), \ INIT_ID(append), \ INIT_ID(argdefs), \ + INIT_ID(args), \ INIT_ID(arguments), \ INIT_ID(argv), \ INIT_ID(as_integer_ratio), \ @@ -871,6 +874,7 @@ extern "C" { INIT_ID(d), \ INIT_ID(data), \ INIT_ID(database), \ + INIT_ID(day), \ INIT_ID(decode), \ INIT_ID(decoder), \ INIT_ID(default), \ @@ -910,6 +914,8 @@ extern "C" { INIT_ID(errors), \ INIT_ID(event), \ INIT_ID(eventmask), \ + INIT_ID(exc_type), \ + INIT_ID(exc_value), \ INIT_ID(excepthook), \ INIT_ID(exception), \ INIT_ID(existing_file_name), \ @@ -932,12 +938,14 @@ extern "C" { INIT_ID(fileno), \ INIT_ID(filepath), \ INIT_ID(fillvalue), \ + INIT_ID(filter), \ INIT_ID(filters), \ INIT_ID(final), \ INIT_ID(find_class), \ INIT_ID(fix_imports), \ INIT_ID(flags), \ INIT_ID(flush), \ + INIT_ID(fold), \ INIT_ID(follow_symlinks), \ INIT_ID(format), \ INIT_ID(from_param), \ @@ -968,6 +976,7 @@ extern "C" { INIT_ID(headers), \ INIT_ID(hi), \ INIT_ID(hook), \ + INIT_ID(hour), \ INIT_ID(id), \ INIT_ID(ident), \ INIT_ID(identity_hint), \ @@ -1013,6 +1022,7 @@ extern "C" { INIT_ID(kw), \ INIT_ID(kw1), \ INIT_ID(kw2), \ + INIT_ID(kwdefaults), \ INIT_ID(lambda), \ INIT_ID(last), \ INIT_ID(last_exc), \ @@ -1041,6 +1051,7 @@ extern "C" { INIT_ID(max_length), \ INIT_ID(maxdigits), \ INIT_ID(maxevents), \ + INIT_ID(maxlen), \ INIT_ID(maxmem), \ INIT_ID(maxsplit), \ INIT_ID(maxvalue), \ @@ -1050,11 +1061,14 @@ extern "C" { INIT_ID(metaclass), \ INIT_ID(metadata), \ INIT_ID(method), \ + INIT_ID(microsecond), \ + INIT_ID(minute), \ INIT_ID(mod), \ INIT_ID(mode), \ INIT_ID(module), \ INIT_ID(module_globals), \ INIT_ID(modules), \ + INIT_ID(month), \ INIT_ID(mro), \ INIT_ID(msg), \ INIT_ID(mycmp), \ @@ -1159,6 +1173,7 @@ extern "C" { INIT_ID(salt), \ INIT_ID(sched_priority), \ INIT_ID(scheduler), \ + INIT_ID(second), \ INIT_ID(seek), \ INIT_ID(seekable), \ INIT_ID(selectors), \ @@ -1225,6 +1240,7 @@ extern "C" { INIT_ID(timetuple), \ INIT_ID(top), \ INIT_ID(trace_callback), \ + INIT_ID(traceback), \ INIT_ID(trailers), \ INIT_ID(translate), \ INIT_ID(true), \ @@ -1234,6 +1250,7 @@ extern "C" { INIT_ID(type), \ INIT_ID(type_params), \ INIT_ID(tz), \ + INIT_ID(tzinfo), \ INIT_ID(tzname), \ INIT_ID(uid), \ INIT_ID(unlink), \ diff --git a/Include/internal/pycore_sliceobject.h b/Include/internal/pycore_sliceobject.h index 98665c3859d574..89086f67683a2f 100644 --- a/Include/internal/pycore_sliceobject.h +++ b/Include/internal/pycore_sliceobject.h @@ -11,8 +11,6 @@ extern "C" { /* runtime lifecycle */ -extern void _PySlice_Fini(PyInterpreterState *); - extern PyObject * _PyBuildSlice_ConsumeRefs(PyObject *start, PyObject *stop); diff --git a/Include/internal/pycore_symtable.h b/Include/internal/pycore_symtable.h index 1d782ca2c96e05..b44393b5644673 100644 --- a/Include/internal/pycore_symtable.h +++ b/Include/internal/pycore_symtable.h @@ -109,18 +109,18 @@ extern PyObject* _Py_Mangle(PyObject *p, PyObject *name); /* Flags for def-use information */ -#define DEF_GLOBAL 1 /* global stmt */ -#define DEF_LOCAL 2 /* assignment in code block */ -#define DEF_PARAM 2<<1 /* formal parameter */ -#define DEF_NONLOCAL 2<<2 /* nonlocal stmt */ -#define USE 2<<3 /* name is used */ -#define DEF_FREE 2<<4 /* name used but not defined in nested block */ -#define DEF_FREE_CLASS 2<<5 /* free variable from class's method */ -#define DEF_IMPORT 2<<6 /* assignment occurred via import */ -#define DEF_ANNOT 2<<7 /* this name is annotated */ -#define DEF_COMP_ITER 2<<8 /* this name is a comprehension iteration variable */ -#define DEF_TYPE_PARAM 2<<9 /* this name is a type parameter */ -#define DEF_COMP_CELL 2<<10 /* this name is a cell in an inlined comprehension */ +#define DEF_GLOBAL 1 /* global stmt */ +#define DEF_LOCAL 2 /* assignment in code block */ +#define DEF_PARAM (2<<1) /* formal parameter */ +#define DEF_NONLOCAL (2<<2) /* nonlocal stmt */ +#define USE (2<<3) /* name is used */ +#define DEF_FREE (2<<4) /* name used but not defined in nested block */ +#define DEF_FREE_CLASS (2<<5) /* free variable from class's method */ +#define DEF_IMPORT (2<<6) /* assignment occurred via import */ +#define DEF_ANNOT (2<<7) /* this name is annotated */ +#define DEF_COMP_ITER (2<<8) /* this name is a comprehension iteration variable */ +#define DEF_TYPE_PARAM (2<<9) /* this name is a type parameter */ +#define DEF_COMP_CELL (2<<10) /* this name is a cell in an inlined comprehension */ #define DEF_BOUND (DEF_LOCAL | DEF_PARAM | DEF_IMPORT) diff --git a/Include/internal/pycore_tstate.h b/Include/internal/pycore_tstate.h index 17f3e865641773..77a1dc59163d21 100644 --- a/Include/internal/pycore_tstate.h +++ b/Include/internal/pycore_tstate.h @@ -8,6 +8,10 @@ extern "C" { # error "this header requires Py_BUILD_CORE define" #endif +#include "pycore_freelist.h" // struct _Py_freelist_state +#include "pycore_mimalloc.h" // struct _mimalloc_thread_state +#include "pycore_brc.h" // struct _brc_thread_state + // Every PyThreadState is actually allocated as a _PyThreadStateImpl. The // PyThreadState fields are exposed as part of the C API, although most fields @@ -16,7 +20,12 @@ typedef struct _PyThreadStateImpl { // semi-public fields are in PyThreadState. PyThreadState base; - // TODO: add private fields here +#ifdef Py_GIL_DISABLED + struct _mimalloc_thread_state mimalloc; + struct _Py_freelist_state freelist_state; + struct _brc_thread_state brc; +#endif + } _PyThreadStateImpl; diff --git a/Include/internal/pycore_tuple.h b/Include/internal/pycore_tuple.h index 4fa7a12206bcb2..4605f355ccbc38 100644 --- a/Include/internal/pycore_tuple.h +++ b/Include/internal/pycore_tuple.h @@ -14,59 +14,15 @@ extern void _PyTuple_DebugMallocStats(FILE *out); /* runtime lifecycle */ extern PyStatus _PyTuple_InitGlobalObjects(PyInterpreterState *); -extern void _PyTuple_Fini(PyInterpreterState *); /* other API */ -// PyTuple_MAXSAVESIZE - largest tuple to save on free list -// PyTuple_MAXFREELIST - maximum number of tuples of each size to save - -#if defined(PyTuple_MAXSAVESIZE) && PyTuple_MAXSAVESIZE <= 0 - // A build indicated that tuple freelists should not be used. -# define PyTuple_NFREELISTS 0 -# undef PyTuple_MAXSAVESIZE -# undef PyTuple_MAXFREELIST - -#elif !defined(WITH_FREELISTS) -# define PyTuple_NFREELISTS 0 -# undef PyTuple_MAXSAVESIZE -# undef PyTuple_MAXFREELIST - -#else - // We are using a freelist for tuples. -# ifndef PyTuple_MAXSAVESIZE -# define PyTuple_MAXSAVESIZE 20 -# endif -# define PyTuple_NFREELISTS PyTuple_MAXSAVESIZE -# ifndef PyTuple_MAXFREELIST -# define PyTuple_MAXFREELIST 2000 -# endif -#endif - -struct _Py_tuple_state { -#if PyTuple_NFREELISTS > 0 - /* There is one freelist for each size from 1 to PyTuple_MAXSAVESIZE. - The empty tuple is handled separately. - - Each tuple stored in the array is the head of the linked list - (and the next available tuple) for that size. The actual tuple - object is used as the linked list node, with its first item - (ob_item[0]) pointing to the next node (i.e. the previous head). - Each linked list is initially NULL. */ - PyTupleObject *free_list[PyTuple_NFREELISTS]; - int numfree[PyTuple_NFREELISTS]; -#else - char _unused; // Empty structs are not allowed. -#endif -}; - #define _PyTuple_ITEMS(op) _Py_RVALUE(_PyTuple_CAST(op)->ob_item) extern PyObject *_PyTuple_FromArray(PyObject *const *, Py_ssize_t); extern PyObject *_PyTuple_FromArraySteal(PyObject *const *, Py_ssize_t); - typedef struct { PyObject_HEAD Py_ssize_t it_index; diff --git a/Include/internal/pycore_unicodeobject_generated.h b/Include/internal/pycore_unicodeobject_generated.h index be9baa3eebecfc..c8458b4e36ccc9 100644 --- a/Include/internal/pycore_unicodeobject_generated.h +++ b/Include/internal/pycore_unicodeobject_generated.h @@ -669,12 +669,21 @@ _PyUnicode_InitStaticStrings(PyInterpreterState *interp) { string = &_Py_ID(aggregate_class); assert(_PyUnicode_CheckConsistency(string, 1)); _PyUnicode_InternInPlace(interp, &string); + string = &_Py_ID(alias); + assert(_PyUnicode_CheckConsistency(string, 1)); + _PyUnicode_InternInPlace(interp, &string); + string = &_Py_ID(allow_code); + assert(_PyUnicode_CheckConsistency(string, 1)); + _PyUnicode_InternInPlace(interp, &string); string = &_Py_ID(append); assert(_PyUnicode_CheckConsistency(string, 1)); _PyUnicode_InternInPlace(interp, &string); string = &_Py_ID(argdefs); assert(_PyUnicode_CheckConsistency(string, 1)); _PyUnicode_InternInPlace(interp, &string); + string = &_Py_ID(args); + assert(_PyUnicode_CheckConsistency(string, 1)); + _PyUnicode_InternInPlace(interp, &string); string = &_Py_ID(arguments); assert(_PyUnicode_CheckConsistency(string, 1)); _PyUnicode_InternInPlace(interp, &string); @@ -927,6 +936,9 @@ _PyUnicode_InitStaticStrings(PyInterpreterState *interp) { string = &_Py_ID(database); assert(_PyUnicode_CheckConsistency(string, 1)); _PyUnicode_InternInPlace(interp, &string); + string = &_Py_ID(day); + assert(_PyUnicode_CheckConsistency(string, 1)); + _PyUnicode_InternInPlace(interp, &string); string = &_Py_ID(decode); assert(_PyUnicode_CheckConsistency(string, 1)); _PyUnicode_InternInPlace(interp, &string); @@ -1044,6 +1056,12 @@ _PyUnicode_InitStaticStrings(PyInterpreterState *interp) { string = &_Py_ID(eventmask); assert(_PyUnicode_CheckConsistency(string, 1)); _PyUnicode_InternInPlace(interp, &string); + string = &_Py_ID(exc_type); + assert(_PyUnicode_CheckConsistency(string, 1)); + _PyUnicode_InternInPlace(interp, &string); + string = &_Py_ID(exc_value); + assert(_PyUnicode_CheckConsistency(string, 1)); + _PyUnicode_InternInPlace(interp, &string); string = &_Py_ID(excepthook); assert(_PyUnicode_CheckConsistency(string, 1)); _PyUnicode_InternInPlace(interp, &string); @@ -1110,6 +1128,9 @@ _PyUnicode_InitStaticStrings(PyInterpreterState *interp) { string = &_Py_ID(fillvalue); assert(_PyUnicode_CheckConsistency(string, 1)); _PyUnicode_InternInPlace(interp, &string); + string = &_Py_ID(filter); + assert(_PyUnicode_CheckConsistency(string, 1)); + _PyUnicode_InternInPlace(interp, &string); string = &_Py_ID(filters); assert(_PyUnicode_CheckConsistency(string, 1)); _PyUnicode_InternInPlace(interp, &string); @@ -1128,6 +1149,9 @@ _PyUnicode_InitStaticStrings(PyInterpreterState *interp) { string = &_Py_ID(flush); assert(_PyUnicode_CheckConsistency(string, 1)); _PyUnicode_InternInPlace(interp, &string); + string = &_Py_ID(fold); + assert(_PyUnicode_CheckConsistency(string, 1)); + _PyUnicode_InternInPlace(interp, &string); string = &_Py_ID(follow_symlinks); assert(_PyUnicode_CheckConsistency(string, 1)); _PyUnicode_InternInPlace(interp, &string); @@ -1218,6 +1242,9 @@ _PyUnicode_InitStaticStrings(PyInterpreterState *interp) { string = &_Py_ID(hook); assert(_PyUnicode_CheckConsistency(string, 1)); _PyUnicode_InternInPlace(interp, &string); + string = &_Py_ID(hour); + assert(_PyUnicode_CheckConsistency(string, 1)); + _PyUnicode_InternInPlace(interp, &string); string = &_Py_ID(id); assert(_PyUnicode_CheckConsistency(string, 1)); _PyUnicode_InternInPlace(interp, &string); @@ -1353,6 +1380,9 @@ _PyUnicode_InitStaticStrings(PyInterpreterState *interp) { string = &_Py_ID(kw2); assert(_PyUnicode_CheckConsistency(string, 1)); _PyUnicode_InternInPlace(interp, &string); + string = &_Py_ID(kwdefaults); + assert(_PyUnicode_CheckConsistency(string, 1)); + _PyUnicode_InternInPlace(interp, &string); string = &_Py_ID(lambda); assert(_PyUnicode_CheckConsistency(string, 1)); _PyUnicode_InternInPlace(interp, &string); @@ -1437,6 +1467,9 @@ _PyUnicode_InitStaticStrings(PyInterpreterState *interp) { string = &_Py_ID(maxevents); assert(_PyUnicode_CheckConsistency(string, 1)); _PyUnicode_InternInPlace(interp, &string); + string = &_Py_ID(maxlen); + assert(_PyUnicode_CheckConsistency(string, 1)); + _PyUnicode_InternInPlace(interp, &string); string = &_Py_ID(maxmem); assert(_PyUnicode_CheckConsistency(string, 1)); _PyUnicode_InternInPlace(interp, &string); @@ -1464,6 +1497,12 @@ _PyUnicode_InitStaticStrings(PyInterpreterState *interp) { string = &_Py_ID(method); assert(_PyUnicode_CheckConsistency(string, 1)); _PyUnicode_InternInPlace(interp, &string); + string = &_Py_ID(microsecond); + assert(_PyUnicode_CheckConsistency(string, 1)); + _PyUnicode_InternInPlace(interp, &string); + string = &_Py_ID(minute); + assert(_PyUnicode_CheckConsistency(string, 1)); + _PyUnicode_InternInPlace(interp, &string); string = &_Py_ID(mod); assert(_PyUnicode_CheckConsistency(string, 1)); _PyUnicode_InternInPlace(interp, &string); @@ -1479,6 +1518,9 @@ _PyUnicode_InitStaticStrings(PyInterpreterState *interp) { string = &_Py_ID(modules); assert(_PyUnicode_CheckConsistency(string, 1)); _PyUnicode_InternInPlace(interp, &string); + string = &_Py_ID(month); + assert(_PyUnicode_CheckConsistency(string, 1)); + _PyUnicode_InternInPlace(interp, &string); string = &_Py_ID(mro); assert(_PyUnicode_CheckConsistency(string, 1)); _PyUnicode_InternInPlace(interp, &string); @@ -1791,6 +1833,9 @@ _PyUnicode_InitStaticStrings(PyInterpreterState *interp) { string = &_Py_ID(scheduler); assert(_PyUnicode_CheckConsistency(string, 1)); _PyUnicode_InternInPlace(interp, &string); + string = &_Py_ID(second); + assert(_PyUnicode_CheckConsistency(string, 1)); + _PyUnicode_InternInPlace(interp, &string); string = &_Py_ID(seek); assert(_PyUnicode_CheckConsistency(string, 1)); _PyUnicode_InternInPlace(interp, &string); @@ -1989,6 +2034,9 @@ _PyUnicode_InitStaticStrings(PyInterpreterState *interp) { string = &_Py_ID(trace_callback); assert(_PyUnicode_CheckConsistency(string, 1)); _PyUnicode_InternInPlace(interp, &string); + string = &_Py_ID(traceback); + assert(_PyUnicode_CheckConsistency(string, 1)); + _PyUnicode_InternInPlace(interp, &string); string = &_Py_ID(trailers); assert(_PyUnicode_CheckConsistency(string, 1)); _PyUnicode_InternInPlace(interp, &string); @@ -2016,6 +2064,9 @@ _PyUnicode_InitStaticStrings(PyInterpreterState *interp) { string = &_Py_ID(tz); assert(_PyUnicode_CheckConsistency(string, 1)); _PyUnicode_InternInPlace(interp, &string); + string = &_Py_ID(tzinfo); + assert(_PyUnicode_CheckConsistency(string, 1)); + _PyUnicode_InternInPlace(interp, &string); string = &_Py_ID(tzname); assert(_PyUnicode_CheckConsistency(string, 1)); _PyUnicode_InternInPlace(interp, &string); diff --git a/Include/internal/pycore_uop_ids.h b/Include/internal/pycore_uop_ids.h index 4a9a00ba352d33..b2476e1c6e5c4b 100644 --- a/Include/internal/pycore_uop_ids.h +++ b/Include/internal/pycore_uop_ids.h @@ -229,9 +229,15 @@ extern "C" { #define _GUARD_IS_NOT_NONE_POP 376 #define _JUMP_TO_TOP 377 #define _SAVE_RETURN_OFFSET 378 -#define _INSERT 379 -#define _CHECK_VALIDITY 380 -#define MAX_UOP_ID 380 +#define _CHECK_VALIDITY 379 +#define _LOAD_CONST_INLINE 380 +#define _LOAD_CONST_INLINE_BORROW 381 +#define _LOAD_CONST_INLINE_WITH_NULL 382 +#define _LOAD_CONST_INLINE_BORROW_WITH_NULL 383 +#define _CHECK_GLOBALS 384 +#define _CHECK_BUILTINS 385 +#define _INTERNAL_INCREMENT_OPT_COUNTER 386 +#define MAX_UOP_ID 386 #ifdef __cplusplus } diff --git a/Include/internal/pycore_uop_metadata.h b/Include/internal/pycore_uop_metadata.h index 300bd3baa7b377..2b5b37e6b8d6a4 100644 --- a/Include/internal/pycore_uop_metadata.h +++ b/Include/internal/pycore_uop_metadata.h @@ -19,36 +19,36 @@ const uint16_t _PyUop_Flags[MAX_UOP_ID+1] = { [_NOP] = 0, [_RESUME_CHECK] = HAS_DEOPT_FLAG, [_LOAD_FAST_CHECK] = HAS_ARG_FLAG | HAS_LOCAL_FLAG | HAS_ERROR_FLAG, - [_LOAD_FAST] = HAS_ARG_FLAG | HAS_LOCAL_FLAG, + [_LOAD_FAST] = HAS_ARG_FLAG | HAS_LOCAL_FLAG | HAS_PURE_FLAG, [_LOAD_FAST_AND_CLEAR] = HAS_ARG_FLAG | HAS_LOCAL_FLAG, [_LOAD_FAST_LOAD_FAST] = HAS_ARG_FLAG | HAS_LOCAL_FLAG, - [_LOAD_CONST] = HAS_ARG_FLAG | HAS_CONST_FLAG, + [_LOAD_CONST] = HAS_ARG_FLAG | HAS_CONST_FLAG | HAS_PURE_FLAG, [_STORE_FAST] = HAS_ARG_FLAG | HAS_LOCAL_FLAG, [_STORE_FAST_LOAD_FAST] = HAS_ARG_FLAG | HAS_LOCAL_FLAG, [_STORE_FAST_STORE_FAST] = HAS_ARG_FLAG | HAS_LOCAL_FLAG, - [_POP_TOP] = 0, - [_PUSH_NULL] = 0, - [_END_SEND] = 0, + [_POP_TOP] = HAS_PURE_FLAG, + [_PUSH_NULL] = HAS_PURE_FLAG, + [_END_SEND] = HAS_PURE_FLAG, [_UNARY_NEGATIVE] = HAS_ERROR_FLAG | HAS_ESCAPES_FLAG, - [_UNARY_NOT] = 0, + [_UNARY_NOT] = HAS_PURE_FLAG, [_TO_BOOL] = HAS_ERROR_FLAG | HAS_ESCAPES_FLAG, - [_TO_BOOL_BOOL] = HAS_DEOPT_FLAG, + [_TO_BOOL_BOOL] = HAS_DEOPT_FLAG | HAS_PASSTHROUGH_FLAG, [_TO_BOOL_INT] = HAS_DEOPT_FLAG, [_TO_BOOL_LIST] = HAS_DEOPT_FLAG, [_TO_BOOL_NONE] = HAS_DEOPT_FLAG, [_TO_BOOL_STR] = HAS_DEOPT_FLAG, [_TO_BOOL_ALWAYS_TRUE] = HAS_DEOPT_FLAG, [_UNARY_INVERT] = HAS_ERROR_FLAG | HAS_ESCAPES_FLAG, - [_GUARD_BOTH_INT] = HAS_DEOPT_FLAG, - [_BINARY_OP_MULTIPLY_INT] = HAS_ERROR_FLAG, - [_BINARY_OP_ADD_INT] = HAS_ERROR_FLAG, - [_BINARY_OP_SUBTRACT_INT] = HAS_ERROR_FLAG, - [_GUARD_BOTH_FLOAT] = HAS_DEOPT_FLAG, - [_BINARY_OP_MULTIPLY_FLOAT] = 0, - [_BINARY_OP_ADD_FLOAT] = 0, - [_BINARY_OP_SUBTRACT_FLOAT] = 0, - [_GUARD_BOTH_UNICODE] = HAS_DEOPT_FLAG, - [_BINARY_OP_ADD_UNICODE] = HAS_ERROR_FLAG | HAS_ESCAPES_FLAG, + [_GUARD_BOTH_INT] = HAS_DEOPT_FLAG | HAS_PASSTHROUGH_FLAG, + [_BINARY_OP_MULTIPLY_INT] = HAS_ERROR_FLAG | HAS_PURE_FLAG, + [_BINARY_OP_ADD_INT] = HAS_ERROR_FLAG | HAS_PURE_FLAG, + [_BINARY_OP_SUBTRACT_INT] = HAS_ERROR_FLAG | HAS_PURE_FLAG, + [_GUARD_BOTH_FLOAT] = HAS_DEOPT_FLAG | HAS_PASSTHROUGH_FLAG, + [_BINARY_OP_MULTIPLY_FLOAT] = HAS_PURE_FLAG, + [_BINARY_OP_ADD_FLOAT] = HAS_PURE_FLAG, + [_BINARY_OP_SUBTRACT_FLOAT] = HAS_PURE_FLAG, + [_GUARD_BOTH_UNICODE] = HAS_DEOPT_FLAG | HAS_PASSTHROUGH_FLAG, + [_BINARY_OP_ADD_UNICODE] = HAS_ERROR_FLAG | HAS_ESCAPES_FLAG | HAS_PURE_FLAG, [_BINARY_SUBSCR] = HAS_ERROR_FLAG | HAS_ESCAPES_FLAG, [_BINARY_SLICE] = HAS_ERROR_FLAG | HAS_ESCAPES_FLAG, [_STORE_SLICE] = HAS_ERROR_FLAG | HAS_ESCAPES_FLAG, @@ -112,17 +112,17 @@ const uint16_t _PyUop_Flags[MAX_UOP_ID+1] = { [_LOAD_SUPER_ATTR_ATTR] = HAS_ARG_FLAG | HAS_NAME_FLAG | HAS_DEOPT_FLAG | HAS_ERROR_FLAG | HAS_ESCAPES_FLAG, [_LOAD_SUPER_ATTR_METHOD] = HAS_ARG_FLAG | HAS_NAME_FLAG | HAS_DEOPT_FLAG | HAS_ERROR_FLAG | HAS_ESCAPES_FLAG, [_LOAD_ATTR] = HAS_ARG_FLAG | HAS_NAME_FLAG | HAS_ERROR_FLAG | HAS_ESCAPES_FLAG, - [_GUARD_TYPE_VERSION] = HAS_DEOPT_FLAG, - [_CHECK_MANAGED_OBJECT_HAS_VALUES] = HAS_DEOPT_FLAG, + [_GUARD_TYPE_VERSION] = HAS_DEOPT_FLAG | HAS_PASSTHROUGH_FLAG, + [_CHECK_MANAGED_OBJECT_HAS_VALUES] = HAS_DEOPT_FLAG | HAS_PASSTHROUGH_FLAG, [_LOAD_ATTR_INSTANCE_VALUE] = HAS_ARG_FLAG | HAS_DEOPT_FLAG, - [_CHECK_ATTR_MODULE] = HAS_DEOPT_FLAG, + [_CHECK_ATTR_MODULE] = HAS_DEOPT_FLAG | HAS_PASSTHROUGH_FLAG, [_LOAD_ATTR_MODULE] = HAS_ARG_FLAG | HAS_DEOPT_FLAG, - [_CHECK_ATTR_WITH_HINT] = HAS_DEOPT_FLAG | HAS_ESCAPES_FLAG, + [_CHECK_ATTR_WITH_HINT] = HAS_DEOPT_FLAG | HAS_ESCAPES_FLAG | HAS_PASSTHROUGH_FLAG, [_LOAD_ATTR_WITH_HINT] = HAS_ARG_FLAG | HAS_NAME_FLAG | HAS_DEOPT_FLAG | HAS_ESCAPES_FLAG, [_LOAD_ATTR_SLOT] = HAS_ARG_FLAG | HAS_DEOPT_FLAG, - [_CHECK_ATTR_CLASS] = HAS_DEOPT_FLAG, + [_CHECK_ATTR_CLASS] = HAS_DEOPT_FLAG | HAS_PASSTHROUGH_FLAG, [_LOAD_ATTR_CLASS] = HAS_ARG_FLAG, - [_GUARD_DORV_VALUES] = HAS_DEOPT_FLAG, + [_GUARD_DORV_VALUES] = HAS_DEOPT_FLAG | HAS_PASSTHROUGH_FLAG, [_STORE_ATTR_INSTANCE_VALUE] = HAS_ESCAPES_FLAG, [_STORE_ATTR_SLOT] = HAS_ESCAPES_FLAG, [_COMPARE_OP] = HAS_ARG_FLAG | HAS_ERROR_FLAG | HAS_ESCAPES_FLAG, @@ -142,34 +142,34 @@ const uint16_t _PyUop_Flags[MAX_UOP_ID+1] = { [_GET_ITER] = HAS_ERROR_FLAG | HAS_ESCAPES_FLAG, [_GET_YIELD_FROM_ITER] = HAS_ERROR_FLAG | HAS_ESCAPES_FLAG, [_FOR_ITER_TIER_TWO] = HAS_DEOPT_FLAG | HAS_ERROR_FLAG | HAS_ESCAPES_FLAG, - [_ITER_CHECK_LIST] = HAS_DEOPT_FLAG, - [_GUARD_NOT_EXHAUSTED_LIST] = HAS_DEOPT_FLAG, + [_ITER_CHECK_LIST] = HAS_DEOPT_FLAG | HAS_PASSTHROUGH_FLAG, + [_GUARD_NOT_EXHAUSTED_LIST] = HAS_DEOPT_FLAG | HAS_PASSTHROUGH_FLAG, [_ITER_NEXT_LIST] = 0, - [_ITER_CHECK_TUPLE] = HAS_DEOPT_FLAG, - [_GUARD_NOT_EXHAUSTED_TUPLE] = HAS_DEOPT_FLAG, + [_ITER_CHECK_TUPLE] = HAS_DEOPT_FLAG | HAS_PASSTHROUGH_FLAG, + [_GUARD_NOT_EXHAUSTED_TUPLE] = HAS_DEOPT_FLAG | HAS_PASSTHROUGH_FLAG, [_ITER_NEXT_TUPLE] = 0, - [_ITER_CHECK_RANGE] = HAS_DEOPT_FLAG, - [_GUARD_NOT_EXHAUSTED_RANGE] = HAS_DEOPT_FLAG, + [_ITER_CHECK_RANGE] = HAS_DEOPT_FLAG | HAS_PASSTHROUGH_FLAG, + [_GUARD_NOT_EXHAUSTED_RANGE] = HAS_DEOPT_FLAG | HAS_PASSTHROUGH_FLAG, [_ITER_NEXT_RANGE] = HAS_ERROR_FLAG | HAS_ESCAPES_FLAG, [_BEFORE_ASYNC_WITH] = HAS_ERROR_FLAG | HAS_ESCAPES_FLAG, [_BEFORE_WITH] = HAS_ERROR_FLAG | HAS_ESCAPES_FLAG, [_WITH_EXCEPT_START] = HAS_ERROR_FLAG | HAS_ESCAPES_FLAG, [_PUSH_EXC_INFO] = 0, - [_GUARD_DORV_VALUES_INST_ATTR_FROM_DICT] = HAS_DEOPT_FLAG, - [_GUARD_KEYS_VERSION] = HAS_DEOPT_FLAG, + [_GUARD_DORV_VALUES_INST_ATTR_FROM_DICT] = HAS_DEOPT_FLAG | HAS_PASSTHROUGH_FLAG, + [_GUARD_KEYS_VERSION] = HAS_DEOPT_FLAG | HAS_PASSTHROUGH_FLAG, [_LOAD_ATTR_METHOD_WITH_VALUES] = HAS_ARG_FLAG | HAS_ESCAPES_FLAG, [_LOAD_ATTR_METHOD_NO_DICT] = HAS_ARG_FLAG | HAS_ESCAPES_FLAG, [_LOAD_ATTR_NONDESCRIPTOR_WITH_VALUES] = HAS_ARG_FLAG, [_LOAD_ATTR_NONDESCRIPTOR_NO_DICT] = HAS_ARG_FLAG, - [_CHECK_ATTR_METHOD_LAZY_DICT] = HAS_DEOPT_FLAG, + [_CHECK_ATTR_METHOD_LAZY_DICT] = HAS_DEOPT_FLAG | HAS_PASSTHROUGH_FLAG, [_LOAD_ATTR_METHOD_LAZY_DICT] = HAS_ARG_FLAG | HAS_ESCAPES_FLAG, - [_CHECK_CALL_BOUND_METHOD_EXACT_ARGS] = HAS_ARG_FLAG | HAS_DEOPT_FLAG, + [_CHECK_CALL_BOUND_METHOD_EXACT_ARGS] = HAS_ARG_FLAG | HAS_DEOPT_FLAG | HAS_PASSTHROUGH_FLAG, [_INIT_CALL_BOUND_METHOD_EXACT_ARGS] = HAS_ARG_FLAG, [_CHECK_PEP_523] = HAS_DEOPT_FLAG, - [_CHECK_FUNCTION_EXACT_ARGS] = HAS_ARG_FLAG | HAS_DEOPT_FLAG, - [_CHECK_STACK_SPACE] = HAS_ARG_FLAG | HAS_DEOPT_FLAG, - [_INIT_CALL_PY_EXACT_ARGS] = HAS_ARG_FLAG | HAS_ESCAPES_FLAG, - [_PUSH_FRAME] = 0, + [_CHECK_FUNCTION_EXACT_ARGS] = HAS_ARG_FLAG | HAS_DEOPT_FLAG | HAS_PASSTHROUGH_FLAG, + [_CHECK_STACK_SPACE] = HAS_ARG_FLAG | HAS_DEOPT_FLAG | HAS_PASSTHROUGH_FLAG, + [_INIT_CALL_PY_EXACT_ARGS] = HAS_ARG_FLAG | HAS_ESCAPES_FLAG | HAS_PURE_FLAG, + [_PUSH_FRAME] = HAS_ESCAPES_FLAG, [_CALL_TYPE_1] = HAS_ARG_FLAG | HAS_DEOPT_FLAG, [_CALL_STR_1] = HAS_ARG_FLAG | HAS_EVAL_BREAK_FLAG | HAS_DEOPT_FLAG | HAS_ERROR_FLAG | HAS_ESCAPES_FLAG, [_CALL_TUPLE_1] = HAS_ARG_FLAG | HAS_EVAL_BREAK_FLAG | HAS_DEOPT_FLAG | HAS_ERROR_FLAG | HAS_ESCAPES_FLAG, @@ -190,9 +190,9 @@ const uint16_t _PyUop_Flags[MAX_UOP_ID+1] = { [_CONVERT_VALUE] = HAS_ARG_FLAG | HAS_ERROR_FLAG, [_FORMAT_SIMPLE] = HAS_ERROR_FLAG | HAS_ESCAPES_FLAG, [_FORMAT_WITH_SPEC] = HAS_ERROR_FLAG | HAS_ESCAPES_FLAG, - [_COPY] = HAS_ARG_FLAG, + [_COPY] = HAS_ARG_FLAG | HAS_PURE_FLAG, [_BINARY_OP] = HAS_ARG_FLAG | HAS_ERROR_FLAG, - [_SWAP] = HAS_ARG_FLAG, + [_SWAP] = HAS_ARG_FLAG | HAS_PURE_FLAG, [_GUARD_IS_TRUE_POP] = HAS_DEOPT_FLAG, [_GUARD_IS_FALSE_POP] = HAS_DEOPT_FLAG, [_GUARD_IS_NONE_POP] = HAS_DEOPT_FLAG, @@ -201,8 +201,14 @@ const uint16_t _PyUop_Flags[MAX_UOP_ID+1] = { [_SET_IP] = HAS_ARG_FLAG | HAS_ESCAPES_FLAG, [_SAVE_RETURN_OFFSET] = HAS_ARG_FLAG, [_EXIT_TRACE] = HAS_DEOPT_FLAG, - [_INSERT] = HAS_ARG_FLAG, [_CHECK_VALIDITY] = HAS_DEOPT_FLAG, + [_LOAD_CONST_INLINE] = 0, + [_LOAD_CONST_INLINE_BORROW] = 0, + [_LOAD_CONST_INLINE_WITH_NULL] = 0, + [_LOAD_CONST_INLINE_BORROW_WITH_NULL] = 0, + [_CHECK_GLOBALS] = HAS_DEOPT_FLAG, + [_CHECK_BUILTINS] = HAS_DEOPT_FLAG, + [_INTERNAL_INCREMENT_OPT_COUNTER] = 0, }; const char *const _PyOpcode_uop_name[MAX_UOP_ID+1] = { @@ -248,10 +254,12 @@ const char *const _PyOpcode_uop_name[MAX_UOP_ID+1] = { [_CHECK_ATTR_METHOD_LAZY_DICT] = "_CHECK_ATTR_METHOD_LAZY_DICT", [_CHECK_ATTR_MODULE] = "_CHECK_ATTR_MODULE", [_CHECK_ATTR_WITH_HINT] = "_CHECK_ATTR_WITH_HINT", + [_CHECK_BUILTINS] = "_CHECK_BUILTINS", [_CHECK_CALL_BOUND_METHOD_EXACT_ARGS] = "_CHECK_CALL_BOUND_METHOD_EXACT_ARGS", [_CHECK_EG_MATCH] = "_CHECK_EG_MATCH", [_CHECK_EXC_MATCH] = "_CHECK_EXC_MATCH", [_CHECK_FUNCTION_EXACT_ARGS] = "_CHECK_FUNCTION_EXACT_ARGS", + [_CHECK_GLOBALS] = "_CHECK_GLOBALS", [_CHECK_MANAGED_OBJECT_HAS_VALUES] = "_CHECK_MANAGED_OBJECT_HAS_VALUES", [_CHECK_PEP_523] = "_CHECK_PEP_523", [_CHECK_STACK_SPACE] = "_CHECK_STACK_SPACE", @@ -302,7 +310,7 @@ const char *const _PyOpcode_uop_name[MAX_UOP_ID+1] = { [_GUARD_TYPE_VERSION] = "_GUARD_TYPE_VERSION", [_INIT_CALL_BOUND_METHOD_EXACT_ARGS] = "_INIT_CALL_BOUND_METHOD_EXACT_ARGS", [_INIT_CALL_PY_EXACT_ARGS] = "_INIT_CALL_PY_EXACT_ARGS", - [_INSERT] = "_INSERT", + [_INTERNAL_INCREMENT_OPT_COUNTER] = "_INTERNAL_INCREMENT_OPT_COUNTER", [_IS_NONE] = "_IS_NONE", [_IS_OP] = "_IS_OP", [_ITER_CHECK_LIST] = "_ITER_CHECK_LIST", @@ -328,6 +336,10 @@ const char *const _PyOpcode_uop_name[MAX_UOP_ID+1] = { [_LOAD_ATTR_WITH_HINT] = "_LOAD_ATTR_WITH_HINT", [_LOAD_BUILD_CLASS] = "_LOAD_BUILD_CLASS", [_LOAD_CONST] = "_LOAD_CONST", + [_LOAD_CONST_INLINE] = "_LOAD_CONST_INLINE", + [_LOAD_CONST_INLINE_BORROW] = "_LOAD_CONST_INLINE_BORROW", + [_LOAD_CONST_INLINE_BORROW_WITH_NULL] = "_LOAD_CONST_INLINE_BORROW_WITH_NULL", + [_LOAD_CONST_INLINE_WITH_NULL] = "_LOAD_CONST_INLINE_WITH_NULL", [_LOAD_DEREF] = "_LOAD_DEREF", [_LOAD_FAST] = "_LOAD_FAST", [_LOAD_FAST_AND_CLEAR] = "_LOAD_FAST_AND_CLEAR", diff --git a/Include/internal/pycore_uops.h b/Include/internal/pycore_uops.h deleted file mode 100644 index 153884f4bd2902..00000000000000 --- a/Include/internal/pycore_uops.h +++ /dev/null @@ -1,35 +0,0 @@ -#ifndef Py_INTERNAL_UOPS_H -#define Py_INTERNAL_UOPS_H -#ifdef __cplusplus -extern "C" { -#endif - -#ifndef Py_BUILD_CORE -# error "this header requires Py_BUILD_CORE define" -#endif - -#include "pycore_frame.h" // _PyInterpreterFrame - -#define _Py_UOP_MAX_TRACE_LENGTH 512 - -typedef struct { - uint16_t opcode; - uint16_t oparg; - uint32_t target; - uint64_t operand; // A cache entry -} _PyUOpInstruction; - -typedef struct { - _PyExecutorObject base; - _PyUOpInstruction trace[1]; -} _PyUOpExecutorObject; - -_Py_CODEUNIT *_PyUOpExecute( - _PyExecutorObject *executor, - _PyInterpreterFrame *frame, - PyObject **stack_pointer); - -#ifdef __cplusplus -} -#endif -#endif /* !Py_INTERNAL_UOPS_H */ diff --git a/Include/internal/pycore_weakref.h b/Include/internal/pycore_weakref.h index eacbe14c903289..dea267b49039e7 100644 --- a/Include/internal/pycore_weakref.h +++ b/Include/internal/pycore_weakref.h @@ -9,48 +9,66 @@ extern "C" { #endif #include "pycore_critical_section.h" // Py_BEGIN_CRITICAL_SECTION() +#include "pycore_object.h" // _Py_REF_IS_MERGED() -static inline PyObject* _PyWeakref_GET_REF(PyObject *ref_obj) { +static inline int _is_dead(PyObject *obj) +{ + // Explanation for the Py_REFCNT() check: when a weakref's target is part + // of a long chain of deallocations which triggers the trashcan mechanism, + // clearing the weakrefs can be delayed long after the target's refcount + // has dropped to zero. In the meantime, code accessing the weakref will + // be able to "see" the target object even though it is supposed to be + // unreachable. See issue gh-60806. +#if defined(Py_GIL_DISABLED) + Py_ssize_t shared = _Py_atomic_load_ssize_relaxed(&obj->ob_ref_shared); + return shared == _Py_REF_SHARED(0, _Py_REF_MERGED); +#else + return (Py_REFCNT(obj) == 0); +#endif +} + +static inline PyObject* _PyWeakref_GET_REF(PyObject *ref_obj) +{ assert(PyWeakref_Check(ref_obj)); + PyObject *ret = NULL; + Py_BEGIN_CRITICAL_SECTION(ref_obj); PyWeakReference *ref = _Py_CAST(PyWeakReference*, ref_obj); PyObject *obj = ref->wr_object; if (obj == Py_None) { // clear_weakref() was called - return NULL; + goto end; } - // Explanation for the Py_REFCNT() check: when a weakref's target is part - // of a long chain of deallocations which triggers the trashcan mechanism, - // clearing the weakrefs can be delayed long after the target's refcount - // has dropped to zero. In the meantime, code accessing the weakref will - // be able to "see" the target object even though it is supposed to be - // unreachable. See issue gh-60806. - Py_ssize_t refcnt = Py_REFCNT(obj); - if (refcnt == 0) { - return NULL; + if (_is_dead(obj)) { + goto end; } - - assert(refcnt > 0); - return Py_NewRef(obj); +#if !defined(Py_GIL_DISABLED) + assert(Py_REFCNT(obj) > 0); +#endif + ret = Py_NewRef(obj); +end: + Py_END_CRITICAL_SECTION(); + return ret; } -static inline int _PyWeakref_IS_DEAD(PyObject *ref_obj) { +static inline int _PyWeakref_IS_DEAD(PyObject *ref_obj) +{ assert(PyWeakref_Check(ref_obj)); - int is_dead; + int ret = 0; Py_BEGIN_CRITICAL_SECTION(ref_obj); PyWeakReference *ref = _Py_CAST(PyWeakReference*, ref_obj); PyObject *obj = ref->wr_object; if (obj == Py_None) { // clear_weakref() was called - is_dead = 1; + ret = 1; } else { // See _PyWeakref_GET_REF() for the rationale of this test - is_dead = (Py_REFCNT(obj) == 0); + ret = _is_dead(obj); } Py_END_CRITICAL_SECTION(); - return is_dead; + return ret; } extern Py_ssize_t _PyWeakref_GetWeakrefCount(PyWeakReference *head); diff --git a/Include/listobject.h b/Include/listobject.h index 6b7041ba0b05d5..4e4084b43483a2 100644 --- a/Include/listobject.h +++ b/Include/listobject.h @@ -29,6 +29,7 @@ PyAPI_FUNC(PyObject *) PyList_New(Py_ssize_t size); PyAPI_FUNC(Py_ssize_t) PyList_Size(PyObject *); PyAPI_FUNC(PyObject *) PyList_GetItem(PyObject *, Py_ssize_t); +PyAPI_FUNC(PyObject *) PyList_GetItemRef(PyObject *, Py_ssize_t); PyAPI_FUNC(int) PyList_SetItem(PyObject *, Py_ssize_t, PyObject *); PyAPI_FUNC(int) PyList_Insert(PyObject *, Py_ssize_t, PyObject *); PyAPI_FUNC(int) PyList_Append(PyObject *, PyObject *); diff --git a/Include/object.h b/Include/object.h index 48f1ddf7510887..05187fe5dc4f20 100644 --- a/Include/object.h +++ b/Include/object.h @@ -212,7 +212,10 @@ struct _object { struct _PyMutex { uint8_t v; }; struct _object { - uintptr_t ob_tid; // thread id (or zero) + // ob_tid stores the thread id (or zero). It is also used by the GC and the + // trashcan mechanism as a linked list pointer and by the GC to store the + // computed "gc_refs" refcount. + uintptr_t ob_tid; uint16_t _padding; struct _PyMutex ob_mutex; // per-object lock uint8_t ob_gc_bits; // gc-related state @@ -426,7 +429,11 @@ static inline void Py_SET_TYPE(PyObject *ob, PyTypeObject *type) { static inline void Py_SET_SIZE(PyVarObject *ob, Py_ssize_t size) { assert(ob->ob_base.ob_type != &PyLong_Type); assert(ob->ob_base.ob_type != &PyBool_Type); +#ifdef Py_GIL_DISABLED + _Py_atomic_store_ssize_relaxed(&ob->ob_size, size); +#else ob->ob_size = size; +#endif } #if !defined(Py_LIMITED_API) || Py_LIMITED_API+0 < 0x030b0000 # define Py_SET_SIZE(ob, size) Py_SET_SIZE(_PyVarObject_CAST(ob), (size)) diff --git a/Include/patchlevel.h b/Include/patchlevel.h index fad79ecfda7b28..5f9b720f8b1671 100644 --- a/Include/patchlevel.h +++ b/Include/patchlevel.h @@ -20,10 +20,10 @@ #define PY_MINOR_VERSION 13 #define PY_MICRO_VERSION 0 #define PY_RELEASE_LEVEL PY_RELEASE_LEVEL_ALPHA -#define PY_RELEASE_SERIAL 2 +#define PY_RELEASE_SERIAL 3 /* Version as a string */ -#define PY_VERSION "3.13.0a2+" +#define PY_VERSION "3.13.0a3+" /*--end constants--*/ /* Version as a single 4-byte hex number, e.g. 0x010502B2 == 1.5.2b2. diff --git a/Include/pyerrors.h b/Include/pyerrors.h index 5d0028c116e2d8..68d7985dac8876 100644 --- a/Include/pyerrors.h +++ b/Include/pyerrors.h @@ -108,6 +108,7 @@ PyAPI_DATA(PyObject *) PyExc_NotImplementedError; PyAPI_DATA(PyObject *) PyExc_SyntaxError; PyAPI_DATA(PyObject *) PyExc_IndentationError; PyAPI_DATA(PyObject *) PyExc_TabError; +PyAPI_DATA(PyObject *) PyExc_IncompleteInputError; PyAPI_DATA(PyObject *) PyExc_ReferenceError; PyAPI_DATA(PyObject *) PyExc_SystemError; PyAPI_DATA(PyObject *) PyExc_SystemExit; diff --git a/Include/pymacro.h b/Include/pymacro.h index 9d264fe6eea1d4..cd6fc4eba9c2ed 100644 --- a/Include/pymacro.h +++ b/Include/pymacro.h @@ -160,6 +160,9 @@ Py_FatalError("Unreachable C code path reached") #endif +#define _Py_CONTAINER_OF(ptr, type, member) \ + (type*)((char*)ptr - offsetof(type, member)) + // Prevent using an expression as a l-value. // For example, "int x; _Py_RVALUE(x) = 1;" fails with a compiler error. #define _Py_RVALUE(EXPR) ((void)0, (EXPR)) diff --git a/Include/pyport.h b/Include/pyport.h index 328471085f959d..9d7ef0061806ad 100644 --- a/Include/pyport.h +++ b/Include/pyport.h @@ -563,6 +563,11 @@ extern "C" { # define _Py_ADDRESS_SANITIZER # endif # endif +# if __has_feature(thread_sanitizer) +# if !defined(_Py_THREAD_SANITIZER) +# define _Py_THREAD_SANITIZER +# endif +# endif #elif defined(__GNUC__) # if defined(__SANITIZE_ADDRESS__) # define _Py_ADDRESS_SANITIZER diff --git a/LICENSE b/LICENSE index f26bcf4d2de6eb..14603b95c2e23b 100644 --- a/LICENSE +++ b/LICENSE @@ -83,10 +83,8 @@ grants Licensee a nonexclusive, royalty-free, world-wide license to reproduce, analyze, test, perform and/or display publicly, prepare derivative works, distribute, and otherwise use Python alone or in any derivative version, provided, however, that PSF's License Agreement and PSF's notice of copyright, -i.e., "Copyright (c) 2001, 2002, 2003, 2004, 2005, 2006, 2007, 2008, 2009, 2010, -2011, 2012, 2013, 2014, 2015, 2016, 2017, 2018, 2019, 2020, 2021, 2022, 2023 Python Software Foundation; -All Rights Reserved" are retained in Python alone or in any derivative version -prepared by Licensee. +i.e., "Copyright (c) 2001-2024 Python Software Foundation; All Rights Reserved" +are retained in Python alone or in any derivative version prepared by Licensee. 3. In the event Licensee prepares a derivative work that is based on or incorporates Python or any part thereof, and wants to make diff --git a/Lib/_pydatetime.py b/Lib/_pydatetime.py index bca2acf1fc88cf..54c12d3b2f3f16 100644 --- a/Lib/_pydatetime.py +++ b/Lib/_pydatetime.py @@ -1809,7 +1809,7 @@ def fromtimestamp(cls, timestamp, tz=None): def utcfromtimestamp(cls, t): """Construct a naive UTC datetime from a POSIX timestamp.""" import warnings - warnings.warn("datetime.utcfromtimestamp() is deprecated and scheduled " + warnings.warn("datetime.datetime.utcfromtimestamp() is deprecated and scheduled " "for removal in a future version. Use timezone-aware " "objects to represent datetimes in UTC: " "datetime.datetime.fromtimestamp(t, datetime.UTC).", @@ -1827,8 +1827,8 @@ def now(cls, tz=None): def utcnow(cls): "Construct a UTC datetime from time.time()." import warnings - warnings.warn("datetime.utcnow() is deprecated and scheduled for " - "removal in a future version. Instead, Use timezone-aware " + warnings.warn("datetime.datetime.utcnow() is deprecated and scheduled for " + "removal in a future version. Use timezone-aware " "objects to represent datetimes in UTC: " "datetime.datetime.now(datetime.UTC).", DeprecationWarning, @@ -2347,6 +2347,9 @@ def __new__(cls, offset, name=_Omitted): "timedelta(hours=24).") return cls._create(offset, name) + def __init_subclass__(cls): + raise TypeError("type 'datetime.timezone' is not an acceptable base type") + @classmethod def _create(cls, offset, name=None): self = tzinfo.__new__(cls) diff --git a/Lib/_pyio.py b/Lib/_pyio.py index 32698abac78d25..8a0d0dc4b1a0b8 100644 --- a/Lib/_pyio.py +++ b/Lib/_pyio.py @@ -1495,6 +1495,11 @@ def __init__(self, file, mode='r', closefd=True, opener=None): if isinstance(file, float): raise TypeError('integer argument expected, got float') if isinstance(file, int): + if isinstance(file, bool): + import warnings + warnings.warn("bool is used as a file descriptor", + RuntimeWarning, stacklevel=2) + file = int(file) fd = file if fd < 0: raise ValueError('negative file descriptor') @@ -2198,8 +2203,9 @@ def write(self, s): self.buffer.write(b) if self._line_buffering and (haslf or "\r" in s): self.flush() - self._set_decoded_chars('') - self._snapshot = None + if self._snapshot is not None: + self._set_decoded_chars('') + self._snapshot = None if self._decoder: self._decoder.reset() return length @@ -2513,8 +2519,9 @@ def read(self, size=None): # Read everything. result = (self._get_decoded_chars() + decoder.decode(self.buffer.read(), final=True)) - self._set_decoded_chars('') - self._snapshot = None + if self._snapshot is not None: + self._set_decoded_chars('') + self._snapshot = None return result else: # Keep reading chunks until we have size characters to return. diff --git a/Lib/_strptime.py b/Lib/_strptime.py index 77ccdc9e1d789b..798cf9f9d3fffe 100644 --- a/Lib/_strptime.py +++ b/Lib/_strptime.py @@ -342,8 +342,6 @@ def _strptime(data_string, format="%a %b %d %H:%M:%S %Y"): tz = -1 gmtoff = None gmtoff_fraction = 0 - # Default to -1 to signify that values not known; not critical to have, - # though iso_week = week_of_year = None week_of_year_start = None # weekday and julian defaulted to None so as to signal need to calculate @@ -470,17 +468,17 @@ def _strptime(data_string, format="%a %b %d %H:%M:%S %Y"): # Deal with the cases where ambiguities arise # don't assume default values for ISO week/year - if year is None and iso_year is not None: - if iso_week is None or weekday is None: - raise ValueError("ISO year directive '%G' must be used with " - "the ISO week directive '%V' and a weekday " - "directive ('%A', '%a', '%w', or '%u').") + if iso_year is not None: if julian is not None: raise ValueError("Day of the year directive '%j' is not " "compatible with ISO year directive '%G'. " "Use '%Y' instead.") - elif week_of_year is None and iso_week is not None: - if weekday is None: + elif iso_week is None or weekday is None: + raise ValueError("ISO year directive '%G' must be used with " + "the ISO week directive '%V' and a weekday " + "directive ('%A', '%a', '%w', or '%u').") + elif iso_week is not None: + if year is None or weekday is None: raise ValueError("ISO week directive '%V' must be used with " "the ISO year directive '%G' and a weekday " "directive ('%A', '%a', '%w', or '%u').") @@ -490,11 +488,12 @@ def _strptime(data_string, format="%a %b %d %H:%M:%S %Y"): "instead.") leap_year_fix = False - if year is None and month == 2 and day == 29: - year = 1904 # 1904 is first leap year of 20th century - leap_year_fix = True - elif year is None: - year = 1900 + if year is None: + if month == 2 and day == 29: + year = 1904 # 1904 is first leap year of 20th century + leap_year_fix = True + else: + year = 1900 # If we know the week of the year and what day of that week, we can figure # out the Julian day of the year. diff --git a/Lib/argparse.py b/Lib/argparse.py index a32884db80d1ea..04ee3b19aca755 100644 --- a/Lib/argparse.py +++ b/Lib/argparse.py @@ -564,22 +564,18 @@ def _format_action_invocation(self, action): return metavar else: - parts = [] # if the Optional doesn't take a value, format is: # -s, --long if action.nargs == 0: - parts.extend(action.option_strings) + return ', '.join(action.option_strings) # if the Optional takes a value, format is: - # -s ARGS, --long ARGS + # -s, --long ARGS else: default = self._get_default_metavar_for_optional(action) args_string = self._format_args(action, default) - for option_string in action.option_strings: - parts.append('%s %s' % (option_string, args_string)) - - return ', '.join(parts) + return ', '.join(action.option_strings) + ' ' + args_string def _metavar_formatter(self, action, default_metavar): if action.metavar is not None: @@ -847,7 +843,8 @@ def __init__(self, choices=None, required=False, help=None, - metavar=None): + metavar=None, + deprecated=False): self.option_strings = option_strings self.dest = dest self.nargs = nargs @@ -858,6 +855,7 @@ def __init__(self, self.required = required self.help = help self.metavar = metavar + self.deprecated = deprecated def _get_kwargs(self): names = [ @@ -871,6 +869,7 @@ def _get_kwargs(self): 'required', 'help', 'metavar', + 'deprecated', ] return [(name, getattr(self, name)) for name in names] @@ -893,7 +892,8 @@ def __init__(self, choices=_deprecated_default, required=False, help=None, - metavar=_deprecated_default): + metavar=_deprecated_default, + deprecated=False): _option_strings = [] for option_string in option_strings: @@ -931,7 +931,8 @@ def __init__(self, choices=choices, required=required, help=help, - metavar=metavar) + metavar=metavar, + deprecated=deprecated) def __call__(self, parser, namespace, values, option_string=None): @@ -954,7 +955,8 @@ def __init__(self, choices=None, required=False, help=None, - metavar=None): + metavar=None, + deprecated=False): if nargs == 0: raise ValueError('nargs for store actions must be != 0; if you ' 'have nothing to store, actions such as store ' @@ -971,7 +973,8 @@ def __init__(self, choices=choices, required=required, help=help, - metavar=metavar) + metavar=metavar, + deprecated=deprecated) def __call__(self, parser, namespace, values, option_string=None): setattr(namespace, self.dest, values) @@ -986,7 +989,8 @@ def __init__(self, default=None, required=False, help=None, - metavar=None): + metavar=None, + deprecated=False): super(_StoreConstAction, self).__init__( option_strings=option_strings, dest=dest, @@ -994,7 +998,8 @@ def __init__(self, const=const, default=default, required=required, - help=help) + help=help, + deprecated=deprecated) def __call__(self, parser, namespace, values, option_string=None): setattr(namespace, self.dest, self.const) @@ -1007,14 +1012,16 @@ def __init__(self, dest, default=False, required=False, - help=None): + help=None, + deprecated=False): super(_StoreTrueAction, self).__init__( option_strings=option_strings, dest=dest, const=True, - default=default, + deprecated=deprecated, required=required, - help=help) + help=help, + default=default) class _StoreFalseAction(_StoreConstAction): @@ -1024,14 +1031,16 @@ def __init__(self, dest, default=True, required=False, - help=None): + help=None, + deprecated=False): super(_StoreFalseAction, self).__init__( option_strings=option_strings, dest=dest, const=False, default=default, required=required, - help=help) + help=help, + deprecated=deprecated) class _AppendAction(Action): @@ -1046,7 +1055,8 @@ def __init__(self, choices=None, required=False, help=None, - metavar=None): + metavar=None, + deprecated=False): if nargs == 0: raise ValueError('nargs for append actions must be != 0; if arg ' 'strings are not supplying the value to append, ' @@ -1063,7 +1073,8 @@ def __init__(self, choices=choices, required=required, help=help, - metavar=metavar) + metavar=metavar, + deprecated=deprecated) def __call__(self, parser, namespace, values, option_string=None): items = getattr(namespace, self.dest, None) @@ -1081,7 +1092,8 @@ def __init__(self, default=None, required=False, help=None, - metavar=None): + metavar=None, + deprecated=False): super(_AppendConstAction, self).__init__( option_strings=option_strings, dest=dest, @@ -1090,7 +1102,8 @@ def __init__(self, default=default, required=required, help=help, - metavar=metavar) + metavar=metavar, + deprecated=deprecated) def __call__(self, parser, namespace, values, option_string=None): items = getattr(namespace, self.dest, None) @@ -1106,14 +1119,16 @@ def __init__(self, dest, default=None, required=False, - help=None): + help=None, + deprecated=False): super(_CountAction, self).__init__( option_strings=option_strings, dest=dest, nargs=0, default=default, required=required, - help=help) + help=help, + deprecated=deprecated) def __call__(self, parser, namespace, values, option_string=None): count = getattr(namespace, self.dest, None) @@ -1128,13 +1143,15 @@ def __init__(self, option_strings, dest=SUPPRESS, default=SUPPRESS, - help=None): + help=None, + deprecated=False): super(_HelpAction, self).__init__( option_strings=option_strings, dest=dest, default=default, nargs=0, - help=help) + help=help, + deprecated=deprecated) def __call__(self, parser, namespace, values, option_string=None): parser.print_help() @@ -1148,7 +1165,8 @@ def __init__(self, version=None, dest=SUPPRESS, default=SUPPRESS, - help="show program's version number and exit"): + help="show program's version number and exit", + deprecated=False): super(_VersionAction, self).__init__( option_strings=option_strings, dest=dest, @@ -1192,6 +1210,7 @@ def __init__(self, self._parser_class = parser_class self._name_parser_map = {} self._choices_actions = [] + self._deprecated = set() super(_SubParsersAction, self).__init__( option_strings=option_strings, @@ -1202,7 +1221,7 @@ def __init__(self, help=help, metavar=metavar) - def add_parser(self, name, **kwargs): + def add_parser(self, name, *, deprecated=False, **kwargs): # set prog from the existing prefix if kwargs.get('prog') is None: kwargs['prog'] = '%s %s' % (self._prog_prefix, name) @@ -1230,6 +1249,10 @@ def add_parser(self, name, **kwargs): for alias in aliases: self._name_parser_map[alias] = parser + if deprecated: + self._deprecated.add(name) + self._deprecated.update(aliases) + return parser def _get_subactions(self): @@ -1245,13 +1268,17 @@ def __call__(self, parser, namespace, values, option_string=None): # select the parser try: - parser = self._name_parser_map[parser_name] + subparser = self._name_parser_map[parser_name] except KeyError: args = {'parser_name': parser_name, 'choices': ', '.join(self._name_parser_map)} msg = _('unknown parser %(parser_name)r (choices: %(choices)s)') % args raise ArgumentError(self, msg) + if parser_name in self._deprecated: + parser._warning(_("command '%(parser_name)s' is deprecated") % + {'parser_name': parser_name}) + # parse all the remaining options into the namespace # store any unrecognized options on the object, so that the top # level parser can decide what to do with them @@ -1259,7 +1286,7 @@ def __call__(self, parser, namespace, values, option_string=None): # In case this subparser defines new defaults, we parse them # in a new namespace object and then update the original # namespace for the relevant parts. - subnamespace, arg_strings = parser.parse_known_args(arg_strings, None) + subnamespace, arg_strings = subparser.parse_known_args(arg_strings, None) for key, value in vars(subnamespace).items(): setattr(namespace, key, value) @@ -1979,6 +2006,7 @@ def _parse_known_args(self, arg_strings, namespace): # converts arg strings to the appropriate and then takes the action seen_actions = set() seen_non_default_actions = set() + warned = set() def take_action(action, argument_strings, option_string=None): seen_actions.add(action) @@ -2074,6 +2102,10 @@ def consume_optional(start_index): # the Optional's string args stopped assert action_tuples for action, args, option_string in action_tuples: + if action.deprecated and option_string not in warned: + self._warning(_("option '%(option)s' is deprecated") % + {'option': option_string}) + warned.add(option_string) take_action(action, args, option_string) return stop @@ -2093,6 +2125,10 @@ def consume_positionals(start_index): for action, arg_count in zip(positionals, arg_counts): args = arg_strings[start_index: start_index + arg_count] start_index += arg_count + if args and action.deprecated and action.dest not in warned: + self._warning(_("argument '%(argument_name)s' is deprecated") % + {'argument_name': action.dest}) + warned.add(action.dest) take_action(action, args) # slice off the Positionals that we just parsed and return the @@ -2489,7 +2525,7 @@ def parse_known_intermixed_args(self, args=None, namespace=None): # ======================== def _get_values(self, action, arg_strings): # for everything but PARSER, REMAINDER args, strip out first '--' - if action.nargs not in [PARSER, REMAINDER]: + if not action.option_strings and action.nargs not in [PARSER, REMAINDER]: try: arg_strings.remove('--') except ValueError: @@ -2654,3 +2690,7 @@ def error(self, message): self.print_usage(_sys.stderr) args = {'prog': self.prog, 'message': message} self.exit(2, _('%(prog)s: error: %(message)s\n') % args) + + def _warning(self, message): + args = {'prog': self.prog, 'message': message} + self._print_message(_('%(prog)s: warning: %(message)s\n') % args, _sys.stderr) diff --git a/Lib/ast.py b/Lib/ast.py index f7888d18859ae4..43703a8325cc5e 100644 --- a/Lib/ast.py +++ b/Lib/ast.py @@ -728,12 +728,11 @@ class _Unparser(NodeVisitor): output source code for the abstract syntax; original formatting is disregarded.""" - def __init__(self, *, _avoid_backslashes=False): + def __init__(self): self._source = [] self._precedences = {} self._type_ignores = {} self._indent = 0 - self._avoid_backslashes = _avoid_backslashes self._in_try_star = False def interleave(self, inter, f, seq): @@ -1270,14 +1269,14 @@ def visit_JoinedStr(self, node): quote_type = quote_types[0] self.write(f"{quote_type}{value}{quote_type}") - def _write_fstring_inner(self, node, scape_newlines=False): + def _write_fstring_inner(self, node, escape_newlines=False): if isinstance(node, JoinedStr): # for both the f-string itself, and format_spec for value in node.values: - self._write_fstring_inner(value, scape_newlines=scape_newlines) + self._write_fstring_inner(value, escape_newlines=escape_newlines) elif isinstance(node, Constant) and isinstance(node.value, str): value = node.value.replace("{", "{{").replace("}", "}}") - if scape_newlines: + if escape_newlines: value = value.replace("\n", "\\n") self.write(value) elif isinstance(node, FormattedValue): @@ -1303,7 +1302,7 @@ def unparse_inner(inner): self.write(":") self._write_fstring_inner( node.format_spec, - scape_newlines=True + escape_newlines=True ) def visit_Name(self, node): @@ -1324,8 +1323,6 @@ def _write_constant(self, value): .replace("inf", _INFSTR) .replace("nan", f"({_INFSTR}-{_INFSTR})") ) - elif self._avoid_backslashes and isinstance(value, str): - self._write_str_avoiding_backslashes(value) else: self.write(repr(value)) @@ -1812,8 +1809,7 @@ def main(): import argparse parser = argparse.ArgumentParser(prog='python -m ast') - parser.add_argument('infile', type=argparse.FileType(mode='rb'), nargs='?', - default='-', + parser.add_argument('infile', nargs='?', default='-', help='the file to parse; defaults to stdin') parser.add_argument('-m', '--mode', default='exec', choices=('exec', 'single', 'eval', 'func_type'), @@ -1827,9 +1823,14 @@ def main(): help='indentation of nodes (number of spaces)') args = parser.parse_args() - with args.infile as infile: - source = infile.read() - tree = parse(source, args.infile.name, args.mode, type_comments=args.no_type_comments) + if args.infile == '-': + name = '<stdin>' + source = sys.stdin.buffer.read() + else: + name = args.infile + with open(args.infile, 'rb') as infile: + source = infile.read() + tree = parse(source, name, args.mode, type_comments=args.no_type_comments) print(dump(tree, include_attributes=args.include_attributes, indent=args.indent)) if __name__ == '__main__': diff --git a/Lib/asyncio/base_events.py b/Lib/asyncio/base_events.py index a8870b636d1df5..aadc4f478f8b56 100644 --- a/Lib/asyncio/base_events.py +++ b/Lib/asyncio/base_events.py @@ -16,6 +16,7 @@ import collections import collections.abc import concurrent.futures +import errno import functools import heapq import itertools @@ -1339,9 +1340,9 @@ async def create_datagram_endpoint(self, protocol_factory, allow_broadcast=None, sock=None): """Create datagram connection.""" if sock is not None: - if sock.type != socket.SOCK_DGRAM: + if sock.type == socket.SOCK_STREAM: raise ValueError( - f'A UDP Socket was expected, got {sock!r}') + f'A datagram socket was expected, got {sock!r}') if (local_addr or remote_addr or family or proto or flags or reuse_port or allow_broadcast): @@ -1585,9 +1586,22 @@ async def create_server( try: sock.bind(sa) except OSError as err: - raise OSError(err.errno, 'error while attempting ' - 'to bind on address %r: %s' - % (sa, err.strerror.lower())) from None + msg = ('error while attempting ' + 'to bind on address %r: %s' + % (sa, err.strerror.lower())) + if err.errno == errno.EADDRNOTAVAIL: + # Assume the family is not enabled (bpo-30945) + sockets.pop() + sock.close() + if self._debug: + logger.warning(msg) + continue + raise OSError(err.errno, msg) from None + + if not sockets: + raise OSError('could not bind on any address out of %r' + % ([info[4] for info in infos],)) + completed = True finally: if not completed: diff --git a/Lib/asyncio/base_subprocess.py b/Lib/asyncio/base_subprocess.py index 4c9b0dd5653c0c..6dbde2b696ad1f 100644 --- a/Lib/asyncio/base_subprocess.py +++ b/Lib/asyncio/base_subprocess.py @@ -115,7 +115,8 @@ def close(self): try: self._proc.kill() - except ProcessLookupError: + except (ProcessLookupError, PermissionError): + # the process may have already exited or may be running setuid pass # Don't clear the _proc reference yet: _post_init() may still run diff --git a/Lib/asyncio/constants.py b/Lib/asyncio/constants.py index f0ce0433a7a8a6..b60c1e4236af1f 100644 --- a/Lib/asyncio/constants.py +++ b/Lib/asyncio/constants.py @@ -1,3 +1,7 @@ +# Contains code from https://github.com/MagicStack/uvloop/tree/v0.16.0 +# SPDX-License-Identifier: PSF-2.0 AND (MIT OR Apache-2.0) +# SPDX-FileCopyrightText: Copyright (c) 2015-2021 MagicStack Inc. http://magic.io + import enum # After the connection is lost, log warnings after this many write()s. diff --git a/Lib/asyncio/events.py b/Lib/asyncio/events.py index ebc3836bdc0c4d..072a99fee123c3 100644 --- a/Lib/asyncio/events.py +++ b/Lib/asyncio/events.py @@ -1,5 +1,9 @@ """Event loop and event loop policy.""" +# Contains code from https://github.com/MagicStack/uvloop/tree/v0.16.0 +# SPDX-License-Identifier: PSF-2.0 AND (MIT OR Apache-2.0) +# SPDX-FileCopyrightText: Copyright (c) 2015-2021 MagicStack Inc. http://magic.io + __all__ = ( 'AbstractEventLoopPolicy', 'AbstractEventLoop', 'AbstractServer', diff --git a/Lib/asyncio/futures.py b/Lib/asyncio/futures.py index 97fc4e3fcb60ee..5d35321db7943b 100644 --- a/Lib/asyncio/futures.py +++ b/Lib/asyncio/futures.py @@ -138,9 +138,6 @@ def _make_cancelled_error(self): exc = exceptions.CancelledError() else: exc = exceptions.CancelledError(self._cancel_message) - exc.__context__ = self._cancelled_exc - # Remove the reference since we don't need this anymore. - self._cancelled_exc = None return exc def cancel(self, msg=None): @@ -272,9 +269,13 @@ def set_exception(self, exception): raise exceptions.InvalidStateError(f'{self._state}: {self!r}') if isinstance(exception, type): exception = exception() - if type(exception) is StopIteration: - raise TypeError("StopIteration interacts badly with generators " - "and cannot be raised into a Future") + if isinstance(exception, StopIteration): + new_exc = RuntimeError("StopIteration interacts badly with " + "generators and cannot be raised into a " + "Future") + new_exc.__cause__ = exception + new_exc.__context__ = exception + exception = new_exc self._exception = exception self._exception_tb = exception.__traceback__ self._state = _FINISHED diff --git a/Lib/asyncio/locks.py b/Lib/asyncio/locks.py index ce5d8d5bfb2e81..aaee8ff0702923 100644 --- a/Lib/asyncio/locks.py +++ b/Lib/asyncio/locks.py @@ -24,25 +24,23 @@ class Lock(_ContextManagerMixin, mixins._LoopBoundMixin): """Primitive lock objects. A primitive lock is a synchronization primitive that is not owned - by a particular coroutine when locked. A primitive lock is in one + by a particular task when locked. A primitive lock is in one of two states, 'locked' or 'unlocked'. It is created in the unlocked state. It has two basic methods, acquire() and release(). When the state is unlocked, acquire() changes the state to locked and returns immediately. When the state is locked, acquire() blocks until a call to release() in - another coroutine changes it to unlocked, then the acquire() call + another task changes it to unlocked, then the acquire() call resets it to locked and returns. The release() method should only be called in the locked state; it changes the state to unlocked and returns immediately. If an attempt is made to release an unlocked lock, a RuntimeError will be raised. - When more than one coroutine is blocked in acquire() waiting for - the state to turn to unlocked, only one coroutine proceeds when a - release() call resets the state to unlocked; first coroutine which - is blocked in acquire() is being processed. - - acquire() is a coroutine and should be called with 'await'. + When more than one task is blocked in acquire() waiting for + the state to turn to unlocked, only one task proceeds when a + release() call resets the state to unlocked; successive release() + calls will unblock tasks in FIFO order. Locks also support the asynchronous context management protocol. 'async with lock' statement should be used. @@ -95,6 +93,8 @@ async def acquire(self): This method blocks until the lock is unlocked, then sets it to locked and returns True. """ + # Implement fair scheduling, where thread always waits + # its turn. Jumping the queue if all are cancelled is an optimization. if (not self._locked and (self._waiters is None or all(w.cancelled() for w in self._waiters))): self._locked = True @@ -105,19 +105,22 @@ async def acquire(self): fut = self._get_loop().create_future() self._waiters.append(fut) - # Finally block should be called before the CancelledError - # handling as we don't want CancelledError to call - # _wake_up_first() and attempt to wake up itself. try: try: await fut finally: self._waiters.remove(fut) except exceptions.CancelledError: + # Currently the only exception designed be able to occur here. + + # Ensure the lock invariant: If lock is not claimed (or about + # to be claimed by us) and there is a Task in waiters, + # ensure that the Task at the head will run. if not self._locked: self._wake_up_first() raise + # assert self._locked is False self._locked = True return True @@ -125,7 +128,7 @@ def release(self): """Release a lock. When the lock is locked, reset it to unlocked, and return. - If any other coroutines are blocked waiting for the lock to become + If any other tasks are blocked waiting for the lock to become unlocked, allow exactly one of them to proceed. When invoked on an unlocked lock, a RuntimeError is raised. @@ -139,7 +142,7 @@ def release(self): raise RuntimeError('Lock is not acquired.') def _wake_up_first(self): - """Wake up the first waiter if it isn't done.""" + """Ensure that the first waiter will wake up.""" if not self._waiters: return try: @@ -147,9 +150,7 @@ def _wake_up_first(self): except StopIteration: return - # .done() necessarily means that a waiter will wake up later on and - # either take the lock, or, if it was cancelled and lock wasn't - # taken already, will hit this again and wake up a new waiter. + # .done() means that the waiter is already set to wake up. if not fut.done(): fut.set_result(True) @@ -179,8 +180,8 @@ def is_set(self): return self._value def set(self): - """Set the internal flag to true. All coroutines waiting for it to - become true are awakened. Coroutine that call wait() once the flag is + """Set the internal flag to true. All tasks waiting for it to + become true are awakened. Tasks that call wait() once the flag is true will not block at all. """ if not self._value: @@ -191,7 +192,7 @@ def set(self): fut.set_result(True) def clear(self): - """Reset the internal flag to false. Subsequently, coroutines calling + """Reset the internal flag to false. Subsequently, tasks calling wait() will block until set() is called to set the internal flag to true again.""" self._value = False @@ -200,7 +201,7 @@ async def wait(self): """Block until the internal flag is true. If the internal flag is true on entry, return True - immediately. Otherwise, block until another coroutine calls + immediately. Otherwise, block until another task calls set() to set the flag to true, then return True. """ if self._value: @@ -219,8 +220,8 @@ class Condition(_ContextManagerMixin, mixins._LoopBoundMixin): """Asynchronous equivalent to threading.Condition. This class implements condition variable objects. A condition variable - allows one or more coroutines to wait until they are notified by another - coroutine. + allows one or more tasks to wait until they are notified by another + task. A new Lock object is created and used as the underlying lock. """ @@ -247,45 +248,64 @@ def __repr__(self): async def wait(self): """Wait until notified. - If the calling coroutine has not acquired the lock when this + If the calling task has not acquired the lock when this method is called, a RuntimeError is raised. This method releases the underlying lock, and then blocks until it is awakened by a notify() or notify_all() call for - the same condition variable in another coroutine. Once + the same condition variable in another task. Once awakened, it re-acquires the lock and returns True. + + This method may return spuriously, + which is why the caller should always + re-check the state and be prepared to wait() again. """ if not self.locked(): raise RuntimeError('cannot wait on un-acquired lock') + fut = self._get_loop().create_future() self.release() try: - fut = self._get_loop().create_future() - self._waiters.append(fut) try: - await fut - return True - finally: - self._waiters.remove(fut) - - finally: - # Must reacquire lock even if wait is cancelled - cancelled = False - while True: + self._waiters.append(fut) try: - await self.acquire() - break - except exceptions.CancelledError: - cancelled = True + await fut + return True + finally: + self._waiters.remove(fut) - if cancelled: - raise exceptions.CancelledError + finally: + # Must re-acquire lock even if wait is cancelled. + # We only catch CancelledError here, since we don't want any + # other (fatal) errors with the future to cause us to spin. + err = None + while True: + try: + await self.acquire() + break + except exceptions.CancelledError as e: + err = e + + if err is not None: + try: + raise err # Re-raise most recent exception instance. + finally: + err = None # Break reference cycles. + except BaseException: + # Any error raised out of here _may_ have occurred after this Task + # believed to have been successfully notified. + # Make sure to notify another Task instead. This may result + # in a "spurious wakeup", which is allowed as part of the + # Condition Variable protocol. + self._notify(1) + raise async def wait_for(self, predicate): """Wait until a predicate becomes true. - The predicate should be a callable which result will be - interpreted as a boolean value. The final predicate value is + The predicate should be a callable whose result will be + interpreted as a boolean value. The method will repeatedly + wait() until it evaluates to true. The final predicate value is the return value. """ result = predicate() @@ -295,20 +315,22 @@ async def wait_for(self, predicate): return result def notify(self, n=1): - """By default, wake up one coroutine waiting on this condition, if any. - If the calling coroutine has not acquired the lock when this method + """By default, wake up one task waiting on this condition, if any. + If the calling task has not acquired the lock when this method is called, a RuntimeError is raised. - This method wakes up at most n of the coroutines waiting for the - condition variable; it is a no-op if no coroutines are waiting. + This method wakes up n of the tasks waiting for the condition + variable; if fewer than n are waiting, they are all awoken. - Note: an awakened coroutine does not actually return from its + Note: an awakened task does not actually return from its wait() call until it can reacquire the lock. Since notify() does not release the lock, its caller should. """ if not self.locked(): raise RuntimeError('cannot notify on un-acquired lock') + self._notify(n) + def _notify(self, n): idx = 0 for fut in self._waiters: if idx >= n: @@ -357,6 +379,7 @@ def __repr__(self): def locked(self): """Returns True if semaphore cannot be acquired immediately.""" + # Due to state, or FIFO rules (must allow others to run first). return self._value == 0 or ( any(not w.cancelled() for w in (self._waiters or ()))) @@ -365,11 +388,12 @@ async def acquire(self): If the internal counter is larger than zero on entry, decrement it by one and return True immediately. If it is - zero on entry, block, waiting until some other coroutine has + zero on entry, block, waiting until some other task has called release() to make it larger than 0, and then return True. """ if not self.locked(): + # Maintain FIFO, wait for others to start even if _value > 0. self._value -= 1 return True @@ -378,29 +402,34 @@ async def acquire(self): fut = self._get_loop().create_future() self._waiters.append(fut) - # Finally block should be called before the CancelledError - # handling as we don't want CancelledError to call - # _wake_up_first() and attempt to wake up itself. try: try: await fut finally: self._waiters.remove(fut) except exceptions.CancelledError: - if not fut.cancelled(): + # Currently the only exception designed be able to occur here. + if fut.done() and not fut.cancelled(): + # Our Future was successfully set to True via _wake_up_next(), + # but we are not about to successfully acquire(). Therefore we + # must undo the bookkeeping already done and attempt to wake + # up someone else. self._value += 1 - self._wake_up_next() raise - if self._value > 0: - self._wake_up_next() + finally: + # New waiters may have arrived but had to wait due to FIFO. + # Wake up as many as are allowed. + while self._value > 0: + if not self._wake_up_next(): + break # There was no-one to wake up. return True def release(self): """Release a semaphore, incrementing the internal counter by one. - When it was zero on entry and another coroutine is waiting for it to - become larger than zero again, wake up that coroutine. + When it was zero on entry and another task is waiting for it to + become larger than zero again, wake up that task. """ self._value += 1 self._wake_up_next() @@ -408,13 +437,15 @@ def release(self): def _wake_up_next(self): """Wake up the first waiter that isn't done.""" if not self._waiters: - return + return False for fut in self._waiters: if not fut.done(): self._value -= 1 fut.set_result(True) - return + # `fut` is now `done()` and not `cancelled()`. + return True + return False class BoundedSemaphore(Semaphore): diff --git a/Lib/asyncio/selector_events.py b/Lib/asyncio/selector_events.py index dcd5e0aa345029..10fbdd76e93f79 100644 --- a/Lib/asyncio/selector_events.py +++ b/Lib/asyncio/selector_events.py @@ -235,6 +235,10 @@ async def _accept_connection2( await waiter except BaseException: transport.close() + # gh-109534: When an exception is raised by the SSLProtocol object the + # exception set in this future can keep the protocol object alive and + # cause a reference cycle. + waiter = None raise # It's now up to the protocol to handle the connection. diff --git a/Lib/asyncio/sslproto.py b/Lib/asyncio/sslproto.py index cbb6527d0b28e0..fa99d4533aa0a6 100644 --- a/Lib/asyncio/sslproto.py +++ b/Lib/asyncio/sslproto.py @@ -1,3 +1,7 @@ +# Contains code from https://github.com/MagicStack/uvloop/tree/v0.16.0 +# SPDX-License-Identifier: PSF-2.0 AND (MIT OR Apache-2.0) +# SPDX-FileCopyrightText: Copyright (c) 2015-2021 MagicStack Inc. http://magic.io + import collections import enum import warnings @@ -575,6 +579,7 @@ def _on_handshake_complete(self, handshake_exc): peercert = sslobj.getpeercert() except Exception as exc: + handshake_exc = None self._set_state(SSLProtocolState.UNWRAPPED) if isinstance(exc, ssl.CertificateError): msg = 'SSL handshake failed on verifying the certificate' diff --git a/Lib/asyncio/staggered.py b/Lib/asyncio/staggered.py index 451a53a16f3831..e180cde0243b15 100644 --- a/Lib/asyncio/staggered.py +++ b/Lib/asyncio/staggered.py @@ -3,7 +3,6 @@ __all__ = 'staggered_race', import contextlib -import typing from . import events from . import exceptions as exceptions_mod @@ -11,16 +10,7 @@ from . import tasks -async def staggered_race( - coro_fns: typing.Iterable[typing.Callable[[], typing.Awaitable]], - delay: typing.Optional[float], - *, - loop: events.AbstractEventLoop = None, -) -> typing.Tuple[ - typing.Any, - typing.Optional[int], - typing.List[typing.Optional[Exception]] -]: +async def staggered_race(coro_fns, delay, *, loop=None): """Run coroutines with staggered start times and take the first to finish. This method takes an iterable of coroutine functions. The first one is diff --git a/Lib/asyncio/streams.py b/Lib/asyncio/streams.py index ffb48b9cd6cb70..df58b7a799a5ad 100644 --- a/Lib/asyncio/streams.py +++ b/Lib/asyncio/streams.py @@ -246,6 +246,9 @@ def connection_made(self, transport): self._stream_writer) if coroutines.iscoroutine(res): def callback(task): + if task.cancelled(): + transport.close() + return exc = task.exception() if exc is not None: self._loop.call_exception_handler({ diff --git a/Lib/asyncio/taskgroups.py b/Lib/asyncio/taskgroups.py index cb9c1ce4d7d1d2..f322b1f6653f6a 100644 --- a/Lib/asyncio/taskgroups.py +++ b/Lib/asyncio/taskgroups.py @@ -73,8 +73,10 @@ async def __aexit__(self, et, exc, tb): self._base_error is None): self._base_error = exc - propagate_cancellation_error = \ - exc if et is exceptions.CancelledError else None + if et is not None and issubclass(et, exceptions.CancelledError): + propagate_cancellation_error = exc + else: + propagate_cancellation_error = None if self._parent_cancel_requested: # If this flag is set we *must* call uncancel(). if self._parent_task.uncancel() == 0: @@ -130,10 +132,10 @@ async def __aexit__(self, et, exc, tb): # Propagate CancelledError if there is one, except if there # are other errors -- those have priority. - if propagate_cancellation_error and not self._errors: + if propagate_cancellation_error is not None and not self._errors: raise propagate_cancellation_error - if et is not None and et is not exceptions.CancelledError: + if et is not None and not issubclass(et, exceptions.CancelledError): self._errors.append(exc) if self._errors: diff --git a/Lib/asyncio/timeouts.py b/Lib/asyncio/timeouts.py index 30042abb3ad804..e6f5100691d362 100644 --- a/Lib/asyncio/timeouts.py +++ b/Lib/asyncio/timeouts.py @@ -109,10 +109,16 @@ async def __aexit__( if self._state is _State.EXPIRING: self._state = _State.EXPIRED - if self._task.uncancel() <= self._cancelling and exc_type is exceptions.CancelledError: + if self._task.uncancel() <= self._cancelling and exc_type is not None: # Since there are no new cancel requests, we're # handling this. - raise TimeoutError from exc_val + if issubclass(exc_type, exceptions.CancelledError): + raise TimeoutError from exc_val + elif exc_val is not None: + self._insert_timeout_error(exc_val) + if isinstance(exc_val, ExceptionGroup): + for exc in exc_val.exceptions: + self._insert_timeout_error(exc) elif self._state is _State.ENTERED: self._state = _State.EXITED @@ -125,6 +131,16 @@ def _on_timeout(self) -> None: # drop the reference early self._timeout_handler = None + @staticmethod + def _insert_timeout_error(exc_val: BaseException) -> None: + while exc_val.__context__ is not None: + if isinstance(exc_val.__context__, exceptions.CancelledError): + te = TimeoutError() + te.__context__ = te.__cause__ = exc_val.__context__ + exc_val.__context__ = te + break + exc_val = exc_val.__context__ + def timeout(delay: Optional[float]) -> Timeout: """Timeout async context manager. diff --git a/Lib/calendar.py b/Lib/calendar.py index 03469d8ac96bcd..833ce331b14a0c 100644 --- a/Lib/calendar.py +++ b/Lib/calendar.py @@ -734,10 +734,15 @@ def main(args=None): choices=("text", "html"), help="output type (text or html)" ) + parser.add_argument( + "-f", "--first-weekday", + type=int, default=0, + help="weekday (0 is Monday, 6 is Sunday) to start each week (default 0)" + ) parser.add_argument( "year", nargs='?', type=int, - help="year number (1-9999)" + help="year number" ) parser.add_argument( "month", @@ -761,6 +766,7 @@ def main(args=None): cal = LocaleHTMLCalendar(locale=locale) else: cal = HTMLCalendar() + cal.setfirstweekday(options.first_weekday) encoding = options.encoding if encoding is None: encoding = sys.getdefaultencoding() @@ -775,6 +781,7 @@ def main(args=None): cal = LocaleTextCalendar(locale=locale) else: cal = TextCalendar() + cal.setfirstweekday(options.first_weekday) optdict = dict(w=options.width, l=options.lines) if options.month is None: optdict["c"] = options.spacing diff --git a/Lib/cmd.py b/Lib/cmd.py index 2e358d6cd5a02d..a37d16cd7bde16 100644 --- a/Lib/cmd.py +++ b/Lib/cmd.py @@ -42,7 +42,7 @@ functions respectively. """ -import string, sys +import inspect, string, sys __all__ = ["Cmd"] @@ -305,6 +305,7 @@ def do_help(self, arg): except AttributeError: try: doc=getattr(self, 'do_' + arg).__doc__ + doc = inspect.cleandoc(doc) if doc: self.stdout.write("%s\n"%str(doc)) return diff --git a/Lib/codeop.py b/Lib/codeop.py index 91146be2c438e2..6ad60e7f85098d 100644 --- a/Lib/codeop.py +++ b/Lib/codeop.py @@ -65,9 +65,10 @@ def _maybe_compile(compiler, source, filename, symbol): try: compiler(source + "\n", filename, symbol) return None + except IncompleteInputError as e: + return None except SyntaxError as e: - if "incomplete input" in str(e): - return None + pass # fallthrough return compiler(source, filename, symbol, incomplete_input=False) diff --git a/Lib/concurrent/futures/process.py b/Lib/concurrent/futures/process.py index ffaffdb8b3d0aa..ca843e11eeb83d 100644 --- a/Lib/concurrent/futures/process.py +++ b/Lib/concurrent/futures/process.py @@ -190,16 +190,6 @@ def _on_queue_feeder_error(self, e, obj): super()._on_queue_feeder_error(e, obj) -def _get_chunks(*iterables, chunksize): - """ Iterates over zip()ed iterables in chunks. """ - it = zip(*iterables) - while True: - chunk = tuple(itertools.islice(it, chunksize)) - if not chunk: - return - yield chunk - - def _process_chunk(fn, chunk): """ Processes a chunk of an iterable passed to map. @@ -847,7 +837,7 @@ def map(self, fn, *iterables, timeout=None, chunksize=1): raise ValueError("chunksize must be >= 1.") results = super().map(partial(_process_chunk, fn), - _get_chunks(*iterables, chunksize=chunksize), + itertools.batched(zip(*iterables), chunksize), timeout=timeout) return _chain_from_iterable_of_lists(results) diff --git a/Lib/csv.py b/Lib/csv.py index 77f30c8d2b1f61..75e35b23236795 100644 --- a/Lib/csv.py +++ b/Lib/csv.py @@ -1,28 +1,90 @@ -""" -csv.py - read/write/investigate CSV files +r""" +CSV parsing and writing. + +This module provides classes that assist in the reading and writing +of Comma Separated Value (CSV) files, and implements the interface +described by PEP 305. Although many CSV files are simple to parse, +the format is not formally defined by a stable specification and +is subtle enough that parsing lines of a CSV file with something +like line.split(",") is bound to fail. The module supports three +basic APIs: reading, writing, and registration of dialects. + + +DIALECT REGISTRATION: + +Readers and writers support a dialect argument, which is a convenient +handle on a group of settings. When the dialect argument is a string, +it identifies one of the dialects previously registered with the module. +If it is a class or instance, the attributes of the argument are used as +the settings for the reader or writer: + + class excel: + delimiter = ',' + quotechar = '"' + escapechar = None + doublequote = True + skipinitialspace = False + lineterminator = '\r\n' + quoting = QUOTE_MINIMAL + +SETTINGS: + + * quotechar - specifies a one-character string to use as the + quoting character. It defaults to '"'. + * delimiter - specifies a one-character string to use as the + field separator. It defaults to ','. + * skipinitialspace - specifies how to interpret spaces which + immediately follow a delimiter. It defaults to False, which + means that spaces immediately following a delimiter is part + of the following field. + * lineterminator - specifies the character sequence which should + terminate rows. + * quoting - controls when quotes should be generated by the writer. + It can take on any of the following module constants: + + csv.QUOTE_MINIMAL means only when required, for example, when a + field contains either the quotechar or the delimiter + csv.QUOTE_ALL means that quotes are always placed around fields. + csv.QUOTE_NONNUMERIC means that quotes are always placed around + fields which do not parse as integers or floating point + numbers. + csv.QUOTE_STRINGS means that quotes are always placed around + fields which are strings. Note that the Python value None + is not a string. + csv.QUOTE_NOTNULL means that quotes are only placed around fields + that are not the Python value None. + csv.QUOTE_NONE means that quotes are never placed around fields. + * escapechar - specifies a one-character string used to escape + the delimiter when quoting is set to QUOTE_NONE. + * doublequote - controls the handling of quotes inside fields. When + True, two consecutive quotes are interpreted as one during read, + and when writing, each quote character embedded in the data is + written as two quotes """ import re import types -from _csv import Error, __version__, writer, reader, register_dialect, \ +from _csv import Error, writer, reader, register_dialect, \ unregister_dialect, get_dialect, list_dialects, \ field_size_limit, \ QUOTE_MINIMAL, QUOTE_ALL, QUOTE_NONNUMERIC, QUOTE_NONE, \ - QUOTE_STRINGS, QUOTE_NOTNULL, \ - __doc__ + QUOTE_STRINGS, QUOTE_NOTNULL from _csv import Dialect as _Dialect from io import StringIO __all__ = ["QUOTE_MINIMAL", "QUOTE_ALL", "QUOTE_NONNUMERIC", "QUOTE_NONE", "QUOTE_STRINGS", "QUOTE_NOTNULL", - "Error", "Dialect", "__doc__", "excel", "excel_tab", + "Error", "Dialect", "excel", "excel_tab", "field_size_limit", "reader", "writer", "register_dialect", "get_dialect", "list_dialects", "Sniffer", - "unregister_dialect", "__version__", "DictReader", "DictWriter", + "unregister_dialect", "DictReader", "DictWriter", "unix_dialect"] +__version__ = "1.0" + + class Dialect: """Describe a CSV dialect. @@ -51,8 +113,8 @@ def _validate(self): try: _Dialect(self) except TypeError as e: - # We do this for compatibility with py2.3 - raise Error(str(e)) + # Re-raise to get a traceback showing more user code. + raise Error(str(e)) from None class excel(Dialect): """Describe the usual properties of Excel-generated CSV files.""" diff --git a/Lib/ctypes/_endian.py b/Lib/ctypes/_endian.py index 3febb3118b8230..6382dd22b8acc8 100644 --- a/Lib/ctypes/_endian.py +++ b/Lib/ctypes/_endian.py @@ -15,8 +15,8 @@ def _other_endian(typ): # if typ is array if isinstance(typ, _array_type): return _other_endian(typ._type_) * typ._length_ - # if typ is structure - if issubclass(typ, Structure): + # if typ is structure or union + if issubclass(typ, (Structure, Union)): return typ raise TypeError("This type does not support other endian: %s" % typ) diff --git a/Lib/ctypes/util.py b/Lib/ctypes/util.py index 0c2510e1619c8e..c550883e7c7d4b 100644 --- a/Lib/ctypes/util.py +++ b/Lib/ctypes/util.py @@ -96,8 +96,11 @@ def find_library(name): def _is_elf(filename): "Return True if the given file is an ELF file" elf_header = b'\x7fELF' - with open(filename, 'br') as thefile: - return thefile.read(4) == elf_header + try: + with open(filename, 'br') as thefile: + return thefile.read(4) == elf_header + except FileNotFoundError: + return False def _findLib_gcc(name): # Run GCC's linker with the -t (aka --trace) option and examine the diff --git a/Lib/dataclasses.py b/Lib/dataclasses.py index 2fba32b5ffbc1e..3335821f3f32e9 100644 --- a/Lib/dataclasses.py +++ b/Lib/dataclasses.py @@ -1332,58 +1332,69 @@ class C: def _asdict_inner(obj, dict_factory): - if type(obj) in _ATOMIC_TYPES: + obj_type = type(obj) + if obj_type in _ATOMIC_TYPES: return obj - elif _is_dataclass_instance(obj): - # fast path for the common case + elif hasattr(obj_type, _FIELDS): + # dataclass instance: fast path for the common case if dict_factory is dict: return { f.name: _asdict_inner(getattr(obj, f.name), dict) for f in fields(obj) } else: - result = [] - for f in fields(obj): - value = _asdict_inner(getattr(obj, f.name), dict_factory) - result.append((f.name, value)) - return dict_factory(result) - elif isinstance(obj, tuple) and hasattr(obj, '_fields'): - # obj is a namedtuple. Recurse into it, but the returned - # object is another namedtuple of the same type. This is - # similar to how other list- or tuple-derived classes are - # treated (see below), but we just need to create them - # differently because a namedtuple's __init__ needs to be - # called differently (see bpo-34363). - - # I'm not using namedtuple's _asdict() - # method, because: - # - it does not recurse in to the namedtuple fields and - # convert them to dicts (using dict_factory). - # - I don't actually want to return a dict here. The main - # use case here is json.dumps, and it handles converting - # namedtuples to lists. Admittedly we're losing some - # information here when we produce a json list instead of a - # dict. Note that if we returned dicts here instead of - # namedtuples, we could no longer call asdict() on a data - # structure where a namedtuple was used as a dict key. - - return type(obj)(*[_asdict_inner(v, dict_factory) for v in obj]) - elif isinstance(obj, (list, tuple)): - # Assume we can create an object of this type by passing in a - # generator (which is not true for namedtuples, handled - # above). - return type(obj)(_asdict_inner(v, dict_factory) for v in obj) - elif isinstance(obj, dict): - if hasattr(type(obj), 'default_factory'): + return dict_factory([ + (f.name, _asdict_inner(getattr(obj, f.name), dict_factory)) + for f in fields(obj) + ]) + # handle the builtin types first for speed; subclasses handled below + elif obj_type is list: + return [_asdict_inner(v, dict_factory) for v in obj] + elif obj_type is dict: + return { + _asdict_inner(k, dict_factory): _asdict_inner(v, dict_factory) + for k, v in obj.items() + } + elif obj_type is tuple: + return tuple([_asdict_inner(v, dict_factory) for v in obj]) + elif issubclass(obj_type, tuple): + if hasattr(obj, '_fields'): + # obj is a namedtuple. Recurse into it, but the returned + # object is another namedtuple of the same type. This is + # similar to how other list- or tuple-derived classes are + # treated (see below), but we just need to create them + # differently because a namedtuple's __init__ needs to be + # called differently (see bpo-34363). + + # I'm not using namedtuple's _asdict() + # method, because: + # - it does not recurse in to the namedtuple fields and + # convert them to dicts (using dict_factory). + # - I don't actually want to return a dict here. The main + # use case here is json.dumps, and it handles converting + # namedtuples to lists. Admittedly we're losing some + # information here when we produce a json list instead of a + # dict. Note that if we returned dicts here instead of + # namedtuples, we could no longer call asdict() on a data + # structure where a namedtuple was used as a dict key. + return obj_type(*[_asdict_inner(v, dict_factory) for v in obj]) + else: + return obj_type(_asdict_inner(v, dict_factory) for v in obj) + elif issubclass(obj_type, dict): + if hasattr(obj_type, 'default_factory'): # obj is a defaultdict, which has a different constructor from # dict as it requires the default_factory as its first arg. - result = type(obj)(getattr(obj, 'default_factory')) + result = obj_type(obj.default_factory) for k, v in obj.items(): result[_asdict_inner(k, dict_factory)] = _asdict_inner(v, dict_factory) return result - return type(obj)((_asdict_inner(k, dict_factory), - _asdict_inner(v, dict_factory)) - for k, v in obj.items()) + return obj_type((_asdict_inner(k, dict_factory), + _asdict_inner(v, dict_factory)) + for k, v in obj.items()) + elif issubclass(obj_type, list): + # Assume we can create an object of this type by passing in a + # generator + return obj_type(_asdict_inner(v, dict_factory) for v in obj) else: return copy.deepcopy(obj) @@ -1416,11 +1427,10 @@ def _astuple_inner(obj, tuple_factory): if type(obj) in _ATOMIC_TYPES: return obj elif _is_dataclass_instance(obj): - result = [] - for f in fields(obj): - value = _astuple_inner(getattr(obj, f.name), tuple_factory) - result.append(value) - return tuple_factory(result) + return tuple_factory([ + _astuple_inner(getattr(obj, f.name), tuple_factory) + for f in fields(obj) + ]) elif isinstance(obj, tuple) and hasattr(obj, '_fields'): # obj is a namedtuple. Recurse into it, but the returned # object is another namedtuple of the same type. This is @@ -1558,14 +1568,14 @@ class C: return _replace(obj, **changes) -def _replace(obj, /, **changes): +def _replace(self, /, **changes): # We're going to mutate 'changes', but that's okay because it's a - # new dict, even if called with 'replace(obj, **my_changes)'. + # new dict, even if called with 'replace(self, **my_changes)'. # It's an error to have init=False fields in 'changes'. - # If a field is not in 'changes', read its value from the provided obj. + # If a field is not in 'changes', read its value from the provided 'self'. - for f in getattr(obj, _FIELDS).values(): + for f in getattr(self, _FIELDS).values(): # Only consider normal fields or InitVars. if f._field_type is _FIELD_CLASSVAR: continue @@ -1582,11 +1592,11 @@ def _replace(obj, /, **changes): if f._field_type is _FIELD_INITVAR and f.default is MISSING: raise TypeError(f"InitVar {f.name!r} " f'must be specified with replace()') - changes[f.name] = getattr(obj, f.name) + changes[f.name] = getattr(self, f.name) # Create the new object, which calls __init__() and # __post_init__() (if defined), using all of the init fields we've # added and/or left in 'changes'. If there are values supplied in # changes that aren't fields, this will correctly raise a # TypeError. - return obj.__class__(**changes) + return self.__class__(**changes) diff --git a/Lib/dbm/dumb.py b/Lib/dbm/dumb.py index 754624ccc8f500..def120ffc3778b 100644 --- a/Lib/dbm/dumb.py +++ b/Lib/dbm/dumb.py @@ -98,7 +98,8 @@ def _update(self, flag): except OSError: if flag not in ('c', 'n'): raise - self._modified = True + with self._io.open(self._dirfile, 'w', encoding="Latin-1") as f: + self._chmod(self._dirfile) else: with f: for line in f: @@ -134,6 +135,7 @@ def _commit(self): # position; UTF-8, though, does care sometimes. entry = "%r, %r\n" % (key.decode('Latin-1'), pos_and_siz_pair) f.write(entry) + self._modified = False sync = _commit diff --git a/Lib/dis.py b/Lib/dis.py index 1a2f1032d500af..f05ea1a24f45a7 100644 --- a/Lib/dis.py +++ b/Lib/dis.py @@ -1032,11 +1032,16 @@ def main(): help='show inline caches') parser.add_argument('-O', '--show-offsets', action='store_true', help='show instruction offsets') - parser.add_argument('infile', type=argparse.FileType('rb'), nargs='?', default='-') + parser.add_argument('infile', nargs='?', default='-') args = parser.parse_args() - with args.infile as infile: - source = infile.read() - code = compile(source, args.infile.name, "exec") + if args.infile == '-': + name = '<stdin>' + source = sys.stdin.buffer.read() + else: + name = args.infile + with open(args.infile, 'rb') as infile: + source = infile.read() + code = compile(source, name, "exec") dis(code, show_caches=args.show_caches, show_offsets=args.show_offsets) if __name__ == "__main__": diff --git a/Lib/email/_header_value_parser.py b/Lib/email/_header_value_parser.py index 0d6bd812475eea..5b653f66c18554 100644 --- a/Lib/email/_header_value_parser.py +++ b/Lib/email/_header_value_parser.py @@ -2766,6 +2766,7 @@ def _refold_parse_tree(parse_tree, *, policy): encoding = 'utf-8' if policy.utf8 else 'us-ascii' lines = [''] last_ew = None + last_charset = None wrap_as_ew_blocked = 0 want_encoding = False end_ew_not_allowed = Terminal('', 'wrap_as_ew_blocked') @@ -2820,8 +2821,14 @@ def _refold_parse_tree(parse_tree, *, policy): else: # It's a terminal, wrap it as an encoded word, possibly # combining it with previously encoded words if allowed. + if (last_ew is not None and + charset != last_charset and + (last_charset == 'unknown-8bit' or + last_charset == 'utf-8' and charset != 'us-ascii')): + last_ew = None last_ew = _fold_as_ew(tstr, lines, maxlen, last_ew, part.ew_combine_allowed, charset) + last_charset = charset want_encoding = False continue if len(tstr) <= maxlen - len(lines[-1]): diff --git a/Lib/email/_parseaddr.py b/Lib/email/_parseaddr.py index febe411355d6be..0f1bf8e4253ec4 100644 --- a/Lib/email/_parseaddr.py +++ b/Lib/email/_parseaddr.py @@ -13,7 +13,7 @@ 'quote', ] -import time, calendar +import time SPACE = ' ' EMPTYSTRING = '' @@ -194,6 +194,9 @@ def mktime_tz(data): # No zone info, so localtime is better assumption than GMT return time.mktime(data[:8] + (-1,)) else: + # Delay the import, since mktime_tz is rarely used + import calendar + t = calendar.timegm(data) return t - data[9] diff --git a/Lib/email/policy.py b/Lib/email/policy.py index 611deb50bb5290..8816c84ed175a7 100644 --- a/Lib/email/policy.py +++ b/Lib/email/policy.py @@ -210,8 +210,15 @@ def _fold(self, name, value, refold_binary=False): self.refold_source == 'long' and (lines and len(lines[0])+len(name)+2 > maxlen or any(len(x) > maxlen for x in lines[1:]))) - if refold or refold_binary and _has_surrogates(value): + + if not refold: + if not self.utf8: + refold = not value.isascii() + elif refold_binary: + refold = _has_surrogates(value) + if refold: return self.header_factory(name, ''.join(lines)).fold(policy=self) + return name + ': ' + self.linesep.join(lines) + self.linesep diff --git a/Lib/ensurepip/__init__.py b/Lib/ensurepip/__init__.py index a09bf3201e1fb7..e8dd253bb55520 100644 --- a/Lib/ensurepip/__init__.py +++ b/Lib/ensurepip/__init__.py @@ -1,78 +1,64 @@ -import collections import os -import os.path import subprocess import sys import sysconfig import tempfile +from contextlib import nullcontext from importlib import resources +from pathlib import Path +from shutil import copy2 __all__ = ["version", "bootstrap"] -_PACKAGE_NAMES = ('pip',) -_PIP_VERSION = "23.3.2" -_PROJECTS = [ - ("pip", _PIP_VERSION, "py3"), -] - -# Packages bundled in ensurepip._bundled have wheel_name set. -# Packages from WHEEL_PKG_DIR have wheel_path set. -_Package = collections.namedtuple('Package', - ('version', 'wheel_name', 'wheel_path')) +_PIP_VERSION = "24.0" # Directory of system wheel packages. Some Linux distribution packaging # policies recommend against bundling dependencies. For example, Fedora # installs wheel packages in the /usr/share/python-wheels/ directory and don't # install the ensurepip._bundled package. -_WHEEL_PKG_DIR = sysconfig.get_config_var('WHEEL_PKG_DIR') +if (_pkg_dir := sysconfig.get_config_var('WHEEL_PKG_DIR')) is not None: + _WHEEL_PKG_DIR = Path(_pkg_dir).resolve() +else: + _WHEEL_PKG_DIR = None + +def _find_wheel_pkg_dir_pip(): + if _WHEEL_PKG_DIR is None: + # NOTE: The compile-time `WHEEL_PKG_DIR` is unset so there is no place + # NOTE: for looking up the wheels. + return None -def _find_packages(path): - packages = {} + dist_matching_wheels = _WHEEL_PKG_DIR.glob('pip-*.whl') try: - filenames = os.listdir(path) - except OSError: - # Ignore: path doesn't exist or permission error - filenames = () - # Make the code deterministic if a directory contains multiple wheel files - # of the same package, but don't attempt to implement correct version - # comparison since this case should not happen. - filenames = sorted(filenames) - for filename in filenames: - # filename is like 'pip-21.2.4-py3-none-any.whl' - if not filename.endswith(".whl"): - continue - for name in _PACKAGE_NAMES: - prefix = name + '-' - if filename.startswith(prefix): - break - else: - continue - - # Extract '21.2.4' from 'pip-21.2.4-py3-none-any.whl' - version = filename.removeprefix(prefix).partition('-')[0] - wheel_path = os.path.join(path, filename) - packages[name] = _Package(version, None, wheel_path) - return packages - - -def _get_packages(): - global _PACKAGES, _WHEEL_PKG_DIR - if _PACKAGES is not None: - return _PACKAGES - - packages = {} - for name, version, py_tag in _PROJECTS: - wheel_name = f"{name}-{version}-{py_tag}-none-any.whl" - packages[name] = _Package(version, wheel_name, None) - if _WHEEL_PKG_DIR: - dir_packages = _find_packages(_WHEEL_PKG_DIR) - # only used the wheel package directory if all packages are found there - if all(name in dir_packages for name in _PACKAGE_NAMES): - packages = dir_packages - _PACKAGES = packages - return packages -_PACKAGES = None + last_matching_dist_wheel = sorted(dist_matching_wheels)[-1] + except IndexError: + # NOTE: `WHEEL_PKG_DIR` does not contain any wheel files for `pip`. + return None + + return nullcontext(last_matching_dist_wheel) + + +def _get_pip_whl_path_ctx(): + # Prefer pip from the wheel package directory, if present. + if (alternative_pip_wheel_path := _find_wheel_pkg_dir_pip()) is not None: + return alternative_pip_wheel_path + + return resources.as_file( + resources.files('ensurepip') + / '_bundled' + / f'pip-{_PIP_VERSION}-py3-none-any.whl' + ) + + +def _get_pip_version(): + with _get_pip_whl_path_ctx() as bundled_wheel_path: + wheel_name = bundled_wheel_path.name + return ( + # Extract '21.2.4' from 'pip-21.2.4-py3-none-any.whl' + wheel_name. + removeprefix('pip-'). + partition('-')[0] + ) def _run_pip(args, additional_paths=None): @@ -105,7 +91,7 @@ def version(): """ Returns a string specifying the bundled version of pip. """ - return _get_packages()['pip'].version + return _get_pip_version() def _disable_pip_configuration_settings(): @@ -167,24 +153,10 @@ def _bootstrap(*, root=None, upgrade=False, user=False, with tempfile.TemporaryDirectory() as tmpdir: # Put our bundled wheels into a temporary directory and construct the # additional paths that need added to sys.path - additional_paths = [] - for name, package in _get_packages().items(): - if package.wheel_name: - # Use bundled wheel package - wheel_name = package.wheel_name - wheel_path = resources.files("ensurepip") / "_bundled" / wheel_name - whl = wheel_path.read_bytes() - else: - # Use the wheel package directory - with open(package.wheel_path, "rb") as fp: - whl = fp.read() - wheel_name = os.path.basename(package.wheel_path) - - filename = os.path.join(tmpdir, wheel_name) - with open(filename, "wb") as fp: - fp.write(whl) - - additional_paths.append(filename) + tmpdir_path = Path(tmpdir) + with _get_pip_whl_path_ctx() as bundled_wheel_path: + tmp_wheel_path = tmpdir_path / bundled_wheel_path.name + copy2(bundled_wheel_path, tmp_wheel_path) # Construct the arguments to be passed to the pip command args = ["install", "--no-cache-dir", "--no-index", "--find-links", tmpdir] @@ -197,7 +169,8 @@ def _bootstrap(*, root=None, upgrade=False, user=False, if verbosity: args += ["-" + "v" * verbosity] - return _run_pip([*args, *_PACKAGE_NAMES], additional_paths) + return _run_pip([*args, "pip"], [os.fsdecode(tmp_wheel_path)]) + def _uninstall_helper(*, verbosity=0): """Helper to support a clean default uninstall process on Windows @@ -227,7 +200,7 @@ def _uninstall_helper(*, verbosity=0): if verbosity: args += ["-" + "v" * verbosity] - return _run_pip([*args, *reversed(_PACKAGE_NAMES)]) + return _run_pip([*args, "pip"]) def _main(argv=None): diff --git a/Lib/ensurepip/_bundled/pip-23.3.2-py3-none-any.whl b/Lib/ensurepip/_bundled/pip-24.0-py3-none-any.whl similarity index 83% rename from Lib/ensurepip/_bundled/pip-23.3.2-py3-none-any.whl rename to Lib/ensurepip/_bundled/pip-24.0-py3-none-any.whl index ae78b8a6ce0737..2e6aa9d2cb9923 100644 Binary files a/Lib/ensurepip/_bundled/pip-23.3.2-py3-none-any.whl and b/Lib/ensurepip/_bundled/pip-24.0-py3-none-any.whl differ diff --git a/Lib/enum.py b/Lib/enum.py index a8a50a58380375..98a8966f5eb159 100644 --- a/Lib/enum.py +++ b/Lib/enum.py @@ -409,10 +409,11 @@ def __setitem__(self, key, value): if isinstance(value, auto): single = True value = (value, ) - if type(value) is tuple and any(isinstance(v, auto) for v in value): + if isinstance(value, tuple) and any(isinstance(v, auto) for v in value): # insist on an actual tuple, no subclasses, in keeping with only supporting # top-level auto() usage (not contained in any other data structure) auto_valued = [] + t = type(value) for v in value: if isinstance(v, auto): non_auto_store = False @@ -427,7 +428,12 @@ def __setitem__(self, key, value): if single: value = auto_valued[0] else: - value = tuple(auto_valued) + try: + # accepts iterable as multiple arguments? + value = t(auto_valued) + except TypeError: + # then pass them in singlely + value = t(*auto_valued) self._member_names[key] = None if non_auto_store: self._last_values.append(value) diff --git a/Lib/fileinput.py b/Lib/fileinput.py index 1b25f28f3d3432..3dba3d2fbfa967 100644 --- a/Lib/fileinput.py +++ b/Lib/fileinput.py @@ -53,7 +53,7 @@ sequence must be accessed in strictly sequential order; sequence access and readline() cannot be mixed. -Optional in-place filtering: if the keyword argument inplace=1 is +Optional in-place filtering: if the keyword argument inplace=True is passed to input() or to the FileInput constructor, the file is moved to a backup file and standard output is directed to the input file. This makes it possible to write a filter that rewrites its input file diff --git a/Lib/fractions.py b/Lib/fractions.py index 6532d5d54e3c35..389ab386b6a8a4 100644 --- a/Lib/fractions.py +++ b/Lib/fractions.py @@ -55,17 +55,17 @@ def _hash_algorithm(numerator, denominator): return -2 if result == -1 else result _RATIONAL_FORMAT = re.compile(r""" - \A\s* # optional whitespace at the start, - (?P<sign>[-+]?) # an optional sign, then - (?=\d|\.\d) # lookahead for digit or .digit - (?P<num>\d*|\d+(_\d+)*) # numerator (possibly empty) - (?: # followed by - (?:\s*/\s*(?P<denom>\d+(_\d+)*))? # an optional denominator - | # or - (?:\.(?P<decimal>d*|\d+(_\d+)*))? # an optional fractional part - (?:E(?P<exp>[-+]?\d+(_\d+)*))? # and optional exponent + \A\s* # optional whitespace at the start, + (?P<sign>[-+]?) # an optional sign, then + (?=\d|\.\d) # lookahead for digit or .digit + (?P<num>\d*|\d+(_\d+)*) # numerator (possibly empty) + (?: # followed by + (?:\s*/\s*(?P<denom>\d+(_\d+)*))? # an optional denominator + | # or + (?:\.(?P<decimal>\d*|\d+(_\d+)*))? # an optional fractional part + (?:E(?P<exp>[-+]?\d+(_\d+)*))? # and optional exponent ) - \s*\Z # and optional whitespace to finish + \s*\Z # and optional whitespace to finish """, re.VERBOSE | re.IGNORECASE) diff --git a/Lib/ftplib.py b/Lib/ftplib.py index a56e0c3085701b..10c5d1ea08ab11 100644 --- a/Lib/ftplib.py +++ b/Lib/ftplib.py @@ -900,11 +900,17 @@ def ftpcp(source, sourcename, target, targetname = '', type = 'I'): def test(): '''Test program. - Usage: ftp [-d] [-r[file]] host [-l[dir]] [-d[dir]] [-p] [file] ... + Usage: ftplib [-d] [-r[file]] host [-l[dir]] [-d[dir]] [-p] [file] ... - -d dir - -l list - -p password + Options: + -d increase debugging level + -r[file] set alternate ~/.netrc file + + Commands: + -l[dir] list directory + -d[dir] change the current directory + -p toggle passive and active mode + file retrieve the file and write it to stdout ''' if len(sys.argv) < 2: @@ -930,15 +936,14 @@ def test(): netrcobj = netrc.netrc(rcfile) except OSError: if rcfile is not None: - sys.stderr.write("Could not open account file" - " -- using anonymous login.") + print("Could not open account file -- using anonymous login.", + file=sys.stderr) else: try: userid, acct, passwd = netrcobj.authenticators(host) - except KeyError: + except (KeyError, TypeError): # no account for host - sys.stderr.write( - "No account -- using anonymous login.") + print("No account -- using anonymous login.", file=sys.stderr) ftp.login(userid, passwd, acct) for file in sys.argv[2:]: if file[:2] == '-l': @@ -951,7 +956,9 @@ def test(): ftp.set_pasv(not ftp.passiveserver) else: ftp.retrbinary('RETR ' + file, \ - sys.stdout.write, 1024) + sys.stdout.buffer.write, 1024) + sys.stdout.buffer.flush() + sys.stdout.flush() ftp.quit() diff --git a/Lib/gzip.py b/Lib/gzip.py index 177f9080dc5af8..fda93e0261e028 100644 --- a/Lib/gzip.py +++ b/Lib/gzip.py @@ -349,7 +349,7 @@ def closed(self): def close(self): fileobj = self.fileobj - if fileobj is None: + if fileobj is None or self._buffer.closed: return try: if self.mode == WRITE: diff --git a/Lib/idlelib/News3.txt b/Lib/idlelib/News3.txt index 308865d968814c..241b1f48e5c1d8 100644 --- a/Lib/idlelib/News3.txt +++ b/Lib/idlelib/News3.txt @@ -4,7 +4,20 @@ Released on 2024-10-xx ========================= +gh-96905: In idlelib code, stop redefining built-ins 'dict' and 'object'. + +gh-72284: Improve the lists of features, editor key bindings, +and shell key bingings in the IDLE doc. + +gh-113903: Fix rare failure of test.test_idle, in test_configdialog. + +gh-113729: Fix the "Help -> IDLE Doc" menu bug in 3.11.7 and 3.12.1. + +gh-57795: Enter selected text into the Find box when opening +a Replace dialog. Patch by Roger Serwy and Zackery Spytz. + gh-113269: Fix test_editor hang on macOS Catalina. +Patch by Terry Reedy. gh-112939: Fix processing unsaved files when quitting IDLE on macOS. Patch by Ronald Oussoren and Christopher Chavez. diff --git a/Lib/idlelib/debugger.py b/Lib/idlelib/debugger.py index f487b4c4b16a60..d90dbcd11f9f61 100644 --- a/Lib/idlelib/debugger.py +++ b/Lib/idlelib/debugger.py @@ -508,11 +508,11 @@ def show_source(self, index): class NamespaceViewer: "Global/local namespace viewer for debugger GUI." - def __init__(self, master, title, dict=None): + def __init__(self, master, title, odict=None): # XXX odict never passed. width = 0 height = 40 - if dict: - height = 20*len(dict) # XXX 20 == observed height of Entry widget + if odict: + height = 20*len(odict) # XXX 20 == observed height of Entry widget self.master = master self.title = title import reprlib @@ -533,24 +533,24 @@ def __init__(self, master, title, dict=None): canvas["yscrollcommand"] = vbar.set self.subframe = subframe = Frame(canvas) self.sfid = canvas.create_window(0, 0, window=subframe, anchor="nw") - self.load_dict(dict) + self.load_dict(odict) - dict = -1 + prev_odict = -1 # Needed for initial comparison below. - def load_dict(self, dict, force=0, rpc_client=None): - if dict is self.dict and not force: + def load_dict(self, odict, force=0, rpc_client=None): + if odict is self.prev_odict and not force: return subframe = self.subframe frame = self.frame for c in list(subframe.children.values()): c.destroy() - self.dict = None - if not dict: + self.prev_odict = None + if not odict: l = Label(subframe, text="None") l.grid(row=0, column=0) else: #names = sorted(dict) - ### + # # Because of (temporary) limitations on the dict_keys type (not yet # public or pickleable), have the subprocess to send a list of # keys, not a dict_keys object. sorted() will take a dict_keys @@ -560,12 +560,12 @@ def load_dict(self, dict, force=0, rpc_client=None): # interpreter gets into a loop requesting non-existing dict[0], # dict[1], dict[2], etc from the debugger_r.DictProxy. # TODO recheck above; see debugger_r 159ff, debugobj 60. - keys_list = dict.keys() + keys_list = odict.keys() names = sorted(keys_list) - ### + row = 0 for name in names: - value = dict[name] + value = odict[name] svalue = self.repr.repr(value) # repr(value) # Strip extra quotes caused by calling repr on the (already) # repr'd value sent across the RPC interface: @@ -577,7 +577,7 @@ def load_dict(self, dict, force=0, rpc_client=None): l.insert(0, svalue) l.grid(row=row, column=1, sticky="nw") row = row+1 - self.dict = dict + self.prev_odict = odict # XXX Could we use a <Configure> callback for the following? subframe.update_idletasks() # Alas! width = subframe.winfo_reqwidth() diff --git a/Lib/idlelib/debugger_r.py b/Lib/idlelib/debugger_r.py index 26204438858d8a..ad3355d9f82765 100644 --- a/Lib/idlelib/debugger_r.py +++ b/Lib/idlelib/debugger_r.py @@ -125,16 +125,16 @@ def frame_attr(self, fid, name): def frame_globals(self, fid): frame = frametable[fid] - dict = frame.f_globals - did = id(dict) - dicttable[did] = dict + gdict = frame.f_globals + did = id(gdict) + dicttable[did] = gdict return did def frame_locals(self, fid): frame = frametable[fid] - dict = frame.f_locals - did = id(dict) - dicttable[did] = dict + ldict = frame.f_locals + did = id(ldict) + dicttable[did] = ldict return did def frame_code(self, fid): @@ -158,20 +158,17 @@ def code_filename(self, cid): def dict_keys(self, did): raise NotImplementedError("dict_keys not public or pickleable") -## dict = dicttable[did] -## return dict.keys() +## return dicttable[did].keys() - ### Needed until dict_keys is type is finished and pickealable. + ### Needed until dict_keys type is finished and pickleable. + # xxx finished. pickleable? ### Will probably need to extend rpc.py:SocketIO._proxify at that time. def dict_keys_list(self, did): - dict = dicttable[did] - return list(dict.keys()) + return list(dicttable[did].keys()) def dict_item(self, did, key): - dict = dicttable[did] - value = dict[key] - value = reprlib.repr(value) ### can't pickle module 'builtins' - return value + value = dicttable[did][key] + return reprlib.repr(value) # Can't pickle module 'builtins'. #----------end class IdbAdapter---------- diff --git a/Lib/idlelib/debugobj.py b/Lib/idlelib/debugobj.py index 156377f8ed26ac..fb448ece2fa25e 100644 --- a/Lib/idlelib/debugobj.py +++ b/Lib/idlelib/debugobj.py @@ -1,3 +1,5 @@ +"""Define tree items for debug stackviewer, which is only user. +""" # XXX TO DO: # - popup menu # - support partial or total redisplay @@ -17,9 +19,9 @@ myrepr.maxother = 100 class ObjectTreeItem(TreeItem): - def __init__(self, labeltext, object, setfunction=None): + def __init__(self, labeltext, object_, setfunction=None): self.labeltext = labeltext - self.object = object + self.object = object_ self.setfunction = setfunction def GetLabelText(self): return self.labeltext @@ -51,8 +53,8 @@ def GetSubList(self): item = make_objecttreeitem( str(key) + " =", value, - lambda value, key=key, object=self.object: - setattr(object, key, value)) + lambda value, key=key, object_=self.object: + setattr(object_, key, value)) sublist.append(item) return sublist @@ -85,8 +87,8 @@ def GetSubList(self): value = self.object[key] except KeyError: continue - def setfunction(value, key=key, object=self.object): - object[key] = value + def setfunction(value, key=key, object_=self.object): + object_[key] = value item = make_objecttreeitem(f"{key!r}:", value, setfunction) sublist.append(item) return sublist @@ -111,13 +113,13 @@ def keys(self): type: ClassTreeItem, } -def make_objecttreeitem(labeltext, object, setfunction=None): - t = type(object) +def make_objecttreeitem(labeltext, object_, setfunction=None): + t = type(object_) if t in dispatch: c = dispatch[t] else: c = ObjectTreeItem - return c(labeltext, object, setfunction) + return c(labeltext, object_, setfunction) def _debug_object_browser(parent): # htest # diff --git a/Lib/idlelib/editor.py b/Lib/idlelib/editor.py index 6ad383f460c7ee..8ee8eba64367a5 100644 --- a/Lib/idlelib/editor.py +++ b/Lib/idlelib/editor.py @@ -166,8 +166,9 @@ def __init__(self, flist=None, filename=None, key=None, root=None): text.bind("<3>",self.right_menu_event) text.bind('<MouseWheel>', wheel_event) - text.bind('<Button-4>', wheel_event) - text.bind('<Button-5>', wheel_event) + if text._windowingsystem == 'x11': + text.bind('<Button-4>', wheel_event) + text.bind('<Button-5>', wheel_event) text.bind('<Configure>', self.handle_winconfig) text.bind("<<cut>>", self.cut) text.bind("<<copy>>", self.copy) diff --git a/Lib/idlelib/help.html b/Lib/idlelib/help.html index 722406b81a8ae6..827d230b54e159 100644 --- a/Lib/idlelib/help.html +++ b/Lib/idlelib/help.html @@ -1,33 +1,31 @@ - <!DOCTYPE html> <html lang="en"> <head> <meta charset="utf-8" /> - <meta name="viewport" content="width=device-width, initial-scale=1.0" /><meta name="generator" content="Docutils 0.18.1: http://docutils.sourceforge.net/" /> + <meta name="viewport" content="width=device-width, initial-scale=1.0" /><meta name="viewport" content="width=device-width, initial-scale=1" /> - <title>IDLE — Python 3.12.0a0 documentation + IDLE — Python 3.13.0a2 documentation - + + - - - + - + @@ -41,35 +39,48 @@ } } - + + + +
-

Next topic

@@ -1117,7 +1123,7 @@

Navigation

next |
  • - previous |
  • python logo
    • @@ -1130,7 +1136,7 @@

    Navigation

  • - 3.12.0a0 Documentation » + 3.13.0a2 Documentation »
  • The Python Standard Library »
    • @@ -1141,19 +1147,26 @@

    Navigation

    | +
  • + |
    • - © Copyright 2001-2022, Python Software Foundation. + © Copyright 2001-2024, Python Software Foundation.
    This page is licensed under the Python Software Foundation License Version 2.
    @@ -1167,11 +1180,11 @@

    Navigation



    - Last updated on Sep 03, 2022. + Last updated on Jan 17, 2024 (06:57 UTC). Found a bug?
    - Created using Sphinx 5.0.2. + Created using Sphinx 7.0.1.
    diff --git a/Lib/idlelib/help.py b/Lib/idlelib/help.py index 3cc7e36e35555b..bdf4b2b29f11a2 100644 --- a/Lib/idlelib/help.py +++ b/Lib/idlelib/help.py @@ -102,7 +102,7 @@ def handle_starttag(self, tag, attrs): if self.level > 0: self.nested_dl = True elif tag == 'li': - s = '\n* ' if self.simplelist else '\n\n* ' + s = '\n* ' elif tag == 'dt': s = '\n\n' if not self.nested_dl else '\n' # Avoid extra line. self.nested_dl = False @@ -241,12 +241,13 @@ def __init__(self, parent, filename, title): Toplevel.__init__(self, parent) self.wm_title(title) self.protocol("WM_DELETE_WINDOW", self.destroy) - HelpFrame(self, filename).grid(column=0, row=0, sticky='nsew') + self.frame = HelpFrame(self, filename) + self.frame.grid(column=0, row=0, sticky='nsew') self.grid_columnconfigure(0, weight=1) self.grid_rowconfigure(0, weight=1) -def copy_strip(): +def copy_strip(): # pragma: no cover """Copy idle.html to idlelib/help.html, stripping trailing whitespace. Files with trailing whitespace cannot be pushed to the git cpython @@ -279,13 +280,13 @@ def copy_strip(): print(f'{src} copied to {dst}') -def _helpwindow(parent): +def show_idlehelp(parent): "Create HelpWindow; called from Idle Help event handler." filename = join(abspath(dirname(__file__)), 'help.html') - if not isfile(filename): + if not isfile(filename): # pragma: no cover # Try copy_strip, present message. return - HelpWindow(parent, filename, 'IDLE Help (%s)' % python_version()) + return HelpWindow(parent, filename, 'IDLE Doc (%s)' % python_version()) if __name__ == '__main__': @@ -293,4 +294,4 @@ def _helpwindow(parent): main('idlelib.idle_test.test_help', verbosity=2, exit=False) from idlelib.idle_test.htest import run - run(_helpwindow) + run(show_idlehelp) diff --git a/Lib/idlelib/idle_test/htest.py b/Lib/idlelib/idle_test/htest.py index 997f85ff5a78b2..a7293774eecaeb 100644 --- a/Lib/idlelib/idle_test/htest.py +++ b/Lib/idlelib/idle_test/htest.py @@ -190,13 +190,6 @@ ", [Cancel], or [X] prints None to shell" } -_helpwindow_spec = { - 'file': 'help', - 'kwds': {}, - 'msg': "If the help text displays, this works.\n" - "Text is selectable. Window is scrollable." - } - _io_binding_spec = { 'file': 'iomenu', 'kwds': {}, @@ -299,6 +292,13 @@ "Its only action is to close." } +show_idlehelp_spec = { + 'file': 'help', + 'kwds': {}, + 'msg': "If the help text displays, this works.\n" + "Text is selectable. Window is scrollable." + } + _sidebar_number_scrolling_spec = { 'file': 'sidebar', 'kwds': {}, diff --git a/Lib/idlelib/idle_test/test_calltip.py b/Lib/idlelib/idle_test/test_calltip.py index 1ccb63b9dbd65f..28c196a42672fc 100644 --- a/Lib/idlelib/idle_test/test_calltip.py +++ b/Lib/idlelib/idle_test/test_calltip.py @@ -7,6 +7,7 @@ import types import re from idlelib.idle_test.mock_tk import Text +from test.support import MISSING_C_DOCSTRINGS # Test Class TC is used in multiple get_argspec test methods @@ -50,6 +51,8 @@ class Get_argspecTest(unittest.TestCase): # but a red buildbot is better than a user crash (as has happened). # For a simple mismatch, change the expected output to the actual. + @unittest.skipIf(MISSING_C_DOCSTRINGS, + "Signature information for builtins requires docstrings") def test_builtins(self): def tiptest(obj, out): @@ -76,6 +79,7 @@ class SB: __call__ = None tiptest(list.append, '(self, object, /)' + append_doc) tiptest(List.append, '(self, object, /)' + append_doc) tiptest([].append, '(object, /)' + append_doc) + # The use of 'object' above matches the signature text. tiptest(types.MethodType, '(function, instance, /)\n' @@ -143,6 +147,8 @@ def f(): pass f.__doc__ = 'a'*300 self.assertEqual(get_spec(f), f"()\n{'a'*(calltip._MAX_COLS-3) + '...'}") + @unittest.skipIf(MISSING_C_DOCSTRINGS, + "Signature information for builtins requires docstrings") def test_multiline_docstring(self): # Test fewer lines than max. self.assertEqual(get_spec(range), @@ -157,6 +163,7 @@ def test_multiline_docstring(self): bytes(int) -> bytes object of size given by the parameter initialized with null bytes bytes() -> empty bytes object''') + def test_multiline_docstring_2(self): # Test more than max lines def f(): pass f.__doc__ = 'a\n' * 15 diff --git a/Lib/idlelib/idle_test/test_configdialog.py b/Lib/idlelib/idle_test/test_configdialog.py index 6f8518a9bb19d0..5099d093382445 100644 --- a/Lib/idlelib/idle_test/test_configdialog.py +++ b/Lib/idlelib/idle_test/test_configdialog.py @@ -420,20 +420,14 @@ def test_highlight_target_text_mouse(self): # Set highlight_target through clicking highlight_sample. eq = self.assertEqual d = self.page - - elem = {} - count = 0 hs = d.highlight_sample hs.focus_force() - hs.see(1.0) - hs.update_idletasks() - def tag_to_element(elem): - for element, tag in d.theme_elements.items(): - elem[tag] = element - - def click_it(start): - x, y, dx, dy = hs.bbox(start) + def click_char(index): + "Simulate click on character at *index*." + hs.see(index) + hs.update_idletasks() + x, y, dx, dy = hs.bbox(index) x += dx // 2 y += dy // 2 hs.event_generate('', x=0, y=0) @@ -441,17 +435,20 @@ def click_it(start): hs.event_generate('', x=x, y=y) hs.event_generate('', x=x, y=y) - # Flip theme_elements to make the tag the key. - tag_to_element(elem) + # Reverse theme_elements to make the tag the key. + elem = {tag: element for element, tag in d.theme_elements.items()} # If highlight_sample has a tag that isn't in theme_elements, there # will be a KeyError in the test run. + count = 0 for tag in hs.tag_names(): - for start_index in hs.tag_ranges(tag)[0::2]: - count += 1 - click_it(start_index) + try: + click_char(hs.tag_nextrange(tag, "1.0")[0]) eq(d.highlight_target.get(), elem[tag]) + count += 1 eq(d.set_highlight_target.called, count) + except IndexError: + pass # Skip unused theme_elements tag, like 'sel'. def test_highlight_sample_double_click(self): # Test double click on highlight_sample. diff --git a/Lib/idlelib/idle_test/test_help.py b/Lib/idlelib/idle_test/test_help.py index b542659981894d..c528d4e77f8ca7 100644 --- a/Lib/idlelib/idle_test/test_help.py +++ b/Lib/idlelib/idle_test/test_help.py @@ -1,4 +1,4 @@ -"Test help, coverage 87%." +"Test help, coverage 94%." from idlelib import help import unittest @@ -8,25 +8,27 @@ from tkinter import Tk -class HelpFrameTest(unittest.TestCase): +class IdleDocTest(unittest.TestCase): @classmethod def setUpClass(cls): "By itself, this tests that file parsed without exception." cls.root = root = Tk() root.withdraw() - helpfile = join(dirname(dirname(abspath(__file__))), 'help.html') - cls.frame = help.HelpFrame(root, helpfile) + cls.window = help.show_idlehelp(root) @classmethod def tearDownClass(cls): - del cls.frame + del cls.window cls.root.update_idletasks() cls.root.destroy() del cls.root - def test_line1(self): - text = self.frame.text + def test_1window(self): + self.assertIn('IDLE Doc', self.window.wm_title()) + + def test_4text(self): + text = self.window.frame.text self.assertEqual(text.get('1.0', '1.end'), ' IDLE ') diff --git a/Lib/idlelib/idle_test/test_sidebar.py b/Lib/idlelib/idle_test/test_sidebar.py index fb52b3a0179553..605e7a892570d7 100644 --- a/Lib/idlelib/idle_test/test_sidebar.py +++ b/Lib/idlelib/idle_test/test_sidebar.py @@ -690,16 +690,22 @@ def test_mousewheel(self): last_lineno = get_end_linenumber(text) self.assertIsNotNone(text.dlineinfo(text.index(f'{last_lineno}.0'))) - # Scroll up using the event. - # The meaning of delta is platform-dependent. - delta = -1 if sys.platform == 'darwin' else 120 - sidebar.canvas.event_generate('', x=0, y=0, delta=delta) + # Delta for , whose meaning is platform-dependent. + delta = 1 if sidebar.canvas._windowingsystem == 'aqua' else 120 + + # Scroll up. + if sidebar.canvas._windowingsystem == 'x11': + sidebar.canvas.event_generate('', x=0, y=0) + else: + sidebar.canvas.event_generate('', x=0, y=0, delta=delta) yield - if sys.platform != 'darwin': # .update_idletasks() does not work. - self.assertIsNone(text.dlineinfo(text.index(f'{last_lineno}.0'))) + self.assertIsNone(text.dlineinfo(text.index(f'{last_lineno}.0'))) - # Scroll back down using the event. - sidebar.canvas.event_generate('', x=0, y=0) + # Scroll back down. + if sidebar.canvas._windowingsystem == 'x11': + sidebar.canvas.event_generate('', x=0, y=0) + else: + sidebar.canvas.event_generate('', x=0, y=0, delta=-delta) yield self.assertIsNotNone(text.dlineinfo(text.index(f'{last_lineno}.0'))) diff --git a/Lib/idlelib/replace.py b/Lib/idlelib/replace.py index 7997f24f1b0fa6..3716d841568d30 100644 --- a/Lib/idlelib/replace.py +++ b/Lib/idlelib/replace.py @@ -25,7 +25,8 @@ def replace(text, insert_tags=None): if not hasattr(engine, "_replacedialog"): engine._replacedialog = ReplaceDialog(root, engine) dialog = engine._replacedialog - dialog.open(text, insert_tags=insert_tags) + searchphrase = text.get("sel.first", "sel.last") + dialog.open(text, searchphrase, insert_tags=insert_tags) class ReplaceDialog(SearchDialogBase): @@ -51,27 +52,17 @@ def __init__(self, root, engine): self.replvar = StringVar(root) self.insert_tags = None - def open(self, text, insert_tags=None): + def open(self, text, searchphrase=None, *, insert_tags=None): """Make dialog visible on top of others and ready to use. - Also, highlight the currently selected text and set the - search to include the current selection (self.ok). + Also, set the search to include the current selection + (self.ok). Args: text: Text widget being searched. + searchphrase: String phrase to search. """ - SearchDialogBase.open(self, text) - try: - first = text.index("sel.first") - except TclError: - first = None - try: - last = text.index("sel.last") - except TclError: - last = None - first = first or text.index("insert") - last = last or first - self.show_hit(first, last) + SearchDialogBase.open(self, text, searchphrase) self.ok = True self.insert_tags = insert_tags diff --git a/Lib/idlelib/rpc.py b/Lib/idlelib/rpc.py index b08b80c9004551..3f0b2230dd185d 100644 --- a/Lib/idlelib/rpc.py +++ b/Lib/idlelib/rpc.py @@ -158,8 +158,8 @@ def debug(self, *args): s = s + " " + str(a) print(s, file=sys.__stderr__) - def register(self, oid, object): - self.objtable[oid] = object + def register(self, oid, object_): + self.objtable[oid] = object_ def unregister(self, oid): try: diff --git a/Lib/idlelib/stackviewer.py b/Lib/idlelib/stackviewer.py index 977c56ef15f2ae..95042d4debdc03 100644 --- a/Lib/idlelib/stackviewer.py +++ b/Lib/idlelib/stackviewer.py @@ -106,8 +106,8 @@ def GetSubList(self): value = self.object[key] except KeyError: continue - def setfunction(value, key=key, object=self.object): - object[key] = value + def setfunction(value, key=key, object_=self.object): + object_[key] = value item = make_objecttreeitem(key + " =", value, setfunction) sublist.append(item) return sublist diff --git a/Lib/idlelib/tree.py b/Lib/idlelib/tree.py index 9c2eb47b24aec9..0726d7e23660f6 100644 --- a/Lib/idlelib/tree.py +++ b/Lib/idlelib/tree.py @@ -285,8 +285,9 @@ def drawtext(self): self.label.bind("<1>", self.select_or_edit) self.label.bind("", self.flip) self.label.bind("", lambda e: wheel_event(e, self.canvas)) - self.label.bind("", lambda e: wheel_event(e, self.canvas)) - self.label.bind("", lambda e: wheel_event(e, self.canvas)) + if self.label._windowingsystem == 'x11': + self.label.bind("", lambda e: wheel_event(e, self.canvas)) + self.label.bind("", lambda e: wheel_event(e, self.canvas)) self.text_id = id def select_or_edit(self, event=None): @@ -460,8 +461,9 @@ def __init__(self, master, **opts): self.canvas.bind("", self.unit_up) self.canvas.bind("", self.unit_down) self.canvas.bind("", wheel_event) - self.canvas.bind("", wheel_event) - self.canvas.bind("", wheel_event) + if self.canvas._windowingsystem == 'x11': + self.canvas.bind("", wheel_event) + self.canvas.bind("", wheel_event) #if isinstance(master, Toplevel) or isinstance(master, Tk): self.canvas.bind("", self.zoom_height) self.canvas.focus_set() diff --git a/Lib/importlib/_bootstrap.py b/Lib/importlib/_bootstrap.py index d942045f3de666..6d6292f9559253 100644 --- a/Lib/importlib/_bootstrap.py +++ b/Lib/importlib/_bootstrap.py @@ -53,7 +53,7 @@ def _new_module(name): # For a list that can have a weakref to it. class _List(list): - pass + __slots__ = ("__weakref__",) # Copied from weakref.py with some simplifications and modifications unique to diff --git a/Lib/importlib/_bootstrap_external.py b/Lib/importlib/_bootstrap_external.py index d537598ac16433..2a9aef03179f6f 100644 --- a/Lib/importlib/_bootstrap_external.py +++ b/Lib/importlib/_bootstrap_external.py @@ -462,6 +462,9 @@ def _write_atomic(path, data, mode=0o666): # Python 3.13a1 3563 (Add CALL_KW and remove KW_NAMES) # Python 3.13a1 3564 (Removed oparg from YIELD_VALUE, changed oparg values of RESUME) # Python 3.13a1 3565 (Oparg of YIELD_VALUE indicates whether it is in a yield-from) +# Python 3.13a1 3566 (Emit JUMP_NO_INTERRUPT instead of JUMP for non-loop no-lineno cases) +# Python 3.13a1 3567 (Reimplement line number propagation by the compiler) +# Python 3.13a1 3568 (Change semantics of END_FOR) # Python 3.14 will start with 3600 @@ -478,7 +481,7 @@ def _write_atomic(path, data, mode=0o666): # Whenever MAGIC_NUMBER is changed, the ranges in the magic_values array # in PC/launcher.c must also be updated. -MAGIC_NUMBER = (3565).to_bytes(2, 'little') + b'\r\n' +MAGIC_NUMBER = (3568).to_bytes(2, 'little') + b'\r\n' _RAW_MAGIC_NUMBER = int.from_bytes(MAGIC_NUMBER, 'little') # For import.c diff --git a/Lib/importlib/metadata/__init__.py b/Lib/importlib/metadata/__init__.py index 7b142e786e829e..c612fbefee2e80 100644 --- a/Lib/importlib/metadata/__init__.py +++ b/Lib/importlib/metadata/__init__.py @@ -1,7 +1,6 @@ import os import re import abc -import csv import sys import json import email @@ -478,6 +477,10 @@ def make_file(name, hash=None, size_str=None): @pass_none def make_files(lines): + # Delay csv import, since Distribution.files is not as widely used + # as other parts of importlib.metadata + import csv + return starmap(make_file, csv.reader(lines)) @pass_none diff --git a/Lib/importlib/resources/__init__.py b/Lib/importlib/resources/__init__.py index e6b60c18caa052..ae83cd07c4d4fb 100644 --- a/Lib/importlib/resources/__init__.py +++ b/Lib/importlib/resources/__init__.py @@ -4,6 +4,7 @@ as_file, files, Package, + Anchor, ) from .abc import ResourceReader @@ -11,6 +12,7 @@ __all__ = [ 'Package', + 'Anchor', 'ResourceReader', 'as_file', 'files', diff --git a/Lib/json/tool.py b/Lib/json/tool.py index 0490b8c0be11df..fdfc3372bcca02 100644 --- a/Lib/json/tool.py +++ b/Lib/json/tool.py @@ -13,7 +13,6 @@ import argparse import json import sys -from pathlib import Path def main(): @@ -22,11 +21,9 @@ def main(): 'to validate and pretty-print JSON objects.') parser = argparse.ArgumentParser(prog=prog, description=description) parser.add_argument('infile', nargs='?', - type=argparse.FileType(encoding="utf-8"), help='a JSON file to be validated or pretty-printed', - default=sys.stdin) + default='-') parser.add_argument('outfile', nargs='?', - type=Path, help='write the output of infile to outfile', default=None) parser.add_argument('--sort-keys', action='store_true', default=False, @@ -59,23 +56,30 @@ def main(): dump_args['indent'] = None dump_args['separators'] = ',', ':' - with options.infile as infile: + try: + if options.infile == '-': + infile = sys.stdin + else: + infile = open(options.infile, encoding='utf-8') try: if options.json_lines: objs = (json.loads(line) for line in infile) else: objs = (json.load(infile),) + finally: + if infile is not sys.stdin: + infile.close() - if options.outfile is None: - out = sys.stdout - else: - out = options.outfile.open('w', encoding='utf-8') - with out as outfile: - for obj in objs: - json.dump(obj, outfile, **dump_args) - outfile.write('\n') - except ValueError as e: - raise SystemExit(e) + if options.outfile is None: + outfile = sys.stdout + else: + outfile = open(options.outfile, 'w', encoding='utf-8') + with outfile: + for obj in objs: + json.dump(obj, outfile, **dump_args) + outfile.write('\n') + except ValueError as e: + raise SystemExit(e) if __name__ == '__main__': diff --git a/Lib/logging/__init__.py b/Lib/logging/__init__.py index eb7e020d1edfc0..684b58d5548f91 100644 --- a/Lib/logging/__init__.py +++ b/Lib/logging/__init__.py @@ -1493,7 +1493,7 @@ def debug(self, msg, *args, **kwargs): To pass exception information, use the keyword argument exc_info with a true value, e.g. - logger.debug("Houston, we have a %s", "thorny problem", exc_info=1) + logger.debug("Houston, we have a %s", "thorny problem", exc_info=True) """ if self.isEnabledFor(DEBUG): self._log(DEBUG, msg, args, **kwargs) @@ -1505,7 +1505,7 @@ def info(self, msg, *args, **kwargs): To pass exception information, use the keyword argument exc_info with a true value, e.g. - logger.info("Houston, we have a %s", "notable problem", exc_info=1) + logger.info("Houston, we have a %s", "notable problem", exc_info=True) """ if self.isEnabledFor(INFO): self._log(INFO, msg, args, **kwargs) @@ -1517,7 +1517,7 @@ def warning(self, msg, *args, **kwargs): To pass exception information, use the keyword argument exc_info with a true value, e.g. - logger.warning("Houston, we have a %s", "bit of a problem", exc_info=1) + logger.warning("Houston, we have a %s", "bit of a problem", exc_info=True) """ if self.isEnabledFor(WARNING): self._log(WARNING, msg, args, **kwargs) @@ -1529,7 +1529,7 @@ def error(self, msg, *args, **kwargs): To pass exception information, use the keyword argument exc_info with a true value, e.g. - logger.error("Houston, we have a %s", "major problem", exc_info=1) + logger.error("Houston, we have a %s", "major problem", exc_info=True) """ if self.isEnabledFor(ERROR): self._log(ERROR, msg, args, **kwargs) @@ -1547,7 +1547,7 @@ def critical(self, msg, *args, **kwargs): To pass exception information, use the keyword argument exc_info with a true value, e.g. - logger.critical("Houston, we have a %s", "major disaster", exc_info=1) + logger.critical("Houston, we have a %s", "major disaster", exc_info=True) """ if self.isEnabledFor(CRITICAL): self._log(CRITICAL, msg, args, **kwargs) @@ -1565,7 +1565,7 @@ def log(self, level, msg, *args, **kwargs): To pass exception information, use the keyword argument exc_info with a true value, e.g. - logger.log(level, "We have a %s", "mysterious problem", exc_info=1) + logger.log(level, "We have a %s", "mysterious problem", exc_info=True) """ if not isinstance(level, int): if raiseExceptions: diff --git a/Lib/logging/config.py b/Lib/logging/config.py index 4b520e3b1e0da6..de06090942d965 100644 --- a/Lib/logging/config.py +++ b/Lib/logging/config.py @@ -1,4 +1,4 @@ -# Copyright 2001-2022 by Vinay Sajip. All Rights Reserved. +# Copyright 2001-2023 by Vinay Sajip. All Rights Reserved. # # Permission to use, copy, modify, and distribute this software and its # documentation for any purpose and without fee is hereby granted, @@ -734,7 +734,7 @@ def _configure_queue_handler(self, klass, **kwargs): lklass = kwargs['listener'] else: lklass = logging.handlers.QueueListener - listener = lklass(q, *kwargs['handlers'], respect_handler_level=rhl) + listener = lklass(q, *kwargs.get('handlers', []), respect_handler_level=rhl) handler = klass(q) handler.listener = listener return handler @@ -776,11 +776,12 @@ def configure_handler(self, config): raise ValueError('Unable to set target handler %r' % tn) from e elif issubclass(klass, logging.handlers.QueueHandler): # Another special case for handler which refers to other handlers - if 'handlers' not in config: - raise ValueError('No handlers specified for a QueueHandler') + # if 'handlers' not in config: + # raise ValueError('No handlers specified for a QueueHandler') if 'queue' in config: + from multiprocessing.queues import Queue as MPQueue qspec = config['queue'] - if not isinstance(qspec, queue.Queue): + if not isinstance(qspec, (queue.Queue, MPQueue)): if isinstance(qspec, str): q = self.resolve(qspec) if not callable(q): @@ -813,18 +814,19 @@ def configure_handler(self, config): if not callable(listener): raise TypeError('Invalid listener specifier %r' % lspec) config['listener'] = listener - hlist = [] - try: - for hn in config['handlers']: - h = self.config['handlers'][hn] - if not isinstance(h, logging.Handler): - config.update(config_copy) # restore for deferred cfg - raise TypeError('Required handler %r ' - 'is not configured yet' % hn) - hlist.append(h) - except Exception as e: - raise ValueError('Unable to set required handler %r' % hn) from e - config['handlers'] = hlist + if 'handlers' in config: + hlist = [] + try: + for hn in config['handlers']: + h = self.config['handlers'][hn] + if not isinstance(h, logging.Handler): + config.update(config_copy) # restore for deferred cfg + raise TypeError('Required handler %r ' + 'is not configured yet' % hn) + hlist.append(h) + except Exception as e: + raise ValueError('Unable to set required handler %r' % hn) from e + config['handlers'] = hlist elif issubclass(klass, logging.handlers.SMTPHandler) and\ 'mailhost' in config: config['mailhost'] = self.as_tuple(config['mailhost']) diff --git a/Lib/logging/handlers.py b/Lib/logging/handlers.py index 9840b7b0aeba88..e7f1322e4ba3d9 100644 --- a/Lib/logging/handlers.py +++ b/Lib/logging/handlers.py @@ -1586,6 +1586,7 @@ def stop(self): Note that if you don't call this before your application exits, there may be some records still left on the queue, which won't be processed. """ - self.enqueue_sentinel() - self._thread.join() - self._thread = None + if self._thread: # see gh-114706 - allow calling this more than once + self.enqueue_sentinel() + self._thread.join() + self._thread = None diff --git a/Lib/mailbox.py b/Lib/mailbox.py index 36afaded705d0a..746811bd559412 100644 --- a/Lib/mailbox.py +++ b/Lib/mailbox.py @@ -590,6 +590,8 @@ def _refresh(self): for subdir in self._toc_mtimes: path = self._paths[subdir] for entry in os.listdir(path): + if entry.startswith('.'): + continue p = os.path.join(path, entry) if os.path.isdir(p): continue @@ -828,10 +830,11 @@ def get_message(self, key): """Return a Message representation or raise a KeyError.""" start, stop = self._lookup(key) self._file.seek(start) - from_line = self._file.readline().replace(linesep, b'') + from_line = self._file.readline().replace(linesep, b'').decode('ascii') string = self._file.read(stop - self._file.tell()) msg = self._message_factory(string.replace(linesep, b'\n')) - msg.set_from(from_line[5:].decode('ascii')) + msg.set_unixfrom(from_line) + msg.set_from(from_line[5:]) return msg def get_string(self, key, from_=False): @@ -1139,10 +1142,24 @@ def __len__(self): """Return a count of messages in the mailbox.""" return len(list(self.iterkeys())) + def _open_mh_sequences_file(self, text): + mode = '' if text else 'b' + kwargs = {'encoding': 'ASCII'} if text else {} + path = os.path.join(self._path, '.mh_sequences') + while True: + try: + return open(path, 'r+' + mode, **kwargs) + except FileNotFoundError: + pass + try: + return open(path, 'x+' + mode, **kwargs) + except FileExistsError: + pass + def lock(self): """Lock the mailbox.""" if not self._locked: - self._file = open(os.path.join(self._path, '.mh_sequences'), 'rb+') + self._file = self._open_mh_sequences_file(text=False) _lock_file(self._file) self._locked = True @@ -1196,7 +1213,11 @@ def remove_folder(self, folder): def get_sequences(self): """Return a name-to-key-list dictionary to define each sequence.""" results = {} - with open(os.path.join(self._path, '.mh_sequences'), 'r', encoding='ASCII') as f: + try: + f = open(os.path.join(self._path, '.mh_sequences'), 'r', encoding='ASCII') + except FileNotFoundError: + return results + with f: all_keys = set(self.keys()) for line in f: try: @@ -1219,7 +1240,7 @@ def get_sequences(self): def set_sequences(self, sequences): """Set sequences using the given name-to-key-list dictionary.""" - f = open(os.path.join(self._path, '.mh_sequences'), 'r+', encoding='ASCII') + f = self._open_mh_sequences_file(text=True) try: os.close(os.open(f.name, os.O_WRONLY | os.O_TRUNC)) for name, keys in sequences.items(): diff --git a/Lib/mimetypes.py b/Lib/mimetypes.py index 37228de4828de5..51b99701c9d727 100644 --- a/Lib/mimetypes.py +++ b/Lib/mimetypes.py @@ -528,6 +528,7 @@ def _default_mime_types(): '.tiff' : 'image/tiff', '.tif' : 'image/tiff', '.ico' : 'image/vnd.microsoft.icon', + '.webp' : 'image/webp', '.ras' : 'image/x-cmu-raster', '.pnm' : 'image/x-portable-anymap', '.pbm' : 'image/x-portable-bitmap', @@ -587,7 +588,6 @@ def _default_mime_types(): '.pict': 'image/pict', '.pct' : 'image/pict', '.pic' : 'image/pict', - '.webp': 'image/webp', '.xul' : 'text/xul', } diff --git a/Lib/multiprocessing/managers.py b/Lib/multiprocessing/managers.py index 96cebc6eabec89..76b915de74d94e 100644 --- a/Lib/multiprocessing/managers.py +++ b/Lib/multiprocessing/managers.py @@ -156,7 +156,7 @@ def __init__(self, registry, address, authkey, serializer): Listener, Client = listener_client[serializer] # do authentication later - self.listener = Listener(address=address, backlog=16) + self.listener = Listener(address=address, backlog=128) self.address = self.listener.address self.id_to_obj = {'0': (None, ())} diff --git a/Lib/multiprocessing/resource_sharer.py b/Lib/multiprocessing/resource_sharer.py index 66076509a1202e..b8afb0fbed3a3c 100644 --- a/Lib/multiprocessing/resource_sharer.py +++ b/Lib/multiprocessing/resource_sharer.py @@ -123,7 +123,7 @@ def _start(self): from .connection import Listener assert self._listener is None, "Already have Listener" util.debug('starting listener and thread for sending handles') - self._listener = Listener(authkey=process.current_process().authkey) + self._listener = Listener(authkey=process.current_process().authkey, backlog=128) self._address = self._listener.address t = threading.Thread(target=self._serve) t.daemon = True diff --git a/Lib/multiprocessing/util.py b/Lib/multiprocessing/util.py index 28c77df1c32ea8..32871850ddec8b 100644 --- a/Lib/multiprocessing/util.py +++ b/Lib/multiprocessing/util.py @@ -43,19 +43,19 @@ def sub_debug(msg, *args): if _logger: - _logger.log(SUBDEBUG, msg, *args) + _logger.log(SUBDEBUG, msg, *args, stacklevel=2) def debug(msg, *args): if _logger: - _logger.log(DEBUG, msg, *args) + _logger.log(DEBUG, msg, *args, stacklevel=2) def info(msg, *args): if _logger: - _logger.log(INFO, msg, *args) + _logger.log(INFO, msg, *args, stacklevel=2) def sub_warning(msg, *args): if _logger: - _logger.log(SUBWARNING, msg, *args) + _logger.log(SUBWARNING, msg, *args, stacklevel=2) def get_logger(): ''' diff --git a/Lib/ntpath.py b/Lib/ntpath.py index 3061a4a5ef4c56..e7cbfe17ecb3c8 100644 --- a/Lib/ntpath.py +++ b/Lib/ntpath.py @@ -26,8 +26,8 @@ __all__ = ["normcase","isabs","join","splitdrive","splitroot","split","splitext", "basename","dirname","commonprefix","getsize","getmtime", "getatime","getctime", "islink","exists","lexists","isdir","isfile", - "ismount", "expanduser","expandvars","normpath","abspath", - "curdir","pardir","sep","pathsep","defpath","altsep", + "ismount","isreserved","expanduser","expandvars","normpath", + "abspath","curdir","pardir","sep","pathsep","defpath","altsep", "extsep","devnull","realpath","supports_unicode_filenames","relpath", "samefile", "sameopenfile", "samestat", "commonpath", "isjunction"] @@ -77,12 +77,6 @@ def normcase(s): return s.replace('/', '\\').lower() -# Return whether a path is absolute. -# Trivial in Posix, harder on Windows. -# For Windows it is absolute if it starts with a slash or backslash (current -# volume), or if a pathname after the volume-letter-and-colon or UNC-resource -# starts with a slash or backslash. - def isabs(s): """Test whether a path is absolute""" s = os.fspath(s) @@ -90,16 +84,15 @@ def isabs(s): sep = b'\\' altsep = b'/' colon_sep = b':\\' + double_sep = b'\\\\' else: sep = '\\' altsep = '/' colon_sep = ':\\' + double_sep = '\\\\' s = s[:3].replace(altsep, sep) # Absolute: UNC, device, and paths with a drive and root. - # LEGACY BUG: isabs("/x") should be false since the path has no drive. - if s.startswith(sep) or s.startswith(colon_sep, 1): - return True - return False + return s.startswith(colon_sep, 1) or s.startswith(double_sep) # Join two (or more) paths. @@ -337,6 +330,42 @@ def ismount(path): return False +_reserved_chars = frozenset( + {chr(i) for i in range(32)} | + {'"', '*', ':', '<', '>', '?', '|', '/', '\\'} +) + +_reserved_names = frozenset( + {'CON', 'PRN', 'AUX', 'NUL', 'CONIN$', 'CONOUT$'} | + {f'COM{c}' for c in '123456789\xb9\xb2\xb3'} | + {f'LPT{c}' for c in '123456789\xb9\xb2\xb3'} +) + +def isreserved(path): + """Return true if the pathname is reserved by the system.""" + # Refer to "Naming Files, Paths, and Namespaces": + # https://docs.microsoft.com/en-us/windows/win32/fileio/naming-a-file + path = os.fsdecode(splitroot(path)[2]).replace(altsep, sep) + return any(_isreservedname(name) for name in reversed(path.split(sep))) + +def _isreservedname(name): + """Return true if the filename is reserved by the system.""" + # Trailing dots and spaces are reserved. + if name.endswith(('.', ' ')) and name not in ('.', '..'): + return True + # Wildcards, separators, colon, and pipe (*?"<>/\:|) are reserved. + # ASCII control characters (0-31) are reserved. + # Colon is reserved for file streams (e.g. "name:stream[:type]"). + if _reserved_chars.intersection(name): + return True + # DOS device names are reserved (e.g. "nul" or "nul .txt"). The rules + # are complex and vary across Windows versions. On the side of + # caution, return True for names that may not be reserved. + if name.partition('.')[0].rstrip(' ').upper() in _reserved_names: + return True + return False + + # Expand paths beginning with '~' or '~user'. # '~' means $HOME; '~user' means that user's home directory. # If the path doesn't begin with '~', or if the user or $HOME is unknown, diff --git a/Lib/os.py b/Lib/os.py index 8c4b93250918eb..7f38e14e7bdd96 100644 --- a/Lib/os.py +++ b/Lib/os.py @@ -131,6 +131,7 @@ def _add(str, fn): _set = set() _add("HAVE_FCHDIR", "chdir") _add("HAVE_FCHMOD", "chmod") + _add("MS_WINDOWS", "chmod") _add("HAVE_FCHOWN", "chown") _add("HAVE_FDOPENDIR", "listdir") _add("HAVE_FDOPENDIR", "scandir") diff --git a/Lib/pathlib/__init__.py b/Lib/pathlib/__init__.py index b020d2db350da8..65ce836765c42b 100644 --- a/Lib/pathlib/__init__.py +++ b/Lib/pathlib/__init__.py @@ -9,6 +9,10 @@ import ntpath import os import posixpath +import sys +import warnings +from itertools import chain +from _collections_abc import Sequence try: import pwd @@ -29,6 +33,35 @@ ] +class _PathParents(Sequence): + """This object provides sequence-like access to the logical ancestors + of a path. Don't try to construct it yourself.""" + __slots__ = ('_path', '_drv', '_root', '_tail') + + def __init__(self, path): + self._path = path + self._drv = path.drive + self._root = path.root + self._tail = path._tail + + def __len__(self): + return len(self._tail) + + def __getitem__(self, idx): + if isinstance(idx, slice): + return tuple(self[i] for i in range(*idx.indices(len(self)))) + + if idx >= len(self) or idx < -len(self): + raise IndexError(idx) + if idx < 0: + idx += len(self) + return self._path._from_parsed_parts(self._drv, self._root, + self._tail[:-idx - 1]) + + def __repr__(self): + return "<{}.parents>".format(type(self._path).__name__) + + UnsupportedOperation = _abc.UnsupportedOperation @@ -43,6 +76,24 @@ class PurePath(_abc.PurePathBase): """ __slots__ = ( + # The `_raw_paths` slot stores unnormalized string paths. This is set + # in the `__init__()` method. + '_raw_paths', + + # The `_drv`, `_root` and `_tail_cached` slots store parsed and + # normalized parts of the path. They are set when any of the `drive`, + # `root` or `_tail` properties are accessed for the first time. The + # three-part division corresponds to the result of + # `os.path.splitroot()`, except that the tail is further split on path + # separators (i.e. it is a list of strings), and that the root and + # tail are normalized. + '_drv', '_root', '_tail_cached', + + # The `_str` slot stores the string representation of the path, + # computed from the drive, root and tail when `__str__()` is called + # for the first time. It's used to implement `_str_normcase` + '_str', + # The `_str_normcase_cached` slot stores the string path with # normalized case. It is set when the `_str_normcase` property is # accessed for the first time. It's used to implement `__eq__()` @@ -59,6 +110,7 @@ class PurePath(_abc.PurePathBase): # path. It's set when `__hash__()` is called for the first time. '_hash', ) + pathmod = os.path def __new__(cls, *args, **kwargs): """Construct a PurePath from one or several strings and or existing @@ -92,13 +144,35 @@ def __init__(self, *args): paths.append(path) # Avoid calling super().__init__, as an optimisation self._raw_paths = paths - self._resolving = False + + def joinpath(self, *pathsegments): + """Combine this path with one or several arguments, and return a + new path representing either a subpath (if all arguments are relative + paths) or a totally different path (if one of the arguments is + anchored). + """ + return self.with_segments(self, *pathsegments) + + def __truediv__(self, key): + try: + return self.with_segments(self, key) + except TypeError: + return NotImplemented + + def __rtruediv__(self, key): + try: + return self.with_segments(key, self) + except TypeError: + return NotImplemented def __reduce__(self): # Using the parts tuple helps share interned path parts # when pickling related paths. return (self.__class__, self.parts) + def __repr__(self): + return "{}({!r})".format(self.__class__.__name__, self.as_posix()) + def __fspath__(self): return str(self) @@ -160,6 +234,204 @@ def __ge__(self, other): return NotImplemented return self._parts_normcase >= other._parts_normcase + def __str__(self): + """Return the string representation of the path, suitable for + passing to system calls.""" + try: + return self._str + except AttributeError: + self._str = self._format_parsed_parts(self.drive, self.root, + self._tail) or '.' + return self._str + + @classmethod + def _format_parsed_parts(cls, drv, root, tail): + if drv or root: + return drv + root + cls.pathmod.sep.join(tail) + elif tail and cls.pathmod.splitdrive(tail[0])[0]: + tail = ['.'] + tail + return cls.pathmod.sep.join(tail) + + def _from_parsed_parts(self, drv, root, tail): + path_str = self._format_parsed_parts(drv, root, tail) + path = self.with_segments(path_str) + path._str = path_str or '.' + path._drv = drv + path._root = root + path._tail_cached = tail + return path + + @classmethod + def _parse_path(cls, path): + if not path: + return '', '', [] + sep = cls.pathmod.sep + altsep = cls.pathmod.altsep + if altsep: + path = path.replace(altsep, sep) + drv, root, rel = cls.pathmod.splitroot(path) + if not root and drv.startswith(sep) and not drv.endswith(sep): + drv_parts = drv.split(sep) + if len(drv_parts) == 4 and drv_parts[2] not in '?.': + # e.g. //server/share + root = sep + elif len(drv_parts) == 6: + # e.g. //?/unc/server/share + root = sep + parsed = [sys.intern(str(x)) for x in rel.split(sep) if x and x != '.'] + return drv, root, parsed + + @property + def _raw_path(self): + """The joined but unnormalized path.""" + paths = self._raw_paths + if len(paths) == 0: + path = '' + elif len(paths) == 1: + path = paths[0] + else: + path = self.pathmod.join(*paths) + return path + + @property + def drive(self): + """The drive prefix (letter or UNC path), if any.""" + try: + return self._drv + except AttributeError: + self._drv, self._root, self._tail_cached = self._parse_path(self._raw_path) + return self._drv + + @property + def root(self): + """The root of the path, if any.""" + try: + return self._root + except AttributeError: + self._drv, self._root, self._tail_cached = self._parse_path(self._raw_path) + return self._root + + @property + def _tail(self): + try: + return self._tail_cached + except AttributeError: + self._drv, self._root, self._tail_cached = self._parse_path(self._raw_path) + return self._tail_cached + + @property + def anchor(self): + """The concatenation of the drive and root, or ''.""" + return self.drive + self.root + + @property + def parts(self): + """An object providing sequence-like access to the + components in the filesystem path.""" + if self.drive or self.root: + return (self.drive + self.root,) + tuple(self._tail) + else: + return tuple(self._tail) + + @property + def parent(self): + """The logical parent of the path.""" + drv = self.drive + root = self.root + tail = self._tail + if not tail: + return self + return self._from_parsed_parts(drv, root, tail[:-1]) + + @property + def parents(self): + """A sequence of this path's logical parents.""" + # The value of this property should not be cached on the path object, + # as doing so would introduce a reference cycle. + return _PathParents(self) + + @property + def name(self): + """The final path component, if any.""" + tail = self._tail + if not tail: + return '' + return tail[-1] + + def with_name(self, name): + """Return a new path with the file name changed.""" + m = self.pathmod + if not name or m.sep in name or (m.altsep and m.altsep in name) or name == '.': + raise ValueError(f"Invalid name {name!r}") + tail = self._tail.copy() + if not tail: + raise ValueError(f"{self!r} has an empty name") + tail[-1] = name + return self._from_parsed_parts(self.drive, self.root, tail) + + def relative_to(self, other, /, *_deprecated, walk_up=False): + """Return the relative path to another path identified by the passed + arguments. If the operation is not possible (because this is not + related to the other path), raise ValueError. + + The *walk_up* parameter controls whether `..` may be used to resolve + the path. + """ + if _deprecated: + msg = ("support for supplying more than one positional argument " + "to pathlib.PurePath.relative_to() is deprecated and " + "scheduled for removal in Python 3.14") + warnings.warn(msg, DeprecationWarning, stacklevel=2) + other = self.with_segments(other, *_deprecated) + elif not isinstance(other, PurePath): + other = self.with_segments(other) + for step, path in enumerate(chain([other], other.parents)): + if path == self or path in self.parents: + break + elif not walk_up: + raise ValueError(f"{str(self)!r} is not in the subpath of {str(other)!r}") + elif path.name == '..': + raise ValueError(f"'..' segment in {str(other)!r} cannot be walked") + else: + raise ValueError(f"{str(self)!r} and {str(other)!r} have different anchors") + parts = ['..'] * step + self._tail[len(path._tail):] + return self._from_parsed_parts('', '', parts) + + def is_relative_to(self, other, /, *_deprecated): + """Return True if the path is relative to another path or False. + """ + if _deprecated: + msg = ("support for supplying more than one argument to " + "pathlib.PurePath.is_relative_to() is deprecated and " + "scheduled for removal in Python 3.14") + warnings.warn(msg, DeprecationWarning, stacklevel=2) + other = self.with_segments(other, *_deprecated) + elif not isinstance(other, PurePath): + other = self.with_segments(other) + return other == self or other in self.parents + + def is_absolute(self): + """True if the path is absolute (has both a root and, if applicable, + a drive).""" + if self.pathmod is posixpath: + # Optimization: work with raw paths on POSIX. + for path in self._raw_paths: + if path.startswith('/'): + return True + return False + return self.pathmod.isabs(self) + + def is_reserved(self): + """Return True if the path contains one of the special names reserved + by the system, if any.""" + msg = ("pathlib.PurePath.is_reserved() is deprecated and scheduled " + "for removal in Python 3.15. Use os.path.isreserved() to " + "detect reserved paths on Windows.") + warnings.warn(msg, DeprecationWarning, stacklevel=2) + if self.pathmod is ntpath: + return self.pathmod.isreserved(self) + return False + def as_uri(self): """Return the path as a URI.""" if not self.is_absolute(): @@ -181,6 +453,28 @@ def as_uri(self): from urllib.parse import quote_from_bytes return prefix + quote_from_bytes(os.fsencode(path)) + @property + def _pattern_stack(self): + """Stack of path components, to be used with patterns in glob().""" + parts = self._tail.copy() + pattern = self._raw_path + if self.anchor: + raise NotImplementedError("Non-relative patterns are unsupported") + elif not parts: + raise ValueError("Unacceptable pattern: {!r}".format(pattern)) + elif pattern[-1] in (self.pathmod.sep, self.pathmod.altsep): + # GH-65238: pathlib doesn't preserve trailing slash. Add it back. + parts.append('') + parts.reverse() + return parts + + @property + def _pattern_str(self): + """The path expressed as a string, for use in pattern-matching.""" + # The string representation of an empty path is a single dot ('.'). Empty + # paths shouldn't match wildcards, so we change it to the empty string. + path_str = str(self) + return '' if path_str == '.' else path_str # Subclassing os.PathLike makes isinstance() checks slower, # which in turn makes Path construction slower. Register instead! @@ -220,13 +514,11 @@ class Path(_abc.PathBase, PurePath): as_uri = PurePath.as_uri @classmethod - def _unsupported(cls, method_name): - msg = f"{cls.__name__}.{method_name}() is unsupported on this system" - raise UnsupportedOperation(msg) + def _unsupported_msg(cls, attribute): + return f"{cls.__name__}.{attribute} is unsupported on this system" def __init__(self, *args, **kwargs): if kwargs: - import warnings msg = ("support for supplying keyword arguments to pathlib.PurePath " "is deprecated and scheduled for removal in Python {remove}") warnings._deprecated("pathlib.PurePath(**kwargs)", msg, remove=(3, 14)) @@ -266,6 +558,24 @@ def open(self, mode='r', buffering=-1, encoding=None, encoding = io.text_encoding(encoding) return io.open(self, mode, buffering, encoding, errors, newline) + def read_text(self, encoding=None, errors=None, newline=None): + """ + Open the file in text mode, read it, and close the file. + """ + # Call io.text_encoding() here to ensure any warning is raised at an + # appropriate stack level. + encoding = io.text_encoding(encoding) + return _abc.PathBase.read_text(self, encoding, errors, newline) + + def write_text(self, data, encoding=None, errors=None, newline=None): + """ + Open the file in text mode, write to it, and close the file. + """ + # Call io.text_encoding() here to ensure any warning is raised at an + # appropriate stack level. + encoding = io.text_encoding(encoding) + return _abc.PathBase.write_text(self, data, encoding, errors, newline) + def iterdir(self): """Yield path objects of the directory contents. @@ -277,6 +587,62 @@ def iterdir(self): def _scandir(self): return os.scandir(self) + def _make_child_entry(self, entry): + # Transform an entry yielded from _scandir() into a path object. + path_str = entry.name if str(self) == '.' else entry.path + path = self.with_segments(path_str) + path._str = path_str + path._drv = self.drive + path._root = self.root + path._tail_cached = self._tail + [entry.name] + return path + + def _make_child_relpath(self, name): + if not name: + return self + path_str = str(self) + tail = self._tail + if tail: + path_str = f'{path_str}{self.pathmod.sep}{name}' + elif path_str != '.': + path_str = f'{path_str}{name}' + else: + path_str = name + path = self.with_segments(path_str) + path._str = path_str + path._drv = self.drive + path._root = self.root + path._tail_cached = tail + [name] + return path + + def glob(self, pattern, *, case_sensitive=None, follow_symlinks=None): + """Iterate over this subtree and yield all existing files (of any + kind, including directories) matching the given relative pattern. + """ + sys.audit("pathlib.Path.glob", self, pattern) + if not isinstance(pattern, PurePath): + pattern = self.with_segments(pattern) + return _abc.PathBase.glob( + self, pattern, case_sensitive=case_sensitive, follow_symlinks=follow_symlinks) + + def rglob(self, pattern, *, case_sensitive=None, follow_symlinks=None): + """Recursively yield all existing files (of any kind, including + directories) matching the given relative pattern, anywhere in + this subtree. + """ + sys.audit("pathlib.Path.rglob", self, pattern) + if not isinstance(pattern, PurePath): + pattern = self.with_segments(pattern) + pattern = '**' / pattern + return _abc.PathBase.glob( + self, pattern, case_sensitive=case_sensitive, follow_symlinks=follow_symlinks) + + def walk(self, top_down=True, on_error=None, follow_symlinks=False): + """Walk the directory tree from this directory, similar to os.walk().""" + sys.audit("pathlib.Path.walk", self, on_error, follow_symlinks) + return _abc.PathBase.walk( + self, top_down=top_down, on_error=on_error, follow_symlinks=follow_symlinks) + def absolute(self): """Return an absolute version of this path No normalization or symlink resolution is performed. diff --git a/Lib/pathlib/_abc.py b/Lib/pathlib/_abc.py index 4808d0e61f7038..e4b1201a3703c3 100644 --- a/Lib/pathlib/_abc.py +++ b/Lib/pathlib/_abc.py @@ -1,30 +1,24 @@ +""" +Abstract base classes for rich path objects. + +This module is published as a PyPI package called "pathlib-abc". + +This module is also a *PRIVATE* part of the Python standard library, where +it's developed alongside pathlib. If it finds success and maturity as a PyPI +package, it could become a public part of the standard library. + +Two base classes are defined here -- PurePathBase and PathBase -- that +resemble pathlib's PurePath and Path respectively. +""" + import functools -import io -import ntpath -import os -import posixpath -import sys -import warnings -from _collections_abc import Sequence from errno import ENOENT, ENOTDIR, EBADF, ELOOP, EINVAL -from itertools import chain from stat import S_ISDIR, S_ISLNK, S_ISREG, S_ISSOCK, S_ISBLK, S_ISCHR, S_ISFIFO # # Internals # -# Maximum number of symlinks to follow in PathBase.resolve() -_MAX_SYMLINKS = 40 - -# Reference for Windows paths can be found at -# https://learn.microsoft.com/en-gb/windows/win32/fileio/naming-a-file . -_WIN_RESERVED_NAMES = frozenset( - {'CON', 'PRN', 'AUX', 'NUL', 'CONIN$', 'CONOUT$'} | - {f'COM{c}' for c in '123456789\xb9\xb2\xb3'} | - {f'LPT{c}' for c in '123456789\xb9\xb2\xb3'} -) - _WINERROR_NOT_READY = 21 # drive exists but is not accessible _WINERROR_INVALID_NAME = 123 # fix for bpo-35306 _WINERROR_CANT_RESOLVE_FILENAME = 1921 # broken symlink pointing to itself @@ -53,8 +47,8 @@ def _is_case_sensitive(pathmod): re = glob = None -@functools.lru_cache(maxsize=256) -def _compile_pattern(pat, sep, case_sensitive): +@functools.lru_cache(maxsize=512) +def _compile_pattern(pat, sep, case_sensitive, recursive=True): """Compile given glob pattern to a re.Pattern object (observing case sensitivity).""" global re, glob @@ -62,13 +56,16 @@ def _compile_pattern(pat, sep, case_sensitive): import re, glob flags = re.NOFLAG if case_sensitive else re.IGNORECASE - regex = glob.translate(pat, recursive=True, include_hidden=True, seps=sep) - # The string representation of an empty path is a single dot ('.'). Empty - # paths shouldn't match wildcards, so we consume it with an atomic group. - regex = r'(\.\Z)?+' + regex + regex = glob.translate(pat, recursive=recursive, include_hidden=True, seps=sep) return re.compile(regex, flags=flags).match +def _select_special(paths, part): + """Yield special literal children of the given paths.""" + for path in paths: + yield path._make_child_relpath(part) + + def _select_children(parent_paths, dir_only, follow_symlinks, match): """Yield direct children of given paths, filtering by name and type.""" if follow_symlinks is None: @@ -89,9 +86,8 @@ def _select_children(parent_paths, dir_only, follow_symlinks, match): continue except OSError: continue - name = entry.name - if match(name): - yield parent_path._make_child_relpath(name) + if match(entry.name): + yield parent_path._make_child_entry(entry) def _select_recursive(parent_paths, dir_only, follow_symlinks): @@ -99,7 +95,7 @@ def _select_recursive(parent_paths, dir_only, follow_symlinks): if follow_symlinks is None: follow_symlinks = False for parent_path in parent_paths: - paths = [parent_path] + paths = [parent_path._make_child_relpath('')] while paths: path = paths.pop() yield path @@ -114,12 +110,12 @@ def _select_recursive(parent_paths, dir_only, follow_symlinks): for entry in entries: try: if entry.is_dir(follow_symlinks=follow_symlinks): - paths.append(path._make_child_relpath(entry.name)) + paths.append(path._make_child_entry(entry)) continue except OSError: pass if not dir_only: - yield path._make_child_relpath(entry.name) + yield path._make_child_entry(entry) def _select_unique(paths): @@ -142,33 +138,50 @@ class UnsupportedOperation(NotImplementedError): pass -class _PathParents(Sequence): - """This object provides sequence-like access to the logical ancestors - of a path. Don't try to construct it yourself.""" - __slots__ = ('_path', '_drv', '_root', '_tail') +class PathModuleBase: + """Base class for path modules, which do low-level path manipulation. + + Path modules provide a subset of the os.path API, specifically those + functions needed to provide PurePathBase functionality. Each PurePathBase + subclass references its path module via a 'pathmod' class attribute. - def __init__(self, path): - self._path = path - self._drv = path.drive - self._root = path.root - self._tail = path._tail + Every method in this base class raises an UnsupportedOperation exception. + """ - def __len__(self): - return len(self._tail) + @classmethod + def _unsupported_msg(cls, attribute): + return f"{cls.__name__}.{attribute} is unsupported" - def __getitem__(self, idx): - if isinstance(idx, slice): - return tuple(self[i] for i in range(*idx.indices(len(self)))) + @property + def sep(self): + """The character used to separate path components.""" + raise UnsupportedOperation(self._unsupported_msg('sep')) - if idx >= len(self) or idx < -len(self): - raise IndexError(idx) - if idx < 0: - idx += len(self) - return self._path._from_parsed_parts(self._drv, self._root, - self._tail[:-idx - 1]) + def join(self, path, *paths): + """Join path segments.""" + raise UnsupportedOperation(self._unsupported_msg('join()')) - def __repr__(self): - return "<{}.parents>".format(type(self._path).__name__) + def split(self, path): + """Split the path into a pair (head, tail), where *head* is everything + before the final path separator, and *tail* is everything after. + Either part may be empty. + """ + raise UnsupportedOperation(self._unsupported_msg('split()')) + + def splitdrive(self, path): + """Split the path into a 2-item tuple (drive, tail), where *drive* is + a device name or mount point, and *tail* is everything after the + drive. Either part may be empty.""" + raise UnsupportedOperation(self._unsupported_msg('splitdrive()')) + + def normcase(self, path): + """Normalize the case of the path.""" + raise UnsupportedOperation(self._unsupported_msg('normcase()')) + + def isabs(self, path): + """Returns whether the path is absolute, i.e. unaffected by the + current directory or drive.""" + raise UnsupportedOperation(self._unsupported_msg('isabs()')) class PurePathBase: @@ -181,33 +194,22 @@ class PurePathBase: """ __slots__ = ( - # The `_raw_paths` slot stores unnormalized string paths. This is set - # in the `__init__()` method. - '_raw_paths', - - # The `_drv`, `_root` and `_tail_cached` slots store parsed and - # normalized parts of the path. They are set when any of the `drive`, - # `root` or `_tail` properties are accessed for the first time. The - # three-part division corresponds to the result of - # `os.path.splitroot()`, except that the tail is further split on path - # separators (i.e. it is a list of strings), and that the root and - # tail are normalized. - '_drv', '_root', '_tail_cached', - - # The `_str` slot stores the string representation of the path, - # computed from the drive, root and tail when `__str__()` is called - # for the first time. It's used to implement `_str_normcase` - '_str', + # The `_raw_path` slot store a joined string path. This is set in the + # `__init__()` method. + '_raw_path', # The '_resolving' slot stores a boolean indicating whether the path # is being processed by `PathBase.resolve()`. This prevents duplicate # work from occurring when `resolve()` calls `stat()` or `readlink()`. '_resolving', ) - pathmod = os.path + pathmod = PathModuleBase() - def __init__(self, *paths): - self._raw_paths = paths + def __init__(self, path, *paths): + self._raw_path = self.pathmod.join(path, *paths) if paths else path + if not isinstance(self._raw_path, str): + raise TypeError( + f"path should be a str, not {type(self._raw_path).__name__!r}") self._resolving = False def with_segments(self, *pathsegments): @@ -217,113 +219,35 @@ def with_segments(self, *pathsegments): """ return type(self)(*pathsegments) - @classmethod - def _parse_path(cls, path): - if not path: - return '', '', [] - sep = cls.pathmod.sep - altsep = cls.pathmod.altsep - if altsep: - path = path.replace(altsep, sep) - drv, root, rel = cls.pathmod.splitroot(path) - if not root and drv.startswith(sep) and not drv.endswith(sep): - drv_parts = drv.split(sep) - if len(drv_parts) == 4 and drv_parts[2] not in '?.': - # e.g. //server/share - root = sep - elif len(drv_parts) == 6: - # e.g. //?/unc/server/share - root = sep - parsed = [sys.intern(str(x)) for x in rel.split(sep) if x and x != '.'] - return drv, root, parsed - - def _load_parts(self): - paths = self._raw_paths - if len(paths) == 0: - path = '' - elif len(paths) == 1: - path = paths[0] - else: - path = self.pathmod.join(*paths) - drv, root, tail = self._parse_path(path) - self._drv = drv - self._root = root - self._tail_cached = tail - - def _from_parsed_parts(self, drv, root, tail): - path_str = self._format_parsed_parts(drv, root, tail) - path = self.with_segments(path_str) - path._str = path_str or '.' - path._drv = drv - path._root = root - path._tail_cached = tail - return path - - @classmethod - def _format_parsed_parts(cls, drv, root, tail): - if drv or root: - return drv + root + cls.pathmod.sep.join(tail) - elif tail and cls.pathmod.splitdrive(tail[0])[0]: - tail = ['.'] + tail - return cls.pathmod.sep.join(tail) - def __str__(self): """Return the string representation of the path, suitable for passing to system calls.""" - try: - return self._str - except AttributeError: - self._str = self._format_parsed_parts(self.drive, self.root, - self._tail) or '.' - return self._str + return self._raw_path def as_posix(self): """Return the string representation of the path with forward (/) slashes.""" return str(self).replace(self.pathmod.sep, '/') - def __repr__(self): - return "{}({!r})".format(self.__class__.__name__, self.as_posix()) - @property def drive(self): """The drive prefix (letter or UNC path), if any.""" - try: - return self._drv - except AttributeError: - self._load_parts() - return self._drv + return self.pathmod.splitdrive(self.anchor)[0] @property def root(self): """The root of the path, if any.""" - try: - return self._root - except AttributeError: - self._load_parts() - return self._root - - @property - def _tail(self): - try: - return self._tail_cached - except AttributeError: - self._load_parts() - return self._tail_cached + return self.pathmod.splitdrive(self.anchor)[1] @property def anchor(self): """The concatenation of the drive and root, or ''.""" - anchor = self.drive + self.root - return anchor + return self._stack[0] @property def name(self): """The final path component, if any.""" - tail = self._tail - if not tail: - return '' - return tail[-1] + return self.pathmod.split(self._raw_path)[1] @property def suffix(self): @@ -364,14 +288,10 @@ def stem(self): def with_name(self, name): """Return a new path with the file name changed.""" - m = self.pathmod - if not name or m.sep in name or (m.altsep and m.altsep in name) or name == '.': + split = self.pathmod.split + if split(name)[0]: raise ValueError(f"Invalid name {name!r}") - tail = self._tail.copy() - if not tail: - raise ValueError(f"{self!r} has an empty name") - tail[-1] = name - return self._from_parsed_parts(self.drive, self.root, tail) + return self.with_segments(split(self._raw_path)[0], name) def with_stem(self, stem): """Return a new path with the stem changed.""" @@ -382,14 +302,17 @@ def with_suffix(self, suffix): has no suffix, add given suffix. If the given suffix is an empty string, remove the suffix from the path. """ + stem = self.stem if not suffix: - return self.with_name(self.stem) + return self.with_name(stem) + elif not stem: + raise ValueError(f"{self!r} has an empty name") elif suffix.startswith('.') and len(suffix) > 1: - return self.with_name(self.stem + suffix) + return self.with_name(stem + suffix) else: raise ValueError(f"Invalid suffix {suffix!r}") - def relative_to(self, other, /, *_deprecated, walk_up=False): + def relative_to(self, other, *, walk_up=False): """Return the relative path to another path identified by the passed arguments. If the operation is not possible (because this is not related to the other path), raise ValueError. @@ -397,49 +320,51 @@ def relative_to(self, other, /, *_deprecated, walk_up=False): The *walk_up* parameter controls whether `..` may be used to resolve the path. """ - if _deprecated: - msg = ("support for supplying more than one positional argument " - "to pathlib.PurePath.relative_to() is deprecated and " - "scheduled for removal in Python {remove}") - warnings._deprecated("pathlib.PurePath.relative_to(*args)", msg, - remove=(3, 14)) - other = self.with_segments(other, *_deprecated) - elif not isinstance(other, PurePathBase): + if not isinstance(other, PurePathBase): other = self.with_segments(other) - for step, path in enumerate(chain([other], other.parents)): - if path == self or path in self.parents: - break + anchor0, parts0 = self._stack + anchor1, parts1 = other._stack + if anchor0 != anchor1: + raise ValueError(f"{self._raw_path!r} and {other._raw_path!r} have different anchors") + while parts0 and parts1 and parts0[-1] == parts1[-1]: + parts0.pop() + parts1.pop() + for part in parts1: + if not part or part == '.': + pass elif not walk_up: - raise ValueError(f"{str(self)!r} is not in the subpath of {str(other)!r}") - elif path.name == '..': - raise ValueError(f"'..' segment in {str(other)!r} cannot be walked") - else: - raise ValueError(f"{str(self)!r} and {str(other)!r} have different anchors") - parts = ['..'] * step + self._tail[len(path._tail):] - return self._from_parsed_parts('', '', parts) + raise ValueError(f"{self._raw_path!r} is not in the subpath of {other._raw_path!r}") + elif part == '..': + raise ValueError(f"'..' segment in {other._raw_path!r} cannot be walked") + else: + parts0.append('..') + return self.with_segments('', *reversed(parts0)) - def is_relative_to(self, other, /, *_deprecated): + def is_relative_to(self, other): """Return True if the path is relative to another path or False. """ - if _deprecated: - msg = ("support for supplying more than one argument to " - "pathlib.PurePath.is_relative_to() is deprecated and " - "scheduled for removal in Python {remove}") - warnings._deprecated("pathlib.PurePath.is_relative_to(*args)", - msg, remove=(3, 14)) - other = self.with_segments(other, *_deprecated) - elif not isinstance(other, PurePathBase): + if not isinstance(other, PurePathBase): other = self.with_segments(other) - return other == self or other in self.parents + anchor0, parts0 = self._stack + anchor1, parts1 = other._stack + if anchor0 != anchor1: + return False + while parts0 and parts1 and parts0[-1] == parts1[-1]: + parts0.pop() + parts1.pop() + for part in parts1: + if part and part != '.': + return False + return True @property def parts(self): """An object providing sequence-like access to the components in the filesystem path.""" - if self.drive or self.root: - return (self.drive + self.root,) + tuple(self._tail) - else: - return tuple(self._tail) + anchor, parts = self._stack + if anchor: + parts.append(anchor) + return tuple(reversed(parts)) def joinpath(self, *pathsegments): """Combine this path with one or several arguments, and return a @@ -447,88 +372,116 @@ def joinpath(self, *pathsegments): paths) or a totally different path (if one of the arguments is anchored). """ - return self.with_segments(*self._raw_paths, *pathsegments) + return self.with_segments(self._raw_path, *pathsegments) def __truediv__(self, key): try: - return self.joinpath(key) + return self.with_segments(self._raw_path, key) except TypeError: return NotImplemented def __rtruediv__(self, key): try: - return self.with_segments(key, *self._raw_paths) + return self.with_segments(key, self._raw_path) except TypeError: return NotImplemented + @property + def _stack(self): + """ + Split the path into a 2-tuple (anchor, parts), where *anchor* is the + uppermost parent of the path (equivalent to path.parents[-1]), and + *parts* is a reversed list of parts following the anchor. + """ + split = self.pathmod.split + path = self._raw_path + parent, name = split(path) + names = [] + while path != parent: + names.append(name) + path = parent + parent, name = split(path) + return path, names + @property def parent(self): """The logical parent of the path.""" - drv = self.drive - root = self.root - tail = self._tail - if not tail: - return self - path = self._from_parsed_parts(drv, root, tail[:-1]) - path._resolving = self._resolving - return path + path = self._raw_path + parent = self.pathmod.split(path)[0] + if path != parent: + parent = self.with_segments(parent) + parent._resolving = self._resolving + return parent + return self @property def parents(self): """A sequence of this path's logical parents.""" - # The value of this property should not be cached on the path object, - # as doing so would introduce a reference cycle. - return _PathParents(self) + split = self.pathmod.split + path = self._raw_path + parent = split(path)[0] + parents = [] + while path != parent: + parents.append(self.with_segments(parent)) + path = parent + parent = split(path)[0] + return tuple(parents) def is_absolute(self): """True if the path is absolute (has both a root and, if applicable, a drive).""" - if self.pathmod is ntpath: - # ntpath.isabs() is defective - see GH-44626. - return bool(self.drive and self.root) - elif self.pathmod is posixpath: - # Optimization: work with raw paths on POSIX. - for path in self._raw_paths: - if path.startswith('/'): - return True - return False - else: - return self.pathmod.isabs(str(self)) + return self.pathmod.isabs(self._raw_path) - def is_reserved(self): - """Return True if the path contains one of the special names reserved - by the system, if any.""" - if self.pathmod is posixpath or not self._tail: - return False + @property + def _pattern_stack(self): + """Stack of path components, to be used with patterns in glob().""" + anchor, parts = self._stack + if anchor: + raise NotImplementedError("Non-relative patterns are unsupported") + return parts - # NOTE: the rules for reserved names seem somewhat complicated - # (e.g. r"..\NUL" is reserved but not r"foo\NUL" if "foo" does not - # exist). We err on the side of caution and return True for paths - # which are not considered reserved by Windows. - if self.drive.startswith('\\\\'): - # UNC paths are never reserved. - return False - name = self._tail[-1].partition('.')[0].partition(':')[0].rstrip(' ') - return name.upper() in _WIN_RESERVED_NAMES + @property + def _pattern_str(self): + """The path expressed as a string, for use in pattern-matching.""" + return str(self) def match(self, path_pattern, *, case_sensitive=None): """ - Return True if this path matches the given pattern. + Return True if this path matches the given pattern. If the pattern is + relative, matching is done from the right; otherwise, the entire path + is matched. The recursive wildcard '**' is *not* supported by this + method. """ if not isinstance(path_pattern, PurePathBase): path_pattern = self.with_segments(path_pattern) if case_sensitive is None: case_sensitive = _is_case_sensitive(self.pathmod) sep = path_pattern.pathmod.sep - pattern_str = str(path_pattern) - if path_pattern.drive or path_pattern.root: - pass - elif path_pattern._tail: - pattern_str = f'**{sep}{pattern_str}' - else: + path_parts = self.parts[::-1] + pattern_parts = path_pattern.parts[::-1] + if not pattern_parts: raise ValueError("empty pattern") - match = _compile_pattern(pattern_str, sep, case_sensitive) - return match(str(self)) is not None + if len(path_parts) < len(pattern_parts): + return False + if len(path_parts) > len(pattern_parts) and path_pattern.anchor: + return False + for path_part, pattern_part in zip(path_parts, pattern_parts): + match = _compile_pattern(pattern_part, sep, case_sensitive, recursive=False) + if match(path_part) is None: + return False + return True + + def full_match(self, pattern, *, case_sensitive=None): + """ + Return True if this path matches the given glob-style pattern. The + pattern is matched against the entire path. + """ + if not isinstance(pattern, PurePathBase): + pattern = self.with_segments(pattern) + if case_sensitive is None: + case_sensitive = _is_case_sensitive(self.pathmod) + match = _compile_pattern(pattern._pattern_str, pattern.pathmod.sep, case_sensitive) + return match(self._pattern_str) is not None @@ -547,17 +500,19 @@ class PathBase(PurePathBase): """ __slots__ = () + # Maximum number of symlinks to follow in resolve() + _max_symlinks = 40 + @classmethod - def _unsupported(cls, method_name): - msg = f"{cls.__name__}.{method_name}() is unsupported" - raise UnsupportedOperation(msg) + def _unsupported_msg(cls, attribute): + return f"{cls.__name__}.{attribute} is unsupported" def stat(self, *, follow_symlinks=True): """ Return the result of the stat() system call on this path, like os.stat() does. """ - self._unsupported("stat") + raise UnsupportedOperation(self._unsupported_msg('stat()')) def lstat(self): """ @@ -746,7 +701,7 @@ def open(self, mode='r', buffering=-1, encoding=None, Open the file pointed by this path and return a file object, as the built-in open() function does. """ - self._unsupported("open") + raise UnsupportedOperation(self._unsupported_msg('open()')) def read_bytes(self): """ @@ -759,7 +714,6 @@ def read_text(self, encoding=None, errors=None, newline=None): """ Open the file in text mode, read it, and close the file. """ - encoding = io.text_encoding(encoding) with self.open(mode='r', encoding=encoding, errors=errors, newline=newline) as f: return f.read() @@ -779,7 +733,6 @@ def write_text(self, data, encoding=None, errors=None, newline=None): if not isinstance(data, str): raise TypeError('data must be str, not %s' % data.__class__.__name__) - encoding = io.text_encoding(encoding) with self.open(mode='w', encoding=encoding, errors=errors, newline=newline) as f: return f.write(data) @@ -789,7 +742,7 @@ def iterdir(self): The children are yielded in arbitrary order, and the special entries '.' and '..' are not included. """ - self._unsupported("iterdir") + raise UnsupportedOperation(self._unsupported_msg('iterdir()')) def _scandir(self): # Emulate os.scandir(), which returns an object that can be used as a @@ -797,97 +750,46 @@ def _scandir(self): from contextlib import nullcontext return nullcontext(self.iterdir()) + def _make_child_entry(self, entry): + # Transform an entry yielded from _scandir() into a path object. + return entry + def _make_child_relpath(self, name): - path_str = str(self) - tail = self._tail - if tail: - path_str = f'{path_str}{self.pathmod.sep}{name}' - elif path_str != '.': - path_str = f'{path_str}{name}' - else: - path_str = name - path = self.with_segments(path_str) - path._str = path_str - path._drv = self.drive - path._root = self.root - path._tail_cached = tail + [name] - return path + return self.joinpath(name) def glob(self, pattern, *, case_sensitive=None, follow_symlinks=None): """Iterate over this subtree and yield all existing files (of any kind, including directories) matching the given relative pattern. """ - sys.audit("pathlib.Path.glob", self, pattern) - return self._glob(pattern, case_sensitive, follow_symlinks) - - def rglob(self, pattern, *, case_sensitive=None, follow_symlinks=None): - """Recursively yield all existing files (of any kind, including - directories) matching the given relative pattern, anywhere in - this subtree. - """ - sys.audit("pathlib.Path.rglob", self, pattern) - return self._glob(f'**/{pattern}', case_sensitive, follow_symlinks) - - def _glob(self, pattern, case_sensitive, follow_symlinks): - path_pattern = self.with_segments(pattern) - if path_pattern.drive or path_pattern.root: - raise NotImplementedError("Non-relative patterns are unsupported") - elif not path_pattern._tail: - raise ValueError("Unacceptable pattern: {!r}".format(pattern)) - - pattern_parts = path_pattern._tail.copy() - if pattern[-1] in (self.pathmod.sep, self.pathmod.altsep): - # GH-65238: pathlib doesn't preserve trailing slash. Add it back. - pattern_parts.append('') - if pattern_parts[-1] == '**': - # GH-70303: '**' only matches directories. Add trailing slash. - warnings.warn( - "Pattern ending '**' will match files and directories in a " - "future Python release. Add a trailing slash to match only " - "directories and remove this warning.", - FutureWarning, 3) - pattern_parts.append('') - + if not isinstance(pattern, PurePathBase): + pattern = self.with_segments(pattern) if case_sensitive is None: # TODO: evaluate case-sensitivity of each directory in _select_children(). case_sensitive = _is_case_sensitive(self.pathmod) - # If symlinks are handled consistently, and the pattern does not - # contain '..' components, then we can use a 'walk-and-match' strategy - # when expanding '**' wildcards. When a '**' wildcard is encountered, - # all following pattern parts are immediately consumed and used to - # build a `re.Pattern` object. This pattern is used to filter the - # recursive walk. As a result, pattern parts following a '**' wildcard - # do not perform any filesystem access, which can be much faster! - filter_paths = follow_symlinks is not None and '..' not in pattern_parts + stack = pattern._pattern_stack + specials = ('', '.', '..') + filter_paths = False deduplicate_paths = False sep = self.pathmod.sep paths = iter([self] if self.is_dir() else []) - part_idx = 0 - while part_idx < len(pattern_parts): - part = pattern_parts[part_idx] - part_idx += 1 - if part == '': - # Trailing slash. - pass - elif part == '..': - paths = (path._make_child_relpath('..') for path in paths) + while stack: + part = stack.pop() + if part in specials: + paths = _select_special(paths, part) elif part == '**': # Consume adjacent '**' components. - while part_idx < len(pattern_parts) and pattern_parts[part_idx] == '**': - part_idx += 1 + while stack and stack[-1] == '**': + stack.pop() - if filter_paths and part_idx < len(pattern_parts) and pattern_parts[part_idx] != '': - dir_only = pattern_parts[-1] == '' - paths = _select_recursive(paths, dir_only, follow_symlinks) + # Consume adjacent non-special components and enable post-walk + # regex filtering, provided we're treating symlinks consistently. + if follow_symlinks is not None: + while stack and stack[-1] not in specials: + filter_paths = True + stack.pop() - # Filter out paths that don't match pattern. - prefix_len = len(str(self._make_child_relpath('_'))) - 1 - match = _compile_pattern(str(path_pattern), sep, case_sensitive) - paths = (path for path in paths if match(str(path), prefix_len)) - return paths - - dir_only = part_idx < len(pattern_parts) + dir_only = bool(stack) paths = _select_recursive(paths, dir_only, follow_symlinks) if deduplicate_paths: # De-duplicate if we've already seen a '**' component. @@ -896,14 +798,28 @@ def _glob(self, pattern, case_sensitive, follow_symlinks): elif '**' in part: raise ValueError("Invalid pattern: '**' can only be an entire path component") else: - dir_only = part_idx < len(pattern_parts) + dir_only = bool(stack) match = _compile_pattern(part, sep, case_sensitive) paths = _select_children(paths, dir_only, follow_symlinks, match) + if filter_paths: + # Filter out paths that don't match pattern. + prefix_len = len(str(self._make_child_relpath('_'))) - 1 + match = _compile_pattern(pattern._pattern_str, sep, case_sensitive) + paths = (path for path in paths if match(path._pattern_str, prefix_len)) return paths + def rglob(self, pattern, *, case_sensitive=None, follow_symlinks=None): + """Recursively yield all existing files (of any kind, including + directories) matching the given relative pattern, anywhere in + this subtree. + """ + if not isinstance(pattern, PurePathBase): + pattern = self.with_segments(pattern) + pattern = '**' / pattern + return self.glob(pattern, case_sensitive=case_sensitive, follow_symlinks=follow_symlinks) + def walk(self, top_down=True, on_error=None, follow_symlinks=False): """Walk the directory tree from this directory, similar to os.walk().""" - sys.audit("pathlib.Path.walk", self, on_error, follow_symlinks) paths = [self] while paths: @@ -927,6 +843,8 @@ def walk(self, top_down=True, on_error=None, follow_symlinks=False): with scandir_obj as scandir_it: dirnames = [] filenames = [] + if not top_down: + paths.append((path, dirnames, filenames)) for entry in scandir_it: try: is_dir = entry.is_dir(follow_symlinks=follow_symlinks) @@ -935,16 +853,15 @@ def walk(self, top_down=True, on_error=None, follow_symlinks=False): is_dir = False if is_dir: + if not top_down: + paths.append(path._make_child_entry(entry)) dirnames.append(entry.name) else: filenames.append(entry.name) if top_down: yield path, dirnames, filenames - else: - paths.append((path, dirnames, filenames)) - - paths += [path._make_child_relpath(d) for d in reversed(dirnames)] + paths += [path._make_child_relpath(d) for d in reversed(dirnames)] def absolute(self): """Return an absolute version of this path @@ -952,7 +869,7 @@ def absolute(self): Use resolve() to resolve symlinks and remove '..' segments. """ - self._unsupported("absolute") + raise UnsupportedOperation(self._unsupported_msg('absolute()')) @classmethod def cwd(cls): @@ -961,13 +878,13 @@ def cwd(cls): # enable users to replace the implementation of 'absolute()' in a # subclass and benefit from the new behaviour here. This works because # os.path.abspath('.') == os.getcwd(). - return cls().absolute() + return cls('').absolute() def expanduser(self): """ Return a new path with expanded ~ and ~user constructs (as returned by os.path.expanduser) """ - self._unsupported("expanduser") + raise UnsupportedOperation(self._unsupported_msg('expanduser()')) @classmethod def home(cls): @@ -979,19 +896,9 @@ def readlink(self): """ Return the path to which the symbolic link points. """ - self._unsupported("readlink") + raise UnsupportedOperation(self._unsupported_msg('readlink()')) readlink._supported = False - def _split_stack(self): - """ - Split the path into a 2-tuple (anchor, parts), where *anchor* is the - uppermost parent of the path (equivalent to path.parents[-1]), and - *parts* is a reversed list of parts following the anchor. - """ - if not self._tail: - return self, [] - return self._from_parsed_parts(self.drive, self.root, []), self._tail[::-1] - def resolve(self, strict=False): """ Make the path absolute, resolving all symlinks on the way and also @@ -999,11 +906,15 @@ def resolve(self, strict=False): """ if self._resolving: return self - path, parts = self._split_stack() + path_root, parts = self._stack + path = self.with_segments(path_root) try: path = path.absolute() except UnsupportedOperation: - pass + path_tail = [] + else: + path_root, path_tail = path._stack + path_tail.reverse() # If the user has *not* overridden the `readlink()` method, then symlinks are unsupported # and (in non-strict mode) we can improve performance by not calling `stat()`. @@ -1011,52 +922,56 @@ def resolve(self, strict=False): link_count = 0 while parts: part = parts.pop() + if not part or part == '.': + continue if part == '..': - if not path._tail: - if path.root: + if not path_tail: + if path_root: # Delete '..' segment immediately following root continue - elif path._tail[-1] != '..': + elif path_tail[-1] != '..': # Delete '..' segment and its predecessor - path = path.parent + path_tail.pop() continue - next_path = path._make_child_relpath(part) + path_tail.append(part) if querying and part != '..': - next_path._resolving = True + path = self.with_segments(path_root + self.pathmod.sep.join(path_tail)) + path._resolving = True try: - st = next_path.stat(follow_symlinks=False) + st = path.stat(follow_symlinks=False) if S_ISLNK(st.st_mode): # Like Linux and macOS, raise OSError(errno.ELOOP) if too many symlinks are # encountered during resolution. link_count += 1 - if link_count >= _MAX_SYMLINKS: - raise OSError(ELOOP, "Too many symbolic links in path", str(self)) - target, target_parts = next_path.readlink()._split_stack() + if link_count >= self._max_symlinks: + raise OSError(ELOOP, "Too many symbolic links in path", self._raw_path) + target_root, target_parts = path.readlink()._stack # If the symlink target is absolute (like '/etc/hosts'), set the current # path to its uppermost parent (like '/'). - if target.root: - path = target + if target_root: + path_root = target_root + path_tail.clear() + else: + path_tail.pop() # Add the symlink target's reversed tail parts (like ['hosts', 'etc']) to # the stack of unresolved path parts. parts.extend(target_parts) continue elif parts and not S_ISDIR(st.st_mode): - raise NotADirectoryError(ENOTDIR, "Not a directory", str(self)) + raise NotADirectoryError(ENOTDIR, "Not a directory", self._raw_path) except OSError: if strict: raise else: querying = False - next_path._resolving = False - path = next_path - return path + return self.with_segments(path_root + self.pathmod.sep.join(path_tail)) def symlink_to(self, target, target_is_directory=False): """ Make this path a symlink pointing to the target path. Note the order of arguments (link, target) is the reverse of os.symlink. """ - self._unsupported("symlink_to") + raise UnsupportedOperation(self._unsupported_msg('symlink_to()')) def hardlink_to(self, target): """ @@ -1064,19 +979,19 @@ def hardlink_to(self, target): Note the order of arguments (self, target) is the reverse of os.link's. """ - self._unsupported("hardlink_to") + raise UnsupportedOperation(self._unsupported_msg('hardlink_to()')) def touch(self, mode=0o666, exist_ok=True): """ Create this file with the given access mode, if it doesn't exist. """ - self._unsupported("touch") + raise UnsupportedOperation(self._unsupported_msg('touch()')) def mkdir(self, mode=0o777, parents=False, exist_ok=False): """ Create a new directory at this given path. """ - self._unsupported("mkdir") + raise UnsupportedOperation(self._unsupported_msg('mkdir()')) def rename(self, target): """ @@ -1088,7 +1003,7 @@ def rename(self, target): Returns the new Path instance pointing to the target path. """ - self._unsupported("rename") + raise UnsupportedOperation(self._unsupported_msg('rename()')) def replace(self, target): """ @@ -1100,13 +1015,13 @@ def replace(self, target): Returns the new Path instance pointing to the target path. """ - self._unsupported("replace") + raise UnsupportedOperation(self._unsupported_msg('replace()')) def chmod(self, mode, *, follow_symlinks=True): """ Change the permissions of the path, like os.chmod(). """ - self._unsupported("chmod") + raise UnsupportedOperation(self._unsupported_msg('chmod()')) def lchmod(self, mode): """ @@ -1120,31 +1035,31 @@ def unlink(self, missing_ok=False): Remove this file or link. If the path is a directory, use rmdir() instead. """ - self._unsupported("unlink") + raise UnsupportedOperation(self._unsupported_msg('unlink()')) def rmdir(self): """ Remove this directory. The directory must be empty. """ - self._unsupported("rmdir") + raise UnsupportedOperation(self._unsupported_msg('rmdir()')) def owner(self, *, follow_symlinks=True): """ Return the login name of the file owner. """ - self._unsupported("owner") + raise UnsupportedOperation(self._unsupported_msg('owner()')) def group(self, *, follow_symlinks=True): """ Return the group name of the file gid. """ - self._unsupported("group") + raise UnsupportedOperation(self._unsupported_msg('group()')) @classmethod def from_uri(cls, uri): """Return a new path from the given 'file' URI.""" - cls._unsupported("from_uri") + raise UnsupportedOperation(cls._unsupported_msg('from_uri()')) def as_uri(self): """Return the path as a URI.""" - self._unsupported("as_uri") + raise UnsupportedOperation(self._unsupported_msg('as_uri()')) diff --git a/Lib/pdb.py b/Lib/pdb.py index 83b7fefec63636..0754e8b628cf57 100755 --- a/Lib/pdb.py +++ b/Lib/pdb.py @@ -76,6 +76,7 @@ import dis import code import glob +import token import codeop import pprint import signal @@ -96,17 +97,47 @@ class Restart(Exception): __all__ = ["run", "pm", "Pdb", "runeval", "runctx", "runcall", "set_trace", "post_mortem", "help"] + +def find_first_executable_line(code): + """ Try to find the first executable line of the code object. + + Equivalently, find the line number of the instruction that's + after RESUME + + Return code.co_firstlineno if no executable line is found. + """ + prev = None + for instr in dis.get_instructions(code): + if prev is not None and prev.opname == 'RESUME': + if instr.positions.lineno is not None: + return instr.positions.lineno + return code.co_firstlineno + prev = instr + return code.co_firstlineno + def find_function(funcname, filename): cre = re.compile(r'def\s+%s\s*[(]' % re.escape(funcname)) try: fp = tokenize.open(filename) except OSError: return None + funcdef = "" + funcstart = None # consumer of this info expects the first line to be 1 with fp: for lineno, line in enumerate(fp, start=1): if cre.match(line): - return funcname, filename, lineno + funcstart, funcdef = lineno, line + elif funcdef: + funcdef += line + + if funcdef: + try: + funccode = compile(funcdef, filename, 'exec').co_consts[0] + except SyntaxError: + continue + lineno_offset = find_first_executable_line(funccode) + return funcname, filename, funcstart + lineno_offset - 1 return None def lasti2lineno(code, lasti): @@ -232,6 +263,8 @@ class Pdb(bdb.Bdb, cmd.Cmd): # but in case there are recursions, we stop at 999. MAX_CHAINED_EXCEPTION_DEPTH = 999 + _file_mtime_table = {} + def __init__(self, completekey='tab', stdin=None, stdout=None, skip=None, nosigint=False, readrc=True): bdb.Bdb.__init__(self, skip=skip) @@ -436,6 +469,20 @@ def _cmdloop(self): except KeyboardInterrupt: self.message('--KeyboardInterrupt--') + def _validate_file_mtime(self): + """Check if the source file of the current frame has been modified since + the last time we saw it. If so, give a warning.""" + try: + filename = self.curframe.f_code.co_filename + mtime = os.path.getmtime(filename) + except Exception: + return + if (filename in self._file_mtime_table and + mtime != self._file_mtime_table[filename]): + self.message(f"*** WARNING: file '{filename}' was edited, " + "running stale code until the program is rerun") + self._file_mtime_table[filename] = mtime + # Called before loop, handles display expressions # Set up convenience variable containers def preloop(self): @@ -601,6 +648,39 @@ def default(self, line): except: self._error_exc() + def _replace_convenience_variables(self, line): + """Replace the convenience variables in 'line' with their values. + e.g. $foo is replaced by __pdb_convenience_variables["foo"]. + Note: such pattern in string literals will be skipped""" + + if "$" not in line: + return line + + dollar_start = dollar_end = -1 + replace_variables = [] + try: + for t in tokenize.generate_tokens(io.StringIO(line).readline): + token_type, token_string, start, end, _ = t + if token_type == token.OP and token_string == '$': + dollar_start, dollar_end = start, end + elif start == dollar_end and token_type == token.NAME: + # line is a one-line command so we only care about column + replace_variables.append((dollar_start[1], end[1], token_string)) + except tokenize.TokenError: + return line + + if not replace_variables: + return line + + last_end = 0 + line_pieces = [] + for start, end, name in replace_variables: + line_pieces.append(line[last_end:start] + f'__pdb_convenience_variables["{name}"]') + last_end = end + line_pieces.append(line[last_end:]) + + return ''.join(line_pieces) + def precmd(self, line): """Handle alias expansion and ';;' separator.""" if not line.strip(): @@ -635,7 +715,7 @@ def precmd(self, line): line = line[:marker].rstrip() # Replace all the convenience variables - line = re.sub(r'\$([a-zA-Z_][a-zA-Z0-9_]*)', r'__pdb_convenience_variables["\1"]', line) + line = self._replace_convenience_variables(line) return line @@ -647,6 +727,7 @@ def onecmd(self, line): a breakpoint command list definition. """ if not self.commands_defining: + self._validate_file_mtime() return cmd.Cmd.onecmd(self, line) else: return self.handle_command_def(line) @@ -924,7 +1005,7 @@ def do_break(self, arg, temporary = 0): #use co_name to identify the bkpt (function names #could be aliased, but co_name is invariant) funcname = code.co_name - lineno = self._find_first_executable_line(code) + lineno = find_first_executable_line(code) filename = code.co_filename except: # last thing to try @@ -1027,23 +1108,6 @@ def checkline(self, filename, lineno): return 0 return lineno - def _find_first_executable_line(self, code): - """ Try to find the first executable line of the code object. - - Equivalently, find the line number of the instruction that's - after RESUME - - Return code.co_firstlineno if no executable line is found. - """ - prev = None - for instr in dis.get_instructions(code): - if prev is not None and prev.opname == 'RESUME': - if instr.positions.lineno is not None: - return instr.positions.lineno - return code.co_firstlineno - prev = instr - return code.co_firstlineno - def do_enable(self, arg): """enable bpnumber [bpnumber ...] @@ -1987,6 +2051,10 @@ def _run(self, target: Union[_ModuleTarget, _ScriptTarget]): __main__.__dict__.clear() __main__.__dict__.update(target.namespace) + # Clear the mtime table for program reruns, assume all the files + # are up to date. + self._file_mtime_table.clear() + self.run(target.code) def _format_exc(self, exc: BaseException): diff --git a/Lib/pickle.py b/Lib/pickle.py index 4f5ad5b71e8899..33c97c8c5efb28 100644 --- a/Lib/pickle.py +++ b/Lib/pickle.py @@ -857,13 +857,13 @@ def save_str(self, obj): else: self.write(BINUNICODE + pack(" 1: + name = '' if arg == '-' else arg + preamble = args.preamble.format(name=name) + output.write(preamble + '\n') + if arg == '-': + dis(sys.stdin.buffer, output, memo, args.indentlevel, annotate) + else: + with open(arg, 'rb') as f: + dis(f, output, memo, args.indentlevel, annotate) + finally: + if output is not sys.stdout: + output.close() diff --git a/Lib/platform.py b/Lib/platform.py index 75aa55510858fd..b56472235ee9e4 100755 --- a/Lib/platform.py +++ b/Lib/platform.py @@ -752,6 +752,8 @@ def architecture(executable=sys.executable, bits='', linkage=''): # Linkage if 'ELF' in fileout: linkage = 'ELF' + elif 'Mach-O' in fileout: + linkage = "Mach-O" elif 'PE' in fileout: # E.g. Windows uses this format if 'Windows' in fileout: diff --git a/Lib/plistlib.py b/Lib/plistlib.py index 3292c30d5fb29b..67e832db217319 100644 --- a/Lib/plistlib.py +++ b/Lib/plistlib.py @@ -140,7 +140,7 @@ def _decode_base64(s): _dateParser = re.compile(r"(?P\d\d\d\d)(?:-(?P\d\d)(?:-(?P\d\d)(?:T(?P\d\d)(?::(?P\d\d)(?::(?P\d\d))?)?)?)?)?Z", re.ASCII) -def _date_from_string(s): +def _date_from_string(s, aware_datetime): order = ('year', 'month', 'day', 'hour', 'minute', 'second') gd = _dateParser.match(s).groupdict() lst = [] @@ -149,10 +149,14 @@ def _date_from_string(s): if val is None: break lst.append(int(val)) + if aware_datetime: + return datetime.datetime(*lst, tzinfo=datetime.UTC) return datetime.datetime(*lst) -def _date_to_string(d): +def _date_to_string(d, aware_datetime): + if aware_datetime: + d = d.astimezone(datetime.UTC) return '%04d-%02d-%02dT%02d:%02d:%02dZ' % ( d.year, d.month, d.day, d.hour, d.minute, d.second @@ -171,11 +175,12 @@ def _escape(text): return text class _PlistParser: - def __init__(self, dict_type): + def __init__(self, dict_type, aware_datetime=False): self.stack = [] self.current_key = None self.root = None self._dict_type = dict_type + self._aware_datetime = aware_datetime def parse(self, fileobj): self.parser = ParserCreate() @@ -277,7 +282,8 @@ def end_data(self): self.add_object(_decode_base64(self.get_data())) def end_date(self): - self.add_object(_date_from_string(self.get_data())) + self.add_object(_date_from_string(self.get_data(), + aware_datetime=self._aware_datetime)) class _DumbXMLWriter: @@ -321,13 +327,14 @@ def writeln(self, line): class _PlistWriter(_DumbXMLWriter): def __init__( self, file, indent_level=0, indent=b"\t", writeHeader=1, - sort_keys=True, skipkeys=False): + sort_keys=True, skipkeys=False, aware_datetime=False): if writeHeader: file.write(PLISTHEADER) _DumbXMLWriter.__init__(self, file, indent_level, indent) self._sort_keys = sort_keys self._skipkeys = skipkeys + self._aware_datetime = aware_datetime def write(self, value): self.writeln("") @@ -360,7 +367,8 @@ def write_value(self, value): self.write_bytes(value) elif isinstance(value, datetime.datetime): - self.simple_element("date", _date_to_string(value)) + self.simple_element("date", + _date_to_string(value, self._aware_datetime)) elif isinstance(value, (tuple, list)): self.write_array(value) @@ -461,8 +469,9 @@ class _BinaryPlistParser: see also: http://opensource.apple.com/source/CF/CF-744.18/CFBinaryPList.c """ - def __init__(self, dict_type): + def __init__(self, dict_type, aware_datetime=False): self._dict_type = dict_type + self._aware_datime = aware_datetime def parse(self, fp): try: @@ -556,8 +565,11 @@ def _read_object(self, ref): f = struct.unpack('>d', self._fp.read(8))[0] # timestamp 0 of binary plists corresponds to 1/1/2001 # (year of Mac OS X 10.0), instead of 1/1/1970. - result = (datetime.datetime(2001, 1, 1) + - datetime.timedelta(seconds=f)) + if self._aware_datime: + epoch = datetime.datetime(2001, 1, 1, tzinfo=datetime.UTC) + else: + epoch = datetime.datetime(2001, 1, 1) + result = epoch + datetime.timedelta(seconds=f) elif tokenH == 0x40: # data s = self._get_size(tokenL) @@ -588,7 +600,8 @@ def _read_object(self, ref): obj_refs = self._read_refs(s) result = [] self._objects[ref] = result - result.extend(self._read_object(x) for x in obj_refs) + for x in obj_refs: + result.append(self._read_object(x)) # tokenH == 0xB0 is documented as 'ordset', but is not actually # implemented in the Apple reference code. @@ -629,10 +642,11 @@ def _count_to_size(count): _scalars = (str, int, float, datetime.datetime, bytes) class _BinaryPlistWriter (object): - def __init__(self, fp, sort_keys, skipkeys): + def __init__(self, fp, sort_keys, skipkeys, aware_datetime=False): self._fp = fp self._sort_keys = sort_keys self._skipkeys = skipkeys + self._aware_datetime = aware_datetime def write(self, value): @@ -778,7 +792,12 @@ def _write_object(self, value): self._fp.write(struct.pack('>Bd', 0x23, value)) elif isinstance(value, datetime.datetime): - f = (value - datetime.datetime(2001, 1, 1)).total_seconds() + if self._aware_datetime: + dt = value.astimezone(datetime.UTC) + offset = dt - datetime.datetime(2001, 1, 1, tzinfo=datetime.UTC) + f = offset.total_seconds() + else: + f = (value - datetime.datetime(2001, 1, 1)).total_seconds() self._fp.write(struct.pack('>Bd', 0x33, f)) elif isinstance(value, (bytes, bytearray)): @@ -862,7 +881,7 @@ def _is_fmt_binary(header): } -def load(fp, *, fmt=None, dict_type=dict): +def load(fp, *, fmt=None, dict_type=dict, aware_datetime=False): """Read a .plist file. 'fp' should be a readable and binary file object. Return the unpacked root object (which usually is a dictionary). """ @@ -880,32 +899,41 @@ def load(fp, *, fmt=None, dict_type=dict): else: P = _FORMATS[fmt]['parser'] - p = P(dict_type=dict_type) + p = P(dict_type=dict_type, aware_datetime=aware_datetime) return p.parse(fp) -def loads(value, *, fmt=None, dict_type=dict): +def loads(value, *, fmt=None, dict_type=dict, aware_datetime=False): """Read a .plist file from a bytes object. Return the unpacked root object (which usually is a dictionary). """ + if isinstance(value, str): + if fmt == FMT_BINARY: + raise TypeError("value must be bytes-like object when fmt is " + "FMT_BINARY") + value = value.encode() fp = BytesIO(value) - return load(fp, fmt=fmt, dict_type=dict_type) + return load(fp, fmt=fmt, dict_type=dict_type, aware_datetime=aware_datetime) -def dump(value, fp, *, fmt=FMT_XML, sort_keys=True, skipkeys=False): +def dump(value, fp, *, fmt=FMT_XML, sort_keys=True, skipkeys=False, + aware_datetime=False): """Write 'value' to a .plist file. 'fp' should be a writable, binary file object. """ if fmt not in _FORMATS: raise ValueError("Unsupported format: %r"%(fmt,)) - writer = _FORMATS[fmt]["writer"](fp, sort_keys=sort_keys, skipkeys=skipkeys) + writer = _FORMATS[fmt]["writer"](fp, sort_keys=sort_keys, skipkeys=skipkeys, + aware_datetime=aware_datetime) writer.write(value) -def dumps(value, *, fmt=FMT_XML, skipkeys=False, sort_keys=True): +def dumps(value, *, fmt=FMT_XML, skipkeys=False, sort_keys=True, + aware_datetime=False): """Return a bytes object with the contents for a .plist file. """ fp = BytesIO() - dump(value, fp, fmt=fmt, skipkeys=skipkeys, sort_keys=sort_keys) + dump(value, fp, fmt=fmt, skipkeys=skipkeys, sort_keys=sort_keys, + aware_datetime=aware_datetime) return fp.getvalue() diff --git a/Lib/pydoc.py b/Lib/pydoc.py index 83c74a75cd1c00..96aa1dfc1aacf6 100755 --- a/Lib/pydoc.py +++ b/Lib/pydoc.py @@ -345,6 +345,8 @@ def sort_attributes(attrs, object): def ispackage(path): """Guess whether a path refers to a package directory.""" + warnings.warn('The pydoc.ispackage() function is deprecated', + DeprecationWarning, stacklevel=2) if os.path.isdir(path): for ext in ('.py', '.pyc'): if os.path.isfile(os.path.join(path, '__init__' + ext)): diff --git a/Lib/pydoc_data/topics.py b/Lib/pydoc_data/topics.py index 7c1bdc4dff2ec4..d453c1f038ef07 100644 --- a/Lib/pydoc_data/topics.py +++ b/Lib/pydoc_data/topics.py @@ -1,5 +1,5 @@ # -*- coding: utf-8 -*- -# Autogenerated by Sphinx on Wed Nov 22 11:44:32 2023 +# Autogenerated by Sphinx on Wed Jan 17 13:09:41 2024 # as part of the release process. topics = {'assert': 'The "assert" statement\n' '**********************\n' @@ -864,19 +864,25 @@ '*instance* of the\n' ' owner class.\n' '\n' - 'The attribute "__objclass__" is interpreted by the ' - '"inspect" module as\n' - 'specifying the class where this object was defined ' + 'Instances of descriptors may also have the ' + '"__objclass__" attribute\n' + 'present:\n' + '\n' + 'object.__objclass__\n' + '\n' + ' The attribute "__objclass__" is interpreted by the ' + '"inspect" module\n' + ' as specifying the class where this object was defined ' '(setting this\n' - 'appropriately can assist in runtime introspection of ' + ' appropriately can assist in runtime introspection of ' 'dynamic class\n' - 'attributes). For callables, it may indicate that an ' + ' attributes). For callables, it may indicate that an ' 'instance of the\n' - 'given type (or a subclass) is expected or required as ' + ' given type (or a subclass) is expected or required as ' 'the first\n' - 'positional argument (for example, CPython sets this ' + ' positional argument (for example, CPython sets this ' 'attribute for\n' - 'unbound methods that are implemented in C).\n' + ' unbound methods that are implemented in C).\n' '\n' '\n' 'Invoking Descriptors\n' @@ -1111,16 +1117,23 @@ 'attribute references, which most objects do. This ' 'object is then\n' 'asked to produce the attribute whose name is the ' - 'identifier. This\n' - 'production can be customized by overriding the ' - '"__getattr__()" method.\n' - 'If this attribute is not available, the exception ' - '"AttributeError" is\n' - 'raised. Otherwise, the type and value of the object ' - 'produced is\n' - 'determined by the object. Multiple evaluations of ' - 'the same attribute\n' - 'reference may yield different objects.\n', + 'identifier. The type\n' + 'and value produced is determined by the object. ' + 'Multiple evaluations\n' + 'of the same attribute reference may yield different ' + 'objects.\n' + '\n' + 'This production can be customized by overriding the\n' + '"__getattribute__()" method or the "__getattr__()" ' + 'method. The\n' + '"__getattribute__()" method is called first and ' + 'either returns a value\n' + 'or raises "AttributeError" if the attribute is not ' + 'available.\n' + '\n' + 'If an "AttributeError" is raised and the object has ' + 'a "__getattr__()"\n' + 'method, that method is called as a fallback.\n', 'augassign': 'Augmented assignment statements\n' '*******************************\n' '\n' @@ -3045,8 +3058,7 @@ 'standard\n' 'Python grammar. Triple-quoted strings are supported. Raw ' 'strings and\n' - 'byte strings are supported. Formatted string literals are not\n' - 'supported.\n' + 'byte strings are supported. f-strings are not supported.\n' '\n' 'The forms "signed_number \'+\' NUMBER" and "signed_number \'-\' ' 'NUMBER"\n' @@ -3700,7 +3712,7 @@ ' **PEP 526** - Syntax for Variable Annotations\n' ' Ability to type hint variable declarations, including ' 'class\n' - ' variables and instance variables\n' + ' variables and instance variables.\n' '\n' ' **PEP 563** - Postponed Evaluation of Annotations\n' ' Support for forward references within annotations by ' @@ -3708,6 +3720,11 @@ ' annotations in a string form at runtime instead of eager\n' ' evaluation.\n' '\n' + ' **PEP 318** - Decorators for Functions and Methods\n' + ' Function and method decorators were introduced. Class ' + 'decorators\n' + ' were introduced in **PEP 3129**.\n' + '\n' '\n' 'Class definitions\n' '=================\n' @@ -4824,8 +4841,8 @@ 'denial-of-service caused\n' ' by carefully chosen inputs that exploit the worst ' 'case\n' - ' performance of a dict insertion, O(n^2) complexity. ' - 'See\n' + ' performance of a dict insertion, *O*(*n*^2) ' + 'complexity. See\n' ' http://ocert.org/advisories/ocert-2011-003.html for\n' ' details.Changing hash values affects the iteration ' 'order of sets.\n' @@ -5565,10 +5582,31 @@ 'whose\n' ' global namespace contains all the (global and local) names ' 'found in\n' - ' the current scope.\n' + ' the current scope. Use "exit()" or "quit()" to exit the ' + 'interpreter\n' + ' and return to the debugger.\n' + '\n' + ' Note:\n' + '\n' + ' Because interact creates a new global namespace with the ' + 'current\n' + ' global and local namespace for execution, assignment to ' + 'variables\n' + ' will not affect the original namespaces. However, ' + 'modification to\n' + ' the mutable objects will be reflected in the original ' + 'namespaces.\n' '\n' ' New in version 3.2.\n' '\n' + ' New in version 3.13: "exit()" and "quit()" can be used to ' + 'exit\n' + ' "interact" command.\n' + '\n' + ' Changed in version 3.13: "interact" directs its output to ' + 'the\n' + ' debugger’s output channel rather than "sys.stderr".\n' + '\n' 'alias [name [command]]\n' '\n' ' Create an alias called *name* that executes *command*. The\n' @@ -6554,7 +6592,7 @@ ' index_string ::= ' '+\n' ' conversion ::= "r" | "s" | "a"\n' - ' format_spec ::= \n' + ' format_spec ::= format-spec:format_spec\n' '\n' 'In less formal terms, the replacement field can start with ' 'a\n' @@ -6689,12 +6727,11 @@ 'contained\n' 'within a format string to define how individual values are ' 'presented\n' - '(see Format String Syntax and Formatted string literals). ' - 'They can\n' - 'also be passed directly to the built-in "format()" ' - 'function. Each\n' - 'formattable type may define how the format specification is ' - 'to be\n' + '(see Format String Syntax and f-strings). They can also be ' + 'passed\n' + 'directly to the built-in "format()" function. Each ' + 'formattable type\n' + 'may define how the format specification is to be ' 'interpreted.\n' '\n' 'Most built-in types implement the following options for ' @@ -7509,13 +7546,18 @@ ' **PEP 526** - Syntax for Variable Annotations\n' ' Ability to type hint variable declarations, including ' 'class\n' - ' variables and instance variables\n' + ' variables and instance variables.\n' '\n' ' **PEP 563** - Postponed Evaluation of Annotations\n' ' Support for forward references within annotations by ' 'preserving\n' ' annotations in a string form at runtime instead of eager\n' - ' evaluation.\n', + ' evaluation.\n' + '\n' + ' **PEP 318** - Decorators for Functions and Methods\n' + ' Function and method decorators were introduced. Class ' + 'decorators\n' + ' were introduced in **PEP 3129**.\n', 'global': 'The "global" statement\n' '**********************\n' '\n' @@ -9212,15 +9254,13 @@ '\n' 'A traceback object is normally created automatically when an ' 'exception\n' - 'is raised and attached to it as the "__traceback__" attribute, ' - 'which\n' - 'is writable. You can create an exception and set your own traceback ' - 'in\n' - 'one step using the "with_traceback()" exception method (which ' - 'returns\n' - 'the same exception instance, with its traceback set to its ' - 'argument),\n' - 'like so:\n' + 'is raised and attached to it as the "__traceback__" attribute. You ' + 'can\n' + 'create an exception and set your own traceback in one step using ' + 'the\n' + '"with_traceback()" exception method (which returns the same ' + 'exception\n' + 'instance, with its traceback set to its argument), like so:\n' '\n' ' raise Exception("foo occurred").with_traceback(tracebackobj)\n' '\n' @@ -9246,6 +9286,8 @@ ' ...\n' ' Traceback (most recent call last):\n' ' File "", line 2, in \n' + ' print(1 / 0)\n' + ' ~~^~~\n' ' ZeroDivisionError: division by zero\n' '\n' ' The above exception was the direct cause of the following ' @@ -9253,6 +9295,7 @@ '\n' ' Traceback (most recent call last):\n' ' File "", line 4, in \n' + ' raise RuntimeError("Something bad happened") from exc\n' ' RuntimeError: Something bad happened\n' '\n' 'A similar mechanism works implicitly if a new exception is raised ' @@ -9271,6 +9314,8 @@ ' ...\n' ' Traceback (most recent call last):\n' ' File "", line 2, in \n' + ' print(1 / 0)\n' + ' ~~^~~\n' ' ZeroDivisionError: division by zero\n' '\n' ' During handling of the above exception, another exception ' @@ -9278,6 +9323,7 @@ '\n' ' Traceback (most recent call last):\n' ' File "", line 4, in \n' + ' raise RuntimeError("Something bad happened")\n' ' RuntimeError: Something bad happened\n' '\n' 'Exception chaining can be explicitly suppressed by specifying ' @@ -9466,23 +9512,20 @@ '\n' ' Called to implement evaluation of "self[key]". For ' '*sequence*\n' - ' types, the accepted keys should be integers and slice ' - 'objects.\n' - ' Note that the special interpretation of negative ' - 'indexes (if the\n' - ' class wishes to emulate a *sequence* type) is up to ' - 'the\n' - ' "__getitem__()" method. If *key* is of an inappropriate ' - 'type,\n' - ' "TypeError" may be raised; if of a value outside the ' - 'set of indexes\n' - ' for the sequence (after any special interpretation of ' - 'negative\n' - ' values), "IndexError" should be raised. For *mapping* ' - 'types, if\n' - ' *key* is missing (not in the container), "KeyError" ' - 'should be\n' - ' raised.\n' + ' types, the accepted keys should be integers. ' + 'Optionally, they may\n' + ' support "slice" objects as well. Negative index ' + 'support is also\n' + ' optional. If *key* is of an inappropriate type, ' + '"TypeError" may be\n' + ' raised; if *key* is a value outside the set of indexes ' + 'for the\n' + ' sequence (after any special interpretation of negative ' + 'values),\n' + ' "IndexError" should be raised. For *mapping* types, if ' + '*key* is\n' + ' missing (not in the container), "KeyError" should be ' + 'raised.\n' '\n' ' Note:\n' '\n' @@ -10204,8 +10247,8 @@ ' intended to provide protection against a ' 'denial-of-service caused\n' ' by carefully chosen inputs that exploit the worst case\n' - ' performance of a dict insertion, O(n^2) complexity. ' - 'See\n' + ' performance of a dict insertion, *O*(*n*^2) ' + 'complexity. See\n' ' http://ocert.org/advisories/ocert-2011-003.html for\n' ' details.Changing hash values affects the iteration ' 'order of sets.\n' @@ -10483,19 +10526,25 @@ 'of the\n' ' owner class.\n' '\n' - 'The attribute "__objclass__" is interpreted by the "inspect" ' - 'module as\n' - 'specifying the class where this object was defined (setting ' - 'this\n' - 'appropriately can assist in runtime introspection of dynamic ' - 'class\n' - 'attributes). For callables, it may indicate that an instance ' - 'of the\n' - 'given type (or a subclass) is expected or required as the ' + 'Instances of descriptors may also have the "__objclass__" ' + 'attribute\n' + 'present:\n' + '\n' + 'object.__objclass__\n' + '\n' + ' The attribute "__objclass__" is interpreted by the ' + '"inspect" module\n' + ' as specifying the class where this object was defined ' + '(setting this\n' + ' appropriately can assist in runtime introspection of ' + 'dynamic class\n' + ' attributes). For callables, it may indicate that an ' + 'instance of the\n' + ' given type (or a subclass) is expected or required as the ' 'first\n' - 'positional argument (for example, CPython sets this ' + ' positional argument (for example, CPython sets this ' 'attribute for\n' - 'unbound methods that are implemented in C).\n' + ' unbound methods that are implemented in C).\n' '\n' '\n' 'Invoking Descriptors\n' @@ -10742,7 +10791,7 @@ '\n' ' Keyword arguments which are given to a new class are ' 'passed to the\n' - ' parent’s class "__init_subclass__". For compatibility ' + ' parent class’s "__init_subclass__". For compatibility ' 'with other\n' ' classes using "__init_subclass__", one should take out ' 'the needed\n' @@ -11429,22 +11478,20 @@ '\n' ' Called to implement evaluation of "self[key]". For ' '*sequence*\n' - ' types, the accepted keys should be integers and slice ' - 'objects.\n' - ' Note that the special interpretation of negative indexes ' - '(if the\n' - ' class wishes to emulate a *sequence* type) is up to the\n' - ' "__getitem__()" method. If *key* is of an inappropriate ' - 'type,\n' - ' "TypeError" may be raised; if of a value outside the set ' - 'of indexes\n' - ' for the sequence (after any special interpretation of ' - 'negative\n' - ' values), "IndexError" should be raised. For *mapping* ' - 'types, if\n' - ' *key* is missing (not in the container), "KeyError" ' - 'should be\n' - ' raised.\n' + ' types, the accepted keys should be integers. Optionally, ' + 'they may\n' + ' support "slice" objects as well. Negative index support ' + 'is also\n' + ' optional. If *key* is of an inappropriate type, ' + '"TypeError" may be\n' + ' raised; if *key* is a value outside the set of indexes ' + 'for the\n' + ' sequence (after any special interpretation of negative ' + 'values),\n' + ' "IndexError" should be raised. For *mapping* types, if ' + '*key* is\n' + ' missing (not in the container), "KeyError" should be ' + 'raised.\n' '\n' ' Note:\n' '\n' @@ -13043,12 +13090,11 @@ '\n' 'A string literal with "\'f\'" or "\'F\'" in its prefix is a ' '*formatted\n' - 'string literal*; see Formatted string literals. The "\'f\'" may ' - 'be\n' - 'combined with "\'r\'", but not with "\'b\'" or "\'u\'", therefore ' - 'raw\n' - 'formatted strings are possible, but formatted bytes literals are ' - 'not.\n' + 'string literal*; see f-strings. The "\'f\'" may be combined with ' + '"\'r\'",\n' + 'but not with "\'b\'" or "\'u\'", therefore raw formatted strings ' + 'are\n' + 'possible, but formatted bytes literals are not.\n' '\n' 'In triple-quoted literals, unescaped newlines and quotes are ' 'allowed\n' @@ -13952,130 +13998,117 @@ 'function’s\n' 'formal parameter list.\n' '\n' - 'Special attributes:\n' '\n' - '+---------------------------+---------------------------------+-------------+\n' - '| Attribute | Meaning ' - '| |\n' - '|===========================|=================================|=============|\n' - '| "__doc__" | The function’s documentation | ' - 'Writable |\n' - '| | string, or "None" if ' - '| |\n' - '| | unavailable; not inherited by ' - '| |\n' - '| | subclasses. ' - '| |\n' - '+---------------------------+---------------------------------+-------------+\n' - '| "__name__" | The function’s name. | ' - 'Writable |\n' - '+---------------------------+---------------------------------+-------------+\n' - '| "__qualname__" | The function’s *qualified | ' - 'Writable |\n' - '| | name*. New in version 3.3. ' - '| |\n' - '+---------------------------+---------------------------------+-------------+\n' - '| "__module__" | The name of the module the | ' - 'Writable |\n' - '| | function was defined in, or ' - '| |\n' - '| | "None" if unavailable. ' - '| |\n' - '+---------------------------+---------------------------------+-------------+\n' - '| "__defaults__" | A tuple containing default | ' - 'Writable |\n' - '| | argument values for those ' - '| |\n' - '| | arguments that have defaults, ' - '| |\n' - '| | or "None" if no arguments have ' - '| |\n' - '| | a default value. ' - '| |\n' - '+---------------------------+---------------------------------+-------------+\n' - '| "__code__" | The code object representing | ' - 'Writable |\n' - '| | the compiled function body. ' - '| |\n' - '+---------------------------+---------------------------------+-------------+\n' - '| "__globals__" | A reference to the dictionary | ' - 'Read-only |\n' - '| | that holds the function’s ' - '| |\n' - '| | global variables — the global ' - '| |\n' - '| | namespace of the module in ' - '| |\n' - '| | which the function was defined. ' - '| |\n' - '+---------------------------+---------------------------------+-------------+\n' - '| "__dict__" | The namespace supporting | ' - 'Writable |\n' - '| | arbitrary function attributes. ' - '| |\n' - '+---------------------------+---------------------------------+-------------+\n' - '| "__closure__" | "None" or a tuple of cells that | ' - 'Read-only |\n' - '| | contain bindings for the ' - '| |\n' - '| | function’s free variables. See ' - '| |\n' - '| | below for information on the ' - '| |\n' - '| | "cell_contents" attribute. ' - '| |\n' - '+---------------------------+---------------------------------+-------------+\n' - '| "__annotations__" | A dict containing annotations | ' - 'Writable |\n' - '| | of parameters. The keys of the ' - '| |\n' - '| | dict are the parameter names, ' - '| |\n' - '| | and "\'return\'" for the return ' - '| |\n' - '| | annotation, if provided. For ' - '| |\n' - '| | more information on working ' - '| |\n' - '| | with this attribute, see ' - '| |\n' - '| | Annotations Best Practices. ' - '| |\n' - '+---------------------------+---------------------------------+-------------+\n' - '| "__kwdefaults__" | A dict containing defaults for | ' - 'Writable |\n' - '| | keyword-only parameters. ' - '| |\n' - '+---------------------------+---------------------------------+-------------+\n' - '| "__type_params__" | A tuple containing the type | ' - 'Writable |\n' - '| | parameters of a generic ' - '| |\n' - '| | function. ' - '| |\n' - '+---------------------------+---------------------------------+-------------+\n' - '\n' - 'Most of the attributes labelled “Writable” check the type of the\n' - 'assigned value.\n' + 'Special read-only attributes\n' + '~~~~~~~~~~~~~~~~~~~~~~~~~~~~\n' + '\n' + '+----------------------------------------------------+----------------------------------------------------+\n' + '| Attribute | ' + 'Meaning |\n' + '|====================================================|====================================================|\n' + '| function.__globals__ | A reference ' + 'to the "dictionary" that holds the |\n' + '| | function’s ' + 'global variables – the global namespace |\n' + '| | of the ' + 'module in which the function was defined. |\n' + '+----------------------------------------------------+----------------------------------------------------+\n' + '| function.__closure__ | "None" or a ' + '"tuple" of cells that contain bindings |\n' + '| | for the ' + 'function’s free variables. A cell object |\n' + '| | has the ' + 'attribute "cell_contents". This can be |\n' + '| | used to get ' + 'the value of the cell, as well as set |\n' + '| | the ' + 'value. |\n' + '+----------------------------------------------------+----------------------------------------------------+\n' + '\n' + '\n' + 'Special writable attributes\n' + '~~~~~~~~~~~~~~~~~~~~~~~~~~~\n' + '\n' + 'Most of these attributes check the type of the assigned value:\n' + '\n' + '+----------------------------------------------------+----------------------------------------------------+\n' + '| Attribute | ' + 'Meaning |\n' + '|====================================================|====================================================|\n' + '| function.__doc__ | The ' + 'function’s documentation string, or "None" if |\n' + '| | unavailable. ' + 'Not inherited by subclasses. |\n' + '+----------------------------------------------------+----------------------------------------------------+\n' + '| function.__name__ | The ' + 'function’s name. See also: "__name__ |\n' + '| | ' + 'attributes". |\n' + '+----------------------------------------------------+----------------------------------------------------+\n' + '| function.__qualname__ | The ' + 'function’s *qualified name*. See also: |\n' + '| | ' + '"__qualname__ attributes". New in version 3.3. |\n' + '+----------------------------------------------------+----------------------------------------------------+\n' + '| function.__module__ | The name of ' + 'the module the function was defined |\n' + '| | in, or ' + '"None" if unavailable. |\n' + '+----------------------------------------------------+----------------------------------------------------+\n' + '| function.__defaults__ | A "tuple" ' + 'containing default *parameter* values |\n' + '| | for those ' + 'parameters that have defaults, or "None" |\n' + '| | if no ' + 'parameters have a default value. |\n' + '+----------------------------------------------------+----------------------------------------------------+\n' + '| function.__code__ | The code ' + 'object representing the compiled function |\n' + '| | ' + 'body. |\n' + '+----------------------------------------------------+----------------------------------------------------+\n' + '| function.__dict__ | The ' + 'namespace supporting arbitrary function |\n' + '| | attributes. ' + 'See also: "__dict__ attributes". |\n' + '+----------------------------------------------------+----------------------------------------------------+\n' + '| function.__annotations__ | A ' + '"dictionary" containing annotations of |\n' + '| | ' + '*parameters*. The keys of the dictionary are the |\n' + '| | parameter ' + 'names, and "\'return\'" for the return |\n' + '| | annotation, ' + 'if provided. See also: Annotations |\n' + '| | Best ' + 'Practices. |\n' + '+----------------------------------------------------+----------------------------------------------------+\n' + '| function.__kwdefaults__ | A ' + '"dictionary" containing defaults for keyword- |\n' + '| | only ' + '*parameters*. |\n' + '+----------------------------------------------------+----------------------------------------------------+\n' + '| function.__type_params__ | A "tuple" ' + 'containing the type parameters of a |\n' + '| | generic ' + 'function. New in version 3.12. |\n' + '+----------------------------------------------------+----------------------------------------------------+\n' '\n' 'Function objects also support getting and setting arbitrary\n' 'attributes, which can be used, for example, to attach metadata to\n' 'functions. Regular attribute dot-notation is used to get and set ' 'such\n' - 'attributes. *Note that the current implementation only supports\n' - 'function attributes on user-defined functions. Function attributes ' - 'on\n' - 'built-in functions may be supported in the future.*\n' + 'attributes.\n' '\n' - 'A cell object has the attribute "cell_contents". This can be used ' - 'to\n' - 'get the value of the cell, as well as set the value.\n' + '**CPython implementation detail:** CPython’s current ' + 'implementation\n' + 'only supports function attributes on user-defined functions. ' + 'Function\n' + 'attributes on built-in functions may be supported in the future.\n' '\n' 'Additional information about a function’s definition can be ' 'retrieved\n' - 'from its code object; see the description of internal types below. ' - 'The\n' - '"cell" type can be accessed in the "types" module.\n' + 'from its code object (accessible via the "__code__" attribute).\n' '\n' '\n' 'Instance methods\n' @@ -14085,14 +14118,34 @@ 'any\n' 'callable object (normally a user-defined function).\n' '\n' - 'Special read-only attributes: "__self__" is the class instance ' - 'object,\n' - '"__func__" is the function object; "__doc__" is the method’s\n' - 'documentation (same as "__func__.__doc__"); "__name__" is the ' - 'method\n' - 'name (same as "__func__.__name__"); "__module__" is the name of ' - 'the\n' - 'module the method was defined in, or "None" if unavailable.\n' + 'Special read-only attributes:\n' + '\n' + '+----------------------------------------------------+----------------------------------------------------+\n' + '| method.__self__ | Refers to ' + 'the class instance object to which the |\n' + '| | method is ' + 'bound |\n' + '+----------------------------------------------------+----------------------------------------------------+\n' + '| method.__func__ | Refers to ' + 'the original function object |\n' + '+----------------------------------------------------+----------------------------------------------------+\n' + '| method.__doc__ | The method’s ' + 'documentation (same as |\n' + '| | ' + '"method.__func__.__doc__"). A "string" if the |\n' + '| | original ' + 'function had a docstring, else "None". |\n' + '+----------------------------------------------------+----------------------------------------------------+\n' + '| method.__name__ | The name of ' + 'the method (same as |\n' + '| | ' + '"method.__func__.__name__") |\n' + '+----------------------------------------------------+----------------------------------------------------+\n' + '| method.__module__ | The name of ' + 'the module the method was defined in, |\n' + '| | or "None" if ' + 'unavailable. |\n' + '+----------------------------------------------------+----------------------------------------------------+\n' '\n' 'Methods also support accessing (but not setting) the arbitrary\n' 'function attributes on the underlying function object.\n' @@ -14101,24 +14154,20 @@ 'attribute\n' 'of a class (perhaps via an instance of that class), if that ' 'attribute\n' - 'is a user-defined function object or a class method object.\n' + 'is a user-defined function object or a "classmethod" object.\n' '\n' 'When an instance method object is created by retrieving a ' 'user-defined\n' 'function object from a class via one of its instances, its ' '"__self__"\n' - 'attribute is the instance, and the method object is said to be ' - 'bound.\n' - 'The new method’s "__func__" attribute is the original function ' - 'object.\n' + 'attribute is the instance, and the method object is said to be\n' + '*bound*. The new method’s "__func__" attribute is the original\n' + 'function object.\n' '\n' - 'When an instance method object is created by retrieving a class ' - 'method\n' - 'object from a class or instance, its "__self__" attribute is the ' - 'class\n' - 'itself, and its "__func__" attribute is the function object ' - 'underlying\n' - 'the class method.\n' + 'When an instance method object is created by retrieving a\n' + '"classmethod" object from a class or instance, its "__self__"\n' + 'attribute is the class itself, and its "__func__" attribute is the\n' + 'function object underlying the class method.\n' '\n' 'When an instance method object is called, the underlying function\n' '("__func__") is called, inserting the class instance ("__self__") ' @@ -14129,7 +14178,7 @@ 'of\n' '"C", calling "x.f(1)" is equivalent to calling "C.f(x, 1)".\n' '\n' - 'When an instance method object is derived from a class method ' + 'When an instance method object is derived from a "classmethod" ' 'object,\n' 'the “class instance” stored in "__self__" will actually be the ' 'class\n' @@ -14217,13 +14266,18 @@ 'of built-in functions are "len()" and "math.sin()" ("math" is a\n' 'standard built-in module). The number and type of the arguments ' 'are\n' - 'determined by the C function. Special read-only attributes: ' - '"__doc__"\n' - 'is the function’s documentation string, or "None" if unavailable;\n' - '"__name__" is the function’s name; "__self__" is set to "None" ' - '(but\n' - 'see the next item); "__module__" is the name of the module the\n' - 'function was defined in or "None" if unavailable.\n' + 'determined by the C function. Special read-only attributes:\n' + '\n' + '* "__doc__" is the function’s documentation string, or "None" if\n' + ' unavailable. See "function.__doc__".\n' + '\n' + '* "__name__" is the function’s name. See "function.__name__".\n' + '\n' + '* "__self__" is set to "None" (but see the next item).\n' + '\n' + '* "__module__" is the name of the module the function was defined ' + 'in\n' + ' or "None" if unavailable. See "function.__module__".\n' '\n' '\n' 'Built-in methods\n' @@ -14235,7 +14289,9 @@ 'argument. An example of a built-in method is "alist.append()",\n' 'assuming *alist* is a list object. In this case, the special ' 'read-only\n' - 'attribute "__self__" is set to the object denoted by *alist*.\n' + 'attribute "__self__" is set to the object denoted by *alist*. (The\n' + 'attribute has the same semantics as it does with "other instance\n' + 'methods".)\n' '\n' '\n' 'Classes\n' @@ -14267,16 +14323,15 @@ 'statement, or by calling functions such as ' '"importlib.import_module()"\n' 'and built-in "__import__()". A module object has a namespace\n' - 'implemented by a dictionary object (this is the dictionary ' - 'referenced\n' - 'by the "__globals__" attribute of functions defined in the ' - 'module).\n' - 'Attribute references are translated to lookups in this dictionary,\n' - 'e.g., "m.x" is equivalent to "m.__dict__["x"]". A module object ' - 'does\n' - 'not contain the code object used to initialize the module (since ' - 'it\n' - 'isn’t needed once the initialization is done).\n' + 'implemented by a "dictionary" object (this is the dictionary\n' + 'referenced by the "__globals__" attribute of functions defined in ' + 'the\n' + 'module). Attribute references are translated to lookups in this\n' + 'dictionary, e.g., "m.x" is equivalent to "m.__dict__["x"]". A ' + 'module\n' + 'object does not contain the code object used to initialize the ' + 'module\n' + '(since it isn’t needed once the initialization is done).\n' '\n' 'Attribute assignment updates the module’s namespace dictionary, ' 'e.g.,\n' @@ -14350,14 +14405,13 @@ 'a\n' 'class method object, it is transformed into an instance method ' 'object\n' - 'whose "__self__" attribute is "C". When it would yield a static\n' - 'method object, it is transformed into the object wrapped by the ' - 'static\n' - 'method object. See section Implementing Descriptors for another way ' - 'in\n' - 'which attributes retrieved from a class may differ from those ' - 'actually\n' - 'contained in its "__dict__".\n' + 'whose "__self__" attribute is "C". When it would yield a\n' + '"staticmethod" object, it is transformed into the object wrapped ' + 'by\n' + 'the static method object. See section Implementing Descriptors for\n' + 'another way in which attributes retrieved from a class may differ ' + 'from\n' + 'those actually contained in its "__dict__".\n' '\n' 'Class attribute assignments update the class’s dictionary, never ' 'the\n' @@ -14480,43 +14534,106 @@ 'code objects are immutable and contain no references (directly or\n' 'indirectly) to mutable objects.\n' '\n' - 'Special read-only attributes: "co_name" gives the function name;\n' - '"co_qualname" gives the fully qualified function name; ' - '"co_argcount"\n' - 'is the total number of positional arguments (including ' - 'positional-only\n' - 'arguments and arguments with default values); "co_posonlyargcount" ' - 'is\n' - 'the number of positional-only arguments (including arguments with\n' - 'default values); "co_kwonlyargcount" is the number of keyword-only\n' - 'arguments (including arguments with default values); "co_nlocals" ' - 'is\n' - 'the number of local variables used by the function (including\n' - 'arguments); "co_varnames" is a tuple containing the names of the ' - 'local\n' - 'variables (starting with the argument names); "co_cellvars" is a ' - 'tuple\n' - 'containing the names of local variables that are referenced by ' - 'nested\n' - 'functions; "co_freevars" is a tuple containing the names of free\n' - 'variables; "co_code" is a string representing the sequence of ' - 'bytecode\n' - 'instructions; "co_consts" is a tuple containing the literals used ' - 'by\n' - 'the bytecode; "co_names" is a tuple containing the names used by ' - 'the\n' - 'bytecode; "co_filename" is the filename from which the code was\n' - 'compiled; "co_firstlineno" is the first line number of the ' - 'function;\n' - '"co_lnotab" is a string encoding the mapping from bytecode offsets ' - 'to\n' - 'line numbers (for details see the source code of the interpreter, ' - 'is\n' - 'deprecated since 3.12 and may be removed in 3.14); "co_stacksize" ' - 'is\n' - 'the required stack size; "co_flags" is an integer encoding a number ' - 'of\n' - 'flags for the interpreter.\n' + '\n' + 'Special read-only attributes\n' + '~~~~~~~~~~~~~~~~~~~~~~~~~~~~\n' + '\n' + '+----------------------------------------------------+----------------------------------------------------+\n' + '| codeobject.co_name | The function ' + 'name |\n' + '+----------------------------------------------------+----------------------------------------------------+\n' + '| codeobject.co_qualname | The fully ' + 'qualified function name |\n' + '+----------------------------------------------------+----------------------------------------------------+\n' + '| codeobject.co_argcount | The total ' + 'number of positional *parameters* |\n' + '| | (including ' + 'positional-only parameters and |\n' + '| | parameters ' + 'with default values) that the function |\n' + '| | ' + 'has |\n' + '+----------------------------------------------------+----------------------------------------------------+\n' + '| codeobject.co_posonlyargcount | The number ' + 'of positional-only *parameters* |\n' + '| | (including ' + 'arguments with default values) that the |\n' + '| | function ' + 'has |\n' + '+----------------------------------------------------+----------------------------------------------------+\n' + '| codeobject.co_kwonlyargcount | The number ' + 'of keyword-only *parameters* (including |\n' + '| | arguments ' + 'with default values) that the function |\n' + '| | ' + 'has |\n' + '+----------------------------------------------------+----------------------------------------------------+\n' + '| codeobject.co_nlocals | The number ' + 'of local variables used by the function |\n' + '| | (including ' + 'parameters) |\n' + '+----------------------------------------------------+----------------------------------------------------+\n' + '| codeobject.co_varnames | A "tuple" ' + 'containing the names of the local |\n' + '| | variables in ' + 'the function (starting with the |\n' + '| | parameter ' + 'names) |\n' + '+----------------------------------------------------+----------------------------------------------------+\n' + '| codeobject.co_cellvars | A "tuple" ' + 'containing the names of local variables |\n' + '| | that are ' + 'referenced by nested functions inside the |\n' + '| | ' + 'function |\n' + '+----------------------------------------------------+----------------------------------------------------+\n' + '| codeobject.co_freevars | A "tuple" ' + 'containing the names of free variables |\n' + '| | in the ' + 'function |\n' + '+----------------------------------------------------+----------------------------------------------------+\n' + '| codeobject.co_code | A string ' + 'representing the sequence of *bytecode* |\n' + '| | instructions ' + 'in the function |\n' + '+----------------------------------------------------+----------------------------------------------------+\n' + '| codeobject.co_consts | A "tuple" ' + 'containing the literals used by the |\n' + '| | *bytecode* ' + 'in the function |\n' + '+----------------------------------------------------+----------------------------------------------------+\n' + '| codeobject.co_names | A "tuple" ' + 'containing the names used by the |\n' + '| | *bytecode* ' + 'in the function |\n' + '+----------------------------------------------------+----------------------------------------------------+\n' + '| codeobject.co_filename | The name of ' + 'the file from which the code was |\n' + '| | ' + 'compiled |\n' + '+----------------------------------------------------+----------------------------------------------------+\n' + '| codeobject.co_firstlineno | The line ' + 'number of the first line of the function |\n' + '+----------------------------------------------------+----------------------------------------------------+\n' + '| codeobject.co_lnotab | A string ' + 'encoding the mapping from *bytecode* |\n' + '| | offsets to ' + 'line numbers. For details, see the |\n' + '| | source code ' + 'of the interpreter. Deprecated since |\n' + '| | version ' + '3.12: This attribute of code objects is |\n' + '| | deprecated, ' + 'and may be removed in Python 3.14. |\n' + '+----------------------------------------------------+----------------------------------------------------+\n' + '| codeobject.co_stacksize | The required ' + 'stack size of the code object |\n' + '+----------------------------------------------------+----------------------------------------------------+\n' + '| codeobject.co_flags | An "integer" ' + 'encoding a number of flags for the |\n' + '| | ' + 'interpreter. |\n' + '+----------------------------------------------------+----------------------------------------------------+\n' '\n' 'The following flag bits are defined for "co_flags": bit "0x04" is ' 'set\n' @@ -14525,7 +14642,9 @@ 'number of positional arguments; bit "0x08" is set if the function ' 'uses\n' 'the "**keywords" syntax to accept arbitrary keyword arguments; bit\n' - '"0x20" is set if the function is a generator.\n' + '"0x20" is set if the function is a generator. See Code Objects Bit\n' + 'Flags for details on the semantics of each flags that might be\n' + 'present.\n' '\n' 'Future feature declarations ("from __future__ import division") ' 'also\n' @@ -14544,16 +14663,19 @@ 'is the documentation string of the function, or "None" if ' 'undefined.\n' '\n' + '\n' + 'Methods on code objects\n' + '~~~~~~~~~~~~~~~~~~~~~~~\n' + '\n' 'codeobject.co_positions()\n' '\n' - ' Returns an iterable over the source code positions of each ' - 'bytecode\n' - ' instruction in the code object.\n' + ' Returns an iterable over the source code positions of each\n' + ' *bytecode* instruction in the code object.\n' '\n' - ' The iterator returns tuples containing the "(start_line, ' - 'end_line,\n' - ' start_column, end_column)". The *i-th* tuple corresponds to the\n' - ' position of the source code that compiled to the *i-th*\n' + ' The iterator returns "tuple"s containing the "(start_line,\n' + ' end_line, start_column, end_column)". The *i-th* tuple ' + 'corresponds\n' + ' to the position of the source code that compiled to the *i-th*\n' ' instruction. Column information is 0-indexed utf-8 byte offsets ' 'on\n' ' the given source line.\n' @@ -14591,51 +14713,153 @@ 'the\n' ' "PYTHONNODEBUGRANGES" environment variable can be used.\n' '\n' + 'codeobject.co_lines()\n' '\n' - 'Frame objects\n' - '-------------\n' + ' Returns an iterator that yields information about successive ' + 'ranges\n' + ' of *bytecode*s. Each item yielded is a "(start, end, lineno)"\n' + ' "tuple":\n' '\n' - 'Frame objects represent execution frames. They may occur in ' - 'traceback\n' - 'objects (see below), and are also passed to registered trace\n' - 'functions.\n' + ' * "start" (an "int") represents the offset (inclusive) of the ' + 'start\n' + ' of the *bytecode* range\n' '\n' - 'Special read-only attributes: "f_back" is to the previous stack ' - 'frame\n' - '(towards the caller), or "None" if this is the bottom stack frame;\n' - '"f_code" is the code object being executed in this frame; ' - '"f_locals"\n' - 'is the dictionary used to look up local variables; "f_globals" is ' - 'used\n' - 'for global variables; "f_builtins" is used for built-in ' - '(intrinsic)\n' - 'names; "f_lasti" gives the precise instruction (this is an index ' - 'into\n' - 'the bytecode string of the code object).\n' + ' * "end" (an "int") represents the offset (exclusive) of the end ' + 'of\n' + ' the *bytecode* range\n' '\n' - 'Accessing "f_code" raises an auditing event "object.__getattr__" ' - 'with\n' - 'arguments "obj" and ""f_code"".\n' + ' * "lineno" is an "int" representing the line number of the\n' + ' *bytecode* range, or "None" if the bytecodes in the given ' + 'range\n' + ' have no line number\n' '\n' - 'Special writable attributes: "f_trace", if not "None", is a ' - 'function\n' - 'called for various events during code execution (this is used by ' + ' The items yielded will have the following properties:\n' + '\n' + ' * The first range yielded will have a "start" of 0.\n' + '\n' + ' * The "(start, end)" ranges will be non-decreasing and ' + 'consecutive.\n' + ' That is, for any pair of "tuple"s, the "start" of the second ' + 'will\n' + ' be equal to the "end" of the first.\n' + '\n' + ' * No range will be backwards: "end >= start" for all triples.\n' + '\n' + ' * The last "tuple" yielded will have "end" equal to the size of ' 'the\n' - 'debugger). Normally an event is triggered for each new source line ' - '-\n' - 'this can be disabled by setting "f_trace_lines" to "False".\n' - '\n' - 'Implementations *may* allow per-opcode events to be requested by\n' - 'setting "f_trace_opcodes" to "True". Note that this may lead to\n' - 'undefined interpreter behaviour if exceptions raised by the trace\n' - 'function escape to the function being traced.\n' - '\n' - '"f_lineno" is the current line number of the frame — writing to ' - 'this\n' - 'from within a trace function jumps to the given line (only for the\n' - 'bottom-most frame). A debugger can implement a Jump command (aka ' - 'Set\n' - 'Next Statement) by writing to f_lineno.\n' + ' *bytecode*.\n' + '\n' + ' Zero-width ranges, where "start == end", are allowed. ' + 'Zero-width\n' + ' ranges are used for lines that are present in the source code, ' + 'but\n' + ' have been eliminated by the *bytecode* compiler.\n' + '\n' + ' New in version 3.10.\n' + '\n' + ' See also:\n' + '\n' + ' **PEP 626** - Precise line numbers for debugging and other ' + 'tools.\n' + ' The PEP that introduced the "co_lines()" method.\n' + '\n' + '\n' + 'Frame objects\n' + '-------------\n' + '\n' + 'Frame objects represent execution frames. They may occur in ' + 'traceback\n' + 'objects, and are also passed to registered trace functions.\n' + '\n' + '\n' + 'Special read-only attributes\n' + '~~~~~~~~~~~~~~~~~~~~~~~~~~~~\n' + '\n' + '+----------------------------------------------------+----------------------------------------------------+\n' + '| frame.f_back | Points to ' + 'the previous stack frame (towards the |\n' + '| | caller), or ' + '"None" if this is the bottom stack |\n' + '| | ' + 'frame |\n' + '+----------------------------------------------------+----------------------------------------------------+\n' + '| frame.f_code | The code ' + 'object being executed in this frame. |\n' + '| | Accessing ' + 'this attribute raises an auditing event |\n' + '| | ' + '"object.__getattr__" with arguments "obj" and |\n' + '| | ' + '""f_code"". |\n' + '+----------------------------------------------------+----------------------------------------------------+\n' + '| frame.f_locals | The ' + 'dictionary used by the frame to look up local |\n' + '| | ' + 'variables |\n' + '+----------------------------------------------------+----------------------------------------------------+\n' + '| frame.f_globals | The ' + 'dictionary used by the frame to look up global |\n' + '| | ' + 'variables |\n' + '+----------------------------------------------------+----------------------------------------------------+\n' + '| frame.f_builtins | The ' + 'dictionary used by the frame to look up built- |\n' + '| | in ' + '(intrinsic) names |\n' + '+----------------------------------------------------+----------------------------------------------------+\n' + '| frame.f_lasti | The “precise ' + 'instruction” of the frame object |\n' + '| | (this is an ' + 'index into the *bytecode* string of |\n' + '| | the code ' + 'object) |\n' + '+----------------------------------------------------+----------------------------------------------------+\n' + '\n' + '\n' + 'Special writable attributes\n' + '~~~~~~~~~~~~~~~~~~~~~~~~~~~\n' + '\n' + '+----------------------------------------------------+----------------------------------------------------+\n' + '| frame.f_trace | If not ' + '"None", this is a function called for |\n' + '| | various ' + 'events during code execution (this is used |\n' + '| | by ' + 'debuggers). Normally an event is triggered for |\n' + '| | each new ' + 'source line (see "f_trace_lines"). |\n' + '+----------------------------------------------------+----------------------------------------------------+\n' + '| frame.f_trace_lines | Set this ' + 'attribute to "False" to disable |\n' + '| | triggering a ' + 'tracing event for each source line. |\n' + '+----------------------------------------------------+----------------------------------------------------+\n' + '| frame.f_trace_opcodes | Set this ' + 'attribute to "True" to allow per-opcode |\n' + '| | events to be ' + 'requested. Note that this may lead to |\n' + '| | undefined ' + 'interpreter behaviour if exceptions |\n' + '| | raised by ' + 'the trace function escape to the |\n' + '| | function ' + 'being traced. |\n' + '+----------------------------------------------------+----------------------------------------------------+\n' + '| frame.f_lineno | The current ' + 'line number of the frame – writing to |\n' + '| | this from ' + 'within a trace function jumps to the |\n' + '| | given line ' + '(only for the bottom-most frame). A |\n' + '| | debugger can ' + 'implement a Jump command (aka Set |\n' + '| | Next ' + 'Statement) by writing to this attribute. |\n' + '+----------------------------------------------------+----------------------------------------------------+\n' + '\n' + '\n' + 'Frame object methods\n' + '~~~~~~~~~~~~~~~~~~~~\n' '\n' 'Frame objects support one method:\n' '\n' @@ -14643,7 +14867,7 @@ '\n' ' This method clears all references to local variables held by ' 'the\n' - ' frame. Also, if the frame belonged to a generator, the ' + ' frame. Also, if the frame belonged to a *generator*, the ' 'generator\n' ' is finalized. This helps break reference cycles involving ' 'frame\n' @@ -14664,11 +14888,14 @@ 'Traceback objects\n' '-----------------\n' '\n' - 'Traceback objects represent a stack trace of an exception. A\n' + 'Traceback objects represent the stack trace of an exception. A\n' 'traceback object is implicitly created when an exception occurs, ' 'and\n' 'may also be explicitly created by calling "types.TracebackType".\n' '\n' + 'Changed in version 3.7: Traceback objects can now be explicitly\n' + 'instantiated from Python code.\n' + '\n' 'For implicitly created tracebacks, when the search for an ' 'exception\n' 'handler unwinds the execution stack, at each unwound level a ' @@ -14691,30 +14918,40 @@ 'linked\n' 'to form a full stack trace.\n' '\n' - 'Special read-only attributes: "tb_frame" points to the execution ' - 'frame\n' - 'of the current level; "tb_lineno" gives the line number where the\n' - 'exception occurred; "tb_lasti" indicates the precise instruction. ' - 'The\n' - 'line number and last instruction in the traceback may differ from ' + 'Special read-only attributes:\n' + '\n' + '+----------------------------------------------------+----------------------------------------------------+\n' + '| traceback.tb_frame | Points to ' + 'the execution frame of the current |\n' + '| | level. ' + 'Accessing this attribute raises an |\n' + '| | auditing ' + 'event "object.__getattr__" with arguments |\n' + '| | "obj" and ' + '""tb_frame"". |\n' + '+----------------------------------------------------+----------------------------------------------------+\n' + '| traceback.tb_lineno | Gives the ' + 'line number where the exception occurred |\n' + '+----------------------------------------------------+----------------------------------------------------+\n' + '| traceback.tb_lasti | Indicates ' + 'the “precise instruction”. |\n' + '+----------------------------------------------------+----------------------------------------------------+\n' + '\n' + 'The line number and last instruction in the traceback may differ ' + 'from\n' + 'the line number of its frame object if the exception occurred in a\n' + '"try" statement with no matching except clause or with a "finally"\n' + 'clause.\n' + '\n' + 'traceback.tb_next\n' + '\n' + ' The special writable attribute "tb_next" is the next level in ' 'the\n' - 'line number of its frame object if the exception occurred in a ' - '"try"\n' - 'statement with no matching except clause or with a finally clause.\n' - '\n' - 'Accessing "tb_frame" raises an auditing event "object.__getattr__"\n' - 'with arguments "obj" and ""tb_frame"".\n' - '\n' - 'Special writable attribute: "tb_next" is the next level in the ' - 'stack\n' - 'trace (towards the frame where the exception occurred), or "None" ' - 'if\n' - 'there is no next level.\n' + ' stack trace (towards the frame where the exception occurred), ' + 'or\n' + ' "None" if there is no next level.\n' '\n' - 'Changed in version 3.7: Traceback objects can now be explicitly\n' - 'instantiated from Python code, and the "tb_next" attribute of ' - 'existing\n' - 'instances can be updated.\n' + ' Changed in version 3.7: This attribute is now writable\n' '\n' '\n' 'Slice objects\n' @@ -15291,7 +15528,7 @@ 'notation.\n' 'There are two flavors: built-in methods (such as "append()" ' 'on lists)\n' - 'and class instance methods. Built-in methods are described ' + 'and class instance method. Built-in methods are described ' 'with the\n' 'types that support them.\n' '\n' @@ -15299,8 +15536,8 @@ 'namespace)\n' 'through an instance, you get a special object: a *bound ' 'method* (also\n' - 'called *instance method*) object. When called, it will add ' - 'the "self"\n' + 'called instance method) object. When called, it will add the ' + '"self"\n' 'argument to the argument list. Bound methods have two ' 'special read-\n' 'only attributes: "m.__self__" is the object on which the ' @@ -15315,7 +15552,7 @@ 'arbitrary\n' 'attributes. However, since method attributes are actually ' 'stored on\n' - 'the underlying function object ("meth.__func__"), setting ' + 'the underlying function object ("method.__func__"), setting ' 'method\n' 'attributes on bound methods is disallowed. Attempting to ' 'set an\n' @@ -15340,7 +15577,7 @@ ' >>> c.method.whoami\n' " 'my name is method'\n" '\n' - 'See The standard type hierarchy for more information.\n', + 'See Instance methods for more information.\n', 'typesmodules': 'Modules\n' '*******\n' '\n' diff --git a/Lib/queue.py b/Lib/queue.py index 55f50088460f9e..467ff4fcecb134 100644 --- a/Lib/queue.py +++ b/Lib/queue.py @@ -25,6 +25,10 @@ class Full(Exception): pass +class ShutDown(Exception): + '''Raised when put/get with shut-down queue.''' + + class Queue: '''Create a queue object with a given maximum size. @@ -54,6 +58,9 @@ def __init__(self, maxsize=0): self.all_tasks_done = threading.Condition(self.mutex) self.unfinished_tasks = 0 + # Queue shutdown state + self.is_shutdown = False + def task_done(self): '''Indicate that a formerly enqueued task is complete. @@ -67,6 +74,8 @@ def task_done(self): Raises a ValueError if called more times than there were items placed in the queue. + + Raises ShutDown if the queue has been shut down immediately. ''' with self.all_tasks_done: unfinished = self.unfinished_tasks - 1 @@ -84,6 +93,8 @@ def join(self): to indicate the item was retrieved and all work on it is complete. When the count of unfinished tasks drops to zero, join() unblocks. + + Raises ShutDown if the queue has been shut down immediately. ''' with self.all_tasks_done: while self.unfinished_tasks: @@ -129,8 +140,12 @@ def put(self, item, block=True, timeout=None): Otherwise ('block' is false), put an item on the queue if a free slot is immediately available, else raise the Full exception ('timeout' is ignored in that case). + + Raises ShutDown if the queue has been shut down. ''' with self.not_full: + if self.is_shutdown: + raise ShutDown if self.maxsize > 0: if not block: if self._qsize() >= self.maxsize: @@ -138,6 +153,8 @@ def put(self, item, block=True, timeout=None): elif timeout is None: while self._qsize() >= self.maxsize: self.not_full.wait() + if self.is_shutdown: + raise ShutDown elif timeout < 0: raise ValueError("'timeout' must be a non-negative number") else: @@ -147,6 +164,8 @@ def put(self, item, block=True, timeout=None): if remaining <= 0.0: raise Full self.not_full.wait(remaining) + if self.is_shutdown: + raise ShutDown self._put(item) self.unfinished_tasks += 1 self.not_empty.notify() @@ -161,14 +180,21 @@ def get(self, block=True, timeout=None): Otherwise ('block' is false), return an item if one is immediately available, else raise the Empty exception ('timeout' is ignored in that case). + + Raises ShutDown if the queue has been shut down and is empty, + or if the queue has been shut down immediately. ''' with self.not_empty: + if self.is_shutdown and not self._qsize(): + raise ShutDown if not block: if not self._qsize(): raise Empty elif timeout is None: while not self._qsize(): self.not_empty.wait() + if self.is_shutdown and not self._qsize(): + raise ShutDown elif timeout < 0: raise ValueError("'timeout' must be a non-negative number") else: @@ -178,6 +204,8 @@ def get(self, block=True, timeout=None): if remaining <= 0.0: raise Empty self.not_empty.wait(remaining) + if self.is_shutdown and not self._qsize(): + raise ShutDown item = self._get() self.not_full.notify() return item @@ -198,6 +226,28 @@ def get_nowait(self): ''' return self.get(block=False) + def shutdown(self, immediate=False): + '''Shut-down the queue, making queue gets and puts raise. + + By default, gets will only raise once the queue is empty. Set + 'immediate' to True to make gets raise immediately instead. + + All blocked callers of put() will be unblocked, and also get() + and join() if 'immediate'. The ShutDown exception is raised. + ''' + with self.mutex: + self.is_shutdown = True + if immediate: + n_items = self._qsize() + while self._qsize(): + self._get() + if self.unfinished_tasks > 0: + self.unfinished_tasks -= 1 + self.not_empty.notify_all() + # release all blocked threads in `join()` + self.all_tasks_done.notify_all() + self.not_full.notify_all() + # Override these methods to implement other queue organizations # (e.g. stack or priority queue). # These will only be called with appropriate locks held diff --git a/Lib/runpy.py b/Lib/runpy.py index 42f896c9cd5094..ef54d3282eee06 100644 --- a/Lib/runpy.py +++ b/Lib/runpy.py @@ -247,17 +247,17 @@ def _get_main_module_details(error=ImportError): sys.modules[main_name] = saved_main -def _get_code_from_file(run_name, fname): +def _get_code_from_file(fname): # Check for a compiled file first from pkgutil import read_code - decoded_path = os.path.abspath(os.fsdecode(fname)) - with io.open_code(decoded_path) as f: + code_path = os.path.abspath(fname) + with io.open_code(code_path) as f: code = read_code(f) if code is None: # That didn't work, so try it as normal source code - with io.open_code(decoded_path) as f: + with io.open_code(code_path) as f: code = compile(f.read(), fname, 'exec') - return code, fname + return code def run_path(path_name, init_globals=None, run_name=None): """Execute code located at the specified filesystem location. @@ -279,12 +279,13 @@ def run_path(path_name, init_globals=None, run_name=None): pkg_name = run_name.rpartition(".")[0] from pkgutil import get_importer importer = get_importer(path_name) + path_name = os.fsdecode(path_name) if isinstance(importer, type(None)): # Not a valid sys.path entry, so run the code directly # execfile() doesn't help as we want to allow compiled files - code, fname = _get_code_from_file(run_name, path_name) + code = _get_code_from_file(path_name) return _run_module_code(code, init_globals, run_name, - pkg_name=pkg_name, script_name=fname) + pkg_name=pkg_name, script_name=path_name) else: # Finder is defined for path, so add it to # the start of sys.path diff --git a/Lib/shutil.py b/Lib/shutil.py index dc3aac3e07f910..c19ea0607208af 100644 --- a/Lib/shutil.py +++ b/Lib/shutil.py @@ -306,7 +306,12 @@ def copymode(src, dst, *, follow_symlinks=True): else: return else: - stat_func, chmod_func = _stat, os.chmod + stat_func = _stat + if os.name == 'nt' and os.path.islink(dst): + def chmod_func(*args): + os.chmod(*args, follow_symlinks=True) + else: + chmod_func = os.chmod st = stat_func(src) chmod_func(dst, stat.S_IMODE(st.st_mode)) @@ -856,12 +861,12 @@ def move(src, dst, copy_function=copy2): similar to the Unix "mv" command. Return the file or directory's destination. - If the destination is a directory or a symlink to a directory, the source - is moved inside the directory. The destination path must not already - exist. + If dst is an existing directory or a symlink to a directory, then src is + moved inside that directory. The destination path in that directory must + not already exist. - If the destination already exists but is not a directory, it may be - overwritten depending on os.rename() semantics. + If dst already exists but is not a directory, it may be overwritten + depending on os.rename() semantics. If the destination is on our current filesystem, then rename() is used. Otherwise, src is copied to the destination and then removed. Symlinks are @@ -880,7 +885,7 @@ def move(src, dst, copy_function=copy2): sys.audit("shutil.move", src, dst) real_dst = dst if os.path.isdir(dst): - if _samefile(src, dst): + if _samefile(src, dst) and not os.path.islink(src): # We might be on a case insensitive filesystem, # perform the rename anyway. os.rename(src, dst) diff --git a/Lib/signal.py b/Lib/signal.py index 50b215b29d2fad..c8cd3d4f597ca5 100644 --- a/Lib/signal.py +++ b/Lib/signal.py @@ -22,9 +22,11 @@ def _int_to_enum(value, enum_klass): - """Convert a numeric value to an IntEnum member. - If it's not a known member, return the numeric value itself. + """Convert a possible numeric value to an IntEnum member. + If it's not a known member, return the value itself. """ + if not isinstance(value, int): + return value try: return enum_klass(value) except ValueError: diff --git a/Lib/site.py b/Lib/site.py index 2517b7e5f1d22a..0631f3f6115ec0 100644 --- a/Lib/site.py +++ b/Lib/site.py @@ -74,6 +74,7 @@ import builtins import _sitebuiltins import io +import stat # Prefixes for site-packages; add additional prefixes like /usr/local here PREFIXES = [sys.prefix, sys.exec_prefix] @@ -168,6 +169,14 @@ def addpackage(sitedir, name, known_paths): else: reset = False fullname = os.path.join(sitedir, name) + try: + st = os.lstat(fullname) + except OSError: + return + if ((getattr(st, 'st_flags', 0) & stat.UF_HIDDEN) or + (getattr(st, 'st_file_attributes', 0) & stat.FILE_ATTRIBUTE_HIDDEN)): + _trace(f"Skipping hidden .pth file: {fullname!r}") + return _trace(f"Processing .pth file: {fullname!r}") try: # locale encoding is not ideal especially on Windows. But we have used @@ -221,7 +230,8 @@ def addsitedir(sitedir, known_paths=None): names = os.listdir(sitedir) except OSError: return - names = [name for name in names if name.endswith(".pth")] + names = [name for name in names + if name.endswith(".pth") and not name.startswith(".")] for name in sorted(names): addpackage(sitedir, name, known_paths) if reset: @@ -260,6 +270,10 @@ def check_enableusersite(): # # See https://bugs.python.org/issue29585 +# Copy of sysconfig._get_implementation() +def _get_implementation(): + return 'Python' + # Copy of sysconfig._getuserbase() def _getuserbase(): env_base = os.environ.get("PYTHONUSERBASE", None) @@ -275,7 +289,7 @@ def joinuser(*args): if os.name == "nt": base = os.environ.get("APPDATA") or "~" - return joinuser(base, "Python") + return joinuser(base, _get_implementation()) if sys.platform == "darwin" and sys._framework: return joinuser("~", "Library", sys._framework, @@ -288,12 +302,14 @@ def joinuser(*args): def _get_path(userbase): version = sys.version_info + implementation = _get_implementation() + implementation_lower = implementation.lower() if os.name == 'nt': ver_nodot = sys.winver.replace('.', '') - return f'{userbase}\\Python{ver_nodot}\\site-packages' + return f'{userbase}\\{implementation}{ver_nodot}\\site-packages' if sys.platform == 'darwin' and sys._framework: - return f'{userbase}/lib/python/site-packages' + return f'{userbase}/lib/{implementation_lower}/site-packages' return f'{userbase}/lib/python{version[0]}.{version[1]}/site-packages' @@ -361,6 +377,8 @@ def getsitepackages(prefixes=None): continue seen.add(prefix) + implementation = _get_implementation().lower() + ver = sys.version_info if os.sep == '/': libdirs = [sys.platlibdir] if sys.platlibdir != "lib": @@ -368,7 +386,7 @@ def getsitepackages(prefixes=None): for libdir in libdirs: path = os.path.join(prefix, libdir, - "python%d.%d" % sys.version_info[:2], + f"{implementation}{ver[0]}.{ver[1]}", "site-packages") sitepackages.append(path) else: @@ -425,6 +443,20 @@ def setcopyright(): def sethelper(): builtins.help = _sitebuiltins._Helper() + +def gethistoryfile(): + """Check if the PYTHON_HISTORY environment variable is set and define + it as the .python_history file. If PYTHON_HISTORY is not set, use the + default .python_history file. + """ + if not sys.flags.ignore_environment: + history = os.environ.get("PYTHON_HISTORY") + if history: + return history + return os.path.join(os.path.expanduser('~'), + '.python_history') + + def enablerlcompleter(): """Enable default readline configuration on interactive prompts, by registering a sys.__interactivehook__. @@ -459,13 +491,13 @@ def register_readline(): pass if readline.get_current_history_length() == 0: - # If no history was loaded, default to .python_history. + # If no history was loaded, default to .python_history, + # or PYTHON_HISTORY. # The guard is necessary to avoid doubling history size at # each interpreter exit when readline was already configured # through a PYTHONSTARTUP hook, see: # http://bugs.python.org/issue5845#msg198636 - history = os.path.join(os.path.expanduser('~'), - '.python_history') + history = gethistoryfile() try: readline.read_history_file(history) except OSError: diff --git a/Lib/socket.py b/Lib/socket.py index 5f0a1f40e25b94..77986fc2e48099 100644 --- a/Lib/socket.py +++ b/Lib/socket.py @@ -382,7 +382,7 @@ def _sendfile_use_sendfile(self, file, offset=0, count=None): if timeout and not selector_select(timeout): raise TimeoutError('timed out') if count: - blocksize = count - total_sent + blocksize = min(count - total_sent, blocksize) if blocksize <= 0: break try: diff --git a/Lib/sqlite3/dump.py b/Lib/sqlite3/dump.py index ead3360ce67608..9dcce7dc76ced4 100644 --- a/Lib/sqlite3/dump.py +++ b/Lib/sqlite3/dump.py @@ -15,7 +15,7 @@ def _quote_value(value): return "'{0}'".format(value.replace("'", "''")) -def _iterdump(connection): +def _iterdump(connection, *, filter=None): """ Returns an iterator to the dump of the database in an SQL text format. @@ -26,17 +26,29 @@ def _iterdump(connection): writeable_schema = False cu = connection.cursor() + # Disable foreign key constraints, if there is any foreign key violation. + violations = cu.execute("PRAGMA foreign_key_check").fetchall() + if violations: + yield('PRAGMA foreign_keys=OFF;') yield('BEGIN TRANSACTION;') + if filter: + # Return database objects which match the filter pattern. + filter_name_clause = 'AND "name" LIKE ?' + params = [filter] + else: + filter_name_clause = "" + params = [] # sqlite_master table contains the SQL CREATE statements for the database. - q = """ + q = f""" SELECT "name", "type", "sql" FROM "sqlite_master" WHERE "sql" NOT NULL AND "type" == 'table' + {filter_name_clause} ORDER BY "name" """ - schema_res = cu.execute(q) + schema_res = cu.execute(q, params) sqlite_sequence = [] for table_name, type, sql in schema_res.fetchall(): if table_name == 'sqlite_sequence': @@ -78,13 +90,14 @@ def _iterdump(connection): yield("{0};".format(row[0])) # Now when the type is 'index', 'trigger', or 'view' - q = """ + q = f""" SELECT "name", "type", "sql" FROM "sqlite_master" WHERE "sql" NOT NULL AND "type" IN ('index', 'trigger', 'view') + {filter_name_clause} """ - schema_res = cu.execute(q) + schema_res = cu.execute(q, params) for name, type, sql in schema_res.fetchall(): yield('{0};'.format(sql)) diff --git a/Lib/ssl.py b/Lib/ssl.py index d01484964b6895..03d0121891ff4c 100644 --- a/Lib/ssl.py +++ b/Lib/ssl.py @@ -110,7 +110,7 @@ try: from _ssl import RAND_egd except ImportError: - # LibreSSL does not provide RAND_egd + # RAND_egd is not supported on some platforms pass @@ -994,71 +994,67 @@ def _create(cls, sock, server_side=False, do_handshake_on_connect=True, if context.check_hostname and not server_hostname: raise ValueError("check_hostname requires server_hostname") + sock_timeout = sock.gettimeout() kwargs = dict( family=sock.family, type=sock.type, proto=sock.proto, fileno=sock.fileno() ) self = cls.__new__(cls, **kwargs) super(SSLSocket, self).__init__(**kwargs) - sock_timeout = sock.gettimeout() sock.detach() - - self._context = context - self._session = session - self._closed = False - self._sslobj = None - self.server_side = server_side - self.server_hostname = context._encode_hostname(server_hostname) - self.do_handshake_on_connect = do_handshake_on_connect - self.suppress_ragged_eofs = suppress_ragged_eofs - - # See if we are connected + # Now SSLSocket is responsible for closing the file descriptor. try: - self.getpeername() - except OSError as e: - if e.errno != errno.ENOTCONN: - raise - connected = False - blocking = self.getblocking() - self.setblocking(False) + self._context = context + self._session = session + self._closed = False + self._sslobj = None + self.server_side = server_side + self.server_hostname = context._encode_hostname(server_hostname) + self.do_handshake_on_connect = do_handshake_on_connect + self.suppress_ragged_eofs = suppress_ragged_eofs + + # See if we are connected try: - # We are not connected so this is not supposed to block, but - # testing revealed otherwise on macOS and Windows so we do - # the non-blocking dance regardless. Our raise when any data - # is found means consuming the data is harmless. - notconn_pre_handshake_data = self.recv(1) + self.getpeername() except OSError as e: - # EINVAL occurs for recv(1) on non-connected on unix sockets. - if e.errno not in (errno.ENOTCONN, errno.EINVAL): + if e.errno != errno.ENOTCONN: raise - notconn_pre_handshake_data = b'' - self.setblocking(blocking) - if notconn_pre_handshake_data: - # This prevents pending data sent to the socket before it was - # closed from escaping to the caller who could otherwise - # presume it came through a successful TLS connection. - reason = "Closed before TLS handshake with data in recv buffer." - notconn_pre_handshake_data_error = SSLError(e.errno, reason) - # Add the SSLError attributes that _ssl.c always adds. - notconn_pre_handshake_data_error.reason = reason - notconn_pre_handshake_data_error.library = None - try: - self.close() - except OSError: - pass + connected = False + blocking = self.getblocking() + self.setblocking(False) try: - raise notconn_pre_handshake_data_error - finally: - # Explicitly break the reference cycle. - notconn_pre_handshake_data_error = None - else: - connected = True + # We are not connected so this is not supposed to block, but + # testing revealed otherwise on macOS and Windows so we do + # the non-blocking dance regardless. Our raise when any data + # is found means consuming the data is harmless. + notconn_pre_handshake_data = self.recv(1) + except OSError as e: + # EINVAL occurs for recv(1) on non-connected on unix sockets. + if e.errno not in (errno.ENOTCONN, errno.EINVAL): + raise + notconn_pre_handshake_data = b'' + self.setblocking(blocking) + if notconn_pre_handshake_data: + # This prevents pending data sent to the socket before it was + # closed from escaping to the caller who could otherwise + # presume it came through a successful TLS connection. + reason = "Closed before TLS handshake with data in recv buffer." + notconn_pre_handshake_data_error = SSLError(e.errno, reason) + # Add the SSLError attributes that _ssl.c always adds. + notconn_pre_handshake_data_error.reason = reason + notconn_pre_handshake_data_error.library = None + try: + raise notconn_pre_handshake_data_error + finally: + # Explicitly break the reference cycle. + notconn_pre_handshake_data_error = None + else: + connected = True - self.settimeout(sock_timeout) # Must come after setblocking() calls. - self._connected = connected - if connected: - # create the SSL object - try: + self.settimeout(sock_timeout) # Must come after setblocking() calls. + self._connected = connected + if connected: + # create the SSL object self._sslobj = self._context._wrap_socket( self, server_side, self.server_hostname, owner=self, session=self._session, @@ -1069,9 +1065,12 @@ def _create(cls, sock, server_side=False, do_handshake_on_connect=True, # non-blocking raise ValueError("do_handshake_on_connect should not be specified for non-blocking sockets") self.do_handshake() - except (OSError, ValueError): + except: + try: self.close() - raise + except OSError: + pass + raise return self @property diff --git a/Lib/stat.py b/Lib/stat.py index 52cadbf04f6c88..9167ab185944fb 100644 --- a/Lib/stat.py +++ b/Lib/stat.py @@ -2,6 +2,7 @@ Suggested usage: from stat import * """ +import sys # Indices for stat struct members in the tuple returned by os.stat() @@ -110,19 +111,25 @@ def S_ISWHT(mode): S_IXOTH = 0o0001 # execute by others # Names for file flags - +UF_SETTABLE = 0x0000ffff # owner settable flags UF_NODUMP = 0x00000001 # do not dump file UF_IMMUTABLE = 0x00000002 # file may not be changed UF_APPEND = 0x00000004 # file may only be appended to UF_OPAQUE = 0x00000008 # directory is opaque when viewed through a union stack UF_NOUNLINK = 0x00000010 # file may not be renamed or deleted -UF_COMPRESSED = 0x00000020 # OS X: file is hfs-compressed -UF_HIDDEN = 0x00008000 # OS X: file should not be displayed +UF_COMPRESSED = 0x00000020 # macOS: file is compressed +UF_TRACKED = 0x00000040 # macOS: used for handling document IDs +UF_DATAVAULT = 0x00000080 # macOS: entitlement needed for I/O +UF_HIDDEN = 0x00008000 # macOS: file should not be displayed +SF_SETTABLE = 0xffff0000 # superuser settable flags SF_ARCHIVED = 0x00010000 # file may be archived SF_IMMUTABLE = 0x00020000 # file may not be changed SF_APPEND = 0x00040000 # file may only be appended to +SF_RESTRICTED = 0x00080000 # macOS: entitlement needed for writing SF_NOUNLINK = 0x00100000 # file may not be renamed or deleted SF_SNAPSHOT = 0x00200000 # file is a snapshot file +SF_FIRMLINK = 0x00800000 # macOS: file is a firmlink +SF_DATALESS = 0x40000000 # macOS: file is a dataless object _filemode_table = ( diff --git a/Lib/subprocess.py b/Lib/subprocess.py index d5bd9a9e31aa04..20db7747d5db13 100644 --- a/Lib/subprocess.py +++ b/Lib/subprocess.py @@ -1944,16 +1944,21 @@ def _execute_child(self, args, executable, preexec_fn, close_fds, SubprocessError) if issubclass(child_exception_type, OSError) and hex_errno: errno_num = int(hex_errno, 16) - child_exec_never_called = (err_msg == "noexec") - if child_exec_never_called: + if err_msg == "noexec:chdir": err_msg = "" # The error must be from chdir(cwd). err_filename = cwd + elif err_msg == "noexec": + err_msg = "" + err_filename = None else: err_filename = orig_executable if errno_num != 0: err_msg = os.strerror(errno_num) - raise child_exception_type(errno_num, err_msg, err_filename) + if err_filename is not None: + raise child_exception_type(errno_num, err_msg, err_filename) + else: + raise child_exception_type(errno_num, err_msg) raise child_exception_type(err_msg) diff --git a/Lib/sysconfig/__init__.py b/Lib/sysconfig/__init__.py index deb438c705f3a0..07ab27c7fb0c35 100644 --- a/Lib/sysconfig/__init__.py +++ b/Lib/sysconfig/__init__.py @@ -26,24 +26,24 @@ _INSTALL_SCHEMES = { 'posix_prefix': { - 'stdlib': '{installed_base}/{platlibdir}/python{py_version_short}', - 'platstdlib': '{platbase}/{platlibdir}/python{py_version_short}', - 'purelib': '{base}/lib/python{py_version_short}/site-packages', - 'platlib': '{platbase}/{platlibdir}/python{py_version_short}/site-packages', + 'stdlib': '{installed_base}/{platlibdir}/{implementation_lower}{py_version_short}', + 'platstdlib': '{platbase}/{platlibdir}/{implementation_lower}{py_version_short}', + 'purelib': '{base}/lib/{implementation_lower}{py_version_short}/site-packages', + 'platlib': '{platbase}/{platlibdir}/{implementation_lower}{py_version_short}/site-packages', 'include': - '{installed_base}/include/python{py_version_short}{abiflags}', + '{installed_base}/include/{implementation_lower}{py_version_short}{abiflags}', 'platinclude': - '{installed_platbase}/include/python{py_version_short}{abiflags}', + '{installed_platbase}/include/{implementation_lower}{py_version_short}{abiflags}', 'scripts': '{base}/bin', 'data': '{base}', }, 'posix_home': { - 'stdlib': '{installed_base}/lib/python', - 'platstdlib': '{base}/lib/python', - 'purelib': '{base}/lib/python', - 'platlib': '{base}/lib/python', - 'include': '{installed_base}/include/python', - 'platinclude': '{installed_base}/include/python', + 'stdlib': '{installed_base}/lib/{implementation_lower}', + 'platstdlib': '{base}/lib/{implementation_lower}', + 'purelib': '{base}/lib/{implementation_lower}', + 'platlib': '{base}/lib/{implementation_lower}', + 'include': '{installed_base}/include/{implementation_lower}', + 'platinclude': '{installed_base}/include/{implementation_lower}', 'scripts': '{base}/bin', 'data': '{base}', }, @@ -75,14 +75,14 @@ # Downstream distributors who patch posix_prefix/nt scheme are encouraged to # leave the following schemes unchanged 'posix_venv': { - 'stdlib': '{installed_base}/{platlibdir}/python{py_version_short}', - 'platstdlib': '{platbase}/{platlibdir}/python{py_version_short}', - 'purelib': '{base}/lib/python{py_version_short}/site-packages', - 'platlib': '{platbase}/{platlibdir}/python{py_version_short}/site-packages', + 'stdlib': '{installed_base}/{platlibdir}/{implementation_lower}{py_version_short}', + 'platstdlib': '{platbase}/{platlibdir}/{implementation_lower}{py_version_short}', + 'purelib': '{base}/lib/{implementation_lower}{py_version_short}/site-packages', + 'platlib': '{platbase}/{platlibdir}/{implementation_lower}{py_version_short}/site-packages', 'include': - '{installed_base}/include/python{py_version_short}{abiflags}', + '{installed_base}/include/{implementation_lower}{py_version_short}{abiflags}', 'platinclude': - '{installed_platbase}/include/python{py_version_short}{abiflags}', + '{installed_platbase}/include/{implementation_lower}{py_version_short}{abiflags}', 'scripts': '{base}/bin', 'data': '{base}', }, @@ -104,6 +104,8 @@ else: _INSTALL_SCHEMES['venv'] = _INSTALL_SCHEMES['posix_venv'] +def _get_implementation(): + return 'Python' # NOTE: site.py has copy of this function. # Sync it when modify this function. @@ -121,7 +123,7 @@ def joinuser(*args): if os.name == "nt": base = os.environ.get("APPDATA") or "~" - return joinuser(base, "Python") + return joinuser(base, _get_implementation()) if sys.platform == "darwin" and sys._framework: return joinuser("~", "Library", sys._framework, @@ -135,29 +137,29 @@ def joinuser(*args): _INSTALL_SCHEMES |= { # NOTE: When modifying "purelib" scheme, update site._get_path() too. 'nt_user': { - 'stdlib': '{userbase}/Python{py_version_nodot_plat}', - 'platstdlib': '{userbase}/Python{py_version_nodot_plat}', - 'purelib': '{userbase}/Python{py_version_nodot_plat}/site-packages', - 'platlib': '{userbase}/Python{py_version_nodot_plat}/site-packages', - 'include': '{userbase}/Python{py_version_nodot_plat}/Include', - 'scripts': '{userbase}/Python{py_version_nodot_plat}/Scripts', + 'stdlib': '{userbase}/{implementation}{py_version_nodot_plat}', + 'platstdlib': '{userbase}/{implementation}{py_version_nodot_plat}', + 'purelib': '{userbase}/{implementation}{py_version_nodot_plat}/site-packages', + 'platlib': '{userbase}/{implementation}{py_version_nodot_plat}/site-packages', + 'include': '{userbase}/{implementation}{py_version_nodot_plat}/Include', + 'scripts': '{userbase}/{implementation}{py_version_nodot_plat}/Scripts', 'data': '{userbase}', }, 'posix_user': { - 'stdlib': '{userbase}/{platlibdir}/python{py_version_short}', - 'platstdlib': '{userbase}/{platlibdir}/python{py_version_short}', - 'purelib': '{userbase}/lib/python{py_version_short}/site-packages', - 'platlib': '{userbase}/lib/python{py_version_short}/site-packages', - 'include': '{userbase}/include/python{py_version_short}', + 'stdlib': '{userbase}/{platlibdir}/{implementation_lower}{py_version_short}', + 'platstdlib': '{userbase}/{platlibdir}/{implementation_lower}{py_version_short}', + 'purelib': '{userbase}/lib/{implementation_lower}{py_version_short}/site-packages', + 'platlib': '{userbase}/lib/{implementation_lower}{py_version_short}/site-packages', + 'include': '{userbase}/include/{implementation_lower}{py_version_short}', 'scripts': '{userbase}/bin', 'data': '{userbase}', }, 'osx_framework_user': { - 'stdlib': '{userbase}/lib/python', - 'platstdlib': '{userbase}/lib/python', - 'purelib': '{userbase}/lib/python/site-packages', - 'platlib': '{userbase}/lib/python/site-packages', - 'include': '{userbase}/include/python{py_version_short}', + 'stdlib': '{userbase}/lib/{implementation_lower}', + 'platstdlib': '{userbase}/lib/{implementation_lower}', + 'purelib': '{userbase}/lib/{implementation_lower}/site-packages', + 'platlib': '{userbase}/lib/{implementation_lower}/site-packages', + 'include': '{userbase}/include/{implementation_lower}{py_version_short}', 'scripts': '{userbase}/bin', 'data': '{userbase}', }, @@ -459,6 +461,8 @@ def _init_config_vars(): _CONFIG_VARS['platbase'] = _EXEC_PREFIX _CONFIG_VARS['projectbase'] = _PROJECT_BASE _CONFIG_VARS['platlibdir'] = sys.platlibdir + _CONFIG_VARS['implementation'] = _get_implementation() + _CONFIG_VARS['implementation_lower'] = _get_implementation().lower() try: _CONFIG_VARS['abiflags'] = sys.abiflags except AttributeError: diff --git a/Lib/tarfile.py b/Lib/tarfile.py index 5ada0ad626bda8..9775040cbe372c 100755 --- a/Lib/tarfile.py +++ b/Lib/tarfile.py @@ -330,10 +330,11 @@ def write(self, s): class _Stream: """Class that serves as an adapter between TarFile and a stream-like object. The stream-like object only - needs to have a read() or write() method and is accessed - blockwise. Use of gzip or bzip2 compression is possible. - A stream-like object could be for example: sys.stdin, - sys.stdout, a socket, a tape device etc. + needs to have a read() or write() method that works with bytes, + and the method is accessed blockwise. + Use of gzip or bzip2 compression is possible. + A stream-like object could be for example: sys.stdin.buffer, + sys.stdout.buffer, a socket, a tape device etc. _Stream is intended to be used only internally. """ @@ -2455,7 +2456,8 @@ def makedir(self, tarinfo, targetpath): # later in _extract_member(). os.mkdir(targetpath, 0o700) except FileExistsError: - pass + if not os.path.isdir(targetpath): + raise def makefile(self, tarinfo, targetpath): """Make a file called targetpath. diff --git a/Lib/test/.ruff.toml b/Lib/test/.ruff.toml index 74ab215ee8ee28..1c9bac507209b1 100644 --- a/Lib/test/.ruff.toml +++ b/Lib/test/.ruff.toml @@ -1,7 +1,4 @@ fix = true -select = [ - "F811", # Redefinition of unused variable (useful for finding test methods with the same name) -] extend-exclude = [ # Excluded (run with the other AC files in its own separate ruff job in pre-commit) "test_clinic.py", @@ -20,5 +17,9 @@ extend-exclude = [ "test_import/__init__.py", "test_pkg.py", "test_yield_from.py", - "time_hashlib.py", +] + +[lint] +select = [ + "F811", # Redefinition of unused variable (useful for finding test methods with the same name) ] diff --git a/Lib/test/_test_multiprocessing.py b/Lib/test/_test_multiprocessing.py index a94eb6c0ae4b8e..c0d3ca50f17d69 100644 --- a/Lib/test/_test_multiprocessing.py +++ b/Lib/test/_test_multiprocessing.py @@ -2693,13 +2693,19 @@ def test_make_pool(self): p.join() def test_terminate(self): + # Simulate slow tasks which take "forever" to complete + sleep_time = support.LONG_TIMEOUT + if self.TYPE == 'threads': - self.skipTest("Threads cannot be terminated") + # Thread pool workers can't be forced to quit, so if the first + # task starts early enough, we will end up waiting for it. + # Sleep for a shorter time, so the test doesn't block. + sleep_time = 1 - # Simulate slow tasks which take "forever" to complete p = self.Pool(3) - args = [support.LONG_TIMEOUT for i in range(10_000)] + args = [sleep_time for i in range(10_000)] result = p.map_async(time.sleep, args, chunksize=1) + time.sleep(0.2) # give some tasks a chance to start p.terminate() p.join() @@ -4724,6 +4730,29 @@ def test_level(self): root_logger.setLevel(root_level) logger.setLevel(level=LOG_LEVEL) + def test_filename(self): + logger = multiprocessing.get_logger() + original_level = logger.level + try: + logger.setLevel(util.DEBUG) + stream = io.StringIO() + handler = logging.StreamHandler(stream) + logging_format = '[%(levelname)s] [%(filename)s] %(message)s' + handler.setFormatter(logging.Formatter(logging_format)) + logger.addHandler(handler) + logger.info('1') + util.info('2') + logger.debug('3') + filename = os.path.basename(__file__) + log_record = stream.getvalue() + self.assertIn(f'[INFO] [{filename}] 1', log_record) + self.assertIn(f'[INFO] [{filename}] 2', log_record) + self.assertIn(f'[DEBUG] [{filename}] 3', log_record) + finally: + logger.setLevel(original_level) + logger.removeHandler(handler) + handler.close() + # class _TestLoggingProcessName(BaseTestCase): # diff --git a/Lib/test/archiver_tests.py b/Lib/test/archiver_tests.py new file mode 100644 index 00000000000000..1a4bbb9e5706c5 --- /dev/null +++ b/Lib/test/archiver_tests.py @@ -0,0 +1,155 @@ +"""Tests common to tarfile and zipfile.""" + +import os +import sys + +from test.support import os_helper + +class OverwriteTests: + + def setUp(self): + os.makedirs(self.testdir) + self.addCleanup(os_helper.rmtree, self.testdir) + + def create_file(self, path, content=b''): + with open(path, 'wb') as f: + f.write(content) + + def open(self, path): + raise NotImplementedError + + def extractall(self, ar): + raise NotImplementedError + + + def test_overwrite_file_as_file(self): + target = os.path.join(self.testdir, 'test') + self.create_file(target, b'content') + with self.open(self.ar_with_file) as ar: + self.extractall(ar) + self.assertTrue(os.path.isfile(target)) + with open(target, 'rb') as f: + self.assertEqual(f.read(), b'newcontent') + + def test_overwrite_dir_as_dir(self): + target = os.path.join(self.testdir, 'test') + os.mkdir(target) + with self.open(self.ar_with_dir) as ar: + self.extractall(ar) + self.assertTrue(os.path.isdir(target)) + + def test_overwrite_dir_as_implicit_dir(self): + target = os.path.join(self.testdir, 'test') + os.mkdir(target) + with self.open(self.ar_with_implicit_dir) as ar: + self.extractall(ar) + self.assertTrue(os.path.isdir(target)) + self.assertTrue(os.path.isfile(os.path.join(target, 'file'))) + with open(os.path.join(target, 'file'), 'rb') as f: + self.assertEqual(f.read(), b'newcontent') + + def test_overwrite_dir_as_file(self): + target = os.path.join(self.testdir, 'test') + os.mkdir(target) + with self.open(self.ar_with_file) as ar: + with self.assertRaises(PermissionError if sys.platform == 'win32' + else IsADirectoryError): + self.extractall(ar) + self.assertTrue(os.path.isdir(target)) + + def test_overwrite_file_as_dir(self): + target = os.path.join(self.testdir, 'test') + self.create_file(target, b'content') + with self.open(self.ar_with_dir) as ar: + with self.assertRaises(FileExistsError): + self.extractall(ar) + self.assertTrue(os.path.isfile(target)) + with open(target, 'rb') as f: + self.assertEqual(f.read(), b'content') + + def test_overwrite_file_as_implicit_dir(self): + target = os.path.join(self.testdir, 'test') + self.create_file(target, b'content') + with self.open(self.ar_with_implicit_dir) as ar: + with self.assertRaises(FileNotFoundError if sys.platform == 'win32' + else NotADirectoryError): + self.extractall(ar) + self.assertTrue(os.path.isfile(target)) + with open(target, 'rb') as f: + self.assertEqual(f.read(), b'content') + + @os_helper.skip_unless_symlink + def test_overwrite_file_symlink_as_file(self): + # XXX: It is potential security vulnerability. + target = os.path.join(self.testdir, 'test') + target2 = os.path.join(self.testdir, 'test2') + self.create_file(target2, b'content') + os.symlink('test2', target) + with self.open(self.ar_with_file) as ar: + self.extractall(ar) + self.assertTrue(os.path.islink(target)) + self.assertTrue(os.path.isfile(target2)) + with open(target2, 'rb') as f: + self.assertEqual(f.read(), b'newcontent') + + @os_helper.skip_unless_symlink + def test_overwrite_broken_file_symlink_as_file(self): + # XXX: It is potential security vulnerability. + target = os.path.join(self.testdir, 'test') + target2 = os.path.join(self.testdir, 'test2') + os.symlink('test2', target) + with self.open(self.ar_with_file) as ar: + self.extractall(ar) + self.assertTrue(os.path.islink(target)) + self.assertTrue(os.path.isfile(target2)) + with open(target2, 'rb') as f: + self.assertEqual(f.read(), b'newcontent') + + @os_helper.skip_unless_symlink + def test_overwrite_dir_symlink_as_dir(self): + # XXX: It is potential security vulnerability. + target = os.path.join(self.testdir, 'test') + target2 = os.path.join(self.testdir, 'test2') + os.mkdir(target2) + os.symlink('test2', target, target_is_directory=True) + with self.open(self.ar_with_dir) as ar: + self.extractall(ar) + self.assertTrue(os.path.islink(target)) + self.assertTrue(os.path.isdir(target2)) + + @os_helper.skip_unless_symlink + def test_overwrite_dir_symlink_as_implicit_dir(self): + # XXX: It is potential security vulnerability. + target = os.path.join(self.testdir, 'test') + target2 = os.path.join(self.testdir, 'test2') + os.mkdir(target2) + os.symlink('test2', target, target_is_directory=True) + with self.open(self.ar_with_implicit_dir) as ar: + self.extractall(ar) + self.assertTrue(os.path.islink(target)) + self.assertTrue(os.path.isdir(target2)) + self.assertTrue(os.path.isfile(os.path.join(target2, 'file'))) + with open(os.path.join(target2, 'file'), 'rb') as f: + self.assertEqual(f.read(), b'newcontent') + + @os_helper.skip_unless_symlink + def test_overwrite_broken_dir_symlink_as_dir(self): + target = os.path.join(self.testdir, 'test') + target2 = os.path.join(self.testdir, 'test2') + os.symlink('test2', target, target_is_directory=True) + with self.open(self.ar_with_dir) as ar: + with self.assertRaises(FileExistsError): + self.extractall(ar) + self.assertTrue(os.path.islink(target)) + self.assertFalse(os.path.exists(target2)) + + @os_helper.skip_unless_symlink + def test_overwrite_broken_dir_symlink_as_implicit_dir(self): + target = os.path.join(self.testdir, 'test') + target2 = os.path.join(self.testdir, 'test2') + os.symlink('test2', target, target_is_directory=True) + with self.open(self.ar_with_implicit_dir) as ar: + with self.assertRaises(FileExistsError): + self.extractall(ar) + self.assertTrue(os.path.islink(target)) + self.assertFalse(os.path.exists(target2)) diff --git a/Lib/test/audit-tests.py b/Lib/test/audit-tests.py index ce4a11b119c900..de7d0da560a1c7 100644 --- a/Lib/test/audit-tests.py +++ b/Lib/test/audit-tests.py @@ -487,7 +487,13 @@ def hook(event, args): print(event, args[0]) sys.addaudithook(hook) - _wmi.exec_query("SELECT * FROM Win32_OperatingSystem") + try: + _wmi.exec_query("SELECT * FROM Win32_OperatingSystem") + except WindowsError as e: + # gh-112278: WMI may be slow response when first called, but we still + # get the audit event, so just ignore the timeout + if e.winerror != 258: + raise def test_syslog(): import syslog diff --git a/Lib/test/clinic.test.c b/Lib/test/clinic.test.c index b15aeb898d35a1..168f6f73f6186f 100644 --- a/Lib/test/clinic.test.c +++ b/Lib/test/clinic.test.c @@ -4625,7 +4625,7 @@ Test_cls_no_params_impl(TestObj *self, PyTypeObject *cls); static PyObject * Test_cls_no_params(TestObj *self, PyTypeObject *cls, PyObject *const *args, Py_ssize_t nargs, PyObject *kwnames) { - if (nargs) { + if (nargs || (kwnames && PyTuple_GET_SIZE(kwnames))) { PyErr_SetString(PyExc_TypeError, "cls_no_params() takes no arguments"); return NULL; } @@ -4634,7 +4634,7 @@ Test_cls_no_params(TestObj *self, PyTypeObject *cls, PyObject *const *args, Py_s static PyObject * Test_cls_no_params_impl(TestObj *self, PyTypeObject *cls) -/*[clinic end generated code: output=cc8845f22cff3dcb input=e7e2e4e344e96a11]*/ +/*[clinic end generated code: output=4d68b4652c144af3 input=e7e2e4e344e96a11]*/ /*[clinic input] diff --git a/Lib/test/datetimetester.py b/Lib/test/datetimetester.py index 8bda17358db87f..53ad5e57ada017 100644 --- a/Lib/test/datetimetester.py +++ b/Lib/test/datetimetester.py @@ -301,6 +301,10 @@ def test_inheritance(self): self.assertIsInstance(timezone.utc, tzinfo) self.assertIsInstance(self.EST, tzinfo) + def test_cannot_subclass(self): + with self.assertRaises(TypeError): + class MyTimezone(timezone): pass + def test_utcoffset(self): dummy = self.DT for h in [0, 1.5, 12]: diff --git a/Lib/test/exception_hierarchy.txt b/Lib/test/exception_hierarchy.txt index 1eca123be0fecb..217ee15d4c8af5 100644 --- a/Lib/test/exception_hierarchy.txt +++ b/Lib/test/exception_hierarchy.txt @@ -44,6 +44,7 @@ BaseException ├── StopAsyncIteration ├── StopIteration ├── SyntaxError + │ └── IncompleteInputError │ └── IndentationError │ └── TabError ├── SystemError diff --git a/Lib/test/libregrtest/findtests.py b/Lib/test/libregrtest/findtests.py index 78343775bc5b99..ee890b5b1db4cd 100644 --- a/Lib/test/libregrtest/findtests.py +++ b/Lib/test/libregrtest/findtests.py @@ -19,6 +19,7 @@ SPLITTESTDIRS: set[TestName] = { "test_asyncio", "test_concurrent_futures", + "test_doctests", "test_future_stmt", "test_gdb", "test_inspect", diff --git a/Lib/test/libregrtest/refleak.py b/Lib/test/libregrtest/refleak.py index 5836a8421cb42d..7da16cf721f097 100644 --- a/Lib/test/libregrtest/refleak.py +++ b/Lib/test/libregrtest/refleak.py @@ -5,6 +5,7 @@ from test import support from test.support import os_helper +from test.support import refleak_helper from .runtests import HuntRefleak from .utils import clear_caches @@ -96,7 +97,12 @@ def get_pooled_int(value): support.gc_collect() for i in rep_range: - results = test_func() + current = refleak_helper._hunting_for_refleaks + refleak_helper._hunting_for_refleaks = True + try: + results = test_func() + finally: + refleak_helper._hunting_for_refleaks = current dash_R_cleanup(fs, ps, pic, zdc, abcs) support.gc_collect() diff --git a/Lib/test/libregrtest/utils.py b/Lib/test/libregrtest/utils.py index 26481e71221ade..b30025d962413c 100644 --- a/Lib/test/libregrtest/utils.py +++ b/Lib/test/libregrtest/utils.py @@ -340,6 +340,9 @@ def get_build_info(): # --with-undefined-behavior-sanitizer if support.check_sanitizer(ub=True): sanitizers.append("UBSAN") + # --with-thread-sanitizer + if support.check_sanitizer(thread=True): + sanitizers.append("TSAN") if sanitizers: build.append('+'.join(sanitizers)) @@ -634,6 +637,7 @@ def display_header(use_resources: tuple[str, ...], asan = support.check_sanitizer(address=True) msan = support.check_sanitizer(memory=True) ubsan = support.check_sanitizer(ub=True) + tsan = support.check_sanitizer(thread=True) sanitizers = [] if asan: sanitizers.append("address") @@ -641,12 +645,15 @@ def display_header(use_resources: tuple[str, ...], sanitizers.append("memory") if ubsan: sanitizers.append("undefined behavior") + if tsan: + sanitizers.append("thread") if sanitizers: print(f"== sanitizers: {', '.join(sanitizers)}") for sanitizer, env_var in ( (asan, "ASAN_OPTIONS"), (msan, "MSAN_OPTIONS"), (ubsan, "UBSAN_OPTIONS"), + (tsan, "TSAN_OPTIONS"), ): options= os.environ.get(env_var) if sanitizer and options is not None: diff --git a/Lib/test/pickletester.py b/Lib/test/pickletester.py index fd446c8145850c..93e7dbbd103934 100644 --- a/Lib/test/pickletester.py +++ b/Lib/test/pickletester.py @@ -1825,6 +1825,14 @@ def test_unicode_high_plane(self): t2 = self.loads(p) self.assert_is_copy(t, t2) + def test_unicode_memoization(self): + # Repeated str is re-used (even when escapes added). + for proto in protocols: + for s in '', 'xyz', 'xyz\n', 'x\\yz', 'x\xa1yz\r': + p = self.dumps((s, s), proto) + s1, s2 = self.loads(p) + self.assertIs(s1, s2) + def test_bytes(self): for proto in protocols: for s in b'', b'xyz', b'xyz'*100: @@ -2429,7 +2437,7 @@ def test_bad_getattr(self): # Issue #3514: crash when there is an infinite loop in __getattr__ x = BadGetattr() for proto in range(2): - with support.infinite_recursion(): + with support.infinite_recursion(25): self.assertRaises(RuntimeError, self.dumps, x, proto) for proto in range(2, pickle.HIGHEST_PROTOCOL + 1): s = self.dumps(x, proto) diff --git a/Lib/test/pydocfodder.py b/Lib/test/pydocfodder.py index d0750e5a43c0c0..a3ef2231243954 100644 --- a/Lib/test/pydocfodder.py +++ b/Lib/test/pydocfodder.py @@ -2,8 +2,8 @@ import types -class A_new: - "A new-style class." +class A: + "A class." def A_method(self): "Method defined in A." @@ -41,8 +41,8 @@ def _delx(self): A_int_alias = int -class B_new(A_new): - "A new-style class, derived from A_new." +class B(A): + "A class, derived from A." def AB_method(self): "Method defined in A and B." @@ -61,8 +61,8 @@ def BD_method(self): def BCD_method(self): "Method defined in B, C and D." -class C_new(A_new): - "A new-style class, derived from A_new." +class C(A): + "A class, derived from A." def AC_method(self): "Method defined in A and C." @@ -81,8 +81,8 @@ def C_method(self): def CD_method(self): "Method defined in C and D." -class D_new(B_new, C_new): - """A new-style class, derived from B_new and C_new. +class D(B, C): + """A class, derived from B and C. """ def AD_method(self): diff --git a/Lib/test/pythoninfo.py b/Lib/test/pythoninfo.py index 49e41ca6cdaf98..814358746d6d8a 100644 --- a/Lib/test/pythoninfo.py +++ b/Lib/test/pythoninfo.py @@ -865,26 +865,36 @@ def collect_subprocess(info_add): def collect_windows(info_add): - try: - import ctypes - except ImportError: - return - - if not hasattr(ctypes, 'WinDLL'): + if sys.platform != "win32": + # Code specific to Windows return - ntdll = ctypes.WinDLL('ntdll') - BOOLEAN = ctypes.c_ubyte - + # windows.RtlAreLongPathsEnabled: RtlAreLongPathsEnabled() + # windows.is_admin: IsUserAnAdmin() try: - RtlAreLongPathsEnabled = ntdll.RtlAreLongPathsEnabled - except AttributeError: - res = '' + import ctypes + if not hasattr(ctypes, 'WinDLL'): + raise ImportError + except ImportError: + pass else: - RtlAreLongPathsEnabled.restype = BOOLEAN - RtlAreLongPathsEnabled.argtypes = () - res = bool(RtlAreLongPathsEnabled()) - info_add('windows.RtlAreLongPathsEnabled', res) + ntdll = ctypes.WinDLL('ntdll') + BOOLEAN = ctypes.c_ubyte + try: + RtlAreLongPathsEnabled = ntdll.RtlAreLongPathsEnabled + except AttributeError: + res = '' + else: + RtlAreLongPathsEnabled.restype = BOOLEAN + RtlAreLongPathsEnabled.argtypes = () + res = bool(RtlAreLongPathsEnabled()) + info_add('windows.RtlAreLongPathsEnabled', res) + + shell32 = ctypes.windll.shell32 + IsUserAnAdmin = shell32.IsUserAnAdmin + IsUserAnAdmin.restype = BOOLEAN + IsUserAnAdmin.argtypes = () + info_add('windows.is_admin', IsUserAnAdmin()) try: import _winapi @@ -893,6 +903,7 @@ def collect_windows(info_add): except (ImportError, AttributeError): pass + # windows.version_caption: "wmic os get Caption,Version /value" command import subprocess try: # When wmic.exe output is redirected to a pipe, @@ -919,12 +930,15 @@ def collect_windows(info_add): if line: info_add('windows.version', line) + # windows.ver: "ver" command try: proc = subprocess.Popen(["ver"], shell=True, stdout=subprocess.PIPE, stderr=subprocess.PIPE, text=True) output = proc.communicate()[0] + if proc.returncode == 0xc0000142: + return if proc.returncode: output = "" except OSError: @@ -935,6 +949,22 @@ def collect_windows(info_add): if line: info_add('windows.ver', line) + # windows.developer_mode: get AllowDevelopmentWithoutDevLicense registry + import winreg + try: + key = winreg.OpenKey( + winreg.HKEY_LOCAL_MACHINE, + r"SOFTWARE\Microsoft\Windows\CurrentVersion\AppModelUnlock") + subkey = "AllowDevelopmentWithoutDevLicense" + try: + value, value_type = winreg.QueryValueEx(key, subkey) + finally: + winreg.CloseKey(key) + except OSError: + pass + else: + info_add('windows.developer_mode', "enabled" if value else "disabled") + def collect_fips(info_add): try: diff --git a/Lib/test/shadowed_super.py b/Lib/test/shadowed_super.py deleted file mode 100644 index 2a62f667e93818..00000000000000 --- a/Lib/test/shadowed_super.py +++ /dev/null @@ -1,7 +0,0 @@ -class super: - msg = "truly super" - - -class C: - def method(self): - return super().msg diff --git a/Lib/test/support/__init__.py b/Lib/test/support/__init__.py index b605951320dc8b..5b091fb2fd32dc 100644 --- a/Lib/test/support/__init__.py +++ b/Lib/test/support/__init__.py @@ -43,7 +43,7 @@ "requires_limited_api", "requires_specialization", # sys "MS_WINDOWS", "is_jython", "is_android", "is_emscripten", "is_wasi", - "check_impl_detail", "unix_shell", "setswitchinterval", + "is_apple_mobile", "check_impl_detail", "unix_shell", "setswitchinterval", # os "get_pagesize", # network @@ -392,10 +392,10 @@ def skip_if_buildbot(reason=None): isbuildbot = False return unittest.skipIf(isbuildbot, reason) -def check_sanitizer(*, address=False, memory=False, ub=False): +def check_sanitizer(*, address=False, memory=False, ub=False, thread=False): """Returns True if Python is compiled with sanitizer support""" - if not (address or memory or ub): - raise ValueError('At least one of address, memory, or ub must be True') + if not (address or memory or ub or thread): + raise ValueError('At least one of address, memory, ub or thread must be True') cflags = sysconfig.get_config_var('CFLAGS') or '' @@ -412,18 +412,23 @@ def check_sanitizer(*, address=False, memory=False, ub=False): '-fsanitize=undefined' in cflags or '--with-undefined-behavior-sanitizer' in config_args ) + thread_sanitizer = ( + '-fsanitize=thread' in cflags or + '--with-thread-sanitizer' in config_args + ) return ( (memory and memory_sanitizer) or (address and address_sanitizer) or - (ub and ub_sanitizer) + (ub and ub_sanitizer) or + (thread and thread_sanitizer) ) -def skip_if_sanitizer(reason=None, *, address=False, memory=False, ub=False): +def skip_if_sanitizer(reason=None, *, address=False, memory=False, ub=False, thread=False): """Decorator raising SkipTest if running with a sanitizer active.""" if not reason: reason = 'not working with sanitizers active' - skip = check_sanitizer(address=address, memory=memory, ub=ub) + skip = check_sanitizer(address=address, memory=memory, ub=ub, thread=thread) return unittest.skipIf(skip, reason) # gh-89363: True if fork() can hang if Python is built with Address Sanitizer @@ -432,7 +437,7 @@ def skip_if_sanitizer(reason=None, *, address=False, memory=False, ub=False): def set_sanitizer_env_var(env, option): - for name in ('ASAN_OPTIONS', 'MSAN_OPTIONS', 'UBSAN_OPTIONS'): + for name in ('ASAN_OPTIONS', 'MSAN_OPTIONS', 'UBSAN_OPTIONS', 'TSAN_OPTIONS'): if name in env: env[name] += f':{option}' else: @@ -517,7 +522,7 @@ def requires_debug_ranges(reason='requires co_positions / debug_ranges'): is_android = hasattr(sys, 'getandroidapilevel') -if sys.platform not in ('win32', 'vxworks'): +if sys.platform not in {"win32", "vxworks", "ios", "tvos", "watchos"}: unix_shell = '/system/bin/sh' if is_android else '/bin/sh' else: unix_shell = None @@ -527,19 +532,35 @@ def requires_debug_ranges(reason='requires co_positions / debug_ranges'): is_emscripten = sys.platform == "emscripten" is_wasi = sys.platform == "wasi" -has_fork_support = hasattr(os, "fork") and not is_emscripten and not is_wasi +# Apple mobile platforms (iOS/tvOS/watchOS) are POSIX-like but do not +# have subprocess or fork support. +is_apple_mobile = sys.platform in {"ios", "tvos", "watchos"} +is_apple = is_apple_mobile or sys.platform == "darwin" + +has_fork_support = hasattr(os, "fork") and not ( + is_emscripten + or is_wasi + or is_apple_mobile +) def requires_fork(): return unittest.skipUnless(has_fork_support, "requires working os.fork()") -has_subprocess_support = not is_emscripten and not is_wasi +has_subprocess_support = not ( + is_emscripten + or is_wasi + or is_apple_mobile +) def requires_subprocess(): """Used for subprocess, os.spawn calls, fd inheritance""" return unittest.skipUnless(has_subprocess_support, "requires subprocess support") # Emscripten's socket emulation and WASI sockets have limitations. -has_socket_support = not is_emscripten and not is_wasi +has_socket_support = not ( + is_emscripten + or is_wasi +) def requires_working_socket(*, module=False): """Skip tests or modules that require working sockets @@ -2122,19 +2143,11 @@ def set_recursion_limit(limit): sys.setrecursionlimit(original_limit) def infinite_recursion(max_depth=None): - """Set a lower limit for tests that interact with infinite recursions - (e.g test_ast.ASTHelpers_Test.test_recursion_direct) since on some - debug windows builds, due to not enough functions being inlined the - stack size might not handle the default recursion limit (1000). See - bpo-11105 for details.""" if max_depth is None: - if not python_is_optimized() or Py_DEBUG: - # Python built without compiler optimizations or in debug mode - # usually consumes more stack memory per function call. - # Unoptimized number based on what works under a WASI debug build. - max_depth = 50 - else: - max_depth = 100 + # Pick a number large enough to cause problems + # but not take too long for code that can handle + # very deep recursion. + max_depth = 20_000 elif max_depth < 3: raise ValueError("max_depth must be at least 3, got {max_depth}") depth = get_recursion_depth() @@ -2196,7 +2209,9 @@ def _findwheel(pkgname): If set, the wheels are searched for in WHEEL_PKG_DIR (see ensurepip). Otherwise, they are searched for in the test directory. """ - wheel_dir = sysconfig.get_config_var('WHEEL_PKG_DIR') or TEST_HOME_DIR + wheel_dir = sysconfig.get_config_var('WHEEL_PKG_DIR') or os.path.join( + TEST_HOME_DIR, 'wheeldata', + ) filenames = os.listdir(wheel_dir) filenames = sorted(filenames, reverse=True) # approximate "newest" first for filename in filenames: @@ -2373,8 +2388,6 @@ def adjust_int_max_str_digits(max_digits): finally: sys.set_int_max_str_digits(current) -#For recursion tests, easily exceeds default recursion limit -EXCEEDS_RECURSION_LIMIT = 5000 def _get_c_recursion_limit(): try: @@ -2382,11 +2395,17 @@ def _get_c_recursion_limit(): return _testcapi.Py_C_RECURSION_LIMIT except (ImportError, AttributeError): # Originally taken from Include/cpython/pystate.h . - return 1500 + if sys.platform == 'win32': + return 4000 + else: + return 10000 # The default C recursion limit. Py_C_RECURSION_LIMIT = _get_c_recursion_limit() +#For recursion tests, easily exceeds default recursion limit +EXCEEDS_RECURSION_LIMIT = Py_C_RECURSION_LIMIT * 3 + #Windows doesn't have os.uname() but it doesn't support s390x. skip_on_s390x = unittest.skipIf(hasattr(os, 'uname') and os.uname().machine == 's390x', 'skipped on s390x') diff --git a/Lib/test/support/os_helper.py b/Lib/test/support/os_helper.py index 20f38fd36a8876..22787e32b5f3ab 100644 --- a/Lib/test/support/os_helper.py +++ b/Lib/test/support/os_helper.py @@ -22,8 +22,8 @@ # TESTFN_UNICODE is a non-ascii filename TESTFN_UNICODE = TESTFN_ASCII + "-\xe0\xf2\u0258\u0141\u011f" -if sys.platform == 'darwin': - # In Mac OS X's VFS API file names are, by definition, canonically +if support.is_apple: + # On Apple's VFS API file names are, by definition, canonically # decomposed Unicode, encoded using UTF-8. See QA1173: # http://developer.apple.com/mac/library/qa/qa2001/qa1173.html import unicodedata @@ -48,8 +48,8 @@ 'encoding (%s). Unicode filename tests may not be effective' % (TESTFN_UNENCODABLE, sys.getfilesystemencoding())) TESTFN_UNENCODABLE = None -# macOS and Emscripten deny unencodable filenames (invalid utf-8) -elif sys.platform not in {'darwin', 'emscripten', 'wasi'}: +# Apple and Emscripten deny unencodable filenames (invalid utf-8) +elif not support.is_apple and sys.platform not in {"emscripten", "wasi"}: try: # ascii and utf-8 cannot encode the byte 0xff b'\xff'.decode(sys.getfilesystemencoding()) diff --git a/Lib/test/support/pty_helper.py b/Lib/test/support/pty_helper.py index 11037d22516448..6587fd40333c51 100644 --- a/Lib/test/support/pty_helper.py +++ b/Lib/test/support/pty_helper.py @@ -58,3 +58,23 @@ def terminate(proc): input = b"" # Stop writing if not input: sel.modify(master, selectors.EVENT_READ) + + +###################################################################### +## Fake stdin (for testing interactive debugging) +###################################################################### + +class FakeInput: + """ + A fake input stream for pdb's interactive debugger. Whenever a + line is read, print it (to simulate the user typing it), and then + return it. The set of lines to return is specified in the + constructor; they should not have trailing newlines. + """ + def __init__(self, lines): + self.lines = lines + + def readline(self): + line = self.lines.pop(0) + print(line) + return line + '\n' diff --git a/Lib/test/support/refleak_helper.py b/Lib/test/support/refleak_helper.py new file mode 100644 index 00000000000000..2f86c93a1e2e58 --- /dev/null +++ b/Lib/test/support/refleak_helper.py @@ -0,0 +1,8 @@ +""" +Utilities for changing test behaviour while hunting +for refleaks +""" + +_hunting_for_refleaks = False +def hunting_for_refleaks(): + return _hunting_for_refleaks diff --git a/Lib/test/smtpd.py b/Lib/test/support/smtpd.py similarity index 100% rename from Lib/test/smtpd.py rename to Lib/test/support/smtpd.py diff --git a/Lib/test/test_abstract_numbers.py b/Lib/test/test_abstract_numbers.py index 2e06f0d16fdd05..72232b670cdb89 100644 --- a/Lib/test/test_abstract_numbers.py +++ b/Lib/test/test_abstract_numbers.py @@ -1,14 +1,34 @@ """Unit tests for numbers.py.""" +import abc import math import operator import unittest -from numbers import Complex, Real, Rational, Integral +from numbers import Complex, Real, Rational, Integral, Number + + +def concretize(cls): + def not_implemented(*args, **kwargs): + raise NotImplementedError() + + for name in dir(cls): + try: + value = getattr(cls, name) + if value.__isabstractmethod__: + setattr(cls, name, not_implemented) + except AttributeError: + pass + abc.update_abstractmethods(cls) + return cls + class TestNumbers(unittest.TestCase): def test_int(self): self.assertTrue(issubclass(int, Integral)) + self.assertTrue(issubclass(int, Rational)) + self.assertTrue(issubclass(int, Real)) self.assertTrue(issubclass(int, Complex)) + self.assertTrue(issubclass(int, Number)) self.assertEqual(7, int(7).real) self.assertEqual(0, int(7).imag) @@ -18,8 +38,11 @@ def test_int(self): self.assertEqual(1, int(7).denominator) def test_float(self): + self.assertFalse(issubclass(float, Integral)) self.assertFalse(issubclass(float, Rational)) self.assertTrue(issubclass(float, Real)) + self.assertTrue(issubclass(float, Complex)) + self.assertTrue(issubclass(float, Number)) self.assertEqual(7.3, float(7.3).real) self.assertEqual(0, float(7.3).imag) @@ -27,8 +50,11 @@ def test_float(self): self.assertEqual(-7.3, float(-7.3).conjugate()) def test_complex(self): + self.assertFalse(issubclass(complex, Integral)) + self.assertFalse(issubclass(complex, Rational)) self.assertFalse(issubclass(complex, Real)) self.assertTrue(issubclass(complex, Complex)) + self.assertTrue(issubclass(complex, Number)) c1, c2 = complex(3, 2), complex(4,1) # XXX: This is not ideal, but see the comment in math_trunc(). @@ -40,5 +66,135 @@ def test_complex(self): self.assertRaises(TypeError, int, c1) +class TestNumbersDefaultMethods(unittest.TestCase): + def test_complex(self): + @concretize + class MyComplex(Complex): + def __init__(self, real, imag): + self.r = real + self.i = imag + + @property + def real(self): + return self.r + + @property + def imag(self): + return self.i + + def __add__(self, other): + if isinstance(other, Complex): + return MyComplex(self.imag + other.imag, + self.real + other.real) + raise NotImplementedError + + def __neg__(self): + return MyComplex(-self.real, -self.imag) + + def __eq__(self, other): + if isinstance(other, Complex): + return self.imag == other.imag and self.real == other.real + if isinstance(other, Number): + return self.imag == 0 and self.real == other.real + + # test __bool__ + self.assertTrue(bool(MyComplex(1, 1))) + self.assertTrue(bool(MyComplex(0, 1))) + self.assertTrue(bool(MyComplex(1, 0))) + self.assertFalse(bool(MyComplex(0, 0))) + + # test __sub__ + self.assertEqual(MyComplex(2, 3) - complex(1, 2), MyComplex(1, 1)) + + # test __rsub__ + self.assertEqual(complex(2, 3) - MyComplex(1, 2), MyComplex(1, 1)) + + def test_real(self): + @concretize + class MyReal(Real): + def __init__(self, n): + self.n = n + + def __pos__(self): + return self.n + + def __float__(self): + return float(self.n) + + def __floordiv__(self, other): + return self.n // other + + def __rfloordiv__(self, other): + return other // self.n + + def __mod__(self, other): + return self.n % other + + def __rmod__(self, other): + return other % self.n + + # test __divmod__ + self.assertEqual(divmod(MyReal(3), 2), (1, 1)) + + # test __rdivmod__ + self.assertEqual(divmod(3, MyReal(2)), (1, 1)) + + # test __complex__ + self.assertEqual(complex(MyReal(1)), 1+0j) + + # test real + self.assertEqual(MyReal(3).real, 3) + + # test imag + self.assertEqual(MyReal(3).imag, 0) + + # test conjugate + self.assertEqual(MyReal(123).conjugate(), 123) + + + def test_rational(self): + @concretize + class MyRational(Rational): + def __init__(self, numerator, denominator): + self.n = numerator + self.d = denominator + + @property + def numerator(self): + return self.n + + @property + def denominator(self): + return self.d + + # test__float__ + self.assertEqual(float(MyRational(5, 2)), 2.5) + + + def test_integral(self): + @concretize + class MyIntegral(Integral): + def __init__(self, n): + self.n = n + + def __pos__(self): + return self.n + + def __int__(self): + return self.n + + # test __index__ + self.assertEqual(operator.index(MyIntegral(123)), 123) + + # test __float__ + self.assertEqual(float(MyIntegral(123)), 123.0) + + # test numerator + self.assertEqual(MyIntegral(123).numerator, 123) + + # test denominator + self.assertEqual(MyIntegral(123).denominator, 1) + + if __name__ == "__main__": unittest.main() diff --git a/Lib/test/test_argparse.py b/Lib/test/test_argparse.py index 7c1f5d36999a3d..86d6e81a71642b 100644 --- a/Lib/test/test_argparse.py +++ b/Lib/test/test_argparse.py @@ -3922,7 +3922,7 @@ class TestHelpUsageWithParentheses(HelpTestCase): options: -h, --help show this help message and exit - -p {1 (option A), 2 (option B)}, --optional {1 (option A), 2 (option B)} + -p, --optional {1 (option A), 2 (option B)} ''' version = '' @@ -4405,8 +4405,8 @@ class TestHelpAlternatePrefixChars(HelpTestCase): help = usage + '''\ options: - ^^foo foo help - ;b BAR, ;;bar BAR bar help + ^^foo foo help + ;b, ;;bar BAR bar help ''' version = '' @@ -5099,7 +5099,8 @@ def test_optional(self): string = ( "Action(option_strings=['--foo', '-a', '-b'], dest='b', " "nargs='+', const=None, default=42, type='int', " - "choices=[1, 2, 3], required=False, help='HELP', metavar='METAVAR')") + "choices=[1, 2, 3], required=False, help='HELP', " + "metavar='METAVAR', deprecated=False)") self.assertStringEqual(option, string) def test_argument(self): @@ -5116,7 +5117,8 @@ def test_argument(self): string = ( "Action(option_strings=[], dest='x', nargs='?', " "const=None, default=2.5, type=%r, choices=[0.5, 1.5, 2.5], " - "required=True, help='H HH H', metavar='MV MV MV')" % float) + "required=True, help='H HH H', metavar='MV MV MV', " + "deprecated=False)" % float) self.assertStringEqual(argument, string) def test_namespace(self): @@ -5308,6 +5310,139 @@ def spam(string_to_convert): args = parser.parse_args('--foo spam!'.split()) self.assertEqual(NS(foo='foo_converted'), args) + +# ============================================== +# Check that deprecated arguments output warning +# ============================================== + +class TestDeprecatedArguments(TestCase): + + def test_deprecated_option(self): + parser = argparse.ArgumentParser() + parser.add_argument('-f', '--foo', deprecated=True) + + with captured_stderr() as stderr: + parser.parse_args(['--foo', 'spam']) + stderr = stderr.getvalue() + self.assertRegex(stderr, "warning: option '--foo' is deprecated") + self.assertEqual(stderr.count('is deprecated'), 1) + + with captured_stderr() as stderr: + parser.parse_args(['-f', 'spam']) + stderr = stderr.getvalue() + self.assertRegex(stderr, "warning: option '-f' is deprecated") + self.assertEqual(stderr.count('is deprecated'), 1) + + with captured_stderr() as stderr: + parser.parse_args(['--foo', 'spam', '-f', 'ham']) + stderr = stderr.getvalue() + self.assertRegex(stderr, "warning: option '--foo' is deprecated") + self.assertRegex(stderr, "warning: option '-f' is deprecated") + self.assertEqual(stderr.count('is deprecated'), 2) + + with captured_stderr() as stderr: + parser.parse_args(['--foo', 'spam', '--foo', 'ham']) + stderr = stderr.getvalue() + self.assertRegex(stderr, "warning: option '--foo' is deprecated") + self.assertEqual(stderr.count('is deprecated'), 1) + + def test_deprecated_boolean_option(self): + parser = argparse.ArgumentParser() + parser.add_argument('-f', '--foo', action=argparse.BooleanOptionalAction, deprecated=True) + + with captured_stderr() as stderr: + parser.parse_args(['--foo']) + stderr = stderr.getvalue() + self.assertRegex(stderr, "warning: option '--foo' is deprecated") + self.assertEqual(stderr.count('is deprecated'), 1) + + with captured_stderr() as stderr: + parser.parse_args(['-f']) + stderr = stderr.getvalue() + self.assertRegex(stderr, "warning: option '-f' is deprecated") + self.assertEqual(stderr.count('is deprecated'), 1) + + with captured_stderr() as stderr: + parser.parse_args(['--no-foo']) + stderr = stderr.getvalue() + self.assertRegex(stderr, "warning: option '--no-foo' is deprecated") + self.assertEqual(stderr.count('is deprecated'), 1) + + with captured_stderr() as stderr: + parser.parse_args(['--foo', '--no-foo']) + stderr = stderr.getvalue() + self.assertRegex(stderr, "warning: option '--foo' is deprecated") + self.assertRegex(stderr, "warning: option '--no-foo' is deprecated") + self.assertEqual(stderr.count('is deprecated'), 2) + + def test_deprecated_arguments(self): + parser = argparse.ArgumentParser() + parser.add_argument('foo', nargs='?', deprecated=True) + parser.add_argument('bar', nargs='?', deprecated=True) + + with captured_stderr() as stderr: + parser.parse_args([]) + stderr = stderr.getvalue() + self.assertEqual(stderr.count('is deprecated'), 0) + + with captured_stderr() as stderr: + parser.parse_args(['spam']) + stderr = stderr.getvalue() + self.assertRegex(stderr, "warning: argument 'foo' is deprecated") + self.assertEqual(stderr.count('is deprecated'), 1) + + with captured_stderr() as stderr: + parser.parse_args(['spam', 'ham']) + stderr = stderr.getvalue() + self.assertRegex(stderr, "warning: argument 'foo' is deprecated") + self.assertRegex(stderr, "warning: argument 'bar' is deprecated") + self.assertEqual(stderr.count('is deprecated'), 2) + + def test_deprecated_varargument(self): + parser = argparse.ArgumentParser() + parser.add_argument('foo', nargs='*', deprecated=True) + + with captured_stderr() as stderr: + parser.parse_args([]) + stderr = stderr.getvalue() + self.assertEqual(stderr.count('is deprecated'), 0) + + with captured_stderr() as stderr: + parser.parse_args(['spam']) + stderr = stderr.getvalue() + self.assertRegex(stderr, "warning: argument 'foo' is deprecated") + self.assertEqual(stderr.count('is deprecated'), 1) + + with captured_stderr() as stderr: + parser.parse_args(['spam', 'ham']) + stderr = stderr.getvalue() + self.assertRegex(stderr, "warning: argument 'foo' is deprecated") + self.assertEqual(stderr.count('is deprecated'), 1) + + def test_deprecated_subparser(self): + parser = argparse.ArgumentParser() + subparsers = parser.add_subparsers() + subparsers.add_parser('foo', aliases=['baz'], deprecated=True) + subparsers.add_parser('bar') + + with captured_stderr() as stderr: + parser.parse_args(['bar']) + stderr = stderr.getvalue() + self.assertEqual(stderr.count('is deprecated'), 0) + + with captured_stderr() as stderr: + parser.parse_args(['foo']) + stderr = stderr.getvalue() + self.assertRegex(stderr, "warning: command 'foo' is deprecated") + self.assertEqual(stderr.count('is deprecated'), 1) + + with captured_stderr() as stderr: + parser.parse_args(['baz']) + stderr = stderr.getvalue() + self.assertRegex(stderr, "warning: command 'baz' is deprecated") + self.assertEqual(stderr.count('is deprecated'), 1) + + # ================================================================== # Check semantics regarding the default argument and type conversion # ================================================================== @@ -5405,6 +5540,22 @@ def test_zero_or_more_optional(self): args = parser.parse_args([]) self.assertEqual(NS(x=[]), args) + def test_double_dash(self): + parser = argparse.ArgumentParser() + parser.add_argument('-f', '--foo', nargs='*') + parser.add_argument('bar', nargs='*') + + args = parser.parse_args(['--foo=--']) + self.assertEqual(NS(foo=['--'], bar=[]), args) + args = parser.parse_args(['--foo', '--']) + self.assertEqual(NS(foo=[], bar=[]), args) + args = parser.parse_args(['-f--']) + self.assertEqual(NS(foo=['--'], bar=[]), args) + args = parser.parse_args(['-f', '--']) + self.assertEqual(NS(foo=[], bar=[]), args) + args = parser.parse_args(['--foo', 'a', 'b', '--', 'c', 'd']) + self.assertEqual(NS(foo=['a', 'b'], bar=['c', 'd']), args) + # =========================== # parse_intermixed_args tests diff --git a/Lib/test/test_ast.py b/Lib/test/test_ast.py index 64fcb02309de77..3789ac22e3899c 100644 --- a/Lib/test/test_ast.py +++ b/Lib/test/test_ast.py @@ -1126,7 +1126,7 @@ def next(self): def test_ast_recursion_limit(self): fail_depth = support.EXCEEDS_RECURSION_LIMIT crash_depth = 100_000 - success_depth = 1200 + success_depth = int(support.Py_C_RECURSION_LIMIT * 0.8) if _testinternalcapi is not None: remaining = _testinternalcapi.get_c_recursion_remaining() success_depth = min(success_depth, remaining) diff --git a/Lib/test/test_asyncgen.py b/Lib/test/test_asyncgen.py index a49630112af510..39605dca3886c8 100644 --- a/Lib/test/test_asyncgen.py +++ b/Lib/test/test_asyncgen.py @@ -379,7 +379,10 @@ async def async_gen_wrapper(): def test_async_gen_exception_12(self): async def gen(): - await anext(me) + with self.assertWarnsRegex(RuntimeWarning, + f"coroutine method 'asend' of '{gen.__qualname__}' " + f"was never awaited"): + await anext(me) yield 123 me = gen() @@ -395,7 +398,12 @@ async def gen(): yield 123 with self.assertWarns(DeprecationWarning): - gen().athrow(GeneratorExit, GeneratorExit(), None) + x = gen().athrow(GeneratorExit, GeneratorExit(), None) + with self.assertWarnsRegex(RuntimeWarning, + f"coroutine method 'athrow' of '{gen.__qualname__}' " + f"was never awaited"): + del x + gc_collect() def test_async_gen_api_01(self): async def gen(): @@ -1564,6 +1572,11 @@ async def main(): self.assertIsInstance(message['exception'], ZeroDivisionError) self.assertIn('unhandled exception during asyncio.run() shutdown', message['message']) + with self.assertWarnsRegex(RuntimeWarning, + f"coroutine method 'aclose' of '{async_iterate.__qualname__}' " + f"was never awaited"): + del message, messages + gc_collect() def test_async_gen_expression_01(self): async def arange(n): @@ -1617,6 +1630,10 @@ async def main(): asyncio.run(main()) self.assertEqual([], messages) + with self.assertWarnsRegex(RuntimeWarning, + f"coroutine method 'aclose' of '{async_iterate.__qualname__}' " + f"was never awaited"): + gc_collect() def test_async_gen_await_same_anext_coro_twice(self): async def async_iterate(): @@ -1701,6 +1718,14 @@ def test_asend(self): async def gen(): yield 1 + # gh-113753: asend objects allocated from a free-list should warn. + # Ensure there is a finalized 'asend' object ready to be reused. + try: + g = gen() + g.asend(None).send(None) + except StopIteration: + pass + msg = f"coroutine method 'asend' of '{gen.__qualname__}' was never awaited" with self.assertWarnsRegex(RuntimeWarning, msg): g = gen() diff --git a/Lib/test/test_asyncio/test_base_events.py b/Lib/test/test_asyncio/test_base_events.py index c2080977e9d587..82071edb252570 100644 --- a/Lib/test/test_asyncio/test_base_events.py +++ b/Lib/test/test_asyncio/test_base_events.py @@ -1232,7 +1232,7 @@ def test_create_datagram_endpoint_wrong_sock(self): with sock: coro = self.loop.create_datagram_endpoint(MyProto, sock=sock) with self.assertRaisesRegex(ValueError, - 'A UDP Socket was expected'): + 'A datagram socket was expected'): self.loop.run_until_complete(coro) def test_create_connection_no_host_port_sock(self): diff --git a/Lib/test/test_asyncio/test_events.py b/Lib/test/test_asyncio/test_events.py index b25c0975736e20..c92c88bd5b2429 100644 --- a/Lib/test/test_asyncio/test_events.py +++ b/Lib/test/test_asyncio/test_events.py @@ -1815,6 +1815,7 @@ def check_killed(self, returncode): else: self.assertEqual(-signal.SIGKILL, returncode) + @support.requires_subprocess() def test_subprocess_exec(self): prog = os.path.join(os.path.dirname(__file__), 'echo.py') @@ -1836,6 +1837,7 @@ def test_subprocess_exec(self): self.check_killed(proto.returncode) self.assertEqual(b'Python The Winner', proto.data[1]) + @support.requires_subprocess() def test_subprocess_interactive(self): prog = os.path.join(os.path.dirname(__file__), 'echo.py') @@ -1863,6 +1865,7 @@ def test_subprocess_interactive(self): self.loop.run_until_complete(proto.completed) self.check_killed(proto.returncode) + @support.requires_subprocess() def test_subprocess_shell(self): connect = self.loop.subprocess_shell( functools.partial(MySubprocessProtocol, self.loop), @@ -1879,6 +1882,7 @@ def test_subprocess_shell(self): self.assertEqual(proto.data[2], b'') transp.close() + @support.requires_subprocess() def test_subprocess_exitcode(self): connect = self.loop.subprocess_shell( functools.partial(MySubprocessProtocol, self.loop), @@ -1890,6 +1894,7 @@ def test_subprocess_exitcode(self): self.assertEqual(7, proto.returncode) transp.close() + @support.requires_subprocess() def test_subprocess_close_after_finish(self): connect = self.loop.subprocess_shell( functools.partial(MySubprocessProtocol, self.loop), @@ -1904,6 +1909,7 @@ def test_subprocess_close_after_finish(self): self.assertEqual(7, proto.returncode) self.assertIsNone(transp.close()) + @support.requires_subprocess() def test_subprocess_kill(self): prog = os.path.join(os.path.dirname(__file__), 'echo.py') @@ -1920,6 +1926,7 @@ def test_subprocess_kill(self): self.check_killed(proto.returncode) transp.close() + @support.requires_subprocess() def test_subprocess_terminate(self): prog = os.path.join(os.path.dirname(__file__), 'echo.py') @@ -1937,6 +1944,7 @@ def test_subprocess_terminate(self): transp.close() @unittest.skipIf(sys.platform == 'win32', "Don't have SIGHUP") + @support.requires_subprocess() def test_subprocess_send_signal(self): # bpo-31034: Make sure that we get the default signal handler (killing # the process). The parent process may have decided to ignore SIGHUP, @@ -1961,6 +1969,7 @@ def test_subprocess_send_signal(self): finally: signal.signal(signal.SIGHUP, old_handler) + @support.requires_subprocess() def test_subprocess_stderr(self): prog = os.path.join(os.path.dirname(__file__), 'echo2.py') @@ -1982,6 +1991,7 @@ def test_subprocess_stderr(self): self.assertTrue(proto.data[2].startswith(b'ERR:test'), proto.data[2]) self.assertEqual(0, proto.returncode) + @support.requires_subprocess() def test_subprocess_stderr_redirect_to_stdout(self): prog = os.path.join(os.path.dirname(__file__), 'echo2.py') @@ -2007,6 +2017,7 @@ def test_subprocess_stderr_redirect_to_stdout(self): transp.close() self.assertEqual(0, proto.returncode) + @support.requires_subprocess() def test_subprocess_close_client_stream(self): prog = os.path.join(os.path.dirname(__file__), 'echo3.py') @@ -2041,6 +2052,7 @@ def test_subprocess_close_client_stream(self): self.loop.run_until_complete(proto.completed) self.check_killed(proto.returncode) + @support.requires_subprocess() def test_subprocess_wait_no_same_group(self): # start the new process in a new session connect = self.loop.subprocess_shell( @@ -2053,6 +2065,7 @@ def test_subprocess_wait_no_same_group(self): self.assertEqual(7, proto.returncode) transp.close() + @support.requires_subprocess() def test_subprocess_exec_invalid_args(self): async def connect(**kwds): await self.loop.subprocess_exec( @@ -2066,6 +2079,7 @@ async def connect(**kwds): with self.assertRaises(ValueError): self.loop.run_until_complete(connect(shell=True)) + @support.requires_subprocess() def test_subprocess_shell_invalid_args(self): async def connect(cmd=None, **kwds): diff --git a/Lib/test/test_asyncio/test_futures.py b/Lib/test/test_asyncio/test_futures.py index 2184b2091f84ee..d3e8efec1c04c2 100644 --- a/Lib/test/test_asyncio/test_futures.py +++ b/Lib/test/test_asyncio/test_futures.py @@ -270,10 +270,6 @@ def test_exception(self): f = self._new_future(loop=self.loop) self.assertRaises(asyncio.InvalidStateError, f.exception) - # StopIteration cannot be raised into a Future - CPython issue26221 - self.assertRaisesRegex(TypeError, "StopIteration .* cannot be raised", - f.set_exception, StopIteration) - f.set_exception(exc) self.assertFalse(f.cancelled()) self.assertTrue(f.done()) @@ -283,6 +279,25 @@ def test_exception(self): self.assertRaises(asyncio.InvalidStateError, f.set_exception, None) self.assertFalse(f.cancel()) + def test_stop_iteration_exception(self, stop_iteration_class=StopIteration): + exc = stop_iteration_class() + f = self._new_future(loop=self.loop) + f.set_exception(exc) + self.assertFalse(f.cancelled()) + self.assertTrue(f.done()) + self.assertRaises(RuntimeError, f.result) + exc = f.exception() + cause = exc.__cause__ + self.assertIsInstance(exc, RuntimeError) + self.assertRegex(str(exc), 'StopIteration .* cannot be raised') + self.assertIsInstance(cause, stop_iteration_class) + + def test_stop_iteration_subclass_exception(self): + class MyStopIteration(StopIteration): + pass + + self.test_stop_iteration_exception(MyStopIteration) + def test_exception_class(self): f = self._new_future(loop=self.loop) f.set_exception(RuntimeError) diff --git a/Lib/test/test_asyncio/test_locks.py b/Lib/test/test_asyncio/test_locks.py index f6c6a282429a21..a0884bffe6b0de 100644 --- a/Lib/test/test_asyncio/test_locks.py +++ b/Lib/test/test_asyncio/test_locks.py @@ -758,6 +758,155 @@ async def test_timeout_in_block(self): with self.assertRaises(asyncio.TimeoutError): await asyncio.wait_for(condition.wait(), timeout=0.5) + async def test_cancelled_error_wakeup(self): + # Test that a cancelled error, received when awaiting wakeup, + # will be re-raised un-modified. + wake = False + raised = None + cond = asyncio.Condition() + + async def func(): + nonlocal raised + async with cond: + with self.assertRaises(asyncio.CancelledError) as err: + await cond.wait_for(lambda: wake) + raised = err.exception + raise raised + + task = asyncio.create_task(func()) + await asyncio.sleep(0) + # Task is waiting on the condition, cancel it there. + task.cancel(msg="foo") + with self.assertRaises(asyncio.CancelledError) as err: + await task + self.assertEqual(err.exception.args, ("foo",)) + # We should have got the _same_ exception instance as the one + # originally raised. + self.assertIs(err.exception, raised) + + async def test_cancelled_error_re_aquire(self): + # Test that a cancelled error, received when re-aquiring lock, + # will be re-raised un-modified. + wake = False + raised = None + cond = asyncio.Condition() + + async def func(): + nonlocal raised + async with cond: + with self.assertRaises(asyncio.CancelledError) as err: + await cond.wait_for(lambda: wake) + raised = err.exception + raise raised + + task = asyncio.create_task(func()) + await asyncio.sleep(0) + # Task is waiting on the condition + await cond.acquire() + wake = True + cond.notify() + await asyncio.sleep(0) + # Task is now trying to re-acquire the lock, cancel it there. + task.cancel(msg="foo") + cond.release() + with self.assertRaises(asyncio.CancelledError) as err: + await task + self.assertEqual(err.exception.args, ("foo",)) + # We should have got the _same_ exception instance as the one + # originally raised. + self.assertIs(err.exception, raised) + + async def test_cancelled_wakeup(self): + # Test that a task cancelled at the "same" time as it is woken + # up as part of a Condition.notify() does not result in a lost wakeup. + # This test simulates a cancel while the target task is awaiting initial + # wakeup on the wakeup queue. + condition = asyncio.Condition() + state = 0 + async def consumer(): + nonlocal state + async with condition: + while True: + await condition.wait_for(lambda: state != 0) + if state < 0: + return + state -= 1 + + # create two consumers + c = [asyncio.create_task(consumer()) for _ in range(2)] + # wait for them to settle + await asyncio.sleep(0) + async with condition: + # produce one item and wake up one + state += 1 + condition.notify(1) + + # Cancel it while it is awaiting to be run. + # This cancellation could come from the outside + c[0].cancel() + + # now wait for the item to be consumed + # if it doesn't means that our "notify" didn"t take hold. + # because it raced with a cancel() + try: + async with asyncio.timeout(0.01): + await condition.wait_for(lambda: state == 0) + except TimeoutError: + pass + self.assertEqual(state, 0) + + # clean up + state = -1 + condition.notify_all() + await c[1] + + async def test_cancelled_wakeup_relock(self): + # Test that a task cancelled at the "same" time as it is woken + # up as part of a Condition.notify() does not result in a lost wakeup. + # This test simulates a cancel while the target task is acquiring the lock + # again. + condition = asyncio.Condition() + state = 0 + async def consumer(): + nonlocal state + async with condition: + while True: + await condition.wait_for(lambda: state != 0) + if state < 0: + return + state -= 1 + + # create two consumers + c = [asyncio.create_task(consumer()) for _ in range(2)] + # wait for them to settle + await asyncio.sleep(0) + async with condition: + # produce one item and wake up one + state += 1 + condition.notify(1) + + # now we sleep for a bit. This allows the target task to wake up and + # settle on re-aquiring the lock + await asyncio.sleep(0) + + # Cancel it while awaiting the lock + # This cancel could come the outside. + c[0].cancel() + + # now wait for the item to be consumed + # if it doesn't means that our "notify" didn"t take hold. + # because it raced with a cancel() + try: + async with asyncio.timeout(0.01): + await condition.wait_for(lambda: state == 0) + except TimeoutError: + pass + self.assertEqual(state, 0) + + # clean up + state = -1 + condition.notify_all() + await c[1] class SemaphoreTests(unittest.IsolatedAsyncioTestCase): @@ -1044,6 +1193,62 @@ async def c3(result): await asyncio.gather(*tasks, return_exceptions=True) self.assertEqual([2, 3], result) + async def test_acquire_fifo_order_4(self): + # Test that a successfule `acquire()` will wake up multiple Tasks + # that were waiting in the Semaphore queue due to FIFO rules. + sem = asyncio.Semaphore(0) + result = [] + count = 0 + + async def c1(result): + # First task immediatlly waits for semaphore. It will be awoken by c2. + self.assertEqual(sem._value, 0) + await sem.acquire() + # We should have woken up all waiting tasks now. + self.assertEqual(sem._value, 0) + # Create a fourth task. It should run after c3, not c2. + nonlocal t4 + t4 = asyncio.create_task(c4(result)) + result.append(1) + return True + + async def c2(result): + # The second task begins by releasing semaphore three times, + # for c1, c2, and c3. + sem.release() + sem.release() + sem.release() + self.assertEqual(sem._value, 2) + # It is locked, because c1 hasn't woken up yet. + self.assertTrue(sem.locked()) + await sem.acquire() + result.append(2) + return True + + async def c3(result): + await sem.acquire() + self.assertTrue(sem.locked()) + result.append(3) + return True + + async def c4(result): + result.append(4) + return True + + t1 = asyncio.create_task(c1(result)) + t2 = asyncio.create_task(c2(result)) + t3 = asyncio.create_task(c3(result)) + t4 = None + + await asyncio.sleep(0) + # Three tasks are in the queue, the first hasn't woken up yet. + self.assertEqual(sem._value, 2) + self.assertEqual(len(sem._waiters), 3) + await asyncio.sleep(0) + + tasks = [t1, t2, t3, t4] + await asyncio.gather(*tasks) + self.assertEqual([1, 2, 3, 4], result) class BarrierTests(unittest.IsolatedAsyncioTestCase): diff --git a/Lib/test/test_asyncio/test_runners.py b/Lib/test/test_asyncio/test_runners.py index 13493d3c806d6a..266f057f0776c3 100644 --- a/Lib/test/test_asyncio/test_runners.py +++ b/Lib/test/test_asyncio/test_runners.py @@ -495,6 +495,24 @@ async def coro(): self.assertEqual(1, policy.set_event_loop.call_count) runner.close() + def test_no_repr_is_call_on_the_task_result(self): + # See https://github.com/python/cpython/issues/112559. + class MyResult: + def __init__(self): + self.repr_count = 0 + def __repr__(self): + self.repr_count += 1 + return super().__repr__() + + async def coro(): + return MyResult() + + + with asyncio.Runner() as runner: + result = runner.run(coro()) + + self.assertEqual(0, result.repr_count) + if __name__ == '__main__': unittest.main() diff --git a/Lib/test/test_asyncio/test_server.py b/Lib/test/test_asyncio/test_server.py index f22cf3026e244b..918faac909b9bf 100644 --- a/Lib/test/test_asyncio/test_server.py +++ b/Lib/test/test_asyncio/test_server.py @@ -200,13 +200,13 @@ async def test_unix_server_sock_cleanup(self): async def serve(*args): pass - sock = socket.socket(socket.AF_UNIX, socket.SOCK_STREAM) - sock.bind(addr) + with socket.socket(socket.AF_UNIX, socket.SOCK_STREAM) as sock: + sock.bind(addr) - srv = await asyncio.start_unix_server(serve, sock=sock) + srv = await asyncio.start_unix_server(serve, sock=sock) - srv.close() - self.assertFalse(os.path.exists(addr)) + srv.close() + self.assertFalse(os.path.exists(addr)) @socket_helper.skip_unless_bind_unix_socket async def test_unix_server_cleanup_gone(self): @@ -215,14 +215,14 @@ async def test_unix_server_cleanup_gone(self): async def serve(*args): pass - sock = socket.socket(socket.AF_UNIX, socket.SOCK_STREAM) - sock.bind(addr) + with socket.socket(socket.AF_UNIX, socket.SOCK_STREAM) as sock: + sock.bind(addr) - srv = await asyncio.start_unix_server(serve, sock=sock) + srv = await asyncio.start_unix_server(serve, sock=sock) - os.unlink(addr) + os.unlink(addr) - srv.close() + srv.close() @socket_helper.skip_unless_bind_unix_socket async def test_unix_server_cleanup_replaced(self): @@ -234,11 +234,11 @@ async def serve(*args): srv = await asyncio.start_unix_server(serve, addr) os.unlink(addr) - sock = socket.socket(socket.AF_UNIX, socket.SOCK_STREAM) - sock.bind(addr) + with socket.socket(socket.AF_UNIX, socket.SOCK_STREAM) as sock: + sock.bind(addr) - srv.close() - self.assertTrue(os.path.exists(addr)) + srv.close() + self.assertTrue(os.path.exists(addr)) @socket_helper.skip_unless_bind_unix_socket async def test_unix_server_cleanup_prevented(self): diff --git a/Lib/test/test_asyncio/test_ssl.py b/Lib/test/test_asyncio/test_ssl.py index e9cc735613fb8e..e072ede29ee3c7 100644 --- a/Lib/test/test_asyncio/test_ssl.py +++ b/Lib/test/test_asyncio/test_ssl.py @@ -1,3 +1,7 @@ +# Contains code from https://github.com/MagicStack/uvloop/tree/v0.16.0 +# SPDX-License-Identifier: PSF-2.0 AND (MIT OR Apache-2.0) +# SPDX-FileCopyrightText: Copyright (c) 2015-2021 MagicStack Inc. http://magic.io + import asyncio import contextlib import gc diff --git a/Lib/test/test_asyncio/test_streams.py b/Lib/test/test_asyncio/test_streams.py index b408cd1f7da205..210990593adfa9 100644 --- a/Lib/test/test_asyncio/test_streams.py +++ b/Lib/test/test_asyncio/test_streams.py @@ -10,7 +10,6 @@ import unittest from unittest import mock import warnings -from test.support import socket_helper try: import ssl except ImportError: @@ -18,6 +17,7 @@ import asyncio from test.test_asyncio import utils as test_utils +from test.support import requires_subprocess, socket_helper def tearDownModule(): @@ -770,6 +770,7 @@ async def client(addr): self.assertEqual(msg2, b"hello world 2!\n") @unittest.skipIf(sys.platform == 'win32', "Don't have pipes") + @requires_subprocess() def test_read_all_from_pipe_reader(self): # See asyncio issue 168. This test is derived from the example # subprocess_attach_read_pipe.py, but we configure the @@ -1129,7 +1130,7 @@ async def inner(httpd): self.assertEqual(messages, []) - def test_unhandled_exceptions(self) -> None: + def _basetest_unhandled_exceptions(self, handle_echo): port = socket_helper.find_unused_port() messages = [] @@ -1143,9 +1144,6 @@ async def client(): await wr.wait_closed() async def main(): - async def handle_echo(reader, writer): - raise Exception('test') - server = await asyncio.start_server( handle_echo, 'localhost', port) await server.start_serving() @@ -1154,11 +1152,20 @@ async def handle_echo(reader, writer): await server.wait_closed() self.loop.run_until_complete(main()) + return messages + def test_unhandled_exception(self): + async def handle_echo(reader, writer): + raise Exception('test') + messages = self._basetest_unhandled_exceptions(handle_echo) self.assertEqual(messages[0]['message'], - 'Unhandled exception in client_connected_cb') - # Break explicitly reference cycle - messages = None + 'Unhandled exception in client_connected_cb') + + def test_unhandled_cancel(self): + async def handle_echo(reader, writer): + asyncio.current_task().cancel() + messages = self._basetest_unhandled_exceptions(handle_echo) + self.assertEqual(messages, []) if __name__ == '__main__': diff --git a/Lib/test/test_asyncio/test_subprocess.py b/Lib/test/test_asyncio/test_subprocess.py index 179c8cb8cc17cf..f50a9ebc031ba8 100644 --- a/Lib/test/test_asyncio/test_subprocess.py +++ b/Lib/test/test_asyncio/test_subprocess.py @@ -47,6 +47,7 @@ def _start(self, *args, **kwargs): self._proc.pid = -1 +@support.requires_subprocess() class SubprocessTransportTests(test_utils.TestCase): def setUp(self): super().setUp() @@ -110,6 +111,7 @@ def test_subprocess_repr(self): transport.close() +@support.requires_subprocess() class SubprocessMixin: def test_stdin_stdout(self): @@ -207,7 +209,7 @@ def test_kill(self): def test_kill_issue43884(self): if sys.platform == 'win32': - blocking_shell_command = f'{sys.executable} -c "import time; time.sleep(2)"' + blocking_shell_command = f'"{sys.executable}" -c "import time; time.sleep(2)"' else: blocking_shell_command = 'sleep 1; sleep 1' creationflags = 0 @@ -745,7 +747,10 @@ async def check_stdout_output(self, coro, output): def test_create_subprocess_env_shell(self) -> None: async def main() -> None: - cmd = f'''{sys.executable} -c "import os, sys; sys.stdout.write(os.getenv('FOO'))"''' + executable = sys.executable + if sys.platform == "win32": + executable = f'"{executable}"' + cmd = f'''{executable} -c "import os, sys; sys.stdout.write(os.getenv('FOO'))"''' env = os.environ.copy() env["FOO"] = "bar" proc = await asyncio.create_subprocess_shell( @@ -975,8 +980,13 @@ async def in_thread(): async def main(): # asyncio.Runner did not call asyncio.set_event_loop() - with self.assertRaises(RuntimeError): - asyncio.get_event_loop_policy().get_event_loop() + with warnings.catch_warnings(): + warnings.simplefilter('error', DeprecationWarning) + # get_event_loop() raises DeprecationWarning if + # set_event_loop() was never called and RuntimeError if + # it was called at least once. + with self.assertRaises((RuntimeError, DeprecationWarning)): + asyncio.get_event_loop_policy().get_event_loop() return await asyncio.to_thread(asyncio.run, in_thread()) with self.assertWarns(DeprecationWarning): asyncio.set_child_watcher(asyncio.PidfdChildWatcher()) diff --git a/Lib/test/test_asyncio/test_timeouts.py b/Lib/test/test_asyncio/test_timeouts.py index f54e79e4d8e600..1f7f9ee696a525 100644 --- a/Lib/test/test_asyncio/test_timeouts.py +++ b/Lib/test/test_asyncio/test_timeouts.py @@ -116,15 +116,68 @@ async def test_foreign_exception_passed(self): raise KeyError self.assertFalse(cm.expired()) + async def test_timeout_exception_context(self): + with self.assertRaises(TimeoutError) as cm: + async with asyncio.timeout(0.01): + try: + 1/0 + finally: + await asyncio.sleep(1) + e = cm.exception + # Expect TimeoutError caused by CancelledError raised during handling + # of ZeroDivisionError. + e2 = e.__cause__ + self.assertIsInstance(e2, asyncio.CancelledError) + self.assertIs(e.__context__, e2) + self.assertIsNone(e2.__cause__) + self.assertIsInstance(e2.__context__, ZeroDivisionError) + async def test_foreign_exception_on_timeout(self): async def crash(): try: await asyncio.sleep(1) finally: 1/0 - with self.assertRaises(ZeroDivisionError): + with self.assertRaises(ZeroDivisionError) as cm: async with asyncio.timeout(0.01): await crash() + e = cm.exception + # Expect ZeroDivisionError raised during handling of TimeoutError + # caused by CancelledError. + self.assertIsNone(e.__cause__) + e2 = e.__context__ + self.assertIsInstance(e2, TimeoutError) + e3 = e2.__cause__ + self.assertIsInstance(e3, asyncio.CancelledError) + self.assertIs(e2.__context__, e3) + + async def test_foreign_exception_on_timeout_2(self): + with self.assertRaises(ZeroDivisionError) as cm: + async with asyncio.timeout(0.01): + try: + try: + raise ValueError + finally: + await asyncio.sleep(1) + finally: + try: + raise KeyError + finally: + 1/0 + e = cm.exception + # Expect ZeroDivisionError raised during handling of KeyError + # raised during handling of TimeoutError caused by CancelledError. + self.assertIsNone(e.__cause__) + e2 = e.__context__ + self.assertIsInstance(e2, KeyError) + self.assertIsNone(e2.__cause__) + e3 = e2.__context__ + self.assertIsInstance(e3, TimeoutError) + e4 = e3.__cause__ + self.assertIsInstance(e4, asyncio.CancelledError) + self.assertIsNone(e4.__cause__) + self.assertIsInstance(e4.__context__, ValueError) + self.assertIs(e3.__context__, e4) async def test_foreign_cancel_doesnt_timeout_if_not_expired(self): with self.assertRaises(asyncio.CancelledError): @@ -219,14 +272,30 @@ async def test_repr_disabled(self): self.assertEqual(repr(cm), r"") async def test_nested_timeout_in_finally(self): - with self.assertRaises(TimeoutError): + with self.assertRaises(TimeoutError) as cm1: async with asyncio.timeout(0.01): try: await asyncio.sleep(1) finally: - with self.assertRaises(TimeoutError): + with self.assertRaises(TimeoutError) as cm2: async with asyncio.timeout(0.01): await asyncio.sleep(10) + e1 = cm1.exception + # Expect TimeoutError caused by CancelledError. + e12 = e1.__cause__ + self.assertIsInstance(e12, asyncio.CancelledError) + self.assertIsNone(e12.__cause__) + self.assertIsNone(e12.__context__) + self.assertIs(e1.__context__, e12) + e2 = cm2.exception + # Expect TimeoutError caused by CancelledError raised during + # handling of other CancelledError (which is the same as in + # the above chain). + e22 = e2.__cause__ + self.assertIsInstance(e22, asyncio.CancelledError) + self.assertIsNone(e22.__cause__) + self.assertIs(e22.__context__, e12) + self.assertIs(e2.__context__, e22) async def test_timeout_after_cancellation(self): try: @@ -235,7 +304,7 @@ async def test_timeout_after_cancellation(self): except asyncio.CancelledError: pass finally: - with self.assertRaises(TimeoutError): + with self.assertRaises(TimeoutError) as cm: async with asyncio.timeout(0.0): await asyncio.sleep(1) # some cleanup @@ -251,13 +320,6 @@ async def test_cancel_in_timeout_after_cancellation(self): asyncio.current_task().cancel() await asyncio.sleep(2) # some cleanup - async def test_timeout_exception_cause (self): - with self.assertRaises(asyncio.TimeoutError) as exc: - async with asyncio.timeout(0): - await asyncio.sleep(1) - cause = exc.exception.__cause__ - assert isinstance(cause, asyncio.CancelledError) - async def test_timeout_already_entered(self): async with asyncio.timeout(0.01) as cm: with self.assertRaisesRegex(RuntimeError, "has already been entered"): @@ -303,6 +365,47 @@ async def test_timeout_without_task(self): with self.assertRaisesRegex(RuntimeError, "has not been entered"): cm.reschedule(0.02) + async def test_timeout_taskgroup(self): + async def task(): + try: + await asyncio.sleep(2) # Will be interrupted after 0.01 second + finally: + 1/0 # Crash in cleanup + + with self.assertRaises(ExceptionGroup) as cm: + async with asyncio.timeout(0.01): + async with asyncio.TaskGroup() as tg: + tg.create_task(task()) + try: + raise ValueError + finally: + await asyncio.sleep(1) + eg = cm.exception + # Expect ExceptionGroup raised during handling of TimeoutError caused + # by CancelledError raised during handling of ValueError. + self.assertIsNone(eg.__cause__) + e_1 = eg.__context__ + self.assertIsInstance(e_1, TimeoutError) + e_2 = e_1.__cause__ + self.assertIsInstance(e_2, asyncio.CancelledError) + self.assertIsNone(e_2.__cause__) + self.assertIsInstance(e_2.__context__, ValueError) + self.assertIs(e_1.__context__, e_2) + + self.assertEqual(len(eg.exceptions), 1, eg) + e1 = eg.exceptions[0] + # Expect ZeroDivisionError raised during handling of TimeoutError + # caused by CancelledError (it is a different CancelledError). + self.assertIsInstance(e1, ZeroDivisionError) + self.assertIsNone(e1.__cause__) + e2 = e1.__context__ + self.assertIsInstance(e2, TimeoutError) + e3 = e2.__cause__ + self.assertIsInstance(e3, asyncio.CancelledError) + self.assertIsNone(e3.__context__) + self.assertIsNone(e3.__cause__) + self.assertIs(e2.__context__, e3) + if __name__ == '__main__': unittest.main() diff --git a/Lib/test/test_asyncio/test_unix_events.py b/Lib/test/test_asyncio/test_unix_events.py index d2c8cba6acfa31..59ef9f5f58cabc 100644 --- a/Lib/test/test_asyncio/test_unix_events.py +++ b/Lib/test/test_asyncio/test_unix_events.py @@ -1874,7 +1874,7 @@ async def runner(): wsock.close() -@unittest.skipUnless(hasattr(os, 'fork'), 'requires os.fork()') +@support.requires_fork() class TestFork(unittest.IsolatedAsyncioTestCase): async def test_fork_not_share_event_loop(self): diff --git a/Lib/test/test_base64.py b/Lib/test/test_base64.py index fa03fa1d61ceab..f6171d3ed4efd7 100644 --- a/Lib/test/test_base64.py +++ b/Lib/test/test_base64.py @@ -586,6 +586,7 @@ def test_a85decode(self): eq(base64.a85decode(b'y+', b"www.python.org") @@ -689,6 +690,8 @@ def test_a85decode_errors(self): self.assertRaises(ValueError, base64.a85decode, b's8W', adobe=False) self.assertRaises(ValueError, base64.a85decode, b's8W-', adobe=False) self.assertRaises(ValueError, base64.a85decode, b's8W-"', adobe=False) + self.assertRaises(ValueError, base64.a85decode, b'aaaay', + foldspaces=True) def test_b85decode_errors(self): illegal = list(range(33)) + \ diff --git a/Lib/test/test_buffer.py b/Lib/test/test_buffer.py index 72a06d6af450e3..5b1b95b9c82064 100644 --- a/Lib/test/test_buffer.py +++ b/Lib/test/test_buffer.py @@ -4585,6 +4585,33 @@ def test_c_buffer(self): buf.__release_buffer__(mv) self.assertEqual(buf.references, 0) + @unittest.skipIf(_testcapi is None, "requires _testcapi") + def test_c_buffer_invalid_flags(self): + buf = _testcapi.testBuf() + self.assertRaises(SystemError, buf.__buffer__, PyBUF_READ) + self.assertRaises(SystemError, buf.__buffer__, PyBUF_WRITE) + + @unittest.skipIf(_testcapi is None, "requires _testcapi") + def test_c_fill_buffer_invalid_flags(self): + # PyBuffer_FillInfo + source = b"abc" + self.assertRaises(SystemError, _testcapi.buffer_fill_info, + source, 0, PyBUF_READ) + self.assertRaises(SystemError, _testcapi.buffer_fill_info, + source, 0, PyBUF_WRITE) + + @unittest.skipIf(_testcapi is None, "requires _testcapi") + def test_c_fill_buffer_readonly_and_writable(self): + source = b"abc" + with _testcapi.buffer_fill_info(source, 1, PyBUF_SIMPLE) as m: + self.assertEqual(bytes(m), b"abc") + self.assertTrue(m.readonly) + with _testcapi.buffer_fill_info(source, 0, PyBUF_WRITABLE) as m: + self.assertEqual(bytes(m), b"abc") + self.assertFalse(m.readonly) + self.assertRaises(BufferError, _testcapi.buffer_fill_info, + source, 1, PyBUF_WRITABLE) + def test_inheritance(self): class A(bytearray): def __buffer__(self, flags): diff --git a/Lib/test/test_builtin.py b/Lib/test/test_builtin.py index e15492783aeec1..7a3ab2274a58f2 100644 --- a/Lib/test/test_builtin.py +++ b/Lib/test/test_builtin.py @@ -611,6 +611,14 @@ def __dir__(self): self.assertIsInstance(res, list) self.assertTrue(res == ["a", "b", "c"]) + # dir(obj__dir__iterable) + class Foo(object): + def __dir__(self): + return {"b", "c", "a"} + res = dir(Foo()) + self.assertIsInstance(res, list) + self.assertEqual(sorted(res), ["a", "b", "c"]) + # dir(obj__dir__not_sequence) class Foo(object): def __dir__(self): @@ -629,8 +637,8 @@ def __dir__(self): def test___ne__(self): self.assertFalse(None.__ne__(None)) - self.assertTrue(None.__ne__(0)) - self.assertTrue(None.__ne__("abc")) + self.assertIs(None.__ne__(0), NotImplemented) + self.assertIs(None.__ne__("abc"), NotImplemented) def test_divmod(self): self.assertEqual(divmod(12, 7), (1, 5)) diff --git a/Lib/test/test_c_locale_coercion.py b/Lib/test/test_c_locale_coercion.py index 71f934756e26a1..7334a325ba22f0 100644 --- a/Lib/test/test_c_locale_coercion.py +++ b/Lib/test/test_c_locale_coercion.py @@ -112,12 +112,16 @@ class EncodingDetails(_EncodingDetails): ]) @classmethod - def get_expected_details(cls, coercion_expected, fs_encoding, stream_encoding, env_vars): + def get_expected_details(cls, coercion_expected, fs_encoding, stream_encoding, stream_errors, env_vars): """Returns expected child process details for a given encoding""" _stream = stream_encoding + ":{}" - # stdin and stdout should use surrogateescape either because the - # coercion triggered, or because the C locale was detected - stream_info = 2*[_stream.format("surrogateescape")] + if stream_errors is None: + # stdin and stdout should use surrogateescape either because the + # coercion triggered, or because the C locale was detected + stream_errors = "surrogateescape" + + stream_info = [_stream.format(stream_errors)] * 2 + # stderr should always use backslashreplace stream_info.append(_stream.format("backslashreplace")) expected_lang = env_vars.get("LANG", "not set") @@ -210,6 +214,7 @@ def _check_child_encoding_details(self, env_vars, expected_fs_encoding, expected_stream_encoding, + expected_stream_errors, expected_warnings, coercion_expected): """Check the C locale handling for the given process environment @@ -225,6 +230,7 @@ def _check_child_encoding_details(self, coercion_expected, expected_fs_encoding, expected_stream_encoding, + expected_stream_errors, env_vars ) self.assertEqual(encoding_details, expected_details) @@ -257,6 +263,7 @@ def test_external_target_locale_configuration(self): "LC_CTYPE": "", "LC_ALL": "", "PYTHONCOERCECLOCALE": "", + "PYTHONIOENCODING": "", } for env_var in ("LANG", "LC_CTYPE"): for locale_to_set in AVAILABLE_TARGETS: @@ -273,10 +280,43 @@ def test_external_target_locale_configuration(self): self._check_child_encoding_details(var_dict, expected_fs_encoding, expected_stream_encoding, + expected_stream_errors=None, expected_warnings=None, coercion_expected=False) + def test_with_ioencoding(self): + # Explicitly setting a target locale should give the same behaviour as + # is seen when implicitly coercing to that target locale + self.maxDiff = None + + expected_fs_encoding = "utf-8" + expected_stream_encoding = "utf-8" + base_var_dict = { + "LANG": "", + "LC_CTYPE": "", + "LC_ALL": "", + "PYTHONCOERCECLOCALE": "", + "PYTHONIOENCODING": "UTF-8", + } + for env_var in ("LANG", "LC_CTYPE"): + for locale_to_set in AVAILABLE_TARGETS: + # XXX (ncoghlan): LANG=UTF-8 doesn't appear to work as + # expected, so skip that combination for now + # See https://bugs.python.org/issue30672 for discussion + if env_var == "LANG" and locale_to_set == "UTF-8": + continue + + with self.subTest(env_var=env_var, + configured_locale=locale_to_set): + var_dict = base_var_dict.copy() + var_dict[env_var] = locale_to_set + self._check_child_encoding_details(var_dict, + expected_fs_encoding, + expected_stream_encoding, + expected_stream_errors="strict", + expected_warnings=None, + coercion_expected=False) @support.cpython_only @unittest.skipUnless(sysconfig.get_config_var("PY_COERCE_C_LOCALE"), @@ -316,6 +356,7 @@ def _check_c_locale_coercion(self, "LC_CTYPE": "", "LC_ALL": "", "PYTHONCOERCECLOCALE": "", + "PYTHONIOENCODING": "", } base_var_dict.update(extra_vars) if coerce_c_locale is not None: @@ -340,6 +381,7 @@ def _check_c_locale_coercion(self, self._check_child_encoding_details(base_var_dict, fs_encoding, stream_encoding, + None, _expected_warnings, _coercion_expected) @@ -348,13 +390,15 @@ def _check_c_locale_coercion(self, for env_var in ("LANG", "LC_CTYPE"): with self.subTest(env_var=env_var, nominal_locale=locale_to_set, - PYTHONCOERCECLOCALE=coerce_c_locale): + PYTHONCOERCECLOCALE=coerce_c_locale, + PYTHONIOENCODING=""): var_dict = base_var_dict.copy() var_dict[env_var] = locale_to_set # Check behaviour on successful coercion self._check_child_encoding_details(var_dict, fs_encoding, stream_encoding, + None, expected_warnings, coercion_expected) diff --git a/Lib/test/test_call.py b/Lib/test/test_call.py index b1c78d7136fc9b..2a6a5d287b04ee 100644 --- a/Lib/test/test_call.py +++ b/Lib/test/test_call.py @@ -1,5 +1,6 @@ import unittest -from test.support import cpython_only, requires_limited_api, skip_on_s390x +from test.support import (cpython_only, is_wasi, requires_limited_api, Py_DEBUG, + set_recursion_limit, skip_on_s390x) try: import _testcapi except ImportError: @@ -154,7 +155,7 @@ def test_varargs16_kw(self): min, 0, default=1, key=2, foo=3) def test_varargs17_kw(self): - msg = r"'foo' is an invalid keyword argument for print\(\)$" + msg = r"print\(\) got an unexpected keyword argument 'foo'$" self.assertRaisesRegex(TypeError, msg, print, 0, sep=1, end=2, file=3, flush=4, foo=5) @@ -927,7 +928,7 @@ def check_suggestion_includes(self, message): self.assertIn(f"Did you mean '{message}'?", str(cm.exception)) @contextlib.contextmanager - def check_suggestion_not_pressent(self): + def check_suggestion_not_present(self): with self.assertRaises(TypeError) as cm: yield self.assertNotIn("Did you mean", str(cm.exception)) @@ -945,7 +946,7 @@ def foo(blech=None, /, aaa=None, *args, late1=None): for keyword, suggestion in cases: with self.subTest(keyword): - ctx = self.check_suggestion_includes(suggestion) if suggestion else self.check_suggestion_not_pressent() + ctx = self.check_suggestion_includes(suggestion) if suggestion else self.check_suggestion_not_present() with ctx: foo(**{keyword:None}) @@ -986,10 +987,37 @@ def case_change_over_substitution(BLuch=None, Luch = None, fluch = None): with self.check_suggestion_includes(suggestion): func(bluch=None) + def test_unexpected_keyword_suggestion_via_getargs(self): + with self.check_suggestion_includes("maxsplit"): + "foo".split(maxsplt=1) + + self.assertRaisesRegex( + TypeError, r"split\(\) got an unexpected keyword argument 'blech'$", + "foo".split, blech=1 + ) + with self.check_suggestion_not_present(): + "foo".split(blech=1) + with self.check_suggestion_not_present(): + "foo".split(more_noise=1, maxsplt=1) + + # Also test the vgetargskeywords path + with self.check_suggestion_includes("name"): + ImportError(namez="oops") + + self.assertRaisesRegex( + TypeError, r"ImportError\(\) got an unexpected keyword argument 'blech'$", + ImportError, blech=1 + ) + with self.check_suggestion_not_present(): + ImportError(blech=1) + with self.check_suggestion_not_present(): + ImportError(blech=1, namez="oops") + @cpython_only class TestRecursion(unittest.TestCase): @skip_on_s390x + @unittest.skipIf(is_wasi and Py_DEBUG, "requires deep stack") def test_super_deep(self): def recurse(n): @@ -1010,9 +1038,7 @@ def c_py_recurse(m): if m: _testcapi.pyobject_vectorcall(py_recurse, (1000, m), ()) - depth = sys.getrecursionlimit() - sys.setrecursionlimit(100_000) - try: + with set_recursion_limit(100_000): recurse(90_000) with self.assertRaises(RecursionError): recurse(101_000) @@ -1022,8 +1048,6 @@ def c_py_recurse(m): c_py_recurse(90) with self.assertRaises(RecursionError): c_py_recurse(100_000) - finally: - sys.setrecursionlimit(depth) class TestFunctionWithManyArgs(unittest.TestCase): diff --git a/Lib/test/test_capi/test_complex.py b/Lib/test/test_capi/test_complex.py index d6fc1f077c40aa..a5b59558e7f851 100644 --- a/Lib/test/test_capi/test_complex.py +++ b/Lib/test/test_capi/test_complex.py @@ -77,8 +77,14 @@ def test_realasdouble(self): self.assertEqual(realasdouble(FloatSubclass(4.25)), 4.25) # Test types with __complex__ dunder method - # Function doesn't support classes with __complex__ dunder, see #109598 - self.assertRaises(TypeError, realasdouble, Complex()) + self.assertEqual(realasdouble(Complex()), 4.25) + self.assertRaises(TypeError, realasdouble, BadComplex()) + with self.assertWarns(DeprecationWarning): + self.assertEqual(realasdouble(BadComplex2()), 4.25) + with warnings.catch_warnings(): + warnings.simplefilter("error", DeprecationWarning) + self.assertRaises(DeprecationWarning, realasdouble, BadComplex2()) + self.assertRaises(RuntimeError, realasdouble, BadComplex3()) # Test types with __float__ dunder method self.assertEqual(realasdouble(Float()), 4.25) @@ -104,11 +110,22 @@ def test_imagasdouble(self): self.assertEqual(imagasdouble(FloatSubclass(4.25)), 0.0) # Test types with __complex__ dunder method - # Function doesn't support classes with __complex__ dunder, see #109598 - self.assertEqual(imagasdouble(Complex()), 0.0) + self.assertEqual(imagasdouble(Complex()), 0.5) + self.assertRaises(TypeError, imagasdouble, BadComplex()) + with self.assertWarns(DeprecationWarning): + self.assertEqual(imagasdouble(BadComplex2()), 0.5) + with warnings.catch_warnings(): + warnings.simplefilter("error", DeprecationWarning) + self.assertRaises(DeprecationWarning, imagasdouble, BadComplex2()) + self.assertRaises(RuntimeError, imagasdouble, BadComplex3()) + + # Test types with __float__ dunder method + self.assertEqual(imagasdouble(Float()), 0.0) + self.assertRaises(TypeError, imagasdouble, BadFloat()) + with self.assertWarns(DeprecationWarning): + self.assertEqual(imagasdouble(BadFloat2()), 0.0) - # Function returns 0.0 anyway, see #109598 - self.assertEqual(imagasdouble(object()), 0.0) + self.assertRaises(TypeError, imagasdouble, object()) # CRASHES imagasdouble(NULL) diff --git a/Lib/test/test_capi/test_dict.py b/Lib/test/test_capi/test_dict.py index 57a7238588eae0..cca6145bc90c04 100644 --- a/Lib/test/test_capi/test_dict.py +++ b/Lib/test/test_capi/test_dict.py @@ -339,6 +339,28 @@ def test_dict_setdefault(self): # CRASHES setdefault({}, 'a', NULL) # CRASHES setdefault(NULL, 'a', 5) + def test_dict_setdefaultref(self): + setdefault = _testcapi.dict_setdefaultref + dct = {} + self.assertEqual(setdefault(dct, 'a', 5), 5) + self.assertEqual(dct, {'a': 5}) + self.assertEqual(setdefault(dct, 'a', 8), 5) + self.assertEqual(dct, {'a': 5}) + + dct2 = DictSubclass() + self.assertEqual(setdefault(dct2, 'a', 5), 5) + self.assertEqual(dct2, {'a': 5}) + self.assertEqual(setdefault(dct2, 'a', 8), 5) + self.assertEqual(dct2, {'a': 5}) + + self.assertRaises(TypeError, setdefault, {}, [], 5) # unhashable + self.assertRaises(SystemError, setdefault, UserDict(), 'a', 5) + self.assertRaises(SystemError, setdefault, [1], 0, 5) + self.assertRaises(SystemError, setdefault, 42, 'a', 5) + # CRASHES setdefault({}, NULL, 5) + # CRASHES setdefault({}, 'a', NULL) + # CRASHES setdefault(NULL, 'a', 5) + def test_mapping_keys_valuesitems(self): class BadMapping(dict): def keys(self): diff --git a/Lib/test/test_capi/test_getargs.py b/Lib/test/test_capi/test_getargs.py index 9b6aef27625ad0..12039803ba543e 100644 --- a/Lib/test/test_capi/test_getargs.py +++ b/Lib/test/test_capi/test_getargs.py @@ -667,7 +667,7 @@ def test_invalid_keyword(self): try: getargs_keywords((1,2),3,arg5=10,arg666=666) except TypeError as err: - self.assertEqual(str(err), "'arg666' is an invalid keyword argument for this function") + self.assertEqual(str(err), "this function got an unexpected keyword argument 'arg666'") else: self.fail('TypeError should have been raised') @@ -675,7 +675,7 @@ def test_surrogate_keyword(self): try: getargs_keywords((1,2), 3, (4,(5,6)), (7,8,9), **{'\uDC80': 10}) except TypeError as err: - self.assertEqual(str(err), "'\udc80' is an invalid keyword argument for this function") + self.assertEqual(str(err), "this function got an unexpected keyword argument '\udc80'") else: self.fail('TypeError should have been raised') @@ -742,12 +742,12 @@ def test_too_many_args(self): def test_invalid_keyword(self): # extraneous keyword arg with self.assertRaisesRegex(TypeError, - "'monster' is an invalid keyword argument for this function"): + "this function got an unexpected keyword argument 'monster'"): getargs_keyword_only(1, 2, monster=666) def test_surrogate_keyword(self): with self.assertRaisesRegex(TypeError, - "'\udc80' is an invalid keyword argument for this function"): + "this function got an unexpected keyword argument '\udc80'"): getargs_keyword_only(1, 2, **{'\uDC80': 10}) def test_weird_str_subclass(self): @@ -761,7 +761,7 @@ def __hash__(self): "invalid keyword argument for this function"): getargs_keyword_only(1, 2, **{BadStr("keyword_only"): 3}) with self.assertRaisesRegex(TypeError, - "invalid keyword argument for this function"): + "this function got an unexpected keyword argument"): getargs_keyword_only(1, 2, **{BadStr("monster"): 666}) def test_weird_str_subclass2(self): @@ -774,7 +774,7 @@ def __hash__(self): "invalid keyword argument for this function"): getargs_keyword_only(1, 2, **{BadStr("keyword_only"): 3}) with self.assertRaisesRegex(TypeError, - "invalid keyword argument for this function"): + "this function got an unexpected keyword argument"): getargs_keyword_only(1, 2, **{BadStr("monster"): 666}) @@ -807,7 +807,7 @@ def test_required_args(self): def test_empty_keyword(self): with self.assertRaisesRegex(TypeError, - "'' is an invalid keyword argument for this function"): + "this function got an unexpected keyword argument ''"): self.getargs(1, 2, **{'': 666}) @@ -1204,7 +1204,7 @@ def test_basic(self): "function missing required argument 'a'"): parse((), {}, 'O', ['a']) with self.assertRaisesRegex(TypeError, - "'b' is an invalid keyword argument"): + "this function got an unexpected keyword argument 'b'"): parse((), {'b': 1}, '|O', ['a']) with self.assertRaisesRegex(TypeError, fr"argument for function given by name \('a'\) " @@ -1278,10 +1278,10 @@ def test_nonascii_keywords(self): fr"and position \(1\)"): parse((1,), {name: 2}, 'O|O', [name, 'b']) with self.assertRaisesRegex(TypeError, - f"'{name}' is an invalid keyword argument"): + f"this function got an unexpected keyword argument '{name}'"): parse((), {name: 1}, '|O', ['b']) with self.assertRaisesRegex(TypeError, - "'b' is an invalid keyword argument"): + "this function got an unexpected keyword argument 'b'"): parse((), {'b': 1}, '|O', [name]) invalid = name.encode() + (name.encode()[:-1] or b'\x80') @@ -1301,17 +1301,17 @@ def test_nonascii_keywords(self): for name2 in ('b', 'ë', 'ĉ', 'Ɐ', '𐀁'): with self.subTest(name2=name2): with self.assertRaisesRegex(TypeError, - f"'{name2}' is an invalid keyword argument"): + f"this function got an unexpected keyword argument '{name2}'"): parse((), {name2: 1}, '|O', [name]) name2 = name.encode().decode('latin1') if name2 != name: with self.assertRaisesRegex(TypeError, - f"'{name2}' is an invalid keyword argument"): + f"this function got an unexpected keyword argument '{name2}'"): parse((), {name2: 1}, '|O', [name]) name3 = name + '3' with self.assertRaisesRegex(TypeError, - f"'{name2}' is an invalid keyword argument"): + f"this function got an unexpected keyword argument '{name2}'"): parse((), {name2: 1, name3: 2}, '|OO', [name, name3]) def test_nested_tuple(self): diff --git a/Lib/test/test_capi/test_list.py b/Lib/test/test_capi/test_list.py index eb03d51d3def37..dceb4fce3c077b 100644 --- a/Lib/test/test_capi/test_list.py +++ b/Lib/test/test_capi/test_list.py @@ -82,10 +82,8 @@ def test_list_get_size(self): # CRASHES size(UserList()) # CRASHES size(NULL) - - def test_list_getitem(self): - # Test PyList_GetItem() - getitem = _testcapi.list_getitem + def check_list_get_item(self, getitem, exctype): + # Common test cases for PyList_GetItem() and PyList_GetItemRef() lst = [1, 2, 3] self.assertEqual(getitem(lst, 0), 1) self.assertEqual(getitem(lst, 2), 3) @@ -93,12 +91,19 @@ def test_list_getitem(self): self.assertRaises(IndexError, getitem, lst, -1) self.assertRaises(IndexError, getitem, lst, PY_SSIZE_T_MIN) self.assertRaises(IndexError, getitem, lst, PY_SSIZE_T_MAX) - self.assertRaises(SystemError, getitem, 42, 1) - self.assertRaises(SystemError, getitem, (1, 2, 3), 1) - self.assertRaises(SystemError, getitem, {1: 2}, 1) - + self.assertRaises(exctype, getitem, 42, 1) + self.assertRaises(exctype, getitem, (1, 2, 3), 1) + self.assertRaises(exctype, getitem, {1: 2}, 1) # CRASHES getitem(NULL, 1) + def test_list_getitem(self): + # Test PyList_GetItem() + self.check_list_get_item(_testcapi.list_getitem, SystemError) + + def test_list_get_item_ref(self): + # Test PyList_GetItemRef() + self.check_list_get_item(_testcapi.list_get_item_ref, TypeError) + def test_list_get_item(self): # Test PyList_GET_ITEM() get_item = _testcapi.list_get_item @@ -112,7 +117,6 @@ def test_list_get_item(self): # CRASHES get_item(21, 2) # CRASHES get_item(NULL, 1) - def test_list_setitem(self): # Test PyList_SetItem() setitem = _testcapi.list_setitem diff --git a/Lib/test/test_capi/test_misc.py b/Lib/test/test_capi/test_misc.py index 123813b949fb7d..67fbef4f269814 100644 --- a/Lib/test/test_capi/test_misc.py +++ b/Lib/test/test_capi/test_misc.py @@ -469,6 +469,8 @@ def __del__(self): del L self.assertEqual(PyList.num, 0) + @unittest.skipIf(MISSING_C_DOCSTRINGS, + "Signature information for builtins requires docstrings") def test_heap_ctype_doc_and_text_signature(self): self.assertEqual(_testcapi.HeapDocCType.__doc__, "somedoc") self.assertEqual(_testcapi.HeapDocCType.__text_signature__, "(arg1, arg2)") diff --git a/Lib/test/test_capi/test_structmembers.py b/Lib/test/test_capi/test_structmembers.py index 2cf46b203478dc..a294c3b13a5c30 100644 --- a/Lib/test/test_capi/test_structmembers.py +++ b/Lib/test/test_capi/test_structmembers.py @@ -14,6 +14,13 @@ PY_SSIZE_T_MAX, PY_SSIZE_T_MIN, ) + +class Index: + def __init__(self, value): + self.value = value + def __index__(self): + return self.value + # There are two classes: one using and another using # `Py_`-prefixed API. They should behave the same in Python @@ -38,75 +45,115 @@ class ReadWriteTests: def setUp(self): self.ts = _make_test_object(self.cls) + def _test_write(self, name, value, expected=None): + if expected is None: + expected = value + ts = self.ts + setattr(ts, name, value) + self.assertEqual(getattr(ts, name), expected) + + def _test_warn(self, name, value, expected=None): + ts = self.ts + self.assertWarns(RuntimeWarning, setattr, ts, name, value) + if expected is not None: + self.assertEqual(getattr(ts, name), expected) + + def _test_overflow(self, name, value): + ts = self.ts + self.assertRaises(OverflowError, setattr, ts, name, value) + + def _test_int_range(self, name, minval, maxval, *, hardlimit=None, + indexlimit=None): + if hardlimit is None: + hardlimit = (minval, maxval) + ts = self.ts + self._test_write(name, minval) + self._test_write(name, maxval) + hardminval, hardmaxval = hardlimit + self._test_overflow(name, hardminval-1) + self._test_overflow(name, hardmaxval+1) + self._test_overflow(name, 2**1000) + self._test_overflow(name, -2**1000) + if hardminval < minval: + self._test_warn(name, hardminval) + self._test_warn(name, minval-1, maxval) + if maxval < hardmaxval: + self._test_warn(name, maxval+1, minval) + self._test_warn(name, hardmaxval) + + if indexlimit is None: + indexlimit = hardlimit + if not indexlimit: + self.assertRaises(TypeError, setattr, ts, name, Index(minval)) + self.assertRaises(TypeError, setattr, ts, name, Index(maxval)) + else: + hardminindexval, hardmaxindexval = indexlimit + self._test_write(name, Index(minval), minval) + if minval < hardminindexval: + self._test_write(name, Index(hardminindexval), hardminindexval) + if maxval < hardmaxindexval: + self._test_write(name, Index(maxval), maxval) + else: + self._test_write(name, Index(hardmaxindexval), hardmaxindexval) + self._test_overflow(name, Index(hardminindexval-1)) + if name in ('T_UINT', 'T_ULONG'): + self.assertRaises(TypeError, setattr, self.ts, name, + Index(hardmaxindexval+1)) + self.assertRaises(TypeError, setattr, self.ts, name, + Index(2**1000)) + else: + self._test_overflow(name, Index(hardmaxindexval+1)) + self._test_overflow(name, Index(2**1000)) + self._test_overflow(name, Index(-2**1000)) + if hardminindexval < minval and name != 'T_ULONGLONG': + self._test_warn(name, Index(hardminindexval)) + self._test_warn(name, Index(minval-1)) + if maxval < hardmaxindexval: + self._test_warn(name, Index(maxval+1)) + self._test_warn(name, Index(hardmaxindexval)) + def test_bool(self): ts = self.ts ts.T_BOOL = True - self.assertEqual(ts.T_BOOL, True) + self.assertIs(ts.T_BOOL, True) ts.T_BOOL = False - self.assertEqual(ts.T_BOOL, False) + self.assertIs(ts.T_BOOL, False) self.assertRaises(TypeError, setattr, ts, 'T_BOOL', 1) + self.assertRaises(TypeError, setattr, ts, 'T_BOOL', 0) + self.assertRaises(TypeError, setattr, ts, 'T_BOOL', None) def test_byte(self): - ts = self.ts - ts.T_BYTE = CHAR_MAX - self.assertEqual(ts.T_BYTE, CHAR_MAX) - ts.T_BYTE = CHAR_MIN - self.assertEqual(ts.T_BYTE, CHAR_MIN) - ts.T_UBYTE = UCHAR_MAX - self.assertEqual(ts.T_UBYTE, UCHAR_MAX) + self._test_int_range('T_BYTE', CHAR_MIN, CHAR_MAX, + hardlimit=(LONG_MIN, LONG_MAX)) + self._test_int_range('T_UBYTE', 0, UCHAR_MAX, + hardlimit=(LONG_MIN, LONG_MAX)) def test_short(self): - ts = self.ts - ts.T_SHORT = SHRT_MAX - self.assertEqual(ts.T_SHORT, SHRT_MAX) - ts.T_SHORT = SHRT_MIN - self.assertEqual(ts.T_SHORT, SHRT_MIN) - ts.T_USHORT = USHRT_MAX - self.assertEqual(ts.T_USHORT, USHRT_MAX) + self._test_int_range('T_SHORT', SHRT_MIN, SHRT_MAX, + hardlimit=(LONG_MIN, LONG_MAX)) + self._test_int_range('T_USHORT', 0, USHRT_MAX, + hardlimit=(LONG_MIN, LONG_MAX)) def test_int(self): - ts = self.ts - ts.T_INT = INT_MAX - self.assertEqual(ts.T_INT, INT_MAX) - ts.T_INT = INT_MIN - self.assertEqual(ts.T_INT, INT_MIN) - ts.T_UINT = UINT_MAX - self.assertEqual(ts.T_UINT, UINT_MAX) + self._test_int_range('T_INT', INT_MIN, INT_MAX, + hardlimit=(LONG_MIN, LONG_MAX)) + self._test_int_range('T_UINT', 0, UINT_MAX, + hardlimit=(LONG_MIN, ULONG_MAX), + indexlimit=(LONG_MIN, LONG_MAX)) def test_long(self): - ts = self.ts - ts.T_LONG = LONG_MAX - self.assertEqual(ts.T_LONG, LONG_MAX) - ts.T_LONG = LONG_MIN - self.assertEqual(ts.T_LONG, LONG_MIN) - ts.T_ULONG = ULONG_MAX - self.assertEqual(ts.T_ULONG, ULONG_MAX) + self._test_int_range('T_LONG', LONG_MIN, LONG_MAX) + self._test_int_range('T_ULONG', 0, ULONG_MAX, + hardlimit=(LONG_MIN, ULONG_MAX), + indexlimit=(LONG_MIN, LONG_MAX)) def test_py_ssize_t(self): - ts = self.ts - ts.T_PYSSIZET = PY_SSIZE_T_MAX - self.assertEqual(ts.T_PYSSIZET, PY_SSIZE_T_MAX) - ts.T_PYSSIZET = PY_SSIZE_T_MIN - self.assertEqual(ts.T_PYSSIZET, PY_SSIZE_T_MIN) + self._test_int_range('T_PYSSIZET', PY_SSIZE_T_MIN, PY_SSIZE_T_MAX, indexlimit=False) def test_longlong(self): - ts = self.ts - if not hasattr(ts, "T_LONGLONG"): - self.skipTest("long long not present") - - ts.T_LONGLONG = LLONG_MAX - self.assertEqual(ts.T_LONGLONG, LLONG_MAX) - ts.T_LONGLONG = LLONG_MIN - self.assertEqual(ts.T_LONGLONG, LLONG_MIN) - - ts.T_ULONGLONG = ULLONG_MAX - self.assertEqual(ts.T_ULONGLONG, ULLONG_MAX) - - ## make sure these will accept a plain int as well as a long - ts.T_LONGLONG = 3 - self.assertEqual(ts.T_LONGLONG, 3) - ts.T_ULONGLONG = 4 - self.assertEqual(ts.T_ULONGLONG, 4) + self._test_int_range('T_LONGLONG', LLONG_MIN, LLONG_MAX) + self._test_int_range('T_ULONGLONG', 0, ULLONG_MAX, + indexlimit=(LONG_MIN, LONG_MAX)) def test_bad_assignments(self): ts = self.ts @@ -116,10 +163,9 @@ def test_bad_assignments(self): 'T_SHORT', 'T_USHORT', 'T_INT', 'T_UINT', 'T_LONG', 'T_ULONG', + 'T_LONGLONG', 'T_ULONGLONG', 'T_PYSSIZET' ] - if hasattr(ts, 'T_LONGLONG'): - integer_attributes.extend(['T_LONGLONG', 'T_ULONGLONG']) # issue8014: this produced 'bad argument to internal function' # internal error @@ -139,46 +185,6 @@ class ReadWriteTests_OldAPI(ReadWriteTests, unittest.TestCase): class ReadWriteTests_NewAPI(ReadWriteTests, unittest.TestCase): cls = _test_structmembersType_NewAPI -class TestWarnings: - def setUp(self): - self.ts = _make_test_object(self.cls) - - def test_byte_max(self): - ts = self.ts - with warnings_helper.check_warnings(('', RuntimeWarning)): - ts.T_BYTE = CHAR_MAX+1 - - def test_byte_min(self): - ts = self.ts - with warnings_helper.check_warnings(('', RuntimeWarning)): - ts.T_BYTE = CHAR_MIN-1 - - def test_ubyte_max(self): - ts = self.ts - with warnings_helper.check_warnings(('', RuntimeWarning)): - ts.T_UBYTE = UCHAR_MAX+1 - - def test_short_max(self): - ts = self.ts - with warnings_helper.check_warnings(('', RuntimeWarning)): - ts.T_SHORT = SHRT_MAX+1 - - def test_short_min(self): - ts = self.ts - with warnings_helper.check_warnings(('', RuntimeWarning)): - ts.T_SHORT = SHRT_MIN-1 - - def test_ushort_max(self): - ts = self.ts - with warnings_helper.check_warnings(('', RuntimeWarning)): - ts.T_USHORT = USHRT_MAX+1 - -class TestWarnings_OldAPI(TestWarnings, unittest.TestCase): - cls = _test_structmembersType_OldAPI - -class TestWarnings_NewAPI(TestWarnings, unittest.TestCase): - cls = _test_structmembersType_NewAPI - if __name__ == "__main__": unittest.main() diff --git a/Lib/test/test_capi/test_watchers.py b/Lib/test/test_capi/test_watchers.py index 5981712c80c3a9..ae062b1bda26b7 100644 --- a/Lib/test/test_capi/test_watchers.py +++ b/Lib/test/test_capi/test_watchers.py @@ -151,8 +151,8 @@ def test_watch_out_of_range_watcher_id(self): def test_watch_unassigned_watcher_id(self): d = {} - with self.assertRaisesRegex(ValueError, r"No dict watcher set for ID 1"): - self.watch(1, d) + with self.assertRaisesRegex(ValueError, r"No dict watcher set for ID 3"): + self.watch(3, d) def test_unwatch_non_dict(self): with self.watcher() as wid: @@ -168,8 +168,8 @@ def test_unwatch_out_of_range_watcher_id(self): def test_unwatch_unassigned_watcher_id(self): d = {} - with self.assertRaisesRegex(ValueError, r"No dict watcher set for ID 1"): - self.unwatch(1, d) + with self.assertRaisesRegex(ValueError, r"No dict watcher set for ID 3"): + self.unwatch(3, d) def test_clear_out_of_range_watcher_id(self): with self.assertRaisesRegex(ValueError, r"Invalid dict watcher ID -1"): @@ -178,8 +178,8 @@ def test_clear_out_of_range_watcher_id(self): self.clear_watcher(8) # DICT_MAX_WATCHERS = 8 def test_clear_unassigned_watcher_id(self): - with self.assertRaisesRegex(ValueError, r"No dict watcher set for ID 1"): - self.clear_watcher(1) + with self.assertRaisesRegex(ValueError, r"No dict watcher set for ID 3"): + self.clear_watcher(3) class TestTypeWatchers(unittest.TestCase): diff --git a/Lib/test/test_class.py b/Lib/test/test_class.py index 1531aad4f1f779..d59271435e9eb0 100644 --- a/Lib/test/test_class.py +++ b/Lib/test/test_class.py @@ -771,6 +771,22 @@ def add_one_level(): with self.assertRaises(RecursionError): add_one_level() + def testMetaclassCallOptimization(self): + calls = 0 + + class TypeMetaclass(type): + def __call__(cls, *args, **kwargs): + nonlocal calls + calls += 1 + return type.__call__(cls, *args, **kwargs) + + class Type(metaclass=TypeMetaclass): + def __init__(self, obj): + self._obj = obj + + for i in range(100): + Type(i) + self.assertEqual(calls, 100) if __name__ == '__main__': unittest.main() diff --git a/Lib/test/test_clinic.py b/Lib/test/test_clinic.py index f3f73a174aeb13..e987ce54605497 100644 --- a/Lib/test/test_clinic.py +++ b/Lib/test/test_clinic.py @@ -16,12 +16,13 @@ test_tools.skip_if_missing('clinic') with test_tools.imports_under_tool('clinic'): + import libclinic import clinic from clinic import DSLParser def _make_clinic(*, filename='clinic_tests'): - clang = clinic.CLanguage(None) + clang = clinic.CLanguage(filename) c = clinic.Clinic(clang, filename=filename, limited_capi=False) c.block_parser = clinic.BlockParser('', clang) return c @@ -3287,6 +3288,26 @@ def test_cloned_func_with_converter_exception_message(self): func = getattr(ac_tester, name) self.assertEqual(func(), name) + def test_meth_method_no_params(self): + obj = ac_tester.TestClass() + meth = obj.meth_method_no_params + check = partial(self.assertRaisesRegex, TypeError, "no arguments") + check(meth, 1) + check(meth, a=1) + + def test_meth_method_no_params_capi(self): + from _testcapi import pyobject_vectorcall + obj = ac_tester.TestClass() + meth = obj.meth_method_no_params + pyobject_vectorcall(meth, None, None) + pyobject_vectorcall(meth, (), None) + pyobject_vectorcall(meth, (), ()) + pyobject_vectorcall(meth, None, ()) + + check = partial(self.assertRaisesRegex, TypeError, "no arguments") + check(pyobject_vectorcall, meth, (1,), None) + check(pyobject_vectorcall, meth, (1,), ("a",)) + def test_depr_star_new(self): cls = ac_tester.DeprStarNew cls() @@ -3729,7 +3750,7 @@ def test_strip_leading_and_trailing_blank_lines(self): ) for lines, expected in dataset: with self.subTest(lines=lines, expected=expected): - out = clinic.strip_leading_and_trailing_blank_lines(lines) + out = libclinic.normalize_snippet(lines) self.assertEqual(out, expected) def test_normalize_snippet(self): @@ -3758,63 +3779,33 @@ def test_normalize_snippet(self): expected_outputs = {0: zero_indent, 4: four_indent, 8: eight_indent} for indent, expected in expected_outputs.items(): with self.subTest(indent=indent): - actual = clinic.normalize_snippet(snippet, indent=indent) + actual = libclinic.normalize_snippet(snippet, indent=indent) self.assertEqual(actual, expected) - def test_accumulator(self): - acc = clinic.text_accumulator() - self.assertEqual(acc.output(), "") - acc.append("a") - self.assertEqual(acc.output(), "a") - self.assertEqual(acc.output(), "") - acc.append("b") - self.assertEqual(acc.output(), "b") - self.assertEqual(acc.output(), "") - acc.append("c") - acc.append("d") - self.assertEqual(acc.output(), "cd") - self.assertEqual(acc.output(), "") - - def test_quoted_for_c_string(self): + def test_escaped_docstring(self): dataset = ( # input, expected - (r"abc", r"abc"), - (r"\abc", r"\\abc"), - (r"\a\bc", r"\\a\\bc"), - (r"\a\\bc", r"\\a\\\\bc"), - (r'"abc"', r'\"abc\"'), - (r"'a'", r"\'a\'"), + (r"abc", r'"abc"'), + (r"\abc", r'"\\abc"'), + (r"\a\bc", r'"\\a\\bc"'), + (r"\a\\bc", r'"\\a\\\\bc"'), + (r'"abc"', r'"\"abc\""'), + (r"'a'", r'"\'a\'"'), ) for line, expected in dataset: with self.subTest(line=line, expected=expected): - out = clinic.quoted_for_c_string(line) + out = libclinic.docstring_for_c_string(line) self.assertEqual(out, expected) - def test_rstrip_lines(self): - lines = ( - "a \n" - "b\n" - " c\n" - " d \n" - ) - expected = ( - "a\n" - "b\n" - " c\n" - " d\n" - ) - out = clinic.rstrip_lines(lines) - self.assertEqual(out, expected) - def test_format_escape(self): line = "{}, {a}" expected = "{{}}, {{a}}" - out = clinic.format_escape(line) + out = libclinic.format_escape(line) self.assertEqual(out, expected) def test_indent_all_lines(self): # Blank lines are expected to be unchanged. - self.assertEqual(clinic.indent_all_lines("", prefix="bar"), "") + self.assertEqual(libclinic.indent_all_lines("", prefix="bar"), "") lines = ( "one\n" @@ -3824,7 +3815,7 @@ def test_indent_all_lines(self): "barone\n" "bartwo" ) - out = clinic.indent_all_lines(lines, prefix="bar") + out = libclinic.indent_all_lines(lines, prefix="bar") self.assertEqual(out, expected) # If last line is empty, expect it to be unchanged. @@ -3840,12 +3831,12 @@ def test_indent_all_lines(self): "bartwo\n" "" ) - out = clinic.indent_all_lines(lines, prefix="bar") + out = libclinic.indent_all_lines(lines, prefix="bar") self.assertEqual(out, expected) def test_suffix_all_lines(self): # Blank lines are expected to be unchanged. - self.assertEqual(clinic.suffix_all_lines("", suffix="foo"), "") + self.assertEqual(libclinic.suffix_all_lines("", suffix="foo"), "") lines = ( "one\n" @@ -3855,7 +3846,7 @@ def test_suffix_all_lines(self): "onefoo\n" "twofoo" ) - out = clinic.suffix_all_lines(lines, suffix="foo") + out = libclinic.suffix_all_lines(lines, suffix="foo") self.assertEqual(out, expected) # If last line is empty, expect it to be unchanged. @@ -3871,7 +3862,7 @@ def test_suffix_all_lines(self): "twofoo\n" "" ) - out = clinic.suffix_all_lines(lines, suffix="foo") + out = libclinic.suffix_all_lines(lines, suffix="foo") self.assertEqual(out, expected) @@ -3949,7 +3940,7 @@ def test_Function_and_Parameter_reprs(self): self.assertEqual(repr(parameter), "") def test_Monitor_repr(self): - monitor = clinic.cpp.Monitor() + monitor = libclinic.cpp.Monitor("test.c") self.assertRegex(repr(monitor), r"") monitor.line_number = 42 diff --git a/Lib/test/test_cmd_line_script.py b/Lib/test/test_cmd_line_script.py index 48754d5a63da3b..3a5a8abf81e43d 100644 --- a/Lib/test/test_cmd_line_script.py +++ b/Lib/test/test_cmd_line_script.py @@ -14,8 +14,7 @@ import textwrap from test import support -from test.support import import_helper -from test.support import os_helper +from test.support import import_helper, is_apple, os_helper from test.support.script_helper import ( make_pkg, make_script, make_zip_pkg, make_zip_script, assert_python_ok, assert_python_failure, spawn_python, kill_python) @@ -557,12 +556,17 @@ def test_pep_409_verbiage(self): self.assertTrue(text[3].startswith('NameError')) def test_non_ascii(self): - # Mac OS X denies the creation of a file with an invalid UTF-8 name. + # Apple platforms deny the creation of a file with an invalid UTF-8 name. # Windows allows creating a name with an arbitrary bytes name, but # Python cannot a undecodable bytes argument to a subprocess. - # WASI does not permit invalid UTF-8 names. - if (os_helper.TESTFN_UNDECODABLE - and sys.platform not in ('win32', 'darwin', 'emscripten', 'wasi')): + # Emscripten/WASI does not permit invalid UTF-8 names. + if ( + os_helper.TESTFN_UNDECODABLE + and sys.platform not in { + "win32", "emscripten", "wasi" + } + and not is_apple + ): name = os.fsdecode(os_helper.TESTFN_UNDECODABLE) elif os_helper.TESTFN_NONASCII: name = os_helper.TESTFN_NONASCII diff --git a/Lib/test/test_code.py b/Lib/test/test_code.py index d8fb826edeb681..46bebfc7af675b 100644 --- a/Lib/test/test_code.py +++ b/Lib/test/test_code.py @@ -865,6 +865,7 @@ def __init__(self, f, test): self.test = test def run(self): del self.f + gc_collect() self.test.assertEqual(LAST_FREED, 500) SetExtra(f.__code__, FREE_INDEX, ctypes.c_voidp(500)) diff --git a/Lib/test/test_code_module.py b/Lib/test/test_code_module.py index 747c0f9683c19c..259778a5cade98 100644 --- a/Lib/test/test_code_module.py +++ b/Lib/test/test_code_module.py @@ -160,6 +160,7 @@ def setUp(self): self.console = code.InteractiveConsole(local_exit=True) self.mock_sys() + @unittest.skipIf(sys.flags.no_site, "exit() isn't defined unless there's a site module") def test_exit(self): # default exit message self.infunc.side_effect = ["exit()"] diff --git a/Lib/test/test_codeop.py b/Lib/test/test_codeop.py index 2abb6c6d935b7e..787bd1b6a79e20 100644 --- a/Lib/test/test_codeop.py +++ b/Lib/test/test_codeop.py @@ -223,6 +223,9 @@ def test_incomplete(self): ai("(x for x in") ai("(x for x in (") + ai('a = f"""') + ai('a = \\') + def test_invalid(self): ai = self.assertInvalid ai("a b") diff --git a/Lib/test/test_compile.py b/Lib/test/test_compile.py index 906e16cc9437fb..ebb479f2de7c63 100644 --- a/Lib/test/test_compile.py +++ b/Lib/test/test_compile.py @@ -448,6 +448,28 @@ def test_condition_expression_with_dead_blocks_compiles(self): # See gh-113054 compile('if (5 if 5 else T): 0', '', 'exec') + def test_condition_expression_with_redundant_comparisons_compiles(self): + # See gh-113054, gh-114083 + exprs = [ + 'if 9<9<9and 9or 9:9', + 'if 9<9<9and 9or 9or 9:9', + 'if 9<9<9and 9or 9or 9or 9:9', + 'if 9<9<9and 9or 9or 9or 9or 9:9', + ] + for expr in exprs: + with self.subTest(expr=expr): + with self.assertWarns(SyntaxWarning): + compile(expr, '', 'exec') + + def test_dead_code_with_except_handler_compiles(self): + compile(textwrap.dedent(""" + if None: + with CM: + x = 1 + else: + x = 2 + """), '', 'exec') + def test_compile_invalid_namedexpr(self): # gh-109351 m = ast.Module( @@ -609,12 +631,10 @@ def test_yet_more_evil_still_undecodable(self): @support.cpython_only @unittest.skipIf(support.is_wasi, "exhausts limited stack on WASI") def test_compiler_recursion_limit(self): - # Expected limit is Py_C_RECURSION_LIMIT * 2 - # Duplicating the limit here is a little ugly. - # Perhaps it should be exposed somewhere... - fail_depth = Py_C_RECURSION_LIMIT * 2 + 1 + # Expected limit is Py_C_RECURSION_LIMIT + fail_depth = Py_C_RECURSION_LIMIT + 1 crash_depth = Py_C_RECURSION_LIMIT * 100 - success_depth = int(Py_C_RECURSION_LIMIT * 1.8) + success_depth = int(Py_C_RECURSION_LIMIT * 0.8) def check_limit(prefix, repeated, mode="single"): expect_ok = prefix + repeated * success_depth @@ -1084,19 +1104,72 @@ async def test(aseq): code_lines = self.get_code_lines(test.__code__) self.assertEqual(expected_lines, code_lines) - def test_lineno_of_backward_jump(self): + def check_line_numbers(self, code, opnames=None): + # Check that all instructions whose op matches opnames + # have a line number. opnames can be a single name, or + # a sequence of names. If it is None, match all ops. + + if isinstance(opnames, str): + opnames = (opnames, ) + for inst in dis.Bytecode(code): + if opnames and inst.opname in opnames: + self.assertIsNotNone(inst.positions.lineno) + + def test_line_number_synthetic_jump_multiple_predecessors(self): + def f(): + for x in it: + try: + if C1: + yield 2 + except OSError: + pass + + self.check_line_numbers(f.__code__, 'JUMP_BACKWARD') + + def test_line_number_synthetic_jump_multiple_predecessors_nested(self): + def f(): + for x in it: + try: + X = 3 + except OSError: + try: + if C3: + X = 4 + except OSError: + pass + return 42 + + self.check_line_numbers(f.__code__, 'JUMP_BACKWARD') + + def test_line_number_synthetic_jump_multiple_predecessors_more_nested(self): + def f(): + for x in it: + try: + X = 3 + except OSError: + try: + if C3: + if C4: + X = 4 + except OSError: + try: + if C3: + if C4: + X = 5 + except OSError: + pass + return 42 + + self.check_line_numbers(f.__code__, 'JUMP_BACKWARD') + + def test_lineno_of_backward_jump_conditional_in_loop(self): # Issue gh-107901 def f(): for i in x: if y: pass - linenos = list(inst.positions.lineno - for inst in dis.get_instructions(f.__code__) - if inst.opname == 'JUMP_BACKWARD') - - self.assertTrue(len(linenos) > 0) - self.assertTrue(all(l is not None for l in linenos)) + self.check_line_numbers(f.__code__, 'JUMP_BACKWARD') def test_big_dict_literal(self): # The compiler has a flushing point in "compiler_dict" that calls compiles diff --git a/Lib/test/test_compileall.py b/Lib/test/test_compileall.py index 83a9532aecfac8..0ec6013dc11e3e 100644 --- a/Lib/test/test_compileall.py +++ b/Lib/test/test_compileall.py @@ -369,7 +369,9 @@ def test_strip_only_invalid(self): script = script_helper.make_script(path, "test", "1 / 0") bc = importlib.util.cache_from_source(script) stripdir = os.path.join(self.directory, *(fullpath[:2] + ['fake'])) - compileall.compile_dir(path, quiet=True, stripdir=stripdir) + with support.captured_stdout() as out: + compileall.compile_dir(path, quiet=True, stripdir=stripdir) + self.assertIn("not a valid prefix", out.getvalue()) rc, out, err = script_helper.assert_python_failure(bc) expected_not_in = os.path.join(self.directory, *fullpath[2:]) self.assertIn( diff --git a/Lib/test/test_compiler_codegen.py b/Lib/test/test_compiler_codegen.py index 6d7731ddba02c5..dbeadd9ca47c63 100644 --- a/Lib/test/test_compiler_codegen.py +++ b/Lib/test/test_compiler_codegen.py @@ -21,7 +21,7 @@ def test_if_expression(self): ('TO_BOOL', 0, 1), ('POP_JUMP_IF_FALSE', false_lbl := self.Label(), 1), ('LOAD_CONST', 1, 1), - ('JUMP', exit_lbl := self.Label()), + ('JUMP_NO_INTERRUPT', exit_lbl := self.Label()), false_lbl, ('LOAD_CONST', 2, 1), exit_lbl, @@ -49,6 +49,7 @@ def test_for_loop(self): ('JUMP', loop_lbl), exit_lbl, ('END_FOR', None), + ('POP_TOP', None), ('LOAD_CONST', 0), ('RETURN_VALUE', None), ] diff --git a/Lib/test/test_concurrent_futures/executor.py b/Lib/test/test_concurrent_futures/executor.py index 1e7d4344740943..6a79fe69ec37cf 100644 --- a/Lib/test/test_concurrent_futures/executor.py +++ b/Lib/test/test_concurrent_futures/executor.py @@ -1,8 +1,10 @@ import threading import time +import unittest import weakref from concurrent import futures from test import support +from test.support import Py_GIL_DISABLED def mul(x, y): @@ -83,10 +85,21 @@ def test_no_stale_references(self): my_object_collected = threading.Event() my_object_callback = weakref.ref( my_object, lambda obj: my_object_collected.set()) - # Deliberately discarding the future. - self.executor.submit(my_object.my_method) + fut = self.executor.submit(my_object.my_method) del my_object + if Py_GIL_DISABLED: + # Due to biased reference counting, my_object might only be + # deallocated while the thread that created it runs -- if the + # thread is paused waiting on an event, it may not merge the + # refcount of the queued object. For that reason, we wait for the + # task to finish (so that it's no longer referenced) and force a + # GC to ensure that it is collected. + fut.result() # Wait for the task to finish. + support.gc_collect() + else: + del fut # Deliberately discard the future. + collected = my_object_collected.wait(timeout=support.SHORT_TIMEOUT) self.assertTrue(collected, "Stale reference not collected within timeout.") diff --git a/Lib/test/test_concurrent_futures/test_process_pool.py b/Lib/test/test_concurrent_futures/test_process_pool.py index 3e61b0c9387c6f..7fc59a05f3deac 100644 --- a/Lib/test/test_concurrent_futures/test_process_pool.py +++ b/Lib/test/test_concurrent_futures/test_process_pool.py @@ -98,6 +98,7 @@ def test_ressources_gced_in_workers(self): # explicitly destroy the object to ensure that EventfulGCObj.__del__() # is called while manager is still running. + support.gc_collect() obj = None support.gc_collect() diff --git a/Lib/test/test_coroutines.py b/Lib/test/test_coroutines.py index 25c981d1511bc1..d848bfbd46c83b 100644 --- a/Lib/test/test_coroutines.py +++ b/Lib/test/test_coroutines.py @@ -953,11 +953,12 @@ async def b(): def test_corotype_1(self): ct = types.CoroutineType - self.assertIn('into coroutine', ct.send.__doc__) - self.assertIn('inside coroutine', ct.close.__doc__) - self.assertIn('in coroutine', ct.throw.__doc__) - self.assertIn('of the coroutine', ct.__dict__['__name__'].__doc__) - self.assertIn('of the coroutine', ct.__dict__['__qualname__'].__doc__) + if not support.MISSING_C_DOCSTRINGS: + self.assertIn('into coroutine', ct.send.__doc__) + self.assertIn('inside coroutine', ct.close.__doc__) + self.assertIn('in coroutine', ct.throw.__doc__) + self.assertIn('of the coroutine', ct.__dict__['__name__'].__doc__) + self.assertIn('of the coroutine', ct.__dict__['__qualname__'].__doc__) self.assertEqual(ct.__name__, 'coroutine') async def f(): pass diff --git a/Lib/test/test_csv.py b/Lib/test/test_csv.py index 97b9bba24bcbca..21a4cb586ff665 100644 --- a/Lib/test/test_csv.py +++ b/Lib/test/test_csv.py @@ -28,14 +28,20 @@ class Test_Csv(unittest.TestCase): in TestDialectRegistry. """ def _test_arg_valid(self, ctor, arg): + ctor(arg) self.assertRaises(TypeError, ctor) self.assertRaises(TypeError, ctor, None) - self.assertRaises(TypeError, ctor, arg, bad_attr = 0) - self.assertRaises(TypeError, ctor, arg, delimiter = 0) - self.assertRaises(TypeError, ctor, arg, delimiter = 'XX') + self.assertRaises(TypeError, ctor, arg, bad_attr=0) + self.assertRaises(TypeError, ctor, arg, delimiter='') + self.assertRaises(TypeError, ctor, arg, escapechar='') + self.assertRaises(TypeError, ctor, arg, quotechar='') + self.assertRaises(TypeError, ctor, arg, delimiter='^^') + self.assertRaises(TypeError, ctor, arg, escapechar='^^') + self.assertRaises(TypeError, ctor, arg, quotechar='^^') self.assertRaises(csv.Error, ctor, arg, 'foo') self.assertRaises(TypeError, ctor, arg, delimiter=None) self.assertRaises(TypeError, ctor, arg, delimiter=1) + self.assertRaises(TypeError, ctor, arg, escapechar=1) self.assertRaises(TypeError, ctor, arg, quotechar=1) self.assertRaises(TypeError, ctor, arg, lineterminator=None) self.assertRaises(TypeError, ctor, arg, lineterminator=1) @@ -46,6 +52,40 @@ def _test_arg_valid(self, ctor, arg): quoting=csv.QUOTE_ALL, quotechar=None) self.assertRaises(TypeError, ctor, arg, quoting=csv.QUOTE_NONE, quotechar='') + self.assertRaises(ValueError, ctor, arg, delimiter='\n') + self.assertRaises(ValueError, ctor, arg, escapechar='\n') + self.assertRaises(ValueError, ctor, arg, quotechar='\n') + self.assertRaises(ValueError, ctor, arg, delimiter='\r') + self.assertRaises(ValueError, ctor, arg, escapechar='\r') + self.assertRaises(ValueError, ctor, arg, quotechar='\r') + ctor(arg, delimiter=' ') + ctor(arg, escapechar=' ') + ctor(arg, quotechar=' ') + ctor(arg, delimiter='\t', skipinitialspace=True) + ctor(arg, escapechar='\t', skipinitialspace=True) + ctor(arg, quotechar='\t', skipinitialspace=True) + self.assertRaises(ValueError, ctor, arg, + delimiter=' ', skipinitialspace=True) + self.assertRaises(ValueError, ctor, arg, + escapechar=' ', skipinitialspace=True) + self.assertRaises(ValueError, ctor, arg, + quotechar=' ', skipinitialspace=True) + ctor(arg, delimiter='^') + ctor(arg, escapechar='^') + ctor(arg, quotechar='^') + self.assertRaises(ValueError, ctor, arg, delimiter='^', escapechar='^') + self.assertRaises(ValueError, ctor, arg, delimiter='^', quotechar='^') + self.assertRaises(ValueError, ctor, arg, escapechar='^', quotechar='^') + ctor(arg, delimiter='\x85') + ctor(arg, escapechar='\x85') + ctor(arg, quotechar='\x85') + ctor(arg, lineterminator='\x85') + self.assertRaises(ValueError, ctor, arg, + delimiter='\x85', lineterminator='\x85') + self.assertRaises(ValueError, ctor, arg, + escapechar='\x85', lineterminator='\x85') + self.assertRaises(ValueError, ctor, arg, + quotechar='\x85', lineterminator='\x85') def test_reader_arg_valid(self): self._test_arg_valid(csv.reader, []) @@ -297,13 +337,18 @@ def test_read_oddinputs(self): [b'abc'], None) def test_read_eol(self): - self._read_test(['a,b'], [['a','b']]) - self._read_test(['a,b\n'], [['a','b']]) - self._read_test(['a,b\r\n'], [['a','b']]) - self._read_test(['a,b\r'], [['a','b']]) - self.assertRaises(csv.Error, self._read_test, ['a,b\rc,d'], []) - self.assertRaises(csv.Error, self._read_test, ['a,b\nc,d'], []) - self.assertRaises(csv.Error, self._read_test, ['a,b\r\nc,d'], []) + self._read_test(['a,b', 'c,d'], [['a','b'], ['c','d']]) + self._read_test(['a,b\n', 'c,d\n'], [['a','b'], ['c','d']]) + self._read_test(['a,b\r\n', 'c,d\r\n'], [['a','b'], ['c','d']]) + self._read_test(['a,b\r', 'c,d\r'], [['a','b'], ['c','d']]) + + errmsg = "with newline=''" + with self.assertRaisesRegex(csv.Error, errmsg): + next(csv.reader(['a,b\rc,d'])) + with self.assertRaisesRegex(csv.Error, errmsg): + next(csv.reader(['a,b\nc,d'])) + with self.assertRaisesRegex(csv.Error, errmsg): + next(csv.reader(['a,b\r\nc,d'])) def test_read_eof(self): self._read_test(['a,"'], [['a', '']]) @@ -347,10 +392,26 @@ def test_read_quoting(self): # will this fail where locale uses comma for decimals? self._read_test([',3,"5",7.3, 9'], [['', 3, '5', 7.3, 9]], quoting=csv.QUOTE_NONNUMERIC) + self._read_test([',3,"5",7.3, 9'], [[None, '3', '5', '7.3', ' 9']], + quoting=csv.QUOTE_NOTNULL) + self._read_test([',3,"5",7.3, 9'], [[None, 3, '5', 7.3, 9]], + quoting=csv.QUOTE_STRINGS) + + self._read_test([',,"",'], [['', '', '', '']]) + self._read_test([',,"",'], [['', '', '', '']], + quoting=csv.QUOTE_NONNUMERIC) + self._read_test([',,"",'], [[None, None, '', None]], + quoting=csv.QUOTE_NOTNULL) + self._read_test([',,"",'], [[None, None, '', None]], + quoting=csv.QUOTE_STRINGS) + self._read_test(['"a\nb", 7'], [['a\nb', ' 7']]) self.assertRaises(ValueError, self._read_test, ['abc,3'], [[]], quoting=csv.QUOTE_NONNUMERIC) + self.assertRaises(ValueError, self._read_test, + ['abc,3'], [[]], + quoting=csv.QUOTE_STRINGS) self._read_test(['1,@,3,@,5'], [['1', ',3,', '5']], quotechar='@') self._read_test(['1,\0,3,\0,5'], [['1', ',3,', '5']], quotechar='\0') @@ -358,6 +419,15 @@ def test_read_skipinitialspace(self): self._read_test(['no space, space, spaces,\ttab'], [['no space', 'space', 'spaces', '\ttab']], skipinitialspace=True) + self._read_test([' , , '], + [['', '', '']], + skipinitialspace=True) + self._read_test([' , , '], + [[None, None, None]], + skipinitialspace=True, quoting=csv.QUOTE_NOTNULL) + self._read_test([' , , '], + [[None, None, None]], + skipinitialspace=True, quoting=csv.QUOTE_STRINGS) def test_read_bigfield(self): # This exercises the buffer realloc functionality and field size @@ -530,14 +600,6 @@ class unspecified(): finally: csv.unregister_dialect('testC') - def test_bad_dialect(self): - # Unknown parameter - self.assertRaises(TypeError, csv.reader, [], bad_attr = 0) - # Bad values - self.assertRaises(TypeError, csv.reader, [], delimiter = None) - self.assertRaises(TypeError, csv.reader, [], quoting = -1) - self.assertRaises(TypeError, csv.reader, [], quoting = 100) - def test_copy(self): for name in csv.list_dialects(): dialect = csv.get_dialect(name) @@ -1083,10 +1145,15 @@ class mydialect(csv.Dialect): '"lineterminator" must be a string') def test_invalid_chars(self): - def create_invalid(field_name, value): + def create_invalid(field_name, value, **kwargs): class mydialect(csv.Dialect): - pass + delimiter = ',' + quoting = csv.QUOTE_ALL + quotechar = '"' + lineterminator = '\r\n' setattr(mydialect, field_name, value) + for field_name, value in kwargs.items(): + setattr(mydialect, field_name, value) d = mydialect() for field_name in ("delimiter", "escapechar", "quotechar"): @@ -1095,6 +1162,10 @@ class mydialect(csv.Dialect): self.assertRaises(csv.Error, create_invalid, field_name, "abc") self.assertRaises(csv.Error, create_invalid, field_name, b'x') self.assertRaises(csv.Error, create_invalid, field_name, 5) + self.assertRaises(ValueError, create_invalid, field_name, "\n") + self.assertRaises(ValueError, create_invalid, field_name, "\r") + self.assertRaises(ValueError, create_invalid, field_name, " ", + skipinitialspace=True) class TestSniffer(unittest.TestCase): @@ -1411,8 +1482,7 @@ def test_ordered_dict_reader(self): class MiscTestCase(unittest.TestCase): def test__all__(self): - extra = {'__doc__', '__version__'} - support.check__all__(self, csv, ('csv', '_csv'), extra=extra) + support.check__all__(self, csv, ('csv', '_csv')) def test_subclassable(self): # issue 44089 diff --git a/Lib/test/test_ctypes/_support.py b/Lib/test/test_ctypes/_support.py new file mode 100644 index 00000000000000..e4c2b33825ae8f --- /dev/null +++ b/Lib/test/test_ctypes/_support.py @@ -0,0 +1,24 @@ +# Some classes and types are not export to _ctypes module directly. + +import ctypes +from _ctypes import Structure, Union, _Pointer, Array, _SimpleCData, CFuncPtr + + +_CData = Structure.__base__ +assert _CData.__name__ == "_CData" + +class _X(Structure): + _fields_ = [("x", ctypes.c_int)] +CField = type(_X.x) + +# metaclasses +PyCStructType = type(Structure) +UnionType = type(Union) +PyCPointerType = type(_Pointer) +PyCArrayType = type(Array) +PyCSimpleType = type(_SimpleCData) +PyCFuncPtrType = type(CFuncPtr) + +# type flags +Py_TPFLAGS_DISALLOW_INSTANTIATION = 1 << 7 +Py_TPFLAGS_IMMUTABLETYPE = 1 << 8 diff --git a/Lib/test/test_ctypes/test_arrays.py b/Lib/test/test_ctypes/test_arrays.py index 6b6cebd3e20285..774316e227ff73 100644 --- a/Lib/test/test_ctypes/test_arrays.py +++ b/Lib/test/test_ctypes/test_arrays.py @@ -7,6 +7,8 @@ c_char, c_wchar, c_byte, c_ubyte, c_short, c_ushort, c_int, c_uint, c_long, c_ulonglong, c_float, c_double, c_longdouble) from test.support import bigmemtest, _2G +from ._support import (_CData, PyCArrayType, Py_TPFLAGS_DISALLOW_INSTANTIATION, + Py_TPFLAGS_IMMUTABLETYPE) formats = "bBhHiIlLqQfd" @@ -23,6 +25,18 @@ def ARRAY(*args): class ArrayTestCase(unittest.TestCase): + def test_inheritance_hierarchy(self): + self.assertEqual(Array.mro(), [Array, _CData, object]) + + self.assertEqual(PyCArrayType.__name__, "PyCArrayType") + self.assertEqual(type(PyCArrayType), type) + + def test_type_flags(self): + for cls in Array, PyCArrayType: + with self.subTest(cls=cls): + self.assertTrue(cls.__flags__ & Py_TPFLAGS_IMMUTABLETYPE) + self.assertFalse(cls.__flags__ & Py_TPFLAGS_DISALLOW_INSTANTIATION) + def test_simple(self): # create classes holding simple numeric types, and check # various properties. diff --git a/Lib/test/test_ctypes/test_as_parameter.py b/Lib/test/test_ctypes/test_as_parameter.py index a1a8745e737fa2..ca75e748256083 100644 --- a/Lib/test/test_ctypes/test_as_parameter.py +++ b/Lib/test/test_ctypes/test_as_parameter.py @@ -221,5 +221,16 @@ class AsParamPropertyWrapperTestCase(BasicWrapTestCase): wrap = AsParamPropertyWrapper +class AsParamNestedWrapperTestCase(BasicWrapTestCase): + """Test that _as_parameter_ is evaluated recursively. + + The _as_parameter_ attribute can be another object which + defines its own _as_parameter_ attribute. + """ + + def wrap(self, param): + return AsParamWrapper(AsParamWrapper(AsParamWrapper(param))) + + if __name__ == '__main__': unittest.main() diff --git a/Lib/test/test_ctypes/test_byteswap.py b/Lib/test/test_ctypes/test_byteswap.py index b97b57646ecd71..78eff0392c4548 100644 --- a/Lib/test/test_ctypes/test_byteswap.py +++ b/Lib/test/test_ctypes/test_byteswap.py @@ -363,6 +363,24 @@ class TestUnion(parent): self.assertEqual(s.point.x, 1) self.assertEqual(s.point.y, 2) + def test_build_struct_union_opposite_system_byteorder(self): + # gh-105102 + if sys.byteorder == "little": + _Structure = BigEndianStructure + _Union = BigEndianUnion + else: + _Structure = LittleEndianStructure + _Union = LittleEndianUnion + + class S1(_Structure): + _fields_ = [("a", c_byte), ("b", c_byte)] + + class U1(_Union): + _fields_ = [("s1", S1), ("ab", c_short)] + + class S2(_Structure): + _fields_ = [("u1", U1), ("c", c_byte)] + if __name__ == "__main__": unittest.main() diff --git a/Lib/test/test_ctypes/test_find.py b/Lib/test/test_ctypes/test_find.py index 66ff23e72b5e10..7732ff37308848 100644 --- a/Lib/test/test_ctypes/test_find.py +++ b/Lib/test/test_ctypes/test_find.py @@ -125,6 +125,9 @@ def test_find_library_with_ld(self): unittest.mock.patch("ctypes.util._findLib_gcc", lambda *args: None): self.assertNotEqual(find_library('c'), None) + def test_gh114257(self): + self.assertIsNone(find_library("libc")) + if __name__ == "__main__": unittest.main() diff --git a/Lib/test/test_ctypes/test_funcptr.py b/Lib/test/test_ctypes/test_funcptr.py index 2ad40647e0cfbb..0eed39484fb39e 100644 --- a/Lib/test/test_ctypes/test_funcptr.py +++ b/Lib/test/test_ctypes/test_funcptr.py @@ -3,6 +3,8 @@ import unittest from ctypes import (CDLL, Structure, CFUNCTYPE, sizeof, _CFuncPtr, c_void_p, c_char_p, c_char, c_int, c_uint, c_long) +from ._support import (_CData, PyCFuncPtrType, Py_TPFLAGS_DISALLOW_INSTANTIATION, + Py_TPFLAGS_IMMUTABLETYPE) try: @@ -15,6 +17,18 @@ class CFuncPtrTestCase(unittest.TestCase): + def test_inheritance_hierarchy(self): + self.assertEqual(_CFuncPtr.mro(), [_CFuncPtr, _CData, object]) + + self.assertEqual(PyCFuncPtrType.__name__, "PyCFuncPtrType") + self.assertEqual(type(PyCFuncPtrType), type) + + def test_type_flags(self): + for cls in _CFuncPtr, PyCFuncPtrType: + with self.subTest(cls=cls): + self.assertTrue(_CFuncPtr.__flags__ & Py_TPFLAGS_IMMUTABLETYPE) + self.assertFalse(_CFuncPtr.__flags__ & Py_TPFLAGS_DISALLOW_INSTANTIATION) + def test_basic(self): X = WINFUNCTYPE(c_int, c_int, c_int) diff --git a/Lib/test/test_ctypes/test_loading.py b/Lib/test/test_ctypes/test_loading.py index 22db97b818c17e..59d7f51935f3cd 100644 --- a/Lib/test/test_ctypes/test_loading.py +++ b/Lib/test/test_ctypes/test_loading.py @@ -141,7 +141,7 @@ def test_load_hasattr(self): def test_load_dll_with_flags(self): _sqlite3 = import_helper.import_module("_sqlite3") src = _sqlite3.__file__ - if src.lower().endswith("_d.pyd"): + if os.path.basename(src).partition(".")[0].lower().endswith("_d"): ext = "_d.dll" else: ext = ".dll" diff --git a/Lib/test/test_ctypes/test_pointers.py b/Lib/test/test_ctypes/test_pointers.py index 8410174358c19d..8cf2114c282cab 100644 --- a/Lib/test/test_ctypes/test_pointers.py +++ b/Lib/test/test_ctypes/test_pointers.py @@ -10,6 +10,8 @@ c_byte, c_ubyte, c_short, c_ushort, c_int, c_uint, c_long, c_ulong, c_longlong, c_ulonglong, c_float, c_double) +from ._support import (_CData, PyCPointerType, Py_TPFLAGS_DISALLOW_INSTANTIATION, + Py_TPFLAGS_IMMUTABLETYPE) ctype_types = [c_byte, c_ubyte, c_short, c_ushort, c_int, c_uint, @@ -19,6 +21,18 @@ class PointersTestCase(unittest.TestCase): + def test_inheritance_hierarchy(self): + self.assertEqual(_Pointer.mro(), [_Pointer, _CData, object]) + + self.assertEqual(PyCPointerType.__name__, "PyCPointerType") + self.assertEqual(type(PyCPointerType), type) + + def test_type_flags(self): + for cls in _Pointer, PyCPointerType: + with self.subTest(cls=cls): + self.assertTrue(_Pointer.__flags__ & Py_TPFLAGS_IMMUTABLETYPE) + self.assertFalse(_Pointer.__flags__ & Py_TPFLAGS_DISALLOW_INSTANTIATION) + def test_pointer_crash(self): class A(POINTER(c_ulong)): diff --git a/Lib/test/test_ctypes/test_simplesubclasses.py b/Lib/test/test_ctypes/test_simplesubclasses.py index 6072b62de5d53a..c96798e67f23f7 100644 --- a/Lib/test/test_ctypes/test_simplesubclasses.py +++ b/Lib/test/test_ctypes/test_simplesubclasses.py @@ -1,5 +1,7 @@ import unittest -from ctypes import Structure, CFUNCTYPE, c_int +from ctypes import Structure, CFUNCTYPE, c_int, _SimpleCData +from ._support import (_CData, PyCSimpleType, Py_TPFLAGS_DISALLOW_INSTANTIATION, + Py_TPFLAGS_IMMUTABLETYPE) class MyInt(c_int): @@ -10,6 +12,19 @@ def __eq__(self, other): class Test(unittest.TestCase): + def test_inheritance_hierarchy(self): + self.assertEqual(_SimpleCData.mro(), [_SimpleCData, _CData, object]) + + self.assertEqual(PyCSimpleType.__name__, "PyCSimpleType") + self.assertEqual(type(PyCSimpleType), type) + + self.assertEqual(c_int.mro(), [c_int, _SimpleCData, _CData, object]) + + def test_type_flags(self): + for cls in _SimpleCData, PyCSimpleType: + with self.subTest(cls=cls): + self.assertTrue(_SimpleCData.__flags__ & Py_TPFLAGS_IMMUTABLETYPE) + self.assertFalse(_SimpleCData.__flags__ & Py_TPFLAGS_DISALLOW_INSTANTIATION) def test_compare(self): self.assertEqual(MyInt(3), MyInt(3)) diff --git a/Lib/test/test_ctypes/test_struct_fields.py b/Lib/test/test_ctypes/test_struct_fields.py index f60dfe5b42ef65..f474a02fa8db06 100644 --- a/Lib/test/test_ctypes/test_struct_fields.py +++ b/Lib/test/test_ctypes/test_struct_fields.py @@ -1,5 +1,7 @@ import unittest from ctypes import Structure, Union, sizeof, c_char, c_int +from ._support import (CField, Py_TPFLAGS_DISALLOW_INSTANTIATION, + Py_TPFLAGS_IMMUTABLETYPE) class StructFieldsTestCase(unittest.TestCase): @@ -12,7 +14,6 @@ class StructFieldsTestCase(unittest.TestCase): # 4. The type is subclassed # # When they are finalized, assigning _fields_ is no longer allowed. - def test_1_A(self): class X(Structure): pass @@ -56,11 +57,15 @@ class X(Structure): self.assertEqual(bytes(x), b'a\x00###') def test_6(self): - class X(Structure): - _fields_ = [("x", c_int)] - CField = type(X.x) self.assertRaises(TypeError, CField) + def test_cfield_type_flags(self): + self.assertTrue(CField.__flags__ & Py_TPFLAGS_DISALLOW_INSTANTIATION) + self.assertTrue(CField.__flags__ & Py_TPFLAGS_IMMUTABLETYPE) + + def test_cfield_inheritance_hierarchy(self): + self.assertEqual(CField.mro(), [CField, object]) + def test_gh99275(self): class BrokenStructure(Structure): def __init_subclass__(cls, **kwargs): diff --git a/Lib/test/test_ctypes/test_structures.py b/Lib/test/test_ctypes/test_structures.py index 21039f04947507..98bc4bdcac9306 100644 --- a/Lib/test/test_ctypes/test_structures.py +++ b/Lib/test/test_ctypes/test_structures.py @@ -1,5 +1,5 @@ import _ctypes_test -import platform +from platform import architecture as _architecture import struct import sys import unittest @@ -8,9 +8,12 @@ c_uint8, c_uint16, c_uint32, c_short, c_ushort, c_int, c_uint, c_long, c_ulong, c_longlong, c_ulonglong, c_float, c_double) +from ctypes.util import find_library from struct import calcsize from collections import namedtuple from test import support +from ._support import (_CData, PyCStructType, Py_TPFLAGS_DISALLOW_INSTANTIATION, + Py_TPFLAGS_IMMUTABLETYPE) class SubclassesTest(unittest.TestCase): @@ -69,6 +72,19 @@ class StructureTestCase(unittest.TestCase): "d": c_double, } + def test_inheritance_hierarchy(self): + self.assertEqual(Structure.mro(), [Structure, _CData, object]) + + self.assertEqual(PyCStructType.__name__, "PyCStructType") + self.assertEqual(type(PyCStructType), type) + + + def test_type_flags(self): + for cls in Structure, PyCStructType: + with self.subTest(cls=cls): + self.assertTrue(Structure.__flags__ & Py_TPFLAGS_IMMUTABLETYPE) + self.assertFalse(Structure.__flags__ & Py_TPFLAGS_DISALLOW_INSTANTIATION) + def test_simple_structs(self): for code, tp in self.formats.items(): class X(Structure): @@ -472,6 +488,66 @@ class X(Structure): self.assertEqual(s.first, got.first) self.assertEqual(s.second, got.second) + def _test_issue18060(self, Vector): + # The call to atan2() should succeed if the + # class fields were correctly cloned in the + # subclasses. Otherwise, it will segfault. + if sys.platform == 'win32': + libm = CDLL(find_library('msvcrt.dll')) + else: + libm = CDLL(find_library('m')) + + libm.atan2.argtypes = [Vector] + libm.atan2.restype = c_double + + arg = Vector(y=0.0, x=-1.0) + self.assertAlmostEqual(libm.atan2(arg), 3.141592653589793) + + @unittest.skipIf(_architecture() == ('64bit', 'WindowsPE'), "can't test Windows x64 build") + @unittest.skipUnless(sys.byteorder == 'little', "can't test on this platform") + def test_issue18060_a(self): + # This test case calls + # PyCStructUnionType_update_stgdict() for each + # _fields_ assignment, and PyCStgDict_clone() + # for the Mid and Vector class definitions. + class Base(Structure): + _fields_ = [('y', c_double), + ('x', c_double)] + class Mid(Base): + pass + Mid._fields_ = [] + class Vector(Mid): pass + self._test_issue18060(Vector) + + @unittest.skipIf(_architecture() == ('64bit', 'WindowsPE'), "can't test Windows x64 build") + @unittest.skipUnless(sys.byteorder == 'little', "can't test on this platform") + def test_issue18060_b(self): + # This test case calls + # PyCStructUnionType_update_stgdict() for each + # _fields_ assignment. + class Base(Structure): + _fields_ = [('y', c_double), + ('x', c_double)] + class Mid(Base): + _fields_ = [] + class Vector(Mid): + _fields_ = [] + self._test_issue18060(Vector) + + @unittest.skipIf(_architecture() == ('64bit', 'WindowsPE'), "can't test Windows x64 build") + @unittest.skipUnless(sys.byteorder == 'little', "can't test on this platform") + def test_issue18060_c(self): + # This test case calls + # PyCStructUnionType_update_stgdict() for each + # _fields_ assignment. + class Base(Structure): + _fields_ = [('y', c_double)] + class Mid(Base): + _fields_ = [] + class Vector(Mid): + _fields_ = [('x', c_double)] + self._test_issue18060(Vector) + def test_array_in_struct(self): # See bpo-22273 diff --git a/Lib/test/test_ctypes/test_unions.py b/Lib/test/test_ctypes/test_unions.py new file mode 100644 index 00000000000000..cf5344bdf19165 --- /dev/null +++ b/Lib/test/test_ctypes/test_unions.py @@ -0,0 +1,18 @@ +import unittest +from ctypes import Union +from ._support import (_CData, UnionType, Py_TPFLAGS_DISALLOW_INSTANTIATION, + Py_TPFLAGS_IMMUTABLETYPE) + + +class ArrayTestCase(unittest.TestCase): + def test_inheritance_hierarchy(self): + self.assertEqual(Union.mro(), [Union, _CData, object]) + + self.assertEqual(UnionType.__name__, "UnionType") + self.assertEqual(type(UnionType), type) + + def test_type_flags(self): + for cls in Union, UnionType: + with self.subTest(cls=Union): + self.assertTrue(Union.__flags__ & Py_TPFLAGS_IMMUTABLETYPE) + self.assertFalse(Union.__flags__ & Py_TPFLAGS_DISALLOW_INSTANTIATION) diff --git a/Lib/test/test_ctypes/test_win32.py b/Lib/test/test_ctypes/test_win32.py index 01e624f76f0685..4aaecd8d38f98f 100644 --- a/Lib/test/test_ctypes/test_win32.py +++ b/Lib/test/test_ctypes/test_win32.py @@ -9,6 +9,7 @@ _pointer_type_cache, c_void_p, c_char, c_int, c_long) from test import support +from ._support import Py_TPFLAGS_DISALLOW_INSTANTIATION, Py_TPFLAGS_IMMUTABLETYPE @unittest.skipUnless(sys.platform == "win32", 'Windows-specific test') @@ -73,6 +74,11 @@ def test_COMError(self): self.assertEqual(ex.text, "text") self.assertEqual(ex.details, ("details",)) + self.assertEqual(COMError.mro(), + [COMError, Exception, BaseException, object]) + self.assertFalse(COMError.__flags__ & Py_TPFLAGS_DISALLOW_INSTANTIATION) + self.assertTrue(COMError.__flags__ & Py_TPFLAGS_IMMUTABLETYPE) + @unittest.skipUnless(sys.platform == "win32", 'Windows-specific test') class TestWinError(unittest.TestCase): diff --git a/Lib/test/test_curses.py b/Lib/test/test_curses.py index 31bc108e7712ea..83d10dd8579074 100644 --- a/Lib/test/test_curses.py +++ b/Lib/test/test_curses.py @@ -8,7 +8,7 @@ from unittest.mock import MagicMock from test.support import (requires, verbose, SaveSignals, cpython_only, - check_disallow_instantiation) + check_disallow_instantiation, MISSING_C_DOCSTRINGS) from test.support.import_helper import import_module # Optionally test curses module. This currently requires that the @@ -1142,6 +1142,8 @@ def test_encoding(self): with self.assertRaises(TypeError): del stdscr.encoding + @unittest.skipIf(MISSING_C_DOCSTRINGS, + "Signature information for builtins requires docstrings") def test_issue21088(self): stdscr = self.stdscr # diff --git a/Lib/test/test_dbm_dumb.py b/Lib/test/test_dbm_dumb.py index a481175b3bfdbd..672f9092207cf6 100644 --- a/Lib/test/test_dbm_dumb.py +++ b/Lib/test/test_dbm_dumb.py @@ -246,9 +246,27 @@ def test_missing_data(self): _delete_files() with self.assertRaises(FileNotFoundError): dumbdbm.open(_fname, value) + self.assertFalse(os.path.exists(_fname + '.dat')) self.assertFalse(os.path.exists(_fname + '.dir')) self.assertFalse(os.path.exists(_fname + '.bak')) + for value in ('c', 'n'): + _delete_files() + with dumbdbm.open(_fname, value) as f: + self.assertTrue(os.path.exists(_fname + '.dat')) + self.assertTrue(os.path.exists(_fname + '.dir')) + self.assertFalse(os.path.exists(_fname + '.bak')) + self.assertFalse(os.path.exists(_fname + '.bak')) + + for value in ('c', 'n'): + _delete_files() + with dumbdbm.open(_fname, value) as f: + f['key'] = 'value' + self.assertTrue(os.path.exists(_fname + '.dat')) + self.assertTrue(os.path.exists(_fname + '.dir')) + self.assertFalse(os.path.exists(_fname + '.bak')) + self.assertTrue(os.path.exists(_fname + '.bak')) + def test_missing_index(self): with dumbdbm.open(_fname, 'n') as f: pass @@ -259,6 +277,60 @@ def test_missing_index(self): self.assertFalse(os.path.exists(_fname + '.dir')) self.assertFalse(os.path.exists(_fname + '.bak')) + for value in ('c', 'n'): + with dumbdbm.open(_fname, value) as f: + self.assertTrue(os.path.exists(_fname + '.dir')) + self.assertFalse(os.path.exists(_fname + '.bak')) + self.assertFalse(os.path.exists(_fname + '.bak')) + os.unlink(_fname + '.dir') + + for value in ('c', 'n'): + with dumbdbm.open(_fname, value) as f: + f['key'] = 'value' + self.assertTrue(os.path.exists(_fname + '.dir')) + self.assertFalse(os.path.exists(_fname + '.bak')) + self.assertTrue(os.path.exists(_fname + '.bak')) + os.unlink(_fname + '.dir') + os.unlink(_fname + '.bak') + + def test_sync_empty_unmodified(self): + with dumbdbm.open(_fname, 'n') as f: + pass + os.unlink(_fname + '.dir') + for value in ('c', 'n'): + with dumbdbm.open(_fname, value) as f: + self.assertTrue(os.path.exists(_fname + '.dir')) + self.assertFalse(os.path.exists(_fname + '.bak')) + f.sync() + self.assertTrue(os.path.exists(_fname + '.dir')) + self.assertFalse(os.path.exists(_fname + '.bak')) + os.unlink(_fname + '.dir') + f.sync() + self.assertFalse(os.path.exists(_fname + '.dir')) + self.assertFalse(os.path.exists(_fname + '.bak')) + self.assertFalse(os.path.exists(_fname + '.dir')) + self.assertFalse(os.path.exists(_fname + '.bak')) + + def test_sync_nonempty_unmodified(self): + with dumbdbm.open(_fname, 'n') as f: + pass + os.unlink(_fname + '.dir') + for value in ('c', 'n'): + with dumbdbm.open(_fname, value) as f: + f['key'] = 'value' + self.assertTrue(os.path.exists(_fname + '.dir')) + self.assertFalse(os.path.exists(_fname + '.bak')) + f.sync() + self.assertTrue(os.path.exists(_fname + '.dir')) + self.assertTrue(os.path.exists(_fname + '.bak')) + os.unlink(_fname + '.dir') + os.unlink(_fname + '.bak') + f.sync() + self.assertFalse(os.path.exists(_fname + '.dir')) + self.assertFalse(os.path.exists(_fname + '.bak')) + self.assertFalse(os.path.exists(_fname + '.dir')) + self.assertFalse(os.path.exists(_fname + '.bak')) + def test_invalid_flag(self): for flag in ('x', 'rf', None): with self.assertRaisesRegex(ValueError, diff --git a/Lib/test/test_decimal.py b/Lib/test/test_decimal.py index 7a5fe62b467372..1423bc61c7f690 100644 --- a/Lib/test/test_decimal.py +++ b/Lib/test/test_decimal.py @@ -41,6 +41,7 @@ darwin_malloc_err_warning, is_emscripten) from test.support.import_helper import import_fresh_module from test.support import threading_helper +from test.support import warnings_helper import random import inspect import threading @@ -1237,7 +1238,14 @@ def test_deprecated_N_format(self): else: self.assertRaises(ValueError, format, h, 'N') self.assertRaises(ValueError, format, h, '010.3N') - + with warnings_helper.check_no_warnings(self): + self.assertEqual(format(h, 'N>10.3'), 'NN6.63E-34') + self.assertEqual(format(h, 'N>10.3n'), 'NN6.63e-34') + self.assertEqual(format(h, 'N>10.3e'), 'N6.626e-34') + self.assertEqual(format(h, 'N>10.3f'), 'NNNNN0.000') + self.assertRaises(ValueError, format, h, '>Nf') + self.assertRaises(ValueError, format, h, '10Nf') + self.assertRaises(ValueError, format, h, 'Nx') @run_with_locale('LC_ALL', 'ps_AF') def test_wide_char_separator_decimal_point(self): diff --git a/Lib/test/test_descr.py b/Lib/test/test_descr.py index fd0af9b30a0a71..beeab6cb7f254c 100644 --- a/Lib/test/test_descr.py +++ b/Lib/test/test_descr.py @@ -4734,6 +4734,20 @@ class X(object): with self.assertRaises(AttributeError): del X.__abstractmethods__ + def test_gh55664(self): + # gh-55664: issue a warning when the + # __dict__ of a class contains non-string keys + with self.assertWarnsRegex(RuntimeWarning, 'MyClass'): + MyClass = type('MyClass', (), {1: 2}) + + class meta(type): + def __new__(mcls, name, bases, ns): + ns[1] = 2 + return super().__new__(mcls, name, bases, ns) + + with self.assertWarnsRegex(RuntimeWarning, 'MyClass'): + MyClass = meta('MyClass', (), {}) + def test_proxy_call(self): class FakeStr: __class__ = str @@ -5151,7 +5165,8 @@ class Base2(object): mykey = 'from Base2' mykey2 = 'from Base2' - X = type('X', (Base,), {MyKey(): 5}) + with self.assertWarnsRegex(RuntimeWarning, 'X'): + X = type('X', (Base,), {MyKey(): 5}) # mykey is read from Base self.assertEqual(X.mykey, 'from Base') # mykey2 is read from Base2 because MyKey.__eq__ has set __bases__ diff --git a/Lib/test/test_dis.py b/Lib/test/test_dis.py index e4c7e9ba4649f7..a5917da346dded 100644 --- a/Lib/test/test_dis.py +++ b/Lib/test/test_dis.py @@ -158,6 +158,7 @@ def bug708901(): %3d JUMP_BACKWARD 5 (to L1) %3d L2: END_FOR + POP_TOP RETURN_CONST 0 (None) """ % (bug708901.__code__.co_firstlineno, bug708901.__code__.co_firstlineno + 1, @@ -577,14 +578,10 @@ async def _asyncwith(c): RETURN_CONST 0 (None) %4d L12: CLEANUP_THROW - - -- L13: JUMP_BACKWARD 26 (to L5) - -%4d L14: CLEANUP_THROW - - -- L15: JUMP_BACKWARD 11 (to L11) - -%4d L16: PUSH_EXC_INFO + L13: JUMP_BACKWARD_NO_INTERRUPT 25 (to L5) + L14: CLEANUP_THROW + L15: JUMP_BACKWARD_NO_INTERRUPT 9 (to L11) + L16: PUSH_EXC_INFO WITH_EXCEPT_START GET_AWAITABLE 2 LOAD_CONST 0 (None) @@ -630,8 +627,6 @@ async def _asyncwith(c): _asyncwith.__code__.co_firstlineno + 1, _asyncwith.__code__.co_firstlineno + 3, _asyncwith.__code__.co_firstlineno + 1, - _asyncwith.__code__.co_firstlineno + 1, - _asyncwith.__code__.co_firstlineno + 1, _asyncwith.__code__.co_firstlineno + 3, ) @@ -797,6 +792,7 @@ def foo(x): POP_TOP JUMP_BACKWARD 12 (to L2) L3: END_FOR + POP_TOP RETURN_CONST 0 (None) -- L4: CALL_INTRINSIC_1 3 (INTRINSIC_STOPITERATION_ERROR) @@ -849,6 +845,7 @@ def loop_test(): JUMP_BACKWARD 16 (to L1) %3d L2: END_FOR + POP_TOP RETURN_CONST 0 (None) """ % (loop_test.__code__.co_firstlineno, loop_test.__code__.co_firstlineno + 1, @@ -1654,122 +1651,123 @@ def _prepare_test_cases(): ] expected_opinfo_jumpy = [ - Instruction(opname='RESUME', opcode=149, arg=0, argval=0, argrepr='', offset=0, start_offset=0, starts_line=True, line_number=1, label=None, positions=None), - Instruction(opname='LOAD_GLOBAL', opcode=91, arg=1, argval='range', argrepr='range + NULL', offset=2, start_offset=2, starts_line=True, line_number=3, label=None, positions=None), - Instruction(opname='LOAD_CONST', opcode=83, arg=1, argval=10, argrepr='10', offset=12, start_offset=12, starts_line=False, line_number=3, label=None, positions=None), - Instruction(opname='CALL', opcode=53, arg=1, argval=1, argrepr='', offset=14, start_offset=14, starts_line=False, line_number=3, label=None, positions=None), - Instruction(opname='GET_ITER', opcode=19, arg=None, argval=None, argrepr='', offset=22, start_offset=22, starts_line=False, line_number=3, label=None, positions=None), - Instruction(opname='FOR_ITER', opcode=72, arg=30, argval=88, argrepr='to L4', offset=24, start_offset=24, starts_line=False, line_number=3, label=1, positions=None), - Instruction(opname='STORE_FAST', opcode=110, arg=0, argval='i', argrepr='i', offset=28, start_offset=28, starts_line=False, line_number=3, label=None, positions=None), - Instruction(opname='LOAD_GLOBAL', opcode=91, arg=3, argval='print', argrepr='print + NULL', offset=30, start_offset=30, starts_line=True, line_number=4, label=None, positions=None), - Instruction(opname='LOAD_FAST', opcode=85, arg=0, argval='i', argrepr='i', offset=40, start_offset=40, starts_line=False, line_number=4, label=None, positions=None), - Instruction(opname='CALL', opcode=53, arg=1, argval=1, argrepr='', offset=42, start_offset=42, starts_line=False, line_number=4, label=None, positions=None), - Instruction(opname='POP_TOP', opcode=32, arg=None, argval=None, argrepr='', offset=50, start_offset=50, starts_line=False, line_number=4, label=None, positions=None), - Instruction(opname='LOAD_FAST', opcode=85, arg=0, argval='i', argrepr='i', offset=52, start_offset=52, starts_line=True, line_number=5, label=None, positions=None), - Instruction(opname='LOAD_CONST', opcode=83, arg=2, argval=4, argrepr='4', offset=54, start_offset=54, starts_line=False, line_number=5, label=None, positions=None), - Instruction(opname='COMPARE_OP', opcode=58, arg=18, argval='<', argrepr='bool(<)', offset=56, start_offset=56, starts_line=False, line_number=5, label=None, positions=None), - Instruction(opname='POP_JUMP_IF_FALSE', opcode=97, arg=2, argval=68, argrepr='to L2', offset=60, start_offset=60, starts_line=False, line_number=5, label=None, positions=None), - Instruction(opname='JUMP_BACKWARD', opcode=77, arg=22, argval=24, argrepr='to L1', offset=64, start_offset=64, starts_line=True, line_number=6, label=None, positions=None), - Instruction(opname='LOAD_FAST', opcode=85, arg=0, argval='i', argrepr='i', offset=68, start_offset=68, starts_line=True, line_number=7, label=2, positions=None), - Instruction(opname='LOAD_CONST', opcode=83, arg=3, argval=6, argrepr='6', offset=70, start_offset=70, starts_line=False, line_number=7, label=None, positions=None), - Instruction(opname='COMPARE_OP', opcode=58, arg=148, argval='>', argrepr='bool(>)', offset=72, start_offset=72, starts_line=False, line_number=7, label=None, positions=None), - Instruction(opname='POP_JUMP_IF_TRUE', opcode=100, arg=2, argval=84, argrepr='to L3', offset=76, start_offset=76, starts_line=False, line_number=7, label=None, positions=None), - Instruction(opname='JUMP_BACKWARD', opcode=77, arg=30, argval=24, argrepr='to L1', offset=80, start_offset=80, starts_line=False, line_number=7, label=None, positions=None), - Instruction(opname='POP_TOP', opcode=32, arg=None, argval=None, argrepr='', offset=84, start_offset=84, starts_line=True, line_number=8, label=3, positions=None), - Instruction(opname='JUMP_FORWARD', opcode=79, arg=12, argval=112, argrepr='to L5', offset=86, start_offset=86, starts_line=False, line_number=8, label=None, positions=None), - Instruction(opname='END_FOR', opcode=11, arg=None, argval=None, argrepr='', offset=88, start_offset=88, starts_line=True, line_number=3, label=4, positions=None), - Instruction(opname='LOAD_GLOBAL', opcode=91, arg=3, argval='print', argrepr='print + NULL', offset=90, start_offset=90, starts_line=True, line_number=10, label=None, positions=None), - Instruction(opname='LOAD_CONST', opcode=83, arg=4, argval='I can haz else clause?', argrepr="'I can haz else clause?'", offset=100, start_offset=100, starts_line=False, line_number=10, label=None, positions=None), - Instruction(opname='CALL', opcode=53, arg=1, argval=1, argrepr='', offset=102, start_offset=102, starts_line=False, line_number=10, label=None, positions=None), - Instruction(opname='POP_TOP', opcode=32, arg=None, argval=None, argrepr='', offset=110, start_offset=110, starts_line=False, line_number=10, label=None, positions=None), - Instruction(opname='LOAD_FAST_CHECK', opcode=87, arg=0, argval='i', argrepr='i', offset=112, start_offset=112, starts_line=True, line_number=11, label=5, positions=None), - Instruction(opname='TO_BOOL', opcode=40, arg=None, argval=None, argrepr='', offset=114, start_offset=114, starts_line=False, line_number=11, label=None, positions=None), - Instruction(opname='POP_JUMP_IF_FALSE', opcode=97, arg=40, argval=206, argrepr='to L9', offset=122, start_offset=122, starts_line=False, line_number=11, label=None, positions=None), - Instruction(opname='LOAD_GLOBAL', opcode=91, arg=3, argval='print', argrepr='print + NULL', offset=126, start_offset=126, starts_line=True, line_number=12, label=6, positions=None), - Instruction(opname='LOAD_FAST', opcode=85, arg=0, argval='i', argrepr='i', offset=136, start_offset=136, starts_line=False, line_number=12, label=None, positions=None), - Instruction(opname='CALL', opcode=53, arg=1, argval=1, argrepr='', offset=138, start_offset=138, starts_line=False, line_number=12, label=None, positions=None), - Instruction(opname='POP_TOP', opcode=32, arg=None, argval=None, argrepr='', offset=146, start_offset=146, starts_line=False, line_number=12, label=None, positions=None), - Instruction(opname='LOAD_FAST', opcode=85, arg=0, argval='i', argrepr='i', offset=148, start_offset=148, starts_line=True, line_number=13, label=None, positions=None), - Instruction(opname='LOAD_CONST', opcode=83, arg=5, argval=1, argrepr='1', offset=150, start_offset=150, starts_line=False, line_number=13, label=None, positions=None), - Instruction(opname='BINARY_OP', opcode=45, arg=23, argval=23, argrepr='-=', offset=152, start_offset=152, starts_line=False, line_number=13, label=None, positions=None), - Instruction(opname='STORE_FAST', opcode=110, arg=0, argval='i', argrepr='i', offset=156, start_offset=156, starts_line=False, line_number=13, label=None, positions=None), - Instruction(opname='LOAD_FAST', opcode=85, arg=0, argval='i', argrepr='i', offset=158, start_offset=158, starts_line=True, line_number=14, label=None, positions=None), - Instruction(opname='LOAD_CONST', opcode=83, arg=3, argval=6, argrepr='6', offset=160, start_offset=160, starts_line=False, line_number=14, label=None, positions=None), - Instruction(opname='COMPARE_OP', opcode=58, arg=148, argval='>', argrepr='bool(>)', offset=162, start_offset=162, starts_line=False, line_number=14, label=None, positions=None), - Instruction(opname='POP_JUMP_IF_FALSE', opcode=97, arg=2, argval=174, argrepr='to L7', offset=166, start_offset=166, starts_line=False, line_number=14, label=None, positions=None), - Instruction(opname='JUMP_BACKWARD', opcode=77, arg=31, argval=112, argrepr='to L5', offset=170, start_offset=170, starts_line=True, line_number=15, label=None, positions=None), - Instruction(opname='LOAD_FAST', opcode=85, arg=0, argval='i', argrepr='i', offset=174, start_offset=174, starts_line=True, line_number=16, label=7, positions=None), - Instruction(opname='LOAD_CONST', opcode=83, arg=2, argval=4, argrepr='4', offset=176, start_offset=176, starts_line=False, line_number=16, label=None, positions=None), - Instruction(opname='COMPARE_OP', opcode=58, arg=18, argval='<', argrepr='bool(<)', offset=178, start_offset=178, starts_line=False, line_number=16, label=None, positions=None), - Instruction(opname='POP_JUMP_IF_FALSE', opcode=97, arg=1, argval=188, argrepr='to L8', offset=182, start_offset=182, starts_line=False, line_number=16, label=None, positions=None), - Instruction(opname='JUMP_FORWARD', opcode=79, arg=20, argval=228, argrepr='to L10', offset=186, start_offset=186, starts_line=True, line_number=17, label=None, positions=None), - Instruction(opname='LOAD_FAST', opcode=85, arg=0, argval='i', argrepr='i', offset=188, start_offset=188, starts_line=True, line_number=11, label=8, positions=None), - Instruction(opname='TO_BOOL', opcode=40, arg=None, argval=None, argrepr='', offset=190, start_offset=190, starts_line=False, line_number=11, label=None, positions=None), - Instruction(opname='POP_JUMP_IF_FALSE', opcode=97, arg=2, argval=206, argrepr='to L9', offset=198, start_offset=198, starts_line=False, line_number=11, label=None, positions=None), - Instruction(opname='JUMP_BACKWARD', opcode=77, arg=40, argval=126, argrepr='to L6', offset=202, start_offset=202, starts_line=False, line_number=11, label=None, positions=None), - Instruction(opname='LOAD_GLOBAL', opcode=91, arg=3, argval='print', argrepr='print + NULL', offset=206, start_offset=206, starts_line=True, line_number=19, label=9, positions=None), - Instruction(opname='LOAD_CONST', opcode=83, arg=6, argval='Who let lolcatz into this test suite?', argrepr="'Who let lolcatz into this test suite?'", offset=216, start_offset=216, starts_line=False, line_number=19, label=None, positions=None), - Instruction(opname='CALL', opcode=53, arg=1, argval=1, argrepr='', offset=218, start_offset=218, starts_line=False, line_number=19, label=None, positions=None), - Instruction(opname='POP_TOP', opcode=32, arg=None, argval=None, argrepr='', offset=226, start_offset=226, starts_line=False, line_number=19, label=None, positions=None), - Instruction(opname='NOP', opcode=30, arg=None, argval=None, argrepr='', offset=228, start_offset=228, starts_line=True, line_number=20, label=10, positions=None), - Instruction(opname='LOAD_CONST', opcode=83, arg=5, argval=1, argrepr='1', offset=230, start_offset=230, starts_line=True, line_number=21, label=None, positions=None), - Instruction(opname='LOAD_CONST', opcode=83, arg=7, argval=0, argrepr='0', offset=232, start_offset=232, starts_line=False, line_number=21, label=None, positions=None), - Instruction(opname='BINARY_OP', opcode=45, arg=11, argval=11, argrepr='/', offset=234, start_offset=234, starts_line=False, line_number=21, label=None, positions=None), - Instruction(opname='POP_TOP', opcode=32, arg=None, argval=None, argrepr='', offset=238, start_offset=238, starts_line=False, line_number=21, label=None, positions=None), - Instruction(opname='LOAD_FAST', opcode=85, arg=0, argval='i', argrepr='i', offset=240, start_offset=240, starts_line=True, line_number=25, label=None, positions=None), - Instruction(opname='BEFORE_WITH', opcode=2, arg=None, argval=None, argrepr='', offset=242, start_offset=242, starts_line=False, line_number=25, label=None, positions=None), - Instruction(opname='STORE_FAST', opcode=110, arg=1, argval='dodgy', argrepr='dodgy', offset=244, start_offset=244, starts_line=False, line_number=25, label=None, positions=None), - Instruction(opname='LOAD_GLOBAL', opcode=91, arg=3, argval='print', argrepr='print + NULL', offset=246, start_offset=246, starts_line=True, line_number=26, label=None, positions=None), - Instruction(opname='LOAD_CONST', opcode=83, arg=8, argval='Never reach this', argrepr="'Never reach this'", offset=256, start_offset=256, starts_line=False, line_number=26, label=None, positions=None), - Instruction(opname='CALL', opcode=53, arg=1, argval=1, argrepr='', offset=258, start_offset=258, starts_line=False, line_number=26, label=None, positions=None), - Instruction(opname='POP_TOP', opcode=32, arg=None, argval=None, argrepr='', offset=266, start_offset=266, starts_line=False, line_number=26, label=None, positions=None), - Instruction(opname='LOAD_CONST', opcode=83, arg=0, argval=None, argrepr='None', offset=268, start_offset=268, starts_line=True, line_number=25, label=None, positions=None), - Instruction(opname='LOAD_CONST', opcode=83, arg=0, argval=None, argrepr='None', offset=270, start_offset=270, starts_line=False, line_number=25, label=None, positions=None), - Instruction(opname='LOAD_CONST', opcode=83, arg=0, argval=None, argrepr='None', offset=272, start_offset=272, starts_line=False, line_number=25, label=None, positions=None), - Instruction(opname='CALL', opcode=53, arg=2, argval=2, argrepr='', offset=274, start_offset=274, starts_line=False, line_number=25, label=None, positions=None), - Instruction(opname='POP_TOP', opcode=32, arg=None, argval=None, argrepr='', offset=282, start_offset=282, starts_line=False, line_number=25, label=None, positions=None), - Instruction(opname='LOAD_GLOBAL', opcode=91, arg=3, argval='print', argrepr='print + NULL', offset=284, start_offset=284, starts_line=True, line_number=28, label=11, positions=None), - Instruction(opname='LOAD_CONST', opcode=83, arg=10, argval="OK, now we're done", argrepr='"OK, now we\'re done"', offset=294, start_offset=294, starts_line=False, line_number=28, label=None, positions=None), - Instruction(opname='CALL', opcode=53, arg=1, argval=1, argrepr='', offset=296, start_offset=296, starts_line=False, line_number=28, label=None, positions=None), - Instruction(opname='POP_TOP', opcode=32, arg=None, argval=None, argrepr='', offset=304, start_offset=304, starts_line=False, line_number=28, label=None, positions=None), - Instruction(opname='RETURN_CONST', opcode=103, arg=0, argval=None, argrepr='None', offset=306, start_offset=306, starts_line=False, line_number=28, label=None, positions=None), - Instruction(opname='PUSH_EXC_INFO', opcode=33, arg=None, argval=None, argrepr='', offset=308, start_offset=308, starts_line=True, line_number=25, label=None, positions=None), - Instruction(opname='WITH_EXCEPT_START', opcode=44, arg=None, argval=None, argrepr='', offset=310, start_offset=310, starts_line=False, line_number=25, label=None, positions=None), - Instruction(opname='TO_BOOL', opcode=40, arg=None, argval=None, argrepr='', offset=312, start_offset=312, starts_line=False, line_number=25, label=None, positions=None), - Instruction(opname='POP_JUMP_IF_TRUE', opcode=100, arg=1, argval=326, argrepr='to L12', offset=320, start_offset=320, starts_line=False, line_number=25, label=None, positions=None), - Instruction(opname='RERAISE', opcode=102, arg=2, argval=2, argrepr='', offset=324, start_offset=324, starts_line=False, line_number=25, label=None, positions=None), - Instruction(opname='POP_TOP', opcode=32, arg=None, argval=None, argrepr='', offset=326, start_offset=326, starts_line=False, line_number=25, label=12, positions=None), - Instruction(opname='POP_EXCEPT', opcode=31, arg=None, argval=None, argrepr='', offset=328, start_offset=328, starts_line=False, line_number=25, label=None, positions=None), - Instruction(opname='POP_TOP', opcode=32, arg=None, argval=None, argrepr='', offset=330, start_offset=330, starts_line=False, line_number=25, label=None, positions=None), - Instruction(opname='POP_TOP', opcode=32, arg=None, argval=None, argrepr='', offset=332, start_offset=332, starts_line=False, line_number=25, label=None, positions=None), - Instruction(opname='JUMP_BACKWARD', opcode=77, arg=27, argval=284, argrepr='to L11', offset=334, start_offset=334, starts_line=False, line_number=25, label=None, positions=None), - Instruction(opname='COPY', opcode=61, arg=3, argval=3, argrepr='', offset=338, start_offset=338, starts_line=True, line_number=None, label=None, positions=None), - Instruction(opname='POP_EXCEPT', opcode=31, arg=None, argval=None, argrepr='', offset=340, start_offset=340, starts_line=False, line_number=None, label=None, positions=None), - Instruction(opname='RERAISE', opcode=102, arg=1, argval=1, argrepr='', offset=342, start_offset=342, starts_line=False, line_number=None, label=None, positions=None), - Instruction(opname='PUSH_EXC_INFO', opcode=33, arg=None, argval=None, argrepr='', offset=344, start_offset=344, starts_line=False, line_number=None, label=None, positions=None), - Instruction(opname='LOAD_GLOBAL', opcode=91, arg=4, argval='ZeroDivisionError', argrepr='ZeroDivisionError', offset=346, start_offset=346, starts_line=True, line_number=22, label=None, positions=None), - Instruction(opname='CHECK_EXC_MATCH', opcode=7, arg=None, argval=None, argrepr='', offset=356, start_offset=356, starts_line=False, line_number=22, label=None, positions=None), - Instruction(opname='POP_JUMP_IF_FALSE', opcode=97, arg=15, argval=392, argrepr='to L13', offset=358, start_offset=358, starts_line=False, line_number=22, label=None, positions=None), - Instruction(opname='POP_TOP', opcode=32, arg=None, argval=None, argrepr='', offset=362, start_offset=362, starts_line=False, line_number=22, label=None, positions=None), - Instruction(opname='LOAD_GLOBAL', opcode=91, arg=3, argval='print', argrepr='print + NULL', offset=364, start_offset=364, starts_line=True, line_number=23, label=None, positions=None), - Instruction(opname='LOAD_CONST', opcode=83, arg=9, argval='Here we go, here we go, here we go...', argrepr="'Here we go, here we go, here we go...'", offset=374, start_offset=374, starts_line=False, line_number=23, label=None, positions=None), - Instruction(opname='CALL', opcode=53, arg=1, argval=1, argrepr='', offset=376, start_offset=376, starts_line=False, line_number=23, label=None, positions=None), - Instruction(opname='POP_TOP', opcode=32, arg=None, argval=None, argrepr='', offset=384, start_offset=384, starts_line=False, line_number=23, label=None, positions=None), - Instruction(opname='POP_EXCEPT', opcode=31, arg=None, argval=None, argrepr='', offset=386, start_offset=386, starts_line=False, line_number=23, label=None, positions=None), - Instruction(opname='JUMP_BACKWARD', opcode=77, arg=54, argval=284, argrepr='to L11', offset=388, start_offset=388, starts_line=False, line_number=23, label=None, positions=None), - Instruction(opname='RERAISE', opcode=102, arg=0, argval=0, argrepr='', offset=392, start_offset=392, starts_line=True, line_number=22, label=13, positions=None), - Instruction(opname='COPY', opcode=61, arg=3, argval=3, argrepr='', offset=394, start_offset=394, starts_line=True, line_number=None, label=None, positions=None), - Instruction(opname='POP_EXCEPT', opcode=31, arg=None, argval=None, argrepr='', offset=396, start_offset=396, starts_line=False, line_number=None, label=None, positions=None), - Instruction(opname='RERAISE', opcode=102, arg=1, argval=1, argrepr='', offset=398, start_offset=398, starts_line=False, line_number=None, label=None, positions=None), - Instruction(opname='PUSH_EXC_INFO', opcode=33, arg=None, argval=None, argrepr='', offset=400, start_offset=400, starts_line=False, line_number=None, label=None, positions=None), - Instruction(opname='LOAD_GLOBAL', opcode=91, arg=3, argval='print', argrepr='print + NULL', offset=402, start_offset=402, starts_line=True, line_number=28, label=None, positions=None), - Instruction(opname='LOAD_CONST', opcode=83, arg=10, argval="OK, now we're done", argrepr='"OK, now we\'re done"', offset=412, start_offset=412, starts_line=False, line_number=28, label=None, positions=None), - Instruction(opname='CALL', opcode=53, arg=1, argval=1, argrepr='', offset=414, start_offset=414, starts_line=False, line_number=28, label=None, positions=None), - Instruction(opname='POP_TOP', opcode=32, arg=None, argval=None, argrepr='', offset=422, start_offset=422, starts_line=False, line_number=28, label=None, positions=None), - Instruction(opname='RERAISE', opcode=102, arg=0, argval=0, argrepr='', offset=424, start_offset=424, starts_line=False, line_number=28, label=None, positions=None), - Instruction(opname='COPY', opcode=61, arg=3, argval=3, argrepr='', offset=426, start_offset=426, starts_line=True, line_number=None, label=None, positions=None), - Instruction(opname='POP_EXCEPT', opcode=31, arg=None, argval=None, argrepr='', offset=428, start_offset=428, starts_line=False, line_number=None, label=None, positions=None), - Instruction(opname='RERAISE', opcode=102, arg=1, argval=1, argrepr='', offset=430, start_offset=430, starts_line=False, line_number=None, label=None, positions=None), + Instruction(opname='RESUME', opcode=149, arg=0, argval=0, argrepr='', offset=0, start_offset=0, starts_line=True, line_number=1, label=None, positions=None, cache_info=None), + Instruction(opname='LOAD_GLOBAL', opcode=91, arg=1, argval='range', argrepr='range + NULL', offset=2, start_offset=2, starts_line=True, line_number=3, label=None, positions=None, cache_info=[('counter', 1, b'\x00\x00'), ('index', 1, b'\x00\x00'), ('module_keys_version', 1, b'\x00\x00'), ('builtin_keys_version', 1, b'\x00\x00')]), + Instruction(opname='LOAD_CONST', opcode=83, arg=1, argval=10, argrepr='10', offset=12, start_offset=12, starts_line=False, line_number=3, label=None, positions=None, cache_info=None), + Instruction(opname='CALL', opcode=53, arg=1, argval=1, argrepr='', offset=14, start_offset=14, starts_line=False, line_number=3, label=None, positions=None, cache_info=[('counter', 1, b'\x00\x00'), ('func_version', 2, b'\x00\x00\x00\x00')]), + Instruction(opname='GET_ITER', opcode=19, arg=None, argval=None, argrepr='', offset=22, start_offset=22, starts_line=False, line_number=3, label=None, positions=None, cache_info=None), + Instruction(opname='FOR_ITER', opcode=72, arg=30, argval=88, argrepr='to L4', offset=24, start_offset=24, starts_line=False, line_number=3, label=1, positions=None, cache_info=[('counter', 1, b'\x00\x00')]), + Instruction(opname='STORE_FAST', opcode=110, arg=0, argval='i', argrepr='i', offset=28, start_offset=28, starts_line=False, line_number=3, label=None, positions=None, cache_info=None), + Instruction(opname='LOAD_GLOBAL', opcode=91, arg=3, argval='print', argrepr='print + NULL', offset=30, start_offset=30, starts_line=True, line_number=4, label=None, positions=None, cache_info=[('counter', 1, b'\x00\x00'), ('index', 1, b'\x00\x00'), ('module_keys_version', 1, b'\x00\x00'), ('builtin_keys_version', 1, b'\x00\x00')]), + Instruction(opname='LOAD_FAST', opcode=85, arg=0, argval='i', argrepr='i', offset=40, start_offset=40, starts_line=False, line_number=4, label=None, positions=None, cache_info=None), + Instruction(opname='CALL', opcode=53, arg=1, argval=1, argrepr='', offset=42, start_offset=42, starts_line=False, line_number=4, label=None, positions=None, cache_info=[('counter', 1, b'\x00\x00'), ('func_version', 2, b'\x00\x00\x00\x00')]), + Instruction(opname='POP_TOP', opcode=32, arg=None, argval=None, argrepr='', offset=50, start_offset=50, starts_line=False, line_number=4, label=None, positions=None, cache_info=None), + Instruction(opname='LOAD_FAST', opcode=85, arg=0, argval='i', argrepr='i', offset=52, start_offset=52, starts_line=True, line_number=5, label=None, positions=None, cache_info=None), + Instruction(opname='LOAD_CONST', opcode=83, arg=2, argval=4, argrepr='4', offset=54, start_offset=54, starts_line=False, line_number=5, label=None, positions=None, cache_info=None), + Instruction(opname='COMPARE_OP', opcode=58, arg=18, argval='<', argrepr='bool(<)', offset=56, start_offset=56, starts_line=False, line_number=5, label=None, positions=None, cache_info=[('counter', 1, b'\x00\x00')]), + Instruction(opname='POP_JUMP_IF_FALSE', opcode=97, arg=2, argval=68, argrepr='to L2', offset=60, start_offset=60, starts_line=False, line_number=5, label=None, positions=None, cache_info=[('counter', 1, b'\x00\x00')]), + Instruction(opname='JUMP_BACKWARD', opcode=77, arg=22, argval=24, argrepr='to L1', offset=64, start_offset=64, starts_line=True, line_number=6, label=None, positions=None, cache_info=[('counter', 1, b'\x00\x00')]), + Instruction(opname='LOAD_FAST', opcode=85, arg=0, argval='i', argrepr='i', offset=68, start_offset=68, starts_line=True, line_number=7, label=2, positions=None, cache_info=None), + Instruction(opname='LOAD_CONST', opcode=83, arg=3, argval=6, argrepr='6', offset=70, start_offset=70, starts_line=False, line_number=7, label=None, positions=None, cache_info=None), + Instruction(opname='COMPARE_OP', opcode=58, arg=148, argval='>', argrepr='bool(>)', offset=72, start_offset=72, starts_line=False, line_number=7, label=None, positions=None, cache_info=[('counter', 1, b'\x00\x00')]), + Instruction(opname='POP_JUMP_IF_TRUE', opcode=100, arg=2, argval=84, argrepr='to L3', offset=76, start_offset=76, starts_line=False, line_number=7, label=None, positions=None, cache_info=[('counter', 1, b'\x00\x00')]), + Instruction(opname='JUMP_BACKWARD', opcode=77, arg=30, argval=24, argrepr='to L1', offset=80, start_offset=80, starts_line=False, line_number=7, label=None, positions=None, cache_info=[('counter', 1, b'\x00\x00')]), + Instruction(opname='POP_TOP', opcode=32, arg=None, argval=None, argrepr='', offset=84, start_offset=84, starts_line=True, line_number=8, label=3, positions=None, cache_info=None), + Instruction(opname='JUMP_FORWARD', opcode=79, arg=13, argval=114, argrepr='to L5', offset=86, start_offset=86, starts_line=False, line_number=8, label=None, positions=None, cache_info=None), + Instruction(opname='END_FOR', opcode=11, arg=None, argval=None, argrepr='', offset=88, start_offset=88, starts_line=True, line_number=3, label=4, positions=None, cache_info=None), + Instruction(opname='POP_TOP', opcode=32, arg=None, argval=None, argrepr='', offset=90, start_offset=90, starts_line=False, line_number=3, label=None, positions=None, cache_info=None), + Instruction(opname='LOAD_GLOBAL', opcode=91, arg=3, argval='print', argrepr='print + NULL', offset=92, start_offset=92, starts_line=True, line_number=10, label=None, positions=None, cache_info=[('counter', 1, b'\x00\x00'), ('index', 1, b'\x00\x00'), ('module_keys_version', 1, b'\x00\x00'), ('builtin_keys_version', 1, b'\x00\x00')]), + Instruction(opname='LOAD_CONST', opcode=83, arg=4, argval='I can haz else clause?', argrepr="'I can haz else clause?'", offset=102, start_offset=102, starts_line=False, line_number=10, label=None, positions=None, cache_info=None), + Instruction(opname='CALL', opcode=53, arg=1, argval=1, argrepr='', offset=104, start_offset=104, starts_line=False, line_number=10, label=None, positions=None, cache_info=[('counter', 1, b'\x00\x00'), ('func_version', 2, b'\x00\x00\x00\x00')]), + Instruction(opname='POP_TOP', opcode=32, arg=None, argval=None, argrepr='', offset=112, start_offset=112, starts_line=False, line_number=10, label=None, positions=None, cache_info=None), + Instruction(opname='LOAD_FAST_CHECK', opcode=87, arg=0, argval='i', argrepr='i', offset=114, start_offset=114, starts_line=True, line_number=11, label=5, positions=None, cache_info=None), + Instruction(opname='TO_BOOL', opcode=40, arg=None, argval=None, argrepr='', offset=116, start_offset=116, starts_line=False, line_number=11, label=None, positions=None, cache_info=[('counter', 1, b'\x00\x00'), ('version', 2, b'\x00\x00\x00\x00')]), + Instruction(opname='POP_JUMP_IF_FALSE', opcode=97, arg=40, argval=208, argrepr='to L9', offset=124, start_offset=124, starts_line=False, line_number=11, label=None, positions=None, cache_info=[('counter', 1, b'\x00\x00')]), + Instruction(opname='LOAD_GLOBAL', opcode=91, arg=3, argval='print', argrepr='print + NULL', offset=128, start_offset=128, starts_line=True, line_number=12, label=6, positions=None, cache_info=[('counter', 1, b'\x00\x00'), ('index', 1, b'\x00\x00'), ('module_keys_version', 1, b'\x00\x00'), ('builtin_keys_version', 1, b'\x00\x00')]), + Instruction(opname='LOAD_FAST', opcode=85, arg=0, argval='i', argrepr='i', offset=138, start_offset=138, starts_line=False, line_number=12, label=None, positions=None, cache_info=None), + Instruction(opname='CALL', opcode=53, arg=1, argval=1, argrepr='', offset=140, start_offset=140, starts_line=False, line_number=12, label=None, positions=None, cache_info=[('counter', 1, b'\x00\x00'), ('func_version', 2, b'\x00\x00\x00\x00')]), + Instruction(opname='POP_TOP', opcode=32, arg=None, argval=None, argrepr='', offset=148, start_offset=148, starts_line=False, line_number=12, label=None, positions=None, cache_info=None), + Instruction(opname='LOAD_FAST', opcode=85, arg=0, argval='i', argrepr='i', offset=150, start_offset=150, starts_line=True, line_number=13, label=None, positions=None, cache_info=None), + Instruction(opname='LOAD_CONST', opcode=83, arg=5, argval=1, argrepr='1', offset=152, start_offset=152, starts_line=False, line_number=13, label=None, positions=None, cache_info=None), + Instruction(opname='BINARY_OP', opcode=45, arg=23, argval=23, argrepr='-=', offset=154, start_offset=154, starts_line=False, line_number=13, label=None, positions=None, cache_info=[('counter', 1, b'\x00\x00')]), + Instruction(opname='STORE_FAST', opcode=110, arg=0, argval='i', argrepr='i', offset=158, start_offset=158, starts_line=False, line_number=13, label=None, positions=None, cache_info=None), + Instruction(opname='LOAD_FAST', opcode=85, arg=0, argval='i', argrepr='i', offset=160, start_offset=160, starts_line=True, line_number=14, label=None, positions=None, cache_info=None), + Instruction(opname='LOAD_CONST', opcode=83, arg=3, argval=6, argrepr='6', offset=162, start_offset=162, starts_line=False, line_number=14, label=None, positions=None, cache_info=None), + Instruction(opname='COMPARE_OP', opcode=58, arg=148, argval='>', argrepr='bool(>)', offset=164, start_offset=164, starts_line=False, line_number=14, label=None, positions=None, cache_info=[('counter', 1, b'\x00\x00')]), + Instruction(opname='POP_JUMP_IF_FALSE', opcode=97, arg=2, argval=176, argrepr='to L7', offset=168, start_offset=168, starts_line=False, line_number=14, label=None, positions=None, cache_info=[('counter', 1, b'\x00\x00')]), + Instruction(opname='JUMP_BACKWARD', opcode=77, arg=31, argval=114, argrepr='to L5', offset=172, start_offset=172, starts_line=True, line_number=15, label=None, positions=None, cache_info=[('counter', 1, b'\x00\x00')]), + Instruction(opname='LOAD_FAST', opcode=85, arg=0, argval='i', argrepr='i', offset=176, start_offset=176, starts_line=True, line_number=16, label=7, positions=None, cache_info=None), + Instruction(opname='LOAD_CONST', opcode=83, arg=2, argval=4, argrepr='4', offset=178, start_offset=178, starts_line=False, line_number=16, label=None, positions=None, cache_info=None), + Instruction(opname='COMPARE_OP', opcode=58, arg=18, argval='<', argrepr='bool(<)', offset=180, start_offset=180, starts_line=False, line_number=16, label=None, positions=None, cache_info=[('counter', 1, b'\x00\x00')]), + Instruction(opname='POP_JUMP_IF_FALSE', opcode=97, arg=1, argval=190, argrepr='to L8', offset=184, start_offset=184, starts_line=False, line_number=16, label=None, positions=None, cache_info=[('counter', 1, b'\x00\x00')]), + Instruction(opname='JUMP_FORWARD', opcode=79, arg=20, argval=230, argrepr='to L10', offset=188, start_offset=188, starts_line=True, line_number=17, label=None, positions=None, cache_info=None), + Instruction(opname='LOAD_FAST', opcode=85, arg=0, argval='i', argrepr='i', offset=190, start_offset=190, starts_line=True, line_number=11, label=8, positions=None, cache_info=None), + Instruction(opname='TO_BOOL', opcode=40, arg=None, argval=None, argrepr='', offset=192, start_offset=192, starts_line=False, line_number=11, label=None, positions=None, cache_info=[('counter', 1, b'\x00\x00'), ('version', 2, b'\x00\x00\x00\x00')]), + Instruction(opname='POP_JUMP_IF_FALSE', opcode=97, arg=2, argval=208, argrepr='to L9', offset=200, start_offset=200, starts_line=False, line_number=11, label=None, positions=None, cache_info=[('counter', 1, b'\x00\x00')]), + Instruction(opname='JUMP_BACKWARD', opcode=77, arg=40, argval=128, argrepr='to L6', offset=204, start_offset=204, starts_line=False, line_number=11, label=None, positions=None, cache_info=[('counter', 1, b'\x00\x00')]), + Instruction(opname='LOAD_GLOBAL', opcode=91, arg=3, argval='print', argrepr='print + NULL', offset=208, start_offset=208, starts_line=True, line_number=19, label=9, positions=None, cache_info=[('counter', 1, b'\x00\x00'), ('index', 1, b'\x00\x00'), ('module_keys_version', 1, b'\x00\x00'), ('builtin_keys_version', 1, b'\x00\x00')]), + Instruction(opname='LOAD_CONST', opcode=83, arg=6, argval='Who let lolcatz into this test suite?', argrepr="'Who let lolcatz into this test suite?'", offset=218, start_offset=218, starts_line=False, line_number=19, label=None, positions=None, cache_info=None), + Instruction(opname='CALL', opcode=53, arg=1, argval=1, argrepr='', offset=220, start_offset=220, starts_line=False, line_number=19, label=None, positions=None, cache_info=[('counter', 1, b'\x00\x00'), ('func_version', 2, b'\x00\x00\x00\x00')]), + Instruction(opname='POP_TOP', opcode=32, arg=None, argval=None, argrepr='', offset=228, start_offset=228, starts_line=False, line_number=19, label=None, positions=None, cache_info=None), + Instruction(opname='NOP', opcode=30, arg=None, argval=None, argrepr='', offset=230, start_offset=230, starts_line=True, line_number=20, label=10, positions=None, cache_info=None), + Instruction(opname='LOAD_CONST', opcode=83, arg=5, argval=1, argrepr='1', offset=232, start_offset=232, starts_line=True, line_number=21, label=None, positions=None, cache_info=None), + Instruction(opname='LOAD_CONST', opcode=83, arg=7, argval=0, argrepr='0', offset=234, start_offset=234, starts_line=False, line_number=21, label=None, positions=None, cache_info=None), + Instruction(opname='BINARY_OP', opcode=45, arg=11, argval=11, argrepr='/', offset=236, start_offset=236, starts_line=False, line_number=21, label=None, positions=None, cache_info=[('counter', 1, b'\x00\x00')]), + Instruction(opname='POP_TOP', opcode=32, arg=None, argval=None, argrepr='', offset=240, start_offset=240, starts_line=False, line_number=21, label=None, positions=None, cache_info=None), + Instruction(opname='LOAD_FAST', opcode=85, arg=0, argval='i', argrepr='i', offset=242, start_offset=242, starts_line=True, line_number=25, label=None, positions=None, cache_info=None), + Instruction(opname='BEFORE_WITH', opcode=2, arg=None, argval=None, argrepr='', offset=244, start_offset=244, starts_line=False, line_number=25, label=None, positions=None, cache_info=None), + Instruction(opname='STORE_FAST', opcode=110, arg=1, argval='dodgy', argrepr='dodgy', offset=246, start_offset=246, starts_line=False, line_number=25, label=None, positions=None, cache_info=None), + Instruction(opname='LOAD_GLOBAL', opcode=91, arg=3, argval='print', argrepr='print + NULL', offset=248, start_offset=248, starts_line=True, line_number=26, label=None, positions=None, cache_info=[('counter', 1, b'\x00\x00'), ('index', 1, b'\x00\x00'), ('module_keys_version', 1, b'\x00\x00'), ('builtin_keys_version', 1, b'\x00\x00')]), + Instruction(opname='LOAD_CONST', opcode=83, arg=8, argval='Never reach this', argrepr="'Never reach this'", offset=258, start_offset=258, starts_line=False, line_number=26, label=None, positions=None, cache_info=None), + Instruction(opname='CALL', opcode=53, arg=1, argval=1, argrepr='', offset=260, start_offset=260, starts_line=False, line_number=26, label=None, positions=None, cache_info=[('counter', 1, b'\x00\x00'), ('func_version', 2, b'\x00\x00\x00\x00')]), + Instruction(opname='POP_TOP', opcode=32, arg=None, argval=None, argrepr='', offset=268, start_offset=268, starts_line=False, line_number=26, label=None, positions=None, cache_info=None), + Instruction(opname='LOAD_CONST', opcode=83, arg=0, argval=None, argrepr='None', offset=270, start_offset=270, starts_line=True, line_number=25, label=None, positions=None, cache_info=None), + Instruction(opname='LOAD_CONST', opcode=83, arg=0, argval=None, argrepr='None', offset=272, start_offset=272, starts_line=False, line_number=25, label=None, positions=None, cache_info=None), + Instruction(opname='LOAD_CONST', opcode=83, arg=0, argval=None, argrepr='None', offset=274, start_offset=274, starts_line=False, line_number=25, label=None, positions=None, cache_info=None), + Instruction(opname='CALL', opcode=53, arg=2, argval=2, argrepr='', offset=276, start_offset=276, starts_line=False, line_number=25, label=None, positions=None, cache_info=[('counter', 1, b'\x00\x00'), ('func_version', 2, b'\x00\x00\x00\x00')]), + Instruction(opname='POP_TOP', opcode=32, arg=None, argval=None, argrepr='', offset=284, start_offset=284, starts_line=False, line_number=25, label=None, positions=None, cache_info=None), + Instruction(opname='LOAD_GLOBAL', opcode=91, arg=3, argval='print', argrepr='print + NULL', offset=286, start_offset=286, starts_line=True, line_number=28, label=11, positions=None, cache_info=[('counter', 1, b'\x00\x00'), ('index', 1, b'\x00\x00'), ('module_keys_version', 1, b'\x00\x00'), ('builtin_keys_version', 1, b'\x00\x00')]), + Instruction(opname='LOAD_CONST', opcode=83, arg=10, argval="OK, now we're done", argrepr='"OK, now we\'re done"', offset=296, start_offset=296, starts_line=False, line_number=28, label=None, positions=None, cache_info=None), + Instruction(opname='CALL', opcode=53, arg=1, argval=1, argrepr='', offset=298, start_offset=298, starts_line=False, line_number=28, label=None, positions=None, cache_info=[('counter', 1, b'\x00\x00'), ('func_version', 2, b'\x00\x00\x00\x00')]), + Instruction(opname='POP_TOP', opcode=32, arg=None, argval=None, argrepr='', offset=306, start_offset=306, starts_line=False, line_number=28, label=None, positions=None, cache_info=None), + Instruction(opname='RETURN_CONST', opcode=103, arg=0, argval=None, argrepr='None', offset=308, start_offset=308, starts_line=False, line_number=28, label=None, positions=None, cache_info=None), + Instruction(opname='PUSH_EXC_INFO', opcode=33, arg=None, argval=None, argrepr='', offset=310, start_offset=310, starts_line=True, line_number=25, label=None, positions=None, cache_info=None), + Instruction(opname='WITH_EXCEPT_START', opcode=44, arg=None, argval=None, argrepr='', offset=312, start_offset=312, starts_line=False, line_number=25, label=None, positions=None, cache_info=None), + Instruction(opname='TO_BOOL', opcode=40, arg=None, argval=None, argrepr='', offset=314, start_offset=314, starts_line=False, line_number=25, label=None, positions=None, cache_info=[('counter', 1, b'\x00\x00'), ('version', 2, b'\x00\x00\x00\x00')]), + Instruction(opname='POP_JUMP_IF_TRUE', opcode=100, arg=1, argval=328, argrepr='to L12', offset=322, start_offset=322, starts_line=False, line_number=25, label=None, positions=None, cache_info=[('counter', 1, b'\x00\x00')]), + Instruction(opname='RERAISE', opcode=102, arg=2, argval=2, argrepr='', offset=326, start_offset=326, starts_line=False, line_number=25, label=None, positions=None, cache_info=None), + Instruction(opname='POP_TOP', opcode=32, arg=None, argval=None, argrepr='', offset=328, start_offset=328, starts_line=False, line_number=25, label=12, positions=None, cache_info=None), + Instruction(opname='POP_EXCEPT', opcode=31, arg=None, argval=None, argrepr='', offset=330, start_offset=330, starts_line=False, line_number=25, label=None, positions=None, cache_info=None), + Instruction(opname='POP_TOP', opcode=32, arg=None, argval=None, argrepr='', offset=332, start_offset=332, starts_line=False, line_number=25, label=None, positions=None, cache_info=None), + Instruction(opname='POP_TOP', opcode=32, arg=None, argval=None, argrepr='', offset=334, start_offset=334, starts_line=False, line_number=25, label=None, positions=None, cache_info=None), + Instruction(opname='JUMP_BACKWARD_NO_INTERRUPT', opcode=78, arg=26, argval=286, argrepr='to L11', offset=336, start_offset=336, starts_line=False, line_number=25, label=None, positions=None, cache_info=None), + Instruction(opname='COPY', opcode=61, arg=3, argval=3, argrepr='', offset=338, start_offset=338, starts_line=True, line_number=None, label=None, positions=None, cache_info=None), + Instruction(opname='POP_EXCEPT', opcode=31, arg=None, argval=None, argrepr='', offset=340, start_offset=340, starts_line=False, line_number=None, label=None, positions=None, cache_info=None), + Instruction(opname='RERAISE', opcode=102, arg=1, argval=1, argrepr='', offset=342, start_offset=342, starts_line=False, line_number=None, label=None, positions=None, cache_info=None), + Instruction(opname='PUSH_EXC_INFO', opcode=33, arg=None, argval=None, argrepr='', offset=344, start_offset=344, starts_line=False, line_number=None, label=None, positions=None, cache_info=None), + Instruction(opname='LOAD_GLOBAL', opcode=91, arg=4, argval='ZeroDivisionError', argrepr='ZeroDivisionError', offset=346, start_offset=346, starts_line=True, line_number=22, label=None, positions=None, cache_info=[('counter', 1, b'\x00\x00'), ('index', 1, b'\x00\x00'), ('module_keys_version', 1, b'\x00\x00'), ('builtin_keys_version', 1, b'\x00\x00')]), + Instruction(opname='CHECK_EXC_MATCH', opcode=7, arg=None, argval=None, argrepr='', offset=356, start_offset=356, starts_line=False, line_number=22, label=None, positions=None, cache_info=None), + Instruction(opname='POP_JUMP_IF_FALSE', opcode=97, arg=14, argval=390, argrepr='to L13', offset=358, start_offset=358, starts_line=False, line_number=22, label=None, positions=None, cache_info=[('counter', 1, b'\x00\x00')]), + Instruction(opname='POP_TOP', opcode=32, arg=None, argval=None, argrepr='', offset=362, start_offset=362, starts_line=False, line_number=22, label=None, positions=None, cache_info=None), + Instruction(opname='LOAD_GLOBAL', opcode=91, arg=3, argval='print', argrepr='print + NULL', offset=364, start_offset=364, starts_line=True, line_number=23, label=None, positions=None, cache_info=[('counter', 1, b'\x00\x00'), ('index', 1, b'\x00\x00'), ('module_keys_version', 1, b'\x00\x00'), ('builtin_keys_version', 1, b'\x00\x00')]), + Instruction(opname='LOAD_CONST', opcode=83, arg=9, argval='Here we go, here we go, here we go...', argrepr="'Here we go, here we go, here we go...'", offset=374, start_offset=374, starts_line=False, line_number=23, label=None, positions=None, cache_info=None), + Instruction(opname='CALL', opcode=53, arg=1, argval=1, argrepr='', offset=376, start_offset=376, starts_line=False, line_number=23, label=None, positions=None, cache_info=[('counter', 1, b'\x00\x00'), ('func_version', 2, b'\x00\x00\x00\x00')]), + Instruction(opname='POP_TOP', opcode=32, arg=None, argval=None, argrepr='', offset=384, start_offset=384, starts_line=False, line_number=23, label=None, positions=None, cache_info=None), + Instruction(opname='POP_EXCEPT', opcode=31, arg=None, argval=None, argrepr='', offset=386, start_offset=386, starts_line=False, line_number=23, label=None, positions=None, cache_info=None), + Instruction(opname='JUMP_BACKWARD_NO_INTERRUPT', opcode=78, arg=52, argval=286, argrepr='to L11', offset=388, start_offset=388, starts_line=False, line_number=23, label=None, positions=None, cache_info=None), + Instruction(opname='RERAISE', opcode=102, arg=0, argval=0, argrepr='', offset=390, start_offset=390, starts_line=True, line_number=22, label=13, positions=None, cache_info=None), + Instruction(opname='COPY', opcode=61, arg=3, argval=3, argrepr='', offset=392, start_offset=392, starts_line=True, line_number=None, label=None, positions=None, cache_info=None), + Instruction(opname='POP_EXCEPT', opcode=31, arg=None, argval=None, argrepr='', offset=394, start_offset=394, starts_line=False, line_number=None, label=None, positions=None, cache_info=None), + Instruction(opname='RERAISE', opcode=102, arg=1, argval=1, argrepr='', offset=396, start_offset=396, starts_line=False, line_number=None, label=None, positions=None, cache_info=None), + Instruction(opname='PUSH_EXC_INFO', opcode=33, arg=None, argval=None, argrepr='', offset=398, start_offset=398, starts_line=False, line_number=None, label=None, positions=None, cache_info=None), + Instruction(opname='LOAD_GLOBAL', opcode=91, arg=3, argval='print', argrepr='print + NULL', offset=400, start_offset=400, starts_line=True, line_number=28, label=None, positions=None, cache_info=[('counter', 1, b'\x00\x00'), ('index', 1, b'\x00\x00'), ('module_keys_version', 1, b'\x00\x00'), ('builtin_keys_version', 1, b'\x00\x00')]), + Instruction(opname='LOAD_CONST', opcode=83, arg=10, argval="OK, now we're done", argrepr='"OK, now we\'re done"', offset=410, start_offset=410, starts_line=False, line_number=28, label=None, positions=None, cache_info=None), + Instruction(opname='CALL', opcode=53, arg=1, argval=1, argrepr='', offset=412, start_offset=412, starts_line=False, line_number=28, label=None, positions=None, cache_info=[('counter', 1, b'\x00\x00'), ('func_version', 2, b'\x00\x00\x00\x00')]), + Instruction(opname='POP_TOP', opcode=32, arg=None, argval=None, argrepr='', offset=420, start_offset=420, starts_line=False, line_number=28, label=None, positions=None, cache_info=None), + Instruction(opname='RERAISE', opcode=102, arg=0, argval=0, argrepr='', offset=422, start_offset=422, starts_line=False, line_number=28, label=None, positions=None, cache_info=None), + Instruction(opname='COPY', opcode=61, arg=3, argval=3, argrepr='', offset=424, start_offset=424, starts_line=True, line_number=None, label=None, positions=None, cache_info=None), + Instruction(opname='POP_EXCEPT', opcode=31, arg=None, argval=None, argrepr='', offset=426, start_offset=426, starts_line=False, line_number=None, label=None, positions=None, cache_info=None), + Instruction(opname='RERAISE', opcode=102, arg=1, argval=1, argrepr='', offset=428, start_offset=428, starts_line=False, line_number=None, label=None, positions=None, cache_info=None), ] # One last piece of inspect fodder to check the default line number handling diff --git a/Lib/test/test_doctest/__init__.py b/Lib/test/test_doctest/__init__.py new file mode 100644 index 00000000000000..4b16ecc31156a5 --- /dev/null +++ b/Lib/test/test_doctest/__init__.py @@ -0,0 +1,5 @@ +import os +from test.support import load_package_tests + +def load_tests(*args): + return load_package_tests(os.path.dirname(__file__), *args) diff --git a/Lib/test/doctest_aliases.py b/Lib/test/test_doctest/doctest_aliases.py similarity index 100% rename from Lib/test/doctest_aliases.py rename to Lib/test/test_doctest/doctest_aliases.py diff --git a/Lib/test/doctest_lineno.py b/Lib/test/test_doctest/doctest_lineno.py similarity index 100% rename from Lib/test/doctest_lineno.py rename to Lib/test/test_doctest/doctest_lineno.py diff --git a/Lib/test/sample_doctest.py b/Lib/test/test_doctest/sample_doctest.py similarity index 91% rename from Lib/test/sample_doctest.py rename to Lib/test/test_doctest/sample_doctest.py index 89eb5cb7cf1d97..049f737a0a44ac 100644 --- a/Lib/test/sample_doctest.py +++ b/Lib/test/test_doctest/sample_doctest.py @@ -32,8 +32,8 @@ def bar(): def test_silly_setup(): """ - >>> import test.test_doctest - >>> test.test_doctest.sillySetup + >>> import test.test_doctest.test_doctest + >>> test.test_doctest.test_doctest.sillySetup True """ diff --git a/Lib/test/sample_doctest_no_docstrings.py b/Lib/test/test_doctest/sample_doctest_no_docstrings.py similarity index 100% rename from Lib/test/sample_doctest_no_docstrings.py rename to Lib/test/test_doctest/sample_doctest_no_docstrings.py diff --git a/Lib/test/sample_doctest_no_doctests.py b/Lib/test/test_doctest/sample_doctest_no_doctests.py similarity index 100% rename from Lib/test/sample_doctest_no_doctests.py rename to Lib/test/test_doctest/sample_doctest_no_doctests.py diff --git a/Lib/test/test_doctest.py b/Lib/test/test_doctest/test_doctest.py similarity index 94% rename from Lib/test/test_doctest.py rename to Lib/test/test_doctest/test_doctest.py index 46a51007f9644d..7015255db1f7f0 100644 --- a/Lib/test/test_doctest.py +++ b/Lib/test/test_doctest/test_doctest.py @@ -4,6 +4,7 @@ from test import support from test.support import import_helper +from test.support.pty_helper import FakeInput # used in doctests import doctest import functools import os @@ -77,6 +78,15 @@ def get(self): """ return self.val + def setter(self, val): + """ + >>> s = SampleClass(-5) + >>> s.setter(1) + >>> print(s.val) + 1 + """ + self.val = val + def a_staticmethod(v): """ >>> print(SampleClass.a_staticmethod(10)) @@ -95,7 +105,7 @@ def a_classmethod(cls, v): return v+2 a_classmethod = classmethod(a_classmethod) - a_property = property(get, doc=""" + a_property = property(get, setter, doc=""" >>> print(SampleClass(22).a_property) 22 """) @@ -156,25 +166,6 @@ def get(self): """ return self.val -###################################################################### -## Fake stdin (for testing interactive debugging) -###################################################################### - -class _FakeInput: - """ - A fake input stream for pdb's interactive debugger. Whenever a - line is read, print it (to simulate the user typing it), and then - return it. The set of lines to return is specified in the - constructor; they should not have trailing newlines. - """ - def __init__(self, lines): - self.lines = lines - - def readline(self): - line = self.lines.pop(0) - print(line) - return line+'\n' - ###################################################################### ## Test Cases ###################################################################### @@ -468,14 +459,14 @@ def basics(): r""" We'll simulate a __file__ attr that ends in pyc: - >>> import test.test_doctest - >>> old = test.test_doctest.__file__ - >>> test.test_doctest.__file__ = 'test_doctest.pyc' + >>> from test.test_doctest import test_doctest + >>> old = test_doctest.__file__ + >>> test_doctest.__file__ = 'test_doctest.pyc' >>> tests = finder.find(sample_func) >>> print(tests) # doctest: +ELLIPSIS - [] + [] The exact name depends on how test_doctest was invoked, so allow for leading path components. @@ -483,7 +474,7 @@ def basics(): r""" >>> tests[0].filename # doctest: +ELLIPSIS '...test_doctest.py' - >>> test.test_doctest.__file__ = old + >>> test_doctest.__file__ = old >>> e = tests[0].examples[0] @@ -537,6 +528,7 @@ def basics(): r""" 1 SampleClass.a_staticmethod 1 SampleClass.double 1 SampleClass.get + 3 SampleClass.setter New-style classes are also supported: @@ -576,10 +568,10 @@ def basics(): r""" ... 'c': triple}}) >>> finder = doctest.DocTestFinder() - >>> # Use module=test.test_doctest, to prevent doctest from + >>> # Use module=test_doctest, to prevent doctest from >>> # ignoring the objects since they weren't defined in m. - >>> import test.test_doctest - >>> tests = finder.find(m, module=test.test_doctest) + >>> from test.test_doctest import test_doctest + >>> tests = finder.find(m, module=test_doctest) >>> for t in tests: ... print('%2s %s' % (len(t.examples), t.name)) 1 some_module @@ -593,23 +585,38 @@ def basics(): r""" 1 some_module.SampleClass.a_staticmethod 1 some_module.SampleClass.double 1 some_module.SampleClass.get + 3 some_module.SampleClass.setter 1 some_module.__test__.c 2 some_module.__test__.d 1 some_module.sample_func +However, doctest will ignore imported objects from other modules +(without proper `module=`): + + >>> import types + >>> m = types.ModuleType('poluted_namespace') + >>> m.__dict__.update({ + ... 'sample_func': sample_func, + ... 'SampleClass': SampleClass, + ... }) + + >>> finder = doctest.DocTestFinder() + >>> finder.find(m) + [] + Duplicate Removal ~~~~~~~~~~~~~~~~~ If a single object is listed twice (under different names), then tests will only be generated for it once: - >>> from test import doctest_aliases + >>> from test.test_doctest import doctest_aliases >>> assert doctest_aliases.TwoNames.f >>> assert doctest_aliases.TwoNames.g >>> tests = excl_empty_finder.find(doctest_aliases) >>> print(len(tests)) 2 >>> print(tests[0].name) - test.doctest_aliases.TwoNames + test.test_doctest.doctest_aliases.TwoNames TwoNames.f and TwoNames.g are bound to the same object. We can't guess which will be found in doctest's traversal of @@ -635,6 +642,7 @@ def basics(): r""" 1 SampleClass.a_staticmethod 1 SampleClass.double 1 SampleClass.get + 3 SampleClass.setter By default, that excluded objects with no doctests. exclude_empty=False tells it to include (empty) tests for objects with no doctests. This feature @@ -656,28 +664,29 @@ def basics(): r""" 1 SampleClass.a_staticmethod 1 SampleClass.double 1 SampleClass.get + 3 SampleClass.setter When used with `exclude_empty=False` we are also interested in line numbers of doctests that are empty. It used to be broken for quite some time until `bpo-28249`. - >>> from test import doctest_lineno + >>> from test.test_doctest import doctest_lineno >>> tests = doctest.DocTestFinder(exclude_empty=False).find(doctest_lineno) >>> for t in tests: ... print('%5s %s' % (t.lineno, t.name)) - None test.doctest_lineno - 22 test.doctest_lineno.ClassWithDocstring - 30 test.doctest_lineno.ClassWithDoctest - None test.doctest_lineno.ClassWithoutDocstring - None test.doctest_lineno.MethodWrapper - 53 test.doctest_lineno.MethodWrapper.classmethod_with_doctest - 39 test.doctest_lineno.MethodWrapper.method_with_docstring - 45 test.doctest_lineno.MethodWrapper.method_with_doctest - None test.doctest_lineno.MethodWrapper.method_without_docstring - 61 test.doctest_lineno.MethodWrapper.property_with_doctest - 4 test.doctest_lineno.func_with_docstring - 12 test.doctest_lineno.func_with_doctest - None test.doctest_lineno.func_without_docstring + None test.test_doctest.doctest_lineno + 22 test.test_doctest.doctest_lineno.ClassWithDocstring + 30 test.test_doctest.doctest_lineno.ClassWithDoctest + None test.test_doctest.doctest_lineno.ClassWithoutDocstring + None test.test_doctest.doctest_lineno.MethodWrapper + 53 test.test_doctest.doctest_lineno.MethodWrapper.classmethod_with_doctest + 39 test.test_doctest.doctest_lineno.MethodWrapper.method_with_docstring + 45 test.test_doctest.doctest_lineno.MethodWrapper.method_with_doctest + None test.test_doctest.doctest_lineno.MethodWrapper.method_without_docstring + 61 test.test_doctest.doctest_lineno.MethodWrapper.property_with_doctest + 4 test.test_doctest.doctest_lineno.func_with_docstring + 12 test.test_doctest.doctest_lineno.func_with_doctest + None test.test_doctest.doctest_lineno.func_without_docstring Turning off Recursion ~~~~~~~~~~~~~~~~~~~~~ @@ -1924,9 +1933,9 @@ def test_testsource(): r""" example code is converted to regular Python code. The surrounding words and expected output are converted to comments: - >>> import test.test_doctest - >>> name = 'test.test_doctest.sample_func' - >>> print(doctest.testsource(test.test_doctest, name)) + >>> from test.test_doctest import test_doctest + >>> name = 'test.test_doctest.test_doctest.sample_func' + >>> print(doctest.testsource(test_doctest, name)) # Blah blah # print(sample_func(22)) @@ -1936,8 +1945,8 @@ def test_testsource(): r""" # Yee ha! - >>> name = 'test.test_doctest.SampleNewStyleClass' - >>> print(doctest.testsource(test.test_doctest, name)) + >>> name = 'test.test_doctest.test_doctest.SampleNewStyleClass' + >>> print(doctest.testsource(test_doctest, name)) print('1\n2\n3') # Expected: ## 1 @@ -1945,8 +1954,8 @@ def test_testsource(): r""" ## 3 - >>> name = 'test.test_doctest.SampleClass.a_classmethod' - >>> print(doctest.testsource(test.test_doctest, name)) + >>> name = 'test.test_doctest.test_doctest.SampleClass.a_classmethod' + >>> print(doctest.testsource(test_doctest, name)) print(SampleClass.a_classmethod(10)) # Expected: ## 12 @@ -1969,7 +1978,7 @@ def test_debug(): r""" Create some fake stdin input, to feed to the debugger: >>> real_stdin = sys.stdin - >>> sys.stdin = _FakeInput(['next', 'print(x)', 'continue']) + >>> sys.stdin = FakeInput(['next', 'print(x)', 'continue']) Run the debugger on the docstring, and then restore sys.stdin. @@ -2012,7 +2021,7 @@ def test_pdb_set_trace(): captures our debugger input: >>> real_stdin = sys.stdin - >>> sys.stdin = _FakeInput([ + >>> sys.stdin = FakeInput([ ... 'print(x)', # print data defined by the example ... 'continue', # stop debugging ... '']) @@ -2039,7 +2048,7 @@ def test_pdb_set_trace(): ... ''' >>> test = parser.get_doctest(doc, globals(), "foo-bar@baz", "foo-bar@baz.py", 0) >>> real_stdin = sys.stdin - >>> sys.stdin = _FakeInput([ + >>> sys.stdin = FakeInput([ ... 'print(y)', # print data defined in the function ... 'up', # out of function ... 'print(x)', # print data defined by the example @@ -2051,7 +2060,7 @@ def test_pdb_set_trace(): ... finally: ... sys.stdin = real_stdin --Return-- - > (3)calls_set_trace()->None + > (3)calls_set_trace()->None -> import pdb; pdb.set_trace() (Pdb) print(y) 2 @@ -2076,7 +2085,7 @@ def test_pdb_set_trace(): ... ''' >>> test = parser.get_doctest(doc, globals(), "foo-bar@baz", "foo-bar@baz.py", 0) >>> real_stdin = sys.stdin - >>> sys.stdin = _FakeInput([ + >>> sys.stdin = FakeInput([ ... 'list', # list source from example 2 ... 'next', # return from g() ... 'list', # list source from example 1 @@ -2148,7 +2157,7 @@ def test_pdb_set_trace_nested(): >>> runner = doctest.DocTestRunner(verbose=False) >>> test = parser.get_doctest(doc, globals(), "foo-bar@baz", "foo-bar@baz.py", 0) >>> real_stdin = sys.stdin - >>> sys.stdin = _FakeInput([ + >>> sys.stdin = FakeInput([ ... 'print(y)', # print data defined in the function ... 'step', 'step', 'step', 'step', 'step', 'step', 'print(z)', ... 'up', 'print(x)', @@ -2162,39 +2171,39 @@ def test_pdb_set_trace_nested(): ... finally: ... sys.stdin = real_stdin ... # doctest: +REPORT_NDIFF - > (5)calls_set_trace() + > (5)calls_set_trace() -> self.f1() (Pdb) print(y) 1 (Pdb) step --Call-- - > (7)f1() + > (7)f1() -> def f1(self): (Pdb) step - > (8)f1() + > (8)f1() -> x = 1 (Pdb) step - > (9)f1() + > (9)f1() -> self.f2() (Pdb) step --Call-- - > (11)f2() + > (11)f2() -> def f2(self): (Pdb) step - > (12)f2() + > (12)f2() -> z = 1 (Pdb) step - > (13)f2() + > (13)f2() -> z = 2 (Pdb) print(z) 1 (Pdb) up - > (9)f1() + > (9)f1() -> self.f2() (Pdb) print(x) 1 (Pdb) up - > (5)calls_set_trace() + > (5)calls_set_trace() -> self.f1() (Pdb) print(y) 1 @@ -2214,39 +2223,39 @@ def test_DocTestSuite(): by passing a module object: >>> import unittest - >>> import test.sample_doctest - >>> suite = doctest.DocTestSuite(test.sample_doctest) + >>> import test.test_doctest.sample_doctest + >>> suite = doctest.DocTestSuite(test.test_doctest.sample_doctest) >>> suite.run(unittest.TestResult()) We can also supply the module by name: - >>> suite = doctest.DocTestSuite('test.sample_doctest') + >>> suite = doctest.DocTestSuite('test.test_doctest.sample_doctest') >>> suite.run(unittest.TestResult()) The module need not contain any doctest examples: - >>> suite = doctest.DocTestSuite('test.sample_doctest_no_doctests') + >>> suite = doctest.DocTestSuite('test.test_doctest.sample_doctest_no_doctests') >>> suite.run(unittest.TestResult()) The module need not contain any docstrings either: - >>> suite = doctest.DocTestSuite('test.sample_doctest_no_docstrings') + >>> suite = doctest.DocTestSuite('test.test_doctest.sample_doctest_no_docstrings') >>> suite.run(unittest.TestResult()) We can use the current module: - >>> suite = test.sample_doctest.test_suite() + >>> suite = test.test_doctest.sample_doctest.test_suite() >>> suite.run(unittest.TestResult()) We can also provide a DocTestFinder: >>> finder = doctest.DocTestFinder() - >>> suite = doctest.DocTestSuite('test.sample_doctest', + >>> suite = doctest.DocTestSuite('test.test_doctest.sample_doctest', ... test_finder=finder) >>> suite.run(unittest.TestResult()) @@ -2254,7 +2263,7 @@ def test_DocTestSuite(): The DocTestFinder need not return any tests: >>> finder = doctest.DocTestFinder() - >>> suite = doctest.DocTestSuite('test.sample_doctest_no_docstrings', + >>> suite = doctest.DocTestSuite('test.test_doctest.sample_doctest_no_docstrings', ... test_finder=finder) >>> suite.run(unittest.TestResult()) @@ -2263,14 +2272,14 @@ def test_DocTestSuite(): used instead of the module globals. Here we'll pass an empty globals, triggering an extra error: - >>> suite = doctest.DocTestSuite('test.sample_doctest', globs={}) + >>> suite = doctest.DocTestSuite('test.test_doctest.sample_doctest', globs={}) >>> suite.run(unittest.TestResult()) Alternatively, we can provide extra globals. Here we'll make an error go away by providing an extra global variable: - >>> suite = doctest.DocTestSuite('test.sample_doctest', + >>> suite = doctest.DocTestSuite('test.test_doctest.sample_doctest', ... extraglobs={'y': 1}) >>> suite.run(unittest.TestResult()) @@ -2278,7 +2287,7 @@ def test_DocTestSuite(): You can pass option flags. Here we'll cause an extra error by disabling the blank-line feature: - >>> suite = doctest.DocTestSuite('test.sample_doctest', + >>> suite = doctest.DocTestSuite('test.test_doctest.sample_doctest', ... optionflags=doctest.DONT_ACCEPT_BLANKLINE) >>> suite.run(unittest.TestResult()) @@ -2286,27 +2295,27 @@ def test_DocTestSuite(): You can supply setUp and tearDown functions: >>> def setUp(t): - ... import test.test_doctest - ... test.test_doctest.sillySetup = True + ... from test.test_doctest import test_doctest + ... test_doctest.sillySetup = True >>> def tearDown(t): - ... import test.test_doctest - ... del test.test_doctest.sillySetup + ... from test.test_doctest import test_doctest + ... del test_doctest.sillySetup Here, we installed a silly variable that the test expects: - >>> suite = doctest.DocTestSuite('test.sample_doctest', + >>> suite = doctest.DocTestSuite('test.test_doctest.sample_doctest', ... setUp=setUp, tearDown=tearDown) >>> suite.run(unittest.TestResult()) But the tearDown restores sanity: - >>> import test.test_doctest - >>> test.test_doctest.sillySetup + >>> from test.test_doctest import test_doctest + >>> test_doctest.sillySetup Traceback (most recent call last): ... - AttributeError: module 'test.test_doctest' has no attribute 'sillySetup' + AttributeError: module 'test.test_doctest.test_doctest' has no attribute 'sillySetup' The setUp and tearDown functions are passed test objects. Here we'll use the setUp function to supply the missing variable y: @@ -2314,7 +2323,7 @@ def test_DocTestSuite(): >>> def setUp(test): ... test.globs['y'] = 1 - >>> suite = doctest.DocTestSuite('test.sample_doctest', setUp=setUp) + >>> suite = doctest.DocTestSuite('test.test_doctest.sample_doctest', setUp=setUp) >>> suite.run(unittest.TestResult()) @@ -2345,7 +2354,7 @@ def test_DocFileSuite(): >>> suite = doctest.DocFileSuite('test_doctest.txt', ... 'test_doctest2.txt', ... 'test_doctest4.txt', - ... package='test') + ... package='test.test_doctest') >>> suite.run(unittest.TestResult()) @@ -2361,7 +2370,7 @@ def test_DocFileSuite(): ... suite = doctest.DocFileSuite('test_doctest.txt', ... 'test_doctest2.txt', ... 'test_doctest4.txt', - ... package='test') + ... package='test.test_doctest') ... suite.run(unittest.TestResult()) ... finally: ... if added_loader: @@ -2371,16 +2380,17 @@ def test_DocFileSuite(): '/' should be used as a path separator. It will be converted to a native separator at run time: - >>> suite = doctest.DocFileSuite('../test/test_doctest.txt') + >>> suite = doctest.DocFileSuite('../test_doctest/test_doctest.txt') >>> suite.run(unittest.TestResult()) If DocFileSuite is used from an interactive session, then files are resolved relative to the directory of sys.argv[0]: - >>> import types, os.path, test.test_doctest + >>> import types, os.path + >>> from test.test_doctest import test_doctest >>> save_argv = sys.argv - >>> sys.argv = [test.test_doctest.__file__] + >>> sys.argv = [test_doctest.__file__] >>> suite = doctest.DocFileSuite('test_doctest.txt', ... package=types.ModuleType('__main__')) >>> sys.argv = save_argv @@ -2390,7 +2400,7 @@ def test_DocFileSuite(): working directory): >>> # Get the absolute path of the test package. - >>> test_doctest_path = os.path.abspath(test.test_doctest.__file__) + >>> test_doctest_path = os.path.abspath(test_doctest.__file__) >>> test_pkg_path = os.path.split(test_doctest_path)[0] >>> # Use it to find the absolute path of test_doctest.txt. @@ -2430,12 +2440,12 @@ def test_DocFileSuite(): And, you can provide setUp and tearDown functions: >>> def setUp(t): - ... import test.test_doctest - ... test.test_doctest.sillySetup = True + ... from test.test_doctest import test_doctest + ... test_doctest.sillySetup = True >>> def tearDown(t): - ... import test.test_doctest - ... del test.test_doctest.sillySetup + ... from test.test_doctest import test_doctest + ... del test_doctest.sillySetup Here, we installed a silly variable that the test expects: @@ -2448,11 +2458,11 @@ def test_DocFileSuite(): But the tearDown restores sanity: - >>> import test.test_doctest - >>> test.test_doctest.sillySetup + >>> from test.test_doctest import test_doctest + >>> test_doctest.sillySetup Traceback (most recent call last): ... - AttributeError: module 'test.test_doctest' has no attribute 'sillySetup' + AttributeError: module 'test.test_doctest.test_doctest' has no attribute 'sillySetup' The setUp and tearDown functions are passed test objects. Here, we'll use a setUp function to set the favorite color in @@ -3212,8 +3222,8 @@ def test_run_doctestsuite_multiple_times(): http://bugs.python.org/issue9736 >>> import unittest - >>> import test.sample_doctest - >>> suite = doctest.DocTestSuite(test.sample_doctest) + >>> import test.test_doctest.sample_doctest + >>> suite = doctest.DocTestSuite(test.test_doctest.sample_doctest) >>> suite.run(unittest.TestResult()) >>> suite.run(unittest.TestResult()) @@ -3390,4 +3400,4 @@ def load_tests(loader, tests, pattern): if __name__ == '__main__': - unittest.main(module='test.test_doctest') + unittest.main(module='test.test_doctest.test_doctest') diff --git a/Lib/test/test_doctest.txt b/Lib/test/test_doctest/test_doctest.txt similarity index 100% rename from Lib/test/test_doctest.txt rename to Lib/test/test_doctest/test_doctest.txt diff --git a/Lib/test/test_doctest2.py b/Lib/test/test_doctest/test_doctest2.py similarity index 100% rename from Lib/test/test_doctest2.py rename to Lib/test/test_doctest/test_doctest2.py diff --git a/Lib/test/test_doctest2.txt b/Lib/test/test_doctest/test_doctest2.txt similarity index 77% rename from Lib/test/test_doctest2.txt rename to Lib/test/test_doctest/test_doctest2.txt index 2e14856c27d8b3..76dab94a9c0470 100644 --- a/Lib/test/test_doctest2.txt +++ b/Lib/test/test_doctest/test_doctest2.txt @@ -2,8 +2,8 @@ This is a sample doctest in a text file. In this example, we'll rely on some silly setup: - >>> import test.test_doctest - >>> test.test_doctest.sillySetup + >>> import test.test_doctest.test_doctest + >>> test.test_doctest.test_doctest.sillySetup True This test also has some (random) encoded (utf-8) unicode text: diff --git a/Lib/test/test_doctest3.txt b/Lib/test/test_doctest/test_doctest3.txt similarity index 100% rename from Lib/test/test_doctest3.txt rename to Lib/test/test_doctest/test_doctest3.txt diff --git a/Lib/test/test_doctest4.txt b/Lib/test/test_doctest/test_doctest4.txt similarity index 100% rename from Lib/test/test_doctest4.txt rename to Lib/test/test_doctest/test_doctest4.txt diff --git a/Lib/test/test_dynamic.py b/Lib/test/test_dynamic.py index 0aa3be6a1bde6a..3928bbab4423c2 100644 --- a/Lib/test/test_dynamic.py +++ b/Lib/test/test_dynamic.py @@ -4,7 +4,7 @@ import sys import unittest -from test.support import swap_item, swap_attr +from test.support import is_wasi, Py_DEBUG, swap_item, swap_attr class RebindBuiltinsTests(unittest.TestCase): @@ -134,6 +134,7 @@ def test_eval_gives_lambda_custom_globals(self): self.assertEqual(foo(), 7) + @unittest.skipIf(is_wasi and Py_DEBUG, "stack depth too shallow in pydebug WASI") def test_load_global_specialization_failure_keeps_oparg(self): # https://github.com/python/cpython/issues/91625 class MyGlobals(dict): diff --git a/Lib/test/test_email/test__header_value_parser.py b/Lib/test/test_email/test__header_value_parser.py index 854f2ff009c618..bdb0e55f21069f 100644 --- a/Lib/test/test_email/test__header_value_parser.py +++ b/Lib/test/test_email/test__header_value_parser.py @@ -2915,6 +2915,45 @@ def test_ews_combined_before_wrap(self): "mich. And that's\n" " all I'm sayin.\n") + def test_unicode_after_unknown_not_combined(self): + self._test(parser.get_unstructured("=?unknown-8bit?q?=A4?=\xa4"), + "=?unknown-8bit?q?=A4?==?utf-8?q?=C2=A4?=\n") + prefix = "0123456789 "*5 + self._test(parser.get_unstructured(prefix + "=?unknown-8bit?q?=A4?=\xa4"), + prefix + "=?unknown-8bit?q?=A4?=\n =?utf-8?q?=C2=A4?=\n") + + def test_ascii_after_unknown_not_combined(self): + self._test(parser.get_unstructured("=?unknown-8bit?q?=A4?=abc"), + "=?unknown-8bit?q?=A4?=abc\n") + prefix = "0123456789 "*5 + self._test(parser.get_unstructured(prefix + "=?unknown-8bit?q?=A4?=abc"), + prefix + "=?unknown-8bit?q?=A4?=\n =?utf-8?q?abc?=\n") + + def test_unknown_after_unicode_not_combined(self): + self._test(parser.get_unstructured("\xa4" + "=?unknown-8bit?q?=A4?="), + "=?utf-8?q?=C2=A4?==?unknown-8bit?q?=A4?=\n") + prefix = "0123456789 "*5 + self._test(parser.get_unstructured(prefix + "\xa4=?unknown-8bit?q?=A4?="), + prefix + "=?utf-8?q?=C2=A4?=\n =?unknown-8bit?q?=A4?=\n") + + def test_unknown_after_ascii_not_combined(self): + self._test(parser.get_unstructured("abc" + "=?unknown-8bit?q?=A4?="), + "abc=?unknown-8bit?q?=A4?=\n") + prefix = "0123456789 "*5 + self._test(parser.get_unstructured(prefix + "abcd=?unknown-8bit?q?=A4?="), + prefix + "abcd\n =?unknown-8bit?q?=A4?=\n") + + def test_unknown_after_unknown(self): + self._test(parser.get_unstructured("=?unknown-8bit?q?=C2?=" + "=?unknown-8bit?q?=A4?="), + "=?unknown-8bit?q?=C2=A4?=\n") + prefix = "0123456789 "*5 + self._test(parser.get_unstructured(prefix + "=?unknown-8bit?q?=C2?=" + "=?unknown-8bit?q?=A4?="), + prefix + "=?unknown-8bit?q?=C2?=\n =?unknown-8bit?q?=A4?=\n") + # XXX Need test of an encoded word so long that it needs to be wrapped def test_simple_address(self): diff --git a/Lib/test/test_email/test_policy.py b/Lib/test/test_email/test_policy.py index e87c275549406d..c6b9c80efe1b54 100644 --- a/Lib/test/test_email/test_policy.py +++ b/Lib/test/test_email/test_policy.py @@ -135,6 +135,23 @@ def test_policy_addition(self): for attr, value in expected.items(): self.assertEqual(getattr(added, attr), value) + def test_fold_utf8(self): + expected_ascii = 'Subject: =?utf-8?q?=C3=A1?=\n' + expected_utf8 = 'Subject: á\n' + + msg = email.message.EmailMessage() + s = 'á' + msg['Subject'] = s + + p_ascii = email.policy.default.clone() + p_utf8 = email.policy.default.clone(utf8=True) + + self.assertEqual(p_ascii.fold('Subject', msg['Subject']), expected_ascii) + self.assertEqual(p_utf8.fold('Subject', msg['Subject']), expected_utf8) + + self.assertEqual(p_ascii.fold('Subject', s), expected_ascii) + self.assertEqual(p_utf8.fold('Subject', s), expected_utf8) + def test_fold_zero_max_line_length(self): expected = 'Subject: =?utf-8?q?=C3=A1?=\n' diff --git a/Lib/test/test_email/test_utils.py b/Lib/test/test_email/test_utils.py index c9d973df0a2192..d04b3909efa643 100644 --- a/Lib/test/test_email/test_utils.py +++ b/Lib/test/test_email/test_utils.py @@ -143,12 +143,12 @@ def test_localtime_epoch_notz_daylight_false(self): t2 = utils.localtime(t0.replace(tzinfo=None)) self.assertEqual(t1, t2) - @unittest.skipUnless("Europe/Kyiv" in zoneinfo.available_timezones(), - "Can't find a Kyiv timezone database") @test.support.run_with_tz('Europe/Kyiv') def test_variable_tzname(self): t0 = datetime.datetime(1984, 1, 1, tzinfo=datetime.timezone.utc) t1 = utils.localtime(t0) + if t1.tzname() in ('Europe', 'UTC'): + self.skipTest("Can't find a Kyiv timezone database") self.assertEqual(t1.tzname(), 'MSK') t0 = datetime.datetime(1994, 1, 1, tzinfo=datetime.timezone.utc) t1 = utils.localtime(t0) diff --git a/Lib/test/test_ensurepip.py b/Lib/test/test_ensurepip.py index 69ab2a4feaa938..a4b36a90d8815e 100644 --- a/Lib/test/test_ensurepip.py +++ b/Lib/test/test_ensurepip.py @@ -6,6 +6,8 @@ import test.support import unittest import unittest.mock +from importlib.resources.abc import Traversable +from pathlib import Path import ensurepip import ensurepip._uninstall @@ -20,41 +22,35 @@ def test_version(self): # Test version() with tempfile.TemporaryDirectory() as tmpdir: self.touch(tmpdir, "pip-1.2.3b1-py2.py3-none-any.whl") - with (unittest.mock.patch.object(ensurepip, '_PACKAGES', None), - unittest.mock.patch.object(ensurepip, '_WHEEL_PKG_DIR', tmpdir)): + with unittest.mock.patch.object(ensurepip, '_WHEEL_PKG_DIR', Path(tmpdir)): self.assertEqual(ensurepip.version(), '1.2.3b1') - def test_get_packages_no_dir(self): - # Test _get_packages() without a wheel package directory - with (unittest.mock.patch.object(ensurepip, '_PACKAGES', None), - unittest.mock.patch.object(ensurepip, '_WHEEL_PKG_DIR', None)): - packages = ensurepip._get_packages() - - # when bundled wheel packages are used, we get _PIP_VERSION + def test_version_no_dir(self): + # Test version() without a wheel package directory + with unittest.mock.patch.object(ensurepip, '_WHEEL_PKG_DIR', None): + # when the bundled pip wheel is used, we get _PIP_VERSION self.assertEqual(ensurepip._PIP_VERSION, ensurepip.version()) - # use bundled wheel packages - self.assertIsNotNone(packages['pip'].wheel_name) + def test_selected_wheel_path_no_dir(self): + pip_filename = f'pip-{ensurepip._PIP_VERSION}-py3-none-any.whl' + with unittest.mock.patch.object(ensurepip, '_WHEEL_PKG_DIR', None): + with ensurepip._get_pip_whl_path_ctx() as bundled_wheel_path: + self.assertEqual(pip_filename, bundled_wheel_path.name) - def test_get_packages_with_dir(self): - # Test _get_packages() with a wheel package directory + def test_selected_wheel_path_with_dir(self): + # Test _get_pip_whl_path_ctx() with a wheel package directory pip_filename = "pip-20.2.2-py2.py3-none-any.whl" with tempfile.TemporaryDirectory() as tmpdir: self.touch(tmpdir, pip_filename) - # not used, make sure that it's ignored + # not used, make sure that they're ignored + self.touch(tmpdir, "pip-1.2.3-py2.py3-none-any.whl") self.touch(tmpdir, "wheel-0.34.2-py2.py3-none-any.whl") + self.touch(tmpdir, "pip-script.py") - with (unittest.mock.patch.object(ensurepip, '_PACKAGES', None), - unittest.mock.patch.object(ensurepip, '_WHEEL_PKG_DIR', tmpdir)): - packages = ensurepip._get_packages() - - self.assertEqual(packages['pip'].version, '20.2.2') - self.assertEqual(packages['pip'].wheel_path, - os.path.join(tmpdir, pip_filename)) - - # wheel package is ignored - self.assertEqual(sorted(packages), ['pip']) + with unittest.mock.patch.object(ensurepip, '_WHEEL_PKG_DIR', Path(tmpdir)): + with ensurepip._get_pip_whl_path_ctx() as bundled_wheel_path: + self.assertEqual(pip_filename, bundled_wheel_path.name) class EnsurepipMixin: @@ -69,7 +65,7 @@ def setUp(self): real_devnull = os.devnull os_patch = unittest.mock.patch("ensurepip.os") patched_os = os_patch.start() - # But expose os.listdir() used by _find_packages() + # But expose os.listdir() used by _find_wheel_pkg_dir_pip() patched_os.listdir = os.listdir self.addCleanup(os_patch.stop) patched_os.devnull = real_devnull diff --git a/Lib/test/test_enum.py b/Lib/test/test_enum.py index f99d4ca204b5a7..39c1ae0ad5a078 100644 --- a/Lib/test/test_enum.py +++ b/Lib/test/test_enum.py @@ -2344,6 +2344,40 @@ class SomeTuple(tuple, Enum): globals()['SomeTuple'] = SomeTuple test_pickle_dump_load(self.assertIs, SomeTuple.first) + def test_tuple_subclass_with_auto_1(self): + from collections import namedtuple + T = namedtuple('T', 'index desc') + class SomeEnum(T, Enum): + __qualname__ = 'SomeEnum' # needed for pickle protocol 4 + first = auto(), 'for the money' + second = auto(), 'for the show' + third = auto(), 'for the music' + self.assertIs(type(SomeEnum.first), SomeEnum) + self.assertEqual(SomeEnum.third.value, (3, 'for the music')) + self.assertIsInstance(SomeEnum.third.value, T) + self.assertEqual(SomeEnum.first.index, 1) + self.assertEqual(SomeEnum.second.desc, 'for the show') + globals()['SomeEnum'] = SomeEnum + globals()['T'] = T + test_pickle_dump_load(self.assertIs, SomeEnum.first) + + def test_tuple_subclass_with_auto_2(self): + from collections import namedtuple + T = namedtuple('T', 'index desc') + class SomeEnum(Enum): + __qualname__ = 'SomeEnum' # needed for pickle protocol 4 + first = T(auto(), 'for the money') + second = T(auto(), 'for the show') + third = T(auto(), 'for the music') + self.assertIs(type(SomeEnum.first), SomeEnum) + self.assertEqual(SomeEnum.third.value, (3, 'for the music')) + self.assertIsInstance(SomeEnum.third.value, T) + self.assertEqual(SomeEnum.first.value.index, 1) + self.assertEqual(SomeEnum.second.value.desc, 'for the show') + globals()['SomeEnum'] = SomeEnum + globals()['T'] = T + test_pickle_dump_load(self.assertIs, SomeEnum.first) + def test_duplicate_values_give_unique_enum_items(self): class AutoNumber(Enum): first = () @@ -3202,6 +3236,37 @@ class NTEnum(Enum): [TTuple(id=0, a=0, blist=[]), TTuple(id=1, a=2, blist=[4]), TTuple(id=2, a=4, blist=[0, 1, 2])], ) + self.assertRaises(AttributeError, getattr, NTEnum.NONE, 'id') + # + class NTCEnum(TTuple, Enum): + NONE = 0, 0, [] + A = 1, 2, [4] + B = 2, 4, [0, 1, 2] + self.assertEqual(repr(NTCEnum.NONE), "") + self.assertEqual(NTCEnum.NONE.value, TTuple(id=0, a=0, blist=[])) + self.assertEqual(NTCEnum.NONE.id, 0) + self.assertEqual(NTCEnum.A.a, 2) + self.assertEqual(NTCEnum.B.blist, [0, 1 ,2]) + self.assertEqual( + [x.value for x in NTCEnum], + [TTuple(id=0, a=0, blist=[]), TTuple(id=1, a=2, blist=[4]), TTuple(id=2, a=4, blist=[0, 1, 2])], + ) + # + class NTDEnum(Enum): + def __new__(cls, id, a, blist): + member = object.__new__(cls) + member.id = id + member.a = a + member.blist = blist + return member + NONE = TTuple(0, 0, []) + A = TTuple(1, 2, [4]) + B = TTuple(2, 4, [0, 1, 2]) + self.assertEqual(repr(NTDEnum.NONE), "") + self.assertEqual(NTDEnum.NONE.id, 0) + self.assertEqual(NTDEnum.A.a, 2) + self.assertEqual(NTDEnum.B.blist, [0, 1 ,2]) + def test_flag_with_custom_new(self): class FlagFromChar(IntFlag): def __new__(cls, c): diff --git a/Lib/test/test_exceptions.py b/Lib/test/test_exceptions.py index c57488e44aecc6..c7e76414ff0715 100644 --- a/Lib/test/test_exceptions.py +++ b/Lib/test/test_exceptions.py @@ -1917,7 +1917,7 @@ def test_attributes(self): self.assertEqual(exc.name, 'somename') self.assertEqual(exc.path, 'somepath') - msg = "'invalid' is an invalid keyword argument for ImportError" + msg = r"ImportError\(\) got an unexpected keyword argument 'invalid'" with self.assertRaisesRegex(TypeError, msg): ImportError('test', invalid='keyword') diff --git a/Lib/test/test_fcntl.py b/Lib/test/test_fcntl.py index 203dd6fe57dcd9..6d734d052454d3 100644 --- a/Lib/test/test_fcntl.py +++ b/Lib/test/test_fcntl.py @@ -6,7 +6,9 @@ import struct import sys import unittest -from test.support import verbose, cpython_only, get_pagesize +from test.support import ( + cpython_only, get_pagesize, is_apple, requires_subprocess, verbose +) from test.support.import_helper import import_module from test.support.os_helper import TESTFN, unlink @@ -56,8 +58,10 @@ def get_lockdata(): else: start_len = "qq" - if (sys.platform.startswith(('netbsd', 'freebsd', 'openbsd')) - or sys.platform == 'darwin'): + if ( + sys.platform.startswith(('netbsd', 'freebsd', 'openbsd')) + or is_apple + ): if struct.calcsize('l') == 8: off_t = 'l' pid_t = 'i' @@ -157,6 +161,7 @@ def test_flock(self): self.assertRaises(TypeError, fcntl.flock, 'spam', fcntl.LOCK_SH) @unittest.skipIf(platform.system() == "AIX", "AIX returns PermissionError") + @requires_subprocess() def test_lockf_exclusive(self): self.f = open(TESTFN, 'wb+') cmd = fcntl.LOCK_EX | fcntl.LOCK_NB @@ -169,6 +174,7 @@ def test_lockf_exclusive(self): self.assertEqual(p.exitcode, 0) @unittest.skipIf(platform.system() == "AIX", "AIX returns PermissionError") + @requires_subprocess() def test_lockf_share(self): self.f = open(TESTFN, 'wb+') cmd = fcntl.LOCK_SH | fcntl.LOCK_NB diff --git a/Lib/test/test_fileinput.py b/Lib/test/test_fileinput.py index 786d9186634305..b3ad41d2588c4c 100644 --- a/Lib/test/test_fileinput.py +++ b/Lib/test/test_fileinput.py @@ -151,7 +151,7 @@ def test_buffer_sizes(self): print('6. Inplace') savestdout = sys.stdout try: - fi = FileInput(files=(t1, t2, t3, t4), inplace=1, encoding="utf-8") + fi = FileInput(files=(t1, t2, t3, t4), inplace=True, encoding="utf-8") for line in fi: line = line[:-1].upper() print(line) @@ -256,7 +256,7 @@ def test_detached_stdin_binary_mode(self): def test_file_opening_hook(self): try: # cannot use openhook and inplace mode - fi = FileInput(inplace=1, openhook=lambda f, m: None) + fi = FileInput(inplace=True, openhook=lambda f, m: None) self.fail("FileInput should raise if both inplace " "and openhook arguments are given") except ValueError: diff --git a/Lib/test/test_fileio.py b/Lib/test/test_fileio.py index f490485cdaf3eb..06d5a8abf32083 100644 --- a/Lib/test/test_fileio.py +++ b/Lib/test/test_fileio.py @@ -174,6 +174,16 @@ def testRepr(self): self.assertEqual(repr(self.f), "<%s.FileIO [closed]>" % (self.modulename,)) + def test_subclass_repr(self): + class TestSubclass(self.FileIO): + pass + + f = TestSubclass(TESTFN) + with f: + self.assertIn(TestSubclass.__name__, repr(f)) + + self.assertIn(TestSubclass.__name__, repr(f)) + def testReprNoCloseFD(self): fd = os.open(TESTFN, os.O_RDONLY) try: @@ -474,6 +484,14 @@ def testInvalidFd(self): import msvcrt self.assertRaises(OSError, msvcrt.get_osfhandle, make_bad_fd()) + def testBooleanFd(self): + for fd in False, True: + with self.assertWarnsRegex(RuntimeWarning, + 'bool is used as a file descriptor') as cm: + f = self.FileIO(fd, closefd=False) + f.close() + self.assertEqual(cm.filename, __file__) + def testBadModeArgument(self): # verify that we get a sensible error message for bad mode argument bad_mode = "qwerty" diff --git a/Lib/test/test_fractions.py b/Lib/test/test_fractions.py index 84779526ce0eb0..af3cb214ab0ac1 100644 --- a/Lib/test/test_fractions.py +++ b/Lib/test/test_fractions.py @@ -261,6 +261,30 @@ def testFromString(self): self.assertRaisesMessage( ValueError, "Invalid literal for Fraction: '1.1e+1__1'", F, "1.1e+1__1") + self.assertRaisesMessage( + ValueError, "Invalid literal for Fraction: '123.dd'", + F, "123.dd") + self.assertRaisesMessage( + ValueError, "Invalid literal for Fraction: '123.5_dd'", + F, "123.5_dd") + self.assertRaisesMessage( + ValueError, "Invalid literal for Fraction: 'dd.5'", + F, "dd.5") + self.assertRaisesMessage( + ValueError, "Invalid literal for Fraction: '7_dd'", + F, "7_dd") + self.assertRaisesMessage( + ValueError, "Invalid literal for Fraction: '1/dd'", + F, "1/dd") + self.assertRaisesMessage( + ValueError, "Invalid literal for Fraction: '1/123_dd'", + F, "1/123_dd") + self.assertRaisesMessage( + ValueError, "Invalid literal for Fraction: '789edd'", + F, "789edd") + self.assertRaisesMessage( + ValueError, "Invalid literal for Fraction: '789e2_dd'", + F, "789e2_dd") # Test catastrophic backtracking. val = "9"*50 + "_" self.assertRaisesMessage( diff --git a/Lib/test/test_frame.py b/Lib/test/test_frame.py index 7f17666a8d9697..baed03d92b9e56 100644 --- a/Lib/test/test_frame.py +++ b/Lib/test/test_frame.py @@ -55,6 +55,27 @@ class C: # The reference was released by .clear() self.assertIs(None, wr()) + def test_clear_locals_after_f_locals_access(self): + # see gh-113939 + class C: + pass + + wr = None + def inner(): + nonlocal wr + c = C() + wr = weakref.ref(c) + 1/0 + + try: + inner() + except ZeroDivisionError as exc: + support.gc_collect() + self.assertIsNotNone(wr()) + exc.__traceback__.tb_next.tb_frame.clear() + support.gc_collect() + self.assertIsNone(wr()) + def test_clear_does_not_clear_specials(self): class C: pass diff --git a/Lib/test/test_ftplib.py b/Lib/test/test_ftplib.py index 2f191ea7a44c16..81115e9db888cf 100644 --- a/Lib/test/test_ftplib.py +++ b/Lib/test/test_ftplib.py @@ -18,6 +18,7 @@ from unittest import TestCase, skipUnless from test import support +from test.support import requires_subprocess from test.support import threading_helper from test.support import socket_helper from test.support import warnings_helper @@ -900,6 +901,7 @@ def retr(): @skipUnless(ssl, "SSL not available") +@requires_subprocess() class TestTLS_FTPClassMixin(TestFTPClass): """Repeat TestFTPClass tests starting the TLS layer for both control and data connections first. @@ -916,6 +918,7 @@ def setUp(self, encoding=DEFAULT_ENCODING): @skipUnless(ssl, "SSL not available") +@requires_subprocess() class TestTLS_FTPClass(TestCase): """Specific TLS_FTP class tests.""" diff --git a/Lib/test/test_functools.py b/Lib/test/test_functools.py index e4de2c5ede15f1..7c66b906d308ba 100644 --- a/Lib/test/test_functools.py +++ b/Lib/test/test_functools.py @@ -939,6 +939,8 @@ def mycmp(x, y): self.assertRaises(TypeError, hash, k) self.assertNotIsInstance(k, collections.abc.Hashable) + @unittest.skipIf(support.MISSING_C_DOCSTRINGS, + "Signature information for builtins requires docstrings") def test_cmp_to_signature(self): self.assertEqual(str(Signature.from_callable(self.cmp_to_key)), '(mycmp)') @@ -1862,6 +1864,26 @@ def orig(): ... self.assertEqual(str(Signature.from_callable(lru.cache_info)), '()') self.assertEqual(str(Signature.from_callable(lru.cache_clear)), '()') + @support.skip_on_s390x + @unittest.skipIf(support.is_wasi, "WASI has limited C stack") + def test_lru_recursion(self): + + @self.module.lru_cache + def fib(n): + if n <= 1: + return n + return fib(n-1) + fib(n-2) + + if not support.Py_DEBUG: + depth = support.Py_C_RECURSION_LIMIT*2//7 + with support.infinite_recursion(): + fib(depth) + if self.module == c_functools: + fib.cache_clear() + with support.infinite_recursion(): + with self.assertRaises(RecursionError): + fib(10000) + @py_functools.lru_cache() def py_cached_func(x, y): diff --git a/Lib/test/test_gc.py b/Lib/test/test_gc.py index db7cb9ace6e5f3..b01f344cb14a1a 100644 --- a/Lib/test/test_gc.py +++ b/Lib/test/test_gc.py @@ -1,7 +1,7 @@ import unittest import unittest.mock from test.support import (verbose, refcount_test, - cpython_only, requires_subprocess) + cpython_only, requires_subprocess, Py_GIL_DISABLED) from test.support.import_helper import import_module from test.support.os_helper import temp_dir, TESTFN, unlink from test.support.script_helper import assert_python_ok, make_script @@ -815,6 +815,15 @@ def test_freeze(self): self.assertEqual(gc.get_freeze_count(), 0) def test_get_objects(self): + gc.collect() + l = [] + l.append(l) + self.assertTrue( + any(l is element for element in gc.get_objects()) + ) + + @unittest.skipIf(Py_GIL_DISABLED, 'need generational GC') + def test_get_objects_generations(self): gc.collect() l = [] l.append(l) @@ -1225,7 +1234,7 @@ def test_refcount_errors(self): p.stderr.close() # Verify that stderr has a useful error message: self.assertRegex(stderr, - br'gcmodule\.c:[0-9]+: gc_decref: Assertion "gc_get_refs\(g\) > 0" failed.') + br'gc.*\.c:[0-9]+: .*: Assertion "gc_get_refs\(.+\) .*" failed.') self.assertRegex(stderr, br'refcount is too small') # "address : 0x7fb5062efc18" diff --git a/Lib/test/test_generated_cases.py b/Lib/test/test_generated_cases.py index 3b2f579be684b7..ca1228ee7008a9 100644 --- a/Lib/test/test_generated_cases.py +++ b/Lib/test/test_generated_cases.py @@ -9,7 +9,7 @@ def skip_if_different_mount_drives(): - if sys.platform != 'win32': + if sys.platform != "win32": return ROOT = os.path.dirname(os.path.dirname(__file__)) root_drive = os.path.splitroot(ROOT)[0] @@ -22,11 +22,13 @@ def skip_if_different_mount_drives(): f"directory have different mount drives " f"({cwd_drive} and {root_drive})" ) + + skip_if_different_mount_drives() -test_tools.skip_if_missing('cases_generator') -with test_tools.imports_under_tool('cases_generator'): +test_tools.skip_if_missing("cases_generator") +with test_tools.imports_under_tool("cases_generator"): from analyzer import StackItem import parser from stack import Stack @@ -39,13 +41,14 @@ def handle_stderr(): else: return support.captured_stderr() + class TestEffects(unittest.TestCase): def test_effect_sizes(self): stack = Stack() inputs = [ - x:= StackItem("x", None, "", "1"), - y:= StackItem("y", None, "", "oparg"), - z:= StackItem("z", None, "", "oparg*2"), + x := StackItem("x", None, "", "1"), + y := StackItem("y", None, "", "oparg"), + z := StackItem("z", None, "", "oparg*2"), ] outputs = [ StackItem("x", None, "", "1"), @@ -96,9 +99,7 @@ def run_cases_test(self, input: str, expected: str): with handle_stderr(): tier1_generator.generate_tier1_from_files( - [self.temp_input_filename], - self.temp_output_filename, - False + [self.temp_input_filename], self.temp_output_filename, False ) with open(self.temp_output_filename) as temp_output: @@ -750,7 +751,7 @@ def test_override_op(self): def test_annotated_inst(self): input = """ - guard inst(OP, (--)) { + pure inst(OP, (--)) { ham(); } """ @@ -767,7 +768,7 @@ def test_annotated_inst(self): def test_annotated_op(self): input = """ - guard op(OP, (--)) { + pure op(OP, (--)) { spam(); } macro(M) = OP; @@ -784,7 +785,7 @@ def test_annotated_op(self): self.run_cases_test(input, output) input = """ - guard register specializing op(OP, (--)) { + pure register specializing op(OP, (--)) { spam(); } macro(M) = OP; diff --git a/Lib/test/test_genericpath.py b/Lib/test/test_genericpath.py index 4f311c2d498e9f..f407ee3caf154c 100644 --- a/Lib/test/test_genericpath.py +++ b/Lib/test/test_genericpath.py @@ -7,9 +7,9 @@ import sys import unittest import warnings -from test.support import is_emscripten -from test.support import os_helper -from test.support import warnings_helper +from test.support import ( + is_apple, is_emscripten, os_helper, warnings_helper +) from test.support.script_helper import assert_python_ok from test.support.os_helper import FakePath @@ -165,6 +165,12 @@ def test_exists_fd(self): os.close(w) self.assertFalse(self.pathmodule.exists(r)) + def test_exists_bool(self): + for fd in False, True: + with self.assertWarnsRegex(RuntimeWarning, + 'bool is used as a file descriptor'): + self.pathmodule.exists(fd) + def test_isdir(self): filename = os_helper.TESTFN bfilename = os.fsencode(filename) @@ -483,12 +489,16 @@ def test_abspath_issue3426(self): self.assertIsInstance(abspath(path), str) def test_nonascii_abspath(self): - if (os_helper.TESTFN_UNDECODABLE - # macOS and Emscripten deny the creation of a directory with an - # invalid UTF-8 name. Windows allows creating a directory with an - # arbitrary bytes name, but fails to enter this directory - # (when the bytes name is used). - and sys.platform not in ('win32', 'darwin', 'emscripten', 'wasi')): + if ( + os_helper.TESTFN_UNDECODABLE + # Apple platforms and Emscripten/WASI deny the creation of a + # directory with an invalid UTF-8 name. Windows allows creating a + # directory with an arbitrary bytes name, but fails to enter this + # directory (when the bytes name is used). + and sys.platform not in { + "win32", "emscripten", "wasi" + } and not is_apple + ): name = os_helper.TESTFN_UNDECODABLE elif os_helper.TESTFN_NONASCII: name = os_helper.TESTFN_NONASCII diff --git a/Lib/test/test_httpservers.py b/Lib/test/test_httpservers.py index 9fa6ecf9c08e27..d762ec6102ab8a 100644 --- a/Lib/test/test_httpservers.py +++ b/Lib/test/test_httpservers.py @@ -30,8 +30,9 @@ import unittest from test import support -from test.support import os_helper -from test.support import threading_helper +from test.support import ( + is_apple, os_helper, requires_subprocess, threading_helper +) support.requires_working_socket(module=True) @@ -410,8 +411,8 @@ def close_conn(): reader.close() return body - @unittest.skipIf(sys.platform == 'darwin', - 'undecodable name cannot always be decoded on macOS') + @unittest.skipIf(is_apple, + 'undecodable name cannot always be decoded on Apple platforms') @unittest.skipIf(sys.platform == 'win32', 'undecodable name cannot be decoded on win32') @unittest.skipUnless(os_helper.TESTFN_UNDECODABLE, @@ -422,11 +423,11 @@ def test_undecodable_filename(self): with open(os.path.join(self.tempdir, filename), 'wb') as f: f.write(os_helper.TESTFN_UNDECODABLE) response = self.request(self.base_url + '/') - if sys.platform == 'darwin': - # On Mac OS the HFS+ filesystem replaces bytes that aren't valid - # UTF-8 into a percent-encoded value. + if is_apple: + # On Apple platforms the HFS+ filesystem replaces bytes that + # aren't valid UTF-8 into a percent-encoded value. for name in os.listdir(self.tempdir): - if name != 'test': # Ignore a filename created in setUp(). + if name != 'test': # Ignore a filename created in setUp(). filename = name break body = self.check_status_and_reason(response, HTTPStatus.OK) @@ -697,6 +698,7 @@ def test_html_escape_filename(self): @unittest.skipIf(hasattr(os, 'geteuid') and os.geteuid() == 0, "This test can't be run reliably as root (issue #13308).") +@requires_subprocess() class CGIHTTPServerTestCase(BaseTestCase): class request_handler(NoLogRequestHandler, CGIHTTPRequestHandler): _test_case_self = None # populated by each setUp() method call. diff --git a/Lib/test/test_importlib/extension/test_loader.py b/Lib/test/test_importlib/extension/test_loader.py index 64c8a5485106e3..84a0680e4ec653 100644 --- a/Lib/test/test_importlib/extension/test_loader.py +++ b/Lib/test/test_importlib/extension/test_loader.py @@ -9,6 +9,7 @@ import warnings import importlib.util import importlib +from test.support import MISSING_C_DOCSTRINGS class LoaderTests: @@ -373,7 +374,8 @@ def test_nonascii(self): with self.subTest(name): module = self.load_module_by_name(name) self.assertEqual(module.__name__, name) - self.assertEqual(module.__doc__, "Module named in %s" % lang) + if not MISSING_C_DOCSTRINGS: + self.assertEqual(module.__doc__, "Module named in %s" % lang) (Frozen_MultiPhaseExtensionModuleTests, diff --git a/Lib/test/test_importlib/test_main.py b/Lib/test/test_importlib/test_main.py index 1d3817151edf64..0a769b89841234 100644 --- a/Lib/test/test_importlib/test_main.py +++ b/Lib/test/test_importlib/test_main.py @@ -4,6 +4,7 @@ import warnings import importlib.metadata import contextlib +from test.support import os_helper try: import pyfakefs.fake_filesystem_unittest as ffs @@ -403,6 +404,7 @@ def test_packages_distributions_all_module_types(self): assert not any(name.endswith('.dist-info') for name in distributions) + @os_helper.skip_unless_symlink def test_packages_distributions_symlinked_top_level(self) -> None: """ Distribution is resolvable from a simple top-level symlink in RECORD. diff --git a/Lib/test/test_inspect/test_inspect.py b/Lib/test/test_inspect/test_inspect.py index 09d50859970c99..4611f62b293ff9 100644 --- a/Lib/test/test_inspect/test_inspect.py +++ b/Lib/test/test_inspect/test_inspect.py @@ -3990,6 +3990,8 @@ def foo(a, *, b:1): pass foo_sig = MySignature.from_callable(foo) self.assertIsInstance(foo_sig, MySignature) + @unittest.skipIf(MISSING_C_DOCSTRINGS, + "Signature information for builtins requires docstrings") def test_signature_from_callable_class(self): # A regression test for a class inheriting its signature from `object`. class MySignature(inspect.Signature): pass @@ -4080,7 +4082,8 @@ def test_signature_eval_str(self): par('c', PORK, annotation="'MyClass'"), ))) - self.assertEqual(signature_func(isa.UnannotatedClass), sig()) + if not MISSING_C_DOCSTRINGS: + self.assertEqual(signature_func(isa.UnannotatedClass), sig()) self.assertEqual(signature_func(isa.unannotated_function), sig( parameters=( diff --git a/Lib/test/test_int.py b/Lib/test/test_int.py index 5545ee39d8e942..0bf55facad9fed 100644 --- a/Lib/test/test_int.py +++ b/Lib/test/test_int.py @@ -90,6 +90,7 @@ def test_basic(self): self.assertRaises(TypeError, int, 1, 12) + self.assertRaises(TypeError, int, "10", 2, 1) self.assertEqual(int('0o123', 0), 83) self.assertEqual(int('0x123', 16), 291) diff --git a/Lib/test/test_interpreters/__main__.py b/Lib/test/test_interpreters/__main__.py index 8641229877b2be..40a23a297ec2b4 100644 --- a/Lib/test/test_interpreters/__main__.py +++ b/Lib/test/test_interpreters/__main__.py @@ -1,4 +1,4 @@ from . import load_tests import unittest -nittest.main() +unittest.main() diff --git a/Lib/test/test_io.py b/Lib/test/test_io.py index 09cced9baef99b..a24579dcc878cf 100644 --- a/Lib/test/test_io.py +++ b/Lib/test/test_io.py @@ -39,11 +39,9 @@ from test import support from test.support.script_helper import ( assert_python_ok, assert_python_failure, run_python_until_end) -from test.support import import_helper -from test.support import os_helper -from test.support import threading_helper -from test.support import warnings_helper -from test.support import skip_if_sanitizer +from test.support import ( + import_helper, is_apple, os_helper, skip_if_sanitizer, threading_helper, warnings_helper +) from test.support.os_helper import FakePath import codecs @@ -606,10 +604,10 @@ def test_raw_bytes_io(self): self.read_ops(f, True) def test_large_file_ops(self): - # On Windows and Mac OSX this test consumes large resources; It takes - # a long time to build the >2 GiB file and takes >2 GiB of disk space - # therefore the resource must be enabled to run this test. - if sys.platform[:3] == 'win' or sys.platform == 'darwin': + # On Windows and Apple platforms this test consumes large resources; It + # takes a long time to build the >2 GiB file and takes >2 GiB of disk + # space therefore the resource must be enabled to run this test. + if sys.platform[:3] == 'win' or is_apple: support.requires( 'largefile', 'test requires %s bytes and a long time to run' % self.LARGE) @@ -1654,7 +1652,8 @@ def test_truncate_on_read_only(self): class CBufferedReaderTest(BufferedReaderTest, SizeofTest): tp = io.BufferedReader - @skip_if_sanitizer(memory=True, address=True, reason= "sanitizer defaults to crashing " + @skip_if_sanitizer(memory=True, address=True, thread=True, + reason="sanitizer defaults to crashing " "instead of returning NULL for malloc failure.") def test_constructor(self): BufferedReaderTest.test_constructor(self) @@ -2021,7 +2020,8 @@ def test_slow_close_from_thread(self): class CBufferedWriterTest(BufferedWriterTest, SizeofTest): tp = io.BufferedWriter - @skip_if_sanitizer(memory=True, address=True, reason= "sanitizer defaults to crashing " + @skip_if_sanitizer(memory=True, address=True, thread=True, + reason="sanitizer defaults to crashing " "instead of returning NULL for malloc failure.") def test_constructor(self): BufferedWriterTest.test_constructor(self) @@ -2497,6 +2497,28 @@ def test_interleaved_read_write(self): f.flush() self.assertEqual(raw.getvalue(), b'a2c') + def test_read1_after_write(self): + with self.BytesIO(b'abcdef') as raw: + with self.tp(raw, 3) as f: + f.write(b"1") + self.assertEqual(f.read1(1), b'b') + f.flush() + self.assertEqual(raw.getvalue(), b'1bcdef') + with self.BytesIO(b'abcdef') as raw: + with self.tp(raw, 3) as f: + f.write(b"1") + self.assertEqual(f.read1(), b'bcd') + f.flush() + self.assertEqual(raw.getvalue(), b'1bcdef') + with self.BytesIO(b'abcdef') as raw: + with self.tp(raw, 3) as f: + f.write(b"1") + # XXX: read(100) returns different numbers of bytes + # in Python and C implementations. + self.assertEqual(f.read1(100)[:3], b'bcd') + f.flush() + self.assertEqual(raw.getvalue(), b'1bcdef') + def test_interleaved_readline_write(self): with self.BytesIO(b'ab\ncdef\ng\n') as raw: with self.tp(raw) as f: @@ -2509,6 +2531,36 @@ def test_interleaved_readline_write(self): f.flush() self.assertEqual(raw.getvalue(), b'1b\n2def\n3\n') + def test_xxx(self): + with self.BytesIO(b'abcdefgh') as raw: + with self.tp(raw) as f: + f.write(b'123') + self.assertEqual(f.read(), b'defgh') + f.write(b'456') + f.flush() + self.assertEqual(raw.getvalue(), b'123defgh456') + with self.BytesIO(b'abcdefgh') as raw: + with self.tp(raw) as f: + f.write(b'123') + self.assertEqual(f.read(3), b'def') + f.write(b'456') + f.flush() + self.assertEqual(raw.getvalue(), b'123def456') + with self.BytesIO(b'abcdefgh') as raw: + with self.tp(raw) as f: + f.write(b'123') + self.assertEqual(f.read1(), b'defgh') + f.write(b'456') + f.flush() + self.assertEqual(raw.getvalue(), b'123defgh456') + with self.BytesIO(b'abcdefgh') as raw: + with self.tp(raw) as f: + f.write(b'123') + self.assertEqual(f.read1(3), b'def') + f.write(b'456') + f.flush() + self.assertEqual(raw.getvalue(), b'123def456') + # You can't construct a BufferedRandom over a non-seekable stream. test_unseekable = None @@ -2520,7 +2572,8 @@ def test_interleaved_readline_write(self): class CBufferedRandomTest(BufferedRandomTest, SizeofTest): tp = io.BufferedRandom - @skip_if_sanitizer(memory=True, address=True, reason= "sanitizer defaults to crashing " + @skip_if_sanitizer(memory=True, address=True, thread=True, + reason="sanitizer defaults to crashing " "instead of returning NULL for malloc failure.") def test_constructor(self): BufferedRandomTest.test_constructor(self) @@ -2803,6 +2856,13 @@ def test_recursive_repr(self): with self.assertRaises(RuntimeError): repr(t) # Should not crash + def test_subclass_repr(self): + class TestSubclass(self.TextIOWrapper): + pass + + f = TestSubclass(self.StringIO()) + self.assertIn(TestSubclass.__name__, repr(f)) + def test_line_buffering(self): r = self.BytesIO() b = self.BufferedWriter(r, 1000) @@ -3642,10 +3702,8 @@ def _check_create_at_shutdown(self, **kwargs): codecs.lookup('utf-8') class C: - def __init__(self): - self.buf = io.BytesIO() def __del__(self): - io.TextIOWrapper(self.buf, **{kwargs}) + io.TextIOWrapper(io.BytesIO(), **{kwargs}) print("ok") c = C() """.format(iomod=iomod, kwargs=kwargs) @@ -3877,6 +3935,14 @@ def test_issue25862(self): t.write('x') t.tell() + def test_issue35928(self): + p = self.BufferedRWPair(self.BytesIO(b'foo\nbar\n'), self.BytesIO()) + f = self.TextIOWrapper(p) + res = f.readline() + self.assertEqual(res, 'foo\n') + f.write(res) + self.assertEqual(res + f.readline(), 'foo\nbar\n') + class MemviewBytesIO(io.BytesIO): '''A BytesIO object whose read method returns memoryviews diff --git a/Lib/test/test_isinstance.py b/Lib/test/test_isinstance.py index bf9332e40aeaf2..7f759fb3317146 100644 --- a/Lib/test/test_isinstance.py +++ b/Lib/test/test_isinstance.py @@ -310,7 +310,7 @@ class X: @property def __bases__(self): return self.__bases__ - with support.infinite_recursion(): + with support.infinite_recursion(25): self.assertRaises(RecursionError, issubclass, X(), int) self.assertRaises(RecursionError, issubclass, int, X()) self.assertRaises(RecursionError, isinstance, 1, X()) @@ -344,7 +344,7 @@ class B: pass A.__getattr__ = B.__getattr__ = X.__getattr__ return (A(), B()) - with support.infinite_recursion(): + with support.infinite_recursion(25): self.assertRaises(RecursionError, issubclass, X(), int) diff --git a/Lib/test/test_json/test_recursion.py b/Lib/test/test_json/test_recursion.py index 9919d7fbe54ef7..164ff2013eb552 100644 --- a/Lib/test/test_json/test_recursion.py +++ b/Lib/test/test_json/test_recursion.py @@ -85,10 +85,10 @@ def test_highly_nested_objects_encoding(self): for x in range(100000): l, d = [l], {'k':d} with self.assertRaises(RecursionError): - with support.infinite_recursion(): + with support.infinite_recursion(5000): self.dumps(l) with self.assertRaises(RecursionError): - with support.infinite_recursion(): + with support.infinite_recursion(5000): self.dumps(d) def test_endless_recursion(self): @@ -99,7 +99,7 @@ def default(self, o): return [o] with self.assertRaises(RecursionError): - with support.infinite_recursion(): + with support.infinite_recursion(1000): EndlessJSONEncoder(check_circular=False).encode(5j) diff --git a/Lib/test/test_launcher.py b/Lib/test/test_launcher.py index bcd4ed63bf25a0..2528a51240fbf7 100644 --- a/Lib/test/test_launcher.py +++ b/Lib/test/test_launcher.py @@ -19,8 +19,10 @@ PY_EXE = "py.exe" +DEBUG_BUILD = False if sys.executable.casefold().endswith("_d.exe".casefold()): PY_EXE = "py_d.exe" + DEBUG_BUILD = True # Registry data to create. On removal, everything beneath top-level names will # be deleted. @@ -88,6 +90,12 @@ "test-command=TEST_EXE.exe", ]) + +def quote(s): + s = str(s) + return f'"{s}"' if " " in s else s + + def create_registry_data(root, data): def _create_registry_data(root, key, value): if isinstance(value, dict): @@ -232,7 +240,7 @@ def run_py(self, args, env=None, allow_fail=False, expect_returncode=0, argv=Non p.stdin.close() p.wait(10) out = p.stdout.read().decode("utf-8", "replace") - err = p.stderr.read().decode("ascii", "replace") + err = p.stderr.read().decode("ascii", "replace").replace("\uFFFD", "?") if p.returncode != expect_returncode and support.verbose and not allow_fail: print("++ COMMAND ++") print([self.py_exe, *args]) @@ -273,7 +281,7 @@ def script(self, content, encoding="utf-8"): def fake_venv(self): venv = Path.cwd() / "Scripts" venv.mkdir(exist_ok=True, parents=True) - venv_exe = (venv / Path(sys.executable).name) + venv_exe = (venv / ("python_d.exe" if DEBUG_BUILD else "python.exe")) venv_exe.touch() try: yield venv_exe, {"VIRTUAL_ENV": str(venv.parent)} @@ -521,6 +529,9 @@ def test_virtualenv_in_list(self): self.assertEqual(str(venv_exe), m.group(1)) break else: + if support.verbose: + print(data["stdout"]) + print(data["stderr"]) self.fail("did not find active venv path") data = self.run_py(["-0"], env=env) @@ -537,10 +548,10 @@ def test_virtualenv_with_env(self): data1 = self.run_py([], env={**env, "PY_PYTHON": "PythonTestSuite/3"}) data2 = self.run_py(["-V:PythonTestSuite/3"], env={**env, "PY_PYTHON": "PythonTestSuite/3"}) # Compare stdout, because stderr goes via ascii - self.assertEqual(data1["stdout"].strip(), str(venv_exe)) + self.assertEqual(data1["stdout"].strip(), quote(venv_exe)) self.assertEqual(data1["SearchInfo.lowPriorityTag"], "True") # Ensure passing the argument doesn't trigger the same behaviour - self.assertNotEqual(data2["stdout"].strip(), str(venv_exe)) + self.assertNotEqual(data2["stdout"].strip(), quote(venv_exe)) self.assertNotEqual(data2["SearchInfo.lowPriorityTag"], "True") def test_py_shebang(self): @@ -549,7 +560,7 @@ def test_py_shebang(self): data = self.run_py([script, "-postarg"]) self.assertEqual("PythonTestSuite", data["SearchInfo.company"]) self.assertEqual("3.100", data["SearchInfo.tag"]) - self.assertEqual(f"X.Y.exe -prearg {script} -postarg", data["stdout"].strip()) + self.assertEqual(f"X.Y.exe -prearg {quote(script)} -postarg", data["stdout"].strip()) def test_python_shebang(self): with self.py_ini(TEST_PY_DEFAULTS): @@ -557,7 +568,7 @@ def test_python_shebang(self): data = self.run_py([script, "-postarg"]) self.assertEqual("PythonTestSuite", data["SearchInfo.company"]) self.assertEqual("3.100", data["SearchInfo.tag"]) - self.assertEqual(f"X.Y.exe -prearg {script} -postarg", data["stdout"].strip()) + self.assertEqual(f"X.Y.exe -prearg {quote(script)} -postarg", data["stdout"].strip()) def test_py2_shebang(self): with self.py_ini(TEST_PY_DEFAULTS): @@ -565,7 +576,8 @@ def test_py2_shebang(self): data = self.run_py([script, "-postarg"]) self.assertEqual("PythonTestSuite", data["SearchInfo.company"]) self.assertEqual("3.100-32", data["SearchInfo.tag"]) - self.assertEqual(f"X.Y-32.exe -prearg {script} -postarg", data["stdout"].strip()) + self.assertEqual(f"X.Y-32.exe -prearg {quote(script)} -postarg", + data["stdout"].strip()) def test_py3_shebang(self): with self.py_ini(TEST_PY_DEFAULTS): @@ -573,7 +585,8 @@ def test_py3_shebang(self): data = self.run_py([script, "-postarg"]) self.assertEqual("PythonTestSuite", data["SearchInfo.company"]) self.assertEqual("3.100-arm64", data["SearchInfo.tag"]) - self.assertEqual(f"X.Y-arm64.exe -X fake_arg_for_test -prearg {script} -postarg", data["stdout"].strip()) + self.assertEqual(f"X.Y-arm64.exe -X fake_arg_for_test -prearg {quote(script)} -postarg", + data["stdout"].strip()) def test_py_shebang_nl(self): with self.py_ini(TEST_PY_DEFAULTS): @@ -581,7 +594,8 @@ def test_py_shebang_nl(self): data = self.run_py([script, "-postarg"]) self.assertEqual("PythonTestSuite", data["SearchInfo.company"]) self.assertEqual("3.100", data["SearchInfo.tag"]) - self.assertEqual(f"X.Y.exe -prearg {script} -postarg", data["stdout"].strip()) + self.assertEqual(f"X.Y.exe -prearg {quote(script)} -postarg", + data["stdout"].strip()) def test_py2_shebang_nl(self): with self.py_ini(TEST_PY_DEFAULTS): @@ -589,7 +603,8 @@ def test_py2_shebang_nl(self): data = self.run_py([script, "-postarg"]) self.assertEqual("PythonTestSuite", data["SearchInfo.company"]) self.assertEqual("3.100-32", data["SearchInfo.tag"]) - self.assertEqual(f"X.Y-32.exe -prearg {script} -postarg", data["stdout"].strip()) + self.assertEqual(f"X.Y-32.exe -prearg {quote(script)} -postarg", + data["stdout"].strip()) def test_py3_shebang_nl(self): with self.py_ini(TEST_PY_DEFAULTS): @@ -597,7 +612,8 @@ def test_py3_shebang_nl(self): data = self.run_py([script, "-postarg"]) self.assertEqual("PythonTestSuite", data["SearchInfo.company"]) self.assertEqual("3.100-arm64", data["SearchInfo.tag"]) - self.assertEqual(f"X.Y-arm64.exe -X fake_arg_for_test -prearg {script} -postarg", data["stdout"].strip()) + self.assertEqual(f"X.Y-arm64.exe -X fake_arg_for_test -prearg {quote(script)} -postarg", + data["stdout"].strip()) def test_py_shebang_short_argv0(self): with self.py_ini(TEST_PY_DEFAULTS): @@ -616,25 +632,31 @@ def test_py_handle_64_in_ini(self): self.assertEqual("True", data["SearchInfo.oldStyleTag"]) def test_search_path(self): - stem = Path(sys.executable).stem + exe = Path("arbitrary-exe-name.exe").absolute() + exe.touch() + self.addCleanup(exe.unlink) with self.py_ini(TEST_PY_DEFAULTS): - with self.script(f"#! /usr/bin/env {stem} -prearg") as script: + with self.script(f"#! /usr/bin/env {exe.stem} -prearg") as script: data = self.run_py( [script, "-postarg"], - env={"PATH": f"{Path(sys.executable).parent};{os.getenv('PATH')}"}, + env={"PATH": f"{exe.parent};{os.getenv('PATH')}"}, ) - self.assertEqual(f"{sys.executable} -prearg {script} -postarg", data["stdout"].strip()) + self.assertEqual(f"{quote(exe)} -prearg {quote(script)} -postarg", + data["stdout"].strip()) def test_search_path_exe(self): # Leave the .exe on the name to ensure we don't add it a second time - name = Path(sys.executable).name + exe = Path("arbitrary-exe-name.exe").absolute() + exe.touch() + self.addCleanup(exe.unlink) with self.py_ini(TEST_PY_DEFAULTS): - with self.script(f"#! /usr/bin/env {name} -prearg") as script: + with self.script(f"#! /usr/bin/env {exe.name} -prearg") as script: data = self.run_py( [script, "-postarg"], - env={"PATH": f"{Path(sys.executable).parent};{os.getenv('PATH')}"}, + env={"PATH": f"{exe.parent};{os.getenv('PATH')}"}, ) - self.assertEqual(f"{sys.executable} -prearg {script} -postarg", data["stdout"].strip()) + self.assertEqual(f"{quote(exe)} -prearg {quote(script)} -postarg", + data["stdout"].strip()) def test_recursive_search_path(self): stem = self.get_py_exe().stem @@ -645,7 +667,7 @@ def test_recursive_search_path(self): env={"PATH": f"{self.get_py_exe().parent};{os.getenv('PATH')}"}, ) # The recursive search is ignored and we get normal "py" behavior - self.assertEqual(f"X.Y.exe {script}", data["stdout"].strip()) + self.assertEqual(f"X.Y.exe {quote(script)}", data["stdout"].strip()) def test_install(self): data = self.run_py(["-V:3.10"], env={"PYLAUNCHER_ALWAYS_INSTALL": "1"}, expect_returncode=111) @@ -665,7 +687,7 @@ def test_literal_shebang_absolute(self): with self.script("#! C:/some_random_app -witharg") as script: data = self.run_py([script]) self.assertEqual( - f"C:\\some_random_app -witharg {script}", + f"C:\\some_random_app -witharg {quote(script)}", data["stdout"].strip(), ) @@ -673,7 +695,7 @@ def test_literal_shebang_relative(self): with self.script("#! ..\\some_random_app -witharg") as script: data = self.run_py([script]) self.assertEqual( - f"{script.parent.parent}\\some_random_app -witharg {script}", + f"{quote(script.parent.parent / 'some_random_app')} -witharg {quote(script)}", data["stdout"].strip(), ) @@ -681,14 +703,14 @@ def test_literal_shebang_quoted(self): with self.script('#! "some random app" -witharg') as script: data = self.run_py([script]) self.assertEqual( - f'"{script.parent}\\some random app" -witharg {script}', + f"{quote(script.parent / 'some random app')} -witharg {quote(script)}", data["stdout"].strip(), ) with self.script('#! some" random "app -witharg') as script: data = self.run_py([script]) self.assertEqual( - f'"{script.parent}\\some random app" -witharg {script}', + f"{quote(script.parent / 'some random app')} -witharg {quote(script)}", data["stdout"].strip(), ) @@ -696,7 +718,7 @@ def test_literal_shebang_quoted_escape(self): with self.script('#! some\\" random "app -witharg') as script: data = self.run_py([script]) self.assertEqual( - f'"{script.parent}\\some\\ random app" -witharg {script}', + f"{quote(script.parent / 'some/ random app')} -witharg {quote(script)}", data["stdout"].strip(), ) @@ -705,7 +727,7 @@ def test_literal_shebang_command(self): with self.script('#! test-command arg1') as script: data = self.run_py([script]) self.assertEqual( - f"TEST_EXE.exe arg1 {script}", + f"TEST_EXE.exe arg1 {quote(script)}", data["stdout"].strip(), ) @@ -714,7 +736,7 @@ def test_literal_shebang_invalid_template(self): data = self.run_py([script]) expect = script.parent / "/usr/bin/not-python" self.assertEqual( - f"{expect} arg1 {script}", + f"{quote(expect)} arg1 {quote(script)}", data["stdout"].strip(), ) @@ -727,15 +749,18 @@ def test_shebang_command_in_venv(self): data = self.run_py([script], expect_returncode=103) with self.fake_venv() as (venv_exe, env): - # Put a real Python (ourselves) on PATH as a distraction. + # Put a "normal" Python on PATH as a distraction. # The active VIRTUAL_ENV should be preferred when the name isn't an # exact match. - env["PATH"] = f"{Path(sys.executable).parent};{os.environ['PATH']}" + exe = Path(Path(venv_exe).name).absolute() + exe.touch() + self.addCleanup(exe.unlink) + env["PATH"] = f"{exe.parent};{os.environ['PATH']}" with self.script(f'#! /usr/bin/env {stem} arg1') as script: data = self.run_py([script], env=env) - self.assertEqual(data["stdout"].strip(), f"{venv_exe} arg1 {script}") + self.assertEqual(data["stdout"].strip(), f"{quote(venv_exe)} arg1 {quote(script)}") - with self.script(f'#! /usr/bin/env {Path(sys.executable).stem} arg1') as script: + with self.script(f'#! /usr/bin/env {exe.stem} arg1') as script: data = self.run_py([script], env=env) - self.assertEqual(data["stdout"].strip(), f"{sys.executable} arg1 {script}") + self.assertEqual(data["stdout"].strip(), f"{quote(exe)} arg1 {quote(script)}") diff --git a/Lib/test/test_listcomps.py b/Lib/test/test_listcomps.py index f95a78aff0c711..2868dd01545b95 100644 --- a/Lib/test/test_listcomps.py +++ b/Lib/test/test_listcomps.py @@ -156,6 +156,18 @@ def method(self): self.assertEqual(C.y, [4, 4, 4, 4, 4]) self.assertIs(C().method(), C) + def test_references_super(self): + code = """ + res = [super for x in [1]] + """ + self._check_in_scopes(code, outputs={"res": [super]}) + + def test_references___class__(self): + code = """ + res = [__class__ for x in [1]] + """ + self._check_in_scopes(code, raises=NameError) + def test_inner_cell_shadows_outer(self): code = """ items = [(lambda: i) for i in range(5)] diff --git a/Lib/test/test_logging.py b/Lib/test/test_logging.py index 89be432e4b78c0..888523227c2ac4 100644 --- a/Lib/test/test_logging.py +++ b/Lib/test/test_logging.py @@ -43,11 +43,13 @@ import tempfile from test.support.script_helper import assert_python_ok, assert_python_failure from test import support +from test.support import import_helper from test.support import os_helper from test.support import socket_helper from test.support import threading_helper from test.support import warnings_helper from test.support import asyncore +from test.support import smtpd from test.support.logging_helper import TestHandler import textwrap import threading @@ -62,9 +64,6 @@ from socketserver import (ThreadingUDPServer, DatagramRequestHandler, ThreadingTCPServer, StreamRequestHandler) -with warnings.catch_warnings(): - from . import smtpd - try: import win32evtlog, win32evtlogutil, pywintypes except ImportError: @@ -3922,6 +3921,26 @@ def test_90195(self): # Logger should be enabled, since explicitly mentioned self.assertFalse(logger.disabled) + def test_111615(self): + # See gh-111615 + import_helper.import_module('_multiprocessing') # see gh-113692 + mp = import_helper.import_module('multiprocessing') + + config = { + 'version': 1, + 'handlers': { + 'sink': { + 'class': 'logging.handlers.QueueHandler', + 'queue': mp.get_context('spawn').Queue(), + }, + }, + 'root': { + 'handlers': ['sink'], + 'level': 'DEBUG', + }, + } + logging.config.dictConfig(config) + class ManagerTest(BaseTest): def test_manager_loggerclass(self): logged = [] @@ -4070,6 +4089,7 @@ def test_queue_listener(self): self.que_logger.critical(self.next_message()) finally: listener.stop() + listener.stop() # gh-114706 - ensure no crash if called again self.assertTrue(handler.matches(levelno=logging.WARNING, message='1')) self.assertTrue(handler.matches(levelno=logging.ERROR, message='2')) self.assertTrue(handler.matches(levelno=logging.CRITICAL, message='3')) diff --git a/Lib/test/test_lzma.py b/Lib/test/test_lzma.py index 13b200912f6abd..65e6488c5d7b10 100644 --- a/Lib/test/test_lzma.py +++ b/Lib/test/test_lzma.py @@ -1401,6 +1401,14 @@ def test__decode_filter_properties(self): self.assertEqual(filterspec["lc"], 3) self.assertEqual(filterspec["dict_size"], 8 << 20) + # see gh-104282 + filters = [lzma.FILTER_X86, lzma.FILTER_POWERPC, + lzma.FILTER_IA64, lzma.FILTER_ARM, + lzma.FILTER_ARMTHUMB, lzma.FILTER_SPARC] + for f in filters: + filterspec = lzma._decode_filter_properties(f, b"") + self.assertEqual(filterspec, {"id": f}) + def test_filter_properties_roundtrip(self): spec1 = lzma._decode_filter_properties( lzma.FILTER_LZMA1, b"]\x00\x00\x80\x00") diff --git a/Lib/test/test_mailbox.py b/Lib/test/test_mailbox.py index 23fcbfac1f9c89..c52c014185bec7 100644 --- a/Lib/test/test_mailbox.py +++ b/Lib/test/test_mailbox.py @@ -681,6 +681,20 @@ def test_initialize_existing(self): self._box = mailbox.Maildir(self._path) self._check_basics() + def test_filename_leading_dot(self): + self.tearDown() + for subdir in '', 'tmp', 'new', 'cur': + os.mkdir(os.path.normpath(os.path.join(self._path, subdir))) + for subdir in 'tmp', 'new', 'cur': + fname = os.path.join(self._path, subdir, '.foo' + subdir) + with open(fname, 'wb') as f: + f.write(b"@") + self._box = mailbox.Maildir(self._path) + self.assertNotIn('.footmp', self._box) + self.assertNotIn('.foonew', self._box) + self.assertNotIn('.foocur', self._box) + self.assertEqual(list(self._box.iterkeys()), []) + def _check_basics(self, factory=None): # (Used by test_open_new() and test_open_existing().) self.assertEqual(self._box._path, os.path.abspath(self._path)) @@ -1113,12 +1127,14 @@ def test_add_from_string(self): # Add a string starting with 'From ' to the mailbox key = self._box.add('From foo@bar blah\nFrom: foo\n\n0\n') self.assertEqual(self._box[key].get_from(), 'foo@bar blah') + self.assertEqual(self._box[key].get_unixfrom(), 'From foo@bar blah') self.assertEqual(self._box[key].get_payload(), '0\n') def test_add_from_bytes(self): # Add a byte string starting with 'From ' to the mailbox key = self._box.add(b'From foo@bar blah\nFrom: foo\n\n0\n') self.assertEqual(self._box[key].get_from(), 'foo@bar blah') + self.assertEqual(self._box[key].get_unixfrom(), 'From foo@bar blah') self.assertEqual(self._box[key].get_payload(), '0\n') def test_add_mbox_or_mmdf_message(self): @@ -1333,6 +1349,28 @@ def test_sequences(self): self._box.remove(key1) self.assertEqual(self._box.get_sequences(), {'flagged':[key0]}) + self._box.set_sequences({'foo':[key0]}) + self.assertEqual(self._box.get_sequences(), {'foo':[key0]}) + + def test_no_dot_mh_sequences_file(self): + path = os.path.join(self._path, 'foo.bar') + os.mkdir(path) + box = self._factory(path) + self.assertEqual(os.listdir(path), []) + self.assertEqual(box.get_sequences(), {}) + self.assertEqual(os.listdir(path), []) + box.set_sequences({}) + self.assertEqual(os.listdir(path), ['.mh_sequences']) + + def test_lock_unlock_no_dot_mh_sequences_file(self): + path = os.path.join(self._path, 'foo.bar') + os.mkdir(path) + box = self._factory(path) + self.assertEqual(os.listdir(path), []) + box.lock() + box.unlock() + self.assertEqual(os.listdir(path), ['.mh_sequences']) + def test_issue2625(self): msg0 = mailbox.MHMessage(self._template % 0) msg0.add_sequence('foo') @@ -1631,18 +1669,23 @@ def test_initialize_with_unixfrom(self): msg = mailbox.Message(_sample_message) msg.set_unixfrom('From foo@bar blah') msg = mailbox.mboxMessage(msg) - self.assertEqual(msg.get_from(), 'foo@bar blah', msg.get_from()) + self.assertEqual(msg.get_from(), 'foo@bar blah') + self.assertEqual(msg.get_unixfrom(), 'From foo@bar blah') def test_from(self): # Get and set "From " line msg = mailbox.mboxMessage(_sample_message) self._check_from(msg) + self.assertIsNone(msg.get_unixfrom()) msg.set_from('foo bar') self.assertEqual(msg.get_from(), 'foo bar') + self.assertIsNone(msg.get_unixfrom()) msg.set_from('foo@bar', True) self._check_from(msg, 'foo@bar') + self.assertIsNone(msg.get_unixfrom()) msg.set_from('blah@temp', time.localtime()) self._check_from(msg, 'blah@temp') + self.assertIsNone(msg.get_unixfrom()) def test_flags(self): # Use get_flags(), set_flags(), add_flag(), remove_flag() @@ -1830,6 +1873,7 @@ def test_maildir_to_mboxmmdf(self): self.assertEqual(msg.get_flags(), result) self.assertEqual(msg.get_from(), 'MAILER-DAEMON %s' % time.asctime(time.gmtime(0.0))) + self.assertIsNone(msg.get_unixfrom()) msg_maildir.set_subdir('cur') self.assertEqual(class_(msg_maildir).get_flags(), 'RODFA') @@ -1878,10 +1922,12 @@ def test_mboxmmdf_to_mboxmmdf(self): msg_mboxMMDF = class_(_sample_message) msg_mboxMMDF.set_flags('RODFA') msg_mboxMMDF.set_from('foo@bar') + self.assertIsNone(msg_mboxMMDF.get_unixfrom()) for class2_ in (mailbox.mboxMessage, mailbox.MMDFMessage): msg2 = class2_(msg_mboxMMDF) self.assertEqual(msg2.get_flags(), 'RODFA') self.assertEqual(msg2.get_from(), 'foo@bar') + self.assertIsNone(msg2.get_unixfrom()) def test_mboxmmdf_to_mh(self): # Convert mboxMessage and MMDFMessage to MHMessage diff --git a/Lib/test/test_marshal.py b/Lib/test/test_marshal.py index 3d9d6d5d0aca34..615568e6af2102 100644 --- a/Lib/test/test_marshal.py +++ b/Lib/test/test_marshal.py @@ -1,5 +1,5 @@ from test import support -from test.support import os_helper, requires_debug_ranges +from test.support import is_apple_mobile, os_helper, requires_debug_ranges from test.support.script_helper import assert_python_ok import array import io @@ -129,6 +129,32 @@ def test_different_filenames(self): self.assertEqual(co1.co_filename, "f1") self.assertEqual(co2.co_filename, "f2") + def test_no_allow_code(self): + data = {'a': [({0},)]} + dump = marshal.dumps(data, allow_code=False) + self.assertEqual(marshal.loads(dump, allow_code=False), data) + + f = io.BytesIO() + marshal.dump(data, f, allow_code=False) + f.seek(0) + self.assertEqual(marshal.load(f, allow_code=False), data) + + co = ExceptionTestCase.test_exceptions.__code__ + data = {'a': [({co, 0},)]} + dump = marshal.dumps(data, allow_code=True) + self.assertEqual(marshal.loads(dump, allow_code=True), data) + with self.assertRaises(ValueError): + marshal.dumps(data, allow_code=False) + with self.assertRaises(ValueError): + marshal.loads(dump, allow_code=False) + + marshal.dump(data, io.BytesIO(), allow_code=True) + self.assertEqual(marshal.load(io.BytesIO(dump), allow_code=True), data) + with self.assertRaises(ValueError): + marshal.dump(data, io.BytesIO(), allow_code=False) + with self.assertRaises(ValueError): + marshal.load(io.BytesIO(dump), allow_code=False) + @requires_debug_ranges() def test_minimal_linetable_with_no_debug_ranges(self): # Make sure when demarshalling objects with `-X no_debug_ranges` @@ -260,7 +286,7 @@ def test_recursion_limit(self): #if os.name == 'nt' and support.Py_DEBUG: if os.name == 'nt': MAX_MARSHAL_STACK_DEPTH = 1000 - elif sys.platform == 'wasi': + elif sys.platform == 'wasi' or is_apple_mobile: MAX_MARSHAL_STACK_DEPTH = 1500 else: MAX_MARSHAL_STACK_DEPTH = 2000 diff --git a/Lib/test/test_mimetypes.py b/Lib/test/test_mimetypes.py index d64aee71fc48b1..01bba0ac2eed5a 100644 --- a/Lib/test/test_mimetypes.py +++ b/Lib/test/test_mimetypes.py @@ -96,14 +96,12 @@ def test_non_standard_types(self): # First try strict eq(self.db.guess_type('foo.xul', strict=True), (None, None)) eq(self.db.guess_extension('image/jpg', strict=True), None) - eq(self.db.guess_extension('image/webp', strict=True), None) # And then non-strict eq(self.db.guess_type('foo.xul', strict=False), ('text/xul', None)) eq(self.db.guess_type('foo.XUL', strict=False), ('text/xul', None)) eq(self.db.guess_type('foo.invalid', strict=False), (None, None)) eq(self.db.guess_extension('image/jpg', strict=False), '.jpg') eq(self.db.guess_extension('image/JPG', strict=False), '.jpg') - eq(self.db.guess_extension('image/webp', strict=False), '.webp') def test_filename_with_url_delimiters(self): # bpo-38449: URL delimiters cases should be handled also. @@ -183,6 +181,7 @@ def check_extensions(): self.assertEqual(mimetypes.guess_extension('application/xml'), '.xsl') self.assertEqual(mimetypes.guess_extension('audio/mpeg'), '.mp3') self.assertEqual(mimetypes.guess_extension('image/avif'), '.avif') + self.assertEqual(mimetypes.guess_extension('image/webp'), '.webp') self.assertEqual(mimetypes.guess_extension('image/jpeg'), '.jpg') self.assertEqual(mimetypes.guess_extension('image/tiff'), '.tiff') self.assertEqual(mimetypes.guess_extension('message/rfc822'), '.eml') diff --git a/Lib/test/test_mmap.py b/Lib/test/test_mmap.py index 866ede5b83dff0..ac759757d24659 100644 --- a/Lib/test/test_mmap.py +++ b/Lib/test/test_mmap.py @@ -1,9 +1,10 @@ from test.support import ( - requires, _2G, _4G, gc_collect, cpython_only, is_emscripten + requires, _2G, _4G, gc_collect, cpython_only, is_emscripten, is_apple, ) from test.support.import_helper import import_module from test.support.os_helper import TESTFN, unlink import unittest +import errno import os import re import itertools @@ -266,6 +267,62 @@ def test_access_parameter(self): self.assertRaises(TypeError, m.write_byte, 0) m.close() + @unittest.skipIf(os.name == 'nt', 'trackfd not present on Windows') + def test_trackfd_parameter(self): + size = 64 + with open(TESTFN, "wb") as f: + f.write(b"a"*size) + for close_original_fd in True, False: + with self.subTest(close_original_fd=close_original_fd): + with open(TESTFN, "r+b") as f: + with mmap.mmap(f.fileno(), size, trackfd=False) as m: + if close_original_fd: + f.close() + self.assertEqual(len(m), size) + with self.assertRaises(OSError) as err_cm: + m.size() + self.assertEqual(err_cm.exception.errno, errno.EBADF) + with self.assertRaises(ValueError): + m.resize(size * 2) + with self.assertRaises(ValueError): + m.resize(size // 2) + self.assertEqual(m.closed, False) + + # Smoke-test other API + m.write_byte(ord('X')) + m[2] = ord('Y') + m.flush() + with open(TESTFN, "rb") as f: + self.assertEqual(f.read(4), b'XaYa') + self.assertEqual(m.tell(), 1) + m.seek(0) + self.assertEqual(m.tell(), 0) + self.assertEqual(m.read_byte(), ord('X')) + + self.assertEqual(m.closed, True) + self.assertEqual(os.stat(TESTFN).st_size, size) + + @unittest.skipIf(os.name == 'nt', 'trackfd not present on Windows') + def test_trackfd_neg1(self): + size = 64 + with mmap.mmap(-1, size, trackfd=False) as m: + with self.assertRaises(OSError): + m.size() + with self.assertRaises(ValueError): + m.resize(size // 2) + self.assertEqual(len(m), size) + m[0] = ord('a') + assert m[0] == ord('a') + + @unittest.skipIf(os.name != 'nt', 'trackfd only fails on Windows') + def test_no_trackfd_parameter_on_windows(self): + # 'trackffd' is an invalid keyword argument for this function + size = 64 + with self.assertRaises(TypeError): + mmap.mmap(-1, size, trackfd=True) + with self.assertRaises(TypeError): + mmap.mmap(-1, size, trackfd=False) + def test_bad_file_desc(self): # Try opening a bad file descriptor... self.assertRaises(OSError, mmap.mmap, -2, 4096) @@ -672,14 +729,16 @@ def test_tagname(self): m2.close() m1.close() + with self.assertRaisesRegex(TypeError, 'tagname'): + mmap.mmap(-1, 8, tagname=1) + @cpython_only @unittest.skipUnless(os.name == 'nt', 'requires Windows') def test_sizeof(self): m1 = mmap.mmap(-1, 100) tagname = random_tagname() m2 = mmap.mmap(-1, 100, tagname=tagname) - self.assertEqual(sys.getsizeof(m2), - sys.getsizeof(m1) + len(tagname) + 1) + self.assertGreater(sys.getsizeof(m2), sys.getsizeof(m1)) @unittest.skipUnless(os.name == 'nt', 'requires Windows') def test_crasher_on_windows(self): @@ -1008,7 +1067,7 @@ def tearDown(self): unlink(TESTFN) def _make_test_file(self, num_zeroes, tail): - if sys.platform[:3] == 'win' or sys.platform == 'darwin': + if sys.platform[:3] == 'win' or is_apple: requires('largefile', 'test requires %s bytes and a long time to run' % str(0x180000000)) f = open(TESTFN, 'w+b') diff --git a/Lib/test/test_module/__init__.py b/Lib/test/test_module/__init__.py index d49c44df4d839d..98d1cbe824df12 100644 --- a/Lib/test/test_module/__init__.py +++ b/Lib/test/test_module/__init__.py @@ -30,7 +30,7 @@ def test_uninitialized(self): self.fail("__name__ = %s" % repr(s)) except AttributeError: pass - self.assertEqual(foo.__doc__, ModuleType.__doc__) + self.assertEqual(foo.__doc__, ModuleType.__doc__ or '') def test_uninitialized_missing_getattr(self): # Issue 8297 diff --git a/Lib/test/test_monitoring.py b/Lib/test/test_monitoring.py index a2efbc95f556c6..60b6326bfbad5e 100644 --- a/Lib/test/test_monitoring.py +++ b/Lib/test/test_monitoring.py @@ -750,7 +750,7 @@ class UnwindRecorder(ExceptionRecorder): event_type = E.PY_UNWIND def __call__(self, code, offset, exc): - self.events.append(("unwind", type(exc))) + self.events.append(("unwind", type(exc), code.co_name)) class ExceptionHandledRecorder(ExceptionRecorder): @@ -766,8 +766,27 @@ class ThrowRecorder(ExceptionRecorder): def __call__(self, code, offset, exc): self.events.append(("throw", type(exc))) -class ExceptionMonitoringTest(CheckEvents): +class CallRecorder: + + event_type = E.CALL + + def __init__(self, events): + self.events = events + + def __call__(self, code, offset, func, arg): + self.events.append(("call", func.__name__, arg)) + +class ReturnRecorder: + + event_type = E.PY_RETURN + + def __init__(self, events): + self.events = events + def __call__(self, code, offset, val): + self.events.append(("return", code.co_name, val)) + +class ExceptionMonitoringTest(CheckEvents): exception_recorders = ( ExceptionRecorder, @@ -936,26 +955,48 @@ def func(): ) self.assertEqual(events[0], ("throw", IndexError)) -class LineRecorder: + def test_no_unwind_for_shim_frame(self): - event_type = E.LINE + class B: + def __init__(self): + raise ValueError() + + def f(): + try: + return B() + except ValueError: + pass + for _ in range(100): + f() + recorders = ( + ReturnRecorder, + UnwindRecorder + ) + events = self.get_events(f, TEST_TOOL, recorders) + adaptive_insts = dis.get_instructions(f, adaptive=True) + self.assertIn( + "CALL_ALLOC_AND_ENTER_INIT", + [i.opname for i in adaptive_insts] + ) + #There should be only one unwind event + expected = [ + ('unwind', ValueError, '__init__'), + ('return', 'f', None), + ] - def __init__(self, events): - self.events = events + self.assertEqual(events, expected) - def __call__(self, code, line): - self.events.append(("line", code.co_name, line - code.co_firstlineno)) +class LineRecorder: -class CallRecorder: + event_type = E.LINE - event_type = E.CALL def __init__(self, events): self.events = events - def __call__(self, code, offset, func, arg): - self.events.append(("call", func.__name__, arg)) + def __call__(self, code, line): + self.events.append(("line", code.co_name, line - code.co_firstlineno)) class CEventRecorder: @@ -1351,15 +1392,6 @@ class BranchRecorder(JumpRecorder): event_type = E.BRANCH name = "branch" -class ReturnRecorder: - - event_type = E.PY_RETURN - - def __init__(self, events): - self.events = events - - def __call__(self, code, offset, val): - self.events.append(("return", val)) JUMP_AND_BRANCH_RECORDERS = JumpRecorder, BranchRecorder @@ -1434,9 +1466,8 @@ def func(): ('branch', 'func', 4, 4), ('line', 'func', 5), ('line', 'meth', 1), - ('jump', 'func', 5, 5), - ('jump', 'func', 5, '[offset=114]'), - ('branch', 'func', '[offset=120]', '[offset=124]'), + ('jump', 'func', 5, '[offset=118]'), + ('branch', 'func', '[offset=122]', '[offset=126]'), ('line', 'get_events', 11)]) self.check_events(func, recorders = FLOW_AND_LINE_RECORDERS, expected = [ @@ -1449,11 +1480,10 @@ def func(): ('branch', 'func', 4, 4), ('line', 'func', 5), ('line', 'meth', 1), - ('return', None), - ('jump', 'func', 5, 5), - ('jump', 'func', 5, '[offset=114]'), - ('branch', 'func', '[offset=120]', '[offset=124]'), - ('return', None), + ('return', 'meth', None), + ('jump', 'func', 5, '[offset=118]'), + ('branch', 'func', '[offset=122]', '[offset=126]'), + ('return', 'func', None), ('line', 'get_events', 11)]) class TestLoadSuperAttr(CheckEvents): diff --git a/Lib/test/test_ntpath.py b/Lib/test/test_ntpath.py index bf990ed36fbcae..9cb03e3cd5de8d 100644 --- a/Lib/test/test_ntpath.py +++ b/Lib/test/test_ntpath.py @@ -227,10 +227,18 @@ def test_split(self): tester('ntpath.split("//conky/mountpoint/")', ('//conky/mountpoint/', '')) def test_isabs(self): + tester('ntpath.isabs("foo\\bar")', 0) + tester('ntpath.isabs("foo/bar")', 0) tester('ntpath.isabs("c:\\")', 1) + tester('ntpath.isabs("c:\\foo\\bar")', 1) + tester('ntpath.isabs("c:/foo/bar")', 1) tester('ntpath.isabs("\\\\conky\\mountpoint\\")', 1) - tester('ntpath.isabs("\\foo")', 1) - tester('ntpath.isabs("\\foo\\bar")', 1) + + # gh-44626: paths with only a drive or root are not absolute. + tester('ntpath.isabs("\\foo\\bar")', 0) + tester('ntpath.isabs("/foo/bar")', 0) + tester('ntpath.isabs("c:foo\\bar")', 0) + tester('ntpath.isabs("c:foo/bar")', 0) # gh-96290: normal UNC paths and device paths without trailing backslashes tester('ntpath.isabs("\\\\conky\\mountpoint")', 1) @@ -973,6 +981,62 @@ def test_ismount(self): self.assertTrue(ntpath.ismount(b"\\\\localhost\\c$")) self.assertTrue(ntpath.ismount(b"\\\\localhost\\c$\\")) + def test_isreserved(self): + self.assertFalse(ntpath.isreserved('')) + self.assertFalse(ntpath.isreserved('.')) + self.assertFalse(ntpath.isreserved('..')) + self.assertFalse(ntpath.isreserved('/')) + self.assertFalse(ntpath.isreserved('/foo/bar')) + # A name that ends with a space or dot is reserved. + self.assertTrue(ntpath.isreserved('foo.')) + self.assertTrue(ntpath.isreserved('foo ')) + # ASCII control characters are reserved. + self.assertTrue(ntpath.isreserved('\foo')) + # Wildcard characters, colon, and pipe are reserved. + self.assertTrue(ntpath.isreserved('foo*bar')) + self.assertTrue(ntpath.isreserved('foo?bar')) + self.assertTrue(ntpath.isreserved('foo"bar')) + self.assertTrue(ntpath.isreserved('foobar')) + self.assertTrue(ntpath.isreserved('foo:bar')) + self.assertTrue(ntpath.isreserved('foo|bar')) + # Case-insensitive DOS-device names are reserved. + self.assertTrue(ntpath.isreserved('nul')) + self.assertTrue(ntpath.isreserved('aux')) + self.assertTrue(ntpath.isreserved('prn')) + self.assertTrue(ntpath.isreserved('con')) + self.assertTrue(ntpath.isreserved('conin$')) + self.assertTrue(ntpath.isreserved('conout$')) + # COM/LPT + 1-9 or + superscript 1-3 are reserved. + self.assertTrue(ntpath.isreserved('COM1')) + self.assertTrue(ntpath.isreserved('LPT9')) + self.assertTrue(ntpath.isreserved('com\xb9')) + self.assertTrue(ntpath.isreserved('com\xb2')) + self.assertTrue(ntpath.isreserved('lpt\xb3')) + # DOS-device name matching ignores characters after a dot or + # a colon and also ignores trailing spaces. + self.assertTrue(ntpath.isreserved('NUL.txt')) + self.assertTrue(ntpath.isreserved('PRN ')) + self.assertTrue(ntpath.isreserved('AUX .txt')) + self.assertTrue(ntpath.isreserved('COM1:bar')) + self.assertTrue(ntpath.isreserved('LPT9 :bar')) + # DOS-device names are only matched at the beginning + # of a path component. + self.assertFalse(ntpath.isreserved('bar.com9')) + self.assertFalse(ntpath.isreserved('bar.lpt9')) + # The entire path is checked, except for the drive. + self.assertTrue(ntpath.isreserved('c:/bar/baz/NUL')) + self.assertTrue(ntpath.isreserved('c:/NUL/bar/baz')) + self.assertFalse(ntpath.isreserved('//./NUL')) + # Bytes are supported. + self.assertFalse(ntpath.isreserved(b'')) + self.assertFalse(ntpath.isreserved(b'.')) + self.assertFalse(ntpath.isreserved(b'..')) + self.assertFalse(ntpath.isreserved(b'/')) + self.assertFalse(ntpath.isreserved(b'/foo/bar')) + self.assertTrue(ntpath.isreserved(b'foo.')) + self.assertTrue(ntpath.isreserved(b'nul')) + def assertEqualCI(self, s1, s2): """Assert that two strings are equal ignoring case differences.""" self.assertEqual(s1.lower(), s2.lower()) diff --git a/Lib/test/test_optimizer.py b/Lib/test/test_optimizer.py new file mode 100644 index 00000000000000..c8554c40df4b2d --- /dev/null +++ b/Lib/test/test_optimizer.py @@ -0,0 +1,78 @@ +import unittest +import types +from test.support import import_helper + + +_testinternalcapi = import_helper.import_module("_testinternalcapi") + + +class TestRareEventCounters(unittest.TestCase): + def test_set_class(self): + class A: + pass + class B: + pass + a = A() + + orig_counter = _testinternalcapi.get_rare_event_counters()["set_class"] + a.__class__ = B + self.assertEqual( + orig_counter + 1, + _testinternalcapi.get_rare_event_counters()["set_class"] + ) + + def test_set_bases(self): + class A: + pass + class B: + pass + class C(B): + pass + + orig_counter = _testinternalcapi.get_rare_event_counters()["set_bases"] + C.__bases__ = (A,) + self.assertEqual( + orig_counter + 1, + _testinternalcapi.get_rare_event_counters()["set_bases"] + ) + + def test_set_eval_frame_func(self): + orig_counter = _testinternalcapi.get_rare_event_counters()["set_eval_frame_func"] + _testinternalcapi.set_eval_frame_record([]) + self.assertEqual( + orig_counter + 1, + _testinternalcapi.get_rare_event_counters()["set_eval_frame_func"] + ) + _testinternalcapi.set_eval_frame_default() + + def test_builtin_dict(self): + orig_counter = _testinternalcapi.get_rare_event_counters()["builtin_dict"] + if isinstance(__builtins__, types.ModuleType): + builtins = __builtins__.__dict__ + else: + builtins = __builtins__ + builtins["FOO"] = 42 + self.assertEqual( + orig_counter + 1, + _testinternalcapi.get_rare_event_counters()["builtin_dict"] + ) + del builtins["FOO"] + + def test_func_modification(self): + def func(x=0): + pass + + for attribute in ( + "__code__", + "__defaults__", + "__kwdefaults__" + ): + orig_counter = _testinternalcapi.get_rare_event_counters()["func_modification"] + setattr(func, attribute, getattr(func, attribute)) + self.assertEqual( + orig_counter + 1, + _testinternalcapi.get_rare_event_counters()["func_modification"] + ) + +if __name__ == "__main__": + unittest.main() diff --git a/Lib/test/test_os.py b/Lib/test/test_os.py index c66c5797471413..2c8823ae47c726 100644 --- a/Lib/test/test_os.py +++ b/Lib/test/test_os.py @@ -2195,12 +2195,15 @@ def test_chmod(self): class TestInvalidFD(unittest.TestCase): singles = ["fchdir", "dup", "fdatasync", "fstat", "fstatvfs", "fsync", "tcgetpgrp", "ttyname"] + singles_fildes = {"fchdir", "fdatasync", "fsync"} #singles.append("close") #We omit close because it doesn't raise an exception on some platforms def get_single(f): def helper(self): if hasattr(os, f): self.check(getattr(os, f)) + if f in self.singles_fildes: + self.check_bool(getattr(os, f)) return helper for f in singles: locals()["test_"+f] = get_single(f) @@ -2214,8 +2217,16 @@ def check(self, f, *args, **kwargs): self.fail("%r didn't raise an OSError with a bad file descriptor" % f) + def check_bool(self, f, *args, **kwargs): + with warnings.catch_warnings(): + warnings.simplefilter("error", RuntimeWarning) + for fd in False, True: + with self.assertRaises(RuntimeWarning): + f(fd, *args, **kwargs) + def test_fdopen(self): self.check(os.fdopen, encoding="utf-8") + self.check_bool(os.fdopen, encoding="utf-8") @unittest.skipUnless(hasattr(os, 'isatty'), 'test needs os.isatty()') def test_isatty(self): @@ -2277,11 +2288,14 @@ def test_fchown(self): def test_fpathconf(self): self.check(os.pathconf, "PC_NAME_MAX") self.check(os.fpathconf, "PC_NAME_MAX") + self.check_bool(os.pathconf, "PC_NAME_MAX") + self.check_bool(os.fpathconf, "PC_NAME_MAX") @unittest.skipUnless(hasattr(os, 'ftruncate'), 'test needs os.ftruncate()') def test_ftruncate(self): self.check(os.truncate, 0) self.check(os.ftruncate, 0) + self.check_bool(os.truncate, 0) @unittest.skipUnless(hasattr(os, 'lseek'), 'test needs os.lseek()') def test_lseek(self): @@ -3085,6 +3099,65 @@ def test_stat_unlink_race(self): except subprocess.TimeoutExpired: proc.terminate() + @support.requires_subprocess() + def test_stat_inaccessible_file(self): + filename = os_helper.TESTFN + ICACLS = os.path.expandvars(r"%SystemRoot%\System32\icacls.exe") + + with open(filename, "wb") as f: + f.write(b'Test data') + + stat1 = os.stat(filename) + + try: + # Remove all permissions from the file + subprocess.check_output([ICACLS, filename, "/inheritance:r"], + stderr=subprocess.STDOUT) + except subprocess.CalledProcessError as ex: + if support.verbose: + print(ICACLS, filename, "/inheritance:r", "failed.") + print(ex.stdout.decode("oem", "replace").rstrip()) + try: + os.unlink(filename) + except OSError: + pass + self.skipTest("Unable to create inaccessible file") + + def cleanup(): + # Give delete permission. We are the file owner, so we can do this + # even though we removed all permissions earlier. + subprocess.check_output([ICACLS, filename, "/grant", "Everyone:(D)"], + stderr=subprocess.STDOUT) + os.unlink(filename) + + self.addCleanup(cleanup) + + if support.verbose: + print("File:", filename) + print("stat with access:", stat1) + + # First test - we shouldn't raise here, because we still have access to + # the directory and can extract enough information from its metadata. + stat2 = os.stat(filename) + + if support.verbose: + print(" without access:", stat2) + + # We may not get st_dev/st_ino, so ensure those are 0 or match + self.assertIn(stat2.st_dev, (0, stat1.st_dev)) + self.assertIn(stat2.st_ino, (0, stat1.st_ino)) + + # st_mode and st_size should match (for a normal file, at least) + self.assertEqual(stat1.st_mode, stat2.st_mode) + self.assertEqual(stat1.st_size, stat2.st_size) + + # st_ctime and st_mtime should be the same + self.assertEqual(stat1.st_ctime, stat2.st_ctime) + self.assertEqual(stat1.st_mtime, stat2.st_mtime) + + # st_atime should be the same or later + self.assertGreaterEqual(stat1.st_atime, stat2.st_atime) + @os_helper.skip_unless_symlink class NonLocalSymlinkTests(unittest.TestCase): @@ -3789,6 +3862,7 @@ def test_does_not_crash(self): self.assertGreaterEqual(size.columns, 0) self.assertGreaterEqual(size.lines, 0) + @support.requires_subprocess() def test_stty_match(self): """Check if stty returns the same results @@ -4477,13 +4551,105 @@ def test_dup2(self): self.assertEqual(os.dup2(fd, fd3, inheritable=False), fd3) self.assertFalse(os.get_inheritable(fd3)) - @unittest.skipUnless(hasattr(os, 'openpty'), "need os.openpty()") +@unittest.skipUnless(hasattr(os, 'openpty'), "need os.openpty()") +class PseudoterminalTests(unittest.TestCase): + def open_pty(self): + """Open a pty fd-pair, and schedule cleanup for it""" + main_fd, second_fd = os.openpty() + self.addCleanup(os.close, main_fd) + self.addCleanup(os.close, second_fd) + return main_fd, second_fd + def test_openpty(self): - master_fd, slave_fd = os.openpty() - self.addCleanup(os.close, master_fd) - self.addCleanup(os.close, slave_fd) - self.assertEqual(os.get_inheritable(master_fd), False) - self.assertEqual(os.get_inheritable(slave_fd), False) + main_fd, second_fd = self.open_pty() + self.assertEqual(os.get_inheritable(main_fd), False) + self.assertEqual(os.get_inheritable(second_fd), False) + + @unittest.skipUnless(hasattr(os, 'ptsname'), "need os.ptsname()") + @unittest.skipUnless(hasattr(os, 'O_RDWR'), "need os.O_RDWR") + @unittest.skipUnless(hasattr(os, 'O_NOCTTY'), "need os.O_NOCTTY") + def test_open_via_ptsname(self): + main_fd, second_fd = self.open_pty() + second_path = os.ptsname(main_fd) + reopened_second_fd = os.open(second_path, os.O_RDWR|os.O_NOCTTY) + self.addCleanup(os.close, reopened_second_fd) + os.write(reopened_second_fd, b'foo') + self.assertEqual(os.read(main_fd, 3), b'foo') + + @unittest.skipUnless(hasattr(os, 'posix_openpt'), "need os.posix_openpt()") + @unittest.skipUnless(hasattr(os, 'grantpt'), "need os.grantpt()") + @unittest.skipUnless(hasattr(os, 'unlockpt'), "need os.unlockpt()") + @unittest.skipUnless(hasattr(os, 'ptsname'), "need os.ptsname()") + @unittest.skipUnless(hasattr(os, 'O_RDWR'), "need os.O_RDWR") + @unittest.skipUnless(hasattr(os, 'O_NOCTTY'), "need os.O_NOCTTY") + def test_posix_pty_functions(self): + mother_fd = os.posix_openpt(os.O_RDWR|os.O_NOCTTY) + self.addCleanup(os.close, mother_fd) + os.grantpt(mother_fd) + os.unlockpt(mother_fd) + son_path = os.ptsname(mother_fd) + son_fd = os.open(son_path, os.O_RDWR|os.O_NOCTTY) + self.addCleanup(os.close, son_fd) + self.assertEqual(os.ptsname(mother_fd), os.ttyname(son_fd)) + + @unittest.skipUnless(hasattr(os, 'spawnl'), "need os.spawnl()") + @support.requires_subprocess() + def test_pipe_spawnl(self): + # gh-77046: On Windows, os.pipe() file descriptors must be created with + # _O_NOINHERIT to make them non-inheritable. UCRT has no public API to + # get (_osfile(fd) & _O_NOINHERIT), so use a functional test. + # + # Make sure that fd is not inherited by a child process created by + # os.spawnl(): get_osfhandle() and dup() must fail with EBADF. + + fd, fd2 = os.pipe() + self.addCleanup(os.close, fd) + self.addCleanup(os.close, fd2) + + code = textwrap.dedent(f""" + import errno + import os + import test.support + try: + import msvcrt + except ImportError: + msvcrt = None + + fd = {fd} + + with test.support.SuppressCrashReport(): + if msvcrt is not None: + try: + handle = msvcrt.get_osfhandle(fd) + except OSError as exc: + if exc.errno != errno.EBADF: + raise + # get_osfhandle(fd) failed with EBADF as expected + else: + raise Exception("get_osfhandle() must fail") + + try: + fd3 = os.dup(fd) + except OSError as exc: + if exc.errno != errno.EBADF: + raise + # os.dup(fd) failed with EBADF as expected + else: + os.close(fd3) + raise Exception("dup must fail") + """) + + filename = os_helper.TESTFN + self.addCleanup(os_helper.unlink, os_helper.TESTFN) + with open(filename, "w") as fp: + print(code, file=fp, end="") + + executable = sys.executable + cmd = [executable, filename] + if os.name == "nt" and " " in cmd[0]: + cmd[0] = f'"{cmd[0]}"' + exitcode = os.spawnl(os.P_WAIT, executable, *cmd) + self.assertEqual(exitcode, 0) class PathTConverterTests(unittest.TestCase): diff --git a/Lib/test/test_pathlib/test_pathlib.py b/Lib/test/test_pathlib/test_pathlib.py index 00cfdd37e568a6..2b166451243775 100644 --- a/Lib/test/test_pathlib/test_pathlib.py +++ b/Lib/test/test_pathlib/test_pathlib.py @@ -2,8 +2,10 @@ import os import sys import errno +import ntpath import pathlib import pickle +import posixpath import socket import stat import tempfile @@ -13,9 +15,11 @@ from test.support import import_helper from test.support import is_emscripten, is_wasi +from test.support import set_recursion_limit from test.support import os_helper from test.support.os_helper import TESTFN, FakePath from test.test_pathlib import test_pathlib_abc +from test.test_pathlib.test_pathlib_abc import needs_posix, needs_windows, needs_symlinks try: import grp, pwd @@ -23,11 +27,6 @@ grp = pwd = None -only_nt = unittest.skipIf(os.name != 'nt', - 'test requires a Windows-compatible system') -only_posix = unittest.skipIf(os.name == 'nt', - 'test requires a POSIX-compatible system') - root_in_posix = False if hasattr(os, 'geteuid'): root_in_posix = (os.geteuid() == 0) @@ -39,6 +38,66 @@ class PurePathTest(test_pathlib_abc.DummyPurePathTest): cls = pathlib.PurePath + # Make sure any symbolic links in the base test path are resolved. + base = os.path.realpath(TESTFN) + + # Keys are canonical paths, values are list of tuples of arguments + # supposed to produce equal paths. + equivalences = { + 'a/b': [ + ('a', 'b'), ('a/', 'b'), ('a', 'b/'), ('a/', 'b/'), + ('a/b/',), ('a//b',), ('a//b//',), + # Empty components get removed. + ('', 'a', 'b'), ('a', '', 'b'), ('a', 'b', ''), + ], + '/b/c/d': [ + ('a', '/b/c', 'd'), ('/a', '/b/c', 'd'), + # Empty components get removed. + ('/', 'b', '', 'c/d'), ('/', '', 'b/c/d'), ('', '/b/c/d'), + ], + } + + def test_concrete_class(self): + if self.cls is pathlib.PurePath: + expected = pathlib.PureWindowsPath if os.name == 'nt' else pathlib.PurePosixPath + else: + expected = self.cls + p = self.cls('a') + self.assertIs(type(p), expected) + + def test_concrete_pathmod(self): + if self.cls is pathlib.PurePosixPath: + expected = posixpath + elif self.cls is pathlib.PureWindowsPath: + expected = ntpath + else: + expected = os.path + p = self.cls('a') + self.assertIs(p.pathmod, expected) + + def test_different_pathmods_unequal(self): + p = self.cls('a') + if p.pathmod is posixpath: + q = pathlib.PureWindowsPath('a') + else: + q = pathlib.PurePosixPath('a') + self.assertNotEqual(p, q) + + def test_different_pathmods_unordered(self): + p = self.cls('a') + if p.pathmod is posixpath: + q = pathlib.PureWindowsPath('a') + else: + q = pathlib.PurePosixPath('a') + with self.assertRaises(TypeError): + p < q + with self.assertRaises(TypeError): + p <= q + with self.assertRaises(TypeError): + p > q + with self.assertRaises(TypeError): + p >= q + def test_constructor_nested(self): P = self.cls P(FakePath("a/b/c")) @@ -48,6 +107,47 @@ def test_constructor_nested(self): self.assertEqual(P(P('a'), P('b'), P('c')), P(FakePath("a/b/c"))) self.assertEqual(P(P('./a:b')), P('./a:b')) + def _check_parse_path(self, raw_path, *expected): + sep = self.pathmod.sep + actual = self.cls._parse_path(raw_path.replace('/', sep)) + self.assertEqual(actual, expected) + if altsep := self.pathmod.altsep: + actual = self.cls._parse_path(raw_path.replace('/', altsep)) + self.assertEqual(actual, expected) + + def test_parse_path_common(self): + check = self._check_parse_path + sep = self.pathmod.sep + check('', '', '', []) + check('a', '', '', ['a']) + check('a/', '', '', ['a']) + check('a/b', '', '', ['a', 'b']) + check('a/b/', '', '', ['a', 'b']) + check('a/b/c/d', '', '', ['a', 'b', 'c', 'd']) + check('a/b//c/d', '', '', ['a', 'b', 'c', 'd']) + check('a/b/c/d', '', '', ['a', 'b', 'c', 'd']) + check('.', '', '', []) + check('././b', '', '', ['b']) + check('a/./b', '', '', ['a', 'b']) + check('a/./.', '', '', ['a']) + check('/a/b', '', sep, ['a', 'b']) + + def test_empty_path(self): + # The empty path points to '.' + p = self.cls('') + self.assertEqual(str(p), '.') + # Special case for the empty path. + self._check_str('.', ('',)) + + def test_parts_interning(self): + P = self.cls + p = P('/usr/bin/foo') + q = P('/usr/local/bin') + # 'usr' + self.assertIs(p.parts[1], q.parts[1]) + # 'bin' + self.assertIs(p.parts[2], q.parts[3]) + def test_join_nested(self): P = self.cls p = P('a/b').joinpath(P('c')) @@ -60,14 +160,28 @@ def test_div_nested(self): def test_pickling_common(self): P = self.cls - p = P('/a/b') - for proto in range(0, pickle.HIGHEST_PROTOCOL + 1): - dumped = pickle.dumps(p, proto) - pp = pickle.loads(dumped) - self.assertIs(pp.__class__, p.__class__) - self.assertEqual(pp, p) - self.assertEqual(hash(pp), hash(p)) - self.assertEqual(str(pp), str(p)) + for pathstr in ('a', 'a/', 'a/b', 'a/b/c', '/', '/a/b', '/a/b/c', 'a/b/c/'): + with self.subTest(pathstr=pathstr): + p = P(pathstr) + for proto in range(0, pickle.HIGHEST_PROTOCOL + 1): + dumped = pickle.dumps(p, proto) + pp = pickle.loads(dumped) + self.assertIs(pp.__class__, p.__class__) + self.assertEqual(pp, p) + self.assertEqual(hash(pp), hash(p)) + self.assertEqual(str(pp), str(p)) + + def test_repr_common(self): + for pathstr in ('a', 'a/b', 'a/b/c', '/', '/a/b', '/a/b/c'): + with self.subTest(pathstr=pathstr): + p = self.cls(pathstr) + clsname = p.__class__.__name__ + r = repr(p) + # The repr() is in the form ClassName("forward-slashes path"). + self.assertTrue(r.startswith(clsname + '('), r) + self.assertTrue(r.endswith(')'), r) + inner = r[len(clsname) + 1 : -1] + self.assertEqual(eval(inner), p.as_posix()) def test_fspath_common(self): P = self.cls @@ -75,7 +189,7 @@ def test_fspath_common(self): self._check_str(p.__fspath__(), ('a/b',)) self._check_str(os.fspath(p), ('a/b',)) - def test_bytes(self): + def test_bytes_exc_message(self): P = self.cls message = (r"argument should be a str or an os\.PathLike object " r"where __fspath__ returns a str, not 'bytes'") @@ -85,28 +199,45 @@ def test_bytes(self): P(b'a', 'b') with self.assertRaisesRegex(TypeError, message): P('a', b'b') - with self.assertRaises(TypeError): - P('a').joinpath(b'b') - with self.assertRaises(TypeError): - P('a') / b'b' - with self.assertRaises(TypeError): - b'a' / P('b') - with self.assertRaises(TypeError): - P('a').match(b'b') - with self.assertRaises(TypeError): - P('a').relative_to(b'b') - with self.assertRaises(TypeError): - P('a').with_name(b'b') - with self.assertRaises(TypeError): - P('a').with_stem(b'b') - with self.assertRaises(TypeError): - P('a').with_suffix(b'b') def test_as_bytes_common(self): sep = os.fsencode(self.sep) P = self.cls self.assertEqual(bytes(P('a/b')), b'a' + sep + b'b') + def test_eq_common(self): + P = self.cls + self.assertEqual(P('a/b'), P('a/b')) + self.assertEqual(P('a/b'), P('a', 'b')) + self.assertNotEqual(P('a/b'), P('a')) + self.assertNotEqual(P('a/b'), P('/a/b')) + self.assertNotEqual(P('a/b'), P()) + self.assertNotEqual(P('/a/b'), P('/')) + self.assertNotEqual(P(), P('/')) + self.assertNotEqual(P(), "") + self.assertNotEqual(P(), {}) + self.assertNotEqual(P(), int) + + def test_equivalences(self, equivalences=None): + if equivalences is None: + equivalences = self.equivalences + for k, tuples in equivalences.items(): + canon = k.replace('/', self.sep) + posix = k.replace(self.sep, '/') + if canon != posix: + tuples = tuples + [ + tuple(part.replace('/', self.sep) for part in t) + for t in tuples + ] + tuples.append((posix, )) + pcanon = self.cls(canon) + for t in tuples: + p = self.cls(*t) + self.assertEqual(p, pcanon, "failed with args {}".format(t)) + self.assertEqual(hash(p), hash(pcanon)) + self.assertEqual(str(p), canon) + self.assertEqual(p.as_posix(), posix) + def test_ordering_common(self): # Ordering is tuple-alike. def assertLess(a, b): @@ -153,11 +284,59 @@ def test_repr_roundtrips(self): self.assertEqual(q, p) self.assertEqual(repr(q), r) + def test_name_empty(self): + P = self.cls + self.assertEqual(P('').name, '') + self.assertEqual(P('.').name, '') + self.assertEqual(P('/a/b/.').name, 'b') -class PurePosixPathTest(PurePathTest): - cls = pathlib.PurePosixPath + def test_stem_empty(self): + P = self.cls + self.assertEqual(P('').stem, '') + self.assertEqual(P('.').stem, '') + + def test_with_name_empty(self): + P = self.cls + self.assertRaises(ValueError, P('').with_name, 'd.xml') + self.assertRaises(ValueError, P('.').with_name, 'd.xml') + self.assertRaises(ValueError, P('/').with_name, 'd.xml') + self.assertRaises(ValueError, P('a/b').with_name, '') + self.assertRaises(ValueError, P('a/b').with_name, '.') + + def test_with_stem_empty(self): + P = self.cls + self.assertRaises(ValueError, P('').with_stem, 'd') + self.assertRaises(ValueError, P('.').with_stem, 'd') + self.assertRaises(ValueError, P('/').with_stem, 'd') + self.assertRaises(ValueError, P('a/b').with_stem, '') + self.assertRaises(ValueError, P('a/b').with_stem, '.') + + def test_relative_to_several_args(self): + P = self.cls + p = P('a/b') + with self.assertWarns(DeprecationWarning): + p.relative_to('a', 'b') + p.relative_to('a', 'b', walk_up=True) + + def test_is_relative_to_several_args(self): + P = self.cls + p = P('a/b') + with self.assertWarns(DeprecationWarning): + p.is_relative_to('a', 'b') + + def test_is_reserved_deprecated(self): + P = self.cls + p = P('a/b') + with self.assertWarns(DeprecationWarning): + p.is_reserved() - def test_parse_path(self): + def test_match_empty(self): + P = self.cls + self.assertRaises(ValueError, P('a').match, '') + self.assertRaises(ValueError, P('a').match, '.') + + @needs_posix + def test_parse_path_posix(self): check = self._check_parse_path # Collapsing of excess leading slashes, except for the double-slash # special case. @@ -169,25 +348,21 @@ def test_parse_path(self): check('c:\\a', '', '', ['c:\\a',]) check('\\a', '', '', ['\\a',]) - def test_root(self): - P = self.cls - self.assertEqual(P('/a/b').root, '/') - self.assertEqual(P('///a/b').root, '/') - # POSIX special case for two leading slashes. - self.assertEqual(P('//a/b').root, '//') - - def test_eq(self): + @needs_posix + def test_eq_posix(self): P = self.cls self.assertNotEqual(P('a/b'), P('A/b')) self.assertEqual(P('/a'), P('///a')) self.assertNotEqual(P('/a'), P('//a')) - def test_as_uri(self): + @needs_posix + def test_as_uri_posix(self): P = self.cls self.assertEqual(P('/').as_uri(), 'file:///') self.assertEqual(P('/a/b.c').as_uri(), 'file:///a/b.c') self.assertEqual(P('/a/b%#c').as_uri(), 'file:///a/b%25%23c') + @needs_posix def test_as_uri_non_ascii(self): from urllib.parse import quote_from_bytes P = self.cls @@ -198,61 +373,14 @@ def test_as_uri_non_ascii(self): self.assertEqual(P('/a/b\xe9').as_uri(), 'file:///a/b' + quote_from_bytes(os.fsencode('\xe9'))) - def test_match(self): - P = self.cls - self.assertFalse(P('A.py').match('a.PY')) - - def test_is_absolute(self): - P = self.cls - self.assertFalse(P().is_absolute()) - self.assertFalse(P('a').is_absolute()) - self.assertFalse(P('a/b/').is_absolute()) - self.assertTrue(P('/').is_absolute()) - self.assertTrue(P('/a').is_absolute()) - self.assertTrue(P('/a/b/').is_absolute()) - self.assertTrue(P('//a').is_absolute()) - self.assertTrue(P('//a/b').is_absolute()) - - def test_is_reserved(self): - P = self.cls - self.assertIs(False, P('').is_reserved()) - self.assertIs(False, P('/').is_reserved()) - self.assertIs(False, P('/foo/bar').is_reserved()) - self.assertIs(False, P('/dev/con/PRN/NUL').is_reserved()) - - def test_join(self): - P = self.cls - p = P('//a') - pp = p.joinpath('b') - self.assertEqual(pp, P('//a/b')) - pp = P('/a').joinpath('//c') - self.assertEqual(pp, P('//c')) - pp = P('//a').joinpath('/c') - self.assertEqual(pp, P('/c')) - - def test_div(self): - # Basically the same as joinpath(). - P = self.cls - p = P('//a') - pp = p / 'b' - self.assertEqual(pp, P('//a/b')) - pp = P('/a') / '//c' - self.assertEqual(pp, P('//c')) - pp = P('//a') / '/c' - self.assertEqual(pp, P('/c')) - + @needs_posix def test_parse_windows_path(self): P = self.cls p = P('c:', 'a', 'b') pp = P(pathlib.PureWindowsPath('c:\\a\\b')) self.assertEqual(p, pp) - -class PureWindowsPathTest(PurePathTest): - cls = pathlib.PureWindowsPath - - equivalences = PurePathTest.equivalences.copy() - equivalences.update({ + windows_equivalences = { './a:b': [ ('./a:b',) ], 'c:a': [ ('c:', 'a'), ('c:', 'a/'), ('.', 'c:', 'a') ], 'c:/a': [ @@ -263,9 +391,14 @@ class PureWindowsPathTest(PurePathTest): '//a/b/c': [ ('//a/b', 'c'), ('//a/b/', 'c'), ], - }) + } - def test_parse_path(self): + @needs_windows + def test_equivalences_windows(self): + self.test_equivalences(self.windows_equivalences) + + @needs_windows + def test_parse_path_windows(self): check = self._check_parse_path # First part is anchored. check('c:', 'c:', '', []) @@ -313,31 +446,8 @@ def test_parse_path(self): check('D:a/c:b', 'D:', '', ['a', 'c:b']) check('D:/a/c:b', 'D:', '\\', ['a', 'c:b']) - def test_str(self): - p = self.cls('a/b/c') - self.assertEqual(str(p), 'a\\b\\c') - p = self.cls('c:/a/b/c') - self.assertEqual(str(p), 'c:\\a\\b\\c') - p = self.cls('//a/b') - self.assertEqual(str(p), '\\\\a\\b\\') - p = self.cls('//a/b/c') - self.assertEqual(str(p), '\\\\a\\b\\c') - p = self.cls('//a/b/c/d') - self.assertEqual(str(p), '\\\\a\\b\\c\\d') - - def test_str_subclass(self): - self._check_str_subclass('.\\a:b') - self._check_str_subclass('c:') - self._check_str_subclass('c:a') - self._check_str_subclass('c:a\\b.txt') - self._check_str_subclass('c:\\') - self._check_str_subclass('c:\\a') - self._check_str_subclass('c:\\a\\b.txt') - self._check_str_subclass('\\\\some\\share') - self._check_str_subclass('\\\\some\\share\\a') - self._check_str_subclass('\\\\some\\share\\a\\b.txt') - - def test_eq(self): + @needs_windows + def test_eq_windows(self): P = self.cls self.assertEqual(P('c:a/b'), P('c:a/b')) self.assertEqual(P('c:a/b'), P('c:', 'a', 'b')) @@ -350,7 +460,8 @@ def test_eq(self): self.assertEqual(P('//Some/SHARE/a/B'), P('//somE/share/A/b')) self.assertEqual(P('\u0130'), P('i\u0307')) - def test_as_uri(self): + @needs_windows + def test_as_uri_windows(self): P = self.cls with self.assertRaises(ValueError): P('/a/b').as_uri() @@ -366,34 +477,8 @@ def test_as_uri(self): self.assertEqual(P('//some/share/a/b%#c\xe9').as_uri(), 'file://some/share/a/b%25%23c%C3%A9') - def test_match(self): - P = self.cls - # Absolute patterns. - self.assertTrue(P('c:/b.py').match('*:/*.py')) - self.assertTrue(P('c:/b.py').match('c:/*.py')) - self.assertFalse(P('d:/b.py').match('c:/*.py')) # wrong drive - self.assertFalse(P('b.py').match('/*.py')) - self.assertFalse(P('b.py').match('c:*.py')) - self.assertFalse(P('b.py').match('c:/*.py')) - self.assertFalse(P('c:b.py').match('/*.py')) - self.assertFalse(P('c:b.py').match('c:/*.py')) - self.assertFalse(P('/b.py').match('c:*.py')) - self.assertFalse(P('/b.py').match('c:/*.py')) - # UNC patterns. - self.assertTrue(P('//some/share/a.py').match('//*/*/*.py')) - self.assertTrue(P('//some/share/a.py').match('//some/share/*.py')) - self.assertFalse(P('//other/share/a.py').match('//some/share/*.py')) - self.assertFalse(P('//some/share/a/b.py').match('//some/share/*.py')) - # Case-insensitivity. - self.assertTrue(P('B.py').match('b.PY')) - self.assertTrue(P('c:/a/B.Py').match('C:/A/*.pY')) - self.assertTrue(P('//Some/Share/B.Py').match('//somE/sharE/*.pY')) - # Path anchor doesn't match pattern anchor - self.assertFalse(P('c:/b.py').match('/*.py')) # 'c:/' vs '/' - self.assertFalse(P('c:/b.py').match('c:*.py')) # 'c:/' vs 'c:' - self.assertFalse(P('//some/share/a.py').match('/*.py')) # '//some/share/' vs '/' - - def test_ordering_common(self): + @needs_windows + def test_ordering_windows(self): # Case-insensitivity. def assertOrderedEqual(a, b): self.assertLessEqual(a, b) @@ -410,507 +495,13 @@ def assertOrderedEqual(a, b): self.assertFalse(p < q) self.assertFalse(p > q) - def test_parts(self): - P = self.cls - p = P('c:a/b') - parts = p.parts - self.assertEqual(parts, ('c:', 'a', 'b')) - p = P('c:/a/b') - parts = p.parts - self.assertEqual(parts, ('c:\\', 'a', 'b')) - p = P('//a/b/c/d') - parts = p.parts - self.assertEqual(parts, ('\\\\a\\b\\', 'c', 'd')) - - def test_parent(self): - # Anchored - P = self.cls - p = P('z:a/b/c') - self.assertEqual(p.parent, P('z:a/b')) - self.assertEqual(p.parent.parent, P('z:a')) - self.assertEqual(p.parent.parent.parent, P('z:')) - self.assertEqual(p.parent.parent.parent.parent, P('z:')) - p = P('z:/a/b/c') - self.assertEqual(p.parent, P('z:/a/b')) - self.assertEqual(p.parent.parent, P('z:/a')) - self.assertEqual(p.parent.parent.parent, P('z:/')) - self.assertEqual(p.parent.parent.parent.parent, P('z:/')) - p = P('//a/b/c/d') - self.assertEqual(p.parent, P('//a/b/c')) - self.assertEqual(p.parent.parent, P('//a/b')) - self.assertEqual(p.parent.parent.parent, P('//a/b')) - - def test_parents(self): - # Anchored - P = self.cls - p = P('z:a/b/') - par = p.parents - self.assertEqual(len(par), 2) - self.assertEqual(par[0], P('z:a')) - self.assertEqual(par[1], P('z:')) - self.assertEqual(par[0:1], (P('z:a'),)) - self.assertEqual(par[:-1], (P('z:a'),)) - self.assertEqual(par[:2], (P('z:a'), P('z:'))) - self.assertEqual(par[1:], (P('z:'),)) - self.assertEqual(par[::2], (P('z:a'),)) - self.assertEqual(par[::-1], (P('z:'), P('z:a'))) - self.assertEqual(list(par), [P('z:a'), P('z:')]) - with self.assertRaises(IndexError): - par[2] - p = P('z:/a/b/') - par = p.parents - self.assertEqual(len(par), 2) - self.assertEqual(par[0], P('z:/a')) - self.assertEqual(par[1], P('z:/')) - self.assertEqual(par[0:1], (P('z:/a'),)) - self.assertEqual(par[0:-1], (P('z:/a'),)) - self.assertEqual(par[:2], (P('z:/a'), P('z:/'))) - self.assertEqual(par[1:], (P('z:/'),)) - self.assertEqual(par[::2], (P('z:/a'),)) - self.assertEqual(par[::-1], (P('z:/'), P('z:/a'),)) - self.assertEqual(list(par), [P('z:/a'), P('z:/')]) - with self.assertRaises(IndexError): - par[2] - p = P('//a/b/c/d') - par = p.parents - self.assertEqual(len(par), 2) - self.assertEqual(par[0], P('//a/b/c')) - self.assertEqual(par[1], P('//a/b')) - self.assertEqual(par[0:1], (P('//a/b/c'),)) - self.assertEqual(par[0:-1], (P('//a/b/c'),)) - self.assertEqual(par[:2], (P('//a/b/c'), P('//a/b'))) - self.assertEqual(par[1:], (P('//a/b'),)) - self.assertEqual(par[::2], (P('//a/b/c'),)) - self.assertEqual(par[::-1], (P('//a/b'), P('//a/b/c'))) - self.assertEqual(list(par), [P('//a/b/c'), P('//a/b')]) - with self.assertRaises(IndexError): - par[2] - - def test_drive(self): - P = self.cls - self.assertEqual(P('c:').drive, 'c:') - self.assertEqual(P('c:a/b').drive, 'c:') - self.assertEqual(P('c:/').drive, 'c:') - self.assertEqual(P('c:/a/b/').drive, 'c:') - self.assertEqual(P('//a/b').drive, '\\\\a\\b') - self.assertEqual(P('//a/b/').drive, '\\\\a\\b') - self.assertEqual(P('//a/b/c/d').drive, '\\\\a\\b') - self.assertEqual(P('./c:a').drive, '') - - def test_root(self): - P = self.cls - self.assertEqual(P('c:').root, '') - self.assertEqual(P('c:a/b').root, '') - self.assertEqual(P('c:/').root, '\\') - self.assertEqual(P('c:/a/b/').root, '\\') - self.assertEqual(P('//a/b').root, '\\') - self.assertEqual(P('//a/b/').root, '\\') - self.assertEqual(P('//a/b/c/d').root, '\\') - - def test_anchor(self): - P = self.cls - self.assertEqual(P('c:').anchor, 'c:') - self.assertEqual(P('c:a/b').anchor, 'c:') - self.assertEqual(P('c:/').anchor, 'c:\\') - self.assertEqual(P('c:/a/b/').anchor, 'c:\\') - self.assertEqual(P('//a/b').anchor, '\\\\a\\b\\') - self.assertEqual(P('//a/b/').anchor, '\\\\a\\b\\') - self.assertEqual(P('//a/b/c/d').anchor, '\\\\a\\b\\') - - def test_name(self): - P = self.cls - self.assertEqual(P('c:').name, '') - self.assertEqual(P('c:/').name, '') - self.assertEqual(P('c:a/b').name, 'b') - self.assertEqual(P('c:/a/b').name, 'b') - self.assertEqual(P('c:a/b.py').name, 'b.py') - self.assertEqual(P('c:/a/b.py').name, 'b.py') - self.assertEqual(P('//My.py/Share.php').name, '') - self.assertEqual(P('//My.py/Share.php/a/b').name, 'b') - - def test_suffix(self): - P = self.cls - self.assertEqual(P('c:').suffix, '') - self.assertEqual(P('c:/').suffix, '') - self.assertEqual(P('c:a/b').suffix, '') - self.assertEqual(P('c:/a/b').suffix, '') - self.assertEqual(P('c:a/b.py').suffix, '.py') - self.assertEqual(P('c:/a/b.py').suffix, '.py') - self.assertEqual(P('c:a/.hgrc').suffix, '') - self.assertEqual(P('c:/a/.hgrc').suffix, '') - self.assertEqual(P('c:a/.hg.rc').suffix, '.rc') - self.assertEqual(P('c:/a/.hg.rc').suffix, '.rc') - self.assertEqual(P('c:a/b.tar.gz').suffix, '.gz') - self.assertEqual(P('c:/a/b.tar.gz').suffix, '.gz') - self.assertEqual(P('c:a/Some name. Ending with a dot.').suffix, '') - self.assertEqual(P('c:/a/Some name. Ending with a dot.').suffix, '') - self.assertEqual(P('//My.py/Share.php').suffix, '') - self.assertEqual(P('//My.py/Share.php/a/b').suffix, '') - - def test_suffixes(self): - P = self.cls - self.assertEqual(P('c:').suffixes, []) - self.assertEqual(P('c:/').suffixes, []) - self.assertEqual(P('c:a/b').suffixes, []) - self.assertEqual(P('c:/a/b').suffixes, []) - self.assertEqual(P('c:a/b.py').suffixes, ['.py']) - self.assertEqual(P('c:/a/b.py').suffixes, ['.py']) - self.assertEqual(P('c:a/.hgrc').suffixes, []) - self.assertEqual(P('c:/a/.hgrc').suffixes, []) - self.assertEqual(P('c:a/.hg.rc').suffixes, ['.rc']) - self.assertEqual(P('c:/a/.hg.rc').suffixes, ['.rc']) - self.assertEqual(P('c:a/b.tar.gz').suffixes, ['.tar', '.gz']) - self.assertEqual(P('c:/a/b.tar.gz').suffixes, ['.tar', '.gz']) - self.assertEqual(P('//My.py/Share.php').suffixes, []) - self.assertEqual(P('//My.py/Share.php/a/b').suffixes, []) - self.assertEqual(P('c:a/Some name. Ending with a dot.').suffixes, []) - self.assertEqual(P('c:/a/Some name. Ending with a dot.').suffixes, []) - - def test_stem(self): - P = self.cls - self.assertEqual(P('c:').stem, '') - self.assertEqual(P('c:.').stem, '') - self.assertEqual(P('c:..').stem, '..') - self.assertEqual(P('c:/').stem, '') - self.assertEqual(P('c:a/b').stem, 'b') - self.assertEqual(P('c:a/b.py').stem, 'b') - self.assertEqual(P('c:a/.hgrc').stem, '.hgrc') - self.assertEqual(P('c:a/.hg.rc').stem, '.hg') - self.assertEqual(P('c:a/b.tar.gz').stem, 'b.tar') - self.assertEqual(P('c:a/Some name. Ending with a dot.').stem, - 'Some name. Ending with a dot.') - - def test_with_name(self): - P = self.cls - self.assertEqual(P('c:a/b').with_name('d.xml'), P('c:a/d.xml')) - self.assertEqual(P('c:/a/b').with_name('d.xml'), P('c:/a/d.xml')) - self.assertEqual(P('c:a/Dot ending.').with_name('d.xml'), P('c:a/d.xml')) - self.assertEqual(P('c:/a/Dot ending.').with_name('d.xml'), P('c:/a/d.xml')) - self.assertRaises(ValueError, P('c:').with_name, 'd.xml') - self.assertRaises(ValueError, P('c:/').with_name, 'd.xml') - self.assertRaises(ValueError, P('//My/Share').with_name, 'd.xml') - self.assertEqual(str(P('a').with_name('d:')), '.\\d:') - self.assertEqual(str(P('a').with_name('d:e')), '.\\d:e') - self.assertEqual(P('c:a/b').with_name('d:'), P('c:a/d:')) - self.assertEqual(P('c:a/b').with_name('d:e'), P('c:a/d:e')) - self.assertRaises(ValueError, P('c:a/b').with_name, 'd:/e') - self.assertRaises(ValueError, P('c:a/b').with_name, '//My/Share') - - def test_with_stem(self): - P = self.cls - self.assertEqual(P('c:a/b').with_stem('d'), P('c:a/d')) - self.assertEqual(P('c:/a/b').with_stem('d'), P('c:/a/d')) - self.assertEqual(P('c:a/Dot ending.').with_stem('d'), P('c:a/d')) - self.assertEqual(P('c:/a/Dot ending.').with_stem('d'), P('c:/a/d')) - self.assertRaises(ValueError, P('c:').with_stem, 'd') - self.assertRaises(ValueError, P('c:/').with_stem, 'd') - self.assertRaises(ValueError, P('//My/Share').with_stem, 'd') - self.assertEqual(str(P('a').with_stem('d:')), '.\\d:') - self.assertEqual(str(P('a').with_stem('d:e')), '.\\d:e') - self.assertEqual(P('c:a/b').with_stem('d:'), P('c:a/d:')) - self.assertEqual(P('c:a/b').with_stem('d:e'), P('c:a/d:e')) - self.assertRaises(ValueError, P('c:a/b').with_stem, 'd:/e') - self.assertRaises(ValueError, P('c:a/b').with_stem, '//My/Share') - - def test_with_suffix(self): - P = self.cls - self.assertEqual(P('c:a/b').with_suffix('.gz'), P('c:a/b.gz')) - self.assertEqual(P('c:/a/b').with_suffix('.gz'), P('c:/a/b.gz')) - self.assertEqual(P('c:a/b.py').with_suffix('.gz'), P('c:a/b.gz')) - self.assertEqual(P('c:/a/b.py').with_suffix('.gz'), P('c:/a/b.gz')) - # Path doesn't have a "filename" component. - self.assertRaises(ValueError, P('').with_suffix, '.gz') - self.assertRaises(ValueError, P('.').with_suffix, '.gz') - self.assertRaises(ValueError, P('/').with_suffix, '.gz') - self.assertRaises(ValueError, P('//My/Share').with_suffix, '.gz') - # Invalid suffix. - self.assertRaises(ValueError, P('c:a/b').with_suffix, 'gz') - self.assertRaises(ValueError, P('c:a/b').with_suffix, '/') - self.assertRaises(ValueError, P('c:a/b').with_suffix, '\\') - self.assertRaises(ValueError, P('c:a/b').with_suffix, 'c:') - self.assertRaises(ValueError, P('c:a/b').with_suffix, '/.gz') - self.assertRaises(ValueError, P('c:a/b').with_suffix, '\\.gz') - self.assertRaises(ValueError, P('c:a/b').with_suffix, 'c:.gz') - self.assertRaises(ValueError, P('c:a/b').with_suffix, 'c/d') - self.assertRaises(ValueError, P('c:a/b').with_suffix, 'c\\d') - self.assertRaises(ValueError, P('c:a/b').with_suffix, '.c/d') - self.assertRaises(ValueError, P('c:a/b').with_suffix, '.c\\d') - - def test_relative_to(self): - P = self.cls - p = P('C:Foo/Bar') - self.assertEqual(p.relative_to(P('c:')), P('Foo/Bar')) - self.assertEqual(p.relative_to('c:'), P('Foo/Bar')) - self.assertEqual(p.relative_to(P('c:foO')), P('Bar')) - self.assertEqual(p.relative_to('c:foO'), P('Bar')) - self.assertEqual(p.relative_to('c:foO/'), P('Bar')) - self.assertEqual(p.relative_to(P('c:foO/baR')), P()) - self.assertEqual(p.relative_to('c:foO/baR'), P()) - self.assertEqual(p.relative_to(P('c:'), walk_up=True), P('Foo/Bar')) - self.assertEqual(p.relative_to('c:', walk_up=True), P('Foo/Bar')) - self.assertEqual(p.relative_to(P('c:foO'), walk_up=True), P('Bar')) - self.assertEqual(p.relative_to('c:foO', walk_up=True), P('Bar')) - self.assertEqual(p.relative_to('c:foO/', walk_up=True), P('Bar')) - self.assertEqual(p.relative_to(P('c:foO/baR'), walk_up=True), P()) - self.assertEqual(p.relative_to('c:foO/baR', walk_up=True), P()) - self.assertEqual(p.relative_to(P('C:Foo/Bar/Baz'), walk_up=True), P('..')) - self.assertEqual(p.relative_to(P('C:Foo/Baz'), walk_up=True), P('../Bar')) - self.assertEqual(p.relative_to(P('C:Baz/Bar'), walk_up=True), P('../../Foo/Bar')) - # Unrelated paths. - self.assertRaises(ValueError, p.relative_to, P()) - self.assertRaises(ValueError, p.relative_to, '') - self.assertRaises(ValueError, p.relative_to, P('d:')) - self.assertRaises(ValueError, p.relative_to, P('/')) - self.assertRaises(ValueError, p.relative_to, P('Foo')) - self.assertRaises(ValueError, p.relative_to, P('/Foo')) - self.assertRaises(ValueError, p.relative_to, P('C:/Foo')) - self.assertRaises(ValueError, p.relative_to, P('C:Foo/Bar/Baz')) - self.assertRaises(ValueError, p.relative_to, P('C:Foo/Baz')) - self.assertRaises(ValueError, p.relative_to, P(), walk_up=True) - self.assertRaises(ValueError, p.relative_to, '', walk_up=True) - self.assertRaises(ValueError, p.relative_to, P('d:'), walk_up=True) - self.assertRaises(ValueError, p.relative_to, P('/'), walk_up=True) - self.assertRaises(ValueError, p.relative_to, P('Foo'), walk_up=True) - self.assertRaises(ValueError, p.relative_to, P('/Foo'), walk_up=True) - self.assertRaises(ValueError, p.relative_to, P('C:/Foo'), walk_up=True) - p = P('C:/Foo/Bar') - self.assertEqual(p.relative_to(P('c:/')), P('Foo/Bar')) - self.assertEqual(p.relative_to('c:/'), P('Foo/Bar')) - self.assertEqual(p.relative_to(P('c:/foO')), P('Bar')) - self.assertEqual(p.relative_to('c:/foO'), P('Bar')) - self.assertEqual(p.relative_to('c:/foO/'), P('Bar')) - self.assertEqual(p.relative_to(P('c:/foO/baR')), P()) - self.assertEqual(p.relative_to('c:/foO/baR'), P()) - self.assertEqual(p.relative_to(P('c:/'), walk_up=True), P('Foo/Bar')) - self.assertEqual(p.relative_to('c:/', walk_up=True), P('Foo/Bar')) - self.assertEqual(p.relative_to(P('c:/foO'), walk_up=True), P('Bar')) - self.assertEqual(p.relative_to('c:/foO', walk_up=True), P('Bar')) - self.assertEqual(p.relative_to('c:/foO/', walk_up=True), P('Bar')) - self.assertEqual(p.relative_to(P('c:/foO/baR'), walk_up=True), P()) - self.assertEqual(p.relative_to('c:/foO/baR', walk_up=True), P()) - self.assertEqual(p.relative_to('C:/Baz', walk_up=True), P('../Foo/Bar')) - self.assertEqual(p.relative_to('C:/Foo/Bar/Baz', walk_up=True), P('..')) - self.assertEqual(p.relative_to('C:/Foo/Baz', walk_up=True), P('../Bar')) - # Unrelated paths. - self.assertRaises(ValueError, p.relative_to, 'c:') - self.assertRaises(ValueError, p.relative_to, P('c:')) - self.assertRaises(ValueError, p.relative_to, P('C:/Baz')) - self.assertRaises(ValueError, p.relative_to, P('C:/Foo/Bar/Baz')) - self.assertRaises(ValueError, p.relative_to, P('C:/Foo/Baz')) - self.assertRaises(ValueError, p.relative_to, P('C:Foo')) - self.assertRaises(ValueError, p.relative_to, P('d:')) - self.assertRaises(ValueError, p.relative_to, P('d:/')) - self.assertRaises(ValueError, p.relative_to, P('/')) - self.assertRaises(ValueError, p.relative_to, P('/Foo')) - self.assertRaises(ValueError, p.relative_to, P('//C/Foo')) - self.assertRaises(ValueError, p.relative_to, 'c:', walk_up=True) - self.assertRaises(ValueError, p.relative_to, P('c:'), walk_up=True) - self.assertRaises(ValueError, p.relative_to, P('C:Foo'), walk_up=True) - self.assertRaises(ValueError, p.relative_to, P('d:'), walk_up=True) - self.assertRaises(ValueError, p.relative_to, P('d:/'), walk_up=True) - self.assertRaises(ValueError, p.relative_to, P('/'), walk_up=True) - self.assertRaises(ValueError, p.relative_to, P('/Foo'), walk_up=True) - self.assertRaises(ValueError, p.relative_to, P('//C/Foo'), walk_up=True) - # UNC paths. - p = P('//Server/Share/Foo/Bar') - self.assertEqual(p.relative_to(P('//sErver/sHare')), P('Foo/Bar')) - self.assertEqual(p.relative_to('//sErver/sHare'), P('Foo/Bar')) - self.assertEqual(p.relative_to('//sErver/sHare/'), P('Foo/Bar')) - self.assertEqual(p.relative_to(P('//sErver/sHare/Foo')), P('Bar')) - self.assertEqual(p.relative_to('//sErver/sHare/Foo'), P('Bar')) - self.assertEqual(p.relative_to('//sErver/sHare/Foo/'), P('Bar')) - self.assertEqual(p.relative_to(P('//sErver/sHare/Foo/Bar')), P()) - self.assertEqual(p.relative_to('//sErver/sHare/Foo/Bar'), P()) - self.assertEqual(p.relative_to(P('//sErver/sHare'), walk_up=True), P('Foo/Bar')) - self.assertEqual(p.relative_to('//sErver/sHare', walk_up=True), P('Foo/Bar')) - self.assertEqual(p.relative_to('//sErver/sHare/', walk_up=True), P('Foo/Bar')) - self.assertEqual(p.relative_to(P('//sErver/sHare/Foo'), walk_up=True), P('Bar')) - self.assertEqual(p.relative_to('//sErver/sHare/Foo', walk_up=True), P('Bar')) - self.assertEqual(p.relative_to('//sErver/sHare/Foo/', walk_up=True), P('Bar')) - self.assertEqual(p.relative_to(P('//sErver/sHare/Foo/Bar'), walk_up=True), P()) - self.assertEqual(p.relative_to('//sErver/sHare/Foo/Bar', walk_up=True), P()) - self.assertEqual(p.relative_to(P('//sErver/sHare/bar'), walk_up=True), P('../Foo/Bar')) - self.assertEqual(p.relative_to('//sErver/sHare/bar', walk_up=True), P('../Foo/Bar')) - # Unrelated paths. - self.assertRaises(ValueError, p.relative_to, P('/Server/Share/Foo')) - self.assertRaises(ValueError, p.relative_to, P('c:/Server/Share/Foo')) - self.assertRaises(ValueError, p.relative_to, P('//z/Share/Foo')) - self.assertRaises(ValueError, p.relative_to, P('//Server/z/Foo')) - self.assertRaises(ValueError, p.relative_to, P('/Server/Share/Foo'), walk_up=True) - self.assertRaises(ValueError, p.relative_to, P('c:/Server/Share/Foo'), walk_up=True) - self.assertRaises(ValueError, p.relative_to, P('//z/Share/Foo'), walk_up=True) - self.assertRaises(ValueError, p.relative_to, P('//Server/z/Foo'), walk_up=True) - - def test_is_relative_to(self): - P = self.cls - p = P('C:Foo/Bar') - self.assertTrue(p.is_relative_to(P('c:'))) - self.assertTrue(p.is_relative_to('c:')) - self.assertTrue(p.is_relative_to(P('c:foO'))) - self.assertTrue(p.is_relative_to('c:foO')) - self.assertTrue(p.is_relative_to('c:foO/')) - self.assertTrue(p.is_relative_to(P('c:foO/baR'))) - self.assertTrue(p.is_relative_to('c:foO/baR')) - # Unrelated paths. - self.assertFalse(p.is_relative_to(P())) - self.assertFalse(p.is_relative_to('')) - self.assertFalse(p.is_relative_to(P('d:'))) - self.assertFalse(p.is_relative_to(P('/'))) - self.assertFalse(p.is_relative_to(P('Foo'))) - self.assertFalse(p.is_relative_to(P('/Foo'))) - self.assertFalse(p.is_relative_to(P('C:/Foo'))) - self.assertFalse(p.is_relative_to(P('C:Foo/Bar/Baz'))) - self.assertFalse(p.is_relative_to(P('C:Foo/Baz'))) - p = P('C:/Foo/Bar') - self.assertTrue(p.is_relative_to(P('c:/'))) - self.assertTrue(p.is_relative_to(P('c:/foO'))) - self.assertTrue(p.is_relative_to('c:/foO/')) - self.assertTrue(p.is_relative_to(P('c:/foO/baR'))) - self.assertTrue(p.is_relative_to('c:/foO/baR')) - # Unrelated paths. - self.assertFalse(p.is_relative_to('c:')) - self.assertFalse(p.is_relative_to(P('C:/Baz'))) - self.assertFalse(p.is_relative_to(P('C:/Foo/Bar/Baz'))) - self.assertFalse(p.is_relative_to(P('C:/Foo/Baz'))) - self.assertFalse(p.is_relative_to(P('C:Foo'))) - self.assertFalse(p.is_relative_to(P('d:'))) - self.assertFalse(p.is_relative_to(P('d:/'))) - self.assertFalse(p.is_relative_to(P('/'))) - self.assertFalse(p.is_relative_to(P('/Foo'))) - self.assertFalse(p.is_relative_to(P('//C/Foo'))) - # UNC paths. - p = P('//Server/Share/Foo/Bar') - self.assertTrue(p.is_relative_to(P('//sErver/sHare'))) - self.assertTrue(p.is_relative_to('//sErver/sHare')) - self.assertTrue(p.is_relative_to('//sErver/sHare/')) - self.assertTrue(p.is_relative_to(P('//sErver/sHare/Foo'))) - self.assertTrue(p.is_relative_to('//sErver/sHare/Foo')) - self.assertTrue(p.is_relative_to('//sErver/sHare/Foo/')) - self.assertTrue(p.is_relative_to(P('//sErver/sHare/Foo/Bar'))) - self.assertTrue(p.is_relative_to('//sErver/sHare/Foo/Bar')) - # Unrelated paths. - self.assertFalse(p.is_relative_to(P('/Server/Share/Foo'))) - self.assertFalse(p.is_relative_to(P('c:/Server/Share/Foo'))) - self.assertFalse(p.is_relative_to(P('//z/Share/Foo'))) - self.assertFalse(p.is_relative_to(P('//Server/z/Foo'))) - - def test_is_absolute(self): - P = self.cls - # Under NT, only paths with both a drive and a root are absolute. - self.assertFalse(P().is_absolute()) - self.assertFalse(P('a').is_absolute()) - self.assertFalse(P('a/b/').is_absolute()) - self.assertFalse(P('/').is_absolute()) - self.assertFalse(P('/a').is_absolute()) - self.assertFalse(P('/a/b/').is_absolute()) - self.assertFalse(P('c:').is_absolute()) - self.assertFalse(P('c:a').is_absolute()) - self.assertFalse(P('c:a/b/').is_absolute()) - self.assertTrue(P('c:/').is_absolute()) - self.assertTrue(P('c:/a').is_absolute()) - self.assertTrue(P('c:/a/b/').is_absolute()) - # UNC paths are absolute by definition. - self.assertTrue(P('//a/b').is_absolute()) - self.assertTrue(P('//a/b/').is_absolute()) - self.assertTrue(P('//a/b/c').is_absolute()) - self.assertTrue(P('//a/b/c/d').is_absolute()) - - def test_join(self): - P = self.cls - p = P('C:/a/b') - pp = p.joinpath('x/y') - self.assertEqual(pp, P('C:/a/b/x/y')) - pp = p.joinpath('/x/y') - self.assertEqual(pp, P('C:/x/y')) - # Joining with a different drive => the first path is ignored, even - # if the second path is relative. - pp = p.joinpath('D:x/y') - self.assertEqual(pp, P('D:x/y')) - pp = p.joinpath('D:/x/y') - self.assertEqual(pp, P('D:/x/y')) - pp = p.joinpath('//host/share/x/y') - self.assertEqual(pp, P('//host/share/x/y')) - # Joining with the same drive => the first path is appended to if - # the second path is relative. - pp = p.joinpath('c:x/y') - self.assertEqual(pp, P('C:/a/b/x/y')) - pp = p.joinpath('c:/x/y') - self.assertEqual(pp, P('C:/x/y')) - # Joining with files with NTFS data streams => the filename should - # not be parsed as a drive letter - pp = p.joinpath(P('./d:s')) - self.assertEqual(pp, P('C:/a/b/d:s')) - pp = p.joinpath(P('./dd:s')) - self.assertEqual(pp, P('C:/a/b/dd:s')) - pp = p.joinpath(P('E:d:s')) - self.assertEqual(pp, P('E:d:s')) - # Joining onto a UNC path with no root - pp = P('//').joinpath('server') - self.assertEqual(pp, P('//server')) - pp = P('//server').joinpath('share') - self.assertEqual(pp, P('//server/share')) - pp = P('//./BootPartition').joinpath('Windows') - self.assertEqual(pp, P('//./BootPartition/Windows')) - - def test_div(self): - # Basically the same as joinpath(). - P = self.cls - p = P('C:/a/b') - self.assertEqual(p / 'x/y', P('C:/a/b/x/y')) - self.assertEqual(p / 'x' / 'y', P('C:/a/b/x/y')) - self.assertEqual(p / '/x/y', P('C:/x/y')) - self.assertEqual(p / '/x' / 'y', P('C:/x/y')) - # Joining with a different drive => the first path is ignored, even - # if the second path is relative. - self.assertEqual(p / 'D:x/y', P('D:x/y')) - self.assertEqual(p / 'D:' / 'x/y', P('D:x/y')) - self.assertEqual(p / 'D:/x/y', P('D:/x/y')) - self.assertEqual(p / 'D:' / '/x/y', P('D:/x/y')) - self.assertEqual(p / '//host/share/x/y', P('//host/share/x/y')) - # Joining with the same drive => the first path is appended to if - # the second path is relative. - self.assertEqual(p / 'c:x/y', P('C:/a/b/x/y')) - self.assertEqual(p / 'c:/x/y', P('C:/x/y')) - # Joining with files with NTFS data streams => the filename should - # not be parsed as a drive letter - self.assertEqual(p / P('./d:s'), P('C:/a/b/d:s')) - self.assertEqual(p / P('./dd:s'), P('C:/a/b/dd:s')) - self.assertEqual(p / P('E:d:s'), P('E:d:s')) - - def test_is_reserved(self): - P = self.cls - self.assertIs(False, P('').is_reserved()) - self.assertIs(False, P('/').is_reserved()) - self.assertIs(False, P('/foo/bar').is_reserved()) - # UNC paths are never reserved. - self.assertIs(False, P('//my/share/nul/con/aux').is_reserved()) - # Case-insensitive DOS-device names are reserved. - self.assertIs(True, P('nul').is_reserved()) - self.assertIs(True, P('aux').is_reserved()) - self.assertIs(True, P('prn').is_reserved()) - self.assertIs(True, P('con').is_reserved()) - self.assertIs(True, P('conin$').is_reserved()) - self.assertIs(True, P('conout$').is_reserved()) - # COM/LPT + 1-9 or + superscript 1-3 are reserved. - self.assertIs(True, P('COM1').is_reserved()) - self.assertIs(True, P('LPT9').is_reserved()) - self.assertIs(True, P('com\xb9').is_reserved()) - self.assertIs(True, P('com\xb2').is_reserved()) - self.assertIs(True, P('lpt\xb3').is_reserved()) - # DOS-device name mataching ignores characters after a dot or - # a colon and also ignores trailing spaces. - self.assertIs(True, P('NUL.txt').is_reserved()) - self.assertIs(True, P('PRN ').is_reserved()) - self.assertIs(True, P('AUX .txt').is_reserved()) - self.assertIs(True, P('COM1:bar').is_reserved()) - self.assertIs(True, P('LPT9 :bar').is_reserved()) - # DOS-device names are only matched at the beginning - # of a path component. - self.assertIs(False, P('bar.com9').is_reserved()) - self.assertIs(False, P('bar.lpt9').is_reserved()) - # Only the last path component matters. - self.assertIs(True, P('c:/baz/con/NUL').is_reserved()) - self.assertIs(False, P('c:/NUL/con/baz').is_reserved()) + +class PurePosixPathTest(PurePathTest): + cls = pathlib.PurePosixPath + + +class PureWindowsPathTest(PurePathTest): + cls = pathlib.PureWindowsPath class PurePathSubclassTest(PurePathTest): @@ -944,6 +535,20 @@ def tempdir(self): self.addCleanup(os_helper.rmtree, d) return d + def test_matches_pathbase_api(self): + our_names = {name for name in dir(self.cls) if name[0] != '_'} + our_names.remove('is_reserved') # only present in PurePath + path_names = {name for name in dir(pathlib._abc.PathBase) if name[0] != '_'} + self.assertEqual(our_names, path_names) + for attr_name in our_names: + if attr_name == 'pathmod': + # On Windows, Path.pathmod is ntpath, but PathBase.pathmod is + # posixpath, and so their docstrings differ. + continue + our_attr = getattr(self.cls, attr_name) + path_attr = getattr(pathlib._abc.PathBase, attr_name) + self.assertEqual(our_attr.__doc__, path_attr.__doc__) + def test_concrete_class(self): if self.cls is pathlib.Path: expected = pathlib.WindowsPath if os.name == 'nt' else pathlib.PosixPath @@ -1085,7 +690,7 @@ def test_chmod(self): self.assertEqual(p.stat().st_mode, new_mode) # On Windows, os.chmod does not follow symlinks (issue #15411) - @only_posix + @needs_posix @os_helper.skip_unless_working_chmod def test_chmod_follow_symlinks_true(self): p = self.cls(self.base) / 'linkA' @@ -1354,7 +959,7 @@ def test_mkdir_exist_ok_root(self): self.cls('/').resolve().mkdir(exist_ok=True) self.cls('/').resolve().mkdir(parents=True, exist_ok=True) - @only_nt # XXX: not sure how to test this on POSIX. + @needs_windows # XXX: not sure how to test this on POSIX. def test_mkdir_with_unknown_drive(self): for d in 'ZYXWVUTSRQPONMLKJIHGFEDCBA': p = self.cls(d + ':\\') @@ -1419,9 +1024,8 @@ def my_mkdir(path, mode=0o777): self.assertNotIn(str(p12), concurrently_created) self.assertTrue(p.exists()) + @needs_symlinks def test_symlink_to(self): - if not self.can_symlink: - self.skipTest("symlinks required") P = self.cls(self.base) target = P / 'fileA' # Symlinking a path target. @@ -1587,12 +1191,75 @@ def test_walk_many_open_files(self): self.assertEqual(next(it), expected) path = path / 'd' + def test_walk_above_recursion_limit(self): + recursion_limit = 40 + # directory_depth > recursion_limit + directory_depth = recursion_limit + 10 + base = self.cls(self.base, 'deep') + path = base.joinpath(*(['d'] * directory_depth)) + path.mkdir(parents=True) -@only_posix -class PosixPathTest(PathTest, PurePosixPathTest): - cls = pathlib.PosixPath + with set_recursion_limit(recursion_limit): + list(base.walk()) + list(base.walk(top_down=False)) + + def test_glob_empty_pattern(self): + p = self.cls('') + with self.assertRaisesRegex(ValueError, 'Unacceptable pattern'): + list(p.glob('')) + with self.assertRaisesRegex(ValueError, 'Unacceptable pattern'): + list(p.glob('.')) + with self.assertRaisesRegex(ValueError, 'Unacceptable pattern'): + list(p.glob('./')) - def test_absolute(self): + def test_glob_many_open_files(self): + depth = 30 + P = self.cls + p = base = P(self.base) / 'deep' + p.mkdir() + for _ in range(depth): + p /= 'd' + p.mkdir() + pattern = '/'.join(['*'] * depth) + iters = [base.glob(pattern) for j in range(100)] + for it in iters: + self.assertEqual(next(it), p) + iters = [base.rglob('d') for j in range(100)] + p = base + for i in range(depth): + p = p / 'd' + for it in iters: + self.assertEqual(next(it), p) + + def test_glob_above_recursion_limit(self): + recursion_limit = 50 + # directory_depth > recursion_limit + directory_depth = recursion_limit + 10 + base = self.cls(self.base, 'deep') + path = base.joinpath(*(['d'] * directory_depth)) + path.mkdir(parents=True) + + with set_recursion_limit(recursion_limit): + list(base.glob('**/')) + + def test_glob_pathlike(self): + P = self.cls + p = P(self.base) + pattern = "dir*/file*" + expect = {p / "dirB/fileB", p / "dirC/fileC"} + self.assertEqual(expect, set(p.glob(P(pattern)))) + self.assertEqual(expect, set(p.glob(FakePath(pattern)))) + + def test_rglob_pathlike(self): + P = self.cls + p = P(self.base, "dirC") + pattern = "**/file*" + expect = {p / "fileC", p / "dirD/fileD"} + self.assertEqual(expect, set(p.rglob(P(pattern)))) + self.assertEqual(expect, set(p.rglob(FakePath(pattern)))) + + @needs_posix + def test_absolute_posix(self): P = self.cls self.assertEqual(str(P('/').absolute()), '/') self.assertEqual(str(P('/a').absolute()), '/a') @@ -1607,6 +1274,7 @@ def test_absolute(self): is_emscripten or is_wasi, "umask is not implemented on Emscripten/WASI." ) + @needs_posix def test_open_mode(self): old_mask = os.umask(0) self.addCleanup(os.umask, old_mask) @@ -1621,6 +1289,7 @@ def test_open_mode(self): st = os.stat(self.pathmod.join(self.base, 'other_new_file')) self.assertEqual(stat.S_IMODE(st.st_mode), 0o644) + @needs_posix def test_resolve_root(self): current_directory = os.getcwd() try: @@ -1634,6 +1303,7 @@ def test_resolve_root(self): is_emscripten or is_wasi, "umask is not implemented on Emscripten/WASI." ) + @needs_posix def test_touch_mode(self): old_mask = os.umask(0) self.addCleanup(os.umask, old_mask) @@ -1649,27 +1319,12 @@ def test_touch_mode(self): st = os.stat(self.pathmod.join(self.base, 'masked_new_file')) self.assertEqual(stat.S_IMODE(st.st_mode), 0o750) - def test_glob(self): - P = self.cls - p = P(self.base) - given = set(p.glob("FILEa")) - expect = set() if not os_helper.fs_is_case_insensitive(self.base) else given - self.assertEqual(given, expect) - self.assertEqual(set(p.glob("FILEa*")), set()) - - def test_rglob(self): - P = self.cls - p = P(self.base, "dirC") - given = set(p.rglob("FILEd")) - expect = set() if not os_helper.fs_is_case_insensitive(self.base) else given - self.assertEqual(given, expect) - self.assertEqual(set(p.rglob("FILEd*")), set()) - @unittest.skipUnless(hasattr(pwd, 'getpwall'), 'pwd module does not expose getpwall()') @unittest.skipIf(sys.platform == "vxworks", "no home directory on VxWorks") - def test_expanduser(self): + @needs_posix + def test_expanduser_posix(self): P = self.cls import_helper.import_module('pwd') import pwd @@ -1724,6 +1379,7 @@ def test_expanduser(self): @unittest.skipIf(sys.platform != "darwin", "Bad file descriptor in /dev/fd affects only macOS") + @needs_posix def test_handling_bad_descriptor(self): try: file_descriptors = list(pathlib.Path('/dev/fd').rglob("*"))[3:] @@ -1745,7 +1401,8 @@ def test_handling_bad_descriptor(self): self.fail("Bad file descriptor not handled.") raise - def test_from_uri(self): + @needs_posix + def test_from_uri_posix(self): P = self.cls self.assertEqual(P.from_uri('file:/foo/bar'), P('/foo/bar')) self.assertEqual(P.from_uri('file://foo/bar'), P('//foo/bar')) @@ -1758,17 +1415,14 @@ def test_from_uri(self): self.assertRaises(ValueError, P.from_uri, 'file:foo/bar') self.assertRaises(ValueError, P.from_uri, 'http://foo/bar') - def test_from_uri_pathname2url(self): + @needs_posix + def test_from_uri_pathname2url_posix(self): P = self.cls self.assertEqual(P.from_uri('file:' + pathname2url('/foo/bar')), P('/foo/bar')) self.assertEqual(P.from_uri('file:' + pathname2url('//foo/bar')), P('//foo/bar')) - -@only_nt -class WindowsPathTest(PathTest, PureWindowsPathTest): - cls = pathlib.WindowsPath - - def test_absolute(self): + @needs_windows + def test_absolute_windows(self): P = self.cls # Simple absolute paths. @@ -1813,23 +1467,8 @@ def test_absolute(self): self.assertEqual(str(P(other_drive).absolute()), other_cwd) self.assertEqual(str(P(other_drive + 'foo').absolute()), other_cwd + '\\foo') - def test_glob(self): - P = self.cls - p = P(self.base) - self.assertEqual(set(p.glob("FILEa")), { P(self.base, "fileA") }) - self.assertEqual(set(p.glob("*a\\")), { P(self.base, "dirA/") }) - self.assertEqual(set(p.glob("F*a")), { P(self.base, "fileA") }) - self.assertEqual(set(map(str, p.glob("FILEa"))), {f"{p}\\fileA"}) - self.assertEqual(set(map(str, p.glob("F*a"))), {f"{p}\\fileA"}) - - def test_rglob(self): - P = self.cls - p = P(self.base, "dirC") - self.assertEqual(set(p.rglob("FILEd")), { P(self.base, "dirC/dirD/fileD") }) - self.assertEqual(set(p.rglob("*\\")), { P(self.base, "dirC/dirD/") }) - self.assertEqual(set(map(str, p.rglob("FILEd"))), {f"{p}\\dirD\\fileD"}) - - def test_expanduser(self): + @needs_windows + def test_expanduser_windows(self): P = self.cls with os_helper.EnvironmentVarGuard() as env: env.pop('HOME', None) @@ -1882,7 +1521,8 @@ def check(): env['HOME'] = 'C:\\Users\\eve' check() - def test_from_uri(self): + @needs_windows + def test_from_uri_windows(self): P = self.cls # DOS drive paths self.assertEqual(P.from_uri('file:c:/path/to/file'), P('c:/path/to/file')) @@ -1903,22 +1543,35 @@ def test_from_uri(self): self.assertRaises(ValueError, P.from_uri, 'file:foo/bar') self.assertRaises(ValueError, P.from_uri, 'http://foo/bar') - def test_from_uri_pathname2url(self): + @needs_windows + def test_from_uri_pathname2url_windows(self): P = self.cls self.assertEqual(P.from_uri('file:' + pathname2url(r'c:\path\to\file')), P('c:/path/to/file')) self.assertEqual(P.from_uri('file:' + pathname2url(r'\\server\path\to\file')), P('//server/path/to/file')) - def test_owner(self): + @needs_windows + def test_owner_windows(self): P = self.cls with self.assertRaises(pathlib.UnsupportedOperation): P('c:/').owner() - def test_group(self): + @needs_windows + def test_group_windows(self): P = self.cls with self.assertRaises(pathlib.UnsupportedOperation): P('c:/').group() +@unittest.skipIf(os.name == 'nt', 'test requires a POSIX-compatible system') +class PosixPathTest(PathTest, PurePosixPathTest): + cls = pathlib.PosixPath + + +@unittest.skipIf(os.name != 'nt', 'test requires a Windows-compatible system') +class WindowsPathTest(PathTest, PureWindowsPathTest): + cls = pathlib.WindowsPath + + class PathSubclassTest(PathTest): class cls(pathlib.Path): pass diff --git a/Lib/test/test_pathlib/test_pathlib_abc.py b/Lib/test/test_pathlib/test_pathlib_abc.py index a272973d9c1d61..1d30deca8f7a1b 100644 --- a/Lib/test/test_pathlib/test_pathlib_abc.py +++ b/Lib/test/test_pathlib/test_pathlib_abc.py @@ -1,22 +1,56 @@ -import collections.abc +import collections import io import os import errno -import pathlib -import pickle -import posixpath import stat import unittest -from test.support import set_recursion_limit +from pathlib._abc import UnsupportedOperation, PathModuleBase, PurePathBase, PathBase +import posixpath + from test.support.os_helper import TESTFN +_tests_needing_posix = set() +_tests_needing_windows = set() +_tests_needing_symlinks = set() + + +def needs_posix(fn): + """Decorator that marks a test as requiring a POSIX-flavoured path class.""" + _tests_needing_posix.add(fn.__name__) + return fn + +def needs_windows(fn): + """Decorator that marks a test as requiring a Windows-flavoured path class.""" + _tests_needing_windows.add(fn.__name__) + return fn + +def needs_symlinks(fn): + """Decorator that marks a test as requiring a path class that supports symlinks.""" + _tests_needing_symlinks.add(fn.__name__) + return fn + + class UnsupportedOperationTest(unittest.TestCase): def test_is_notimplemented(self): - self.assertTrue(issubclass(pathlib.UnsupportedOperation, NotImplementedError)) - self.assertTrue(isinstance(pathlib.UnsupportedOperation(), NotImplementedError)) + self.assertTrue(issubclass(UnsupportedOperation, NotImplementedError)) + self.assertTrue(isinstance(UnsupportedOperation(), NotImplementedError)) + +class PathModuleBaseTest(unittest.TestCase): + cls = PathModuleBase + + def test_unsupported_operation(self): + m = self.cls() + e = UnsupportedOperation + with self.assertRaises(e): + m.sep + self.assertRaises(e, m.join, 'foo') + self.assertRaises(e, m.split, 'foo') + self.assertRaises(e, m.splitdrive, 'foo') + self.assertRaises(e, m.normcase, 'foo') + self.assertRaises(e, m.isabs, 'foo') # # Tests for the pure classes. @@ -24,13 +58,50 @@ def test_is_notimplemented(self): class PurePathBaseTest(unittest.TestCase): - cls = pathlib._abc.PurePathBase + cls = PurePathBase + + def test_unsupported_operation_pure(self): + p = self.cls('foo') + e = UnsupportedOperation + with self.assertRaises(e): + p.drive + with self.assertRaises(e): + p.root + with self.assertRaises(e): + p.anchor + with self.assertRaises(e): + p.parts + with self.assertRaises(e): + p.parent + with self.assertRaises(e): + p.parents + with self.assertRaises(e): + p.name + with self.assertRaises(e): + p.stem + with self.assertRaises(e): + p.suffix + with self.assertRaises(e): + p.suffixes + with self.assertRaises(e): + p / 'bar' + with self.assertRaises(e): + 'bar' / p + self.assertRaises(e, p.joinpath, 'bar') + self.assertRaises(e, p.with_name, 'bar') + self.assertRaises(e, p.with_stem, 'bar') + self.assertRaises(e, p.with_suffix, '.txt') + self.assertRaises(e, p.relative_to, '') + self.assertRaises(e, p.is_relative_to, '') + self.assertRaises(e, p.is_absolute) + self.assertRaises(e, p.match, '*') def test_magic_methods(self): P = self.cls self.assertFalse(hasattr(P, '__fspath__')) self.assertFalse(hasattr(P, '__bytes__')) self.assertIs(P.__reduce__, object.__reduce__) + self.assertIs(P.__repr__, object.__repr__) self.assertIs(P.__hash__, object.__hash__) self.assertIs(P.__eq__, object.__eq__) self.assertIs(P.__lt__, object.__lt__) @@ -38,8 +109,14 @@ def test_magic_methods(self): self.assertIs(P.__gt__, object.__gt__) self.assertIs(P.__ge__, object.__ge__) + def test_pathmod(self): + self.assertIsInstance(self.cls.pathmod, PathModuleBase) + + +class DummyPurePath(PurePathBase): + __slots__ = () + pathmod = posixpath -class DummyPurePath(pathlib._abc.PurePathBase): def __eq__(self, other): if not isinstance(other, DummyPurePath): return NotImplemented @@ -48,30 +125,22 @@ def __eq__(self, other): def __hash__(self): return hash(str(self)) + def __repr__(self): + return "{}({!r})".format(self.__class__.__name__, self.as_posix()) + class DummyPurePathTest(unittest.TestCase): cls = DummyPurePath - # Make sure any symbolic links in the base test path are resolved. - base = os.path.realpath(TESTFN) - - # Keys are canonical paths, values are list of tuples of arguments - # supposed to produce equal paths. - equivalences = { - 'a/b': [ - ('a', 'b'), ('a/', 'b'), ('a', 'b/'), ('a/', 'b/'), - ('a/b/',), ('a//b',), ('a//b//',), - # Empty components get removed. - ('', 'a', 'b'), ('a', '', 'b'), ('a', 'b', ''), - ], - '/b/c/d': [ - ('a', '/b/c', 'd'), ('/a', '/b/c', 'd'), - # Empty components get removed. - ('/', 'b', '', 'c/d'), ('/', '', 'b/c/d'), ('', '/b/c/d'), - ], - } + # Use a base path that's unrelated to any real filesystem path. + base = f'/this/path/kills/fascists/{TESTFN}' def setUp(self): + name = self.id().split('.')[-1] + if name in _tests_needing_posix and self.cls.pathmod is not posixpath: + self.skipTest('requires POSIX-flavoured path class') + if name in _tests_needing_windows and self.cls.pathmod is posixpath: + self.skipTest('requires Windows-flavoured path class') p = self.cls('a') self.pathmod = p.pathmod self.sep = self.pathmod.sep @@ -86,36 +155,30 @@ def test_constructor_common(self): P('a/b/c') P('/a/b/c') - def test_concrete_class(self): - if self.cls is pathlib.PurePath: - expected = pathlib.PureWindowsPath if os.name == 'nt' else pathlib.PurePosixPath - else: - expected = self.cls - p = self.cls('a') - self.assertIs(type(p), expected) - - def test_different_pathmods_unequal(self): - p = self.cls('a') - if p.pathmod is posixpath: - q = pathlib.PureWindowsPath('a') - else: - q = pathlib.PurePosixPath('a') - self.assertNotEqual(p, q) - - def test_different_pathmods_unordered(self): - p = self.cls('a') - if p.pathmod is posixpath: - q = pathlib.PureWindowsPath('a') - else: - q = pathlib.PurePosixPath('a') + def test_bytes(self): + P = self.cls + with self.assertRaises(TypeError): + P(b'a') with self.assertRaises(TypeError): - p < q + P(b'a', 'b') with self.assertRaises(TypeError): - p <= q + P('a', b'b') with self.assertRaises(TypeError): - p > q + P('a').joinpath(b'b') with self.assertRaises(TypeError): - p >= q + P('a') / b'b' + with self.assertRaises(TypeError): + b'a' / P('b') + with self.assertRaises(TypeError): + P('a').match(b'b') + with self.assertRaises(TypeError): + P('a').relative_to(b'b') + with self.assertRaises(TypeError): + P('a').with_name(b'b') + with self.assertRaises(TypeError): + P('a').with_stem(b'b') + with self.assertRaises(TypeError): + P('a').with_suffix(b'b') def _check_str_subclass(self, *args): # Issue #21127: it should be possible to construct a PurePath object @@ -136,6 +199,19 @@ def test_str_subclass_common(self): self._check_str_subclass('a/b.txt') self._check_str_subclass('/a/b.txt') + @needs_windows + def test_str_subclass_windows(self): + self._check_str_subclass('.\\a:b') + self._check_str_subclass('c:') + self._check_str_subclass('c:a') + self._check_str_subclass('c:a\\b.txt') + self._check_str_subclass('c:\\') + self._check_str_subclass('c:\\a') + self._check_str_subclass('c:\\a\\b.txt') + self._check_str_subclass('\\\\some\\share') + self._check_str_subclass('\\\\some\\share\\a') + self._check_str_subclass('\\\\some\\share\\a\\b.txt') + def test_with_segments_common(self): class P(self.cls): def __init__(self, *pathsegments, session_id): @@ -157,31 +233,6 @@ def with_segments(self, *pathsegments): for parent in p.parents: self.assertEqual(42, parent.session_id) - def _check_parse_path(self, raw_path, *expected): - sep = self.pathmod.sep - actual = self.cls._parse_path(raw_path.replace('/', sep)) - self.assertEqual(actual, expected) - if altsep := self.pathmod.altsep: - actual = self.cls._parse_path(raw_path.replace('/', altsep)) - self.assertEqual(actual, expected) - - def test_parse_path_common(self): - check = self._check_parse_path - sep = self.pathmod.sep - check('', '', '', []) - check('a', '', '', ['a']) - check('a/', '', '', ['a']) - check('a/b', '', '', ['a', 'b']) - check('a/b/', '', '', ['a', 'b']) - check('a/b/c/d', '', '', ['a', 'b', 'c', 'd']) - check('a/b//c/d', '', '', ['a', 'b', 'c', 'd']) - check('a/b/c/d', '', '', ['a', 'b', 'c', 'd']) - check('.', '', '', []) - check('././b', '', '', ['b']) - check('a/./b', '', '', ['a', 'b']) - check('a/./.', '', '', ['a']) - check('/a/b', '', sep, ['a', 'b']) - def test_join_common(self): P = self.cls p = P('a/b') @@ -193,6 +244,55 @@ def test_join_common(self): pp = p.joinpath('/c') self.assertEqual(pp, P('/c')) + @needs_posix + def test_join_posix(self): + P = self.cls + p = P('//a') + pp = p.joinpath('b') + self.assertEqual(pp, P('//a/b')) + pp = P('/a').joinpath('//c') + self.assertEqual(pp, P('//c')) + pp = P('//a').joinpath('/c') + self.assertEqual(pp, P('/c')) + + @needs_windows + def test_join_windows(self): + P = self.cls + p = P('C:/a/b') + pp = p.joinpath('x/y') + self.assertEqual(pp, P('C:/a/b/x/y')) + pp = p.joinpath('/x/y') + self.assertEqual(pp, P('C:/x/y')) + # Joining with a different drive => the first path is ignored, even + # if the second path is relative. + pp = p.joinpath('D:x/y') + self.assertEqual(pp, P('D:x/y')) + pp = p.joinpath('D:/x/y') + self.assertEqual(pp, P('D:/x/y')) + pp = p.joinpath('//host/share/x/y') + self.assertEqual(pp, P('//host/share/x/y')) + # Joining with the same drive => the first path is appended to if + # the second path is relative. + pp = p.joinpath('c:x/y') + self.assertEqual(pp, P('C:/a/b/x/y')) + pp = p.joinpath('c:/x/y') + self.assertEqual(pp, P('C:/x/y')) + # Joining with files with NTFS data streams => the filename should + # not be parsed as a drive letter + pp = p.joinpath(P('./d:s')) + self.assertEqual(pp, P('C:/a/b/d:s')) + pp = p.joinpath(P('./dd:s')) + self.assertEqual(pp, P('C:/a/b/dd:s')) + pp = p.joinpath(P('E:d:s')) + self.assertEqual(pp, P('E:d:s')) + # Joining onto a UNC path with no root + pp = P('//').joinpath('server') + self.assertEqual(pp, P('//server')) + pp = P('//server').joinpath('share') + self.assertEqual(pp, P('//server/share')) + pp = P('//./BootPartition').joinpath('Windows') + self.assertEqual(pp, P('//./BootPartition/Windows')) + def test_div_common(self): # Basically the same as joinpath(). P = self.cls @@ -209,6 +309,44 @@ def test_div_common(self): pp = p/ '/c' self.assertEqual(pp, P('/c')) + @needs_posix + def test_div_posix(self): + # Basically the same as joinpath(). + P = self.cls + p = P('//a') + pp = p / 'b' + self.assertEqual(pp, P('//a/b')) + pp = P('/a') / '//c' + self.assertEqual(pp, P('//c')) + pp = P('//a') / '/c' + self.assertEqual(pp, P('/c')) + + @needs_windows + def test_div_windows(self): + # Basically the same as joinpath(). + P = self.cls + p = P('C:/a/b') + self.assertEqual(p / 'x/y', P('C:/a/b/x/y')) + self.assertEqual(p / 'x' / 'y', P('C:/a/b/x/y')) + self.assertEqual(p / '/x/y', P('C:/x/y')) + self.assertEqual(p / '/x' / 'y', P('C:/x/y')) + # Joining with a different drive => the first path is ignored, even + # if the second path is relative. + self.assertEqual(p / 'D:x/y', P('D:x/y')) + self.assertEqual(p / 'D:' / 'x/y', P('D:x/y')) + self.assertEqual(p / 'D:/x/y', P('D:/x/y')) + self.assertEqual(p / 'D:' / '/x/y', P('D:/x/y')) + self.assertEqual(p / '//host/share/x/y', P('//host/share/x/y')) + # Joining with the same drive => the first path is appended to if + # the second path is relative. + self.assertEqual(p / 'c:x/y', P('C:/a/b/x/y')) + self.assertEqual(p / 'c:/x/y', P('C:/x/y')) + # Joining with files with NTFS data streams => the filename should + # not be parsed as a drive letter + self.assertEqual(p / P('./d:s'), P('C:/a/b/d:s')) + self.assertEqual(p / P('./dd:s'), P('C:/a/b/dd:s')) + self.assertEqual(p / P('E:d:s'), P('E:d:s')) + def _check_str(self, expected, args): p = self.cls(*args) self.assertEqual(str(p), expected.replace('/', self.sep)) @@ -217,45 +355,33 @@ def test_str_common(self): # Canonicalized paths roundtrip. for pathstr in ('a', 'a/b', 'a/b/c', '/', '/a/b', '/a/b/c'): self._check_str(pathstr, (pathstr,)) - # Special case for the empty path. - self._check_str('.', ('',)) # Other tests for str() are in test_equivalences(). + @needs_windows + def test_str_windows(self): + p = self.cls('a/b/c') + self.assertEqual(str(p), 'a\\b\\c') + p = self.cls('c:/a/b/c') + self.assertEqual(str(p), 'c:\\a\\b\\c') + p = self.cls('//a/b') + self.assertEqual(str(p), '\\\\a\\b\\') + p = self.cls('//a/b/c') + self.assertEqual(str(p), '\\\\a\\b\\c') + p = self.cls('//a/b/c/d') + self.assertEqual(str(p), '\\\\a\\b\\c\\d') + def test_as_posix_common(self): P = self.cls for pathstr in ('a', 'a/b', 'a/b/c', '/', '/a/b', '/a/b/c'): self.assertEqual(P(pathstr).as_posix(), pathstr) # Other tests for as_posix() are in test_equivalences(). - def test_repr_common(self): - for pathstr in ('a', 'a/b', 'a/b/c', '/', '/a/b', '/a/b/c'): - with self.subTest(pathstr=pathstr): - p = self.cls(pathstr) - clsname = p.__class__.__name__ - r = repr(p) - # The repr() is in the form ClassName("forward-slashes path"). - self.assertTrue(r.startswith(clsname + '('), r) - self.assertTrue(r.endswith(')'), r) - inner = r[len(clsname) + 1 : -1] - self.assertEqual(eval(inner), p.as_posix()) - - def test_eq_common(self): - P = self.cls - self.assertEqual(P('a/b'), P('a/b')) - self.assertEqual(P('a/b'), P('a', 'b')) - self.assertNotEqual(P('a/b'), P('a')) - self.assertNotEqual(P('a/b'), P('/a/b')) - self.assertNotEqual(P('a/b'), P()) - self.assertNotEqual(P('/a/b'), P('/')) - self.assertNotEqual(P(), P('/')) - self.assertNotEqual(P(), "") - self.assertNotEqual(P(), {}) - self.assertNotEqual(P(), int) + def test_match_empty(self): + P = self.cls + self.assertRaises(ValueError, P('a').match, '') def test_match_common(self): P = self.cls - self.assertRaises(ValueError, P('a').match, '') - self.assertRaises(ValueError, P('a').match, '.') # Simple relative pattern. self.assertTrue(P('b.py').match('b.py')) self.assertTrue(P('a/b.py').match('b.py')) @@ -287,39 +413,123 @@ def test_match_common(self): self.assertFalse(P('/ab.py').match('/a/*.py')) self.assertFalse(P('/a/b/c.py').match('/a/*.py')) # Multi-part glob-style pattern. - self.assertTrue(P('a').match('**')) - self.assertTrue(P('c.py').match('**')) - self.assertTrue(P('a/b/c.py').match('**')) - self.assertTrue(P('/a/b/c.py').match('**')) - self.assertTrue(P('/a/b/c.py').match('/**')) - self.assertTrue(P('/a/b/c.py').match('**/')) - self.assertTrue(P('/a/b/c.py').match('/a/**')) - self.assertTrue(P('/a/b/c.py').match('**/*.py')) - self.assertTrue(P('/a/b/c.py').match('/**/*.py')) + self.assertFalse(P('/a/b/c.py').match('/**/*.py')) self.assertTrue(P('/a/b/c.py').match('/a/**/*.py')) - self.assertTrue(P('/a/b/c.py').match('/a/b/**/*.py')) - self.assertTrue(P('/a/b/c.py').match('/**/**/**/**/*.py')) - self.assertFalse(P('c.py').match('**/a.py')) - self.assertFalse(P('c.py').match('c/**')) - self.assertFalse(P('a/b/c.py').match('**/a')) - self.assertFalse(P('a/b/c.py').match('**/a/b')) - self.assertFalse(P('a/b/c.py').match('**/a/b/c')) - self.assertFalse(P('a/b/c.py').match('**/a/b/c.')) - self.assertFalse(P('a/b/c.py').match('**/a/b/c./**')) - self.assertFalse(P('a/b/c.py').match('**/a/b/c./**')) - self.assertFalse(P('a/b/c.py').match('/a/b/c.py/**')) - self.assertFalse(P('a/b/c.py').match('/**/a/b/c.py')) - self.assertRaises(ValueError, P('a').match, '**a/b/c') - self.assertRaises(ValueError, P('a').match, 'a/b/c**') # Case-sensitive flag self.assertFalse(P('A.py').match('a.PY', case_sensitive=True)) self.assertTrue(P('A.py').match('a.PY', case_sensitive=False)) self.assertFalse(P('c:/a/B.Py').match('C:/A/*.pY', case_sensitive=True)) self.assertTrue(P('/a/b/c.py').match('/A/*/*.Py', case_sensitive=False)) # Matching against empty path - self.assertFalse(P().match('*')) - self.assertTrue(P().match('**')) - self.assertFalse(P().match('**/*')) + self.assertFalse(P('').match('*')) + self.assertFalse(P('').match('**')) + self.assertFalse(P('').match('**/*')) + + @needs_posix + def test_match_posix(self): + P = self.cls + self.assertFalse(P('A.py').match('a.PY')) + + @needs_windows + def test_match_windows(self): + P = self.cls + # Absolute patterns. + self.assertTrue(P('c:/b.py').match('*:/*.py')) + self.assertTrue(P('c:/b.py').match('c:/*.py')) + self.assertFalse(P('d:/b.py').match('c:/*.py')) # wrong drive + self.assertFalse(P('b.py').match('/*.py')) + self.assertFalse(P('b.py').match('c:*.py')) + self.assertFalse(P('b.py').match('c:/*.py')) + self.assertFalse(P('c:b.py').match('/*.py')) + self.assertFalse(P('c:b.py').match('c:/*.py')) + self.assertFalse(P('/b.py').match('c:*.py')) + self.assertFalse(P('/b.py').match('c:/*.py')) + # UNC patterns. + self.assertTrue(P('//some/share/a.py').match('//*/*/*.py')) + self.assertTrue(P('//some/share/a.py').match('//some/share/*.py')) + self.assertFalse(P('//other/share/a.py').match('//some/share/*.py')) + self.assertFalse(P('//some/share/a/b.py').match('//some/share/*.py')) + # Case-insensitivity. + self.assertTrue(P('B.py').match('b.PY')) + self.assertTrue(P('c:/a/B.Py').match('C:/A/*.pY')) + self.assertTrue(P('//Some/Share/B.Py').match('//somE/sharE/*.pY')) + # Path anchor doesn't match pattern anchor + self.assertFalse(P('c:/b.py').match('/*.py')) # 'c:/' vs '/' + self.assertFalse(P('c:/b.py').match('c:*.py')) # 'c:/' vs 'c:' + self.assertFalse(P('//some/share/a.py').match('/*.py')) # '//some/share/' vs '/' + + def test_full_match_common(self): + P = self.cls + # Simple relative pattern. + self.assertTrue(P('b.py').full_match('b.py')) + self.assertFalse(P('a/b.py').full_match('b.py')) + self.assertFalse(P('/a/b.py').full_match('b.py')) + self.assertFalse(P('a.py').full_match('b.py')) + self.assertFalse(P('b/py').full_match('b.py')) + self.assertFalse(P('/a.py').full_match('b.py')) + self.assertFalse(P('b.py/c').full_match('b.py')) + # Wildcard relative pattern. + self.assertTrue(P('b.py').full_match('*.py')) + self.assertFalse(P('a/b.py').full_match('*.py')) + self.assertFalse(P('/a/b.py').full_match('*.py')) + self.assertFalse(P('b.pyc').full_match('*.py')) + self.assertFalse(P('b./py').full_match('*.py')) + self.assertFalse(P('b.py/c').full_match('*.py')) + # Multi-part relative pattern. + self.assertTrue(P('ab/c.py').full_match('a*/*.py')) + self.assertFalse(P('/d/ab/c.py').full_match('a*/*.py')) + self.assertFalse(P('a.py').full_match('a*/*.py')) + self.assertFalse(P('/dab/c.py').full_match('a*/*.py')) + self.assertFalse(P('ab/c.py/d').full_match('a*/*.py')) + # Absolute pattern. + self.assertTrue(P('/b.py').full_match('/*.py')) + self.assertFalse(P('b.py').full_match('/*.py')) + self.assertFalse(P('a/b.py').full_match('/*.py')) + self.assertFalse(P('/a/b.py').full_match('/*.py')) + # Multi-part absolute pattern. + self.assertTrue(P('/a/b.py').full_match('/a/*.py')) + self.assertFalse(P('/ab.py').full_match('/a/*.py')) + self.assertFalse(P('/a/b/c.py').full_match('/a/*.py')) + # Multi-part glob-style pattern. + self.assertTrue(P('a').full_match('**')) + self.assertTrue(P('c.py').full_match('**')) + self.assertTrue(P('a/b/c.py').full_match('**')) + self.assertTrue(P('/a/b/c.py').full_match('**')) + self.assertTrue(P('/a/b/c.py').full_match('/**')) + self.assertTrue(P('/a/b/c.py').full_match('/a/**')) + self.assertTrue(P('/a/b/c.py').full_match('**/*.py')) + self.assertTrue(P('/a/b/c.py').full_match('/**/*.py')) + self.assertTrue(P('/a/b/c.py').full_match('/a/**/*.py')) + self.assertTrue(P('/a/b/c.py').full_match('/a/b/**/*.py')) + self.assertTrue(P('/a/b/c.py').full_match('/**/**/**/**/*.py')) + self.assertFalse(P('c.py').full_match('**/a.py')) + self.assertFalse(P('c.py').full_match('c/**')) + self.assertFalse(P('a/b/c.py').full_match('**/a')) + self.assertFalse(P('a/b/c.py').full_match('**/a/b')) + self.assertFalse(P('a/b/c.py').full_match('**/a/b/c')) + self.assertFalse(P('a/b/c.py').full_match('**/a/b/c.')) + self.assertFalse(P('a/b/c.py').full_match('**/a/b/c./**')) + self.assertFalse(P('a/b/c.py').full_match('**/a/b/c./**')) + self.assertFalse(P('a/b/c.py').full_match('/a/b/c.py/**')) + self.assertFalse(P('a/b/c.py').full_match('/**/a/b/c.py')) + self.assertRaises(ValueError, P('a').full_match, '**a/b/c') + self.assertRaises(ValueError, P('a').full_match, 'a/b/c**') + # Case-sensitive flag + self.assertFalse(P('A.py').full_match('a.PY', case_sensitive=True)) + self.assertTrue(P('A.py').full_match('a.PY', case_sensitive=False)) + self.assertFalse(P('c:/a/B.Py').full_match('C:/A/*.pY', case_sensitive=True)) + self.assertTrue(P('/a/b/c.py').full_match('/A/*/*.Py', case_sensitive=False)) + # Matching against empty path + self.assertFalse(P('').full_match('*')) + self.assertTrue(P('').full_match('**')) + self.assertFalse(P('').full_match('**/*')) + # Matching with empty pattern + self.assertTrue(P('').full_match('')) + self.assertTrue(P('.').full_match('.')) + self.assertFalse(P('/').full_match('')) + self.assertFalse(P('/').full_match('.')) + self.assertFalse(P('foo').full_match('')) + self.assertFalse(P('foo').full_match('.')) def test_parts_common(self): # `parts` returns a tuple. @@ -333,23 +543,18 @@ def test_parts_common(self): parts = p.parts self.assertEqual(parts, (sep, 'a', 'b')) - def test_equivalences(self): - for k, tuples in self.equivalences.items(): - canon = k.replace('/', self.sep) - posix = k.replace(self.sep, '/') - if canon != posix: - tuples = tuples + [ - tuple(part.replace('/', self.sep) for part in t) - for t in tuples - ] - tuples.append((posix, )) - pcanon = self.cls(canon) - for t in tuples: - p = self.cls(*t) - self.assertEqual(p, pcanon, "failed with args {}".format(t)) - self.assertEqual(hash(p), hash(pcanon)) - self.assertEqual(str(p), canon) - self.assertEqual(p.as_posix(), posix) + @needs_windows + def test_parts_windows(self): + P = self.cls + p = P('c:a/b') + parts = p.parts + self.assertEqual(parts, ('c:', 'a', 'b')) + p = P('c:/a/b') + parts = p.parts + self.assertEqual(parts, ('c:\\', 'a', 'b')) + p = P('//a/b/c/d') + parts = p.parts + self.assertEqual(parts, ('\\\\a\\b\\', 'c', 'd')) def test_parent_common(self): # Relative @@ -357,8 +562,8 @@ def test_parent_common(self): p = P('a/b/c') self.assertEqual(p.parent, P('a/b')) self.assertEqual(p.parent.parent, P('a')) - self.assertEqual(p.parent.parent.parent, P()) - self.assertEqual(p.parent.parent.parent.parent, P()) + self.assertEqual(p.parent.parent.parent, P('')) + self.assertEqual(p.parent.parent.parent.parent, P('')) # Anchored p = P('/a/b/c') self.assertEqual(p.parent, P('/a/b')) @@ -366,6 +571,25 @@ def test_parent_common(self): self.assertEqual(p.parent.parent.parent, P('/')) self.assertEqual(p.parent.parent.parent.parent, P('/')) + @needs_windows + def test_parent_windows(self): + # Anchored + P = self.cls + p = P('z:a/b/c') + self.assertEqual(p.parent, P('z:a/b')) + self.assertEqual(p.parent.parent, P('z:a')) + self.assertEqual(p.parent.parent.parent, P('z:')) + self.assertEqual(p.parent.parent.parent.parent, P('z:')) + p = P('z:/a/b/c') + self.assertEqual(p.parent, P('z:/a/b')) + self.assertEqual(p.parent.parent, P('z:/a')) + self.assertEqual(p.parent.parent.parent, P('z:/')) + self.assertEqual(p.parent.parent.parent.parent, P('z:/')) + p = P('//a/b/c/d') + self.assertEqual(p.parent, P('//a/b/c')) + self.assertEqual(p.parent.parent, P('//a/b')) + self.assertEqual(p.parent.parent.parent, P('//a/b')) + def test_parents_common(self): # Relative P = self.cls @@ -374,17 +598,17 @@ def test_parents_common(self): self.assertEqual(len(par), 3) self.assertEqual(par[0], P('a/b')) self.assertEqual(par[1], P('a')) - self.assertEqual(par[2], P('.')) - self.assertEqual(par[-1], P('.')) + self.assertEqual(par[2], P('')) + self.assertEqual(par[-1], P('')) self.assertEqual(par[-2], P('a')) self.assertEqual(par[-3], P('a/b')) self.assertEqual(par[0:1], (P('a/b'),)) self.assertEqual(par[:2], (P('a/b'), P('a'))) self.assertEqual(par[:-1], (P('a/b'), P('a'))) - self.assertEqual(par[1:], (P('a'), P('.'))) - self.assertEqual(par[::2], (P('a/b'), P('.'))) - self.assertEqual(par[::-1], (P('.'), P('a'), P('a/b'))) - self.assertEqual(list(par), [P('a/b'), P('a'), P('.')]) + self.assertEqual(par[1:], (P('a'), P(''))) + self.assertEqual(par[::2], (P('a/b'), P(''))) + self.assertEqual(par[::-1], (P(''), P('a'), P('a/b'))) + self.assertEqual(list(par), [P('a/b'), P('a'), P('')]) with self.assertRaises(IndexError): par[-4] with self.assertRaises(IndexError): @@ -413,12 +637,71 @@ def test_parents_common(self): with self.assertRaises(IndexError): par[3] + @needs_windows + def test_parents_windows(self): + # Anchored + P = self.cls + p = P('z:a/b/') + par = p.parents + self.assertEqual(len(par), 2) + self.assertEqual(par[0], P('z:a')) + self.assertEqual(par[1], P('z:')) + self.assertEqual(par[0:1], (P('z:a'),)) + self.assertEqual(par[:-1], (P('z:a'),)) + self.assertEqual(par[:2], (P('z:a'), P('z:'))) + self.assertEqual(par[1:], (P('z:'),)) + self.assertEqual(par[::2], (P('z:a'),)) + self.assertEqual(par[::-1], (P('z:'), P('z:a'))) + self.assertEqual(list(par), [P('z:a'), P('z:')]) + with self.assertRaises(IndexError): + par[2] + p = P('z:/a/b/') + par = p.parents + self.assertEqual(len(par), 2) + self.assertEqual(par[0], P('z:/a')) + self.assertEqual(par[1], P('z:/')) + self.assertEqual(par[0:1], (P('z:/a'),)) + self.assertEqual(par[0:-1], (P('z:/a'),)) + self.assertEqual(par[:2], (P('z:/a'), P('z:/'))) + self.assertEqual(par[1:], (P('z:/'),)) + self.assertEqual(par[::2], (P('z:/a'),)) + self.assertEqual(par[::-1], (P('z:/'), P('z:/a'),)) + self.assertEqual(list(par), [P('z:/a'), P('z:/')]) + with self.assertRaises(IndexError): + par[2] + p = P('//a/b/c/d') + par = p.parents + self.assertEqual(len(par), 2) + self.assertEqual(par[0], P('//a/b/c')) + self.assertEqual(par[1], P('//a/b')) + self.assertEqual(par[0:1], (P('//a/b/c'),)) + self.assertEqual(par[0:-1], (P('//a/b/c'),)) + self.assertEqual(par[:2], (P('//a/b/c'), P('//a/b'))) + self.assertEqual(par[1:], (P('//a/b'),)) + self.assertEqual(par[::2], (P('//a/b/c'),)) + self.assertEqual(par[::-1], (P('//a/b'), P('//a/b/c'))) + self.assertEqual(list(par), [P('//a/b/c'), P('//a/b')]) + with self.assertRaises(IndexError): + par[2] + def test_drive_common(self): P = self.cls self.assertEqual(P('a/b').drive, '') self.assertEqual(P('/a/b').drive, '') self.assertEqual(P('').drive, '') + @needs_windows + def test_drive_windows(self): + P = self.cls + self.assertEqual(P('c:').drive, 'c:') + self.assertEqual(P('c:a/b').drive, 'c:') + self.assertEqual(P('c:/').drive, 'c:') + self.assertEqual(P('c:/a/b/').drive, 'c:') + self.assertEqual(P('//a/b').drive, '\\\\a\\b') + self.assertEqual(P('//a/b/').drive, '\\\\a\\b') + self.assertEqual(P('//a/b/c/d').drive, '\\\\a\\b') + self.assertEqual(P('./c:a').drive, '') + def test_root_common(self): P = self.cls sep = self.sep @@ -427,6 +710,24 @@ def test_root_common(self): self.assertEqual(P('/').root, sep) self.assertEqual(P('/a/b').root, sep) + @needs_posix + def test_root_posix(self): + P = self.cls + self.assertEqual(P('/a/b').root, '/') + # POSIX special case for two leading slashes. + self.assertEqual(P('//a/b').root, '//') + + @needs_windows + def test_root_windows(self): + P = self.cls + self.assertEqual(P('c:').root, '') + self.assertEqual(P('c:a/b').root, '') + self.assertEqual(P('c:/').root, '\\') + self.assertEqual(P('c:/a/b/').root, '\\') + self.assertEqual(P('//a/b').root, '\\') + self.assertEqual(P('//a/b/').root, '\\') + self.assertEqual(P('//a/b/c/d').root, '\\') + def test_anchor_common(self): P = self.cls sep = self.sep @@ -435,17 +736,43 @@ def test_anchor_common(self): self.assertEqual(P('/').anchor, sep) self.assertEqual(P('/a/b').anchor, sep) - def test_name_common(self): + @needs_windows + def test_anchor_windows(self): + P = self.cls + self.assertEqual(P('c:').anchor, 'c:') + self.assertEqual(P('c:a/b').anchor, 'c:') + self.assertEqual(P('c:/').anchor, 'c:\\') + self.assertEqual(P('c:/a/b/').anchor, 'c:\\') + self.assertEqual(P('//a/b').anchor, '\\\\a\\b\\') + self.assertEqual(P('//a/b/').anchor, '\\\\a\\b\\') + self.assertEqual(P('//a/b/c/d').anchor, '\\\\a\\b\\') + + def test_name_empty(self): P = self.cls self.assertEqual(P('').name, '') - self.assertEqual(P('.').name, '') + self.assertEqual(P('.').name, '.') + self.assertEqual(P('/a/b/.').name, '.') + + def test_name_common(self): + P = self.cls self.assertEqual(P('/').name, '') self.assertEqual(P('a/b').name, 'b') self.assertEqual(P('/a/b').name, 'b') - self.assertEqual(P('/a/b/.').name, 'b') self.assertEqual(P('a/b.py').name, 'b.py') self.assertEqual(P('/a/b.py').name, 'b.py') + @needs_windows + def test_name_windows(self): + P = self.cls + self.assertEqual(P('c:').name, '') + self.assertEqual(P('c:/').name, '') + self.assertEqual(P('c:a/b').name, 'b') + self.assertEqual(P('c:/a/b').name, 'b') + self.assertEqual(P('c:a/b.py').name, 'b.py') + self.assertEqual(P('c:/a/b.py').name, 'b.py') + self.assertEqual(P('//My.py/Share.php').name, '') + self.assertEqual(P('//My.py/Share.php/a/b').name, 'b') + def test_suffix_common(self): P = self.cls self.assertEqual(P('').suffix, '') @@ -466,6 +793,26 @@ def test_suffix_common(self): self.assertEqual(P('a/Some name. Ending with a dot.').suffix, '') self.assertEqual(P('/a/Some name. Ending with a dot.').suffix, '') + @needs_windows + def test_suffix_windows(self): + P = self.cls + self.assertEqual(P('c:').suffix, '') + self.assertEqual(P('c:/').suffix, '') + self.assertEqual(P('c:a/b').suffix, '') + self.assertEqual(P('c:/a/b').suffix, '') + self.assertEqual(P('c:a/b.py').suffix, '.py') + self.assertEqual(P('c:/a/b.py').suffix, '.py') + self.assertEqual(P('c:a/.hgrc').suffix, '') + self.assertEqual(P('c:/a/.hgrc').suffix, '') + self.assertEqual(P('c:a/.hg.rc').suffix, '.rc') + self.assertEqual(P('c:/a/.hg.rc').suffix, '.rc') + self.assertEqual(P('c:a/b.tar.gz').suffix, '.gz') + self.assertEqual(P('c:/a/b.tar.gz').suffix, '.gz') + self.assertEqual(P('c:a/Some name. Ending with a dot.').suffix, '') + self.assertEqual(P('c:/a/Some name. Ending with a dot.').suffix, '') + self.assertEqual(P('//My.py/Share.php').suffix, '') + self.assertEqual(P('//My.py/Share.php/a/b').suffix, '') + def test_suffixes_common(self): P = self.cls self.assertEqual(P('').suffixes, []) @@ -485,10 +832,33 @@ def test_suffixes_common(self): self.assertEqual(P('a/Some name. Ending with a dot.').suffixes, []) self.assertEqual(P('/a/Some name. Ending with a dot.').suffixes, []) - def test_stem_common(self): + @needs_windows + def test_suffixes_windows(self): + P = self.cls + self.assertEqual(P('c:').suffixes, []) + self.assertEqual(P('c:/').suffixes, []) + self.assertEqual(P('c:a/b').suffixes, []) + self.assertEqual(P('c:/a/b').suffixes, []) + self.assertEqual(P('c:a/b.py').suffixes, ['.py']) + self.assertEqual(P('c:/a/b.py').suffixes, ['.py']) + self.assertEqual(P('c:a/.hgrc').suffixes, []) + self.assertEqual(P('c:/a/.hgrc').suffixes, []) + self.assertEqual(P('c:a/.hg.rc').suffixes, ['.rc']) + self.assertEqual(P('c:/a/.hg.rc').suffixes, ['.rc']) + self.assertEqual(P('c:a/b.tar.gz').suffixes, ['.tar', '.gz']) + self.assertEqual(P('c:/a/b.tar.gz').suffixes, ['.tar', '.gz']) + self.assertEqual(P('//My.py/Share.php').suffixes, []) + self.assertEqual(P('//My.py/Share.php/a/b').suffixes, []) + self.assertEqual(P('c:a/Some name. Ending with a dot.').suffixes, []) + self.assertEqual(P('c:/a/Some name. Ending with a dot.').suffixes, []) + + def test_stem_empty(self): P = self.cls self.assertEqual(P('').stem, '') - self.assertEqual(P('.').stem, '') + self.assertEqual(P('.').stem, '.') + + def test_stem_common(self): + P = self.cls self.assertEqual(P('..').stem, '..') self.assertEqual(P('/').stem, '') self.assertEqual(P('a/b').stem, 'b') @@ -499,6 +869,20 @@ def test_stem_common(self): self.assertEqual(P('a/Some name. Ending with a dot.').stem, 'Some name. Ending with a dot.') + @needs_windows + def test_stem_windows(self): + P = self.cls + self.assertEqual(P('c:').stem, '') + self.assertEqual(P('c:.').stem, '') + self.assertEqual(P('c:..').stem, '..') + self.assertEqual(P('c:/').stem, '') + self.assertEqual(P('c:a/b').stem, 'b') + self.assertEqual(P('c:a/b.py').stem, 'b') + self.assertEqual(P('c:a/.hgrc').stem, '.hgrc') + self.assertEqual(P('c:a/.hg.rc').stem, '.hg') + self.assertEqual(P('c:a/b.tar.gz').stem, 'b.tar') + self.assertEqual(P('c:a/Some name. Ending with a dot.').stem, + 'Some name. Ending with a dot.') def test_with_name_common(self): P = self.cls self.assertEqual(P('a/b').with_name('d.xml'), P('a/d.xml')) @@ -507,11 +891,34 @@ def test_with_name_common(self): self.assertEqual(P('/a/b.py').with_name('d.xml'), P('/a/d.xml')) self.assertEqual(P('a/Dot ending.').with_name('d.xml'), P('a/d.xml')) self.assertEqual(P('/a/Dot ending.').with_name('d.xml'), P('/a/d.xml')) - self.assertRaises(ValueError, P('').with_name, 'd.xml') - self.assertRaises(ValueError, P('.').with_name, 'd.xml') - self.assertRaises(ValueError, P('/').with_name, 'd.xml') - self.assertRaises(ValueError, P('a/b').with_name, '') - self.assertRaises(ValueError, P('a/b').with_name, '.') + + @needs_windows + def test_with_name_windows(self): + P = self.cls + self.assertEqual(P('c:a/b').with_name('d.xml'), P('c:a/d.xml')) + self.assertEqual(P('c:/a/b').with_name('d.xml'), P('c:/a/d.xml')) + self.assertEqual(P('c:a/Dot ending.').with_name('d.xml'), P('c:a/d.xml')) + self.assertEqual(P('c:/a/Dot ending.').with_name('d.xml'), P('c:/a/d.xml')) + self.assertRaises(ValueError, P('c:').with_name, 'd.xml') + self.assertRaises(ValueError, P('c:/').with_name, 'd.xml') + self.assertRaises(ValueError, P('//My/Share').with_name, 'd.xml') + self.assertEqual(str(P('a').with_name('d:')), '.\\d:') + self.assertEqual(str(P('a').with_name('d:e')), '.\\d:e') + self.assertEqual(P('c:a/b').with_name('d:'), P('c:a/d:')) + self.assertEqual(P('c:a/b').with_name('d:e'), P('c:a/d:e')) + self.assertRaises(ValueError, P('c:a/b').with_name, 'd:/e') + self.assertRaises(ValueError, P('c:a/b').with_name, '//My/Share') + + def test_with_name_empty(self): + P = self.cls + self.assertEqual(P('').with_name('d.xml'), P('d.xml')) + self.assertEqual(P('.').with_name('d.xml'), P('d.xml')) + self.assertEqual(P('/').with_name('d.xml'), P('/d.xml')) + self.assertEqual(P('a/b').with_name(''), P('a/')) + self.assertEqual(P('a/b').with_name('.'), P('a/.')) + + def test_with_name_seps(self): + P = self.cls self.assertRaises(ValueError, P('a/b').with_name, '/c') self.assertRaises(ValueError, P('a/b').with_name, 'c/') self.assertRaises(ValueError, P('a/b').with_name, 'c/d') @@ -525,11 +932,34 @@ def test_with_stem_common(self): self.assertEqual(P('/a/b.tar.gz').with_stem('d'), P('/a/d.gz')) self.assertEqual(P('a/Dot ending.').with_stem('d'), P('a/d')) self.assertEqual(P('/a/Dot ending.').with_stem('d'), P('/a/d')) - self.assertRaises(ValueError, P('').with_stem, 'd') - self.assertRaises(ValueError, P('.').with_stem, 'd') - self.assertRaises(ValueError, P('/').with_stem, 'd') - self.assertRaises(ValueError, P('a/b').with_stem, '') - self.assertRaises(ValueError, P('a/b').with_stem, '.') + + @needs_windows + def test_with_stem_windows(self): + P = self.cls + self.assertEqual(P('c:a/b').with_stem('d'), P('c:a/d')) + self.assertEqual(P('c:/a/b').with_stem('d'), P('c:/a/d')) + self.assertEqual(P('c:a/Dot ending.').with_stem('d'), P('c:a/d')) + self.assertEqual(P('c:/a/Dot ending.').with_stem('d'), P('c:/a/d')) + self.assertRaises(ValueError, P('c:').with_stem, 'd') + self.assertRaises(ValueError, P('c:/').with_stem, 'd') + self.assertRaises(ValueError, P('//My/Share').with_stem, 'd') + self.assertEqual(str(P('a').with_stem('d:')), '.\\d:') + self.assertEqual(str(P('a').with_stem('d:e')), '.\\d:e') + self.assertEqual(P('c:a/b').with_stem('d:'), P('c:a/d:')) + self.assertEqual(P('c:a/b').with_stem('d:e'), P('c:a/d:e')) + self.assertRaises(ValueError, P('c:a/b').with_stem, 'd:/e') + self.assertRaises(ValueError, P('c:a/b').with_stem, '//My/Share') + + def test_with_stem_empty(self): + P = self.cls + self.assertEqual(P('').with_stem('d'), P('d')) + self.assertEqual(P('.').with_stem('d'), P('d')) + self.assertEqual(P('/').with_stem('d'), P('/d')) + self.assertEqual(P('a/b').with_stem(''), P('a/')) + self.assertEqual(P('a/b').with_stem('.'), P('a/.')) + + def test_with_stem_seps(self): + P = self.cls self.assertRaises(ValueError, P('a/b').with_stem, '/c') self.assertRaises(ValueError, P('a/b').with_stem, 'c/') self.assertRaises(ValueError, P('a/b').with_stem, 'c/d') @@ -543,10 +973,40 @@ def test_with_suffix_common(self): # Stripping suffix. self.assertEqual(P('a/b.py').with_suffix(''), P('a/b')) self.assertEqual(P('/a/b').with_suffix(''), P('/a/b')) + + @needs_windows + def test_with_suffix_windows(self): + P = self.cls + self.assertEqual(P('c:a/b').with_suffix('.gz'), P('c:a/b.gz')) + self.assertEqual(P('c:/a/b').with_suffix('.gz'), P('c:/a/b.gz')) + self.assertEqual(P('c:a/b.py').with_suffix('.gz'), P('c:a/b.gz')) + self.assertEqual(P('c:/a/b.py').with_suffix('.gz'), P('c:/a/b.gz')) # Path doesn't have a "filename" component. self.assertRaises(ValueError, P('').with_suffix, '.gz') self.assertRaises(ValueError, P('.').with_suffix, '.gz') self.assertRaises(ValueError, P('/').with_suffix, '.gz') + self.assertRaises(ValueError, P('//My/Share').with_suffix, '.gz') + # Invalid suffix. + self.assertRaises(ValueError, P('c:a/b').with_suffix, 'gz') + self.assertRaises(ValueError, P('c:a/b').with_suffix, '/') + self.assertRaises(ValueError, P('c:a/b').with_suffix, '\\') + self.assertRaises(ValueError, P('c:a/b').with_suffix, 'c:') + self.assertRaises(ValueError, P('c:a/b').with_suffix, '/.gz') + self.assertRaises(ValueError, P('c:a/b').with_suffix, '\\.gz') + self.assertRaises(ValueError, P('c:a/b').with_suffix, 'c:.gz') + self.assertRaises(ValueError, P('c:a/b').with_suffix, 'c/d') + self.assertRaises(ValueError, P('c:a/b').with_suffix, 'c\\d') + self.assertRaises(ValueError, P('c:a/b').with_suffix, '.c/d') + self.assertRaises(ValueError, P('c:a/b').with_suffix, '.c\\d') + + def test_with_suffix_empty(self): + P = self.cls + # Path doesn't have a "filename" component. + self.assertRaises(ValueError, P('').with_suffix, '.gz') + self.assertRaises(ValueError, P('/').with_suffix, '.gz') + + def test_with_suffix_seps(self): + P = self.cls # Invalid suffix. self.assertRaises(ValueError, P('a/b').with_suffix, 'gz') self.assertRaises(ValueError, P('a/b').with_suffix, '/') @@ -562,30 +1022,26 @@ def test_relative_to_common(self): p = P('a/b') self.assertRaises(TypeError, p.relative_to) self.assertRaises(TypeError, p.relative_to, b'a') - self.assertEqual(p.relative_to(P()), P('a/b')) + self.assertEqual(p.relative_to(P('')), P('a/b')) self.assertEqual(p.relative_to(''), P('a/b')) self.assertEqual(p.relative_to(P('a')), P('b')) self.assertEqual(p.relative_to('a'), P('b')) self.assertEqual(p.relative_to('a/'), P('b')) - self.assertEqual(p.relative_to(P('a/b')), P()) - self.assertEqual(p.relative_to('a/b'), P()) - self.assertEqual(p.relative_to(P(), walk_up=True), P('a/b')) + self.assertEqual(p.relative_to(P('a/b')), P('')) + self.assertEqual(p.relative_to('a/b'), P('')) + self.assertEqual(p.relative_to(P(''), walk_up=True), P('a/b')) self.assertEqual(p.relative_to('', walk_up=True), P('a/b')) self.assertEqual(p.relative_to(P('a'), walk_up=True), P('b')) self.assertEqual(p.relative_to('a', walk_up=True), P('b')) self.assertEqual(p.relative_to('a/', walk_up=True), P('b')) - self.assertEqual(p.relative_to(P('a/b'), walk_up=True), P()) - self.assertEqual(p.relative_to('a/b', walk_up=True), P()) + self.assertEqual(p.relative_to(P('a/b'), walk_up=True), P('')) + self.assertEqual(p.relative_to('a/b', walk_up=True), P('')) self.assertEqual(p.relative_to(P('a/c'), walk_up=True), P('../b')) self.assertEqual(p.relative_to('a/c', walk_up=True), P('../b')) self.assertEqual(p.relative_to(P('a/b/c'), walk_up=True), P('..')) self.assertEqual(p.relative_to('a/b/c', walk_up=True), P('..')) self.assertEqual(p.relative_to(P('c'), walk_up=True), P('../a/b')) self.assertEqual(p.relative_to('c', walk_up=True), P('../a/b')) - # With several args. - with self.assertWarns(DeprecationWarning): - p.relative_to('a', 'b') - p.relative_to('a', 'b', walk_up=True) # Unrelated paths. self.assertRaises(ValueError, p.relative_to, P('c')) self.assertRaises(ValueError, p.relative_to, P('a/b/c')) @@ -605,15 +1061,15 @@ def test_relative_to_common(self): self.assertEqual(p.relative_to(P('/a')), P('b')) self.assertEqual(p.relative_to('/a'), P('b')) self.assertEqual(p.relative_to('/a/'), P('b')) - self.assertEqual(p.relative_to(P('/a/b')), P()) - self.assertEqual(p.relative_to('/a/b'), P()) + self.assertEqual(p.relative_to(P('/a/b')), P('')) + self.assertEqual(p.relative_to('/a/b'), P('')) self.assertEqual(p.relative_to(P('/'), walk_up=True), P('a/b')) self.assertEqual(p.relative_to('/', walk_up=True), P('a/b')) self.assertEqual(p.relative_to(P('/a'), walk_up=True), P('b')) self.assertEqual(p.relative_to('/a', walk_up=True), P('b')) self.assertEqual(p.relative_to('/a/', walk_up=True), P('b')) - self.assertEqual(p.relative_to(P('/a/b'), walk_up=True), P()) - self.assertEqual(p.relative_to('/a/b', walk_up=True), P()) + self.assertEqual(p.relative_to(P('/a/b'), walk_up=True), P('')) + self.assertEqual(p.relative_to('/a/b', walk_up=True), P('')) self.assertEqual(p.relative_to(P('/a/c'), walk_up=True), P('../b')) self.assertEqual(p.relative_to('/a/c', walk_up=True), P('../b')) self.assertEqual(p.relative_to(P('/a/b/c'), walk_up=True), P('..')) @@ -624,7 +1080,7 @@ def test_relative_to_common(self): self.assertRaises(ValueError, p.relative_to, P('/c')) self.assertRaises(ValueError, p.relative_to, P('/a/b/c')) self.assertRaises(ValueError, p.relative_to, P('/a/c')) - self.assertRaises(ValueError, p.relative_to, P()) + self.assertRaises(ValueError, p.relative_to, P('')) self.assertRaises(ValueError, p.relative_to, '') self.assertRaises(ValueError, p.relative_to, P('a')) self.assertRaises(ValueError, p.relative_to, P("../a")) @@ -636,20 +1092,123 @@ def test_relative_to_common(self): self.assertRaises(ValueError, p.relative_to, P("a/.."), walk_up=True) self.assertRaises(ValueError, p.relative_to, P("/a/.."), walk_up=True) + @needs_windows + def test_relative_to_windows(self): + P = self.cls + p = P('C:Foo/Bar') + self.assertEqual(p.relative_to(P('c:')), P('Foo/Bar')) + self.assertEqual(p.relative_to('c:'), P('Foo/Bar')) + self.assertEqual(p.relative_to(P('c:foO')), P('Bar')) + self.assertEqual(p.relative_to('c:foO'), P('Bar')) + self.assertEqual(p.relative_to('c:foO/'), P('Bar')) + self.assertEqual(p.relative_to(P('c:foO/baR')), P()) + self.assertEqual(p.relative_to('c:foO/baR'), P()) + self.assertEqual(p.relative_to(P('c:'), walk_up=True), P('Foo/Bar')) + self.assertEqual(p.relative_to('c:', walk_up=True), P('Foo/Bar')) + self.assertEqual(p.relative_to(P('c:foO'), walk_up=True), P('Bar')) + self.assertEqual(p.relative_to('c:foO', walk_up=True), P('Bar')) + self.assertEqual(p.relative_to('c:foO/', walk_up=True), P('Bar')) + self.assertEqual(p.relative_to(P('c:foO/baR'), walk_up=True), P()) + self.assertEqual(p.relative_to('c:foO/baR', walk_up=True), P()) + self.assertEqual(p.relative_to(P('C:Foo/Bar/Baz'), walk_up=True), P('..')) + self.assertEqual(p.relative_to(P('C:Foo/Baz'), walk_up=True), P('../Bar')) + self.assertEqual(p.relative_to(P('C:Baz/Bar'), walk_up=True), P('../../Foo/Bar')) + # Unrelated paths. + self.assertRaises(ValueError, p.relative_to, P()) + self.assertRaises(ValueError, p.relative_to, '') + self.assertRaises(ValueError, p.relative_to, P('d:')) + self.assertRaises(ValueError, p.relative_to, P('/')) + self.assertRaises(ValueError, p.relative_to, P('Foo')) + self.assertRaises(ValueError, p.relative_to, P('/Foo')) + self.assertRaises(ValueError, p.relative_to, P('C:/Foo')) + self.assertRaises(ValueError, p.relative_to, P('C:Foo/Bar/Baz')) + self.assertRaises(ValueError, p.relative_to, P('C:Foo/Baz')) + self.assertRaises(ValueError, p.relative_to, P(), walk_up=True) + self.assertRaises(ValueError, p.relative_to, '', walk_up=True) + self.assertRaises(ValueError, p.relative_to, P('d:'), walk_up=True) + self.assertRaises(ValueError, p.relative_to, P('/'), walk_up=True) + self.assertRaises(ValueError, p.relative_to, P('Foo'), walk_up=True) + self.assertRaises(ValueError, p.relative_to, P('/Foo'), walk_up=True) + self.assertRaises(ValueError, p.relative_to, P('C:/Foo'), walk_up=True) + p = P('C:/Foo/Bar') + self.assertEqual(p.relative_to(P('c:/')), P('Foo/Bar')) + self.assertEqual(p.relative_to('c:/'), P('Foo/Bar')) + self.assertEqual(p.relative_to(P('c:/foO')), P('Bar')) + self.assertEqual(p.relative_to('c:/foO'), P('Bar')) + self.assertEqual(p.relative_to('c:/foO/'), P('Bar')) + self.assertEqual(p.relative_to(P('c:/foO/baR')), P()) + self.assertEqual(p.relative_to('c:/foO/baR'), P()) + self.assertEqual(p.relative_to(P('c:/'), walk_up=True), P('Foo/Bar')) + self.assertEqual(p.relative_to('c:/', walk_up=True), P('Foo/Bar')) + self.assertEqual(p.relative_to(P('c:/foO'), walk_up=True), P('Bar')) + self.assertEqual(p.relative_to('c:/foO', walk_up=True), P('Bar')) + self.assertEqual(p.relative_to('c:/foO/', walk_up=True), P('Bar')) + self.assertEqual(p.relative_to(P('c:/foO/baR'), walk_up=True), P()) + self.assertEqual(p.relative_to('c:/foO/baR', walk_up=True), P()) + self.assertEqual(p.relative_to('C:/Baz', walk_up=True), P('../Foo/Bar')) + self.assertEqual(p.relative_to('C:/Foo/Bar/Baz', walk_up=True), P('..')) + self.assertEqual(p.relative_to('C:/Foo/Baz', walk_up=True), P('../Bar')) + # Unrelated paths. + self.assertRaises(ValueError, p.relative_to, 'c:') + self.assertRaises(ValueError, p.relative_to, P('c:')) + self.assertRaises(ValueError, p.relative_to, P('C:/Baz')) + self.assertRaises(ValueError, p.relative_to, P('C:/Foo/Bar/Baz')) + self.assertRaises(ValueError, p.relative_to, P('C:/Foo/Baz')) + self.assertRaises(ValueError, p.relative_to, P('C:Foo')) + self.assertRaises(ValueError, p.relative_to, P('d:')) + self.assertRaises(ValueError, p.relative_to, P('d:/')) + self.assertRaises(ValueError, p.relative_to, P('/')) + self.assertRaises(ValueError, p.relative_to, P('/Foo')) + self.assertRaises(ValueError, p.relative_to, P('//C/Foo')) + self.assertRaises(ValueError, p.relative_to, 'c:', walk_up=True) + self.assertRaises(ValueError, p.relative_to, P('c:'), walk_up=True) + self.assertRaises(ValueError, p.relative_to, P('C:Foo'), walk_up=True) + self.assertRaises(ValueError, p.relative_to, P('d:'), walk_up=True) + self.assertRaises(ValueError, p.relative_to, P('d:/'), walk_up=True) + self.assertRaises(ValueError, p.relative_to, P('/'), walk_up=True) + self.assertRaises(ValueError, p.relative_to, P('/Foo'), walk_up=True) + self.assertRaises(ValueError, p.relative_to, P('//C/Foo'), walk_up=True) + # UNC paths. + p = P('//Server/Share/Foo/Bar') + self.assertEqual(p.relative_to(P('//sErver/sHare')), P('Foo/Bar')) + self.assertEqual(p.relative_to('//sErver/sHare'), P('Foo/Bar')) + self.assertEqual(p.relative_to('//sErver/sHare/'), P('Foo/Bar')) + self.assertEqual(p.relative_to(P('//sErver/sHare/Foo')), P('Bar')) + self.assertEqual(p.relative_to('//sErver/sHare/Foo'), P('Bar')) + self.assertEqual(p.relative_to('//sErver/sHare/Foo/'), P('Bar')) + self.assertEqual(p.relative_to(P('//sErver/sHare/Foo/Bar')), P()) + self.assertEqual(p.relative_to('//sErver/sHare/Foo/Bar'), P()) + self.assertEqual(p.relative_to(P('//sErver/sHare'), walk_up=True), P('Foo/Bar')) + self.assertEqual(p.relative_to('//sErver/sHare', walk_up=True), P('Foo/Bar')) + self.assertEqual(p.relative_to('//sErver/sHare/', walk_up=True), P('Foo/Bar')) + self.assertEqual(p.relative_to(P('//sErver/sHare/Foo'), walk_up=True), P('Bar')) + self.assertEqual(p.relative_to('//sErver/sHare/Foo', walk_up=True), P('Bar')) + self.assertEqual(p.relative_to('//sErver/sHare/Foo/', walk_up=True), P('Bar')) + self.assertEqual(p.relative_to(P('//sErver/sHare/Foo/Bar'), walk_up=True), P()) + self.assertEqual(p.relative_to('//sErver/sHare/Foo/Bar', walk_up=True), P()) + self.assertEqual(p.relative_to(P('//sErver/sHare/bar'), walk_up=True), P('../Foo/Bar')) + self.assertEqual(p.relative_to('//sErver/sHare/bar', walk_up=True), P('../Foo/Bar')) + # Unrelated paths. + self.assertRaises(ValueError, p.relative_to, P('/Server/Share/Foo')) + self.assertRaises(ValueError, p.relative_to, P('c:/Server/Share/Foo')) + self.assertRaises(ValueError, p.relative_to, P('//z/Share/Foo')) + self.assertRaises(ValueError, p.relative_to, P('//Server/z/Foo')) + self.assertRaises(ValueError, p.relative_to, P('/Server/Share/Foo'), walk_up=True) + self.assertRaises(ValueError, p.relative_to, P('c:/Server/Share/Foo'), walk_up=True) + self.assertRaises(ValueError, p.relative_to, P('//z/Share/Foo'), walk_up=True) + self.assertRaises(ValueError, p.relative_to, P('//Server/z/Foo'), walk_up=True) + def test_is_relative_to_common(self): P = self.cls p = P('a/b') self.assertRaises(TypeError, p.is_relative_to) self.assertRaises(TypeError, p.is_relative_to, b'a') - self.assertTrue(p.is_relative_to(P())) + self.assertTrue(p.is_relative_to(P(''))) self.assertTrue(p.is_relative_to('')) self.assertTrue(p.is_relative_to(P('a'))) self.assertTrue(p.is_relative_to('a/')) self.assertTrue(p.is_relative_to(P('a/b'))) self.assertTrue(p.is_relative_to('a/b')) - # With several args. - with self.assertWarns(DeprecationWarning): - p.is_relative_to('a', 'b') # Unrelated paths. self.assertFalse(p.is_relative_to(P('c'))) self.assertFalse(p.is_relative_to(P('a/b/c'))) @@ -667,22 +1226,114 @@ def test_is_relative_to_common(self): self.assertFalse(p.is_relative_to(P('/c'))) self.assertFalse(p.is_relative_to(P('/a/b/c'))) self.assertFalse(p.is_relative_to(P('/a/c'))) - self.assertFalse(p.is_relative_to(P())) + self.assertFalse(p.is_relative_to(P(''))) self.assertFalse(p.is_relative_to('')) self.assertFalse(p.is_relative_to(P('a'))) + @needs_windows + def test_is_relative_to_windows(self): + P = self.cls + p = P('C:Foo/Bar') + self.assertTrue(p.is_relative_to(P('c:'))) + self.assertTrue(p.is_relative_to('c:')) + self.assertTrue(p.is_relative_to(P('c:foO'))) + self.assertTrue(p.is_relative_to('c:foO')) + self.assertTrue(p.is_relative_to('c:foO/')) + self.assertTrue(p.is_relative_to(P('c:foO/baR'))) + self.assertTrue(p.is_relative_to('c:foO/baR')) + # Unrelated paths. + self.assertFalse(p.is_relative_to(P())) + self.assertFalse(p.is_relative_to('')) + self.assertFalse(p.is_relative_to(P('d:'))) + self.assertFalse(p.is_relative_to(P('/'))) + self.assertFalse(p.is_relative_to(P('Foo'))) + self.assertFalse(p.is_relative_to(P('/Foo'))) + self.assertFalse(p.is_relative_to(P('C:/Foo'))) + self.assertFalse(p.is_relative_to(P('C:Foo/Bar/Baz'))) + self.assertFalse(p.is_relative_to(P('C:Foo/Baz'))) + p = P('C:/Foo/Bar') + self.assertTrue(p.is_relative_to(P('c:/'))) + self.assertTrue(p.is_relative_to(P('c:/foO'))) + self.assertTrue(p.is_relative_to('c:/foO/')) + self.assertTrue(p.is_relative_to(P('c:/foO/baR'))) + self.assertTrue(p.is_relative_to('c:/foO/baR')) + # Unrelated paths. + self.assertFalse(p.is_relative_to('c:')) + self.assertFalse(p.is_relative_to(P('C:/Baz'))) + self.assertFalse(p.is_relative_to(P('C:/Foo/Bar/Baz'))) + self.assertFalse(p.is_relative_to(P('C:/Foo/Baz'))) + self.assertFalse(p.is_relative_to(P('C:Foo'))) + self.assertFalse(p.is_relative_to(P('d:'))) + self.assertFalse(p.is_relative_to(P('d:/'))) + self.assertFalse(p.is_relative_to(P('/'))) + self.assertFalse(p.is_relative_to(P('/Foo'))) + self.assertFalse(p.is_relative_to(P('//C/Foo'))) + # UNC paths. + p = P('//Server/Share/Foo/Bar') + self.assertTrue(p.is_relative_to(P('//sErver/sHare'))) + self.assertTrue(p.is_relative_to('//sErver/sHare')) + self.assertTrue(p.is_relative_to('//sErver/sHare/')) + self.assertTrue(p.is_relative_to(P('//sErver/sHare/Foo'))) + self.assertTrue(p.is_relative_to('//sErver/sHare/Foo')) + self.assertTrue(p.is_relative_to('//sErver/sHare/Foo/')) + self.assertTrue(p.is_relative_to(P('//sErver/sHare/Foo/Bar'))) + self.assertTrue(p.is_relative_to('//sErver/sHare/Foo/Bar')) + # Unrelated paths. + self.assertFalse(p.is_relative_to(P('/Server/Share/Foo'))) + self.assertFalse(p.is_relative_to(P('c:/Server/Share/Foo'))) + self.assertFalse(p.is_relative_to(P('//z/Share/Foo'))) + self.assertFalse(p.is_relative_to(P('//Server/z/Foo'))) + + @needs_posix + def test_is_absolute_posix(self): + P = self.cls + self.assertFalse(P('').is_absolute()) + self.assertFalse(P('a').is_absolute()) + self.assertFalse(P('a/b/').is_absolute()) + self.assertTrue(P('/').is_absolute()) + self.assertTrue(P('/a').is_absolute()) + self.assertTrue(P('/a/b/').is_absolute()) + self.assertTrue(P('//a').is_absolute()) + self.assertTrue(P('//a/b').is_absolute()) + + @needs_windows + def test_is_absolute_windows(self): + P = self.cls + # Under NT, only paths with both a drive and a root are absolute. + self.assertFalse(P().is_absolute()) + self.assertFalse(P('a').is_absolute()) + self.assertFalse(P('a/b/').is_absolute()) + self.assertFalse(P('/').is_absolute()) + self.assertFalse(P('/a').is_absolute()) + self.assertFalse(P('/a/b/').is_absolute()) + self.assertFalse(P('c:').is_absolute()) + self.assertFalse(P('c:a').is_absolute()) + self.assertFalse(P('c:a/b/').is_absolute()) + self.assertTrue(P('c:/').is_absolute()) + self.assertTrue(P('c:/a').is_absolute()) + self.assertTrue(P('c:/a/b/').is_absolute()) + # UNC paths are absolute by definition. + self.assertTrue(P('//').is_absolute()) + self.assertTrue(P('//a').is_absolute()) + self.assertTrue(P('//a/b').is_absolute()) + self.assertTrue(P('//a/b/').is_absolute()) + self.assertTrue(P('//a/b/c').is_absolute()) + self.assertTrue(P('//a/b/c/d').is_absolute()) + self.assertTrue(P('//?/UNC/').is_absolute()) + self.assertTrue(P('//?/UNC/spam').is_absolute()) + # # Tests for the virtual classes. # class PathBaseTest(PurePathBaseTest): - cls = pathlib._abc.PathBase + cls = PathBase def test_unsupported_operation(self): P = self.cls - p = self.cls() - e = pathlib.UnsupportedOperation + p = self.cls('') + e = UnsupportedOperation self.assertRaises(e, p.stat) self.assertRaises(e, p.lstat) self.assertRaises(e, p.exists) @@ -724,23 +1375,14 @@ def test_unsupported_operation(self): self.assertRaises(e, p.as_uri) def test_as_uri_common(self): - e = pathlib.UnsupportedOperation - self.assertRaises(e, self.cls().as_uri) + e = UnsupportedOperation + self.assertRaises(e, self.cls('').as_uri) def test_fspath_common(self): - self.assertRaises(TypeError, os.fspath, self.cls()) + self.assertRaises(TypeError, os.fspath, self.cls('')) def test_as_bytes_common(self): - self.assertRaises(TypeError, bytes, self.cls()) - - def test_matches_path_api(self): - our_names = {name for name in dir(self.cls) if name[0] != '_'} - path_names = {name for name in dir(pathlib.Path) if name[0] != '_'} - self.assertEqual(our_names, path_names) - for attr_name in our_names: - our_attr = getattr(self.cls, attr_name) - path_attr = getattr(pathlib.Path, attr_name) - self.assertEqual(our_attr.__doc__, path_attr.__doc__) + self.assertRaises(TypeError, bytes, self.cls('')) class DummyPathIO(io.BytesIO): @@ -758,11 +1400,19 @@ def close(self): super().close() -class DummyPath(pathlib._abc.PathBase): +DummyPathStatResult = collections.namedtuple( + 'DummyPathStatResult', + 'st_mode st_ino st_dev st_nlink st_uid st_gid st_size st_atime st_mtime st_ctime') + + +class DummyPath(PathBase): """ Simple implementation of PathBase that keeps files and directories in memory. """ + __slots__ = () + pathmod = posixpath + _files = {} _directories = {} _symlinks = {} @@ -775,6 +1425,9 @@ def __eq__(self, other): def __hash__(self): return hash(str(self)) + def __repr__(self): + return "{}({!r})".format(self.__class__.__name__, self.as_posix()) + def stat(self, *, follow_symlinks=True): if follow_symlinks: path = str(self.resolve()) @@ -788,7 +1441,7 @@ def stat(self, *, follow_symlinks=True): st_mode = stat.S_IFLNK else: raise FileNotFoundError(errno.ENOENT, "Not found", str(self)) - return os.stat_result((st_mode, hash(str(self)), 0, 0, 0, 0, 0, 0, 0, 0)) + return DummyPathStatResult(st_mode, hash(str(self)), 0, 0, 0, 0, 0, 0, 0, 0) def open(self, mode='r', buffering=-1, encoding=None, errors=None, newline=None): @@ -874,6 +1527,9 @@ class DummyPathTest(DummyPurePathTest): def setUp(self): super().setUp() + name = self.id().split('.')[-1] + if name in _tests_needing_symlinks and not self.can_symlink: + self.skipTest('requires symlinks') pathmod = self.cls.pathmod p = self.cls(self.base) p.mkdir(parents=True) @@ -917,11 +1573,13 @@ def assertFileNotFound(self, func, *args, **kwargs): self.assertEqual(cm.exception.errno, errno.ENOENT) def assertEqualNormCase(self, path_a, path_b): - self.assertEqual(os.path.normcase(path_a), os.path.normcase(path_b)) + normcase = self.pathmod.normcase + self.assertEqual(normcase(path_a), normcase(path_b)) def test_samefile(self): - fileA_path = os.path.join(self.base, 'fileA') - fileB_path = os.path.join(self.base, 'dirB', 'fileB') + pathmod = self.pathmod + fileA_path = pathmod.join(self.base, 'fileA') + fileB_path = pathmod.join(self.base, 'dirB', 'fileB') p = self.cls(fileA_path) pp = self.cls(fileA_path) q = self.cls(fileB_path) @@ -930,7 +1588,7 @@ def test_samefile(self): self.assertFalse(p.samefile(fileB_path)) self.assertFalse(p.samefile(q)) # Test the non-existent file case - non_existent = os.path.join(self.base, 'foo') + non_existent = pathmod.join(self.base, 'foo') r = self.cls(non_existent) self.assertRaises(FileNotFoundError, p.samefile, r) self.assertRaises(FileNotFoundError, p.samefile, non_existent) @@ -939,11 +1597,6 @@ def test_samefile(self): self.assertRaises(FileNotFoundError, r.samefile, r) self.assertRaises(FileNotFoundError, r.samefile, non_existent) - def test_empty_path(self): - # The empty path points to '.' - p = self.cls('') - self.assertEqual(str(p), '.') - def test_exists(self): P = self.cls p = P(self.base) @@ -1034,9 +1687,8 @@ def test_iterdir(self): expected += ['linkA', 'linkB', 'brokenLink', 'brokenLinkLoop'] self.assertEqual(paths, { P(self.base, q) for q in expected }) + @needs_symlinks def test_iterdir_symlink(self): - if not self.can_symlink: - self.skipTest("symlinks required") # __iter__ on a symlink to a directory. P = self.cls p = P(self.base, 'linkB') @@ -1085,10 +1737,31 @@ def _check(glob, expected): else: _check(p.glob("*/"), ["dirA/", "dirB/", "dirC/", "dirE/", "linkB/"]) + @needs_posix + def test_glob_posix(self): + P = self.cls + p = P(self.base) + given = set(p.glob("FILEa")) + expect = set() + self.assertEqual(given, expect) + self.assertEqual(set(p.glob("FILEa*")), set()) + + @needs_windows + def test_glob_windows(self): + P = self.cls + p = P(self.base) + self.assertEqual(set(p.glob("FILEa")), { P(self.base, "fileA") }) + self.assertEqual(set(p.glob("*a\\")), { P(self.base, "dirA/") }) + self.assertEqual(set(p.glob("F*a")), { P(self.base, "fileA") }) + self.assertEqual(set(map(str, p.glob("FILEa"))), {f"{p}\\fileA"}) + self.assertEqual(set(map(str, p.glob("F*a"))), {f"{p}\\fileA"}) + def test_glob_empty_pattern(self): - p = self.cls() - with self.assertRaisesRegex(ValueError, 'Unacceptable pattern'): - list(p.glob('')) + P = self.cls + p = P(self.base) + self.assertEqual(list(p.glob("")), [p]) + self.assertEqual(list(p.glob(".")), [p / "."]) + self.assertEqual(list(p.glob("./")), [p / "./"]) def test_glob_case_sensitive(self): P = self.cls @@ -1102,12 +1775,11 @@ def _check(path, pattern, case_sensitive, expected): _check(path, "dirb/file*", True, []) _check(path, "dirb/file*", False, ["dirB/fileB"]) + @needs_symlinks def test_glob_follow_symlinks_common(self): - if not self.can_symlink: - self.skipTest("symlinks required") def _check(path, glob, expected): actual = {path for path in path.glob(glob, follow_symlinks=True) - if "linkD" not in path.parent.parts} # exclude symlink loop. + if path.parts.count("linkD") <= 1} # exclude symlink loop. self.assertEqual(actual, { P(self.base, q) for q in expected }) P = self.cls p = P(self.base) @@ -1117,20 +1789,31 @@ def _check(path, glob, expected): _check(p, "*B/*", ["dirB/fileB", "dirB/linkD", "linkB/fileB", "linkB/linkD"]) _check(p, "*/fileB", ["dirB/fileB", "linkB/fileB"]) _check(p, "*/", ["dirA/", "dirB/", "dirC/", "dirE/", "linkB/"]) - _check(p, "dir*/*/..", ["dirC/dirD/..", "dirA/linkC/.."]) + _check(p, "dir*/*/..", ["dirC/dirD/..", "dirA/linkC/..", "dirB/linkD/.."]) + _check(p, "dir*/**", [ + "dirA/", "dirA/linkC", "dirA/linkC/fileB", "dirA/linkC/linkD", "dirA/linkC/linkD/fileB", + "dirB/", "dirB/fileB", "dirB/linkD", "dirB/linkD/fileB", + "dirC/", "dirC/fileC", "dirC/dirD", "dirC/dirD/fileD", "dirC/novel.txt", + "dirE/"]) _check(p, "dir*/**/", ["dirA/", "dirA/linkC/", "dirA/linkC/linkD/", "dirB/", "dirB/linkD/", "dirC/", "dirC/dirD/", "dirE/"]) _check(p, "dir*/**/..", ["dirA/..", "dirA/linkC/..", "dirB/..", + "dirB/linkD/..", "dirA/linkC/linkD/..", "dirC/..", "dirC/dirD/..", "dirE/.."]) + _check(p, "dir*/*/**", [ + "dirA/linkC/", "dirA/linkC/linkD", "dirA/linkC/fileB", "dirA/linkC/linkD/fileB", + "dirB/linkD/", "dirB/linkD/fileB", + "dirC/dirD/", "dirC/dirD/fileD"]) _check(p, "dir*/*/**/", ["dirA/linkC/", "dirA/linkC/linkD/", "dirB/linkD/", "dirC/dirD/"]) - _check(p, "dir*/*/**/..", ["dirA/linkC/..", "dirC/dirD/.."]) + _check(p, "dir*/*/**/..", ["dirA/linkC/..", "dirA/linkC/linkD/..", + "dirB/linkD/..", "dirC/dirD/.."]) _check(p, "dir*/**/fileC", ["dirC/fileC"]) _check(p, "dir*/*/../dirD/**/", ["dirC/dirD/../dirD/"]) + _check(p, "*/dirD/**", ["dirC/dirD/", "dirC/dirD/fileD"]) _check(p, "*/dirD/**/", ["dirC/dirD/"]) + @needs_symlinks def test_glob_no_follow_symlinks_common(self): - if not self.can_symlink: - self.skipTest("symlinks required") def _check(path, glob, expected): actual = {path for path in path.glob(glob, follow_symlinks=False)} self.assertEqual(actual, { P(self.base, q) for q in expected }) @@ -1143,12 +1826,20 @@ def _check(path, glob, expected): _check(p, "*/fileB", ["dirB/fileB"]) _check(p, "*/", ["dirA/", "dirB/", "dirC/", "dirE/"]) _check(p, "dir*/*/..", ["dirC/dirD/.."]) + _check(p, "dir*/**", [ + "dirA/", "dirA/linkC", + "dirB/", "dirB/fileB", "dirB/linkD", + "dirC/", "dirC/fileC", "dirC/dirD", "dirC/dirD/fileD", "dirC/novel.txt", + "dirE/"]) _check(p, "dir*/**/", ["dirA/", "dirB/", "dirC/", "dirC/dirD/", "dirE/"]) _check(p, "dir*/**/..", ["dirA/..", "dirB/..", "dirC/..", "dirC/dirD/..", "dirE/.."]) + _check(p, "dir*/*/**", ["dirC/dirD/", "dirC/dirD/fileD"]) _check(p, "dir*/*/**/", ["dirC/dirD/"]) _check(p, "dir*/*/**/..", ["dirC/dirD/.."]) _check(p, "dir*/**/fileC", ["dirC/fileC"]) + _check(p, "dir*/*/../dirD/**", ["dirC/dirD/../dirD/", "dirC/dirD/../dirD/fileD"]) _check(p, "dir*/*/../dirD/**/", ["dirC/dirD/../dirD/"]) + _check(p, "*/dirD/**", ["dirC/dirD/", "dirC/dirD/fileD"]) _check(p, "*/dirD/**/", ["dirC/dirD/"]) def test_rglob_common(self): @@ -1178,39 +1869,61 @@ def _check(glob, expected): "dirA/", "dirA/linkC/", "dirB/", "dirB/linkD/", "dirC/", "dirC/dirD/", "dirE/", "linkB/", ]) - _check(p.rglob(""), ["./", "dirA/", "dirB/", "dirC/", "dirE/", "dirC/dirD/"]) + _check(p.rglob(""), ["", "dirA/", "dirB/", "dirC/", "dirE/", "dirC/dirD/"]) p = P(self.base, "dirC") _check(p.rglob("*"), ["dirC/fileC", "dirC/novel.txt", "dirC/dirD", "dirC/dirD/fileD"]) _check(p.rglob("file*"), ["dirC/fileC", "dirC/dirD/fileD"]) _check(p.rglob("**/file*"), ["dirC/fileC", "dirC/dirD/fileD"]) + _check(p.rglob("dir*/**"), ["dirC/dirD/", "dirC/dirD/fileD"]) _check(p.rglob("dir*/**/"), ["dirC/dirD/"]) _check(p.rglob("*/*"), ["dirC/dirD/fileD"]) _check(p.rglob("*/"), ["dirC/dirD/"]) _check(p.rglob(""), ["dirC/", "dirC/dirD/"]) + _check(p.rglob("**"), [ + "dirC/", "dirC/fileC", "dirC/dirD", "dirC/dirD/fileD", "dirC/novel.txt"]) _check(p.rglob("**/"), ["dirC/", "dirC/dirD/"]) # gh-91616, a re module regression _check(p.rglob("*.txt"), ["dirC/novel.txt"]) _check(p.rglob("*.*"), ["dirC/novel.txt"]) + @needs_posix + def test_rglob_posix(self): + P = self.cls + p = P(self.base, "dirC") + given = set(p.rglob("FILEd")) + expect = set() + self.assertEqual(given, expect) + self.assertEqual(set(p.rglob("FILEd*")), set()) + + @needs_windows + def test_rglob_windows(self): + P = self.cls + p = P(self.base, "dirC") + self.assertEqual(set(p.rglob("FILEd")), { P(self.base, "dirC/dirD/fileD") }) + self.assertEqual(set(p.rglob("*\\")), { P(self.base, "dirC/dirD/") }) + self.assertEqual(set(map(str, p.rglob("FILEd"))), {f"{p}\\dirD\\fileD"}) + + @needs_symlinks def test_rglob_follow_symlinks_common(self): - if not self.can_symlink: - self.skipTest("symlinks required") def _check(path, glob, expected): actual = {path for path in path.rglob(glob, follow_symlinks=True) - if 'linkD' not in path.parent.parts} # exclude symlink loop. + if path.parts.count("linkD") <= 1} # exclude symlink loop. self.assertEqual(actual, { P(self.base, q) for q in expected }) P = self.cls p = P(self.base) - _check(p, "fileB", ["dirB/fileB", "dirA/linkC/fileB", "linkB/fileB"]) + _check(p, "fileB", ["dirB/fileB", "dirA/linkC/fileB", "linkB/fileB", + "dirA/linkC/linkD/fileB", "dirB/linkD/fileB", "linkB/linkD/fileB"]) _check(p, "*/fileA", []) - _check(p, "*/fileB", ["dirB/fileB", "dirA/linkC/fileB", "linkB/fileB"]) + _check(p, "*/fileB", ["dirB/fileB", "dirA/linkC/fileB", "linkB/fileB", + "dirA/linkC/linkD/fileB", "dirB/linkD/fileB", "linkB/linkD/fileB"]) _check(p, "file*", ["fileA", "dirA/linkC/fileB", "dirB/fileB", + "dirA/linkC/linkD/fileB", "dirB/linkD/fileB", "linkB/linkD/fileB", "dirC/fileC", "dirC/dirD/fileD", "linkB/fileB"]) _check(p, "*/", ["dirA/", "dirA/linkC/", "dirA/linkC/linkD/", "dirB/", "dirB/linkD/", "dirC/", "dirC/dirD/", "dirE/", "linkB/", "linkB/linkD/"]) - _check(p, "", ["./", "dirA/", "dirA/linkC/", "dirA/linkC/linkD/", "dirB/", "dirB/linkD/", + _check(p, "", ["", "dirA/", "dirA/linkC/", "dirA/linkC/linkD/", "dirB/", "dirB/linkD/", "dirC/", "dirE/", "dirC/dirD/", "linkB/", "linkB/linkD/"]) p = P(self.base, "dirC") @@ -1224,9 +1937,8 @@ def _check(path, glob, expected): _check(p, "*.txt", ["dirC/novel.txt"]) _check(p, "*.*", ["dirC/novel.txt"]) + @needs_symlinks def test_rglob_no_follow_symlinks_common(self): - if not self.can_symlink: - self.skipTest("symlinks required") def _check(path, glob, expected): actual = {path for path in path.rglob(glob, follow_symlinks=False)} self.assertEqual(actual, { P(self.base, q) for q in expected }) @@ -1237,7 +1949,7 @@ def _check(path, glob, expected): _check(p, "*/fileB", ["dirB/fileB"]) _check(p, "file*", ["fileA", "dirB/fileB", "dirC/fileC", "dirC/dirD/fileD", ]) _check(p, "*/", ["dirA/", "dirB/", "dirC/", "dirC/dirD/", "dirE/"]) - _check(p, "", ["./", "dirA/", "dirB/", "dirC/", "dirE/", "dirC/dirD/"]) + _check(p, "", ["", "dirA/", "dirB/", "dirC/", "dirE/", "dirC/dirD/"]) p = P(self.base, "dirC") _check(p, "*", ["dirC/fileC", "dirC/novel.txt", @@ -1250,10 +1962,9 @@ def _check(path, glob, expected): _check(p, "*.txt", ["dirC/novel.txt"]) _check(p, "*.*", ["dirC/novel.txt"]) + @needs_symlinks def test_rglob_symlink_loop(self): # Don't get fooled by symlink loops (Issue #26012). - if not self.can_symlink: - self.skipTest("symlinks required") P = self.cls p = P(self.base) given = set(p.rglob('*')) @@ -1270,25 +1981,6 @@ def test_rglob_symlink_loop(self): } self.assertEqual(given, {p / x for x in expect}) - def test_glob_many_open_files(self): - depth = 30 - P = self.cls - p = base = P(self.base) / 'deep' - p.mkdir() - for _ in range(depth): - p /= 'd' - p.mkdir() - pattern = '/'.join(['*'] * depth) - iters = [base.glob(pattern) for j in range(100)] - for it in iters: - self.assertEqual(next(it), p) - iters = [base.rglob('d') for j in range(100)] - p = base - for i in range(depth): - p = p / 'd' - for it in iters: - self.assertEqual(next(it), p) - def test_glob_dotdot(self): # ".." is not special in globs. P = self.cls @@ -1302,10 +1994,9 @@ def test_glob_dotdot(self): self.assertEqual(set(p.glob("xyzzy/..")), set()) self.assertEqual(set(p.glob("/".join([".."] * 50))), { P(self.base, *[".."] * 50)}) + @needs_symlinks def test_glob_permissions(self): # See bpo-38894 - if not self.can_symlink: - self.skipTest("symlinks required") P = self.cls base = P(self.base) / 'permissions' base.mkdir() @@ -1322,43 +2013,17 @@ def test_glob_permissions(self): self.assertEqual(len(set(base.glob("*/fileC"))), 50) self.assertEqual(len(set(base.glob("*/file*"))), 50) + @needs_symlinks def test_glob_long_symlink(self): # See gh-87695 - if not self.can_symlink: - self.skipTest("symlinks required") base = self.cls(self.base) / 'long_symlink' base.mkdir() bad_link = base / 'bad_link' bad_link.symlink_to("bad" * 200) self.assertEqual(sorted(base.glob('**/*')), [bad_link]) - def test_glob_above_recursion_limit(self): - recursion_limit = 50 - # directory_depth > recursion_limit - directory_depth = recursion_limit + 10 - base = self.cls(self.base, 'deep') - path = base.joinpath(*(['d'] * directory_depth)) - path.mkdir(parents=True) - - with set_recursion_limit(recursion_limit): - list(base.glob('**/')) - - def test_glob_recursive_no_trailing_slash(self): - P = self.cls - p = P(self.base) - with self.assertWarns(FutureWarning): - p.glob('**') - with self.assertWarns(FutureWarning): - p.glob('*/**') - with self.assertWarns(FutureWarning): - p.rglob('**') - with self.assertWarns(FutureWarning): - p.rglob('*/**') - - + @needs_symlinks def test_readlink(self): - if not self.can_symlink: - self.skipTest("symlinks required") P = self.cls(self.base) self.assertEqual((P / 'linkA').readlink(), self.cls('fileA')) self.assertEqual((P / 'brokenLink').readlink(), @@ -1372,7 +2037,7 @@ def test_readlink(self): def test_readlink_unsupported(self): P = self.cls(self.base) p = P / 'fileA' - with self.assertRaises(pathlib.UnsupportedOperation): + with self.assertRaises(UnsupportedOperation): q.readlink(p) def _check_resolve(self, p, expected, strict=True): @@ -1382,23 +2047,23 @@ def _check_resolve(self, p, expected, strict=True): # This can be used to check both relative and absolute resolutions. _check_resolve_relative = _check_resolve_absolute = _check_resolve + @needs_symlinks def test_resolve_common(self): - if not self.can_symlink: - self.skipTest("symlinks required") P = self.cls p = P(self.base, 'foo') with self.assertRaises(OSError) as cm: p.resolve(strict=True) self.assertEqual(cm.exception.errno, errno.ENOENT) # Non-strict + pathmod = self.pathmod self.assertEqualNormCase(str(p.resolve(strict=False)), - os.path.join(self.base, 'foo')) + pathmod.join(self.base, 'foo')) p = P(self.base, 'foo', 'in', 'spam') self.assertEqualNormCase(str(p.resolve(strict=False)), - os.path.join(self.base, 'foo', 'in', 'spam')) + pathmod.join(self.base, 'foo', 'in', 'spam')) p = P(self.base, '..', 'foo', 'in', 'spam') self.assertEqualNormCase(str(p.resolve(strict=False)), - os.path.abspath(os.path.join('foo', 'in', 'spam'))) + pathmod.join(pathmod.dirname(self.base), 'foo', 'in', 'spam')) # These are all relative symlinks. p = P(self.base, 'dirB', 'fileB') self._check_resolve_relative(p, p) @@ -1413,7 +2078,7 @@ def test_resolve_common(self): self._check_resolve_relative(p, P(self.base, 'dirB', 'fileB', 'foo', 'in', 'spam'), False) p = P(self.base, 'dirA', 'linkC', '..', 'foo', 'in', 'spam') - if os.name == 'nt' and isinstance(p, pathlib.Path): + if self.cls.pathmod is not posixpath: # In Windows, if linkY points to dirB, 'dirA\linkY\..' # resolves to 'dirA' without resolving linkY first. self._check_resolve_relative(p, P(self.base, 'dirA', 'foo', 'in', @@ -1433,7 +2098,7 @@ def test_resolve_common(self): self._check_resolve_relative(p, P(self.base, 'dirB', 'foo', 'in', 'spam'), False) p = P(self.base, 'dirA', 'linkX', 'linkY', '..', 'foo', 'in', 'spam') - if os.name == 'nt' and isinstance(p, pathlib.Path): + if self.cls.pathmod is not posixpath: # In Windows, if linkY points to dirB, 'dirA\linkY\..' # resolves to 'dirA' without resolving linkY first. self._check_resolve_relative(p, P(d, 'foo', 'in', 'spam'), False) @@ -1442,14 +2107,14 @@ def test_resolve_common(self): # resolves to 'dirB/..' first before resolving to parent of dirB. self._check_resolve_relative(p, P(self.base, 'foo', 'in', 'spam'), False) + @needs_symlinks def test_resolve_dot(self): # See http://web.archive.org/web/20200623062557/https://bitbucket.org/pitrou/pathlib/issues/9/ - if not self.can_symlink: - self.skipTest("symlinks required") + pathmod = self.pathmod p = self.cls(self.base) p.joinpath('0').symlink_to('.', target_is_directory=True) - p.joinpath('1').symlink_to(os.path.join('0', '0'), target_is_directory=True) - p.joinpath('2').symlink_to(os.path.join('1', '1'), target_is_directory=True) + p.joinpath('1').symlink_to(pathmod.join('0', '0'), target_is_directory=True) + p.joinpath('2').symlink_to(pathmod.join('1', '1'), target_is_directory=True) q = p / '2' self.assertEqual(q.resolve(strict=True), p) r = q / '3' / '4' @@ -1463,11 +2128,9 @@ def _check_symlink_loop(self, *args): path.resolve(strict=True) self.assertEqual(cm.exception.errno, errno.ELOOP) + @needs_posix + @needs_symlinks def test_resolve_loop(self): - if not self.can_symlink: - self.skipTest("symlinks required") - if os.name == 'nt' and issubclass(self.cls, pathlib.Path): - self.skipTest("symlink loops work differently with concrete Windows paths") # Loops with relative symlinks. self.cls(self.base, 'linkX').symlink_to('linkX/inside') self._check_symlink_loop(self.base, 'linkX') @@ -1509,9 +2172,8 @@ def test_stat(self): self.assertEqual(statA.st_dev, statC.st_dev) # other attributes not used by pathlib. + @needs_symlinks def test_stat_no_follow_symlinks(self): - if not self.can_symlink: - self.skipTest("symlinks required") p = self.cls(self.base) / 'linkA' st = p.stat() self.assertNotEqual(st, p.stat(follow_symlinks=False)) @@ -1521,9 +2183,8 @@ def test_stat_no_follow_symlinks_nosymlink(self): st = p.stat() self.assertEqual(st, p.stat(follow_symlinks=False)) + @needs_symlinks def test_lstat(self): - if not self.can_symlink: - self.skipTest("symlinks required") p = self.cls(self.base)/ 'linkA' st = p.stat() self.assertNotEqual(st, p.lstat()) @@ -1655,31 +2316,13 @@ def test_is_char_device_false(self): self.assertIs((P / 'fileA\udfff').is_char_device(), False) self.assertIs((P / 'fileA\x00').is_char_device(), False) - def test_pickling_common(self): - p = self.cls(self.base, 'fileA') - for proto in range(0, pickle.HIGHEST_PROTOCOL + 1): - dumped = pickle.dumps(p, proto) - pp = pickle.loads(dumped) - self.assertEqual(pp.stat(), p.stat()) - - def test_parts_interning(self): - P = self.cls - p = P('/usr/bin/foo') - q = P('/usr/local/bin') - # 'usr' - self.assertIs(p.parts[1], q.parts[1]) - # 'bin' - self.assertIs(p.parts[2], q.parts[3]) - def _check_complex_symlinks(self, link0_target): - if not self.can_symlink: - self.skipTest("symlinks required") - # Test solving a non-looping chain of symlinks (issue #19887). + pathmod = self.pathmod P = self.cls(self.base) - P.joinpath('link1').symlink_to(os.path.join('link0', 'link0'), target_is_directory=True) - P.joinpath('link2').symlink_to(os.path.join('link1', 'link1'), target_is_directory=True) - P.joinpath('link3').symlink_to(os.path.join('link2', 'link2'), target_is_directory=True) + P.joinpath('link1').symlink_to(pathmod.join('link0', 'link0'), target_is_directory=True) + P.joinpath('link2').symlink_to(pathmod.join('link1', 'link1'), target_is_directory=True) + P.joinpath('link3').symlink_to(pathmod.join('link2', 'link2'), target_is_directory=True) P.joinpath('link0').symlink_to(link0_target, target_is_directory=True) # Resolve absolute paths. @@ -1698,8 +2341,8 @@ def _check_complex_symlinks(self, link0_target): # Resolve relative paths. try: - self.cls().absolute() - except pathlib.UnsupportedOperation: + self.cls('').absolute() + except UnsupportedOperation: return old_path = os.getcwd() os.chdir(self.base) @@ -1719,14 +2362,17 @@ def _check_complex_symlinks(self, link0_target): finally: os.chdir(old_path) + @needs_symlinks def test_complex_symlinks_absolute(self): self._check_complex_symlinks(self.base) + @needs_symlinks def test_complex_symlinks_relative(self): self._check_complex_symlinks('.') + @needs_symlinks def test_complex_symlinks_relative_dot_dot(self): - self._check_complex_symlinks(os.path.join('dirA', '..')) + self._check_complex_symlinks(self.pathmod.join('dirA', '..')) def setUpWalk(self): # Build: @@ -1840,9 +2486,8 @@ def test_walk_bottom_up(self): raise AssertionError(f"Unexpected path: {path}") self.assertTrue(seen_testfn) + @needs_symlinks def test_walk_follow_symlinks(self): - if not self.can_symlink: - self.skipTest("symlinks required") self.setUpWalk() walk_it = self.walk_path.walk(follow_symlinks=True) for root, dirs, files in walk_it: @@ -1853,9 +2498,8 @@ def test_walk_follow_symlinks(self): else: self.fail("Didn't follow symlink with follow_symlinks=True") + @needs_symlinks def test_walk_symlink_location(self): - if not self.can_symlink: - self.skipTest("symlinks required") self.setUpWalk() # Tests whether symlinks end up in filenames or dirnames depending # on the `follow_symlinks` argument. @@ -1875,19 +2519,13 @@ def test_walk_symlink_location(self): else: self.fail("symlink not found") - def test_walk_above_recursion_limit(self): - recursion_limit = 40 - # directory_depth > recursion_limit - directory_depth = recursion_limit + 10 - base = self.cls(self.base, 'deep') - path = base.joinpath(*(['d'] * directory_depth)) - path.mkdir(parents=True) - - with set_recursion_limit(recursion_limit): - list(base.walk()) - list(base.walk(top_down=False)) class DummyPathWithSymlinks(DummyPath): + __slots__ = () + + # Reduce symlink traversal limit to make tests run faster. + _max_symlinks = 20 + def readlink(self): path = str(self.parent.resolve() / self.name) if path in self._symlinks: diff --git a/Lib/test/test_pdb.py b/Lib/test/test_pdb.py index d53fe3c611bc35..2b0795cdad707e 100644 --- a/Lib/test/test_pdb.py +++ b/Lib/test/test_pdb.py @@ -16,11 +16,12 @@ from test import support from test.support import os_helper from test.support.import_helper import import_module -from test.support.pty_helper import run_pty -# This little helper class is essential for testing pdb under doctest. -from test.test_doctest import _FakeInput +from test.support.pty_helper import run_pty, FakeInput from unittest.mock import patch +# gh-114275: WASI fails to run asyncio tests, similar skip than test_asyncio. +SKIP_ASYNCIO_TESTS = (not support.has_socket_support) + class PdbTestInput(object): """Context manager that makes testing Pdb in doctests easier.""" @@ -30,7 +31,7 @@ def __init__(self, input): def __enter__(self): self.real_stdin = sys.stdin - sys.stdin = _FakeInput(self.input) + sys.stdin = FakeInput(self.input) self.orig_trace = sys.gettrace() if hasattr(sys, 'gettrace') else None def __exit__(self, *exc): @@ -778,58 +779,62 @@ def test_pdb_where_command(): (Pdb) continue """ -def test_pdb_interact_command(): - """Test interact command - - >>> g = 0 - >>> dict_g = {} - - >>> def test_function(): - ... x = 1 - ... lst_local = [] - ... import pdb; pdb.Pdb(nosigint=True, readrc=False).set_trace() - >>> with PdbTestInput([ # doctest: +ELLIPSIS, +NORMALIZE_WHITESPACE - ... 'interact', - ... 'x', - ... 'g', - ... 'x = 2', - ... 'g = 3', - ... 'dict_g["a"] = True', - ... 'lst_local.append(x)', - ... 'exit()', - ... 'p x', - ... 'p g', - ... 'p dict_g', - ... 'p lst_local', - ... 'continue', - ... ]): - ... test_function() - --Return-- - > (4)test_function()->None - -> import pdb; pdb.Pdb(nosigint=True, readrc=False).set_trace() - (Pdb) interact - *pdb interact start* - ... x - 1 - ... g - 0 - ... x = 2 - ... g = 3 - ... dict_g["a"] = True - ... lst_local.append(x) - ... exit() - *exit from pdb interact command* - (Pdb) p x - 1 - (Pdb) p g - 0 - (Pdb) p dict_g - {'a': True} - (Pdb) p lst_local - [2] - (Pdb) continue - """ +# skip this test if sys.flags.no_site = True; +# exit() isn't defined unless there's a site module. +if not sys.flags.no_site: + def test_pdb_interact_command(): + """Test interact command + + >>> g = 0 + >>> dict_g = {} + + >>> def test_function(): + ... x = 1 + ... lst_local = [] + ... import pdb; pdb.Pdb(nosigint=True, readrc=False).set_trace() + + >>> with PdbTestInput([ # doctest: +ELLIPSIS, +NORMALIZE_WHITESPACE + ... 'interact', + ... 'x', + ... 'g', + ... 'x = 2', + ... 'g = 3', + ... 'dict_g["a"] = True', + ... 'lst_local.append(x)', + ... 'exit()', + ... 'p x', + ... 'p g', + ... 'p dict_g', + ... 'p lst_local', + ... 'continue', + ... ]): + ... test_function() + --Return-- + > (4)test_function()->None + -> import pdb; pdb.Pdb(nosigint=True, readrc=False).set_trace() + (Pdb) interact + *pdb interact start* + ... x + 1 + ... g + 0 + ... x = 2 + ... g = 3 + ... dict_g["a"] = True + ... lst_local.append(x) + ... exit() + *exit from pdb interact command* + (Pdb) p x + 1 + (Pdb) p g + 0 + (Pdb) p dict_g + {'a': True} + (Pdb) p lst_local + [2] + (Pdb) continue + """ def test_convenience_variables(): """Test convenience variables @@ -847,9 +852,12 @@ def test_convenience_variables(): >>> with PdbTestInput([ # doctest: +ELLIPSIS, +NORMALIZE_WHITESPACE ... '$_frame.f_lineno', # Check frame convenience variable + ... '$ _frame', # This should be a syntax error ... '$a = 10', # Set a convenience variable ... '$a', # Print its value + ... 'p "$a"', # Print the string $a ... 'p $a + 2', # Do some calculation + ... 'p f"$a = {$a}"', # Make sure $ in string is not converted and f-string works ... 'u', # Switch frame ... '$_frame.f_lineno', # Make sure the frame changed ... '$a', # Make sure the value persists @@ -869,11 +877,17 @@ def test_convenience_variables(): -> try: (Pdb) $_frame.f_lineno 3 + (Pdb) $ _frame + *** SyntaxError: invalid syntax (Pdb) $a = 10 (Pdb) $a 10 + (Pdb) p "$a" + '$a' (Pdb) p $a + 2 12 + (Pdb) p f"$a = {$a}" + '$a = 10' (Pdb) u > (2)test_function() -> util_function() @@ -1686,122 +1700,123 @@ def test_pdb_next_command_for_generator(): finished """ -def test_pdb_next_command_for_coroutine(): - """Testing skip unwindng stack on yield for coroutines for "next" command - - >>> import asyncio - - >>> async def test_coro(): - ... await asyncio.sleep(0) - ... await asyncio.sleep(0) - ... await asyncio.sleep(0) - - >>> async def test_main(): - ... import pdb; pdb.Pdb(nosigint=True, readrc=False).set_trace() - ... await test_coro() - - >>> def test_function(): - ... loop = asyncio.new_event_loop() - ... loop.run_until_complete(test_main()) - ... loop.close() - ... asyncio.set_event_loop_policy(None) - ... print("finished") - - >>> with PdbTestInput(['step', - ... 'step', - ... 'next', - ... 'next', - ... 'next', - ... 'step', - ... 'continue']): - ... test_function() - > (3)test_main() - -> await test_coro() - (Pdb) step - --Call-- - > (1)test_coro() - -> async def test_coro(): - (Pdb) step - > (2)test_coro() - -> await asyncio.sleep(0) - (Pdb) next - > (3)test_coro() - -> await asyncio.sleep(0) - (Pdb) next - > (4)test_coro() - -> await asyncio.sleep(0) - (Pdb) next - Internal StopIteration - > (3)test_main() - -> await test_coro() - (Pdb) step - --Return-- - > (3)test_main()->None - -> await test_coro() - (Pdb) continue - finished - """ - -def test_pdb_next_command_for_asyncgen(): - """Testing skip unwindng stack on yield for coroutines for "next" command - - >>> import asyncio - - >>> async def agen(): - ... yield 1 - ... await asyncio.sleep(0) - ... yield 2 - - >>> async def test_coro(): - ... async for x in agen(): - ... print(x) - - >>> async def test_main(): - ... import pdb; pdb.Pdb(nosigint=True, readrc=False).set_trace() - ... await test_coro() - - >>> def test_function(): - ... loop = asyncio.new_event_loop() - ... loop.run_until_complete(test_main()) - ... loop.close() - ... asyncio.set_event_loop_policy(None) - ... print("finished") +if not SKIP_ASYNCIO_TESTS: + def test_pdb_next_command_for_coroutine(): + """Testing skip unwindng stack on yield for coroutines for "next" command + + >>> import asyncio + + >>> async def test_coro(): + ... await asyncio.sleep(0) + ... await asyncio.sleep(0) + ... await asyncio.sleep(0) + + >>> async def test_main(): + ... import pdb; pdb.Pdb(nosigint=True, readrc=False).set_trace() + ... await test_coro() + + >>> def test_function(): + ... loop = asyncio.new_event_loop() + ... loop.run_until_complete(test_main()) + ... loop.close() + ... asyncio.set_event_loop_policy(None) + ... print("finished") + + >>> with PdbTestInput(['step', + ... 'step', + ... 'next', + ... 'next', + ... 'next', + ... 'step', + ... 'continue']): + ... test_function() + > (3)test_main() + -> await test_coro() + (Pdb) step + --Call-- + > (1)test_coro() + -> async def test_coro(): + (Pdb) step + > (2)test_coro() + -> await asyncio.sleep(0) + (Pdb) next + > (3)test_coro() + -> await asyncio.sleep(0) + (Pdb) next + > (4)test_coro() + -> await asyncio.sleep(0) + (Pdb) next + Internal StopIteration + > (3)test_main() + -> await test_coro() + (Pdb) step + --Return-- + > (3)test_main()->None + -> await test_coro() + (Pdb) continue + finished + """ - >>> with PdbTestInput(['step', - ... 'step', - ... 'next', - ... 'next', - ... 'step', - ... 'next', - ... 'continue']): - ... test_function() - > (3)test_main() - -> await test_coro() - (Pdb) step - --Call-- - > (1)test_coro() - -> async def test_coro(): - (Pdb) step - > (2)test_coro() - -> async for x in agen(): - (Pdb) next - > (3)test_coro() - -> print(x) - (Pdb) next - 1 - > (2)test_coro() - -> async for x in agen(): - (Pdb) step - --Call-- - > (2)agen() - -> yield 1 - (Pdb) next - > (3)agen() - -> await asyncio.sleep(0) - (Pdb) continue - 2 - finished - """ + def test_pdb_next_command_for_asyncgen(): + """Testing skip unwindng stack on yield for coroutines for "next" command + + >>> import asyncio + + >>> async def agen(): + ... yield 1 + ... await asyncio.sleep(0) + ... yield 2 + + >>> async def test_coro(): + ... async for x in agen(): + ... print(x) + + >>> async def test_main(): + ... import pdb; pdb.Pdb(nosigint=True, readrc=False).set_trace() + ... await test_coro() + + >>> def test_function(): + ... loop = asyncio.new_event_loop() + ... loop.run_until_complete(test_main()) + ... loop.close() + ... asyncio.set_event_loop_policy(None) + ... print("finished") + + >>> with PdbTestInput(['step', + ... 'step', + ... 'next', + ... 'next', + ... 'step', + ... 'next', + ... 'continue']): + ... test_function() + > (3)test_main() + -> await test_coro() + (Pdb) step + --Call-- + > (1)test_coro() + -> async def test_coro(): + (Pdb) step + > (2)test_coro() + -> async for x in agen(): + (Pdb) next + > (3)test_coro() + -> print(x) + (Pdb) next + 1 + > (2)test_coro() + -> async for x in agen(): + (Pdb) step + --Call-- + > (2)agen() + -> yield 1 + (Pdb) next + > (3)agen() + -> await asyncio.sleep(0) + (Pdb) continue + 2 + finished + """ def test_pdb_return_command_for_generator(): """Testing no unwindng stack on yield for generators @@ -1858,47 +1873,48 @@ def test_pdb_return_command_for_generator(): finished """ -def test_pdb_return_command_for_coroutine(): - """Testing no unwindng stack on yield for coroutines for "return" command - - >>> import asyncio - - >>> async def test_coro(): - ... await asyncio.sleep(0) - ... await asyncio.sleep(0) - ... await asyncio.sleep(0) - - >>> async def test_main(): - ... import pdb; pdb.Pdb(nosigint=True, readrc=False).set_trace() - ... await test_coro() - - >>> def test_function(): - ... loop = asyncio.new_event_loop() - ... loop.run_until_complete(test_main()) - ... loop.close() - ... asyncio.set_event_loop_policy(None) - ... print("finished") - - >>> with PdbTestInput(['step', - ... 'step', - ... 'next', - ... 'continue']): - ... test_function() - > (3)test_main() - -> await test_coro() - (Pdb) step - --Call-- - > (1)test_coro() - -> async def test_coro(): - (Pdb) step - > (2)test_coro() - -> await asyncio.sleep(0) - (Pdb) next - > (3)test_coro() - -> await asyncio.sleep(0) - (Pdb) continue - finished - """ +if not SKIP_ASYNCIO_TESTS: + def test_pdb_return_command_for_coroutine(): + """Testing no unwindng stack on yield for coroutines for "return" command + + >>> import asyncio + + >>> async def test_coro(): + ... await asyncio.sleep(0) + ... await asyncio.sleep(0) + ... await asyncio.sleep(0) + + >>> async def test_main(): + ... import pdb; pdb.Pdb(nosigint=True, readrc=False).set_trace() + ... await test_coro() + + >>> def test_function(): + ... loop = asyncio.new_event_loop() + ... loop.run_until_complete(test_main()) + ... loop.close() + ... asyncio.set_event_loop_policy(None) + ... print("finished") + + >>> with PdbTestInput(['step', + ... 'step', + ... 'next', + ... 'continue']): + ... test_function() + > (3)test_main() + -> await test_coro() + (Pdb) step + --Call-- + > (1)test_coro() + -> async def test_coro(): + (Pdb) step + > (2)test_coro() + -> await asyncio.sleep(0) + (Pdb) next + > (3)test_coro() + -> await asyncio.sleep(0) + (Pdb) continue + finished + """ def test_pdb_until_command_for_generator(): """Testing no unwindng stack on yield for generators @@ -1944,52 +1960,53 @@ def test_pdb_until_command_for_generator(): finished """ -def test_pdb_until_command_for_coroutine(): - """Testing no unwindng stack for coroutines - for "until" command if target breakpoint is not reached - - >>> import asyncio - - >>> async def test_coro(): - ... print(0) - ... await asyncio.sleep(0) - ... print(1) - ... await asyncio.sleep(0) - ... print(2) - ... await asyncio.sleep(0) - ... print(3) - - >>> async def test_main(): - ... import pdb; pdb.Pdb(nosigint=True, readrc=False).set_trace() - ... await test_coro() - - >>> def test_function(): - ... loop = asyncio.new_event_loop() - ... loop.run_until_complete(test_main()) - ... loop.close() - ... asyncio.set_event_loop_policy(None) - ... print("finished") - - >>> with PdbTestInput(['step', - ... 'until 8', - ... 'continue']): - ... test_function() - > (3)test_main() - -> await test_coro() - (Pdb) step - --Call-- - > (1)test_coro() - -> async def test_coro(): - (Pdb) until 8 - 0 - 1 - 2 - > (8)test_coro() - -> print(3) - (Pdb) continue - 3 - finished - """ +if not SKIP_ASYNCIO_TESTS: + def test_pdb_until_command_for_coroutine(): + """Testing no unwindng stack for coroutines + for "until" command if target breakpoint is not reached + + >>> import asyncio + + >>> async def test_coro(): + ... print(0) + ... await asyncio.sleep(0) + ... print(1) + ... await asyncio.sleep(0) + ... print(2) + ... await asyncio.sleep(0) + ... print(3) + + >>> async def test_main(): + ... import pdb; pdb.Pdb(nosigint=True, readrc=False).set_trace() + ... await test_coro() + + >>> def test_function(): + ... loop = asyncio.new_event_loop() + ... loop.run_until_complete(test_main()) + ... loop.close() + ... asyncio.set_event_loop_policy(None) + ... print("finished") + + >>> with PdbTestInput(['step', + ... 'until 8', + ... 'continue']): + ... test_function() + > (3)test_main() + -> await test_coro() + (Pdb) step + --Call-- + > (1)test_coro() + -> async def test_coro(): + (Pdb) until 8 + 0 + 1 + 2 + > (8)test_coro() + -> print(3) + (Pdb) continue + 3 + finished + """ def test_pdb_next_command_in_generator_for_loop(): """The next command on returning from a generator controlled by a for loop. @@ -2648,7 +2665,7 @@ def quux(): pass """.encode(), 'bœr', - ('bœr', 4), + ('bœr', 5), ) def test_find_function_found_with_encoding_cookie(self): @@ -2665,7 +2682,7 @@ def quux(): pass """.encode('iso-8859-15'), 'bœr', - ('bœr', 5), + ('bœr', 6), ) def test_find_function_found_with_bom(self): @@ -2675,9 +2692,34 @@ def bœr(): pass """.encode(), 'bœr', - ('bœr', 1), + ('bœr', 2), ) + def test_find_function_first_executable_line(self): + code = textwrap.dedent("""\ + def foo(): pass + + def bar(): + pass # line 4 + + def baz(): + # comment + pass # line 8 + + def mul(): + # code on multiple lines + code = compile( # line 12 + 'def f()', + '', + 'exec', + ) + """).encode() + + self._assert_find_function(code, 'foo', ('foo', 1)) + self._assert_find_function(code, 'bar', ('bar', 4)) + self._assert_find_function(code, 'baz', ('baz', 8)) + self._assert_find_function(code, 'mul', ('mul', 12)) + def test_issue7964(self): # open the file as binary so we can force \r\n newline with open(os_helper.TESTFN, 'wb') as f: @@ -3043,6 +3085,87 @@ def test_blocks_at_first_code_line(self): self.assertTrue(any("__main__.py(4)()" in l for l in stdout.splitlines()), stdout) + def test_file_modified_after_execution(self): + script = """ + print("hello") + """ + + commands = """ + filename = $_frame.f_code.co_filename + f = open(filename, "w") + f.write("print('goodbye')") + f.close() + ll + """ + + stdout, stderr = self.run_pdb_script(script, commands) + self.assertIn("WARNING:", stdout) + self.assertIn("was edited", stdout) + + def test_file_modified_after_execution_with_multiple_instances(self): + script = """ + import pdb; pdb.Pdb().set_trace() + with open(__file__, "w") as f: + f.write("print('goodbye')\\n" * 5) + import pdb; pdb.Pdb().set_trace() + """ + + commands = """ + continue + continue + """ + + filename = 'main.py' + with open(filename, 'w') as f: + f.write(textwrap.dedent(script)) + self.addCleanup(os_helper.unlink, filename) + self.addCleanup(os_helper.rmtree, '__pycache__') + cmd = [sys.executable, filename] + with subprocess.Popen( + cmd, + stdout=subprocess.PIPE, + stdin=subprocess.PIPE, + stderr=subprocess.STDOUT, + env = {**os.environ, 'PYTHONIOENCODING': 'utf-8'}, + ) as proc: + stdout, _ = proc.communicate(str.encode(commands)) + stdout = stdout and bytes.decode(stdout) + + self.assertEqual(proc.returncode, 0) + self.assertIn("WARNING:", stdout) + self.assertIn("was edited", stdout) + + def test_file_modified_after_execution_with_restart(self): + script = """ + import random + # Any code with a source to step into so this script is not checked + # for changes when it's being changed + random.randint(1, 4) + print("hello") + """ + + commands = """ + ll + n + s + filename = $_frame.f_back.f_code.co_filename + def change_file(content, filename): + with open(filename, "w") as f: + f.write(f"print({content})") + + change_file('world', filename) + restart + ll + """ + + stdout, stderr = self.run_pdb_script(script, commands) + # Make sure the code is running correctly and the file is edited + self.assertIn("hello", stdout) + self.assertIn("world", stdout) + # The file was edited, but restart should clear the state and consider + # the file as up to date + self.assertNotIn("WARNING:", stdout) + def test_relative_imports(self): self.module_name = 't_main' os_helper.rmtree(self.module_name) diff --git a/Lib/test/test_peepholer.py b/Lib/test/test_peepholer.py index b825b27060e86b..2ea186c85c8823 100644 --- a/Lib/test/test_peepholer.py +++ b/Lib/test/test_peepholer.py @@ -1131,6 +1131,47 @@ def test_no_dead_store_elimination_in_different_lineno(self): ] self.cfg_optimization_test(insts, expected_insts, consts=list(range(3)), nlocals=1) + def test_unconditional_jump_threading(self): + + def get_insts(lno1, lno2, op1, op2): + return [ + lbl2 := self.Label(), + ('LOAD_NAME', 0, 10), + (op1, lbl1 := self.Label(), lno1), + ('LOAD_NAME', 1, 20), + lbl1, + (op2, lbl2, lno2), + ] + + + for op1 in ('JUMP', 'JUMP_NO_INTERRUPT'): + for op2 in ('JUMP', 'JUMP_NO_INTERRUPT'): + # different lines + lno1, lno2 = (4, 5) + with self.subTest(lno = (lno1, lno2), ops = (op1, op2)): + insts = get_insts(lno1, lno2, op1, op2) + op = 'JUMP' if 'JUMP' in (op1, op2) else 'JUMP_NO_INTERRUPT' + expected_insts = [ + ('LOAD_NAME', 0, 10), + ('NOP', 0, 4), + (op, 0, 5), + ] + self.cfg_optimization_test(insts, expected_insts, consts=list(range(5))) + + # Threading + for lno1, lno2 in [(-1, -1), (-1, 5), (6, -1), (7, 7)]: + with self.subTest(lno = (lno1, lno2), ops = (op1, op2)): + insts = get_insts(lno1, lno2, op1, op2) + lno = lno1 if lno1 != -1 else lno2 + if lno == -1: + lno = 10 # Propagated from the line before + + op = 'JUMP' if 'JUMP' in (op1, op2) else 'JUMP_NO_INTERRUPT' + expected_insts = [ + ('LOAD_NAME', 0, 10), + (op, 0, lno), + ] + self.cfg_optimization_test(insts, expected_insts, consts=list(range(5))) if __name__ == "__main__": unittest.main() diff --git a/Lib/test/test_pickle.py b/Lib/test/test_pickle.py index 1a55da39bdc58d..5e187e5189d117 100644 --- a/Lib/test/test_pickle.py +++ b/Lib/test/test_pickle.py @@ -122,6 +122,7 @@ class PyIdPersPicklerTests(AbstractIdentityPersistentPicklerTests, pickler = pickle._Pickler unpickler = pickle._Unpickler + persistent_load_error = pickle.UnpicklingError @support.cpython_only def test_pickler_reference_cycle(self): @@ -176,7 +177,6 @@ class DispatchTable: support.gc_collect() self.assertIsNone(table_ref()) - @support.cpython_only def test_unpickler_reference_cycle(self): def check(Unpickler): @@ -206,6 +206,28 @@ def persistent_load(pid): return pid check(PersUnpickler) + def test_pickler_super(self): + class PersPickler(self.pickler): + def persistent_id(subself, obj): + self.assertIsNone(super().persistent_id(obj)) + return obj + + for proto in range(pickle.HIGHEST_PROTOCOL + 1): + f = io.BytesIO() + pickler = PersPickler(f, proto) + pickler.dump('abc') + self.assertEqual(self.loads(f.getvalue()), 'abc') + + def test_unpickler_super(self): + class PersUnpickler(self.unpickler): + def persistent_load(subself, pid): + with self.assertRaises(self.persistent_load_error): + super().persistent_load(pid) + return pid + + for proto in range(pickle.HIGHEST_PROTOCOL + 1): + unpickler = PersUnpickler(io.BytesIO(self.dumps('abc', proto))) + self.assertEqual(unpickler.load(), 'abc') class PyPicklerUnpicklerObjectTests(AbstractPicklerUnpicklerObjectTests, unittest.TestCase): @@ -256,6 +278,7 @@ class CPersPicklerTests(PyPersPicklerTests): class CIdPersPicklerTests(PyIdPersPicklerTests): pickler = _pickle.Pickler unpickler = _pickle.Unpickler + persistent_load_error = _pickle.UnpicklingError class CDumpPickle_LoadPickle(PyPicklerTests): pickler = _pickle.Pickler @@ -326,7 +349,7 @@ class SizeofTests(unittest.TestCase): check_sizeof = support.check_sizeof def test_pickler(self): - basesize = support.calcobjsize('7P2n3i2n3i2P') + basesize = support.calcobjsize('6P2n3i2n3i2P') p = _pickle.Pickler(io.BytesIO()) self.assertEqual(object.__sizeof__(p), basesize) MT_size = struct.calcsize('3nP0n') @@ -343,7 +366,7 @@ def test_pickler(self): 0) # Write buffer is cleared after every dump(). def test_unpickler(self): - basesize = support.calcobjsize('2P2n2P 2P2n2i5P 2P3n8P2n2i') + basesize = support.calcobjsize('2P2nP 2P2n2i5P 2P3n8P2n2i') unpickler = _pickle.Unpickler P = struct.calcsize('P') # Size of memo table entry. n = struct.calcsize('n') # Size of mark table entry. @@ -379,7 +402,9 @@ def recurse(deep): check_unpickler(recurse(1), 32, 20) check_unpickler(recurse(20), 32, 20) check_unpickler(recurse(50), 64, 60) - check_unpickler(recurse(100), 128, 140) + if not (support.is_wasi and support.Py_DEBUG): + # stack depth too shallow in pydebug WASI. + check_unpickler(recurse(100), 128, 140) u = unpickler(io.BytesIO(pickle.dumps('a', 0)), encoding='ASCII', errors='strict') @@ -542,7 +567,8 @@ def test_exceptions(self): RecursionError, EncodingWarning, BaseExceptionGroup, - ExceptionGroup): + ExceptionGroup, + IncompleteInputError): continue if exc is not OSError and issubclass(exc, OSError): self.assertEqual(reverse_mapping('builtins', name), diff --git a/Lib/test/test_platform.py b/Lib/test/test_platform.py index 216973350319fe..648e18d0150ef0 100644 --- a/Lib/test/test_platform.py +++ b/Lib/test/test_platform.py @@ -472,7 +472,8 @@ def test_macos(self): 'root:xnu-4570.71.2~1/RELEASE_X86_64'), 'x86_64', 'i386') arch = ('64bit', '') - with mock.patch.object(platform, 'uname', return_value=uname), \ + with mock.patch.object(sys, "platform", "darwin"), \ + mock.patch.object(platform, 'uname', return_value=uname), \ mock.patch.object(platform, 'architecture', return_value=arch): for mac_ver, expected_terse, expected in [ # darwin: mac_ver() returns empty strings diff --git a/Lib/test/test_plistlib.py b/Lib/test/test_plistlib.py index b08ababa341cfe..001f86f2893f2f 100644 --- a/Lib/test/test_plistlib.py +++ b/Lib/test/test_plistlib.py @@ -13,6 +13,8 @@ import subprocess import binascii import collections +import time +import zoneinfo from test import support from test.support import os_helper from io import BytesIO @@ -508,6 +510,19 @@ def test_bytes(self): data2 = plistlib.dumps(pl2) self.assertEqual(data, data2) + def test_loads_str_with_xml_fmt(self): + pl = self._create() + b = plistlib.dumps(pl) + s = b.decode() + self.assertIsInstance(s, str) + pl2 = plistlib.loads(s) + self.assertEqual(pl, pl2) + + def test_loads_str_with_binary_fmt(self): + msg = "value must be bytes-like object when fmt is FMT_BINARY" + with self.assertRaisesRegex(TypeError, msg): + plistlib.loads('test', fmt=plistlib.FMT_BINARY) + def test_indentation_array(self): data = [[[[[[[[{'test': b'aaaaaa'}]]]]]]]] self.assertEqual(plistlib.loads(plistlib.dumps(data)), data) @@ -838,6 +853,54 @@ def test_xml_plist_with_entity_decl(self): "XML entity declarations are not supported"): plistlib.loads(XML_PLIST_WITH_ENTITY, fmt=plistlib.FMT_XML) + def test_load_aware_datetime(self): + dt = plistlib.loads(b"2023-12-10T08:03:30Z", + aware_datetime=True) + self.assertEqual(dt.tzinfo, datetime.UTC) + + @unittest.skipUnless("America/Los_Angeles" in zoneinfo.available_timezones(), + "Can't find timezone datebase") + def test_dump_aware_datetime(self): + dt = datetime.datetime(2345, 6, 7, 8, 9, 10, + tzinfo=zoneinfo.ZoneInfo("America/Los_Angeles")) + for fmt in ALL_FORMATS: + s = plistlib.dumps(dt, fmt=fmt, aware_datetime=True) + loaded_dt = plistlib.loads(s, fmt=fmt, aware_datetime=True) + self.assertEqual(loaded_dt.tzinfo, datetime.UTC) + self.assertEqual(loaded_dt, dt) + + def test_dump_utc_aware_datetime(self): + dt = datetime.datetime(2345, 6, 7, 8, 9, 10, tzinfo=datetime.UTC) + for fmt in ALL_FORMATS: + s = plistlib.dumps(dt, fmt=fmt, aware_datetime=True) + loaded_dt = plistlib.loads(s, fmt=fmt, aware_datetime=True) + self.assertEqual(loaded_dt.tzinfo, datetime.UTC) + self.assertEqual(loaded_dt, dt) + + @unittest.skipUnless("America/Los_Angeles" in zoneinfo.available_timezones(), + "Can't find timezone datebase") + def test_dump_aware_datetime_without_aware_datetime_option(self): + dt = datetime.datetime(2345, 6, 7, 8, + tzinfo=zoneinfo.ZoneInfo("America/Los_Angeles")) + s = plistlib.dumps(dt, fmt=plistlib.FMT_XML, aware_datetime=False) + self.assertIn(b"2345-06-07T08:00:00Z", s) + + def test_dump_utc_aware_datetime_without_aware_datetime_option(self): + dt = datetime.datetime(2345, 6, 7, 8, tzinfo=datetime.UTC) + s = plistlib.dumps(dt, fmt=plistlib.FMT_XML, aware_datetime=False) + self.assertIn(b"2345-06-07T08:00:00Z", s) + + def test_dump_naive_datetime_with_aware_datetime_option(self): + # Save a naive datetime with aware_datetime set to true. This will lead + # to having different time as compared to the current machine's + # timezone, which is UTC. + dt = datetime.datetime(2003, 6, 7, 8, tzinfo=None) + for fmt in ALL_FORMATS: + s = plistlib.dumps(dt, fmt=fmt, aware_datetime=True) + parsed = plistlib.loads(s, aware_datetime=False) + expected = dt.astimezone(datetime.UTC).replace(tzinfo=None) + self.assertEqual(parsed, expected) + class TestBinaryPlistlib(unittest.TestCase): @@ -908,12 +971,12 @@ def test_cycles(self): self.assertIs(b['x'], b) def test_deep_nesting(self): - for N in [300, 100000]: + for N in [50, 300, 100_000]: chunks = [b'\xa1' + (i + 1).to_bytes(4, 'big') for i in range(N)] try: result = self.decode(*chunks, b'\x54seed', offset_size=4, ref_size=4) except RecursionError: - pass + self.assertGreater(N, sys.getrecursionlimit()) else: for i in range(N): self.assertIsInstance(result, list) @@ -962,6 +1025,28 @@ def test_invalid_binary(self): with self.assertRaises(plistlib.InvalidFileException): plistlib.loads(b'bplist00' + data, fmt=plistlib.FMT_BINARY) + def test_load_aware_datetime(self): + data = (b'bplist003B\x04>\xd0d\x00\x00\x00\x08\x00\x00\x00\x00\x00\x00' + b'\x01\x01\x00\x00\x00\x00\x00\x00\x00\x01\x00\x00\x00\x00\x00' + b'\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x11') + self.assertEqual(plistlib.loads(data, aware_datetime=True), + datetime.datetime(2345, 6, 7, 8, tzinfo=datetime.UTC)) + + @unittest.skipUnless("America/Los_Angeles" in zoneinfo.available_timezones(), + "Can't find timezone datebase") + def test_dump_aware_datetime_without_aware_datetime_option(self): + dt = datetime.datetime(2345, 6, 7, 8, + tzinfo=zoneinfo.ZoneInfo("America/Los_Angeles")) + msg = "can't subtract offset-naive and offset-aware datetimes" + with self.assertRaisesRegex(TypeError, msg): + plistlib.dumps(dt, fmt=plistlib.FMT_BINARY, aware_datetime=False) + + def test_dump_utc_aware_datetime_without_aware_datetime_option(self): + dt = datetime.datetime(2345, 6, 7, 8, tzinfo=datetime.UTC) + msg = "can't subtract offset-naive and offset-aware datetimes" + with self.assertRaisesRegex(TypeError, msg): + plistlib.dumps(dt, fmt=plistlib.FMT_BINARY, aware_datetime=False) + class TestKeyedArchive(unittest.TestCase): def test_keyed_archive_data(self): @@ -1072,5 +1157,6 @@ def test_octal_and_hex(self): self.assertEqual(p.get("HexType"), 16777228) self.assertEqual(p.get("IntType"), 83) + if __name__ == '__main__': unittest.main() diff --git a/Lib/test/test_posix.py b/Lib/test/test_posix.py index 55cc5e4c6e4f03..a45f620e18dc1d 100644 --- a/Lib/test/test_posix.py +++ b/Lib/test/test_posix.py @@ -1,7 +1,7 @@ "Test posix functions" from test import support -from test.support import import_helper +from test.support import is_apple from test.support import os_helper from test.support import warnings_helper from test.support.script_helper import assert_python_ok @@ -781,9 +781,10 @@ def check_stat(uid, gid): check_stat(uid, gid) self.assertRaises(OSError, chown_func, first_param, 0, -1) check_stat(uid, gid) - if 0 not in os.getgroups(): - self.assertRaises(OSError, chown_func, first_param, -1, 0) - check_stat(uid, gid) + if hasattr(os, 'getgroups'): + if 0 not in os.getgroups(): + self.assertRaises(OSError, chown_func, first_param, -1, 0) + check_stat(uid, gid) # test illegal types for t in str, float: self.assertRaises(TypeError, chown_func, first_param, t(uid), gid) @@ -936,6 +937,7 @@ def test_utime(self): posix.utime(os_helper.TESTFN, (now, now)) def check_chmod(self, chmod_func, target, **kwargs): + closefd = not isinstance(target, int) mode = os.stat(target).st_mode try: new_mode = mode & ~(stat.S_IWOTH | stat.S_IWGRP | stat.S_IWUSR) @@ -943,7 +945,7 @@ def check_chmod(self, chmod_func, target, **kwargs): self.assertEqual(os.stat(target).st_mode, new_mode) if stat.S_ISREG(mode): try: - with open(target, 'wb+'): + with open(target, 'wb+', closefd=closefd): pass except PermissionError: pass @@ -951,10 +953,10 @@ def check_chmod(self, chmod_func, target, **kwargs): chmod_func(target, new_mode, **kwargs) self.assertEqual(os.stat(target).st_mode, new_mode) if stat.S_ISREG(mode): - with open(target, 'wb+'): + with open(target, 'wb+', closefd=closefd): pass finally: - posix.chmod(target, mode) + chmod_func(target, mode) @os_helper.skip_unless_working_chmod def test_chmod_file(self): @@ -971,6 +973,12 @@ def test_chmod_dir(self): target = self.tempdir() self.check_chmod(posix.chmod, target) + @os_helper.skip_unless_working_chmod + def test_fchmod_file(self): + with open(os_helper.TESTFN, 'wb+') as f: + self.check_chmod(posix.fchmod, f.fileno()) + self.check_chmod(posix.chmod, f.fileno()) + @unittest.skipUnless(hasattr(posix, 'lchmod'), 'test needs os.lchmod()') def test_lchmod_file(self): self.check_chmod(posix.lchmod, os_helper.TESTFN) @@ -1249,8 +1257,8 @@ def test_sched_priority(self): self.assertIsInstance(lo, int) self.assertIsInstance(hi, int) self.assertGreaterEqual(hi, lo) - # OSX evidently just returns 15 without checking the argument. - if sys.platform != "darwin": + # Apple plaforms return 15 without checking the argument. + if not is_apple: self.assertRaises(OSError, posix.sched_get_priority_min, -23) self.assertRaises(OSError, posix.sched_get_priority_max, -23) @@ -1506,6 +1514,13 @@ def test_stat_dir_fd(self): self.assertRaises(OverflowError, posix.stat, name, dir_fd=10**20) + for fd in False, True: + with self.assertWarnsRegex(RuntimeWarning, + 'bool is used as a file descriptor') as cm: + with self.assertRaises(OSError): + posix.stat('nonexisting', dir_fd=fd) + self.assertEqual(cm.filename, __file__) + @unittest.skipUnless(os.utime in os.supports_dir_fd, "test needs dir_fd support in os.utime()") def test_utime_dir_fd(self): with self.prepare_file() as (dir_fd, name, fullname): @@ -2021,11 +2036,13 @@ def test_dup2(self): @unittest.skipUnless(hasattr(os, 'posix_spawn'), "test needs os.posix_spawn") +@support.requires_subprocess() class TestPosixSpawn(unittest.TestCase, _PosixSpawnMixin): spawn_func = getattr(posix, 'posix_spawn', None) @unittest.skipUnless(hasattr(os, 'posix_spawnp'), "test needs os.posix_spawnp") +@support.requires_subprocess() class TestPosixSpawnP(unittest.TestCase, _PosixSpawnMixin): spawn_func = getattr(posix, 'posix_spawnp', None) diff --git a/Lib/test/test_pty.py b/Lib/test/test_pty.py index f31a68c5d84e03..3f2bac0155fd9e 100644 --- a/Lib/test/test_pty.py +++ b/Lib/test/test_pty.py @@ -1,11 +1,17 @@ -from test.support import verbose, reap_children +import sys +import unittest +from test.support import ( + is_apple_mobile, is_emscripten, is_wasi, reap_children, verbose +) from test.support.import_helper import import_module +from test.support.os_helper import TESTFN, unlink -# Skip these tests if termios or fcntl are not available +# Skip these tests if termios is not available import_module('termios') -# fcntl is a proxy for not being one of the wasm32 platforms even though we -# don't use this module... a proper check for what crashes those is needed. -import_module("fcntl") + +# Skip tests on WASM platforms, plus iOS/tvOS/watchOS +if is_apple_mobile or is_emscripten or is_wasi: + raise unittest.SkipTest(f"pty tests not required on {sys.platform}") import errno import os @@ -16,7 +22,6 @@ import signal import socket import io # readline -import unittest import warnings TEST_STRING_1 = b"I wish to buy a fish license.\n" @@ -292,7 +297,26 @@ def test_master_read(self): self.assertEqual(data, b"") def test_spawn_doesnt_hang(self): - pty.spawn([sys.executable, '-c', 'print("hi there")']) + self.addCleanup(unlink, TESTFN) + with open(TESTFN, 'wb') as f: + STDOUT_FILENO = 1 + dup_stdout = os.dup(STDOUT_FILENO) + os.dup2(f.fileno(), STDOUT_FILENO) + buf = b'' + def master_read(fd): + nonlocal buf + data = os.read(fd, 1024) + buf += data + return data + try: + pty.spawn([sys.executable, '-c', 'print("hi there")'], + master_read) + finally: + os.dup2(dup_stdout, STDOUT_FILENO) + os.close(dup_stdout) + self.assertEqual(buf, b'hi there\r\n') + with open(TESTFN, 'rb') as f: + self.assertEqual(f.read(), b'hi there\r\n') class SmallPtyTests(unittest.TestCase): """These tests don't spawn children or hang.""" diff --git a/Lib/test/test_pydoc.py b/Lib/test/test_pydoc.py index eb50510e12b7b6..99b19d01783a10 100644 --- a/Lib/test/test_pydoc.py +++ b/Lib/test/test_pydoc.py @@ -32,7 +32,7 @@ from test.support import threading_helper from test.support import (reap_children, captured_output, captured_stdout, captured_stderr, is_emscripten, is_wasi, - requires_docstrings) + requires_docstrings, MISSING_C_DOCSTRINGS) from test.support.os_helper import (TESTFN, rmtree, unlink) from test import pydoc_mod @@ -745,14 +745,18 @@ def test_splitdoc_with_description(self): def test_is_package_when_not_package(self): with os_helper.temp_cwd() as test_dir: - self.assertFalse(pydoc.ispackage(test_dir)) + with self.assertWarns(DeprecationWarning) as cm: + self.assertFalse(pydoc.ispackage(test_dir)) + self.assertEqual(cm.filename, __file__) def test_is_package_when_is_package(self): with os_helper.temp_cwd() as test_dir: init_path = os.path.join(test_dir, '__init__.py') open(init_path, 'w').close() - self.assertTrue(pydoc.ispackage(test_dir)) + with self.assertWarns(DeprecationWarning) as cm: + self.assertTrue(pydoc.ispackage(test_dir)) os.remove(init_path) + self.assertEqual(cm.filename, __file__) def test_allmethods(self): # issue 17476: allmethods was no longer returning unbound methods. @@ -906,12 +910,13 @@ class A(builtins.object) | ---------------------------------------------------------------------- | Data descriptors defined here: | - | __dict__ - | dictionary for instance variables + | __dict__%s | - | __weakref__ - | list of weak references to the object -''' % __name__) + | __weakref__%s +''' % (__name__, + '' if MISSING_C_DOCSTRINGS else '\n | dictionary for instance variables', + '' if MISSING_C_DOCSTRINGS else '\n | list of weak references to the object', + )) def func( arg1: Callable[[Annotated[int, 'Some doc']], str], @@ -1154,13 +1159,15 @@ def test_generic_alias(self): doc = pydoc.render_doc(typing.List[int], renderer=pydoc.plaintext) self.assertIn('_GenericAlias in module typing', doc) self.assertIn('List = class list(object)', doc) - self.assertIn(list.__doc__.strip().splitlines()[0], doc) + if not MISSING_C_DOCSTRINGS: + self.assertIn(list.__doc__.strip().splitlines()[0], doc) self.assertEqual(pydoc.describe(list[int]), 'GenericAlias') doc = pydoc.render_doc(list[int], renderer=pydoc.plaintext) self.assertIn('GenericAlias in module builtins', doc) self.assertIn('\nclass list(object)', doc) - self.assertIn(list.__doc__.strip().splitlines()[0], doc) + if not MISSING_C_DOCSTRINGS: + self.assertIn(list.__doc__.strip().splitlines()[0], doc) def test_union_type(self): self.assertEqual(pydoc.describe(typing.Union[int, str]), '_UnionGenericAlias') @@ -1174,7 +1181,8 @@ def test_union_type(self): doc = pydoc.render_doc(int | str, renderer=pydoc.plaintext) self.assertIn('UnionType in module types object', doc) self.assertIn('\nclass UnionType(builtins.object)', doc) - self.assertIn(types.UnionType.__doc__.strip().splitlines()[0], doc) + if not MISSING_C_DOCSTRINGS: + self.assertIn(types.UnionType.__doc__.strip().splitlines()[0], doc) def test_special_form(self): self.assertEqual(pydoc.describe(typing.NoReturn), '_SpecialForm') @@ -1327,6 +1335,7 @@ def test_bound_builtin_classmethod_o(self): "__class_getitem__(object, /) method of builtins.type instance") @support.cpython_only + @requires_docstrings def test_module_level_callable_unrepresentable_default(self): import _testcapi builtin = _testcapi.func_with_unrepresentable_signature @@ -1334,6 +1343,7 @@ def test_module_level_callable_unrepresentable_default(self): "func_with_unrepresentable_signature(a, b=)") @support.cpython_only + @requires_docstrings def test_builtin_staticmethod_unrepresentable_default(self): self.assertEqual(self._get_summary_line(str.maketrans), "maketrans(x, y=, z=, /)") @@ -1343,6 +1353,7 @@ def test_builtin_staticmethod_unrepresentable_default(self): "staticmeth(a, b=)") @support.cpython_only + @requires_docstrings def test_unbound_builtin_method_unrepresentable_default(self): self.assertEqual(self._get_summary_line(dict.pop), "pop(self, key, default=, /)") @@ -1352,6 +1363,7 @@ def test_unbound_builtin_method_unrepresentable_default(self): "meth(self, /, a, b=)") @support.cpython_only + @requires_docstrings def test_bound_builtin_method_unrepresentable_default(self): self.assertEqual(self._get_summary_line({}.pop), "pop(key, default=, /) " @@ -1363,6 +1375,7 @@ def test_bound_builtin_method_unrepresentable_default(self): "method of _testcapi.DocStringUnrepresentableSignatureTest instance") @support.cpython_only + @requires_docstrings def test_unbound_builtin_classmethod_unrepresentable_default(self): import _testcapi cls = _testcapi.DocStringUnrepresentableSignatureTest @@ -1371,6 +1384,7 @@ def test_unbound_builtin_classmethod_unrepresentable_default(self): "classmeth(type, /, a, b=)") @support.cpython_only + @requires_docstrings def test_bound_builtin_classmethod_unrepresentable_default(self): import _testcapi cls = _testcapi.DocStringUnrepresentableSignatureTest diff --git a/Lib/test/test_queue.py b/Lib/test/test_queue.py index 33113a72e6b6a9..e3d4d566cdda48 100644 --- a/Lib/test/test_queue.py +++ b/Lib/test/test_queue.py @@ -241,6 +241,384 @@ def test_shrinking_queue(self): with self.assertRaises(self.queue.Full): q.put_nowait(4) + def test_shutdown_empty(self): + q = self.type2test() + q.shutdown() + with self.assertRaises(self.queue.ShutDown): + q.put("data") + with self.assertRaises(self.queue.ShutDown): + q.get() + + def test_shutdown_nonempty(self): + q = self.type2test() + q.put("data") + q.shutdown() + q.get() + with self.assertRaises(self.queue.ShutDown): + q.get() + + def test_shutdown_immediate(self): + q = self.type2test() + q.put("data") + q.shutdown(immediate=True) + with self.assertRaises(self.queue.ShutDown): + q.get() + + def test_shutdown_allowed_transitions(self): + # allowed transitions would be from alive via shutdown to immediate + q = self.type2test() + self.assertFalse(q.is_shutdown) + + q.shutdown() + self.assertTrue(q.is_shutdown) + + q.shutdown(immediate=True) + self.assertTrue(q.is_shutdown) + + q.shutdown(immediate=False) + + def _shutdown_all_methods_in_one_thread(self, immediate): + q = self.type2test(2) + q.put("L") + q.put_nowait("O") + q.shutdown(immediate) + + with self.assertRaises(self.queue.ShutDown): + q.put("E") + with self.assertRaises(self.queue.ShutDown): + q.put_nowait("W") + if immediate: + with self.assertRaises(self.queue.ShutDown): + q.get() + with self.assertRaises(self.queue.ShutDown): + q.get_nowait() + with self.assertRaises(ValueError): + q.task_done() + q.join() + else: + self.assertIn(q.get(), "LO") + q.task_done() + self.assertIn(q.get(), "LO") + q.task_done() + q.join() + # on shutdown(immediate=False) + # when queue is empty, should raise ShutDown Exception + with self.assertRaises(self.queue.ShutDown): + q.get() # p.get(True) + with self.assertRaises(self.queue.ShutDown): + q.get_nowait() # p.get(False) + with self.assertRaises(self.queue.ShutDown): + q.get(True, 1.0) + + def test_shutdown_all_methods_in_one_thread(self): + return self._shutdown_all_methods_in_one_thread(False) + + def test_shutdown_immediate_all_methods_in_one_thread(self): + return self._shutdown_all_methods_in_one_thread(True) + + def _write_msg_thread(self, q, n, results, delay, + i_when_exec_shutdown, + event_start, event_end): + event_start.wait() + for i in range(1, n+1): + try: + q.put((i, "YDLO")) + results.append(True) + except self.queue.ShutDown: + results.append(False) + # triggers shutdown of queue + if i == i_when_exec_shutdown: + event_end.set() + time.sleep(delay) + # end of all puts + q.join() + + def _read_msg_thread(self, q, nb, results, delay, event_start): + event_start.wait() + block = True + while nb: + time.sleep(delay) + try: + # Get at least one message + q.get(block) + block = False + q.task_done() + results.append(True) + nb -= 1 + except self.queue.ShutDown: + results.append(False) + nb -= 1 + except self.queue.Empty: + pass + q.join() + + def _shutdown_thread(self, q, event_end, immediate): + event_end.wait() + q.shutdown(immediate) + q.join() + + def _join_thread(self, q, delay, event_start): + event_start.wait() + time.sleep(delay) + q.join() + + def _shutdown_all_methods_in_many_threads(self, immediate): + q = self.type2test() + ps = [] + ev_start = threading.Event() + ev_exec_shutdown = threading.Event() + res_puts = [] + res_gets = [] + delay = 1e-4 + read_process = 4 + nb_msgs = read_process * 16 + nb_msgs_r = nb_msgs // read_process + when_exec_shutdown = nb_msgs // 2 + lprocs = ( + (self._write_msg_thread, 1, (q, nb_msgs, res_puts, delay, + when_exec_shutdown, + ev_start, ev_exec_shutdown)), + (self._read_msg_thread, read_process, (q, nb_msgs_r, + res_gets, delay*2, + ev_start)), + (self._join_thread, 2, (q, delay*2, ev_start)), + (self._shutdown_thread, 1, (q, ev_exec_shutdown, immediate)), + ) + # start all threds + for func, n, args in lprocs: + for i in range(n): + ps.append(threading.Thread(target=func, args=args)) + ps[-1].start() + # set event in order to run q.shutdown() + ev_start.set() + + if not immediate: + assert(len(res_gets) == len(res_puts)) + assert(res_gets.count(True) == res_puts.count(True)) + else: + assert(len(res_gets) <= len(res_puts)) + assert(res_gets.count(True) <= res_puts.count(True)) + + for thread in ps[1:]: + thread.join() + + def test_shutdown_all_methods_in_many_threads(self): + return self._shutdown_all_methods_in_many_threads(False) + + def test_shutdown_immediate_all_methods_in_many_threads(self): + return self._shutdown_all_methods_in_many_threads(True) + + def _get(self, q, go, results, shutdown=False): + go.wait() + try: + msg = q.get() + results.append(not shutdown) + return not shutdown + except self.queue.ShutDown: + results.append(shutdown) + return shutdown + + def _get_shutdown(self, q, go, results): + return self._get(q, go, results, True) + + def _get_task_done(self, q, go, results): + go.wait() + try: + msg = q.get() + q.task_done() + results.append(True) + return msg + except self.queue.ShutDown: + results.append(False) + return False + + def _put(self, q, msg, go, results, shutdown=False): + go.wait() + try: + q.put(msg) + results.append(not shutdown) + return not shutdown + except self.queue.ShutDown: + results.append(shutdown) + return shutdown + + def _put_shutdown(self, q, msg, go, results): + return self._put(q, msg, go, results, True) + + def _join(self, q, results, shutdown=False): + try: + q.join() + results.append(not shutdown) + return not shutdown + except self.queue.ShutDown: + results.append(shutdown) + return shutdown + + def _join_shutdown(self, q, results): + return self._join(q, results, True) + + def _shutdown_get(self, immediate): + q = self.type2test(2) + results = [] + go = threading.Event() + q.put("Y") + q.put("D") + # queue full + + if immediate: + thrds = ( + (self._get_shutdown, (q, go, results)), + (self._get_shutdown, (q, go, results)), + ) + else: + thrds = ( + # on shutdown(immediate=False) + # one of these threads shoud raise Shutdown + (self._get, (q, go, results)), + (self._get, (q, go, results)), + (self._get, (q, go, results)), + ) + threads = [] + for func, params in thrds: + threads.append(threading.Thread(target=func, args=params)) + threads[-1].start() + q.shutdown(immediate) + go.set() + for t in threads: + t.join() + if immediate: + self.assertListEqual(results, [True, True]) + else: + self.assertListEqual(sorted(results), [False] + [True]*(len(thrds)-1)) + + def test_shutdown_get(self): + return self._shutdown_get(False) + + def test_shutdown_immediate_get(self): + return self._shutdown_get(True) + + def _shutdown_put(self, immediate): + q = self.type2test(2) + results = [] + go = threading.Event() + q.put("Y") + q.put("D") + # queue fulled + + thrds = ( + (self._put_shutdown, (q, "E", go, results)), + (self._put_shutdown, (q, "W", go, results)), + ) + threads = [] + for func, params in thrds: + threads.append(threading.Thread(target=func, args=params)) + threads[-1].start() + q.shutdown() + go.set() + for t in threads: + t.join() + + self.assertEqual(results, [True]*len(thrds)) + + def test_shutdown_put(self): + return self._shutdown_put(False) + + def test_shutdown_immediate_put(self): + return self._shutdown_put(True) + + def _shutdown_join(self, immediate): + q = self.type2test() + results = [] + q.put("Y") + go = threading.Event() + nb = q.qsize() + + thrds = ( + (self._join, (q, results)), + (self._join, (q, results)), + ) + threads = [] + for func, params in thrds: + threads.append(threading.Thread(target=func, args=params)) + threads[-1].start() + if not immediate: + res = [] + for i in range(nb): + threads.append(threading.Thread(target=self._get_task_done, args=(q, go, res))) + threads[-1].start() + q.shutdown(immediate) + go.set() + for t in threads: + t.join() + + self.assertEqual(results, [True]*len(thrds)) + + def test_shutdown_immediate_join(self): + return self._shutdown_join(True) + + def test_shutdown_join(self): + return self._shutdown_join(False) + + def _shutdown_put_join(self, immediate): + q = self.type2test(2) + results = [] + go = threading.Event() + q.put("Y") + nb = q.qsize() + # queue not fulled + + thrds = ( + (self._put_shutdown, (q, "E", go, results)), + (self._join, (q, results)), + ) + threads = [] + for func, params in thrds: + threads.append(threading.Thread(target=func, args=params)) + threads[-1].start() + self.assertEqual(q.unfinished_tasks, nb) + for i in range(nb): + t = threading.Thread(target=q.task_done) + t.start() + threads.append(t) + q.shutdown(immediate) + go.set() + for t in threads: + t.join() + + self.assertEqual(results, [True]*len(thrds)) + + def test_shutdown_immediate_put_join(self): + return self._shutdown_put_join(True) + + def test_shutdown_put_join(self): + return self._shutdown_put_join(False) + + def test_shutdown_get_task_done_join(self): + q = self.type2test(2) + results = [] + go = threading.Event() + q.put("Y") + q.put("D") + self.assertEqual(q.unfinished_tasks, q.qsize()) + + thrds = ( + (self._get_task_done, (q, go, results)), + (self._get_task_done, (q, go, results)), + (self._join, (q, results)), + (self._join, (q, results)), + ) + threads = [] + for func, params in thrds: + threads.append(threading.Thread(target=func, args=params)) + threads[-1].start() + go.set() + q.shutdown(False) + for t in threads: + t.join() + + self.assertEqual(results, [True]*len(thrds)) + + class QueueTest(BaseQueueTestMixin): def setUp(self): diff --git a/Lib/test/test_regrtest.py b/Lib/test/test_regrtest.py index e828941f6c779d..89562fa5eac62c 100644 --- a/Lib/test/test_regrtest.py +++ b/Lib/test/test_regrtest.py @@ -845,6 +845,8 @@ def test_tools_buildbot_test(self): test_args.append('-x64') # 64-bit build if not support.Py_DEBUG: test_args.append('+d') # Release build, use python.exe + if sysconfig.get_config_var("Py_GIL_DISABLED"): + test_args.append('--disable-gil') self.run_batch(script, *test_args, *self.tests) @unittest.skipUnless(sys.platform == 'win32', 'Windows only') @@ -862,6 +864,8 @@ def test_pcbuild_rt(self): rt_args.append('-x64') # 64-bit build if support.Py_DEBUG: rt_args.append('-d') # Debug build, use python_d.exe + if sysconfig.get_config_var("Py_GIL_DISABLED"): + rt_args.append('--disable-gil') self.run_batch(script, *rt_args, *self.regrtest_args, *self.tests) diff --git a/Lib/test/test_richcmp.py b/Lib/test/test_richcmp.py index 6fb31c80d7e670..5f449cdc05c6ba 100644 --- a/Lib/test/test_richcmp.py +++ b/Lib/test/test_richcmp.py @@ -221,7 +221,7 @@ def do(bad): self.assertRaises(Exc, func, Bad()) @support.no_tracing - @support.infinite_recursion() + @support.infinite_recursion(25) def test_recursion(self): # Check that comparison for recursive objects fails gracefully from collections import UserList diff --git a/Lib/test/test_rlcompleter.py b/Lib/test/test_rlcompleter.py index 7347fca71be2fe..273ce2cf5c7dd2 100644 --- a/Lib/test/test_rlcompleter.py +++ b/Lib/test/test_rlcompleter.py @@ -2,6 +2,7 @@ from unittest.mock import patch import builtins import rlcompleter +from test.support import MISSING_C_DOCSTRINGS class CompleteMe: """ Trivial class used in testing rlcompleter.Completer. """ @@ -40,12 +41,12 @@ def test_global_matches(self): # test with a customized namespace self.assertEqual(self.completer.global_matches('CompleteM'), - ['CompleteMe()']) + ['CompleteMe(' if MISSING_C_DOCSTRINGS else 'CompleteMe()']) self.assertEqual(self.completer.global_matches('eg'), ['egg(']) # XXX: see issue5256 self.assertEqual(self.completer.global_matches('CompleteM'), - ['CompleteMe()']) + ['CompleteMe(' if MISSING_C_DOCSTRINGS else 'CompleteMe()']) def test_attr_matches(self): # test with builtins namespace diff --git a/Lib/test/test_runpy.py b/Lib/test/test_runpy.py index 628c8cae38a751..9d76764c75be3e 100644 --- a/Lib/test/test_runpy.py +++ b/Lib/test/test_runpy.py @@ -12,7 +12,8 @@ import textwrap import unittest import warnings -from test.support import no_tracing, verbose, requires_subprocess, requires_resource +from test.support import (infinite_recursion, no_tracing, verbose, + requires_subprocess, requires_resource) from test.support.import_helper import forget, make_legacy_pyc, unload from test.support.os_helper import create_empty_file, temp_dir from test.support.script_helper import make_script, make_zip_script @@ -661,8 +662,10 @@ def test_basic_script_with_path_object(self): mod_name = 'script' script_name = pathlib.Path(self._make_test_script(script_dir, mod_name)) - self._check_script(script_name, "", script_name, - script_name, expect_spec=False) + self._check_script(script_name, "", + os.fsdecode(script_name), + os.fsdecode(script_name), + expect_spec=False) def test_basic_script_no_suffix(self): with temp_dir() as script_dir: @@ -741,7 +744,8 @@ def test_main_recursion_error(self): "runpy.run_path(%r)\n") % dummy_dir script_name = self._make_test_script(script_dir, mod_name, source) zip_name, fname = make_zip_script(script_dir, 'test_zip', script_name) - self.assertRaises(RecursionError, run_path, zip_name) + with infinite_recursion(25): + self.assertRaises(RecursionError, run_path, zip_name) def test_encoding(self): with temp_dir() as script_dir: diff --git a/Lib/test/test_selectors.py b/Lib/test/test_selectors.py index 677349c2bfca93..643775597c56c6 100644 --- a/Lib/test/test_selectors.py +++ b/Lib/test/test_selectors.py @@ -6,8 +6,7 @@ import socket import sys from test import support -from test.support import os_helper -from test.support import socket_helper +from test.support import is_apple, os_helper, socket_helper from time import sleep import unittest import unittest.mock @@ -526,7 +525,7 @@ def test_above_fd_setsize(self): try: fds = s.select() except OSError as e: - if e.errno == errno.EINVAL and sys.platform == 'darwin': + if e.errno == errno.EINVAL and is_apple: # unexplainable errors on macOS don't need to fail the test self.skipTest("Invalid argument error calling poll()") raise diff --git a/Lib/test/test_shutil.py b/Lib/test/test_shutil.py index 5ce8e5d77fbbf3..d96dad4eb9475d 100644 --- a/Lib/test/test_shutil.py +++ b/Lib/test/test_shutil.py @@ -1101,19 +1101,18 @@ def test_copymode_follow_symlinks(self): shutil.copymode(src, dst) self.assertEqual(os.stat(src).st_mode, os.stat(dst).st_mode) # On Windows, os.chmod does not follow symlinks (issue #15411) - if os.name != 'nt': - # follow src link - os.chmod(dst, stat.S_IRWXO) - shutil.copymode(src_link, dst) - self.assertEqual(os.stat(src).st_mode, os.stat(dst).st_mode) - # follow dst link - os.chmod(dst, stat.S_IRWXO) - shutil.copymode(src, dst_link) - self.assertEqual(os.stat(src).st_mode, os.stat(dst).st_mode) - # follow both links - os.chmod(dst, stat.S_IRWXO) - shutil.copymode(src_link, dst_link) - self.assertEqual(os.stat(src).st_mode, os.stat(dst).st_mode) + # follow src link + os.chmod(dst, stat.S_IRWXO) + shutil.copymode(src_link, dst) + self.assertEqual(os.stat(src).st_mode, os.stat(dst).st_mode) + # follow dst link + os.chmod(dst, stat.S_IRWXO) + shutil.copymode(src, dst_link) + self.assertEqual(os.stat(src).st_mode, os.stat(dst).st_mode) + # follow both links + os.chmod(dst, stat.S_IRWXO) + shutil.copymode(src_link, dst_link) + self.assertEqual(os.stat(src).st_mode, os.stat(dst).st_mode) @unittest.skipUnless(hasattr(os, 'lchmod'), 'requires os.lchmod') @os_helper.skip_unless_symlink @@ -2149,6 +2148,7 @@ def check_chown(path, uid=None, gid=None): check_chown(dirname, uid, gid) +@support.requires_subprocess() class TestWhich(BaseTest, unittest.TestCase): def setUp(self): @@ -2689,6 +2689,35 @@ def test_move_dir_caseinsensitive(self): finally: os.rmdir(dst_dir) + # bpo-26791: Check that a symlink to a directory can + # be moved into that directory. + @mock_rename + def _test_move_symlink_to_dir_into_dir(self, dst): + src = os.path.join(self.src_dir, 'linktodir') + dst_link = os.path.join(self.dst_dir, 'linktodir') + os.symlink(self.dst_dir, src, target_is_directory=True) + shutil.move(src, dst) + self.assertTrue(os.path.islink(dst_link)) + self.assertTrue(os.path.samefile(self.dst_dir, dst_link)) + self.assertFalse(os.path.exists(src)) + + # Repeat the move operation with the destination + # symlink already in place (should raise shutil.Error). + os.symlink(self.dst_dir, src, target_is_directory=True) + with self.assertRaises(shutil.Error): + shutil.move(src, dst) + self.assertTrue(os.path.samefile(self.dst_dir, dst_link)) + self.assertTrue(os.path.exists(src)) + + @os_helper.skip_unless_symlink + def test_move_symlink_to_dir_into_dir(self): + self._test_move_symlink_to_dir_into_dir(self.dst_dir) + + @os_helper.skip_unless_symlink + def test_move_symlink_to_dir_into_symlink_to_dir(self): + dst = os.path.join(self.src_dir, 'otherlinktodir') + os.symlink(self.dst_dir, dst, target_is_directory=True) + self._test_move_symlink_to_dir_into_dir(dst) @os_helper.skip_unless_dac_override @unittest.skipUnless(hasattr(os, 'lchflags') @@ -3153,6 +3182,7 @@ def test_bad_environ(self): self.assertGreaterEqual(size.lines, 0) @unittest.skipUnless(os.isatty(sys.__stdout__.fileno()), "not on tty") + @support.requires_subprocess() @unittest.skipUnless(hasattr(os, 'get_terminal_size'), 'need os.get_terminal_size()') def test_stty_match(self): diff --git a/Lib/test/test_signal.py b/Lib/test/test_signal.py index acb7e9d4c6074d..61fb047caf6dab 100644 --- a/Lib/test/test_signal.py +++ b/Lib/test/test_signal.py @@ -1,5 +1,6 @@ import enum import errno +import functools import inspect import os import random @@ -12,9 +13,10 @@ import time import unittest from test import support -from test.support import os_helper +from test.support import ( + is_apple, is_apple_mobile, os_helper, threading_helper +) from test.support.script_helper import assert_python_ok, spawn_python -from test.support import threading_helper try: import _testcapi except ImportError: @@ -76,6 +78,9 @@ class PosixTests(unittest.TestCase): def trivial_signal_handler(self, *args): pass + def create_handler_with_partial(self, argument): + return functools.partial(self.trivial_signal_handler, argument) + def test_out_of_range_signal_number_raises_error(self): self.assertRaises(ValueError, signal.getsignal, 4242) @@ -96,6 +101,28 @@ def test_getsignal(self): signal.signal(signal.SIGHUP, hup) self.assertEqual(signal.getsignal(signal.SIGHUP), hup) + def test_no_repr_is_called_on_signal_handler(self): + # See https://github.com/python/cpython/issues/112559. + + class MyArgument: + def __init__(self): + self.repr_count = 0 + + def __repr__(self): + self.repr_count += 1 + return super().__repr__() + + argument = MyArgument() + self.assertEqual(0, argument.repr_count) + + handler = self.create_handler_with_partial(argument) + hup = signal.signal(signal.SIGHUP, handler) + self.assertIsInstance(hup, signal.Handlers) + self.assertEqual(signal.getsignal(signal.SIGHUP), handler) + signal.signal(signal.SIGHUP, hup) + self.assertEqual(signal.getsignal(signal.SIGHUP), hup) + self.assertEqual(0, argument.repr_count) + def test_strsignal(self): self.assertIn("Interrupt", signal.strsignal(signal.SIGINT)) self.assertIn("Terminated", signal.strsignal(signal.SIGTERM)) @@ -806,7 +833,7 @@ def test_itimer_real(self): self.assertEqual(self.hndl_called, True) # Issue 3864, unknown if this affects earlier versions of freebsd also - @unittest.skipIf(sys.platform in ('netbsd5',), + @unittest.skipIf(sys.platform in ('netbsd5',) or is_apple_mobile, 'itimer not reliable (does not mix well with threading) on some BSDs.') def test_itimer_virtual(self): self.itimer = signal.ITIMER_VIRTUAL @@ -1318,7 +1345,7 @@ def handler(signum, frame): # Python handler self.assertEqual(len(sigs), N, "Some signals were lost") - @unittest.skipIf(sys.platform == "darwin", "crashes due to system bug (FB13453490)") + @unittest.skipIf(is_apple, "crashes due to system bug (FB13453490)") @unittest.skipUnless(hasattr(signal, "SIGUSR1"), "test needs SIGUSR1") @threading_helper.requires_working_threading() diff --git a/Lib/test/test_site.py b/Lib/test/test_site.py index 33d0975bda8eaa..0502181854f52b 100644 --- a/Lib/test/test_site.py +++ b/Lib/test/test_site.py @@ -7,6 +7,7 @@ import unittest import test.support from test import support +from test.support.script_helper import assert_python_ok from test.support import os_helper from test.support import socket_helper from test.support import captured_stderr @@ -18,6 +19,7 @@ import os import re import shutil +import stat import subprocess import sys import sysconfig @@ -194,6 +196,45 @@ def test_addsitedir(self): finally: pth_file.cleanup() + def test_addsitedir_dotfile(self): + pth_file = PthFile('.dotfile') + pth_file.cleanup(prep=True) + try: + pth_file.create() + site.addsitedir(pth_file.base_dir, set()) + self.assertNotIn(site.makepath(pth_file.good_dir_path)[0], sys.path) + self.assertIn(pth_file.base_dir, sys.path) + finally: + pth_file.cleanup() + + @unittest.skipUnless(hasattr(os, 'chflags'), 'test needs os.chflags()') + def test_addsitedir_hidden_flags(self): + pth_file = PthFile() + pth_file.cleanup(prep=True) + try: + pth_file.create() + st = os.stat(pth_file.file_path) + os.chflags(pth_file.file_path, st.st_flags | stat.UF_HIDDEN) + site.addsitedir(pth_file.base_dir, set()) + self.assertNotIn(site.makepath(pth_file.good_dir_path)[0], sys.path) + self.assertIn(pth_file.base_dir, sys.path) + finally: + pth_file.cleanup() + + @unittest.skipUnless(sys.platform == 'win32', 'test needs Windows') + @support.requires_subprocess() + def test_addsitedir_hidden_file_attribute(self): + pth_file = PthFile() + pth_file.cleanup(prep=True) + try: + pth_file.create() + subprocess.check_call(['attrib', '+H', pth_file.file_path]) + site.addsitedir(pth_file.base_dir, set()) + self.assertNotIn(site.makepath(pth_file.good_dir_path)[0], sys.path) + self.assertIn(pth_file.base_dir, sys.path) + finally: + pth_file.cleanup() + # This tests _getuserbase, hence the double underline # to distinguish from a test for getuserbase def test__getuserbase(self): @@ -338,6 +379,19 @@ def test_no_home_directory(self): mock_addsitedir.assert_not_called() self.assertFalse(known_paths) + def test_gethistoryfile(self): + filename = 'file' + rc, out, err = assert_python_ok('-c', + f'import site; assert site.gethistoryfile() == "{filename}"', + PYTHON_HISTORY=filename) + self.assertEqual(rc, 0) + + # Check that PYTHON_HISTORY is ignored in isolated mode. + rc, out, err = assert_python_ok('-I', '-c', + f'import site; assert site.gethistoryfile() != "{filename}"', + PYTHON_HISTORY=filename) + self.assertEqual(rc, 0) + def test_trace(self): message = "bla-bla-bla" for verbose, out in (True, message + "\n"), (False, ""): @@ -641,10 +695,24 @@ def _calc_sys_path_for_underpth_nosite(self, sys_prefix, lines): sys_path.append(abs_path) return sys_path + def _get_pth_lines(self, libpath: str, *, import_site: bool): + pth_lines = ['fake-path-name'] + # include 200 lines of `libpath` in _pth lines (or fewer + # if the `libpath` is long enough to get close to 32KB + # see https://github.com/python/cpython/issues/113628) + encoded_libpath_length = len(libpath.encode("utf-8")) + repetitions = min(200, 30000 // encoded_libpath_length) + if repetitions <= 2: + self.skipTest( + f"Python stdlib path is too long ({encoded_libpath_length:,} bytes)") + pth_lines.extend(libpath for _ in range(repetitions)) + pth_lines.extend(['', '# comment']) + if import_site: + pth_lines.append('import site') + return pth_lines + @support.requires_subprocess() def test_underpth_basic(self): - libpath = test.support.STDLIB_DIR - exe_prefix = os.path.dirname(sys.executable) pth_lines = ['#.', '# ..', *sys.path, '.', '..'] exe_file = self._create_underpth_exe(pth_lines) sys_path = self._calc_sys_path_for_underpth_nosite( @@ -666,12 +734,7 @@ def test_underpth_basic(self): def test_underpth_nosite_file(self): libpath = test.support.STDLIB_DIR exe_prefix = os.path.dirname(sys.executable) - pth_lines = [ - 'fake-path-name', - *[libpath for _ in range(200)], - '', - '# comment', - ] + pth_lines = self._get_pth_lines(libpath, import_site=False) exe_file = self._create_underpth_exe(pth_lines) sys_path = self._calc_sys_path_for_underpth_nosite( os.path.dirname(exe_file), @@ -695,13 +758,8 @@ def test_underpth_nosite_file(self): def test_underpth_file(self): libpath = test.support.STDLIB_DIR exe_prefix = os.path.dirname(sys.executable) - exe_file = self._create_underpth_exe([ - 'fake-path-name', - *[libpath for _ in range(200)], - '', - '# comment', - 'import site' - ]) + exe_file = self._create_underpth_exe( + self._get_pth_lines(libpath, import_site=True)) sys_prefix = os.path.dirname(exe_file) env = os.environ.copy() env['PYTHONPATH'] = 'from-env' @@ -720,13 +778,8 @@ def test_underpth_file(self): def test_underpth_dll_file(self): libpath = test.support.STDLIB_DIR exe_prefix = os.path.dirname(sys.executable) - exe_file = self._create_underpth_exe([ - 'fake-path-name', - *[libpath for _ in range(200)], - '', - '# comment', - 'import site' - ], exe_pth=False) + exe_file = self._create_underpth_exe( + self._get_pth_lines(libpath, import_site=True), exe_pth=False) sys_prefix = os.path.dirname(exe_file) env = os.environ.copy() env['PYTHONPATH'] = 'from-env' diff --git a/Lib/test/test_smtplib.py b/Lib/test/test_smtplib.py index f2e02dab1c3ca5..4c9fc14bd43f54 100644 --- a/Lib/test/test_smtplib.py +++ b/Lib/test/test_smtplib.py @@ -22,10 +22,9 @@ from test.support import socket_helper from test.support import threading_helper from test.support import asyncore +from test.support import smtpd from unittest.mock import Mock -from . import smtpd - support.requires_working_socket(module=True) diff --git a/Lib/test/test_socket.py b/Lib/test/test_socket.py index 4eb5af99d6674c..17964234992062 100644 --- a/Lib/test/test_socket.py +++ b/Lib/test/test_socket.py @@ -1,8 +1,8 @@ import unittest from test import support -from test.support import os_helper -from test.support import socket_helper -from test.support import threading_helper +from test.support import ( + is_apple, os_helper, refleak_helper, socket_helper, threading_helper +) import _thread as thread import array @@ -52,6 +52,35 @@ except ImportError: _socket = None +def skipForRefleakHuntinIf(condition, issueref): + if not condition: + def decorator(f): + f.client_skip = lambda f: f + return f + + else: + def decorator(f): + @contextlib.wraps(f) + def wrapper(*args, **kwds): + if refleak_helper.hunting_for_refleaks(): + raise unittest.SkipTest(f"ignore while hunting for refleaks, see {issueref}") + + return f(*args, **kwds) + + def client_skip(f): + @contextlib.wraps(f) + def wrapper(*args, **kwds): + if refleak_helper.hunting_for_refleaks(): + return + + return f(*args, **kwds) + + return wrapper + wrapper.client_skip = client_skip + return wrapper + + return decorator + def get_cid(): if fcntl is None: return None @@ -1166,8 +1195,11 @@ def testGetServBy(self): # Find one service that exists, then check all the related interfaces. # I've ordered this by protocols that have both a tcp and udp # protocol, at least for modern Linuxes. - if (sys.platform.startswith(('freebsd', 'netbsd', 'gnukfreebsd')) - or sys.platform in ('linux', 'darwin')): + if ( + sys.platform.startswith(('freebsd', 'netbsd', 'gnukfreebsd')) + or sys.platform == 'linux' + or is_apple + ): # avoid the 'echo' service on this platform, as there is an # assumption breaking non-standard port/protocol entry services = ('daytime', 'qotd', 'domain') @@ -3678,7 +3710,7 @@ def testFDPassCMSG_LEN(self): def _testFDPassCMSG_LEN(self): self.createAndSendFDs(1) - @unittest.skipIf(sys.platform == "darwin", "skipping, see issue #12958") + @unittest.skipIf(is_apple, "skipping, see issue #12958") @unittest.skipIf(AIX, "skipping, see issue #22397") @requireAttrs(socket, "CMSG_SPACE") def testFDPassSeparate(self): @@ -3689,7 +3721,7 @@ def testFDPassSeparate(self): maxcmsgs=2) @testFDPassSeparate.client_skip - @unittest.skipIf(sys.platform == "darwin", "skipping, see issue #12958") + @unittest.skipIf(is_apple, "skipping, see issue #12958") @unittest.skipIf(AIX, "skipping, see issue #22397") def _testFDPassSeparate(self): fd0, fd1 = self.newFDs(2) @@ -3702,7 +3734,7 @@ def _testFDPassSeparate(self): array.array("i", [fd1]))]), len(MSG)) - @unittest.skipIf(sys.platform == "darwin", "skipping, see issue #12958") + @unittest.skipIf(is_apple, "skipping, see issue #12958") @unittest.skipIf(AIX, "skipping, see issue #22397") @requireAttrs(socket, "CMSG_SPACE") def testFDPassSeparateMinSpace(self): @@ -3716,7 +3748,7 @@ def testFDPassSeparateMinSpace(self): maxcmsgs=2, ignoreflags=socket.MSG_CTRUNC) @testFDPassSeparateMinSpace.client_skip - @unittest.skipIf(sys.platform == "darwin", "skipping, see issue #12958") + @unittest.skipIf(is_apple, "skipping, see issue #12958") @unittest.skipIf(AIX, "skipping, see issue #22397") def _testFDPassSeparateMinSpace(self): fd0, fd1 = self.newFDs(2) @@ -3740,7 +3772,7 @@ def sendAncillaryIfPossible(self, msg, ancdata): nbytes = self.sendmsgToServer([msg]) self.assertEqual(nbytes, len(msg)) - @unittest.skipIf(sys.platform == "darwin", "see issue #24725") + @unittest.skipIf(is_apple, "skipping, see issue #12958") def testFDPassEmpty(self): # Try to pass an empty FD array. Can receive either no array # or an empty array. @@ -3814,6 +3846,7 @@ def checkTruncatedHeader(self, result, ignoreflags=0): self.checkFlags(flags, eor=True, checkset=socket.MSG_CTRUNC, ignore=ignoreflags) + @skipForRefleakHuntinIf(sys.platform == "darwin", "#80931") def testCmsgTruncNoBufSize(self): # Check that no ancillary data is received when no buffer size # is specified. @@ -3823,26 +3856,32 @@ def testCmsgTruncNoBufSize(self): # received. ignoreflags=socket.MSG_CTRUNC) + @testCmsgTruncNoBufSize.client_skip def _testCmsgTruncNoBufSize(self): self.createAndSendFDs(1) + @skipForRefleakHuntinIf(sys.platform == "darwin", "#80931") def testCmsgTrunc0(self): # Check that no ancillary data is received when buffer size is 0. self.checkTruncatedHeader(self.doRecvmsg(self.serv_sock, len(MSG), 0), ignoreflags=socket.MSG_CTRUNC) + @testCmsgTrunc0.client_skip def _testCmsgTrunc0(self): self.createAndSendFDs(1) # Check that no ancillary data is returned for various non-zero # (but still too small) buffer sizes. + @skipForRefleakHuntinIf(sys.platform == "darwin", "#80931") def testCmsgTrunc1(self): self.checkTruncatedHeader(self.doRecvmsg(self.serv_sock, len(MSG), 1)) + @testCmsgTrunc1.client_skip def _testCmsgTrunc1(self): self.createAndSendFDs(1) + @skipForRefleakHuntinIf(sys.platform == "darwin", "#80931") def testCmsgTrunc2Int(self): # The cmsghdr structure has at least three members, two of # which are ints, so we still shouldn't see any ancillary @@ -3850,13 +3889,16 @@ def testCmsgTrunc2Int(self): self.checkTruncatedHeader(self.doRecvmsg(self.serv_sock, len(MSG), SIZEOF_INT * 2)) + @testCmsgTrunc2Int.client_skip def _testCmsgTrunc2Int(self): self.createAndSendFDs(1) + @skipForRefleakHuntinIf(sys.platform == "darwin", "#80931") def testCmsgTruncLen0Minus1(self): self.checkTruncatedHeader(self.doRecvmsg(self.serv_sock, len(MSG), socket.CMSG_LEN(0) - 1)) + @testCmsgTruncLen0Minus1.client_skip def _testCmsgTruncLen0Minus1(self): self.createAndSendFDs(1) @@ -3887,29 +3929,38 @@ def checkTruncatedArray(self, ancbuf, maxdata, mindata=0): len(cmsg_data) - (len(cmsg_data) % fds.itemsize)]) self.checkFDs(fds) + @skipForRefleakHuntinIf(sys.platform == "darwin", "#80931") def testCmsgTruncLen0(self): self.checkTruncatedArray(ancbuf=socket.CMSG_LEN(0), maxdata=0) + @testCmsgTruncLen0.client_skip def _testCmsgTruncLen0(self): self.createAndSendFDs(1) + @skipForRefleakHuntinIf(sys.platform == "darwin", "#80931") def testCmsgTruncLen0Plus1(self): self.checkTruncatedArray(ancbuf=socket.CMSG_LEN(0) + 1, maxdata=1) + @testCmsgTruncLen0Plus1.client_skip def _testCmsgTruncLen0Plus1(self): self.createAndSendFDs(2) + @skipForRefleakHuntinIf(sys.platform == "darwin", "#80931") def testCmsgTruncLen1(self): self.checkTruncatedArray(ancbuf=socket.CMSG_LEN(SIZEOF_INT), maxdata=SIZEOF_INT) + @testCmsgTruncLen1.client_skip def _testCmsgTruncLen1(self): self.createAndSendFDs(2) + + @skipForRefleakHuntinIf(sys.platform == "darwin", "#80931") def testCmsgTruncLen2Minus1(self): self.checkTruncatedArray(ancbuf=socket.CMSG_LEN(2 * SIZEOF_INT) - 1, maxdata=(2 * SIZEOF_INT) - 1) + @testCmsgTruncLen2Minus1.client_skip def _testCmsgTruncLen2Minus1(self): self.createAndSendFDs(2) diff --git a/Lib/test/test_sqlite3/test_dbapi.py b/Lib/test/test_sqlite3/test_dbapi.py index f3efe0f52f4fd7..588272448bbfda 100644 --- a/Lib/test/test_sqlite3/test_dbapi.py +++ b/Lib/test/test_sqlite3/test_dbapi.py @@ -31,7 +31,7 @@ from test.support import ( SHORT_TIMEOUT, check_disallow_instantiation, requires_subprocess, - is_emscripten, is_wasi + is_apple, is_emscripten, is_wasi ) from test.support import gc_collect from test.support import threading_helper @@ -667,7 +667,7 @@ def test_open_with_path_like_object(self): cx.execute(self._sql) @unittest.skipIf(sys.platform == "win32", "skipped on Windows") - @unittest.skipIf(sys.platform == "darwin", "skipped on macOS") + @unittest.skipIf(is_apple, "skipped on Apple platforms") @unittest.skipIf(is_emscripten or is_wasi, "not supported on Emscripten/WASI") @unittest.skipUnless(TESTFN_UNDECODABLE, "only works if there are undecodable paths") def test_open_with_undecodable_path(self): @@ -713,7 +713,7 @@ def test_open_uri_readonly(self): cx.execute(self._sql) @unittest.skipIf(sys.platform == "win32", "skipped on Windows") - @unittest.skipIf(sys.platform == "darwin", "skipped on macOS") + @unittest.skipIf(is_apple, "skipped on Apple platforms") @unittest.skipIf(is_emscripten or is_wasi, "not supported on Emscripten/WASI") @unittest.skipUnless(TESTFN_UNDECODABLE, "only works if there are undecodable paths") def test_open_undecodable_uri(self): diff --git a/Lib/test/test_sqlite3/test_dump.py b/Lib/test/test_sqlite3/test_dump.py index 14a18c1ad37102..7261b7f0dc93d0 100644 --- a/Lib/test/test_sqlite3/test_dump.py +++ b/Lib/test/test_sqlite3/test_dump.py @@ -20,7 +20,8 @@ def test_table_dump(self): , "CREATE TABLE t1(id integer primary key, s1 text, " \ "t1_i1 integer not null, i2 integer, unique (s1), " \ - "constraint t1_idx1 unique (i2));" + "constraint t1_idx1 unique (i2), " \ + "constraint t1_i1_idx1 unique (t1_i1));" , "INSERT INTO \"t1\" VALUES(1,'foo',10,20);" , @@ -30,6 +31,9 @@ def test_table_dump(self): "t2_i2 integer, primary key (id)," \ "foreign key(t2_i1) references t1(t1_i1));" , + # Foreign key violation. + "INSERT INTO \"t2\" VALUES(1,2,3);" + , "CREATE TRIGGER trigger_1 update of t1_i1 on t1 " \ "begin " \ "update t2 set t2_i1 = new.t1_i1 where t2_i1 = old.t1_i1; " \ @@ -41,11 +45,85 @@ def test_table_dump(self): [self.cu.execute(s) for s in expected_sqls] i = self.cx.iterdump() actual_sqls = [s for s in i] - expected_sqls = ['BEGIN TRANSACTION;'] + expected_sqls + \ - ['COMMIT;'] + expected_sqls = [ + "PRAGMA foreign_keys=OFF;", + "BEGIN TRANSACTION;", + *expected_sqls, + "COMMIT;", + ] [self.assertEqual(expected_sqls[i], actual_sqls[i]) for i in range(len(expected_sqls))] + def test_table_dump_filter(self): + all_table_sqls = [ + """CREATE TABLE "some_table_2" ("id_1" INTEGER);""", + """INSERT INTO "some_table_2" VALUES(3);""", + """INSERT INTO "some_table_2" VALUES(4);""", + """CREATE TABLE "test_table_1" ("id_2" INTEGER);""", + """INSERT INTO "test_table_1" VALUES(1);""", + """INSERT INTO "test_table_1" VALUES(2);""", + ] + all_views_sqls = [ + """CREATE VIEW "view_1" AS SELECT * FROM "some_table_2";""", + """CREATE VIEW "view_2" AS SELECT * FROM "test_table_1";""", + ] + # Create database structure. + for sql in [*all_table_sqls, *all_views_sqls]: + self.cu.execute(sql) + # %_table_% matches all tables. + dump_sqls = list(self.cx.iterdump(filter="%_table_%")) + self.assertEqual( + dump_sqls, + ["BEGIN TRANSACTION;", *all_table_sqls, "COMMIT;"], + ) + # view_% matches all views. + dump_sqls = list(self.cx.iterdump(filter="view_%")) + self.assertEqual( + dump_sqls, + ["BEGIN TRANSACTION;", *all_views_sqls, "COMMIT;"], + ) + # %_1 matches tables and views with the _1 suffix. + dump_sqls = list(self.cx.iterdump(filter="%_1")) + self.assertEqual( + dump_sqls, + [ + "BEGIN TRANSACTION;", + """CREATE TABLE "test_table_1" ("id_2" INTEGER);""", + """INSERT INTO "test_table_1" VALUES(1);""", + """INSERT INTO "test_table_1" VALUES(2);""", + """CREATE VIEW "view_1" AS SELECT * FROM "some_table_2";""", + "COMMIT;" + ], + ) + # some_% matches some_table_2. + dump_sqls = list(self.cx.iterdump(filter="some_%")) + self.assertEqual( + dump_sqls, + [ + "BEGIN TRANSACTION;", + """CREATE TABLE "some_table_2" ("id_1" INTEGER);""", + """INSERT INTO "some_table_2" VALUES(3);""", + """INSERT INTO "some_table_2" VALUES(4);""", + "COMMIT;" + ], + ) + # Only single object. + dump_sqls = list(self.cx.iterdump(filter="view_2")) + self.assertEqual( + dump_sqls, + [ + "BEGIN TRANSACTION;", + """CREATE VIEW "view_2" AS SELECT * FROM "test_table_1";""", + "COMMIT;" + ], + ) + # % matches all objects. + dump_sqls = list(self.cx.iterdump(filter="%")) + self.assertEqual( + dump_sqls, + ["BEGIN TRANSACTION;", *all_table_sqls, *all_views_sqls, "COMMIT;"], + ) + def test_dump_autoincrement(self): expected = [ 'CREATE TABLE "t1" (id integer primary key autoincrement);', diff --git a/Lib/test/test_ssl.py b/Lib/test/test_ssl.py index 3fdfa2960503b8..1b18230d83577d 100644 --- a/Lib/test/test_ssl.py +++ b/Lib/test/test_ssl.py @@ -2206,14 +2206,15 @@ def _test_get_server_certificate(test, host, port, cert=None): sys.stdout.write("\nVerified certificate for %s:%s is\n%s\n" % (host, port ,pem)) def _test_get_server_certificate_fail(test, host, port): - try: - pem = ssl.get_server_certificate((host, port), ca_certs=CERTFILE) - except ssl.SSLError as x: - #should fail - if support.verbose: - sys.stdout.write("%s\n" % x) - else: - test.fail("Got server certificate %s for %s:%s!" % (pem, host, port)) + with warnings_helper.check_no_resource_warning(test): + try: + pem = ssl.get_server_certificate((host, port), ca_certs=CERTFILE) + except ssl.SSLError as x: + #should fail + if support.verbose: + sys.stdout.write("%s\n" % x) + else: + test.fail("Got server certificate %s for %s:%s!" % (pem, host, port)) from test.ssl_servers import make_https_server @@ -3026,6 +3027,16 @@ def test_check_hostname_idn(self): server_hostname="python.example.org") as s: with self.assertRaises(ssl.CertificateError): s.connect((HOST, server.port)) + with ThreadedEchoServer(context=server_context, chatty=True) as server: + with warnings_helper.check_no_resource_warning(self): + with self.assertRaises(UnicodeError): + context.wrap_socket(socket.socket(), + server_hostname='.pythontest.net') + with ThreadedEchoServer(context=server_context, chatty=True) as server: + with warnings_helper.check_no_resource_warning(self): + with self.assertRaises(UnicodeDecodeError): + context.wrap_socket(socket.socket(), + server_hostname=b'k\xf6nig.idn.pythontest.net') def test_wrong_cert_tls12(self): """Connecting when the server rejects the client's certificate @@ -4983,7 +4994,8 @@ def call_after_accept(conn_to_client): self.assertIsNone(wrap_error.library, msg="attr must exist") finally: # gh-108342: Explicitly break the reference cycle - wrap_error = None + with warnings_helper.check_no_resource_warning(self): + wrap_error = None server = None def test_https_client_non_tls_response_ignored(self): @@ -5032,7 +5044,8 @@ def call_after_accept(conn_to_client): # socket; that fails if the connection is broken. It may seem pointless # to test this. It serves as an illustration of something that we never # want to happen... properly not happening. - with self.assertRaises(OSError): + with warnings_helper.check_no_resource_warning(self), \ + self.assertRaises(OSError): connection.request("HEAD", "/test", headers={"Host": "localhost"}) response = connection.getresponse() diff --git a/Lib/test/test_stable_abi_ctypes.py b/Lib/test/test_stable_abi_ctypes.py index 4976ac3642bbe4..8bd373976426ef 100644 --- a/Lib/test/test_stable_abi_ctypes.py +++ b/Lib/test/test_stable_abi_ctypes.py @@ -9,6 +9,13 @@ from _testcapi import get_feature_macros feature_macros = get_feature_macros() + +# Stable ABI is incompatible with Py_TRACE_REFS builds due to PyObject +# layout differences. +# See https://github.com/python/cpython/issues/88299#issuecomment-1113366226 +if feature_macros['Py_TRACE_REFS']: + raise unittest.SkipTest("incompatible with Py_TRACE_REFS.") + ctypes_test = import_module('ctypes') class TestStableABIAvailability(unittest.TestCase): @@ -254,6 +261,7 @@ def test_windows_feature_macros(self): "PyExc_IOError", "PyExc_ImportError", "PyExc_ImportWarning", + "PyExc_IncompleteInputError", "PyExc_IndentationError", "PyExc_IndexError", "PyExc_InterruptedError", @@ -364,6 +372,7 @@ def test_windows_feature_macros(self): "PyList_Append", "PyList_AsTuple", "PyList_GetItem", + "PyList_GetItemRef", "PyList_GetSlice", "PyList_Insert", "PyList_New", @@ -441,7 +450,9 @@ def test_windows_feature_macros(self): "PyModule_AddObjectRef", "PyModule_AddStringConstant", "PyModule_AddType", + "PyModule_Create2", "PyModule_ExecDef", + "PyModule_FromDefAndSpec2", "PyModule_GetDef", "PyModule_GetDict", "PyModule_GetFilename", @@ -911,6 +922,13 @@ def test_windows_feature_macros(self): "_Py_TrueStruct", "_Py_VaBuildValue_SizeT", ) +if feature_macros['HAVE_FORK']: + SYMBOL_NAMES += ( + 'PyOS_AfterFork', + 'PyOS_AfterFork_Child', + 'PyOS_AfterFork_Parent', + 'PyOS_BeforeFork', + ) if feature_macros['MS_WINDOWS']: SYMBOL_NAMES += ( 'PyErr_SetExcFromWindowsErr', @@ -926,17 +944,6 @@ def test_windows_feature_macros(self): 'PyUnicode_DecodeMBCSStateful', 'PyUnicode_EncodeCodePage', ) -if feature_macros['HAVE_FORK']: - SYMBOL_NAMES += ( - 'PyOS_AfterFork', - 'PyOS_AfterFork_Child', - 'PyOS_AfterFork_Parent', - 'PyOS_BeforeFork', - ) -if feature_macros['USE_STACKCHECK']: - SYMBOL_NAMES += ( - 'PyOS_CheckStack', - ) if feature_macros['PY_HAVE_THREAD_NATIVE_ID']: SYMBOL_NAMES += ( 'PyThread_get_thread_native_id', @@ -946,14 +953,23 @@ def test_windows_feature_macros(self): '_Py_NegativeRefcount', '_Py_RefTotal', ) +if feature_macros['Py_TRACE_REFS']: + SYMBOL_NAMES += ( + ) +if feature_macros['USE_STACKCHECK']: + SYMBOL_NAMES += ( + 'PyOS_CheckStack', + ) EXPECTED_FEATURE_MACROS = set(['HAVE_FORK', 'MS_WINDOWS', 'PY_HAVE_THREAD_NATIVE_ID', 'Py_REF_DEBUG', + 'Py_TRACE_REFS', 'USE_STACKCHECK']) WINDOWS_FEATURE_MACROS = {'HAVE_FORK': False, 'MS_WINDOWS': True, 'PY_HAVE_THREAD_NATIVE_ID': True, 'Py_REF_DEBUG': 'maybe', + 'Py_TRACE_REFS': 'maybe', 'USE_STACKCHECK': 'maybe'} diff --git a/Lib/test/test_stat.py b/Lib/test/test_stat.py index a0d0f61e5a192c..49013a4bcd8af6 100644 --- a/Lib/test/test_stat.py +++ b/Lib/test/test_stat.py @@ -2,8 +2,7 @@ import os import socket import sys -from test.support import os_helper -from test.support import socket_helper +from test.support import is_apple, os_helper, socket_helper from test.support.import_helper import import_fresh_module from test.support.os_helper import TESTFN @@ -15,8 +14,10 @@ class TestFilemode: statmod = None file_flags = {'SF_APPEND', 'SF_ARCHIVED', 'SF_IMMUTABLE', 'SF_NOUNLINK', - 'SF_SNAPSHOT', 'UF_APPEND', 'UF_COMPRESSED', 'UF_HIDDEN', - 'UF_IMMUTABLE', 'UF_NODUMP', 'UF_NOUNLINK', 'UF_OPAQUE'} + 'SF_SNAPSHOT', 'SF_SETTABLE', 'SF_RESTRICTED', 'SF_FIRMLINK', + 'SF_DATALESS', 'UF_APPEND', 'UF_COMPRESSED', 'UF_HIDDEN', + 'UF_IMMUTABLE', 'UF_NODUMP', 'UF_NOUNLINK', 'UF_OPAQUE', + 'UF_SETTABLE', 'UF_TRACKED', 'UF_DATAVAULT'} formats = {'S_IFBLK', 'S_IFCHR', 'S_IFDIR', 'S_IFIFO', 'S_IFLNK', 'S_IFREG', 'S_IFSOCK', 'S_IFDOOR', 'S_IFPORT', 'S_IFWHT'} @@ -239,6 +240,18 @@ def test_module_attributes(self): self.assertTrue(callable(func)) self.assertEqual(func(0), 0) + def test_flags_consistent(self): + self.assertFalse(self.statmod.UF_SETTABLE & self.statmod.SF_SETTABLE) + + for flag in self.file_flags: + if flag.startswith("UF"): + self.assertTrue(getattr(self.statmod, flag) & self.statmod.UF_SETTABLE, f"{flag} not in UF_SETTABLE") + elif is_apple and self.statmod is c_stat and flag == 'SF_DATALESS': + self.assertTrue(self.statmod.SF_DATALESS & self.statmod.SF_SYNTHETIC, "SF_DATALESS not in SF_SYNTHETIC") + self.assertFalse(self.statmod.SF_DATALESS & self.statmod.SF_SETTABLE, "SF_DATALESS in SF_SETTABLE") + else: + self.assertTrue(getattr(self.statmod, flag) & self.statmod.SF_SETTABLE, f"{flag} notin SF_SETTABLE") + @unittest.skipUnless(sys.platform == "win32", "FILE_ATTRIBUTE_* constants are Win32 specific") def test_file_attribute_constants(self): @@ -247,6 +260,66 @@ def test_file_attribute_constants(self): modvalue = getattr(self.statmod, key) self.assertEqual(value, modvalue, key) + @unittest.skipUnless(sys.platform == "darwin", "macOS system check") + def test_macosx_attribute_values(self): + self.assertEqual(self.statmod.UF_SETTABLE, 0x0000ffff) + self.assertEqual(self.statmod.UF_NODUMP, 0x00000001) + self.assertEqual(self.statmod.UF_IMMUTABLE, 0x00000002) + self.assertEqual(self.statmod.UF_APPEND, 0x00000004) + self.assertEqual(self.statmod.UF_OPAQUE, 0x00000008) + self.assertEqual(self.statmod.UF_COMPRESSED, 0x00000020) + self.assertEqual(self.statmod.UF_TRACKED, 0x00000040) + self.assertEqual(self.statmod.UF_DATAVAULT, 0x00000080) + self.assertEqual(self.statmod.UF_HIDDEN, 0x00008000) + + if self.statmod is c_stat: + self.assertEqual(self.statmod.SF_SUPPORTED, 0x009f0000) + self.assertEqual(self.statmod.SF_SETTABLE, 0x3fff0000) + self.assertEqual(self.statmod.SF_SYNTHETIC, 0xc0000000) + else: + self.assertEqual(self.statmod.SF_SETTABLE, 0xffff0000) + self.assertEqual(self.statmod.SF_ARCHIVED, 0x00010000) + self.assertEqual(self.statmod.SF_IMMUTABLE, 0x00020000) + self.assertEqual(self.statmod.SF_APPEND, 0x00040000) + self.assertEqual(self.statmod.SF_RESTRICTED, 0x00080000) + self.assertEqual(self.statmod.SF_NOUNLINK, 0x00100000) + self.assertEqual(self.statmod.SF_FIRMLINK, 0x00800000) + self.assertEqual(self.statmod.SF_DATALESS, 0x40000000) + + self.assertFalse(isinstance(self.statmod.S_IFMT, int)) + self.assertEqual(self.statmod.S_IFIFO, 0o010000) + self.assertEqual(self.statmod.S_IFCHR, 0o020000) + self.assertEqual(self.statmod.S_IFDIR, 0o040000) + self.assertEqual(self.statmod.S_IFBLK, 0o060000) + self.assertEqual(self.statmod.S_IFREG, 0o100000) + self.assertEqual(self.statmod.S_IFLNK, 0o120000) + self.assertEqual(self.statmod.S_IFSOCK, 0o140000) + + if self.statmod is c_stat: + self.assertEqual(self.statmod.S_IFWHT, 0o160000) + + self.assertEqual(self.statmod.S_IRWXU, 0o000700) + self.assertEqual(self.statmod.S_IRUSR, 0o000400) + self.assertEqual(self.statmod.S_IWUSR, 0o000200) + self.assertEqual(self.statmod.S_IXUSR, 0o000100) + self.assertEqual(self.statmod.S_IRWXG, 0o000070) + self.assertEqual(self.statmod.S_IRGRP, 0o000040) + self.assertEqual(self.statmod.S_IWGRP, 0o000020) + self.assertEqual(self.statmod.S_IXGRP, 0o000010) + self.assertEqual(self.statmod.S_IRWXO, 0o000007) + self.assertEqual(self.statmod.S_IROTH, 0o000004) + self.assertEqual(self.statmod.S_IWOTH, 0o000002) + self.assertEqual(self.statmod.S_IXOTH, 0o000001) + self.assertEqual(self.statmod.S_ISUID, 0o004000) + self.assertEqual(self.statmod.S_ISGID, 0o002000) + self.assertEqual(self.statmod.S_ISVTX, 0o001000) + + self.assertFalse(hasattr(self.statmod, "S_ISTXT")) + self.assertEqual(self.statmod.S_IREAD, self.statmod.S_IRUSR) + self.assertEqual(self.statmod.S_IWRITE, self.statmod.S_IWUSR) + self.assertEqual(self.statmod.S_IEXEC, self.statmod.S_IXUSR) + + @unittest.skipIf(c_stat is None, 'need _stat extension') class TestFilemodeCStat(TestFilemode, unittest.TestCase): diff --git a/Lib/test/test_strptime.py b/Lib/test/test_strptime.py index 810c5a36e02f41..05c8afc907ad3c 100644 --- a/Lib/test/test_strptime.py +++ b/Lib/test/test_strptime.py @@ -224,35 +224,55 @@ def test_ValueError(self): else: self.fail("'%s' did not raise ValueError" % bad_format) + msg_week_no_year_or_weekday = r"ISO week directive '%V' must be used with " \ + r"the ISO year directive '%G' and a weekday directive " \ + r"\('%A', '%a', '%w', or '%u'\)." + msg_week_not_compatible = r"ISO week directive '%V' is incompatible with " \ + r"the year directive '%Y'. Use the ISO year '%G' instead." + msg_julian_not_compatible = r"Day of the year directive '%j' is not " \ + r"compatible with ISO year directive '%G'. Use '%Y' instead." + msg_year_no_week_or_weekday = r"ISO year directive '%G' must be used with " \ + r"the ISO week directive '%V' and a weekday directive " \ + r"\('%A', '%a', '%w', or '%u'\)." + + locale_time = _strptime.LocaleTime() + # Ambiguous or incomplete cases using ISO year/week/weekday directives - # 1. ISO week (%V) is specified, but the year is specified with %Y - # instead of %G - with self.assertRaises(ValueError): - _strptime._strptime("1999 50", "%Y %V") - # 2. ISO year (%G) and ISO week (%V) are specified, but weekday is not - with self.assertRaises(ValueError): - _strptime._strptime("1999 51", "%G %V") - # 3. ISO year (%G) and weekday are specified, but ISO week (%V) is not - for w in ('A', 'a', 'w', 'u'): - with self.assertRaises(ValueError): - _strptime._strptime("1999 51","%G %{}".format(w)) - # 4. ISO year is specified alone (e.g. time.strptime('2015', '%G')) - with self.assertRaises(ValueError): - _strptime._strptime("2015", "%G") - # 5. Julian/ordinal day (%j) is specified with %G, but not %Y - with self.assertRaises(ValueError): - _strptime._strptime("1999 256", "%G %j") - # 6. Invalid ISO weeks - invalid_iso_weeks = [ - "2019-00-1", - "2019-54-1", - "2021-53-1", + subtests = [ + # 1. ISO week (%V) is specified, but the year is specified with %Y + # instead of %G + ("1999 50", "%Y %V", msg_week_no_year_or_weekday), + ("1999 50 5", "%Y %V %u", msg_week_not_compatible), + # 2. ISO year (%G) and ISO week (%V) are specified, but weekday is not + ("1999 51", "%G %V", msg_year_no_week_or_weekday), + # 3. ISO year (%G) and weekday are specified, but ISO week (%V) is not + ("1999 {}".format(locale_time.f_weekday[5]), "%G %A", + msg_year_no_week_or_weekday), + ("1999 {}".format(locale_time.a_weekday[5]), "%G %a", + msg_year_no_week_or_weekday), + ("1999 5", "%G %w", msg_year_no_week_or_weekday), + ("1999 5", "%G %u", msg_year_no_week_or_weekday), + # 4. ISO year is specified alone (e.g. time.strptime('2015', '%G')) + ("2015", "%G", msg_year_no_week_or_weekday), + # 5. Julian/ordinal day (%j) is specified with %G, but not %Y + ("1999 256", "%G %j", msg_julian_not_compatible), + ("1999 50 5 256", "%G %V %u %j", msg_julian_not_compatible), + # ISO week specified alone + ("50", "%V", msg_week_no_year_or_weekday), + # ISO year is unspecified, falling back to year + ("50 5", "%V %u", msg_week_no_year_or_weekday), + # 6. Invalid ISO weeks + ("2019-00-1", "%G-%V-%u", + "time data '2019-00-1' does not match format '%G-%V-%u'"), + ("2019-54-1", "%G-%V-%u", + "time data '2019-54-1' does not match format '%G-%V-%u'"), + ("2021-53-1", "%G-%V-%u", "Invalid week: 53"), ] - for invalid_iso_dtstr in invalid_iso_weeks: - with self.subTest(invalid_iso_dtstr): - with self.assertRaises(ValueError): - _strptime._strptime(invalid_iso_dtstr, "%G-%V-%u") + for (data_string, format, message) in subtests: + with self.subTest(data_string=data_string, format=format): + with self.assertRaisesRegex(ValueError, message): + _strptime._strptime(data_string, format) def test_strptime_exception_context(self): # check that this doesn't chain exceptions needlessly (see #17572) diff --git a/Lib/test/test_subprocess.py b/Lib/test/test_subprocess.py index 6d3228bf92f8ca..c44a778d5bbefe 100644 --- a/Lib/test/test_subprocess.py +++ b/Lib/test/test_subprocess.py @@ -791,6 +791,19 @@ def test_env(self): stdout, stderr = p.communicate() self.assertEqual(stdout, b"orange") + @unittest.skipUnless(sys.platform == "win32", "Windows only issue") + def test_win32_duplicate_envs(self): + newenv = os.environ.copy() + newenv["fRUit"] = "cherry" + newenv["fruit"] = "lemon" + newenv["FRUIT"] = "orange" + newenv["frUit"] = "banana" + with subprocess.Popen(["CMD", "/c", "SET", "fruit"], + stdout=subprocess.PIPE, + env=newenv) as p: + stdout, _ = p.communicate() + self.assertEqual(stdout.strip(), b"frUit=banana") + # Windows requires at least the SYSTEMROOT environment variable to start # Python @unittest.skipIf(sys.platform == 'win32', @@ -822,6 +835,26 @@ def is_env_var_to_ignore(n): if not is_env_var_to_ignore(k)] self.assertEqual(child_env_names, []) + @unittest.skipIf(sysconfig.get_config_var('Py_ENABLE_SHARED') == 1, + 'The Python shared library cannot be loaded ' + 'without some system environments.') + @unittest.skipIf(check_sanitizer(address=True), + 'AddressSanitizer adds to the environment.') + def test_one_environment_variable(self): + newenv = {'fruit': 'orange'} + cmd = [sys.executable, '-c', + 'import sys,os;' + 'sys.stdout.write("fruit="+os.getenv("fruit"))'] + if sys.platform == "win32": + cmd = ["CMD", "/c", "SET", "fruit"] + with subprocess.Popen(cmd, stdout=subprocess.PIPE, stderr=subprocess.PIPE, env=newenv) as p: + stdout, stderr = p.communicate() + if p.returncode and support.verbose: + print("STDOUT:", stdout.decode("ascii", "replace")) + print("STDERR:", stderr.decode("ascii", "replace")) + self.assertEqual(p.returncode, 0) + self.assertEqual(stdout.strip(), b"fruit=orange") + def test_invalid_cmd(self): # null character in the command name cmd = sys.executable + '\0' @@ -862,6 +895,19 @@ def test_invalid_env(self): stdout, stderr = p.communicate() self.assertEqual(stdout, b"orange=lemon") + @unittest.skipUnless(sys.platform == "win32", "Windows only issue") + def test_win32_invalid_env(self): + # '=' in the environment variable name + newenv = os.environ.copy() + newenv["FRUIT=VEGETABLE"] = "cabbage" + with self.assertRaises(ValueError): + subprocess.Popen(ZERO_RETURN_CMD, env=newenv) + + newenv = os.environ.copy() + newenv["==FRUIT"] = "cabbage" + with self.assertRaises(ValueError): + subprocess.Popen(ZERO_RETURN_CMD, env=newenv) + def test_communicate_stdin(self): p = subprocess.Popen([sys.executable, "-c", 'import sys;' @@ -1945,9 +1991,9 @@ def test_process_group_0(self): @unittest.skipUnless(hasattr(os, 'setreuid'), 'no setreuid on platform') def test_user(self): - # For code coverage of the user parameter. We don't care if we get an - # EPERM error from it depending on the test execution environment, that - # still indicates that it was called. + # For code coverage of the user parameter. We don't care if we get a + # permission error from it depending on the test execution environment, + # that still indicates that it was called. uid = os.geteuid() test_users = [65534 if uid != 65534 else 65533, uid] @@ -1971,11 +2017,11 @@ def test_user(self): "import os; print(os.getuid())"], user=user, close_fds=close_fds) - except PermissionError: # (EACCES, EPERM) - pass - except OSError as e: - if e.errno not in (errno.EACCES, errno.EPERM): - raise + except PermissionError as e: # (EACCES, EPERM) + if e.errno == errno.EACCES: + self.assertEqual(e.filename, sys.executable) + else: + self.assertIsNone(e.filename) else: if isinstance(user, str): user_uid = pwd.getpwnam(user).pw_uid @@ -2019,8 +2065,8 @@ def test_group(self): "import os; print(os.getgid())"], group=group, close_fds=close_fds) - except PermissionError: # (EACCES, EPERM) - pass + except PermissionError as e: # (EACCES, EPERM) + self.assertIsNone(e.filename) else: if isinstance(group, str): group_gid = grp.getgrnam(group).gr_gid @@ -2068,7 +2114,8 @@ def _test_extra_groups_impl(self, *, gid, group_list): [sys.executable, "-c", "import os, sys, json; json.dump(os.getgroups(), sys.stdout)"], extra_groups=group_list) - except PermissionError: + except PermissionError as e: + self.assertIsNone(e.filename) self.skipTest("setgroup() EPERM; this test may require root.") else: parent_groups = os.getgroups() diff --git a/Lib/test/test_super.py b/Lib/test/test_super.py index 43162c540b55ae..256b416caaa584 100644 --- a/Lib/test/test_super.py +++ b/Lib/test/test_super.py @@ -1,8 +1,9 @@ """Unit tests for zero-argument super() & related machinery.""" +import textwrap import unittest from unittest.mock import patch -from test import shadowed_super +from test.support import import_helper ADAPTIVE_WARMUP_DELAY = 2 @@ -342,7 +343,20 @@ def test_super_argtype(self): super(1, int) def test_shadowed_global(self): + source = textwrap.dedent( + """ + class super: + msg = "truly super" + + class C: + def method(self): + return super().msg + """, + ) + with import_helper.ready_to_import(name="shadowed_super", source=source): + import shadowed_super self.assertEqual(shadowed_super.C().method(), "truly super") + import_helper.unload("shadowed_super") def test_shadowed_local(self): class super: @@ -396,6 +410,33 @@ def method(self): with self.assertRaisesRegex(TypeError, "argument 1 must be a type"): C().method() + def test_supercheck_fail(self): + class C: + def method(self, type_, obj): + return super(type_, obj).method() + + c = C() + err_msg = ( + r"super\(type, obj\): obj \({} {}\) is not " + r"an instance or subtype of type \({}\)." + ) + + cases = ( + (int, c, int.__name__, C.__name__, "instance of"), + # obj is instance of type + (C, list(), C.__name__, list.__name__, "instance of"), + # obj is type itself + (C, list, C.__name__, list.__name__, "type"), + ) + + for case in cases: + with self.subTest(case=case): + type_, obj, type_str, obj_str, instance_or_type = case + regex = err_msg.format(instance_or_type, obj_str, type_str) + + with self.assertRaisesRegex(TypeError, regex): + c.method(type_, obj) + def test_super___class__(self): class C: def method(self): diff --git a/Lib/test/test_support.py b/Lib/test/test_support.py index c34b0e5e015702..d160cbf0645b47 100644 --- a/Lib/test/test_support.py +++ b/Lib/test/test_support.py @@ -630,7 +630,7 @@ def recursive_function(depth): if depth: recursive_function(depth - 1) - for max_depth in (5, 25, 250): + for max_depth in (5, 25, 250, 2500): with support.infinite_recursion(max_depth): available = support.get_recursion_available() diff --git a/Lib/test/test_syntax.py b/Lib/test/test_syntax.py index 8b3ca69c9fe155..ee9b873d9023f0 100644 --- a/Lib/test/test_syntax.py +++ b/Lib/test/test_syntax.py @@ -259,6 +259,36 @@ Traceback (most recent call last): SyntaxError: invalid syntax +Comprehensions without 'in' keyword: + +>>> [x for x if range(1)] +Traceback (most recent call last): +SyntaxError: 'in' expected after for-loop variables + +>>> tuple(x for x if range(1)) +Traceback (most recent call last): +SyntaxError: 'in' expected after for-loop variables + +>>> [x for x() in a] +Traceback (most recent call last): +SyntaxError: cannot assign to function call + +>>> [x for a, b, (c + 1, d()) in y] +Traceback (most recent call last): +SyntaxError: cannot assign to expression + +>>> [x for a, b, (c + 1, d()) if y] +Traceback (most recent call last): +SyntaxError: 'in' expected after for-loop variables + +>>> [x for x+1 in y] +Traceback (most recent call last): +SyntaxError: cannot assign to expression + +>>> [x for x+1, x() in y] +Traceback (most recent call last): +SyntaxError: cannot assign to expression + Comprehensions creating tuples without parentheses should produce a specialized error message: @@ -2360,6 +2390,8 @@ def test_error_parenthesis(self): """ self._check_error(code, "parenthesis '\\)' does not match opening parenthesis '\\['") + self._check_error("match y:\n case e(e=v,v,", " was never closed") + # Examples with dencodings s = b'# coding=latin\n(aaaaaaaaaaaaaaaaa\naaaaaaaaaaa\xb5' self._check_error(s, r"'\(' was never closed") diff --git a/Lib/test/test_sys.py b/Lib/test/test_sys.py index 6c87dfabad9f0f..71671a5a984256 100644 --- a/Lib/test/test_sys.py +++ b/Lib/test/test_sys.py @@ -1392,6 +1392,7 @@ def setUp(self): self.longdigit = sys.int_info.sizeof_digit import _testinternalcapi self.gc_headsize = _testinternalcapi.SIZEOF_PYGC_HEAD + self.managed_pre_header_size = _testinternalcapi.SIZEOF_MANAGED_PRE_HEADER check_sizeof = test.support.check_sizeof @@ -1427,7 +1428,7 @@ class OverflowSizeof(int): def __sizeof__(self): return int(self) self.assertEqual(sys.getsizeof(OverflowSizeof(sys.maxsize)), - sys.maxsize + self.gc_headsize*2) + sys.maxsize + self.gc_headsize + self.managed_pre_header_size) with self.assertRaises(OverflowError): sys.getsizeof(OverflowSizeof(sys.maxsize + 1)) with self.assertRaises(ValueError): @@ -1650,7 +1651,7 @@ def delx(self): del self.__x # type # static type: PyTypeObject fmt = 'P2nPI13Pl4Pn9Pn12PIPc' - s = vsize('2P' + fmt) + s = vsize(fmt) check(int, s) # class s = vsize(fmt + # PyTypeObject diff --git a/Lib/test/test_sys_settrace.py b/Lib/test/test_sys_settrace.py index fc5ca72236b1fb..125f40227118f6 100644 --- a/Lib/test/test_sys_settrace.py +++ b/Lib/test/test_sys_settrace.py @@ -7,7 +7,7 @@ import gc from functools import wraps import asyncio -from test.support import import_helper +from test.support import import_helper, requires_subprocess import contextlib import os import tempfile @@ -1810,6 +1810,7 @@ def compare_events(self, line_offset, events, expected_events): def make_tracer(): return Tracer(trace_opcode_events=True) + @requires_subprocess() def test_trace_opcodes_after_settrace(self): """Make sure setting f_trace_opcodes after starting trace works even if it's the first time f_trace_opcodes is being set. GH-103615""" @@ -3037,10 +3038,8 @@ def test_trace_unpack_long_sequence(self): self.assertEqual(counts, {'call': 1, 'line': 301, 'return': 1}) def test_trace_lots_of_globals(self): - count = 1000 - if _testinternalcapi is not None: - remaining = _testinternalcapi.get_c_recursion_remaining() - count = min(count, remaining) + + count = min(1000, int(support.Py_C_RECURSION_LIMIT * 0.8)) code = """if 1: def f(): diff --git a/Lib/test/test_sysconfig.py b/Lib/test/test_sysconfig.py index a19c04b1b2cde5..bb87bf00dc2d1a 100644 --- a/Lib/test/test_sysconfig.py +++ b/Lib/test/test_sysconfig.py @@ -43,6 +43,7 @@ def setUp(self): self.name = os.name self.platform = sys.platform self.version = sys.version + self._framework = sys._framework self.sep = os.sep self.join = os.path.join self.isabs = os.path.isabs @@ -66,6 +67,7 @@ def tearDown(self): os.name = self.name sys.platform = self.platform sys.version = self.version + sys._framework = self._framework os.sep = self.sep os.path.join = self.join os.path.isabs = self.isabs @@ -139,7 +141,7 @@ def test_get_preferred_schemes(self): # Mac, framework build. os.name = 'posix' sys.platform = 'darwin' - sys._framework = True + sys._framework = "MyPython" self.assertIsInstance(schemes, dict) self.assertEqual(set(schemes), expected_schemes) @@ -152,17 +154,21 @@ def test_posix_venv_scheme(self): 'python%d.%d' % sys.version_info[:2], 'site-packages') - # Resolve the paths in prefix - binpath = os.path.join(sys.prefix, binpath) - incpath = os.path.join(sys.prefix, incpath) - libpath = os.path.join(sys.prefix, libpath) + # Resolve the paths in an imaginary venv/ directory + binpath = os.path.join('venv', binpath) + incpath = os.path.join('venv', incpath) + libpath = os.path.join('venv', libpath) - self.assertEqual(binpath, sysconfig.get_path('scripts', scheme='posix_venv')) - self.assertEqual(libpath, sysconfig.get_path('purelib', scheme='posix_venv')) + # Mimic the venv module, set all bases to the venv directory + bases = ('base', 'platbase', 'installed_base', 'installed_platbase') + vars = {base: 'venv' for base in bases} + + self.assertEqual(binpath, sysconfig.get_path('scripts', scheme='posix_venv', vars=vars)) + self.assertEqual(libpath, sysconfig.get_path('purelib', scheme='posix_venv', vars=vars)) # The include directory on POSIX isn't exactly the same as before, # but it is "within" - sysconfig_includedir = sysconfig.get_path('include', scheme='posix_venv') + sysconfig_includedir = sysconfig.get_path('include', scheme='posix_venv', vars=vars) self.assertTrue(sysconfig_includedir.startswith(incpath + os.sep)) def test_nt_venv_scheme(self): @@ -172,14 +178,19 @@ def test_nt_venv_scheme(self): incpath = 'Include' libpath = os.path.join('Lib', 'site-packages') - # Resolve the paths in prefix - binpath = os.path.join(sys.prefix, binpath) - incpath = os.path.join(sys.prefix, incpath) - libpath = os.path.join(sys.prefix, libpath) + # Resolve the paths in an imaginary venv\ directory + venv = 'venv' + binpath = os.path.join(venv, binpath) + incpath = os.path.join(venv, incpath) + libpath = os.path.join(venv, libpath) + + # Mimic the venv module, set all bases to the venv directory + bases = ('base', 'platbase', 'installed_base', 'installed_platbase') + vars = {base: 'venv' for base in bases} - self.assertEqual(binpath, sysconfig.get_path('scripts', scheme='nt_venv')) - self.assertEqual(incpath, sysconfig.get_path('include', scheme='nt_venv')) - self.assertEqual(libpath, sysconfig.get_path('purelib', scheme='nt_venv')) + self.assertEqual(binpath, sysconfig.get_path('scripts', scheme='nt_venv', vars=vars)) + self.assertEqual(incpath, sysconfig.get_path('include', scheme='nt_venv', vars=vars)) + self.assertEqual(libpath, sysconfig.get_path('purelib', scheme='nt_venv', vars=vars)) def test_venv_scheme(self): if sys.platform == 'win32': @@ -413,7 +424,10 @@ def test_library(self): else: self.assertTrue(library.startswith(f'libpython{major}.{minor}')) self.assertTrue(library.endswith('.a')) - self.assertTrue(ldlibrary.startswith(f'libpython{major}.{minor}')) + if sys.platform == 'darwin' and sys._framework: + self.skipTest('gh-110824: skip LDLIBRARY test for framework build') + else: + self.assertTrue(ldlibrary.startswith(f'libpython{major}.{minor}')) @unittest.skipUnless(sys.platform == "darwin", "test only relevant on MacOSX") @requires_subprocess() diff --git a/Lib/test/test_tarfile.py b/Lib/test/test_tarfile.py index da5009126b3815..51f070e96047a6 100644 --- a/Lib/test/test_tarfile.py +++ b/Lib/test/test_tarfile.py @@ -15,6 +15,7 @@ import unittest.mock import tarfile +from test import archiver_tests from test import support from test.support import os_helper from test.support import script_helper @@ -4135,6 +4136,38 @@ def valueerror_filter(tarinfo, path): self.expect_exception(TypeError) # errorlevel is not int +class OverwriteTests(archiver_tests.OverwriteTests, unittest.TestCase): + testdir = os.path.join(TEMPDIR, "testoverwrite") + + @classmethod + def setUpClass(cls): + p = cls.ar_with_file = os.path.join(TEMPDIR, 'tar-with-file.tar') + cls.addClassCleanup(os_helper.unlink, p) + with tarfile.open(p, 'w') as tar: + t = tarfile.TarInfo('test') + t.size = 10 + tar.addfile(t, io.BytesIO(b'newcontent')) + + p = cls.ar_with_dir = os.path.join(TEMPDIR, 'tar-with-dir.tar') + cls.addClassCleanup(os_helper.unlink, p) + with tarfile.open(p, 'w') as tar: + tar.addfile(tar.gettarinfo(os.curdir, 'test')) + + p = os.path.join(TEMPDIR, 'tar-with-implicit-dir.tar') + cls.ar_with_implicit_dir = p + cls.addClassCleanup(os_helper.unlink, p) + with tarfile.open(p, 'w') as tar: + t = tarfile.TarInfo('test/file') + t.size = 10 + tar.addfile(t, io.BytesIO(b'newcontent')) + + def open(self, path): + return tarfile.open(path, 'r') + + def extractall(self, ar): + ar.extractall(self.testdir, filter='fully_trusted') + + def setUpModule(): os_helper.unlink(TEMPDIR) os.makedirs(TEMPDIR) diff --git a/Lib/test/test_threading.py b/Lib/test/test_threading.py index 3060af44fd7e3d..1ab223b81e939e 100644 --- a/Lib/test/test_threading.py +++ b/Lib/test/test_threading.py @@ -115,6 +115,7 @@ def tearDown(self): class ThreadTests(BaseTestCase): + maxDiff = 9999 @cpython_only def test_name(self): @@ -170,11 +171,21 @@ def test_args_argument(self): t.start() t.join() - @cpython_only - def test_disallow_instantiation(self): - # Ensure that the type disallows instantiation (bpo-43916) - lock = threading.Lock() - test.support.check_disallow_instantiation(self, type(lock)) + def test_lock_no_args(self): + threading.Lock() # works + self.assertRaises(TypeError, threading.Lock, 1) + self.assertRaises(TypeError, threading.Lock, a=1) + self.assertRaises(TypeError, threading.Lock, 1, 2, a=1, b=2) + + def test_lock_no_subclass(self): + # Intentionally disallow subclasses of threading.Lock because they have + # never been allowed, so why start now just because the type is public? + with self.assertRaises(TypeError): + class MyLock(threading.Lock): pass + + def test_lock_or_none(self): + import types + self.assertIsInstance(threading.Lock | None, types.UnionType) # Create a bunch of threads, let each do some work, wait until all are # done. @@ -226,8 +237,6 @@ def f(): tid = _thread.start_new_thread(f, ()) done.wait() self.assertEqual(ident[0], tid) - # Kill the "immortal" _DummyThread - del threading._active[ident[0]] # run with a small(ish) thread stack size (256 KiB) def test_various_ops_small_stack(self): @@ -255,11 +264,29 @@ def test_various_ops_large_stack(self): def test_foreign_thread(self): # Check that a "foreign" thread can use the threading module. + dummy_thread = None + error = None def f(mutex): - # Calling current_thread() forces an entry for the foreign - # thread to get made in the threading._active map. - threading.current_thread() - mutex.release() + try: + nonlocal dummy_thread + nonlocal error + # Calling current_thread() forces an entry for the foreign + # thread to get made in the threading._active map. + dummy_thread = threading.current_thread() + tid = dummy_thread.ident + self.assertIn(tid, threading._active) + self.assertIsInstance(dummy_thread, threading._DummyThread) + self.assertIs(threading._active.get(tid), dummy_thread) + # gh-29376 + self.assertTrue( + dummy_thread.is_alive(), + 'Expected _DummyThread to be considered alive.' + ) + self.assertIn('_DummyThread', repr(dummy_thread)) + except BaseException as e: + error = e + finally: + mutex.release() mutex = threading.Lock() mutex.acquire() @@ -267,20 +294,25 @@ def f(mutex): tid = _thread.start_new_thread(f, (mutex,)) # Wait for the thread to finish. mutex.acquire() - self.assertIn(tid, threading._active) - self.assertIsInstance(threading._active[tid], threading._DummyThread) - #Issue 29376 - self.assertTrue(threading._active[tid].is_alive()) - self.assertRegex(repr(threading._active[tid]), '_DummyThread') - + if error is not None: + raise error + self.assertEqual(tid, dummy_thread.ident) # Issue gh-106236: with self.assertRaises(RuntimeError): - threading._active[tid].join() - threading._active[tid]._started.clear() + dummy_thread.join() + dummy_thread._started.clear() with self.assertRaises(RuntimeError): - threading._active[tid].is_alive() - - del threading._active[tid] + dummy_thread.is_alive() + # Busy wait for the following condition: after the thread dies, the + # related dummy thread must be removed from threading._active. + timeout = 5 + timeout_at = time.monotonic() + timeout + while time.monotonic() < timeout_at: + if threading._active.get(dummy_thread.ident) is not dummy_thread: + break + time.sleep(.1) + else: + self.fail('It was expected that the created threading._DummyThread was removed from threading._active.') # PyThreadState_SetAsyncExc() is a CPython-only gimmick, not (currently) # exposed at the Python level. This test relies on ctypes to get at it. @@ -676,19 +708,25 @@ def test_main_thread_after_fork(self): import os, threading from test import support + ident = threading.get_ident() pid = os.fork() if pid == 0: + print("current ident", threading.get_ident() == ident) main = threading.main_thread() - print(main.name) - print(main.ident == threading.current_thread().ident) - print(main.ident == threading.get_ident()) + print("main", main.name) + print("main ident", main.ident == ident) + print("current is main", threading.current_thread() is main) else: support.wait_process(pid, exitcode=0) """ _, out, err = assert_python_ok("-c", code) data = out.decode().replace('\r', '') self.assertEqual(err, b"") - self.assertEqual(data, "MainThread\nTrue\nTrue\n") + self.assertEqual(data, + "current ident True\n" + "main MainThread\n" + "main ident True\n" + "current is main True\n") @skip_unless_reliable_fork @unittest.skipUnless(hasattr(os, 'waitpid'), "test needs os.waitpid()") @@ -698,15 +736,17 @@ def test_main_thread_after_fork_from_nonmain_thread(self): from test import support def func(): + ident = threading.get_ident() with warnings.catch_warnings(record=True) as ws: warnings.filterwarnings( "always", category=DeprecationWarning) pid = os.fork() if pid == 0: + print("current ident", threading.get_ident() == ident) main = threading.main_thread() - print(main.name) - print(main.ident == threading.current_thread().ident) - print(main.ident == threading.get_ident()) + print("main", main.name, type(main).__name__) + print("main ident", main.ident == ident) + print("current is main", threading.current_thread() is main) # stdout is fully buffered because not a tty, # we have to flush before exit. sys.stdout.flush() @@ -722,7 +762,80 @@ def func(): _, out, err = assert_python_ok("-c", code) data = out.decode().replace('\r', '') self.assertEqual(err.decode('utf-8'), "") - self.assertEqual(data, "Thread-1 (func)\nTrue\nTrue\n") + self.assertEqual(data, + "current ident True\n" + "main Thread-1 (func) Thread\n" + "main ident True\n" + "current is main True\n" + ) + + @unittest.skipIf(sys.platform in platforms_to_skip, "due to known OS bug") + @support.requires_fork() + @unittest.skipUnless(hasattr(os, 'waitpid'), "test needs os.waitpid()") + def test_main_thread_after_fork_from_foreign_thread(self, create_dummy=False): + code = """if 1: + import os, threading, sys, traceback, _thread + from test import support + + def func(lock): + ident = threading.get_ident() + if %s: + # call current_thread() before fork to allocate DummyThread + current = threading.current_thread() + print("current", current.name, type(current).__name__) + print("ident in _active", ident in threading._active) + # flush before fork, so child won't flush it again + sys.stdout.flush() + pid = os.fork() + if pid == 0: + print("current ident", threading.get_ident() == ident) + main = threading.main_thread() + print("main", main.name, type(main).__name__) + print("main ident", main.ident == ident) + print("current is main", threading.current_thread() is main) + print("_dangling", [t.name for t in list(threading._dangling)]) + # stdout is fully buffered because not a tty, + # we have to flush before exit. + sys.stdout.flush() + try: + threading._shutdown() + os._exit(0) + except: + traceback.print_exc() + sys.stderr.flush() + os._exit(1) + else: + try: + support.wait_process(pid, exitcode=0) + except Exception: + # avoid 'could not acquire lock for + # <_io.BufferedWriter name=''> at interpreter shutdown,' + traceback.print_exc() + sys.stderr.flush() + finally: + lock.release() + + join_lock = _thread.allocate_lock() + join_lock.acquire() + th = _thread.start_new_thread(func, (join_lock,)) + join_lock.acquire() + """ % create_dummy + # "DeprecationWarning: This process is multi-threaded, use of fork() + # may lead to deadlocks in the child" + _, out, err = assert_python_ok("-W", "ignore::DeprecationWarning", "-c", code) + data = out.decode().replace('\r', '') + self.assertEqual(err.decode(), "") + self.assertEqual(data, + ("current Dummy-1 _DummyThread\n" if create_dummy else "") + + f"ident in _active {create_dummy!s}\n" + + "current ident True\n" + "main MainThread _MainThread\n" + "main ident True\n" + "current is main True\n" + "_dangling ['MainThread']\n") + + def test_main_thread_after_fork_from_dummy_thread(self, create_dummy=False): + self.test_main_thread_after_fork_from_foreign_thread(create_dummy=True) def test_main_thread_during_shutdown(self): # bpo-31516: current_thread() should still point to the main thread diff --git a/Lib/test/test_tkinter/support.py b/Lib/test/test_tkinter/support.py index a37705f0ae6feb..ebb9e00ff91bf0 100644 --- a/Lib/test/test_tkinter/support.py +++ b/Lib/test/test_tkinter/support.py @@ -14,7 +14,7 @@ def setUpClass(cls): # Some window managers can maximize new windows. cls.root.wm_state('normal') try: - cls.root.wm_attributes('-zoomed', False) + cls.root.wm_attributes(zoomed=False) except tkinter.TclError: pass diff --git a/Lib/test/test_tkinter/test_misc.py b/Lib/test/test_tkinter/test_misc.py index 6639eaaa59936a..81a20b698a72eb 100644 --- a/Lib/test/test_tkinter/test_misc.py +++ b/Lib/test/test_tkinter/test_misc.py @@ -281,6 +281,18 @@ def assertApprox(col1, col2): with self.assertRaises(tkinter.TclError): rgb((111, 78, 55)) + def test_winfo_pathname(self): + t = tkinter.Toplevel(self.root) + w = tkinter.Button(t) + wid = w.winfo_id() + self.assertIsInstance(wid, int) + self.assertEqual(self.root.winfo_pathname(hex(wid)), str(w)) + self.assertEqual(self.root.winfo_pathname(hex(wid), displayof=None), str(w)) + self.assertEqual(self.root.winfo_pathname(hex(wid), displayof=t), str(w)) + self.assertEqual(self.root.winfo_pathname(wid), str(w)) + self.assertEqual(self.root.winfo_pathname(wid, displayof=None), str(w)) + self.assertEqual(self.root.winfo_pathname(wid, displayof=t), str(w)) + def test_event_repr_defaults(self): e = tkinter.Event() e.serial = 12345 @@ -425,6 +437,61 @@ def test_info_patchlevel(self): self.assertTrue(str(vi).startswith(f'{vi.major}.{vi.minor}')) +class WmTest(AbstractTkTest, unittest.TestCase): + + def test_wm_attribute(self): + w = self.root + attributes = w.wm_attributes(return_python_dict=True) + self.assertIsInstance(attributes, dict) + attributes2 = w.wm_attributes() + self.assertIsInstance(attributes2, tuple) + self.assertEqual(attributes2[::2], + tuple('-' + k for k in attributes)) + self.assertEqual(attributes2[1::2], tuple(attributes.values())) + # silently deprecated + attributes3 = w.wm_attributes(None) + if self.wantobjects: + self.assertEqual(attributes3, attributes2) + else: + self.assertIsInstance(attributes3, str) + + for name in attributes: + self.assertEqual(w.wm_attributes(name), attributes[name]) + # silently deprecated + for name in attributes: + self.assertEqual(w.wm_attributes('-' + name), attributes[name]) + + self.assertIn('alpha', attributes) + self.assertIn('fullscreen', attributes) + self.assertIn('topmost', attributes) + if w._windowingsystem == "win32": + self.assertIn('disabled', attributes) + self.assertIn('toolwindow', attributes) + self.assertIn('transparentcolor', attributes) + if w._windowingsystem == "aqua": + self.assertIn('modified', attributes) + self.assertIn('notify', attributes) + self.assertIn('titlepath', attributes) + self.assertIn('transparent', attributes) + if w._windowingsystem == "x11": + self.assertIn('type', attributes) + self.assertIn('zoomed', attributes) + + w.wm_attributes(alpha=0.5) + self.assertEqual(w.wm_attributes('alpha'), + 0.5 if self.wantobjects else '0.5') + w.wm_attributes(alpha=1.0) + self.assertEqual(w.wm_attributes('alpha'), + 1.0 if self.wantobjects else '1.0') + # silently deprecated + w.wm_attributes('-alpha', 0.5) + self.assertEqual(w.wm_attributes('alpha'), + 0.5 if self.wantobjects else '0.5') + w.wm_attributes(alpha=1.0) + self.assertEqual(w.wm_attributes('alpha'), + 1.0 if self.wantobjects else '1.0') + + class BindTest(AbstractTkTest, unittest.TestCase): def setUp(self): @@ -694,6 +761,101 @@ def test3(e): pass self.assertCommandExist(funcid2) self.assertCommandExist(funcid3) + def _test_tag_bind(self, w): + tag = 'sel' + event = '' + w.pack() + self.assertRaises(TypeError, w.tag_bind) + tag_bind = w._tag_bind if isinstance(w, tkinter.Text) else w.tag_bind + if isinstance(w, tkinter.Text): + self.assertRaises(TypeError, w.tag_bind, tag) + self.assertRaises(TypeError, w.tag_bind, tag, event) + self.assertEqual(tag_bind(tag), ()) + self.assertEqual(tag_bind(tag, event), '') + def test1(e): pass + def test2(e): pass + + funcid = w.tag_bind(tag, event, test1) + self.assertEqual(tag_bind(tag), (event,)) + script = tag_bind(tag, event) + self.assertIn(funcid, script) + self.assertCommandExist(funcid) + + funcid2 = w.tag_bind(tag, event, test2, add=True) + script = tag_bind(tag, event) + self.assertIn(funcid, script) + self.assertIn(funcid2, script) + self.assertCommandExist(funcid) + self.assertCommandExist(funcid2) + + def _test_tag_unbind(self, w): + tag = 'sel' + event = '' + w.pack() + tag_bind = w._tag_bind if isinstance(w, tkinter.Text) else w.tag_bind + self.assertEqual(tag_bind(tag), ()) + self.assertEqual(tag_bind(tag, event), '') + def test1(e): pass + def test2(e): pass + + funcid = w.tag_bind(tag, event, test1) + funcid2 = w.tag_bind(tag, event, test2, add=True) + + self.assertRaises(TypeError, w.tag_unbind, tag) + w.tag_unbind(tag, event) + self.assertEqual(tag_bind(tag, event), '') + self.assertEqual(tag_bind(tag), ()) + + def _test_tag_bind_rebind(self, w): + tag = 'sel' + event = '' + w.pack() + tag_bind = w._tag_bind if isinstance(w, tkinter.Text) else w.tag_bind + self.assertEqual(tag_bind(tag), ()) + self.assertEqual(tag_bind(tag, event), '') + def test1(e): pass + def test2(e): pass + def test3(e): pass + + funcid = w.tag_bind(tag, event, test1) + funcid2 = w.tag_bind(tag, event, test2, add=True) + script = tag_bind(tag, event) + self.assertIn(funcid2, script) + self.assertIn(funcid, script) + self.assertCommandExist(funcid) + self.assertCommandExist(funcid2) + + funcid3 = w.tag_bind(tag, event, test3) + script = tag_bind(tag, event) + self.assertNotIn(funcid, script) + self.assertNotIn(funcid2, script) + self.assertIn(funcid3, script) + self.assertCommandExist(funcid3) + + def test_canvas_tag_bind(self): + c = tkinter.Canvas(self.frame) + self._test_tag_bind(c) + + def test_canvas_tag_unbind(self): + c = tkinter.Canvas(self.frame) + self._test_tag_unbind(c) + + def test_canvas_tag_bind_rebind(self): + c = tkinter.Canvas(self.frame) + self._test_tag_bind_rebind(c) + + def test_text_tag_bind(self): + t = tkinter.Text(self.frame) + self._test_tag_bind(t) + + def test_text_tag_unbind(self): + t = tkinter.Text(self.frame) + self._test_tag_unbind(t) + + def test_text_tag_bind_rebind(self): + t = tkinter.Text(self.frame) + self._test_tag_bind_rebind(t) + def test_bindtags(self): f = self.frame self.assertEqual(self.root.bindtags(), ('.', 'Tk', 'all')) diff --git a/Lib/test/test_tools/test_freeze.py b/Lib/test/test_tools/test_freeze.py index 671ec2961e7f8f..0e7ed67de71067 100644 --- a/Lib/test/test_tools/test_freeze.py +++ b/Lib/test/test_tools/test_freeze.py @@ -14,6 +14,8 @@ @support.requires_zlib() @unittest.skipIf(sys.platform.startswith('win'), 'not supported on Windows') +@unittest.skipIf(sys.platform == 'darwin' and sys._framework, + 'not supported for frameworks builds on macOS') @support.skip_if_buildbot('not all buildbots have enough space') # gh-103053: Skip test if Python is built with Profile Guided Optimization # (PGO), since the test is just too slow in this case. diff --git a/Lib/test/test_traceback.py b/Lib/test/test_traceback.py index a6708119b81191..372fc48bf81a6a 100644 --- a/Lib/test/test_traceback.py +++ b/Lib/test/test_traceback.py @@ -2209,6 +2209,20 @@ def __repr__(self): err_msg = "b'please do not show me as numbers'" self.assertEqual(self.get_report(e), vanilla + err_msg + '\n') + # an exception with a broken __getattr__ raising a non expected error + class BrokenException(Exception): + broken = False + def __getattr__(self, name): + if self.broken: + raise ValueError(f'no {name}') + + e = BrokenException(123) + vanilla = self.get_report(e) + e.broken = True + self.assertEqual( + self.get_report(e), + vanilla + "Ignored error getting __notes__: ValueError('no __notes__')\n") + def test_exception_with_multiple_notes(self): for e in [ValueError(42), SyntaxError('bad syntax')]: with self.subTest(e=e): diff --git a/Lib/test/test_tty.py b/Lib/test/test_tty.py index af20864aac361e..4cb730c226f134 100644 --- a/Lib/test/test_tty.py +++ b/Lib/test/test_tty.py @@ -19,7 +19,6 @@ def setUp(self): self.addCleanup(termios.tcsetattr, self.fd, termios.TCSAFLUSH, self.mode) def check_cbreak(self, mode): - self.assertEqual(mode[0] & termios.ICRNL, 0) self.assertEqual(mode[3] & termios.ECHO, 0) self.assertEqual(mode[3] & termios.ICANON, 0) self.assertEqual(mode[6][termios.VMIN], 1) @@ -56,6 +55,14 @@ def test_cfmakecbreak(self): self.assertEqual(mode[2], self.mode[2]) self.assertEqual(mode[4], self.mode[4]) self.assertEqual(mode[5], self.mode[5]) + mode[tty.IFLAG] |= termios.ICRNL + tty.cfmakecbreak(mode) + self.assertEqual(mode[tty.IFLAG] & termios.ICRNL, termios.ICRNL, + msg="ICRNL should not be cleared by cbreak") + mode[tty.IFLAG] &= ~termios.ICRNL + tty.cfmakecbreak(mode) + self.assertEqual(mode[tty.IFLAG] & termios.ICRNL, 0, + msg="ICRNL should not be set by cbreak") def test_setraw(self): mode0 = termios.tcgetattr(self.fd) @@ -74,6 +81,9 @@ def test_setcbreak(self): self.assertEqual(mode1, mode0) mode2 = termios.tcgetattr(self.fd) self.check_cbreak(mode2) + ICRNL = termios.ICRNL + self.assertEqual(mode2[tty.IFLAG] & ICRNL, mode0[tty.IFLAG] & ICRNL, + msg="ICRNL should not be altered by cbreak") mode3 = tty.setcbreak(self.fd, termios.TCSANOW) self.assertEqual(mode3, mode2) tty.setcbreak(self.stream) diff --git a/Lib/test/test_type_cache.py b/Lib/test/test_type_cache.py index 72587ecc11b6f3..58572c6f4d3157 100644 --- a/Lib/test/test_type_cache.py +++ b/Lib/test/test_type_cache.py @@ -1,5 +1,6 @@ """ Tests for the internal type cache in CPython. """ import unittest +import dis from test import support from test.support import import_helper try: @@ -8,8 +9,11 @@ _clear_type_cache = None # Skip this test if the _testcapi module isn't available. -type_get_version = import_helper.import_module('_testcapi').type_get_version -type_assign_version = import_helper.import_module('_testcapi').type_assign_version +_testcapi = import_helper.import_module("_testcapi") +type_get_version = _testcapi.type_get_version +type_assign_specific_version_unsafe = _testcapi.type_assign_specific_version_unsafe +type_assign_version = _testcapi.type_assign_version +type_modified = _testcapi.type_modified @support.cpython_only @@ -56,6 +60,196 @@ class C: self.assertNotEqual(type_get_version(C), 0) self.assertNotEqual(type_get_version(C), c_ver) + def test_type_assign_specific_version(self): + """meta-test for type_assign_specific_version_unsafe""" + class C: + pass + + type_assign_version(C) + orig_version = type_get_version(C) + if orig_version == 0: + self.skipTest("Could not assign a valid type version") + + type_modified(C) + type_assign_specific_version_unsafe(C, orig_version + 5) + type_assign_version(C) # this should do nothing + + new_version = type_get_version(C) + self.assertEqual(new_version, orig_version + 5) + + _clear_type_cache() + + def test_per_class_limit(self): + class C: + x = 0 + + type_assign_version(C) + orig_version = type_get_version(C) + for i in range(1001): + C.x = i + type_assign_version(C) + + new_version = type_get_version(C) + self.assertEqual(new_version, 0) + + +@support.cpython_only +class TypeCacheWithSpecializationTests(unittest.TestCase): + def tearDown(self): + _clear_type_cache() + + def _assign_valid_version_or_skip(self, type_): + type_modified(type_) + type_assign_version(type_) + if type_get_version(type_) == 0: + self.skipTest("Could not assign valid type version") + + def _assign_and_check_version_0(self, user_type): + type_modified(user_type) + type_assign_specific_version_unsafe(user_type, 0) + self.assertEqual(type_get_version(user_type), 0) + + def _all_opnames(self, func): + return set(instr.opname for instr in dis.Bytecode(func, adaptive=True)) + + def _check_specialization(self, func, arg, opname, *, should_specialize): + for _ in range(100): + func(arg) + + if should_specialize: + self.assertNotIn(opname, self._all_opnames(func)) + else: + self.assertIn(opname, self._all_opnames(func)) + + def test_class_load_attr_specialization_user_type(self): + class A: + def foo(self): + pass + + self._assign_valid_version_or_skip(A) + + def load_foo_1(type_): + type_.foo + + self._check_specialization(load_foo_1, A, "LOAD_ATTR", should_specialize=True) + del load_foo_1 + + self._assign_and_check_version_0(A) + + def load_foo_2(type_): + return type_.foo + + self._check_specialization(load_foo_2, A, "LOAD_ATTR", should_specialize=False) + + def test_class_load_attr_specialization_static_type(self): + self._assign_valid_version_or_skip(str) + self._assign_valid_version_or_skip(bytes) + + def get_capitalize_1(type_): + return type_.capitalize + + self._check_specialization(get_capitalize_1, str, "LOAD_ATTR", should_specialize=True) + self.assertEqual(get_capitalize_1(str)('hello'), 'Hello') + self.assertEqual(get_capitalize_1(bytes)(b'hello'), b'Hello') + del get_capitalize_1 + + # Permanently overflow the static type version counter, and force str and bytes + # to have tp_version_tag == 0 + for _ in range(2**16): + type_modified(str) + type_assign_version(str) + type_modified(bytes) + type_assign_version(bytes) + + self.assertEqual(type_get_version(str), 0) + self.assertEqual(type_get_version(bytes), 0) + + def get_capitalize_2(type_): + return type_.capitalize + + self._check_specialization(get_capitalize_2, str, "LOAD_ATTR", should_specialize=False) + self.assertEqual(get_capitalize_2(str)('hello'), 'Hello') + self.assertEqual(get_capitalize_2(bytes)(b'hello'), b'Hello') + + def test_property_load_attr_specialization_user_type(self): + class G: + @property + def x(self): + return 9 + + self._assign_valid_version_or_skip(G) + + def load_x_1(instance): + instance.x + + self._check_specialization(load_x_1, G(), "LOAD_ATTR", should_specialize=True) + del load_x_1 + + self._assign_and_check_version_0(G) + + def load_x_2(instance): + instance.x + + self._check_specialization(load_x_2, G(), "LOAD_ATTR", should_specialize=False) + + def test_store_attr_specialization_user_type(self): + class B: + __slots__ = ("bar",) + + self._assign_valid_version_or_skip(B) + + def store_bar_1(type_): + type_.bar = 10 + + self._check_specialization(store_bar_1, B(), "STORE_ATTR", should_specialize=True) + del store_bar_1 + + self._assign_and_check_version_0(B) + + def store_bar_2(type_): + type_.bar = 10 + + self._check_specialization(store_bar_2, B(), "STORE_ATTR", should_specialize=False) + + def test_class_call_specialization_user_type(self): + class F: + def __init__(self): + pass + + self._assign_valid_version_or_skip(F) + + def call_class_1(type_): + type_() + + self._check_specialization(call_class_1, F, "CALL", should_specialize=True) + del call_class_1 + + self._assign_and_check_version_0(F) + + def call_class_2(type_): + type_() + + self._check_specialization(call_class_2, F, "CALL", should_specialize=False) + + def test_to_bool_specialization_user_type(self): + class H: + pass + + self._assign_valid_version_or_skip(H) + + def to_bool_1(instance): + not instance + + self._check_specialization(to_bool_1, H(), "TO_BOOL", should_specialize=True) + del to_bool_1 + + self._assign_and_check_version_0(H) + + def to_bool_2(instance): + not instance + + self._check_specialization(to_bool_2, H(), "TO_BOOL", should_specialize=False) + if __name__ == "__main__": unittest.main() diff --git a/Lib/test/test_types.py b/Lib/test/test_types.py index da32c4ea6477ce..1acb2a4d81adf3 100644 --- a/Lib/test/test_types.py +++ b/Lib/test/test_types.py @@ -1,6 +1,6 @@ # Python test set -- part 6, built-in types -from test.support import run_with_locale, cpython_only +from test.support import run_with_locale, cpython_only, MISSING_C_DOCSTRINGS import collections.abc from collections import namedtuple import copy @@ -598,6 +598,8 @@ def test_slot_wrapper_types(self): self.assertIsInstance(object.__lt__, types.WrapperDescriptorType) self.assertIsInstance(int.__lt__, types.WrapperDescriptorType) + @unittest.skipIf(MISSING_C_DOCSTRINGS, + "Signature information for builtins requires docstrings") def test_dunder_get_signature(self): sig = inspect.signature(object.__init__.__get__) self.assertEqual(list(sig.parameters), ["instance", "owner"]) @@ -2261,5 +2263,38 @@ def coro(): 'close', 'throw'})) +class FunctionTests(unittest.TestCase): + def test_function_type_defaults(self): + def ex(a, /, b, *, c): + return a + b + c + + func = types.FunctionType( + ex.__code__, {}, "func", (1, 2), None, {'c': 3}, + ) + + self.assertEqual(func(), 6) + self.assertEqual(func.__defaults__, (1, 2)) + self.assertEqual(func.__kwdefaults__, {'c': 3}) + + func = types.FunctionType( + ex.__code__, {}, "func", None, None, None, + ) + self.assertEqual(func.__defaults__, None) + self.assertEqual(func.__kwdefaults__, None) + + def test_function_type_wrong_defaults(self): + def ex(a, /, b, *, c): + return a + b + c + + with self.assertRaisesRegex(TypeError, 'arg 4'): + types.FunctionType( + ex.__code__, {}, "func", 1, None, {'c': 3}, + ) + with self.assertRaisesRegex(TypeError, 'arg 6'): + types.FunctionType( + ex.__code__, {}, "func", None, None, 3, + ) + + if __name__ == '__main__': unittest.main() diff --git a/Lib/test/test_typing.py b/Lib/test/test_typing.py index 2d10c39840ddf3..c3a092f3af3009 100644 --- a/Lib/test/test_typing.py +++ b/Lib/test/test_typing.py @@ -3448,8 +3448,8 @@ def meth(self): pass self.assertNotIn("__protocol_attrs__", vars(NonP)) self.assertNotIn("__protocol_attrs__", vars(NonPR)) - self.assertNotIn("__callable_proto_members_only__", vars(NonP)) - self.assertNotIn("__callable_proto_members_only__", vars(NonPR)) + self.assertNotIn("__non_callable_proto_members__", vars(NonP)) + self.assertNotIn("__non_callable_proto_members__", vars(NonPR)) self.assertEqual(get_protocol_members(P), {"x"}) self.assertEqual(get_protocol_members(PR), {"meth"}) @@ -4105,6 +4105,7 @@ def method(self) -> None: ... self.assertNotIsInstance(42, ProtocolWithMixedMembers) def test_protocol_issubclass_error_message(self): + @runtime_checkable class Vec2D(Protocol): x: float y: float @@ -4120,6 +4121,39 @@ def square_norm(self) -> float: with self.assertRaisesRegex(TypeError, re.escape(expected_error_message)): issubclass(int, Vec2D) + def test_nonruntime_protocol_interaction_with_evil_classproperty(self): + class classproperty: + def __get__(self, instance, type): + raise RuntimeError("NO") + + class Commentable(Protocol): + evil = classproperty() + + # recognised as a protocol attr, + # but not actually accessed by the protocol metaclass + # (which would raise RuntimeError) for non-runtime protocols. + # See gh-113320 + self.assertEqual(get_protocol_members(Commentable), {"evil"}) + + def test_runtime_protocol_interaction_with_evil_classproperty(self): + class CustomError(Exception): pass + + class classproperty: + def __get__(self, instance, type): + raise CustomError + + with self.assertRaises(TypeError) as cm: + @runtime_checkable + class Commentable(Protocol): + evil = classproperty() + + exc = cm.exception + self.assertEqual( + exc.args[0], + "Failed to determine whether protocol member 'evil' is a method member" + ) + self.assertIs(type(exc.__cause__), CustomError) + class GenericTests(BaseTestCase): @@ -4289,6 +4323,16 @@ class C(B[int]): c.bar = 'abc' self.assertEqual(c.__dict__, {'bar': 'abc'}) + def test_setattr_exceptions(self): + class Immutable[T]: + def __setattr__(self, key, value): + raise RuntimeError("immutable") + + # gh-115165: This used to cause RuntimeError to be raised + # when we tried to set `__orig_class__` on the `Immutable` instance + # returned by the `Immutable[int]()` call + self.assertIsInstance(Immutable[int](), Immutable) + def test_subscripted_generics_as_proxies(self): T = TypeVar('T') class C(Generic[T]): @@ -4886,6 +4930,75 @@ class B(Generic[S]): ... class C(List[int], B): ... self.assertEqual(C.__mro__, (C, list, B, Generic, object)) + def test_multiple_inheritance_non_type_with___mro_entries__(self): + class GoodEntries: + def __mro_entries__(self, bases): + return (object,) + + class A(List[int], GoodEntries()): ... + + self.assertEqual(A.__mro__, (A, list, Generic, object)) + + def test_multiple_inheritance_non_type_without___mro_entries__(self): + # Error should be from the type machinery, not from typing.py + with self.assertRaisesRegex(TypeError, r"^bases must be types"): + class A(List[int], object()): ... + + def test_multiple_inheritance_non_type_bad___mro_entries__(self): + class BadEntries: + def __mro_entries__(self, bases): + return None + + # Error should be from the type machinery, not from typing.py + with self.assertRaisesRegex( + TypeError, + r"^__mro_entries__ must return a tuple", + ): + class A(List[int], BadEntries()): ... + + def test_multiple_inheritance___mro_entries___returns_non_type(self): + class BadEntries: + def __mro_entries__(self, bases): + return (object(),) + + # Error should be from the type machinery, not from typing.py + with self.assertRaisesRegex( + TypeError, + r"^bases must be types", + ): + class A(List[int], BadEntries()): ... + + def test_multiple_inheritance_with_genericalias(self): + class A(typing.Sized, list[int]): ... + + self.assertEqual( + A.__mro__, + (A, collections.abc.Sized, Generic, list, object), + ) + + def test_multiple_inheritance_with_genericalias_2(self): + T = TypeVar("T") + + class BaseSeq(typing.Sequence[T]): ... + class MySeq(List[T], BaseSeq[T]): ... + + self.assertEqual( + MySeq.__mro__, + ( + MySeq, + list, + BaseSeq, + collections.abc.Sequence, + collections.abc.Reversible, + collections.abc.Collection, + collections.abc.Sized, + collections.abc.Iterable, + collections.abc.Container, + Generic, + object, + ), + ) + def test_init_subclass_super_called(self): class FinalException(Exception): pass @@ -5650,7 +5763,7 @@ def fun(x: a): pass def cmp(o1, o2): return o1 == o2 - with infinite_recursion(): + with infinite_recursion(25): r1 = namespace1() r2 = namespace2() self.assertIsNot(r1, r2) @@ -8458,6 +8571,17 @@ def test_instantiate_generic(self): self.assertEqual(MyCount([4, 4, 5]), {4: 2, 5: 1}) self.assertEqual(MyCount[int]([4, 4, 5]), {4: 2, 5: 1}) + def test_instantiate_immutable(self): + class C: + def __setattr__(self, key, value): + raise Exception("should be ignored") + + A = Annotated[C, "a decoration"] + # gh-115165: This used to cause RuntimeError to be raised + # when we tried to set `__orig_class__` on the `C` instance + # returned by the `A()` call + self.assertIsInstance(A(), C) + def test_cannot_instantiate_forward(self): A = Annotated["int", (5, 6)] with self.assertRaises(TypeError): diff --git a/Lib/test/test_unicode_file_functions.py b/Lib/test/test_unicode_file_functions.py index 47619c8807bafe..25c16e3a0b7e43 100644 --- a/Lib/test/test_unicode_file_functions.py +++ b/Lib/test/test_unicode_file_functions.py @@ -5,7 +5,7 @@ import unittest import warnings from unicodedata import normalize -from test.support import os_helper +from test.support import is_apple, os_helper from test import support @@ -23,13 +23,13 @@ '10_\u1fee\u1ffd', ] -# Mac OS X decomposes Unicode names, using Normal Form D. +# Apple platforms decompose Unicode names, using Normal Form D. # http://developer.apple.com/mac/library/qa/qa2001/qa1173.html # "However, most volume formats do not follow the exact specification for # these normal forms. For example, HFS Plus uses a variant of Normal Form D # in which U+2000 through U+2FFF, U+F900 through U+FAFF, and U+2F800 through # U+2FAFF are not decomposed." -if sys.platform != 'darwin': +if not is_apple: filenames.extend([ # Specific code points: NFC(fn), NFD(fn), NFKC(fn) and NFKD(fn) all different '11_\u0385\u03d3\u03d4', @@ -119,11 +119,11 @@ def test_open(self): os.stat(name) self._apply_failure(os.listdir, name, self._listdir_failure) - # Skip the test on darwin, because darwin does normalize the filename to + # Skip the test on Apple platforms, because they don't normalize the filename to # NFD (a variant of Unicode NFD form). Normalize the filename to NFC, NFKC, # NFKD in Python is useless, because darwin will normalize it later and so # open(), os.stat(), etc. don't raise any exception. - @unittest.skipIf(sys.platform == 'darwin', 'irrelevant test on Mac OS X') + @unittest.skipIf(is_apple, 'irrelevant test on Apple platforms') @unittest.skipIf( support.is_emscripten or support.is_wasi, "test fails on Emscripten/WASI when host platform is macOS." @@ -142,10 +142,10 @@ def test_normalize(self): self._apply_failure(os.remove, name) self._apply_failure(os.listdir, name) - # Skip the test on darwin, because darwin uses a normalization different + # Skip the test on Apple platforms, because they use a normalization different # than Python NFD normalization: filenames are different even if we use # Python NFD normalization. - @unittest.skipIf(sys.platform == 'darwin', 'irrelevant test on Mac OS X') + @unittest.skipIf(is_apple, 'irrelevant test on Apple platforms') def test_listdir(self): sf0 = set(self.files) with warnings.catch_warnings(): diff --git a/Lib/test/test_unittest/test_discovery.py b/Lib/test/test_unittest/test_discovery.py index dcb72d73efceab..004898ed431834 100644 --- a/Lib/test/test_unittest/test_discovery.py +++ b/Lib/test/test_unittest/test_discovery.py @@ -571,7 +571,7 @@ def _get_module_from_name(name): result = unittest.TestResult() suite.run(result) self.assertEqual(len(result.skipped), 1) - self.assertEqual(result.testsRun, 0) + self.assertEqual(result.testsRun, 1) self.assertEqual(import_calls, ['my_package']) # Check picklability diff --git a/Lib/test/test_unittest/test_program.py b/Lib/test/test_unittest/test_program.py index f6d52f93e4a25f..7241cf59f73d4f 100644 --- a/Lib/test/test_unittest/test_program.py +++ b/Lib/test/test_unittest/test_program.py @@ -167,6 +167,18 @@ def test_ExitAsDefault(self): 'expected failures=1, unexpected successes=1)\n') self.assertTrue(out.endswith(expected)) + def test_ExitSkippedSuite(self): + stream = BufferedWriter() + with self.assertRaises(SystemExit) as cm: + unittest.main( + argv=["foobar", "-k", "testSkipped"], + testRunner=unittest.TextTestRunner(stream=stream), + testLoader=self.TestLoader(self.FooBar)) + self.assertEqual(cm.exception.code, 0) + out = stream.getvalue() + expected = '\n\nOK (skipped=1)\n' + self.assertTrue(out.endswith(expected)) + def test_ExitEmptySuite(self): stream = BufferedWriter() with self.assertRaises(SystemExit) as cm: @@ -447,8 +459,8 @@ def _join(name): def testParseArgsAbsolutePathsThatCannotBeConverted(self): program = self.program - # even on Windows '/...' is considered absolute by os.path.abspath - argv = ['progname', '/foo/bar/baz.py', '/green/red.py'] + drive = os.path.splitdrive(os.getcwd())[0] + argv = ['progname', f'{drive}/foo/bar/baz.py', f'{drive}/green/red.py'] self._patch_isfile(argv) program.createTests = lambda: None diff --git a/Lib/test/test_unittest/test_skipping.py b/Lib/test/test_unittest/test_skipping.py index 1a6af06d32b433..f146dcac18ecc0 100644 --- a/Lib/test/test_unittest/test_skipping.py +++ b/Lib/test/test_unittest/test_skipping.py @@ -103,16 +103,16 @@ def test_dont_skip(self): pass result = LoggingResult(events) self.assertIs(suite.run(result), result) self.assertEqual(len(result.skipped), 1) - expected = ['addSkip', 'stopTest', 'startTest', - 'addSuccess', 'stopTest'] + expected = ['startTest', 'addSkip', 'stopTest', + 'startTest', 'addSuccess', 'stopTest'] self.assertEqual(events, expected) - self.assertEqual(result.testsRun, 1) + self.assertEqual(result.testsRun, 2) self.assertEqual(result.skipped, [(test_do_skip, "testing")]) self.assertTrue(result.wasSuccessful()) events = [] result = test_do_skip.run() - self.assertEqual(events, ['startTestRun', 'addSkip', + self.assertEqual(events, ['startTestRun', 'startTest', 'addSkip', 'stopTest', 'stopTestRun']) self.assertEqual(result.skipped, [(test_do_skip, "testing")]) @@ -135,13 +135,13 @@ def test_1(self): test = Foo("test_1") suite = unittest.TestSuite([test]) self.assertIs(suite.run(result), result) - self.assertEqual(events, ['addSkip', 'stopTest']) + self.assertEqual(events, ['startTest', 'addSkip', 'stopTest']) self.assertEqual(result.skipped, [(test, "testing")]) self.assertEqual(record, []) events = [] result = test.run() - self.assertEqual(events, ['startTestRun', 'addSkip', + self.assertEqual(events, ['startTestRun', 'startTest', 'addSkip', 'stopTest', 'stopTestRun']) self.assertEqual(result.skipped, [(test, "testing")]) self.assertEqual(record, []) diff --git a/Lib/test/test_unittest/testmock/testmock.py b/Lib/test/test_unittest/testmock/testmock.py index 6af8acc3b0617e..1725406bcfb9e4 100644 --- a/Lib/test/test_unittest/testmock/testmock.py +++ b/Lib/test/test_unittest/testmock/testmock.py @@ -1547,25 +1547,33 @@ def f(x=None): pass mock = Mock(spec=f) mock(1) - with self.assertRaisesRegex( - AssertionError, - '^{}$'.format( - re.escape('Calls not found.\n' - 'Expected: [call()]\n' - ' Actual: [call(1)]'))) as cm: + with self.assertRaises(AssertionError) as cm: mock.assert_has_calls([call()]) + self.assertEqual(str(cm.exception), + 'Calls not found.\n' + 'Expected: [call()]\n' + ' Actual: [call(1)]' + ) self.assertIsNone(cm.exception.__cause__) + uncalled_mock = Mock() + with self.assertRaises(AssertionError) as cm: + uncalled_mock.assert_has_calls([call()]) + self.assertEqual(str(cm.exception), + 'Calls not found.\n' + 'Expected: [call()]\n' + ' Actual: []' + ) + self.assertIsNone(cm.exception.__cause__) - with self.assertRaisesRegex( - AssertionError, - '^{}$'.format( - re.escape( - 'Error processing expected calls.\n' - "Errors: [None, TypeError('too many positional arguments')]\n" - "Expected: [call(), call(1, 2)]\n" - ' Actual: [call(1)]'))) as cm: + with self.assertRaises(AssertionError) as cm: mock.assert_has_calls([call(), call(1, 2)]) + self.assertEqual(str(cm.exception), + 'Error processing expected calls.\n' + "Errors: [None, TypeError('too many positional arguments')]\n" + 'Expected: [call(), call(1, 2)]\n' + ' Actual: [call(1)]' + ) self.assertIsInstance(cm.exception.__cause__, TypeError) def test_assert_any_call(self): diff --git a/Lib/test/test_urllib2.py b/Lib/test/test_urllib2.py index 99c9e24994732f..fa528a675892b5 100644 --- a/Lib/test/test_urllib2.py +++ b/Lib/test/test_urllib2.py @@ -1,6 +1,7 @@ import unittest from test import support from test.support import os_helper +from test.support import requires_subprocess from test.support import warnings_helper from test import test_urllib from unittest import mock @@ -998,6 +999,7 @@ def test_http_body_fileobj(self): file_obj.close() + @requires_subprocess() def test_http_body_pipe(self): # A file reading from a pipe. # A pipe cannot be seek'ed. There is no way to determine the diff --git a/Lib/test/test_userdict.py b/Lib/test/test_userdict.py index 9a03f2d04ce970..61e79f553e8ec9 100644 --- a/Lib/test/test_userdict.py +++ b/Lib/test/test_userdict.py @@ -215,7 +215,7 @@ class G(collections.UserDict): # Decorate existing test with recursion limit, because # the test is for C structure, but `UserDict` is a Python structure. - test_repr_deep = support.infinite_recursion()( + test_repr_deep = support.infinite_recursion(25)( mapping_tests.TestHashMappingProtocol.test_repr_deep, ) diff --git a/Lib/test/test_userlist.py b/Lib/test/test_userlist.py index 76d253753528b0..312702c8e398b9 100644 --- a/Lib/test/test_userlist.py +++ b/Lib/test/test_userlist.py @@ -69,7 +69,7 @@ def test_userlist_copy(self): # Decorate existing test with recursion limit, because # the test is for C structure, but `UserList` is a Python structure. - test_repr_deep = support.infinite_recursion()( + test_repr_deep = support.infinite_recursion(25)( list_tests.CommonTest.test_repr_deep, ) diff --git a/Lib/test/test_venv.py b/Lib/test/test_venv.py index 8ecb23ff384362..ba31beb81e80b0 100644 --- a/Lib/test/test_venv.py +++ b/Lib/test/test_venv.py @@ -19,8 +19,8 @@ import tempfile from test.support import (captured_stdout, captured_stderr, skip_if_broken_multiprocessing_synchronize, verbose, - requires_subprocess, is_emscripten, is_wasi, - requires_venv_with_pip, TEST_HOME_DIR, + requires_subprocess, is_apple_mobile, is_emscripten, + is_wasi, requires_venv_with_pip, TEST_HOME_DIR, requires_resource, copy_python_src_ignore) from test.support.os_helper import (can_symlink, EnvironmentVarGuard, rmtree) import unittest @@ -39,8 +39,10 @@ or sys._base_executable != sys.executable, 'cannot run venv.create from within a venv on this platform') -if is_emscripten or is_wasi: - raise unittest.SkipTest("venv is not available on Emscripten/WASI.") +# Skip tests on WASM platforms, plus iOS/tvOS/watchOS +if is_apple_mobile or is_emscripten or is_wasi: + raise unittest.SkipTest(f"venv tests not required on {sys.platform}") + @requires_subprocess() def check_output(cmd, encoding=None): @@ -169,7 +171,7 @@ def test_config_file_command_key(self): ('--clear', 'clear', True), ('--upgrade', 'upgrade', True), ('--upgrade-deps', 'upgrade_deps', True), - ('--prompt', 'prompt', True), + ('--prompt="foobar"', 'prompt', 'foobar'), ('--without-scm-ignore-files', 'scm_ignore_files', frozenset()), ] for opt, attr, value in options: @@ -201,7 +203,7 @@ def test_prompt(self): self.run_with_capture(builder.create, self.env_dir) context = builder.ensure_directories(self.env_dir) data = self.get_text_file_contents('pyvenv.cfg') - self.assertEqual(context.prompt, '(%s) ' % env_name) + self.assertEqual(context.prompt, env_name) self.assertNotIn("prompt = ", data) rmtree(self.env_dir) @@ -209,7 +211,7 @@ def test_prompt(self): self.run_with_capture(builder.create, self.env_dir) context = builder.ensure_directories(self.env_dir) data = self.get_text_file_contents('pyvenv.cfg') - self.assertEqual(context.prompt, '(My prompt) ') + self.assertEqual(context.prompt, 'My prompt') self.assertIn("prompt = 'My prompt'\n", data) rmtree(self.env_dir) @@ -218,13 +220,19 @@ def test_prompt(self): self.run_with_capture(builder.create, self.env_dir) context = builder.ensure_directories(self.env_dir) data = self.get_text_file_contents('pyvenv.cfg') - self.assertEqual(context.prompt, '(%s) ' % cwd) + self.assertEqual(context.prompt, cwd) self.assertIn("prompt = '%s'\n" % cwd, data) def test_upgrade_dependencies(self): builder = venv.EnvBuilder() - bin_path = 'Scripts' if sys.platform == 'win32' else 'bin' + bin_path = 'bin' python_exe = os.path.split(sys.executable)[1] + if sys.platform == 'win32': + bin_path = 'Scripts' + if os.path.normcase(os.path.splitext(python_exe)[0]).endswith('_d'): + python_exe = 'python_d.exe' + else: + python_exe = 'python.exe' with tempfile.TemporaryDirectory() as fake_env_dir: expect_exe = os.path.normcase( os.path.join(fake_env_dir, bin_path, python_exe) @@ -283,7 +291,9 @@ def test_sysconfig(self): # build environment ('is_python_build()', str(sysconfig.is_python_build())), ('get_makefile_filename()', sysconfig.get_makefile_filename()), - ('get_config_h_filename()', sysconfig.get_config_h_filename())): + ('get_config_h_filename()', sysconfig.get_config_h_filename()), + ('get_config_var("Py_GIL_DISABLED")', + str(sysconfig.get_config_var("Py_GIL_DISABLED")))): with self.subTest(call): cmd[2] = 'import sysconfig; print(sysconfig.%s)' % call out, err = check_output(cmd, encoding='utf-8') @@ -315,7 +325,9 @@ def test_sysconfig_symlinks(self): # build environment ('is_python_build()', str(sysconfig.is_python_build())), ('get_makefile_filename()', sysconfig.get_makefile_filename()), - ('get_config_h_filename()', sysconfig.get_config_h_filename())): + ('get_config_h_filename()', sysconfig.get_config_h_filename()), + ('get_config_var("Py_GIL_DISABLED")', + str(sysconfig.get_config_var("Py_GIL_DISABLED")))): with self.subTest(call): cmd[2] = 'import sysconfig; print(sysconfig.%s)' % call out, err = check_output(cmd, encoding='utf-8') @@ -324,7 +336,8 @@ def test_sysconfig_symlinks(self): ('executable', self.envpy()), # Usually compare to sys.executable, but if we're running in our own # venv then we really need to compare to our base executable - ('_base_executable', sys._base_executable), + # HACK: Test fails on POSIX with unversioned binary (PR gh-113033) + #('_base_executable', sys._base_executable), ): with self.subTest(attr): cmd[2] = f'import sys; print(sys.{attr})' diff --git a/Lib/test/test_webbrowser.py b/Lib/test/test_webbrowser.py index ca481c57c3d972..8c074cb28a87e3 100644 --- a/Lib/test/test_webbrowser.py +++ b/Lib/test/test_webbrowser.py @@ -307,6 +307,7 @@ def test_get(self): webbrowser.get('fakebrowser') self.assertIsNotNone(webbrowser._tryorder) + @unittest.skipIf(" " in sys.executable, "test assumes no space in path (GH-114452)") def test_synthesize(self): webbrowser = import_helper.import_fresh_module('webbrowser') name = os.path.basename(sys.executable).lower() diff --git a/Lib/test/test_winconsoleio.py b/Lib/test/test_winconsoleio.py index 70a85552cc03b0..209e4464e1a5c0 100644 --- a/Lib/test/test_winconsoleio.py +++ b/Lib/test/test_winconsoleio.py @@ -6,7 +6,7 @@ import sys import tempfile import unittest -from test.support import os_helper +from test.support import os_helper, requires_resource if sys.platform != 'win32': raise unittest.SkipTest("test only relevant on win32") @@ -98,6 +98,16 @@ def test_open_name(self): self.assertIsInstance(f, ConIO) f.close() + def test_subclass_repr(self): + class TestSubclass(ConIO): + pass + + f = TestSubclass("CON") + with f: + self.assertIn(TestSubclass.__name__, repr(f)) + + self.assertIn(TestSubclass.__name__, repr(f)) + @unittest.skipIf(sys.getwindowsversion()[:2] <= (6, 1), "test does not work on Windows 7 and earlier") def test_conin_conout_names(self): @@ -140,6 +150,7 @@ def assertStdinRoundTrip(self, text): sys.stdin = old_stdin self.assertEqual(actual, text) + @requires_resource('console') def test_input(self): # ASCII self.assertStdinRoundTrip('abc123') @@ -154,6 +165,7 @@ def test_input_nonbmp(self): # Non-BMP self.assertStdinRoundTrip('\U00100000\U0010ffff\U0010fffd') + @requires_resource('console') def test_partial_reads(self): # Test that reading less than 1 full character works when stdin # contains multibyte UTF-8 sequences @@ -189,6 +201,7 @@ def test_partial_surrogate_reads(self): self.assertEqual(actual, expected, 'stdin.read({})'.format(read_count)) + @requires_resource('console') def test_ctrl_z(self): with open('CONIN$', 'rb', buffering=0) as stdin: source = '\xC4\x1A\r\n'.encode('utf-16-le') diff --git a/Lib/test/test_xml_etree.py b/Lib/test/test_xml_etree.py index b9e7937b0bbc00..a435ec7822ea0c 100644 --- a/Lib/test/test_xml_etree.py +++ b/Lib/test/test_xml_etree.py @@ -536,7 +536,9 @@ def test_iterparse(self): iterparse = ET.iterparse context = iterparse(SIMPLE_XMLFILE) + self.assertIsNone(context.root) action, elem = next(context) + self.assertIsNone(context.root) self.assertEqual((action, elem.tag), ('end', 'element')) self.assertEqual([(action, elem.tag) for action, elem in context], [ ('end', 'element'), @@ -553,6 +555,17 @@ def test_iterparse(self): ('end', '{namespace}root'), ]) + with open(SIMPLE_XMLFILE, 'rb') as source: + context = iterparse(source) + action, elem = next(context) + self.assertEqual((action, elem.tag), ('end', 'element')) + self.assertEqual([(action, elem.tag) for action, elem in context], [ + ('end', 'element'), + ('end', 'empty-element'), + ('end', 'root'), + ]) + self.assertEqual(context.root.tag, 'root') + events = () context = iterparse(SIMPLE_XMLFILE, events) self.assertEqual([(action, elem.tag) for action, elem in context], []) @@ -644,12 +657,81 @@ def test_iterparse(self): # Not exhausting the iterator still closes the resource (bpo-43292) with warnings_helper.check_no_resource_warning(self): - it = iterparse(TESTFN) + it = iterparse(SIMPLE_XMLFILE) del it + with warnings_helper.check_no_resource_warning(self): + it = iterparse(SIMPLE_XMLFILE) + it.close() + del it + + with warnings_helper.check_no_resource_warning(self): + it = iterparse(SIMPLE_XMLFILE) + action, elem = next(it) + self.assertEqual((action, elem.tag), ('end', 'element')) + del it, elem + + with warnings_helper.check_no_resource_warning(self): + it = iterparse(SIMPLE_XMLFILE) + action, elem = next(it) + it.close() + self.assertEqual((action, elem.tag), ('end', 'element')) + del it, elem + with self.assertRaises(FileNotFoundError): iterparse("nonexistent") + def test_iterparse_close(self): + iterparse = ET.iterparse + + it = iterparse(SIMPLE_XMLFILE) + it.close() + with self.assertRaises(StopIteration): + next(it) + it.close() # idempotent + + with open(SIMPLE_XMLFILE, 'rb') as source: + it = iterparse(source) + it.close() + self.assertFalse(source.closed) + with self.assertRaises(StopIteration): + next(it) + it.close() # idempotent + + it = iterparse(SIMPLE_XMLFILE) + action, elem = next(it) + self.assertEqual((action, elem.tag), ('end', 'element')) + it.close() + with self.assertRaises(StopIteration): + next(it) + it.close() # idempotent + + with open(SIMPLE_XMLFILE, 'rb') as source: + it = iterparse(source) + action, elem = next(it) + self.assertEqual((action, elem.tag), ('end', 'element')) + it.close() + self.assertFalse(source.closed) + with self.assertRaises(StopIteration): + next(it) + it.close() # idempotent + + it = iterparse(SIMPLE_XMLFILE) + list(it) + it.close() + with self.assertRaises(StopIteration): + next(it) + it.close() # idempotent + + with open(SIMPLE_XMLFILE, 'rb') as source: + it = iterparse(source) + list(it) + it.close() + self.assertFalse(source.closed) + with self.assertRaises(StopIteration): + next(it) + it.close() # idempotent + def test_writefile(self): elem = ET.Element("tag") elem.text = "text" @@ -3042,8 +3124,7 @@ def test_basic(self): # With an explicit parser too (issue #9708) sourcefile = serialize(doc, to_string=False) parser = ET.XMLParser(target=ET.TreeBuilder()) - self.assertEqual(next(ET.iterparse(sourcefile, parser=parser))[0], - 'end') + self.assertEqual(next(ET.iterparse(sourcefile, parser=parser))[0], 'end') tree = ET.ElementTree(None) self.assertRaises(AttributeError, tree.iter) diff --git a/Lib/test/test_zipfile/_path/test_path.py b/Lib/test/test_zipfile/_path/test_path.py index c66cb3cba69ebd..171ab6fdb5fc28 100644 --- a/Lib/test/test_zipfile/_path/test_path.py +++ b/Lib/test/test_zipfile/_path/test_path.py @@ -577,3 +577,15 @@ def test_getinfo_missing(self, alpharep): zipfile.Path(alpharep) with self.assertRaises(KeyError): alpharep.getinfo('does-not-exist') + + def test_root_folder_in_zipfile(self): + """ + gh-112795: Some tools or self constructed codes will add '/' folder to + the zip file, this is a strange behavior, but we should support it. + """ + in_memory_file = io.BytesIO() + zf = zipfile.ZipFile(in_memory_file, "w") + zf.mkdir('/') + zf.writestr('./a.txt', 'aaa') + tmpdir = pathlib.Path(self.fixtures.enter_context(temp_dir())) + zf.extractall(tmpdir) diff --git a/Lib/test/test_zipfile/test_core.py b/Lib/test/test_zipfile/test_core.py index a51764b9297363..087fa8d65cc336 100644 --- a/Lib/test/test_zipfile/test_core.py +++ b/Lib/test/test_zipfile/test_core.py @@ -18,6 +18,7 @@ from tempfile import TemporaryFile from random import randint, random, randbytes +from test import archiver_tests from test.support import script_helper from test.support import ( findfile, requires_zlib, requires_bz2, requires_lzma, @@ -315,7 +316,7 @@ def test_writestr_compresslevel(self): # Compression level follows the constructor. a_info = zipfp.getinfo('a.txt') self.assertEqual(a_info.compress_type, self.compression) - self.assertEqual(a_info._compresslevel, 1) + self.assertEqual(a_info.compress_level, 1) # Compression level is overridden. b_info = zipfp.getinfo('b.txt') @@ -408,7 +409,7 @@ def test_per_file_compresslevel(self): one_info = zipfp.getinfo('compress_1') nine_info = zipfp.getinfo('compress_9') self.assertEqual(one_info._compresslevel, 1) - self.assertEqual(nine_info._compresslevel, 9) + self.assertEqual(nine_info.compress_level, 9) def test_writing_errors(self): class BrokenFile(io.BytesIO): @@ -1687,6 +1688,33 @@ def _test_extract_hackers_arcnames(self, hacknames): unlink(TESTFN2) +class OverwriteTests(archiver_tests.OverwriteTests, unittest.TestCase): + testdir = TESTFN + + @classmethod + def setUpClass(cls): + p = cls.ar_with_file = TESTFN + '-with-file.zip' + cls.addClassCleanup(unlink, p) + with zipfile.ZipFile(p, 'w') as zipfp: + zipfp.writestr('test', b'newcontent') + + p = cls.ar_with_dir = TESTFN + '-with-dir.zip' + cls.addClassCleanup(unlink, p) + with zipfile.ZipFile(p, 'w') as zipfp: + zipfp.mkdir('test') + + p = cls.ar_with_implicit_dir = TESTFN + '-with-implicit-dir.zip' + cls.addClassCleanup(unlink, p) + with zipfile.ZipFile(p, 'w') as zipfp: + zipfp.writestr('test/file', b'newcontent') + + def open(self, path): + return zipfile.ZipFile(path, 'r') + + def extractall(self, ar): + ar.extractall(self.testdir) + + class OtherTests(unittest.TestCase): def test_open_via_zip_info(self): # Create the ZIP archive @@ -2272,6 +2300,66 @@ def test_decompress_without_3rd_party_library(self): with zipfile.ZipFile(zip_file) as zf: self.assertRaises(RuntimeError, zf.extract, 'a.txt') + @requires_zlib() + def test_full_overlap(self): + data = ( + b'PK\x03\x04\x14\x00\x00\x00\x08\x00\xa0lH\x05\xe2\x1e' + b'8\xbb\x10\x00\x00\x00\t\x04\x00\x00\x01\x00\x00\x00a\xed' + b'\xc0\x81\x08\x00\x00\x00\xc00\xd6\xfbK\\d\x0b`P' + b'K\x01\x02\x14\x00\x14\x00\x00\x00\x08\x00\xa0lH\x05\xe2' + b'\x1e8\xbb\x10\x00\x00\x00\t\x04\x00\x00\x01\x00\x00\x00\x00' + b'\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00aPK' + b'\x01\x02\x14\x00\x14\x00\x00\x00\x08\x00\xa0lH\x05\xe2\x1e' + b'8\xbb\x10\x00\x00\x00\t\x04\x00\x00\x01\x00\x00\x00\x00\x00' + b'\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00bPK\x05' + b'\x06\x00\x00\x00\x00\x02\x00\x02\x00^\x00\x00\x00/\x00\x00' + b'\x00\x00\x00' + ) + with zipfile.ZipFile(io.BytesIO(data), 'r') as zipf: + self.assertEqual(zipf.namelist(), ['a', 'b']) + zi = zipf.getinfo('a') + self.assertEqual(zi.header_offset, 0) + self.assertEqual(zi.compress_size, 16) + self.assertEqual(zi.file_size, 1033) + zi = zipf.getinfo('b') + self.assertEqual(zi.header_offset, 0) + self.assertEqual(zi.compress_size, 16) + self.assertEqual(zi.file_size, 1033) + self.assertEqual(len(zipf.read('a')), 1033) + with self.assertRaisesRegex(zipfile.BadZipFile, 'File name.*differ'): + zipf.read('b') + + @requires_zlib() + def test_quoted_overlap(self): + data = ( + b'PK\x03\x04\x14\x00\x00\x00\x08\x00\xa0lH\x05Y\xfc' + b'8\x044\x00\x00\x00(\x04\x00\x00\x01\x00\x00\x00a\x00' + b'\x1f\x00\xe0\xffPK\x03\x04\x14\x00\x00\x00\x08\x00\xa0l' + b'H\x05\xe2\x1e8\xbb\x10\x00\x00\x00\t\x04\x00\x00\x01\x00' + b'\x00\x00b\xed\xc0\x81\x08\x00\x00\x00\xc00\xd6\xfbK\\' + b'd\x0b`PK\x01\x02\x14\x00\x14\x00\x00\x00\x08\x00\xa0' + b'lH\x05Y\xfc8\x044\x00\x00\x00(\x04\x00\x00\x01' + b'\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00' + b'\x00aPK\x01\x02\x14\x00\x14\x00\x00\x00\x08\x00\xa0l' + b'H\x05\xe2\x1e8\xbb\x10\x00\x00\x00\t\x04\x00\x00\x01\x00' + b'\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00$\x00\x00\x00' + b'bPK\x05\x06\x00\x00\x00\x00\x02\x00\x02\x00^\x00\x00' + b'\x00S\x00\x00\x00\x00\x00' + ) + with zipfile.ZipFile(io.BytesIO(data), 'r') as zipf: + self.assertEqual(zipf.namelist(), ['a', 'b']) + zi = zipf.getinfo('a') + self.assertEqual(zi.header_offset, 0) + self.assertEqual(zi.compress_size, 52) + self.assertEqual(zi.file_size, 1064) + zi = zipf.getinfo('b') + self.assertEqual(zi.header_offset, 36) + self.assertEqual(zi.compress_size, 16) + self.assertEqual(zi.file_size, 1033) + with self.assertRaisesRegex(zipfile.BadZipFile, 'Overlapped entries'): + zipf.read('a') + self.assertEqual(len(zipf.read('b')), 1033) + def tearDown(self): unlink(TESTFN) unlink(TESTFN2) @@ -2899,7 +2987,7 @@ def test_create_directory_with_write(self): directory = os.path.join(TESTFN2, "directory2") os.mkdir(directory) - mode = os.stat(directory).st_mode + mode = os.stat(directory).st_mode & 0xFFFF zf.write(directory, arcname="directory2/") zinfo = zf.filelist[1] self.assertEqual(zinfo.filename, "directory2/") @@ -2951,6 +3039,17 @@ def test_from_dir(self): self.assertEqual(zi.compress_type, zipfile.ZIP_STORED) self.assertEqual(zi.file_size, 0) + def test_compresslevel_property(self): + zinfo = zipfile.ZipInfo("xxx") + self.assertFalse(zinfo._compresslevel) + self.assertFalse(zinfo.compress_level) + zinfo._compresslevel = 99 # test the legacy @property.setter + self.assertEqual(zinfo.compress_level, 99) + self.assertEqual(zinfo._compresslevel, 99) + zinfo.compress_level = 8 + self.assertEqual(zinfo.compress_level, 8) + self.assertEqual(zinfo._compresslevel, 8) + class CommandLineTest(unittest.TestCase): diff --git a/Lib/test/test_zipimport_support.py b/Lib/test/test_zipimport_support.py index 7bf50a33728e53..71039d2a8e7ab9 100644 --- a/Lib/test/test_zipimport_support.py +++ b/Lib/test/test_zipimport_support.py @@ -29,8 +29,9 @@ # test_cmd_line_script (covers the zipimport support in runpy) # Retrieve some helpers from other test cases -from test import (test_doctest, sample_doctest, sample_doctest_no_doctests, - sample_doctest_no_docstrings) +from test.test_doctest import (test_doctest, + sample_doctest, sample_doctest_no_doctests, + sample_doctest_no_docstrings) def _run_object_doctest(obj, module): @@ -100,18 +101,18 @@ def test_doctest_issue4197(self): # everything still works correctly test_src = inspect.getsource(test_doctest) test_src = test_src.replace( - "from test import test_doctest", + "from test.test_doctest import test_doctest", "import test_zipped_doctest as test_doctest") - test_src = test_src.replace("test.test_doctest", + test_src = test_src.replace("test.test_doctest.test_doctest", "test_zipped_doctest") - test_src = test_src.replace("test.sample_doctest", + test_src = test_src.replace("test.test_doctest.sample_doctest", "sample_zipped_doctest") # The sample doctest files rewritten to include in the zipped version. sample_sources = {} for mod in [sample_doctest, sample_doctest_no_doctests, sample_doctest_no_docstrings]: src = inspect.getsource(mod) - src = src.replace("test.test_doctest", "test_zipped_doctest") + src = src.replace("test.test_doctest.test_doctest", "test_zipped_doctest") # Rewrite the module name so that, for example, # "test.sample_doctest" becomes "sample_zipped_doctest". mod_name = mod.__name__.split(".")[-1] diff --git a/Lib/test/test_zoneinfo/test_zoneinfo.py b/Lib/test/test_zoneinfo/test_zoneinfo.py index 3766ceac8385f2..8414721555731e 100644 --- a/Lib/test/test_zoneinfo/test_zoneinfo.py +++ b/Lib/test/test_zoneinfo/test_zoneinfo.py @@ -17,9 +17,10 @@ from datetime import date, datetime, time, timedelta, timezone from functools import cached_property +from test.support import MISSING_C_DOCSTRINGS from test.test_zoneinfo import _support as test_support from test.test_zoneinfo._support import OS_ENV_LOCK, TZPATH_TEST_LOCK, ZoneInfoTestBase -from test.support.import_helper import import_module +from test.support.import_helper import import_module, CleanImport lzma = import_module('lzma') py_zoneinfo, c_zoneinfo = test_support.get_modules() @@ -35,6 +36,7 @@ TEMP_DIR = None DATA_DIR = pathlib.Path(__file__).parent / "data" ZONEINFO_JSON = DATA_DIR / "zoneinfo_data.json" +DRIVE = os.path.splitdrive('x:')[0] # Useful constants ZERO = timedelta(0) @@ -404,6 +406,8 @@ def test_time_fixed_offset(self): class CZoneInfoTest(ZoneInfoTest): module = c_zoneinfo + @unittest.skipIf(MISSING_C_DOCSTRINGS, + "Signature information for builtins requires docstrings") def test_signatures(self): """Ensure that C module has valid method signatures.""" import inspect @@ -1676,8 +1680,8 @@ def test_env_variable(self): """Tests that the environment variable works with reset_tzpath.""" new_paths = [ ("", []), - ("/etc/zoneinfo", ["/etc/zoneinfo"]), - (f"/a/b/c{os.pathsep}/d/e/f", ["/a/b/c", "/d/e/f"]), + (f"{DRIVE}/etc/zoneinfo", [f"{DRIVE}/etc/zoneinfo"]), + (f"{DRIVE}/a/b/c{os.pathsep}{DRIVE}/d/e/f", [f"{DRIVE}/a/b/c", f"{DRIVE}/d/e/f"]), ] for new_path_var, expected_result in new_paths: @@ -1691,22 +1695,22 @@ def test_env_variable_relative_paths(self): test_cases = [ [("path/to/somewhere",), ()], [ - ("/usr/share/zoneinfo", "path/to/somewhere",), - ("/usr/share/zoneinfo",), + (f"{DRIVE}/usr/share/zoneinfo", "path/to/somewhere",), + (f"{DRIVE}/usr/share/zoneinfo",), ], [("../relative/path",), ()], [ - ("/usr/share/zoneinfo", "../relative/path",), - ("/usr/share/zoneinfo",), + (f"{DRIVE}/usr/share/zoneinfo", "../relative/path",), + (f"{DRIVE}/usr/share/zoneinfo",), ], [("path/to/somewhere", "../relative/path",), ()], [ ( - "/usr/share/zoneinfo", + f"{DRIVE}/usr/share/zoneinfo", "path/to/somewhere", "../relative/path", ), - ("/usr/share/zoneinfo",), + (f"{DRIVE}/usr/share/zoneinfo",), ], ] @@ -1716,17 +1720,30 @@ def test_env_variable_relative_paths(self): with self.subTest("warning", path_var=path_var): # Note: Per PEP 615 the warning is implementation-defined # behavior, other implementations need not warn. - with self.assertWarns(self.module.InvalidTZPathWarning): + with self.assertWarns(self.module.InvalidTZPathWarning) as w: self.module.reset_tzpath() + self.assertEqual(w.warnings[0].filename, __file__) tzpath = self.module.TZPATH with self.subTest("filtered", path_var=path_var): self.assertSequenceEqual(tzpath, expected_paths) + def test_env_variable_relative_paths_warning_location(self): + path_var = "path/to/somewhere" + + with self.python_tzpath_context(path_var): + with CleanImport("zoneinfo", "zoneinfo._tzpath"): + with self.assertWarns(RuntimeWarning) as w: + import zoneinfo + InvalidTZPathWarning = zoneinfo.InvalidTZPathWarning + self.assertIsInstance(w.warnings[0].message, InvalidTZPathWarning) + # It should represent the current file: + self.assertEqual(w.warnings[0].filename, __file__) + def test_reset_tzpath_kwarg(self): - self.module.reset_tzpath(to=["/a/b/c"]) + self.module.reset_tzpath(to=[f"{DRIVE}/a/b/c"]) - self.assertSequenceEqual(self.module.TZPATH, ("/a/b/c",)) + self.assertSequenceEqual(self.module.TZPATH, (f"{DRIVE}/a/b/c",)) def test_reset_tzpath_relative_paths(self): bad_values = [ @@ -1755,8 +1772,8 @@ def test_tzpath_type_error(self): self.module.reset_tzpath(bad_value) def test_tzpath_attribute(self): - tzpath_0 = ["/one", "/two"] - tzpath_1 = ["/three"] + tzpath_0 = [f"{DRIVE}/one", f"{DRIVE}/two"] + tzpath_1 = [f"{DRIVE}/three"] with self.tzpath_context(tzpath_0): query_0 = self.module.TZPATH diff --git a/Lib/test/time_hashlib.py b/Lib/test/time_hashlib.py deleted file mode 100644 index 55ebac62912fe1..00000000000000 --- a/Lib/test/time_hashlib.py +++ /dev/null @@ -1,88 +0,0 @@ -# It's intended that this script be run by hand. It runs speed tests on -# hashlib functions; it does not test for correctness. - -import sys -import time -import hashlib - - -def creatorFunc(): - raise RuntimeError("eek, creatorFunc not overridden") - -def test_scaled_msg(scale, name): - iterations = 106201//scale * 20 - longStr = b'Z'*scale - - localCF = creatorFunc - start = time.perf_counter() - for f in range(iterations): - x = localCF(longStr).digest() - end = time.perf_counter() - - print(('%2.2f' % (end-start)), "seconds", iterations, "x", len(longStr), "bytes", name) - -def test_create(): - start = time.perf_counter() - for f in range(20000): - d = creatorFunc() - end = time.perf_counter() - - print(('%2.2f' % (end-start)), "seconds", '[20000 creations]') - -def test_zero(): - start = time.perf_counter() - for f in range(20000): - x = creatorFunc().digest() - end = time.perf_counter() - - print(('%2.2f' % (end-start)), "seconds", '[20000 "" digests]') - - - -hName = sys.argv[1] - -# -# setup our creatorFunc to test the requested hash -# -if hName in ('_md5', '_sha'): - exec('import '+hName) - exec('creatorFunc = '+hName+'.new') - print("testing speed of old", hName, "legacy interface") -elif hName == '_hashlib' and len(sys.argv) > 3: - import _hashlib - exec('creatorFunc = _hashlib.%s' % sys.argv[2]) - print("testing speed of _hashlib.%s" % sys.argv[2], getattr(_hashlib, sys.argv[2])) -elif hName == '_hashlib' and len(sys.argv) == 3: - import _hashlib - exec('creatorFunc = lambda x=_hashlib.new : x(%r)' % sys.argv[2]) - print("testing speed of _hashlib.new(%r)" % sys.argv[2]) -elif hasattr(hashlib, hName) and hasattr(getattr(hashlib, hName), '__call__'): - creatorFunc = getattr(hashlib, hName) - print("testing speed of hashlib."+hName, getattr(hashlib, hName)) -else: - exec("creatorFunc = lambda x=hashlib.new : x(%r)" % hName) - print("testing speed of hashlib.new(%r)" % hName) - -try: - test_create() -except ValueError: - print() - print("pass argument(s) naming the hash to run a speed test on:") - print(" '_md5' and '_sha' test the legacy builtin md5 and sha") - print(" '_hashlib' 'openssl_hName' 'fast' tests the builtin _hashlib") - print(" '_hashlib' 'hName' tests builtin _hashlib.new(shaFOO)") - print(" 'hName' tests the hashlib.hName() implementation if it exists") - print(" otherwise it uses hashlib.new(hName).") - print() - raise - -test_zero() -test_scaled_msg(scale=106201, name='[huge data]') -test_scaled_msg(scale=10620, name='[large data]') -test_scaled_msg(scale=1062, name='[medium data]') -test_scaled_msg(scale=424, name='[4*small data]') -test_scaled_msg(scale=336, name='[3*small data]') -test_scaled_msg(scale=212, name='[2*small data]') -test_scaled_msg(scale=106, name='[small data]') -test_scaled_msg(scale=creatorFunc().digest_size, name='[digest_size data]') -test_scaled_msg(scale=10, name='[tiny data]') diff --git a/Lib/test/setuptools-67.6.1-py3-none-any.whl b/Lib/test/wheeldata/setuptools-67.6.1-py3-none-any.whl similarity index 100% rename from Lib/test/setuptools-67.6.1-py3-none-any.whl rename to Lib/test/wheeldata/setuptools-67.6.1-py3-none-any.whl diff --git a/Lib/test/wheel-0.40.0-py3-none-any.whl b/Lib/test/wheeldata/wheel-0.40.0-py3-none-any.whl similarity index 100% rename from Lib/test/wheel-0.40.0-py3-none-any.whl rename to Lib/test/wheeldata/wheel-0.40.0-py3-none-any.whl diff --git a/Lib/threading.py b/Lib/threading.py index 85aff58968082d..b6ff00acadd58f 100644 --- a/Lib/threading.py +++ b/Lib/threading.py @@ -3,9 +3,7 @@ import os as _os import sys as _sys import _thread -import functools import warnings -import _weakref from time import monotonic as _time from _weakrefset import WeakSet @@ -37,6 +35,7 @@ _start_joinable_thread = _thread.start_joinable_thread _daemon_threads_allowed = _thread.daemon_threads_allowed _allocate_lock = _thread.allocate_lock +_LockType = _thread.LockType _set_sentinel = _thread._set_sentinel get_ident = _thread.get_ident _is_main_interpreter = _thread._is_main_interpreter @@ -54,6 +53,13 @@ TIMEOUT_MAX = _thread.TIMEOUT_MAX del _thread +# get thread-local implementation, either from the thread +# module, or from the python fallback + +try: + from _thread import _local as local +except ImportError: + from _threading_local import local # Support for profile and trace hooks @@ -108,7 +114,7 @@ def gettrace(): # Synchronization classes -Lock = _allocate_lock +Lock = _LockType def RLock(*args, **kwargs): """Factory function that returns a new reentrant lock. @@ -943,7 +949,6 @@ def _after_fork(self, new_ident=None): # This thread is alive. self._ident = new_ident if self._handle is not None: - self._handle.after_fork_alive() assert self._handle.ident == new_ident # bpo-42350: If the fork happens when the thread is already stopped # (ex: after threading._shutdown() has been called), _tstate_lock @@ -959,9 +964,7 @@ def _after_fork(self, new_ident=None): self._is_stopped = True self._tstate_lock = None self._join_lock = None - if self._handle is not None: - self._handle.after_fork_dead() - self._handle = None + self._handle = None def __repr__(self): assert self._initialized, "Thread.__init__() was not called" @@ -1476,10 +1479,36 @@ def __init__(self): _active[self._ident] = self +# Helper thread-local instance to detect when a _DummyThread +# is collected. Not a part of the public API. +_thread_local_info = local() + + +class _DeleteDummyThreadOnDel: + ''' + Helper class to remove a dummy thread from threading._active on __del__. + ''' + + def __init__(self, dummy_thread): + self._dummy_thread = dummy_thread + self._tident = dummy_thread.ident + # Put the thread on a thread local variable so that when + # the related thread finishes this instance is collected. + # + # Note: no other references to this instance may be created. + # If any client code creates a reference to this instance, + # the related _DummyThread will be kept forever! + _thread_local_info._track_dummy_thread_ref = self + + def __del__(self): + with _active_limbo_lock: + if _active.get(self._tident) is self._dummy_thread: + _active.pop(self._tident, None) + + # Dummy thread class to represent threads not started here. -# These aren't garbage collected when they die, nor can they be waited for. -# If they invoke anything in threading.py that calls current_thread(), they -# leave an entry in the _active dict forever after. +# These should be added to `_active` and removed automatically +# when they die, although they can't be waited for. # Their purpose is to return *something* from current_thread(). # They are marked as daemon threads so we won't wait for them # when we exit (conform previous semantics). @@ -1489,13 +1518,13 @@ class _DummyThread(Thread): def __init__(self): Thread.__init__(self, name=_newname("Dummy-%d"), daemon=_daemon_threads_allowed()) - self._started.set() self._set_ident() if _HAVE_THREAD_NATIVE_ID: self._set_native_id() with _active_limbo_lock: _active[self._ident] = self + _DeleteDummyThreadOnDel(self) def _stop(self): pass @@ -1508,6 +1537,14 @@ def is_alive(self): def join(self, timeout=None): raise RuntimeError("cannot join a dummy thread") + def _after_fork(self, new_ident=None): + if new_ident is not None: + self.__class__ = _MainThread + self._name = 'MainThread' + self._daemonic = False + self._set_tstate_lock() + Thread._after_fork(self, new_ident=new_ident) + # Global API functions @@ -1589,8 +1626,7 @@ def _register_atexit(func, *arg, **kwargs): if _SHUTTING_DOWN: raise RuntimeError("can't register atexit after shutdown") - call = functools.partial(func, *arg, **kwargs) - _threading_atexits.append(call) + _threading_atexits.append(lambda: func(*arg, **kwargs)) from _thread import stack_size @@ -1669,14 +1705,6 @@ def main_thread(): # XXX Figure this out for subinterpreters. (See gh-75698.) return _main_thread -# get thread-local implementation, either from the thread -# module, or from the python fallback - -try: - from _thread import _local as local -except ImportError: - from _threading_local import local - def _after_fork(): """ diff --git a/Lib/tkinter/__init__.py b/Lib/tkinter/__init__.py index 124882420c255c..2be9da2cfb9299 100644 --- a/Lib/tkinter/__init__.py +++ b/Lib/tkinter/__init__.py @@ -1260,6 +1260,8 @@ def winfo_parent(self): def winfo_pathname(self, id, displayof=0): """Return the pathname of the widget given by ID.""" + if isinstance(id, int): + id = hex(id) args = ('winfo', 'pathname') \ + self._displayof(displayof) + (id,) return self.tk.call(args) @@ -1535,16 +1537,19 @@ def unbind(self, sequence, funcid=None): Otherwise destroy the current binding for SEQUENCE, leaving SEQUENCE unbound. """ + self._unbind(('bind', self._w, sequence), funcid) + + def _unbind(self, what, funcid=None): if funcid is None: - self.tk.call('bind', self._w, sequence, '') + self.tk.call(*what, '') else: - lines = self.tk.call('bind', self._w, sequence).split('\n') + lines = self.tk.call(what).split('\n') prefix = f'if {{"[{funcid} ' keep = '\n'.join(line for line in lines if not line.startswith(prefix)) if not keep.strip(): keep = '' - self.tk.call('bind', self._w, sequence, keep) + self.tk.call(*what, keep) self.deletecommand(funcid) def bind_all(self, sequence=None, func=None, add=None): @@ -1556,7 +1561,7 @@ def bind_all(self, sequence=None, func=None, add=None): def unbind_all(self, sequence): """Unbind for all widgets for event SEQUENCE all functions.""" - self.tk.call('bind', 'all' , sequence, '') + self._root()._unbind(('bind', 'all', sequence)) def bind_class(self, className, sequence=None, func=None, add=None): """Bind to widgets with bindtag CLASSNAME at event @@ -1571,7 +1576,7 @@ def bind_class(self, className, sequence=None, func=None, add=None): def unbind_class(self, className, sequence): """Unbind for all widgets with bindtag CLASSNAME for event SEQUENCE all functions.""" - self.tk.call('bind', className , sequence, '') + self._root()._unbind(('bind', className, sequence)) def mainloop(self, n=0): """Call the mainloop of Tk.""" @@ -2103,26 +2108,39 @@ def wm_aspect(self, aspect = wm_aspect - def wm_attributes(self, *args): - """This subcommand returns or sets platform specific attributes - - The first form returns a list of the platform specific flags and - their values. The second form returns the value for the specific - option. The third form sets one or more of the values. The values - are as follows: - - On Windows, -disabled gets or sets whether the window is in a - disabled state. -toolwindow gets or sets the style of the window - to toolwindow (as defined in the MSDN). -topmost gets or sets - whether this is a topmost window (displays above all other - windows). + def wm_attributes(self, *args, return_python_dict=False, **kwargs): + """Return or sets platform specific attributes. - On Macintosh, XXXXX + When called with a single argument return_python_dict=True, + return a dict of the platform specific attributes and their values. + When called without arguments or with a single argument + return_python_dict=False, return a tuple containing intermixed + attribute names with the minus prefix and their values. - On Unix, there are currently no special attribute values. + When called with a single string value, return the value for the + specific option. When called with keyword arguments, set the + corresponding attributes. """ - args = ('wm', 'attributes', self._w) + args - return self.tk.call(args) + if not kwargs: + if not args: + res = self.tk.call('wm', 'attributes', self._w) + if return_python_dict: + return _splitdict(self.tk, res) + else: + return self.tk.splitlist(res) + if len(args) == 1 and args[0] is not None: + option = args[0] + if option[0] == '-': + # TODO: deprecate + option = option[1:] + return self.tk.call('wm', 'attributes', self._w, '-' + option) + # TODO: deprecate + return self.tk.call('wm', 'attributes', self._w, *args) + elif args: + raise TypeError('wm_attribute() options have been specified as ' + 'positional and keyword arguments') + else: + self.tk.call('wm', 'attributes', self._w, *self._options(kwargs)) attributes = wm_attributes @@ -2883,9 +2901,7 @@ def bbox(self, *args): def tag_unbind(self, tagOrId, sequence, funcid=None): """Unbind for all items with TAGORID for event SEQUENCE the function identified with FUNCID.""" - self.tk.call(self._w, 'bind', tagOrId, sequence, '') - if funcid: - self.deletecommand(funcid) + self._unbind((self._w, 'bind', tagOrId, sequence), funcid) def tag_bind(self, tagOrId, sequence=None, func=None, add=None): """Bind to all items with TAGORID at event SEQUENCE a call to function FUNC. @@ -3995,9 +4011,7 @@ def tag_add(self, tagName, index1, *args): def tag_unbind(self, tagName, sequence, funcid=None): """Unbind for all characters with TAGNAME for event SEQUENCE the function identified with FUNCID.""" - self.tk.call(self._w, 'tag', 'bind', tagName, sequence, '') - if funcid: - self.deletecommand(funcid) + return self._unbind((self._w, 'tag', 'bind', tagName, sequence), funcid) def tag_bind(self, tagName, sequence, func, add=None): """Bind to all characters with TAGNAME at event SEQUENCE a call to function FUNC. @@ -4008,6 +4022,11 @@ def tag_bind(self, tagName, sequence, func, add=None): return self._bind((self._w, 'tag', 'bind', tagName), sequence, func, add) + def _tag_bind(self, tagName, sequence=None, func=None, add=None): + # For tests only + return self._bind((self._w, 'tag', 'bind', tagName), + sequence, func, add) + def tag_cget(self, tagName, option): """Return the value of OPTION for tag TAGNAME.""" if option[:1] != '-': @@ -4713,7 +4732,7 @@ def panes(self): def _test(): root = Tk() - text = "This is Tcl/Tk version %s" % TclVersion + text = "This is Tcl/Tk %s" % root.globalgetvar('tk_patchLevel') text += "\nThis should be a cedilla: \xe7" label = Label(root, text=text) label.pack() diff --git a/Lib/tkinter/simpledialog.py b/Lib/tkinter/simpledialog.py index 538bbfc318d704..0f0dc66460f798 100644 --- a/Lib/tkinter/simpledialog.py +++ b/Lib/tkinter/simpledialog.py @@ -262,7 +262,7 @@ def _setup_dialog(w): w.tk.call("::tk::unsupported::MacWindowStyle", "style", w, "moveableModal", "") elif w._windowingsystem == "x11": - w.wm_attributes("-type", "dialog") + w.wm_attributes(type="dialog") # -------------------------------------------------------------------- # convenience dialogues diff --git a/Lib/trace.py b/Lib/trace.py index 7cb6f897634b14..7886959fa64f68 100755 --- a/Lib/trace.py +++ b/Lib/trace.py @@ -265,8 +265,7 @@ def write_results(self, show_missing=True, summary=False, coverdir=None, *, modulename = _modname(filename) else: dir = coverdir - if not os.path.exists(dir): - os.makedirs(dir) + os.makedirs(dir, exist_ok=True) modulename = _fullmodname(filename) # If desired, get a list of the line numbers which represent diff --git a/Lib/traceback.py b/Lib/traceback.py index 1cf008c7e9da97..d27c7a726d2bb6 100644 --- a/Lib/traceback.py +++ b/Lib/traceback.py @@ -1051,7 +1051,11 @@ def __init__(self, exc_type, exc_value, exc_traceback, *, limit=None, # Capture now to permit freeing resources: only complication is in the # unofficial API _format_final_exc_line self._str = _safe_string(exc_value, 'exception') - self.__notes__ = getattr(exc_value, '__notes__', None) + try: + self.__notes__ = getattr(exc_value, '__notes__', None) + except Exception as e: + self.__notes__ = [ + f'Ignored error getting __notes__: {_safe_string(e, '__notes__', repr)}'] self._is_syntax_error = False self._have_exc_type = exc_type is not None @@ -1497,6 +1501,13 @@ def _compute_suggestion_error(exc_value, tb, wrong_name): if hasattr(self, wrong_name): return f"self.{wrong_name}" + try: + import _suggestions + except ImportError: + pass + else: + return _suggestions._generate_suggestions(d, wrong_name) + # Compute closest match if len(d) > _MAX_CANDIDATE_ITEMS: diff --git a/Lib/tty.py b/Lib/tty.py index 283e5c334f5751..5a49e0400425f3 100644 --- a/Lib/tty.py +++ b/Lib/tty.py @@ -45,9 +45,6 @@ def cfmakeraw(mode): def cfmakecbreak(mode): """Make termios mode cbreak.""" - # Do not map CR to NL on input. - mode[IFLAG] &= ~(ICRNL) - # Do not echo characters; disable canonical input. mode[LFLAG] &= ~(ECHO | ICANON) diff --git a/Lib/typing.py b/Lib/typing.py index d7d793539b35b1..914ddeaf504cd0 100644 --- a/Lib/typing.py +++ b/Lib/typing.py @@ -1127,7 +1127,9 @@ def __call__(self, *args, **kwargs): result = self.__origin__(*args, **kwargs) try: result.__orig_class__ = self - except AttributeError: + # Some objects raise TypeError (or something even more exotic) + # if you try to set attributes on them; we guard against that here + except Exception: pass return result @@ -1135,9 +1137,29 @@ def __mro_entries__(self, bases): res = [] if self.__origin__ not in bases: res.append(self.__origin__) + + # Check if any base that occurs after us in `bases` is either itself a + # subclass of Generic, or something which will add a subclass of Generic + # to `__bases__` via its `__mro_entries__`. If not, add Generic + # ourselves. The goal is to ensure that Generic (or a subclass) will + # appear exactly once in the final bases tuple. If we let it appear + # multiple times, we risk "can't form a consistent MRO" errors. i = bases.index(self) for b in bases[i+1:]: - if isinstance(b, _BaseGenericAlias) or issubclass(b, Generic): + if isinstance(b, _BaseGenericAlias): + break + if not isinstance(b, type): + meth = getattr(b, "__mro_entries__", None) + new_bases = meth(bases) if meth else None + if ( + isinstance(new_bases, tuple) and + any( + isinstance(b2, type) and issubclass(b2, Generic) + for b2 in new_bases + ) + ): + break + elif issubclass(b, Generic): break else: res.append(Generic) @@ -1670,7 +1692,7 @@ class _TypingEllipsis: _TYPING_INTERNALS = frozenset({ '__parameters__', '__orig_bases__', '__orig_class__', '_is_protocol', '_is_runtime_protocol', '__protocol_attrs__', - '__callable_proto_members_only__', '__type_params__', + '__non_callable_proto_members__', '__type_params__', }) _SPECIAL_NAMES = frozenset({ @@ -1833,11 +1855,6 @@ def __init__(cls, *args, **kwargs): super().__init__(*args, **kwargs) if getattr(cls, "_is_protocol", False): cls.__protocol_attrs__ = _get_protocol_attrs(cls) - # PEP 544 prohibits using issubclass() - # with protocols that have non-method members. - cls.__callable_proto_members_only__ = all( - callable(getattr(cls, attr, None)) for attr in cls.__protocol_attrs__ - ) def __subclasscheck__(cls, other): if cls is Protocol: @@ -1846,25 +1863,23 @@ def __subclasscheck__(cls, other): getattr(cls, '_is_protocol', False) and not _allow_reckless_class_checks() ): + if not getattr(cls, '_is_runtime_protocol', False): + _type_check_issubclass_arg_1(other) + raise TypeError( + "Instance and class checks can only be used with " + "@runtime_checkable protocols" + ) if ( - not cls.__callable_proto_members_only__ + # this attribute is set by @runtime_checkable: + cls.__non_callable_proto_members__ and cls.__dict__.get("__subclasshook__") is _proto_hook ): _type_check_issubclass_arg_1(other) - non_method_attrs = sorted( - attr for attr in cls.__protocol_attrs__ - if not callable(getattr(cls, attr, None)) - ) + non_method_attrs = sorted(cls.__non_callable_proto_members__) raise TypeError( "Protocols with non-method members don't support issubclass()." f" Non-method members: {str(non_method_attrs)[1:-1]}." ) - if not getattr(cls, '_is_runtime_protocol', False): - _type_check_issubclass_arg_1(other) - raise TypeError( - "Instance and class checks can only be used with " - "@runtime_checkable protocols" - ) return _abc_subclasscheck(cls, other) def __instancecheck__(cls, instance): @@ -1892,7 +1907,8 @@ def __instancecheck__(cls, instance): val = getattr_static(instance, attr) except AttributeError: break - if val is None and callable(getattr(cls, attr, None)): + # this attribute is set by @runtime_checkable: + if val is None and attr not in cls.__non_callable_proto_members__: break else: return True @@ -2114,6 +2130,22 @@ def close(self): ... raise TypeError('@runtime_checkable can be only applied to protocol classes,' ' got %r' % cls) cls._is_runtime_protocol = True + # PEP 544 prohibits using issubclass() + # with protocols that have non-method members. + # See gh-113320 for why we compute this attribute here, + # rather than in `_ProtocolMeta.__init__` + cls.__non_callable_proto_members__ = set() + for attr in cls.__protocol_attrs__: + try: + is_callable = callable(getattr(cls, attr, None)) + except Exception as e: + raise TypeError( + f"Failed to determine whether protocol member {attr!r} " + "is a method member" + ) from e + else: + if not is_callable: + cls.__non_callable_proto_members__.add(attr) return cls diff --git a/Lib/unittest/case.py b/Lib/unittest/case.py index 811557498bb30e..001b640dc43ad6 100644 --- a/Lib/unittest/case.py +++ b/Lib/unittest/case.py @@ -606,6 +606,7 @@ def run(self, result=None): else: stopTestRun = None + result.startTest(self) try: testMethod = getattr(self, self._testMethodName) if (getattr(self.__class__, "__unittest_skip__", False) or @@ -616,9 +617,6 @@ def run(self, result=None): _addSkip(result, self, skip_why) return result - # Increase the number of tests only if it hasn't been skipped - result.startTest(self) - expecting_failure = ( getattr(self, "__unittest_expecting_failure__", False) or getattr(testMethod, "__unittest_expecting_failure__", False) diff --git a/Lib/unittest/main.py b/Lib/unittest/main.py index d29a9f91fcca42..c3869de3f6f18e 100644 --- a/Lib/unittest/main.py +++ b/Lib/unittest/main.py @@ -269,7 +269,7 @@ def runTests(self): testRunner = self.testRunner self.result = testRunner.run(self.test) if self.exit: - if self.result.testsRun == 0: + if self.result.testsRun == 0 and len(self.result.skipped) == 0: sys.exit(_NO_TESTS_EXITCODE) elif self.result.wasSuccessful(): sys.exit(0) diff --git a/Lib/unittest/mock.py b/Lib/unittest/mock.py index c6b46eea657a21..93f4d9743ed2fa 100644 --- a/Lib/unittest/mock.py +++ b/Lib/unittest/mock.py @@ -1010,8 +1010,8 @@ def assert_has_calls(self, calls, any_order=False): for e in expected]) raise AssertionError( f'{problem}\n' - f'Expected: {_CallList(calls)}' - f'{self._calls_repr(prefix=" Actual").rstrip(".")}' + f'Expected: {_CallList(calls)}\n' + f' Actual: {safe_repr(self.mock_calls)}' ) from cause return @@ -1085,7 +1085,7 @@ def _get_child_mock(self, /, **kw): return klass(**kw) - def _calls_repr(self, prefix="Calls"): + def _calls_repr(self): """Renders self.mock_calls as a string. Example: "\nCalls: [call(1), call(2)]." @@ -1095,7 +1095,7 @@ def _calls_repr(self, prefix="Calls"): """ if not self.mock_calls: return "" - return f"\n{prefix}: {safe_repr(self.mock_calls)}." + return f"\nCalls: {safe_repr(self.mock_calls)}." # Denylist for forbidden attribute names in safe mode @@ -2229,8 +2229,11 @@ def __get__(self, obj, _type=None): return self.create_mock() -_CODE_ATTRS = dir(CodeType) -_CODE_SIG = inspect.signature(partial(CodeType.__init__, None)) +try: + _CODE_SIG = inspect.signature(partial(CodeType.__init__, None)) + _CODE_ATTRS = dir(CodeType) +except ValueError: + _CODE_SIG = None class AsyncMockMixin(Base): @@ -2250,9 +2253,12 @@ def __init__(self, /, *args, **kwargs): self.__dict__['_mock_await_count'] = 0 self.__dict__['_mock_await_args'] = None self.__dict__['_mock_await_args_list'] = _CallList() - code_mock = NonCallableMock(spec_set=_CODE_ATTRS) - code_mock.__dict__["_spec_class"] = CodeType - code_mock.__dict__["_spec_signature"] = _CODE_SIG + if _CODE_SIG: + code_mock = NonCallableMock(spec_set=_CODE_ATTRS) + code_mock.__dict__["_spec_class"] = CodeType + code_mock.__dict__["_spec_signature"] = _CODE_SIG + else: + code_mock = NonCallableMock(spec_set=CodeType) code_mock.co_flags = ( inspect.CO_COROUTINE + inspect.CO_VARARGS diff --git a/Lib/unittest/result.py b/Lib/unittest/result.py index 9e56f658027f4d..3ace0a5b7bf2ef 100644 --- a/Lib/unittest/result.py +++ b/Lib/unittest/result.py @@ -97,12 +97,10 @@ def _restoreStdout(self): sys.stdout = self._original_stdout sys.stderr = self._original_stderr - if self._stdout_buffer is not None: - self._stdout_buffer.seek(0) - self._stdout_buffer.truncate() - if self._stderr_buffer is not None: - self._stderr_buffer.seek(0) - self._stderr_buffer.truncate() + self._stdout_buffer.seek(0) + self._stdout_buffer.truncate() + self._stderr_buffer.seek(0) + self._stderr_buffer.truncate() def stopTestRun(self): """Called once after all tests are executed. diff --git a/Lib/unittest/runner.py b/Lib/unittest/runner.py index e3c020e0ace96d..2bcadf0c998bd9 100644 --- a/Lib/unittest/runner.py +++ b/Lib/unittest/runner.py @@ -274,7 +274,7 @@ def run(self, test): infos.append("failures=%d" % failed) if errored: infos.append("errors=%d" % errored) - elif run == 0: + elif run == 0 and not skipped: self.stream.write("NO TESTS RAN") else: self.stream.write("OK") diff --git a/Lib/urllib/request.py b/Lib/urllib/request.py index 1d03259b918c33..bca594420f6d9d 100644 --- a/Lib/urllib/request.py +++ b/Lib/urllib/request.py @@ -2490,7 +2490,7 @@ def getproxies_environment(): # select only environment variables which end in (after making lowercase) _proxy proxies = {} environment = [] - for name in os.environ.keys(): + for name in os.environ: # fast screen underscore position before more expensive case-folding if len(name) > 5 and name[-6] == "_" and name[-5:].lower() == "proxy": value = os.environ[name] diff --git a/Lib/venv/__init__.py b/Lib/venv/__init__.py index d960bf3bd82ac5..4856594755ae57 100644 --- a/Lib/venv/__init__.py +++ b/Lib/venv/__init__.py @@ -129,8 +129,7 @@ def create_if_needed(d): context = types.SimpleNamespace() context.env_dir = env_dir context.env_name = os.path.split(env_dir)[1] - prompt = self.prompt if self.prompt is not None else context.env_name - context.prompt = '(%s) ' % prompt + context.prompt = self.prompt if self.prompt is not None else context.env_name create_if_needed(env_dir) executable = sys._base_executable if not executable: # see gh-96861 @@ -139,6 +138,11 @@ def create_if_needed(d): 'check that your PATH environment variable is ' 'correctly set.') dirname, exename = os.path.split(os.path.abspath(executable)) + if sys.platform == 'win32': + # Always create the simplest name in the venv. It will either be a + # link back to executable, or a copy of the appropriate launcher + _d = '_d' if os.path.splitext(exename)[0].endswith('_d') else '' + exename = f'python{_d}.exe' context.executable = executable context.python_dir = dirname context.python_exe = exename @@ -222,67 +226,26 @@ def create_configuration(self, context): args = ' '.join(args) f.write(f'command = {sys.executable} -m venv {args}\n') - if os.name != 'nt': - def symlink_or_copy(self, src, dst, relative_symlinks_ok=False): - """ - Try symlinking a file, and if that fails, fall back to copying. - """ - force_copy = not self.symlinks - if not force_copy: - try: - if not os.path.islink(dst): # can't link to itself! - if relative_symlinks_ok: - assert os.path.dirname(src) == os.path.dirname(dst) - os.symlink(os.path.basename(src), dst) - else: - os.symlink(src, dst) - except Exception: # may need to use a more specific exception - logger.warning('Unable to symlink %r to %r', src, dst) - force_copy = True - if force_copy: - shutil.copyfile(src, dst) - else: - def symlink_or_copy(self, src, dst, relative_symlinks_ok=False): - """ - Try symlinking a file, and if that fails, fall back to copying. - """ - bad_src = os.path.lexists(src) and not os.path.exists(src) - if self.symlinks and not bad_src and not os.path.islink(dst): - try: + def symlink_or_copy(self, src, dst, relative_symlinks_ok=False): + """ + Try symlinking a file, and if that fails, fall back to copying. + (Unused on Windows, because we can't just copy a failed symlink file: we + switch to a different set of files instead.) + """ + assert os.name != 'nt' + force_copy = not self.symlinks + if not force_copy: + try: + if not os.path.islink(dst): # can't link to itself! if relative_symlinks_ok: assert os.path.dirname(src) == os.path.dirname(dst) os.symlink(os.path.basename(src), dst) else: os.symlink(src, dst) - return - except Exception: # may need to use a more specific exception - logger.warning('Unable to symlink %r to %r', src, dst) - - # On Windows, we rewrite symlinks to our base python.exe into - # copies of venvlauncher.exe - basename, ext = os.path.splitext(os.path.basename(src)) - srcfn = os.path.join(os.path.dirname(__file__), - "scripts", - "nt", - basename + ext) - # Builds or venv's from builds need to remap source file - # locations, as we do not put them into Lib/venv/scripts - if sysconfig.is_python_build() or not os.path.isfile(srcfn): - if basename.endswith('_d'): - ext = '_d' + ext - basename = basename[:-2] - if basename == 'python': - basename = 'venvlauncher' - elif basename == 'pythonw': - basename = 'venvwlauncher' - src = os.path.join(os.path.dirname(src), basename + ext) - else: - src = srcfn - if not os.path.exists(src): - if not bad_src: - logger.warning('Unable to copy %r', src) - return - + except Exception: # may need to use a more specific exception + logger.warning('Unable to symlink %r to %r', src, dst) + force_copy = True + if force_copy: shutil.copyfile(src, dst) def create_git_ignore_file(self, context): @@ -298,22 +261,23 @@ def create_git_ignore_file(self, context): 'see https://docs.python.org/3/library/venv.html\n') file.write('*\n') - def setup_python(self, context): - """ - Set up a Python executable in the environment. + if os.name != 'nt': + def setup_python(self, context): + """ + Set up a Python executable in the environment. - :param context: The information for the environment creation request - being processed. - """ - binpath = context.bin_path - path = context.env_exe - copier = self.symlink_or_copy - dirname = context.python_dir - if os.name != 'nt': + :param context: The information for the environment creation request + being processed. + """ + binpath = context.bin_path + path = context.env_exe + copier = self.symlink_or_copy + dirname = context.python_dir copier(context.executable, path) if not os.path.islink(path): os.chmod(path, 0o755) - for suffix in ('python', 'python3', f'python3.{sys.version_info[1]}'): + for suffix in ('python', 'python3', + f'python3.{sys.version_info[1]}'): path = os.path.join(binpath, suffix) if not os.path.exists(path): # Issue 18807: make copies if @@ -321,30 +285,105 @@ def setup_python(self, context): copier(context.env_exe, path, relative_symlinks_ok=True) if not os.path.islink(path): os.chmod(path, 0o755) - else: - if self.symlinks: - # For symlinking, we need a complete copy of the root directory - # If symlinks fail, you'll get unnecessary copies of files, but - # we assume that if you've opted into symlinks on Windows then - # you know what you're doing. - suffixes = [ - f for f in os.listdir(dirname) if - os.path.normcase(os.path.splitext(f)[1]) in ('.exe', '.dll') - ] - if sysconfig.is_python_build(): - suffixes = [ - f for f in suffixes if - os.path.normcase(f).startswith(('python', 'vcruntime')) - ] + + else: + def setup_python(self, context): + """ + Set up a Python executable in the environment. + + :param context: The information for the environment creation request + being processed. + """ + binpath = context.bin_path + dirname = context.python_dir + exename = os.path.basename(context.env_exe) + exe_stem = os.path.splitext(exename)[0] + exe_d = '_d' if os.path.normcase(exe_stem).endswith('_d') else '' + if sysconfig.is_python_build(): + scripts = dirname + else: + scripts = os.path.join(os.path.dirname(__file__), + 'scripts', 'nt') + if not sysconfig.get_config_var("Py_GIL_DISABLED"): + python_exe = os.path.join(dirname, f'python{exe_d}.exe') + pythonw_exe = os.path.join(dirname, f'pythonw{exe_d}.exe') + link_sources = { + 'python.exe': python_exe, + f'python{exe_d}.exe': python_exe, + 'pythonw.exe': pythonw_exe, + f'pythonw{exe_d}.exe': pythonw_exe, + } + python_exe = os.path.join(scripts, f'venvlauncher{exe_d}.exe') + pythonw_exe = os.path.join(scripts, f'venvwlauncher{exe_d}.exe') + copy_sources = { + 'python.exe': python_exe, + f'python{exe_d}.exe': python_exe, + 'pythonw.exe': pythonw_exe, + f'pythonw{exe_d}.exe': pythonw_exe, + } else: - suffixes = {'python.exe', 'python_d.exe', 'pythonw.exe', 'pythonw_d.exe'} - base_exe = os.path.basename(context.env_exe) - suffixes.add(base_exe) + exe_t = f'3.{sys.version_info[1]}t' + python_exe = os.path.join(dirname, f'python{exe_t}{exe_d}.exe') + pythonw_exe = os.path.join(dirname, f'pythonw{exe_t}{exe_d}.exe') + link_sources = { + 'python.exe': python_exe, + f'python{exe_d}.exe': python_exe, + f'python{exe_t}.exe': python_exe, + f'python{exe_t}{exe_d}.exe': python_exe, + 'pythonw.exe': pythonw_exe, + f'pythonw{exe_d}.exe': pythonw_exe, + f'pythonw{exe_t}.exe': pythonw_exe, + f'pythonw{exe_t}{exe_d}.exe': pythonw_exe, + } + python_exe = os.path.join(scripts, f'venvlaunchert{exe_d}.exe') + pythonw_exe = os.path.join(scripts, f'venvwlaunchert{exe_d}.exe') + copy_sources = { + 'python.exe': python_exe, + f'python{exe_d}.exe': python_exe, + f'python{exe_t}.exe': python_exe, + f'python{exe_t}{exe_d}.exe': python_exe, + 'pythonw.exe': pythonw_exe, + f'pythonw{exe_d}.exe': pythonw_exe, + f'pythonw{exe_t}.exe': pythonw_exe, + f'pythonw{exe_t}{exe_d}.exe': pythonw_exe, + } + + do_copies = True + if self.symlinks: + do_copies = False + # For symlinking, we need all the DLLs to be available alongside + # the executables. + link_sources.update({ + f: os.path.join(dirname, f) for f in os.listdir(dirname) + if os.path.normcase(f).startswith(('python', 'vcruntime')) + and os.path.normcase(os.path.splitext(f)[1]) == '.dll' + }) + + to_unlink = [] + for dest, src in link_sources.items(): + dest = os.path.join(binpath, dest) + try: + os.symlink(src, dest) + to_unlink.append(dest) + except OSError: + logger.warning('Unable to symlink %r to %r', src, dst) + do_copies = True + for f in to_unlink: + try: + os.unlink(f) + except OSError: + logger.warning('Failed to clean up symlink %r', + f) + logger.warning('Retrying with copies') + break - for suffix in suffixes: - src = os.path.join(dirname, suffix) - if os.path.lexists(src): - copier(src, os.path.join(binpath, suffix)) + if do_copies: + for dest, src in copy_sources.items(): + dest = os.path.join(binpath, dest) + try: + shutil.copy2(src, dest) + except OSError: + logger.warning('Unable to copy %r to %r', src, dest) if sysconfig.is_python_build(): # copy init.tcl @@ -437,6 +476,14 @@ def install_scripts(self, context, path): """ binpath = context.bin_path plen = len(path) + if os.name == 'nt': + def skip_file(f): + f = os.path.normcase(f) + return (f.startswith(('python', 'venv')) + and f.endswith(('.exe', '.pdb'))) + else: + def skip_file(f): + return False for root, dirs, files in os.walk(path): if root == path: # at top-level, remove irrelevant dirs for d in dirs[:]: @@ -444,8 +491,7 @@ def install_scripts(self, context, path): dirs.remove(d) continue # ignore files in top level for f in files: - if (os.name == 'nt' and f.startswith('python') - and f.endswith(('.exe', '.pdb'))): + if skip_file(f): continue srcfile = os.path.join(root, f) suffix = root[plen:].split(os.sep)[2:] @@ -456,20 +502,25 @@ def install_scripts(self, context, path): if not os.path.exists(dstdir): os.makedirs(dstdir) dstfile = os.path.join(dstdir, f) + if os.name == 'nt' and srcfile.endswith(('.exe', '.pdb')): + shutil.copy2(srcfile, dstfile) + continue with open(srcfile, 'rb') as f: data = f.read() - if not srcfile.endswith(('.exe', '.pdb')): - try: - data = data.decode('utf-8') - data = self.replace_variables(data, context) - data = data.encode('utf-8') - except UnicodeError as e: - data = None - logger.warning('unable to copy script %r, ' - 'may be binary: %s', srcfile, e) - if data is not None: + try: + new_data = ( + self.replace_variables(data.decode('utf-8'), context) + .encode('utf-8') + ) + except UnicodeError as e: + logger.warning('unable to copy script %r, ' + 'may be binary: %s', srcfile, e) + continue + if new_data == data: + shutil.copy2(srcfile, dstfile) + else: with open(dstfile, 'wb') as f: - f.write(data) + f.write(new_data) shutil.copymode(srcfile, dstfile) def upgrade_dependencies(self, context): diff --git a/Lib/venv/scripts/common/activate b/Lib/venv/scripts/common/activate index a4e0609045a9d5..cbd4873f012246 100644 --- a/Lib/venv/scripts/common/activate +++ b/Lib/venv/scripts/common/activate @@ -66,7 +66,7 @@ fi if [ -z "${VIRTUAL_ENV_DISABLE_PROMPT:-}" ] ; then _OLD_VIRTUAL_PS1="${PS1:-}" - PS1="__VENV_PROMPT__${PS1:-}" + PS1="(__VENV_PROMPT__) ${PS1:-}" export PS1 fi diff --git a/Lib/venv/scripts/nt/activate.bat b/Lib/venv/scripts/nt/activate.bat index c1c3c82ee37f10..dd5ea8eb67b90a 100644 --- a/Lib/venv/scripts/nt/activate.bat +++ b/Lib/venv/scripts/nt/activate.bat @@ -15,8 +15,8 @@ if not defined PROMPT set PROMPT=$P$G if defined _OLD_VIRTUAL_PROMPT set PROMPT=%_OLD_VIRTUAL_PROMPT% if defined _OLD_VIRTUAL_PYTHONHOME set PYTHONHOME=%_OLD_VIRTUAL_PYTHONHOME% -set _OLD_VIRTUAL_PROMPT=%PROMPT% -set PROMPT=__VENV_PROMPT__%PROMPT% +set "_OLD_VIRTUAL_PROMPT=%PROMPT%" +set "PROMPT=(__VENV_PROMPT__) %PROMPT%" if defined PYTHONHOME set _OLD_VIRTUAL_PYTHONHOME=%PYTHONHOME% set PYTHONHOME= diff --git a/Lib/venv/scripts/posix/activate.csh b/Lib/venv/scripts/posix/activate.csh index 9caf138a919a86..c707f1988b0acc 100644 --- a/Lib/venv/scripts/posix/activate.csh +++ b/Lib/venv/scripts/posix/activate.csh @@ -19,7 +19,7 @@ setenv VIRTUAL_ENV_PROMPT "__VENV_PROMPT__" set _OLD_VIRTUAL_PROMPT="$prompt" if (! "$?VIRTUAL_ENV_DISABLE_PROMPT") then - set prompt = "__VENV_PROMPT__$prompt" + set prompt = "(__VENV_PROMPT__) $prompt" endif alias pydoc python -m pydoc diff --git a/Lib/venv/scripts/posix/activate.fish b/Lib/venv/scripts/posix/activate.fish index 565df23d1e2a13..25c42756789bbc 100644 --- a/Lib/venv/scripts/posix/activate.fish +++ b/Lib/venv/scripts/posix/activate.fish @@ -57,7 +57,7 @@ if test -z "$VIRTUAL_ENV_DISABLE_PROMPT" set -l old_status $status # Output the venv prompt; color taken from the blue of the Python logo. - printf "%s%s%s" (set_color 4B8BBE) "__VENV_PROMPT__" (set_color normal) + printf "%s(%s)%s " (set_color 4B8BBE) "__VENV_PROMPT__" (set_color normal) # Restore the return status of the previous command. echo "exit $old_status" | . diff --git a/Lib/warnings.py b/Lib/warnings.py index b8ff078569d2ce..4ad6ad027192e8 100644 --- a/Lib/warnings.py +++ b/Lib/warnings.py @@ -58,15 +58,16 @@ def _formatwarnmsg_impl(msg): # catch Exception, not only ImportError and RecursionError. except Exception: # don't suggest to enable tracemalloc if it's not available - tracing = True + suggest_tracemalloc = False tb = None else: - tracing = tracemalloc.is_tracing() try: + suggest_tracemalloc = not tracemalloc.is_tracing() tb = tracemalloc.get_object_traceback(msg.source) except Exception: # When a warning is logged during Python shutdown, tracemalloc # and the import machinery don't work anymore + suggest_tracemalloc = False tb = None if tb is not None: @@ -85,7 +86,7 @@ def _formatwarnmsg_impl(msg): if line: line = line.strip() s += ' %s\n' % line - elif not tracing: + elif suggest_tracemalloc: s += (f'{category}: Enable tracemalloc to get the object ' f'allocation traceback\n') return s diff --git a/Lib/webbrowser.py b/Lib/webbrowser.py index 6f9c6a6de177e6..636e8ca459d109 100755 --- a/Lib/webbrowser.py +++ b/Lib/webbrowser.py @@ -574,6 +574,7 @@ def __init__(self, name='default'): super().__init__(name) def open(self, url, new=0, autoraise=True): + sys.audit("webbrowser.open", url) if self.name == 'default': script = 'open location "%s"' % url.replace('"', '%22') # opens in default browser else: diff --git a/Lib/xml/etree/ElementTree.py b/Lib/xml/etree/ElementTree.py index 42574eefd81beb..a37fead41b750e 100644 --- a/Lib/xml/etree/ElementTree.py +++ b/Lib/xml/etree/ElementTree.py @@ -99,6 +99,7 @@ import collections import collections.abc import contextlib +import weakref from . import ElementPath @@ -1223,13 +1224,14 @@ def iterparse(source, events=None, parser=None): # parser argument of iterparse is removed, this can be killed. pullparser = XMLPullParser(events=events, _parser=parser) - def iterator(source): + if not hasattr(source, "read"): + source = open(source, "rb") + close_source = True + else: close_source = False + + def iterator(source): try: - if not hasattr(source, "read"): - source = open(source, "rb") - close_source = True - yield None while True: yield from pullparser.read_events() # load event buffer @@ -1239,18 +1241,30 @@ def iterator(source): pullparser.feed(data) root = pullparser._close_and_return_root() yield from pullparser.read_events() - it.root = root + it = wr() + if it is not None: + it.root = root finally: if close_source: source.close() + gen = iterator(source) class IterParseIterator(collections.abc.Iterator): - __next__ = iterator(source).__next__ + __next__ = gen.__next__ + def close(self): + if close_source: + source.close() + gen.close() + + def __del__(self): + # TODO: Emit a ResourceWarning if it was not explicitly closed. + # (When the close() method will be supported in all maintained Python versions.) + if close_source: + source.close() + it = IterParseIterator() it.root = None - del iterator, IterParseIterator - - next(it) + wr = weakref.ref(it) return it diff --git a/Lib/zipfile/__init__.py b/Lib/zipfile/__init__.py index fe629ed1cf2fc5..8005b4b34ccf76 100644 --- a/Lib/zipfile/__init__.py +++ b/Lib/zipfile/__init__.py @@ -371,7 +371,7 @@ def _sanitize_filename(filename): return filename -class ZipInfo (object): +class ZipInfo: """Class with attributes describing each file in the ZIP archive.""" __slots__ = ( @@ -379,7 +379,7 @@ class ZipInfo (object): 'filename', 'date_time', 'compress_type', - '_compresslevel', + 'compress_level', 'comment', 'extra', 'create_system', @@ -395,6 +395,7 @@ class ZipInfo (object): 'compress_size', 'file_size', '_raw_time', + '_end_offset', ) def __init__(self, filename="NoName", date_time=(1980,1,1,0,0,0)): @@ -412,7 +413,7 @@ def __init__(self, filename="NoName", date_time=(1980,1,1,0,0,0)): # Standard values: self.compress_type = ZIP_STORED # Type of compression for the file - self._compresslevel = None # Level for the compressor + self.compress_level = None # Level for the compressor self.comment = b"" # Comment for each file self.extra = b"" # ZIP extra data if sys.platform == 'win32': @@ -429,10 +430,20 @@ def __init__(self, filename="NoName", date_time=(1980,1,1,0,0,0)): self.external_attr = 0 # External file attributes self.compress_size = 0 # Size of the compressed file self.file_size = 0 # Size of the uncompressed file + self._end_offset = None # Start of the next local header or central directory # Other attributes are set by class ZipFile: # header_offset Byte offset to the file header # CRC CRC-32 of the uncompressed file + # Maintain backward compatibility with the old protected attribute name. + @property + def _compresslevel(self): + return self.compress_level + + @_compresslevel.setter + def _compresslevel(self, value): + self.compress_level = value + def __repr__(self): result = ['<%s filename=%r' % (self.__class__.__name__, self.filename)] if self.compress_type != ZIP_STORED: @@ -1189,7 +1200,7 @@ def __init__(self, zf, zinfo, zip64): self._zip64 = zip64 self._zipfile = zf self._compressor = _get_compressor(zinfo.compress_type, - zinfo._compresslevel) + zinfo.compress_level) self._file_size = 0 self._compress_size = 0 self._crc = 0 @@ -1488,6 +1499,12 @@ def _RealGetContents(self): if self.debug > 2: print("total", total) + end_offset = self.start_dir + for zinfo in sorted(self.filelist, + key=lambda zinfo: zinfo.header_offset, + reverse=True): + zinfo._end_offset = end_offset + end_offset = zinfo.header_offset def namelist(self): """Return a list of file names in the archive.""" @@ -1595,7 +1612,7 @@ def open(self, name, mode="r", pwd=None, *, force_zip64=False): elif mode == 'w': zinfo = ZipInfo(name) zinfo.compress_type = self.compression - zinfo._compresslevel = self.compresslevel + zinfo.compress_level = self.compresslevel else: # Get info object for name zinfo = self.getinfo(name) @@ -1644,6 +1661,10 @@ def open(self, name, mode="r", pwd=None, *, force_zip64=False): 'File name in directory %r and header %r differ.' % (zinfo.orig_filename, fname)) + if (zinfo._end_offset is not None and + zef_file.tell() + zinfo.compress_size > zinfo._end_offset): + raise BadZipFile(f"Overlapped entries: {zinfo.orig_filename!r} (possible zip bomb)") + # check for encrypted flag & handle password is_encrypted = zinfo.flag_bits & _MASK_ENCRYPTED if is_encrypted: @@ -1772,7 +1793,7 @@ def _extract_member(self, member, targetpath, pwd): # filter illegal characters on Windows arcname = self._sanitize_windows_name(arcname, os.path.sep) - if not arcname: + if not arcname and not member.is_dir(): raise ValueError("Empty filename.") targetpath = os.path.join(targetpath, arcname) @@ -1843,9 +1864,9 @@ def write(self, filename, arcname=None, zinfo.compress_type = self.compression if compresslevel is not None: - zinfo._compresslevel = compresslevel + zinfo.compress_level = compresslevel else: - zinfo._compresslevel = self.compresslevel + zinfo.compress_level = self.compresslevel with open(filename, "rb") as src, self.open(zinfo, 'w') as dest: shutil.copyfileobj(src, dest, 1024*8) @@ -1863,7 +1884,7 @@ def writestr(self, zinfo_or_arcname, data, zinfo = ZipInfo(filename=zinfo_or_arcname, date_time=time.localtime(time.time())[:6]) zinfo.compress_type = self.compression - zinfo._compresslevel = self.compresslevel + zinfo.compress_level = self.compresslevel if zinfo.filename.endswith('/'): zinfo.external_attr = 0o40775 << 16 # drwxrwxr-x zinfo.external_attr |= 0x10 # MS-DOS directory flag @@ -1884,7 +1905,7 @@ def writestr(self, zinfo_or_arcname, data, zinfo.compress_type = compress_type if compresslevel is not None: - zinfo._compresslevel = compresslevel + zinfo.compress_level = compresslevel zinfo.file_size = len(data) # Uncompressed size with self._lock: diff --git a/Lib/zoneinfo/_tzpath.py b/Lib/zoneinfo/_tzpath.py index 4985dce2dc36d0..5db17bea045d8c 100644 --- a/Lib/zoneinfo/_tzpath.py +++ b/Lib/zoneinfo/_tzpath.py @@ -2,7 +2,7 @@ import sysconfig -def reset_tzpath(to=None): +def _reset_tzpath(to=None, stacklevel=4): global TZPATH tzpaths = to @@ -18,17 +18,22 @@ def reset_tzpath(to=None): base_tzpath = tzpaths else: env_var = os.environ.get("PYTHONTZPATH", None) - if env_var is not None: - base_tzpath = _parse_python_tzpath(env_var) - else: - base_tzpath = _parse_python_tzpath( - sysconfig.get_config_var("TZPATH") - ) + if env_var is None: + env_var = sysconfig.get_config_var("TZPATH") + base_tzpath = _parse_python_tzpath(env_var, stacklevel) TZPATH = tuple(base_tzpath) -def _parse_python_tzpath(env_var): +def reset_tzpath(to=None): + """Reset global TZPATH.""" + # We need `_reset_tzpath` helper function because it produces a warning, + # it is used as both a module-level call and a public API. + # This is how we equalize the stacklevel for both calls. + _reset_tzpath(to) + + +def _parse_python_tzpath(env_var, stacklevel): if not env_var: return () @@ -45,6 +50,7 @@ def _parse_python_tzpath(env_var): "Invalid paths specified in PYTHONTZPATH environment variable. " + msg, InvalidTZPathWarning, + stacklevel=stacklevel, ) return new_tzpath @@ -172,4 +178,4 @@ class InvalidTZPathWarning(RuntimeWarning): TZPATH = () -reset_tzpath() +_reset_tzpath(stacklevel=5) diff --git a/Mac/BuildScript/build-installer.py b/Mac/BuildScript/build-installer.py index 32de56bcf13086..0af90563cbbb2b 100755 --- a/Mac/BuildScript/build-installer.py +++ b/Mac/BuildScript/build-installer.py @@ -246,9 +246,9 @@ def library_recipes(): result.extend([ dict( - name="OpenSSL 3.0.11", - url="https://www.openssl.org/source/openssl-3.0.11.tar.gz", - checksum='b3425d3bb4a2218d0697eb41f7fc0cdede016ed19ca49d168b78e8d947887f55', + name="OpenSSL 3.0.13", + url="https://www.openssl.org/source/openssl-3.0.13.tar.gz", + checksum='88525753f79d3bec27d2fa7c66aa0b92b3aa9498dafd93d7cfa4b3780cdae313', buildrecipe=build_universal_openssl, configure=None, install=None, @@ -359,9 +359,9 @@ def library_recipes(): ), ), dict( - name="SQLite 3.44.2", - url="https://sqlite.org/2023/sqlite-autoconf-3440200.tar.gz", - checksum="c02f40fd4f809ced95096250adc5764a", + name="SQLite 3.45.1", + url="https://sqlite.org/2024/sqlite-autoconf-3450100.tar.gz", + checksum="cd9c27841b7a5932c9897651e20b86c701dd740556989b01ca596fcfa3d49a0a", extra_cflags=('-Os ' '-DSQLITE_ENABLE_FTS5 ' '-DSQLITE_ENABLE_FTS4 ' diff --git a/Mac/IDLE/IDLE.app/Contents/Info.plist b/Mac/IDLE/IDLE.app/Contents/Info.plist index 20b97b67f41d1a..8549e405e2a65a 100644 --- a/Mac/IDLE/IDLE.app/Contents/Info.plist +++ b/Mac/IDLE/IDLE.app/Contents/Info.plist @@ -37,7 +37,7 @@ CFBundleExecutable IDLE CFBundleGetInfoString - %version%, © 2001-2023 Python Software Foundation + %version%, © 2001-2024 Python Software Foundation CFBundleIconFile IDLE.icns CFBundleIdentifier diff --git a/Mac/PythonLauncher/Info.plist.in b/Mac/PythonLauncher/Info.plist.in index b7cddac0729fc2..233694788ac2b7 100644 --- a/Mac/PythonLauncher/Info.plist.in +++ b/Mac/PythonLauncher/Info.plist.in @@ -40,9 +40,9 @@ CFBundleExecutable Python Launcher NSHumanReadableCopyright - Copyright © 2001-2023 Python Software Foundation + Copyright © 2001-2024 Python Software Foundation CFBundleGetInfoString - %VERSION%, © 2001-2023 Python Software Foundation + %VERSION%, © 2001-2024 Python Software Foundation CFBundleIconFile PythonLauncher.icns CFBundleIdentifier diff --git a/Mac/Resources/app/Info.plist.in b/Mac/Resources/app/Info.plist.in index 8362b19b361b62..a1fc1511c40e96 100644 --- a/Mac/Resources/app/Info.plist.in +++ b/Mac/Resources/app/Info.plist.in @@ -20,7 +20,7 @@ CFBundleExecutable Python CFBundleGetInfoString - %version%, (c) 2001-2023 Python Software Foundation. + %version%, (c) 2001-2024 Python Software Foundation. CFBundleHelpBookFolder Documentation @@ -37,7 +37,7 @@ CFBundleInfoDictionaryVersion 6.0 CFBundleLongVersionString - %version%, (c) 2001-2023 Python Software Foundation. + %version%, (c) 2001-2024 Python Software Foundation. CFBundleName Python CFBundlePackageType @@ -55,7 +55,7 @@ NSAppleScriptEnabled NSHumanReadableCopyright - (c) 2001-2023 Python Software Foundation. + (c) 2001-2024 Python Software Foundation. NSHighResolutionCapable CFBundleAllowMixedLocalizations diff --git a/Mac/Resources/framework/Info.plist.in b/Mac/Resources/framework/Info.plist.in index 238441ce2c76c7..4c42971ed90ee4 100644 --- a/Mac/Resources/framework/Info.plist.in +++ b/Mac/Resources/framework/Info.plist.in @@ -17,9 +17,9 @@ CFBundlePackageType FMWK CFBundleShortVersionString - %VERSION%, (c) 2001-2023 Python Software Foundation. + %VERSION%, (c) 2001-2024 Python Software Foundation. CFBundleLongVersionString - %VERSION%, (c) 2001-2023 Python Software Foundation. + %VERSION%, (c) 2001-2024 Python Software Foundation. CFBundleSignature ???? CFBundleVersion diff --git a/Makefile.pre.in b/Makefile.pre.in index 6a64547e97d266..4dabe328ce0362 100644 --- a/Makefile.pre.in +++ b/Makefile.pre.in @@ -405,6 +405,7 @@ PYTHON_OBJS= \ Python/ast_opt.o \ Python/ast_unparse.o \ Python/bltinmodule.o \ + Python/brc.o \ Python/ceval.o \ Python/codecs.o \ Python/compile.o \ @@ -417,6 +418,9 @@ PYTHON_OBJS= \ Python/frame.o \ Python/frozenmain.o \ Python/future.o \ + Python/gc.o \ + Python/gc_free_threading.o \ + Python/gc_gil.o \ Python/getargs.o \ Python/getcompiler.o \ Python/getcopyright.o \ @@ -430,12 +434,14 @@ PYTHON_OBJS= \ Python/initconfig.o \ Python/instrumentation.o \ Python/intrinsics.o \ + Python/jit.o \ Python/legacy_tracing.o \ Python/lock.o \ Python/marshal.o \ Python/modsupport.o \ Python/mysnprintf.o \ Python/mystrtoul.o \ + Python/object_stack.o \ Python/optimizer.o \ Python/optimizer_analysis.o \ Python/parking_lot.o \ @@ -546,7 +552,7 @@ LINK_PYTHON_OBJS=@LINK_PYTHON_OBJS@ # On some systems, object files that reference DTrace probes need to be modified # in-place by dtrace(1). DTRACE_DEPS = \ - Python/ceval.o Python/import.o Python/sysmodule.o Modules/gcmodule.o + Python/ceval.o Python/gc.o Python/import.o Python/sysmodule.o ########################################################################## # decimal's libmpdec @@ -931,6 +937,262 @@ python.html: $(srcdir)/Tools/wasm/python.html python.worker.js python.worker.js: $(srcdir)/Tools/wasm/python.worker.js @cp $(srcdir)/Tools/wasm/python.worker.js $@ +############################################################################ +# Header files + +PYTHON_HEADERS= \ + $(srcdir)/Include/Python.h \ + $(srcdir)/Include/abstract.h \ + $(srcdir)/Include/bltinmodule.h \ + $(srcdir)/Include/boolobject.h \ + $(srcdir)/Include/bytearrayobject.h \ + $(srcdir)/Include/bytesobject.h \ + $(srcdir)/Include/ceval.h \ + $(srcdir)/Include/codecs.h \ + $(srcdir)/Include/compile.h \ + $(srcdir)/Include/complexobject.h \ + $(srcdir)/Include/descrobject.h \ + $(srcdir)/Include/dictobject.h \ + $(srcdir)/Include/dynamic_annotations.h \ + $(srcdir)/Include/enumobject.h \ + $(srcdir)/Include/errcode.h \ + $(srcdir)/Include/exports.h \ + $(srcdir)/Include/fileobject.h \ + $(srcdir)/Include/fileutils.h \ + $(srcdir)/Include/floatobject.h \ + $(srcdir)/Include/frameobject.h \ + $(srcdir)/Include/genericaliasobject.h \ + $(srcdir)/Include/import.h \ + $(srcdir)/Include/interpreteridobject.h \ + $(srcdir)/Include/intrcheck.h \ + $(srcdir)/Include/iterobject.h \ + $(srcdir)/Include/listobject.h \ + $(srcdir)/Include/longobject.h \ + $(srcdir)/Include/marshal.h \ + $(srcdir)/Include/memoryobject.h \ + $(srcdir)/Include/methodobject.h \ + $(srcdir)/Include/modsupport.h \ + $(srcdir)/Include/moduleobject.h \ + $(srcdir)/Include/object.h \ + $(srcdir)/Include/objimpl.h \ + $(srcdir)/Include/opcode.h \ + $(srcdir)/Include/opcode_ids.h \ + $(srcdir)/Include/osdefs.h \ + $(srcdir)/Include/osmodule.h \ + $(srcdir)/Include/patchlevel.h \ + $(srcdir)/Include/pyatomic.h \ + $(srcdir)/Include/pybuffer.h \ + $(srcdir)/Include/pycapsule.h \ + $(srcdir)/Include/pydtrace.h \ + $(srcdir)/Include/pyerrors.h \ + $(srcdir)/Include/pyexpat.h \ + $(srcdir)/Include/pyframe.h \ + $(srcdir)/Include/pyhash.h \ + $(srcdir)/Include/pylifecycle.h \ + $(srcdir)/Include/pymacconfig.h \ + $(srcdir)/Include/pymacro.h \ + $(srcdir)/Include/pymath.h \ + $(srcdir)/Include/pymem.h \ + $(srcdir)/Include/pyport.h \ + $(srcdir)/Include/pystate.h \ + $(srcdir)/Include/pystats.h \ + $(srcdir)/Include/pystrcmp.h \ + $(srcdir)/Include/pystrtod.h \ + $(srcdir)/Include/pythonrun.h \ + $(srcdir)/Include/pythread.h \ + $(srcdir)/Include/pytypedefs.h \ + $(srcdir)/Include/rangeobject.h \ + $(srcdir)/Include/setobject.h \ + $(srcdir)/Include/sliceobject.h \ + $(srcdir)/Include/structmember.h \ + $(srcdir)/Include/structseq.h \ + $(srcdir)/Include/sysmodule.h \ + $(srcdir)/Include/traceback.h \ + $(srcdir)/Include/tupleobject.h \ + $(srcdir)/Include/typeslots.h \ + $(srcdir)/Include/unicodeobject.h \ + $(srcdir)/Include/warnings.h \ + $(srcdir)/Include/weakrefobject.h \ + \ + pyconfig.h \ + $(PARSER_HEADERS) \ + \ + $(srcdir)/Include/cpython/abstract.h \ + $(srcdir)/Include/cpython/bytearrayobject.h \ + $(srcdir)/Include/cpython/bytesobject.h \ + $(srcdir)/Include/cpython/cellobject.h \ + $(srcdir)/Include/cpython/ceval.h \ + $(srcdir)/Include/cpython/classobject.h \ + $(srcdir)/Include/cpython/code.h \ + $(srcdir)/Include/cpython/compile.h \ + $(srcdir)/Include/cpython/complexobject.h \ + $(srcdir)/Include/cpython/context.h \ + $(srcdir)/Include/cpython/descrobject.h \ + $(srcdir)/Include/cpython/dictobject.h \ + $(srcdir)/Include/cpython/fileobject.h \ + $(srcdir)/Include/cpython/fileutils.h \ + $(srcdir)/Include/cpython/floatobject.h \ + $(srcdir)/Include/cpython/frameobject.h \ + $(srcdir)/Include/cpython/funcobject.h \ + $(srcdir)/Include/cpython/genobject.h \ + $(srcdir)/Include/cpython/import.h \ + $(srcdir)/Include/cpython/initconfig.h \ + $(srcdir)/Include/cpython/interpreteridobject.h \ + $(srcdir)/Include/cpython/listobject.h \ + $(srcdir)/Include/cpython/longintrepr.h \ + $(srcdir)/Include/cpython/longobject.h \ + $(srcdir)/Include/cpython/memoryobject.h \ + $(srcdir)/Include/cpython/methodobject.h \ + $(srcdir)/Include/cpython/object.h \ + $(srcdir)/Include/cpython/objimpl.h \ + $(srcdir)/Include/cpython/odictobject.h \ + $(srcdir)/Include/cpython/optimizer.h \ + $(srcdir)/Include/cpython/picklebufobject.h \ + $(srcdir)/Include/cpython/pthread_stubs.h \ + $(srcdir)/Include/cpython/pyatomic.h \ + $(srcdir)/Include/cpython/pyatomic_gcc.h \ + $(srcdir)/Include/cpython/pyatomic_std.h \ + $(srcdir)/Include/cpython/pyctype.h \ + $(srcdir)/Include/cpython/pydebug.h \ + $(srcdir)/Include/cpython/pyerrors.h \ + $(srcdir)/Include/cpython/pyfpe.h \ + $(srcdir)/Include/cpython/pyframe.h \ + $(srcdir)/Include/cpython/pyhash.h \ + $(srcdir)/Include/cpython/pylifecycle.h \ + $(srcdir)/Include/cpython/pymem.h \ + $(srcdir)/Include/cpython/pystate.h \ + $(srcdir)/Include/cpython/pystats.h \ + $(srcdir)/Include/cpython/pythonrun.h \ + $(srcdir)/Include/cpython/pythread.h \ + $(srcdir)/Include/cpython/setobject.h \ + $(srcdir)/Include/cpython/sysmodule.h \ + $(srcdir)/Include/cpython/traceback.h \ + $(srcdir)/Include/cpython/tracemalloc.h \ + $(srcdir)/Include/cpython/tupleobject.h \ + $(srcdir)/Include/cpython/unicodeobject.h \ + $(srcdir)/Include/cpython/warnings.h \ + $(srcdir)/Include/cpython/weakrefobject.h \ + \ + $(MIMALLOC_HEADERS) \ + \ + $(srcdir)/Include/internal/pycore_abstract.h \ + $(srcdir)/Include/internal/pycore_asdl.h \ + $(srcdir)/Include/internal/pycore_ast.h \ + $(srcdir)/Include/internal/pycore_ast_state.h \ + $(srcdir)/Include/internal/pycore_atexit.h \ + $(srcdir)/Include/internal/pycore_bitutils.h \ + $(srcdir)/Include/internal/pycore_blocks_output_buffer.h \ + $(srcdir)/Include/internal/pycore_brc.h \ + $(srcdir)/Include/internal/pycore_bytes_methods.h \ + $(srcdir)/Include/internal/pycore_bytesobject.h \ + $(srcdir)/Include/internal/pycore_call.h \ + $(srcdir)/Include/internal/pycore_capsule.h \ + $(srcdir)/Include/internal/pycore_ceval.h \ + $(srcdir)/Include/internal/pycore_ceval_state.h \ + $(srcdir)/Include/internal/pycore_code.h \ + $(srcdir)/Include/internal/pycore_codecs.h \ + $(srcdir)/Include/internal/pycore_compile.h \ + $(srcdir)/Include/internal/pycore_complexobject.h \ + $(srcdir)/Include/internal/pycore_condvar.h \ + $(srcdir)/Include/internal/pycore_context.h \ + $(srcdir)/Include/internal/pycore_critical_section.h \ + $(srcdir)/Include/internal/pycore_crossinterp.h \ + $(srcdir)/Include/internal/pycore_descrobject.h \ + $(srcdir)/Include/internal/pycore_dict.h \ + $(srcdir)/Include/internal/pycore_dict_state.h \ + $(srcdir)/Include/internal/pycore_dtoa.h \ + $(srcdir)/Include/internal/pycore_exceptions.h \ + $(srcdir)/Include/internal/pycore_faulthandler.h \ + $(srcdir)/Include/internal/pycore_fileutils.h \ + $(srcdir)/Include/internal/pycore_floatobject.h \ + $(srcdir)/Include/internal/pycore_flowgraph.h \ + $(srcdir)/Include/internal/pycore_format.h \ + $(srcdir)/Include/internal/pycore_frame.h \ + $(srcdir)/Include/internal/pycore_freelist.h \ + $(srcdir)/Include/internal/pycore_function.h \ + $(srcdir)/Include/internal/pycore_gc.h \ + $(srcdir)/Include/internal/pycore_genobject.h \ + $(srcdir)/Include/internal/pycore_getopt.h \ + $(srcdir)/Include/internal/pycore_gil.h \ + $(srcdir)/Include/internal/pycore_global_objects.h \ + $(srcdir)/Include/internal/pycore_global_objects_fini_generated.h \ + $(srcdir)/Include/internal/pycore_global_strings.h \ + $(srcdir)/Include/internal/pycore_hamt.h \ + $(srcdir)/Include/internal/pycore_hashtable.h \ + $(srcdir)/Include/internal/pycore_identifier.h \ + $(srcdir)/Include/internal/pycore_import.h \ + $(srcdir)/Include/internal/pycore_importdl.h \ + $(srcdir)/Include/internal/pycore_initconfig.h \ + $(srcdir)/Include/internal/pycore_instruments.h \ + $(srcdir)/Include/internal/pycore_interp.h \ + $(srcdir)/Include/internal/pycore_intrinsics.h \ + $(srcdir)/Include/internal/pycore_jit.h \ + $(srcdir)/Include/internal/pycore_list.h \ + $(srcdir)/Include/internal/pycore_llist.h \ + $(srcdir)/Include/internal/pycore_lock.h \ + $(srcdir)/Include/internal/pycore_long.h \ + $(srcdir)/Include/internal/pycore_memoryobject.h \ + $(srcdir)/Include/internal/pycore_mimalloc.h \ + $(srcdir)/Include/internal/pycore_modsupport.h \ + $(srcdir)/Include/internal/pycore_moduleobject.h \ + $(srcdir)/Include/internal/pycore_namespace.h \ + $(srcdir)/Include/internal/pycore_object.h \ + $(srcdir)/Include/internal/pycore_object_alloc.h \ + $(srcdir)/Include/internal/pycore_object_stack.h \ + $(srcdir)/Include/internal/pycore_object_state.h \ + $(srcdir)/Include/internal/pycore_obmalloc.h \ + $(srcdir)/Include/internal/pycore_obmalloc_init.h \ + $(srcdir)/Include/internal/pycore_opcode_metadata.h \ + $(srcdir)/Include/internal/pycore_opcode_utils.h \ + $(srcdir)/Include/internal/pycore_optimizer.h \ + $(srcdir)/Include/internal/pycore_parking_lot.h \ + $(srcdir)/Include/internal/pycore_parser.h \ + $(srcdir)/Include/internal/pycore_pathconfig.h \ + $(srcdir)/Include/internal/pycore_pyarena.h \ + $(srcdir)/Include/internal/pycore_pybuffer.h \ + $(srcdir)/Include/internal/pycore_pyerrors.h \ + $(srcdir)/Include/internal/pycore_pyhash.h \ + $(srcdir)/Include/internal/pycore_pylifecycle.h \ + $(srcdir)/Include/internal/pycore_pymath.h \ + $(srcdir)/Include/internal/pycore_pymem.h \ + $(srcdir)/Include/internal/pycore_pymem_init.h \ + $(srcdir)/Include/internal/pycore_pystate.h \ + $(srcdir)/Include/internal/pycore_pystats.h \ + $(srcdir)/Include/internal/pycore_pythonrun.h \ + $(srcdir)/Include/internal/pycore_pythread.h \ + $(srcdir)/Include/internal/pycore_range.h \ + $(srcdir)/Include/internal/pycore_runtime.h \ + $(srcdir)/Include/internal/pycore_runtime_init.h \ + $(srcdir)/Include/internal/pycore_runtime_init_generated.h \ + $(srcdir)/Include/internal/pycore_semaphore.h \ + $(srcdir)/Include/internal/pycore_setobject.h \ + $(srcdir)/Include/internal/pycore_signal.h \ + $(srcdir)/Include/internal/pycore_sliceobject.h \ + $(srcdir)/Include/internal/pycore_strhex.h \ + $(srcdir)/Include/internal/pycore_structseq.h \ + $(srcdir)/Include/internal/pycore_symtable.h \ + $(srcdir)/Include/internal/pycore_sysmodule.h \ + $(srcdir)/Include/internal/pycore_time.h \ + $(srcdir)/Include/internal/pycore_token.h \ + $(srcdir)/Include/internal/pycore_traceback.h \ + $(srcdir)/Include/internal/pycore_tracemalloc.h \ + $(srcdir)/Include/internal/pycore_tstate.h \ + $(srcdir)/Include/internal/pycore_tuple.h \ + $(srcdir)/Include/internal/pycore_typeobject.h \ + $(srcdir)/Include/internal/pycore_typevarobject.h \ + $(srcdir)/Include/internal/pycore_ucnhash.h \ + $(srcdir)/Include/internal/pycore_unicodeobject.h \ + $(srcdir)/Include/internal/pycore_unicodeobject_generated.h \ + $(srcdir)/Include/internal/pycore_unionobject.h \ + $(srcdir)/Include/internal/pycore_uop_ids.h \ + $(srcdir)/Include/internal/pycore_uop_metadata.h \ + $(srcdir)/Include/internal/pycore_warnings.h \ + $(srcdir)/Include/internal/pycore_weakref.h \ + $(DTRACE_HEADERS) \ + @PLATFORM_HEADERS@ \ + \ + $(srcdir)/Python/stdlib_module_names.h + ########################################################################## # Build static libmpdec.a LIBMPDEC_CFLAGS=@LIBMPDEC_CFLAGS@ $(PY_STDMODULE_CFLAGS) $(CCSHARED) @@ -1361,7 +1623,7 @@ regen-unicodedata: regen-all: regen-cases regen-typeslots \ regen-token regen-ast regen-keyword regen-sre regen-frozen \ regen-pegen-metaparser regen-pegen regen-test-frozenmain \ - regen-test-levenshtein regen-global-objects regen-sbom + regen-test-levenshtein regen-global-objects regen-sbom regen-jit @echo @echo "Note: make regen-stdlib-module-names, make regen-limited-abi, " @echo "make regen-configure and make regen-unicodedata should be run manually" @@ -1395,7 +1657,7 @@ Modules/getpath.o: $(srcdir)/Modules/getpath.c Python/frozen_modules/getpath.h M Programs/python.o: $(srcdir)/Programs/python.c $(CC) -c $(PY_CORE_CFLAGS) -o $@ $(srcdir)/Programs/python.c -Programs/_testembed.o: $(srcdir)/Programs/_testembed.c Programs/test_frozenmain.h +Programs/_testembed.o: $(srcdir)/Programs/_testembed.c Programs/test_frozenmain.h $(PYTHON_HEADERS) $(CC) -c $(PY_CORE_CFLAGS) -o $@ $(srcdir)/Programs/_testembed.c Modules/_sre/sre.o: $(srcdir)/Modules/_sre/sre.c $(srcdir)/Modules/_sre/sre.h $(srcdir)/Modules/_sre/sre_constants.h $(srcdir)/Modules/_sre/sre_lib.h @@ -1647,8 +1909,8 @@ Include/pydtrace_probes.h: $(srcdir)/Include/pydtrace.d mv $@.tmp $@ Python/ceval.o: $(srcdir)/Include/pydtrace.h +Python/gc.o: $(srcdir)/Include/pydtrace.h Python/import.o: $(srcdir)/Include/pydtrace.h -Modules/gcmodule.o: $(srcdir)/Include/pydtrace.h Python/pydtrace.o: $(srcdir)/Include/pydtrace.d $(DTRACE_DEPS) $(DTRACE) $(DFLAGS) -o $@ -G -s $< $(DTRACE_DEPS) @@ -1664,242 +1926,6 @@ regen-typeslots: $(srcdir)/Objects/typeslots.inc.new $(UPDATE_FILE) $(srcdir)/Objects/typeslots.inc $(srcdir)/Objects/typeslots.inc.new -############################################################################ -# Header files - -PYTHON_HEADERS= \ - $(srcdir)/Include/Python.h \ - $(srcdir)/Include/abstract.h \ - $(srcdir)/Include/bltinmodule.h \ - $(srcdir)/Include/boolobject.h \ - $(srcdir)/Include/bytearrayobject.h \ - $(srcdir)/Include/bytesobject.h \ - $(srcdir)/Include/ceval.h \ - $(srcdir)/Include/codecs.h \ - $(srcdir)/Include/compile.h \ - $(srcdir)/Include/complexobject.h \ - $(srcdir)/Include/descrobject.h \ - $(srcdir)/Include/dictobject.h \ - $(srcdir)/Include/dynamic_annotations.h \ - $(srcdir)/Include/enumobject.h \ - $(srcdir)/Include/errcode.h \ - $(srcdir)/Include/fileobject.h \ - $(srcdir)/Include/fileutils.h \ - $(srcdir)/Include/floatobject.h \ - $(srcdir)/Include/frameobject.h \ - $(srcdir)/Include/import.h \ - $(srcdir)/Include/interpreteridobject.h \ - $(srcdir)/Include/intrcheck.h \ - $(srcdir)/Include/iterobject.h \ - $(srcdir)/Include/listobject.h \ - $(srcdir)/Include/longobject.h \ - $(srcdir)/Include/marshal.h \ - $(srcdir)/Include/memoryobject.h \ - $(srcdir)/Include/methodobject.h \ - $(srcdir)/Include/modsupport.h \ - $(srcdir)/Include/moduleobject.h \ - $(srcdir)/Include/object.h \ - $(srcdir)/Include/objimpl.h \ - $(srcdir)/Include/opcode.h \ - $(srcdir)/Include/opcode_ids.h \ - $(srcdir)/Include/osdefs.h \ - $(srcdir)/Include/osmodule.h \ - $(srcdir)/Include/patchlevel.h \ - $(srcdir)/Include/pybuffer.h \ - $(srcdir)/Include/pycapsule.h \ - $(srcdir)/Include/pydtrace.h \ - $(srcdir)/Include/pyerrors.h \ - $(srcdir)/Include/pyframe.h \ - $(srcdir)/Include/pyhash.h \ - $(srcdir)/Include/pylifecycle.h \ - $(srcdir)/Include/pymacconfig.h \ - $(srcdir)/Include/pymacro.h \ - $(srcdir)/Include/pymath.h \ - $(srcdir)/Include/pymem.h \ - $(srcdir)/Include/pyport.h \ - $(srcdir)/Include/pystate.h \ - $(srcdir)/Include/pystats.h \ - $(srcdir)/Include/pystrcmp.h \ - $(srcdir)/Include/pystrtod.h \ - $(srcdir)/Include/pythonrun.h \ - $(srcdir)/Include/pythread.h \ - $(srcdir)/Include/pytypedefs.h \ - $(srcdir)/Include/rangeobject.h \ - $(srcdir)/Include/setobject.h \ - $(srcdir)/Include/sliceobject.h \ - $(srcdir)/Include/structmember.h \ - $(srcdir)/Include/structseq.h \ - $(srcdir)/Include/sysmodule.h \ - $(srcdir)/Include/traceback.h \ - $(srcdir)/Include/tupleobject.h \ - $(srcdir)/Include/unicodeobject.h \ - $(srcdir)/Include/warnings.h \ - $(srcdir)/Include/weakrefobject.h \ - \ - pyconfig.h \ - $(PARSER_HEADERS) \ - \ - $(srcdir)/Include/cpython/abstract.h \ - $(srcdir)/Include/cpython/bytearrayobject.h \ - $(srcdir)/Include/cpython/bytesobject.h \ - $(srcdir)/Include/cpython/cellobject.h \ - $(srcdir)/Include/cpython/ceval.h \ - $(srcdir)/Include/cpython/classobject.h \ - $(srcdir)/Include/cpython/code.h \ - $(srcdir)/Include/cpython/compile.h \ - $(srcdir)/Include/cpython/complexobject.h \ - $(srcdir)/Include/cpython/context.h \ - $(srcdir)/Include/cpython/descrobject.h \ - $(srcdir)/Include/cpython/dictobject.h \ - $(srcdir)/Include/cpython/fileobject.h \ - $(srcdir)/Include/cpython/fileutils.h \ - $(srcdir)/Include/cpython/floatobject.h \ - $(srcdir)/Include/cpython/frameobject.h \ - $(srcdir)/Include/cpython/funcobject.h \ - $(srcdir)/Include/cpython/genobject.h \ - $(srcdir)/Include/cpython/import.h \ - $(srcdir)/Include/cpython/initconfig.h \ - $(srcdir)/Include/cpython/interpreteridobject.h \ - $(srcdir)/Include/cpython/listobject.h \ - $(srcdir)/Include/cpython/longintrepr.h \ - $(srcdir)/Include/cpython/longobject.h \ - $(srcdir)/Include/cpython/memoryobject.h \ - $(srcdir)/Include/cpython/methodobject.h \ - $(srcdir)/Include/cpython/object.h \ - $(srcdir)/Include/cpython/objimpl.h \ - $(srcdir)/Include/cpython/odictobject.h \ - $(srcdir)/Include/cpython/optimizer.h \ - $(srcdir)/Include/cpython/picklebufobject.h \ - $(srcdir)/Include/cpython/pthread_stubs.h \ - $(srcdir)/Include/cpython/pyatomic.h \ - $(srcdir)/Include/cpython/pyatomic_gcc.h \ - $(srcdir)/Include/cpython/pyatomic_std.h \ - $(srcdir)/Include/cpython/pyctype.h \ - $(srcdir)/Include/cpython/pydebug.h \ - $(srcdir)/Include/cpython/pyerrors.h \ - $(srcdir)/Include/cpython/pyfpe.h \ - $(srcdir)/Include/cpython/pyframe.h \ - $(srcdir)/Include/cpython/pyhash.h \ - $(srcdir)/Include/cpython/pylifecycle.h \ - $(srcdir)/Include/cpython/pymem.h \ - $(srcdir)/Include/cpython/pystate.h \ - $(srcdir)/Include/cpython/pystats.h \ - $(srcdir)/Include/cpython/pythonrun.h \ - $(srcdir)/Include/cpython/pythread.h \ - $(srcdir)/Include/cpython/setobject.h \ - $(srcdir)/Include/cpython/sysmodule.h \ - $(srcdir)/Include/cpython/traceback.h \ - $(srcdir)/Include/cpython/tracemalloc.h \ - $(srcdir)/Include/cpython/tupleobject.h \ - $(srcdir)/Include/cpython/unicodeobject.h \ - $(srcdir)/Include/cpython/warnings.h \ - $(srcdir)/Include/cpython/weakrefobject.h \ - \ - $(MIMALLOC_HEADERS) \ - \ - $(srcdir)/Include/internal/pycore_abstract.h \ - $(srcdir)/Include/internal/pycore_asdl.h \ - $(srcdir)/Include/internal/pycore_ast.h \ - $(srcdir)/Include/internal/pycore_ast_state.h \ - $(srcdir)/Include/internal/pycore_atexit.h \ - $(srcdir)/Include/internal/pycore_bitutils.h \ - $(srcdir)/Include/internal/pycore_bytes_methods.h \ - $(srcdir)/Include/internal/pycore_bytesobject.h \ - $(srcdir)/Include/internal/pycore_call.h \ - $(srcdir)/Include/internal/pycore_capsule.h \ - $(srcdir)/Include/internal/pycore_ceval.h \ - $(srcdir)/Include/internal/pycore_ceval_state.h \ - $(srcdir)/Include/internal/pycore_code.h \ - $(srcdir)/Include/internal/pycore_codecs.h \ - $(srcdir)/Include/internal/pycore_compile.h \ - $(srcdir)/Include/internal/pycore_complexobject.h \ - $(srcdir)/Include/internal/pycore_condvar.h \ - $(srcdir)/Include/internal/pycore_context.h \ - $(srcdir)/Include/internal/pycore_critical_section.h \ - $(srcdir)/Include/internal/pycore_crossinterp.h \ - $(srcdir)/Include/internal/pycore_dict.h \ - $(srcdir)/Include/internal/pycore_dict_state.h \ - $(srcdir)/Include/internal/pycore_descrobject.h \ - $(srcdir)/Include/internal/pycore_dtoa.h \ - $(srcdir)/Include/internal/pycore_exceptions.h \ - $(srcdir)/Include/internal/pycore_faulthandler.h \ - $(srcdir)/Include/internal/pycore_fileutils.h \ - $(srcdir)/Include/internal/pycore_floatobject.h \ - $(srcdir)/Include/internal/pycore_format.h \ - $(srcdir)/Include/internal/pycore_frame.h \ - $(srcdir)/Include/internal/pycore_function.h \ - $(srcdir)/Include/internal/pycore_genobject.h \ - $(srcdir)/Include/internal/pycore_getopt.h \ - $(srcdir)/Include/internal/pycore_gil.h \ - $(srcdir)/Include/internal/pycore_global_objects.h \ - $(srcdir)/Include/internal/pycore_global_objects_fini_generated.h \ - $(srcdir)/Include/internal/pycore_hamt.h \ - $(srcdir)/Include/internal/pycore_hashtable.h \ - $(srcdir)/Include/internal/pycore_identifier.h \ - $(srcdir)/Include/internal/pycore_import.h \ - $(srcdir)/Include/internal/pycore_initconfig.h \ - $(srcdir)/Include/internal/pycore_interp.h \ - $(srcdir)/Include/internal/pycore_intrinsics.h \ - $(srcdir)/Include/internal/pycore_list.h \ - $(srcdir)/Include/internal/pycore_llist.h \ - $(srcdir)/Include/internal/pycore_lock.h \ - $(srcdir)/Include/internal/pycore_long.h \ - $(srcdir)/Include/internal/pycore_modsupport.h \ - $(srcdir)/Include/internal/pycore_moduleobject.h \ - $(srcdir)/Include/internal/pycore_namespace.h \ - $(srcdir)/Include/internal/pycore_object.h \ - $(srcdir)/Include/internal/pycore_object_state.h \ - $(srcdir)/Include/internal/pycore_obmalloc.h \ - $(srcdir)/Include/internal/pycore_obmalloc_init.h \ - $(srcdir)/Include/internal/pycore_opcode_metadata.h \ - $(srcdir)/Include/internal/pycore_opcode_utils.h \ - $(srcdir)/Include/internal/pycore_optimizer.h \ - $(srcdir)/Include/internal/pycore_parking_lot.h \ - $(srcdir)/Include/internal/pycore_pathconfig.h \ - $(srcdir)/Include/internal/pycore_pyarena.h \ - $(srcdir)/Include/internal/pycore_pybuffer.h \ - $(srcdir)/Include/internal/pycore_pyerrors.h \ - $(srcdir)/Include/internal/pycore_pyhash.h \ - $(srcdir)/Include/internal/pycore_pylifecycle.h \ - $(srcdir)/Include/internal/pycore_pymem.h \ - $(srcdir)/Include/internal/pycore_pymem_init.h \ - $(srcdir)/Include/internal/pycore_pystate.h \ - $(srcdir)/Include/internal/pycore_pystats.h \ - $(srcdir)/Include/internal/pycore_pythonrun.h \ - $(srcdir)/Include/internal/pycore_pythread.h \ - $(srcdir)/Include/internal/pycore_range.h \ - $(srcdir)/Include/internal/pycore_runtime.h \ - $(srcdir)/Include/internal/pycore_runtime_init_generated.h \ - $(srcdir)/Include/internal/pycore_runtime_init.h \ - $(srcdir)/Include/internal/pycore_semaphore.h \ - $(srcdir)/Include/internal/pycore_setobject.h \ - $(srcdir)/Include/internal/pycore_signal.h \ - $(srcdir)/Include/internal/pycore_sliceobject.h \ - $(srcdir)/Include/internal/pycore_strhex.h \ - $(srcdir)/Include/internal/pycore_structseq.h \ - $(srcdir)/Include/internal/pycore_symtable.h \ - $(srcdir)/Include/internal/pycore_sysmodule.h \ - $(srcdir)/Include/internal/pycore_time.h \ - $(srcdir)/Include/internal/pycore_token.h \ - $(srcdir)/Include/internal/pycore_traceback.h \ - $(srcdir)/Include/internal/pycore_tracemalloc.h \ - $(srcdir)/Include/internal/pycore_tstate.h \ - $(srcdir)/Include/internal/pycore_tuple.h \ - $(srcdir)/Include/internal/pycore_typeobject.h \ - $(srcdir)/Include/internal/pycore_typevarobject.h \ - $(srcdir)/Include/internal/pycore_ucnhash.h \ - $(srcdir)/Include/internal/pycore_unionobject.h \ - $(srcdir)/Include/internal/pycore_unicodeobject.h \ - $(srcdir)/Include/internal/pycore_unicodeobject_generated.h \ - $(srcdir)/Include/internal/pycore_uops.h \ - $(srcdir)/Include/internal/pycore_uop_metadata.h \ - $(srcdir)/Include/internal/pycore_warnings.h \ - $(srcdir)/Include/internal/pycore_weakref.h \ - $(DTRACE_HEADERS) \ - @PLATFORM_HEADERS@ \ - \ - $(srcdir)/Python/stdlib_module_names.h - $(LIBRARY_OBJS) $(MODOBJS) Programs/python.o: $(PYTHON_HEADERS) @@ -2218,6 +2244,7 @@ TESTSUBDIRS= idlelib/idle_test \ test/test_cppext \ test/test_ctypes \ test/test_dataclasses \ + test/test_doctest \ test/test_email \ test/test_email/data \ test/test_future_stmt \ @@ -2313,6 +2340,7 @@ TESTSUBDIRS= idlelib/idle_test \ test/tokenizedata \ test/tracedmodules \ test/typinganndata \ + test/wheeldata \ test/xmltestdata \ test/xmltestdata/c14n-20 @@ -2632,6 +2660,23 @@ config.status: $(srcdir)/configure Python/asm_trampoline.o: $(srcdir)/Python/asm_trampoline.S $(CC) -c $(PY_CORE_CFLAGS) -o $@ $< + +JIT_DEPS = \ + $(srcdir)/Tools/jit/*.c \ + $(srcdir)/Tools/jit/*.py \ + $(srcdir)/Python/executor_cases.c.h \ + pyconfig.h + +jit_stencils.h: $(JIT_DEPS) + @REGEN_JIT_COMMAND@ + +Python/jit.o: $(srcdir)/Python/jit.c @JIT_STENCILS_H@ + $(CC) -c $(PY_CORE_CFLAGS) -o $@ $< + +.PHONY: regen-jit +regen-jit: + @REGEN_JIT_COMMAND@ + # Some make's put the object file in the current directory .c.o: $(CC) -c $(PY_CORE_CFLAGS) -o $@ $< @@ -2724,6 +2769,7 @@ clean-retain-profile: pycremoval -rm -f Python/deepfreeze/*.[co] -rm -f Python/frozen_modules/*.h -rm -f Python/frozen_modules/MANIFEST + -rm -f jit_stencils.h -find build -type f -a ! -name '*.gc??' -exec rm -f {} ';' -rm -f Include/pydtrace_probes.h -rm -f profile-gen-stamp @@ -2848,6 +2894,9 @@ Python/thread.o: @THREADHEADERS@ $(srcdir)/Python/condvar.h MODULE_DEPS_STATIC=Modules/config.c MODULE_DEPS_SHARED=$(MODULE_DEPS_STATIC) $(EXPORTSYMS) +MODULE__CURSES_DEPS=$(srcdir)/Include/py_curses.h +MODULE__CURSES_PANEL_DEPS=$(srcdir)/Include/py_curses.h +MODULE__DATETIME_DEPS=$(srcdir)/Include/datetime.h MODULE_CMATH_DEPS=$(srcdir)/Modules/_math.h MODULE_MATH_DEPS=$(srcdir)/Modules/_math.h MODULE_PYEXPAT_DEPS=@LIBEXPAT_INTERNAL@ @@ -2864,7 +2913,7 @@ MODULE__SHA1_DEPS=$(srcdir)/Modules/hashlib.h $(LIBHACL_HEADERS) Modules/_hacl/H MODULE__SHA2_DEPS=$(srcdir)/Modules/hashlib.h $(LIBHACL_SHA2_HEADERS) $(LIBHACL_SHA2_A) MODULE__SHA3_DEPS=$(srcdir)/Modules/hashlib.h $(LIBHACL_HEADERS) Modules/_hacl/Hacl_Hash_SHA3.h Modules/_hacl/Hacl_Hash_SHA3.c MODULE__SOCKET_DEPS=$(srcdir)/Modules/socketmodule.h $(srcdir)/Modules/addrinfo.h $(srcdir)/Modules/getaddrinfo.c $(srcdir)/Modules/getnameinfo.c -MODULE__SSL_DEPS=$(srcdir)/Modules/_ssl.h $(srcdir)/Modules/_ssl/cert.c $(srcdir)/Modules/_ssl/debughelpers.c $(srcdir)/Modules/_ssl/misc.c $(srcdir)/Modules/_ssl_data.h $(srcdir)/Modules/_ssl_data_111.h $(srcdir)/Modules/_ssl_data_300.h $(srcdir)/Modules/socketmodule.h +MODULE__SSL_DEPS=$(srcdir)/Modules/_ssl.h $(srcdir)/Modules/_ssl/cert.c $(srcdir)/Modules/_ssl/debughelpers.c $(srcdir)/Modules/_ssl/misc.c $(srcdir)/Modules/_ssl_data_111.h $(srcdir)/Modules/_ssl_data_300.h $(srcdir)/Modules/socketmodule.h MODULE__TESTCAPI_DEPS=$(srcdir)/Modules/_testcapi/testcapi_long.h $(srcdir)/Modules/_testcapi/parts.h $(srcdir)/Modules/_testcapi/util.h MODULE__TESTINTERNALCAPI_DEPS=$(srcdir)/Modules/_testinternalcapi/parts.h MODULE__SQLITE3_DEPS=$(srcdir)/Modules/_sqlite/connection.h $(srcdir)/Modules/_sqlite/cursor.h $(srcdir)/Modules/_sqlite/microprotocols.h $(srcdir)/Modules/_sqlite/module.h $(srcdir)/Modules/_sqlite/prepare_protocol.h $(srcdir)/Modules/_sqlite/row.h $(srcdir)/Modules/_sqlite/util.h diff --git a/Misc/ACKS b/Misc/ACKS index 6b98be32905391..8a80e02ecba26a 100644 --- a/Misc/ACKS +++ b/Misc/ACKS @@ -31,6 +31,7 @@ Farhan Ahmad Matthew Ahrens Nir Aides Akira +Ege Akman Yaniv Aknin Jyrki Alakuijala Tatiana Al-Chueyr @@ -665,6 +666,7 @@ Eddy De Greef Duane Griffin Grant Griffin Andrea Griffini +Semyon Grigoryev Duncan Grisby Olivier Grisel Fabian Groffen @@ -962,6 +964,7 @@ Carsten Klein Bastian Kleineidam Joel Klimont Bob Kline +Alois Klink Matthias Klose Jeremy Kloth Thomas Kluyver @@ -1048,6 +1051,7 @@ Mark Lawrence Chris Laws Michael Layzell Michael Lazar +Peter Lazorchak Brian Leair Mathieu Leduc-Hamel Amandine Lee diff --git a/Misc/NEWS.d/3.10.0a1.rst b/Misc/NEWS.d/3.10.0a1.rst index 731eed3447d2bc..4842a026aa49f7 100644 --- a/Misc/NEWS.d/3.10.0a1.rst +++ b/Misc/NEWS.d/3.10.0a1.rst @@ -4,7 +4,7 @@ .. release date: 2020-10-05 .. section: Security -Fixes `python3x._pth` being ignored on Windows, caused by the fix for +Fixes ``python3x._pth`` being ignored on Windows, caused by the fix for :issue:`29778` (CVE-2020-15801). .. @@ -217,7 +217,7 @@ Port the :mod:`_opcode` extension module to multi-phase initialization .. nonce: 3-VJiH .. section: Core and Builtins -Fixes the wrong error description in the error raised by using 2 `,` in +Fixes the wrong error description in the error raised by using 2 ``,`` in format string in f-string and :meth:`str.format`. .. @@ -747,7 +747,7 @@ Galindo. Fix a bug where a line with only a line continuation character is not considered a blank line at tokenizer level. In such cases, more than a -single `NEWLINE` token was emitted. The old parser was working around the +single ``NEWLINE`` token was emitted. The old parser was working around the issue, but the new parser threw a :exc:`SyntaxError` for valid input due to this. For example, an empty line following a line continuation character was interpreted as a :exc:`SyntaxError`. @@ -1157,7 +1157,7 @@ The :class:`threading.Thread` constructor now uses the target name if the .. nonce: bnh-VG .. section: Library -fix `tkinter.EventType` Enum so all members are strings, and none are tuples +fix ``tkinter.EventType`` Enum so all members are strings, and none are tuples .. @@ -1220,7 +1220,7 @@ class Previously there was no way to check that without using private API. See the `relevant issue in python/typing -` +`_. .. @@ -1229,8 +1229,8 @@ Previously there was no way to check that without using private API. See the .. nonce: pI_uZQ .. section: Library -Honor `object` overrides in `Enum` class creation (specifically, `__str__`, -`__repr__`, `__format__`, and `__reduce_ex__`). +Honor ``object`` overrides in ``Enum`` class creation (specifically, ``__str__``, +``__repr__``, ``__format__``, and ``__reduce_ex__``). .. @@ -1239,7 +1239,7 @@ Honor `object` overrides in `Enum` class creation (specifically, `__str__`, .. nonce: IpYkEe .. section: Library -`enum.Flag` and `enum.IntFlag` members are now iterable +``enum.Flag`` and ``enum.IntFlag`` members are now iterable. .. @@ -1557,7 +1557,7 @@ activation. .. nonce: wqrj8C .. section: Library -Recursive evaluation of `typing.ForwardRef` in `get_type_hints`. +Recursive evaluation of ``typing.ForwardRef`` in ``get_type_hints``. .. @@ -1851,8 +1851,8 @@ Merry. .. nonce: 1dk8Bu .. section: Library -:mod:`ensurepip` now disables the use of `pip` cache when installing the -bundled versions of `pip` and `setuptools`. Patch by Krzysztof Konopko. +:mod:`ensurepip` now disables the use of ``pip`` cache when installing the +bundled versions of ``pip`` and ``setuptools``. Patch by Krzysztof Konopko. .. @@ -1933,7 +1933,7 @@ Smith and Tal Einat. .. nonce: n7fOwS .. section: Library -Added a `defaults` parameter to :class:`logging.Formatter`, to allow +Added a ``defaults`` parameter to :class:`logging.Formatter`, to allow specifying default values for custom fields. Patch by Asaf Alon and Bar Harel. @@ -2225,7 +2225,7 @@ policy. .. nonce: ps7Yf1 .. section: Library -func:`hashlib.new` passed ``usedforsecurity`` to OpenSSL EVP constructor +:func:`hashlib.new` passed ``usedforsecurity`` to OpenSSL EVP constructor ``_hashlib.new()``. test_hashlib and test_smtplib handle strict security policy better. @@ -2393,8 +2393,8 @@ closes connection during TLS negotiation .. nonce: kOOaHn .. section: Library -fix default `_missing_` so a duplicate `ValueError` is not set as the -`__context__` of the original `ValueError` +fix default ``_missing_`` so a duplicate ``ValueError`` is not set as the +``__context__`` of the original ``ValueError``. .. @@ -2527,7 +2527,7 @@ in Python 3.4 and removed in Python 3.5. .. nonce: BE7zbu .. section: Library -Fix `cgi.parse_multipart` without content_length. Patch by Roger Duran +Fix ``cgi.parse_multipart`` without content_length. Patch by Roger Duran .. diff --git a/Misc/NEWS.d/3.10.0a2.rst b/Misc/NEWS.d/3.10.0a2.rst index 78f4377656b0cc..79f570439b52b8 100644 --- a/Misc/NEWS.d/3.10.0a2.rst +++ b/Misc/NEWS.d/3.10.0a2.rst @@ -88,10 +88,10 @@ Goldschmidt. .. nonce: d4a-8o .. section: Core and Builtins -Fix a bug in the parser, where a curly brace following a `primary` didn't -fail immediately. This led to invalid expressions like `a {b}` to throw a +Fix a bug in the parser, where a curly brace following a ``primary`` didn't +fail immediately. This led to invalid expressions like ``a {b}`` to throw a :exc:`SyntaxError` with a wrong offset, or invalid expressions ending with a -curly brace like `a {` to not fail immediately in the REPL. +curly brace like ``a {`` to not fail immediately in the REPL. .. @@ -214,7 +214,7 @@ Micro optimization for range.index if step is 1. Patch by Donghee Na. .. nonce: qPWjJA .. section: Core and Builtins -Add `sys._current_exceptions()` function to retrieve a dictionary mapping +Add ``sys._current_exceptions()`` function to retrieve a dictionary mapping each thread's identifier to the topmost exception currently active in that thread at the time the function is called. @@ -302,7 +302,7 @@ type to a heap type. Patch by Mohamed Koubaa and Victor Stinner. .. section: Library Fix memory leak in :func:`subprocess.Popen` in case an uid (gid) specified -in `user` (`group`, `extra_groups`) overflows `uid_t` (`gid_t`). +in ``user`` (``group``, ``extra_groups``) overflows ``uid_t`` (``gid_t``). .. @@ -649,7 +649,7 @@ Document __format__ functionality for IP addresses. .. nonce: CzBMit .. section: Documentation -Document the default implementation of `object.__eq__`. +Document the default implementation of ``object.__eq__``. .. @@ -878,7 +878,7 @@ targeting 3.10 or later. .. nonce: sh8IDY .. section: C API -Add `_Py_closerange` function to provide performant closing of a range of +Add ``_Py_closerange`` function to provide performant closing of a range of file descriptors. .. @@ -898,7 +898,7 @@ available again in limited API. .. nonce: ZZ5wJG .. section: C API -Add `PyIter_Send` function to allow sending value into +Add ``PyIter_Send`` function to allow sending value into generator/coroutine/iterator without raising StopIteration exception to signal return. diff --git a/Misc/NEWS.d/3.10.0a3.rst b/Misc/NEWS.d/3.10.0a3.rst index 7112819c1b4118..179cf3e9cfb08c 100644 --- a/Misc/NEWS.d/3.10.0a3.rst +++ b/Misc/NEWS.d/3.10.0a3.rst @@ -247,8 +247,8 @@ fixes union type expressions not de-duplicating ``GenericAlias`` objects. .. nonce: B-Veg7 .. section: Core and Builtins -The import system triggers a `ImportWarning` when it falls back to using -`load_module()`. +The import system triggers a ``ImportWarning`` when it falls back to using +``load_module()``. .. @@ -464,8 +464,8 @@ Support signal module on VxWorks. .. nonce: r9rNCj .. section: Library -We fixed an issue in `pickle.whichmodule` in which importing -`multiprocessing` could change the how pickle identifies which module an +We fixed an issue in ``pickle.whichmodule`` in which importing +``multiprocessing`` could change the how pickle identifies which module an object belongs to, potentially breaking the unpickling of those objects. .. @@ -602,7 +602,7 @@ function when ``os.open`` fails. .. nonce: F363jO .. section: Library -Fix `os.sendfile()` on illumos. +Fix ``os.sendfile()`` on illumos. .. @@ -718,8 +718,8 @@ Improve asyncio.wait function to create the futures set just one time. .. nonce: BzizYV .. section: Library -Update various modules in the stdlib to fall back on `__spec__.loader` when -`__loader__` isn't defined on a module. +Update various modules in the stdlib to fall back on ``__spec__.loader`` when +``__loader__`` isn't defined on a module. .. @@ -728,8 +728,8 @@ Update various modules in the stdlib to fall back on `__spec__.loader` when .. nonce: CAsI3O .. section: Library -The `load_module()` methods found in importlib now trigger a -DeprecationWarning. +The ``load_module()`` methods found in ``importlib`` now trigger a +``DeprecationWarning``. .. @@ -872,7 +872,7 @@ extension during TLS handshake when no custom context is supplied. .. nonce: 5mi7b0 .. section: Library -Add func:`os.eventfd` to provide a low level interface for Linux's event +Add :func:`os.eventfd` to provide a low level interface for Linux's event notification file descriptor. .. diff --git a/Misc/NEWS.d/3.10.0a4.rst b/Misc/NEWS.d/3.10.0a4.rst index 414823f162d85c..398f7e5d3422cb 100644 --- a/Misc/NEWS.d/3.10.0a4.rst +++ b/Misc/NEWS.d/3.10.0a4.rst @@ -516,7 +516,7 @@ Define THREAD_STACK_SIZE for VxWorks. .. nonce: x8TASR .. section: Library -[Enum] `_EnumDict.update()` is now supported +[Enum] ``_EnumDict.update()`` is now supported. .. diff --git a/Misc/NEWS.d/3.10.0a7.rst b/Misc/NEWS.d/3.10.0a7.rst index d9cdfbd04c88d4..74120a3b40c012 100644 --- a/Misc/NEWS.d/3.10.0a7.rst +++ b/Misc/NEWS.d/3.10.0a7.rst @@ -261,7 +261,7 @@ the annotations _Py_NO_SANITIZE_ADDRESS, _Py_NO_SANITIZE_THREAD, and _Py_NO_SANITIZE_MEMORY. Those annotations are no longer needed. To disable the radix tree map, set a preprocessor flag as follows: -`-DWITH_PYMALLOC_RADIX_TREE=0`. +``-DWITH_PYMALLOC_RADIX_TREE=0``. Co-authored-by: Tim Peters @@ -664,11 +664,11 @@ on failure. .. nonce: f1dr_5 .. section: Library -Enum's `repr()` and `str()` have changed: `repr()` is now -*EnumClass.MemberName* and `str()` is *MemberName*. Additionally, stdlib +Enum's ``repr()`` and ``str()`` have changed: ``repr()`` is now +*EnumClass.MemberName* and ``str()`` is *MemberName*. Additionally, stdlib Enum's whose contents are available as module attributes, such as -`RegexFlag.IGNORECASE`, have their `repr()` as *module.name*, e.g. -`re.IGNORECASE`. +``RegexFlag.IGNORECASE``, have their ``repr()`` as *module.name*, e.g. +``re.IGNORECASE``. .. diff --git a/Misc/NEWS.d/3.10.0b1.rst b/Misc/NEWS.d/3.10.0b1.rst index e7b6b93d0b6df3..640f3ee58adbae 100644 --- a/Misc/NEWS.d/3.10.0b1.rst +++ b/Misc/NEWS.d/3.10.0b1.rst @@ -1181,8 +1181,8 @@ Lewis Gaul. .. nonce: NzLVaR .. section: Library -Add `pathlib.Path.hardlink_to()` method that supersedes `link_to()`. The new -method has the same argument order as `symlink_to()`. +Add ``pathlib.Path.hardlink_to()`` method that supersedes ``link_to()``. The new +method has the same argument order as ``symlink_to()``. .. diff --git a/Misc/NEWS.d/3.11.0a1.rst b/Misc/NEWS.d/3.11.0a1.rst index 26c44b6c1af0ed..e8d4a02a11e0f9 100644 --- a/Misc/NEWS.d/3.11.0a1.rst +++ b/Misc/NEWS.d/3.11.0a1.rst @@ -819,7 +819,7 @@ always available when needed. Patch by Mark Shannon. .. nonce: qKnSqV .. section: Core and Builtins -The threading debug (:envvar:`PYTHONTHREADDEBUG` environment variable) is +The threading debug (:envvar:`!PYTHONTHREADDEBUG` environment variable) is deprecated in Python 3.10 and will be removed in Python 3.12. This feature requires a debug build of Python. Patch by Victor Stinner. @@ -1642,9 +1642,9 @@ interval specified with nanosecond precision. .. nonce: UptGAn .. section: Library -Remove from the :mod:`configparser` module: the :class:`SafeConfigParser` -class, the :attr:`filename` property of the :class:`ParsingError` class, the -:meth:`readfp` method of the :class:`ConfigParser` class, deprecated since +Remove from the :mod:`configparser` module: the :class:`!SafeConfigParser` +class, the :attr:`!filename` property of the :class:`~configparser.ParsingError` class, the +:meth:`!readfp` method of the :class:`~configparser.ConfigParser` class, deprecated since Python 3.2. Patch by Hugo van Kemenade. @@ -2808,7 +2808,7 @@ behaves differently than the similar implementation in :mod:`sysconfig`. .. nonce: 3hmkWw .. section: Library -:class:`smtpd.MailmanProxy` is now removed as it is unusable without an +:class:`!smtpd.MailmanProxy` is now removed as it is unusable without an external module, ``mailman``. Patch by Donghee Na. .. @@ -3204,7 +3204,7 @@ deprecated, for removal in 3.14. .. section: Library When :class:`http.server.SimpleHTTPRequestHandler` sends a ``301 (Moved -Permanently)`` for a directory path not ending with `/`, add a +Permanently)`` for a directory path not ending with ``/``, add a ``Content-Length: 0`` header. This improves the behavior for certain clients. @@ -3240,7 +3240,7 @@ Patch by Erlend E. Aasland. .. nonce: m72tlH .. section: Library -AIX: `Lib/_aix_support.get_platform()` may fail in an AIX WPAR. The fileset +AIX: ``Lib/_aix_support.get_platform()`` may fail in an AIX WPAR. The fileset bos.rte appears to have a builddate in both LPAR and WPAR so this fileset is queried rather than bos.mp64. To prevent a similar situation (no builddate in ODM) a value (9988) sufficient for completing a build is provided. Patch @@ -3483,9 +3483,9 @@ Improved reprs of :mod:`threading` synchronization objects: Deprecated the following :mod:`unittest` functions, scheduled for removal in Python 3.13: -* :func:`~!unittest.findTestCases` -* :func:`~!unittest.makeSuite` -* :func:`~!unittest.getTestCaseNames` +* :func:`!findTestCases` +* :func:`!makeSuite` +* :func:`!getTestCaseNames` Use :class:`~unittest.TestLoader` methods instead: @@ -3685,9 +3685,9 @@ equivalents are valid. .. nonce: aJuvQF .. section: Documentation -Removed the othergui.rst file, any references to it, and the list of GUI +Removed the ``othergui.rst`` file, any references to it, and the list of GUI frameworks in the FAQ. In their place I've added links to the Python Wiki -`page on GUI frameworks `. +`page on GUI frameworks `_. .. diff --git a/Misc/NEWS.d/3.11.0a2.rst b/Misc/NEWS.d/3.11.0a2.rst index 503e489b658e4d..a6b5fe54b391c5 100644 --- a/Misc/NEWS.d/3.11.0a2.rst +++ b/Misc/NEWS.d/3.11.0a2.rst @@ -507,7 +507,7 @@ Waygood. .. section: Library Make :func:`inspect.getmodule` catch ``FileNotFoundError`` raised by -:'func:`inspect.getabsfile`, and return ``None`` to indicate that the module +:func:`inspect.getabsfile`, and return ``None`` to indicate that the module could not be determined. .. @@ -897,8 +897,8 @@ was often the case on macOS. .. nonce: F18qcE .. section: Tests -Add more test cases for `@functools.singledispatchmethod` when combined with -`@classmethod` or `@staticmethod`. +Add more test cases for ``@functools.singledispatchmethod`` when combined with +``@classmethod`` or ``@staticmethod``. .. @@ -1189,7 +1189,7 @@ context objects can now be disabled. .. section: C API Exclude :c:func:`PyWeakref_GET_OBJECT` from the limited C API. It never -worked since the :c:type:`PyWeakReference` structure is opaque in the +worked since the :c:type:`!PyWeakReference` structure is opaque in the limited C API. .. diff --git a/Misc/NEWS.d/3.11.0a3.rst b/Misc/NEWS.d/3.11.0a3.rst index a96a59115797ee..2842aad0e163d6 100644 --- a/Misc/NEWS.d/3.11.0a3.rst +++ b/Misc/NEWS.d/3.11.0a3.rst @@ -350,7 +350,7 @@ Galindo. .. section: Library Fix possible crash when getting an attribute of -class:`xml.etree.ElementTree.Element` simultaneously with replacing the +:class:`xml.etree.ElementTree.Element` simultaneously with replacing the ``attrib`` dict. .. diff --git a/Misc/NEWS.d/3.11.0a4.rst b/Misc/NEWS.d/3.11.0a4.rst index 3dd335929d655f..78b682f7a22cc6 100644 --- a/Misc/NEWS.d/3.11.0a4.rst +++ b/Misc/NEWS.d/3.11.0a4.rst @@ -161,7 +161,7 @@ faster due to reference-counting optimizations. Patch by Dennis Sweeney. .. nonce: 7oGp-I .. section: Core and Builtins -:opcode:`PREP_RERAISE_STAR` no longer pushes ``lasti`` to the stack. +:opcode:`!PREP_RERAISE_STAR` no longer pushes ``lasti`` to the stack. .. @@ -170,7 +170,7 @@ faster due to reference-counting optimizations. Patch by Dennis Sweeney. .. nonce: IKx4v6 .. section: Core and Builtins -Remove :opcode:`POP_EXCEPT_AND_RERAISE` and replace it by an equivalent +Remove :opcode:`!POP_EXCEPT_AND_RERAISE` and replace it by an equivalent sequence of other opcodes. .. @@ -510,7 +510,7 @@ Remove special-casing of ``__new__`` in :meth:`enum.Enum.__dir__`. Improve day constants in :mod:`calendar`. -Now all constants (`MONDAY` ... `SUNDAY`) are documented, tested, and added +Now all constants (``MONDAY`` ... ``SUNDAY``) are documented, tested, and added to ``__all__``. .. @@ -1171,7 +1171,7 @@ Replaced deprecated usage of :c:func:`PyImport_ImportModuleNoBlock` with .. nonce: sMgDLz .. section: C API -The :c:func:`PyUnicode_CHECK_INTERNED` macro has been excluded from the +The :c:func:`!PyUnicode_CHECK_INTERNED` macro has been excluded from the limited C API. It was never usable there, because it used internal structures which are not available in the limited C API. Patch by Victor Stinner. diff --git a/Misc/NEWS.d/3.11.0a5.rst b/Misc/NEWS.d/3.11.0a5.rst index 08d94e82ed8ccf..30a462e9bfdcbf 100644 --- a/Misc/NEWS.d/3.11.0a5.rst +++ b/Misc/NEWS.d/3.11.0a5.rst @@ -275,7 +275,7 @@ Do not use POSIX semaphores on NetBSD .. nonce: M9m8Qd .. section: Core and Builtins -Improve the exc:`TypeError` message for non-string second arguments passed +Improve the :exc:`TypeError` message for non-string second arguments passed to the built-in functions :func:`getattr` and :func:`hasattr`. Patch by Géry Ogam. @@ -673,7 +673,7 @@ file .. section: Tests Mocks can no longer be provided as the specs for other Mocks. As a result, -an already-mocked object cannot be passed to `mock.Mock()`. This can uncover +an already-mocked object cannot be passed to ``mock.Mock()``. This can uncover bugs in tests since these Mock-derived Mocks will always pass certain tests (e.g. isinstance) and builtin assert functions (e.g. assert_called_once_with) will unconditionally pass. diff --git a/Misc/NEWS.d/3.11.0a6.rst b/Misc/NEWS.d/3.11.0a6.rst index 52055b3fafd485..2fdceef7746d4e 100644 --- a/Misc/NEWS.d/3.11.0a6.rst +++ b/Misc/NEWS.d/3.11.0a6.rst @@ -248,7 +248,7 @@ Don't un-adapt :opcode:`COMPARE_OP` when collecting specialization stats. .. nonce: RX_AzJ .. section: Core and Builtins -Fix specialization stats gathering for :opcode:`PRECALL` instructions. +Fix specialization stats gathering for :opcode:`!PRECALL` instructions. .. @@ -941,7 +941,7 @@ uvloop library. Make the :class:`configparser.ConfigParser` constructor raise :exc:`TypeError` if the ``interpolation`` parameter is not of type -:class:`configparser.Interpolation` +:class:`!configparser.Interpolation` .. @@ -1034,7 +1034,7 @@ if zlib uses the s390x hardware accelerator. Patch by Victor Stinner. .. nonce: jfciLG .. section: Build -Respect `--with-suffix` when building on case-insensitive file systems. +Respect ``--with-suffix`` when building on case-insensitive file systems. .. @@ -1151,7 +1151,7 @@ Make query dialogs on Windows start with a cursor in the entry box. .. nonce: FhiH5P .. section: IDLE -Apply IDLE syntax highlighting to `.pyi` files. Patch by Alex Waygood and +Apply IDLE syntax highlighting to ``.pyi`` files. Patch by Alex Waygood and Terry Jan Reedy. .. diff --git a/Misc/NEWS.d/3.11.0a7.rst b/Misc/NEWS.d/3.11.0a7.rst index 6e41f9cbd933b5..ec99bd0294ceca 100644 --- a/Misc/NEWS.d/3.11.0a7.rst +++ b/Misc/NEWS.d/3.11.0a7.rst @@ -138,7 +138,7 @@ Replaced :opcode:`JUMP_ABSOLUTE` by the relative jump .. nonce: SwrrFO .. section: Core and Builtins -:c:func:`PyFrame_FastToLocalsWithError` and :c:func:`PyFrame_LocalsToFast` +:c:func:`!PyFrame_FastToLocalsWithError` and :c:func:`!PyFrame_LocalsToFast` are no longer called during profiling nor tracing. C code can access the ``f_locals`` attribute of :c:type:`PyFrameObject` by calling :c:func:`PyFrame_GetLocals`. @@ -295,7 +295,7 @@ oparg) as an adaptive counter. .. nonce: O12Pba .. section: Core and Builtins -Use inline caching for :opcode:`PRECALL` and :opcode:`CALL`, and remove the +Use inline caching for :opcode:`!PRECALL` and :opcode:`CALL`, and remove the internal machinery for managing the (now unused) non-inline caches. .. @@ -717,7 +717,7 @@ Fix :class:`asyncio.Semaphore` re-aquiring FIFO order. .. nonce: uaEDcI .. section: Library -The :mod:`asynchat`, :mod:`asyncore` and :mod:`smtpd` modules have been +The :mod:`!asynchat`, :mod:`!asyncore` and :mod:`!smtpd` modules have been deprecated since at least Python 3.6. Their documentation and deprecation warnings and have now been updated to note they will removed in Python 3.12 (:pep:`594`). @@ -1038,8 +1038,8 @@ Add optional parameter *dir_fd* in :func:`shutil.rmtree`. .. nonce: AixHW7 .. section: Library -:meth:`~!unittest.TestProgram.usageExit` is marked deprecated, to be removed -in 3.13. +:meth:`!unittest.TestProgram.usageExit` is marked as deprecated, +to be removed in Python 3.13. .. @@ -1324,7 +1324,7 @@ extensions. .. section: Tests A test case for :func:`os.sendfile` is converted from deprecated -:mod:`asyncore` (see :pep:`594`) to :mod:`asyncio`. Patch by Oleg Iarygin. +:mod:`!asyncore` (see :pep:`594`) to :mod:`asyncio`. Patch by Oleg Iarygin. .. diff --git a/Misc/NEWS.d/3.11.0b1.rst b/Misc/NEWS.d/3.11.0b1.rst index a4cdda2cafdb43..f9296679655573 100644 --- a/Misc/NEWS.d/3.11.0b1.rst +++ b/Misc/NEWS.d/3.11.0b1.rst @@ -403,8 +403,8 @@ so this led to crashes. The problem is now fixed. .. nonce: 6S_uoU .. section: Core and Builtins -Make opcodes :opcode:`JUMP_IF_TRUE_OR_POP` and -:opcode:`JUMP_IF_FALSE_OR_POP` relative rather than absolute. +Make opcodes :opcode:`!JUMP_IF_TRUE_OR_POP` and +:opcode:`!JUMP_IF_FALSE_OR_POP` relative rather than absolute. .. @@ -437,7 +437,7 @@ cache when applicable. .. section: Core and Builtins Classes and functions that unconditionally declared their docstrings -ignoring the `--without-doc-strings` compilation flag no longer do so. +ignoring the ``--without-doc-strings`` compilation flag no longer do so. The classes affected are :class:`ctypes.UnionType`, :class:`pickle.PickleBuffer`, :class:`testcapi.RecursingInfinitelyError`, @@ -576,7 +576,7 @@ planned). Patch by Alex Waygood. .. section: Library Deprecate nested classes in enum definitions becoming members -- in 3.13 -they will be normal classes; add `member` and `nonmember` functions to allow +they will be normal classes; add ``member`` and ``nonmember`` functions to allow control over results now. .. @@ -785,7 +785,7 @@ avoid :exc:`BrokenPipeError` at garbage collection and at .. nonce: V0YveU .. section: Library -Add `datetime.UTC` alias for `datetime.timezone.utc`. +Add ``datetime.UTC`` alias for ``datetime.timezone.utc``. Patch by Kabir Kwatra. @@ -1657,9 +1657,9 @@ Convert :mod:`csv` to use Argument Clinic for :func:`csv.field_size_limit`, .. nonce: z2WhDQ .. section: Library -Raise an ArgumentError when the same subparser name is added twice to an -`argparse.ArgumentParser`. This is consistent with the (default) behavior -when the same option string is added twice to an ArgumentParser. +Raise an ``ArgumentError`` when the same subparser name is added twice to an +``argparse.ArgumentParser``. This is consistent with the (default) behavior +when the same option string is added twice to an ``ArgumentParser``. .. @@ -1712,7 +1712,7 @@ Aviv Palivoda and Erlend E. Aasland. .. nonce: kTjJLx .. section: Documentation -Add a new `gh` role to the documentation to link to GitHub issues. +Add a new ``gh`` role to the documentation to link to GitHub issues. .. diff --git a/Misc/NEWS.d/3.12.0a1.rst b/Misc/NEWS.d/3.12.0a1.rst index 29d04fa0e175bf..f192bf086ed259 100644 --- a/Misc/NEWS.d/3.12.0a1.rst +++ b/Misc/NEWS.d/3.12.0a1.rst @@ -1462,7 +1462,7 @@ expression, but there's no trailing brace. For example, f"{i=". .. nonce: Jf6gAj .. section: Core and Builtins -Cache the result of :c:func:`PyCode_GetCode` function to restore the O(1) +Cache the result of :c:func:`PyCode_GetCode` function to restore the *O*\ (1) lookup of the :attr:`~types.CodeType.co_code` attribute. .. @@ -3617,7 +3617,7 @@ allow access to handlers by name. .. nonce: uw6x5z .. section: Library -The :mod:`smtpd` module was removed per the schedule in :pep:`594`. +The :mod:`!smtpd` module was removed per the schedule in :pep:`594`. .. diff --git a/Misc/NEWS.d/3.12.0a2.rst b/Misc/NEWS.d/3.12.0a2.rst index 1a04ed473f329d..a9c5038fa489bb 100644 --- a/Misc/NEWS.d/3.12.0a2.rst +++ b/Misc/NEWS.d/3.12.0a2.rst @@ -584,8 +584,8 @@ Use the frame bound builtins when offering a name suggestion in .. nonce: qtm-9T .. section: Library -In :mod:`importlib._bootstrap`, enhance namespace package repr to ``. +In :mod:`importlib._bootstrap`, enhance namespace package repr to ````. .. @@ -695,7 +695,7 @@ Make sure ``patch.dict()`` can be applied on async functions. .. nonce: jUpzF3 .. section: Library -Remove modules :mod:`asyncore` and :mod:`asynchat`, which were deprecated by +Remove modules :mod:`!asyncore` and :mod:`!asynchat`, which were deprecated by :pep:`594`. .. diff --git a/Misc/NEWS.d/3.12.0a4.rst b/Misc/NEWS.d/3.12.0a4.rst index 75246f3f13503e..82faa5ad0b2031 100644 --- a/Misc/NEWS.d/3.12.0a4.rst +++ b/Misc/NEWS.d/3.12.0a4.rst @@ -13,8 +13,8 @@ Fix misleading default value in :func:`input`'s ``__text_signature__``. .. nonce: cmGwxv .. section: Core and Builtins -Remove :opcode:`UNARY_POSITIVE`, :opcode:`ASYNC_GEN_WRAP` and -:opcode:`LIST_TO_TUPLE`, replacing them with intrinsics. +Remove :opcode:`!UNARY_POSITIVE`, :opcode:`!ASYNC_GEN_WRAP` and +:opcode:`!LIST_TO_TUPLE`, replacing them with intrinsics. .. @@ -147,8 +147,8 @@ clinic. .. nonce: yRWQ1y .. section: Core and Builtins -Improve the output of ``co_lines`` by emitting only one entry for each line -range. +Improve the output of :meth:`codeobject.co_lines` by emitting only one entry +for each line range. .. diff --git a/Misc/NEWS.d/3.12.0a6.rst b/Misc/NEWS.d/3.12.0a6.rst index 5bd600cd8b6fc0..cf28bdb9258820 100644 --- a/Misc/NEWS.d/3.12.0a6.rst +++ b/Misc/NEWS.d/3.12.0a6.rst @@ -170,7 +170,7 @@ all as not all platform C libraries generate an error. .. section: Core and Builtins Add :opcode:`CALL_INTRINSIC_2` and use it instead of -:opcode:`PREP_RERAISE_STAR`. +:opcode:`!PREP_RERAISE_STAR`. .. diff --git a/Misc/NEWS.d/3.12.0a7.rst b/Misc/NEWS.d/3.12.0a7.rst index 1ef81747558857..a859be8a047456 100644 --- a/Misc/NEWS.d/3.12.0a7.rst +++ b/Misc/NEWS.d/3.12.0a7.rst @@ -24,7 +24,7 @@ Reduce the number of inline :opcode:`CACHE` entries for .. nonce: PRkGca .. section: Core and Builtins -Removed :opcode:`JUMP_IF_FALSE_OR_POP` and :opcode:`JUMP_IF_TRUE_OR_POP` +Removed :opcode:`!JUMP_IF_FALSE_OR_POP` and :opcode:`!JUMP_IF_TRUE_OR_POP` instructions. .. @@ -429,7 +429,7 @@ an awaitable object. Patch by Kumar Aditya. Speed up setting or deleting mutable attributes on non-dataclass subclasses of frozen dataclasses. Due to the implementation of ``__setattr__`` and ``__delattr__`` for frozen dataclasses, this previously had a time -complexity of ``O(n)``. It now has a time complexity of ``O(1)``. +complexity of *O*\ (*n*). It now has a time complexity of *O*\ (1). .. diff --git a/Misc/NEWS.d/3.12.0b1.rst b/Misc/NEWS.d/3.12.0b1.rst index 007a6ad4ffd4d4..211513d05d0040 100644 --- a/Misc/NEWS.d/3.12.0b1.rst +++ b/Misc/NEWS.d/3.12.0b1.rst @@ -1008,7 +1008,7 @@ Update the bundled copy of pip to version 23.1.2. .. nonce: pst8iT .. section: Library -Make :mod:`dis` display the value of oparg of :opcode:`KW_NAMES`. +Make :mod:`dis` display the value of oparg of :opcode:`!KW_NAMES`. .. diff --git a/Misc/NEWS.d/3.13.0a1.rst b/Misc/NEWS.d/3.13.0a1.rst index 102bddcee5c5c2..d385b6a4504f97 100644 --- a/Misc/NEWS.d/3.13.0a1.rst +++ b/Misc/NEWS.d/3.13.0a1.rst @@ -2276,7 +2276,7 @@ creation. .. nonce: m2H5Bk .. section: Library -Remove unnecessary extra ``__slots__`` in :py:class:`datetime`\'s pure +Remove unnecessary extra ``__slots__`` in :class:`~datetime.datetime`\'s pure python implementation to reduce memory size, as they are defined in the superclass. Patch by James Hilton-Balfe diff --git a/Misc/NEWS.d/3.13.0a2.rst b/Misc/NEWS.d/3.13.0a2.rst index c1b1be523325e8..e5841e14c02efb 100644 --- a/Misc/NEWS.d/3.13.0a2.rst +++ b/Misc/NEWS.d/3.13.0a2.rst @@ -199,7 +199,7 @@ their debugging to ``PYTHON_UOPS`` and ``PYTHON_LLTRACE``. .. nonce: 11h6Mc .. section: Core and Builtins -Speed up :obj:`Traceback` object creation by lazily compute the line number. +Speed up :class:`Traceback` object creation by lazily compute the line number. Patch by Pablo Galindo .. @@ -565,9 +565,9 @@ part of a :exc:`BaseExceptionGroup`, in addition to the recent support for .. section: Library The :class:`mmap.mmap` class now has an :meth:`~mmap.mmap.seekable` method -that can be used where it requires a file-like object with seekable and the -:meth:`~mmap.mmap.seek` method return the new absolute position. Patch by -Donghee Na. +that can be used when a seekable file-like object is required. +The :meth:`~mmap.mmap.seek` method now returns the new absolute position. +Patch by Donghee Na. .. diff --git a/Misc/NEWS.d/3.13.0a3.rst b/Misc/NEWS.d/3.13.0a3.rst new file mode 100644 index 00000000000000..95aa66603de7cb --- /dev/null +++ b/Misc/NEWS.d/3.13.0a3.rst @@ -0,0 +1,2414 @@ +.. date: 2024-01-02-19-52-23 +.. gh-issue: 113659 +.. nonce: DkmnQc +.. release date: 2024-01-17 +.. section: Security + +Skip ``.pth`` files with names starting with a dot or hidden file attribute. + +.. + +.. date: 2023-12-06-14-06-59 +.. gh-issue: 112302 +.. nonce: 3bl20f +.. section: Security + +Created a Software Bill-of-Materials document and tooling for tracking +dependencies. + +.. + +.. date: 2024-01-11-16-54-55 +.. gh-issue: 107901 +.. nonce: Td3JPI +.. section: Core and Builtins + +Compiler duplicates basic blocks that have an eval breaker check, no line +number, and multiple predecessors. + +.. + +.. date: 2024-01-11-14-03-31 +.. gh-issue: 107901 +.. nonce: U65IyC +.. section: Core and Builtins + +A jump leaving an exception handler back to normal code no longer checks the +eval breaker. + +.. + +.. date: 2024-01-11-01-28-25 +.. gh-issue: 113655 +.. nonce: Mfioxp +.. section: Core and Builtins + +Set the C recursion limit to 4000 on Windows, and 10000 on Linux/OSX. This +seems to be near the sweet spot to maintain safety, but not compromise +backwards compatibility. + +.. + +.. date: 2024-01-09-23-01-00 +.. gh-issue: 113710 +.. nonce: pe3flY +.. section: Core and Builtins + +Add typed stack effects to the interpreter DSL, along with various +instruction annotations. + +.. + +.. date: 2024-01-08-14-34-02 +.. gh-issue: 77046 +.. nonce: sDUh2d +.. section: Core and Builtins + +On Windows, file descriptors wrapping Windows handles are now created non +inheritable by default (:pep:`446`). Patch by Zackery Spytz and Victor +Stinner. + +.. + +.. date: 2024-01-08-05-36-59 +.. gh-issue: 113853 +.. nonce: lm-6_a +.. section: Core and Builtins + +Guarantee that all executors make progress. This then guarantees that tier 2 +execution always makes progress. + +.. + +.. date: 2024-01-05-21-28-48 +.. gh-issue: 113753 +.. nonce: 2HNiuq +.. section: Core and Builtins + +Fix an issue where the finalizer of ``PyAsyncGenASend`` objects might not be +called if they were allocated from a free list. + +.. + +.. date: 2024-01-05-00-49-14 +.. gh-issue: 107901 +.. nonce: 6JRrb6 +.. section: Core and Builtins + +Compiler changed so that synthetic jumps which are not at loop end no longer +check the eval breaker. + +.. + +.. date: 2024-01-04-17-15-30 +.. gh-issue: 113703 +.. nonce: Zsk0pY +.. section: Core and Builtins + +Fix a regression in the :mod:`codeop` module that was causing it to +incorrectly identify incomplete f-strings. Patch by Pablo Galindo + +.. + +.. date: 2024-01-03-12-19-37 +.. gh-issue: 89811 +.. nonce: cZOj6d +.. section: Core and Builtins + +Check for a valid ``tp_version_tag`` before performing bytecode +specializations that rely on this value being usable. + +.. + +.. date: 2024-01-02-17-22-57 +.. gh-issue: 111488 +.. nonce: EJH3Oh +.. section: Core and Builtins + +Changed error message in case of no 'in' keyword after 'for' in list +comprehensions + +.. + +.. date: 2024-01-02-11-14-29 +.. gh-issue: 113657 +.. nonce: CQo9vF +.. section: Core and Builtins + +Fix an issue that caused important instruction pointer updates to be +optimized out of tier two traces. + +.. + +.. date: 2024-01-01-23-57-24 +.. gh-issue: 113603 +.. nonce: ySwovr +.. section: Core and Builtins + +Fixed bug where a redundant NOP is not removed, causing an assertion to fail +in the compiler in debug mode. + +.. + +.. date: 2024-01-01-00-07-02 +.. gh-issue: 113602 +.. nonce: cWuTzk +.. section: Core and Builtins + +Fix an error that was causing the parser to try to overwrite existing errors +and crashing in the process. Patch by Pablo Galindo + +.. + +.. date: 2023-12-31-07-46-01 +.. gh-issue: 113486 +.. nonce: uki19C +.. section: Core and Builtins + +No longer issue spurious ``PY_UNWIND`` events for optimized calls to +classes. + +.. + +.. date: 2023-12-20-18-27-11 +.. gh-issue: 113297 +.. nonce: BZyAI_ +.. section: Core and Builtins + +Fix segfault in the compiler on with statement with 19 context managers. + +.. + +.. date: 2023-12-20-08-54-54 +.. gh-issue: 113212 +.. nonce: 62AUlw +.. section: Core and Builtins + +Improve :py:class:`super` error messages. + +.. + +.. date: 2023-12-19-22-03-43 +.. gh-issue: 111375 +.. nonce: M9vuA6 +.. section: Core and Builtins + +Only use ``NULL`` in the exception stack to indicate an exception was +handled. Patch by Carey Metcalfe. + +.. + +.. date: 2023-12-15-16-26-01 +.. gh-issue: 112215 +.. nonce: xJS6_6 +.. section: Core and Builtins + +Increase the C recursion limit by a factor of 3 for non-debug builds, except +for webassembly and s390 platforms which are unchanged. This mitigates some +regressions in 3.12 with deep recursion mixing builtin (C) and Python code. + +.. + +.. date: 2023-12-14-20-08-35 +.. gh-issue: 113054 +.. nonce: e20CtM +.. section: Core and Builtins + +Fixed bug where a redundant NOP is not removed, causing an assertion to fail +in the compiler in debug mode. + +.. + +.. date: 2023-12-13-11-45-53 +.. gh-issue: 106905 +.. nonce: 5dslTN +.. section: Core and Builtins + +Use per AST-parser state rather than global state to track recursion depth +within the AST parser to prevent potential race condition due to +simultaneous parsing. + +The issue primarily showed up in 3.11 by multithreaded users of +:func:`ast.parse`. In 3.12 a change to when garbage collection can be +triggered prevented the race condition from occurring. + +.. + +.. date: 2023-12-12-04-53-19 +.. gh-issue: 108866 +.. nonce: xbJ-9a +.. section: Core and Builtins + +Change the API and contract of ``_PyExecutorObject`` to return the +next_instr pointer, instead of the frame, and to always execute at least one +instruction. + +.. + +.. date: 2023-12-11-19-53-32 +.. gh-issue: 90350 +.. nonce: -FQy3E +.. section: Core and Builtins + +Optimize builtin functions :func:`min` and :func:`max`. + +.. + +.. date: 2023-12-11-00-50-00 +.. gh-issue: 112943 +.. nonce: RHNZie +.. section: Core and Builtins + +Correctly compute end column offsets for multiline tokens in the +:mod:`tokenize` module. Patch by Pablo Galindo + +.. + +.. date: 2023-12-07-13-19-55 +.. gh-issue: 112125 +.. nonce: 4ADN7i +.. section: Core and Builtins + +Fix None.__ne__(None) returning NotImplemented instead of False + +.. + +.. date: 2023-12-07-12-00-04 +.. gh-issue: 74616 +.. nonce: kgTGVb +.. section: Core and Builtins + +:func:`input` now raises a ValueError when output on the terminal if the +prompt contains embedded null characters instead of silently truncating it. + +.. + +.. date: 2023-12-05-20-41-58 +.. gh-issue: 112716 +.. nonce: hOcx0Y +.. section: Core and Builtins + +Fix SystemError in the ``import`` statement and in ``__reduce__()`` methods +of builtin types when ``__builtins__`` is not a dict. + +.. + +.. date: 2023-12-04-23-09-07 +.. gh-issue: 112730 +.. nonce: BXHlFa +.. section: Core and Builtins + +Use color to highlight error locations in tracebacks. Patch by Pablo Galindo + +.. + +.. date: 2023-12-03-19-34-51 +.. gh-issue: 112625 +.. nonce: QWTlwS +.. section: Core and Builtins + +Fixes a bug where a bytearray object could be cleared while iterating over +an argument in the ``bytearray.join()`` method that could result in reading +memory after it was freed. + +.. + +.. date: 2023-12-03-15-29-53 +.. gh-issue: 112660 +.. nonce: gldBvh +.. section: Core and Builtins + +Do not clear unexpected errors during formatting error messages for +ImportError and AttributeError for modules. + +.. + +.. date: 2023-12-01-19-02-21 +.. gh-issue: 105967 +.. nonce: Puq5Cn +.. section: Core and Builtins + +Workaround a bug in Apple's macOS platform zlib library where +:func:`zlib.crc32` and :func:`binascii.crc32` could produce incorrect +results on multi-gigabyte inputs. Including when using :mod:`zipfile` on +zips containing large data. + +.. + +.. date: 2023-12-01-08-16-10 +.. gh-issue: 95754 +.. nonce: ae4gwy +.. section: Core and Builtins + +Provide a better error message when accessing invalid attributes on +partially initialized modules. The origin of the module being accessed is +now included in the message to help with the common issue of shadowing other +modules. + +.. + +.. date: 2023-11-27-18-55-30 +.. gh-issue: 112217 +.. nonce: SwFLMj +.. section: Core and Builtins + +Add check for the type of ``__cause__`` returned from calling the type ``T`` +in ``raise from T``. + +.. + +.. date: 2023-11-26-21-30-11 +.. gh-issue: 111058 +.. nonce: q4DqDY +.. section: Core and Builtins + +Change coro.cr_frame/gen.gi_frame to return ``None`` after the +coroutine/generator has been closed. This fixes a bug where +:func:`~inspect.getcoroutinestate` and :func:`~inspect.getgeneratorstate` +return the wrong state for a closed coroutine/generator. + +.. + +.. date: 2023-11-25-22-58-49 +.. gh-issue: 112388 +.. nonce: MU3cIM +.. section: Core and Builtins + +Fix an error that was causing the parser to try to overwrite tokenizer +errors. Patch by pablo Galindo + +.. + +.. date: 2023-11-25-22-39-44 +.. gh-issue: 112387 +.. nonce: AbBq5W +.. section: Core and Builtins + +Fix error positions for decoded strings with backwards tokenize errors. +Patch by Pablo Galindo + +.. + +.. date: 2023-11-25-20-36-38 +.. gh-issue: 99606 +.. nonce: fDY5hK +.. section: Core and Builtins + +Make code generated for an empty f-string identical to the code of an empty +normal string. + +.. + +.. date: 2023-11-24-14-10-57 +.. gh-issue: 112367 +.. nonce: 9z1IDp +.. section: Core and Builtins + +Avoid undefined behaviour when using the perf trampolines by not freeing the +code arenas until shutdown. Patch by Pablo Galindo + +.. + +.. date: 2023-11-22-13-17-54 +.. gh-issue: 112320 +.. nonce: EddM51 +.. section: Core and Builtins + +The Tier 2 translator now tracks the confidence level for staying "on trace" +(i.e. not exiting back to the Tier 1 interpreter) for branch instructions +based on the number of bits set in the branch "counter". Trace translation +ends when the confidence drops below 1/3rd. + +.. + +.. date: 2023-09-21-11-54-28 +.. gh-issue: 109598 +.. nonce: CRidSy +.. section: Core and Builtins + +:c:func:`PyComplex_RealAsDouble`/:c:func:`PyComplex_ImagAsDouble` now tries +to convert an object to a :class:`complex` instance using its +``__complex__()`` method before falling back to the ``__float__()`` method. +Patch by Sergey B Kirpichev. + +.. + +.. date: 2022-07-07-05-37-53 +.. gh-issue: 94606 +.. nonce: hojJ54 +.. section: Core and Builtins + +Fix UnicodeEncodeError when :func:`email.message.get_payload` reads a +message with a Unicode surrogate character and the message content is not +well-formed for surrogateescape encoding. Patch by Sidney Markowitz. + +.. + +.. bpo: 21861 +.. date: 2022-01-23-18-00-10 +.. nonce: N8E1zw +.. section: Core and Builtins + +Use the object's actual class name in :meth:`_io.FileIO.__repr__`, +:meth:`_io._WindowsConsoleIO` and :meth:`_io.TextIOWrapper.__repr__`, to +make these methods subclass friendly. + +.. + +.. bpo: 45369 +.. date: 2021-10-05-05-00-16 +.. nonce: tluk_X +.. section: Core and Builtins + +Remove LibreSSL workarounds as per :pep:`644`. + +.. + +.. bpo: 34392 +.. date: 2018-08-13-13-25-15 +.. nonce: 9kIlMF +.. section: Core and Builtins + +Added :func:`sys._is_interned`. + +.. + +.. date: 2024-01-15-12-12-54 +.. gh-issue: 114077 +.. nonce: KcVnfj +.. section: Library + +Fix possible :exc:`OverflowError` in :meth:`socket.socket.sendfile` when +pass *count* larger than 2 GiB on 32-bit platform. + +.. + +.. date: 2024-01-13-14-20-31 +.. gh-issue: 111803 +.. nonce: llpLAw +.. section: Library + +:mod:`plistlib` now supports loading more deeply nested lists in binary +format. + +.. + +.. date: 2024-01-13-11-34-29 +.. gh-issue: 114014 +.. nonce: WRHifN +.. section: Library + +Fixed a bug in :class:`fractions.Fraction` where an invalid string using +``d`` in the decimals part creates a different error compared to other +invalid letters/characters. Patch by Jeremiah Gabriel Pascual. + +.. + +.. date: 2024-01-11-22-22-51 +.. gh-issue: 108364 +.. nonce: QH7C-1 +.. section: Library + +:meth:`sqlite3.Connection.iterdump` now ensures that foreign key support is +disabled before dumping the database schema, if there is any foreign key +violation. Patch by Erlend E. Aasland and Mariusz Felisiak. + +.. + +.. date: 2024-01-11-16-58-10 +.. gh-issue: 113971 +.. nonce: skJZ4g +.. section: Library + +The :class:`zipfile.ZipInfo` previously protected ``._compresslevel`` +attribute has been made public as ``.compress_level`` with the old +``_compresslevel`` name remaining available as a property to retain +compatibility. + +.. + +.. date: 2024-01-10-12-03-38 +.. gh-issue: 113877 +.. nonce: RxKlrQ +.. section: Library + +Fix :mod:`tkinter` method ``winfo_pathname()`` on 64-bit Windows. + +.. + +.. date: 2024-01-09-18-07-08 +.. gh-issue: 113868 +.. nonce: DlZG2r +.. section: Library + +Added :data:`mmap.MAP_NORESERVE`, :data:`mmap.MAP_NOEXTEND`, +:data:`mmap.MAP_HASSEMAPHORE`, :data:`mmap.MAP_NOCACHE`, +:data:`mmap.MAP_JIT`, :data:`mmap.MAP_RESILIENT_CODESIGN`, +:data:`mmap.MAP_RESILIENT_MEDIA`, :data:`mmap.MAP_32BIT`, +:data:`mmap.MAP_TRANSLATED_ALLOW_EXECUTE`, :data:`mmap.MAP_UNIX03` and +:data:`mmap.MAP_TPRO`. All of them are ``mmap(2)`` flags on macOS. + +.. + +.. date: 2024-01-09-12-19-55 +.. gh-issue: 113848 +.. nonce: kXoCy0 +.. section: Library + +:func:`asyncio.TaskGroup()` and :func:`asyncio.timeout()` context managers +now handle :exc:`~asyncio.CancelledError` subclasses as well as exact +:exc:`!CancelledError`. + +.. + +.. date: 2024-01-09-08-59-43 +.. gh-issue: 113661 +.. nonce: asvXSx +.. section: Library + +unittest runner: Don't exit 5 if tests were skipped. The intention of +exiting 5 was to detect issues where the test suite wasn't discovered at +all. If we skipped tests, it was correctly discovered. + +.. + +.. date: 2024-01-08-19-38-42 +.. gh-issue: 96037 +.. nonce: Yr2Y1C +.. section: Library + +Insert :exc:`TimeoutError` in the context of the exception that was raised +during exiting an expired :func:`asyncio.timeout` block. + +.. + +.. date: 2024-01-08-14-57-09 +.. gh-issue: 113781 +.. nonce: IoTnwi +.. section: Library + +Silence unraisable AttributeError when warnings are emitted during Python +finalization. + +.. + +.. date: 2024-01-07-23-31-44 +.. gh-issue: 113238 +.. nonce: wFWBfW +.. section: Library + +Add ``Anchor`` to ``importlib.resources`` (in order for the code to comply +with the documentation) + +.. + +.. date: 2024-01-07-13-36-03 +.. gh-issue: 111693 +.. nonce: xN2LuL +.. section: Library + +:func:`asyncio.Condition.wait()` now re-raises the same +:exc:`CancelledError` instance that may have caused it to be interrupted. +Fixed race condition in :func:`asyncio.Semaphore.aquire` when interrupted +with a :exc:`CancelledError`. + +.. + +.. date: 2024-01-07-11-45-56 +.. gh-issue: 113791 +.. nonce: XF5xSW +.. section: Library + +Add ``CLOCK_MONOTONIC_RAW_APPROX`` and ``CLOCK_UPTIME_RAW_APPROX`` to +:mod:`time` on macOS. These are clocks available on macOS 10.12 or later. + +.. + +.. date: 2024-01-07-00-56-41 +.. gh-issue: 112932 +.. nonce: OfhUu7 +.. section: Library + +Restore the ability for :mod:`zipfile` to ``extractall`` from zip files with +a "/" directory entry in them as is commonly added to zips by some wiki or +bug tracker data exporters. + +.. + +.. date: 2024-01-05-21-52-59 +.. gh-issue: 113568 +.. nonce: _0FkpZ +.. section: Library + +Raise deprecation warnings from :class:`pathlib.PurePath` and not its +private base class ``PurePathBase``. + +.. + +.. date: 2024-01-05-12-42-07 +.. gh-issue: 113594 +.. nonce: 4t8HiR +.. section: Library + +Fix :exc:`UnicodeEncodeError` in :mod:`email` when re-fold lines that +contain unknown-8bit encoded part followed by non-unknown-8bit encoded part. + +.. + +.. date: 2024-01-03-14-19-26 +.. gh-issue: 113538 +.. nonce: ahuBCo +.. section: Library + +In :meth:`asyncio.StreamReaderProtocol.connection_made`, there is callback +that logs an error if the task wrapping the "connected callback" fails. This +callback would itself fail if the task was cancelled. Prevent this by +checking whether the task was cancelled first. If so, close the transport +but don't log an error. + +.. + +.. date: 2024-01-02-12-41-59 +.. gh-issue: 113626 +.. nonce: i1PPY_ +.. section: Library + +Add support for the *allow_code* argument in the :mod:`marshal` module. +Passing ``allow_code=False`` prevents serialization and de-serialization of +code objects which is incompatible between Python versions. + +.. + +.. date: 2024-01-01-13-26-02 +.. gh-issue: 85567 +.. nonce: K4U15m +.. section: Library + +Fix resource warnings for unclosed files in :mod:`pickle` and +:mod:`pickletools` command line interfaces. + +.. + +.. date: 2023-12-30-20-30-05 +.. gh-issue: 113537 +.. nonce: v1W5_X +.. section: Library + +Support loads ``str`` in :func:`plistlib.loads`. + +.. + +.. date: 2023-12-29-22-29-34 +.. gh-issue: 89850 +.. nonce: KnxiZA +.. section: Library + +Add default implementations of :meth:`pickle.Pickler.persistent_id` and +:meth:`pickle.Unpickler.persistent_load` methods in the C implementation. +Calling ``super().persistent_id()`` and ``super().persistent_load()`` in +subclasses of the C implementation of :class:`pickle.Pickler` and +:class:`pickle.Unpickler` classes no longer causes infinite recursion. + +.. + +.. date: 2023-12-29-17-57-45 +.. gh-issue: 113569 +.. nonce: qcRCEI +.. section: Library + +Indicate if there were no actual calls in unittest +:meth:`~unittest.mock.Mock.assert_has_calls` failure. + +.. + +.. date: 2023-12-29-17-46-06 +.. gh-issue: 101225 +.. nonce: QaEyxF +.. section: Library + +Increase the backlog for :class:`multiprocessing.connection.Listener` +objects created by :mod:`multiprocessing.manager` and +:mod:`multiprocessing.resource_sharer` to significantly reduce the risk of +getting a connection refused error when creating a +:class:`multiprocessing.connection.Connection` to them. + +.. + +.. date: 2023-12-29-17-30-49 +.. gh-issue: 113568 +.. nonce: UpWNAI +.. section: Library + +Raise audit events from :class:`pathlib.Path` and not its private base class +``PathBase``. + +.. + +.. date: 2023-12-28-14-36-20 +.. gh-issue: 113543 +.. nonce: 2iWkOR +.. section: Library + +Make sure that ``webbrowser.MacOSXOSAScript`` sends ``webbrowser.open`` +audit event. + +.. + +.. date: 2023-12-23-16-51-17 +.. gh-issue: 113028 +.. nonce: 3Jmdoj +.. section: Library + +When a second reference to a string appears in the input to :mod:`pickle`, +and the Python implementation is in use, we are guaranteed that a single +copy gets pickled and a single object is shared when reloaded. Previously, +in protocol 0, when a string contained certain characters (e.g. newline) it +resulted in duplicate objects. + +.. + +.. date: 2023-12-23-16-10-07 +.. gh-issue: 113421 +.. nonce: w7vs08 +.. section: Library + +Fix multiprocessing logger for ``%(filename)s``. + +.. + +.. date: 2023-12-23-13-10-42 +.. gh-issue: 111784 +.. nonce: Nb4L1j +.. section: Library + +Fix segfaults in the ``_elementtree`` module. Fix first segfault during +deallocation of ``_elementtree.XMLParser`` instances by keeping strong +reference to ``pyexpat`` module in module state for capsule lifetime. Fix +second segfault which happens in the same deallocation process by keeping +strong reference to ``_elementtree`` module in ``XMLParser`` structure for +``_elementtree`` module lifetime. + +.. + +.. date: 2023-12-22-20-49-52 +.. gh-issue: 113407 +.. nonce: C_O13_ +.. section: Library + +Fix import of :mod:`unittest.mock` when CPython is built without docstrings. + +.. + +.. date: 2023-12-22-11-30-57 +.. gh-issue: 113320 +.. nonce: Vp5suS +.. section: Library + +Fix regression in Python 3.12 where :class:`~typing.Protocol` classes that +were not marked as :func:`runtime-checkable ` +would be unnecessarily introspected, potentially causing exceptions to be +raised if the protocol had problematic members. Patch by Alex Waygood. + +.. + +.. date: 2023-12-21-23-47-42 +.. gh-issue: 53502 +.. nonce: dercJI +.. section: Library + +Add a new option ``aware_datetime`` in :mod:`plistlib` to loads or dumps +aware datetime. + +.. + +.. date: 2023-12-21-14-55-06 +.. gh-issue: 113358 +.. nonce: nRkiSL +.. section: Library + +Fix rendering tracebacks with exceptions with a broken __getattr__ + +.. + +.. date: 2023-12-20-21-18-51 +.. gh-issue: 113214 +.. nonce: JcV9Mn +.. section: Library + +Fix an ``AttributeError`` during asyncio SSL protocol aborts in SSL-over-SSL +scenarios. + +.. + +.. date: 2023-12-18-09-47-54 +.. gh-issue: 113246 +.. nonce: em930H +.. section: Library + +Update bundled pip to 23.3.2. + +.. + +.. date: 2023-12-17-13-56-30 +.. gh-issue: 87264 +.. nonce: RgfHCv +.. section: Library + +Fixed tarfile list() method to show file type. + +.. + +.. date: 2023-12-17-10-22-55 +.. gh-issue: 112182 +.. nonce: jLWGlr +.. section: Library + +:meth:`asyncio.futures.Future.set_exception()` now transforms +:exc:`StopIteration` into :exc:`RuntimeError` instead of hanging or other +misbehavior. Patch contributed by Jamie Phan. + +.. + +.. date: 2023-12-17-04-43-57 +.. gh-issue: 113225 +.. nonce: dhxhiZ +.. section: Library + +Speed up :meth:`pathlib.Path.glob` by using :attr:`os.DirEntry.path` where +possible. + +.. + +.. date: 2023-12-16-23-56-42 +.. gh-issue: 113149 +.. nonce: 7LWgTS +.. section: Library + +Improve error message when a JSON array or object contains a trailing comma. +Patch by Carson Radtke. + +.. + +.. date: 2023-12-16-10-58-34 +.. gh-issue: 113117 +.. nonce: 0zF7bH +.. section: Library + +The :mod:`subprocess` module can now use the :func:`os.posix_spawn` function +with ``close_fds=True`` on platforms where +``posix_spawn_file_actions_addclosefrom_np`` is available. Patch by Jakub +Kulik. + +.. + +.. date: 2023-12-16-01-10-47 +.. gh-issue: 113199 +.. nonce: oDjnjL +.. section: Library + +Make ``http.client.HTTPResponse.read1`` and +``http.client.HTTPResponse.readline`` close IO after reading all data when +content length is known. Patch by Illia Volochii. + +.. + +.. date: 2023-12-15-21-33-42 +.. gh-issue: 113191 +.. nonce: Il155b +.. section: Library + +Add support of :func:`os.fchmod` and a file descriptor in :func:`os.chmod` +on Windows. + +.. + +.. date: 2023-12-15-20-29-49 +.. gh-issue: 113188 +.. nonce: AvoraB +.. section: Library + +Fix :func:`shutil.copymode` and :func:`shutil.copystat` on Windows. +Previously they worked differenly if *dst* is a symbolic link: they modified +the permission bits of *dst* itself rather than the file it points to if +*follow_symlinks* is true or *src* is not a symbolic link, and did not +modify the permission bits if *follow_symlinks* is false and *src* is a +symbolic link. + +.. + +.. date: 2023-12-15-18-13-59 +.. gh-issue: 113119 +.. nonce: al-569 +.. section: Library + +:func:`os.posix_spawn` now accepts ``env=None``, which makes the newly +spawned process use the current process environment. Patch by Jakub Kulik. + +.. + +.. date: 2023-12-15-18-10-26 +.. gh-issue: 113202 +.. nonce: xv_Ww8 +.. section: Library + +Add a ``strict`` option to ``batched()`` in the ``itertools`` module. + +.. + +.. date: 2023-12-15-12-35-28 +.. gh-issue: 61648 +.. nonce: G-4pz0 +.. section: Library + +Detect line numbers of properties in doctests. + +.. + +.. date: 2023-12-15-09-51-41 +.. gh-issue: 113175 +.. nonce: RHsNwE +.. section: Library + +Sync with importlib_metadata 7.0, including improved type annotations, fixed +issue with symlinked packages in ``package_distributions``, added +``EntryPoints.__repr__``, introduced the ``diagnose`` script, added +``Distribution.origin`` property, and removed deprecated ``EntryPoint`` +access by numeric index (tuple behavior). + +.. + +.. date: 2023-12-13-17-08-21 +.. gh-issue: 59616 +.. nonce: JNlWSs +.. section: Library + +Add support of :func:`os.lchmod` and the *follow_symlinks* argument in +:func:`os.chmod` on Windows. Note that the default value of +*follow_symlinks* in :func:`!os.lchmod` is ``False`` on Windows. + +.. + +.. date: 2023-12-12-20-15-57 +.. gh-issue: 112559 +.. nonce: IgXkje +.. section: Library + +:func:`signal.signal` and :func:`signal.getsignal` no longer call ``repr`` +on callable handlers. :func:`asyncio.run` and :meth:`asyncio.Runner.run` no +longer call ``repr`` on the task results. Patch by Yilei Yang. + +.. + +.. date: 2023-12-12-16-32-55 +.. gh-issue: 112962 +.. nonce: ZZWXZn +.. section: Library + +:mod:`dis` module functions add cache information to the +:class:`~dis.Instruction` instance rather than creating fake +:class:`~dis.Instruction` instances to represent the cache entries. + +.. + +.. date: 2023-12-12-05-48-17 +.. gh-issue: 112989 +.. nonce: ZAa_eq +.. section: Library + +Reduce overhead to connect sockets with :mod:`asyncio` SelectorEventLoop. + +.. + +.. date: 2023-12-11-16-13-15 +.. gh-issue: 112970 +.. nonce: 87jmKP +.. section: Library + +Use :c:func:`!closefrom` on Linux where available (e.g. glibc-2.34), rather +than only FreeBSD. + +.. + +.. date: 2023-12-11-14-12-46 +.. gh-issue: 110190 +.. nonce: e0iEUa +.. section: Library + +Fix ctypes structs with array on PPC64LE platform by setting +``MAX_STRUCT_SIZE`` to 64 in stgdict. Patch by Diego Russo. + +.. + +.. date: 2023-12-08-11-17-17 +.. gh-issue: 112540 +.. nonce: Pm5egX +.. section: Library + +The statistics.geometric_mean() function now returns zero for datasets +containing a zero. Formerly, it would raise an exception. + +.. + +.. date: 2023-12-07-16-55-41 +.. gh-issue: 87286 +.. nonce: MILC9_ +.. section: Library + +Added :const:`LOG_FTP`, :const:`LOG_NETINFO`, :const:`LOG_REMOTEAUTH`, +:const:`LOG_INSTALL`, :const:`LOG_RAS`, and :const:`LOG_LAUNCHD` tot the +:mod:`syslog` module, all of them constants on used on macOS. + +.. + +.. date: 2023-12-06-16-01-33 +.. gh-issue: 112800 +.. nonce: TNsGJ- +.. section: Library + +Fix :mod:`asyncio` ``SubprocessTransport.close()`` not to throw +``PermissionError`` when used with setuid executables. + +.. + +.. date: 2023-12-06-14-06-14 +.. gh-issue: 51944 +.. nonce: -5qq_L +.. section: Library + +Add the following constants to the :mod:`termios` module. These values are +present in macOS system headers: ``ALTWERASE``, ``B14400``, ``B28800``, +``B7200``, ``B76800``, ``CCAR_OFLOW``, ``CCTS_OFLOW``, ``CDSR_OFLOW``, +``CDTR_IFLOW``, ``CIGNORE``, ``CRTS_IFLOW``, ``EXTPROC``, ``IUTF8``, +``MDMBUF``, ``NL2``, ``NL3``, ``NOKERNINFO``, ``ONOEOT``, ``OXTABS``, +``VDSUSP``, ``VSTATUS``. + +.. + +.. date: 2023-12-05-18-57-53 +.. gh-issue: 79325 +.. nonce: P2vMVK +.. section: Library + +Fix an infinite recursion error in :func:`tempfile.TemporaryDirectory` +cleanup on Windows. + +.. + +.. date: 2023-12-05-16-20-40 +.. gh-issue: 94692 +.. nonce: -e5C3c +.. section: Library + +:func:`shutil.rmtree` now only catches OSError exceptions. Previously a +symlink attack resistant version of ``shutil.rmtree()`` could ignore or pass +to the error handler arbitrary exception when invalid arguments were +provided. + +.. + +.. date: 2023-12-05-01-19-28 +.. gh-issue: 112736 +.. nonce: rdHDrU +.. section: Library + +The use of del-safe symbols in ``subprocess`` was refactored to allow for +use in cross-platform build environments. + +.. + +.. date: 2023-12-04-21-30-34 +.. gh-issue: 112727 +.. nonce: jpgNRB +.. section: Library + +Speed up :meth:`pathlib.Path.absolute`. Patch by Barney Gale. + +.. + +.. date: 2023-12-04-16-45-11 +.. gh-issue: 74690 +.. nonce: pQYP5U +.. section: Library + +Speedup :func:`issubclass` checks against simple :func:`runtime-checkable +protocols ` by around 6%. Patch by Alex Waygood. + +.. + +.. date: 2023-12-04-14-05-24 +.. gh-issue: 74690 +.. nonce: eODKRm +.. section: Library + +Speedup :func:`isinstance` checks by roughly 20% for +:func:`runtime-checkable protocols ` that only +have one callable member. Speedup :func:`issubclass` checks for these +protocols by roughly 10%. Patch by Alex Waygood. + +.. + +.. date: 2023-12-03-12-41-48 +.. gh-issue: 112645 +.. nonce: blMsKf +.. section: Library + +Remove deprecation error on passing ``onerror`` to :func:`shutil.rmtree`. + +.. + +.. date: 2023-12-03-11-15-53 +.. gh-issue: 112640 +.. nonce: -FVwP7 +.. section: Library + +Add ``kwdefaults`` parameter to :data:`types.FunctionType` to set default +keyword argument values. + +.. + +.. date: 2023-12-03-01-01-52 +.. gh-issue: 112622 +.. nonce: 1Z8cpx +.. section: Library + +Ensure ``name`` parameter is passed to event loop in +:func:`asyncio.create_task`. + +.. + +.. date: 2023-12-02-12-55-17 +.. gh-issue: 112618 +.. nonce: 7_FT8- +.. section: Library + +Fix a caching bug relating to :data:`typing.Annotated`. ``Annotated[str, +True]`` is no longer identical to ``Annotated[str, 1]``. + +.. + +.. date: 2023-12-01-21-05-46 +.. gh-issue: 112334 +.. nonce: DmNXKh +.. section: Library + +Fixed a performance regression in 3.12's :mod:`subprocess` on Linux where it +would no longer use the fast-path ``vfork()`` system call when it could have +due to a logic bug, instead falling back to the safe but slower ``fork()``. + +Also fixed a second 3.12.0 potential security bug. If a value of +``extra_groups=[]`` was passed to :mod:`subprocess.Popen` or related APIs, +the underlying ``setgroups(0, NULL)`` system call to clear the groups list +would not be made in the child process prior to ``exec()``. + +This was identified via code inspection in the process of fixing the first +bug. + +.. + +.. date: 2023-12-01-18-05-09 +.. gh-issue: 110190 +.. nonce: 5bf-c9 +.. section: Library + +Fix ctypes structs with array on Arm platform by setting ``MAX_STRUCT_SIZE`` +to 32 in stgdict. Patch by Diego Russo. + +.. + +.. date: 2023-12-01-16-09-59 +.. gh-issue: 81194 +.. nonce: FFad1c +.. section: Library + +Fix a crash in :func:`socket.if_indextoname` with specific value (UINT_MAX). +Fix an integer overflow in :func:`socket.if_indextoname` on 64-bit +non-Windows platforms. + +.. + +.. date: 2023-12-01-08-28-09 +.. gh-issue: 112578 +.. nonce: bfNbfi +.. section: Library + +Fix a spurious :exc:`RuntimeWarning` when executing the :mod:`zipfile` +module. + +.. + +.. date: 2023-11-29-10-51-41 +.. gh-issue: 112516 +.. nonce: rFKUKN +.. section: Library + +Update the bundled copy of pip to version 23.3.1. + +.. + +.. date: 2023-11-29-02-26-32 +.. gh-issue: 112510 +.. nonce: j-zXGc +.. section: Library + +Add :data:`readline.backend` for the backend readline uses (``editline`` or +``readline``) + +.. + +.. date: 2023-11-28-20-47-39 +.. gh-issue: 112328 +.. nonce: Z2AxEY +.. section: Library + +[Enum] Make ``EnumDict``, ``EnumDict.member_names``, +``EnumType._add_alias_`` and ``EnumType._add_value_alias_`` public. + +.. + +.. date: 2023-11-28-20-01-33 +.. gh-issue: 112509 +.. nonce: QtoKed +.. section: Library + +Fix edge cases that could cause a key to be present in both the +``__required_keys__`` and ``__optional_keys__`` attributes of a +:class:`typing.TypedDict`. Patch by Jelle Zijlstra. + +.. + +.. date: 2023-11-28-02-39-30 +.. gh-issue: 101336 +.. nonce: ya433z +.. section: Library + +Add ``keep_alive`` keyword parameter for +:meth:`AbstractEventLoop.create_server` and +:meth:`BaseEventLoop.create_server`. + +.. + +.. date: 2023-11-27-12-41-23 +.. gh-issue: 63284 +.. nonce: q2Qi9q +.. section: Library + +Added support for TLS-PSK (pre-shared key) mode to the :mod:`ssl` module. + +.. + +.. date: 2023-11-26-13-44-19 +.. gh-issue: 112414 +.. nonce: kx2E7S +.. section: Library + +Fix regression in Python 3.12 where calling :func:`repr` on a module that +had been imported using a custom :term:`loader` could fail with +:exc:`AttributeError`. Patch by Alex Waygood. + +.. + +.. date: 2023-11-26-13-26-56 +.. gh-issue: 112358 +.. nonce: smhaeZ +.. section: Library + +Revert change to :class:`struct.Struct` initialization that broke some cases +of subclassing. + +.. + +.. date: 2023-11-25-20-29-28 +.. gh-issue: 112405 +.. nonce: cOtzxC +.. section: Library + +Optimize :meth:`pathlib.PurePath.relative_to`. Patch by Alex Waygood. + +.. + +.. date: 2023-11-24-21-00-24 +.. gh-issue: 94722 +.. nonce: GMIQIn +.. section: Library + +Fix bug where comparison between instances of :class:`~doctest.DocTest` +fails if one of them has ``None`` as its lineno. + +.. + +.. date: 2023-11-24-09-27-01 +.. gh-issue: 112361 +.. nonce: kYtnHW +.. section: Library + +Speed up a small handful of :mod:`pathlib` methods by removing some +temporary objects. + +.. + +.. date: 2023-11-23-17-25-27 +.. gh-issue: 112345 +.. nonce: FFApHx +.. section: Library + +Improve error message when trying to call :func:`issubclass` against a +:class:`typing.Protocol` that has non-method members. Patch by Randolf +Scholz. + +.. + +.. date: 2023-11-23-12-37-22 +.. gh-issue: 112137 +.. nonce: kM46Q6 +.. section: Library + +Change :mod:`dis` output to display no-lineno as "--" instead of "None". + +.. + +.. date: 2023-11-23-10-41-21 +.. gh-issue: 112332 +.. nonce: rhTBaa +.. section: Library + +Deprecate the ``exc_type`` field of :class:`traceback.TracebackException`. +Add ``exc_type_str`` to replace it. + +.. + +.. date: 2023-11-22-23-08-47 +.. gh-issue: 81620 +.. nonce: mfZ2Wf +.. section: Library + +Add extra tests for :func:`random.binomialvariate` + +.. + +.. date: 2023-11-22-19-43-54 +.. gh-issue: 112292 +.. nonce: 5nDU87 +.. section: Library + +Fix a crash in :mod:`readline` when imported from a sub interpreter. Patch +by Anthony Shaw + +.. + +.. date: 2023-11-21-02-58-14 +.. gh-issue: 77621 +.. nonce: MYv5XS +.. section: Library + +Slightly improve the import time of the :mod:`pathlib` module by deferring +some imports. Patch by Barney Gale. + +.. + +.. date: 2023-11-16-17-18-09 +.. gh-issue: 112137 +.. nonce: QvjGjN +.. section: Library + +Change :mod:`dis` output to display logical labels for jump targets instead +of offsets. + +.. + +.. date: 2023-11-16-10-42-15 +.. gh-issue: 112139 +.. nonce: WpHosf +.. section: Library + +Add :meth:`Signature.format` to format signatures to string with extra +options. And use it in :mod:`pydoc` to render more readable signatures that +have new lines between parameters. + +.. + +.. date: 2023-11-15-04-53-37 +.. gh-issue: 112105 +.. nonce: I3RcVN +.. section: Library + +Make :func:`readline.set_completer_delims` work with libedit + +.. + +.. date: 2023-11-15-01-36-04 +.. gh-issue: 106922 +.. nonce: qslOVH +.. section: Library + +Display multiple lines with ``traceback`` when errors span multiple lines. + +.. + +.. date: 2023-11-09-11-07-34 +.. gh-issue: 111874 +.. nonce: dzYc3j +.. section: Library + +When creating a :class:`typing.NamedTuple` class, ensure +:func:`~object.__set_name__` is called on all objects that define +``__set_name__`` and exist in the values of the ``NamedTuple`` class's class +dictionary. Patch by Alex Waygood. + +.. + +.. date: 2023-11-08-18-53-07 +.. gh-issue: 68166 +.. nonce: 1iTh4Y +.. section: Library + +Add support of the "vsapi" element type in +:meth:`tkinter.ttk.Style.element_create`. + +.. + +.. date: 2023-11-08-16-11-04 +.. gh-issue: 110275 +.. nonce: Bm6GwR +.. section: Library + +Named tuple's methods ``_replace()`` and ``__replace__()`` now raise +TypeError instead of ValueError for invalid keyword arguments. + +.. + +.. date: 2023-11-05-20-09-27 +.. gh-issue: 99367 +.. nonce: HLaWKo +.. section: Library + +Do not mangle ``sys.path[0]`` in :mod:`pdb` if safe_path is set + +.. + +.. date: 2023-11-02-10-13-31 +.. gh-issue: 111615 +.. nonce: 3SMixi +.. section: Library + +Fix a regression caused by a fix to gh-93162 whereby you couldn't configure +a :class:`QueueHandler` without specifying handlers. + +.. + +.. date: 2023-10-25-16-37-13 +.. gh-issue: 75666 +.. nonce: BpsWut +.. section: Library + +Fix the behavior of :mod:`tkinter` widget's ``unbind()`` method with two +arguments. Previously, ``widget.unbind(sequence, funcid)`` destroyed the +current binding for *sequence*, leaving *sequence* unbound, and deleted the +*funcid* command. Now it removes only *funcid* from the binding for +*sequence*, keeping other commands, and deletes the *funcid* command. It +leaves *sequence* unbound only if *funcid* was the last bound command. + +.. + +.. date: 2023-10-25-13-07-53 +.. gh-issue: 67790 +.. nonce: jMn9Ad +.. section: Library + +Implement basic formatting support (minimum width, alignment, fill) for +:class:`fractions.Fraction`. + +.. + +.. date: 2023-10-23-18-42-26 +.. gh-issue: 111049 +.. nonce: Ys7-o_ +.. section: Library + +Fix crash during garbage collection of the :class:`io.BytesIO` buffer +object. + +.. + +.. date: 2023-10-23-03-49-34 +.. gh-issue: 102980 +.. nonce: aXBd54 +.. section: Library + +Redirect the output of ``interact`` command of :mod:`pdb` to the same +channel as the debugger. Add tests and improve docs. + +.. + +.. date: 2023-10-20-15-28-08 +.. gh-issue: 102988 +.. nonce: dStNO7 +.. section: Library + +:func:`email.utils.getaddresses` and :func:`email.utils.parseaddr` now +return ``('', '')`` 2-tuples in more situations where invalid email +addresses are encountered instead of potentially inaccurate values. Add +optional *strict* parameter to these two functions: use ``strict=False`` to +get the old behavior, accept malformed inputs. ``getattr(email.utils, +'supports_strict_parsing', False)`` can be use to check if the *strict* +paramater is available. Patch by Thomas Dwyer and Victor Stinner to improve +the CVE-2023-27043 fix. + +.. + +.. date: 2023-10-17-16-11-03 +.. gh-issue: 52161 +.. nonce: WBYyCJ +.. section: Library + +:meth:`cmd.Cmd.do_help` now cleans docstrings with :func:`inspect.cleandoc` +before writing them. Patch by Filip Łapkiewicz. + +.. + +.. date: 2023-10-12-18-19-47 +.. gh-issue: 82300 +.. nonce: P8-O38 +.. section: Library + +Add ``track`` parameter to +:class:`multiprocessing.shared_memory.SharedMemory` that allows using shared +memory blocks without having to register with the POSIX resource tracker +that automatically releases them upon process exit. + +.. + +.. date: 2023-10-11-02-34-01 +.. gh-issue: 110109 +.. nonce: RFCmHs +.. section: Library + +Add private ``pathlib._PurePathBase`` class: a base class for +:class:`pathlib.PurePath` that omits certain magic methods. It may be made +public (along with ``_PathBase``) in future. + +.. + +.. date: 2023-09-28-13-15-51 +.. gh-issue: 109858 +.. nonce: 43e2dg +.. section: Library + +Protect :mod:`zipfile` from "quoted-overlap" zipbomb. It now raises +BadZipFile when try to read an entry that overlaps with other entry or +central directory. + +.. + +.. date: 2023-09-23-14-40-51 +.. gh-issue: 109786 +.. nonce: UX3pKv +.. section: Library + +Fix possible reference leaks and crash when re-enter the ``__next__()`` +method of :class:`itertools.pairwise`. + +.. + +.. date: 2023-09-01-15-33-18 +.. gh-issue: 91539 +.. nonce: xoNLEI +.. section: Library + +Small (10 - 20%) and trivial performance improvement of +:func:`urrlib.request.getproxies_environment`, typically useful when there +are many environment variables to go over. + +.. + +.. date: 2023-08-14-21-10-52 +.. gh-issue: 103363 +.. nonce: u64_QI +.. section: Library + +Add *follow_symlinks* keyword-only argument to :meth:`pathlib.Path.owner` +and :meth:`~pathlib.Path.group`, defaulting to ``True``. + +.. + +.. date: 2023-08-07-21-11-24 +.. gh-issue: 102130 +.. nonce: _UyI5i +.. section: Library + +Support tab completion in :mod:`cmd` for ``editline``. + +.. + +.. date: 2023-08-04-18-43-21 +.. gh-issue: 99437 +.. nonce: Et8hu8 +.. section: Library + +:func:`runpy.run_path` now decodes path-like objects, making sure __file__ +and sys.argv[0] of the module being run are always strings. + +.. + +.. date: 2023-04-29-20-49-13 +.. gh-issue: 104003 +.. nonce: -8Ruk2 +.. section: Library + +Add :func:`warnings.deprecated`, a decorator to mark deprecated functions to +static type checkers and to warn on usage of deprecated classes and +functions. See :pep:`702`. Patch by Jelle Zijlstra. + +.. + +.. date: 2023-04-23-11-08-02 +.. gh-issue: 103708 +.. nonce: Y17C7p +.. section: Library + +Make hardcoded python name, a configurable parameter so that different +implementations of python can override it instead of making huge diffs in +sysconfig.py + +.. + +.. date: 2023-04-09-21-05-43 +.. gh-issue: 66515 +.. nonce: 0DS8Ya +.. section: Library + +:class:`mailbox.MH` now supports folders that do not contain a +``.mh_sequences`` file (e.g. Claws Mail IMAP-cache folders). Patch by Serhiy +Storchaka. + +.. + +.. date: 2023-02-08-00-43-29 +.. gh-issue: 83162 +.. nonce: ufdI9F +.. section: Library + +Renamed :exc:`!re.error` to :exc:`PatternError` for clarity, and kept +:exc:`!re.error` for backward compatibility. Patch by Matthias Bussonnier +and Adam Chhina. + +.. + +.. date: 2022-12-01-16-57-44 +.. gh-issue: 91133 +.. nonce: LKMVCV +.. section: Library + +Fix a bug in :class:`tempfile.TemporaryDirectory` cleanup, which now no +longer dereferences symlinks when working around file system permission +errors. + +.. + +.. bpo: 43153 +.. date: 2021-12-06-22-10-53 +.. nonce: J7mjSy +.. section: Library + +On Windows, ``tempfile.TemporaryDirectory`` previously masked a +``PermissionError`` with ``NotADirectoryError`` during directory cleanup. It +now correctly raises ``PermissionError`` if errors are not ignored. Patch by +Andrei Kulakov and Ken Jin. + +.. + +.. bpo: 32731 +.. date: 2021-11-23-22-22-49 +.. nonce: kNOASr +.. section: Library + +:func:`getpass.getuser` now raises :exc:`OSError` for all failures rather +than :exc:`ImportError` on systems lacking the :mod:`pwd` module or +:exc:`KeyError` if the password database is empty. + +.. + +.. bpo: 34321 +.. date: 2021-04-15-10-41-51 +.. nonce: 36m6_l +.. section: Library + +:class:`mmap.mmap` now has a *trackfd* parameter on Unix; if it is +``False``, the file descriptor specified by *fileno* will not be duplicated. + +.. + +.. bpo: 35332 +.. date: 2020-12-14-09-31-13 +.. nonce: s22wAx +.. section: Library + +The :func:`shutil.rmtree` function now ignores errors when calling +:func:`os.close` when *ignore_errors* is ``True``, and :func:`os.close` no +longer retried after error. + +.. + +.. bpo: 35928 +.. date: 2020-10-03-23-47-28 +.. nonce: E0iPAa +.. section: Library + +:class:`io.TextIOWrapper` now correctly handles the decoding buffer after +``read()`` and ``write()``. + +.. + +.. bpo: 26791 +.. date: 2020-08-06-14-43-55 +.. nonce: KxoEfO +.. section: Library + +:func:`shutil.move` now moves a symlink into a directory when that directory +is the target of the symlink. This provides the same behavior as the mv +shell command. The previous behavior raised an exception. Patch by Jeffrey +Kintscher. + +.. + +.. bpo: 41422 +.. date: 2020-07-28-20-48-05 +.. nonce: iMwnMu +.. section: Library + +Fixed memory leaks of :class:`pickle.Pickler` and :class:`pickle.Unpickler` +involving cyclic references via the internal memo mapping. + +.. + +.. bpo: 19821 +.. date: 2020-06-15-23-44-53 +.. nonce: ihBk39 +.. section: Library + +The :func:`!pydoc.ispackage` function has been deprecated. + +.. + +.. bpo: 40262 +.. date: 2020-05-21-23-32-46 +.. nonce: z4fQv1 +.. section: Library + +The :meth:`ssl.SSLSocket.recv_into` method no longer requires the *buffer* +argument to implement ``__len__`` and supports buffers with arbitrary item +size. + +.. + +.. bpo: 39912 +.. date: 2020-03-09-15-08-29 +.. nonce: xPOBBY +.. section: Library + +:func:`warnings.filterwarnings()` and :func:`warnings.simplefilter()` now +raise appropriate exceptions instead of ``AssertionError``. Patch +contributed by Rémi Lapeyre. + +.. + +.. bpo: 37260 +.. date: 2019-06-14-22-37-32 +.. nonce: oecdIf +.. section: Library + +Fixed a race condition in :func:`shutil.rmtree` in which directory entries +removed by another process or thread while ``shutil.rmtree()`` is running +can cause it to raise FileNotFoundError. Patch by Jeffrey Kintscher. + +.. + +.. bpo: 36959 +.. date: 2019-05-18-15-50-14 +.. nonce: ew6WZ4 +.. section: Library + +Fix some error messages for invalid ISO format string combinations in +``strptime()`` that referred to directives not contained in the format +string. Patch by Gordon P. Hemsley. + +.. + +.. bpo: 18060 +.. date: 2019-05-17-07-22-33 +.. nonce: 5mqTQM +.. section: Library + +Fixed a class inheritance issue that can cause segfaults when deriving two +or more levels of subclasses from a base class of Structure or Union. + +.. + +.. bpo: 29779 +.. date: 2019-05-08-13-14-11 +.. nonce: jg33dp +.. section: Library + +Add a new :envvar:`PYTHON_HISTORY` environment variable to set the location +of a ``.python_history`` file. + +.. + +.. bpo: 21360 +.. date: 2019-02-12-16-12-54 +.. nonce: gkSSfx +.. section: Library + +:class:`mailbox.Maildir` now ignores files with a leading dot. + +.. + +.. date: 2023-11-30-02-33-59 +.. gh-issue: 111699 +.. nonce: _O5G_y +.. section: Documentation + +Relocate ``smtpd`` deprecation notice to its own section rather than under +``locale`` in What's New in Python 3.12 document + +.. + +.. date: 2023-10-23-23-43-43 +.. gh-issue: 110746 +.. nonce: yg77IE +.. section: Documentation + +Improved markup for valid options/values for methods ttk.treeview.column and +ttk.treeview.heading, and for Layouts. + +.. + +.. date: 2023-08-01-13-11-39 +.. gh-issue: 95649 +.. nonce: F4KhPS +.. section: Documentation + +Document that the :mod:`asyncio` module contains code taken from `v0.16.0 of +the uvloop project `_, as +well as the required MIT licensing information. + +.. + +.. date: 2024-01-12-14-34-24 +.. gh-issue: 111798 +.. nonce: hd9B_- +.. section: Tests + +Disable ``test_super_deep()`` from ``test_call`` under pydebug builds on +WASI; the stack depth is too small to make the test useful. + +.. + +.. date: 2024-01-12-13-19-12 +.. gh-issue: 111801 +.. nonce: 9hh9DY +.. section: Tests + +Lower the recursion limit in ``test_isinstance`` for +``test_infinitely_many_bases()``. This prevents a stack overflow on a +pydebug build of WASI. + +.. + +.. date: 2024-01-12-12-45-24 +.. gh-issue: 111802 +.. nonce: gN41vt +.. section: Tests + +Specify a low recursion depth for ``test_bad_getattr()`` in +``test.pickletester`` to avoid exhausting the stack under a pydebug build +for WASI. + +.. + +.. date: 2024-01-08-21-15-48 +.. gh-issue: 44626 +.. nonce: DRq-PR +.. section: Tests + +Fix :func:`os.path.isabs` incorrectly returning ``True`` when given a path +that starts with exactly one (back)slash on Windows. + +Fix :meth:`pathlib.PureWindowsPath.is_absolute` incorrectly returning +``False`` for some paths beginning with two (back)slashes. + +.. + +.. date: 2024-01-01-14-40-02 +.. gh-issue: 113633 +.. nonce: VOY5ai +.. section: Tests + +Use module state for the _testcapi extension module. + +.. + +.. date: 2023-12-09-21-27-46 +.. gh-issue: 109980 +.. nonce: y--500 +.. section: Tests + +Fix ``test_tarfile_vs_tar`` in ``test_shutil`` for macOS, where system tar +can include more information in the archive than :mod:`shutil.make_archive`. + +.. + +.. date: 2023-12-05-19-50-03 +.. gh-issue: 112769 +.. nonce: kdLJmS +.. section: Tests + +The tests now correctly compare zlib version when +:const:`zlib.ZLIB_RUNTIME_VERSION` contains non-integer suffixes. For +example zlib-ng defines the version as ``1.3.0.zlib-ng``. + +.. + +.. date: 2023-12-04-15-56-11 +.. gh-issue: 112334 +.. nonce: FFc9Ti +.. section: Tests + +Adds a regression test to verify that ``vfork()`` is used when expected by +:mod:`subprocess` on vfork enabled POSIX systems (Linux). + +.. + +.. date: 2023-09-05-20-46-35 +.. gh-issue: 108927 +.. nonce: TpwWav +.. section: Tests + +Fixed order dependence in running tests in the same process when a test that +has submodules (e.g. test_importlib) follows a test that imports its +submodule (e.g. test_importlib.util) and precedes a test (e.g. test_unittest +or test_compileall) that uses that submodule. + +.. + +.. bpo: 40648 +.. date: 2020-05-16-18-00-21 +.. nonce: p2uPqy +.. section: Tests + +Test modes that file can get with chmod() on Windows. + +.. + +.. date: 2024-01-15-16-58-43 +.. gh-issue: 114013 +.. nonce: FoSeQf +.. section: Build + +Fix ``Tools/wasm/wasi.py`` to not include the path to ``python.wasm`` as +part of ``HOSTRUNNER``. The environment variable is meant to specify how to +run the WASI host only, having ``python.wasm`` and relevant flags appended +to the ``HOSTRUNNER``. This fixes ``make test`` work. + +.. + +.. date: 2023-12-23-09-35-48 +.. gh-issue: 113258 +.. nonce: GlsAyH +.. section: Build + +Changed the Windows build to write out generated frozen modules into the +build tree instead of the source tree. + +.. + +.. date: 2023-12-21-05-35-06 +.. gh-issue: 112305 +.. nonce: VfqQPx +.. section: Build + +Fixed the ``check-clean-src`` step performed on out of tree builds to detect +errant ``$(srcdir)/Python/frozen_modules/*.h`` files and recommend +appropriate source tree cleanup steps to get a working build again. + +.. + +.. date: 2023-12-17-18-23-02 +.. gh-issue: 112536 +.. nonce: 8lr3Ep +.. section: Build + +Add support for thread sanitizer (TSAN) + +.. + +.. date: 2023-12-08-11-33-37 +.. gh-issue: 112867 +.. nonce: ZzDfXQ +.. section: Build + +Fix the build for the case that WITH_PYMALLOC_RADIX_TREE=0 set. + +.. + +.. date: 2023-11-27-13-55-47 +.. gh-issue: 103065 +.. nonce: o72OiA +.. section: Build + +Introduce ``Tools/wasm/wasi.py`` to simplify doing a WASI build. + +.. + +.. bpo: 11102 +.. date: 2020-05-01-23-44-31 +.. nonce: Fw9zeS +.. section: Build + +The :func:`os.major`, :func:`os.makedev`, and :func:`os.minor` functions are +now available on HP-UX v3. + +.. + +.. bpo: 36351 +.. date: 2020-01-11-23-49-17 +.. nonce: ce8BBh +.. section: Build + +Do not set ipv6type when cross-compiling. + +.. + +.. date: 2024-01-15-23-53-25 +.. gh-issue: 114096 +.. nonce: G-Myja +.. section: Windows + +Process privileges that are activated for creating directory junctions are +now restored afterwards, avoiding behaviour changes in other parts of the +program. + +.. + +.. date: 2024-01-04-21-16-31 +.. gh-issue: 111877 +.. nonce: fR-B4c +.. section: Windows + +:func:`os.stat` calls were returning incorrect time values for files that +could not be accessed directly. + +.. + +.. date: 2023-12-19-10-56-46 +.. gh-issue: 111973 +.. nonce: A9Wtsb +.. section: Windows + +Update Windows installer to use SQLite 3.44.2. + +.. + +.. date: 2023-12-14-19-00-29 +.. gh-issue: 113009 +.. nonce: 6LNdjz +.. section: Windows + +:mod:`multiprocessing`: On Windows, fix a race condition in +``Process.terminate()``: no longer set the ``returncode`` attribute to +always call ``WaitForSingleObject()`` in ``Process.wait()``. Previously, +sometimes the process was still running after ``TerminateProcess()`` even if +``GetExitCodeProcess()`` is not ``STILL_ACTIVE``. Patch by Victor Stinner. + +.. + +.. date: 2023-12-12-20-58-09 +.. gh-issue: 86179 +.. nonce: YYSk_6 +.. section: Windows + +Fixes path calculations when launching Python on Windows through a symlink. + +.. + +.. date: 2023-12-11-20-23-04 +.. gh-issue: 71383 +.. nonce: 9pZh6t +.. section: Windows + +Update Tcl/Tk in Windows installer to 8.6.13 with a patch to suppress +incorrect ThemeChanged warnings. + +.. + +.. date: 2023-12-05-22-56-30 +.. gh-issue: 111650 +.. nonce: xlWmvM +.. section: Windows + +Ensures the ``Py_GIL_DISABLED`` preprocessor variable is defined in +:file:`pyconfig.h` so that extension modules written in C are able to use +it. + +.. + +.. date: 2023-12-03-19-22-37 +.. gh-issue: 112278 +.. nonce: FiloCE +.. section: Windows + +Reduce the time cost for some functions in :mod:`platform` on Windows if +current user has no permission to the WMI. + +.. + +.. date: 2023-08-08-01-42-14 +.. gh-issue: 73427 +.. nonce: WOpiNt +.. section: Windows + +Deprecate :func:`sys._enablelegacywindowsfsencoding`. Use +:envvar:`PYTHONLEGACYWINDOWSFSENCODING` instead. Patch by Inada Naoki. + +.. + +.. date: 2023-03-15-23-53-45 +.. gh-issue: 87868 +.. nonce: 4C36oQ +.. section: Windows + +Correctly sort and remove duplicate environment variables in +:py:func:`!_winapi.CreateProcess`. + +.. + +.. bpo: 37308 +.. date: 2019-06-16-11-27-05 +.. nonce: Iz_NU_ +.. section: Windows + +Fix mojibake in :class:`mmap.mmap` when using a non-ASCII *tagname* argument +on Windows. + +.. + +.. date: 2024-01-02-22-25-21 +.. gh-issue: 113666 +.. nonce: xKZoBm +.. section: macOS + +Add the following constants to module :mod:`stat`: ``UF_SETTABLE``, +``UF_TRACKED``, ``UF_DATAVAULT``, ``SF_SUPPORTED``, ``SF_SETTABLE``, +``SF_SYNTHETIC``, ``SF_RESTRICTED``, ``SF_FIRMLINK`` and ``SF_DATALESS``. +The values ``UF_SETTABLE``, ``SF_SUPPORTED``, ``SF_SETTABLE`` and +``SF_SYNTHETIC`` are only available on macOS. + +.. + +.. date: 2023-12-28-12-18-39 +.. gh-issue: 113536 +.. nonce: 0ythg7 +.. section: macOS + +:func:`os.waitid` is now available on macOS + +.. + +.. date: 2023-12-23-22-41-07 +.. gh-issue: 110459 +.. nonce: NaMBJy +.. section: macOS + +Running ``configure ... --with-openssl-rpath=X/Y/Z`` no longer fails to +detect OpenSSL on macOS. + +.. + +.. date: 2023-12-21-11-53-47 +.. gh-issue: 74573 +.. nonce: MA6Vys +.. section: macOS + +Document that :mod:`dbm.ndbm` can silently corrupt DBM files on updates when +exceeding undocumented platform limits, and can crash (segmentation fault) +when reading such a corrupted file. (FB8919203) + +.. + +.. date: 2023-12-21-10-20-41 +.. gh-issue: 65701 +.. nonce: Q2hNbN +.. section: macOS + +The :program:`freeze` tool doesn't work with framework builds of Python. +Document this and bail out early when running the tool with such a build. + +.. + +.. date: 2023-12-21-09-41-42 +.. gh-issue: 87277 +.. nonce: IF6EZZ +.. section: macOS + +webbrowser: Don't look for X11 browsers on macOS. Those are generally not +used and probing for them can result in starting XQuartz even if it isn't +used otherwise. + +.. + +.. date: 2023-12-19-10-50-08 +.. gh-issue: 111973 +.. nonce: HMHJfy +.. section: macOS + +Update macOS installer to use SQLite 3.44.2. + +.. + +.. date: 2023-12-16-11-45-32 +.. gh-issue: 108269 +.. nonce: wVgCHF +.. section: macOS + +Set ``CFBundleAllowMixedLocalizations`` to true in the Info.plist for the +framework, embedded Python.app and IDLE.app with framework installs on +macOS. This allows applications to pick up the user's preferred locale when +that's different from english. + +.. + +.. date: 2023-12-10-20-30-06 +.. gh-issue: 102362 +.. nonce: y8svbF +.. section: macOS + +Make sure the result of :func:`sysconfig.get_plaform` includes at least a +major and minor versions, even if ``MACOSX_DEPLOYMENT_TARGET`` is set to +only a major version during build to match the format expected by pip. + +.. + +.. date: 2023-12-07-15-53-16 +.. gh-issue: 110017 +.. nonce: UMYzMR +.. section: macOS + +Disable a signal handling stress test on macOS due to a bug in macOS +(FB13453490). + +.. + +.. date: 2023-12-07-14-19-46 +.. gh-issue: 110820 +.. nonce: DIxb_F +.. section: macOS + +Make sure the preprocessor definitions for ``ALIGNOF_MAX_ALIGN_T``, +``SIZEOF_LONG_DOUBLE`` and ``HAVE_GCC_ASM_FOR_X64`` are correct for +Universal 2 builds on macOS. + +.. + +.. date: 2023-12-06-12-11-13 +.. gh-issue: 109981 +.. nonce: mOHg10 +.. section: macOS + +Use ``/dev/fd`` on macOS to determine the number of open files in +``test.support.os_helper.fd_count`` to avoid a crash with "guarded" file +descriptors when probing for open files. + +.. + +.. date: 2024-01-17-02-15-33 +.. gh-issue: 72284 +.. nonce: cAQiYO +.. section: IDLE + +Improve the lists of features, editor key bindings, and shell key bingings +in the IDLE doc. + +.. + +.. date: 2024-01-11-21-26-58 +.. gh-issue: 113903 +.. nonce: __GLlQ +.. section: IDLE + +Fix rare failure of test.test_idle, in test_configdialog. + +.. + +.. date: 2024-01-05-12-24-01 +.. gh-issue: 113729 +.. nonce: qpluea +.. section: IDLE + +Fix the "Help -> IDLE Doc" menu bug in 3.11.7 and 3.12.1. + +.. + +.. date: 2023-12-19-00-03-12 +.. gh-issue: 113269 +.. nonce: lrU-IC +.. section: IDLE + +Fix test_editor hang on macOS Catalina. + +.. + +.. date: 2023-12-10-20-01-11 +.. gh-issue: 112898 +.. nonce: 98aWv2 +.. section: IDLE + +Fix processing unsaved files when quitting IDLE on macOS. + +.. + +.. bpo: 13586 +.. date: 2019-12-13-12-26-56 +.. nonce: 1grqsR +.. section: IDLE + +Enter the selected text when opening the "Replace" dialog. + +.. + +.. date: 2023-12-02-02-08-11 +.. gh-issue: 106560 +.. nonce: THvuji +.. section: C API + +Fix redundant declarations in the public C API. Declare PyBool_Type, +PyLong_Type and PySys_Audit() only once. Patch by Victor Stinner. + +.. + +.. date: 2023-11-27-09-44-16 +.. gh-issue: 112438 +.. nonce: GdNZiI +.. section: C API + +Fix support of format units "es", "et", "es#", and "et#" in nested tuples in +:c:func:`PyArg_ParseTuple`-like functions. + +.. + +.. date: 2023-11-15-01-26-59 +.. gh-issue: 111545 +.. nonce: iAoFtA +.. section: C API + +Add :c:func:`Py_HashPointer` function to hash a pointer. Patch by Victor +Stinner. + +.. + +.. date: 2023-06-21-11-53-09 +.. gh-issue: 65210 +.. nonce: PhFRBJ +.. section: C API + +Change the declaration of the *keywords* parameter of +:c:func:`PyArg_ParseTupleAndKeywords` and +:c:func:`PyArg_VaParseTupleAndKeywords` for better compatibility with C++. diff --git a/Misc/NEWS.d/3.5.0a1.rst b/Misc/NEWS.d/3.5.0a1.rst index 96e59206cb1291..26b3d8253dbdee 100644 --- a/Misc/NEWS.d/3.5.0a1.rst +++ b/Misc/NEWS.d/3.5.0a1.rst @@ -251,8 +251,8 @@ and "surrogatepass" error handlers. .. nonce: FM72m- .. section: Core and Builtins -speed up `PyObject_IsInstance` and `PyObject_IsSubclass` in the common case -that the second argument has metaclass `type`. +speed up ``PyObject_IsInstance`` and ``PyObject_IsSubclass`` in the common case +that the second argument has metaclass ``type``. .. @@ -261,8 +261,8 @@ that the second argument has metaclass `type`. .. nonce: ds5wQa .. section: Core and Builtins -Add a new `PyErr_FormatV` function, similar to `PyErr_Format` but accepting -a `va_list` argument. +Add a new ``PyErr_FormatV`` function, similar to ``PyErr_Format`` but accepting +a ``va_list`` argument. .. @@ -1522,7 +1522,7 @@ Fixed fcntl() with integer argument on 64-bit big-endian platforms. .. nonce: 62MLqr .. section: Library -Add an `--sort-keys` option to json.tool CLI. +Add an ``--sort-keys`` option to ``json.tool`` CLI. .. @@ -1745,7 +1745,7 @@ already failed. .. section: Library Make it possible to examine the errors from unittest discovery without -executing the test suite. The new `errors` attribute on TestLoader exposes +executing the test suite. The new ``errors`` attribute on ``TestLoader`` exposes these non-fatal errors encountered during discovery. .. @@ -2342,9 +2342,9 @@ as normal call attributes. .. nonce: Nghn-Y .. section: Library -load_tests() is now unconditionally run when it is present in a package's -__init__.py. TestLoader.loadTestsFromModule() still accepts use_load_tests, -but it is deprecated and ignored. A new keyword-only attribute `pattern` is +``load_tests()`` is now unconditionally run when it is present in a package's +``__init__.py``. ``TestLoader.loadTestsFromModule()`` still accepts use_load_tests, +but it is deprecated and ignored. A new keyword-only attribute ``pattern`` is added and documented. Patch given by Robert Collins, tweaked by Barry Warsaw. @@ -2648,7 +2648,7 @@ module. .. nonce: THJSYB .. section: Library -Changed FeedParser feed() to avoid O(N\ :sup:`2`) behavior when parsing long line. +Changed FeedParser feed() to avoid *O*\ (*n*\ :sup:`2`) behavior when parsing long line. Original patch by Raymond Hettinger. .. @@ -2736,8 +2736,8 @@ Convert posixmodule to use Argument Clinic. .. nonce: YccmZF .. section: Library -Add an *exists_ok* argument to `Pathlib.mkdir()` to mimic `mkdir -p` and -`os.makedirs()` functionality. When true, ignore FileExistsErrors. Patch +Add an *exists_ok* argument to ``Pathlib.mkdir()`` to mimic ``mkdir -p`` and +``os.makedirs()`` functionality. When true, ignore ``FileExistsErrors``. Patch by Berker Peksag. .. @@ -3930,7 +3930,7 @@ has been called. .. nonce: 5CDoox .. section: Library -New keyword argument `unsafe` to Mock. It raises `AttributeError` incase of +New keyword argument ``unsafe`` to Mock. It raises ``AttributeError`` incase of an attribute startswith assert or assret. .. @@ -4173,7 +4173,7 @@ provide better security by default. .. nonce: FP5FY0 .. section: Library -`assertRaisesRegex` and `assertWarnsRegex` now raise a TypeError if the +``assertRaisesRegex`` and ``assertWarnsRegex`` now raise a ``TypeError`` if the second argument is not a string or compiled regex. .. diff --git a/Misc/NEWS.d/3.5.0b4.rst b/Misc/NEWS.d/3.5.0b4.rst index 2b1b98a4316fc1..e42d93689d01a0 100644 --- a/Misc/NEWS.d/3.5.0b4.rst +++ b/Misc/NEWS.d/3.5.0b4.rst @@ -129,7 +129,7 @@ Random.setstate() now validates the value of state last element. .. nonce: HvJf6T .. section: Library -Fixed an issue that caused `inspect.getsource` to return incorrect results +Fixed an issue that caused ``inspect.getsource`` to return incorrect results on nested functions. .. diff --git a/Misc/NEWS.d/3.5.2rc1.rst b/Misc/NEWS.d/3.5.2rc1.rst index 01fcd866a896ae..a7e5c1b130f9e9 100644 --- a/Misc/NEWS.d/3.5.2rc1.rst +++ b/Misc/NEWS.d/3.5.2rc1.rst @@ -710,9 +710,9 @@ Fixed the comparison of plistlib.Data with other types. .. nonce: RMRMtM .. section: Library -Fix an uninitialized variable in `ctypes.util`. +Fix an uninitialized variable in ``ctypes.util``. The bug only occurs on SunOS when the ctypes implementation searches for the -`crle` program. Patch by Xiang Zhang. Tested on SunOS by Kees Bos. +``crle`` program. Patch by Xiang Zhang. Tested on SunOS by Kees Bos. .. diff --git a/Misc/NEWS.d/3.6.0a1.rst b/Misc/NEWS.d/3.6.0a1.rst index 98f1215fb91873..144d217f6098a1 100644 --- a/Misc/NEWS.d/3.6.0a1.rst +++ b/Misc/NEWS.d/3.6.0a1.rst @@ -1113,9 +1113,9 @@ Fixed the comparison of plistlib.Data with other types. .. nonce: RMRMtM .. section: Library -Fix an uninitialized variable in `ctypes.util`. +Fix an uninitialized variable in ``ctypes.util``. The bug only occurs on SunOS when the ctypes implementation searches for the -`crle` program. Patch by Xiang Zhang. Tested on SunOS by Kees Bos. +``crle`` program. Patch by Xiang Zhang. Tested on SunOS by Kees Bos. .. @@ -3915,7 +3915,7 @@ Fix output of python-config --extension-suffix. .. nonce: yLO-r4 .. section: Tools/Demos -The pyvenv script has been deprecated in favour of `python3 -m venv`. +The pyvenv script has been deprecated in favour of ``python3 -m venv``. .. diff --git a/Misc/NEWS.d/3.6.0b1.rst b/Misc/NEWS.d/3.6.0b1.rst index 3fbae5c6a4b3a8..4fb6bdd6f89c9b 100644 --- a/Misc/NEWS.d/3.6.0b1.rst +++ b/Misc/NEWS.d/3.6.0b1.rst @@ -166,7 +166,7 @@ a DeprecationWarning. Patch by Emanuel Barry. .. nonce: aABzcL .. section: Core and Builtins -`dict` implementation is changed like PyPy. It is more compact and preserves +``dict`` implementation is changed like PyPy. It is more compact and preserves insertion order. (Concept developed by Raymond Hettinger and patch by Inada Naoki.) diff --git a/Misc/NEWS.d/3.6.3.rst b/Misc/NEWS.d/3.6.3.rst index 4d591d77ffe545..58fd009aea1fed 100644 --- a/Misc/NEWS.d/3.6.3.rst +++ b/Misc/NEWS.d/3.6.3.rst @@ -4,7 +4,7 @@ .. release date: 2017-10-03 .. section: Library -Re-allow arbitrary iterables in `concurrent.futures.as_completed()`. Fixes +Re-allow arbitrary iterables in ``concurrent.futures.as_completed()``. Fixes regression in 3.6.3rc1. .. diff --git a/Misc/NEWS.d/3.6.3rc1.rst b/Misc/NEWS.d/3.6.3rc1.rst index 4b2aae9dc88441..ebda7665e2b6ea 100644 --- a/Misc/NEWS.d/3.6.3rc1.rst +++ b/Misc/NEWS.d/3.6.3rc1.rst @@ -24,8 +24,8 @@ fixes. .. nonce: 0yiA5Q .. section: Core and Builtins -Fix an assertion failure in `subprocess.Popen()` on Windows, in case the env -argument has a bad keys() method. Patch by Oren Milman. +Fix an assertion failure in ``subprocess.Popen()`` on Windows, in case the env +argument has a bad ``keys()`` method. Patch by Oren Milman. .. @@ -34,7 +34,7 @@ argument has a bad keys() method. Patch by Oren Milman. .. nonce: rS-FlC .. section: Core and Builtins -Fix an assertion failure in `PyErr_WriteUnraisable()` in case of an +Fix an assertion failure in ``PyErr_WriteUnraisable()`` in case of an exception with a bad ``__module__`` attribute. Patch by Oren Milman. .. @@ -95,7 +95,7 @@ plans to remove the functions from sys/types.h. .. nonce: t8QggK .. section: Core and Builtins -Fix an assertion failure in `zipimport.zipimporter.get_data` on Windows, +Fix an assertion failure in ``zipimport.zipimporter.get_data`` on Windows, when the return value of ``pathname.replace('/','\\')`` isn't a string. Patch by Oren Milman. @@ -106,7 +106,7 @@ Patch by Oren Milman. .. nonce: YMduKF .. section: Core and Builtins -Fix an assertion failure in the write() method of `io.TextIOWrapper`, when +Fix an assertion failure in the ``write()`` method of ``io.TextIOWrapper``, when the encoder doesn't return a bytes object. Patch by Oren Milman. .. @@ -116,7 +116,7 @@ the encoder doesn't return a bytes object. Patch by Oren Milman. .. nonce: dRJzqR .. section: Core and Builtins -Fix a crash in some methods of `io.TextIOWrapper`, when the decoder's state +Fix a crash in some methods of ``io.TextIOWrapper``, when the decoder's state is invalid. Patch by Oren Milman. .. @@ -477,7 +477,7 @@ attributes to help the garbage collector to destroy all widgets. .. nonce: 1t2hn5 .. section: Library -Fix `copyreg._slotnames()` mangled attribute calculation for classes whose +Fix ``copyreg._slotnames()`` mangled attribute calculation for classes whose name begins with an underscore. Patch by Shane Harvey. .. @@ -585,7 +585,7 @@ socket.close() now ignores ECONNRESET error. .. nonce: CLvEvV .. section: Library -Fix out of bounds write in `asyncio.CFuture.remove_done_callback()`. +Fix out of bounds write in ``asyncio.CFuture.remove_done_callback()``. .. @@ -933,7 +933,7 @@ Add tests for configdialog keys tab. Patch by Cheryl Sabella. .. nonce: sqE1FS .. section: IDLE -IDLE: Calltips use `inspect.signature` instead of `inspect.getfullargspec`. +IDLE: Calltips use ``inspect.signature`` instead of ``inspect.getfullargspec``. This improves calltips for builtins converted to use Argument Clinic. Patch by Louie Lu. diff --git a/Misc/NEWS.d/3.6.4rc1.rst b/Misc/NEWS.d/3.6.4rc1.rst index ae4534be62c5c4..afa5b8b0efb148 100644 --- a/Misc/NEWS.d/3.6.4rc1.rst +++ b/Misc/NEWS.d/3.6.4rc1.rst @@ -138,7 +138,7 @@ integer with binary base. .. section: Core and Builtins Fixed an assertion failure in Python parser in case of a bad -`unicodedata.normalize()`. Patch by Oren Milman. +``unicodedata.normalize()``. Patch by Oren Milman. .. @@ -147,7 +147,7 @@ Fixed an assertion failure in Python parser in case of a bad .. nonce: wT9Iy7 .. section: Core and Builtins -Raise a `TypeError` with a helpful error message when class creation fails +Raise a ``TypeError`` with a helpful error message when class creation fails due to a metaclass with a bad ``__prepare__()`` method. Patch by Oren Milman. @@ -158,7 +158,7 @@ Milman. .. nonce: OxwINs .. section: Core and Builtins -Fix an assertion failure in `_warnings.warn()` in case of a bad ``__name__`` +Fix an assertion failure in ``_warnings.warn()`` in case of a bad ``__name__`` global. Patch by Oren Milman. .. @@ -168,8 +168,8 @@ global. Patch by Oren Milman. .. nonce: VomaFa .. section: Core and Builtins -Fix an assertion failure in `json`, in case `_json.make_encoder()` received -a bad `encoder()` argument. Patch by Oren Milman. +Fix an assertion failure in ``json``, in case ``_json.make_encoder()`` received +a bad ``encoder()`` argument. Patch by Oren Milman. .. @@ -189,7 +189,7 @@ such a module. Patch by Oren Milman. .. nonce: r7m2sj .. section: Core and Builtins -Fix an assertion failure in `ctypes` class definition, in case the class has +Fix an assertion failure in ``ctypes`` class definition, in case the class has an attribute whose name is specified in ``_anonymous_`` but not in ``_fields_``. Patch by Oren Milman. @@ -200,7 +200,7 @@ an attribute whose name is specified in ``_anonymous_`` but not in .. nonce: o06iKD .. section: Core and Builtins -Fix an assertion failure in `_random.Random.seed()` in case the argument has +Fix an assertion failure in ``_random.Random.seed()`` in case the argument has a bad ``__abs__()`` method. Patch by Oren Milman. .. @@ -220,7 +220,7 @@ string. Patch by Oren Milman. .. nonce: bNE2l- .. section: Core and Builtins -Fix a crash in the ``__setstate__()`` method of `ctypes._CData`, in case of +Fix a crash in the ``__setstate__()`` method of ``ctypes._CData``, in case of a bad ``__dict__``. Patch by Oren Milman. .. @@ -240,8 +240,8 @@ float with a bad as_integer_ratio() method. Patch by Oren Milman. .. nonce: 7lzaKV .. section: Core and Builtins -Fix an assertion failure in `warnings.warn_explicit`, when the return value -of the received loader's get_source() has a bad splitlines() method. Patch +Fix an assertion failure in ``warnings.warn_explicit``, when the return value +of the received loader's ``get_source()`` has a bad ``splitlines()`` method. Patch by Oren Milman. .. @@ -251,8 +251,8 @@ by Oren Milman. .. nonce: j7ZvN_ .. section: Core and Builtins -`PyErr_PrintEx()` clears now the ignored exception that may be raised by -`_PySys_SetObjectId()`, for example when no memory. +``PyErr_PrintEx()`` clears now the ignored exception that may be raised by +``_PySys_SetObjectId()``, for example when no memory. .. @@ -599,8 +599,8 @@ On Windows, faulthandler.enable() now ignores MSC and COM exceptions. .. nonce: XrVMME .. section: Library -Prevent crashes in `_elementtree` due to unsafe cleanup of `Element.text` -and `Element.tail`. Patch by Oren Milman. +Prevent crashes in ``_elementtree`` due to unsafe cleanup of ``Element.text`` +and ``Element.tail``. Patch by Oren Milman. .. @@ -813,8 +813,8 @@ reference leaks. .. nonce: lo7FQX .. section: Tests -Add the `set_nomemory(start, stop)` and `remove_mem_hooks()` functions to -the _testcapi module. +Add the ``set_nomemory(start, stop)`` and ``remove_mem_hooks()`` functions to +the ``_testcapi`` module. .. @@ -897,7 +897,7 @@ Prevent double substitution of prefix in python-config.sh. .. nonce: KUDjno .. section: Build -Avoid wholesale rebuild after `make regen-all` if nothing changed. +Avoid wholesale rebuild after ``make regen-all`` if nothing changed. .. @@ -1123,7 +1123,7 @@ and Py_SetPath() .. nonce: Q3T_8n .. section: C API -The `PyExc_RecursionErrorInst` singleton is removed and -`PyErr_NormalizeException()` does not use it anymore. This singleton is +The ``PyExc_RecursionErrorInst`` singleton is removed and +``PyErr_NormalizeException()`` does not use it anymore. This singleton is persistent and its members being never cleared may cause a segfault during finalization of the interpreter. See also issue #22898. diff --git a/Misc/NEWS.d/3.6.5rc1.rst b/Misc/NEWS.d/3.6.5rc1.rst index 448baed5413ecb..056bacb5267c41 100644 --- a/Misc/NEWS.d/3.6.5rc1.rst +++ b/Misc/NEWS.d/3.6.5rc1.rst @@ -283,7 +283,7 @@ Make sure sys.argv remains as a list when running trace. .. nonce: bvHDOc .. section: Library -Fixed `asyncio.Condition` issue which silently ignored cancellation after +Fixed ``asyncio.Condition`` issue which silently ignored cancellation after notifying and cancelling a conditional lock. Patch by Bar Harel. .. @@ -599,7 +599,7 @@ deprecation. .. nonce: w1m_8r .. section: Documentation -Improve docstrings for `pathlib.PurePath` subclasses. +Improve docstrings for ``pathlib.PurePath`` subclasses. .. diff --git a/Misc/NEWS.d/3.6.6rc1.rst b/Misc/NEWS.d/3.6.6rc1.rst index 9624195c79043b..f70649af055b4e 100644 --- a/Misc/NEWS.d/3.6.6rc1.rst +++ b/Misc/NEWS.d/3.6.6rc1.rst @@ -110,7 +110,7 @@ on Windows. .. nonce: UoC319 .. section: Core and Builtins -Fix a crash in `ctypes.cast()` in case the type argument is a ctypes +Fix a crash in ``ctypes.cast()`` in case the type argument is a ctypes structured data type. Patch by Eryk Sun and Oren Milman. .. @@ -248,7 +248,7 @@ Patch by Zvi Effron .. nonce: taxbVT .. section: Library -Fix race condition with `ReadTransport.resume_reading` in Windows proactor +Fix race condition with ``ReadTransport.resume_reading`` in Windows proactor event loop. .. @@ -346,7 +346,7 @@ tree of tuples or lists with ``line_info=False`` and ``col_info=True``. .. nonce: B56Hc1 .. section: Library -Fix FD leak in `_SelectorSocketTransport` Patch by Vlad Starostin. +Fix FD leak in ``_SelectorSocketTransport`` Patch by Vlad Starostin. .. diff --git a/Misc/NEWS.d/3.7.0a1.rst b/Misc/NEWS.d/3.7.0a1.rst index bee424241fd712..aca79c4cc8c1b8 100644 --- a/Misc/NEWS.d/3.7.0a1.rst +++ b/Misc/NEWS.d/3.7.0a1.rst @@ -86,7 +86,7 @@ information. .. nonce: r7m2sj .. section: Core and Builtins -Fix an assertion failure in `ctypes` class definition, in case the class has +Fix an assertion failure in ``ctypes`` class definition, in case the class has an attribute whose name is specified in ``_anonymous_`` but not in ``_fields_``. Patch by Oren Milman. @@ -97,8 +97,8 @@ an attribute whose name is specified in ``_anonymous_`` but not in .. nonce: 0yiA5Q .. section: Core and Builtins -Fix an assertion failure in `subprocess.Popen()` on Windows, in case the env -argument has a bad keys() method. Patch by Oren Milman. +Fix an assertion failure in ``subprocess.Popen()`` on Windows, in case the env +argument has a bad ``keys()`` method. Patch by Oren Milman. .. @@ -107,7 +107,7 @@ argument has a bad keys() method. Patch by Oren Milman. .. nonce: rS-FlC .. section: Core and Builtins -Fix an assertion failure in `PyErr_WriteUnraisable()` in case of an +Fix an assertion failure in ``PyErr_WriteUnraisable()`` in case of an exception with a bad ``__module__`` attribute. Patch by Oren Milman. .. @@ -224,7 +224,7 @@ plans to remove the functions from sys/types.h. .. nonce: t8QggK .. section: Core and Builtins -Fix an assertion failure in `zipimport.zipimporter.get_data` on Windows, +Fix an assertion failure in ``zipimport.zipimporter.get_data`` on Windows, when the return value of ``pathname.replace('/','\\')`` isn't a string. Patch by Oren Milman. @@ -235,7 +235,7 @@ Patch by Oren Milman. .. nonce: YMduKF .. section: Core and Builtins -Fix an assertion failure in the write() method of `io.TextIOWrapper`, when +Fix an assertion failure in the ``write()`` method of ``io.TextIOWrapper``, when the encoder doesn't return a bytes object. Patch by Oren Milman. .. @@ -245,7 +245,7 @@ the encoder doesn't return a bytes object. Patch by Oren Milman. .. nonce: dRJzqR .. section: Core and Builtins -Fix a crash in some methods of `io.TextIOWrapper`, when the decoder's state +Fix a crash in some methods of ``io.TextIOWrapper``, when the decoder's state is invalid. Patch by Oren Milman. .. @@ -1855,7 +1855,7 @@ docserver attribute to None to break a reference cycle. .. nonce: gwnthq .. section: Library -Many asserts in `multiprocessing` are now more informative, and some error +Many asserts in ``multiprocessing`` are now more informative, and some error types have been changed to more specific ones. .. @@ -1896,7 +1896,7 @@ child exit. .. nonce: -2_YGj .. section: Library -`dis` now works with asynchronous generator and coroutine objects. Patch by +``dis`` now works with asynchronous generator and coroutine objects. Patch by George Collins based on diagnosis by Luciano Ramalho. .. @@ -1906,10 +1906,10 @@ George Collins based on diagnosis by Luciano Ramalho. .. nonce: huQi2Y .. section: Library -There are a number of uninformative asserts in the `multiprocessing` module, +There are a number of uninformative asserts in the ``multiprocessing`` module, as noted in issue 5001. This change fixes two of the most potentially problematic ones, since they are in error-reporting code, in the -`multiprocessing.managers.convert_to_error` function. (It also makes more +``multiprocessing.managers.convert_to_error`` function. (It also makes more informative a ValueError message.) The only potentially problematic change is that the AssertionError is now a TypeError; however, this should also help distinguish it from an AssertionError being *reported* by the @@ -1973,7 +1973,7 @@ attributes to help the garbage collector to destroy all widgets. .. nonce: 1t2hn5 .. section: Library -Fix `copyreg._slotnames()` mangled attribute calculation for classes whose +Fix ``copyreg._slotnames()`` mangled attribute calculation for classes whose name begins with an underscore. Patch by Shane Harvey. .. @@ -1983,7 +1983,7 @@ name begins with an underscore. Patch by Shane Harvey. .. nonce: 2CFVCO .. section: Library -Allow `logging.config.fileConfig` to accept kwargs and/or args. +Allow ``logging.config.fileConfig`` to accept kwargs and/or args. .. @@ -2159,7 +2159,7 @@ socket.close() now ignores ECONNRESET error. .. nonce: CLvEvV .. section: Library -Fix out of bounds write in `asyncio.CFuture.remove_done_callback()`. +Fix out of bounds write in ``asyncio.CFuture.remove_done_callback()``. .. @@ -2406,7 +2406,7 @@ Don't log exceptions if Task/Future "cancel()" method was called. .. nonce: xihJ4Y .. section: Library -Fix path calculation in `imp.load_package()`, fixing it for cases when a +Fix path calculation in ``imp.load_package()``, fixing it for cases when a package is only shipped with bytecodes. Patch by Alexandru Ardelean. .. @@ -4947,8 +4947,8 @@ run test_zipfile64. .. nonce: lo7FQX .. section: Tests -Add the `set_nomemory(start, stop)` and `remove_mem_hooks()` functions to -the _testcapi module. +Add the ``set_nomemory(start, stop)`` and ``remove_mem_hooks()`` functions to +the ``_testcapi`` module. .. @@ -5076,7 +5076,7 @@ on the Android armv7 qemu emulator. .. nonce: 4f-VJK .. section: Build -Allow --with-lto to be used on all builds, not just `make profile-opt`. +Allow ``--with-lto`` to be used on all builds, not just ``make profile-opt``. .. @@ -5819,7 +5819,7 @@ Add tests for configdialog keys tab. Patch by Cheryl Sabella. .. nonce: sqE1FS .. section: IDLE -IDLE: Calltips use `inspect.signature` instead of `inspect.getfullargspec`. +IDLE: Calltips use ``inspect.signature`` instead of ``inspect.getfullargspec``. This improves calltips for builtins converted to use Argument Clinic. Patch by Louie Lu. @@ -6224,9 +6224,9 @@ Added the slice index converter in Argument Clinic. .. nonce: KPFC7o .. section: Tools/Demos -Argument Clinic now uses the converter `bool(accept={int})` rather than -`int` for semantical booleans. This avoids repeating the default value for -Python and C and will help in converting to `bool` in future. +Argument Clinic now uses the converter ``bool(accept={int})`` rather than +``int`` for semantical booleans. This avoids repeating the default value for +Python and C and will help in converting to ``bool`` in future. .. diff --git a/Misc/NEWS.d/3.7.0a2.rst b/Misc/NEWS.d/3.7.0a2.rst index 0f107d8c5f5a93..06ca32d37fa4fa 100644 --- a/Misc/NEWS.d/3.7.0a2.rst +++ b/Misc/NEWS.d/3.7.0a2.rst @@ -65,8 +65,8 @@ integer with binary base. .. nonce: MtgLCn .. section: Core and Builtins -Fix an assertion failure in `zipimporter.get_source()` in case of a bad -`zlib.decompress()`. Patch by Oren Milman. +Fix an assertion failure in ``zipimporter.get_source()`` in case of a bad +``zlib.decompress()``. Patch by Oren Milman. .. @@ -76,7 +76,7 @@ Fix an assertion failure in `zipimporter.get_source()` in case of a bad .. section: Core and Builtins Fixed an assertion failure in Python parser in case of a bad -`unicodedata.normalize()`. Patch by Oren Milman. +``unicodedata.normalize()``. Patch by Oren Milman. .. @@ -85,7 +85,7 @@ Fixed an assertion failure in Python parser in case of a bad .. nonce: wT9Iy7 .. section: Core and Builtins -Raise a `TypeError` with a helpful error message when class creation fails +Raise a ``TypeError`` with a helpful error message when class creation fails due to a metaclass with a bad ``__prepare__()`` method. Patch by Oren Milman. @@ -105,7 +105,7 @@ Importlib was instrumented with two dtrace probes to profile import timing. .. nonce: OxwINs .. section: Core and Builtins -Fix an assertion failure in `_warnings.warn()` in case of a bad ``__name__`` +Fix an assertion failure in ``_warnings.warn()`` in case of a bad ``__name__`` global. Patch by Oren Milman. .. @@ -115,7 +115,7 @@ global. Patch by Oren Milman. .. nonce: pRVTRB .. section: Core and Builtins -Improved the error message logic for object.__new__ and object.__init__. +Improved the error message logic for ``object.__new__`` and ``object.__init__``. .. @@ -124,8 +124,8 @@ Improved the error message logic for object.__new__ and object.__init__. .. nonce: VomaFa .. section: Core and Builtins -Fix an assertion failure in `json`, in case `_json.make_encoder()` received -a bad `encoder()` argument. Patch by Oren Milman. +Fix an assertion failure in ``json``, in case ``_json.make_encoder()`` received +a bad ``encoder()`` argument. Patch by Oren Milman. .. @@ -145,7 +145,7 @@ such a module. Patch by Oren Milman. .. nonce: o06iKD .. section: Core and Builtins -Fix an assertion failure in `_random.Random.seed()` in case the argument has +Fix an assertion failure in ``_random.Random.seed()`` in case the argument has a bad ``__abs__()`` method. Patch by Oren Milman. .. @@ -218,7 +218,7 @@ string. Patch by Oren Milman. .. nonce: bNE2l- .. section: Core and Builtins -Fix a crash in the ``__setstate__()`` method of `ctypes._CData`, in case of +Fix a crash in the ``__setstate__()`` method of ``ctypes._CData``, in case of a bad ``__dict__``. Patch by Oren Milman. .. @@ -238,8 +238,8 @@ float with a bad as_integer_ratio() method. Patch by Oren Milman. .. nonce: 7lzaKV .. section: Core and Builtins -Fix an assertion failure in `warnings.warn_explicit`, when the return value -of the received loader's get_source() has a bad splitlines() method. Patch +Fix an assertion failure in ``warnings.warn_explicit``, when the return value +of the received loader's ``get_source()`` has a bad ``splitlines()`` method. Patch by Oren Milman. .. @@ -286,8 +286,8 @@ On Windows, faulthandler.enable() now ignores MSC and COM exceptions. .. nonce: XrVMME .. section: Library -Prevent crashes in `_elementtree` due to unsafe cleanup of `Element.text` -and `Element.tail`. Patch by Oren Milman. +Prevent crashes in ``_elementtree`` due to unsafe cleanup of ``Element.text`` +and ``Element.tail``. Patch by Oren Milman. .. @@ -307,8 +307,8 @@ compiling. bm_regex_compile benchmark shows 14% performance improvements. .. section: Library The types of compiled regular objects and match objects are now exposed as -`re.Pattern` and `re.Match`. This adds information in pydoc output for the -re module. +``re.Pattern`` and ``re.Match``. This adds information in pydoc output for the +``re`` module. .. @@ -485,10 +485,10 @@ since Python 3.3.) .. nonce: cIMFJW .. section: Library -Reprs of subclasses of some collection and iterator classes (`bytearray`, -`array.array`, `collections.deque`, `collections.defaultdict`, -`itertools.count`, `itertools.repeat`) now contain actual type name insteads -of hardcoded name of the base class. +Reprs of subclasses of some collection and iterator classes (``bytearray``, +``array.array``, ``collections.deque``, ``collections.defaultdict``, +``itertools.count``, ``itertools.repeat``) now contain actual type name instead +of hardcoded names of the base class. .. @@ -584,7 +584,7 @@ Correct PCBuild/ case to PCbuild/ in build scripts and documentation. .. nonce: KUDjno .. section: Build -Avoid wholesale rebuild after `make regen-all` if nothing changed. +Avoid wholesale rebuild after ``make regen-all`` if nothing changed. .. @@ -657,8 +657,8 @@ for code and tests by Guilherme Polo and Cheryl Sabella, respectively. .. nonce: K_EjpO .. section: C API -Make `PyMapping_Keys()`, `PyMapping_Values()` and `PyMapping_Items()` always -return a `list` (rather than a `list` or a `tuple`). Patch by Oren Milman. +Make ``PyMapping_Keys()``, ``PyMapping_Values()`` and ``PyMapping_Items()`` always +return a ``list`` (rather than a ``list`` or a ``tuple``). Patch by Oren Milman. .. diff --git a/Misc/NEWS.d/3.7.0a3.rst b/Misc/NEWS.d/3.7.0a3.rst index a968616f55be68..08cabeda7e9a46 100644 --- a/Misc/NEWS.d/3.7.0a3.rst +++ b/Misc/NEWS.d/3.7.0a3.rst @@ -240,8 +240,8 @@ after shrinking a memory block. .. nonce: j7ZvN_ .. section: Core and Builtins -`PyErr_PrintEx()` clears now the ignored exception that may be raised by -`_PySys_SetObjectId()`, for example when no memory. +``PyErr_PrintEx()`` clears now the ignored exception that may be raised by +``_PySys_SetObjectId()``, for example when no memory. .. @@ -643,7 +643,7 @@ undocumented. .. nonce: RwietE .. section: Library -cProfile command line now accepts `-m module_name` as an alternative to +cProfile command line now accepts ``-m module_name`` as an alternative to script path. Patch by Sanyam Khurana. .. @@ -926,7 +926,7 @@ Fix multiprocessing.Process when stdout and/or stderr is closed or None. .. nonce: -1MBEy .. section: Library -Add `subnet_of` and `superset_of` containment tests to +Add ``subnet_of`` and ``superset_of`` containment tests to :class:`ipaddress.IPv6Network` and :class:`ipaddress.IPv4Network`. Patch by Michel Albert and Cheryl Sabella. @@ -948,7 +948,7 @@ for backward compatibility with Python 2.2, and was deprecated since Python .. nonce: IxCvGB .. section: Library -Add a ``subprocess.Popen(text=False)`` keyword argument to `subprocess` +Add a ``subprocess.Popen(text=False)`` keyword argument to ``subprocess`` functions to be more explicit about when the library should attempt to decode outputs into text. Patch by Andrew Clegg. @@ -1191,7 +1191,7 @@ interruptions. If it crashes, restart it when necessary. .. nonce: AniZuz .. section: Library -Added support for AF_UNIX socket in asyncio `create_datagram_endpoint`. +Added support for AF_UNIX socket in asyncio ``create_datagram_endpoint``. .. @@ -1609,7 +1609,7 @@ comparison functions. .. nonce: Q3T_8n .. section: C API -The `PyExc_RecursionErrorInst` singleton is removed and -`PyErr_NormalizeException()` does not use it anymore. This singleton is +The ``PyExc_RecursionErrorInst`` singleton is removed and +``PyErr_NormalizeException()`` does not use it anymore. This singleton is persistent and its members being never cleared may cause a segfault during finalization of the interpreter. See also issue #22898. diff --git a/Misc/NEWS.d/3.7.0a4.rst b/Misc/NEWS.d/3.7.0a4.rst index ebae046a7a6ba0..f2c6559037d84f 100644 --- a/Misc/NEWS.d/3.7.0a4.rst +++ b/Misc/NEWS.d/3.7.0a4.rst @@ -152,7 +152,7 @@ option or the ``PYTHONWARNINGS`` environment variable. .. nonce: PgGQaB .. section: Core and Builtins -`-X dev` now injects a ``'default'`` entry into sys.warnoptions, ensuring +``-X dev`` now injects a ``'default'`` entry into sys.warnoptions, ensuring that it behaves identically to actually passing ``-Wdefault`` at the command line. @@ -192,7 +192,7 @@ by Ivan Levkivskyi. .. nonce: mDeCLK .. section: Core and Builtins -The `atexit` module now has its callback stored per interpreter. +The ``atexit`` module now has its callback stored per interpreter. .. @@ -792,7 +792,7 @@ to more readily adjust to platform dependent behaviour. .. nonce: ODpc9y .. section: Windows -Implement support for `subprocess.Popen(close_fds=True)` on Windows. Patch +Implement support for ``subprocess.Popen(close_fds=True)`` on Windows. Patch by Segev Finer. .. diff --git a/Misc/NEWS.d/3.7.0b2.rst b/Misc/NEWS.d/3.7.0b2.rst index 9590914599bb86..702dbc960c018d 100644 --- a/Misc/NEWS.d/3.7.0b2.rst +++ b/Misc/NEWS.d/3.7.0b2.rst @@ -214,7 +214,7 @@ helper methods that can be used instead ``_dump_registry``, .. nonce: bvHDOc .. section: Library -Fixed `asyncio.Condition` issue which silently ignored cancellation after +Fixed ``asyncio.Condition`` issue which silently ignored cancellation after notifying and cancelling a conditional lock. Patch by Bar Harel. .. @@ -497,7 +497,7 @@ deprecation. .. nonce: w1m_8r .. section: Documentation -Improve docstrings for `pathlib.PurePath` subclasses. +Improve docstrings for ``pathlib.PurePath`` subclasses. .. diff --git a/Misc/NEWS.d/3.7.0b4.rst b/Misc/NEWS.d/3.7.0b4.rst index 1d4fc921406fd0..b17c7e08d1d408 100644 --- a/Misc/NEWS.d/3.7.0b4.rst +++ b/Misc/NEWS.d/3.7.0b4.rst @@ -371,8 +371,8 @@ suffixes). .. nonce: o7G_UO .. section: Build -By default, modules configured in `Modules/Setup` are no longer built with -`-DPy_BUILD_CORE`. Instead, modules that specifically need that preprocessor +By default, modules configured in ``Modules/Setup`` are no longer built with +``-DPy_BUILD_CORE``. Instead, modules that specifically need that preprocessor definition include it in their individual entries. .. diff --git a/Misc/NEWS.d/3.7.0b5.rst b/Misc/NEWS.d/3.7.0b5.rst index fb29109869188b..459eaddbc91f59 100644 --- a/Misc/NEWS.d/3.7.0b5.rst +++ b/Misc/NEWS.d/3.7.0b5.rst @@ -26,7 +26,7 @@ module_globals is not a dict. .. nonce: kqBNzv .. section: Core and Builtins -The new `os.posix_spawn` added in 3.7.0b1 was removed as we are still +The new ``os.posix_spawn`` added in 3.7.0b1 was removed as we are still working on what the API should look like. Expect this in 3.8 instead. .. @@ -77,7 +77,7 @@ their body. Based on patch by Inada Naoki. .. nonce: UoC319 .. section: Core and Builtins -Fix a crash in `ctypes.cast()` in case the type argument is a ctypes +Fix a crash in ``ctypes.cast()`` in case the type argument is a ctypes structured data type. Patch by Eryk Sun and Oren Milman. .. @@ -254,8 +254,8 @@ default. .. nonce: C6Hnd1 .. section: Library -Do not simplify arguments to `typing.Union`. Now `Union[Manager, Employee]` -is not simplified to `Employee` at runtime. Such simplification previously +Do not simplify arguments to ``typing.Union``. Now ``Union[Manager, Employee]`` +is not simplified to ``Employee`` at runtime. Such simplification previously caused several bugs and limited possibilities for introspection. .. @@ -314,7 +314,7 @@ Patch by Zvi Effron .. nonce: taxbVT .. section: Library -Fix race condition with `ReadTransport.resume_reading` in Windows proactor +Fix race condition with ``ReadTransport.resume_reading`` in Windows proactor event loop. .. @@ -324,7 +324,7 @@ event loop. .. nonce: pj2Mbb .. section: Library -Fix failure in `typing.get_type_hints()` when ClassVar was provided as a +Fix failure in ``typing.get_type_hints()`` when ClassVar was provided as a string forward reference. .. @@ -427,7 +427,7 @@ Donghee Na. .. nonce: B56Hc1 .. section: Library -Fix FD leak in `_SelectorSocketTransport` Patch by Vlad Starostin. +Fix FD leak in ``_SelectorSocketTransport`` Patch by Vlad Starostin. .. @@ -466,7 +466,7 @@ Support arrays >=2GiB in :mod:`ctypes`. Patch by Segev Finer. .. nonce: E5gba1 .. section: Documentation -Document that `asyncio.wait()` does not cancel its futures on timeout. +Document that ``asyncio.wait()`` does not cancel its futures on timeout. .. diff --git a/Misc/NEWS.d/3.8.0a1.rst b/Misc/NEWS.d/3.8.0a1.rst index 2b9dbd5d63a87e..bd9061601fe190 100644 --- a/Misc/NEWS.d/3.8.0a1.rst +++ b/Misc/NEWS.d/3.8.0a1.rst @@ -882,7 +882,7 @@ Update valgrind suppression list to use .. nonce: GIOm_8 .. section: Core and Builtins -Added the "socket" option in the `stat.filemode()` Python implementation to +Added the "socket" option in the ``stat.filemode()`` Python implementation to match the C implementation. .. @@ -1271,8 +1271,8 @@ parentheses in the string representation. .. nonce: tDBciE .. section: Core and Builtins -Added support for the `setpgroup`, `resetids`, `setsigmask`, `setsigdef` and -`scheduler` parameters of `posix_spawn`. Patch by Pablo Galindo. +Added support for the ``setpgroup``, ``resetids``, ``setsigmask``, ``setsigdef`` and +``scheduler`` parameters of ``posix_spawn``. Patch by Pablo Galindo. .. @@ -1735,7 +1735,7 @@ Patch by Zackery Spytz. .. nonce: UoC319 .. section: Core and Builtins -Fix a crash in `ctypes.cast()` in case the type argument is a ctypes +Fix a crash in ``ctypes.cast()`` in case the type argument is a ctypes structured data type. Patch by Eryk Sun and Oren Milman. .. @@ -1745,7 +1745,7 @@ structured data type. Patch by Eryk Sun and Oren Milman. .. nonce: jgYsSA .. section: Core and Builtins -Fix a crash in `os.utime()` in case of a bad ns argument. Patch by Oren +Fix a crash in ``os.utime()`` in case of a bad ns argument. Patch by Oren Milman. .. @@ -2006,8 +2006,8 @@ Improved support of custom data descriptors in :func:`help` and .. nonce: V4kNN3 .. section: Library -The `crypt` module now internally uses the `crypt_r()` library function -instead of `crypt()` when available. +The ``crypt`` module now internally uses the ``crypt_r()`` library function +instead of ``crypt()`` when available. .. @@ -2025,7 +2025,7 @@ Fixed help() on metaclasses. Patch by Sanyam Khurana. .. nonce: PutiOC .. section: Library -Expose ``raise(signum)`` as `raise_signal` +Expose ``raise(signum)`` as ``raise_signal`` .. @@ -2248,7 +2248,7 @@ the pool is still running. .. nonce: abB4BN .. section: Library -When a :class:`Mock` instance was used to wrap an object, if `side_effect` +When a :class:`Mock` instance was used to wrap an object, if ``side_effect`` is used in one of the mocks of it methods, don't call the original implementation and return the result of using the side effect the same way that it is done with return_value. @@ -2377,7 +2377,7 @@ compliant, and will not throw an exception on a trailing '%'. .. nonce: vepCSJ .. section: Library -The function `platform.popen` has been removed, it was deprecated since +The function ``platform.popen`` has been removed, it was deprecated since Python 3.3: use :func:`os.popen` instead. .. @@ -2529,9 +2529,9 @@ Fix incorrect parsing of :class:`_io.IncrementalNewlineDecoder`'s .. nonce: CulMN8 .. section: Library -Remove `StreamReaderProtocol._untrack_reader`. The call to `_untrack_reader` +Remove ``StreamReaderProtocol._untrack_reader``. The call to ``_untrack_reader`` is currently performed too soon, causing the protocol to forget about the -reader before `connection_lost` can run and feed the EOF to the reader. +reader before ``connection_lost`` can run and feed the EOF to the reader. .. @@ -2583,8 +2583,8 @@ polling for new events. .. nonce: ltSrtr .. section: Library -`importlib` no longer logs `wrote ` redundantly after -`(created|could not create) ` is already logged. Patch by +``importlib`` no longer logs ``wrote `` redundantly after +``(created|could not create) `` is already logged. Patch by Quentin Agren. .. @@ -2717,7 +2717,7 @@ Use :func:`socket.CMSG_SPACE` to calculate ancillary data size instead of .. nonce: rWBb43 .. section: Library -The `mailbox.mbox.get_string` function *from_* parameter can now +The ``mailbox.mbox.get_string`` function *from_* parameter can now successfully be set to a non-default value. .. @@ -2799,7 +2799,7 @@ children which are instances of ``Element`` subclasses. .. nonce: z2FbOp .. section: Library -:class:`smtplib.SMTP` objects now always have a `sock` attribute present +:class:`smtplib.SMTP` objects now always have a ``sock`` attribute present .. @@ -2914,7 +2914,7 @@ Fix inspect module polluted ``sys.modules`` when parsing .. nonce: Wo2PoJ .. section: Library -Add `mtime` argument to `gzip.compress` for reproducible output. Patch by +Add ``mtime`` argument to ``gzip.compress`` for reproducible output. Patch by Guo Ci Teo. .. @@ -3019,7 +3019,7 @@ argument. Patch by Andrés Delfino. .. nonce: rSPBW9 .. section: Library -In :class:`QueueHandler`, clear `exc_text` from :class:`LogRecord` to +In :class:`QueueHandler`, clear ``exc_text`` from :class:`LogRecord` to prevent traceback from being written twice. .. @@ -3060,8 +3060,8 @@ and will be removed in future Python versions. .. nonce: CUE8LU .. section: Library -Add deprecation warning when `loop` is used in methods: `asyncio.sleep`, -`asyncio.wait` and `asyncio.wait_for`. +Add deprecation warning when ``loop`` is used in methods: ``asyncio.sleep``, +``asyncio.wait`` and ``asyncio.wait_for``. .. @@ -3191,7 +3191,7 @@ the stream is deleted (garbage collected) without ``close()`` call. .. nonce: 3IPIH5 .. section: Library -`Enum._missing_`: raise `ValueError` if None returned and `TypeError` if +``Enum._missing_``: raise ``ValueError`` if None returned and ``TypeError`` if non-member is returned. .. @@ -3302,8 +3302,8 @@ issues on Windows. .. nonce: xL7-kG .. section: Library -Fix possible mojibake in the error message of `pwd.getpwnam` and -`grp.getgrnam` using string representation because of invisible characters +Fix possible mojibake in the error message of ``pwd.getpwnam`` and +``grp.getgrnam`` using string representation because of invisible characters or trailing whitespaces. Patch by William Grzybowski. .. @@ -3395,8 +3395,8 @@ Zackery Spytz. .. nonce: S0Irst .. section: Library -Fix parsing non-ASCII identifiers in :mod:`lib2to3.pgen2.tokenize` (PEP -3131). +Fix parsing non-ASCII identifiers in :mod:`!lib2to3.pgen2.tokenize` +(:pep:`3131`). .. @@ -4575,8 +4575,8 @@ OpenSSL 1.1.1 .. nonce: nzQgD8 .. section: Library -Release GIL on `grp.getgrnam`, `grp.getgrgid`, `pwd.getpwnam` and -`pwd.getpwuid` if reentrant variants of these functions are available. Patch +Release GIL on ``grp.getgrnam``, ``grp.getgrgid``, ``pwd.getpwnam`` and +``pwd.getpwuid`` if reentrant variants of these functions are available. Patch by William Grzybowski. .. @@ -4656,8 +4656,8 @@ default. .. nonce: C6Hnd1 .. section: Library -Do not simplify arguments to `typing.Union`. Now `Union[Manager, Employee]` -is not simplified to `Employee` at runtime. Such simplification previously +Do not simplify arguments to ``typing.Union``. Now ``Union[Manager, Employee]`` +is not simplified to ``Employee`` at runtime. Such simplification previously caused several bugs and limited possibilities for introspection. .. @@ -4736,7 +4736,7 @@ Patch by Zvi Effron .. nonce: taxbVT .. section: Library -Fix race condition with `ReadTransport.resume_reading` in Windows proactor +Fix race condition with ``ReadTransport.resume_reading`` in Windows proactor event loop. .. @@ -4746,7 +4746,7 @@ event loop. .. nonce: pj2Mbb .. section: Library -Fix failure in `typing.get_type_hints()` when ClassVar was provided as a +Fix failure in ``typing.get_type_hints()`` when ClassVar was provided as a string forward reference. .. @@ -5044,8 +5044,8 @@ functionality. .. nonce: C_K-J9 .. section: Library -`ConfigParser.items()` was fixed so that key-value pairs passed in via -`vars` are not included in the resulting output. +``ConfigParser.items()`` was fixed so that key-value pairs passed in via +:func:`vars` are not included in the resulting output. .. @@ -5150,7 +5150,7 @@ instead of a wrapper function for exit callbacks. .. nonce: B56Hc1 .. section: Library -Fix FD leak in `_SelectorSocketTransport` Patch by Vlad Starostin. +Fix FD leak in ``_SelectorSocketTransport`` Patch by Vlad Starostin. .. @@ -5777,7 +5777,7 @@ helper methods that can be used instead ``_dump_registry``, .. nonce: bvHDOc .. section: Library -Fixed `asyncio.Condition` issue which silently ignored cancellation after +Fixed ``asyncio.Condition`` issue which silently ignored cancellation after notifying and cancelling a conditional lock. Patch by Bar Harel. .. @@ -5992,7 +5992,7 @@ Add Ttk spinbox widget to :mod:`tkinter.ttk`. Patch by Alan D Moore. .. nonce: flC-dE .. section: Library -:mod:`profile` CLI accepts `-m module_name` as an alternative to script +:mod:`profile` CLI accepts ``-m module_name`` as an alternative to script path. .. @@ -6147,8 +6147,8 @@ Support arrays >=2GiB in :mod:`ctypes`. Patch by Segev Finer. .. nonce: pDsFJl .. section: Library -Removed support of arguments in `tkinter.ttk.Treeview.selection`. It was -deprecated in 3.6. Use specialized methods like `selection_set` for +Removed support of arguments in ``tkinter.ttk.Treeview.selection``. It was +deprecated in 3.6. Use specialized methods like ``selection_set`` for changing the selection. .. @@ -6224,7 +6224,7 @@ imported from ``typing`` directly. .. nonce: 2eVOYS .. section: Documentation -Fix the documentation about an unexisting `f_restricted` attribute in the +Fix the documentation about an unexisting ``f_restricted`` attribute in the frame object. Patch by Stéphane Wirtel .. @@ -6311,7 +6311,7 @@ Document how passing coroutines to asyncio.wait() can be confusing. .. nonce: p9PoYv .. section: Documentation -Make clear that ``==`` operator sometimes is equivalent to `is`. The ``<``, +Make clear that ``==`` operator sometimes is equivalent to ``is``. The ``<``, ``<=``, ``>`` and ``>=`` operators are only defined where they make sense. .. @@ -6409,7 +6409,7 @@ Improve the documentation of :func:`asyncio.open_connection`, .. nonce: E5gba1 .. section: Documentation -Document that `asyncio.wait()` does not cancel its futures on timeout. +Document that ``asyncio.wait()`` does not cancel its futures on timeout. .. @@ -6660,7 +6660,7 @@ Stéphane Wirtel .. nonce: w1m_8r .. section: Documentation -Improve docstrings for `pathlib.PurePath` subclasses. +Improve docstrings for ``pathlib.PurePath`` subclasses. .. @@ -6997,7 +6997,7 @@ Use 3072 RSA keys and SHA-256 signature for test certs and keys. .. nonce: H8fCGa .. section: Tests -Remove special condition for AIX in `test_subprocess.test_undecodable_env` +Remove special condition for AIX in ``test_subprocess.test_undecodable_env`` .. @@ -7006,7 +7006,7 @@ Remove special condition for AIX in `test_subprocess.test_undecodable_env` .. nonce: IsRDPB .. section: Tests -Fix `test_utf8_mode.test_cmd_line` for AIX +Fix ``test_utf8_mode.test_cmd_line`` for AIX .. @@ -7034,9 +7034,9 @@ Fix ftplib test for TLS 1.3 by reading from data socket. .. nonce: g7TwYm .. section: Tests -Fix `test_socket` on AIX 6.1 and later IPv6 zone id supports only -supported by inet_pton6_zone() Switch to runtime-based platform.system() to -establish current platform rather than build-time based sys.platform() +Fix ``test_socket`` on AIX 6.1 and later IPv6 zone id supports only +supported by ``inet_pton6_zone()``. Switch to runtime-based ``platform.system()`` to +establish current platform rather than build-time based ``sys.platform()`` .. @@ -7172,8 +7172,8 @@ Fix failing ``test_asyncio`` on macOS 10.12.2+ due to transport of .. nonce: IKDsqu .. section: Tests -Making sure the `SMTPUTF8SimTests` class of tests gets run in -test_smtplib.py. +Making sure the ``SMTPUTF8SimTests`` class of tests gets run in +``test_smtplib.py``. .. @@ -7282,7 +7282,7 @@ CFLAGS to build third-party C extensions through distutils. .. nonce: XZTttb .. section: Build -Fix a compiler error when statically linking `pyexpat` in `Modules/Setup`. +Fix a compiler error when statically linking ``pyexpat`` in ``Modules/Setup``. .. @@ -7520,8 +7520,8 @@ suffixes). .. nonce: o7G_UO .. section: Build -By default, modules configured in `Modules/Setup` are no longer built with -`-DPy_BUILD_CORE`. Instead, modules that specifically need that preprocessor +By default, modules configured in ``Modules/Setup`` are no longer built with +``-DPy_BUILD_CORE``. Instead, modules that specifically need that preprocessor definition include it in their individual entries. .. @@ -8090,7 +8090,7 @@ Fix imports in idlelib.window. .. nonce: QEaANl .. section: IDLE -Proper format `calltip` when the function has no docstring. +Proper format ``calltip`` when the function has no docstring. .. diff --git a/Misc/NEWS.d/3.8.0a2.rst b/Misc/NEWS.d/3.8.0a2.rst index 223126145c77f1..c8620aeea7f133 100644 --- a/Misc/NEWS.d/3.8.0a2.rst +++ b/Misc/NEWS.d/3.8.0a2.rst @@ -4,7 +4,7 @@ .. release date: 2019-02-25 .. section: Core and Builtins -Raise a :exc:`SyntaxError` when assigning a value to `__debug__` with the +Raise a :exc:`SyntaxError` when assigning a value to ``__debug__`` with the Assignment Operator. Contributed by Stéphane Wirtel and Pablo Galindo. .. diff --git a/Misc/NEWS.d/3.8.0a3.rst b/Misc/NEWS.d/3.8.0a3.rst index 66308c6bca2e26..a06efb9abe795d 100644 --- a/Misc/NEWS.d/3.8.0a3.rst +++ b/Misc/NEWS.d/3.8.0a3.rst @@ -324,7 +324,7 @@ Raise ModuleNotFoundError in pyclbr when a module can't be found. Thanks to .. nonce: MDXLw6 .. section: Library -Switch the default format used for writing tars with mod:`tarfile` to the +Switch the default format used for writing tars with :mod:`tarfile` to the modern POSIX.1-2001 pax standard, from the vendor-specific GNU. Contributed by C.A.M. Gerlach. @@ -473,7 +473,7 @@ path string. .. nonce: NA_rXa .. section: Library -Ensure custom :func:`warnings.formatwarning` function can receive `line` as +Ensure custom :func:`warnings.formatwarning` function can receive ``line`` as positional argument. Based on patch by Tashrif Billah. .. @@ -533,10 +533,10 @@ Kumar Akshay. .. nonce: yffB3F .. section: Library -`pprint.pp` has been added to pretty-print objects with dictionary keys +``pprint.pp`` has been added to pretty-print objects with dictionary keys being sorted with their insertion order by default. Parameter *sort_dicts* -has been added to `pprint.pprint`, `pprint.pformat` and -`pprint.PrettyPrinter`. Contributed by Rémi Lapeyre. +has been added to ``pprint.pprint``, ``pprint.pformat`` and +``pprint.PrettyPrinter``. Contributed by Rémi Lapeyre. .. diff --git a/Misc/NEWS.d/3.8.0a4.rst b/Misc/NEWS.d/3.8.0a4.rst index 7e8bfa5c4364a9..fa5eb697d9202d 100644 --- a/Misc/NEWS.d/3.8.0a4.rst +++ b/Misc/NEWS.d/3.8.0a4.rst @@ -168,7 +168,7 @@ decoder. Changing ``dict`` keys during iteration of the dict itself, ``keys()``, ``values()``, or ``items()`` will now be detected in certain corner cases where keys are deleted/added so that the number of keys isn't changed. A -`RuntimeError` will be raised after ``len(dict)`` iterations. Contributed by +``RuntimeError`` will be raised after ``len(dict)`` iterations. Contributed by Thomas Perl. .. @@ -255,7 +255,7 @@ all tags in a namespace. Patch by Stefan Behnel. .. nonce: Lpm-SI .. section: Library -`pathlib.path.link_to()` is now implemented. It creates a hard link pointing +``pathlib.path.link_to()`` is now implemented. It creates a hard link pointing to a path. .. @@ -580,7 +580,7 @@ is a dictionary. .. section: Library Calling ``stop()`` on an unstarted or stopped :func:`unittest.mock.patch` -object will now return `None` instead of raising :exc:`RuntimeError`, making +object will now return ``None`` instead of raising :exc:`RuntimeError`, making the method idempotent. Patch by Karthikeyan Singaravelan. .. @@ -609,9 +609,9 @@ Add time module support and fix test_time faiures for VxWorks. .. nonce: i2Z1XR .. section: Library -Added support for keyword arguments `default_namespace` and -`xml_declaration` in functions ElementTree.tostring() and -ElementTree.tostringlist(). +Added support for keyword arguments ``default_namespace`` and +``xml_declaration`` in functions ``ElementTree.tostring()`` and +``ElementTree.tostringlist()``. .. @@ -744,7 +744,7 @@ Remove stale unix datagram socket before binding .. nonce: _4Q_bi .. section: Library -Implemented Happy Eyeballs in `asyncio.create_connection()`. Added two new +Implemented Happy Eyeballs in ``asyncio.create_connection()``. Added two new arguments, *happy_eyeballs_delay* and *interleave*, to specify Happy Eyeballs behavior. @@ -1405,7 +1405,7 @@ only present in alpha releases of Python 3.8. Patch by Paul Ganssle. .. nonce: wpbWeb .. section: C API -Modify ``PyObject_Init`` to correctly increase the refcount of heap- -allocated Type objects. Also fix the refcounts of the heap-allocated types +Modify ``PyObject_Init`` to correctly increase the refcount of heap-allocated +Type objects. Also fix the refcounts of the heap-allocated types that were either doing this manually or not decreasing the type's refcount in tp_dealloc diff --git a/Misc/NEWS.d/3.8.0b1.rst b/Misc/NEWS.d/3.8.0b1.rst index c5e27b04ef8e8b..4eb0c0451e97b5 100644 --- a/Misc/NEWS.d/3.8.0b1.rst +++ b/Misc/NEWS.d/3.8.0b1.rst @@ -146,8 +146,8 @@ constant expressions inside the f-string). .. nonce: VeVvhJ .. section: Core and Builtins -The `bytes.hex`, `bytearray.hex`, and `memoryview.hex` methods as well as -the `binascii.hexlify` and `b2a_hex` functions now have the ability to +The ``bytes.hex``, ``bytearray.hex``, and ``memoryview.hex`` methods as well as +the ``binascii.hexlify`` and ``b2a_hex`` functions now have the ability to include an optional separator between hex bytes. This functionality was inspired by MicroPython's hexlify implementation. @@ -198,8 +198,8 @@ any error. .. nonce: QwLa3P .. section: Core and Builtins -Only accept text after `# type: ignore` if the first character is ASCII. -This is to disallow things like `# type: ignoreé`. +Only accept text after ``# type: ignore`` if the first character is ASCII. +This is to disallow things like ``# type: ignoreé``. .. @@ -208,9 +208,9 @@ This is to disallow things like `# type: ignoreé`. .. nonce: EFRHZ3 .. section: Core and Builtins -Store text appearing after a `# type: ignore` comment in the AST. For -example a type ignore like `# type: ignore[E1000]` will have the string -`"[E1000]"` stored in its AST node. +Store text appearing after a ``# type: ignore`` comment in the AST. For +example a type ignore like ``# type: ignore[E1000]`` will have the string +``"[E1000]"`` stored in its AST node. .. @@ -414,7 +414,7 @@ Fix incorrect use of ``%p`` in format strings. Patch by Zackery Spytz. .. nonce: RO20OV .. section: Core and Builtins -builtins.help() now prefixes `async` for async functions +``builtins.help()`` now prefixes ``async`` for async functions. .. @@ -443,7 +443,7 @@ Added fix for broken symlinks in combination with pathlib .. section: Core and Builtins Added new trashcan macros to deal with a double deallocation that could -occur when the `tp_dealloc` of a subclass calls the `tp_dealloc` of a base +occur when the ``tp_dealloc`` of a subclass calls the ``tp_dealloc`` of a base class and that base class uses the trashcan mechanism. Patch by Jeroen Demeyer. @@ -768,7 +768,7 @@ Taskaya .. nonce: u7cxu7 .. section: Library -PDB command `args` now display positional only arguments. Patch contributed +PDB command ``args`` now display positional only arguments. Patch contributed by Rémi Lapeyre. .. @@ -778,7 +778,7 @@ by Rémi Lapeyre. .. nonce: JkZORP .. section: Library -PDB command `args` now display keyword only arguments. Patch contributed by +PDB command ``args`` now display keyword only arguments. Patch contributed by Rémi Lapeyre. .. @@ -799,7 +799,7 @@ Add missing names to ``typing.__all__``: ``ChainMap``, ``ForwardRef``, .. section: Library Add SupportsIndex protocol to the typing module to allow type checking to -detect classes that can be passed to `hex()`, `oct()` and `bin()`. +detect classes that can be passed to ``hex()``, ``oct()`` and ``bin()``. .. @@ -1023,10 +1023,10 @@ reading metadata from third-party packages. .. nonce: iigeqk .. section: Library -When using `type_comments=True` in `ast.parse`, treat `# type: ignore` +When using ``type_comments=True`` in ``ast.parse``, treat ``# type: ignore`` followed by a non-alphanumeric character and then arbitrary text as a type ignore, instead of requiring nothing but whitespace or another comment. This -is to permit formations such as `# type: ignore[E1000]`. +is to permit formations such as ``# type: ignore[E1000]``. .. @@ -1066,7 +1066,7 @@ exposed to the user. Patch by Aviv Palivoda. .. nonce: WK8Y-k .. section: Library -In `shutil.copystat()`, first copy extended file attributes and then file +In ``shutil.copystat()``, first copy extended file attributes and then file permissions, since extended attributes can only be set on the destination while it is still writeable. @@ -1120,7 +1120,7 @@ add_done_callback correctly when the Future has already completed. .. nonce: 4payXb .. section: Library -Limit `max_workers` in `ProcessPoolExecutor` to 61 to work around a +Limit ``max_workers`` in ``ProcessPoolExecutor`` to 61 to work around a WaitForMultipleObjects limitation. .. @@ -1676,7 +1676,7 @@ documentation covers it independently. .. nonce: 6hg6J8 .. section: Documentation -Add detail to the documentation on the `pty.spawn` function. +Add detail to the documentation on the ``pty.spawn`` function. .. @@ -1704,7 +1704,7 @@ Added the context variable in glossary. .. nonce: Q7s2FB .. section: Documentation -Clarify that `copy()` is not part of the `MutableSequence` ABC. +Clarify that ``copy()`` is not part of the ``MutableSequence`` ABC. .. @@ -1713,7 +1713,7 @@ Clarify that `copy()` is not part of the `MutableSequence` ABC. .. nonce: jalAaQ .. section: Documentation -Make `codecs.StreamRecoder.writelines` take a list of bytes. +Make ``codecs.StreamRecoder.writelines`` take a list of bytes. .. diff --git a/Misc/NEWS.d/3.9.0a1.rst b/Misc/NEWS.d/3.9.0a1.rst index b365e5fbfb0dbe..66d7fc1f32e705 100644 --- a/Misc/NEWS.d/3.9.0a1.rst +++ b/Misc/NEWS.d/3.9.0a1.rst @@ -1304,7 +1304,7 @@ parameter. Patch by Pablo Galindo. .. nonce: J12cWT .. section: Library -Fixed `hmac.new` and `hmac.HMAC` to raise TypeError instead of ValueError +Fixed ``hmac.new`` and ``hmac.HMAC`` to raise TypeError instead of ValueError when the digestmod parameter, now required in 3.8, is omitted. Also clarified the hmac module documentation and docstrings. @@ -1478,7 +1478,7 @@ Removes _AwaitEvent from AsyncMock. .. section: Library Allow the rare code that wants to send invalid http requests from the -`http.client` library a way to do so. The fixes for bpo-30458 led to +``http.client`` library a way to do so. The fixes for bpo-30458 led to breakage for some projects that were relying on this ability to test their own behavior in the face of bad requests. @@ -1813,9 +1813,9 @@ Update importlib.metadata with changes from `importlib_metadata 0.21 .. nonce: 8zn2o3 .. section: Library -Remove `__code__` check in AsyncMock that incorrectly evaluated function -specs as async objects but failed to evaluate classes with `__await__` but -no `__code__` attribute defined as async objects. +Remove ``__code__`` check in AsyncMock that incorrectly evaluated function +specs as async objects but failed to evaluate classes with ``__await__`` but +no ``__code__`` attribute defined as async objects. .. @@ -1976,11 +1976,11 @@ thread at a time. .. nonce: kP-n4L .. section: Library -Subscripts to the `unittest.mock.call` objects now receive the same chaining +Subscripts to the ``unittest.mock.call`` objects now receive the same chaining mechanism as any other custom attributes, so that the following usage no -longer raises a `TypeError`: +longer raises a ``TypeError``: -call().foo().__getitem__('bar') +``call().foo().__getitem__('bar')`` Patch by blhsing @@ -2210,7 +2210,7 @@ Add C fastpath for statistics.NormalDist.inv_cdf() Patch by Donghee Na .. nonce: Ene6L- .. section: Library -Remove the deprecated method `threading.Thread.isAlive()`. Patch by Donghee +Remove the deprecated method ``threading.Thread.isAlive()``. Patch by Donghee Na. .. @@ -2260,8 +2260,8 @@ directories are added and that duplicates are excluded. .. nonce: xfvdb_ .. section: Library -Renamed and documented `test.bytecode_helper` as -`test.support.bytecode_helper`. Patch by Joannah Nanjekye. +Renamed and documented ``test.bytecode_helper`` as +``test.support.bytecode_helper``. Patch by Joannah Nanjekye. .. @@ -2289,8 +2289,8 @@ many small lines are passed. Patch by Sergey Fedoseev. .. nonce: ycbL2z .. section: Library -`ensurepip` now uses `importlib.resources.read_binary()` to read data -instead of `pkgutil.get_data()`. Patch by Joannah Nanjekye. +``ensurepip`` now uses ``importlib.resources.read_binary()`` to read data +instead of ``pkgutil.get_data()``. Patch by Joannah Nanjekye. .. @@ -2524,8 +2524,8 @@ Make internal attributes for statistics.NormalDist() private. .. nonce: S5am28 .. section: Library -Fix `NonCallableMock._call_matcher` returning tuple instead of `_Call` -object when `self._spec_signature` exists. Patch by Elizabeth Uselton +Fix ``NonCallableMock._call_matcher`` returning tuple instead of ``_Call`` +object when ``self._spec_signature`` exists. Patch by Elizabeth Uselton .. @@ -2673,7 +2673,7 @@ which had a too large value in some situations. .. nonce: 0i1MR- .. section: Library -Fixes a possible hang when using a timeout on `subprocess.run()` while +Fixes a possible hang when using a timeout on ``subprocess.run()`` while capturing output. If the child process spawned its own children or otherwise connected its stdout or stderr handles with another process, we could hang after the timeout was reached and our child was killed when @@ -2708,8 +2708,8 @@ The distutils ``bdist_wininst`` command is deprecated in Python 3.8, use .. nonce: O53a5S .. section: Library -When `Enum.__str__` is overridden in a derived class, the override will be -used by `Enum.__format__` regardless of whether mixin classes are present. +When ``Enum.__str__`` is overridden in a derived class, the override will be +used by ``Enum.__format__`` regardless of whether mixin classes are present. .. @@ -2857,8 +2857,8 @@ Patch by Justin Blanchard. Add formal support for UDPLITE sockets. Support was present before, but it is now easier to detect support with ``hasattr(socket, 'IPPROTO_UDPLITE')`` and there are constants defined for each of the values needed: -:py:obj:`socket.IPPROTO_UDPLITE`, :py:obj:`UDPLITE_SEND_CSCOV`, and -:py:obj:`UDPLITE_RECV_CSCOV`. Patch by Gabe Appleton. +``socket.IPPROTO_UDPLITE``, ``UDPLITE_SEND_CSCOV``, and +``UDPLITE_RECV_CSCOV``. Patch by Gabe Appleton. .. @@ -3195,9 +3195,9 @@ Remove ``Enum._convert`` method, deprecated in 3.8. .. nonce: TTzHxj .. section: Library -`argparse._ActionsContainer.add_argument` now throws error, if someone +``argparse._ActionsContainer.add_argument`` now throws error, if someone accidentally pass FileType class object instead of instance of FileType as -`type` argument +``type`` argument. .. @@ -3458,7 +3458,7 @@ Patch contributed by Rémi Lapeyre. .. section: Library Fixes a bug in :mod:`!cgi` module when a multipart/form-data request has no -`Content-Length` header. +``Content-Length`` header. .. @@ -3566,8 +3566,8 @@ source argument. Patch by Emily Morehouse and Maxwell "5.13b" McKinnon. .. nonce: 0stF0u .. section: Library -Added __format__ to IPv4 and IPv6 classes. Always outputs a fully zero- -padded string. Supports b/x/n modifiers (bin/hex/native format). Native +Added ``__format__`` to IPv4 and IPv6 classes. Always outputs a fully +zero-padded string. Supports b/x/n modifiers (bin/hex/native format). Native format for IPv4 is bin, native format for IPv6 is hex. Also supports '#' and '_' modifiers. @@ -3789,7 +3789,7 @@ verify the maildir folder layout correctness. Patch by Sviatoslav Sydorenko. .. nonce: 7tiFR- .. section: Documentation -Fix `importlib` examples to insert any newly created modules via +Fix ``importlib`` examples to insert any newly created modules via importlib.util.module_from_spec() immediately into sys.modules instead of after calling loader.exec_module(). @@ -3888,7 +3888,7 @@ Make C-API docs clear about what the "main" interpreter is. .. nonce: Iqiqtm .. section: Documentation -The documentation for decimal string formatting using the `:g` specifier has +The documentation for decimal string formatting using the ``:g`` specifier has been updated to reflect the correct exponential notation cutoff point. Original patch contributed by Tuomas Suutari. @@ -4450,8 +4450,8 @@ Fix _hashlib build when Blake2 is disabled, but OpenSSL supports it. .. nonce: buCO84 .. section: Build -Misc/python-config.in now uses `getvar()` for all still existing -`sysconfig.get_config_var()` calls. Patch by Joannah Nanjekye. +Misc/python-config.in now uses ``getvar()`` for all still existing +``sysconfig.get_config_var()`` calls. Patch by Joannah Nanjekye. .. @@ -5715,8 +5715,8 @@ The :c:macro:`METH_FASTCALL` calling convention has been documented. .. nonce: 4tClQT .. section: C API -The new function :c:func:`PyCode_NewWithPosOnlyArgs` allows to create code -objects like :c:func:`PyCode_New`, but with an extra *posonlyargcount* +The new function :c:func:`!PyCode_NewWithPosOnlyArgs` allows to create code +objects like :c:func:`!PyCode_New`, but with an extra *posonlyargcount* parameter for indicating the number of positonal-only arguments. .. diff --git a/Misc/NEWS.d/3.9.0a2.rst b/Misc/NEWS.d/3.9.0a2.rst index a03eb10f1d523a..7d878cfe227552 100644 --- a/Misc/NEWS.d/3.9.0a2.rst +++ b/Misc/NEWS.d/3.9.0a2.rst @@ -403,7 +403,7 @@ locale encoding is not UTF-8. Prevent UnboundLocalError to pop up in parse_message_id. parse_message_id() was improperly using a token defined inside an exception -handler, which was raising `UnboundLocalError` on parsing an invalid value. +handler, which was raising ``UnboundLocalError`` on parsing an invalid value. Patch by Claudiu Popa. .. @@ -444,12 +444,12 @@ random.choices() now raises a ValueError when all the weights are zero. Raise pickle.UnpicklingError when loading an item from memo for invalid input. -The previous code was raising a `KeyError` for both the Python and C +The previous code was raising a ``KeyError`` for both the Python and C implementation. This was caused by the specified index of an invalid input which did not exist in the memo structure, where the pickle stores what -objects it has seen. The malformed input would have caused either a `BINGET` -or `LONG_BINGET` load from the memo, leading to a `KeyError` as the -determined index was bogus. Patch by Claudiu Popa +objects it has seen. The malformed input would have caused either a ``BINGET`` +or ``LONG_BINGET`` load from the memo, leading to a ``KeyError`` as the +determined index was bogus. Patch by Claudiu Popa. .. @@ -458,7 +458,7 @@ determined index was bogus. Patch by Claudiu Popa .. nonce: iKx23z .. section: Library -Calling func:`shutil.copytree` to copy a directory tree from one directory +Calling func:``shutil.copytree`` to copy a directory tree from one directory to another subdirectory resulted in an endless loop and a RecursionError. A fix was added to consume an iterator and create the list of the entries to be copied, avoiding the recursion for newly created directories. Patch by @@ -685,7 +685,7 @@ added. .. section: Documentation Update documentation to state that to activate virtual environments under -fish one should use `source`, not `.` as documented at +fish one should use ``source``, not ``.`` as documented at https://fishshell.com/docs/current/cmds/source.html. .. @@ -884,7 +884,7 @@ Add support for building and releasing Windows ARM64 packages. Fixed a crash on OSX dynamic builds that occurred when re-initializing the posix module after a Py_Finalize if the environment had changed since the -previous `import posix`. Patch by Benoît Hudson. +previous ``import posix``. Patch by Benoît Hudson. .. diff --git a/Misc/NEWS.d/3.9.0a3.rst b/Misc/NEWS.d/3.9.0a3.rst index 8a94848427382b..78194791ab5277 100644 --- a/Misc/NEWS.d/3.9.0a3.rst +++ b/Misc/NEWS.d/3.9.0a3.rst @@ -266,8 +266,8 @@ The :func:`os.unsetenv` function is now also available on Windows. .. nonce: D2tSXk .. section: Library -Fixed a regression with the `ignore` callback of :func:`shutil.copytree`. -The argument types are now str and List[str] again. +Fixed a regression with the ``ignore`` callback of :func:`shutil.copytree`. +The argument types are now ``str`` and ``List[str]`` again. .. @@ -454,7 +454,7 @@ resilients to inaccessible sys.path entries (importlib_metadata v1.4.0). .. nonce: _S5VjC .. section: Library -:class:`~!nntplib.NNTP` and :class:`~!nntplib.NNTP_SSL` now raise a +:class:`!NNTP` and :class:`!NNTP_SSL` now raise a :class:`ValueError` if the given timeout for their constructor is zero to prevent the creation of a non-blocking socket. Patch by Donghee Na. @@ -498,7 +498,7 @@ prevent the creation of a non-blocking socket. Patch by Donghee Na. .. section: Library Updated the Gmane domain from news.gmane.org to news.gmane.io which is used -for examples of :class:`~!nntplib.NNTP` news reader server and nntplib tests. +for examples of :class:`!NNTP` news reader server and nntplib tests. .. @@ -507,8 +507,8 @@ for examples of :class:`~!nntplib.NNTP` news reader server and nntplib tests. .. nonce: ihRT1z .. section: Library -Proxy the `SimpleHTTPRequestHandler.guess_type` to `mimetypes.guess_type` so -the `mimetypes.init` is called lazily to avoid unnecessary costs when +Proxy the ``SimpleHTTPRequestHandler.guess_type`` to ``mimetypes.guess_type`` so +the ``mimetypes.init`` is called lazily to avoid unnecessary costs when :mod:`http.server` module is imported. .. @@ -547,10 +547,10 @@ all options. Giovanni Lombardo contributed part of the patch. .. nonce: nzwGyG .. section: Library -If an exception were to be thrown in `Logger.isEnabledFor` (say, by asyncio -timeouts or stopit) , the `logging` global lock may not be released +If an exception were to be thrown in ``Logger.isEnabledFor`` (say, by asyncio +timeouts or stopit) , the ``logging`` global lock may not be released appropriately, resulting in deadlock. This change wraps that block of code -with `try...finally` to ensure the lock is released. +with ``try...finally`` to ensure the lock is released. .. @@ -571,7 +571,7 @@ new task spawning before exception raising. .. section: Library Correctly parenthesize filter-based statements that contain lambda -expressions in mod:`!lib2to3`. Patch by Donghee Na. +expressions in :mod:`!lib2to3`. Patch by Donghee Na. .. @@ -732,9 +732,9 @@ different process. .. nonce: beZ0Sk .. section: Library -Removes trailing space in formatted currency with `international=True` and a -locale with symbol following value. E.g. `locale.currency(12.34, -international=True)` returned `'12,34 EUR '` instead of `'12,34 EUR'`. +Removes trailing space in formatted currency with ``international=True`` and a +locale with symbol following value. E.g. ``locale.currency(12.34, +international=True)`` returned ``'12,34 EUR '`` instead of ``'12,34 EUR'``. .. @@ -835,7 +835,7 @@ functions are now required to build Python. .. nonce: aBmj13 .. section: Build -Updated the documentation in `./configure --help` to show default values, +Updated the documentation in ``./configure --help`` to show default values, reference documentation where required and add additional explanation where needed. diff --git a/Misc/NEWS.d/3.9.0a4.rst b/Misc/NEWS.d/3.9.0a4.rst index e59435b5509acf..ca0eb2abf1d654 100644 --- a/Misc/NEWS.d/3.9.0a4.rst +++ b/Misc/NEWS.d/3.9.0a4.rst @@ -4,8 +4,8 @@ .. release date: 2020-02-25 .. section: Security -Add audit events to functions in `fcntl`, `msvcrt`, `os`, `resource`, -`shutil`, `signal` and `syslog`. +Add audit events to functions in ``fcntl``, ``msvcrt``, ``os``, ``resource``, +``shutil``, ``signal`` and ``syslog``. .. @@ -81,9 +81,9 @@ Fix regression caused by fix for bpo-39386, that prevented calling .. nonce: itNmC0 .. section: Core and Builtins -Change the ending column offset of `Attribute` nodes constructed in -`ast_for_dotted_name` to point at the end of the current node and not at the -end of the last `NAME` node. +Change the ending column offset of ``Attribute`` nodes constructed in +``ast_for_dotted_name`` to point at the end of the current node and not at the +end of the last ``NAME`` node. .. @@ -269,7 +269,7 @@ codec. .. nonce: qiubSp .. section: Library -Remove obsolete check for `__args__` in bdb.Bdb.format_stack_entry. +Remove obsolete check for ``__args__`` in ``bdb.Bdb.format_stack_entry``. .. @@ -599,7 +599,7 @@ default return values. Patch by Karthikeyan Singaravelan. .. nonce: udRSWE .. section: Library -`inspect.Signature.parameters` and `inspect.BoundArguments.arguments` are +``inspect.Signature.parameters`` and ``inspect.BoundArguments.arguments`` are now dicts instead of OrderedDicts. Patch contributed by Rémi Lapeyre. .. @@ -619,9 +619,9 @@ multiprocessing.Process. .. nonce: e0C5dF .. section: Library -* Add `lazycache` function to `__all__`. -* Use `dict.clear` to clear the cache. -* Refactoring `getline` function and `checkcache` function. +* Add ``lazycache`` function to ``__all__``. +* Use ``dict.clear`` to clear the cache. +* Refactoring ``getline`` function and ``checkcache`` function. .. diff --git a/Misc/NEWS.d/3.9.0a5.rst b/Misc/NEWS.d/3.9.0a5.rst index 6ff05788214723..f0015ac54df307 100644 --- a/Misc/NEWS.d/3.9.0a5.rst +++ b/Misc/NEWS.d/3.9.0a5.rst @@ -691,7 +691,7 @@ Fix AttributeError when calling get_stack on a PyAsyncGenObject Task .. section: Library The :func:`compileall.compile_dir` function's *ddir* parameter and the -compileall command line flag `-d` no longer write the wrong pathname to the +compileall command line flag ``-d`` no longer write the wrong pathname to the generated pyc file for submodules beneath the root of the directory tree being compiled. This fixes a regression introduced with Python 3.5. @@ -1122,7 +1122,7 @@ a different condition than the GIL. .. nonce: Nbl7lF .. section: Tools/Demos -Added support to fix ``getproxies`` in the :mod:`lib2to3.fixes.fix_urllib` +Added support to fix ``getproxies`` in the :mod:`!lib2to3.fixes.fix_urllib` module. Patch by José Roberto Meza Cabrera. .. diff --git a/Misc/NEWS.d/3.9.0a6.rst b/Misc/NEWS.d/3.9.0a6.rst index 519c7f833ebcb8..366a260172efb8 100644 --- a/Misc/NEWS.d/3.9.0a6.rst +++ b/Misc/NEWS.d/3.9.0a6.rst @@ -59,8 +59,8 @@ anything that depends on it. .. section: Core and Builtins Fix the tokenizer to display the correct error message, when there is a -SyntaxError on the last input character and no newline follows. It used to -be `unexpected EOF while parsing`, while it should be `invalid syntax`. +``SyntaxError`` on the last input character and no newline follows. It used to +be ``unexpected EOF while parsing``, while it should be ``invalid syntax``. .. @@ -79,7 +79,7 @@ evaluation for annotations activated. Patch by Batuhan Taskaya. .. nonce: vXPze5 .. section: Core and Builtins -Report a specialized error message, `invalid string prefix`, when the +Report a specialized error message, ``invalid string prefix``, when the tokenizer encounters a string with an invalid prefix. .. @@ -434,7 +434,7 @@ for the correspondent concrete type (``list`` in this case). .. nonce: ux8FUr .. section: Library -func:`inspect.getdoc` no longer returns docstring inherited from the type of +:func:`inspect.getdoc` no longer returns docstring inherited from the type of the object or from parent class if it is a class if it is not defined in the object itself. In :mod:`pydoc` the documentation string is now shown not only for class, function, method etc, but for any object that has its own @@ -504,7 +504,7 @@ extension. This allows the use of functions such as ``json_object``. .. nonce: 4EcyIN .. section: Library -Wait in `KqueueSelector.select` when no fds are registered +Wait in ``KqueueSelector.select`` when no fds are registered .. @@ -701,7 +701,7 @@ per header: use the realm of the first Basic challenge. .. section: Library Removed daemon threads from :mod:`concurrent.futures` by adding an internal -`threading._register_atexit()`, which calls registered functions prior to +``threading._register_atexit()``, which calls registered functions prior to joining all non-daemon threads. This allows for compatibility with subinterpreters, which don't support daemon threads. @@ -781,9 +781,9 @@ Added :pep:`584` operators to :class:`weakref.WeakKeyDictionary`. .. nonce: 56Yokh .. section: Library -Fix linear runtime behaviour of the `__getitem__` and `__setitem__` methods +Fix linear runtime behaviour of the ``__getitem__`` and ``__setitem__`` methods in :class:`multiprocessing.shared_memory.ShareableList`. This avoids -quadratic performance when iterating a `ShareableList`. Patch by Thomas +quadratic performance when iterating a ``ShareableList``. Patch by Thomas Krennwallner. .. @@ -793,9 +793,9 @@ Krennwallner. .. nonce: AxXZNz .. section: Library -Remove undocumented support for *closing* a `pathlib.Path` object via its +Remove undocumented support for *closing* a ``pathlib.Path`` object via its context manager. The context manager magic methods remain, but they are now -a no-op, making `Path` objects immutable. +a no-op, making ``Path`` objects immutable. .. diff --git a/Misc/NEWS.d/3.9.0b1.rst b/Misc/NEWS.d/3.9.0b1.rst index ee87315ad334e9..40fb8474bf9364 100644 --- a/Misc/NEWS.d/3.9.0b1.rst +++ b/Misc/NEWS.d/3.9.0b1.rst @@ -112,8 +112,8 @@ Port :mod:`syslog` to multiphase initialization (:pep:`489`). Reporting a specialised error message for invalid string prefixes, which was introduced in :issue:`40246`, is being reverted due to backwards compatibility concerns for strings that immediately follow a reserved -keyword without whitespace between them. Constructs like `bg="#d00" if clear -else"#fca"` were failing to parse, which is not an acceptable breakage on +keyword without whitespace between them. Constructs like ``bg="#d00" if clear +else"#fca"`` were failing to parse, which is not an acceptable breakage on such short notice. .. @@ -684,7 +684,7 @@ The ``isocalendar()`` methods of :class:`datetime.date` and .. nonce: t6kW_1 .. section: Documentation -Add version of removal for explicit passing of coros to `asyncio.wait()`'s +Add version of removal for explicit passing of coros to ``asyncio.wait()``'s documentation .. diff --git a/Misc/NEWS.d/next/Build/2023-11-27-13-55-47.gh-issue-103065.o72OiA.rst b/Misc/NEWS.d/next/Build/2023-11-27-13-55-47.gh-issue-103065.o72OiA.rst deleted file mode 100644 index e2240b7c656a2f..00000000000000 --- a/Misc/NEWS.d/next/Build/2023-11-27-13-55-47.gh-issue-103065.o72OiA.rst +++ /dev/null @@ -1 +0,0 @@ -Introduce ``Tools/wasm/wasi.py`` to simplify doing a WASI build. diff --git a/Misc/NEWS.d/next/Build/2023-12-08-11-33-37.gh-issue-112867.ZzDfXQ.rst b/Misc/NEWS.d/next/Build/2023-12-08-11-33-37.gh-issue-112867.ZzDfXQ.rst deleted file mode 100644 index a36814854882bb..00000000000000 --- a/Misc/NEWS.d/next/Build/2023-12-08-11-33-37.gh-issue-112867.ZzDfXQ.rst +++ /dev/null @@ -1 +0,0 @@ -Fix the build for the case that WITH_PYMALLOC_RADIX_TREE=0 set. diff --git a/Misc/NEWS.d/next/Build/2023-12-21-05-35-06.gh-issue-112305.VfqQPx.rst b/Misc/NEWS.d/next/Build/2023-12-21-05-35-06.gh-issue-112305.VfqQPx.rst deleted file mode 100644 index 2df3207f4e6f6c..00000000000000 --- a/Misc/NEWS.d/next/Build/2023-12-21-05-35-06.gh-issue-112305.VfqQPx.rst +++ /dev/null @@ -1,3 +0,0 @@ -Fixed the ``check-clean-src`` step performed on out of tree builds to detect -errant ``$(srcdir)/Python/frozen_modules/*.h`` files and recommend -appropriate source tree cleanup steps to get a working build again. diff --git a/Misc/NEWS.d/next/Build/2024-02-01-20-08-11.gh-issue-114875.x_2iZ9.rst b/Misc/NEWS.d/next/Build/2024-02-01-20-08-11.gh-issue-114875.x_2iZ9.rst new file mode 100644 index 00000000000000..20e9d6376b973c --- /dev/null +++ b/Misc/NEWS.d/next/Build/2024-02-01-20-08-11.gh-issue-114875.x_2iZ9.rst @@ -0,0 +1 @@ +Add :c:func:`!getgrent` as a prerequisite for building the :mod:`grp` module. diff --git a/Misc/NEWS.d/next/Build/2024-02-08-17-38-56.gh-issue-113632.y9KIGb.rst b/Misc/NEWS.d/next/Build/2024-02-08-17-38-56.gh-issue-113632.y9KIGb.rst new file mode 100644 index 00000000000000..8b02b1b2cd08c9 --- /dev/null +++ b/Misc/NEWS.d/next/Build/2024-02-08-17-38-56.gh-issue-113632.y9KIGb.rst @@ -0,0 +1,2 @@ +Promote WASI to a tier 2 platform and drop Emscripten from tier 3 in +configure.ac. diff --git a/Misc/NEWS.d/next/Build/2024-02-08-19-36-20.gh-issue-115167.LB9nDK.rst b/Misc/NEWS.d/next/Build/2024-02-08-19-36-20.gh-issue-115167.LB9nDK.rst new file mode 100644 index 00000000000000..c60c4a93fe8906 --- /dev/null +++ b/Misc/NEWS.d/next/Build/2024-02-08-19-36-20.gh-issue-115167.LB9nDK.rst @@ -0,0 +1 @@ +Avoid vendoring ``vcruntime140_threads.dll`` when building with Visual Studio 2022 version 17.8. diff --git a/Misc/NEWS.d/next/C API/2023-06-21-11-53-09.gh-issue-65210.PhFRBJ.rst b/Misc/NEWS.d/next/C API/2023-06-21-11-53-09.gh-issue-65210.PhFRBJ.rst deleted file mode 100644 index a15646f4dad127..00000000000000 --- a/Misc/NEWS.d/next/C API/2023-06-21-11-53-09.gh-issue-65210.PhFRBJ.rst +++ /dev/null @@ -1,3 +0,0 @@ -Change the declaration of the *keywords* parameter of -:c:func:`PyArg_ParseTupleAndKeywords` and -:c:func:`PyArg_VaParseTupleAndKeywords` for better compatibility with C++. diff --git a/Misc/NEWS.d/next/C API/2023-11-15-01-26-59.gh-issue-111545.iAoFtA.rst b/Misc/NEWS.d/next/C API/2023-11-15-01-26-59.gh-issue-111545.iAoFtA.rst deleted file mode 100644 index 7bde2498acf999..00000000000000 --- a/Misc/NEWS.d/next/C API/2023-11-15-01-26-59.gh-issue-111545.iAoFtA.rst +++ /dev/null @@ -1,2 +0,0 @@ -Add :c:func:`Py_HashPointer` function to hash a pointer. Patch by Victor -Stinner. diff --git a/Misc/NEWS.d/next/C API/2023-11-15-13-47-48.gh-issue-112066.22WsqR.rst b/Misc/NEWS.d/next/C API/2023-11-15-13-47-48.gh-issue-112066.22WsqR.rst new file mode 100644 index 00000000000000..ae2b8b2444de97 --- /dev/null +++ b/Misc/NEWS.d/next/C API/2023-11-15-13-47-48.gh-issue-112066.22WsqR.rst @@ -0,0 +1,5 @@ +Add :c:func:`PyDict_SetDefaultRef`: insert a key and value into a dictionary +if the key is not already present. This is similar to +:meth:`dict.setdefault`, but returns an integer value indicating if the key +was already present. It is also similar to :c:func:`PyDict_SetDefault`, but +returns a strong reference instead of a borrowed reference. diff --git a/Misc/NEWS.d/next/C API/2023-11-27-09-44-16.gh-issue-112438.GdNZiI.rst b/Misc/NEWS.d/next/C API/2023-11-27-09-44-16.gh-issue-112438.GdNZiI.rst deleted file mode 100644 index 113119efd6aebb..00000000000000 --- a/Misc/NEWS.d/next/C API/2023-11-27-09-44-16.gh-issue-112438.GdNZiI.rst +++ /dev/null @@ -1,2 +0,0 @@ -Fix support of format units "es", "et", "es#", and "et#" in nested tuples in -:c:func:`PyArg_ParseTuple`-like functions. diff --git a/Misc/NEWS.d/next/C API/2023-12-02-02-08-11.gh-issue-106560.THvuji.rst b/Misc/NEWS.d/next/C API/2023-12-02-02-08-11.gh-issue-106560.THvuji.rst deleted file mode 100644 index 59b461ec47ad64..00000000000000 --- a/Misc/NEWS.d/next/C API/2023-12-02-02-08-11.gh-issue-106560.THvuji.rst +++ /dev/null @@ -1,2 +0,0 @@ -Fix redundant declarations in the public C API. Declare PyBool_Type, -PyLong_Type and PySys_Audit() only once. Patch by Victor Stinner. diff --git a/Misc/NEWS.d/next/C API/2024-01-23-21-45-02.gh-issue-114329.YRaBoe.rst b/Misc/NEWS.d/next/C API/2024-01-23-21-45-02.gh-issue-114329.YRaBoe.rst new file mode 100644 index 00000000000000..62d4ce0cfb8de5 --- /dev/null +++ b/Misc/NEWS.d/next/C API/2024-01-23-21-45-02.gh-issue-114329.YRaBoe.rst @@ -0,0 +1,3 @@ +Add :c:func:`PyList_GetItemRef`, which is similar to +:c:func:`PyList_GetItem` but returns a :term:`strong reference` instead of a +:term:`borrowed reference`. diff --git a/Misc/NEWS.d/next/C API/2024-01-29-12-13-24.gh-issue-114685.B07RME.rst b/Misc/NEWS.d/next/C API/2024-01-29-12-13-24.gh-issue-114685.B07RME.rst new file mode 100644 index 00000000000000..55b02d1d8e1e9f --- /dev/null +++ b/Misc/NEWS.d/next/C API/2024-01-29-12-13-24.gh-issue-114685.B07RME.rst @@ -0,0 +1,3 @@ +:c:func:`PyObject_GetBuffer` now raises a :exc:`SystemError` if called with +:c:macro:`PyBUF_READ` or :c:macro:`PyBUF_WRITE` as flags. These flags should +only be used with the ``PyMemoryView_*`` C API. diff --git a/Misc/NEWS.d/next/C API/2024-01-31-15-43-35.gh-issue-114685.n7aRmX.rst b/Misc/NEWS.d/next/C API/2024-01-31-15-43-35.gh-issue-114685.n7aRmX.rst new file mode 100644 index 00000000000000..76ff00645fe57d --- /dev/null +++ b/Misc/NEWS.d/next/C API/2024-01-31-15-43-35.gh-issue-114685.n7aRmX.rst @@ -0,0 +1,3 @@ +:c:func:`PyBuffer_FillInfo` now raises a :exc:`SystemError` if called with +:c:macro:`PyBUF_READ` or :c:macro:`PyBUF_WRITE` as flags. These flags should +only be used with the ``PyMemoryView_*`` C API. diff --git a/Misc/NEWS.d/next/Core and Builtins/2018-08-13-13-25-15.bpo-34392.9kIlMF.rst b/Misc/NEWS.d/next/Core and Builtins/2018-08-13-13-25-15.bpo-34392.9kIlMF.rst deleted file mode 100644 index bc4fd1ad1f5c7c..00000000000000 --- a/Misc/NEWS.d/next/Core and Builtins/2018-08-13-13-25-15.bpo-34392.9kIlMF.rst +++ /dev/null @@ -1 +0,0 @@ -Added :func:`sys._is_interned`. diff --git a/Misc/NEWS.d/next/Core and Builtins/2022-07-07-05-37-53.gh-issue-94606.hojJ54.rst b/Misc/NEWS.d/next/Core and Builtins/2022-07-07-05-37-53.gh-issue-94606.hojJ54.rst deleted file mode 100644 index 5201ab7d842088..00000000000000 --- a/Misc/NEWS.d/next/Core and Builtins/2022-07-07-05-37-53.gh-issue-94606.hojJ54.rst +++ /dev/null @@ -1,3 +0,0 @@ -Fix UnicodeEncodeError when :func:`email.message.get_payload` reads a message -with a Unicode surrogate character and the message content is not well-formed for -surrogateescape encoding. Patch by Sidney Markowitz. diff --git a/Misc/NEWS.d/next/Core and Builtins/2023-05-16-06-52-34.gh-issue-104530.mJnA0W.rst b/Misc/NEWS.d/next/Core and Builtins/2023-05-16-06-52-34.gh-issue-104530.mJnA0W.rst new file mode 100644 index 00000000000000..8643a25ae51b13 --- /dev/null +++ b/Misc/NEWS.d/next/Core and Builtins/2023-05-16-06-52-34.gh-issue-104530.mJnA0W.rst @@ -0,0 +1 @@ +Use native Win32 condition variables. diff --git a/Misc/NEWS.d/next/Core and Builtins/2023-06-06-19-09-00.gh-issue-55664.vYYl0V.rst b/Misc/NEWS.d/next/Core and Builtins/2023-06-06-19-09-00.gh-issue-55664.vYYl0V.rst new file mode 100644 index 00000000000000..438be985496650 --- /dev/null +++ b/Misc/NEWS.d/next/Core and Builtins/2023-06-06-19-09-00.gh-issue-55664.vYYl0V.rst @@ -0,0 +1 @@ +Add warning when creating :class:`type` using a namespace dictionary with non-string keys. Patched by Daniel Urban and Furkan Onder. diff --git a/Misc/NEWS.d/next/Core and Builtins/2023-11-22-13-17-54.gh-issue-112320.EddM51.rst b/Misc/NEWS.d/next/Core and Builtins/2023-11-22-13-17-54.gh-issue-112320.EddM51.rst deleted file mode 100644 index 0da2fd33b0ea52..00000000000000 --- a/Misc/NEWS.d/next/Core and Builtins/2023-11-22-13-17-54.gh-issue-112320.EddM51.rst +++ /dev/null @@ -1,4 +0,0 @@ -The Tier 2 translator now tracks the confidence level for staying "on trace" -(i.e. not exiting back to the Tier 1 interpreter) for branch instructions -based on the number of bits set in the branch "counter". Trace translation -ends when the confidence drops below 1/3rd. diff --git a/Misc/NEWS.d/next/Core and Builtins/2023-11-24-14-10-57.gh-issue-112367.9z1IDp.rst b/Misc/NEWS.d/next/Core and Builtins/2023-11-24-14-10-57.gh-issue-112367.9z1IDp.rst deleted file mode 100644 index 991e45ad47fabe..00000000000000 --- a/Misc/NEWS.d/next/Core and Builtins/2023-11-24-14-10-57.gh-issue-112367.9z1IDp.rst +++ /dev/null @@ -1,2 +0,0 @@ -Avoid undefined behaviour when using the perf trampolines by not freeing the -code arenas until shutdown. Patch by Pablo Galindo diff --git a/Misc/NEWS.d/next/Core and Builtins/2023-11-25-20-36-38.gh-issue-99606.fDY5hK.rst b/Misc/NEWS.d/next/Core and Builtins/2023-11-25-20-36-38.gh-issue-99606.fDY5hK.rst deleted file mode 100644 index adc0e3a6bbc89a..00000000000000 --- a/Misc/NEWS.d/next/Core and Builtins/2023-11-25-20-36-38.gh-issue-99606.fDY5hK.rst +++ /dev/null @@ -1,2 +0,0 @@ -Make code generated for an empty f-string identical to the code of an empty -normal string. diff --git a/Misc/NEWS.d/next/Core and Builtins/2023-11-25-22-39-44.gh-issue-112387.AbBq5W.rst b/Misc/NEWS.d/next/Core and Builtins/2023-11-25-22-39-44.gh-issue-112387.AbBq5W.rst deleted file mode 100644 index adac11bf4c90a1..00000000000000 --- a/Misc/NEWS.d/next/Core and Builtins/2023-11-25-22-39-44.gh-issue-112387.AbBq5W.rst +++ /dev/null @@ -1,2 +0,0 @@ -Fix error positions for decoded strings with backwards tokenize errors. -Patch by Pablo Galindo diff --git a/Misc/NEWS.d/next/Core and Builtins/2023-11-25-22-58-49.gh-issue-112388.MU3cIM.rst b/Misc/NEWS.d/next/Core and Builtins/2023-11-25-22-58-49.gh-issue-112388.MU3cIM.rst deleted file mode 100644 index 1c82be2febda4f..00000000000000 --- a/Misc/NEWS.d/next/Core and Builtins/2023-11-25-22-58-49.gh-issue-112388.MU3cIM.rst +++ /dev/null @@ -1,2 +0,0 @@ -Fix an error that was causing the parser to try to overwrite tokenizer -errors. Patch by pablo Galindo diff --git a/Misc/NEWS.d/next/Core and Builtins/2023-11-26-21-30-11.gh-issue-111058.q4DqDY.rst b/Misc/NEWS.d/next/Core and Builtins/2023-11-26-21-30-11.gh-issue-111058.q4DqDY.rst deleted file mode 100644 index de5661f911aa82..00000000000000 --- a/Misc/NEWS.d/next/Core and Builtins/2023-11-26-21-30-11.gh-issue-111058.q4DqDY.rst +++ /dev/null @@ -1,3 +0,0 @@ -Change coro.cr_frame/gen.gi_frame to return ``None`` after the coroutine/generator has been closed. -This fixes a bug where :func:`~inspect.getcoroutinestate` and :func:`~inspect.getgeneratorstate` -return the wrong state for a closed coroutine/generator. diff --git a/Misc/NEWS.d/next/Core and Builtins/2023-11-27-18-55-30.gh-issue-112217.SwFLMj.rst b/Misc/NEWS.d/next/Core and Builtins/2023-11-27-18-55-30.gh-issue-112217.SwFLMj.rst deleted file mode 100644 index d4efbab6b2d128..00000000000000 --- a/Misc/NEWS.d/next/Core and Builtins/2023-11-27-18-55-30.gh-issue-112217.SwFLMj.rst +++ /dev/null @@ -1 +0,0 @@ -Add check for the type of ``__cause__`` returned from calling the type ``T`` in ``raise from T``. diff --git a/Misc/NEWS.d/next/Core and Builtins/2023-12-01-08-16-10.gh-issue-95754.ae4gwy.rst b/Misc/NEWS.d/next/Core and Builtins/2023-12-01-08-16-10.gh-issue-95754.ae4gwy.rst deleted file mode 100644 index 0884bc4a4be726..00000000000000 --- a/Misc/NEWS.d/next/Core and Builtins/2023-12-01-08-16-10.gh-issue-95754.ae4gwy.rst +++ /dev/null @@ -1 +0,0 @@ -Provide a better error message when accessing invalid attributes on partially initialized modules. The origin of the module being accessed is now included in the message to help with the common issue of shadowing other modules. diff --git a/Misc/NEWS.d/next/Core and Builtins/2023-12-01-19-02-21.gh-issue-105967.Puq5Cn.rst b/Misc/NEWS.d/next/Core and Builtins/2023-12-01-19-02-21.gh-issue-105967.Puq5Cn.rst deleted file mode 100644 index c69511218e3e16..00000000000000 --- a/Misc/NEWS.d/next/Core and Builtins/2023-12-01-19-02-21.gh-issue-105967.Puq5Cn.rst +++ /dev/null @@ -1,4 +0,0 @@ -Workaround a bug in Apple's macOS platform zlib library where -:func:`zlib.crc32` and :func:`binascii.crc32` could produce incorrect results -on multi-gigabyte inputs. Including when using :mod:`zipfile` on zips -containing large data. diff --git a/Misc/NEWS.d/next/Core and Builtins/2023-12-03-15-29-53.gh-issue-112660.gldBvh.rst b/Misc/NEWS.d/next/Core and Builtins/2023-12-03-15-29-53.gh-issue-112660.gldBvh.rst deleted file mode 100644 index ea9052b3e35c48..00000000000000 --- a/Misc/NEWS.d/next/Core and Builtins/2023-12-03-15-29-53.gh-issue-112660.gldBvh.rst +++ /dev/null @@ -1,2 +0,0 @@ -Do not clear unexpected errors during formatting error messages for -ImportError and AttributeError for modules. diff --git a/Misc/NEWS.d/next/Core and Builtins/2023-12-03-19-34-51.gh-issue-112625.QWTlwS.rst b/Misc/NEWS.d/next/Core and Builtins/2023-12-03-19-34-51.gh-issue-112625.QWTlwS.rst deleted file mode 100644 index 4970e10f3f4dcb..00000000000000 --- a/Misc/NEWS.d/next/Core and Builtins/2023-12-03-19-34-51.gh-issue-112625.QWTlwS.rst +++ /dev/null @@ -1 +0,0 @@ -Fixes a bug where a bytearray object could be cleared while iterating over an argument in the ``bytearray.join()`` method that could result in reading memory after it was freed. diff --git a/Misc/NEWS.d/next/Core and Builtins/2023-12-04-23-09-07.gh-issue-112730.BXHlFa.rst b/Misc/NEWS.d/next/Core and Builtins/2023-12-04-23-09-07.gh-issue-112730.BXHlFa.rst deleted file mode 100644 index 51758dd5f4c318..00000000000000 --- a/Misc/NEWS.d/next/Core and Builtins/2023-12-04-23-09-07.gh-issue-112730.BXHlFa.rst +++ /dev/null @@ -1 +0,0 @@ -Use color to highlight error locations in tracebacks. Patch by Pablo Galindo diff --git a/Misc/NEWS.d/next/Core and Builtins/2023-12-05-20-41-58.gh-issue-112716.hOcx0Y.rst b/Misc/NEWS.d/next/Core and Builtins/2023-12-05-20-41-58.gh-issue-112716.hOcx0Y.rst deleted file mode 100644 index 44d63269c5424a..00000000000000 --- a/Misc/NEWS.d/next/Core and Builtins/2023-12-05-20-41-58.gh-issue-112716.hOcx0Y.rst +++ /dev/null @@ -1,2 +0,0 @@ -Fix SystemError in the ``import`` statement and in ``__reduce__()`` methods -of builtin types when ``__builtins__`` is not a dict. diff --git a/Misc/NEWS.d/next/Core and Builtins/2023-12-07-12-00-04.gh-issue-74616.kgTGVb.rst b/Misc/NEWS.d/next/Core and Builtins/2023-12-07-12-00-04.gh-issue-74616.kgTGVb.rst deleted file mode 100644 index 5c345be9de6d0b..00000000000000 --- a/Misc/NEWS.d/next/Core and Builtins/2023-12-07-12-00-04.gh-issue-74616.kgTGVb.rst +++ /dev/null @@ -1,2 +0,0 @@ -:func:`input` now raises a ValueError when output on the terminal if the -prompt contains embedded null characters instead of silently truncating it. diff --git a/Misc/NEWS.d/next/Core and Builtins/2023-12-07-13-19-55.gh-issue-112125.4ADN7i.rst b/Misc/NEWS.d/next/Core and Builtins/2023-12-07-13-19-55.gh-issue-112125.4ADN7i.rst deleted file mode 100644 index 52cd45029fb8c7..00000000000000 --- a/Misc/NEWS.d/next/Core and Builtins/2023-12-07-13-19-55.gh-issue-112125.4ADN7i.rst +++ /dev/null @@ -1 +0,0 @@ -Fix None.__ne__(None) returning NotImplemented instead of False diff --git a/Misc/NEWS.d/next/Core and Builtins/2023-12-11-00-50-00.gh-issue-112943.RHNZie.rst b/Misc/NEWS.d/next/Core and Builtins/2023-12-11-00-50-00.gh-issue-112943.RHNZie.rst deleted file mode 100644 index 4bc2fe7c26d904..00000000000000 --- a/Misc/NEWS.d/next/Core and Builtins/2023-12-11-00-50-00.gh-issue-112943.RHNZie.rst +++ /dev/null @@ -1,2 +0,0 @@ -Correctly compute end column offsets for multiline tokens in the -:mod:`tokenize` module. Patch by Pablo Galindo diff --git a/Misc/NEWS.d/next/Core and Builtins/2023-12-11-19-53-32.gh-issue-90350.-FQy3E.rst b/Misc/NEWS.d/next/Core and Builtins/2023-12-11-19-53-32.gh-issue-90350.-FQy3E.rst deleted file mode 100644 index 6b7881bbd19f59..00000000000000 --- a/Misc/NEWS.d/next/Core and Builtins/2023-12-11-19-53-32.gh-issue-90350.-FQy3E.rst +++ /dev/null @@ -1 +0,0 @@ -Optimize builtin functions :func:`min` and :func:`max`. diff --git a/Misc/NEWS.d/next/Core and Builtins/2023-12-12-04-53-19.gh-issue-108866.xbJ-9a.rst b/Misc/NEWS.d/next/Core and Builtins/2023-12-12-04-53-19.gh-issue-108866.xbJ-9a.rst deleted file mode 100644 index 96606924d4a3ec..00000000000000 --- a/Misc/NEWS.d/next/Core and Builtins/2023-12-12-04-53-19.gh-issue-108866.xbJ-9a.rst +++ /dev/null @@ -1,3 +0,0 @@ -Change the API and contract of ``_PyExecutorObject`` to return the -next_instr pointer, instead of the frame, and to always execute at least one -instruction. diff --git a/Misc/NEWS.d/next/Core and Builtins/2023-12-14-20-08-35.gh-issue-113054.e20CtM.rst b/Misc/NEWS.d/next/Core and Builtins/2023-12-14-20-08-35.gh-issue-113054.e20CtM.rst deleted file mode 100644 index d0729f9c44754c..00000000000000 --- a/Misc/NEWS.d/next/Core and Builtins/2023-12-14-20-08-35.gh-issue-113054.e20CtM.rst +++ /dev/null @@ -1,2 +0,0 @@ -Fixed bug where a redundant NOP is not removed, causing an assertion to fail -in the compiler in debug mode. diff --git a/Misc/NEWS.d/next/Core and Builtins/2023-12-19-22-03-43.gh-issue-111375.M9vuA6.rst b/Misc/NEWS.d/next/Core and Builtins/2023-12-19-22-03-43.gh-issue-111375.M9vuA6.rst deleted file mode 100644 index fbb517173451f8..00000000000000 --- a/Misc/NEWS.d/next/Core and Builtins/2023-12-19-22-03-43.gh-issue-111375.M9vuA6.rst +++ /dev/null @@ -1,2 +0,0 @@ -Only use ``NULL`` in the exception stack to indicate an exception was -handled. Patch by Carey Metcalfe. diff --git a/Misc/NEWS.d/next/Core and Builtins/2023-12-20-18-27-11.gh-issue-113297.BZyAI_.rst b/Misc/NEWS.d/next/Core and Builtins/2023-12-20-18-27-11.gh-issue-113297.BZyAI_.rst deleted file mode 100644 index b6aee1f241fd23..00000000000000 --- a/Misc/NEWS.d/next/Core and Builtins/2023-12-20-18-27-11.gh-issue-113297.BZyAI_.rst +++ /dev/null @@ -1 +0,0 @@ -Fix segfault in the compiler on with statement with 19 context managers. diff --git a/Misc/NEWS.d/next/Core and Builtins/2023-12-22-13-21-39.gh-issue-113055.47xBMF.rst b/Misc/NEWS.d/next/Core and Builtins/2023-12-22-13-21-39.gh-issue-113055.47xBMF.rst new file mode 100644 index 00000000000000..90f49272218c96 --- /dev/null +++ b/Misc/NEWS.d/next/Core and Builtins/2023-12-22-13-21-39.gh-issue-113055.47xBMF.rst @@ -0,0 +1,5 @@ +Make interp->obmalloc a pointer. For interpreters that share state with the +main interpreter, this points to the same static memory structure. For +interpreters with their own obmalloc state, it is heap allocated. Add +free_obmalloc_arenas() which will free the obmalloc arenas and radix tree +structures for interpreters with their own obmalloc state. diff --git a/Misc/NEWS.d/next/Core and Builtins/2023-12-24-03-25-28.gh-issue-113464.dvjQmA.rst b/Misc/NEWS.d/next/Core and Builtins/2023-12-24-03-25-28.gh-issue-113464.dvjQmA.rst new file mode 100644 index 00000000000000..bdee4d645f61c8 --- /dev/null +++ b/Misc/NEWS.d/next/Core and Builtins/2023-12-24-03-25-28.gh-issue-113464.dvjQmA.rst @@ -0,0 +1,4 @@ +Add an option (``--enable-experimental-jit`` for ``configure``-based builds +or ``--experimental-jit`` for ``PCbuild``-based ones) to build an +*experimental* just-in-time compiler, based on `copy-and-patch +`_ diff --git a/Misc/NEWS.d/next/Core and Builtins/2024-01-11-22-58-45.gh-issue-112050.hDuvDW.rst b/Misc/NEWS.d/next/Core and Builtins/2024-01-11-22-58-45.gh-issue-112050.hDuvDW.rst new file mode 100644 index 00000000000000..e5f3d5ea0cea25 --- /dev/null +++ b/Misc/NEWS.d/next/Core and Builtins/2024-01-11-22-58-45.gh-issue-112050.hDuvDW.rst @@ -0,0 +1 @@ +Convert :class:`collections.deque` to use Argument Clinic. diff --git a/Misc/NEWS.d/next/Core and Builtins/2024-01-12-16-40-07.gh-issue-113939.Yi3L-e.rst b/Misc/NEWS.d/next/Core and Builtins/2024-01-12-16-40-07.gh-issue-113939.Yi3L-e.rst new file mode 100644 index 00000000000000..28b8e4bdda6be4 --- /dev/null +++ b/Misc/NEWS.d/next/Core and Builtins/2024-01-12-16-40-07.gh-issue-113939.Yi3L-e.rst @@ -0,0 +1,4 @@ +frame.clear(): +Clear frame.f_locals as well, and not only the fast locals. +This is relevant once frame.f_locals was accessed, +which would contain also references to all the locals. diff --git a/Misc/NEWS.d/next/Core and Builtins/2024-01-17-00-52-57.gh-issue-113884.CvEjUE.rst b/Misc/NEWS.d/next/Core and Builtins/2024-01-17-00-52-57.gh-issue-113884.CvEjUE.rst new file mode 100644 index 00000000000000..6a39fd2f60ab81 --- /dev/null +++ b/Misc/NEWS.d/next/Core and Builtins/2024-01-17-00-52-57.gh-issue-113884.CvEjUE.rst @@ -0,0 +1 @@ +Make :class:`queue.SimpleQueue` thread safe when the GIL is disabled. diff --git a/Misc/NEWS.d/next/Core and Builtins/2024-01-17-05-09-32.gh-issue-112354.Run9ko.rst b/Misc/NEWS.d/next/Core and Builtins/2024-01-17-05-09-32.gh-issue-112354.Run9ko.rst new file mode 100644 index 00000000000000..ed45ba49c3ad42 --- /dev/null +++ b/Misc/NEWS.d/next/Core and Builtins/2024-01-17-05-09-32.gh-issue-112354.Run9ko.rst @@ -0,0 +1,2 @@ +The ``END_FOR`` instruction now pops only one value. This is to better +support side exits in loops. diff --git a/Misc/NEWS.d/next/Core and Builtins/2024-01-17-23-39-20.gh-issue-114050.Lnv1oq.rst b/Misc/NEWS.d/next/Core and Builtins/2024-01-17-23-39-20.gh-issue-114050.Lnv1oq.rst new file mode 100644 index 00000000000000..c35d2508e6bdda --- /dev/null +++ b/Misc/NEWS.d/next/Core and Builtins/2024-01-17-23-39-20.gh-issue-114050.Lnv1oq.rst @@ -0,0 +1,2 @@ +Fix segmentation fault caused by an incorrect format string +in ``TypeError`` exception when more than two arguments are passed to ``int``. diff --git a/Misc/NEWS.d/next/Core and Builtins/2024-01-18-20-20-37.gh-issue-112529.oVNvDG.rst b/Misc/NEWS.d/next/Core and Builtins/2024-01-18-20-20-37.gh-issue-112529.oVNvDG.rst new file mode 100644 index 00000000000000..b3aa43801da488 --- /dev/null +++ b/Misc/NEWS.d/next/Core and Builtins/2024-01-18-20-20-37.gh-issue-112529.oVNvDG.rst @@ -0,0 +1,3 @@ +The free-threaded build now has its own thread-safe GC implementation that +uses mimalloc to find GC tracked objects. It is non-generational, unlike the +existing GC implementation. diff --git a/Misc/NEWS.d/next/Core and Builtins/2024-01-19-13-18-13.gh-issue-114265.7HAi--.rst b/Misc/NEWS.d/next/Core and Builtins/2024-01-19-13-18-13.gh-issue-114265.7HAi--.rst new file mode 100644 index 00000000000000..74affbbd09ffb4 --- /dev/null +++ b/Misc/NEWS.d/next/Core and Builtins/2024-01-19-13-18-13.gh-issue-114265.7HAi--.rst @@ -0,0 +1 @@ +Compiler propagates line numbers before optimization, leading to more optimization opportunities and removing the need for the ``guarantee_lineno_for_exits`` hack. diff --git a/Misc/NEWS.d/next/Core and Builtins/2024-01-21-17-29-32.gh-issue-114388.UVGO4K.rst b/Misc/NEWS.d/next/Core and Builtins/2024-01-21-17-29-32.gh-issue-114388.UVGO4K.rst new file mode 100644 index 00000000000000..52c2742001d9ca --- /dev/null +++ b/Misc/NEWS.d/next/Core and Builtins/2024-01-21-17-29-32.gh-issue-114388.UVGO4K.rst @@ -0,0 +1,5 @@ +Fix a :exc:`RuntimeWarning` emitted when assign an integer-like value that +is not an instance of :class:`int` to an attribute that corresponds to a C +struct member of :ref:`type ` T_UINT and T_ULONG. Fix a +double :exc:`RuntimeWarning` emitted when assign a negative integer value to +an attribute that corresponds to a C struct member of type T_UINT. diff --git a/Misc/NEWS.d/next/Core and Builtins/2024-01-22-09-49-02.gh-issue-114083.hf1-ku.rst b/Misc/NEWS.d/next/Core and Builtins/2024-01-22-09-49-02.gh-issue-114083.hf1-ku.rst new file mode 100644 index 00000000000000..79be45e87b90d3 --- /dev/null +++ b/Misc/NEWS.d/next/Core and Builtins/2024-01-22-09-49-02.gh-issue-114083.hf1-ku.rst @@ -0,0 +1 @@ +Compiler applies folding of LOAD_CONST with following instruction in a separate pass before other optimisations. This enables jump threading in certain circumstances. diff --git a/Misc/NEWS.d/next/Core and Builtins/2024-01-22-15-10-01.gh-issue-114456.fBFEJF.rst b/Misc/NEWS.d/next/Core and Builtins/2024-01-22-15-10-01.gh-issue-114456.fBFEJF.rst new file mode 100644 index 00000000000000..2b30ad98fb5c79 --- /dev/null +++ b/Misc/NEWS.d/next/Core and Builtins/2024-01-22-15-10-01.gh-issue-114456.fBFEJF.rst @@ -0,0 +1 @@ +Lower the recursion limit under a debug build of WASI. diff --git a/Misc/NEWS.d/next/Core and Builtins/2024-01-25-18-50-49.gh-issue-112529.IbbApA.rst b/Misc/NEWS.d/next/Core and Builtins/2024-01-25-18-50-49.gh-issue-112529.IbbApA.rst new file mode 100644 index 00000000000000..2a6d74fb222702 --- /dev/null +++ b/Misc/NEWS.d/next/Core and Builtins/2024-01-25-18-50-49.gh-issue-112529.IbbApA.rst @@ -0,0 +1,4 @@ +The free-threaded build no longer allocates space for the ``PyGC_Head`` +structure in objects that support cyclic garbage collection. A number of +other fields and data structures are used as replacements, including +``ob_gc_bits``, ``ob_tid``, and mimalloc internal data structures. diff --git a/Misc/NEWS.d/next/Core and Builtins/2024-01-31-09-10-10.gh-issue-107944.XWm1B-.rst b/Misc/NEWS.d/next/Core and Builtins/2024-01-31-09-10-10.gh-issue-107944.XWm1B-.rst new file mode 100644 index 00000000000000..8e3fb786c11055 --- /dev/null +++ b/Misc/NEWS.d/next/Core and Builtins/2024-01-31-09-10-10.gh-issue-107944.XWm1B-.rst @@ -0,0 +1 @@ +Improve error message for function calls with bad keyword arguments via getargs diff --git a/Misc/NEWS.d/next/Core and Builtins/2024-02-01-18-16-52.gh-issue-114806.wrH2J6.rst b/Misc/NEWS.d/next/Core and Builtins/2024-02-01-18-16-52.gh-issue-114806.wrH2J6.rst new file mode 100644 index 00000000000000..795f2529df8207 --- /dev/null +++ b/Misc/NEWS.d/next/Core and Builtins/2024-02-01-18-16-52.gh-issue-114806.wrH2J6.rst @@ -0,0 +1,3 @@ +No longer specialize calls to classes, if those classes have metaclasses. +Fixes bug where the ``__call__`` method of the metaclass was not being +called. diff --git a/Misc/NEWS.d/next/Core and Builtins/2024-02-02-05-27-48.gh-issue-113462.VMml8q.rst b/Misc/NEWS.d/next/Core and Builtins/2024-02-02-05-27-48.gh-issue-113462.VMml8q.rst new file mode 100644 index 00000000000000..1a401ecebf019a --- /dev/null +++ b/Misc/NEWS.d/next/Core and Builtins/2024-02-02-05-27-48.gh-issue-113462.VMml8q.rst @@ -0,0 +1,2 @@ +Limit the number of versions that a single class can use. Prevents a few +wayward classes using up all the version numbers. diff --git a/Misc/NEWS.d/next/Core and Builtins/2024-02-03-01-48-38.gh-issue-114944.4J5ELD.rst b/Misc/NEWS.d/next/Core and Builtins/2024-02-03-01-48-38.gh-issue-114944.4J5ELD.rst new file mode 100644 index 00000000000000..fb41caf7c5f4fa --- /dev/null +++ b/Misc/NEWS.d/next/Core and Builtins/2024-02-03-01-48-38.gh-issue-114944.4J5ELD.rst @@ -0,0 +1 @@ +Fixes a race between ``PyParkingLot_Park`` and ``_PyParkingLot_UnparkAll``. diff --git a/Misc/NEWS.d/next/Core and Builtins/2024-02-03-04-07-18.gh-issue-114887.uLSFmN.rst b/Misc/NEWS.d/next/Core and Builtins/2024-02-03-04-07-18.gh-issue-114887.uLSFmN.rst new file mode 100644 index 00000000000000..b4d8cf4089d723 --- /dev/null +++ b/Misc/NEWS.d/next/Core and Builtins/2024-02-03-04-07-18.gh-issue-114887.uLSFmN.rst @@ -0,0 +1,2 @@ +Changed socket type validation in :meth:`~asyncio.loop.create_datagram_endpoint` to accept all non-stream sockets. +This fixes a regression in compatibility with raw sockets. diff --git a/Misc/NEWS.d/next/Core and Builtins/2024-02-07-00-18-42.gh-issue-112069.jRDRR5.rst b/Misc/NEWS.d/next/Core and Builtins/2024-02-07-00-18-42.gh-issue-112069.jRDRR5.rst new file mode 100644 index 00000000000000..51ba6bd1ddaac3 --- /dev/null +++ b/Misc/NEWS.d/next/Core and Builtins/2024-02-07-00-18-42.gh-issue-112069.jRDRR5.rst @@ -0,0 +1 @@ +Adapt :class:`set` and :class:`frozenset` methods to Argument Clinic. diff --git a/Misc/NEWS.d/next/Core and Builtins/2024-02-07-07-50-12.gh-issue-114828.nSXwMi.rst b/Misc/NEWS.d/next/Core and Builtins/2024-02-07-07-50-12.gh-issue-114828.nSXwMi.rst new file mode 100644 index 00000000000000..b1c63e0a1518fd --- /dev/null +++ b/Misc/NEWS.d/next/Core and Builtins/2024-02-07-07-50-12.gh-issue-114828.nSXwMi.rst @@ -0,0 +1,2 @@ +Fix compilation crashes in uncommon code examples using :func:`super` inside +a comprehension in a class body. diff --git a/Misc/NEWS.d/next/Documentation/2023-10-23-23-43-43.gh-issue-110746.yg77IE.rst b/Misc/NEWS.d/next/Documentation/2023-10-23-23-43-43.gh-issue-110746.yg77IE.rst deleted file mode 100644 index 215db7beb75dcf..00000000000000 --- a/Misc/NEWS.d/next/Documentation/2023-10-23-23-43-43.gh-issue-110746.yg77IE.rst +++ /dev/null @@ -1 +0,0 @@ -Improved markup for valid options/values for methods ttk.treeview.column and ttk.treeview.heading, and for Layouts. diff --git a/Misc/NEWS.d/next/Documentation/2023-11-30-02-33-59.gh-issue-111699._O5G_y.rst b/Misc/NEWS.d/next/Documentation/2023-11-30-02-33-59.gh-issue-111699._O5G_y.rst deleted file mode 100644 index 2d31345e6c2044..00000000000000 --- a/Misc/NEWS.d/next/Documentation/2023-11-30-02-33-59.gh-issue-111699._O5G_y.rst +++ /dev/null @@ -1 +0,0 @@ -Relocate ``smtpd`` deprecation notice to its own section rather than under ``locale`` in What's New in Python 3.12 document diff --git a/Misc/NEWS.d/next/Documentation/2024-01-17-11-40-03.gh-issue-114123.LuueXf.rst b/Misc/NEWS.d/next/Documentation/2024-01-17-11-40-03.gh-issue-114123.LuueXf.rst new file mode 100644 index 00000000000000..1d93a422840077 --- /dev/null +++ b/Misc/NEWS.d/next/Documentation/2024-01-17-11-40-03.gh-issue-114123.LuueXf.rst @@ -0,0 +1,7 @@ +Move the :mod:`csv` module docstring to the :mod:`!csv` module +instead of reexporting it from the internal :mod:`!_csv` module, +and remove ``__doc__`` from ``csv.__all__``. + +Move :attr:`!csv.__version__` to the :mod:`!csv` module +instead of reexporting it from the internal :mod:`!_csv` module, +and remove ``__version__`` from ``csv.__all__``. diff --git a/Misc/NEWS.d/next/IDLE/2023-04-25-03-01-23.gh-issue-103820.LCSpza.rst b/Misc/NEWS.d/next/IDLE/2023-04-25-03-01-23.gh-issue-103820.LCSpza.rst new file mode 100644 index 00000000000000..b9d7faf047b28e --- /dev/null +++ b/Misc/NEWS.d/next/IDLE/2023-04-25-03-01-23.gh-issue-103820.LCSpza.rst @@ -0,0 +1,2 @@ +Revise IDLE bindings so that events from mouse button 4/5 on non-X11 +windowing systems (i.e. Win32 and Aqua) are not mistaken for scrolling. diff --git a/Misc/NEWS.d/next/IDLE/2023-12-10-20-01-11.gh-issue-112898.98aWv2.rst b/Misc/NEWS.d/next/IDLE/2023-12-10-20-01-11.gh-issue-112898.98aWv2.rst deleted file mode 100644 index 1c20e46b1e5f7b..00000000000000 --- a/Misc/NEWS.d/next/IDLE/2023-12-10-20-01-11.gh-issue-112898.98aWv2.rst +++ /dev/null @@ -1 +0,0 @@ -Fix processing unsaved files when quitting IDLE on macOS. diff --git a/Misc/NEWS.d/next/IDLE/2023-12-19-00-03-12.gh-issue-113269.lrU-IC.rst b/Misc/NEWS.d/next/IDLE/2023-12-19-00-03-12.gh-issue-113269.lrU-IC.rst deleted file mode 100644 index 72e75b7910e359..00000000000000 --- a/Misc/NEWS.d/next/IDLE/2023-12-19-00-03-12.gh-issue-113269.lrU-IC.rst +++ /dev/null @@ -1 +0,0 @@ -Fix test_editor hang on macOS Catalina. diff --git a/Misc/NEWS.d/next/IDLE/2024-01-17-23-18-15.gh-issue-96905.UYaxoU.rst b/Misc/NEWS.d/next/IDLE/2024-01-17-23-18-15.gh-issue-96905.UYaxoU.rst new file mode 100644 index 00000000000000..fe7dde64c7c7d5 --- /dev/null +++ b/Misc/NEWS.d/next/IDLE/2024-01-17-23-18-15.gh-issue-96905.UYaxoU.rst @@ -0,0 +1 @@ +In idlelib code, stop redefining built-ins 'dict' and 'object'. diff --git a/Misc/NEWS.d/next/Library/2019-06-14-22-37-32.bpo-37260.oecdIf.rst b/Misc/NEWS.d/next/Library/2019-06-14-22-37-32.bpo-37260.oecdIf.rst deleted file mode 100644 index a5f2c5e8e18919..00000000000000 --- a/Misc/NEWS.d/next/Library/2019-06-14-22-37-32.bpo-37260.oecdIf.rst +++ /dev/null @@ -1,2 +0,0 @@ -Fixed a race condition in :func:`shutil.rmtree` in which directory entries removed by another process or thread while ``shutil.rmtree()`` is running can cause it to raise FileNotFoundError. Patch by Jeffrey Kintscher. - diff --git a/Misc/NEWS.d/next/Library/2020-03-09-15-08-29.bpo-39912.xPOBBY.rst b/Misc/NEWS.d/next/Library/2020-03-09-15-08-29.bpo-39912.xPOBBY.rst deleted file mode 100644 index fb8579725a2d7d..00000000000000 --- a/Misc/NEWS.d/next/Library/2020-03-09-15-08-29.bpo-39912.xPOBBY.rst +++ /dev/null @@ -1,3 +0,0 @@ -:func:`warnings.filterwarnings()` and :func:`warnings.simplefilter()` now raise -appropriate exceptions instead of ``AssertionError``. Patch contributed by -Rémi Lapeyre. diff --git a/Misc/NEWS.d/next/Library/2020-05-21-23-32-46.bpo-40262.z4fQv1.rst b/Misc/NEWS.d/next/Library/2020-05-21-23-32-46.bpo-40262.z4fQv1.rst deleted file mode 100644 index c017a1c8df09d8..00000000000000 --- a/Misc/NEWS.d/next/Library/2020-05-21-23-32-46.bpo-40262.z4fQv1.rst +++ /dev/null @@ -1,2 +0,0 @@ -The :meth:`ssl.SSLSocket.recv_into` method no longer requires the *buffer* -argument to implement ``__len__`` and supports buffers with arbitrary item size. diff --git a/Misc/NEWS.d/next/Library/2020-07-28-20-48-05.bpo-41422.iMwnMu.rst b/Misc/NEWS.d/next/Library/2020-07-28-20-48-05.bpo-41422.iMwnMu.rst deleted file mode 100644 index 8bde68f8f2afc8..00000000000000 --- a/Misc/NEWS.d/next/Library/2020-07-28-20-48-05.bpo-41422.iMwnMu.rst +++ /dev/null @@ -1,2 +0,0 @@ -Fixed memory leaks of :class:`pickle.Pickler` and :class:`pickle.Unpickler` involving cyclic references via the -internal memo mapping. diff --git a/Misc/NEWS.d/next/Library/2020-12-14-09-31-13.bpo-35332.s22wAx.rst b/Misc/NEWS.d/next/Library/2020-12-14-09-31-13.bpo-35332.s22wAx.rst deleted file mode 100644 index 80564b99a079c6..00000000000000 --- a/Misc/NEWS.d/next/Library/2020-12-14-09-31-13.bpo-35332.s22wAx.rst +++ /dev/null @@ -1,3 +0,0 @@ -The :func:`shutil.rmtree` function now ignores errors when calling -:func:`os.close` when *ignore_errors* is ``True``, and -:func:`os.close` no longer retried after error. diff --git a/Misc/NEWS.d/next/Library/2021-11-23-22-22-49.bpo-32731.kNOASr.rst b/Misc/NEWS.d/next/Library/2021-11-23-22-22-49.bpo-32731.kNOASr.rst deleted file mode 100644 index 92f3b870c11131..00000000000000 --- a/Misc/NEWS.d/next/Library/2021-11-23-22-22-49.bpo-32731.kNOASr.rst +++ /dev/null @@ -1,3 +0,0 @@ -:func:`getpass.getuser` now raises :exc:`OSError` for all failures rather -than :exc:`ImportError` on systems lacking the :mod:`pwd` module or -:exc:`KeyError` if the password database is empty. diff --git a/Misc/NEWS.d/next/Library/2021-12-06-22-10-53.bpo-43153.J7mjSy.rst b/Misc/NEWS.d/next/Library/2021-12-06-22-10-53.bpo-43153.J7mjSy.rst deleted file mode 100644 index 7800e0a4869adf..00000000000000 --- a/Misc/NEWS.d/next/Library/2021-12-06-22-10-53.bpo-43153.J7mjSy.rst +++ /dev/null @@ -1,4 +0,0 @@ -On Windows, ``tempfile.TemporaryDirectory`` previously masked a -``PermissionError`` with ``NotADirectoryError`` during directory cleanup. It -now correctly raises ``PermissionError`` if errors are not ignored. Patch by -Andrei Kulakov and Ken Jin. diff --git a/Misc/NEWS.d/next/Library/2022-07-31-01-24-40.gh-issue-88569.eU0--b.rst b/Misc/NEWS.d/next/Library/2022-07-31-01-24-40.gh-issue-88569.eU0--b.rst new file mode 100644 index 00000000000000..31dd985bb5c3b6 --- /dev/null +++ b/Misc/NEWS.d/next/Library/2022-07-31-01-24-40.gh-issue-88569.eU0--b.rst @@ -0,0 +1,4 @@ +Add :func:`os.path.isreserved`, which identifies reserved pathnames such +as "NUL", "AUX" and "CON". This function is only available on Windows. + +Deprecate :meth:`pathlib.PurePath.is_reserved`. diff --git a/Misc/NEWS.d/next/Library/2022-12-01-16-57-44.gh-issue-91133.LKMVCV.rst b/Misc/NEWS.d/next/Library/2022-12-01-16-57-44.gh-issue-91133.LKMVCV.rst deleted file mode 100644 index 7991048fc48e03..00000000000000 --- a/Misc/NEWS.d/next/Library/2022-12-01-16-57-44.gh-issue-91133.LKMVCV.rst +++ /dev/null @@ -1,2 +0,0 @@ -Fix a bug in :class:`tempfile.TemporaryDirectory` cleanup, which now no longer -dereferences symlinks when working around file system permission errors. diff --git a/Misc/NEWS.d/next/Library/2023-02-08-00-43-29.gh-issue-83162.ufdI9F.rst b/Misc/NEWS.d/next/Library/2023-02-08-00-43-29.gh-issue-83162.ufdI9F.rst deleted file mode 100644 index 6074dd7f101a6d..00000000000000 --- a/Misc/NEWS.d/next/Library/2023-02-08-00-43-29.gh-issue-83162.ufdI9F.rst +++ /dev/null @@ -1,3 +0,0 @@ -Renamed :exc:`!re.error` to :exc:`PatternError` for clarity, and kept -:exc:`!re.error` for backward compatibility. Patch by Matthias Bussonnier and -Adam Chhina. diff --git a/Misc/NEWS.d/next/Library/2023-03-08-00-02-30.gh-issue-102512.LiugDr.rst b/Misc/NEWS.d/next/Library/2023-03-08-00-02-30.gh-issue-102512.LiugDr.rst new file mode 100644 index 00000000000000..659cba73cbf34e --- /dev/null +++ b/Misc/NEWS.d/next/Library/2023-03-08-00-02-30.gh-issue-102512.LiugDr.rst @@ -0,0 +1,3 @@ +When :func:`os.fork` is called from a foreign thread (aka ``_DummyThread``), +the type of the thread in a child process is changed to ``_MainThread``. +Also changed its name and daemonic status, it can be now joined. diff --git a/Misc/NEWS.d/next/Library/2023-03-15-03-21-18.gh-issue-85984.Xaq6ZN.rst b/Misc/NEWS.d/next/Library/2023-03-15-03-21-18.gh-issue-85984.Xaq6ZN.rst new file mode 100644 index 00000000000000..0e54a1fe3c8a1c --- /dev/null +++ b/Misc/NEWS.d/next/Library/2023-03-15-03-21-18.gh-issue-85984.Xaq6ZN.rst @@ -0,0 +1,2 @@ +Add POSIX pseudo-terminal functions :func:`os.posix_openpt`, +:func:`os.grantpt`, :func:`os.unlockpt`, and :func:`os.ptsname`. diff --git a/Misc/NEWS.d/next/Library/2023-04-08-11-41-07.gh-issue-101599.PaWNFh.rst b/Misc/NEWS.d/next/Library/2023-04-08-11-41-07.gh-issue-101599.PaWNFh.rst new file mode 100644 index 00000000000000..a1608a1ae0d2fa --- /dev/null +++ b/Misc/NEWS.d/next/Library/2023-04-08-11-41-07.gh-issue-101599.PaWNFh.rst @@ -0,0 +1 @@ +Changed argparse flag options formatting to remove redundancy. diff --git a/Misc/NEWS.d/next/Library/2023-04-29-20-49-13.gh-issue-104003.-8Ruk2.rst b/Misc/NEWS.d/next/Library/2023-04-29-20-49-13.gh-issue-104003.-8Ruk2.rst deleted file mode 100644 index 82d61ca8b8bc97..00000000000000 --- a/Misc/NEWS.d/next/Library/2023-04-29-20-49-13.gh-issue-104003.-8Ruk2.rst +++ /dev/null @@ -1,3 +0,0 @@ -Add :func:`warnings.deprecated`, a decorator to mark deprecated functions to -static type checkers and to warn on usage of deprecated classes and functions. -See :pep:`702`. Patch by Jelle Zijlstra. diff --git a/Misc/NEWS.d/next/Library/2023-05-06-04-57-10.gh-issue-96471.C9wAU7.rst b/Misc/NEWS.d/next/Library/2023-05-06-04-57-10.gh-issue-96471.C9wAU7.rst new file mode 100644 index 00000000000000..0bace8d8bd425c --- /dev/null +++ b/Misc/NEWS.d/next/Library/2023-05-06-04-57-10.gh-issue-96471.C9wAU7.rst @@ -0,0 +1 @@ +Add :py:class:`queue.Queue` termination with :py:meth:`~queue.Queue.shutdown`. diff --git a/Misc/NEWS.d/next/Library/2023-05-08-09-30-00.gh-issue-104282.h4c6Eb.rst b/Misc/NEWS.d/next/Library/2023-05-08-09-30-00.gh-issue-104282.h4c6Eb.rst new file mode 100644 index 00000000000000..569ce66a5b9d5f --- /dev/null +++ b/Misc/NEWS.d/next/Library/2023-05-08-09-30-00.gh-issue-104282.h4c6Eb.rst @@ -0,0 +1,3 @@ +Fix null pointer dereference in :func:`lzma._decode_filter_properties` +due to improper handling of BCJ filters with properties of zero length. +Patch by Radislav Chugunov. diff --git a/Misc/NEWS.d/next/Library/2023-05-30-18-30-11.gh-issue-105102.SnpK04.rst b/Misc/NEWS.d/next/Library/2023-05-30-18-30-11.gh-issue-105102.SnpK04.rst new file mode 100644 index 00000000000000..7ca21afefa3132 --- /dev/null +++ b/Misc/NEWS.d/next/Library/2023-05-30-18-30-11.gh-issue-105102.SnpK04.rst @@ -0,0 +1,2 @@ +Allow :class:`ctypes.Union` to be nested in :class:`ctypes.Structure` when +the system endianness is the opposite of the classes. diff --git a/Misc/NEWS.d/next/Library/2023-06-29-14-26-56.gh-issue-106233.Aqw2HI.rst b/Misc/NEWS.d/next/Library/2023-06-29-14-26-56.gh-issue-106233.Aqw2HI.rst new file mode 100644 index 00000000000000..345c8b20815c95 --- /dev/null +++ b/Misc/NEWS.d/next/Library/2023-06-29-14-26-56.gh-issue-106233.Aqw2HI.rst @@ -0,0 +1,2 @@ +Fix stacklevel in ``InvalidTZPathWarning`` during :mod:`zoneinfo` module +import. diff --git a/Misc/NEWS.d/next/Library/2023-07-23-12-28-26.gh-issue-75705.aB2-Ww.rst b/Misc/NEWS.d/next/Library/2023-07-23-12-28-26.gh-issue-75705.aB2-Ww.rst new file mode 100644 index 00000000000000..272e31d64cfbd9 --- /dev/null +++ b/Misc/NEWS.d/next/Library/2023-07-23-12-28-26.gh-issue-75705.aB2-Ww.rst @@ -0,0 +1 @@ +Set unixfrom envelope in :class:`mailbox.mbox` and :class:`mailbox.MMDF`. diff --git a/Misc/NEWS.d/next/Library/2023-08-07-21-11-24.gh-issue-102130._UyI5i.rst b/Misc/NEWS.d/next/Library/2023-08-07-21-11-24.gh-issue-102130._UyI5i.rst deleted file mode 100644 index f582ad5df39e84..00000000000000 --- a/Misc/NEWS.d/next/Library/2023-08-07-21-11-24.gh-issue-102130._UyI5i.rst +++ /dev/null @@ -1 +0,0 @@ -Support tab completion in :mod:`cmd` for ``editline``. diff --git a/Misc/NEWS.d/next/Library/2023-08-14-21-10-52.gh-issue-103363.u64_QI.rst b/Misc/NEWS.d/next/Library/2023-08-14-21-10-52.gh-issue-103363.u64_QI.rst deleted file mode 100644 index d4a27d624eb5e6..00000000000000 --- a/Misc/NEWS.d/next/Library/2023-08-14-21-10-52.gh-issue-103363.u64_QI.rst +++ /dev/null @@ -1,2 +0,0 @@ -Add *follow_symlinks* keyword-only argument to :meth:`pathlib.Path.owner` -and :meth:`~pathlib.Path.group`, defaulting to ``True``. diff --git a/Misc/NEWS.d/next/Library/2023-09-22-22-17-45.gh-issue-38807.m9McRN.rst b/Misc/NEWS.d/next/Library/2023-09-22-22-17-45.gh-issue-38807.m9McRN.rst new file mode 100644 index 00000000000000..4219723d15b9e6 --- /dev/null +++ b/Misc/NEWS.d/next/Library/2023-09-22-22-17-45.gh-issue-38807.m9McRN.rst @@ -0,0 +1,3 @@ +Fix race condition in :mod:`trace`. Instead of checking if a directory +exists and creating it, directly call :func:`os.makedirs` with the kwarg +``exist_ok=True``. diff --git a/Misc/NEWS.d/next/Library/2023-09-23-14-40-51.gh-issue-109786.UX3pKv.rst b/Misc/NEWS.d/next/Library/2023-09-23-14-40-51.gh-issue-109786.UX3pKv.rst deleted file mode 100644 index 07222fa339d703..00000000000000 --- a/Misc/NEWS.d/next/Library/2023-09-23-14-40-51.gh-issue-109786.UX3pKv.rst +++ /dev/null @@ -1,2 +0,0 @@ -Fix possible reference leaks and crash when re-enter the ``__next__()`` method of -:class:`itertools.pairwise`. diff --git a/Misc/NEWS.d/next/Library/2023-10-04-11-09-30.gh-issue-110345.fZU1ud.rst b/Misc/NEWS.d/next/Library/2023-10-04-11-09-30.gh-issue-110345.fZU1ud.rst new file mode 100644 index 00000000000000..d9ccc0f12de47c --- /dev/null +++ b/Misc/NEWS.d/next/Library/2023-10-04-11-09-30.gh-issue-110345.fZU1ud.rst @@ -0,0 +1 @@ +Show the Tcl/Tk patchlevel (rather than version) in :meth:`tkinter._test`. diff --git a/Misc/NEWS.d/next/Library/2023-10-11-02-34-01.gh-issue-110109.RFCmHs.rst b/Misc/NEWS.d/next/Library/2023-10-11-02-34-01.gh-issue-110109.RFCmHs.rst deleted file mode 100644 index 4f12d128f49fb3..00000000000000 --- a/Misc/NEWS.d/next/Library/2023-10-11-02-34-01.gh-issue-110109.RFCmHs.rst +++ /dev/null @@ -1,3 +0,0 @@ -Add private ``pathlib._PurePathBase`` class: a base class for -:class:`pathlib.PurePath` that omits certain magic methods. It may be made -public (along with ``_PathBase``) in future. diff --git a/Misc/NEWS.d/next/Library/2023-10-12-18-19-47.gh-issue-82300.P8-O38.rst b/Misc/NEWS.d/next/Library/2023-10-12-18-19-47.gh-issue-82300.P8-O38.rst deleted file mode 100644 index d7e6b225489b99..00000000000000 --- a/Misc/NEWS.d/next/Library/2023-10-12-18-19-47.gh-issue-82300.P8-O38.rst +++ /dev/null @@ -1 +0,0 @@ -Add ``track`` parameter to :class:`multiprocessing.shared_memory.SharedMemory` that allows using shared memory blocks without having to register with the POSIX resource tracker that automatically releases them upon process exit. diff --git a/Misc/NEWS.d/next/Library/2023-10-19-02-08-12.gh-issue-111051.8h1Dpk.rst b/Misc/NEWS.d/next/Library/2023-10-19-02-08-12.gh-issue-111051.8h1Dpk.rst new file mode 100644 index 00000000000000..adb3241b89ae3e --- /dev/null +++ b/Misc/NEWS.d/next/Library/2023-10-19-02-08-12.gh-issue-111051.8h1Dpk.rst @@ -0,0 +1 @@ +Added check for file modification during debugging with :mod:`pdb` diff --git a/Misc/NEWS.d/next/Library/2023-10-20-15-28-08.gh-issue-102988.dStNO7.rst b/Misc/NEWS.d/next/Library/2023-10-20-15-28-08.gh-issue-102988.dStNO7.rst deleted file mode 100644 index 3d0e9e4078c934..00000000000000 --- a/Misc/NEWS.d/next/Library/2023-10-20-15-28-08.gh-issue-102988.dStNO7.rst +++ /dev/null @@ -1,8 +0,0 @@ -:func:`email.utils.getaddresses` and :func:`email.utils.parseaddr` now -return ``('', '')`` 2-tuples in more situations where invalid email -addresses are encountered instead of potentially inaccurate values. Add -optional *strict* parameter to these two functions: use ``strict=False`` to -get the old behavior, accept malformed inputs. -``getattr(email.utils, 'supports_strict_parsing', False)`` can be use to check -if the *strict* paramater is available. Patch by Thomas Dwyer and Victor -Stinner to improve the CVE-2023-27043 fix. diff --git a/Misc/NEWS.d/next/Library/2023-10-23-03-49-34.gh-issue-102980.aXBd54.rst b/Misc/NEWS.d/next/Library/2023-10-23-03-49-34.gh-issue-102980.aXBd54.rst deleted file mode 100644 index d4bae4790d6fa4..00000000000000 --- a/Misc/NEWS.d/next/Library/2023-10-23-03-49-34.gh-issue-102980.aXBd54.rst +++ /dev/null @@ -1 +0,0 @@ -Redirect the output of ``interact`` command of :mod:`pdb` to the same channel as the debugger. Add tests and improve docs. diff --git a/Misc/NEWS.d/next/Library/2023-10-23-18-42-26.gh-issue-111049.Ys7-o_.rst b/Misc/NEWS.d/next/Library/2023-10-23-18-42-26.gh-issue-111049.Ys7-o_.rst deleted file mode 100644 index b1de348bea0a58..00000000000000 --- a/Misc/NEWS.d/next/Library/2023-10-23-18-42-26.gh-issue-111049.Ys7-o_.rst +++ /dev/null @@ -1,2 +0,0 @@ -Fix crash during garbage collection of the :class:`io.BytesIO` buffer -object. diff --git a/Misc/NEWS.d/next/Library/2023-10-24-19-19-54.gh-issue-82626._hfLRf.rst b/Misc/NEWS.d/next/Library/2023-10-24-19-19-54.gh-issue-82626._hfLRf.rst new file mode 100644 index 00000000000000..92a66b5bf0f635 --- /dev/null +++ b/Misc/NEWS.d/next/Library/2023-10-24-19-19-54.gh-issue-82626._hfLRf.rst @@ -0,0 +1,2 @@ +Many functions now emit a warning if a boolean value is passed as a file +descriptor argument. diff --git a/Misc/NEWS.d/next/Library/2023-10-25-13-07-53.gh-issue-67790.jMn9Ad.rst b/Misc/NEWS.d/next/Library/2023-10-25-13-07-53.gh-issue-67790.jMn9Ad.rst deleted file mode 100644 index 44c5702a6551b0..00000000000000 --- a/Misc/NEWS.d/next/Library/2023-10-25-13-07-53.gh-issue-67790.jMn9Ad.rst +++ /dev/null @@ -1,2 +0,0 @@ -Implement basic formatting support (minimum width, alignment, fill) for -:class:`fractions.Fraction`. diff --git a/Misc/NEWS.d/next/Library/2023-10-25-16-37-13.gh-issue-75666.BpsWut.rst b/Misc/NEWS.d/next/Library/2023-10-25-16-37-13.gh-issue-75666.BpsWut.rst deleted file mode 100644 index d774cc4f7c687f..00000000000000 --- a/Misc/NEWS.d/next/Library/2023-10-25-16-37-13.gh-issue-75666.BpsWut.rst +++ /dev/null @@ -1,6 +0,0 @@ -Fix the behavior of :mod:`tkinter` widget's ``unbind()`` method with two -arguments. Previously, ``widget.unbind(sequence, funcid)`` destroyed the -current binding for *sequence*, leaving *sequence* unbound, and deleted the -*funcid* command. Now it removes only *funcid* from the binding for -*sequence*, keeping other commands, and deletes the *funcid* command. It -leaves *sequence* unbound only if *funcid* was the last bound command. diff --git a/Misc/NEWS.d/next/Library/2023-10-27-19-24-58.gh-issue-43457.84lx9H.rst b/Misc/NEWS.d/next/Library/2023-10-27-19-24-58.gh-issue-43457.84lx9H.rst new file mode 100644 index 00000000000000..401a532ce03e77 --- /dev/null +++ b/Misc/NEWS.d/next/Library/2023-10-27-19-24-58.gh-issue-43457.84lx9H.rst @@ -0,0 +1,8 @@ +Fix the :mod:`tkinter` widget method :meth:`!wm_attributes`. It now +accepts the attribute name without the minus prefix to get window attributes +and allows to specify attributes and values to set as keyword arguments. +Add new optional keyword argument *return_python_dict*: calling +``w.wm_attributes(return_python_dict=True)`` returns the attributes as +a dict instead of a tuple. +Calling ``w.wm_attributes()`` now returns a tuple instead of string if +*wantobjects* was set to 0. diff --git a/Misc/NEWS.d/next/Library/2023-11-04-22-32-27.gh-issue-111741.f1ufr8.rst b/Misc/NEWS.d/next/Library/2023-11-04-22-32-27.gh-issue-111741.f1ufr8.rst new file mode 100644 index 00000000000000..e43f93a270ce9c --- /dev/null +++ b/Misc/NEWS.d/next/Library/2023-11-04-22-32-27.gh-issue-111741.f1ufr8.rst @@ -0,0 +1 @@ +Recognise ``image/webp`` as a standard format in the :mod:`mimetypes` module. diff --git a/Misc/NEWS.d/next/Library/2023-11-05-20-09-27.gh-issue-99367.HLaWKo.rst b/Misc/NEWS.d/next/Library/2023-11-05-20-09-27.gh-issue-99367.HLaWKo.rst deleted file mode 100644 index 0920da221e423f..00000000000000 --- a/Misc/NEWS.d/next/Library/2023-11-05-20-09-27.gh-issue-99367.HLaWKo.rst +++ /dev/null @@ -1 +0,0 @@ -Do not mangle ``sys.path[0]`` in :mod:`pdb` if safe_path is set diff --git a/Misc/NEWS.d/next/Library/2023-11-08-16-11-04.gh-issue-110275.Bm6GwR.rst b/Misc/NEWS.d/next/Library/2023-11-08-16-11-04.gh-issue-110275.Bm6GwR.rst deleted file mode 100644 index 194dd5cb623f0f..00000000000000 --- a/Misc/NEWS.d/next/Library/2023-11-08-16-11-04.gh-issue-110275.Bm6GwR.rst +++ /dev/null @@ -1,2 +0,0 @@ -Named tuple's methods ``_replace()`` and ``__replace__()`` now raise -TypeError instead of ValueError for invalid keyword arguments. diff --git a/Misc/NEWS.d/next/Library/2023-11-08-18-53-07.gh-issue-68166.1iTh4Y.rst b/Misc/NEWS.d/next/Library/2023-11-08-18-53-07.gh-issue-68166.1iTh4Y.rst deleted file mode 100644 index 30379b8fa1afaf..00000000000000 --- a/Misc/NEWS.d/next/Library/2023-11-08-18-53-07.gh-issue-68166.1iTh4Y.rst +++ /dev/null @@ -1,2 +0,0 @@ -Add support of the "vsapi" element type in -:meth:`tkinter.ttk.Style.element_create`. diff --git a/Misc/NEWS.d/next/Library/2023-11-09-11-07-34.gh-issue-111874.dzYc3j.rst b/Misc/NEWS.d/next/Library/2023-11-09-11-07-34.gh-issue-111874.dzYc3j.rst deleted file mode 100644 index 50408202a7a5a1..00000000000000 --- a/Misc/NEWS.d/next/Library/2023-11-09-11-07-34.gh-issue-111874.dzYc3j.rst +++ /dev/null @@ -1,4 +0,0 @@ -When creating a :class:`typing.NamedTuple` class, ensure -:func:`~object.__set_name__` is called on all objects that define -``__set_name__`` and exist in the values of the ``NamedTuple`` class's class -dictionary. Patch by Alex Waygood. diff --git a/Misc/NEWS.d/next/Library/2023-11-15-01-36-04.gh-issue-106922.qslOVH.rst b/Misc/NEWS.d/next/Library/2023-11-15-01-36-04.gh-issue-106922.qslOVH.rst deleted file mode 100644 index b68e75ab87cd0b..00000000000000 --- a/Misc/NEWS.d/next/Library/2023-11-15-01-36-04.gh-issue-106922.qslOVH.rst +++ /dev/null @@ -1 +0,0 @@ -Display multiple lines with ``traceback`` when errors span multiple lines. diff --git a/Misc/NEWS.d/next/Library/2023-11-15-04-53-37.gh-issue-112105.I3RcVN.rst b/Misc/NEWS.d/next/Library/2023-11-15-04-53-37.gh-issue-112105.I3RcVN.rst deleted file mode 100644 index 4243dcb190434f..00000000000000 --- a/Misc/NEWS.d/next/Library/2023-11-15-04-53-37.gh-issue-112105.I3RcVN.rst +++ /dev/null @@ -1 +0,0 @@ -Make :func:`readline.set_completer_delims` work with libedit diff --git a/Misc/NEWS.d/next/Library/2023-11-16-10-42-15.gh-issue-112139.WpHosf.rst b/Misc/NEWS.d/next/Library/2023-11-16-10-42-15.gh-issue-112139.WpHosf.rst deleted file mode 100644 index 090dc8847d9556..00000000000000 --- a/Misc/NEWS.d/next/Library/2023-11-16-10-42-15.gh-issue-112139.WpHosf.rst +++ /dev/null @@ -1,3 +0,0 @@ -Add :meth:`Signature.format` to format signatures to string with extra options. -And use it in :mod:`pydoc` to render more readable signatures that have new -lines between parameters. diff --git a/Misc/NEWS.d/next/Library/2023-11-16-17-18-09.gh-issue-112137.QvjGjN.rst b/Misc/NEWS.d/next/Library/2023-11-16-17-18-09.gh-issue-112137.QvjGjN.rst deleted file mode 100644 index 6b61d051966846..00000000000000 --- a/Misc/NEWS.d/next/Library/2023-11-16-17-18-09.gh-issue-112137.QvjGjN.rst +++ /dev/null @@ -1 +0,0 @@ -Change :mod:`dis` output to display logical labels for jump targets instead of offsets. diff --git a/Misc/NEWS.d/next/Library/2023-11-18-16-30-21.gh-issue-112240.YXS0tj.rst b/Misc/NEWS.d/next/Library/2023-11-18-16-30-21.gh-issue-112240.YXS0tj.rst new file mode 100644 index 00000000000000..686f0311e80dcb --- /dev/null +++ b/Misc/NEWS.d/next/Library/2023-11-18-16-30-21.gh-issue-112240.YXS0tj.rst @@ -0,0 +1,2 @@ +Add option to calendar module CLI to specify the weekday to start each week. +Patch by Steven Ward. diff --git a/Misc/NEWS.d/next/Library/2023-11-21-02-58-14.gh-issue-77621.MYv5XS.rst b/Misc/NEWS.d/next/Library/2023-11-21-02-58-14.gh-issue-77621.MYv5XS.rst deleted file mode 100644 index f3e6efc389afca..00000000000000 --- a/Misc/NEWS.d/next/Library/2023-11-21-02-58-14.gh-issue-77621.MYv5XS.rst +++ /dev/null @@ -1,2 +0,0 @@ -Slightly improve the import time of the :mod:`pathlib` module by deferring -some imports. Patch by Barney Gale. diff --git a/Misc/NEWS.d/next/Library/2023-11-22-19-43-54.gh-issue-112292.5nDU87.rst b/Misc/NEWS.d/next/Library/2023-11-22-19-43-54.gh-issue-112292.5nDU87.rst deleted file mode 100644 index 8345e33791cde0..00000000000000 --- a/Misc/NEWS.d/next/Library/2023-11-22-19-43-54.gh-issue-112292.5nDU87.rst +++ /dev/null @@ -1,2 +0,0 @@ -Fix a crash in :mod:`readline` when imported from a sub interpreter. Patch -by Anthony Shaw diff --git a/Misc/NEWS.d/next/Library/2023-11-22-23-08-47.gh-issue-81620.mfZ2Wf.rst b/Misc/NEWS.d/next/Library/2023-11-22-23-08-47.gh-issue-81620.mfZ2Wf.rst deleted file mode 100644 index ff35806e4d5ed6..00000000000000 --- a/Misc/NEWS.d/next/Library/2023-11-22-23-08-47.gh-issue-81620.mfZ2Wf.rst +++ /dev/null @@ -1 +0,0 @@ -Add extra tests for :func:`random.binomialvariate` diff --git a/Misc/NEWS.d/next/Library/2023-11-23-10-41-21.gh-issue-112332.rhTBaa.rst b/Misc/NEWS.d/next/Library/2023-11-23-10-41-21.gh-issue-112332.rhTBaa.rst deleted file mode 100644 index bd686ad052e5b2..00000000000000 --- a/Misc/NEWS.d/next/Library/2023-11-23-10-41-21.gh-issue-112332.rhTBaa.rst +++ /dev/null @@ -1,2 +0,0 @@ -Deprecate the ``exc_type`` field of :class:`traceback.TracebackException`. -Add ``exc_type_str`` to replace it. diff --git a/Misc/NEWS.d/next/Library/2023-11-23-12-37-22.gh-issue-112137.kM46Q6.rst b/Misc/NEWS.d/next/Library/2023-11-23-12-37-22.gh-issue-112137.kM46Q6.rst deleted file mode 100644 index 1b2e41ae96ff09..00000000000000 --- a/Misc/NEWS.d/next/Library/2023-11-23-12-37-22.gh-issue-112137.kM46Q6.rst +++ /dev/null @@ -1 +0,0 @@ -Change :mod:`dis` output to display no-lineno as "--" instead of "None". diff --git a/Misc/NEWS.d/next/Library/2023-11-23-17-25-27.gh-issue-112345.FFApHx.rst b/Misc/NEWS.d/next/Library/2023-11-23-17-25-27.gh-issue-112345.FFApHx.rst deleted file mode 100644 index b2b9894e6bef3a..00000000000000 --- a/Misc/NEWS.d/next/Library/2023-11-23-17-25-27.gh-issue-112345.FFApHx.rst +++ /dev/null @@ -1,3 +0,0 @@ -Improve error message when trying to call :func:`issubclass` against a -:class:`typing.Protocol` that has non-method members. -Patch by Randolf Scholz. diff --git a/Misc/NEWS.d/next/Library/2023-11-24-09-27-01.gh-issue-112361.kYtnHW.rst b/Misc/NEWS.d/next/Library/2023-11-24-09-27-01.gh-issue-112361.kYtnHW.rst deleted file mode 100644 index 5a83f93f9fbec8..00000000000000 --- a/Misc/NEWS.d/next/Library/2023-11-24-09-27-01.gh-issue-112361.kYtnHW.rst +++ /dev/null @@ -1,2 +0,0 @@ -Speed up a small handful of :mod:`pathlib` methods by removing some -temporary objects. diff --git a/Misc/NEWS.d/next/Library/2023-11-24-19-08-50.gh-issue-112343.RarGFC.rst b/Misc/NEWS.d/next/Library/2023-11-24-19-08-50.gh-issue-112343.RarGFC.rst new file mode 100644 index 00000000000000..aaa50fce3ac962 --- /dev/null +++ b/Misc/NEWS.d/next/Library/2023-11-24-19-08-50.gh-issue-112343.RarGFC.rst @@ -0,0 +1 @@ +Improve handling of pdb convenience variables to avoid replacing string contents. diff --git a/Misc/NEWS.d/next/Library/2023-11-24-21-00-24.gh-issue-94722.GMIQIn.rst b/Misc/NEWS.d/next/Library/2023-11-24-21-00-24.gh-issue-94722.GMIQIn.rst deleted file mode 100644 index 41bd57f46ed82a..00000000000000 --- a/Misc/NEWS.d/next/Library/2023-11-24-21-00-24.gh-issue-94722.GMIQIn.rst +++ /dev/null @@ -1,2 +0,0 @@ -Fix bug where comparison between instances of :class:`~doctest.DocTest` fails if -one of them has ``None`` as its lineno. diff --git a/Misc/NEWS.d/next/Library/2023-11-25-20-29-28.gh-issue-112405.cOtzxC.rst b/Misc/NEWS.d/next/Library/2023-11-25-20-29-28.gh-issue-112405.cOtzxC.rst deleted file mode 100644 index f6f1bee2a0c38f..00000000000000 --- a/Misc/NEWS.d/next/Library/2023-11-25-20-29-28.gh-issue-112405.cOtzxC.rst +++ /dev/null @@ -1 +0,0 @@ -Optimize :meth:`pathlib.PurePath.relative_to`. Patch by Alex Waygood. diff --git a/Misc/NEWS.d/next/Library/2023-11-26-13-26-56.gh-issue-112358.smhaeZ.rst b/Misc/NEWS.d/next/Library/2023-11-26-13-26-56.gh-issue-112358.smhaeZ.rst deleted file mode 100644 index e473ded46a1309..00000000000000 --- a/Misc/NEWS.d/next/Library/2023-11-26-13-26-56.gh-issue-112358.smhaeZ.rst +++ /dev/null @@ -1,2 +0,0 @@ -Revert change to :class:`struct.Struct` initialization that broke some cases -of subclassing. diff --git a/Misc/NEWS.d/next/Library/2023-11-26-13-44-19.gh-issue-112414.kx2E7S.rst b/Misc/NEWS.d/next/Library/2023-11-26-13-44-19.gh-issue-112414.kx2E7S.rst deleted file mode 100644 index 058e5a33227e5a..00000000000000 --- a/Misc/NEWS.d/next/Library/2023-11-26-13-44-19.gh-issue-112414.kx2E7S.rst +++ /dev/null @@ -1,3 +0,0 @@ -Fix regression in Python 3.12 where calling :func:`repr` on a module that -had been imported using a custom :term:`loader` could fail with -:exc:`AttributeError`. Patch by Alex Waygood. diff --git a/Misc/NEWS.d/next/Library/2023-11-27-12-41-23.gh-issue-63284.q2Qi9q.rst b/Misc/NEWS.d/next/Library/2023-11-27-12-41-23.gh-issue-63284.q2Qi9q.rst deleted file mode 100644 index abb57dccd5a91a..00000000000000 --- a/Misc/NEWS.d/next/Library/2023-11-27-12-41-23.gh-issue-63284.q2Qi9q.rst +++ /dev/null @@ -1 +0,0 @@ -Added support for TLS-PSK (pre-shared key) mode to the :mod:`ssl` module. diff --git a/Misc/NEWS.d/next/Library/2023-11-27-19-54-43.gh-issue-59013.chpQ0e.rst b/Misc/NEWS.d/next/Library/2023-11-27-19-54-43.gh-issue-59013.chpQ0e.rst new file mode 100644 index 00000000000000..a2be2fb8eacf17 --- /dev/null +++ b/Misc/NEWS.d/next/Library/2023-11-27-19-54-43.gh-issue-59013.chpQ0e.rst @@ -0,0 +1 @@ +Set breakpoint on the first executable line of the function, instead of the line of function definition when the user do ``break func`` using :mod:`pdb` diff --git a/Misc/NEWS.d/next/Library/2023-11-28-02-39-30.gh-issue-101336.ya433z.rst b/Misc/NEWS.d/next/Library/2023-11-28-02-39-30.gh-issue-101336.ya433z.rst deleted file mode 100644 index c222febae6b554..00000000000000 --- a/Misc/NEWS.d/next/Library/2023-11-28-02-39-30.gh-issue-101336.ya433z.rst +++ /dev/null @@ -1 +0,0 @@ -Add ``keep_alive`` keyword parameter for :meth:`AbstractEventLoop.create_server` and :meth:`BaseEventLoop.create_server`. diff --git a/Misc/NEWS.d/next/Library/2023-11-28-20-01-33.gh-issue-112509.QtoKed.rst b/Misc/NEWS.d/next/Library/2023-11-28-20-01-33.gh-issue-112509.QtoKed.rst deleted file mode 100644 index a16d67e7776bcb..00000000000000 --- a/Misc/NEWS.d/next/Library/2023-11-28-20-01-33.gh-issue-112509.QtoKed.rst +++ /dev/null @@ -1,3 +0,0 @@ -Fix edge cases that could cause a key to be present in both the -``__required_keys__`` and ``__optional_keys__`` attributes of a -:class:`typing.TypedDict`. Patch by Jelle Zijlstra. diff --git a/Misc/NEWS.d/next/Library/2023-11-28-20-47-39.gh-issue-112328.Z2AxEY.rst b/Misc/NEWS.d/next/Library/2023-11-28-20-47-39.gh-issue-112328.Z2AxEY.rst deleted file mode 100644 index 6e6902486b7bc9..00000000000000 --- a/Misc/NEWS.d/next/Library/2023-11-28-20-47-39.gh-issue-112328.Z2AxEY.rst +++ /dev/null @@ -1,2 +0,0 @@ -[Enum] Make ``EnumDict``, ``EnumDict.member_names``, -``EnumType._add_alias_`` and ``EnumType._add_value_alias_`` public. diff --git a/Misc/NEWS.d/next/Library/2023-11-29-02-26-32.gh-issue-112510.j-zXGc.rst b/Misc/NEWS.d/next/Library/2023-11-29-02-26-32.gh-issue-112510.j-zXGc.rst deleted file mode 100644 index 02de6fa80c1b3e..00000000000000 --- a/Misc/NEWS.d/next/Library/2023-11-29-02-26-32.gh-issue-112510.j-zXGc.rst +++ /dev/null @@ -1 +0,0 @@ -Add :data:`readline.backend` for the backend readline uses (``editline`` or ``readline``) diff --git a/Misc/NEWS.d/next/Library/2023-11-29-10-51-41.gh-issue-112516.rFKUKN.rst b/Misc/NEWS.d/next/Library/2023-11-29-10-51-41.gh-issue-112516.rFKUKN.rst deleted file mode 100644 index 530cf992dcd77a..00000000000000 --- a/Misc/NEWS.d/next/Library/2023-11-29-10-51-41.gh-issue-112516.rFKUKN.rst +++ /dev/null @@ -1 +0,0 @@ -Update the bundled copy of pip to version 23.3.1. diff --git a/Misc/NEWS.d/next/Library/2023-12-01-08-28-09.gh-issue-112578.bfNbfi.rst b/Misc/NEWS.d/next/Library/2023-12-01-08-28-09.gh-issue-112578.bfNbfi.rst deleted file mode 100644 index 1de5b1fe26ce6d..00000000000000 --- a/Misc/NEWS.d/next/Library/2023-12-01-08-28-09.gh-issue-112578.bfNbfi.rst +++ /dev/null @@ -1 +0,0 @@ -Fix a spurious :exc:`RuntimeWarning` when executing the :mod:`zipfile` module. diff --git a/Misc/NEWS.d/next/Library/2023-12-01-16-09-59.gh-issue-81194.FFad1c.rst b/Misc/NEWS.d/next/Library/2023-12-01-16-09-59.gh-issue-81194.FFad1c.rst deleted file mode 100644 index feb7a8643b97f6..00000000000000 --- a/Misc/NEWS.d/next/Library/2023-12-01-16-09-59.gh-issue-81194.FFad1c.rst +++ /dev/null @@ -1,3 +0,0 @@ -Fix a crash in :func:`socket.if_indextoname` with specific value (UINT_MAX). -Fix an integer overflow in :func:`socket.if_indextoname` on 64-bit -non-Windows platforms. diff --git a/Misc/NEWS.d/next/Library/2023-12-01-18-05-09.gh-issue-110190.5bf-c9.rst b/Misc/NEWS.d/next/Library/2023-12-01-18-05-09.gh-issue-110190.5bf-c9.rst deleted file mode 100644 index 730b9d49119805..00000000000000 --- a/Misc/NEWS.d/next/Library/2023-12-01-18-05-09.gh-issue-110190.5bf-c9.rst +++ /dev/null @@ -1 +0,0 @@ -Fix ctypes structs with array on Arm platform by setting ``MAX_STRUCT_SIZE`` to 32 in stgdict. Patch by Diego Russo. diff --git a/Misc/NEWS.d/next/Library/2023-12-01-21-05-46.gh-issue-112334.DmNXKh.rst b/Misc/NEWS.d/next/Library/2023-12-01-21-05-46.gh-issue-112334.DmNXKh.rst deleted file mode 100644 index 3a53a8bf84230f..00000000000000 --- a/Misc/NEWS.d/next/Library/2023-12-01-21-05-46.gh-issue-112334.DmNXKh.rst +++ /dev/null @@ -1,11 +0,0 @@ -Fixed a performance regression in 3.12's :mod:`subprocess` on Linux where it -would no longer use the fast-path ``vfork()`` system call when it could have -due to a logic bug, instead falling back to the safe but slower ``fork()``. - -Also fixed a second 3.12.0 potential security bug. If a value of -``extra_groups=[]`` was passed to :mod:`subprocess.Popen` or related APIs, -the underlying ``setgroups(0, NULL)`` system call to clear the groups list -would not be made in the child process prior to ``exec()``. - -This was identified via code inspection in the process of fixing the first -bug. diff --git a/Misc/NEWS.d/next/Library/2023-12-02-12-55-17.gh-issue-112618.7_FT8-.rst b/Misc/NEWS.d/next/Library/2023-12-02-12-55-17.gh-issue-112618.7_FT8-.rst deleted file mode 100644 index c732de15609c96..00000000000000 --- a/Misc/NEWS.d/next/Library/2023-12-02-12-55-17.gh-issue-112618.7_FT8-.rst +++ /dev/null @@ -1,2 +0,0 @@ -Fix a caching bug relating to :data:`typing.Annotated`. -``Annotated[str, True]`` is no longer identical to ``Annotated[str, 1]``. diff --git a/Misc/NEWS.d/next/Library/2023-12-03-01-01-52.gh-issue-112622.1Z8cpx.rst b/Misc/NEWS.d/next/Library/2023-12-03-01-01-52.gh-issue-112622.1Z8cpx.rst deleted file mode 100644 index 91c88bac334dcb..00000000000000 --- a/Misc/NEWS.d/next/Library/2023-12-03-01-01-52.gh-issue-112622.1Z8cpx.rst +++ /dev/null @@ -1,2 +0,0 @@ -Ensure ``name`` parameter is passed to event loop in -:func:`asyncio.create_task`. diff --git a/Misc/NEWS.d/next/Library/2023-12-03-12-41-48.gh-issue-112645.blMsKf.rst b/Misc/NEWS.d/next/Library/2023-12-03-12-41-48.gh-issue-112645.blMsKf.rst deleted file mode 100644 index 4e8f6ebdb882e0..00000000000000 --- a/Misc/NEWS.d/next/Library/2023-12-03-12-41-48.gh-issue-112645.blMsKf.rst +++ /dev/null @@ -1 +0,0 @@ -Remove deprecation error on passing ``onerror`` to :func:`shutil.rmtree`. diff --git a/Misc/NEWS.d/next/Library/2023-12-04-14-05-24.gh-issue-74690.eODKRm.rst b/Misc/NEWS.d/next/Library/2023-12-04-14-05-24.gh-issue-74690.eODKRm.rst deleted file mode 100644 index 36d793f787302e..00000000000000 --- a/Misc/NEWS.d/next/Library/2023-12-04-14-05-24.gh-issue-74690.eODKRm.rst +++ /dev/null @@ -1,5 +0,0 @@ -Speedup :func:`isinstance` checks by roughly 20% for -:func:`runtime-checkable protocols ` -that only have one callable member. -Speedup :func:`issubclass` checks for these protocols by roughly 10%. -Patch by Alex Waygood. diff --git a/Misc/NEWS.d/next/Library/2023-12-04-16-45-11.gh-issue-74690.pQYP5U.rst b/Misc/NEWS.d/next/Library/2023-12-04-16-45-11.gh-issue-74690.pQYP5U.rst deleted file mode 100644 index 8102f02e941c29..00000000000000 --- a/Misc/NEWS.d/next/Library/2023-12-04-16-45-11.gh-issue-74690.pQYP5U.rst +++ /dev/null @@ -1,2 +0,0 @@ -Speedup :func:`issubclass` checks against simple :func:`runtime-checkable -protocols ` by around 6%. Patch by Alex Waygood. diff --git a/Misc/NEWS.d/next/Library/2023-12-04-21-30-34.gh-issue-112727.jpgNRB.rst b/Misc/NEWS.d/next/Library/2023-12-04-21-30-34.gh-issue-112727.jpgNRB.rst deleted file mode 100644 index bbe7aae5732d9a..00000000000000 --- a/Misc/NEWS.d/next/Library/2023-12-04-21-30-34.gh-issue-112727.jpgNRB.rst +++ /dev/null @@ -1 +0,0 @@ -Speed up :meth:`pathlib.Path.absolute`. Patch by Barney Gale. diff --git a/Misc/NEWS.d/next/Library/2023-12-05-01-19-28.gh-issue-112736.rdHDrU.rst b/Misc/NEWS.d/next/Library/2023-12-05-01-19-28.gh-issue-112736.rdHDrU.rst deleted file mode 100644 index 6c09e622923af8..00000000000000 --- a/Misc/NEWS.d/next/Library/2023-12-05-01-19-28.gh-issue-112736.rdHDrU.rst +++ /dev/null @@ -1 +0,0 @@ -The use of del-safe symbols in ``subprocess`` was refactored to allow for use in cross-platform build environments. diff --git a/Misc/NEWS.d/next/Library/2023-12-05-16-20-40.gh-issue-94692.-e5C3c.rst b/Misc/NEWS.d/next/Library/2023-12-05-16-20-40.gh-issue-94692.-e5C3c.rst deleted file mode 100644 index c67ba6c9ececdb..00000000000000 --- a/Misc/NEWS.d/next/Library/2023-12-05-16-20-40.gh-issue-94692.-e5C3c.rst +++ /dev/null @@ -1,4 +0,0 @@ -:func:`shutil.rmtree` now only catches OSError exceptions. Previously a -symlink attack resistant version of ``shutil.rmtree()`` could ignore or pass -to the error handler arbitrary exception when invalid arguments were -provided. diff --git a/Misc/NEWS.d/next/Library/2023-12-05-18-57-53.gh-issue-79325.P2vMVK.rst b/Misc/NEWS.d/next/Library/2023-12-05-18-57-53.gh-issue-79325.P2vMVK.rst deleted file mode 100644 index f3c32d27b5fe66..00000000000000 --- a/Misc/NEWS.d/next/Library/2023-12-05-18-57-53.gh-issue-79325.P2vMVK.rst +++ /dev/null @@ -1,2 +0,0 @@ -Fix an infinite recursion error in :func:`tempfile.TemporaryDirectory` -cleanup on Windows. diff --git a/Misc/NEWS.d/next/Library/2023-12-06-14-06-14.gh-issue-51944.-5qq_L.rst b/Misc/NEWS.d/next/Library/2023-12-06-14-06-14.gh-issue-51944.-5qq_L.rst deleted file mode 100644 index 821eefa7cffcd5..00000000000000 --- a/Misc/NEWS.d/next/Library/2023-12-06-14-06-14.gh-issue-51944.-5qq_L.rst +++ /dev/null @@ -1,6 +0,0 @@ -Add the following constants to the :mod:`termios` module. These values are -present in macOS system headers: ``ALTWERASE``, ``B14400``, ``B28800``, -``B7200``, ``B76800``, ``CCAR_OFLOW``, ``CCTS_OFLOW``, ``CDSR_OFLOW``, -``CDTR_IFLOW``, ``CIGNORE``, ``CRTS_IFLOW``, ``EXTPROC``, ``IUTF8``, -``MDMBUF``, ``NL2``, ``NL3``, ``NOKERNINFO``, ``ONOEOT``, ``OXTABS``, -``VDSUSP``, ``VSTATUS``. diff --git a/Misc/NEWS.d/next/Library/2023-12-07-16-55-41.gh-issue-87286.MILC9_.rst b/Misc/NEWS.d/next/Library/2023-12-07-16-55-41.gh-issue-87286.MILC9_.rst deleted file mode 100644 index bfeec3c95207cb..00000000000000 --- a/Misc/NEWS.d/next/Library/2023-12-07-16-55-41.gh-issue-87286.MILC9_.rst +++ /dev/null @@ -1,3 +0,0 @@ -Added :const:`LOG_FTP`, :const:`LOG_NETINFO`, :const:`LOG_REMOTEAUTH`, -:const:`LOG_INSTALL`, :const:`LOG_RAS`, and :const:`LOG_LAUNCHD` tot the -:mod:`syslog` module, all of them constants on used on macOS. diff --git a/Misc/NEWS.d/next/Library/2023-12-08-11-17-17.gh-issue-112540.Pm5egX.rst b/Misc/NEWS.d/next/Library/2023-12-08-11-17-17.gh-issue-112540.Pm5egX.rst deleted file mode 100644 index 263b13d1762bf1..00000000000000 --- a/Misc/NEWS.d/next/Library/2023-12-08-11-17-17.gh-issue-112540.Pm5egX.rst +++ /dev/null @@ -1,2 +0,0 @@ -The statistics.geometric_mean() function now returns zero for datasets -containing a zero. Formerly, it would raise an exception. diff --git a/Misc/NEWS.d/next/Library/2023-12-09-23-31-17.gh-issue-112919.S5k9QN.rst b/Misc/NEWS.d/next/Library/2023-12-09-23-31-17.gh-issue-112919.S5k9QN.rst new file mode 100644 index 00000000000000..3e99d480139cbe --- /dev/null +++ b/Misc/NEWS.d/next/Library/2023-12-09-23-31-17.gh-issue-112919.S5k9QN.rst @@ -0,0 +1,2 @@ +Speed-up :func:`datetime.datetime.replace`, :func:`datetime.date.replace` and +:func:`datetime.time.replace`. diff --git a/Misc/NEWS.d/next/Library/2023-12-11-14-12-46.gh-issue-110190.e0iEUa.rst b/Misc/NEWS.d/next/Library/2023-12-11-14-12-46.gh-issue-110190.e0iEUa.rst deleted file mode 100644 index 3bfed1e0f1dc91..00000000000000 --- a/Misc/NEWS.d/next/Library/2023-12-11-14-12-46.gh-issue-110190.e0iEUa.rst +++ /dev/null @@ -1 +0,0 @@ -Fix ctypes structs with array on PPC64LE platform by setting ``MAX_STRUCT_SIZE`` to 64 in stgdict. Patch by Diego Russo. diff --git a/Misc/NEWS.d/next/Library/2023-12-11-16-13-15.gh-issue-112970.87jmKP.rst b/Misc/NEWS.d/next/Library/2023-12-11-16-13-15.gh-issue-112970.87jmKP.rst deleted file mode 100644 index 58ca26af511383..00000000000000 --- a/Misc/NEWS.d/next/Library/2023-12-11-16-13-15.gh-issue-112970.87jmKP.rst +++ /dev/null @@ -1 +0,0 @@ -Use :c:func:`!closefrom` on Linux where available (e.g. glibc-2.34), rather than only FreeBSD. diff --git a/Misc/NEWS.d/next/Library/2023-12-12-05-48-17.gh-issue-112989.ZAa_eq.rst b/Misc/NEWS.d/next/Library/2023-12-12-05-48-17.gh-issue-112989.ZAa_eq.rst deleted file mode 100644 index ceeab8cc7d6bec..00000000000000 --- a/Misc/NEWS.d/next/Library/2023-12-12-05-48-17.gh-issue-112989.ZAa_eq.rst +++ /dev/null @@ -1 +0,0 @@ -Reduce overhead to connect sockets with :mod:`asyncio` SelectorEventLoop. diff --git a/Misc/NEWS.d/next/Library/2023-12-12-16-32-55.gh-issue-112962.ZZWXZn.rst b/Misc/NEWS.d/next/Library/2023-12-12-16-32-55.gh-issue-112962.ZZWXZn.rst deleted file mode 100644 index b99e6bc90ae791..00000000000000 --- a/Misc/NEWS.d/next/Library/2023-12-12-16-32-55.gh-issue-112962.ZZWXZn.rst +++ /dev/null @@ -1,3 +0,0 @@ -:mod:`dis` module functions add cache information to the -:class:`~dis.Instruction` instance rather than creating fake -:class:`~dis.Instruction` instances to represent the cache entries. diff --git a/Misc/NEWS.d/next/Library/2023-12-13-17-08-21.gh-issue-59616.JNlWSs.rst b/Misc/NEWS.d/next/Library/2023-12-13-17-08-21.gh-issue-59616.JNlWSs.rst deleted file mode 100644 index 793ae63b4c1ff5..00000000000000 --- a/Misc/NEWS.d/next/Library/2023-12-13-17-08-21.gh-issue-59616.JNlWSs.rst +++ /dev/null @@ -1,3 +0,0 @@ -Add support of :func:`os.lchmod` and the *follow_symlinks* argument in -:func:`os.chmod` on Windows. Note that the default value of *follow_symlinks* -in :func:`!os.lchmod` is ``False`` on Windows. diff --git a/Misc/NEWS.d/next/Library/2023-12-15-09-51-41.gh-issue-113175.RHsNwE.rst b/Misc/NEWS.d/next/Library/2023-12-15-09-51-41.gh-issue-113175.RHsNwE.rst deleted file mode 100644 index 1b43803d1a7aa4..00000000000000 --- a/Misc/NEWS.d/next/Library/2023-12-15-09-51-41.gh-issue-113175.RHsNwE.rst +++ /dev/null @@ -1,5 +0,0 @@ -Sync with importlib_metadata 7.0, including improved type annotations, fixed -issue with symlinked packages in ``package_distributions``, added -``EntryPoints.__repr__``, introduced the ``diagnose`` script, added -``Distribution.origin`` property, and removed deprecated ``EntryPoint`` -access by numeric index (tuple behavior). diff --git a/Misc/NEWS.d/next/Library/2023-12-15-12-35-28.gh-issue-61648.G-4pz0.rst b/Misc/NEWS.d/next/Library/2023-12-15-12-35-28.gh-issue-61648.G-4pz0.rst deleted file mode 100644 index c841e5c7f7683a..00000000000000 --- a/Misc/NEWS.d/next/Library/2023-12-15-12-35-28.gh-issue-61648.G-4pz0.rst +++ /dev/null @@ -1 +0,0 @@ -Detect line numbers of properties in doctests. diff --git a/Misc/NEWS.d/next/Library/2023-12-15-18-10-26.gh-issue-113202.xv_Ww8.rst b/Misc/NEWS.d/next/Library/2023-12-15-18-10-26.gh-issue-113202.xv_Ww8.rst deleted file mode 100644 index 44f26aef60a33a..00000000000000 --- a/Misc/NEWS.d/next/Library/2023-12-15-18-10-26.gh-issue-113202.xv_Ww8.rst +++ /dev/null @@ -1 +0,0 @@ -Add a ``strict`` option to ``batched()`` in the ``itertools`` module. diff --git a/Misc/NEWS.d/next/Library/2023-12-15-18-13-59.gh-issue-113119.al-569.rst b/Misc/NEWS.d/next/Library/2023-12-15-18-13-59.gh-issue-113119.al-569.rst deleted file mode 100644 index 94087b00515e97..00000000000000 --- a/Misc/NEWS.d/next/Library/2023-12-15-18-13-59.gh-issue-113119.al-569.rst +++ /dev/null @@ -1,2 +0,0 @@ -:func:`os.posix_spawn` now accepts ``env=None``, which makes the newly spawned -process use the current process environment. Patch by Jakub Kulik. diff --git a/Misc/NEWS.d/next/Library/2023-12-16-01-10-47.gh-issue-113199.oDjnjL.rst b/Misc/NEWS.d/next/Library/2023-12-16-01-10-47.gh-issue-113199.oDjnjL.rst deleted file mode 100644 index d8e0b1731d1e3b..00000000000000 --- a/Misc/NEWS.d/next/Library/2023-12-16-01-10-47.gh-issue-113199.oDjnjL.rst +++ /dev/null @@ -1,3 +0,0 @@ -Make ``http.client.HTTPResponse.read1`` and -``http.client.HTTPResponse.readline`` close IO after reading all data when -content length is known. Patch by Illia Volochii. diff --git a/Misc/NEWS.d/next/Library/2023-12-16-10-58-34.gh-issue-113117.0zF7bH.rst b/Misc/NEWS.d/next/Library/2023-12-16-10-58-34.gh-issue-113117.0zF7bH.rst deleted file mode 100644 index 718226a0021efe..00000000000000 --- a/Misc/NEWS.d/next/Library/2023-12-16-10-58-34.gh-issue-113117.0zF7bH.rst +++ /dev/null @@ -1,4 +0,0 @@ -The :mod:`subprocess` module can now use the :func:`os.posix_spawn` function -with ``close_fds=True`` on platforms where -``posix_spawn_file_actions_addclosefrom_np`` is available. -Patch by Jakub Kulik. diff --git a/Misc/NEWS.d/next/Library/2023-12-16-23-56-42.gh-issue-113149.7LWgTS.rst b/Misc/NEWS.d/next/Library/2023-12-16-23-56-42.gh-issue-113149.7LWgTS.rst deleted file mode 100644 index 0faa67fefabeca..00000000000000 --- a/Misc/NEWS.d/next/Library/2023-12-16-23-56-42.gh-issue-113149.7LWgTS.rst +++ /dev/null @@ -1,2 +0,0 @@ -Improve error message when a JSON array or object contains a trailing comma. -Patch by Carson Radtke. diff --git a/Misc/NEWS.d/next/Library/2023-12-17-13-56-30.gh-issue-87264.RgfHCv.rst b/Misc/NEWS.d/next/Library/2023-12-17-13-56-30.gh-issue-87264.RgfHCv.rst deleted file mode 100644 index fa987d4f0af9ba..00000000000000 --- a/Misc/NEWS.d/next/Library/2023-12-17-13-56-30.gh-issue-87264.RgfHCv.rst +++ /dev/null @@ -1 +0,0 @@ -Fixed tarfile list() method to show file type. diff --git a/Misc/NEWS.d/next/Library/2023-12-18-09-47-54.gh-issue-113246.em930H.rst b/Misc/NEWS.d/next/Library/2023-12-18-09-47-54.gh-issue-113246.em930H.rst deleted file mode 100644 index 167bb37c0e0643..00000000000000 --- a/Misc/NEWS.d/next/Library/2023-12-18-09-47-54.gh-issue-113246.em930H.rst +++ /dev/null @@ -1 +0,0 @@ -Update bundled pip to 23.3.2. diff --git a/Misc/NEWS.d/next/Library/2023-12-20-21-18-51.gh-issue-113214.JcV9Mn.rst b/Misc/NEWS.d/next/Library/2023-12-20-21-18-51.gh-issue-113214.JcV9Mn.rst deleted file mode 100644 index 6db74cda166e92..00000000000000 --- a/Misc/NEWS.d/next/Library/2023-12-20-21-18-51.gh-issue-113214.JcV9Mn.rst +++ /dev/null @@ -1 +0,0 @@ -Fix an ``AttributeError`` during asyncio SSL protocol aborts in SSL-over-SSL scenarios. diff --git a/Misc/NEWS.d/next/Library/2024-01-04-20-58-17.gh-issue-113225.-nyJM4.rst b/Misc/NEWS.d/next/Library/2024-01-04-20-58-17.gh-issue-113225.-nyJM4.rst new file mode 100644 index 00000000000000..0c07f42fd065d2 --- /dev/null +++ b/Misc/NEWS.d/next/Library/2024-01-04-20-58-17.gh-issue-113225.-nyJM4.rst @@ -0,0 +1,2 @@ +Speed up :meth:`pathlib.Path.walk` by using :attr:`os.DirEntry.path` where +possible. diff --git a/Misc/NEWS.d/next/Library/2024-01-05-16-27-34.gh-issue-113732.fgDRXA.rst b/Misc/NEWS.d/next/Library/2024-01-05-16-27-34.gh-issue-113732.fgDRXA.rst new file mode 100644 index 00000000000000..7582603dcf95f5 --- /dev/null +++ b/Misc/NEWS.d/next/Library/2024-01-05-16-27-34.gh-issue-113732.fgDRXA.rst @@ -0,0 +1,2 @@ +Fix support of :data:`~csv.QUOTE_NOTNULL` and :data:`~csv.QUOTE_STRINGS` in +:func:`csv.reader`. diff --git a/Misc/NEWS.d/next/Library/2024-01-07-21-04-24.gh-issue-113796.6iNsCR.rst b/Misc/NEWS.d/next/Library/2024-01-07-21-04-24.gh-issue-113796.6iNsCR.rst new file mode 100644 index 00000000000000..e9d4aba9906677 --- /dev/null +++ b/Misc/NEWS.d/next/Library/2024-01-07-21-04-24.gh-issue-113796.6iNsCR.rst @@ -0,0 +1,3 @@ +Add more validation checks in the :class:`csv.Dialect` constructor. +:exc:`ValueError` is now raised if the same character is used in different +roles. diff --git a/Misc/NEWS.d/next/Library/2024-01-11-20-47-49.gh-issue-113951.AzlqFK.rst b/Misc/NEWS.d/next/Library/2024-01-11-20-47-49.gh-issue-113951.AzlqFK.rst new file mode 100644 index 00000000000000..e683472e59b8a4 --- /dev/null +++ b/Misc/NEWS.d/next/Library/2024-01-11-20-47-49.gh-issue-113951.AzlqFK.rst @@ -0,0 +1,7 @@ +Fix the behavior of ``tag_unbind()`` methods of :class:`tkinter.Text` and +:class:`tkinter.Canvas` classes with three arguments. Previously, +``widget.tag_unbind(tag, sequence, funcid)`` destroyed the current binding +for *sequence*, leaving *sequence* unbound, and deleted the *funcid* +command. Now it removes only *funcid* from the binding for *sequence*, +keeping other commands, and deletes the *funcid* command. It leaves +*sequence* unbound only if *funcid* was the last bound command. diff --git a/Misc/NEWS.d/next/Library/2024-01-12-09-35-07.gh-issue-112202.t_0V1m.rst b/Misc/NEWS.d/next/Library/2024-01-12-09-35-07.gh-issue-112202.t_0V1m.rst new file mode 100644 index 00000000000000..9abde13bbf8571 --- /dev/null +++ b/Misc/NEWS.d/next/Library/2024-01-12-09-35-07.gh-issue-112202.t_0V1m.rst @@ -0,0 +1 @@ +Ensure that a :func:`asyncio.Condition.notify` call does not get lost if the awakened ``Task`` is simultaneously cancelled or encounters any other error. diff --git a/Misc/NEWS.d/next/Library/2024-01-12-17-32-36.gh-issue-79634.uTSTRI.rst b/Misc/NEWS.d/next/Library/2024-01-12-17-32-36.gh-issue-79634.uTSTRI.rst new file mode 100644 index 00000000000000..ba19b5209e648e --- /dev/null +++ b/Misc/NEWS.d/next/Library/2024-01-12-17-32-36.gh-issue-79634.uTSTRI.rst @@ -0,0 +1,2 @@ +Accept :term:`path-like objects ` as patterns in +:meth:`pathlib.Path.glob` and :meth:`~pathlib.Path.rglob`. diff --git a/Misc/NEWS.d/next/Library/2024-01-15-18-42-44.gh-issue-109534.wYaLMZ.rst b/Misc/NEWS.d/next/Library/2024-01-15-18-42-44.gh-issue-109534.wYaLMZ.rst new file mode 100644 index 00000000000000..fc9a765a230037 --- /dev/null +++ b/Misc/NEWS.d/next/Library/2024-01-15-18-42-44.gh-issue-109534.wYaLMZ.rst @@ -0,0 +1,3 @@ +Fix a reference leak in +:class:`asyncio.selector_events.BaseSelectorEventLoop` when SSL handshakes +fail. Patch contributed by Jamie Phan. diff --git a/Misc/NEWS.d/next/Library/2024-01-15-19-54-41.gh-issue-114087.Xic5vY.rst b/Misc/NEWS.d/next/Library/2024-01-15-19-54-41.gh-issue-114087.Xic5vY.rst new file mode 100644 index 00000000000000..68b27a7b0c9f6c --- /dev/null +++ b/Misc/NEWS.d/next/Library/2024-01-15-19-54-41.gh-issue-114087.Xic5vY.rst @@ -0,0 +1 @@ +Speed up ``dataclasses.asdict`` up to 1.35x. diff --git a/Misc/NEWS.d/next/Library/2024-01-15-20-21-33.gh-issue-83648.HzD_fY.rst b/Misc/NEWS.d/next/Library/2024-01-15-20-21-33.gh-issue-83648.HzD_fY.rst new file mode 100644 index 00000000000000..bd3e27b4be0cf5 --- /dev/null +++ b/Misc/NEWS.d/next/Library/2024-01-15-20-21-33.gh-issue-83648.HzD_fY.rst @@ -0,0 +1,2 @@ +Support deprecation of options, positional arguments and subcommands in +:mod:`argparse`. diff --git a/Misc/NEWS.d/next/Library/2024-01-16-15-59-06.gh-issue-114149.LJ8IPm.rst b/Misc/NEWS.d/next/Library/2024-01-16-15-59-06.gh-issue-114149.LJ8IPm.rst new file mode 100644 index 00000000000000..1403d78d0d4905 --- /dev/null +++ b/Misc/NEWS.d/next/Library/2024-01-16-15-59-06.gh-issue-114149.LJ8IPm.rst @@ -0,0 +1 @@ +Enum: correctly handle tuple subclasses in custom ``__new__``. diff --git a/Misc/NEWS.d/next/Library/2024-01-17-18-53-51.gh-issue-104522.3NyDf4.rst b/Misc/NEWS.d/next/Library/2024-01-17-18-53-51.gh-issue-104522.3NyDf4.rst new file mode 100644 index 00000000000000..ca980945ea12d3 --- /dev/null +++ b/Misc/NEWS.d/next/Library/2024-01-17-18-53-51.gh-issue-104522.3NyDf4.rst @@ -0,0 +1,3 @@ +:exc:`OSError` raised when run a subprocess now only has *filename* +attribute set to *cwd* if the error was caused by a failed attempt to change +the current directory. diff --git a/Misc/NEWS.d/next/Library/2024-01-18-10-07-52.gh-issue-114198.lK4Iif.rst b/Misc/NEWS.d/next/Library/2024-01-18-10-07-52.gh-issue-114198.lK4Iif.rst new file mode 100644 index 00000000000000..fa047e288f807e --- /dev/null +++ b/Misc/NEWS.d/next/Library/2024-01-18-10-07-52.gh-issue-114198.lK4Iif.rst @@ -0,0 +1,2 @@ +The signature for the ``__replace__`` method on :mod:`dataclasses` now has +the first argument named ``self``, rather than ``obj``. diff --git a/Misc/NEWS.d/next/Library/2024-01-18-22-29-28.gh-issue-101438.1-uUi_.rst b/Misc/NEWS.d/next/Library/2024-01-18-22-29-28.gh-issue-101438.1-uUi_.rst new file mode 100644 index 00000000000000..9b69b5deb1b5a0 --- /dev/null +++ b/Misc/NEWS.d/next/Library/2024-01-18-22-29-28.gh-issue-101438.1-uUi_.rst @@ -0,0 +1,4 @@ +Avoid reference cycle in ElementTree.iterparse. The iterator returned by +``ElementTree.iterparse`` may hold on to a file descriptor. The reference +cycle prevented prompt clean-up of the file descriptor if the returned +iterator was not exhausted. diff --git a/Misc/NEWS.d/next/Library/2024-01-19-12-05-22.gh-issue-114281.H5JQe4.rst b/Misc/NEWS.d/next/Library/2024-01-19-12-05-22.gh-issue-114281.H5JQe4.rst new file mode 100644 index 00000000000000..36c54e8faf214c --- /dev/null +++ b/Misc/NEWS.d/next/Library/2024-01-19-12-05-22.gh-issue-114281.H5JQe4.rst @@ -0,0 +1,3 @@ +Remove type hints from ``Lib/asyncio/staggered.py``. +The annotations in the `typeshed `__ +project should be used instead. diff --git a/Misc/NEWS.d/next/Library/2024-01-19-15-48-06.gh-issue-114328.hixxW3.rst b/Misc/NEWS.d/next/Library/2024-01-19-15-48-06.gh-issue-114328.hixxW3.rst new file mode 100644 index 00000000000000..42262c05fd1fbf --- /dev/null +++ b/Misc/NEWS.d/next/Library/2024-01-19-15-48-06.gh-issue-114328.hixxW3.rst @@ -0,0 +1,4 @@ +The :func:`tty.setcbreak` and new :func:`tty.cfmakecbreak` no longer clears +the terminal input ICRLF flag. This fixes a regression introduced in 3.12 +that no longer matched how OSes define cbreak mode in their ``stty(1)`` +manual pages. diff --git a/Misc/NEWS.d/next/Library/2024-01-19-18-41-02.gh-issue-114321.yj_Xw3.rst b/Misc/NEWS.d/next/Library/2024-01-19-18-41-02.gh-issue-114321.yj_Xw3.rst new file mode 100644 index 00000000000000..dc2934bd81a42a --- /dev/null +++ b/Misc/NEWS.d/next/Library/2024-01-19-18-41-02.gh-issue-114321.yj_Xw3.rst @@ -0,0 +1,2 @@ +Expose more platform specific constants in the :mod:`fcntl` module on Linux, +macOS, FreeBSD and NetBSD. diff --git a/Misc/NEWS.d/next/Library/2024-01-21-16-32-55.gh-issue-114257.bCFld5.rst b/Misc/NEWS.d/next/Library/2024-01-21-16-32-55.gh-issue-114257.bCFld5.rst new file mode 100644 index 00000000000000..6f02ff9e62617d --- /dev/null +++ b/Misc/NEWS.d/next/Library/2024-01-21-16-32-55.gh-issue-114257.bCFld5.rst @@ -0,0 +1,2 @@ +Dismiss the :exc:`FileNotFound` error in :func:`ctypes.util.find_library` and +just return ``None`` on Linux. diff --git a/Misc/NEWS.d/next/Library/2024-01-22-11-43-38.gh-issue-114423.6mMoPH.rst b/Misc/NEWS.d/next/Library/2024-01-22-11-43-38.gh-issue-114423.6mMoPH.rst new file mode 100644 index 00000000000000..7b77b73295d948 --- /dev/null +++ b/Misc/NEWS.d/next/Library/2024-01-22-11-43-38.gh-issue-114423.6mMoPH.rst @@ -0,0 +1 @@ +``_DummyThread`` entries in ``threading._active`` are now automatically removed when the related thread dies. diff --git a/Misc/NEWS.d/next/Library/2024-01-22-12-10-34.gh-issue-75128.4FGlRS.rst b/Misc/NEWS.d/next/Library/2024-01-22-12-10-34.gh-issue-75128.4FGlRS.rst new file mode 100644 index 00000000000000..d875148e89b41b --- /dev/null +++ b/Misc/NEWS.d/next/Library/2024-01-22-12-10-34.gh-issue-75128.4FGlRS.rst @@ -0,0 +1,2 @@ +Ignore an :exc:`OSError` in :meth:`asyncio.BaseEventLoop.create_server` when +IPv6 is available but the interface cannot actually support it. diff --git a/Misc/NEWS.d/next/Library/2024-01-23-11-04-21.gh-issue-113267.xe_Pxe.rst b/Misc/NEWS.d/next/Library/2024-01-23-11-04-21.gh-issue-113267.xe_Pxe.rst new file mode 100644 index 00000000000000..ad8aaf9250f6d8 --- /dev/null +++ b/Misc/NEWS.d/next/Library/2024-01-23-11-04-21.gh-issue-113267.xe_Pxe.rst @@ -0,0 +1,2 @@ +Revert changes in :gh:`106584` which made calls of ``TestResult`` methods +``startTest()`` and ``stopTest()`` unbalanced. diff --git a/Misc/NEWS.d/next/Library/2024-01-23-14-11-49.gh-issue-114315.KeVdzl.rst b/Misc/NEWS.d/next/Library/2024-01-23-14-11-49.gh-issue-114315.KeVdzl.rst new file mode 100644 index 00000000000000..a8a19fc525d019 --- /dev/null +++ b/Misc/NEWS.d/next/Library/2024-01-23-14-11-49.gh-issue-114315.KeVdzl.rst @@ -0,0 +1,2 @@ +Make :class:`threading.Lock` a real class, not a factory function. Add +``__new__`` to ``_thread.lock`` type. diff --git a/Misc/NEWS.d/next/Library/2024-01-23-21-20-40.gh-issue-114492.vKxl5o.rst b/Misc/NEWS.d/next/Library/2024-01-23-21-20-40.gh-issue-114492.vKxl5o.rst new file mode 100644 index 00000000000000..8df8299d0dffcd --- /dev/null +++ b/Misc/NEWS.d/next/Library/2024-01-23-21-20-40.gh-issue-114492.vKxl5o.rst @@ -0,0 +1,2 @@ +Make the result of :func:`termios.tcgetattr` reproducible on Alpine Linux. +Previously it could leave a random garbage in some fields. diff --git a/Misc/NEWS.d/next/Library/2024-01-23-23-13-47.gh-issue-109653.KLBHmT.rst b/Misc/NEWS.d/next/Library/2024-01-23-23-13-47.gh-issue-109653.KLBHmT.rst new file mode 100644 index 00000000000000..76074df9c76fa6 --- /dev/null +++ b/Misc/NEWS.d/next/Library/2024-01-23-23-13-47.gh-issue-109653.KLBHmT.rst @@ -0,0 +1 @@ +Reduce the import time of :mod:`threading` module by ~50%. Patch by Daniel Hollas. diff --git a/Misc/NEWS.d/next/Library/2024-01-24-17-25-18.gh-issue-69893.PQq5fR.rst b/Misc/NEWS.d/next/Library/2024-01-24-17-25-18.gh-issue-69893.PQq5fR.rst new file mode 100644 index 00000000000000..1ebf434c33187b --- /dev/null +++ b/Misc/NEWS.d/next/Library/2024-01-24-17-25-18.gh-issue-69893.PQq5fR.rst @@ -0,0 +1,2 @@ +Add the :meth:`!close` method for the iterator returned by +:func:`xml.etree.ElementTree.iterparse`. diff --git a/Misc/NEWS.d/next/Library/2024-01-24-20-11-46.gh-issue-112451.7YrG4p.rst b/Misc/NEWS.d/next/Library/2024-01-24-20-11-46.gh-issue-112451.7YrG4p.rst new file mode 100644 index 00000000000000..126ca36a3b7cb1 --- /dev/null +++ b/Misc/NEWS.d/next/Library/2024-01-24-20-11-46.gh-issue-112451.7YrG4p.rst @@ -0,0 +1,2 @@ +Prohibit subclassing pure-Python :class:`datetime.timezone`. This is consistent +with C-extension implementation. Patch by Mariusz Felisiak. diff --git a/Misc/NEWS.d/next/Library/2024-01-24-20-51-49.gh-issue-91602.8fOH8l.rst b/Misc/NEWS.d/next/Library/2024-01-24-20-51-49.gh-issue-91602.8fOH8l.rst new file mode 100644 index 00000000000000..21d39df43e035b --- /dev/null +++ b/Misc/NEWS.d/next/Library/2024-01-24-20-51-49.gh-issue-91602.8fOH8l.rst @@ -0,0 +1,3 @@ +Add *filter* keyword-only parameter to +:meth:`sqlite3.Connection.iterdump` for filtering database objects to dump. +Patch by Mariusz Felisiak. diff --git a/Misc/NEWS.d/next/Library/2024-01-25-19-22-17.gh-issue-83383.3GwO9v.rst b/Misc/NEWS.d/next/Library/2024-01-25-19-22-17.gh-issue-83383.3GwO9v.rst new file mode 100644 index 00000000000000..e6336204dfa236 --- /dev/null +++ b/Misc/NEWS.d/next/Library/2024-01-25-19-22-17.gh-issue-83383.3GwO9v.rst @@ -0,0 +1,5 @@ +Synchronization of the :mod:`dbm.dumb` database is now no-op if there was no +modification since opening or last synchronization. +The directory file for a newly created empty :mod:`dbm.dumb` database is now +created immediately after opening instead of deferring this until +synchronizing or closing. diff --git a/Misc/NEWS.d/next/Library/2024-01-26-16-46-21.gh-issue-77749.NY_7TS.rst b/Misc/NEWS.d/next/Library/2024-01-26-16-46-21.gh-issue-77749.NY_7TS.rst new file mode 100644 index 00000000000000..f1c99c09d2dfe1 --- /dev/null +++ b/Misc/NEWS.d/next/Library/2024-01-26-16-46-21.gh-issue-77749.NY_7TS.rst @@ -0,0 +1,2 @@ +:meth:`email.policy.EmailPolicy.fold` now always encodes non-ASCII characters +in headers if :attr:`~email.policy.EmailPolicy.utf8` is false. diff --git a/Misc/NEWS.d/next/Library/2024-01-27-20-11-24.gh-issue-113280.CZPQMf.rst b/Misc/NEWS.d/next/Library/2024-01-27-20-11-24.gh-issue-113280.CZPQMf.rst new file mode 100644 index 00000000000000..3dcdbcf0995616 --- /dev/null +++ b/Misc/NEWS.d/next/Library/2024-01-27-20-11-24.gh-issue-113280.CZPQMf.rst @@ -0,0 +1,2 @@ +Fix a leak of open socket in rare cases when error occurred in +:class:`ssl.SSLSocket` creation. diff --git a/Misc/NEWS.d/next/Library/2024-01-28-00-48-12.gh-issue-109653.vF4exe.rst b/Misc/NEWS.d/next/Library/2024-01-28-00-48-12.gh-issue-109653.vF4exe.rst new file mode 100644 index 00000000000000..fb3382098853b3 --- /dev/null +++ b/Misc/NEWS.d/next/Library/2024-01-28-00-48-12.gh-issue-109653.vF4exe.rst @@ -0,0 +1 @@ +Improve import time of :mod:`importlib.metadata` and :mod:`email.utils`. diff --git a/Misc/NEWS.d/next/Library/2024-01-28-18-38-18.gh-issue-70303._Lt_pj.rst b/Misc/NEWS.d/next/Library/2024-01-28-18-38-18.gh-issue-70303._Lt_pj.rst new file mode 100644 index 00000000000000..dedda24b481241 --- /dev/null +++ b/Misc/NEWS.d/next/Library/2024-01-28-18-38-18.gh-issue-70303._Lt_pj.rst @@ -0,0 +1,2 @@ +Return both files and directories from :meth:`pathlib.Path.glob` if a +pattern ends with "``**``". Previously only directories were returned. diff --git a/Misc/NEWS.d/next/Library/2024-01-28-19-40-40.gh-issue-114678.kYKcJw.rst b/Misc/NEWS.d/next/Library/2024-01-28-19-40-40.gh-issue-114678.kYKcJw.rst new file mode 100644 index 00000000000000..2306af4a39dcf6 --- /dev/null +++ b/Misc/NEWS.d/next/Library/2024-01-28-19-40-40.gh-issue-114678.kYKcJw.rst @@ -0,0 +1,3 @@ +Ensure that deprecation warning for 'N' specifier in :class:`~decimal.Decimal` +format is not raised for cases where 'N' appears in other places +in the format specifier. Based on patch by Stefan Krah. diff --git a/Misc/NEWS.d/next/Library/2024-01-30-15-34-08.gh-issue-110190.Z5PQQX.rst b/Misc/NEWS.d/next/Library/2024-01-30-15-34-08.gh-issue-110190.Z5PQQX.rst new file mode 100644 index 00000000000000..af77e409963e04 --- /dev/null +++ b/Misc/NEWS.d/next/Library/2024-01-30-15-34-08.gh-issue-110190.Z5PQQX.rst @@ -0,0 +1 @@ +Fix ctypes structs with array on Windows ARM64 platform by setting ``MAX_STRUCT_SIZE`` to 32 in stgdict. Patch by Diego Russo diff --git a/Misc/NEWS.d/next/Library/2024-01-31-20-07-11.gh-issue-109475.lmTb9S.rst b/Misc/NEWS.d/next/Library/2024-01-31-20-07-11.gh-issue-109475.lmTb9S.rst new file mode 100644 index 00000000000000..7582cb2bcd7629 --- /dev/null +++ b/Misc/NEWS.d/next/Library/2024-01-31-20-07-11.gh-issue-109475.lmTb9S.rst @@ -0,0 +1,2 @@ +Fix support of explicit option value "--" in :mod:`argparse` (e.g. +``--option=--``). diff --git a/Misc/NEWS.d/next/Library/2024-02-01-10-19-11.gh-issue-114071.vkm2G_.rst b/Misc/NEWS.d/next/Library/2024-02-01-10-19-11.gh-issue-114071.vkm2G_.rst new file mode 100644 index 00000000000000..587ce4d2157637 --- /dev/null +++ b/Misc/NEWS.d/next/Library/2024-02-01-10-19-11.gh-issue-114071.vkm2G_.rst @@ -0,0 +1 @@ +Support tuple subclasses using auto() for enum member value. diff --git a/Misc/NEWS.d/next/Library/2024-02-03-16-59-25.gh-issue-114959.dCfAG2.rst b/Misc/NEWS.d/next/Library/2024-02-03-16-59-25.gh-issue-114959.dCfAG2.rst new file mode 100644 index 00000000000000..5c6eaa7525e3b0 --- /dev/null +++ b/Misc/NEWS.d/next/Library/2024-02-03-16-59-25.gh-issue-114959.dCfAG2.rst @@ -0,0 +1,2 @@ +:mod:`tarfile` no longer ignores errors when trying to extract a directory on +top of a file. diff --git a/Misc/NEWS.d/next/Library/2024-02-03-17-54-17.gh-issue-114965.gHksCK.rst b/Misc/NEWS.d/next/Library/2024-02-03-17-54-17.gh-issue-114965.gHksCK.rst new file mode 100644 index 00000000000000..d59ff991993792 --- /dev/null +++ b/Misc/NEWS.d/next/Library/2024-02-03-17-54-17.gh-issue-114965.gHksCK.rst @@ -0,0 +1 @@ +Update bundled pip to 24.0 diff --git a/Misc/NEWS.d/next/Library/2024-02-04-13-17-33.gh-issue-114628.WJpqqS.rst b/Misc/NEWS.d/next/Library/2024-02-04-13-17-33.gh-issue-114628.WJpqqS.rst new file mode 100644 index 00000000000000..8138adc62c95f3 --- /dev/null +++ b/Misc/NEWS.d/next/Library/2024-02-04-13-17-33.gh-issue-114628.WJpqqS.rst @@ -0,0 +1,2 @@ +When csv.Error is raised when handling TypeError, do not print the TypeError +traceback. diff --git a/Misc/NEWS.d/next/Library/2024-02-08-13-26-14.gh-issue-115059.DqP9dr.rst b/Misc/NEWS.d/next/Library/2024-02-08-13-26-14.gh-issue-115059.DqP9dr.rst new file mode 100644 index 00000000000000..331baedd3b24c5 --- /dev/null +++ b/Misc/NEWS.d/next/Library/2024-02-08-13-26-14.gh-issue-115059.DqP9dr.rst @@ -0,0 +1 @@ +:meth:`io.BufferedRandom.read1` now flushes the underlying write buffer. diff --git a/Misc/NEWS.d/next/Library/2024-02-08-17-04-58.gh-issue-112903.SN_vUs.rst b/Misc/NEWS.d/next/Library/2024-02-08-17-04-58.gh-issue-112903.SN_vUs.rst new file mode 100644 index 00000000000000..e27f5832553c13 --- /dev/null +++ b/Misc/NEWS.d/next/Library/2024-02-08-17-04-58.gh-issue-112903.SN_vUs.rst @@ -0,0 +1,2 @@ +Fix "issubclass() arg 1 must be a class" errors in certain cases of multiple +inheritance with generic aliases (regression in early 3.13 alpha releases). diff --git a/Misc/NEWS.d/next/Library/2024-02-09-07-20-16.gh-issue-115165.yfJLXA.rst b/Misc/NEWS.d/next/Library/2024-02-09-07-20-16.gh-issue-115165.yfJLXA.rst new file mode 100644 index 00000000000000..73d3d001f07f3f --- /dev/null +++ b/Misc/NEWS.d/next/Library/2024-02-09-07-20-16.gh-issue-115165.yfJLXA.rst @@ -0,0 +1,4 @@ +Most exceptions are now ignored when attempting to set the ``__orig_class__`` +attribute on objects returned when calling :mod:`typing` generic aliases +(including generic aliases created using :data:`typing.Annotated`). +Previously only :exc:`AttributeError`` was ignored. Patch by Dave Shawley. diff --git a/Misc/NEWS.d/next/Security/2023-12-06-14-06-59.gh-issue-112302.3bl20f.rst b/Misc/NEWS.d/next/Security/2023-12-06-14-06-59.gh-issue-112302.3bl20f.rst deleted file mode 100644 index 65e4dc3762d3c0..00000000000000 --- a/Misc/NEWS.d/next/Security/2023-12-06-14-06-59.gh-issue-112302.3bl20f.rst +++ /dev/null @@ -1,2 +0,0 @@ -Created a Software Bill-of-Materials document and tooling for tracking -dependencies. diff --git a/Misc/NEWS.d/next/Tests/2020-05-16-18-00-21.bpo-40648.p2uPqy.rst b/Misc/NEWS.d/next/Tests/2020-05-16-18-00-21.bpo-40648.p2uPqy.rst deleted file mode 100644 index 8fbe42d263feb9..00000000000000 --- a/Misc/NEWS.d/next/Tests/2020-05-16-18-00-21.bpo-40648.p2uPqy.rst +++ /dev/null @@ -1 +0,0 @@ -Test modes that file can get with chmod() on Windows. diff --git a/Misc/NEWS.d/next/Tests/2023-06-02-05-04-15.gh-issue-105089.KaZFtU.rst b/Misc/NEWS.d/next/Tests/2023-06-02-05-04-15.gh-issue-105089.KaZFtU.rst new file mode 100644 index 00000000000000..d04ef435dd572d --- /dev/null +++ b/Misc/NEWS.d/next/Tests/2023-06-02-05-04-15.gh-issue-105089.KaZFtU.rst @@ -0,0 +1,4 @@ +Fix +``test.test_zipfile.test_core.TestWithDirectory.test_create_directory_with_write`` +test in AIX by doing a bitwise AND of 0xFFFF on mode , so that it will be in +sync with ``zinfo.external_attr`` diff --git a/Misc/NEWS.d/next/Tests/2023-09-05-20-46-35.gh-issue-108927.TpwWav.rst b/Misc/NEWS.d/next/Tests/2023-09-05-20-46-35.gh-issue-108927.TpwWav.rst deleted file mode 100644 index b1a78370afedb2..00000000000000 --- a/Misc/NEWS.d/next/Tests/2023-09-05-20-46-35.gh-issue-108927.TpwWav.rst +++ /dev/null @@ -1,4 +0,0 @@ -Fixed order dependence in running tests in the same process -when a test that has submodules (e.g. test_importlib) follows a test that -imports its submodule (e.g. test_importlib.util) and precedes a test -(e.g. test_unittest or test_compileall) that uses that submodule. diff --git a/Misc/NEWS.d/next/Tests/2023-12-04-15-56-11.gh-issue-112334.FFc9Ti.rst b/Misc/NEWS.d/next/Tests/2023-12-04-15-56-11.gh-issue-112334.FFc9Ti.rst deleted file mode 100644 index aeaad6e5055522..00000000000000 --- a/Misc/NEWS.d/next/Tests/2023-12-04-15-56-11.gh-issue-112334.FFc9Ti.rst +++ /dev/null @@ -1,2 +0,0 @@ -Adds a regression test to verify that ``vfork()`` is used when expected by -:mod:`subprocess` on vfork enabled POSIX systems (Linux). diff --git a/Misc/NEWS.d/next/Tests/2023-12-05-19-50-03.gh-issue-112769.kdLJmS.rst b/Misc/NEWS.d/next/Tests/2023-12-05-19-50-03.gh-issue-112769.kdLJmS.rst deleted file mode 100644 index 1bbbb26fc322fa..00000000000000 --- a/Misc/NEWS.d/next/Tests/2023-12-05-19-50-03.gh-issue-112769.kdLJmS.rst +++ /dev/null @@ -1,3 +0,0 @@ -The tests now correctly compare zlib version when -:const:`zlib.ZLIB_RUNTIME_VERSION` contains non-integer suffixes. For -example zlib-ng defines the version as ``1.3.0.zlib-ng``. diff --git a/Misc/NEWS.d/next/Tests/2023-12-09-21-27-46.gh-issue-109980.y--500.rst b/Misc/NEWS.d/next/Tests/2023-12-09-21-27-46.gh-issue-109980.y--500.rst deleted file mode 100644 index c475a33919db98..00000000000000 --- a/Misc/NEWS.d/next/Tests/2023-12-09-21-27-46.gh-issue-109980.y--500.rst +++ /dev/null @@ -1,2 +0,0 @@ -Fix ``test_tarfile_vs_tar`` in ``test_shutil`` for macOS, where system tar -can include more information in the archive than :mod:`shutil.make_archive`. diff --git a/Misc/NEWS.d/next/Tests/2024-02-02-13-18-55.gh-issue-114099.C_ycWg.rst b/Misc/NEWS.d/next/Tests/2024-02-02-13-18-55.gh-issue-114099.C_ycWg.rst new file mode 100644 index 00000000000000..487cd5062fc75b --- /dev/null +++ b/Misc/NEWS.d/next/Tests/2024-02-02-13-18-55.gh-issue-114099.C_ycWg.rst @@ -0,0 +1 @@ +Added test exclusions required to run the test suite on iOS. diff --git a/Misc/NEWS.d/next/Tools-Demos/2024-02-05-02-45-51.gh-issue-115015.rgtiDB.rst b/Misc/NEWS.d/next/Tools-Demos/2024-02-05-02-45-51.gh-issue-115015.rgtiDB.rst new file mode 100644 index 00000000000000..d8739d28eb2b73 --- /dev/null +++ b/Misc/NEWS.d/next/Tools-Demos/2024-02-05-02-45-51.gh-issue-115015.rgtiDB.rst @@ -0,0 +1,5 @@ +Fix a bug in Argument Clinic that generated incorrect code for methods with +no parameters that use the :ref:`METH_METHOD | METH_FASTCALL | METH_KEYWORDS +` calling convention. Only the +positional parameter count was checked; any keyword argument passed would be +silently accepted. diff --git a/Misc/NEWS.d/next/Tools-Demos/2024-02-05-19-00-32.gh-issue-109991.yJSEkw.rst b/Misc/NEWS.d/next/Tools-Demos/2024-02-05-19-00-32.gh-issue-109991.yJSEkw.rst new file mode 100644 index 00000000000000..4eb4d39629b9bc --- /dev/null +++ b/Misc/NEWS.d/next/Tools-Demos/2024-02-05-19-00-32.gh-issue-109991.yJSEkw.rst @@ -0,0 +1,2 @@ +Update GitHub CI workflows to use OpenSSL 3.0.13 and multissltests to use +1.1.1w, 3.0.13, 3.1.5, and 3.2.1. diff --git a/Misc/NEWS.d/next/Windows/2023-12-03-19-22-37.gh-issue-112278.FiloCE.rst b/Misc/NEWS.d/next/Windows/2023-12-03-19-22-37.gh-issue-112278.FiloCE.rst deleted file mode 100644 index 0350d105d97375..00000000000000 --- a/Misc/NEWS.d/next/Windows/2023-12-03-19-22-37.gh-issue-112278.FiloCE.rst +++ /dev/null @@ -1,2 +0,0 @@ -Reduce the time cost for some functions in :mod:`platform` on Windows if -current user has no permission to the WMI. diff --git a/Misc/NEWS.d/next/Windows/2023-12-05-22-56-30.gh-issue-111650.xlWmvM.rst b/Misc/NEWS.d/next/Windows/2023-12-05-22-56-30.gh-issue-111650.xlWmvM.rst deleted file mode 100644 index 5a3493356e30be..00000000000000 --- a/Misc/NEWS.d/next/Windows/2023-12-05-22-56-30.gh-issue-111650.xlWmvM.rst +++ /dev/null @@ -1,3 +0,0 @@ -Ensures the ``Py_GIL_DISABLED`` preprocessor variable is defined in -:file:`pyconfig.h` so that extension modules written in C are able to use -it. diff --git a/Misc/NEWS.d/next/Windows/2023-12-11-20-23-04.gh-issue-71383.9pZh6t.rst b/Misc/NEWS.d/next/Windows/2023-12-11-20-23-04.gh-issue-71383.9pZh6t.rst deleted file mode 100644 index cf2883357a962a..00000000000000 --- a/Misc/NEWS.d/next/Windows/2023-12-11-20-23-04.gh-issue-71383.9pZh6t.rst +++ /dev/null @@ -1,2 +0,0 @@ -Update Tcl/Tk in Windows installer to 8.6.13 with a patch to suppress -incorrect ThemeChanged warnings. diff --git a/Misc/NEWS.d/next/Windows/2023-12-12-20-58-09.gh-issue-86179.YYSk_6.rst b/Misc/NEWS.d/next/Windows/2023-12-12-20-58-09.gh-issue-86179.YYSk_6.rst deleted file mode 100644 index c1d96792bdae0b..00000000000000 --- a/Misc/NEWS.d/next/Windows/2023-12-12-20-58-09.gh-issue-86179.YYSk_6.rst +++ /dev/null @@ -1 +0,0 @@ -Fixes path calculations when launching Python on Windows through a symlink. diff --git a/Misc/NEWS.d/next/Windows/2023-12-14-19-00-29.gh-issue-113009.6LNdjz.rst b/Misc/NEWS.d/next/Windows/2023-12-14-19-00-29.gh-issue-113009.6LNdjz.rst deleted file mode 100644 index 6fd7f7f9afdfa2..00000000000000 --- a/Misc/NEWS.d/next/Windows/2023-12-14-19-00-29.gh-issue-113009.6LNdjz.rst +++ /dev/null @@ -1,5 +0,0 @@ -:mod:`multiprocessing`: On Windows, fix a race condition in -``Process.terminate()``: no longer set the ``returncode`` attribute to -always call ``WaitForSingleObject()`` in ``Process.wait()``. Previously, -sometimes the process was still running after ``TerminateProcess()`` even if -``GetExitCodeProcess()`` is not ``STILL_ACTIVE``. Patch by Victor Stinner. diff --git a/Misc/NEWS.d/next/Windows/2023-12-19-10-56-46.gh-issue-111973.A9Wtsb.rst b/Misc/NEWS.d/next/Windows/2023-12-19-10-56-46.gh-issue-111973.A9Wtsb.rst deleted file mode 100644 index 0cefa4e44093f0..00000000000000 --- a/Misc/NEWS.d/next/Windows/2023-12-19-10-56-46.gh-issue-111973.A9Wtsb.rst +++ /dev/null @@ -1 +0,0 @@ -Update Windows installer to use SQLite 3.44.2. diff --git a/Misc/NEWS.d/next/Windows/2023-12-19-22-32-28.gh-issue-112984.F7kFMl.rst b/Misc/NEWS.d/next/Windows/2023-12-19-22-32-28.gh-issue-112984.F7kFMl.rst new file mode 100644 index 00000000000000..429cd5bc923e09 --- /dev/null +++ b/Misc/NEWS.d/next/Windows/2023-12-19-22-32-28.gh-issue-112984.F7kFMl.rst @@ -0,0 +1 @@ +Adds free-threaded binaries to Windows installer as an optional component. diff --git a/Misc/NEWS.d/next/Windows/2024-01-23-00-05-05.gh-issue-100107.lkbP_Q.rst b/Misc/NEWS.d/next/Windows/2024-01-23-00-05-05.gh-issue-100107.lkbP_Q.rst new file mode 100644 index 00000000000000..388d61a2b3bd6d --- /dev/null +++ b/Misc/NEWS.d/next/Windows/2024-01-23-00-05-05.gh-issue-100107.lkbP_Q.rst @@ -0,0 +1 @@ +The ``py.exe`` launcher will no longer attempt to run the Microsoft Store redirector when launching a script containing a ``/usr/bin/env`` shebang diff --git a/Misc/NEWS.d/next/Windows/2024-02-01-14-35-05.gh-issue-111239.SO7SUF.rst b/Misc/NEWS.d/next/Windows/2024-02-01-14-35-05.gh-issue-111239.SO7SUF.rst new file mode 100644 index 00000000000000..ea82c3b941f802 --- /dev/null +++ b/Misc/NEWS.d/next/Windows/2024-02-01-14-35-05.gh-issue-111239.SO7SUF.rst @@ -0,0 +1 @@ +Update Windows builds to use zlib v1.3.1. diff --git a/Misc/NEWS.d/next/Windows/2024-02-05-16-53-12.gh-issue-109991.YqjnDz.rst b/Misc/NEWS.d/next/Windows/2024-02-05-16-53-12.gh-issue-109991.YqjnDz.rst new file mode 100644 index 00000000000000..d9923c35c2726e --- /dev/null +++ b/Misc/NEWS.d/next/Windows/2024-02-05-16-53-12.gh-issue-109991.YqjnDz.rst @@ -0,0 +1 @@ +Update Windows build to use OpenSSL 3.0.13. diff --git a/Misc/NEWS.d/next/Windows/2024-02-06-09-05-13.gh-issue-115009.ShMjZs.rst b/Misc/NEWS.d/next/Windows/2024-02-06-09-05-13.gh-issue-115009.ShMjZs.rst new file mode 100644 index 00000000000000..5bdb6963a24311 --- /dev/null +++ b/Misc/NEWS.d/next/Windows/2024-02-06-09-05-13.gh-issue-115009.ShMjZs.rst @@ -0,0 +1 @@ +Update Windows installer to use SQLite 3.45.1. diff --git a/Misc/NEWS.d/next/macOS/2023-12-06-12-11-13.gh-issue-109981.mOHg10.rst b/Misc/NEWS.d/next/macOS/2023-12-06-12-11-13.gh-issue-109981.mOHg10.rst deleted file mode 100644 index f86ab2c37ee6ec..00000000000000 --- a/Misc/NEWS.d/next/macOS/2023-12-06-12-11-13.gh-issue-109981.mOHg10.rst +++ /dev/null @@ -1,3 +0,0 @@ -Use ``/dev/fd`` on macOS to determine the number of open files in -``test.support.os_helper.fd_count`` to avoid a crash with "guarded" file -descriptors when probing for open files. diff --git a/Misc/NEWS.d/next/macOS/2023-12-07-14-19-46.gh-issue-110820.DIxb_F.rst b/Misc/NEWS.d/next/macOS/2023-12-07-14-19-46.gh-issue-110820.DIxb_F.rst deleted file mode 100644 index 0badace7928745..00000000000000 --- a/Misc/NEWS.d/next/macOS/2023-12-07-14-19-46.gh-issue-110820.DIxb_F.rst +++ /dev/null @@ -1,3 +0,0 @@ -Make sure the preprocessor definitions for ``ALIGNOF_MAX_ALIGN_T``, -``SIZEOF_LONG_DOUBLE`` and ``HAVE_GCC_ASM_FOR_X64`` are correct for -Universal 2 builds on macOS. diff --git a/Misc/NEWS.d/next/macOS/2023-12-07-15-53-16.gh-issue-110017.UMYzMR.rst b/Misc/NEWS.d/next/macOS/2023-12-07-15-53-16.gh-issue-110017.UMYzMR.rst deleted file mode 100644 index eab1746f1ae3f7..00000000000000 --- a/Misc/NEWS.d/next/macOS/2023-12-07-15-53-16.gh-issue-110017.UMYzMR.rst +++ /dev/null @@ -1,2 +0,0 @@ -Disable a signal handling stress test on macOS due to a bug in macOS -(FB13453490). diff --git a/Misc/NEWS.d/next/macOS/2023-12-10-20-30-06.gh-issue-102362.y8svbF.rst b/Misc/NEWS.d/next/macOS/2023-12-10-20-30-06.gh-issue-102362.y8svbF.rst deleted file mode 100644 index 55c5ac01434660..00000000000000 --- a/Misc/NEWS.d/next/macOS/2023-12-10-20-30-06.gh-issue-102362.y8svbF.rst +++ /dev/null @@ -1,3 +0,0 @@ -Make sure the result of :func:`sysconfig.get_plaform` includes at least a -major and minor versions, even if ``MACOSX_DEPLOYMENT_TARGET`` is set to -only a major version during build to match the format expected by pip. diff --git a/Misc/NEWS.d/next/macOS/2023-12-16-11-45-32.gh-issue-108269.wVgCHF.rst b/Misc/NEWS.d/next/macOS/2023-12-16-11-45-32.gh-issue-108269.wVgCHF.rst deleted file mode 100644 index 85598454abcaad..00000000000000 --- a/Misc/NEWS.d/next/macOS/2023-12-16-11-45-32.gh-issue-108269.wVgCHF.rst +++ /dev/null @@ -1,4 +0,0 @@ -Set ``CFBundleAllowMixedLocalizations`` to true in the Info.plist for the -framework, embedded Python.app and IDLE.app with framework installs on -macOS. This allows applications to pick up the user's preferred locale when -that's different from english. diff --git a/Misc/NEWS.d/next/macOS/2023-12-19-10-50-08.gh-issue-111973.HMHJfy.rst b/Misc/NEWS.d/next/macOS/2023-12-19-10-50-08.gh-issue-111973.HMHJfy.rst deleted file mode 100644 index 0cf3abf3b71890..00000000000000 --- a/Misc/NEWS.d/next/macOS/2023-12-19-10-50-08.gh-issue-111973.HMHJfy.rst +++ /dev/null @@ -1 +0,0 @@ -Update macOS installer to use SQLite 3.44.2. diff --git a/Misc/NEWS.d/next/macOS/2023-12-21-09-41-42.gh-issue-87277.IF6EZZ.rst b/Misc/NEWS.d/next/macOS/2023-12-21-09-41-42.gh-issue-87277.IF6EZZ.rst deleted file mode 100644 index 4ae55c0293198a..00000000000000 --- a/Misc/NEWS.d/next/macOS/2023-12-21-09-41-42.gh-issue-87277.IF6EZZ.rst +++ /dev/null @@ -1,3 +0,0 @@ -webbrowser: Don't look for X11 browsers on macOS. Those are generally not -used and probing for them can result in starting XQuartz even if it isn't -used otherwise. diff --git a/Misc/NEWS.d/next/macOS/2023-12-21-10-20-41.gh-issue-65701.Q2hNbN.rst b/Misc/NEWS.d/next/macOS/2023-12-21-10-20-41.gh-issue-65701.Q2hNbN.rst deleted file mode 100644 index 870b84a4d1af80..00000000000000 --- a/Misc/NEWS.d/next/macOS/2023-12-21-10-20-41.gh-issue-65701.Q2hNbN.rst +++ /dev/null @@ -1,2 +0,0 @@ -The :program:`freeze` tool doesn't work with framework builds of Python. -Document this and bail out early when running the tool with such a build. diff --git a/Misc/NEWS.d/next/macOS/2024-01-23-11-35-26.gh-issue-114490.FrQOQ0.rst b/Misc/NEWS.d/next/macOS/2024-01-23-11-35-26.gh-issue-114490.FrQOQ0.rst new file mode 100644 index 00000000000000..abd296f8608518 --- /dev/null +++ b/Misc/NEWS.d/next/macOS/2024-01-23-11-35-26.gh-issue-114490.FrQOQ0.rst @@ -0,0 +1 @@ +Add Mach-O linkage support for :func:`platform.architecture()`. diff --git a/Misc/NEWS.d/next/macOS/2024-02-05-18-30-27.gh-issue-109991.tun6Yu.rst b/Misc/NEWS.d/next/macOS/2024-02-05-18-30-27.gh-issue-109991.tun6Yu.rst new file mode 100644 index 00000000000000..79b45e7d51da3f --- /dev/null +++ b/Misc/NEWS.d/next/macOS/2024-02-05-18-30-27.gh-issue-109991.tun6Yu.rst @@ -0,0 +1 @@ +Update macOS installer to use OpenSSL 3.0.13. diff --git a/Misc/NEWS.d/next/macOS/2024-02-06-09-01-10.gh-issue-115009.ysau7e.rst b/Misc/NEWS.d/next/macOS/2024-02-06-09-01-10.gh-issue-115009.ysau7e.rst new file mode 100644 index 00000000000000..47ec488c3cced2 --- /dev/null +++ b/Misc/NEWS.d/next/macOS/2024-02-06-09-01-10.gh-issue-115009.ysau7e.rst @@ -0,0 +1 @@ +Update macOS installer to use SQLite 3.45.1. diff --git a/Misc/python.man b/Misc/python.man index 14cbd85c60bfa9..0f5dfa2e2289f7 100644 --- a/Misc/python.man +++ b/Misc/python.man @@ -604,6 +604,9 @@ can be set to the callable of your debugger of choice. .IP PYTHON_COLORS If this variable is set to 1, the interpreter will colorize various kinds of output. Setting it to 0 deactivates this behavior. +.IP PYTHON_HISTORY +This environment variable can be used to set the location of a history file +(on Unix, it is \fI~/.python_history\fP by default). .SS Debug-mode variables Setting these variables only has an effect in a debug build of Python, that is, if Python was configured with the diff --git a/Misc/sbom.spdx.json b/Misc/sbom.spdx.json index 5b3cd04ffa7f74..d783d14255e66f 100644 --- a/Misc/sbom.spdx.json +++ b/Misc/sbom.spdx.json @@ -1570,18 +1570,18 @@ "fileName": "Modules/_decimal/libmpdec/vcdiv64.asm" }, { - "SPDXID": "SPDXRef-FILE-Lib-ensurepip-bundled-pip-23.3.2-py3-none-any.whl", + "SPDXID": "SPDXRef-FILE-Lib-ensurepip-bundled-pip-24.0-py3-none-any.whl", "checksums": [ { "algorithm": "SHA1", - "checksumValue": "8e48f55ab2965ee64bd55cc91a8077d184a33e30" + "checksumValue": "e44313ae1e6af3c2bd3b60ab2fa8c34308d00555" }, { "algorithm": "SHA256", - "checksumValue": "5052d7889c1f9d05224cd41741acb7c5d6fa735ab34e339624a614eaaa7e7d76" + "checksumValue": "ba0d021a166865d2265246961bec0152ff124de910c5cc39f1156ce3fa7c69dc" } ], - "fileName": "Lib/ensurepip/_bundled/pip-23.3.2-py3-none-any.whl" + "fileName": "Lib/ensurepip/_bundled/pip-24.0-py3-none-any.whl" } ], "packages": [ @@ -1601,7 +1601,7 @@ "referenceType": "cpe23Type" } ], - "licenseConcluded": "MIT", + "licenseConcluded": "NOASSERTION", "name": "expat", "originator": "Organization: Expat development team", "primaryPackagePurpose": "SOURCE", @@ -1623,7 +1623,7 @@ "referenceType": "cpe23Type" } ], - "licenseConcluded": "Apache-2.0", + "licenseConcluded": "NOASSERTION", "name": "hacl-star", "originator": "Organization: HACL* Developers", "primaryPackagePurpose": "SOURCE", @@ -1645,7 +1645,7 @@ "referenceType": "cpe23Type" } ], - "licenseConcluded": "CC0-1.0", + "licenseConcluded": "NOASSERTION", "name": "libb2", "originator": "Organization: BLAKE2 - fast secure hashing", "primaryPackagePurpose": "SOURCE", @@ -1667,7 +1667,7 @@ "referenceType": "purl" } ], - "licenseConcluded": "MIT", + "licenseConcluded": "NOASSERTION", "name": "macholib", "originator": "Person: Ronald Oussoren (ronaldoussoren@mac.com)", "primaryPackagePurpose": "SOURCE", @@ -1689,41 +1689,665 @@ "referenceType": "cpe23Type" } ], - "licenseConcluded": "BSD-2-Clause", + "licenseConcluded": "NOASSERTION", "name": "mpdecimal", "originator": "Organization: bytereef.org", "primaryPackagePurpose": "SOURCE", "versionInfo": "2.5.1" }, + { + "SPDXID": "SPDXRef-PACKAGE-cachecontrol", + "checksums": [ + { + "algorithm": "SHA256", + "checksumValue": "95dedbec849f46dda3137866dc28b9d133fc9af55f5b805ab1291833e4457aa4" + } + ], + "downloadLocation": "https://files.pythonhosted.org/packages/1d/e3/a22348e6226dcd585d5a4b5f0175b3a16dabfd3912cbeb02f321d00e56c7/cachecontrol-0.13.1-py3-none-any.whl", + "externalRefs": [ + { + "referenceCategory": "PACKAGE_MANAGER", + "referenceLocator": "pkg:pypi/cachecontrol@0.13.1", + "referenceType": "purl" + } + ], + "licenseConcluded": "NOASSERTION", + "name": "cachecontrol", + "primaryPackagePurpose": "SOURCE", + "versionInfo": "0.13.1" + }, + { + "SPDXID": "SPDXRef-PACKAGE-colorama", + "checksums": [ + { + "algorithm": "SHA256", + "checksumValue": "4f1d9991f5acc0ca119f9d443620b77f9d6b33703e51011c16baf57afb285fc6" + } + ], + "downloadLocation": "https://files.pythonhosted.org/packages/d1/d6/3965ed04c63042e047cb6a3e6ed1a63a35087b6a609aa3a15ed8ac56c221/colorama-0.4.6-py2.py3-none-any.whl", + "externalRefs": [ + { + "referenceCategory": "PACKAGE_MANAGER", + "referenceLocator": "pkg:pypi/colorama@0.4.6", + "referenceType": "purl" + } + ], + "licenseConcluded": "NOASSERTION", + "name": "colorama", + "primaryPackagePurpose": "SOURCE", + "versionInfo": "0.4.6" + }, + { + "SPDXID": "SPDXRef-PACKAGE-distlib", + "checksums": [ + { + "algorithm": "SHA256", + "checksumValue": "034db59a0b96f8ca18035f36290806a9a6e6bd9d1ff91e45a7f172eb17e51784" + } + ], + "downloadLocation": "https://files.pythonhosted.org/packages/8e/41/9307e4f5f9976bc8b7fea0b66367734e8faf3ec84bc0d412d8cfabbb66cd/distlib-0.3.8-py2.py3-none-any.whl", + "externalRefs": [ + { + "referenceCategory": "PACKAGE_MANAGER", + "referenceLocator": "pkg:pypi/distlib@0.3.8", + "referenceType": "purl" + } + ], + "licenseConcluded": "NOASSERTION", + "name": "distlib", + "primaryPackagePurpose": "SOURCE", + "versionInfo": "0.3.8" + }, + { + "SPDXID": "SPDXRef-PACKAGE-distro", + "checksums": [ + { + "algorithm": "SHA256", + "checksumValue": "99522ca3e365cac527b44bde033f64c6945d90eb9f769703caaec52b09bbd3ff" + } + ], + "downloadLocation": "https://files.pythonhosted.org/packages/f4/2c/c90a3adaf0ddb70afe193f5ebfb539612af57cffe677c3126be533df3098/distro-1.8.0-py3-none-any.whl", + "externalRefs": [ + { + "referenceCategory": "PACKAGE_MANAGER", + "referenceLocator": "pkg:pypi/distro@1.8.0", + "referenceType": "purl" + } + ], + "licenseConcluded": "NOASSERTION", + "name": "distro", + "primaryPackagePurpose": "SOURCE", + "versionInfo": "1.8.0" + }, + { + "SPDXID": "SPDXRef-PACKAGE-msgpack", + "checksums": [ + { + "algorithm": "SHA256", + "checksumValue": "525228efd79bb831cf6830a732e2e80bc1b05436b086d4264814b4b2955b2fa9" + } + ], + "downloadLocation": "https://files.pythonhosted.org/packages/9f/4a/36d936e54cf71e23ad276564465f6a54fb129e3d61520b76e13e0bb29167/msgpack-1.0.5-cp310-cp310-macosx_10_9_universal2.whl", + "externalRefs": [ + { + "referenceCategory": "PACKAGE_MANAGER", + "referenceLocator": "pkg:pypi/msgpack@1.0.5", + "referenceType": "purl" + } + ], + "licenseConcluded": "NOASSERTION", + "name": "msgpack", + "primaryPackagePurpose": "SOURCE", + "versionInfo": "1.0.5" + }, + { + "SPDXID": "SPDXRef-PACKAGE-packaging", + "checksums": [ + { + "algorithm": "SHA256", + "checksumValue": "ef103e05f519cdc783ae24ea4e2e0f508a9c99b2d4969652eed6a2e1ea5bd522" + } + ], + "downloadLocation": "https://files.pythonhosted.org/packages/05/8e/8de486cbd03baba4deef4142bd643a3e7bbe954a784dc1bb17142572d127/packaging-21.3-py3-none-any.whl", + "externalRefs": [ + { + "referenceCategory": "PACKAGE_MANAGER", + "referenceLocator": "pkg:pypi/packaging@21.3", + "referenceType": "purl" + } + ], + "licenseConcluded": "NOASSERTION", + "name": "packaging", + "primaryPackagePurpose": "SOURCE", + "versionInfo": "21.3" + }, + { + "SPDXID": "SPDXRef-PACKAGE-platformdirs", + "checksums": [ + { + "algorithm": "SHA256", + "checksumValue": "cec7b889196b9144d088e4c57d9ceef7374f6c39694ad1577a0aab50d27ea28c" + } + ], + "downloadLocation": "https://files.pythonhosted.org/packages/9e/d8/563a9fc17153c588c8c2042d2f0f84a89057cdb1c30270f589c88b42d62c/platformdirs-3.8.1-py3-none-any.whl", + "externalRefs": [ + { + "referenceCategory": "PACKAGE_MANAGER", + "referenceLocator": "pkg:pypi/platformdirs@3.8.1", + "referenceType": "purl" + } + ], + "licenseConcluded": "NOASSERTION", + "name": "platformdirs", + "primaryPackagePurpose": "SOURCE", + "versionInfo": "3.8.1" + }, + { + "SPDXID": "SPDXRef-PACKAGE-pyparsing", + "checksums": [ + { + "algorithm": "SHA256", + "checksumValue": "d554a96d1a7d3ddaf7183104485bc19fd80543ad6ac5bdb6426719d766fb06c1" + } + ], + "downloadLocation": "https://files.pythonhosted.org/packages/a4/24/6ae4c9c45cf99d96b06b5d99e25526c060303171fb0aea9da2bfd7dbde93/pyparsing-3.1.0-py3-none-any.whl", + "externalRefs": [ + { + "referenceCategory": "PACKAGE_MANAGER", + "referenceLocator": "pkg:pypi/pyparsing@3.1.0", + "referenceType": "purl" + } + ], + "licenseConcluded": "NOASSERTION", + "name": "pyparsing", + "primaryPackagePurpose": "SOURCE", + "versionInfo": "3.1.0" + }, + { + "SPDXID": "SPDXRef-PACKAGE-pyproject-hooks", + "checksums": [ + { + "algorithm": "SHA256", + "checksumValue": "283c11acd6b928d2f6a7c73fa0d01cb2bdc5f07c57a2eeb6e83d5e56b97976f8" + } + ], + "downloadLocation": "https://files.pythonhosted.org/packages/d5/ea/9ae603de7fbb3df820b23a70f6aff92bf8c7770043254ad8d2dc9d6bcba4/pyproject_hooks-1.0.0-py3-none-any.whl", + "externalRefs": [ + { + "referenceCategory": "PACKAGE_MANAGER", + "referenceLocator": "pkg:pypi/pyproject-hooks@1.0.0", + "referenceType": "purl" + } + ], + "licenseConcluded": "NOASSERTION", + "name": "pyproject-hooks", + "primaryPackagePurpose": "SOURCE", + "versionInfo": "1.0.0" + }, + { + "SPDXID": "SPDXRef-PACKAGE-requests", + "checksums": [ + { + "algorithm": "SHA256", + "checksumValue": "58cd2187c01e70e6e26505bca751777aa9f2ee0b7f4300988b709f44e013003f" + } + ], + "downloadLocation": "https://files.pythonhosted.org/packages/70/8e/0e2d847013cb52cd35b38c009bb167a1a26b2ce6cd6965bf26b47bc0bf44/requests-2.31.0-py3-none-any.whl", + "externalRefs": [ + { + "referenceCategory": "PACKAGE_MANAGER", + "referenceLocator": "pkg:pypi/requests@2.31.0", + "referenceType": "purl" + } + ], + "licenseConcluded": "NOASSERTION", + "name": "requests", + "primaryPackagePurpose": "SOURCE", + "versionInfo": "2.31.0" + }, + { + "SPDXID": "SPDXRef-PACKAGE-certifi", + "checksums": [ + { + "algorithm": "SHA256", + "checksumValue": "92d6037539857d8206b8f6ae472e8b77db8058fec5937a1ef3f54304089edbb9" + } + ], + "downloadLocation": "https://files.pythonhosted.org/packages/4c/dd/2234eab22353ffc7d94e8d13177aaa050113286e93e7b40eae01fbf7c3d9/certifi-2023.7.22-py3-none-any.whl", + "externalRefs": [ + { + "referenceCategory": "PACKAGE_MANAGER", + "referenceLocator": "pkg:pypi/certifi@2023.7.22", + "referenceType": "purl" + } + ], + "licenseConcluded": "NOASSERTION", + "name": "certifi", + "primaryPackagePurpose": "SOURCE", + "versionInfo": "2023.7.22" + }, + { + "SPDXID": "SPDXRef-PACKAGE-chardet", + "checksums": [ + { + "algorithm": "SHA256", + "checksumValue": "362777fb014af596ad31334fde1e8c327dfdb076e1960d1694662d46a6917ab9" + } + ], + "downloadLocation": "https://files.pythonhosted.org/packages/74/8f/8fc49109009e8d2169d94d72e6b1f4cd45c13d147ba7d6170fb41f22b08f/chardet-5.1.0-py3-none-any.whl", + "externalRefs": [ + { + "referenceCategory": "PACKAGE_MANAGER", + "referenceLocator": "pkg:pypi/chardet@5.1.0", + "referenceType": "purl" + } + ], + "licenseConcluded": "NOASSERTION", + "name": "chardet", + "primaryPackagePurpose": "SOURCE", + "versionInfo": "5.1.0" + }, + { + "SPDXID": "SPDXRef-PACKAGE-idna", + "checksums": [ + { + "algorithm": "SHA256", + "checksumValue": "90b77e79eaa3eba6de819a0c442c0b4ceefc341a7a2ab77d7562bf49f425c5c2" + } + ], + "downloadLocation": "https://files.pythonhosted.org/packages/fc/34/3030de6f1370931b9dbb4dad48f6ab1015ab1d32447850b9fc94e60097be/idna-3.4-py3-none-any.whl", + "externalRefs": [ + { + "referenceCategory": "PACKAGE_MANAGER", + "referenceLocator": "pkg:pypi/idna@3.4", + "referenceType": "purl" + } + ], + "licenseConcluded": "NOASSERTION", + "name": "idna", + "primaryPackagePurpose": "SOURCE", + "versionInfo": "3.4" + }, + { + "SPDXID": "SPDXRef-PACKAGE-rich", + "checksums": [ + { + "algorithm": "SHA256", + "checksumValue": "8f87bc7ee54675732fa66a05ebfe489e27264caeeff3728c945d25971b6485ec" + } + ], + "downloadLocation": "https://files.pythonhosted.org/packages/fc/1e/482e5eec0b89b593e81d78f819a9412849814e22225842b598908e7ac560/rich-13.4.2-py3-none-any.whl", + "externalRefs": [ + { + "referenceCategory": "PACKAGE_MANAGER", + "referenceLocator": "pkg:pypi/rich@13.4.2", + "referenceType": "purl" + } + ], + "licenseConcluded": "NOASSERTION", + "name": "rich", + "primaryPackagePurpose": "SOURCE", + "versionInfo": "13.4.2" + }, + { + "SPDXID": "SPDXRef-PACKAGE-pygments", + "checksums": [ + { + "algorithm": "SHA256", + "checksumValue": "db2db3deb4b4179f399a09054b023b6a586b76499d36965813c71aa8ed7b5fd1" + } + ], + "downloadLocation": "https://files.pythonhosted.org/packages/34/a7/37c8d68532ba71549db4212cb036dbd6161b40e463aba336770e80c72f84/Pygments-2.15.1-py3-none-any.whl", + "externalRefs": [ + { + "referenceCategory": "PACKAGE_MANAGER", + "referenceLocator": "pkg:pypi/pygments@2.15.1", + "referenceType": "purl" + } + ], + "licenseConcluded": "NOASSERTION", + "name": "pygments", + "primaryPackagePurpose": "SOURCE", + "versionInfo": "2.15.1" + }, + { + "SPDXID": "SPDXRef-PACKAGE-typing-extensions", + "checksums": [ + { + "algorithm": "SHA256", + "checksumValue": "440d5dd3af93b060174bf433bccd69b0babc3b15b1a8dca43789fd7f61514b36" + } + ], + "downloadLocation": "https://files.pythonhosted.org/packages/ec/6b/63cc3df74987c36fe26157ee12e09e8f9db4de771e0f3404263117e75b95/typing_extensions-4.7.1-py3-none-any.whl", + "externalRefs": [ + { + "referenceCategory": "PACKAGE_MANAGER", + "referenceLocator": "pkg:pypi/typing_extensions@4.7.1", + "referenceType": "purl" + } + ], + "licenseConcluded": "NOASSERTION", + "name": "typing_extensions", + "primaryPackagePurpose": "SOURCE", + "versionInfo": "4.7.1" + }, + { + "SPDXID": "SPDXRef-PACKAGE-resolvelib", + "checksums": [ + { + "algorithm": "SHA256", + "checksumValue": "d2da45d1a8dfee81bdd591647783e340ef3bcb104b54c383f70d422ef5cc7dbf" + } + ], + "downloadLocation": "https://files.pythonhosted.org/packages/d2/fc/e9ccf0521607bcd244aa0b3fbd574f71b65e9ce6a112c83af988bbbe2e23/resolvelib-1.0.1-py2.py3-none-any.whl", + "externalRefs": [ + { + "referenceCategory": "PACKAGE_MANAGER", + "referenceLocator": "pkg:pypi/resolvelib@1.0.1", + "referenceType": "purl" + } + ], + "licenseConcluded": "NOASSERTION", + "name": "resolvelib", + "primaryPackagePurpose": "SOURCE", + "versionInfo": "1.0.1" + }, + { + "SPDXID": "SPDXRef-PACKAGE-setuptools", + "checksums": [ + { + "algorithm": "SHA256", + "checksumValue": "11e52c67415a381d10d6b462ced9cfb97066179f0e871399e006c4ab101fc85f" + } + ], + "downloadLocation": "https://files.pythonhosted.org/packages/c7/42/be1c7bbdd83e1bfb160c94b9cafd8e25efc7400346cf7ccdbdb452c467fa/setuptools-68.0.0-py3-none-any.whl", + "externalRefs": [ + { + "referenceCategory": "PACKAGE_MANAGER", + "referenceLocator": "pkg:pypi/setuptools@68.0.0", + "referenceType": "purl" + } + ], + "licenseConcluded": "NOASSERTION", + "name": "setuptools", + "primaryPackagePurpose": "SOURCE", + "versionInfo": "68.0.0" + }, + { + "SPDXID": "SPDXRef-PACKAGE-six", + "checksums": [ + { + "algorithm": "SHA256", + "checksumValue": "8abb2f1d86890a2dfb989f9a77cfcfd3e47c2a354b01111771326f8aa26e0254" + } + ], + "downloadLocation": "https://files.pythonhosted.org/packages/d9/5a/e7c31adbe875f2abbb91bd84cf2dc52d792b5a01506781dbcf25c91daf11/six-1.16.0-py2.py3-none-any.whl", + "externalRefs": [ + { + "referenceCategory": "PACKAGE_MANAGER", + "referenceLocator": "pkg:pypi/six@1.16.0", + "referenceType": "purl" + } + ], + "licenseConcluded": "NOASSERTION", + "name": "six", + "primaryPackagePurpose": "SOURCE", + "versionInfo": "1.16.0" + }, + { + "SPDXID": "SPDXRef-PACKAGE-tenacity", + "checksums": [ + { + "algorithm": "SHA256", + "checksumValue": "2f277afb21b851637e8f52e6a613ff08734c347dc19ade928e519d7d2d8569b0" + } + ], + "downloadLocation": "https://files.pythonhosted.org/packages/e7/b0/c23bd61e1b32c9b96fbca996c87784e196a812da8d621d8d04851f6c8181/tenacity-8.2.2-py3-none-any.whl", + "externalRefs": [ + { + "referenceCategory": "PACKAGE_MANAGER", + "referenceLocator": "pkg:pypi/tenacity@8.2.2", + "referenceType": "purl" + } + ], + "licenseConcluded": "NOASSERTION", + "name": "tenacity", + "primaryPackagePurpose": "SOURCE", + "versionInfo": "8.2.2" + }, + { + "SPDXID": "SPDXRef-PACKAGE-tomli", + "checksums": [ + { + "algorithm": "SHA256", + "checksumValue": "939de3e7a6161af0c887ef91b7d41a53e7c5a1ca976325f429cb46ea9bc30ecc" + } + ], + "downloadLocation": "https://files.pythonhosted.org/packages/97/75/10a9ebee3fd790d20926a90a2547f0bf78f371b2f13aa822c759680ca7b9/tomli-2.0.1-py3-none-any.whl", + "externalRefs": [ + { + "referenceCategory": "PACKAGE_MANAGER", + "referenceLocator": "pkg:pypi/tomli@2.0.1", + "referenceType": "purl" + } + ], + "licenseConcluded": "NOASSERTION", + "name": "tomli", + "primaryPackagePurpose": "SOURCE", + "versionInfo": "2.0.1" + }, + { + "SPDXID": "SPDXRef-PACKAGE-truststore", + "checksums": [ + { + "algorithm": "SHA256", + "checksumValue": "e37a5642ae9fc48caa8f120b6283d77225d600d224965a672c9e8ef49ce4bb4c" + } + ], + "downloadLocation": "https://files.pythonhosted.org/packages/20/56/7811d5439b6a56374f274a8672d8f18b4deadadeb3a9f0c86424b98b6f96/truststore-0.8.0-py3-none-any.whl", + "externalRefs": [ + { + "referenceCategory": "PACKAGE_MANAGER", + "referenceLocator": "pkg:pypi/truststore@0.8.0", + "referenceType": "purl" + } + ], + "licenseConcluded": "NOASSERTION", + "name": "truststore", + "primaryPackagePurpose": "SOURCE", + "versionInfo": "0.8.0" + }, + { + "SPDXID": "SPDXRef-PACKAGE-webencodings", + "checksums": [ + { + "algorithm": "SHA256", + "checksumValue": "a0af1213f3c2226497a97e2b3aa01a7e4bee4f403f95be16fc9acd2947514a78" + } + ], + "downloadLocation": "https://files.pythonhosted.org/packages/f4/24/2a3e3df732393fed8b3ebf2ec078f05546de641fe1b667ee316ec1dcf3b7/webencodings-0.5.1-py2.py3-none-any.whl", + "externalRefs": [ + { + "referenceCategory": "PACKAGE_MANAGER", + "referenceLocator": "pkg:pypi/webencodings@0.5.1", + "referenceType": "purl" + } + ], + "licenseConcluded": "NOASSERTION", + "name": "webencodings", + "primaryPackagePurpose": "SOURCE", + "versionInfo": "0.5.1" + }, + { + "SPDXID": "SPDXRef-PACKAGE-urllib3", + "checksums": [ + { + "algorithm": "SHA256", + "checksumValue": "94a757d178c9be92ef5539b8840d48dc9cf1b2709c9d6b588232a055c524458b" + } + ], + "downloadLocation": "https://files.pythonhosted.org/packages/48/fe/a5c6cc46e9fe9171d7ecf0f33ee7aae14642f8d74baa7af4d7840f9358be/urllib3-1.26.17-py2.py3-none-any.whl", + "externalRefs": [ + { + "referenceCategory": "PACKAGE_MANAGER", + "referenceLocator": "pkg:pypi/urllib3@1.26.17", + "referenceType": "purl" + } + ], + "licenseConcluded": "NOASSERTION", + "name": "urllib3", + "primaryPackagePurpose": "SOURCE", + "versionInfo": "1.26.17" + }, { "SPDXID": "SPDXRef-PACKAGE-pip", "checksums": [ { "algorithm": "SHA256", - "checksumValue": "5052d7889c1f9d05224cd41741acb7c5d6fa735ab34e339624a614eaaa7e7d76" + "checksumValue": "ba0d021a166865d2265246961bec0152ff124de910c5cc39f1156ce3fa7c69dc" } ], - "downloadLocation": "https://files.pythonhosted.org/packages/15/aa/3f4c7bcee2057a76562a5b33ecbd199be08cdb4443a02e26bd2c3cf6fc39/pip-23.3.2-py3-none-any.whl", + "downloadLocation": "https://files.pythonhosted.org/packages/8a/6a/19e9fe04fca059ccf770861c7d5721ab4c2aebc539889e97c7977528a53b/pip-24.0-py3-none-any.whl", "externalRefs": [ { "referenceCategory": "SECURITY", - "referenceLocator": "cpe:2.3:a:pypa:pip:23.3.2:*:*:*:*:*:*:*", + "referenceLocator": "cpe:2.3:a:pypa:pip:24.0:*:*:*:*:*:*:*", "referenceType": "cpe23Type" }, { "referenceCategory": "PACKAGE_MANAGER", - "referenceLocator": "pkg:pypi/pip@23.3.2", + "referenceLocator": "pkg:pypi/pip@24.0", "referenceType": "purl" } ], - "licenseConcluded": "MIT", + "licenseConcluded": "NOASSERTION", "name": "pip", "originator": "Organization: Python Packaging Authority", "primaryPackagePurpose": "SOURCE", - "versionInfo": "23.3.2" + "versionInfo": "24.0" } ], "relationships": [ + { + "relatedSpdxElement": "SPDXRef-PACKAGE-cachecontrol", + "relationshipType": "DEPENDS_ON", + "spdxElementId": "SPDXRef-PACKAGE-pip" + }, + { + "relatedSpdxElement": "SPDXRef-PACKAGE-certifi", + "relationshipType": "DEPENDS_ON", + "spdxElementId": "SPDXRef-PACKAGE-pip" + }, + { + "relatedSpdxElement": "SPDXRef-PACKAGE-chardet", + "relationshipType": "DEPENDS_ON", + "spdxElementId": "SPDXRef-PACKAGE-pip" + }, + { + "relatedSpdxElement": "SPDXRef-PACKAGE-colorama", + "relationshipType": "DEPENDS_ON", + "spdxElementId": "SPDXRef-PACKAGE-pip" + }, + { + "relatedSpdxElement": "SPDXRef-PACKAGE-distlib", + "relationshipType": "DEPENDS_ON", + "spdxElementId": "SPDXRef-PACKAGE-pip" + }, + { + "relatedSpdxElement": "SPDXRef-PACKAGE-distro", + "relationshipType": "DEPENDS_ON", + "spdxElementId": "SPDXRef-PACKAGE-pip" + }, + { + "relatedSpdxElement": "SPDXRef-PACKAGE-idna", + "relationshipType": "DEPENDS_ON", + "spdxElementId": "SPDXRef-PACKAGE-pip" + }, + { + "relatedSpdxElement": "SPDXRef-PACKAGE-msgpack", + "relationshipType": "DEPENDS_ON", + "spdxElementId": "SPDXRef-PACKAGE-pip" + }, + { + "relatedSpdxElement": "SPDXRef-PACKAGE-packaging", + "relationshipType": "DEPENDS_ON", + "spdxElementId": "SPDXRef-PACKAGE-pip" + }, + { + "relatedSpdxElement": "SPDXRef-PACKAGE-platformdirs", + "relationshipType": "DEPENDS_ON", + "spdxElementId": "SPDXRef-PACKAGE-pip" + }, + { + "relatedSpdxElement": "SPDXRef-PACKAGE-pygments", + "relationshipType": "DEPENDS_ON", + "spdxElementId": "SPDXRef-PACKAGE-pip" + }, + { + "relatedSpdxElement": "SPDXRef-PACKAGE-pyparsing", + "relationshipType": "DEPENDS_ON", + "spdxElementId": "SPDXRef-PACKAGE-pip" + }, + { + "relatedSpdxElement": "SPDXRef-PACKAGE-pyproject-hooks", + "relationshipType": "DEPENDS_ON", + "spdxElementId": "SPDXRef-PACKAGE-pip" + }, + { + "relatedSpdxElement": "SPDXRef-PACKAGE-requests", + "relationshipType": "DEPENDS_ON", + "spdxElementId": "SPDXRef-PACKAGE-pip" + }, + { + "relatedSpdxElement": "SPDXRef-PACKAGE-resolvelib", + "relationshipType": "DEPENDS_ON", + "spdxElementId": "SPDXRef-PACKAGE-pip" + }, + { + "relatedSpdxElement": "SPDXRef-PACKAGE-rich", + "relationshipType": "DEPENDS_ON", + "spdxElementId": "SPDXRef-PACKAGE-pip" + }, + { + "relatedSpdxElement": "SPDXRef-PACKAGE-setuptools", + "relationshipType": "DEPENDS_ON", + "spdxElementId": "SPDXRef-PACKAGE-pip" + }, + { + "relatedSpdxElement": "SPDXRef-PACKAGE-six", + "relationshipType": "DEPENDS_ON", + "spdxElementId": "SPDXRef-PACKAGE-pip" + }, + { + "relatedSpdxElement": "SPDXRef-PACKAGE-tenacity", + "relationshipType": "DEPENDS_ON", + "spdxElementId": "SPDXRef-PACKAGE-pip" + }, + { + "relatedSpdxElement": "SPDXRef-PACKAGE-tomli", + "relationshipType": "DEPENDS_ON", + "spdxElementId": "SPDXRef-PACKAGE-pip" + }, + { + "relatedSpdxElement": "SPDXRef-PACKAGE-truststore", + "relationshipType": "DEPENDS_ON", + "spdxElementId": "SPDXRef-PACKAGE-pip" + }, + { + "relatedSpdxElement": "SPDXRef-PACKAGE-typing-extensions", + "relationshipType": "DEPENDS_ON", + "spdxElementId": "SPDXRef-PACKAGE-pip" + }, + { + "relatedSpdxElement": "SPDXRef-PACKAGE-urllib3", + "relationshipType": "DEPENDS_ON", + "spdxElementId": "SPDXRef-PACKAGE-pip" + }, + { + "relatedSpdxElement": "SPDXRef-PACKAGE-webencodings", + "relationshipType": "DEPENDS_ON", + "spdxElementId": "SPDXRef-PACKAGE-pip" + }, { "relatedSpdxElement": "SPDXRef-FILE-Modules-expat-COPYING", "relationshipType": "CONTAINS", @@ -2285,7 +2909,7 @@ "spdxElementId": "SPDXRef-PACKAGE-mpdecimal" }, { - "relatedSpdxElement": "SPDXRef-FILE-Lib-ensurepip-bundled-pip-23.3.2-py3-none-any.whl", + "relatedSpdxElement": "SPDXRef-FILE-Lib-ensurepip-bundled-pip-24.0-py3-none-any.whl", "relationshipType": "CONTAINS", "spdxElementId": "SPDXRef-PACKAGE-pip" } diff --git a/Misc/stable_abi.toml b/Misc/stable_abi.toml index 22b25dd0ec141f..a9875f6ffd1a56 100644 --- a/Misc/stable_abi.toml +++ b/Misc/stable_abi.toml @@ -78,6 +78,10 @@ [feature_macro.Py_REF_DEBUG] doc = 'when Python is compiled in debug mode (with Py_REF_DEBUG)' windows = 'maybe' +[feature_macro.Py_TRACE_REFS] + # nb. This mode is not compatible with Stable ABI/Limited API. + doc = 'when Python is compiled with Py_TRACE_REFS' + windows = 'maybe' # Mentioned in PEP 384: @@ -2481,3 +2485,7 @@ [function._Py_SetRefcnt] added = '3.13' abi_only = true +[data.PyExc_IncompleteInputError] + added = '3.13' +[function.PyList_GetItemRef] + added = '3.13' diff --git a/Modules/Setup.bootstrap.in b/Modules/Setup.bootstrap.in index cd12c1bd0df8f9..aa4e60e272653b 100644 --- a/Modules/Setup.bootstrap.in +++ b/Modules/Setup.bootstrap.in @@ -11,6 +11,7 @@ faulthandler faulthandler.c posix posixmodule.c _signal signalmodule.c _tracemalloc _tracemalloc.c +_suggestions _suggestions.c # modules used by importlib, deepfreeze, freeze, runpy, and sysconfig _codecs _codecsmodule.c diff --git a/Modules/_asynciomodule.c b/Modules/_asynciomodule.c index 3a11cdc926f138..c1aa849ecf1aad 100644 --- a/Modules/_asynciomodule.c +++ b/Modules/_asynciomodule.c @@ -597,12 +597,27 @@ future_set_exception(asyncio_state *state, FutureObj *fut, PyObject *exc) PyErr_SetString(PyExc_TypeError, "invalid exception object"); return NULL; } - if (Py_IS_TYPE(exc_val, (PyTypeObject *)PyExc_StopIteration)) { + if (PyErr_GivenExceptionMatches(exc_val, PyExc_StopIteration)) { + const char *msg = "StopIteration interacts badly with " + "generators and cannot be raised into a " + "Future"; + PyObject *message = PyUnicode_FromString(msg); + if (message == NULL) { + Py_DECREF(exc_val); + return NULL; + } + PyObject *err = PyObject_CallOneArg(PyExc_RuntimeError, message); + Py_DECREF(message); + if (err == NULL) { + Py_DECREF(exc_val); + return NULL; + } + assert(PyExceptionInstance_Check(err)); + + PyException_SetCause(err, Py_NewRef(exc_val)); + PyException_SetContext(err, Py_NewRef(exc_val)); Py_DECREF(exc_val); - PyErr_SetString(PyExc_TypeError, - "StopIteration interacts badly with generators " - "and cannot be raised into a Future"); - return NULL; + exc_val = err; } assert(!fut->fut_exception); @@ -2754,7 +2769,6 @@ gen_status_from_result(PyObject **result) static PyObject * task_step_impl(asyncio_state *state, TaskObj *task, PyObject *exc) { - int res; int clear_exc = 0; PyObject *result = NULL; PyObject *coro; @@ -2771,20 +2785,7 @@ task_step_impl(asyncio_state *state, TaskObj *task, PyObject *exc) if (task->task_must_cancel) { assert(exc != Py_None); - if (exc) { - /* Check if exc is a CancelledError */ - res = PyObject_IsInstance(exc, state->asyncio_CancelledError); - if (res == -1) { - /* An error occurred, abort */ - goto fail; - } - if (res == 0) { - /* exc is not CancelledError; reset it to NULL */ - exc = NULL; - } - } - - if (!exc) { + if (!exc || !PyErr_GivenExceptionMatches(exc, state->asyncio_CancelledError)) { /* exc was not a CancelledError */ exc = create_cancelled_error(state, (FutureObj*)task); diff --git a/Modules/_collectionsmodule.c b/Modules/_collectionsmodule.c index c8cd53de5e2262..ef77d34b10e47b 100644 --- a/Modules/_collectionsmodule.c +++ b/Modules/_collectionsmodule.c @@ -44,8 +44,11 @@ find_module_state_by_def(PyTypeObject *type) /*[clinic input] module _collections class _tuplegetter "_tuplegetterobject *" "clinic_state()->tuplegetter_type" +class _collections.deque "dequeobject *" "clinic_state()->deque_type" [clinic start generated code]*/ -/*[clinic end generated code: output=da39a3ee5e6b4b0d input=7356042a89862e0e]*/ +/*[clinic end generated code: output=da39a3ee5e6b4b0d input=a033cc2a8476b3f1]*/ + +typedef struct dequeobject dequeobject; /* We can safely assume type to be the defining class, * since tuplegetter is not a base type */ @@ -53,6 +56,12 @@ class _tuplegetter "_tuplegetterobject *" "clinic_state()->tuplegetter_type" #include "clinic/_collectionsmodule.c.h" #undef clinic_state +/*[python input] +class dequeobject_converter(self_converter): + type = "dequeobject *" +[python start generated code]*/ +/*[python end generated code: output=da39a3ee5e6b4b0d input=b6ae4a3ff852be2f]*/ + /* collections module implementation of a deque() datatype Written and maintained by Raymond D. Hettinger */ @@ -121,7 +130,7 @@ typedef struct BLOCK { struct BLOCK *rightlink; } block; -typedef struct { +struct dequeobject { PyObject_VAR_HEAD block *leftblock; block *rightblock; @@ -132,7 +141,7 @@ typedef struct { Py_ssize_t numfreeblocks; block *freeblocks[MAXFREEBLOCKS]; PyObject *weakreflist; -} dequeobject; +}; /* For debug builds, add error checking to track the endpoints * in the chain of links. The goal is to make sure that link @@ -219,8 +228,17 @@ deque_new(PyTypeObject *type, PyObject *args, PyObject *kwds) return (PyObject *)deque; } +/*[clinic input] +_collections.deque.pop as deque_pop + + deque: dequeobject + +Remove and return the rightmost element. +[clinic start generated code]*/ + static PyObject * -deque_pop(dequeobject *deque, PyObject *unused) +deque_pop_impl(dequeobject *deque) +/*[clinic end generated code: output=2e5f7890c4251f07 input=eb6e6d020f877dec]*/ { PyObject *item; block *prevblock; @@ -254,10 +272,17 @@ deque_pop(dequeobject *deque, PyObject *unused) return item; } -PyDoc_STRVAR(pop_doc, "Remove and return the rightmost element."); +/*[clinic input] +_collections.deque.popleft as deque_popleft + + deque: dequeobject + +Remove and return the leftmost element. +[clinic start generated code]*/ static PyObject * -deque_popleft(dequeobject *deque, PyObject *unused) +deque_popleft_impl(dequeobject *deque) +/*[clinic end generated code: output=62b154897097ff68 input=acb41b9af50a9d9b]*/ { PyObject *item; block *prevblock; @@ -292,8 +317,6 @@ deque_popleft(dequeobject *deque, PyObject *unused) return item; } -PyDoc_STRVAR(popleft_doc, "Remove and return the leftmost element."); - /* The deque's size limit is d.maxlen. The limit can be zero or positive. * If there is no limit, then d.maxlen == -1. * @@ -326,7 +349,7 @@ deque_append_internal(dequeobject *deque, PyObject *item, Py_ssize_t maxlen) deque->rightindex++; deque->rightblock->data[deque->rightindex] = item; if (NEEDS_TRIM(deque, maxlen)) { - PyObject *olditem = deque_popleft(deque, NULL); + PyObject *olditem = deque_popleft_impl(deque); Py_DECREF(olditem); } else { deque->state++; @@ -334,16 +357,25 @@ deque_append_internal(dequeobject *deque, PyObject *item, Py_ssize_t maxlen) return 0; } +/*[clinic input] +_collections.deque.append as deque_append + + deque: dequeobject + item: object + / + +Add an element to the right side of the deque. +[clinic start generated code]*/ + static PyObject * deque_append(dequeobject *deque, PyObject *item) +/*[clinic end generated code: output=507b13efc4853ecc input=f112b83c380528e3]*/ { if (deque_append_internal(deque, Py_NewRef(item), deque->maxlen) < 0) return NULL; Py_RETURN_NONE; } -PyDoc_STRVAR(append_doc, "Add an element to the right side of the deque."); - static inline int deque_appendleft_internal(dequeobject *deque, PyObject *item, Py_ssize_t maxlen) { @@ -362,7 +394,7 @@ deque_appendleft_internal(dequeobject *deque, PyObject *item, Py_ssize_t maxlen) deque->leftindex--; deque->leftblock->data[deque->leftindex] = item; if (NEEDS_TRIM(deque, deque->maxlen)) { - PyObject *olditem = deque_pop(deque, NULL); + PyObject *olditem = deque_pop_impl(deque); Py_DECREF(olditem); } else { deque->state++; @@ -370,16 +402,25 @@ deque_appendleft_internal(dequeobject *deque, PyObject *item, Py_ssize_t maxlen) return 0; } +/*[clinic input] +_collections.deque.appendleft as deque_appendleft + + deque: dequeobject + item: object + / + +Add an element to the left side of the deque. +[clinic start generated code]*/ + static PyObject * deque_appendleft(dequeobject *deque, PyObject *item) +/*[clinic end generated code: output=de0335a64800ffd8 input=bbdaa60a3e956062]*/ { if (deque_appendleft_internal(deque, Py_NewRef(item), deque->maxlen) < 0) return NULL; Py_RETURN_NONE; } -PyDoc_STRVAR(appendleft_doc, "Add an element to the left side of the deque."); - static PyObject* finalize_iterator(PyObject *it) { @@ -410,8 +451,19 @@ consume_iterator(PyObject *it) return finalize_iterator(it); } +/*[clinic input] +_collections.deque.extend as deque_extend + + deque: dequeobject + iterable: object + / + +Extend the right side of the deque with elements from the iterable. +[clinic start generated code]*/ + static PyObject * deque_extend(dequeobject *deque, PyObject *iterable) +/*[clinic end generated code: output=a3a6e74d17063f8d input=cfebfd34d5383339]*/ { PyObject *it, *item; PyObject *(*iternext)(PyObject *); @@ -454,11 +506,19 @@ deque_extend(dequeobject *deque, PyObject *iterable) return finalize_iterator(it); } -PyDoc_STRVAR(extend_doc, -"Extend the right side of the deque with elements from the iterable"); +/*[clinic input] +_collections.deque.extendleft as deque_extendleft + + deque: dequeobject + iterable: object + / + +Extend the left side of the deque with elements from the iterable. +[clinic start generated code]*/ static PyObject * deque_extendleft(dequeobject *deque, PyObject *iterable) +/*[clinic end generated code: output=2dba946c50498c67 input=f4820e695a6f9416]*/ { PyObject *it, *item; PyObject *(*iternext)(PyObject *); @@ -501,9 +561,6 @@ deque_extendleft(dequeobject *deque, PyObject *iterable) return finalize_iterator(it); } -PyDoc_STRVAR(extendleft_doc, -"Extend the left side of the deque with elements from the iterable"); - static PyObject * deque_inplace_concat(dequeobject *deque, PyObject *other) { @@ -517,8 +574,17 @@ deque_inplace_concat(dequeobject *deque, PyObject *other) return (PyObject *)deque; } +/*[clinic input] +_collections.deque.copy as deque_copy + + deque: dequeobject + +Return a shallow copy of a deque. +[clinic start generated code]*/ + static PyObject * -deque_copy(PyObject *deque, PyObject *Py_UNUSED(ignored)) +deque_copy_impl(dequeobject *deque) +/*[clinic end generated code: output=6409b3d1ad2898b5 input=0e22f138bc1fcbee]*/ { PyObject *result; dequeobject *old_deque = (dequeobject *)deque; @@ -537,7 +603,7 @@ deque_copy(PyObject *deque, PyObject *Py_UNUSED(ignored)) PyObject *item = old_deque->leftblock->data[old_deque->leftindex]; rv = deque_append(new_deque, item); } else { - rv = deque_extend(new_deque, deque); + rv = deque_extend(new_deque, (PyObject *)deque); } if (rv != NULL) { Py_DECREF(rv); @@ -547,7 +613,8 @@ deque_copy(PyObject *deque, PyObject *Py_UNUSED(ignored)) return NULL; } if (old_deque->maxlen < 0) - result = PyObject_CallOneArg((PyObject *)(Py_TYPE(deque)), deque); + result = PyObject_CallOneArg((PyObject *)(Py_TYPE(deque)), + (PyObject *)deque); else result = PyObject_CallFunction((PyObject *)(Py_TYPE(deque)), "Oi", deque, old_deque->maxlen, NULL); @@ -561,7 +628,18 @@ deque_copy(PyObject *deque, PyObject *Py_UNUSED(ignored)) return result; } -PyDoc_STRVAR(copy_doc, "Return a shallow copy of a deque."); +/*[clinic input] +_collections.deque.__copy__ as deque___copy__ = _collections.deque.copy + +Return a shallow copy of a deque. +[clinic start generated code]*/ + +static PyObject * +deque___copy___impl(dequeobject *deque) +/*[clinic end generated code: output=7c5821504342bf23 input=fce05df783e7912b]*/ +{ + return deque_copy_impl(deque); +} static PyObject * deque_concat(dequeobject *deque, PyObject *other) @@ -580,7 +658,7 @@ deque_concat(dequeobject *deque, PyObject *other) return NULL; } - new_deque = deque_copy((PyObject *)deque, NULL); + new_deque = deque_copy_impl(deque); if (new_deque == NULL) return NULL; result = deque_extend((dequeobject *)new_deque, other); @@ -669,22 +747,29 @@ deque_clear(dequeobject *deque) alternate_method: while (Py_SIZE(deque)) { - item = deque_pop(deque, NULL); + item = deque_pop_impl(deque); assert (item != NULL); Py_DECREF(item); } return 0; } +/*[clinic input] +_collections.deque.clear as deque_clearmethod + + deque: dequeobject + +Remove all elements from the deque. +[clinic start generated code]*/ + static PyObject * -deque_clearmethod(dequeobject *deque, PyObject *Py_UNUSED(ignored)) +deque_clearmethod_impl(dequeobject *deque) +/*[clinic end generated code: output=79b2513e097615c1 input=20488eb932f89f9e]*/ { deque_clear(deque); Py_RETURN_NONE; } -PyDoc_STRVAR(clear_doc, "Remove all elements from the deque."); - static PyObject * deque_inplace_repeat(dequeobject *deque, Py_ssize_t n) { @@ -768,7 +853,7 @@ deque_repeat(dequeobject *deque, Py_ssize_t n) dequeobject *new_deque; PyObject *rv; - new_deque = (dequeobject *)deque_copy((PyObject *) deque, NULL); + new_deque = (dequeobject *)deque_copy_impl(deque); if (new_deque == NULL) return NULL; rv = deque_inplace_repeat(new_deque, n); @@ -925,36 +1010,36 @@ _deque_rotate(dequeobject *deque, Py_ssize_t n) return rv; } -static PyObject * -deque_rotate(dequeobject *deque, PyObject *const *args, Py_ssize_t nargs) -{ - Py_ssize_t n=1; +/*[clinic input] +_collections.deque.rotate as deque_rotate - if (!_PyArg_CheckPositional("deque.rotate", nargs, 0, 1)) { - return NULL; - } - if (nargs) { - PyObject *index = _PyNumber_Index(args[0]); - if (index == NULL) { - return NULL; - } - n = PyLong_AsSsize_t(index); - Py_DECREF(index); - if (n == -1 && PyErr_Occurred()) { - return NULL; - } - } + deque: dequeobject + n: Py_ssize_t = 1 + / +Rotate the deque n steps to the right. If n is negative, rotates left. +[clinic start generated code]*/ + +static PyObject * +deque_rotate_impl(dequeobject *deque, Py_ssize_t n) +/*[clinic end generated code: output=96c2402a371eb15d input=d22070f49cc06c76]*/ +{ if (!_deque_rotate(deque, n)) Py_RETURN_NONE; return NULL; } -PyDoc_STRVAR(rotate_doc, -"Rotate the deque n steps to the right (default n=1). If n is negative, rotates left."); +/*[clinic input] +_collections.deque.reverse as deque_reverse + + deque: dequeobject + +Reverse *IN PLACE*. +[clinic start generated code]*/ static PyObject * -deque_reverse(dequeobject *deque, PyObject *unused) +deque_reverse_impl(dequeobject *deque) +/*[clinic end generated code: output=bdeebc2cf8c1f064 input=f139787f406101c9]*/ { block *leftblock = deque->leftblock; block *rightblock = deque->rightblock; @@ -991,11 +1076,19 @@ deque_reverse(dequeobject *deque, PyObject *unused) Py_RETURN_NONE; } -PyDoc_STRVAR(reverse_doc, -"D.reverse() -- reverse *IN PLACE*"); +/*[clinic input] +_collections.deque.count as deque_count + + deque: dequeobject + value as v: object + / + +Return number of occurrences of value. +[clinic start generated code]*/ static PyObject * deque_count(dequeobject *deque, PyObject *v) +/*[clinic end generated code: output=7405d289d94d7b9b input=1892925260ff5d78]*/ { block *b = deque->leftblock; Py_ssize_t index = deque->leftindex; @@ -1030,9 +1123,6 @@ deque_count(dequeobject *deque, PyObject *v) return PyLong_FromSsize_t(count); } -PyDoc_STRVAR(count_doc, -"D.count(value) -- return number of occurrences of value"); - static int deque_contains(dequeobject *deque, PyObject *v) { @@ -1071,22 +1161,33 @@ deque_len(dequeobject *deque) return Py_SIZE(deque); } +/*[clinic input] +@text_signature "($self, value, [start, [stop]])" +_collections.deque.index as deque_index + + deque: dequeobject + value as v: object + start: object(converter='_PyEval_SliceIndexNotNone', type='Py_ssize_t', c_default='0') = NULL + stop: object(converter='_PyEval_SliceIndexNotNone', type='Py_ssize_t', c_default='Py_SIZE(deque)') = NULL + / + +Return first index of value. + +Raises ValueError if the value is not present. +[clinic start generated code]*/ + static PyObject * -deque_index(dequeobject *deque, PyObject *const *args, Py_ssize_t nargs) +deque_index_impl(dequeobject *deque, PyObject *v, Py_ssize_t start, + Py_ssize_t stop) +/*[clinic end generated code: output=df45132753175ef9 input=140210c099830f64]*/ { - Py_ssize_t i, n, start=0, stop=Py_SIZE(deque); - PyObject *v, *item; + Py_ssize_t i, n; + PyObject *item; block *b = deque->leftblock; Py_ssize_t index = deque->leftindex; size_t start_state = deque->state; int cmp; - if (!_PyArg_ParseStack(args, nargs, "O|O&O&:index", &v, - _PyEval_SliceIndexNotNone, &start, - _PyEval_SliceIndexNotNone, &stop)) { - return NULL; - } - if (start < 0) { start += Py_SIZE(deque); if (start < 0) @@ -1138,10 +1239,6 @@ deque_index(dequeobject *deque, PyObject *const *args, Py_ssize_t nargs) return NULL; } -PyDoc_STRVAR(index_doc, -"D.index(value, [start, [stop]]) -- return first index of value.\n" -"Raises ValueError if the value is not present."); - /* insert(), remove(), and delitem() are implemented in terms of rotate() for simplicity and reasonable performance near the end points. If for some reason these methods become popular, it is not @@ -1150,18 +1247,24 @@ PyDoc_STRVAR(index_doc, boost (by moving each pointer only once instead of twice). */ +/*[clinic input] +_collections.deque.insert as deque_insert + + deque: dequeobject + index: Py_ssize_t + value: object + / + +Insert value before index. +[clinic start generated code]*/ + static PyObject * -deque_insert(dequeobject *deque, PyObject *const *args, Py_ssize_t nargs) +deque_insert_impl(dequeobject *deque, Py_ssize_t index, PyObject *value) +/*[clinic end generated code: output=ef4d2c15d5532b80 input=3e5c1c120d70c0e6]*/ { - Py_ssize_t index; Py_ssize_t n = Py_SIZE(deque); - PyObject *value; PyObject *rv; - if (!_PyArg_ParseStack(args, nargs, "nO:insert", &index, &value)) { - return NULL; - } - if (deque->maxlen == Py_SIZE(deque)) { PyErr_SetString(PyExc_IndexError, "deque already at its maximum size"); return NULL; @@ -1184,12 +1287,6 @@ deque_insert(dequeobject *deque, PyObject *const *args, Py_ssize_t nargs) Py_RETURN_NONE; } -PyDoc_STRVAR(insert_doc, -"D.insert(index, object) -- insert object before index"); - -PyDoc_STRVAR(remove_doc, -"D.remove(value) -- remove first occurrence of value."); - static int valid_index(Py_ssize_t i, Py_ssize_t limit) { @@ -1246,15 +1343,26 @@ deque_del_item(dequeobject *deque, Py_ssize_t i) assert (i >= 0 && i < Py_SIZE(deque)); if (_deque_rotate(deque, -i)) return -1; - item = deque_popleft(deque, NULL); + item = deque_popleft_impl(deque); rv = _deque_rotate(deque, i); assert (item != NULL); Py_DECREF(item); return rv; } +/*[clinic input] +_collections.deque.remove as deque_remove + + deque: dequeobject + value: object + / + +Remove first occurrence of value. +[clinic start generated code]*/ + static PyObject * deque_remove(dequeobject *deque, PyObject *value) +/*[clinic end generated code: output=49e1666d612fe911 input=d972f32d15990880]*/ { PyObject *item; block *b = deque->leftblock; @@ -1375,8 +1483,17 @@ deque_traverse(dequeobject *deque, visitproc visit, void *arg) return 0; } +/*[clinic input] +_collections.deque.__reduce__ as deque___reduce__ + + deque: dequeobject + +Return state information for pickling. +[clinic start generated code]*/ + static PyObject * -deque_reduce(dequeobject *deque, PyObject *Py_UNUSED(ignored)) +deque___reduce___impl(dequeobject *deque) +/*[clinic end generated code: output=cb85d9e0b7d2c5ad input=991a933a5bc7a526]*/ { PyObject *state, *it; @@ -1510,26 +1627,23 @@ deque_richcompare(PyObject *v, PyObject *w, int op) return NULL; } +/*[clinic input] +@text_signature "([iterable[, maxlen]])" +_collections.deque.__init__ as deque_init + + deque: dequeobject + iterable: object = NULL + maxlen as maxlenobj: object = NULL + +A list-like sequence optimized for data accesses near its endpoints. +[clinic start generated code]*/ + static int -deque_init(dequeobject *deque, PyObject *args, PyObject *kwdargs) +deque_init_impl(dequeobject *deque, PyObject *iterable, PyObject *maxlenobj) +/*[clinic end generated code: output=7084a39d71218dcd input=5ebdffc48a2d27ae]*/ + { - PyObject *iterable = NULL; - PyObject *maxlenobj = NULL; Py_ssize_t maxlen = -1; - char *kwlist[] = {"iterable", "maxlen", 0}; - - if (kwdargs == NULL && PyTuple_GET_SIZE(args) <= 2) { - if (PyTuple_GET_SIZE(args) > 0) { - iterable = PyTuple_GET_ITEM(args, 0); - } - if (PyTuple_GET_SIZE(args) > 1) { - maxlenobj = PyTuple_GET_ITEM(args, 1); - } - } else { - if (!PyArg_ParseTupleAndKeywords(args, kwdargs, "|OO:deque", kwlist, - &iterable, &maxlenobj)) - return -1; - } if (maxlenobj != NULL && maxlenobj != Py_None) { maxlen = PyLong_AsSsize_t(maxlenobj); if (maxlen == -1 && PyErr_Occurred()) @@ -1551,8 +1665,17 @@ deque_init(dequeobject *deque, PyObject *args, PyObject *kwdargs) return 0; } +/*[clinic input] +_collections.deque.__sizeof__ as deque___sizeof__ + + deque: dequeobject + +Return the size of the deque in memory, in bytes. +[clinic start generated code]*/ + static PyObject * -deque_sizeof(dequeobject *deque, void *unused) +deque___sizeof___impl(dequeobject *deque) +/*[clinic end generated code: output=4d36e9fb4f30bbaf input=4e7c9a00c03c3290]*/ { size_t res = _PyObject_SIZE(Py_TYPE(deque)); size_t blocks; @@ -1563,9 +1686,6 @@ deque_sizeof(dequeobject *deque, void *unused) return PyLong_FromSize_t(res); } -PyDoc_STRVAR(sizeof_doc, -"D.__sizeof__() -- size of D in memory, in bytes"); - static PyObject * deque_get_maxlen(dequeobject *deque, void *Py_UNUSED(ignored)) { @@ -1574,6 +1694,22 @@ deque_get_maxlen(dequeobject *deque, void *Py_UNUSED(ignored)) return PyLong_FromSsize_t(deque->maxlen); } +static PyObject *deque_reviter(dequeobject *deque); + +/*[clinic input] +_collections.deque.__reversed__ as deque___reversed__ + + deque: dequeobject + +Return a reverse iterator over the deque. +[clinic start generated code]*/ + +static PyObject * +deque___reversed___impl(dequeobject *deque) +/*[clinic end generated code: output=3e7e7e715883cf2e input=3d494c25a6fe5c7e]*/ +{ + return deque_reviter(deque); +} /* deque object ********************************************************/ @@ -1584,47 +1720,26 @@ static PyGetSetDef deque_getset[] = { }; static PyObject *deque_iter(dequeobject *deque); -static PyObject *deque_reviter(dequeobject *deque, PyObject *Py_UNUSED(ignored)); -PyDoc_STRVAR(reversed_doc, - "D.__reversed__() -- return a reverse iterator over the deque"); static PyMethodDef deque_methods[] = { - {"append", (PyCFunction)deque_append, - METH_O, append_doc}, - {"appendleft", (PyCFunction)deque_appendleft, - METH_O, appendleft_doc}, - {"clear", (PyCFunction)deque_clearmethod, - METH_NOARGS, clear_doc}, - {"__copy__", deque_copy, - METH_NOARGS, copy_doc}, - {"copy", deque_copy, - METH_NOARGS, copy_doc}, - {"count", (PyCFunction)deque_count, - METH_O, count_doc}, - {"extend", (PyCFunction)deque_extend, - METH_O, extend_doc}, - {"extendleft", (PyCFunction)deque_extendleft, - METH_O, extendleft_doc}, - {"index", _PyCFunction_CAST(deque_index), - METH_FASTCALL, index_doc}, - {"insert", _PyCFunction_CAST(deque_insert), - METH_FASTCALL, insert_doc}, - {"pop", (PyCFunction)deque_pop, - METH_NOARGS, pop_doc}, - {"popleft", (PyCFunction)deque_popleft, - METH_NOARGS, popleft_doc}, - {"__reduce__", (PyCFunction)deque_reduce, - METH_NOARGS, reduce_doc}, - {"remove", (PyCFunction)deque_remove, - METH_O, remove_doc}, - {"__reversed__", (PyCFunction)deque_reviter, - METH_NOARGS, reversed_doc}, - {"reverse", (PyCFunction)deque_reverse, - METH_NOARGS, reverse_doc}, - {"rotate", _PyCFunction_CAST(deque_rotate), - METH_FASTCALL, rotate_doc}, - {"__sizeof__", (PyCFunction)deque_sizeof, - METH_NOARGS, sizeof_doc}, + DEQUE_APPEND_METHODDEF + DEQUE_APPENDLEFT_METHODDEF + DEQUE_CLEARMETHOD_METHODDEF + DEQUE___COPY___METHODDEF + DEQUE_COPY_METHODDEF + DEQUE_COUNT_METHODDEF + DEQUE_EXTEND_METHODDEF + DEQUE_EXTENDLEFT_METHODDEF + DEQUE_INDEX_METHODDEF + DEQUE_INSERT_METHODDEF + DEQUE_POP_METHODDEF + DEQUE_POPLEFT_METHODDEF + DEQUE___REDUCE___METHODDEF + DEQUE_REMOVE_METHODDEF + DEQUE___REVERSED___METHODDEF + DEQUE_REVERSE_METHODDEF + DEQUE_ROTATE_METHODDEF + DEQUE___SIZEOF___METHODDEF {"__class_getitem__", Py_GenericAlias, METH_O|METH_CLASS, PyDoc_STR("See PEP 585")}, {NULL, NULL} /* sentinel */ @@ -1635,17 +1750,12 @@ static PyMemberDef deque_members[] = { {NULL}, }; -PyDoc_STRVAR(deque_doc, -"deque([iterable[, maxlen]]) --> deque object\n\ -\n\ -A list-like sequence optimized for data accesses near its endpoints."); - static PyType_Slot deque_slots[] = { {Py_tp_dealloc, deque_dealloc}, {Py_tp_repr, deque_repr}, {Py_tp_hash, PyObject_HashNotImplemented}, {Py_tp_getattro, PyObject_GenericGetAttr}, - {Py_tp_doc, (void *)deque_doc}, + {Py_tp_doc, (void *)deque_init__doc__}, {Py_tp_traverse, deque_traverse}, {Py_tp_clear, deque_clear}, {Py_tp_richcompare, deque_richcompare}, @@ -1834,7 +1944,7 @@ static PyType_Spec dequeiter_spec = { /*********************** Deque Reverse Iterator **************************/ static PyObject * -deque_reviter(dequeobject *deque, PyObject *Py_UNUSED(ignored)) +deque_reviter(dequeobject *deque) { dequeiterobject *it; collections_state *state = find_module_state_by_def(Py_TYPE(deque)); @@ -1889,7 +1999,7 @@ dequereviter_new(PyTypeObject *type, PyObject *args, PyObject *kwds) return NULL; assert(type == state->dequereviter_type); - it = (dequeiterobject*)deque_reviter((dequeobject *)deque, NULL); + it = (dequeiterobject *)deque_reviter((dequeobject *)deque); if (!it) return NULL; /* consume items from the queue */ diff --git a/Modules/_csv.c b/Modules/_csv.c index ae6b6457ffad9a..3aa648b8e9cec4 100644 --- a/Modules/_csv.c +++ b/Modules/_csv.c @@ -8,8 +8,6 @@ module instead. */ -#define MODULE_VERSION "1.0" - // clinic/_csv.c.h uses internal pycore_modsupport.h API #ifndef Py_BUILD_CORE_BUILTIN # define Py_BUILD_CORE_MODULE 1 @@ -133,7 +131,7 @@ typedef struct { Py_UCS4 *field; /* temporary buffer */ Py_ssize_t field_size; /* size of allocated buffer */ Py_ssize_t field_len; /* length of current field */ - int numeric_field; /* treat field as numeric */ + bool unquoted_field; /* true if no quotes around the current field */ unsigned long line_num; /* Source-file line number */ } ReaderObj; @@ -333,6 +331,33 @@ dialect_check_quoting(int quoting) return -1; } +static int +dialect_check_char(const char *name, Py_UCS4 c, DialectObj *dialect) +{ + if (c == '\r' || c == '\n' || (dialect->skipinitialspace && c == ' ')) { + PyErr_Format(PyExc_ValueError, "bad %s value", name); + return -1; + } + if (PyUnicode_FindChar( + dialect->lineterminator, c, 0, + PyUnicode_GET_LENGTH(dialect->lineterminator), 1) >= 0) + { + PyErr_Format(PyExc_ValueError, "bad %s or lineterminator value", name); + return -1; + } + return 0; +} + + static int +dialect_check_chars(const char *name1, const char *name2, Py_UCS4 c1, Py_UCS4 c2) +{ + if (c1 == c2 && c1 != NOT_SET) { + PyErr_Format(PyExc_ValueError, "bad %s or %s value", name1, name2); + return -1; + } + return 0; +} + #define D_OFF(x) offsetof(DialectObj, x) static struct PyMemberDef Dialect_memberlist[] = { @@ -510,6 +535,18 @@ dialect_new(PyTypeObject *type, PyObject *args, PyObject *kwargs) PyErr_SetString(PyExc_TypeError, "lineterminator must be set"); goto err; } + if (dialect_check_char("delimiter", self->delimiter, self) || + dialect_check_char("escapechar", self->escapechar, self) || + dialect_check_char("quotechar", self->quotechar, self) || + dialect_check_chars("delimiter", "escapechar", + self->delimiter, self->escapechar) || + dialect_check_chars("delimiter", "quotechar", + self->delimiter, self->quotechar) || + dialect_check_chars("escapechar", "quotechar", + self->escapechar, self->quotechar)) + { + goto err; + } ret = Py_NewRef(self); err: @@ -607,22 +644,33 @@ _call_dialect(_csvstate *module_state, PyObject *dialect_inst, PyObject *kwargs) static int parse_save_field(ReaderObj *self) { + int quoting = self->dialect->quoting; PyObject *field; - field = PyUnicode_FromKindAndData(PyUnicode_4BYTE_KIND, - (void *) self->field, self->field_len); - if (field == NULL) - return -1; - self->field_len = 0; - if (self->numeric_field) { - PyObject *tmp; - - self->numeric_field = 0; - tmp = PyNumber_Float(field); - Py_DECREF(field); - if (tmp == NULL) + if (self->unquoted_field && + self->field_len == 0 && + (quoting == QUOTE_NOTNULL || quoting == QUOTE_STRINGS)) + { + field = Py_NewRef(Py_None); + } + else { + field = PyUnicode_FromKindAndData(PyUnicode_4BYTE_KIND, + (void *) self->field, self->field_len); + if (field == NULL) { return -1; - field = tmp; + } + if (self->unquoted_field && + self->field_len != 0 && + (quoting == QUOTE_NONNUMERIC || quoting == QUOTE_STRINGS)) + { + PyObject *tmp = PyNumber_Float(field); + Py_DECREF(field); + if (tmp == NULL) { + return -1; + } + field = tmp; + } + self->field_len = 0; } if (PyList_Append(self->fields, field) < 0) { Py_DECREF(field); @@ -684,6 +732,7 @@ parse_process_char(ReaderObj *self, _csvstate *module_state, Py_UCS4 c) /* fallthru */ case START_FIELD: /* expecting field */ + self->unquoted_field = true; if (c == '\n' || c == '\r' || c == EOL) { /* save empty field - return [fields] */ if (parse_save_field(self) < 0) @@ -693,10 +742,12 @@ parse_process_char(ReaderObj *self, _csvstate *module_state, Py_UCS4 c) else if (c == dialect->quotechar && dialect->quoting != QUOTE_NONE) { /* start quoted field */ + self->unquoted_field = false; self->state = IN_QUOTED_FIELD; } else if (c == dialect->escapechar) { /* possible escaped character */ + self->unquoted_field = false; self->state = ESCAPED_CHAR; } else if (c == ' ' && dialect->skipinitialspace) @@ -709,8 +760,6 @@ parse_process_char(ReaderObj *self, _csvstate *module_state, Py_UCS4 c) } else { /* begin new unquoted field */ - if (dialect->quoting == QUOTE_NONNUMERIC) - self->numeric_field = 1; if (parse_add_char(self, module_state, c) < 0) return -1; self->state = IN_FIELD; @@ -837,7 +886,8 @@ parse_process_char(ReaderObj *self, _csvstate *module_state, Py_UCS4 c) self->state = START_RECORD; else { PyErr_Format(module_state->error_obj, - "new-line character seen in unquoted field - do you need to open the file in universal-newline mode?"); + "new-line character seen in unquoted field - " + "do you need to open the file with newline=''?"); return -1; } break; @@ -854,7 +904,7 @@ parse_reset(ReaderObj *self) return -1; self->field_len = 0; self->state = START_RECORD; - self->numeric_field = 0; + self->unquoted_field = false; return 0; } @@ -1606,68 +1656,7 @@ PyType_Spec error_spec = { * MODULE */ -PyDoc_STRVAR(csv_module_doc, -"CSV parsing and writing.\n" -"\n" -"This module provides classes that assist in the reading and writing\n" -"of Comma Separated Value (CSV) files, and implements the interface\n" -"described by PEP 305. Although many CSV files are simple to parse,\n" -"the format is not formally defined by a stable specification and\n" -"is subtle enough that parsing lines of a CSV file with something\n" -"like line.split(\",\") is bound to fail. The module supports three\n" -"basic APIs: reading, writing, and registration of dialects.\n" -"\n" -"\n" -"DIALECT REGISTRATION:\n" -"\n" -"Readers and writers support a dialect argument, which is a convenient\n" -"handle on a group of settings. When the dialect argument is a string,\n" -"it identifies one of the dialects previously registered with the module.\n" -"If it is a class or instance, the attributes of the argument are used as\n" -"the settings for the reader or writer:\n" -"\n" -" class excel:\n" -" delimiter = ','\n" -" quotechar = '\"'\n" -" escapechar = None\n" -" doublequote = True\n" -" skipinitialspace = False\n" -" lineterminator = '\\r\\n'\n" -" quoting = QUOTE_MINIMAL\n" -"\n" -"SETTINGS:\n" -"\n" -" * quotechar - specifies a one-character string to use as the\n" -" quoting character. It defaults to '\"'.\n" -" * delimiter - specifies a one-character string to use as the\n" -" field separator. It defaults to ','.\n" -" * skipinitialspace - specifies how to interpret spaces which\n" -" immediately follow a delimiter. It defaults to False, which\n" -" means that spaces immediately following a delimiter is part\n" -" of the following field.\n" -" * lineterminator - specifies the character sequence which should\n" -" terminate rows.\n" -" * quoting - controls when quotes should be generated by the writer.\n" -" It can take on any of the following module constants:\n" -"\n" -" csv.QUOTE_MINIMAL means only when required, for example, when a\n" -" field contains either the quotechar or the delimiter\n" -" csv.QUOTE_ALL means that quotes are always placed around fields.\n" -" csv.QUOTE_NONNUMERIC means that quotes are always placed around\n" -" fields which do not parse as integers or floating point\n" -" numbers.\n" -" csv.QUOTE_STRINGS means that quotes are always placed around\n" -" fields which are strings. Note that the Python value None\n" -" is not a string.\n" -" csv.QUOTE_NOTNULL means that quotes are only placed around fields\n" -" that are not the Python value None.\n" -" csv.QUOTE_NONE means that quotes are never placed around fields.\n" -" * escapechar - specifies a one-character string used to escape\n" -" the delimiter when quoting is set to QUOTE_NONE.\n" -" * doublequote - controls the handling of quotes inside fields. When\n" -" True, two consecutive quotes are interpreted as one during read,\n" -" and when writing, each quote character embedded in the data is\n" -" written as two quotes\n"); +PyDoc_STRVAR(csv_module_doc, "CSV parsing and writing.\n"); PyDoc_STRVAR(csv_reader_doc, " csv_reader = reader(iterable [, dialect='excel']\n" @@ -1740,12 +1729,6 @@ csv_exec(PyObject *module) { return -1; } - /* Add version to the module. */ - if (PyModule_AddStringConstant(module, "__version__", - MODULE_VERSION) == -1) { - return -1; - } - /* Set the field limit */ module_state->field_limit = 128 * 1024; diff --git a/Modules/_ctypes/_ctypes.c b/Modules/_ctypes/_ctypes.c index f909a9496b6526..94245ae41afffc 100644 --- a/Modules/_ctypes/_ctypes.c +++ b/Modules/_ctypes/_ctypes.c @@ -128,15 +128,26 @@ bytes(cdata) #include "pycore_long.h" // _PyLong_GetZero() -ctypes_state global_state; +static PyTypeObject Union_Type; +static PyTypeObject Struct_Type; +static PyTypeObject Simple_Type; + +ctypes_state global_state = { + .PyCStgDict_Type = &PyCStgDict_Type, + .PyCData_Type = &PyCData_Type, + .Struct_Type = &Struct_Type, + .Union_Type = &Union_Type, + .PyCArray_Type = &PyCArray_Type, + .Simple_Type = &Simple_Type, + .PyCPointer_Type = &PyCPointer_Type, + .PyCFuncPtr_Type = &PyCFuncPtr_Type, +}; PyObject *PyExc_ArgError = NULL; /* This dict maps ctypes types to POINTER types */ PyObject *_ctypes_ptrtype_cache = NULL; -static PyTypeObject Simple_Type; - /* a callable object used for unpickling: strong reference to _ctypes._unpickle() function */ static PyObject *_unpickle; @@ -521,14 +532,16 @@ StructUnionType_new(PyTypeObject *type, PyObject *args, PyObject *kwds, int isSt /* keep this for bw compatibility */ int r = PyDict_Contains(result->tp_dict, &_Py_ID(_abstract_)); - if (r > 0) + if (r > 0) { return (PyObject *)result; + } if (r < 0) { Py_DECREF(result); return NULL; } - dict = (StgDictObject *)_PyObject_CallNoArgs((PyObject *)&PyCStgDict_Type); + ctypes_state *st = GLOBAL_STATE(); + dict = (StgDictObject *)_PyObject_CallNoArgs((PyObject *)st->PyCStgDict_Type); if (!dict) { Py_DECREF(result); return NULL; @@ -568,8 +581,9 @@ StructUnionType_new(PyTypeObject *type, PyObject *args, PyObject *kwds, int isSt else { StgDictObject *basedict = PyType_stgdict((PyObject *)result->tp_base); - if (basedict == NULL) + if (basedict == NULL) { return (PyObject *)result; + } /* copy base dict */ if (-1 == PyCStgDict_clone(dict, basedict)) { Py_DECREF(result); @@ -876,20 +890,6 @@ CDataType_repeat(PyObject *self, Py_ssize_t length) return PyCArrayType_from_ctype(self, length); } -static PySequenceMethods CDataType_as_sequence = { - 0, /* inquiry sq_length; */ - 0, /* binaryfunc sq_concat; */ - CDataType_repeat, /* intargfunc sq_repeat; */ - 0, /* intargfunc sq_item; */ - 0, /* intintargfunc sq_slice; */ - 0, /* intobjargproc sq_ass_item; */ - 0, /* intintobjargproc sq_ass_slice; */ - 0, /* objobjproc sq_contains; */ - - 0, /* binaryfunc sq_inplace_concat; */ - 0, /* intargfunc sq_inplace_repeat; */ -}; - static int CDataType_clear(PyTypeObject *self) { @@ -903,8 +903,10 @@ static int CDataType_traverse(PyTypeObject *self, visitproc visit, void *arg) { StgDictObject *dict = PyType_stgdict((PyObject *)self); - if (dict) + if (dict) { Py_VISIT(dict->proto); + } + Py_VISIT(Py_TYPE(self)); return PyType_Type.tp_traverse((PyObject *)self, visit, arg); } @@ -935,91 +937,45 @@ UnionType_setattro(PyObject *self, PyObject *key, PyObject *value) return 0; } +static PyType_Slot pycstruct_type_slots[] = { + {Py_tp_setattro, PyCStructType_setattro}, + {Py_tp_doc, PyDoc_STR("metatype for the CData Objects")}, + {Py_tp_traverse, CDataType_traverse}, + {Py_tp_clear, CDataType_clear}, + {Py_tp_methods, CDataType_methods}, + {Py_tp_new, PyCStructType_new}, -PyTypeObject PyCStructType_Type = { - PyVarObject_HEAD_INIT(NULL, 0) - "_ctypes.PyCStructType", /* tp_name */ - 0, /* tp_basicsize */ - 0, /* tp_itemsize */ - 0, /* tp_dealloc */ - 0, /* tp_vectorcall_offset */ - 0, /* tp_getattr */ - 0, /* tp_setattr */ - 0, /* tp_as_async */ - 0, /* tp_repr */ - 0, /* tp_as_number */ - &CDataType_as_sequence, /* tp_as_sequence */ - 0, /* tp_as_mapping */ - 0, /* tp_hash */ - 0, /* tp_call */ - 0, /* tp_str */ - 0, /* tp_getattro */ - PyCStructType_setattro, /* tp_setattro */ - 0, /* tp_as_buffer */ - Py_TPFLAGS_DEFAULT | Py_TPFLAGS_BASETYPE | Py_TPFLAGS_HAVE_GC, /* tp_flags */ - PyDoc_STR("metatype for the CData Objects"), /* tp_doc */ - (traverseproc)CDataType_traverse, /* tp_traverse */ - (inquiry)CDataType_clear, /* tp_clear */ - 0, /* tp_richcompare */ - 0, /* tp_weaklistoffset */ - 0, /* tp_iter */ - 0, /* tp_iternext */ - CDataType_methods, /* tp_methods */ - 0, /* tp_members */ - 0, /* tp_getset */ - 0, /* tp_base */ - 0, /* tp_dict */ - 0, /* tp_descr_get */ - 0, /* tp_descr_set */ - 0, /* tp_dictoffset */ - 0, /* tp_init */ - 0, /* tp_alloc */ - PyCStructType_new, /* tp_new */ - 0, /* tp_free */ + // Sequence protocol. + {Py_sq_repeat, CDataType_repeat}, + {0, NULL}, }; -static PyTypeObject UnionType_Type = { - PyVarObject_HEAD_INIT(NULL, 0) - "_ctypes.UnionType", /* tp_name */ - 0, /* tp_basicsize */ - 0, /* tp_itemsize */ - 0, /* tp_dealloc */ - 0, /* tp_vectorcall_offset */ - 0, /* tp_getattr */ - 0, /* tp_setattr */ - 0, /* tp_as_async */ - 0, /* tp_repr */ - 0, /* tp_as_number */ - &CDataType_as_sequence, /* tp_as_sequence */ - 0, /* tp_as_mapping */ - 0, /* tp_hash */ - 0, /* tp_call */ - 0, /* tp_str */ - 0, /* tp_getattro */ - UnionType_setattro, /* tp_setattro */ - 0, /* tp_as_buffer */ - Py_TPFLAGS_DEFAULT | Py_TPFLAGS_BASETYPE | Py_TPFLAGS_HAVE_GC, /* tp_flags */ - PyDoc_STR("metatype for the CData Objects"), /* tp_doc */ - (traverseproc)CDataType_traverse, /* tp_traverse */ - (inquiry)CDataType_clear, /* tp_clear */ - 0, /* tp_richcompare */ - 0, /* tp_weaklistoffset */ - 0, /* tp_iter */ - 0, /* tp_iternext */ - CDataType_methods, /* tp_methods */ - 0, /* tp_members */ - 0, /* tp_getset */ - 0, /* tp_base */ - 0, /* tp_dict */ - 0, /* tp_descr_get */ - 0, /* tp_descr_set */ - 0, /* tp_dictoffset */ - 0, /* tp_init */ - 0, /* tp_alloc */ - UnionType_new, /* tp_new */ - 0, /* tp_free */ +PyType_Spec pycstruct_type_spec = { + .name = "_ctypes.PyCStructType", + .flags = (Py_TPFLAGS_DEFAULT | Py_TPFLAGS_BASETYPE | Py_TPFLAGS_HAVE_GC | + Py_TPFLAGS_IMMUTABLETYPE), + .slots = pycstruct_type_slots, }; +static PyType_Slot union_type_slots[] = { + {Py_tp_setattro, UnionType_setattro}, + {Py_tp_doc, PyDoc_STR("metatype for the Union Objects")}, + {Py_tp_traverse, CDataType_traverse}, + {Py_tp_clear, CDataType_clear}, + {Py_tp_methods, CDataType_methods}, + {Py_tp_new, UnionType_new}, + + // Sequence protocol. + {Py_sq_repeat, CDataType_repeat}, + {0, NULL}, +}; + +static PyType_Spec union_type_spec = { + .name = "_ctypes.UnionType", + .flags = (Py_TPFLAGS_DEFAULT | Py_TPFLAGS_BASETYPE | Py_TPFLAGS_HAVE_GC | + Py_TPFLAGS_IMMUTABLETYPE), + .slots = union_type_slots, +}; /******************************************************************/ @@ -1081,16 +1037,19 @@ PyCPointerType_new(PyTypeObject *type, PyObject *args, PyObject *kwds) typedict = PyTuple_GetItem(args, 2); - if (!typedict) + if (!typedict) { return NULL; + } /* stgdict items size, align, length contain info about pointers itself, stgdict->proto has info about the pointed to type! */ + ctypes_state *st = GLOBAL_STATE(); stgdict = (StgDictObject *)_PyObject_CallNoArgs( - (PyObject *)&PyCStgDict_Type); - if (!stgdict) + (PyObject *)st->PyCStgDict_Type); + if (!stgdict) { return NULL; + } stgdict->size = sizeof(void *); stgdict->align = _ctypes_get_fielddesc("P")->pffi_type->alignment; stgdict->length = 1; @@ -1207,7 +1166,8 @@ PyCPointerType_from_param(PyObject *type, PyObject *value) break; } - if (PointerObject_Check(value) || ArrayObject_Check(value)) { + ctypes_state *st = GLOBAL_STATE(); + if (PointerObject_Check(st, value) || ArrayObject_Check(st, value)) { /* Array instances are also pointers when the item types are the same. */ @@ -1234,46 +1194,23 @@ static PyMethodDef PyCPointerType_methods[] = { { NULL, NULL }, }; -PyTypeObject PyCPointerType_Type = { - PyVarObject_HEAD_INIT(NULL, 0) - "_ctypes.PyCPointerType", /* tp_name */ - 0, /* tp_basicsize */ - 0, /* tp_itemsize */ - 0, /* tp_dealloc */ - 0, /* tp_vectorcall_offset */ - 0, /* tp_getattr */ - 0, /* tp_setattr */ - 0, /* tp_as_async */ - 0, /* tp_repr */ - 0, /* tp_as_number */ - &CDataType_as_sequence, /* tp_as_sequence */ - 0, /* tp_as_mapping */ - 0, /* tp_hash */ - 0, /* tp_call */ - 0, /* tp_str */ - 0, /* tp_getattro */ - 0, /* tp_setattro */ - 0, /* tp_as_buffer */ - Py_TPFLAGS_DEFAULT | Py_TPFLAGS_BASETYPE | Py_TPFLAGS_HAVE_GC, /* tp_flags */ - PyDoc_STR("metatype for the Pointer Objects"), /* tp_doc */ - (traverseproc)CDataType_traverse, /* tp_traverse */ - (inquiry)CDataType_clear, /* tp_clear */ - 0, /* tp_richcompare */ - 0, /* tp_weaklistoffset */ - 0, /* tp_iter */ - 0, /* tp_iternext */ - PyCPointerType_methods, /* tp_methods */ - 0, /* tp_members */ - 0, /* tp_getset */ - 0, /* tp_base */ - 0, /* tp_dict */ - 0, /* tp_descr_get */ - 0, /* tp_descr_set */ - 0, /* tp_dictoffset */ - 0, /* tp_init */ - 0, /* tp_alloc */ - PyCPointerType_new, /* tp_new */ - 0, /* tp_free */ +static PyType_Slot pycpointer_type_slots[] = { + {Py_tp_doc, PyDoc_STR("metatype for the Pointer Objects")}, + {Py_tp_traverse, CDataType_traverse}, + {Py_tp_clear, CDataType_clear}, + {Py_tp_methods, PyCPointerType_methods}, + {Py_tp_new, PyCPointerType_new}, + + // Sequence protocol. + {Py_sq_repeat, CDataType_repeat}, + {0, NULL}, +}; + +static PyType_Spec pycpointer_type_spec = { + .name = "_ctypes.PyCPointerType", + .flags = (Py_TPFLAGS_DEFAULT | Py_TPFLAGS_BASETYPE | Py_TPFLAGS_HAVE_GC | + Py_TPFLAGS_IMMUTABLETYPE), + .slots = pycpointer_type_slots, }; @@ -1529,11 +1466,12 @@ PyCArrayType_new(PyTypeObject *type, PyObject *args, PyObject *kwds) goto error; } + ctypes_state *st = GLOBAL_STATE(); stgdict = (StgDictObject *)_PyObject_CallNoArgs( - (PyObject *)&PyCStgDict_Type); - if (!stgdict) + (PyObject *)st->PyCStgDict_Type); + if (!stgdict) { goto error; - + } itemdict = PyType_stgdict(type_attr); if (!itemdict) { PyErr_SetString(PyExc_TypeError, @@ -1606,48 +1544,24 @@ PyCArrayType_new(PyTypeObject *type, PyObject *args, PyObject *kwds) return NULL; } -PyTypeObject PyCArrayType_Type = { - PyVarObject_HEAD_INIT(NULL, 0) - "_ctypes.PyCArrayType", /* tp_name */ - 0, /* tp_basicsize */ - 0, /* tp_itemsize */ - 0, /* tp_dealloc */ - 0, /* tp_vectorcall_offset */ - 0, /* tp_getattr */ - 0, /* tp_setattr */ - 0, /* tp_as_async */ - 0, /* tp_repr */ - 0, /* tp_as_number */ - &CDataType_as_sequence, /* tp_as_sequence */ - 0, /* tp_as_mapping */ - 0, /* tp_hash */ - 0, /* tp_call */ - 0, /* tp_str */ - 0, /* tp_getattro */ - 0, /* tp_setattro */ - 0, /* tp_as_buffer */ - Py_TPFLAGS_DEFAULT | Py_TPFLAGS_BASETYPE, /* tp_flags */ - PyDoc_STR("metatype for the Array Objects"), /* tp_doc */ - 0, /* tp_traverse */ - 0, /* tp_clear */ - 0, /* tp_richcompare */ - 0, /* tp_weaklistoffset */ - 0, /* tp_iter */ - 0, /* tp_iternext */ - CDataType_methods, /* tp_methods */ - 0, /* tp_members */ - 0, /* tp_getset */ - 0, /* tp_base */ - 0, /* tp_dict */ - 0, /* tp_descr_get */ - 0, /* tp_descr_set */ - 0, /* tp_dictoffset */ - 0, /* tp_init */ - 0, /* tp_alloc */ - PyCArrayType_new, /* tp_new */ - 0, /* tp_free */ +static PyType_Slot pycarray_type_slots[] = { + {Py_tp_doc, PyDoc_STR("metatype for the Array Objects")}, + {Py_tp_traverse, CDataType_traverse}, + {Py_tp_methods, CDataType_methods}, + {Py_tp_new, PyCArrayType_new}, + {Py_tp_clear, CDataType_clear}, + + // Sequence protocol. + {Py_sq_repeat, CDataType_repeat}, + {0, NULL}, }; +static PyType_Spec pycarray_type_spec = { + .name = "_ctypes.PyCArrayType", + .flags = (Py_TPFLAGS_DEFAULT | Py_TPFLAGS_BASETYPE | Py_TPFLAGS_HAVE_GC | + Py_TPFLAGS_IMMUTABLETYPE), + .slots = pycarray_type_slots, +}; /******************************************************************/ /* @@ -1692,7 +1606,8 @@ c_wchar_p_from_param(PyObject *type, PyObject *value) if (res) { return Py_NewRef(value); } - if (ArrayObject_Check(value) || PointerObject_Check(value)) { + ctypes_state *st = GLOBAL_STATE(); + if (ArrayObject_Check(st, value) || PointerObject_Check(st, value)) { /* c_wchar array instance or pointer(c_wchar(...)) */ StgDictObject *dt = PyObject_stgdict(value); StgDictObject *dict; @@ -1702,7 +1617,6 @@ c_wchar_p_from_param(PyObject *type, PyObject *value) return Py_NewRef(value); } } - ctypes_state *st = GLOBAL_STATE(); if (PyCArg_CheckExact(st, value)) { /* byref(c_char(...)) */ PyCArgObject *a = (PyCArgObject *)value; @@ -1756,7 +1670,8 @@ c_char_p_from_param(PyObject *type, PyObject *value) if (res) { return Py_NewRef(value); } - if (ArrayObject_Check(value) || PointerObject_Check(value)) { + ctypes_state *st = GLOBAL_STATE(); + if (ArrayObject_Check(st, value) || PointerObject_Check(st, value)) { /* c_char array instance or pointer(c_char(...)) */ StgDictObject *dt = PyObject_stgdict(value); StgDictObject *dict; @@ -1766,7 +1681,6 @@ c_char_p_from_param(PyObject *type, PyObject *value) return Py_NewRef(value); } } - ctypes_state *st = GLOBAL_STATE(); if (PyCArg_CheckExact(st, value)) { /* byref(c_char(...)) */ PyCArgObject *a = (PyCArgObject *)value; @@ -1862,13 +1776,13 @@ c_void_p_from_param(PyObject *type, PyObject *value) /* c_void_p instances */ return Py_NewRef(value); } + ctypes_state *st = GLOBAL_STATE(); /* ctypes array or pointer instance */ - if (ArrayObject_Check(value) || PointerObject_Check(value)) { + if (ArrayObject_Check(st, value) || PointerObject_Check(st, value)) { /* Any array or pointer is accepted */ return Py_NewRef(value); } /* byref(...) */ - ctypes_state *st = GLOBAL_STATE(); if (PyCArg_CheckExact(st, value)) { /* byref(c_xxx()) */ PyCArgObject *a = (PyCArgObject *)value; @@ -1877,7 +1791,7 @@ c_void_p_from_param(PyObject *type, PyObject *value) } } /* function pointer */ - if (PyCFuncPtrObject_Check(value)) { + if (PyCFuncPtrObject_Check(st, value)) { PyCArgObject *parg; PyCFuncPtrObject *func; func = (PyCFuncPtrObject *)value; @@ -1893,7 +1807,11 @@ c_void_p_from_param(PyObject *type, PyObject *value) } /* c_char_p, c_wchar_p */ stgd = PyObject_stgdict(value); - if (stgd && CDataObject_Check(value) && stgd->proto && PyUnicode_Check(stgd->proto)) { + if (stgd + && CDataObject_Check(st, value) + && stgd->proto + && PyUnicode_Check(stgd->proto)) + { PyCArgObject *parg; switch (PyUnicode_AsUTF8(stgd->proto)[0]) { @@ -1975,8 +1893,9 @@ static PyObject *CreateSwappedType(PyTypeObject *type, PyObject *args, PyObject if (result == NULL) return NULL; + ctypes_state *st = GLOBAL_STATE(); stgdict = (StgDictObject *)_PyObject_CallNoArgs( - (PyObject *)&PyCStgDict_Type); + (PyObject *)st->PyCStgDict_Type); if (!stgdict) { Py_DECREF(result); return NULL; @@ -2086,11 +2005,12 @@ PyCSimpleType_new(PyTypeObject *type, PyObject *args, PyObject *kwds) goto error; } + ctypes_state *st = GLOBAL_STATE(); stgdict = (StgDictObject *)_PyObject_CallNoArgs( - (PyObject *)&PyCStgDict_Type); - if (!stgdict) + (PyObject *)st->PyCStgDict_Type); + if (!stgdict) { goto error; - + } stgdict->ffi_type_pointer = *fmt->pffi_type; stgdict->align = fmt->pffi_type->alignment; stgdict->length = 0; @@ -2111,7 +2031,7 @@ PyCSimpleType_new(PyTypeObject *type, PyObject *args, PyObject *kwds) stgdict->paramfunc = PyCSimpleType_paramfunc; /* - if (result->tp_base != &Simple_Type) { + if (result->tp_base != st->Simple_Type) { stgdict->setfunc = NULL; stgdict->getfunc = NULL; } @@ -2131,7 +2051,7 @@ PyCSimpleType_new(PyTypeObject *type, PyObject *args, PyObject *kwds) /* Install from_param class methods in ctypes base classes. Overrides the PyCSimpleType_from_param generic method. */ - if (result->tp_base == &Simple_Type) { + if (result->tp_base == st->Simple_Type) { switch (*proto_str) { case 'z': /* c_char_p */ ml = &c_char_p_method; @@ -2175,7 +2095,10 @@ PyCSimpleType_new(PyTypeObject *type, PyObject *args, PyObject *kwds) } } - if (type == &PyCSimpleType_Type && fmt->setfunc_swapped && fmt->getfunc_swapped) { + if (type == st->PyCSimpleType_Type + && fmt->setfunc_swapped + && fmt->getfunc_swapped) + { PyObject *swapped = CreateSwappedType(type, args, kwds, proto, fmt); StgDictObject *sw_dict; @@ -2292,46 +2215,23 @@ static PyMethodDef PyCSimpleType_methods[] = { { NULL, NULL }, }; -PyTypeObject PyCSimpleType_Type = { - PyVarObject_HEAD_INIT(NULL, 0) - "_ctypes.PyCSimpleType", /* tp_name */ - 0, /* tp_basicsize */ - 0, /* tp_itemsize */ - 0, /* tp_dealloc */ - 0, /* tp_vectorcall_offset */ - 0, /* tp_getattr */ - 0, /* tp_setattr */ - 0, /* tp_as_async */ - 0, /* tp_repr */ - 0, /* tp_as_number */ - &CDataType_as_sequence, /* tp_as_sequence */ - 0, /* tp_as_mapping */ - 0, /* tp_hash */ - 0, /* tp_call */ - 0, /* tp_str */ - 0, /* tp_getattro */ - 0, /* tp_setattro */ - 0, /* tp_as_buffer */ - Py_TPFLAGS_DEFAULT | Py_TPFLAGS_BASETYPE, /* tp_flags */ - PyDoc_STR("metatype for the PyCSimpleType Objects"), /* tp_doc */ - 0, /* tp_traverse */ - 0, /* tp_clear */ - 0, /* tp_richcompare */ - 0, /* tp_weaklistoffset */ - 0, /* tp_iter */ - 0, /* tp_iternext */ - PyCSimpleType_methods, /* tp_methods */ - 0, /* tp_members */ - 0, /* tp_getset */ - 0, /* tp_base */ - 0, /* tp_dict */ - 0, /* tp_descr_get */ - 0, /* tp_descr_set */ - 0, /* tp_dictoffset */ - 0, /* tp_init */ - 0, /* tp_alloc */ - PyCSimpleType_new, /* tp_new */ - 0, /* tp_free */ +static PyType_Slot pycsimple_type_slots[] = { + {Py_tp_doc, PyDoc_STR("metatype for the PyCSimpleType Objects")}, + {Py_tp_methods, PyCSimpleType_methods}, + {Py_tp_new, PyCSimpleType_new}, + {Py_tp_traverse, CDataType_traverse}, + {Py_tp_clear, CDataType_clear}, + + // Sequence protocol. + {Py_sq_repeat, CDataType_repeat}, + {0, NULL}, +}; + +PyType_Spec pycsimple_type_spec = { + .name = "_ctypes.PyCSimpleType", + .flags = (Py_TPFLAGS_DEFAULT | Py_TPFLAGS_BASETYPE | Py_TPFLAGS_HAVE_GC | + Py_TPFLAGS_IMMUTABLETYPE), + .slots = pycsimple_type_slots, }; /******************************************************************/ @@ -2532,11 +2432,12 @@ PyCFuncPtrType_new(PyTypeObject *type, PyObject *args, PyObject *kwds) PyTypeObject *result; StgDictObject *stgdict; + ctypes_state *st = GLOBAL_STATE(); stgdict = (StgDictObject *)_PyObject_CallNoArgs( - (PyObject *)&PyCStgDict_Type); - if (!stgdict) + (PyObject *)st->PyCStgDict_Type); + if (!stgdict) { return NULL; - + } stgdict->paramfunc = PyCFuncPtrType_paramfunc; /* We do NOT expose the function signature in the format string. It is impossible, generally, because the only requirement for the @@ -2575,46 +2476,23 @@ PyCFuncPtrType_new(PyTypeObject *type, PyObject *args, PyObject *kwds) return (PyObject *)result; } -PyTypeObject PyCFuncPtrType_Type = { - PyVarObject_HEAD_INIT(NULL, 0) - "_ctypes.PyCFuncPtrType", /* tp_name */ - 0, /* tp_basicsize */ - 0, /* tp_itemsize */ - 0, /* tp_dealloc */ - 0, /* tp_vectorcall_offset */ - 0, /* tp_getattr */ - 0, /* tp_setattr */ - 0, /* tp_as_async */ - 0, /* tp_repr */ - 0, /* tp_as_number */ - &CDataType_as_sequence, /* tp_as_sequence */ - 0, /* tp_as_mapping */ - 0, /* tp_hash */ - 0, /* tp_call */ - 0, /* tp_str */ - 0, /* tp_getattro */ - 0, /* tp_setattro */ - 0, /* tp_as_buffer */ - Py_TPFLAGS_DEFAULT | Py_TPFLAGS_BASETYPE | Py_TPFLAGS_HAVE_GC, /* tp_flags */ - PyDoc_STR("metatype for C function pointers"), /* tp_doc */ - (traverseproc)CDataType_traverse, /* tp_traverse */ - (inquiry)CDataType_clear, /* tp_clear */ - 0, /* tp_richcompare */ - 0, /* tp_weaklistoffset */ - 0, /* tp_iter */ - 0, /* tp_iternext */ - CDataType_methods, /* tp_methods */ - 0, /* tp_members */ - 0, /* tp_getset */ - 0, /* tp_base */ - 0, /* tp_dict */ - 0, /* tp_descr_get */ - 0, /* tp_descr_set */ - 0, /* tp_dictoffset */ - 0, /* tp_init */ - 0, /* tp_alloc */ - PyCFuncPtrType_new, /* tp_new */ - 0, /* tp_free */ +static PyType_Slot pycfuncptr_type_slots[] = { + {Py_tp_doc, PyDoc_STR("metatype for C function pointers")}, + {Py_tp_traverse, CDataType_traverse}, + {Py_tp_clear, CDataType_clear}, + {Py_tp_methods, CDataType_methods}, + {Py_tp_new, PyCFuncPtrType_new}, + + // Sequence protocol. + {Py_sq_repeat, CDataType_repeat}, + {0, NULL}, +}; + +static PyType_Spec pycfuncptr_type_spec = { + .name = "_ctypes.PyCFuncPtrType", + .flags = (Py_TPFLAGS_DEFAULT | Py_TPFLAGS_BASETYPE | Py_TPFLAGS_HAVE_GC | + Py_TPFLAGS_IMMUTABLETYPE), + .slots = pycfuncptr_type_slots, }; @@ -2770,7 +2648,8 @@ static PyMemberDef PyCData_members[] = { static PyObject * PyCData_item_type(PyObject *type) { - if (PyCArrayTypeObject_Check(type)) { + ctypes_state *st = GLOBAL_STATE(); + if (PyCArrayTypeObject_Check(st, type)) { StgDictObject *stg_dict; PyObject *elem_type; @@ -2979,14 +2858,14 @@ PyCData_FromBaseObj(PyObject *type, PyObject *base, Py_ssize_t index, char *adr) } dict->flags |= DICTFLAG_FINAL; cmem = (CDataObject *)((PyTypeObject *)type)->tp_alloc((PyTypeObject *)type, 0); - if (cmem == NULL) + if (cmem == NULL) { return NULL; - assert(CDataObject_Check(cmem)); - + } + assert(CDataObject_Check(GLOBAL_STATE(), cmem)); cmem->b_length = dict->length; cmem->b_size = dict->size; if (base) { /* use base's buffer */ - assert(CDataObject_Check(base)); + assert(CDataObject_Check(GLOBAL_STATE(), base)); cmem->b_ptr = adr; cmem->b_needsfree = 0; cmem->b_base = (CDataObject *)Py_NewRef(base); @@ -3025,9 +2904,10 @@ PyCData_AtAddress(PyObject *type, void *buf) dict->flags |= DICTFLAG_FINAL; pd = (CDataObject *)((PyTypeObject *)type)->tp_alloc((PyTypeObject *)type, 0); - if (!pd) + if (!pd) { return NULL; - assert(CDataObject_Check(pd)); + } + assert(CDataObject_Check(GLOBAL_STATE(), pd)); pd->b_ptr = (char *)buf; pd->b_length = dict->length; pd->b_size = dict->size; @@ -3042,9 +2922,11 @@ PyCData_AtAddress(PyObject *type, void *buf) int _ctypes_simple_instance(PyObject *obj) { PyTypeObject *type = (PyTypeObject *)obj; + ctypes_state *st = GLOBAL_STATE(); - if (PyCSimpleTypeObject_Check(type)) - return type->tp_base != &Simple_Type; + if (PyCSimpleTypeObject_Check(st, type)) { + return type->tp_base != st->Simple_Type; + } return 0; } @@ -3072,10 +2954,11 @@ _PyCData_set(CDataObject *dst, PyObject *type, SETFUNC setfunc, PyObject *value, CDataObject *src; int err; - if (setfunc) + if (setfunc) { return setfunc(ptr, value, size); - - if (!CDataObject_Check(value)) { + } + ctypes_state *st = GLOBAL_STATE(); + if (!CDataObject_Check(st, value)) { StgDictObject *dict = PyType_stgdict(type); if (dict && dict->setfunc) return dict->setfunc(ptr, value, size); @@ -3097,7 +2980,7 @@ _PyCData_set(CDataObject *dst, PyObject *type, SETFUNC setfunc, PyObject *value, size, ptr); Py_DECREF(ob); return result; - } else if (value == Py_None && PyCPointerTypeObject_Check(type)) { + } else if (value == Py_None && PyCPointerTypeObject_Check(st, type)) { *(void **)ptr = NULL; Py_RETURN_NONE; } else { @@ -3118,7 +3001,7 @@ _PyCData_set(CDataObject *dst, PyObject *type, SETFUNC setfunc, PyObject *value, src->b_ptr, size); - if (PyCPointerTypeObject_Check(type)) { + if (PyCPointerTypeObject_Check(st, type)) { /* XXX */ } @@ -3129,8 +3012,8 @@ _PyCData_set(CDataObject *dst, PyObject *type, SETFUNC setfunc, PyObject *value, return Py_NewRef(value); } - if (PyCPointerTypeObject_Check(type) - && ArrayObject_Check(value)) { + if (PyCPointerTypeObject_Check(st, type) + && ArrayObject_Check(st, value)) { StgDictObject *p1, *p2; PyObject *keep; p1 = PyObject_stgdict(value); @@ -3178,8 +3061,9 @@ PyCData_set(PyObject *dst, PyObject *type, SETFUNC setfunc, PyObject *value, { CDataObject *mem = (CDataObject *)dst; PyObject *result; + ctypes_state *st = GLOBAL_STATE(); - if (!CDataObject_Check(dst)) { + if (!CDataObject_Check(st, dst)) { PyErr_SetString(PyExc_TypeError, "not a ctype instance"); return -1; @@ -3399,13 +3283,14 @@ static int _check_outarg_type(PyObject *arg, Py_ssize_t index) { StgDictObject *dict; + ctypes_state *st = GLOBAL_STATE(); - if (PyCPointerTypeObject_Check(arg)) + if (PyCPointerTypeObject_Check(st, arg)) { return 1; - - if (PyCArrayTypeObject_Check(arg)) + } + if (PyCArrayTypeObject_Check(st, arg)) { return 1; - + } dict = PyType_stgdict(arg); if (dict /* simple pointer types, c_void_p, c_wchar_p, BSTR, ... */ @@ -3774,7 +3659,9 @@ static PyObject * _byref(PyObject *obj) { PyCArgObject *parg; - if (!CDataObject_Check(obj)) { + ctypes_state *st = GLOBAL_STATE(); + + if (!CDataObject_Check(st, obj)) { PyErr_SetString(PyExc_TypeError, "expected CData instance"); return NULL; @@ -3882,6 +3769,7 @@ _build_callargs(PyCFuncPtrObject *self, PyObject *argtypes, inargs_index = 1; } #endif + ctypes_state *st = GLOBAL_STATE(); for (i = 0; i < len; ++i) { PyObject *item = PyTuple_GET_ITEM(paramflags, i); PyObject *ob; @@ -3953,11 +3841,13 @@ _build_callargs(PyCFuncPtrObject *self, PyObject *argtypes, ((PyTypeObject *)ob)->tp_name); goto error; } - if (PyCArrayTypeObject_Check(ob)) + if (PyCArrayTypeObject_Check(st, ob)) { ob = _PyObject_CallNoArgs(ob); - else + } + else { /* Create an instance of the pointed-to type */ ob = _PyObject_CallNoArgs(dict->proto); + } /* XXX Is the following correct any longer? We must not pass a byref() to the array then but @@ -4109,7 +3999,8 @@ PyCFuncPtr_call(PyCFuncPtrObject *self, PyObject *inargs, PyObject *kwds) "native com method call without 'this' parameter"); return NULL; } - if (!CDataObject_Check(this)) { + ctypes_state *st = GLOBAL_STATE(); + if (!CDataObject_Check(st, this)) { PyErr_SetString(PyExc_TypeError, "Expected a COM this pointer as first argument"); return NULL; @@ -4354,10 +4245,10 @@ _init_pos_args(PyObject *self, PyTypeObject *type, return index; } - for (i = 0; - i < dict->length && (i+index) < PyTuple_GET_SIZE(args); + for (i = index; + i < dict->length && i < PyTuple_GET_SIZE(args); ++i) { - PyObject *pair = PySequence_GetItem(fields, i); + PyObject *pair = PySequence_GetItem(fields, i - index); PyObject *name, *val; int res; if (!pair) @@ -4367,7 +4258,7 @@ _init_pos_args(PyObject *self, PyTypeObject *type, Py_DECREF(pair); return -1; } - val = PyTuple_GET_ITEM(args, i + index); + val = PyTuple_GET_ITEM(args, i); if (kwds) { res = PyDict_Contains(kwds, name); if (res != 0) { @@ -4388,7 +4279,7 @@ _init_pos_args(PyObject *self, PyTypeObject *type, if (res == -1) return -1; } - return index + dict->length; + return dict->length; } static int @@ -4880,11 +4771,11 @@ PyCArrayType_from_ctype(PyObject *itemtype, Py_ssize_t length) sprintf(name, "%.200s_Array_%ld", ((PyTypeObject *)itemtype)->tp_name, (long)length); #endif - - result = PyObject_CallFunction((PyObject *)&PyCArrayType_Type, + ctypes_state *st = GLOBAL_STATE(); + result = PyObject_CallFunction((PyObject *)st->PyCArrayType_Type, "s(O){s:n,s:O}", name, - &PyCArray_Type, + st->PyCArray_Type, "_length_", length, "_type_", @@ -4995,8 +4886,9 @@ static PyObject * Simple_repr(CDataObject *self) { PyObject *val, *result; + ctypes_state *st = GLOBAL_STATE(); - if (Py_TYPE(self)->tp_base != &Simple_Type) { + if (Py_TYPE(self)->tp_base != st->Simple_Type) { return PyUnicode_FromFormat("<%s object at %p>", Py_TYPE(self)->tp_name, self); } @@ -5159,7 +5051,8 @@ Pointer_set_contents(CDataObject *self, PyObject *value, void *closure) stgdict = PyObject_stgdict((PyObject *)self); assert(stgdict); /* Cannot be NULL for pointer instances */ assert(stgdict->proto); - if (!CDataObject_Check(value)) { + ctypes_state *st = GLOBAL_STATE(); + if (!CDataObject_Check(st, value)) { int res = PyObject_IsInstance(value, stgdict->proto); if (res == -1) return -1; @@ -5541,11 +5434,14 @@ static int cast_check_pointertype(PyObject *arg) { StgDictObject *dict; + ctypes_state *st = GLOBAL_STATE(); - if (PyCPointerTypeObject_Check(arg)) + if (PyCPointerTypeObject_Check(st, arg)) { return 1; - if (PyCFuncPtrTypeObject_Check(arg)) + } + if (PyCFuncPtrTypeObject_Check(st, arg)) { return 1; + } dict = PyType_stgdict(arg); if (dict != NULL && dict->proto != NULL) { if (PyUnicode_Check(dict->proto) @@ -5578,7 +5474,8 @@ cast(void *ptr, PyObject *src, PyObject *ctype) It must certainly contain the source objects one. It must contain the source object itself. */ - if (CDataObject_Check(src)) { + ctypes_state *st = GLOBAL_STATE(); + if (CDataObject_Check(st, src)) { CDataObject *obj = (CDataObject *)src; CDataObject *container; @@ -5682,32 +5579,37 @@ _ctypes_add_types(PyObject *mod) */ CREATE_TYPE(mod, st->PyCArg_Type, &carg_spec, NULL); CREATE_TYPE(mod, st->PyCThunk_Type, &cthunk_spec, NULL); - TYPE_READY(&PyCData_Type); + TYPE_READY(st->PyCData_Type); /* StgDict is derived from PyDict_Type */ - TYPE_READY_BASE(&PyCStgDict_Type, &PyDict_Type); + TYPE_READY_BASE(st->PyCStgDict_Type, &PyDict_Type); /************************************************* * * Metaclasses */ - TYPE_READY_BASE(&PyCStructType_Type, &PyType_Type); - TYPE_READY_BASE(&UnionType_Type, &PyType_Type); - TYPE_READY_BASE(&PyCPointerType_Type, &PyType_Type); - TYPE_READY_BASE(&PyCArrayType_Type, &PyType_Type); - TYPE_READY_BASE(&PyCSimpleType_Type, &PyType_Type); - TYPE_READY_BASE(&PyCFuncPtrType_Type, &PyType_Type); + CREATE_TYPE(mod, st->PyCStructType_Type, &pycstruct_type_spec, + &PyType_Type); + CREATE_TYPE(mod, st->UnionType_Type, &union_type_spec, &PyType_Type); + CREATE_TYPE(mod, st->PyCPointerType_Type, &pycpointer_type_spec, + &PyType_Type); + CREATE_TYPE(mod, st->PyCArrayType_Type, &pycarray_type_spec, + &PyType_Type); + CREATE_TYPE(mod, st->PyCSimpleType_Type, &pycsimple_type_spec, + &PyType_Type); + CREATE_TYPE(mod, st->PyCFuncPtrType_Type, &pycfuncptr_type_spec, + &PyType_Type); /************************************************* * * Classes using a custom metaclass */ - MOD_ADD_TYPE(&Struct_Type, &PyCStructType_Type, &PyCData_Type); - MOD_ADD_TYPE(&Union_Type, &UnionType_Type, &PyCData_Type); - MOD_ADD_TYPE(&PyCPointer_Type, &PyCPointerType_Type, &PyCData_Type); - MOD_ADD_TYPE(&PyCArray_Type, &PyCArrayType_Type, &PyCData_Type); - MOD_ADD_TYPE(&Simple_Type, &PyCSimpleType_Type, &PyCData_Type); - MOD_ADD_TYPE(&PyCFuncPtr_Type, &PyCFuncPtrType_Type, &PyCData_Type); + MOD_ADD_TYPE(st->Struct_Type, st->PyCStructType_Type, st->PyCData_Type); + MOD_ADD_TYPE(st->Union_Type, st->UnionType_Type, st->PyCData_Type); + MOD_ADD_TYPE(st->PyCPointer_Type, st->PyCPointerType_Type, st->PyCData_Type); + MOD_ADD_TYPE(st->PyCArray_Type, st->PyCArrayType_Type, st->PyCData_Type); + MOD_ADD_TYPE(st->Simple_Type, st->PyCSimpleType_Type, st->PyCData_Type); + MOD_ADD_TYPE(st->PyCFuncPtr_Type, st->PyCFuncPtrType_Type, st->PyCData_Type); /************************************************* * diff --git a/Modules/_ctypes/callbacks.c b/Modules/_ctypes/callbacks.c index 154e9f43983cdb..f70479435915ff 100644 --- a/Modules/_ctypes/callbacks.c +++ b/Modules/_ctypes/callbacks.c @@ -151,6 +151,7 @@ static void _CallPythonObject(void *mem, assert(nargs <= CTYPES_MAX_ARGCOUNT); PyObject **args = alloca(nargs * sizeof(PyObject *)); PyObject **cnvs = PySequence_Fast_ITEMS(converters); + ctypes_state *st = GLOBAL_STATE(); for (i = 0; i < nargs; i++) { PyObject *cnv = cnvs[i]; // borrowed ref StgDictObject *dict; @@ -175,7 +176,7 @@ static void _CallPythonObject(void *mem, PrintError("create argument %zd:\n", i); goto Done; } - if (!CDataObject_Check(obj)) { + if (!CDataObject_Check(st, obj)) { Py_DECREF(obj); PrintError("unexpected result of create argument %zd:\n", i); goto Done; diff --git a/Modules/_ctypes/callproc.c b/Modules/_ctypes/callproc.c index 3b11cd7f58ce4b..97d1dbaae03d4f 100644 --- a/Modules/_ctypes/callproc.c +++ b/Modules/_ctypes/callproc.c @@ -1686,11 +1686,13 @@ sizeof_func(PyObject *self, PyObject *obj) StgDictObject *dict; dict = PyType_stgdict(obj); - if (dict) + if (dict) { return PyLong_FromSsize_t(dict->size); - - if (CDataObject_Check(obj)) + } + ctypes_state *st = GLOBAL_STATE(); + if (CDataObject_Check(st, obj)) { return PyLong_FromSsize_t(((CDataObject *)obj)->b_size); + } PyErr_SetString(PyExc_TypeError, "this type has no size"); return NULL; @@ -1744,7 +1746,8 @@ byref(PyObject *self, PyObject *args) if (offset == -1 && PyErr_Occurred()) return NULL; } - if (!CDataObject_Check(obj)) { + ctypes_state *st = GLOBAL_STATE(); + if (!CDataObject_Check(st, obj)) { PyErr_Format(PyExc_TypeError, "byref() argument must be a ctypes instance, not '%s'", Py_TYPE(obj)->tp_name); @@ -1769,7 +1772,8 @@ PyDoc_STRVAR(addressof_doc, static PyObject * addressof(PyObject *self, PyObject *obj) { - if (!CDataObject_Check(obj)) { + ctypes_state *st = GLOBAL_STATE(); + if (!CDataObject_Check(st, obj)) { PyErr_SetString(PyExc_TypeError, "invalid type"); return NULL; @@ -1925,13 +1929,14 @@ create_pointer_type(PyObject *module, PyObject *cls) // found or error return result; } + ctypes_state *st = GLOBAL_STATE(); // not found if (PyUnicode_CheckExact(cls)) { PyObject *name = PyUnicode_FromFormat("LP_%U", cls); - result = PyObject_CallFunction((PyObject *)Py_TYPE(&PyCPointer_Type), + result = PyObject_CallFunction((PyObject *)Py_TYPE(st->PyCPointer_Type), "N(O){}", name, - &PyCPointer_Type); + st->PyCPointer_Type); if (result == NULL) return result; key = PyLong_FromVoidPtr(result); @@ -1942,10 +1947,10 @@ create_pointer_type(PyObject *module, PyObject *cls) } else if (PyType_Check(cls)) { typ = (PyTypeObject *)cls; PyObject *name = PyUnicode_FromFormat("LP_%s", typ->tp_name); - result = PyObject_CallFunction((PyObject *)Py_TYPE(&PyCPointer_Type), + result = PyObject_CallFunction((PyObject *)Py_TYPE(st->PyCPointer_Type), "N(O){sO}", name, - &PyCPointer_Type, + st->PyCPointer_Type, "_type_", cls); if (result == NULL) return result; diff --git a/Modules/_ctypes/cfield.c b/Modules/_ctypes/cfield.c index bfb40e5c5393fc..1d5b0b14bc39e5 100644 --- a/Modules/_ctypes/cfield.c +++ b/Modules/_ctypes/cfield.c @@ -111,7 +111,7 @@ PyCField_FromDesc(PyObject *desc, Py_ssize_t index, /* Field descriptors for 'c_char * n' are be scpecial cased to return a Python string instead of an Array object instance... */ - if (PyCArrayTypeObject_Check(proto)) { + if (PyCArrayTypeObject_Check(st, proto)) { StgDictObject *adict = PyType_stgdict(proto); StgDictObject *idict; if (adict && adict->proto) { @@ -204,7 +204,8 @@ PyCField_set(CFieldObject *self, PyObject *inst, PyObject *value) { CDataObject *dst; char *ptr; - if (!CDataObject_Check(inst)) { + ctypes_state *st = GLOBAL_STATE(); + if (!CDataObject_Check(st, inst)) { PyErr_SetString(PyExc_TypeError, "not a ctype instance"); return -1; @@ -227,7 +228,8 @@ PyCField_get(CFieldObject *self, PyObject *inst, PyTypeObject *type) if (inst == NULL) { return Py_NewRef(self); } - if (!CDataObject_Check(inst)) { + ctypes_state *st = GLOBAL_STATE(); + if (!CDataObject_Check(st, inst)) { PyErr_SetString(PyExc_TypeError, "not a ctype instance"); return NULL; diff --git a/Modules/_ctypes/ctypes.h b/Modules/_ctypes/ctypes.h index 8891a0a741de7b..1989723f6f3dbb 100644 --- a/Modules/_ctypes/ctypes.h +++ b/Modules/_ctypes/ctypes.h @@ -37,10 +37,24 @@ typedef struct { PyTypeObject *PyCArg_Type; PyTypeObject *PyCField_Type; PyTypeObject *PyCThunk_Type; + PyTypeObject *PyCStgDict_Type; + PyTypeObject *StructParam_Type; + PyTypeObject *PyCStructType_Type; + PyTypeObject *UnionType_Type; + PyTypeObject *PyCPointerType_Type; + PyTypeObject *PyCArrayType_Type; + PyTypeObject *PyCSimpleType_Type; + PyTypeObject *PyCFuncPtrType_Type; + PyTypeObject *PyCData_Type; + PyTypeObject *Struct_Type; + PyTypeObject *Union_Type; + PyTypeObject *PyCArray_Type; + PyTypeObject *Simple_Type; + PyTypeObject *PyCPointer_Type; + PyTypeObject *PyCFuncPtr_Type; #ifdef MS_WIN32 PyTypeObject *PyComError_Type; #endif - PyTypeObject *StructParam_Type; } ctypes_state; extern ctypes_state global_state; @@ -118,7 +132,7 @@ typedef struct { Py_ssize_t b_size; /* size of memory block in bytes */ Py_ssize_t b_length; /* number of references we need */ Py_ssize_t b_index; /* index of this object into base's - b_object list */ + b_object list */ PyObject *b_objects; /* list of references we need to keep */ union value b_value; /* end of tagCDataObject, additional fields follow */ @@ -141,8 +155,8 @@ typedef struct { } PyCFuncPtrObject; extern PyTypeObject PyCStgDict_Type; -#define PyCStgDict_CheckExact(v) Py_IS_TYPE(v, &PyCStgDict_Type) -#define PyCStgDict_Check(v) PyObject_TypeCheck(v, &PyCStgDict_Type) +#define PyCStgDict_CheckExact(st, v) Py_IS_TYPE((v), (st)->PyCStgDict_Type) +#define PyCStgDict_Check(st, v) PyObject_TypeCheck((v), (st)->PyCStgDict_Type) extern int PyCStructUnionType_update_stgdict(PyObject *fields, PyObject *type, int isStruct); extern int PyType_stginfo(PyTypeObject *self, Py_ssize_t *psize, Py_ssize_t *palign, Py_ssize_t *plength); @@ -151,13 +165,12 @@ extern int PyObject_stginfo(PyObject *self, Py_ssize_t *psize, Py_ssize_t *palig extern PyTypeObject PyCData_Type; -#define CDataObject_CheckExact(v) Py_IS_TYPE(v, &PyCData_Type) -#define CDataObject_Check(v) PyObject_TypeCheck(v, &PyCData_Type) +#define CDataObject_CheckExact(st, v) Py_IS_TYPE((v), (st)->PyCData_Type) +#define CDataObject_Check(st, v) PyObject_TypeCheck((v), (st)->PyCData_Type) #define _CDataObject_HasExternalBuffer(v) ((v)->b_ptr != (char *)&(v)->b_value) -extern PyTypeObject PyCSimpleType_Type; -#define PyCSimpleTypeObject_CheckExact(v) Py_IS_TYPE(v, &PyCSimpleType_Type) -#define PyCSimpleTypeObject_Check(v) PyObject_TypeCheck(v, &PyCSimpleType_Type) +#define PyCSimpleTypeObject_CheckExact(st, v) Py_IS_TYPE((v), (st)->PyCSimpleType_Type) +#define PyCSimpleTypeObject_Check(st, v) PyObject_TypeCheck((v), (st)->PyCSimpleType_Type) extern struct fielddesc *_ctypes_get_fielddesc(const char *fmt); @@ -171,21 +184,17 @@ PyCField_FromDesc(PyObject *desc, Py_ssize_t index, extern PyObject *PyCData_AtAddress(PyObject *type, void *buf); extern PyObject *PyCData_FromBytes(PyObject *type, char *data, Py_ssize_t length); -extern PyTypeObject PyCArrayType_Type; extern PyTypeObject PyCArray_Type; -extern PyTypeObject PyCPointerType_Type; extern PyTypeObject PyCPointer_Type; extern PyTypeObject PyCFuncPtr_Type; -extern PyTypeObject PyCFuncPtrType_Type; -extern PyTypeObject PyCStructType_Type; - -#define PyCArrayTypeObject_Check(v) PyObject_TypeCheck(v, &PyCArrayType_Type) -#define ArrayObject_Check(v) PyObject_TypeCheck(v, &PyCArray_Type) -#define PointerObject_Check(v) PyObject_TypeCheck(v, &PyCPointer_Type) -#define PyCPointerTypeObject_Check(v) PyObject_TypeCheck(v, &PyCPointerType_Type) -#define PyCFuncPtrObject_Check(v) PyObject_TypeCheck(v, &PyCFuncPtr_Type) -#define PyCFuncPtrTypeObject_Check(v) PyObject_TypeCheck(v, &PyCFuncPtrType_Type) -#define PyCStructTypeObject_Check(v) PyObject_TypeCheck(v, &PyCStructType_Type) + +#define PyCArrayTypeObject_Check(st, v) PyObject_TypeCheck((v), (st)->PyCArrayType_Type) +#define ArrayObject_Check(st, v) PyObject_TypeCheck((v), (st)->PyCArray_Type) +#define PointerObject_Check(st, v) PyObject_TypeCheck((v), (st)->PyCPointer_Type) +#define PyCPointerTypeObject_Check(st, v) PyObject_TypeCheck((v), (st)->PyCPointerType_Type) +#define PyCFuncPtrObject_Check(st,v) PyObject_TypeCheck((v), (st)->PyCFuncPtr_Type) +#define PyCFuncPtrTypeObject_Check(st, v) PyObject_TypeCheck((v), (st)->PyCFuncPtrType_Type) +#define PyCStructTypeObject_Check(st, v) PyObject_TypeCheck((v), (st)->PyCStructType_Type) extern PyObject * PyCArrayType_from_ctype(PyObject *itemtype, Py_ssize_t length); diff --git a/Modules/_ctypes/stgdict.c b/Modules/_ctypes/stgdict.c index dfdb96b0e7258a..deafa696fdd0d0 100644 --- a/Modules/_ctypes/stgdict.c +++ b/Modules/_ctypes/stgdict.c @@ -184,11 +184,14 @@ PyType_stgdict(PyObject *obj) { PyTypeObject *type; - if (!PyType_Check(obj)) + if (!PyType_Check(obj)) { return NULL; + } + ctypes_state *st = GLOBAL_STATE(); type = (PyTypeObject *)obj; - if (!type->tp_dict || !PyCStgDict_CheckExact(type->tp_dict)) + if (!type->tp_dict || !PyCStgDict_CheckExact(st, type->tp_dict)) { return NULL; + } return (StgDictObject *)type->tp_dict; } @@ -201,8 +204,10 @@ StgDictObject * PyObject_stgdict(PyObject *self) { PyTypeObject *type = Py_TYPE(self); - if (!type->tp_dict || !PyCStgDict_CheckExact(type->tp_dict)) + ctypes_state *st = GLOBAL_STATE(); + if (!type->tp_dict || !PyCStgDict_CheckExact(st, type->tp_dict)) { return NULL; + } return (StgDictObject *)type->tp_dict; } @@ -505,6 +510,7 @@ PyCStructUnionType_update_stgdict(PyObject *type, PyObject *fields, int isStruct if (stgdict->format == NULL) return -1; + ctypes_state *st = GLOBAL_STATE(); for (i = 0; i < len; ++i) { PyObject *name = NULL, *desc = NULL; PyObject *pair = PySequence_GetItem(fields, i); @@ -518,8 +524,9 @@ PyCStructUnionType_update_stgdict(PyObject *type, PyObject *fields, int isStruct Py_XDECREF(pair); return -1; } - if (PyCArrayTypeObject_Check(desc)) + if (PyCArrayTypeObject_Check(st, desc)) { arrays_seen = 1; + } dict = PyType_stgdict(desc); if (dict == NULL) { Py_DECREF(pair); @@ -695,12 +702,12 @@ PyCStructUnionType_update_stgdict(PyObject *type, PyObject *fields, int isStruct stgdict->size = aligned_size; stgdict->align = total_align; - stgdict->length = len; /* ADD ffi_ofs? */ + stgdict->length = ffi_ofs + len; /* * The value of MAX_STRUCT_SIZE depends on the platform Python is running on. */ -#if defined(__aarch64__) || defined(__arm__) +#if defined(__aarch64__) || defined(__arm__) || defined(_M_ARM64) # define MAX_STRUCT_SIZE 32 #elif defined(__powerpc64__) # define MAX_STRUCT_SIZE 64 @@ -806,7 +813,7 @@ PyCStructUnionType_update_stgdict(PyObject *type, PyObject *fields, int isStruct i); return -1; } - if (!PyCArrayTypeObject_Check(desc)) { + if (!PyCArrayTypeObject_Check(st, desc)) { /* Not an array. Just need an ffi_type pointer. */ num_ffi_type_pointers++; } @@ -906,7 +913,7 @@ PyCStructUnionType_update_stgdict(PyObject *type, PyObject *fields, int isStruct return -1; } assert(element_index < (ffi_ofs + len)); /* will be used below */ - if (!PyCArrayTypeObject_Check(desc)) { + if (!PyCArrayTypeObject_Check(st, desc)) { /* Not an array. Just copy over the element ffi_type. */ element_types[element_index++] = &dict->ffi_type_pointer; } diff --git a/Modules/_datetimemodule.c b/Modules/_datetimemodule.c index cb5403e8461ff0..9b8e0a719d9048 100644 --- a/Modules/_datetimemodule.c +++ b/Modules/_datetimemodule.c @@ -61,16 +61,6 @@ static datetime_state _datetime_global_state; #define STATIC_STATE() (&_datetime_global_state) -/*[clinic input] -module datetime -class datetime.datetime "PyDateTime_DateTime *" "&PyDateTime_DateTimeType" -class datetime.date "PyDateTime_Date *" "&PyDateTime_DateType" -class datetime.IsoCalendarDate "PyDateTime_IsoCalendarDate *" "&PyDateTime_IsoCalendarDateType" -[clinic start generated code]*/ -/*[clinic end generated code: output=da39a3ee5e6b4b0d input=81bec0fa19837f63]*/ - -#include "clinic/_datetimemodule.c.h" - /* We require that C int be at least 32 bits, and use int virtually * everywhere. In just a few cases we use a temp long, where a Python * API returns a C long. In such cases, we have to ensure that the @@ -161,6 +151,17 @@ static PyTypeObject PyDateTime_TimeZoneType; static int check_tzinfo_subclass(PyObject *p); +/*[clinic input] +module datetime +class datetime.datetime "PyDateTime_DateTime *" "&PyDateTime_DateTimeType" +class datetime.date "PyDateTime_Date *" "&PyDateTime_DateType" +class datetime.time "PyDateTime_Time *" "&PyDateTime_TimeType" +class datetime.IsoCalendarDate "PyDateTime_IsoCalendarDate *" "&PyDateTime_IsoCalendarDateType" +[clinic start generated code]*/ +/*[clinic end generated code: output=da39a3ee5e6b4b0d input=6f65a48dd22fa40f]*/ + +#include "clinic/_datetimemodule.c.h" + /* --------------------------------------------------------------------------- * Math utilities. @@ -3466,24 +3467,22 @@ date_timetuple(PyDateTime_Date *self, PyObject *Py_UNUSED(ignored)) 0, 0, 0, -1); } +/*[clinic input] +datetime.date.replace + + year: int(c_default="GET_YEAR(self)") = unchanged + month: int(c_default="GET_MONTH(self)") = unchanged + day: int(c_default="GET_DAY(self)") = unchanged + +Return date with new specified fields. +[clinic start generated code]*/ + static PyObject * -date_replace(PyDateTime_Date *self, PyObject *args, PyObject *kw) +datetime_date_replace_impl(PyDateTime_Date *self, int year, int month, + int day) +/*[clinic end generated code: output=2a9430d1e6318aeb input=0d1f02685b3e90f6]*/ { - PyObject *clone; - PyObject *tuple; - int year = GET_YEAR(self); - int month = GET_MONTH(self); - int day = GET_DAY(self); - - if (! PyArg_ParseTupleAndKeywords(args, kw, "|iii:replace", date_kws, - &year, &month, &day)) - return NULL; - tuple = Py_BuildValue("iii", year, month, day); - if (tuple == NULL) - return NULL; - clone = date_new(Py_TYPE(self), tuple, NULL); - Py_DECREF(tuple); - return clone; + return new_date_ex(year, month, day, Py_TYPE(self)); } static Py_hash_t @@ -3596,10 +3595,9 @@ static PyMethodDef date_methods[] = { PyDoc_STR("Return the day of the week represented by the date.\n" "Monday == 0 ... Sunday == 6")}, - {"replace", _PyCFunction_CAST(date_replace), METH_VARARGS | METH_KEYWORDS, - PyDoc_STR("Return date with new specified fields.")}, + DATETIME_DATE_REPLACE_METHODDEF - {"__replace__", _PyCFunction_CAST(date_replace), METH_VARARGS | METH_KEYWORDS}, + {"__replace__", _PyCFunction_CAST(datetime_date_replace), METH_FASTCALL | METH_KEYWORDS}, {"__reduce__", (PyCFunction)date_reduce, METH_NOARGS, PyDoc_STR("__reduce__() -> (cls, state)")}, @@ -4573,36 +4571,28 @@ time_hash(PyDateTime_Time *self) return self->hashcode; } +/*[clinic input] +datetime.time.replace + + hour: int(c_default="TIME_GET_HOUR(self)") = unchanged + minute: int(c_default="TIME_GET_MINUTE(self)") = unchanged + second: int(c_default="TIME_GET_SECOND(self)") = unchanged + microsecond: int(c_default="TIME_GET_MICROSECOND(self)") = unchanged + tzinfo: object(c_default="HASTZINFO(self) ? self->tzinfo : Py_None") = unchanged + * + fold: int(c_default="TIME_GET_FOLD(self)") = unchanged + +Return time with new specified fields. +[clinic start generated code]*/ + static PyObject * -time_replace(PyDateTime_Time *self, PyObject *args, PyObject *kw) +datetime_time_replace_impl(PyDateTime_Time *self, int hour, int minute, + int second, int microsecond, PyObject *tzinfo, + int fold) +/*[clinic end generated code: output=0b89a44c299e4f80 input=9b6a35b1e704b0ca]*/ { - PyObject *clone; - PyObject *tuple; - int hh = TIME_GET_HOUR(self); - int mm = TIME_GET_MINUTE(self); - int ss = TIME_GET_SECOND(self); - int us = TIME_GET_MICROSECOND(self); - PyObject *tzinfo = HASTZINFO(self) ? self->tzinfo : Py_None; - int fold = TIME_GET_FOLD(self); - - if (! PyArg_ParseTupleAndKeywords(args, kw, "|iiiiO$i:replace", - time_kws, - &hh, &mm, &ss, &us, &tzinfo, &fold)) - return NULL; - if (fold != 0 && fold != 1) { - PyErr_SetString(PyExc_ValueError, - "fold must be either 0 or 1"); - return NULL; - } - tuple = Py_BuildValue("iiiiO", hh, mm, ss, us, tzinfo); - if (tuple == NULL) - return NULL; - clone = time_new(Py_TYPE(self), tuple, NULL); - if (clone != NULL) { - TIME_SET_FOLD(clone, fold); - } - Py_DECREF(tuple); - return clone; + return new_time_ex2(hour, minute, second, microsecond, tzinfo, fold, + Py_TYPE(self)); } static PyObject * @@ -4732,10 +4722,9 @@ static PyMethodDef time_methods[] = { {"dst", (PyCFunction)time_dst, METH_NOARGS, PyDoc_STR("Return self.tzinfo.dst(self).")}, - {"replace", _PyCFunction_CAST(time_replace), METH_VARARGS | METH_KEYWORDS, - PyDoc_STR("Return time with new specified fields.")}, + DATETIME_TIME_REPLACE_METHODDEF - {"__replace__", _PyCFunction_CAST(time_replace), METH_VARARGS | METH_KEYWORDS}, + {"__replace__", _PyCFunction_CAST(datetime_time_replace), METH_FASTCALL | METH_KEYWORDS}, {"fromisoformat", (PyCFunction)time_fromisoformat, METH_O | METH_CLASS, PyDoc_STR("string -> time from a string in ISO 8601 format")}, @@ -6042,40 +6031,32 @@ datetime_hash(PyDateTime_DateTime *self) return self->hashcode; } +/*[clinic input] +datetime.datetime.replace + + year: int(c_default="GET_YEAR(self)") = unchanged + month: int(c_default="GET_MONTH(self)") = unchanged + day: int(c_default="GET_DAY(self)") = unchanged + hour: int(c_default="DATE_GET_HOUR(self)") = unchanged + minute: int(c_default="DATE_GET_MINUTE(self)") = unchanged + second: int(c_default="DATE_GET_SECOND(self)") = unchanged + microsecond: int(c_default="DATE_GET_MICROSECOND(self)") = unchanged + tzinfo: object(c_default="HASTZINFO(self) ? self->tzinfo : Py_None") = unchanged + * + fold: int(c_default="DATE_GET_FOLD(self)") = unchanged + +Return datetime with new specified fields. +[clinic start generated code]*/ + static PyObject * -datetime_replace(PyDateTime_DateTime *self, PyObject *args, PyObject *kw) +datetime_datetime_replace_impl(PyDateTime_DateTime *self, int year, + int month, int day, int hour, int minute, + int second, int microsecond, PyObject *tzinfo, + int fold) +/*[clinic end generated code: output=00bc96536833fddb input=9b38253d56d9bcad]*/ { - PyObject *clone; - PyObject *tuple; - int y = GET_YEAR(self); - int m = GET_MONTH(self); - int d = GET_DAY(self); - int hh = DATE_GET_HOUR(self); - int mm = DATE_GET_MINUTE(self); - int ss = DATE_GET_SECOND(self); - int us = DATE_GET_MICROSECOND(self); - PyObject *tzinfo = HASTZINFO(self) ? self->tzinfo : Py_None; - int fold = DATE_GET_FOLD(self); - - if (! PyArg_ParseTupleAndKeywords(args, kw, "|iiiiiiiO$i:replace", - datetime_kws, - &y, &m, &d, &hh, &mm, &ss, &us, - &tzinfo, &fold)) - return NULL; - if (fold != 0 && fold != 1) { - PyErr_SetString(PyExc_ValueError, - "fold must be either 0 or 1"); - return NULL; - } - tuple = Py_BuildValue("iiiiiiiO", y, m, d, hh, mm, ss, us, tzinfo); - if (tuple == NULL) - return NULL; - clone = datetime_new(Py_TYPE(self), tuple, NULL); - if (clone != NULL) { - DATE_SET_FOLD(clone, fold); - } - Py_DECREF(tuple); - return clone; + return new_datetime_ex2(year, month, day, hour, minute, second, + microsecond, tzinfo, fold, Py_TYPE(self)); } static PyObject * @@ -6597,10 +6578,9 @@ static PyMethodDef datetime_methods[] = { {"dst", (PyCFunction)datetime_dst, METH_NOARGS, PyDoc_STR("Return self.tzinfo.dst(self).")}, - {"replace", _PyCFunction_CAST(datetime_replace), METH_VARARGS | METH_KEYWORDS, - PyDoc_STR("Return datetime with new specified fields.")}, + DATETIME_DATETIME_REPLACE_METHODDEF - {"__replace__", _PyCFunction_CAST(datetime_replace), METH_VARARGS | METH_KEYWORDS}, + {"__replace__", _PyCFunction_CAST(datetime_datetime_replace), METH_FASTCALL | METH_KEYWORDS}, {"astimezone", _PyCFunction_CAST(datetime_astimezone), METH_VARARGS | METH_KEYWORDS, PyDoc_STR("tz -> convert to local time in new timezone tz\n")}, diff --git a/Modules/_decimal/_decimal.c b/Modules/_decimal/_decimal.c index 8b93f8e2cbcf0b..127f5f2887d4cd 100644 --- a/Modules/_decimal/_decimal.c +++ b/Modules/_decimal/_decimal.c @@ -3446,6 +3446,14 @@ dec_format(PyObject *dec, PyObject *args) if (fmt == NULL) { return NULL; } + + if (size > 0 && fmt[size-1] == 'N') { + if (PyErr_WarnEx(PyExc_DeprecationWarning, + "Format specifier 'N' is deprecated", 1) < 0) { + return NULL; + } + } + /* NOTE: If https://github.com/python/cpython/pull/29438 lands, the * format string manipulation below can be eliminated by enhancing * the forked mpd_parse_fmt_str(). */ @@ -3593,12 +3601,6 @@ dec_format(PyObject *dec, PyObject *args) if (replace_fillchar) { dec_replace_fillchar(decstring); } - if (strchr(fmt, 'N') != NULL) { - if (PyErr_WarnEx(PyExc_DeprecationWarning, - "Format specifier 'N' is deprecated", 1) < 0) { - goto finish; - } - } result = PyUnicode_DecodeUTF8(decstring, size, NULL); diff --git a/Modules/_elementtree.c b/Modules/_elementtree.c index f9d5793f9b6497..54451081211654 100644 --- a/Modules/_elementtree.c +++ b/Modules/_elementtree.c @@ -98,6 +98,7 @@ typedef struct { PyTypeObject *TreeBuilder_Type; PyTypeObject *XMLParser_Type; + PyObject *expat_capsule; struct PyExpat_CAPI *expat_capi; } elementtreestate; @@ -155,6 +156,7 @@ elementtree_clear(PyObject *m) Py_CLEAR(st->ElementIter_Type); Py_CLEAR(st->TreeBuilder_Type); Py_CLEAR(st->XMLParser_Type); + Py_CLEAR(st->expat_capsule); st->expat_capi = NULL; return 0; @@ -175,6 +177,7 @@ elementtree_traverse(PyObject *m, visitproc visit, void *arg) Py_VISIT(st->ElementIter_Type); Py_VISIT(st->TreeBuilder_Type); Py_VISIT(st->XMLParser_Type); + Py_VISIT(st->expat_capsule); return 0; } @@ -264,7 +267,7 @@ typedef struct { LOCAL(int) create_extra(ElementObject* self, PyObject* attrib) { - self->extra = PyObject_Malloc(sizeof(ElementObjectExtra)); + self->extra = PyMem_Malloc(sizeof(ElementObjectExtra)); if (!self->extra) { PyErr_NoMemory(); return -1; @@ -292,10 +295,11 @@ dealloc_extra(ElementObjectExtra *extra) for (i = 0; i < extra->length; i++) Py_DECREF(extra->children[i]); - if (extra->children != extra->_children) - PyObject_Free(extra->children); + if (extra->children != extra->_children) { + PyMem_Free(extra->children); + } - PyObject_Free(extra); + PyMem_Free(extra); } LOCAL(void) @@ -492,14 +496,16 @@ element_resize(ElementObject* self, Py_ssize_t extra) * "children", which needs at least 4 bytes. Although it's a * false alarm always assume at least one child to be safe. */ - children = PyObject_Realloc(self->extra->children, - size * sizeof(PyObject*)); - if (!children) + children = PyMem_Realloc(self->extra->children, + size * sizeof(PyObject*)); + if (!children) { goto nomemory; + } } else { - children = PyObject_Malloc(size * sizeof(PyObject*)); - if (!children) + children = PyMem_Malloc(size * sizeof(PyObject*)); + if (!children) { goto nomemory; + } /* copy existing children from static area to malloc buffer */ memcpy(children, self->extra->children, self->extra->length * sizeof(PyObject*)); @@ -3041,7 +3047,7 @@ _elementtree_TreeBuilder_start_impl(TreeBuilderObject *self, PyObject *tag, #define EXPAT(st, func) ((st)->expat_capi->func) static XML_Memory_Handling_Suite ExpatMemoryHandler = { - PyObject_Malloc, PyObject_Realloc, PyObject_Free}; + PyMem_Malloc, PyMem_Realloc, PyMem_Free}; typedef struct { PyObject_HEAD @@ -3066,6 +3072,7 @@ typedef struct { PyObject *handle_close; elementtreestate *state; + PyObject *elementtree_module; } XMLParserObject; /* helpers */ @@ -3607,7 +3614,11 @@ xmlparser_new(PyTypeObject *type, PyObject *args, PyObject *kwds) self->handle_start = self->handle_data = self->handle_end = NULL; self->handle_comment = self->handle_pi = self->handle_close = NULL; self->handle_doctype = NULL; - self->state = get_elementtree_state_by_type(type); + self->elementtree_module = PyType_GetModuleByDef(type, &elementtreemodule); + assert(self->elementtree_module != NULL); + Py_INCREF(self->elementtree_module); + // See gh-111784 for explanation why is reference to module needed here. + self->state = get_elementtree_state(self->elementtree_module); } return (PyObject *)self; } @@ -3784,6 +3795,7 @@ xmlparser_gc_clear(XMLParserObject *self) EXPAT(st, ParserFree)(parser); } + Py_CLEAR(self->elementtree_module); Py_CLEAR(self->handle_close); Py_CLEAR(self->handle_pi); Py_CLEAR(self->handle_comment); @@ -4343,7 +4355,10 @@ module_exec(PyObject *m) goto error; /* link against pyexpat */ - st->expat_capi = PyCapsule_Import(PyExpat_CAPSULE_NAME, 0); + if (!(st->expat_capsule = _PyImport_GetModuleAttrString("pyexpat", "expat_CAPI"))) + goto error; + if (!(st->expat_capi = PyCapsule_GetPointer(st->expat_capsule, PyExpat_CAPSULE_NAME))) + goto error; if (st->expat_capi) { /* check that it's usable */ if (strcmp(st->expat_capi->magic, PyExpat_CAPI_MAGIC) != 0 || @@ -4418,9 +4433,7 @@ module_exec(PyObject *m) static struct PyModuleDef_Slot elementtree_slots[] = { {Py_mod_exec, module_exec}, - // XXX gh-103092: fix isolation. - {Py_mod_multiple_interpreters, Py_MOD_MULTIPLE_INTERPRETERS_NOT_SUPPORTED}, - //{Py_mod_multiple_interpreters, Py_MOD_PER_INTERPRETER_GIL_SUPPORTED}, + {Py_mod_multiple_interpreters, Py_MOD_PER_INTERPRETER_GIL_SUPPORTED}, {0, NULL}, }; diff --git a/Modules/_io/bufferedio.c b/Modules/_io/bufferedio.c index f02207ace9f3d2..8ebe9ec7095586 100644 --- a/Modules/_io/bufferedio.c +++ b/Modules/_io/bufferedio.c @@ -1050,6 +1050,16 @@ _io__Buffered_read1_impl(buffered *self, Py_ssize_t n) Py_DECREF(res); return NULL; } + /* Flush the write buffer if necessary */ + if (self->writable) { + PyObject *r = buffered_flush_and_rewind_unlocked(self); + if (r == NULL) { + LEAVE_BUFFERED(self) + Py_DECREF(res); + return NULL; + } + Py_DECREF(r); + } _bufferedreader_reset_buf(self); r = _bufferedreader_raw_read(self, PyBytes_AS_STRING(res), n); LEAVE_BUFFERED(self) diff --git a/Modules/_io/clinic/bufferedio.c.h b/Modules/_io/clinic/bufferedio.c.h index d5bec5f71f5be8..64eddcd314a803 100644 --- a/Modules/_io/clinic/bufferedio.c.h +++ b/Modules/_io/clinic/bufferedio.c.h @@ -96,7 +96,7 @@ _io__BufferedIOBase_detach_impl(PyObject *self, PyTypeObject *cls); static PyObject * _io__BufferedIOBase_detach(PyObject *self, PyTypeObject *cls, PyObject *const *args, Py_ssize_t nargs, PyObject *kwnames) { - if (nargs) { + if (nargs || (kwnames && PyTuple_GET_SIZE(kwnames))) { PyErr_SetString(PyExc_TypeError, "detach() takes no arguments"); return NULL; } @@ -1245,4 +1245,4 @@ _io_BufferedRandom___init__(PyObject *self, PyObject *args, PyObject *kwargs) exit: return return_value; } -/*[clinic end generated code: output=442b05b9a117df6c input=a9049054013a1b77]*/ +/*[clinic end generated code: output=4249187a725a3b3e input=a9049054013a1b77]*/ diff --git a/Modules/_io/clinic/bytesio.c.h b/Modules/_io/clinic/bytesio.c.h index 37023e49087647..620e9e3b84ea19 100644 --- a/Modules/_io/clinic/bytesio.c.h +++ b/Modules/_io/clinic/bytesio.c.h @@ -96,7 +96,7 @@ _io_BytesIO_getbuffer_impl(bytesio *self, PyTypeObject *cls); static PyObject * _io_BytesIO_getbuffer(bytesio *self, PyTypeObject *cls, PyObject *const *args, Py_ssize_t nargs, PyObject *kwnames) { - if (nargs) { + if (nargs || (kwnames && PyTuple_GET_SIZE(kwnames))) { PyErr_SetString(PyExc_TypeError, "getbuffer() takes no arguments"); return NULL; } @@ -534,4 +534,4 @@ _io_BytesIO___init__(PyObject *self, PyObject *args, PyObject *kwargs) exit: return return_value; } -/*[clinic end generated code: output=2be0e05a8871b7e2 input=a9049054013a1b77]*/ +/*[clinic end generated code: output=ef116925b8b9e535 input=a9049054013a1b77]*/ diff --git a/Modules/_io/clinic/fileio.c.h b/Modules/_io/clinic/fileio.c.h index cf3ba28b066cf7..5b5487d63eba90 100644 --- a/Modules/_io/clinic/fileio.c.h +++ b/Modules/_io/clinic/fileio.c.h @@ -27,7 +27,7 @@ _io_FileIO_close_impl(fileio *self, PyTypeObject *cls); static PyObject * _io_FileIO_close(fileio *self, PyTypeObject *cls, PyObject *const *args, Py_ssize_t nargs, PyObject *kwnames) { - if (nargs) { + if (nargs || (kwnames && PyTuple_GET_SIZE(kwnames))) { PyErr_SetString(PyExc_TypeError, "close() takes no arguments"); return NULL; } @@ -528,4 +528,4 @@ _io_FileIO_isatty(fileio *self, PyObject *Py_UNUSED(ignored)) #ifndef _IO_FILEIO_TRUNCATE_METHODDEF #define _IO_FILEIO_TRUNCATE_METHODDEF #endif /* !defined(_IO_FILEIO_TRUNCATE_METHODDEF) */ -/*[clinic end generated code: output=1c0f4a36f76b0c6a input=a9049054013a1b77]*/ +/*[clinic end generated code: output=e3d9446b4087020e input=a9049054013a1b77]*/ diff --git a/Modules/_io/clinic/iobase.c.h b/Modules/_io/clinic/iobase.c.h index 6bdfa1444015ac..bae80a265fab07 100644 --- a/Modules/_io/clinic/iobase.c.h +++ b/Modules/_io/clinic/iobase.c.h @@ -262,7 +262,7 @@ _io__IOBase_fileno_impl(PyObject *self, PyTypeObject *cls); static PyObject * _io__IOBase_fileno(PyObject *self, PyTypeObject *cls, PyObject *const *args, Py_ssize_t nargs, PyObject *kwnames) { - if (nargs) { + if (nargs || (kwnames && PyTuple_GET_SIZE(kwnames))) { PyErr_SetString(PyExc_TypeError, "fileno() takes no arguments"); return NULL; } @@ -438,4 +438,4 @@ _io__RawIOBase_readall(PyObject *self, PyObject *Py_UNUSED(ignored)) { return _io__RawIOBase_readall_impl(self); } -/*[clinic end generated code: output=5a22bc5db0ecaacb input=a9049054013a1b77]*/ +/*[clinic end generated code: output=e7326fbefc52bfba input=a9049054013a1b77]*/ diff --git a/Modules/_io/clinic/textio.c.h b/Modules/_io/clinic/textio.c.h index 23b3cc8d71e098..f04ee729abc9ed 100644 --- a/Modules/_io/clinic/textio.c.h +++ b/Modules/_io/clinic/textio.c.h @@ -27,7 +27,7 @@ _io__TextIOBase_detach_impl(PyObject *self, PyTypeObject *cls); static PyObject * _io__TextIOBase_detach(PyObject *self, PyTypeObject *cls, PyObject *const *args, Py_ssize_t nargs, PyObject *kwnames) { - if (nargs) { + if (nargs || (kwnames && PyTuple_GET_SIZE(kwnames))) { PyErr_SetString(PyExc_TypeError, "detach() takes no arguments"); return NULL; } @@ -1292,4 +1292,4 @@ _io_TextIOWrapper__CHUNK_SIZE_set(textio *self, PyObject *value, void *Py_UNUSED return return_value; } -/*[clinic end generated code: output=d01aa598647c1385 input=a9049054013a1b77]*/ +/*[clinic end generated code: output=93a5a91a22100a28 input=a9049054013a1b77]*/ diff --git a/Modules/_io/clinic/winconsoleio.c.h b/Modules/_io/clinic/winconsoleio.c.h index 6cab295c44611d..4696ecc5c843e6 100644 --- a/Modules/_io/clinic/winconsoleio.c.h +++ b/Modules/_io/clinic/winconsoleio.c.h @@ -29,7 +29,7 @@ _io__WindowsConsoleIO_close_impl(winconsoleio *self, PyTypeObject *cls); static PyObject * _io__WindowsConsoleIO_close(winconsoleio *self, PyTypeObject *cls, PyObject *const *args, Py_ssize_t nargs, PyObject *kwnames) { - if (nargs) { + if (nargs || (kwnames && PyTuple_GET_SIZE(kwnames))) { PyErr_SetString(PyExc_TypeError, "close() takes no arguments"); return NULL; } @@ -457,4 +457,4 @@ _io__WindowsConsoleIO_isatty(winconsoleio *self, PyObject *Py_UNUSED(ignored)) #ifndef _IO__WINDOWSCONSOLEIO_ISATTY_METHODDEF #define _IO__WINDOWSCONSOLEIO_ISATTY_METHODDEF #endif /* !defined(_IO__WINDOWSCONSOLEIO_ISATTY_METHODDEF) */ -/*[clinic end generated code: output=04108fc26b187386 input=a9049054013a1b77]*/ +/*[clinic end generated code: output=2c2bc86713b21dd6 input=a9049054013a1b77]*/ diff --git a/Modules/_io/fileio.c b/Modules/_io/fileio.c index 8a73ea0365b7a3..6bb156e41fe43c 100644 --- a/Modules/_io/fileio.c +++ b/Modules/_io/fileio.c @@ -157,7 +157,7 @@ _io_FileIO_close_impl(fileio *self, PyTypeObject *cls) return res; } - PyObject *exc; + PyObject *exc = NULL; if (res == NULL) { exc = PyErr_GetRaisedException(); } @@ -269,6 +269,13 @@ _io_FileIO___init___impl(fileio *self, PyObject *nameobj, const char *mode, self->fd = -1; } + if (PyBool_Check(nameobj)) { + if (PyErr_WarnEx(PyExc_RuntimeWarning, + "bool is used as a file descriptor", 1)) + { + return -1; + } + } fd = PyLong_AsInt(nameobj); if (fd < 0) { if (!PyErr_Occurred()) { @@ -1100,31 +1107,32 @@ static PyObject * fileio_repr(fileio *self) { PyObject *nameobj, *res; + const char *type_name = Py_TYPE((PyObject *) self)->tp_name; - if (self->fd < 0) - return PyUnicode_FromFormat("<_io.FileIO [closed]>"); + if (self->fd < 0) { + return PyUnicode_FromFormat("<%.100s [closed]>", type_name); + } if (PyObject_GetOptionalAttr((PyObject *) self, &_Py_ID(name), &nameobj) < 0) { return NULL; } if (nameobj == NULL) { res = PyUnicode_FromFormat( - "<_io.FileIO fd=%d mode='%s' closefd=%s>", - self->fd, mode_string(self), self->closefd ? "True" : "False"); + "<%.100s fd=%d mode='%s' closefd=%s>", + type_name, self->fd, mode_string(self), self->closefd ? "True" : "False"); } else { int status = Py_ReprEnter((PyObject *)self); res = NULL; if (status == 0) { res = PyUnicode_FromFormat( - "<_io.FileIO name=%R mode='%s' closefd=%s>", - nameobj, mode_string(self), self->closefd ? "True" : "False"); + "<%.100s name=%R mode='%s' closefd=%s>", + type_name, nameobj, mode_string(self), self->closefd ? "True" : "False"); Py_ReprLeave((PyObject *)self); } else if (status > 0) { PyErr_Format(PyExc_RuntimeError, - "reentrant call inside %s.__repr__", - Py_TYPE(self)->tp_name); + "reentrant call inside %.100s.__repr__", type_name); } Py_DECREF(nameobj); } diff --git a/Modules/_io/iobase.c b/Modules/_io/iobase.c index 4da8e5bd572d74..184e0b7d1aa7f1 100644 --- a/Modules/_io/iobase.c +++ b/Modules/_io/iobase.c @@ -66,12 +66,19 @@ PyDoc_STRVAR(iobase_doc, "with open('spam.txt', 'r') as fp:\n" " fp.write('Spam and eggs!')\n"); -/* Use this macro whenever you want to check the internal `closed` status + +/* Internal methods */ + +/* Use this function whenever you want to check the internal `closed` status of the IOBase object rather than the virtual `closed` attribute as returned by whatever subclass. */ +static int +iobase_is_closed(PyObject *self) +{ + return PyObject_HasAttrWithError(self, &_Py_ID(__IOBase_closed)); +} -/* Internal methods */ static PyObject * iobase_unsupported(_PyIO_State *state, const char *message) { @@ -145,14 +152,6 @@ _io__IOBase_truncate_impl(PyObject *self, PyTypeObject *cls, return iobase_unsupported(state, "truncate"); } -static int -iobase_is_closed(PyObject *self) -{ - /* This gets the derived attribute, which is *not* __IOBase_closed - in most cases! */ - return PyObject_HasAttrWithError(self, &_Py_ID(__IOBase_closed)); -} - /* Flush and close methods */ /*[clinic input] diff --git a/Modules/_io/textio.c b/Modules/_io/textio.c index 4507930c14bb50..d794af8de2b8f0 100644 --- a/Modules/_io/textio.c +++ b/Modules/_io/textio.c @@ -1762,8 +1762,10 @@ _io_TextIOWrapper_write_impl(textio *self, PyObject *text) } } - textiowrapper_set_decoded_chars(self, NULL); - Py_CLEAR(self->snapshot); + if (self->snapshot != NULL) { + textiowrapper_set_decoded_chars(self, NULL); + Py_CLEAR(self->snapshot); + } if (self->decoder) { ret = PyObject_CallMethodNoArgs(self->decoder, &_Py_ID(reset)); @@ -1999,8 +2001,10 @@ _io_TextIOWrapper_read_impl(textio *self, Py_ssize_t n) if (result == NULL) goto fail; - textiowrapper_set_decoded_chars(self, NULL); - Py_CLEAR(self->snapshot); + if (self->snapshot != NULL) { + textiowrapper_set_decoded_chars(self, NULL); + Py_CLEAR(self->snapshot); + } return result; } else { @@ -2944,10 +2948,11 @@ textiowrapper_repr(textio *self) { PyObject *nameobj, *modeobj, *res, *s; int status; + const char *type_name = Py_TYPE(self)->tp_name; CHECK_INITIALIZED(self); - res = PyUnicode_FromString("<_io.TextIOWrapper"); + res = PyUnicode_FromFormat("<%.100s", type_name); if (res == NULL) return NULL; @@ -2955,8 +2960,8 @@ textiowrapper_repr(textio *self) if (status != 0) { if (status > 0) { PyErr_Format(PyExc_RuntimeError, - "reentrant call inside %s.__repr__", - Py_TYPE(self)->tp_name); + "reentrant call inside %.100s.__repr__", + type_name); } goto error; } diff --git a/Modules/_io/winconsoleio.c b/Modules/_io/winconsoleio.c index 6680488b740cfc..54e15555417287 100644 --- a/Modules/_io/winconsoleio.c +++ b/Modules/_io/winconsoleio.c @@ -391,9 +391,9 @@ _io__WindowsConsoleIO___init___impl(winconsoleio *self, PyObject *nameobj, } if (self->writable) - self->fd = _Py_open_osfhandle_noraise(handle, _O_WRONLY | _O_BINARY); + self->fd = _Py_open_osfhandle_noraise(handle, _O_WRONLY | _O_BINARY | _O_NOINHERIT); else - self->fd = _Py_open_osfhandle_noraise(handle, _O_RDONLY | _O_BINARY); + self->fd = _Py_open_osfhandle_noraise(handle, _O_RDONLY | _O_BINARY | _O_NOINHERIT); if (self->fd < 0) { PyErr_SetFromErrnoWithFilenameObject(PyExc_OSError, nameobj); CloseHandle(handle); @@ -1070,15 +1070,22 @@ _io__WindowsConsoleIO_write_impl(winconsoleio *self, PyTypeObject *cls, static PyObject * winconsoleio_repr(winconsoleio *self) { - if (self->fd == -1) - return PyUnicode_FromFormat("<_io._WindowsConsoleIO [closed]>"); - - if (self->readable) - return PyUnicode_FromFormat("<_io._WindowsConsoleIO mode='rb' closefd=%s>", - self->closefd ? "True" : "False"); - if (self->writable) - return PyUnicode_FromFormat("<_io._WindowsConsoleIO mode='wb' closefd=%s>", - self->closefd ? "True" : "False"); + const char *type_name = (Py_TYPE((PyObject *)self)->tp_name); + + if (self->fd == -1) { + return PyUnicode_FromFormat("<%.100s [closed]>", type_name); + } + + if (self->readable) { + return PyUnicode_FromFormat("<%.100s mode='rb' closefd=%s>", + type_name, + self->closefd ? "True" : "False"); + } + if (self->writable) { + return PyUnicode_FromFormat("<%.100s mode='wb' closefd=%s>", + type_name, + self->closefd ? "True" : "False"); + } PyErr_SetString(PyExc_SystemError, "_WindowsConsoleIO has invalid mode"); return NULL; diff --git a/Modules/_json.c b/Modules/_json.c index 24b292ce70e5eb..c55299899e77fe 100644 --- a/Modules/_json.c +++ b/Modules/_json.c @@ -691,11 +691,10 @@ _parse_object_unicode(PyScannerObject *s, PyObject *memo, PyObject *pystr, Py_ss key = scanstring_unicode(pystr, idx + 1, s->strict, &next_idx); if (key == NULL) goto bail; - memokey = PyDict_SetDefault(memo, key, key); - if (memokey == NULL) { + if (PyDict_SetDefaultRef(memo, key, key, &memokey) < 0) { goto bail; } - Py_SETREF(key, Py_NewRef(memokey)); + Py_SETREF(key, memokey); idx = next_idx; /* skip whitespace between key and : delimiter, read :, skip whitespace */ diff --git a/Modules/_lzmamodule.c b/Modules/_lzmamodule.c index eb90c308d16d19..f6bfbfa62687b8 100644 --- a/Modules/_lzmamodule.c +++ b/Modules/_lzmamodule.c @@ -492,7 +492,9 @@ build_filter_spec(const lzma_filter *f) case LZMA_FILTER_ARMTHUMB: case LZMA_FILTER_SPARC: { lzma_options_bcj *options = f->options; - ADD_FIELD(options, start_offset); + if (options) { + ADD_FIELD(options, start_offset); + } break; } default: diff --git a/Modules/_pickle.c b/Modules/_pickle.c index 227e5378e42285..f210c0ca205991 100644 --- a/Modules/_pickle.c +++ b/Modules/_pickle.c @@ -399,64 +399,6 @@ _Pickle_FastCall(PyObject *func, PyObject *obj) /*************************************************************************/ -/* Retrieve and deconstruct a method for avoiding a reference cycle - (pickler -> bound method of pickler -> pickler) */ -static int -init_method_ref(PyObject *self, PyObject *name, - PyObject **method_func, PyObject **method_self) -{ - PyObject *func, *func2; - int ret; - - /* *method_func and *method_self should be consistent. All refcount decrements - should be occurred after setting *method_self and *method_func. */ - ret = PyObject_GetOptionalAttr(self, name, &func); - if (func == NULL) { - *method_self = NULL; - Py_CLEAR(*method_func); - return ret; - } - - if (PyMethod_Check(func) && PyMethod_GET_SELF(func) == self) { - /* Deconstruct a bound Python method */ - *method_self = self; /* borrowed */ - func2 = PyMethod_GET_FUNCTION(func); - Py_XSETREF(*method_func, Py_NewRef(func2)); - Py_DECREF(func); - return 0; - } - else { - *method_self = NULL; - Py_XSETREF(*method_func, func); - return 0; - } -} - -/* Bind a method if it was deconstructed */ -static PyObject * -reconstruct_method(PyObject *func, PyObject *self) -{ - if (self) { - return PyMethod_New(func, self); - } - else { - return Py_NewRef(func); - } -} - -static PyObject * -call_method(PyObject *func, PyObject *self, PyObject *obj) -{ - if (self) { - return PyObject_CallFunctionObjArgs(func, self, obj, NULL); - } - else { - return PyObject_CallOneArg(func, obj); - } -} - -/*************************************************************************/ - /* Internal data type used as the unpickling stack. */ typedef struct { PyObject_VAR_HEAD @@ -668,9 +610,7 @@ typedef struct PicklerObject { PyMemoTable *memo; /* Memo table, keep track of the seen objects to support self-referential objects pickling. */ - PyObject *pers_func; /* persistent_id() method, can be NULL */ - PyObject *pers_func_self; /* borrowed reference to self if pers_func - is an unbound method, NULL otherwise */ + PyObject *persistent_id; /* persistent_id() method, can be NULL */ PyObject *dispatch_table; /* private dispatch_table, can be NULL */ PyObject *reducer_override; /* hook for invoking user-defined callbacks instead of save_global when pickling @@ -712,9 +652,7 @@ typedef struct UnpicklerObject { size_t memo_size; /* Capacity of the memo array */ size_t memo_len; /* Number of objects in the memo */ - PyObject *pers_func; /* persistent_load() method, can be NULL. */ - PyObject *pers_func_self; /* borrowed reference to self if pers_func - is an unbound method, NULL otherwise */ + PyObject *persistent_load; /* persistent_load() method, can be NULL. */ Py_buffer buffer; char *input_buffer; @@ -1167,8 +1105,7 @@ _Pickler_New(PickleState *st) } self->memo = memo; - self->pers_func = NULL; - self->pers_func_self = NULL; + self->persistent_id = NULL; self->dispatch_table = NULL; self->reducer_override = NULL; self->write = NULL; @@ -1662,8 +1599,7 @@ _Unpickler_New(PyObject *module) self->memo = memo; self->memo_size = MEMO_SIZE; self->memo_len = 0; - self->pers_func = NULL; - self->pers_func_self = NULL; + self->persistent_load = NULL; memset(&self->buffer, 0, sizeof(Py_buffer)); self->input_buffer = NULL; self->input_line = NULL; @@ -3929,7 +3865,7 @@ save_pers(PickleState *state, PicklerObject *self, PyObject *obj) const char persid_op = PERSID; const char binpersid_op = BINPERSID; - pid = call_method(self->pers_func, self->pers_func_self, obj); + pid = PyObject_CallOneArg(self->persistent_id, obj); if (pid == NULL) return -1; @@ -4317,7 +4253,7 @@ save(PickleState *st, PicklerObject *self, PyObject *obj, int pers_save) /* The extra pers_save argument is necessary to avoid calling save_pers() on its returned object. */ - if (!pers_save && self->pers_func) { + if (!pers_save && self->persistent_id) { /* save_pers() returns: -1 to signal an error; 0 if it did nothing successfully; @@ -4522,6 +4458,12 @@ save(PickleState *st, PicklerObject *self, PyObject *obj, int pers_save) return status; } +static PyObject * +persistent_id(PyObject *self, PyObject *obj) +{ + Py_RETURN_NONE; +} + static int dump(PickleState *state, PicklerObject *self, PyObject *obj) { @@ -4529,17 +4471,25 @@ dump(PickleState *state, PicklerObject *self, PyObject *obj) int status = -1; PyObject *tmp; - if (PyObject_GetOptionalAttr((PyObject *)self, &_Py_ID(reducer_override), - &tmp) < 0) { - goto error; + /* Cache the persistent_id method. */ + tmp = PyObject_GetAttr((PyObject *)self, &_Py_ID(persistent_id)); + if (tmp == NULL) { + goto error; } - /* Cache the reducer_override method, if it exists. */ - if (tmp != NULL) { - Py_XSETREF(self->reducer_override, tmp); + if (PyCFunction_Check(tmp) && + PyCFunction_GET_SELF(tmp) == (PyObject *)self && + PyCFunction_GET_FUNCTION(tmp) == persistent_id) + { + Py_CLEAR(tmp); } - else { - Py_CLEAR(self->reducer_override); + Py_XSETREF(self->persistent_id, tmp); + + /* Cache the reducer_override method, if it exists. */ + if (PyObject_GetOptionalAttr((PyObject *)self, &_Py_ID(reducer_override), + &tmp) < 0) { + goto error; } + Py_XSETREF(self->reducer_override, tmp); if (self->proto >= 2) { char header[2]; @@ -4565,11 +4515,12 @@ dump(PickleState *state, PicklerObject *self, PyObject *obj) self->framing = 0; /* Break the reference cycle we generated at the beginning this function - * call when setting the reducer_override attribute of the Pickler instance - * to a bound method of the same instance. This is important as the Pickler - * instance holds a reference to each object it has pickled (through its - * memo): thus, these objects won't be garbage-collected as long as the - * Pickler itself is not collected. */ + * call when setting the persistent_id and the reducer_override attributes + * of the Pickler instance to a bound method of the same instance. + * This is important as the Pickler instance holds a reference to each + * object it has pickled (through its memo): thus, these objects won't + * be garbage-collected as long as the Pickler itself is not collected. */ + Py_CLEAR(self->persistent_id); Py_CLEAR(self->reducer_override); return status; } @@ -4662,6 +4613,8 @@ _pickle_Pickler___sizeof___impl(PicklerObject *self) } static struct PyMethodDef Pickler_methods[] = { + {"persistent_id", persistent_id, METH_O, + PyDoc_STR("persistent_id($self, obj, /)\n--\n\n")}, _PICKLE_PICKLER_DUMP_METHODDEF _PICKLE_PICKLER_CLEAR_MEMO_METHODDEF _PICKLE_PICKLER___SIZEOF___METHODDEF @@ -4673,7 +4626,7 @@ Pickler_clear(PicklerObject *self) { Py_CLEAR(self->output_buffer); Py_CLEAR(self->write); - Py_CLEAR(self->pers_func); + Py_CLEAR(self->persistent_id); Py_CLEAR(self->dispatch_table); Py_CLEAR(self->fast_memo); Py_CLEAR(self->reducer_override); @@ -4702,7 +4655,7 @@ Pickler_traverse(PicklerObject *self, visitproc visit, void *arg) { Py_VISIT(Py_TYPE(self)); Py_VISIT(self->write); - Py_VISIT(self->pers_func); + Py_VISIT(self->persistent_id); Py_VISIT(self->dispatch_table); Py_VISIT(self->fast_memo); Py_VISIT(self->reducer_override); @@ -4799,11 +4752,6 @@ _pickle_Pickler___init___impl(PicklerObject *self, PyObject *file, self->fast_nesting = 0; self->fast_memo = NULL; - if (init_method_ref((PyObject *)self, &_Py_ID(persistent_id), - &self->pers_func, &self->pers_func_self) < 0) - { - return -1; - } if (self->dispatch_table != NULL) { return 0; } @@ -5052,36 +5000,6 @@ Pickler_set_memo(PicklerObject *self, PyObject *obj, void *Py_UNUSED(ignored)) return -1; } -static PyObject * -Pickler_get_persid(PicklerObject *self, void *Py_UNUSED(ignored)) -{ - if (self->pers_func == NULL) { - PyErr_SetString(PyExc_AttributeError, "persistent_id"); - return NULL; - } - return reconstruct_method(self->pers_func, self->pers_func_self); -} - -static int -Pickler_set_persid(PicklerObject *self, PyObject *value, void *Py_UNUSED(ignored)) -{ - if (value == NULL) { - PyErr_SetString(PyExc_TypeError, - "attribute deletion is not supported"); - return -1; - } - if (!PyCallable_Check(value)) { - PyErr_SetString(PyExc_TypeError, - "persistent_id must be a callable taking one argument"); - return -1; - } - - self->pers_func_self = NULL; - Py_XSETREF(self->pers_func, Py_NewRef(value)); - - return 0; -} - static PyMemberDef Pickler_members[] = { {"bin", Py_T_INT, offsetof(PicklerObject, bin)}, {"fast", Py_T_INT, offsetof(PicklerObject, fast)}, @@ -5092,8 +5010,6 @@ static PyMemberDef Pickler_members[] = { static PyGetSetDef Pickler_getsets[] = { {"memo", (getter)Pickler_get_memo, (setter)Pickler_set_memo}, - {"persistent_id", (getter)Pickler_get_persid, - (setter)Pickler_set_persid}, {NULL} }; @@ -6056,36 +5972,28 @@ load_persid(PickleState *st, UnpicklerObject *self) Py_ssize_t len; char *s; - if (self->pers_func) { - if ((len = _Unpickler_Readline(st, self, &s)) < 0) - return -1; - if (len < 1) - return bad_readline(st); + if ((len = _Unpickler_Readline(st, self, &s)) < 0) + return -1; + if (len < 1) + return bad_readline(st); - pid = PyUnicode_DecodeASCII(s, len - 1, "strict"); - if (pid == NULL) { - if (PyErr_ExceptionMatches(PyExc_UnicodeDecodeError)) { - PyErr_SetString(st->UnpicklingError, - "persistent IDs in protocol 0 must be " - "ASCII strings"); - } - return -1; + pid = PyUnicode_DecodeASCII(s, len - 1, "strict"); + if (pid == NULL) { + if (PyErr_ExceptionMatches(PyExc_UnicodeDecodeError)) { + PyErr_SetString(st->UnpicklingError, + "persistent IDs in protocol 0 must be " + "ASCII strings"); } - - obj = call_method(self->pers_func, self->pers_func_self, pid); - Py_DECREF(pid); - if (obj == NULL) - return -1; - - PDATA_PUSH(self->stack, obj, -1); - return 0; - } - else { - PyErr_SetString(st->UnpicklingError, - "A load persistent id instruction was encountered, " - "but no persistent_load function was specified."); return -1; } + + obj = PyObject_CallOneArg(self->persistent_load, pid); + Py_DECREF(pid); + if (obj == NULL) + return -1; + + PDATA_PUSH(self->stack, obj, -1); + return 0; } static int @@ -6093,25 +6001,17 @@ load_binpersid(PickleState *st, UnpicklerObject *self) { PyObject *pid, *obj; - if (self->pers_func) { - PDATA_POP(st, self->stack, pid); - if (pid == NULL) - return -1; - - obj = call_method(self->pers_func, self->pers_func_self, pid); - Py_DECREF(pid); - if (obj == NULL) - return -1; + PDATA_POP(st, self->stack, pid); + if (pid == NULL) + return -1; - PDATA_PUSH(self->stack, obj, -1); - return 0; - } - else { - PyErr_SetString(st->UnpicklingError, - "A load persistent id instruction was encountered, " - "but no persistent_load function was specified."); + obj = PyObject_CallOneArg(self->persistent_load, pid); + Py_DECREF(pid); + if (obj == NULL) return -1; - } + + PDATA_PUSH(self->stack, obj, -1); + return 0; } static int @@ -6837,6 +6737,7 @@ static PyObject * load(PickleState *st, UnpicklerObject *self) { PyObject *value = NULL; + PyObject *tmp; char *s = NULL; self->num_marks = 0; @@ -6846,6 +6747,13 @@ load(PickleState *st, UnpicklerObject *self) if (Py_SIZE(self->stack)) Pdata_clear(self->stack, 0); + /* Cache the persistent_load method. */ + tmp = PyObject_GetAttr((PyObject *)self, &_Py_ID(persistent_load)); + if (tmp == NULL) { + goto error; + } + Py_XSETREF(self->persistent_load, tmp); + /* Convenient macros for the dispatch while-switch loop just below. */ #define OP(opcode, load_func) \ case opcode: if (load_func(st, self) < 0) break; continue; @@ -6858,7 +6766,7 @@ load(PickleState *st, UnpicklerObject *self) if (PyErr_ExceptionMatches(st->UnpicklingError)) { PyErr_Format(PyExc_EOFError, "Ran out of input"); } - return NULL; + goto error; } switch ((enum opcode)s[0]) { @@ -6944,7 +6852,7 @@ load(PickleState *st, UnpicklerObject *self) PyErr_Format(st->UnpicklingError, "invalid load key, '\\x%02x'.", c); } - return NULL; + goto error; } } @@ -6952,14 +6860,41 @@ load(PickleState *st, UnpicklerObject *self) } if (PyErr_Occurred()) { - return NULL; + goto error; } if (_Unpickler_SkipConsumed(self) < 0) - return NULL; + goto error; + Py_CLEAR(self->persistent_load); PDATA_POP(st, self->stack, value); return value; + +error: + Py_CLEAR(self->persistent_load); + return NULL; +} + +/*[clinic input] + +_pickle.Unpickler.persistent_load + + cls: defining_class + pid: object + / + +[clinic start generated code]*/ + +static PyObject * +_pickle_Unpickler_persistent_load_impl(UnpicklerObject *self, + PyTypeObject *cls, PyObject *pid) +/*[clinic end generated code: output=9f4706f1330cb14d input=2f9554fae051276e]*/ +{ + PickleState *st = _Pickle_GetStateByClass(cls); + PyErr_SetString(st->UnpicklingError, + "A load persistent id instruction was encountered, " + "but no persistent_load function was specified."); + return NULL; } /*[clinic input] @@ -7128,6 +7063,7 @@ _pickle_Unpickler___sizeof___impl(UnpicklerObject *self) } static struct PyMethodDef Unpickler_methods[] = { + _PICKLE_UNPICKLER_PERSISTENT_LOAD_METHODDEF _PICKLE_UNPICKLER_LOAD_METHODDEF _PICKLE_UNPICKLER_FIND_CLASS_METHODDEF _PICKLE_UNPICKLER___SIZEOF___METHODDEF @@ -7142,7 +7078,7 @@ Unpickler_clear(UnpicklerObject *self) Py_CLEAR(self->read); Py_CLEAR(self->peek); Py_CLEAR(self->stack); - Py_CLEAR(self->pers_func); + Py_CLEAR(self->persistent_load); Py_CLEAR(self->buffers); if (self->buffer.buf != NULL) { PyBuffer_Release(&self->buffer); @@ -7181,7 +7117,7 @@ Unpickler_traverse(UnpicklerObject *self, visitproc visit, void *arg) Py_VISIT(self->read); Py_VISIT(self->peek); Py_VISIT(self->stack); - Py_VISIT(self->pers_func); + Py_VISIT(self->persistent_load); Py_VISIT(self->buffers); PyObject **memo = self->memo; if (memo) { @@ -7247,12 +7183,6 @@ _pickle_Unpickler___init___impl(UnpicklerObject *self, PyObject *file, self->fix_imports = fix_imports; - if (init_method_ref((PyObject *)self, &_Py_ID(persistent_load), - &self->pers_func, &self->pers_func_self) < 0) - { - return -1; - } - PyTypeObject *tp = Py_TYPE(self); PickleState *state = _Pickle_FindStateByType(tp); self->stack = (Pdata *)Pdata_New(state); @@ -7521,41 +7451,8 @@ Unpickler_set_memo(UnpicklerObject *self, PyObject *obj, void *Py_UNUSED(ignored return -1; } -static PyObject * -Unpickler_get_persload(UnpicklerObject *self, void *Py_UNUSED(ignored)) -{ - if (self->pers_func == NULL) { - PyErr_SetString(PyExc_AttributeError, "persistent_load"); - return NULL; - } - return reconstruct_method(self->pers_func, self->pers_func_self); -} - -static int -Unpickler_set_persload(UnpicklerObject *self, PyObject *value, void *Py_UNUSED(ignored)) -{ - if (value == NULL) { - PyErr_SetString(PyExc_TypeError, - "attribute deletion is not supported"); - return -1; - } - if (!PyCallable_Check(value)) { - PyErr_SetString(PyExc_TypeError, - "persistent_load must be a callable taking " - "one argument"); - return -1; - } - - self->pers_func_self = NULL; - Py_XSETREF(self->pers_func, Py_NewRef(value)); - - return 0; -} - static PyGetSetDef Unpickler_getsets[] = { {"memo", (getter)Unpickler_get_memo, (setter)Unpickler_set_memo}, - {"persistent_load", (getter)Unpickler_get_persload, - (setter)Unpickler_set_persload}, {NULL} }; diff --git a/Modules/_posixsubprocess.c b/Modules/_posixsubprocess.c index d0dd8f064e0395..aa1a300e4378dd 100644 --- a/Modules/_posixsubprocess.c +++ b/Modules/_posixsubprocess.c @@ -673,9 +673,10 @@ child_exec(char *const exec_array[], PyObject *preexec_fn, PyObject *preexec_fn_args_tuple) { - int i, saved_errno, reached_preexec = 0; + int i, saved_errno; PyObject *result; - const char* err_msg = ""; + /* Indicate to the parent that the error happened before exec(). */ + const char *err_msg = "noexec"; /* Buffer large enough to hold a hex integer. We can't malloc. */ char hex_errno[sizeof(saved_errno)*2+1]; @@ -735,8 +736,12 @@ child_exec(char *const exec_array[], /* We no longer manually close p2cread, c2pwrite, and errwrite here as * _close_open_fds takes care when it is not already non-inheritable. */ - if (cwd) - POSIX_CALL(chdir(cwd)); + if (cwd) { + if (chdir(cwd) == -1) { + err_msg = "noexec:chdir"; + goto error; + } + } if (child_umask >= 0) umask(child_umask); /* umask() always succeeds. */ @@ -784,7 +789,7 @@ child_exec(char *const exec_array[], #endif /* HAVE_SETREUID */ - reached_preexec = 1; + err_msg = ""; if (preexec_fn != Py_None && preexec_fn_args_tuple) { /* This is where the user has asked us to deadlock their program. */ result = PyObject_Call(preexec_fn, preexec_fn_args_tuple, NULL); @@ -842,16 +847,12 @@ child_exec(char *const exec_array[], } _Py_write_noraise(errpipe_write, cur, hex_errno + sizeof(hex_errno) - cur); _Py_write_noraise(errpipe_write, ":", 1); - if (!reached_preexec) { - /* Indicate to the parent that the error happened before exec(). */ - _Py_write_noraise(errpipe_write, "noexec", 6); - } /* We can't call strerror(saved_errno). It is not async signal safe. * The parent process will look the error message up. */ } else { _Py_write_noraise(errpipe_write, "SubprocessError:0:", 18); - _Py_write_noraise(errpipe_write, err_msg, strlen(err_msg)); } + _Py_write_noraise(errpipe_write, err_msg, strlen(err_msg)); } diff --git a/Modules/_queuemodule.c b/Modules/_queuemodule.c index 81a06cdb79a4f2..18b24855c52ad6 100644 --- a/Modules/_queuemodule.c +++ b/Modules/_queuemodule.c @@ -3,10 +3,12 @@ #endif #include "Python.h" -#include "pycore_ceval.h" // _PyEval_MakePendingCalls() +#include "pycore_ceval.h" // Py_MakePendingCalls() #include "pycore_moduleobject.h" // _PyModule_GetState() +#include "pycore_parking_lot.h" #include "pycore_time.h" // _PyTime_t +#include #include // offsetof() typedef struct { @@ -25,12 +27,173 @@ static struct PyModuleDef queuemodule; #define simplequeue_get_state_by_type(type) \ (simplequeue_get_state(PyType_GetModuleByDef(type, &queuemodule))) +static const Py_ssize_t INITIAL_RING_BUF_CAPACITY = 8; + +typedef struct { + // Where to place the next item + Py_ssize_t put_idx; + + // Where to get the next item + Py_ssize_t get_idx; + + PyObject **items; + + // Total number of items that may be stored + Py_ssize_t items_cap; + + // Number of items stored + Py_ssize_t num_items; +} RingBuf; + +static int +RingBuf_Init(RingBuf *buf) +{ + buf->put_idx = 0; + buf->get_idx = 0; + buf->items_cap = INITIAL_RING_BUF_CAPACITY; + buf->num_items = 0; + buf->items = PyMem_Calloc(buf->items_cap, sizeof(PyObject *)); + if (buf->items == NULL) { + PyErr_NoMemory(); + return -1; + } + return 0; +} + +static PyObject * +RingBuf_At(RingBuf *buf, Py_ssize_t idx) +{ + assert(idx >= 0 && idx < buf->num_items); + return buf->items[(buf->get_idx + idx) % buf->items_cap]; +} + +static void +RingBuf_Fini(RingBuf *buf) +{ + PyObject **items = buf->items; + Py_ssize_t num_items = buf->num_items; + Py_ssize_t cap = buf->items_cap; + Py_ssize_t idx = buf->get_idx; + buf->items = NULL; + buf->put_idx = 0; + buf->get_idx = 0; + buf->num_items = 0; + buf->items_cap = 0; + for (Py_ssize_t n = num_items; n > 0; idx = (idx + 1) % cap, n--) { + Py_DECREF(items[idx]); + } + PyMem_Free(items); +} + +// Resize the underlying items array of buf to the new capacity and arrange +// the items contiguously in the new items array. +// +// Returns -1 on allocation failure or 0 on success. +static int +resize_ringbuf(RingBuf *buf, Py_ssize_t capacity) +{ + Py_ssize_t new_capacity = Py_MAX(INITIAL_RING_BUF_CAPACITY, capacity); + if (new_capacity == buf->items_cap) { + return 0; + } + assert(buf->num_items <= new_capacity); + + PyObject **new_items = PyMem_Calloc(new_capacity, sizeof(PyObject *)); + if (new_items == NULL) { + return -1; + } + + // Copy the "tail" of the old items array. This corresponds to "head" of + // the abstract ring buffer. + Py_ssize_t tail_size = + Py_MIN(buf->num_items, buf->items_cap - buf->get_idx); + if (tail_size > 0) { + memcpy(new_items, buf->items + buf->get_idx, + tail_size * sizeof(PyObject *)); + } + + // Copy the "head" of the old items array, if any. This corresponds to the + // "tail" of the abstract ring buffer. + Py_ssize_t head_size = buf->num_items - tail_size; + if (head_size > 0) { + memcpy(new_items + tail_size, buf->items, + head_size * sizeof(PyObject *)); + } + + PyMem_Free(buf->items); + buf->items = new_items; + buf->items_cap = new_capacity; + buf->get_idx = 0; + buf->put_idx = buf->num_items; + + return 0; +} + +// Returns a strong reference from the head of the buffer. +static PyObject * +RingBuf_Get(RingBuf *buf) +{ + assert(buf->num_items > 0); + + if (buf->num_items < (buf->items_cap / 4)) { + // Items is less than 25% occupied, shrink it by 50%. This allows for + // growth without immediately needing to resize the underlying items + // array. + // + // It's safe it ignore allocation failures here; shrinking is an + // optimization that isn't required for correctness. + (void)resize_ringbuf(buf, buf->items_cap / 2); + } + + PyObject *item = buf->items[buf->get_idx]; + buf->items[buf->get_idx] = NULL; + buf->get_idx = (buf->get_idx + 1) % buf->items_cap; + buf->num_items--; + return item; +} + +// Returns 0 on success or -1 if the buffer failed to grow. +// +// Steals a reference to item. +static int +RingBuf_Put(RingBuf *buf, PyObject *item) +{ + assert(buf->num_items <= buf->items_cap); + + if (buf->num_items == buf->items_cap) { + // Buffer is full, grow it. + if (resize_ringbuf(buf, buf->items_cap * 2) < 0) { + PyErr_NoMemory(); + return -1; + } + } + buf->items[buf->put_idx] = item; + buf->put_idx = (buf->put_idx + 1) % buf->items_cap; + buf->num_items++; + return 0; +} + +static Py_ssize_t +RingBuf_Len(RingBuf *buf) +{ + return buf->num_items; +} + +static bool +RingBuf_IsEmpty(RingBuf *buf) +{ + return buf->num_items == 0; +} + typedef struct { PyObject_HEAD - PyThread_type_lock lock; - int locked; - PyObject *lst; - Py_ssize_t lst_pos; + + // Are there threads waiting for items + bool has_threads_waiting; + + // Items in the queue + RingBuf buf; + PyObject *weakreflist; } simplequeueobject; @@ -43,7 +206,7 @@ class _queue.SimpleQueue "simplequeueobject *" "simplequeue_get_state_by_type(ty static int simplequeue_clear(simplequeueobject *self) { - Py_CLEAR(self->lst); + RingBuf_Fini(&self->buf); return 0; } @@ -53,12 +216,6 @@ simplequeue_dealloc(simplequeueobject *self) PyTypeObject *tp = Py_TYPE(self); PyObject_GC_UnTrack(self); - if (self->lock != NULL) { - /* Unlock the lock so it's safe to free it */ - if (self->locked > 0) - PyThread_release_lock(self->lock); - PyThread_free_lock(self->lock); - } (void)simplequeue_clear(self); if (self->weakreflist != NULL) PyObject_ClearWeakRefs((PyObject *) self); @@ -69,7 +226,10 @@ simplequeue_dealloc(simplequeueobject *self) static int simplequeue_traverse(simplequeueobject *self, visitproc visit, void *arg) { - Py_VISIT(self->lst); + RingBuf *buf = &self->buf; + for (Py_ssize_t i = 0, num_items = buf->num_items; i < num_items; i++) { + Py_VISIT(RingBuf_At(buf, i)); + } Py_VISIT(Py_TYPE(self)); return 0; } @@ -90,15 +250,7 @@ simplequeue_new_impl(PyTypeObject *type) self = (simplequeueobject *) type->tp_alloc(type, 0); if (self != NULL) { self->weakreflist = NULL; - self->lst = PyList_New(0); - self->lock = PyThread_allocate_lock(); - self->lst_pos = 0; - if (self->lock == NULL) { - Py_DECREF(self); - PyErr_SetString(PyExc_MemoryError, "can't allocate lock"); - return NULL; - } - if (self->lst == NULL) { + if (RingBuf_Init(&self->buf) < 0) { Py_DECREF(self); return NULL; } @@ -107,7 +259,29 @@ simplequeue_new_impl(PyTypeObject *type) return (PyObject *) self; } +typedef struct { + bool handed_off; + simplequeueobject *queue; + PyObject *item; +} HandoffData; + +static void +maybe_handoff_item(HandoffData *data, PyObject **item, int has_more_waiters) +{ + if (item == NULL) { + // No threads were waiting + data->handed_off = false; + } + else { + // There was at least one waiting thread, hand off the item + *item = data->item; + data->handed_off = true; + } + data->queue->has_threads_waiting = has_more_waiters; +} + /*[clinic input] +@critical_section _queue.SimpleQueue.put item: object block: bool = True @@ -123,21 +297,28 @@ never blocks. They are provided for compatibility with the Queue class. static PyObject * _queue_SimpleQueue_put_impl(simplequeueobject *self, PyObject *item, int block, PyObject *timeout) -/*[clinic end generated code: output=4333136e88f90d8b input=6e601fa707a782d5]*/ +/*[clinic end generated code: output=4333136e88f90d8b input=a16dbb33363c0fa8]*/ { - /* BEGIN GIL-protected critical section */ - if (PyList_Append(self->lst, item) < 0) - return NULL; - if (self->locked) { - /* A get() may be waiting, wake it up */ - self->locked = 0; - PyThread_release_lock(self->lock); + HandoffData data = { + .handed_off = 0, + .item = Py_NewRef(item), + .queue = self, + }; + if (self->has_threads_waiting) { + // Try to hand the item off directly if there are threads waiting + _PyParkingLot_Unpark(&self->has_threads_waiting, + (_Py_unpark_fn_t *)maybe_handoff_item, &data); + } + if (!data.handed_off) { + if (RingBuf_Put(&self->buf, item) < 0) { + return NULL; + } } - /* END GIL-protected critical section */ Py_RETURN_NONE; } /*[clinic input] +@critical_section _queue.SimpleQueue.put_nowait item: object @@ -150,39 +331,23 @@ for compatibility with the Queue class. static PyObject * _queue_SimpleQueue_put_nowait_impl(simplequeueobject *self, PyObject *item) -/*[clinic end generated code: output=0990536715efb1f1 input=36b1ea96756b2ece]*/ +/*[clinic end generated code: output=0990536715efb1f1 input=ce949cc2cd8a4119]*/ { return _queue_SimpleQueue_put_impl(self, item, 0, Py_None); } static PyObject * -simplequeue_pop_item(simplequeueobject *self) +empty_error(PyTypeObject *cls) { - Py_ssize_t count, n; - PyObject *item; - - n = PyList_GET_SIZE(self->lst); - assert(self->lst_pos < n); - - item = PyList_GET_ITEM(self->lst, self->lst_pos); - Py_INCREF(Py_None); - PyList_SET_ITEM(self->lst, self->lst_pos, Py_None); - self->lst_pos += 1; - count = n - self->lst_pos; - if (self->lst_pos > count) { - /* The list is more than 50% empty, reclaim space at the beginning */ - if (PyList_SetSlice(self->lst, 0, self->lst_pos, NULL)) { - /* Undo pop */ - self->lst_pos -= 1; - PyList_SET_ITEM(self->lst, self->lst_pos, item); - return NULL; - } - self->lst_pos = 0; - } - return item; + PyObject *module = PyType_GetModule(cls); + assert(module != NULL); + simplequeue_state *state = simplequeue_get_state(module); + PyErr_SetNone(state->EmptyError); + return NULL; } /*[clinic input] +@critical_section _queue.SimpleQueue.get cls: defining_class @@ -205,23 +370,15 @@ in that case). static PyObject * _queue_SimpleQueue_get_impl(simplequeueobject *self, PyTypeObject *cls, int block, PyObject *timeout_obj) -/*[clinic end generated code: output=5c2cca914cd1e55b input=5b4047bfbc645ec1]*/ +/*[clinic end generated code: output=5c2cca914cd1e55b input=f7836c65e5839c51]*/ { _PyTime_t endtime = 0; - _PyTime_t timeout; - PyObject *item; - PyLockStatus r; - PY_TIMEOUT_T microseconds; - PyThreadState *tstate = PyThreadState_Get(); // XXX Use PyThread_ParseTimeoutArg(). - if (block == 0) { - /* Non-blocking */ - microseconds = 0; - } - else if (timeout_obj != Py_None) { + if (block != 0 && !Py_IsNone(timeout_obj)) { /* With timeout */ + _PyTime_t timeout; if (_PyTime_FromSecondsObject(&timeout, timeout_obj, _PyTime_ROUND_CEILING) < 0) { return NULL; @@ -231,66 +388,64 @@ _queue_SimpleQueue_get_impl(simplequeueobject *self, PyTypeObject *cls, "'timeout' must be a non-negative number"); return NULL; } - microseconds = _PyTime_AsMicroseconds(timeout, - _PyTime_ROUND_CEILING); - if (microseconds > PY_TIMEOUT_MAX) { - PyErr_SetString(PyExc_OverflowError, - "timeout value is too large"); - return NULL; - } endtime = _PyDeadline_Init(timeout); } - else { - /* Infinitely blocking */ - microseconds = -1; - } - /* put() signals the queue to be non-empty by releasing the lock. - * So we simply try to acquire the lock in a loop, until the condition - * (queue non-empty) becomes true. - */ - while (self->lst_pos == PyList_GET_SIZE(self->lst)) { - /* First a simple non-blocking try without releasing the GIL */ - r = PyThread_acquire_lock_timed(self->lock, 0, 0); - if (r == PY_LOCK_FAILURE && microseconds != 0) { - Py_BEGIN_ALLOW_THREADS - r = PyThread_acquire_lock_timed(self->lock, microseconds, 1); - Py_END_ALLOW_THREADS + for (;;) { + if (!RingBuf_IsEmpty(&self->buf)) { + return RingBuf_Get(&self->buf); } - if (r == PY_LOCK_INTR && _PyEval_MakePendingCalls(tstate) < 0) { - return NULL; + if (!block) { + return empty_error(cls); } - if (r == PY_LOCK_FAILURE) { - PyObject *module = PyType_GetModule(cls); - simplequeue_state *state = simplequeue_get_state(module); - /* Timed out */ - PyErr_SetNone(state->EmptyError); - return NULL; - } - self->locked = 1; - /* Adjust timeout for next iteration (if any) */ - if (microseconds > 0) { - timeout = _PyDeadline_Get(endtime); - microseconds = _PyTime_AsMicroseconds(timeout, - _PyTime_ROUND_CEILING); + int64_t timeout_ns = -1; + if (endtime != 0) { + timeout_ns = _PyDeadline_Get(endtime); + if (timeout_ns < 0) { + return empty_error(cls); + } } - } - /* BEGIN GIL-protected critical section */ - assert(self->lst_pos < PyList_GET_SIZE(self->lst)); - item = simplequeue_pop_item(self); - if (self->locked) { - PyThread_release_lock(self->lock); - self->locked = 0; + bool waiting = 1; + self->has_threads_waiting = waiting; + + PyObject *item = NULL; + int st = _PyParkingLot_Park(&self->has_threads_waiting, &waiting, + sizeof(bool), timeout_ns, &item, + /* detach */ 1); + switch (st) { + case Py_PARK_OK: { + assert(item != NULL); + return item; + } + case Py_PARK_TIMEOUT: { + return empty_error(cls); + } + case Py_PARK_INTR: { + // Interrupted + if (Py_MakePendingCalls() < 0) { + return NULL; + } + break; + } + case Py_PARK_AGAIN: { + // This should be impossible with the current implementation of + // PyParkingLot, but would be possible if critical sections / + // the GIL were released before the thread was added to the + // internal thread queue in the parking lot. + break; + } + default: { + Py_UNREACHABLE(); + } + } } - /* END GIL-protected critical section */ - - return item; } /*[clinic input] +@critical_section _queue.SimpleQueue.get_nowait cls: defining_class @@ -305,12 +460,13 @@ raise the Empty exception. static PyObject * _queue_SimpleQueue_get_nowait_impl(simplequeueobject *self, PyTypeObject *cls) -/*[clinic end generated code: output=620c58e2750f8b8a input=842f732bf04216d3]*/ +/*[clinic end generated code: output=620c58e2750f8b8a input=d48be63633fefae9]*/ { return _queue_SimpleQueue_get_impl(self, cls, 0, Py_None); } /*[clinic input] +@critical_section _queue.SimpleQueue.empty -> bool Return True if the queue is empty, False otherwise (not reliable!). @@ -318,12 +474,13 @@ Return True if the queue is empty, False otherwise (not reliable!). static int _queue_SimpleQueue_empty_impl(simplequeueobject *self) -/*[clinic end generated code: output=1a02a1b87c0ef838 input=1a98431c45fd66f9]*/ +/*[clinic end generated code: output=1a02a1b87c0ef838 input=96cb22df5a67d831]*/ { - return self->lst_pos == PyList_GET_SIZE(self->lst); + return RingBuf_IsEmpty(&self->buf); } /*[clinic input] +@critical_section _queue.SimpleQueue.qsize -> Py_ssize_t Return the approximate size of the queue (not reliable!). @@ -331,9 +488,9 @@ Return the approximate size of the queue (not reliable!). static Py_ssize_t _queue_SimpleQueue_qsize_impl(simplequeueobject *self) -/*[clinic end generated code: output=f9dcd9d0a90e121e input=7a74852b407868a1]*/ +/*[clinic end generated code: output=f9dcd9d0a90e121e input=e218623cb8c16a79]*/ { - return PyList_GET_SIZE(self->lst) - self->lst_pos; + return RingBuf_Len(&self->buf); } static int diff --git a/Modules/_sqlite/clinic/connection.c.h b/Modules/_sqlite/clinic/connection.c.h index db5eb77891e52e..811314b5cd8aed 100644 --- a/Modules/_sqlite/clinic/connection.c.h +++ b/Modules/_sqlite/clinic/connection.c.h @@ -1204,21 +1204,67 @@ pysqlite_connection_interrupt(pysqlite_Connection *self, PyObject *Py_UNUSED(ign } PyDoc_STRVAR(pysqlite_connection_iterdump__doc__, -"iterdump($self, /)\n" +"iterdump($self, /, *, filter=None)\n" "--\n" "\n" -"Returns iterator to the dump of the database in an SQL text format."); +"Returns iterator to the dump of the database in an SQL text format.\n" +"\n" +" filter\n" +" An optional LIKE pattern for database objects to dump"); #define PYSQLITE_CONNECTION_ITERDUMP_METHODDEF \ - {"iterdump", (PyCFunction)pysqlite_connection_iterdump, METH_NOARGS, pysqlite_connection_iterdump__doc__}, + {"iterdump", _PyCFunction_CAST(pysqlite_connection_iterdump), METH_FASTCALL|METH_KEYWORDS, pysqlite_connection_iterdump__doc__}, static PyObject * -pysqlite_connection_iterdump_impl(pysqlite_Connection *self); +pysqlite_connection_iterdump_impl(pysqlite_Connection *self, + PyObject *filter); static PyObject * -pysqlite_connection_iterdump(pysqlite_Connection *self, PyObject *Py_UNUSED(ignored)) +pysqlite_connection_iterdump(pysqlite_Connection *self, PyObject *const *args, Py_ssize_t nargs, PyObject *kwnames) { - return pysqlite_connection_iterdump_impl(self); + PyObject *return_value = NULL; + #if defined(Py_BUILD_CORE) && !defined(Py_BUILD_CORE_MODULE) + + #define NUM_KEYWORDS 1 + static struct { + PyGC_Head _this_is_not_used; + PyObject_VAR_HEAD + PyObject *ob_item[NUM_KEYWORDS]; + } _kwtuple = { + .ob_base = PyVarObject_HEAD_INIT(&PyTuple_Type, NUM_KEYWORDS) + .ob_item = { &_Py_ID(filter), }, + }; + #undef NUM_KEYWORDS + #define KWTUPLE (&_kwtuple.ob_base.ob_base) + + #else // !Py_BUILD_CORE + # define KWTUPLE NULL + #endif // !Py_BUILD_CORE + + static const char * const _keywords[] = {"filter", NULL}; + static _PyArg_Parser _parser = { + .keywords = _keywords, + .fname = "iterdump", + .kwtuple = KWTUPLE, + }; + #undef KWTUPLE + PyObject *argsbuf[1]; + Py_ssize_t noptargs = nargs + (kwnames ? PyTuple_GET_SIZE(kwnames) : 0) - 0; + PyObject *filter = Py_None; + + args = _PyArg_UnpackKeywords(args, nargs, NULL, kwnames, &_parser, 0, 0, 0, argsbuf); + if (!args) { + goto exit; + } + if (!noptargs) { + goto skip_optional_kwonly; + } + filter = args[0]; +skip_optional_kwonly: + return_value = pysqlite_connection_iterdump_impl(self, filter); + +exit: + return return_value; } PyDoc_STRVAR(pysqlite_connection_backup__doc__, @@ -1551,7 +1597,9 @@ deserialize(pysqlite_Connection *self, PyObject *const *args, Py_ssize_t nargs, if (ptr == NULL) { goto exit; } - PyBuffer_FillInfo(&data, args[0], (void *)ptr, len, 1, 0); + if (PyBuffer_FillInfo(&data, args[0], (void *)ptr, len, 1, PyBUF_SIMPLE) < 0) { + goto exit; + } } else { /* any bytes-like object */ if (PyObject_GetBuffer(args[0], &data, PyBUF_SIMPLE) != 0) { @@ -1818,4 +1866,4 @@ getconfig(pysqlite_Connection *self, PyObject *arg) #ifndef DESERIALIZE_METHODDEF #define DESERIALIZE_METHODDEF #endif /* !defined(DESERIALIZE_METHODDEF) */ -/*[clinic end generated code: output=90b5b9c14261b8d7 input=a9049054013a1b77]*/ +/*[clinic end generated code: output=3c6d0b748fac016f input=a9049054013a1b77]*/ diff --git a/Modules/_sqlite/connection.c b/Modules/_sqlite/connection.c index 0a6633972cc5ef..f97afcf5fcf16e 100644 --- a/Modules/_sqlite/connection.c +++ b/Modules/_sqlite/connection.c @@ -1979,12 +1979,17 @@ pysqlite_connection_interrupt_impl(pysqlite_Connection *self) /*[clinic input] _sqlite3.Connection.iterdump as pysqlite_connection_iterdump + * + filter: object = None + An optional LIKE pattern for database objects to dump + Returns iterator to the dump of the database in an SQL text format. [clinic start generated code]*/ static PyObject * -pysqlite_connection_iterdump_impl(pysqlite_Connection *self) -/*[clinic end generated code: output=586997aaf9808768 input=1911ca756066da89]*/ +pysqlite_connection_iterdump_impl(pysqlite_Connection *self, + PyObject *filter) +/*[clinic end generated code: output=fd81069c4bdeb6b0 input=4ae6d9a898f108df]*/ { if (!pysqlite_check_connection(self)) { return NULL; @@ -1998,9 +2003,16 @@ pysqlite_connection_iterdump_impl(pysqlite_Connection *self) } return NULL; } - - PyObject *retval = PyObject_CallOneArg(iterdump, (PyObject *)self); + PyObject *args[3] = {NULL, (PyObject *)self, filter}; + PyObject *kwnames = Py_BuildValue("(s)", "filter"); + if (!kwnames) { + Py_DECREF(iterdump); + return NULL; + } + Py_ssize_t nargsf = 1 | PY_VECTORCALL_ARGUMENTS_OFFSET; + PyObject *retval = PyObject_Vectorcall(iterdump, args + 1, nargsf, kwnames); Py_DECREF(iterdump); + Py_DECREF(kwnames); return retval; } diff --git a/Modules/_sre/clinic/sre.c.h b/Modules/_sre/clinic/sre.c.h index cd3fbbc720bdf1..48336c7a2fca26 100644 --- a/Modules/_sre/clinic/sre.c.h +++ b/Modules/_sre/clinic/sre.c.h @@ -1434,7 +1434,7 @@ _sre_SRE_Scanner_match_impl(ScannerObject *self, PyTypeObject *cls); static PyObject * _sre_SRE_Scanner_match(ScannerObject *self, PyTypeObject *cls, PyObject *const *args, Py_ssize_t nargs, PyObject *kwnames) { - if (nargs) { + if (nargs || (kwnames && PyTuple_GET_SIZE(kwnames))) { PyErr_SetString(PyExc_TypeError, "match() takes no arguments"); return NULL; } @@ -1455,10 +1455,10 @@ _sre_SRE_Scanner_search_impl(ScannerObject *self, PyTypeObject *cls); static PyObject * _sre_SRE_Scanner_search(ScannerObject *self, PyTypeObject *cls, PyObject *const *args, Py_ssize_t nargs, PyObject *kwnames) { - if (nargs) { + if (nargs || (kwnames && PyTuple_GET_SIZE(kwnames))) { PyErr_SetString(PyExc_TypeError, "search() takes no arguments"); return NULL; } return _sre_SRE_Scanner_search_impl(self, cls); } -/*[clinic end generated code: output=ad513f31b99505fa input=a9049054013a1b77]*/ +/*[clinic end generated code: output=c3e711f0b2f43d66 input=a9049054013a1b77]*/ diff --git a/Modules/_sre/sre.c b/Modules/_sre/sre.c index d451974b9cf81e..00fbd9674b8cdd 100644 --- a/Modules/_sre/sre.c +++ b/Modules/_sre/sre.c @@ -39,13 +39,14 @@ static const char copyright[] = " SRE 2.2.2 Copyright (c) 1997-2002 by Secret Labs AB "; #include "Python.h" -#include "pycore_dict.h" // _PyDict_Next() -#include "pycore_long.h" // _PyLong_GetZero() -#include "pycore_moduleobject.h" // _PyModule_GetState() +#include "pycore_critical_section.h" // Py_BEGIN_CRITICAL_SECTION +#include "pycore_dict.h" // _PyDict_Next() +#include "pycore_long.h" // _PyLong_GetZero() +#include "pycore_moduleobject.h" // _PyModule_GetState() -#include "sre.h" // SRE_CODE +#include "sre.h" // SRE_CODE -#include // tolower(), toupper(), isalnum() +#include // tolower(), toupper(), isalnum() #define SRE_CODE_BITS (8 * sizeof(SRE_CODE)) @@ -2349,26 +2350,28 @@ _sre_SRE_Match_groupdict_impl(MatchObject *self, PyObject *default_value) if (!result || !self->pattern->groupindex) return result; + Py_BEGIN_CRITICAL_SECTION(self->pattern->groupindex); while (_PyDict_Next(self->pattern->groupindex, &pos, &key, &value, &hash)) { int status; Py_INCREF(key); value = match_getslice(self, key, default_value); if (!value) { Py_DECREF(key); - goto failed; + Py_CLEAR(result); + goto exit; } status = _PyDict_SetItem_KnownHash(result, key, value, hash); Py_DECREF(value); Py_DECREF(key); - if (status < 0) - goto failed; + if (status < 0) { + Py_CLEAR(result); + goto exit; + } } +exit: + Py_END_CRITICAL_SECTION(); return result; - -failed: - Py_DECREF(result); - return NULL; } /*[clinic input] diff --git a/Modules/_sre/sre_lib.h b/Modules/_sre/sre_lib.h index f5497d9ff2b93f..97fbb0a75e54b6 100644 --- a/Modules/_sre/sre_lib.h +++ b/Modules/_sre/sre_lib.h @@ -1122,7 +1122,7 @@ SRE(match)(SRE_STATE* state, const SRE_CODE* pattern, int toplevel) /* install new repeat context */ /* TODO(https://github.com/python/cpython/issues/67877): Fix this * potential memory leak. */ - ctx->u.rep = (SRE_REPEAT*) PyObject_Malloc(sizeof(*ctx->u.rep)); + ctx->u.rep = (SRE_REPEAT*) PyMem_Malloc(sizeof(*ctx->u.rep)); if (!ctx->u.rep) { PyErr_NoMemory(); RETURN_FAILURE; @@ -1136,7 +1136,7 @@ SRE(match)(SRE_STATE* state, const SRE_CODE* pattern, int toplevel) state->ptr = ptr; DO_JUMP(JUMP_REPEAT, jump_repeat, pattern+pattern[0]); state->repeat = ctx->u.rep->prev; - PyObject_Free(ctx->u.rep); + PyMem_Free(ctx->u.rep); if (ret) { RETURN_ON_ERROR(ret); diff --git a/Modules/_ssl.c b/Modules/_ssl.c index 90b600f4b776a3..bc302909424227 100644 --- a/Modules/_ssl.c +++ b/Modules/_ssl.c @@ -125,10 +125,10 @@ static void _PySSLFixErrno(void) { #include "_ssl_data_31.h" #elif (OPENSSL_VERSION_NUMBER >= 0x30000000L) #include "_ssl_data_300.h" -#elif (OPENSSL_VERSION_NUMBER >= 0x10101000L) && !defined(LIBRESSL_VERSION_NUMBER) +#elif (OPENSSL_VERSION_NUMBER >= 0x10101000L) #include "_ssl_data_111.h" #else -#include "_ssl_data.h" +#error Unsupported OpenSSL version #endif /* OpenSSL API 1.1.0+ does not include version methods */ @@ -867,7 +867,7 @@ newPySSLSocket(PySSLContext *sslctx, PySocketSockObject *sock, } /* bpo43522 and OpenSSL < 1.1.1l: copy hostflags manually */ -#if !defined(LIBRESSL_VERSION_NUMBER) && OPENSSL_VERSION < 0x101010cf +#if OPENSSL_VERSION < 0x101010cf X509_VERIFY_PARAM *ssl_params = SSL_get0_param(self->ssl); X509_VERIFY_PARAM_set_hostflags(ssl_params, sslctx->hostflags); #endif @@ -893,10 +893,8 @@ newPySSLSocket(PySSLContext *sslctx, PySocketSockObject *sock, * only in combination with SSL_VERIFY_PEER flag. */ int mode = SSL_get_verify_mode(self->ssl); if (mode & SSL_VERIFY_PEER) { - int (*verify_cb)(int, X509_STORE_CTX *) = NULL; - verify_cb = SSL_get_verify_callback(self->ssl); mode |= SSL_VERIFY_POST_HANDSHAKE; - SSL_set_verify(self->ssl, mode, verify_cb); + SSL_set_verify(self->ssl, mode, NULL); } } else { /* client socket */ @@ -2997,7 +2995,6 @@ static int _set_verify_mode(PySSLContext *self, enum py_ssl_cert_requirements n) { int mode; - int (*verify_cb)(int, X509_STORE_CTX *) = NULL; switch(n) { case PY_SSL_CERT_NONE: @@ -3018,9 +3015,7 @@ _set_verify_mode(PySSLContext *self, enum py_ssl_cert_requirements n) /* bpo-37428: newPySSLSocket() sets SSL_VERIFY_POST_HANDSHAKE flag for * server sockets and SSL_set_post_handshake_auth() for client. */ - /* keep current verify cb */ - verify_cb = SSL_CTX_get_verify_callback(self->ctx); - SSL_CTX_set_verify(self->ctx, mode, verify_cb); + SSL_CTX_set_verify(self->ctx, mode, NULL); return 0; } diff --git a/Modules/_ssl_data.h b/Modules/_ssl_data.h deleted file mode 100644 index 8f2994f52dfef8..00000000000000 --- a/Modules/_ssl_data.h +++ /dev/null @@ -1,6323 +0,0 @@ -/* File generated by Tools/ssl/make_ssl_data.py */ -/* Generated on 2020-04-13T21:45:54.559159 */ - -static struct py_ssl_library_code library_codes[] = { -#ifdef ERR_LIB_ASN1 - {"ASN1", ERR_LIB_ASN1}, -#endif -#ifdef ERR_LIB_ASYNC - {"ASYNC", ERR_LIB_ASYNC}, -#endif -#ifdef ERR_LIB_BIO - {"BIO", ERR_LIB_BIO}, -#endif -#ifdef ERR_LIB_BN - {"BN", ERR_LIB_BN}, -#endif -#ifdef ERR_LIB_CMS - {"CMS", ERR_LIB_CMS}, -#endif -#ifdef ERR_LIB_COMP - {"COMP", ERR_LIB_COMP}, -#endif -#ifdef ERR_LIB_CONF - {"CONF", ERR_LIB_CONF}, -#endif -#ifdef ERR_LIB_CRYPTO - {"CRYPTO", ERR_LIB_CRYPTO}, -#endif -#ifdef ERR_LIB_CT - {"CT", ERR_LIB_CT}, -#endif -#ifdef ERR_LIB_DH - {"DH", ERR_LIB_DH}, -#endif -#ifdef ERR_LIB_DSA - {"DSA", ERR_LIB_DSA}, -#endif -#ifdef ERR_LIB_EC - {"EC", ERR_LIB_EC}, -#endif -#ifdef ERR_LIB_ENGINE - {"ENGINE", ERR_LIB_ENGINE}, -#endif -#ifdef ERR_LIB_EVP - {"EVP", ERR_LIB_EVP}, -#endif -#ifdef ERR_LIB_KDF - {"KDF", ERR_LIB_KDF}, -#endif -#ifdef ERR_LIB_OCSP - {"OCSP", ERR_LIB_OCSP}, -#endif -#ifdef ERR_LIB_PEM - {"PEM", ERR_LIB_PEM}, -#endif -#ifdef ERR_LIB_PKCS12 - {"PKCS12", ERR_LIB_PKCS12}, -#endif -#ifdef ERR_LIB_PKCS7 - {"PKCS7", ERR_LIB_PKCS7}, -#endif -#ifdef ERR_LIB_RAND - {"RAND", ERR_LIB_RAND}, -#endif -#ifdef ERR_LIB_RSA - {"RSA", ERR_LIB_RSA}, -#endif -#ifdef ERR_LIB_SSL - {"SSL", ERR_LIB_SSL}, -#endif -#ifdef ERR_LIB_TS - {"TS", ERR_LIB_TS}, -#endif -#ifdef ERR_LIB_UI - {"UI", ERR_LIB_UI}, -#endif -#ifdef ERR_LIB_X509 - {"X509", ERR_LIB_X509}, -#endif -#ifdef ERR_LIB_X509V3 - {"X509V3", ERR_LIB_X509V3}, -#endif - { NULL } -}; - -static struct py_ssl_error_code error_codes[] = { - #ifdef ASN1_R_ADDING_OBJECT - {"ADDING_OBJECT", ERR_LIB_ASN1, ASN1_R_ADDING_OBJECT}, - #else - {"ADDING_OBJECT", 13, 171}, - #endif - #ifdef ASN1_R_ASN1_PARSE_ERROR - {"ASN1_PARSE_ERROR", ERR_LIB_ASN1, ASN1_R_ASN1_PARSE_ERROR}, - #else - {"ASN1_PARSE_ERROR", 13, 203}, - #endif - #ifdef ASN1_R_ASN1_SIG_PARSE_ERROR - {"ASN1_SIG_PARSE_ERROR", ERR_LIB_ASN1, ASN1_R_ASN1_SIG_PARSE_ERROR}, - #else - {"ASN1_SIG_PARSE_ERROR", 13, 204}, - #endif - #ifdef ASN1_R_AUX_ERROR - {"AUX_ERROR", ERR_LIB_ASN1, ASN1_R_AUX_ERROR}, - #else - {"AUX_ERROR", 13, 100}, - #endif - #ifdef ASN1_R_BAD_OBJECT_HEADER - {"BAD_OBJECT_HEADER", ERR_LIB_ASN1, ASN1_R_BAD_OBJECT_HEADER}, - #else - {"BAD_OBJECT_HEADER", 13, 102}, - #endif - #ifdef ASN1_R_BMPSTRING_IS_WRONG_LENGTH - {"BMPSTRING_IS_WRONG_LENGTH", ERR_LIB_ASN1, ASN1_R_BMPSTRING_IS_WRONG_LENGTH}, - #else - {"BMPSTRING_IS_WRONG_LENGTH", 13, 214}, - #endif - #ifdef ASN1_R_BN_LIB - {"BN_LIB", ERR_LIB_ASN1, ASN1_R_BN_LIB}, - #else - {"BN_LIB", 13, 105}, - #endif - #ifdef ASN1_R_BOOLEAN_IS_WRONG_LENGTH - {"BOOLEAN_IS_WRONG_LENGTH", ERR_LIB_ASN1, ASN1_R_BOOLEAN_IS_WRONG_LENGTH}, - #else - {"BOOLEAN_IS_WRONG_LENGTH", 13, 106}, - #endif - #ifdef ASN1_R_BUFFER_TOO_SMALL - {"BUFFER_TOO_SMALL", ERR_LIB_ASN1, ASN1_R_BUFFER_TOO_SMALL}, - #else - {"BUFFER_TOO_SMALL", 13, 107}, - #endif - #ifdef ASN1_R_CIPHER_HAS_NO_OBJECT_IDENTIFIER - {"CIPHER_HAS_NO_OBJECT_IDENTIFIER", ERR_LIB_ASN1, ASN1_R_CIPHER_HAS_NO_OBJECT_IDENTIFIER}, - #else - {"CIPHER_HAS_NO_OBJECT_IDENTIFIER", 13, 108}, - #endif - #ifdef ASN1_R_CONTEXT_NOT_INITIALISED - {"CONTEXT_NOT_INITIALISED", ERR_LIB_ASN1, ASN1_R_CONTEXT_NOT_INITIALISED}, - #else - {"CONTEXT_NOT_INITIALISED", 13, 217}, - #endif - #ifdef ASN1_R_DATA_IS_WRONG - {"DATA_IS_WRONG", ERR_LIB_ASN1, ASN1_R_DATA_IS_WRONG}, - #else - {"DATA_IS_WRONG", 13, 109}, - #endif - #ifdef ASN1_R_DECODE_ERROR - {"DECODE_ERROR", ERR_LIB_ASN1, ASN1_R_DECODE_ERROR}, - #else - {"DECODE_ERROR", 13, 110}, - #endif - #ifdef ASN1_R_DEPTH_EXCEEDED - {"DEPTH_EXCEEDED", ERR_LIB_ASN1, ASN1_R_DEPTH_EXCEEDED}, - #else - {"DEPTH_EXCEEDED", 13, 174}, - #endif - #ifdef ASN1_R_DIGEST_AND_KEY_TYPE_NOT_SUPPORTED - {"DIGEST_AND_KEY_TYPE_NOT_SUPPORTED", ERR_LIB_ASN1, ASN1_R_DIGEST_AND_KEY_TYPE_NOT_SUPPORTED}, - #else - {"DIGEST_AND_KEY_TYPE_NOT_SUPPORTED", 13, 198}, - #endif - #ifdef ASN1_R_ENCODE_ERROR - {"ENCODE_ERROR", ERR_LIB_ASN1, ASN1_R_ENCODE_ERROR}, - #else - {"ENCODE_ERROR", 13, 112}, - #endif - #ifdef ASN1_R_ERROR_GETTING_TIME - {"ERROR_GETTING_TIME", ERR_LIB_ASN1, ASN1_R_ERROR_GETTING_TIME}, - #else - {"ERROR_GETTING_TIME", 13, 173}, - #endif - #ifdef ASN1_R_ERROR_LOADING_SECTION - {"ERROR_LOADING_SECTION", ERR_LIB_ASN1, ASN1_R_ERROR_LOADING_SECTION}, - #else - {"ERROR_LOADING_SECTION", 13, 172}, - #endif - #ifdef ASN1_R_ERROR_SETTING_CIPHER_PARAMS - {"ERROR_SETTING_CIPHER_PARAMS", ERR_LIB_ASN1, ASN1_R_ERROR_SETTING_CIPHER_PARAMS}, - #else - {"ERROR_SETTING_CIPHER_PARAMS", 13, 114}, - #endif - #ifdef ASN1_R_EXPECTING_AN_INTEGER - {"EXPECTING_AN_INTEGER", ERR_LIB_ASN1, ASN1_R_EXPECTING_AN_INTEGER}, - #else - {"EXPECTING_AN_INTEGER", 13, 115}, - #endif - #ifdef ASN1_R_EXPECTING_AN_OBJECT - {"EXPECTING_AN_OBJECT", ERR_LIB_ASN1, ASN1_R_EXPECTING_AN_OBJECT}, - #else - {"EXPECTING_AN_OBJECT", 13, 116}, - #endif - #ifdef ASN1_R_EXPLICIT_LENGTH_MISMATCH - {"EXPLICIT_LENGTH_MISMATCH", ERR_LIB_ASN1, ASN1_R_EXPLICIT_LENGTH_MISMATCH}, - #else - {"EXPLICIT_LENGTH_MISMATCH", 13, 119}, - #endif - #ifdef ASN1_R_EXPLICIT_TAG_NOT_CONSTRUCTED - {"EXPLICIT_TAG_NOT_CONSTRUCTED", ERR_LIB_ASN1, ASN1_R_EXPLICIT_TAG_NOT_CONSTRUCTED}, - #else - {"EXPLICIT_TAG_NOT_CONSTRUCTED", 13, 120}, - #endif - #ifdef ASN1_R_FIELD_MISSING - {"FIELD_MISSING", ERR_LIB_ASN1, ASN1_R_FIELD_MISSING}, - #else - {"FIELD_MISSING", 13, 121}, - #endif - #ifdef ASN1_R_FIRST_NUM_TOO_LARGE - {"FIRST_NUM_TOO_LARGE", ERR_LIB_ASN1, ASN1_R_FIRST_NUM_TOO_LARGE}, - #else - {"FIRST_NUM_TOO_LARGE", 13, 122}, - #endif - #ifdef ASN1_R_HEADER_TOO_LONG - {"HEADER_TOO_LONG", ERR_LIB_ASN1, ASN1_R_HEADER_TOO_LONG}, - #else - {"HEADER_TOO_LONG", 13, 123}, - #endif - #ifdef ASN1_R_ILLEGAL_BITSTRING_FORMAT - {"ILLEGAL_BITSTRING_FORMAT", ERR_LIB_ASN1, ASN1_R_ILLEGAL_BITSTRING_FORMAT}, - #else - {"ILLEGAL_BITSTRING_FORMAT", 13, 175}, - #endif - #ifdef ASN1_R_ILLEGAL_BOOLEAN - {"ILLEGAL_BOOLEAN", ERR_LIB_ASN1, ASN1_R_ILLEGAL_BOOLEAN}, - #else - {"ILLEGAL_BOOLEAN", 13, 176}, - #endif - #ifdef ASN1_R_ILLEGAL_CHARACTERS - {"ILLEGAL_CHARACTERS", ERR_LIB_ASN1, ASN1_R_ILLEGAL_CHARACTERS}, - #else - {"ILLEGAL_CHARACTERS", 13, 124}, - #endif - #ifdef ASN1_R_ILLEGAL_FORMAT - {"ILLEGAL_FORMAT", ERR_LIB_ASN1, ASN1_R_ILLEGAL_FORMAT}, - #else - {"ILLEGAL_FORMAT", 13, 177}, - #endif - #ifdef ASN1_R_ILLEGAL_HEX - {"ILLEGAL_HEX", ERR_LIB_ASN1, ASN1_R_ILLEGAL_HEX}, - #else - {"ILLEGAL_HEX", 13, 178}, - #endif - #ifdef ASN1_R_ILLEGAL_IMPLICIT_TAG - {"ILLEGAL_IMPLICIT_TAG", ERR_LIB_ASN1, ASN1_R_ILLEGAL_IMPLICIT_TAG}, - #else - {"ILLEGAL_IMPLICIT_TAG", 13, 179}, - #endif - #ifdef ASN1_R_ILLEGAL_INTEGER - {"ILLEGAL_INTEGER", ERR_LIB_ASN1, ASN1_R_ILLEGAL_INTEGER}, - #else - {"ILLEGAL_INTEGER", 13, 180}, - #endif - #ifdef ASN1_R_ILLEGAL_NEGATIVE_VALUE - {"ILLEGAL_NEGATIVE_VALUE", ERR_LIB_ASN1, ASN1_R_ILLEGAL_NEGATIVE_VALUE}, - #else - {"ILLEGAL_NEGATIVE_VALUE", 13, 226}, - #endif - #ifdef ASN1_R_ILLEGAL_NESTED_TAGGING - {"ILLEGAL_NESTED_TAGGING", ERR_LIB_ASN1, ASN1_R_ILLEGAL_NESTED_TAGGING}, - #else - {"ILLEGAL_NESTED_TAGGING", 13, 181}, - #endif - #ifdef ASN1_R_ILLEGAL_NULL - {"ILLEGAL_NULL", ERR_LIB_ASN1, ASN1_R_ILLEGAL_NULL}, - #else - {"ILLEGAL_NULL", 13, 125}, - #endif - #ifdef ASN1_R_ILLEGAL_NULL_VALUE - {"ILLEGAL_NULL_VALUE", ERR_LIB_ASN1, ASN1_R_ILLEGAL_NULL_VALUE}, - #else - {"ILLEGAL_NULL_VALUE", 13, 182}, - #endif - #ifdef ASN1_R_ILLEGAL_OBJECT - {"ILLEGAL_OBJECT", ERR_LIB_ASN1, ASN1_R_ILLEGAL_OBJECT}, - #else - {"ILLEGAL_OBJECT", 13, 183}, - #endif - #ifdef ASN1_R_ILLEGAL_OPTIONAL_ANY - {"ILLEGAL_OPTIONAL_ANY", ERR_LIB_ASN1, ASN1_R_ILLEGAL_OPTIONAL_ANY}, - #else - {"ILLEGAL_OPTIONAL_ANY", 13, 126}, - #endif - #ifdef ASN1_R_ILLEGAL_OPTIONS_ON_ITEM_TEMPLATE - {"ILLEGAL_OPTIONS_ON_ITEM_TEMPLATE", ERR_LIB_ASN1, ASN1_R_ILLEGAL_OPTIONS_ON_ITEM_TEMPLATE}, - #else - {"ILLEGAL_OPTIONS_ON_ITEM_TEMPLATE", 13, 170}, - #endif - #ifdef ASN1_R_ILLEGAL_PADDING - {"ILLEGAL_PADDING", ERR_LIB_ASN1, ASN1_R_ILLEGAL_PADDING}, - #else - {"ILLEGAL_PADDING", 13, 221}, - #endif - #ifdef ASN1_R_ILLEGAL_TAGGED_ANY - {"ILLEGAL_TAGGED_ANY", ERR_LIB_ASN1, ASN1_R_ILLEGAL_TAGGED_ANY}, - #else - {"ILLEGAL_TAGGED_ANY", 13, 127}, - #endif - #ifdef ASN1_R_ILLEGAL_TIME_VALUE - {"ILLEGAL_TIME_VALUE", ERR_LIB_ASN1, ASN1_R_ILLEGAL_TIME_VALUE}, - #else - {"ILLEGAL_TIME_VALUE", 13, 184}, - #endif - #ifdef ASN1_R_ILLEGAL_ZERO_CONTENT - {"ILLEGAL_ZERO_CONTENT", ERR_LIB_ASN1, ASN1_R_ILLEGAL_ZERO_CONTENT}, - #else - {"ILLEGAL_ZERO_CONTENT", 13, 222}, - #endif - #ifdef ASN1_R_INTEGER_NOT_ASCII_FORMAT - {"INTEGER_NOT_ASCII_FORMAT", ERR_LIB_ASN1, ASN1_R_INTEGER_NOT_ASCII_FORMAT}, - #else - {"INTEGER_NOT_ASCII_FORMAT", 13, 185}, - #endif - #ifdef ASN1_R_INTEGER_TOO_LARGE_FOR_LONG - {"INTEGER_TOO_LARGE_FOR_LONG", ERR_LIB_ASN1, ASN1_R_INTEGER_TOO_LARGE_FOR_LONG}, - #else - {"INTEGER_TOO_LARGE_FOR_LONG", 13, 128}, - #endif - #ifdef ASN1_R_INVALID_BIT_STRING_BITS_LEFT - {"INVALID_BIT_STRING_BITS_LEFT", ERR_LIB_ASN1, ASN1_R_INVALID_BIT_STRING_BITS_LEFT}, - #else - {"INVALID_BIT_STRING_BITS_LEFT", 13, 220}, - #endif - #ifdef ASN1_R_INVALID_BMPSTRING_LENGTH - {"INVALID_BMPSTRING_LENGTH", ERR_LIB_ASN1, ASN1_R_INVALID_BMPSTRING_LENGTH}, - #else - {"INVALID_BMPSTRING_LENGTH", 13, 129}, - #endif - #ifdef ASN1_R_INVALID_DIGIT - {"INVALID_DIGIT", ERR_LIB_ASN1, ASN1_R_INVALID_DIGIT}, - #else - {"INVALID_DIGIT", 13, 130}, - #endif - #ifdef ASN1_R_INVALID_MIME_TYPE - {"INVALID_MIME_TYPE", ERR_LIB_ASN1, ASN1_R_INVALID_MIME_TYPE}, - #else - {"INVALID_MIME_TYPE", 13, 205}, - #endif - #ifdef ASN1_R_INVALID_MODIFIER - {"INVALID_MODIFIER", ERR_LIB_ASN1, ASN1_R_INVALID_MODIFIER}, - #else - {"INVALID_MODIFIER", 13, 186}, - #endif - #ifdef ASN1_R_INVALID_NUMBER - {"INVALID_NUMBER", ERR_LIB_ASN1, ASN1_R_INVALID_NUMBER}, - #else - {"INVALID_NUMBER", 13, 187}, - #endif - #ifdef ASN1_R_INVALID_OBJECT_ENCODING - {"INVALID_OBJECT_ENCODING", ERR_LIB_ASN1, ASN1_R_INVALID_OBJECT_ENCODING}, - #else - {"INVALID_OBJECT_ENCODING", 13, 216}, - #endif - #ifdef ASN1_R_INVALID_SCRYPT_PARAMETERS - {"INVALID_SCRYPT_PARAMETERS", ERR_LIB_ASN1, ASN1_R_INVALID_SCRYPT_PARAMETERS}, - #else - {"INVALID_SCRYPT_PARAMETERS", 13, 227}, - #endif - #ifdef ASN1_R_INVALID_SEPARATOR - {"INVALID_SEPARATOR", ERR_LIB_ASN1, ASN1_R_INVALID_SEPARATOR}, - #else - {"INVALID_SEPARATOR", 13, 131}, - #endif - #ifdef ASN1_R_INVALID_STRING_TABLE_VALUE - {"INVALID_STRING_TABLE_VALUE", ERR_LIB_ASN1, ASN1_R_INVALID_STRING_TABLE_VALUE}, - #else - {"INVALID_STRING_TABLE_VALUE", 13, 218}, - #endif - #ifdef ASN1_R_INVALID_UNIVERSALSTRING_LENGTH - {"INVALID_UNIVERSALSTRING_LENGTH", ERR_LIB_ASN1, ASN1_R_INVALID_UNIVERSALSTRING_LENGTH}, - #else - {"INVALID_UNIVERSALSTRING_LENGTH", 13, 133}, - #endif - #ifdef ASN1_R_INVALID_UTF8STRING - {"INVALID_UTF8STRING", ERR_LIB_ASN1, ASN1_R_INVALID_UTF8STRING}, - #else - {"INVALID_UTF8STRING", 13, 134}, - #endif - #ifdef ASN1_R_INVALID_VALUE - {"INVALID_VALUE", ERR_LIB_ASN1, ASN1_R_INVALID_VALUE}, - #else - {"INVALID_VALUE", 13, 219}, - #endif - #ifdef ASN1_R_LIST_ERROR - {"LIST_ERROR", ERR_LIB_ASN1, ASN1_R_LIST_ERROR}, - #else - {"LIST_ERROR", 13, 188}, - #endif - #ifdef ASN1_R_MIME_NO_CONTENT_TYPE - {"MIME_NO_CONTENT_TYPE", ERR_LIB_ASN1, ASN1_R_MIME_NO_CONTENT_TYPE}, - #else - {"MIME_NO_CONTENT_TYPE", 13, 206}, - #endif - #ifdef ASN1_R_MIME_PARSE_ERROR - {"MIME_PARSE_ERROR", ERR_LIB_ASN1, ASN1_R_MIME_PARSE_ERROR}, - #else - {"MIME_PARSE_ERROR", 13, 207}, - #endif - #ifdef ASN1_R_MIME_SIG_PARSE_ERROR - {"MIME_SIG_PARSE_ERROR", ERR_LIB_ASN1, ASN1_R_MIME_SIG_PARSE_ERROR}, - #else - {"MIME_SIG_PARSE_ERROR", 13, 208}, - #endif - #ifdef ASN1_R_MISSING_EOC - {"MISSING_EOC", ERR_LIB_ASN1, ASN1_R_MISSING_EOC}, - #else - {"MISSING_EOC", 13, 137}, - #endif - #ifdef ASN1_R_MISSING_SECOND_NUMBER - {"MISSING_SECOND_NUMBER", ERR_LIB_ASN1, ASN1_R_MISSING_SECOND_NUMBER}, - #else - {"MISSING_SECOND_NUMBER", 13, 138}, - #endif - #ifdef ASN1_R_MISSING_VALUE - {"MISSING_VALUE", ERR_LIB_ASN1, ASN1_R_MISSING_VALUE}, - #else - {"MISSING_VALUE", 13, 189}, - #endif - #ifdef ASN1_R_MSTRING_NOT_UNIVERSAL - {"MSTRING_NOT_UNIVERSAL", ERR_LIB_ASN1, ASN1_R_MSTRING_NOT_UNIVERSAL}, - #else - {"MSTRING_NOT_UNIVERSAL", 13, 139}, - #endif - #ifdef ASN1_R_MSTRING_WRONG_TAG - {"MSTRING_WRONG_TAG", ERR_LIB_ASN1, ASN1_R_MSTRING_WRONG_TAG}, - #else - {"MSTRING_WRONG_TAG", 13, 140}, - #endif - #ifdef ASN1_R_NESTED_ASN1_STRING - {"NESTED_ASN1_STRING", ERR_LIB_ASN1, ASN1_R_NESTED_ASN1_STRING}, - #else - {"NESTED_ASN1_STRING", 13, 197}, - #endif - #ifdef ASN1_R_NESTED_TOO_DEEP - {"NESTED_TOO_DEEP", ERR_LIB_ASN1, ASN1_R_NESTED_TOO_DEEP}, - #else - {"NESTED_TOO_DEEP", 13, 201}, - #endif - #ifdef ASN1_R_NON_HEX_CHARACTERS - {"NON_HEX_CHARACTERS", ERR_LIB_ASN1, ASN1_R_NON_HEX_CHARACTERS}, - #else - {"NON_HEX_CHARACTERS", 13, 141}, - #endif - #ifdef ASN1_R_NOT_ASCII_FORMAT - {"NOT_ASCII_FORMAT", ERR_LIB_ASN1, ASN1_R_NOT_ASCII_FORMAT}, - #else - {"NOT_ASCII_FORMAT", 13, 190}, - #endif - #ifdef ASN1_R_NOT_ENOUGH_DATA - {"NOT_ENOUGH_DATA", ERR_LIB_ASN1, ASN1_R_NOT_ENOUGH_DATA}, - #else - {"NOT_ENOUGH_DATA", 13, 142}, - #endif - #ifdef ASN1_R_NO_CONTENT_TYPE - {"NO_CONTENT_TYPE", ERR_LIB_ASN1, ASN1_R_NO_CONTENT_TYPE}, - #else - {"NO_CONTENT_TYPE", 13, 209}, - #endif - #ifdef ASN1_R_NO_MATCHING_CHOICE_TYPE - {"NO_MATCHING_CHOICE_TYPE", ERR_LIB_ASN1, ASN1_R_NO_MATCHING_CHOICE_TYPE}, - #else - {"NO_MATCHING_CHOICE_TYPE", 13, 143}, - #endif - #ifdef ASN1_R_NO_MULTIPART_BODY_FAILURE - {"NO_MULTIPART_BODY_FAILURE", ERR_LIB_ASN1, ASN1_R_NO_MULTIPART_BODY_FAILURE}, - #else - {"NO_MULTIPART_BODY_FAILURE", 13, 210}, - #endif - #ifdef ASN1_R_NO_MULTIPART_BOUNDARY - {"NO_MULTIPART_BOUNDARY", ERR_LIB_ASN1, ASN1_R_NO_MULTIPART_BOUNDARY}, - #else - {"NO_MULTIPART_BOUNDARY", 13, 211}, - #endif - #ifdef ASN1_R_NO_SIG_CONTENT_TYPE - {"NO_SIG_CONTENT_TYPE", ERR_LIB_ASN1, ASN1_R_NO_SIG_CONTENT_TYPE}, - #else - {"NO_SIG_CONTENT_TYPE", 13, 212}, - #endif - #ifdef ASN1_R_NULL_IS_WRONG_LENGTH - {"NULL_IS_WRONG_LENGTH", ERR_LIB_ASN1, ASN1_R_NULL_IS_WRONG_LENGTH}, - #else - {"NULL_IS_WRONG_LENGTH", 13, 144}, - #endif - #ifdef ASN1_R_OBJECT_NOT_ASCII_FORMAT - {"OBJECT_NOT_ASCII_FORMAT", ERR_LIB_ASN1, ASN1_R_OBJECT_NOT_ASCII_FORMAT}, - #else - {"OBJECT_NOT_ASCII_FORMAT", 13, 191}, - #endif - #ifdef ASN1_R_ODD_NUMBER_OF_CHARS - {"ODD_NUMBER_OF_CHARS", ERR_LIB_ASN1, ASN1_R_ODD_NUMBER_OF_CHARS}, - #else - {"ODD_NUMBER_OF_CHARS", 13, 145}, - #endif - #ifdef ASN1_R_SECOND_NUMBER_TOO_LARGE - {"SECOND_NUMBER_TOO_LARGE", ERR_LIB_ASN1, ASN1_R_SECOND_NUMBER_TOO_LARGE}, - #else - {"SECOND_NUMBER_TOO_LARGE", 13, 147}, - #endif - #ifdef ASN1_R_SEQUENCE_LENGTH_MISMATCH - {"SEQUENCE_LENGTH_MISMATCH", ERR_LIB_ASN1, ASN1_R_SEQUENCE_LENGTH_MISMATCH}, - #else - {"SEQUENCE_LENGTH_MISMATCH", 13, 148}, - #endif - #ifdef ASN1_R_SEQUENCE_NOT_CONSTRUCTED - {"SEQUENCE_NOT_CONSTRUCTED", ERR_LIB_ASN1, ASN1_R_SEQUENCE_NOT_CONSTRUCTED}, - #else - {"SEQUENCE_NOT_CONSTRUCTED", 13, 149}, - #endif - #ifdef ASN1_R_SEQUENCE_OR_SET_NEEDS_CONFIG - {"SEQUENCE_OR_SET_NEEDS_CONFIG", ERR_LIB_ASN1, ASN1_R_SEQUENCE_OR_SET_NEEDS_CONFIG}, - #else - {"SEQUENCE_OR_SET_NEEDS_CONFIG", 13, 192}, - #endif - #ifdef ASN1_R_SHORT_LINE - {"SHORT_LINE", ERR_LIB_ASN1, ASN1_R_SHORT_LINE}, - #else - {"SHORT_LINE", 13, 150}, - #endif - #ifdef ASN1_R_SIG_INVALID_MIME_TYPE - {"SIG_INVALID_MIME_TYPE", ERR_LIB_ASN1, ASN1_R_SIG_INVALID_MIME_TYPE}, - #else - {"SIG_INVALID_MIME_TYPE", 13, 213}, - #endif - #ifdef ASN1_R_STREAMING_NOT_SUPPORTED - {"STREAMING_NOT_SUPPORTED", ERR_LIB_ASN1, ASN1_R_STREAMING_NOT_SUPPORTED}, - #else - {"STREAMING_NOT_SUPPORTED", 13, 202}, - #endif - #ifdef ASN1_R_STRING_TOO_LONG - {"STRING_TOO_LONG", ERR_LIB_ASN1, ASN1_R_STRING_TOO_LONG}, - #else - {"STRING_TOO_LONG", 13, 151}, - #endif - #ifdef ASN1_R_STRING_TOO_SHORT - {"STRING_TOO_SHORT", ERR_LIB_ASN1, ASN1_R_STRING_TOO_SHORT}, - #else - {"STRING_TOO_SHORT", 13, 152}, - #endif - #ifdef ASN1_R_THE_ASN1_OBJECT_IDENTIFIER_IS_NOT_KNOWN_FOR_THIS_MD - {"THE_ASN1_OBJECT_IDENTIFIER_IS_NOT_KNOWN_FOR_THIS_MD", ERR_LIB_ASN1, ASN1_R_THE_ASN1_OBJECT_IDENTIFIER_IS_NOT_KNOWN_FOR_THIS_MD}, - #else - {"THE_ASN1_OBJECT_IDENTIFIER_IS_NOT_KNOWN_FOR_THIS_MD", 13, 154}, - #endif - #ifdef ASN1_R_TIME_NOT_ASCII_FORMAT - {"TIME_NOT_ASCII_FORMAT", ERR_LIB_ASN1, ASN1_R_TIME_NOT_ASCII_FORMAT}, - #else - {"TIME_NOT_ASCII_FORMAT", 13, 193}, - #endif - #ifdef ASN1_R_TOO_LARGE - {"TOO_LARGE", ERR_LIB_ASN1, ASN1_R_TOO_LARGE}, - #else - {"TOO_LARGE", 13, 223}, - #endif - #ifdef ASN1_R_TOO_LONG - {"TOO_LONG", ERR_LIB_ASN1, ASN1_R_TOO_LONG}, - #else - {"TOO_LONG", 13, 155}, - #endif - #ifdef ASN1_R_TOO_SMALL - {"TOO_SMALL", ERR_LIB_ASN1, ASN1_R_TOO_SMALL}, - #else - {"TOO_SMALL", 13, 224}, - #endif - #ifdef ASN1_R_TYPE_NOT_CONSTRUCTED - {"TYPE_NOT_CONSTRUCTED", ERR_LIB_ASN1, ASN1_R_TYPE_NOT_CONSTRUCTED}, - #else - {"TYPE_NOT_CONSTRUCTED", 13, 156}, - #endif - #ifdef ASN1_R_TYPE_NOT_PRIMITIVE - {"TYPE_NOT_PRIMITIVE", ERR_LIB_ASN1, ASN1_R_TYPE_NOT_PRIMITIVE}, - #else - {"TYPE_NOT_PRIMITIVE", 13, 195}, - #endif - #ifdef ASN1_R_UNEXPECTED_EOC - {"UNEXPECTED_EOC", ERR_LIB_ASN1, ASN1_R_UNEXPECTED_EOC}, - #else - {"UNEXPECTED_EOC", 13, 159}, - #endif - #ifdef ASN1_R_UNIVERSALSTRING_IS_WRONG_LENGTH - {"UNIVERSALSTRING_IS_WRONG_LENGTH", ERR_LIB_ASN1, ASN1_R_UNIVERSALSTRING_IS_WRONG_LENGTH}, - #else - {"UNIVERSALSTRING_IS_WRONG_LENGTH", 13, 215}, - #endif - #ifdef ASN1_R_UNKNOWN_FORMAT - {"UNKNOWN_FORMAT", ERR_LIB_ASN1, ASN1_R_UNKNOWN_FORMAT}, - #else - {"UNKNOWN_FORMAT", 13, 160}, - #endif - #ifdef ASN1_R_UNKNOWN_MESSAGE_DIGEST_ALGORITHM - {"UNKNOWN_MESSAGE_DIGEST_ALGORITHM", ERR_LIB_ASN1, ASN1_R_UNKNOWN_MESSAGE_DIGEST_ALGORITHM}, - #else - {"UNKNOWN_MESSAGE_DIGEST_ALGORITHM", 13, 161}, - #endif - #ifdef ASN1_R_UNKNOWN_OBJECT_TYPE - {"UNKNOWN_OBJECT_TYPE", ERR_LIB_ASN1, ASN1_R_UNKNOWN_OBJECT_TYPE}, - #else - {"UNKNOWN_OBJECT_TYPE", 13, 162}, - #endif - #ifdef ASN1_R_UNKNOWN_PUBLIC_KEY_TYPE - {"UNKNOWN_PUBLIC_KEY_TYPE", ERR_LIB_ASN1, ASN1_R_UNKNOWN_PUBLIC_KEY_TYPE}, - #else - {"UNKNOWN_PUBLIC_KEY_TYPE", 13, 163}, - #endif - #ifdef ASN1_R_UNKNOWN_SIGNATURE_ALGORITHM - {"UNKNOWN_SIGNATURE_ALGORITHM", ERR_LIB_ASN1, ASN1_R_UNKNOWN_SIGNATURE_ALGORITHM}, - #else - {"UNKNOWN_SIGNATURE_ALGORITHM", 13, 199}, - #endif - #ifdef ASN1_R_UNKNOWN_TAG - {"UNKNOWN_TAG", ERR_LIB_ASN1, ASN1_R_UNKNOWN_TAG}, - #else - {"UNKNOWN_TAG", 13, 194}, - #endif - #ifdef ASN1_R_UNSUPPORTED_ANY_DEFINED_BY_TYPE - {"UNSUPPORTED_ANY_DEFINED_BY_TYPE", ERR_LIB_ASN1, ASN1_R_UNSUPPORTED_ANY_DEFINED_BY_TYPE}, - #else - {"UNSUPPORTED_ANY_DEFINED_BY_TYPE", 13, 164}, - #endif - #ifdef ASN1_R_UNSUPPORTED_CIPHER - {"UNSUPPORTED_CIPHER", ERR_LIB_ASN1, ASN1_R_UNSUPPORTED_CIPHER}, - #else - {"UNSUPPORTED_CIPHER", 13, 228}, - #endif - #ifdef ASN1_R_UNSUPPORTED_PUBLIC_KEY_TYPE - {"UNSUPPORTED_PUBLIC_KEY_TYPE", ERR_LIB_ASN1, ASN1_R_UNSUPPORTED_PUBLIC_KEY_TYPE}, - #else - {"UNSUPPORTED_PUBLIC_KEY_TYPE", 13, 167}, - #endif - #ifdef ASN1_R_UNSUPPORTED_TYPE - {"UNSUPPORTED_TYPE", ERR_LIB_ASN1, ASN1_R_UNSUPPORTED_TYPE}, - #else - {"UNSUPPORTED_TYPE", 13, 196}, - #endif - #ifdef ASN1_R_WRONG_INTEGER_TYPE - {"WRONG_INTEGER_TYPE", ERR_LIB_ASN1, ASN1_R_WRONG_INTEGER_TYPE}, - #else - {"WRONG_INTEGER_TYPE", 13, 225}, - #endif - #ifdef ASN1_R_WRONG_PUBLIC_KEY_TYPE - {"WRONG_PUBLIC_KEY_TYPE", ERR_LIB_ASN1, ASN1_R_WRONG_PUBLIC_KEY_TYPE}, - #else - {"WRONG_PUBLIC_KEY_TYPE", 13, 200}, - #endif - #ifdef ASN1_R_WRONG_TAG - {"WRONG_TAG", ERR_LIB_ASN1, ASN1_R_WRONG_TAG}, - #else - {"WRONG_TAG", 13, 168}, - #endif - #ifdef ASYNC_R_FAILED_TO_SET_POOL - {"FAILED_TO_SET_POOL", ERR_LIB_ASYNC, ASYNC_R_FAILED_TO_SET_POOL}, - #else - {"FAILED_TO_SET_POOL", 51, 101}, - #endif - #ifdef ASYNC_R_FAILED_TO_SWAP_CONTEXT - {"FAILED_TO_SWAP_CONTEXT", ERR_LIB_ASYNC, ASYNC_R_FAILED_TO_SWAP_CONTEXT}, - #else - {"FAILED_TO_SWAP_CONTEXT", 51, 102}, - #endif - #ifdef ASYNC_R_INIT_FAILED - {"INIT_FAILED", ERR_LIB_ASYNC, ASYNC_R_INIT_FAILED}, - #else - {"INIT_FAILED", 51, 105}, - #endif - #ifdef ASYNC_R_INVALID_POOL_SIZE - {"INVALID_POOL_SIZE", ERR_LIB_ASYNC, ASYNC_R_INVALID_POOL_SIZE}, - #else - {"INVALID_POOL_SIZE", 51, 103}, - #endif - #ifdef BIO_R_ACCEPT_ERROR - {"ACCEPT_ERROR", ERR_LIB_BIO, BIO_R_ACCEPT_ERROR}, - #else - {"ACCEPT_ERROR", 32, 100}, - #endif - #ifdef BIO_R_ADDRINFO_ADDR_IS_NOT_AF_INET - {"ADDRINFO_ADDR_IS_NOT_AF_INET", ERR_LIB_BIO, BIO_R_ADDRINFO_ADDR_IS_NOT_AF_INET}, - #else - {"ADDRINFO_ADDR_IS_NOT_AF_INET", 32, 141}, - #endif - #ifdef BIO_R_AMBIGUOUS_HOST_OR_SERVICE - {"AMBIGUOUS_HOST_OR_SERVICE", ERR_LIB_BIO, BIO_R_AMBIGUOUS_HOST_OR_SERVICE}, - #else - {"AMBIGUOUS_HOST_OR_SERVICE", 32, 129}, - #endif - #ifdef BIO_R_BAD_FOPEN_MODE - {"BAD_FOPEN_MODE", ERR_LIB_BIO, BIO_R_BAD_FOPEN_MODE}, - #else - {"BAD_FOPEN_MODE", 32, 101}, - #endif - #ifdef BIO_R_BROKEN_PIPE - {"BROKEN_PIPE", ERR_LIB_BIO, BIO_R_BROKEN_PIPE}, - #else - {"BROKEN_PIPE", 32, 124}, - #endif - #ifdef BIO_R_CONNECT_ERROR - {"CONNECT_ERROR", ERR_LIB_BIO, BIO_R_CONNECT_ERROR}, - #else - {"CONNECT_ERROR", 32, 103}, - #endif - #ifdef BIO_R_GETHOSTBYNAME_ADDR_IS_NOT_AF_INET - {"GETHOSTBYNAME_ADDR_IS_NOT_AF_INET", ERR_LIB_BIO, BIO_R_GETHOSTBYNAME_ADDR_IS_NOT_AF_INET}, - #else - {"GETHOSTBYNAME_ADDR_IS_NOT_AF_INET", 32, 107}, - #endif - #ifdef BIO_R_GETSOCKNAME_ERROR - {"GETSOCKNAME_ERROR", ERR_LIB_BIO, BIO_R_GETSOCKNAME_ERROR}, - #else - {"GETSOCKNAME_ERROR", 32, 132}, - #endif - #ifdef BIO_R_GETSOCKNAME_TRUNCATED_ADDRESS - {"GETSOCKNAME_TRUNCATED_ADDRESS", ERR_LIB_BIO, BIO_R_GETSOCKNAME_TRUNCATED_ADDRESS}, - #else - {"GETSOCKNAME_TRUNCATED_ADDRESS", 32, 133}, - #endif - #ifdef BIO_R_GETTING_SOCKTYPE - {"GETTING_SOCKTYPE", ERR_LIB_BIO, BIO_R_GETTING_SOCKTYPE}, - #else - {"GETTING_SOCKTYPE", 32, 134}, - #endif - #ifdef BIO_R_INVALID_ARGUMENT - {"INVALID_ARGUMENT", ERR_LIB_BIO, BIO_R_INVALID_ARGUMENT}, - #else - {"INVALID_ARGUMENT", 32, 125}, - #endif - #ifdef BIO_R_INVALID_SOCKET - {"INVALID_SOCKET", ERR_LIB_BIO, BIO_R_INVALID_SOCKET}, - #else - {"INVALID_SOCKET", 32, 135}, - #endif - #ifdef BIO_R_IN_USE - {"IN_USE", ERR_LIB_BIO, BIO_R_IN_USE}, - #else - {"IN_USE", 32, 123}, - #endif - #ifdef BIO_R_LENGTH_TOO_LONG - {"LENGTH_TOO_LONG", ERR_LIB_BIO, BIO_R_LENGTH_TOO_LONG}, - #else - {"LENGTH_TOO_LONG", 32, 102}, - #endif - #ifdef BIO_R_LISTEN_V6_ONLY - {"LISTEN_V6_ONLY", ERR_LIB_BIO, BIO_R_LISTEN_V6_ONLY}, - #else - {"LISTEN_V6_ONLY", 32, 136}, - #endif - #ifdef BIO_R_LOOKUP_RETURNED_NOTHING - {"LOOKUP_RETURNED_NOTHING", ERR_LIB_BIO, BIO_R_LOOKUP_RETURNED_NOTHING}, - #else - {"LOOKUP_RETURNED_NOTHING", 32, 142}, - #endif - #ifdef BIO_R_MALFORMED_HOST_OR_SERVICE - {"MALFORMED_HOST_OR_SERVICE", ERR_LIB_BIO, BIO_R_MALFORMED_HOST_OR_SERVICE}, - #else - {"MALFORMED_HOST_OR_SERVICE", 32, 130}, - #endif - #ifdef BIO_R_NBIO_CONNECT_ERROR - {"NBIO_CONNECT_ERROR", ERR_LIB_BIO, BIO_R_NBIO_CONNECT_ERROR}, - #else - {"NBIO_CONNECT_ERROR", 32, 110}, - #endif - #ifdef BIO_R_NO_ACCEPT_ADDR_OR_SERVICE_SPECIFIED - {"NO_ACCEPT_ADDR_OR_SERVICE_SPECIFIED", ERR_LIB_BIO, BIO_R_NO_ACCEPT_ADDR_OR_SERVICE_SPECIFIED}, - #else - {"NO_ACCEPT_ADDR_OR_SERVICE_SPECIFIED", 32, 143}, - #endif - #ifdef BIO_R_NO_HOSTNAME_OR_SERVICE_SPECIFIED - {"NO_HOSTNAME_OR_SERVICE_SPECIFIED", ERR_LIB_BIO, BIO_R_NO_HOSTNAME_OR_SERVICE_SPECIFIED}, - #else - {"NO_HOSTNAME_OR_SERVICE_SPECIFIED", 32, 144}, - #endif - #ifdef BIO_R_NO_PORT_DEFINED - {"NO_PORT_DEFINED", ERR_LIB_BIO, BIO_R_NO_PORT_DEFINED}, - #else - {"NO_PORT_DEFINED", 32, 113}, - #endif - #ifdef BIO_R_NO_SUCH_FILE - {"NO_SUCH_FILE", ERR_LIB_BIO, BIO_R_NO_SUCH_FILE}, - #else - {"NO_SUCH_FILE", 32, 128}, - #endif - #ifdef BIO_R_NULL_PARAMETER - {"NULL_PARAMETER", ERR_LIB_BIO, BIO_R_NULL_PARAMETER}, - #else - {"NULL_PARAMETER", 32, 115}, - #endif - #ifdef BIO_R_UNABLE_TO_BIND_SOCKET - {"UNABLE_TO_BIND_SOCKET", ERR_LIB_BIO, BIO_R_UNABLE_TO_BIND_SOCKET}, - #else - {"UNABLE_TO_BIND_SOCKET", 32, 117}, - #endif - #ifdef BIO_R_UNABLE_TO_CREATE_SOCKET - {"UNABLE_TO_CREATE_SOCKET", ERR_LIB_BIO, BIO_R_UNABLE_TO_CREATE_SOCKET}, - #else - {"UNABLE_TO_CREATE_SOCKET", 32, 118}, - #endif - #ifdef BIO_R_UNABLE_TO_KEEPALIVE - {"UNABLE_TO_KEEPALIVE", ERR_LIB_BIO, BIO_R_UNABLE_TO_KEEPALIVE}, - #else - {"UNABLE_TO_KEEPALIVE", 32, 137}, - #endif - #ifdef BIO_R_UNABLE_TO_LISTEN_SOCKET - {"UNABLE_TO_LISTEN_SOCKET", ERR_LIB_BIO, BIO_R_UNABLE_TO_LISTEN_SOCKET}, - #else - {"UNABLE_TO_LISTEN_SOCKET", 32, 119}, - #endif - #ifdef BIO_R_UNABLE_TO_NODELAY - {"UNABLE_TO_NODELAY", ERR_LIB_BIO, BIO_R_UNABLE_TO_NODELAY}, - #else - {"UNABLE_TO_NODELAY", 32, 138}, - #endif - #ifdef BIO_R_UNABLE_TO_REUSEADDR - {"UNABLE_TO_REUSEADDR", ERR_LIB_BIO, BIO_R_UNABLE_TO_REUSEADDR}, - #else - {"UNABLE_TO_REUSEADDR", 32, 139}, - #endif - #ifdef BIO_R_UNAVAILABLE_IP_FAMILY - {"UNAVAILABLE_IP_FAMILY", ERR_LIB_BIO, BIO_R_UNAVAILABLE_IP_FAMILY}, - #else - {"UNAVAILABLE_IP_FAMILY", 32, 145}, - #endif - #ifdef BIO_R_UNINITIALIZED - {"UNINITIALIZED", ERR_LIB_BIO, BIO_R_UNINITIALIZED}, - #else - {"UNINITIALIZED", 32, 120}, - #endif - #ifdef BIO_R_UNKNOWN_INFO_TYPE - {"UNKNOWN_INFO_TYPE", ERR_LIB_BIO, BIO_R_UNKNOWN_INFO_TYPE}, - #else - {"UNKNOWN_INFO_TYPE", 32, 140}, - #endif - #ifdef BIO_R_UNSUPPORTED_IP_FAMILY - {"UNSUPPORTED_IP_FAMILY", ERR_LIB_BIO, BIO_R_UNSUPPORTED_IP_FAMILY}, - #else - {"UNSUPPORTED_IP_FAMILY", 32, 146}, - #endif - #ifdef BIO_R_UNSUPPORTED_METHOD - {"UNSUPPORTED_METHOD", ERR_LIB_BIO, BIO_R_UNSUPPORTED_METHOD}, - #else - {"UNSUPPORTED_METHOD", 32, 121}, - #endif - #ifdef BIO_R_UNSUPPORTED_PROTOCOL_FAMILY - {"UNSUPPORTED_PROTOCOL_FAMILY", ERR_LIB_BIO, BIO_R_UNSUPPORTED_PROTOCOL_FAMILY}, - #else - {"UNSUPPORTED_PROTOCOL_FAMILY", 32, 131}, - #endif - #ifdef BIO_R_WRITE_TO_READ_ONLY_BIO - {"WRITE_TO_READ_ONLY_BIO", ERR_LIB_BIO, BIO_R_WRITE_TO_READ_ONLY_BIO}, - #else - {"WRITE_TO_READ_ONLY_BIO", 32, 126}, - #endif - #ifdef BIO_R_WSASTARTUP - {"WSASTARTUP", ERR_LIB_BIO, BIO_R_WSASTARTUP}, - #else - {"WSASTARTUP", 32, 122}, - #endif - #ifdef BN_R_ARG2_LT_ARG3 - {"ARG2_LT_ARG3", ERR_LIB_BN, BN_R_ARG2_LT_ARG3}, - #else - {"ARG2_LT_ARG3", 3, 100}, - #endif - #ifdef BN_R_BAD_RECIPROCAL - {"BAD_RECIPROCAL", ERR_LIB_BN, BN_R_BAD_RECIPROCAL}, - #else - {"BAD_RECIPROCAL", 3, 101}, - #endif - #ifdef BN_R_BIGNUM_TOO_LONG - {"BIGNUM_TOO_LONG", ERR_LIB_BN, BN_R_BIGNUM_TOO_LONG}, - #else - {"BIGNUM_TOO_LONG", 3, 114}, - #endif - #ifdef BN_R_BITS_TOO_SMALL - {"BITS_TOO_SMALL", ERR_LIB_BN, BN_R_BITS_TOO_SMALL}, - #else - {"BITS_TOO_SMALL", 3, 118}, - #endif - #ifdef BN_R_CALLED_WITH_EVEN_MODULUS - {"CALLED_WITH_EVEN_MODULUS", ERR_LIB_BN, BN_R_CALLED_WITH_EVEN_MODULUS}, - #else - {"CALLED_WITH_EVEN_MODULUS", 3, 102}, - #endif - #ifdef BN_R_DIV_BY_ZERO - {"DIV_BY_ZERO", ERR_LIB_BN, BN_R_DIV_BY_ZERO}, - #else - {"DIV_BY_ZERO", 3, 103}, - #endif - #ifdef BN_R_ENCODING_ERROR - {"ENCODING_ERROR", ERR_LIB_BN, BN_R_ENCODING_ERROR}, - #else - {"ENCODING_ERROR", 3, 104}, - #endif - #ifdef BN_R_EXPAND_ON_STATIC_BIGNUM_DATA - {"EXPAND_ON_STATIC_BIGNUM_DATA", ERR_LIB_BN, BN_R_EXPAND_ON_STATIC_BIGNUM_DATA}, - #else - {"EXPAND_ON_STATIC_BIGNUM_DATA", 3, 105}, - #endif - #ifdef BN_R_INPUT_NOT_REDUCED - {"INPUT_NOT_REDUCED", ERR_LIB_BN, BN_R_INPUT_NOT_REDUCED}, - #else - {"INPUT_NOT_REDUCED", 3, 110}, - #endif - #ifdef BN_R_INVALID_LENGTH - {"INVALID_LENGTH", ERR_LIB_BN, BN_R_INVALID_LENGTH}, - #else - {"INVALID_LENGTH", 3, 106}, - #endif - #ifdef BN_R_INVALID_RANGE - {"INVALID_RANGE", ERR_LIB_BN, BN_R_INVALID_RANGE}, - #else - {"INVALID_RANGE", 3, 115}, - #endif - #ifdef BN_R_INVALID_SHIFT - {"INVALID_SHIFT", ERR_LIB_BN, BN_R_INVALID_SHIFT}, - #else - {"INVALID_SHIFT", 3, 119}, - #endif - #ifdef BN_R_NOT_A_SQUARE - {"NOT_A_SQUARE", ERR_LIB_BN, BN_R_NOT_A_SQUARE}, - #else - {"NOT_A_SQUARE", 3, 111}, - #endif - #ifdef BN_R_NOT_INITIALIZED - {"NOT_INITIALIZED", ERR_LIB_BN, BN_R_NOT_INITIALIZED}, - #else - {"NOT_INITIALIZED", 3, 107}, - #endif - #ifdef BN_R_NO_INVERSE - {"NO_INVERSE", ERR_LIB_BN, BN_R_NO_INVERSE}, - #else - {"NO_INVERSE", 3, 108}, - #endif - #ifdef BN_R_NO_SOLUTION - {"NO_SOLUTION", ERR_LIB_BN, BN_R_NO_SOLUTION}, - #else - {"NO_SOLUTION", 3, 116}, - #endif - #ifdef BN_R_PRIVATE_KEY_TOO_LARGE - {"PRIVATE_KEY_TOO_LARGE", ERR_LIB_BN, BN_R_PRIVATE_KEY_TOO_LARGE}, - #else - {"PRIVATE_KEY_TOO_LARGE", 3, 117}, - #endif - #ifdef BN_R_P_IS_NOT_PRIME - {"P_IS_NOT_PRIME", ERR_LIB_BN, BN_R_P_IS_NOT_PRIME}, - #else - {"P_IS_NOT_PRIME", 3, 112}, - #endif - #ifdef BN_R_TOO_MANY_ITERATIONS - {"TOO_MANY_ITERATIONS", ERR_LIB_BN, BN_R_TOO_MANY_ITERATIONS}, - #else - {"TOO_MANY_ITERATIONS", 3, 113}, - #endif - #ifdef BN_R_TOO_MANY_TEMPORARY_VARIABLES - {"TOO_MANY_TEMPORARY_VARIABLES", ERR_LIB_BN, BN_R_TOO_MANY_TEMPORARY_VARIABLES}, - #else - {"TOO_MANY_TEMPORARY_VARIABLES", 3, 109}, - #endif - #ifdef CMS_R_ADD_SIGNER_ERROR - {"ADD_SIGNER_ERROR", ERR_LIB_CMS, CMS_R_ADD_SIGNER_ERROR}, - #else - {"ADD_SIGNER_ERROR", 46, 99}, - #endif - #ifdef CMS_R_ATTRIBUTE_ERROR - {"ATTRIBUTE_ERROR", ERR_LIB_CMS, CMS_R_ATTRIBUTE_ERROR}, - #else - {"ATTRIBUTE_ERROR", 46, 161}, - #endif - #ifdef CMS_R_CERTIFICATE_ALREADY_PRESENT - {"CERTIFICATE_ALREADY_PRESENT", ERR_LIB_CMS, CMS_R_CERTIFICATE_ALREADY_PRESENT}, - #else - {"CERTIFICATE_ALREADY_PRESENT", 46, 175}, - #endif - #ifdef CMS_R_CERTIFICATE_HAS_NO_KEYID - {"CERTIFICATE_HAS_NO_KEYID", ERR_LIB_CMS, CMS_R_CERTIFICATE_HAS_NO_KEYID}, - #else - {"CERTIFICATE_HAS_NO_KEYID", 46, 160}, - #endif - #ifdef CMS_R_CERTIFICATE_VERIFY_ERROR - {"CERTIFICATE_VERIFY_ERROR", ERR_LIB_CMS, CMS_R_CERTIFICATE_VERIFY_ERROR}, - #else - {"CERTIFICATE_VERIFY_ERROR", 46, 100}, - #endif - #ifdef CMS_R_CIPHER_INITIALISATION_ERROR - {"CIPHER_INITIALISATION_ERROR", ERR_LIB_CMS, CMS_R_CIPHER_INITIALISATION_ERROR}, - #else - {"CIPHER_INITIALISATION_ERROR", 46, 101}, - #endif - #ifdef CMS_R_CIPHER_PARAMETER_INITIALISATION_ERROR - {"CIPHER_PARAMETER_INITIALISATION_ERROR", ERR_LIB_CMS, CMS_R_CIPHER_PARAMETER_INITIALISATION_ERROR}, - #else - {"CIPHER_PARAMETER_INITIALISATION_ERROR", 46, 102}, - #endif - #ifdef CMS_R_CMS_DATAFINAL_ERROR - {"CMS_DATAFINAL_ERROR", ERR_LIB_CMS, CMS_R_CMS_DATAFINAL_ERROR}, - #else - {"CMS_DATAFINAL_ERROR", 46, 103}, - #endif - #ifdef CMS_R_CMS_LIB - {"CMS_LIB", ERR_LIB_CMS, CMS_R_CMS_LIB}, - #else - {"CMS_LIB", 46, 104}, - #endif - #ifdef CMS_R_CONTENTIDENTIFIER_MISMATCH - {"CONTENTIDENTIFIER_MISMATCH", ERR_LIB_CMS, CMS_R_CONTENTIDENTIFIER_MISMATCH}, - #else - {"CONTENTIDENTIFIER_MISMATCH", 46, 170}, - #endif - #ifdef CMS_R_CONTENT_NOT_FOUND - {"CONTENT_NOT_FOUND", ERR_LIB_CMS, CMS_R_CONTENT_NOT_FOUND}, - #else - {"CONTENT_NOT_FOUND", 46, 105}, - #endif - #ifdef CMS_R_CONTENT_TYPE_MISMATCH - {"CONTENT_TYPE_MISMATCH", ERR_LIB_CMS, CMS_R_CONTENT_TYPE_MISMATCH}, - #else - {"CONTENT_TYPE_MISMATCH", 46, 171}, - #endif - #ifdef CMS_R_CONTENT_TYPE_NOT_COMPRESSED_DATA - {"CONTENT_TYPE_NOT_COMPRESSED_DATA", ERR_LIB_CMS, CMS_R_CONTENT_TYPE_NOT_COMPRESSED_DATA}, - #else - {"CONTENT_TYPE_NOT_COMPRESSED_DATA", 46, 106}, - #endif - #ifdef CMS_R_CONTENT_TYPE_NOT_ENVELOPED_DATA - {"CONTENT_TYPE_NOT_ENVELOPED_DATA", ERR_LIB_CMS, CMS_R_CONTENT_TYPE_NOT_ENVELOPED_DATA}, - #else - {"CONTENT_TYPE_NOT_ENVELOPED_DATA", 46, 107}, - #endif - #ifdef CMS_R_CONTENT_TYPE_NOT_SIGNED_DATA - {"CONTENT_TYPE_NOT_SIGNED_DATA", ERR_LIB_CMS, CMS_R_CONTENT_TYPE_NOT_SIGNED_DATA}, - #else - {"CONTENT_TYPE_NOT_SIGNED_DATA", 46, 108}, - #endif - #ifdef CMS_R_CONTENT_VERIFY_ERROR - {"CONTENT_VERIFY_ERROR", ERR_LIB_CMS, CMS_R_CONTENT_VERIFY_ERROR}, - #else - {"CONTENT_VERIFY_ERROR", 46, 109}, - #endif - #ifdef CMS_R_CTRL_ERROR - {"CTRL_ERROR", ERR_LIB_CMS, CMS_R_CTRL_ERROR}, - #else - {"CTRL_ERROR", 46, 110}, - #endif - #ifdef CMS_R_CTRL_FAILURE - {"CTRL_FAILURE", ERR_LIB_CMS, CMS_R_CTRL_FAILURE}, - #else - {"CTRL_FAILURE", 46, 111}, - #endif - #ifdef CMS_R_DECRYPT_ERROR - {"DECRYPT_ERROR", ERR_LIB_CMS, CMS_R_DECRYPT_ERROR}, - #else - {"DECRYPT_ERROR", 46, 112}, - #endif - #ifdef CMS_R_ERROR_GETTING_PUBLIC_KEY - {"ERROR_GETTING_PUBLIC_KEY", ERR_LIB_CMS, CMS_R_ERROR_GETTING_PUBLIC_KEY}, - #else - {"ERROR_GETTING_PUBLIC_KEY", 46, 113}, - #endif - #ifdef CMS_R_ERROR_READING_MESSAGEDIGEST_ATTRIBUTE - {"ERROR_READING_MESSAGEDIGEST_ATTRIBUTE", ERR_LIB_CMS, CMS_R_ERROR_READING_MESSAGEDIGEST_ATTRIBUTE}, - #else - {"ERROR_READING_MESSAGEDIGEST_ATTRIBUTE", 46, 114}, - #endif - #ifdef CMS_R_ERROR_SETTING_KEY - {"ERROR_SETTING_KEY", ERR_LIB_CMS, CMS_R_ERROR_SETTING_KEY}, - #else - {"ERROR_SETTING_KEY", 46, 115}, - #endif - #ifdef CMS_R_ERROR_SETTING_RECIPIENTINFO - {"ERROR_SETTING_RECIPIENTINFO", ERR_LIB_CMS, CMS_R_ERROR_SETTING_RECIPIENTINFO}, - #else - {"ERROR_SETTING_RECIPIENTINFO", 46, 116}, - #endif - #ifdef CMS_R_INVALID_ENCRYPTED_KEY_LENGTH - {"INVALID_ENCRYPTED_KEY_LENGTH", ERR_LIB_CMS, CMS_R_INVALID_ENCRYPTED_KEY_LENGTH}, - #else - {"INVALID_ENCRYPTED_KEY_LENGTH", 46, 117}, - #endif - #ifdef CMS_R_INVALID_KEY_ENCRYPTION_PARAMETER - {"INVALID_KEY_ENCRYPTION_PARAMETER", ERR_LIB_CMS, CMS_R_INVALID_KEY_ENCRYPTION_PARAMETER}, - #else - {"INVALID_KEY_ENCRYPTION_PARAMETER", 46, 176}, - #endif - #ifdef CMS_R_INVALID_KEY_LENGTH - {"INVALID_KEY_LENGTH", ERR_LIB_CMS, CMS_R_INVALID_KEY_LENGTH}, - #else - {"INVALID_KEY_LENGTH", 46, 118}, - #endif - #ifdef CMS_R_MD_BIO_INIT_ERROR - {"MD_BIO_INIT_ERROR", ERR_LIB_CMS, CMS_R_MD_BIO_INIT_ERROR}, - #else - {"MD_BIO_INIT_ERROR", 46, 119}, - #endif - #ifdef CMS_R_MESSAGEDIGEST_ATTRIBUTE_WRONG_LENGTH - {"MESSAGEDIGEST_ATTRIBUTE_WRONG_LENGTH", ERR_LIB_CMS, CMS_R_MESSAGEDIGEST_ATTRIBUTE_WRONG_LENGTH}, - #else - {"MESSAGEDIGEST_ATTRIBUTE_WRONG_LENGTH", 46, 120}, - #endif - #ifdef CMS_R_MESSAGEDIGEST_WRONG_LENGTH - {"MESSAGEDIGEST_WRONG_LENGTH", ERR_LIB_CMS, CMS_R_MESSAGEDIGEST_WRONG_LENGTH}, - #else - {"MESSAGEDIGEST_WRONG_LENGTH", 46, 121}, - #endif - #ifdef CMS_R_MSGSIGDIGEST_ERROR - {"MSGSIGDIGEST_ERROR", ERR_LIB_CMS, CMS_R_MSGSIGDIGEST_ERROR}, - #else - {"MSGSIGDIGEST_ERROR", 46, 172}, - #endif - #ifdef CMS_R_MSGSIGDIGEST_VERIFICATION_FAILURE - {"MSGSIGDIGEST_VERIFICATION_FAILURE", ERR_LIB_CMS, CMS_R_MSGSIGDIGEST_VERIFICATION_FAILURE}, - #else - {"MSGSIGDIGEST_VERIFICATION_FAILURE", 46, 162}, - #endif - #ifdef CMS_R_MSGSIGDIGEST_WRONG_LENGTH - {"MSGSIGDIGEST_WRONG_LENGTH", ERR_LIB_CMS, CMS_R_MSGSIGDIGEST_WRONG_LENGTH}, - #else - {"MSGSIGDIGEST_WRONG_LENGTH", 46, 163}, - #endif - #ifdef CMS_R_NEED_ONE_SIGNER - {"NEED_ONE_SIGNER", ERR_LIB_CMS, CMS_R_NEED_ONE_SIGNER}, - #else - {"NEED_ONE_SIGNER", 46, 164}, - #endif - #ifdef CMS_R_NOT_A_SIGNED_RECEIPT - {"NOT_A_SIGNED_RECEIPT", ERR_LIB_CMS, CMS_R_NOT_A_SIGNED_RECEIPT}, - #else - {"NOT_A_SIGNED_RECEIPT", 46, 165}, - #endif - #ifdef CMS_R_NOT_ENCRYPTED_DATA - {"NOT_ENCRYPTED_DATA", ERR_LIB_CMS, CMS_R_NOT_ENCRYPTED_DATA}, - #else - {"NOT_ENCRYPTED_DATA", 46, 122}, - #endif - #ifdef CMS_R_NOT_KEK - {"NOT_KEK", ERR_LIB_CMS, CMS_R_NOT_KEK}, - #else - {"NOT_KEK", 46, 123}, - #endif - #ifdef CMS_R_NOT_KEY_AGREEMENT - {"NOT_KEY_AGREEMENT", ERR_LIB_CMS, CMS_R_NOT_KEY_AGREEMENT}, - #else - {"NOT_KEY_AGREEMENT", 46, 181}, - #endif - #ifdef CMS_R_NOT_KEY_TRANSPORT - {"NOT_KEY_TRANSPORT", ERR_LIB_CMS, CMS_R_NOT_KEY_TRANSPORT}, - #else - {"NOT_KEY_TRANSPORT", 46, 124}, - #endif - #ifdef CMS_R_NOT_PWRI - {"NOT_PWRI", ERR_LIB_CMS, CMS_R_NOT_PWRI}, - #else - {"NOT_PWRI", 46, 177}, - #endif - #ifdef CMS_R_NOT_SUPPORTED_FOR_THIS_KEY_TYPE - {"NOT_SUPPORTED_FOR_THIS_KEY_TYPE", ERR_LIB_CMS, CMS_R_NOT_SUPPORTED_FOR_THIS_KEY_TYPE}, - #else - {"NOT_SUPPORTED_FOR_THIS_KEY_TYPE", 46, 125}, - #endif - #ifdef CMS_R_NO_CIPHER - {"NO_CIPHER", ERR_LIB_CMS, CMS_R_NO_CIPHER}, - #else - {"NO_CIPHER", 46, 126}, - #endif - #ifdef CMS_R_NO_CONTENT - {"NO_CONTENT", ERR_LIB_CMS, CMS_R_NO_CONTENT}, - #else - {"NO_CONTENT", 46, 127}, - #endif - #ifdef CMS_R_NO_CONTENT_TYPE - {"NO_CONTENT_TYPE", ERR_LIB_CMS, CMS_R_NO_CONTENT_TYPE}, - #else - {"NO_CONTENT_TYPE", 46, 173}, - #endif - #ifdef CMS_R_NO_DEFAULT_DIGEST - {"NO_DEFAULT_DIGEST", ERR_LIB_CMS, CMS_R_NO_DEFAULT_DIGEST}, - #else - {"NO_DEFAULT_DIGEST", 46, 128}, - #endif - #ifdef CMS_R_NO_DIGEST_SET - {"NO_DIGEST_SET", ERR_LIB_CMS, CMS_R_NO_DIGEST_SET}, - #else - {"NO_DIGEST_SET", 46, 129}, - #endif - #ifdef CMS_R_NO_KEY - {"NO_KEY", ERR_LIB_CMS, CMS_R_NO_KEY}, - #else - {"NO_KEY", 46, 130}, - #endif - #ifdef CMS_R_NO_KEY_OR_CERT - {"NO_KEY_OR_CERT", ERR_LIB_CMS, CMS_R_NO_KEY_OR_CERT}, - #else - {"NO_KEY_OR_CERT", 46, 174}, - #endif - #ifdef CMS_R_NO_MATCHING_DIGEST - {"NO_MATCHING_DIGEST", ERR_LIB_CMS, CMS_R_NO_MATCHING_DIGEST}, - #else - {"NO_MATCHING_DIGEST", 46, 131}, - #endif - #ifdef CMS_R_NO_MATCHING_RECIPIENT - {"NO_MATCHING_RECIPIENT", ERR_LIB_CMS, CMS_R_NO_MATCHING_RECIPIENT}, - #else - {"NO_MATCHING_RECIPIENT", 46, 132}, - #endif - #ifdef CMS_R_NO_MATCHING_SIGNATURE - {"NO_MATCHING_SIGNATURE", ERR_LIB_CMS, CMS_R_NO_MATCHING_SIGNATURE}, - #else - {"NO_MATCHING_SIGNATURE", 46, 166}, - #endif - #ifdef CMS_R_NO_MSGSIGDIGEST - {"NO_MSGSIGDIGEST", ERR_LIB_CMS, CMS_R_NO_MSGSIGDIGEST}, - #else - {"NO_MSGSIGDIGEST", 46, 167}, - #endif - #ifdef CMS_R_NO_PASSWORD - {"NO_PASSWORD", ERR_LIB_CMS, CMS_R_NO_PASSWORD}, - #else - {"NO_PASSWORD", 46, 178}, - #endif - #ifdef CMS_R_NO_PRIVATE_KEY - {"NO_PRIVATE_KEY", ERR_LIB_CMS, CMS_R_NO_PRIVATE_KEY}, - #else - {"NO_PRIVATE_KEY", 46, 133}, - #endif - #ifdef CMS_R_NO_PUBLIC_KEY - {"NO_PUBLIC_KEY", ERR_LIB_CMS, CMS_R_NO_PUBLIC_KEY}, - #else - {"NO_PUBLIC_KEY", 46, 134}, - #endif - #ifdef CMS_R_NO_RECEIPT_REQUEST - {"NO_RECEIPT_REQUEST", ERR_LIB_CMS, CMS_R_NO_RECEIPT_REQUEST}, - #else - {"NO_RECEIPT_REQUEST", 46, 168}, - #endif - #ifdef CMS_R_NO_SIGNERS - {"NO_SIGNERS", ERR_LIB_CMS, CMS_R_NO_SIGNERS}, - #else - {"NO_SIGNERS", 46, 135}, - #endif - #ifdef CMS_R_PRIVATE_KEY_DOES_NOT_MATCH_CERTIFICATE - {"PRIVATE_KEY_DOES_NOT_MATCH_CERTIFICATE", ERR_LIB_CMS, CMS_R_PRIVATE_KEY_DOES_NOT_MATCH_CERTIFICATE}, - #else - {"PRIVATE_KEY_DOES_NOT_MATCH_CERTIFICATE", 46, 136}, - #endif - #ifdef CMS_R_RECEIPT_DECODE_ERROR - {"RECEIPT_DECODE_ERROR", ERR_LIB_CMS, CMS_R_RECEIPT_DECODE_ERROR}, - #else - {"RECEIPT_DECODE_ERROR", 46, 169}, - #endif - #ifdef CMS_R_RECIPIENT_ERROR - {"RECIPIENT_ERROR", ERR_LIB_CMS, CMS_R_RECIPIENT_ERROR}, - #else - {"RECIPIENT_ERROR", 46, 137}, - #endif - #ifdef CMS_R_SIGNER_CERTIFICATE_NOT_FOUND - {"SIGNER_CERTIFICATE_NOT_FOUND", ERR_LIB_CMS, CMS_R_SIGNER_CERTIFICATE_NOT_FOUND}, - #else - {"SIGNER_CERTIFICATE_NOT_FOUND", 46, 138}, - #endif - #ifdef CMS_R_SIGNFINAL_ERROR - {"SIGNFINAL_ERROR", ERR_LIB_CMS, CMS_R_SIGNFINAL_ERROR}, - #else - {"SIGNFINAL_ERROR", 46, 139}, - #endif - #ifdef CMS_R_SMIME_TEXT_ERROR - {"SMIME_TEXT_ERROR", ERR_LIB_CMS, CMS_R_SMIME_TEXT_ERROR}, - #else - {"SMIME_TEXT_ERROR", 46, 140}, - #endif - #ifdef CMS_R_STORE_INIT_ERROR - {"STORE_INIT_ERROR", ERR_LIB_CMS, CMS_R_STORE_INIT_ERROR}, - #else - {"STORE_INIT_ERROR", 46, 141}, - #endif - #ifdef CMS_R_TYPE_NOT_COMPRESSED_DATA - {"TYPE_NOT_COMPRESSED_DATA", ERR_LIB_CMS, CMS_R_TYPE_NOT_COMPRESSED_DATA}, - #else - {"TYPE_NOT_COMPRESSED_DATA", 46, 142}, - #endif - #ifdef CMS_R_TYPE_NOT_DATA - {"TYPE_NOT_DATA", ERR_LIB_CMS, CMS_R_TYPE_NOT_DATA}, - #else - {"TYPE_NOT_DATA", 46, 143}, - #endif - #ifdef CMS_R_TYPE_NOT_DIGESTED_DATA - {"TYPE_NOT_DIGESTED_DATA", ERR_LIB_CMS, CMS_R_TYPE_NOT_DIGESTED_DATA}, - #else - {"TYPE_NOT_DIGESTED_DATA", 46, 144}, - #endif - #ifdef CMS_R_TYPE_NOT_ENCRYPTED_DATA - {"TYPE_NOT_ENCRYPTED_DATA", ERR_LIB_CMS, CMS_R_TYPE_NOT_ENCRYPTED_DATA}, - #else - {"TYPE_NOT_ENCRYPTED_DATA", 46, 145}, - #endif - #ifdef CMS_R_TYPE_NOT_ENVELOPED_DATA - {"TYPE_NOT_ENVELOPED_DATA", ERR_LIB_CMS, CMS_R_TYPE_NOT_ENVELOPED_DATA}, - #else - {"TYPE_NOT_ENVELOPED_DATA", 46, 146}, - #endif - #ifdef CMS_R_UNABLE_TO_FINALIZE_CONTEXT - {"UNABLE_TO_FINALIZE_CONTEXT", ERR_LIB_CMS, CMS_R_UNABLE_TO_FINALIZE_CONTEXT}, - #else - {"UNABLE_TO_FINALIZE_CONTEXT", 46, 147}, - #endif - #ifdef CMS_R_UNKNOWN_CIPHER - {"UNKNOWN_CIPHER", ERR_LIB_CMS, CMS_R_UNKNOWN_CIPHER}, - #else - {"UNKNOWN_CIPHER", 46, 148}, - #endif - #ifdef CMS_R_UNKNOWN_DIGEST_ALGORITHM - {"UNKNOWN_DIGEST_ALGORITHM", ERR_LIB_CMS, CMS_R_UNKNOWN_DIGEST_ALGORITHM}, - #else - {"UNKNOWN_DIGEST_ALGORITHM", 46, 149}, - #endif - #ifdef CMS_R_UNKNOWN_ID - {"UNKNOWN_ID", ERR_LIB_CMS, CMS_R_UNKNOWN_ID}, - #else - {"UNKNOWN_ID", 46, 150}, - #endif - #ifdef CMS_R_UNSUPPORTED_COMPRESSION_ALGORITHM - {"UNSUPPORTED_COMPRESSION_ALGORITHM", ERR_LIB_CMS, CMS_R_UNSUPPORTED_COMPRESSION_ALGORITHM}, - #else - {"UNSUPPORTED_COMPRESSION_ALGORITHM", 46, 151}, - #endif - #ifdef CMS_R_UNSUPPORTED_CONTENT_TYPE - {"UNSUPPORTED_CONTENT_TYPE", ERR_LIB_CMS, CMS_R_UNSUPPORTED_CONTENT_TYPE}, - #else - {"UNSUPPORTED_CONTENT_TYPE", 46, 152}, - #endif - #ifdef CMS_R_UNSUPPORTED_KEK_ALGORITHM - {"UNSUPPORTED_KEK_ALGORITHM", ERR_LIB_CMS, CMS_R_UNSUPPORTED_KEK_ALGORITHM}, - #else - {"UNSUPPORTED_KEK_ALGORITHM", 46, 153}, - #endif - #ifdef CMS_R_UNSUPPORTED_KEY_ENCRYPTION_ALGORITHM - {"UNSUPPORTED_KEY_ENCRYPTION_ALGORITHM", ERR_LIB_CMS, CMS_R_UNSUPPORTED_KEY_ENCRYPTION_ALGORITHM}, - #else - {"UNSUPPORTED_KEY_ENCRYPTION_ALGORITHM", 46, 179}, - #endif - #ifdef CMS_R_UNSUPPORTED_RECIPIENTINFO_TYPE - {"UNSUPPORTED_RECIPIENTINFO_TYPE", ERR_LIB_CMS, CMS_R_UNSUPPORTED_RECIPIENTINFO_TYPE}, - #else - {"UNSUPPORTED_RECIPIENTINFO_TYPE", 46, 155}, - #endif - #ifdef CMS_R_UNSUPPORTED_RECIPIENT_TYPE - {"UNSUPPORTED_RECIPIENT_TYPE", ERR_LIB_CMS, CMS_R_UNSUPPORTED_RECIPIENT_TYPE}, - #else - {"UNSUPPORTED_RECIPIENT_TYPE", 46, 154}, - #endif - #ifdef CMS_R_UNSUPPORTED_TYPE - {"UNSUPPORTED_TYPE", ERR_LIB_CMS, CMS_R_UNSUPPORTED_TYPE}, - #else - {"UNSUPPORTED_TYPE", 46, 156}, - #endif - #ifdef CMS_R_UNWRAP_ERROR - {"UNWRAP_ERROR", ERR_LIB_CMS, CMS_R_UNWRAP_ERROR}, - #else - {"UNWRAP_ERROR", 46, 157}, - #endif - #ifdef CMS_R_UNWRAP_FAILURE - {"UNWRAP_FAILURE", ERR_LIB_CMS, CMS_R_UNWRAP_FAILURE}, - #else - {"UNWRAP_FAILURE", 46, 180}, - #endif - #ifdef CMS_R_VERIFICATION_FAILURE - {"VERIFICATION_FAILURE", ERR_LIB_CMS, CMS_R_VERIFICATION_FAILURE}, - #else - {"VERIFICATION_FAILURE", 46, 158}, - #endif - #ifdef CMS_R_WRAP_ERROR - {"WRAP_ERROR", ERR_LIB_CMS, CMS_R_WRAP_ERROR}, - #else - {"WRAP_ERROR", 46, 159}, - #endif - #ifdef COMP_R_ZLIB_DEFLATE_ERROR - {"ZLIB_DEFLATE_ERROR", ERR_LIB_COMP, COMP_R_ZLIB_DEFLATE_ERROR}, - #else - {"ZLIB_DEFLATE_ERROR", 41, 99}, - #endif - #ifdef COMP_R_ZLIB_INFLATE_ERROR - {"ZLIB_INFLATE_ERROR", ERR_LIB_COMP, COMP_R_ZLIB_INFLATE_ERROR}, - #else - {"ZLIB_INFLATE_ERROR", 41, 100}, - #endif - #ifdef COMP_R_ZLIB_NOT_SUPPORTED - {"ZLIB_NOT_SUPPORTED", ERR_LIB_COMP, COMP_R_ZLIB_NOT_SUPPORTED}, - #else - {"ZLIB_NOT_SUPPORTED", 41, 101}, - #endif - #ifdef CONF_R_ERROR_LOADING_DSO - {"ERROR_LOADING_DSO", ERR_LIB_CONF, CONF_R_ERROR_LOADING_DSO}, - #else - {"ERROR_LOADING_DSO", 14, 110}, - #endif - #ifdef CONF_R_LIST_CANNOT_BE_NULL - {"LIST_CANNOT_BE_NULL", ERR_LIB_CONF, CONF_R_LIST_CANNOT_BE_NULL}, - #else - {"LIST_CANNOT_BE_NULL", 14, 115}, - #endif - #ifdef CONF_R_MISSING_CLOSE_SQUARE_BRACKET - {"MISSING_CLOSE_SQUARE_BRACKET", ERR_LIB_CONF, CONF_R_MISSING_CLOSE_SQUARE_BRACKET}, - #else - {"MISSING_CLOSE_SQUARE_BRACKET", 14, 100}, - #endif - #ifdef CONF_R_MISSING_EQUAL_SIGN - {"MISSING_EQUAL_SIGN", ERR_LIB_CONF, CONF_R_MISSING_EQUAL_SIGN}, - #else - {"MISSING_EQUAL_SIGN", 14, 101}, - #endif - #ifdef CONF_R_MISSING_INIT_FUNCTION - {"MISSING_INIT_FUNCTION", ERR_LIB_CONF, CONF_R_MISSING_INIT_FUNCTION}, - #else - {"MISSING_INIT_FUNCTION", 14, 112}, - #endif - #ifdef CONF_R_MODULE_INITIALIZATION_ERROR - {"MODULE_INITIALIZATION_ERROR", ERR_LIB_CONF, CONF_R_MODULE_INITIALIZATION_ERROR}, - #else - {"MODULE_INITIALIZATION_ERROR", 14, 109}, - #endif - #ifdef CONF_R_NO_CLOSE_BRACE - {"NO_CLOSE_BRACE", ERR_LIB_CONF, CONF_R_NO_CLOSE_BRACE}, - #else - {"NO_CLOSE_BRACE", 14, 102}, - #endif - #ifdef CONF_R_NO_CONF - {"NO_CONF", ERR_LIB_CONF, CONF_R_NO_CONF}, - #else - {"NO_CONF", 14, 105}, - #endif - #ifdef CONF_R_NO_CONF_OR_ENVIRONMENT_VARIABLE - {"NO_CONF_OR_ENVIRONMENT_VARIABLE", ERR_LIB_CONF, CONF_R_NO_CONF_OR_ENVIRONMENT_VARIABLE}, - #else - {"NO_CONF_OR_ENVIRONMENT_VARIABLE", 14, 106}, - #endif - #ifdef CONF_R_NO_SECTION - {"NO_SECTION", ERR_LIB_CONF, CONF_R_NO_SECTION}, - #else - {"NO_SECTION", 14, 107}, - #endif - #ifdef CONF_R_NO_SUCH_FILE - {"NO_SUCH_FILE", ERR_LIB_CONF, CONF_R_NO_SUCH_FILE}, - #else - {"NO_SUCH_FILE", 14, 114}, - #endif - #ifdef CONF_R_NO_VALUE - {"NO_VALUE", ERR_LIB_CONF, CONF_R_NO_VALUE}, - #else - {"NO_VALUE", 14, 108}, - #endif - #ifdef CONF_R_NUMBER_TOO_LARGE - {"NUMBER_TOO_LARGE", ERR_LIB_CONF, CONF_R_NUMBER_TOO_LARGE}, - #else - {"NUMBER_TOO_LARGE", 14, 121}, - #endif - #ifdef CONF_R_RECURSIVE_DIRECTORY_INCLUDE - {"RECURSIVE_DIRECTORY_INCLUDE", ERR_LIB_CONF, CONF_R_RECURSIVE_DIRECTORY_INCLUDE}, - #else - {"RECURSIVE_DIRECTORY_INCLUDE", 14, 111}, - #endif - #ifdef CONF_R_SSL_COMMAND_SECTION_EMPTY - {"SSL_COMMAND_SECTION_EMPTY", ERR_LIB_CONF, CONF_R_SSL_COMMAND_SECTION_EMPTY}, - #else - {"SSL_COMMAND_SECTION_EMPTY", 14, 117}, - #endif - #ifdef CONF_R_SSL_COMMAND_SECTION_NOT_FOUND - {"SSL_COMMAND_SECTION_NOT_FOUND", ERR_LIB_CONF, CONF_R_SSL_COMMAND_SECTION_NOT_FOUND}, - #else - {"SSL_COMMAND_SECTION_NOT_FOUND", 14, 118}, - #endif - #ifdef CONF_R_SSL_SECTION_EMPTY - {"SSL_SECTION_EMPTY", ERR_LIB_CONF, CONF_R_SSL_SECTION_EMPTY}, - #else - {"SSL_SECTION_EMPTY", 14, 119}, - #endif - #ifdef CONF_R_SSL_SECTION_NOT_FOUND - {"SSL_SECTION_NOT_FOUND", ERR_LIB_CONF, CONF_R_SSL_SECTION_NOT_FOUND}, - #else - {"SSL_SECTION_NOT_FOUND", 14, 120}, - #endif - #ifdef CONF_R_UNABLE_TO_CREATE_NEW_SECTION - {"UNABLE_TO_CREATE_NEW_SECTION", ERR_LIB_CONF, CONF_R_UNABLE_TO_CREATE_NEW_SECTION}, - #else - {"UNABLE_TO_CREATE_NEW_SECTION", 14, 103}, - #endif - #ifdef CONF_R_UNKNOWN_MODULE_NAME - {"UNKNOWN_MODULE_NAME", ERR_LIB_CONF, CONF_R_UNKNOWN_MODULE_NAME}, - #else - {"UNKNOWN_MODULE_NAME", 14, 113}, - #endif - #ifdef CONF_R_VARIABLE_EXPANSION_TOO_LONG - {"VARIABLE_EXPANSION_TOO_LONG", ERR_LIB_CONF, CONF_R_VARIABLE_EXPANSION_TOO_LONG}, - #else - {"VARIABLE_EXPANSION_TOO_LONG", 14, 116}, - #endif - #ifdef CONF_R_VARIABLE_HAS_NO_VALUE - {"VARIABLE_HAS_NO_VALUE", ERR_LIB_CONF, CONF_R_VARIABLE_HAS_NO_VALUE}, - #else - {"VARIABLE_HAS_NO_VALUE", 14, 104}, - #endif - #ifdef CRYPTO_R_FIPS_MODE_NOT_SUPPORTED - {"FIPS_MODE_NOT_SUPPORTED", ERR_LIB_CRYPTO, CRYPTO_R_FIPS_MODE_NOT_SUPPORTED}, - #else - {"FIPS_MODE_NOT_SUPPORTED", 15, 101}, - #endif - #ifdef CRYPTO_R_ILLEGAL_HEX_DIGIT - {"ILLEGAL_HEX_DIGIT", ERR_LIB_CRYPTO, CRYPTO_R_ILLEGAL_HEX_DIGIT}, - #else - {"ILLEGAL_HEX_DIGIT", 15, 102}, - #endif - #ifdef CRYPTO_R_ODD_NUMBER_OF_DIGITS - {"ODD_NUMBER_OF_DIGITS", ERR_LIB_CRYPTO, CRYPTO_R_ODD_NUMBER_OF_DIGITS}, - #else - {"ODD_NUMBER_OF_DIGITS", 15, 103}, - #endif - #ifdef CT_R_BASE64_DECODE_ERROR - {"BASE64_DECODE_ERROR", ERR_LIB_CT, CT_R_BASE64_DECODE_ERROR}, - #else - {"BASE64_DECODE_ERROR", 50, 108}, - #endif - #ifdef CT_R_INVALID_LOG_ID_LENGTH - {"INVALID_LOG_ID_LENGTH", ERR_LIB_CT, CT_R_INVALID_LOG_ID_LENGTH}, - #else - {"INVALID_LOG_ID_LENGTH", 50, 100}, - #endif - #ifdef CT_R_LOG_CONF_INVALID - {"LOG_CONF_INVALID", ERR_LIB_CT, CT_R_LOG_CONF_INVALID}, - #else - {"LOG_CONF_INVALID", 50, 109}, - #endif - #ifdef CT_R_LOG_CONF_INVALID_KEY - {"LOG_CONF_INVALID_KEY", ERR_LIB_CT, CT_R_LOG_CONF_INVALID_KEY}, - #else - {"LOG_CONF_INVALID_KEY", 50, 110}, - #endif - #ifdef CT_R_LOG_CONF_MISSING_DESCRIPTION - {"LOG_CONF_MISSING_DESCRIPTION", ERR_LIB_CT, CT_R_LOG_CONF_MISSING_DESCRIPTION}, - #else - {"LOG_CONF_MISSING_DESCRIPTION", 50, 111}, - #endif - #ifdef CT_R_LOG_CONF_MISSING_KEY - {"LOG_CONF_MISSING_KEY", ERR_LIB_CT, CT_R_LOG_CONF_MISSING_KEY}, - #else - {"LOG_CONF_MISSING_KEY", 50, 112}, - #endif - #ifdef CT_R_LOG_KEY_INVALID - {"LOG_KEY_INVALID", ERR_LIB_CT, CT_R_LOG_KEY_INVALID}, - #else - {"LOG_KEY_INVALID", 50, 113}, - #endif - #ifdef CT_R_SCT_FUTURE_TIMESTAMP - {"SCT_FUTURE_TIMESTAMP", ERR_LIB_CT, CT_R_SCT_FUTURE_TIMESTAMP}, - #else - {"SCT_FUTURE_TIMESTAMP", 50, 116}, - #endif - #ifdef CT_R_SCT_INVALID - {"SCT_INVALID", ERR_LIB_CT, CT_R_SCT_INVALID}, - #else - {"SCT_INVALID", 50, 104}, - #endif - #ifdef CT_R_SCT_INVALID_SIGNATURE - {"SCT_INVALID_SIGNATURE", ERR_LIB_CT, CT_R_SCT_INVALID_SIGNATURE}, - #else - {"SCT_INVALID_SIGNATURE", 50, 107}, - #endif - #ifdef CT_R_SCT_LIST_INVALID - {"SCT_LIST_INVALID", ERR_LIB_CT, CT_R_SCT_LIST_INVALID}, - #else - {"SCT_LIST_INVALID", 50, 105}, - #endif - #ifdef CT_R_SCT_LOG_ID_MISMATCH - {"SCT_LOG_ID_MISMATCH", ERR_LIB_CT, CT_R_SCT_LOG_ID_MISMATCH}, - #else - {"SCT_LOG_ID_MISMATCH", 50, 114}, - #endif - #ifdef CT_R_SCT_NOT_SET - {"SCT_NOT_SET", ERR_LIB_CT, CT_R_SCT_NOT_SET}, - #else - {"SCT_NOT_SET", 50, 106}, - #endif - #ifdef CT_R_SCT_UNSUPPORTED_VERSION - {"SCT_UNSUPPORTED_VERSION", ERR_LIB_CT, CT_R_SCT_UNSUPPORTED_VERSION}, - #else - {"SCT_UNSUPPORTED_VERSION", 50, 115}, - #endif - #ifdef CT_R_UNRECOGNIZED_SIGNATURE_NID - {"UNRECOGNIZED_SIGNATURE_NID", ERR_LIB_CT, CT_R_UNRECOGNIZED_SIGNATURE_NID}, - #else - {"UNRECOGNIZED_SIGNATURE_NID", 50, 101}, - #endif - #ifdef CT_R_UNSUPPORTED_ENTRY_TYPE - {"UNSUPPORTED_ENTRY_TYPE", ERR_LIB_CT, CT_R_UNSUPPORTED_ENTRY_TYPE}, - #else - {"UNSUPPORTED_ENTRY_TYPE", 50, 102}, - #endif - #ifdef CT_R_UNSUPPORTED_VERSION - {"UNSUPPORTED_VERSION", ERR_LIB_CT, CT_R_UNSUPPORTED_VERSION}, - #else - {"UNSUPPORTED_VERSION", 50, 103}, - #endif - #ifdef DH_R_BAD_GENERATOR - {"BAD_GENERATOR", ERR_LIB_DH, DH_R_BAD_GENERATOR}, - #else - {"BAD_GENERATOR", 5, 101}, - #endif - #ifdef DH_R_BN_DECODE_ERROR - {"BN_DECODE_ERROR", ERR_LIB_DH, DH_R_BN_DECODE_ERROR}, - #else - {"BN_DECODE_ERROR", 5, 109}, - #endif - #ifdef DH_R_BN_ERROR - {"BN_ERROR", ERR_LIB_DH, DH_R_BN_ERROR}, - #else - {"BN_ERROR", 5, 106}, - #endif - #ifdef DH_R_CHECK_INVALID_J_VALUE - {"CHECK_INVALID_J_VALUE", ERR_LIB_DH, DH_R_CHECK_INVALID_J_VALUE}, - #else - {"CHECK_INVALID_J_VALUE", 5, 115}, - #endif - #ifdef DH_R_CHECK_INVALID_Q_VALUE - {"CHECK_INVALID_Q_VALUE", ERR_LIB_DH, DH_R_CHECK_INVALID_Q_VALUE}, - #else - {"CHECK_INVALID_Q_VALUE", 5, 116}, - #endif - #ifdef DH_R_CHECK_PUBKEY_INVALID - {"CHECK_PUBKEY_INVALID", ERR_LIB_DH, DH_R_CHECK_PUBKEY_INVALID}, - #else - {"CHECK_PUBKEY_INVALID", 5, 122}, - #endif - #ifdef DH_R_CHECK_PUBKEY_TOO_LARGE - {"CHECK_PUBKEY_TOO_LARGE", ERR_LIB_DH, DH_R_CHECK_PUBKEY_TOO_LARGE}, - #else - {"CHECK_PUBKEY_TOO_LARGE", 5, 123}, - #endif - #ifdef DH_R_CHECK_PUBKEY_TOO_SMALL - {"CHECK_PUBKEY_TOO_SMALL", ERR_LIB_DH, DH_R_CHECK_PUBKEY_TOO_SMALL}, - #else - {"CHECK_PUBKEY_TOO_SMALL", 5, 124}, - #endif - #ifdef DH_R_CHECK_P_NOT_PRIME - {"CHECK_P_NOT_PRIME", ERR_LIB_DH, DH_R_CHECK_P_NOT_PRIME}, - #else - {"CHECK_P_NOT_PRIME", 5, 117}, - #endif - #ifdef DH_R_CHECK_P_NOT_SAFE_PRIME - {"CHECK_P_NOT_SAFE_PRIME", ERR_LIB_DH, DH_R_CHECK_P_NOT_SAFE_PRIME}, - #else - {"CHECK_P_NOT_SAFE_PRIME", 5, 118}, - #endif - #ifdef DH_R_CHECK_Q_NOT_PRIME - {"CHECK_Q_NOT_PRIME", ERR_LIB_DH, DH_R_CHECK_Q_NOT_PRIME}, - #else - {"CHECK_Q_NOT_PRIME", 5, 119}, - #endif - #ifdef DH_R_DECODE_ERROR - {"DECODE_ERROR", ERR_LIB_DH, DH_R_DECODE_ERROR}, - #else - {"DECODE_ERROR", 5, 104}, - #endif - #ifdef DH_R_INVALID_PARAMETER_NAME - {"INVALID_PARAMETER_NAME", ERR_LIB_DH, DH_R_INVALID_PARAMETER_NAME}, - #else - {"INVALID_PARAMETER_NAME", 5, 110}, - #endif - #ifdef DH_R_INVALID_PARAMETER_NID - {"INVALID_PARAMETER_NID", ERR_LIB_DH, DH_R_INVALID_PARAMETER_NID}, - #else - {"INVALID_PARAMETER_NID", 5, 114}, - #endif - #ifdef DH_R_INVALID_PUBKEY - {"INVALID_PUBKEY", ERR_LIB_DH, DH_R_INVALID_PUBKEY}, - #else - {"INVALID_PUBKEY", 5, 102}, - #endif - #ifdef DH_R_KDF_PARAMETER_ERROR - {"KDF_PARAMETER_ERROR", ERR_LIB_DH, DH_R_KDF_PARAMETER_ERROR}, - #else - {"KDF_PARAMETER_ERROR", 5, 112}, - #endif - #ifdef DH_R_KEYS_NOT_SET - {"KEYS_NOT_SET", ERR_LIB_DH, DH_R_KEYS_NOT_SET}, - #else - {"KEYS_NOT_SET", 5, 108}, - #endif - #ifdef DH_R_MISSING_PUBKEY - {"MISSING_PUBKEY", ERR_LIB_DH, DH_R_MISSING_PUBKEY}, - #else - {"MISSING_PUBKEY", 5, 125}, - #endif - #ifdef DH_R_MODULUS_TOO_LARGE - {"MODULUS_TOO_LARGE", ERR_LIB_DH, DH_R_MODULUS_TOO_LARGE}, - #else - {"MODULUS_TOO_LARGE", 5, 103}, - #endif - #ifdef DH_R_NOT_SUITABLE_GENERATOR - {"NOT_SUITABLE_GENERATOR", ERR_LIB_DH, DH_R_NOT_SUITABLE_GENERATOR}, - #else - {"NOT_SUITABLE_GENERATOR", 5, 120}, - #endif - #ifdef DH_R_NO_PARAMETERS_SET - {"NO_PARAMETERS_SET", ERR_LIB_DH, DH_R_NO_PARAMETERS_SET}, - #else - {"NO_PARAMETERS_SET", 5, 107}, - #endif - #ifdef DH_R_NO_PRIVATE_VALUE - {"NO_PRIVATE_VALUE", ERR_LIB_DH, DH_R_NO_PRIVATE_VALUE}, - #else - {"NO_PRIVATE_VALUE", 5, 100}, - #endif - #ifdef DH_R_PARAMETER_ENCODING_ERROR - {"PARAMETER_ENCODING_ERROR", ERR_LIB_DH, DH_R_PARAMETER_ENCODING_ERROR}, - #else - {"PARAMETER_ENCODING_ERROR", 5, 105}, - #endif - #ifdef DH_R_PEER_KEY_ERROR - {"PEER_KEY_ERROR", ERR_LIB_DH, DH_R_PEER_KEY_ERROR}, - #else - {"PEER_KEY_ERROR", 5, 111}, - #endif - #ifdef DH_R_SHARED_INFO_ERROR - {"SHARED_INFO_ERROR", ERR_LIB_DH, DH_R_SHARED_INFO_ERROR}, - #else - {"SHARED_INFO_ERROR", 5, 113}, - #endif - #ifdef DH_R_UNABLE_TO_CHECK_GENERATOR - {"UNABLE_TO_CHECK_GENERATOR", ERR_LIB_DH, DH_R_UNABLE_TO_CHECK_GENERATOR}, - #else - {"UNABLE_TO_CHECK_GENERATOR", 5, 121}, - #endif - #ifdef DSA_R_BAD_Q_VALUE - {"BAD_Q_VALUE", ERR_LIB_DSA, DSA_R_BAD_Q_VALUE}, - #else - {"BAD_Q_VALUE", 10, 102}, - #endif - #ifdef DSA_R_BN_DECODE_ERROR - {"BN_DECODE_ERROR", ERR_LIB_DSA, DSA_R_BN_DECODE_ERROR}, - #else - {"BN_DECODE_ERROR", 10, 108}, - #endif - #ifdef DSA_R_BN_ERROR - {"BN_ERROR", ERR_LIB_DSA, DSA_R_BN_ERROR}, - #else - {"BN_ERROR", 10, 109}, - #endif - #ifdef DSA_R_DECODE_ERROR - {"DECODE_ERROR", ERR_LIB_DSA, DSA_R_DECODE_ERROR}, - #else - {"DECODE_ERROR", 10, 104}, - #endif - #ifdef DSA_R_INVALID_DIGEST_TYPE - {"INVALID_DIGEST_TYPE", ERR_LIB_DSA, DSA_R_INVALID_DIGEST_TYPE}, - #else - {"INVALID_DIGEST_TYPE", 10, 106}, - #endif - #ifdef DSA_R_INVALID_PARAMETERS - {"INVALID_PARAMETERS", ERR_LIB_DSA, DSA_R_INVALID_PARAMETERS}, - #else - {"INVALID_PARAMETERS", 10, 112}, - #endif - #ifdef DSA_R_MISSING_PARAMETERS - {"MISSING_PARAMETERS", ERR_LIB_DSA, DSA_R_MISSING_PARAMETERS}, - #else - {"MISSING_PARAMETERS", 10, 101}, - #endif - #ifdef DSA_R_MISSING_PRIVATE_KEY - {"MISSING_PRIVATE_KEY", ERR_LIB_DSA, DSA_R_MISSING_PRIVATE_KEY}, - #else - {"MISSING_PRIVATE_KEY", 10, 111}, - #endif - #ifdef DSA_R_MODULUS_TOO_LARGE - {"MODULUS_TOO_LARGE", ERR_LIB_DSA, DSA_R_MODULUS_TOO_LARGE}, - #else - {"MODULUS_TOO_LARGE", 10, 103}, - #endif - #ifdef DSA_R_NO_PARAMETERS_SET - {"NO_PARAMETERS_SET", ERR_LIB_DSA, DSA_R_NO_PARAMETERS_SET}, - #else - {"NO_PARAMETERS_SET", 10, 107}, - #endif - #ifdef DSA_R_PARAMETER_ENCODING_ERROR - {"PARAMETER_ENCODING_ERROR", ERR_LIB_DSA, DSA_R_PARAMETER_ENCODING_ERROR}, - #else - {"PARAMETER_ENCODING_ERROR", 10, 105}, - #endif - #ifdef DSA_R_Q_NOT_PRIME - {"Q_NOT_PRIME", ERR_LIB_DSA, DSA_R_Q_NOT_PRIME}, - #else - {"Q_NOT_PRIME", 10, 113}, - #endif - #ifdef DSA_R_SEED_LEN_SMALL - {"SEED_LEN_SMALL", ERR_LIB_DSA, DSA_R_SEED_LEN_SMALL}, - #else - {"SEED_LEN_SMALL", 10, 110}, - #endif - #ifdef EC_R_ASN1_ERROR - {"ASN1_ERROR", ERR_LIB_EC, EC_R_ASN1_ERROR}, - #else - {"ASN1_ERROR", 16, 115}, - #endif - #ifdef EC_R_BAD_SIGNATURE - {"BAD_SIGNATURE", ERR_LIB_EC, EC_R_BAD_SIGNATURE}, - #else - {"BAD_SIGNATURE", 16, 156}, - #endif - #ifdef EC_R_BIGNUM_OUT_OF_RANGE - {"BIGNUM_OUT_OF_RANGE", ERR_LIB_EC, EC_R_BIGNUM_OUT_OF_RANGE}, - #else - {"BIGNUM_OUT_OF_RANGE", 16, 144}, - #endif - #ifdef EC_R_BUFFER_TOO_SMALL - {"BUFFER_TOO_SMALL", ERR_LIB_EC, EC_R_BUFFER_TOO_SMALL}, - #else - {"BUFFER_TOO_SMALL", 16, 100}, - #endif - #ifdef EC_R_CANNOT_INVERT - {"CANNOT_INVERT", ERR_LIB_EC, EC_R_CANNOT_INVERT}, - #else - {"CANNOT_INVERT", 16, 165}, - #endif - #ifdef EC_R_COORDINATES_OUT_OF_RANGE - {"COORDINATES_OUT_OF_RANGE", ERR_LIB_EC, EC_R_COORDINATES_OUT_OF_RANGE}, - #else - {"COORDINATES_OUT_OF_RANGE", 16, 146}, - #endif - #ifdef EC_R_CURVE_DOES_NOT_SUPPORT_ECDH - {"CURVE_DOES_NOT_SUPPORT_ECDH", ERR_LIB_EC, EC_R_CURVE_DOES_NOT_SUPPORT_ECDH}, - #else - {"CURVE_DOES_NOT_SUPPORT_ECDH", 16, 160}, - #endif - #ifdef EC_R_CURVE_DOES_NOT_SUPPORT_SIGNING - {"CURVE_DOES_NOT_SUPPORT_SIGNING", ERR_LIB_EC, EC_R_CURVE_DOES_NOT_SUPPORT_SIGNING}, - #else - {"CURVE_DOES_NOT_SUPPORT_SIGNING", 16, 159}, - #endif - #ifdef EC_R_D2I_ECPKPARAMETERS_FAILURE - {"D2I_ECPKPARAMETERS_FAILURE", ERR_LIB_EC, EC_R_D2I_ECPKPARAMETERS_FAILURE}, - #else - {"D2I_ECPKPARAMETERS_FAILURE", 16, 117}, - #endif - #ifdef EC_R_DECODE_ERROR - {"DECODE_ERROR", ERR_LIB_EC, EC_R_DECODE_ERROR}, - #else - {"DECODE_ERROR", 16, 142}, - #endif - #ifdef EC_R_DISCRIMINANT_IS_ZERO - {"DISCRIMINANT_IS_ZERO", ERR_LIB_EC, EC_R_DISCRIMINANT_IS_ZERO}, - #else - {"DISCRIMINANT_IS_ZERO", 16, 118}, - #endif - #ifdef EC_R_EC_GROUP_NEW_BY_NAME_FAILURE - {"EC_GROUP_NEW_BY_NAME_FAILURE", ERR_LIB_EC, EC_R_EC_GROUP_NEW_BY_NAME_FAILURE}, - #else - {"EC_GROUP_NEW_BY_NAME_FAILURE", 16, 119}, - #endif - #ifdef EC_R_FIELD_TOO_LARGE - {"FIELD_TOO_LARGE", ERR_LIB_EC, EC_R_FIELD_TOO_LARGE}, - #else - {"FIELD_TOO_LARGE", 16, 143}, - #endif - #ifdef EC_R_GF2M_NOT_SUPPORTED - {"GF2M_NOT_SUPPORTED", ERR_LIB_EC, EC_R_GF2M_NOT_SUPPORTED}, - #else - {"GF2M_NOT_SUPPORTED", 16, 147}, - #endif - #ifdef EC_R_GROUP2PKPARAMETERS_FAILURE - {"GROUP2PKPARAMETERS_FAILURE", ERR_LIB_EC, EC_R_GROUP2PKPARAMETERS_FAILURE}, - #else - {"GROUP2PKPARAMETERS_FAILURE", 16, 120}, - #endif - #ifdef EC_R_I2D_ECPKPARAMETERS_FAILURE - {"I2D_ECPKPARAMETERS_FAILURE", ERR_LIB_EC, EC_R_I2D_ECPKPARAMETERS_FAILURE}, - #else - {"I2D_ECPKPARAMETERS_FAILURE", 16, 121}, - #endif - #ifdef EC_R_INCOMPATIBLE_OBJECTS - {"INCOMPATIBLE_OBJECTS", ERR_LIB_EC, EC_R_INCOMPATIBLE_OBJECTS}, - #else - {"INCOMPATIBLE_OBJECTS", 16, 101}, - #endif - #ifdef EC_R_INVALID_ARGUMENT - {"INVALID_ARGUMENT", ERR_LIB_EC, EC_R_INVALID_ARGUMENT}, - #else - {"INVALID_ARGUMENT", 16, 112}, - #endif - #ifdef EC_R_INVALID_COMPRESSED_POINT - {"INVALID_COMPRESSED_POINT", ERR_LIB_EC, EC_R_INVALID_COMPRESSED_POINT}, - #else - {"INVALID_COMPRESSED_POINT", 16, 110}, - #endif - #ifdef EC_R_INVALID_COMPRESSION_BIT - {"INVALID_COMPRESSION_BIT", ERR_LIB_EC, EC_R_INVALID_COMPRESSION_BIT}, - #else - {"INVALID_COMPRESSION_BIT", 16, 109}, - #endif - #ifdef EC_R_INVALID_CURVE - {"INVALID_CURVE", ERR_LIB_EC, EC_R_INVALID_CURVE}, - #else - {"INVALID_CURVE", 16, 141}, - #endif - #ifdef EC_R_INVALID_DIGEST - {"INVALID_DIGEST", ERR_LIB_EC, EC_R_INVALID_DIGEST}, - #else - {"INVALID_DIGEST", 16, 151}, - #endif - #ifdef EC_R_INVALID_DIGEST_TYPE - {"INVALID_DIGEST_TYPE", ERR_LIB_EC, EC_R_INVALID_DIGEST_TYPE}, - #else - {"INVALID_DIGEST_TYPE", 16, 138}, - #endif - #ifdef EC_R_INVALID_ENCODING - {"INVALID_ENCODING", ERR_LIB_EC, EC_R_INVALID_ENCODING}, - #else - {"INVALID_ENCODING", 16, 102}, - #endif - #ifdef EC_R_INVALID_FIELD - {"INVALID_FIELD", ERR_LIB_EC, EC_R_INVALID_FIELD}, - #else - {"INVALID_FIELD", 16, 103}, - #endif - #ifdef EC_R_INVALID_FORM - {"INVALID_FORM", ERR_LIB_EC, EC_R_INVALID_FORM}, - #else - {"INVALID_FORM", 16, 104}, - #endif - #ifdef EC_R_INVALID_GROUP_ORDER - {"INVALID_GROUP_ORDER", ERR_LIB_EC, EC_R_INVALID_GROUP_ORDER}, - #else - {"INVALID_GROUP_ORDER", 16, 122}, - #endif - #ifdef EC_R_INVALID_KEY - {"INVALID_KEY", ERR_LIB_EC, EC_R_INVALID_KEY}, - #else - {"INVALID_KEY", 16, 116}, - #endif - #ifdef EC_R_INVALID_OUTPUT_LENGTH - {"INVALID_OUTPUT_LENGTH", ERR_LIB_EC, EC_R_INVALID_OUTPUT_LENGTH}, - #else - {"INVALID_OUTPUT_LENGTH", 16, 161}, - #endif - #ifdef EC_R_INVALID_PEER_KEY - {"INVALID_PEER_KEY", ERR_LIB_EC, EC_R_INVALID_PEER_KEY}, - #else - {"INVALID_PEER_KEY", 16, 133}, - #endif - #ifdef EC_R_INVALID_PENTANOMIAL_BASIS - {"INVALID_PENTANOMIAL_BASIS", ERR_LIB_EC, EC_R_INVALID_PENTANOMIAL_BASIS}, - #else - {"INVALID_PENTANOMIAL_BASIS", 16, 132}, - #endif - #ifdef EC_R_INVALID_PRIVATE_KEY - {"INVALID_PRIVATE_KEY", ERR_LIB_EC, EC_R_INVALID_PRIVATE_KEY}, - #else - {"INVALID_PRIVATE_KEY", 16, 123}, - #endif - #ifdef EC_R_INVALID_TRINOMIAL_BASIS - {"INVALID_TRINOMIAL_BASIS", ERR_LIB_EC, EC_R_INVALID_TRINOMIAL_BASIS}, - #else - {"INVALID_TRINOMIAL_BASIS", 16, 137}, - #endif - #ifdef EC_R_KDF_PARAMETER_ERROR - {"KDF_PARAMETER_ERROR", ERR_LIB_EC, EC_R_KDF_PARAMETER_ERROR}, - #else - {"KDF_PARAMETER_ERROR", 16, 148}, - #endif - #ifdef EC_R_KEYS_NOT_SET - {"KEYS_NOT_SET", ERR_LIB_EC, EC_R_KEYS_NOT_SET}, - #else - {"KEYS_NOT_SET", 16, 140}, - #endif - #ifdef EC_R_LADDER_POST_FAILURE - {"LADDER_POST_FAILURE", ERR_LIB_EC, EC_R_LADDER_POST_FAILURE}, - #else - {"LADDER_POST_FAILURE", 16, 136}, - #endif - #ifdef EC_R_LADDER_PRE_FAILURE - {"LADDER_PRE_FAILURE", ERR_LIB_EC, EC_R_LADDER_PRE_FAILURE}, - #else - {"LADDER_PRE_FAILURE", 16, 153}, - #endif - #ifdef EC_R_LADDER_STEP_FAILURE - {"LADDER_STEP_FAILURE", ERR_LIB_EC, EC_R_LADDER_STEP_FAILURE}, - #else - {"LADDER_STEP_FAILURE", 16, 162}, - #endif - #ifdef EC_R_MISSING_PARAMETERS - {"MISSING_PARAMETERS", ERR_LIB_EC, EC_R_MISSING_PARAMETERS}, - #else - {"MISSING_PARAMETERS", 16, 124}, - #endif - #ifdef EC_R_MISSING_PRIVATE_KEY - {"MISSING_PRIVATE_KEY", ERR_LIB_EC, EC_R_MISSING_PRIVATE_KEY}, - #else - {"MISSING_PRIVATE_KEY", 16, 125}, - #endif - #ifdef EC_R_NEED_NEW_SETUP_VALUES - {"NEED_NEW_SETUP_VALUES", ERR_LIB_EC, EC_R_NEED_NEW_SETUP_VALUES}, - #else - {"NEED_NEW_SETUP_VALUES", 16, 157}, - #endif - #ifdef EC_R_NOT_A_NIST_PRIME - {"NOT_A_NIST_PRIME", ERR_LIB_EC, EC_R_NOT_A_NIST_PRIME}, - #else - {"NOT_A_NIST_PRIME", 16, 135}, - #endif - #ifdef EC_R_NOT_IMPLEMENTED - {"NOT_IMPLEMENTED", ERR_LIB_EC, EC_R_NOT_IMPLEMENTED}, - #else - {"NOT_IMPLEMENTED", 16, 126}, - #endif - #ifdef EC_R_NOT_INITIALIZED - {"NOT_INITIALIZED", ERR_LIB_EC, EC_R_NOT_INITIALIZED}, - #else - {"NOT_INITIALIZED", 16, 111}, - #endif - #ifdef EC_R_NO_PARAMETERS_SET - {"NO_PARAMETERS_SET", ERR_LIB_EC, EC_R_NO_PARAMETERS_SET}, - #else - {"NO_PARAMETERS_SET", 16, 139}, - #endif - #ifdef EC_R_NO_PRIVATE_VALUE - {"NO_PRIVATE_VALUE", ERR_LIB_EC, EC_R_NO_PRIVATE_VALUE}, - #else - {"NO_PRIVATE_VALUE", 16, 154}, - #endif - #ifdef EC_R_OPERATION_NOT_SUPPORTED - {"OPERATION_NOT_SUPPORTED", ERR_LIB_EC, EC_R_OPERATION_NOT_SUPPORTED}, - #else - {"OPERATION_NOT_SUPPORTED", 16, 152}, - #endif - #ifdef EC_R_PASSED_NULL_PARAMETER - {"PASSED_NULL_PARAMETER", ERR_LIB_EC, EC_R_PASSED_NULL_PARAMETER}, - #else - {"PASSED_NULL_PARAMETER", 16, 134}, - #endif - #ifdef EC_R_PEER_KEY_ERROR - {"PEER_KEY_ERROR", ERR_LIB_EC, EC_R_PEER_KEY_ERROR}, - #else - {"PEER_KEY_ERROR", 16, 149}, - #endif - #ifdef EC_R_PKPARAMETERS2GROUP_FAILURE - {"PKPARAMETERS2GROUP_FAILURE", ERR_LIB_EC, EC_R_PKPARAMETERS2GROUP_FAILURE}, - #else - {"PKPARAMETERS2GROUP_FAILURE", 16, 127}, - #endif - #ifdef EC_R_POINT_ARITHMETIC_FAILURE - {"POINT_ARITHMETIC_FAILURE", ERR_LIB_EC, EC_R_POINT_ARITHMETIC_FAILURE}, - #else - {"POINT_ARITHMETIC_FAILURE", 16, 155}, - #endif - #ifdef EC_R_POINT_AT_INFINITY - {"POINT_AT_INFINITY", ERR_LIB_EC, EC_R_POINT_AT_INFINITY}, - #else - {"POINT_AT_INFINITY", 16, 106}, - #endif - #ifdef EC_R_POINT_COORDINATES_BLIND_FAILURE - {"POINT_COORDINATES_BLIND_FAILURE", ERR_LIB_EC, EC_R_POINT_COORDINATES_BLIND_FAILURE}, - #else - {"POINT_COORDINATES_BLIND_FAILURE", 16, 163}, - #endif - #ifdef EC_R_POINT_IS_NOT_ON_CURVE - {"POINT_IS_NOT_ON_CURVE", ERR_LIB_EC, EC_R_POINT_IS_NOT_ON_CURVE}, - #else - {"POINT_IS_NOT_ON_CURVE", 16, 107}, - #endif - #ifdef EC_R_RANDOM_NUMBER_GENERATION_FAILED - {"RANDOM_NUMBER_GENERATION_FAILED", ERR_LIB_EC, EC_R_RANDOM_NUMBER_GENERATION_FAILED}, - #else - {"RANDOM_NUMBER_GENERATION_FAILED", 16, 158}, - #endif - #ifdef EC_R_SHARED_INFO_ERROR - {"SHARED_INFO_ERROR", ERR_LIB_EC, EC_R_SHARED_INFO_ERROR}, - #else - {"SHARED_INFO_ERROR", 16, 150}, - #endif - #ifdef EC_R_SLOT_FULL - {"SLOT_FULL", ERR_LIB_EC, EC_R_SLOT_FULL}, - #else - {"SLOT_FULL", 16, 108}, - #endif - #ifdef EC_R_UNDEFINED_GENERATOR - {"UNDEFINED_GENERATOR", ERR_LIB_EC, EC_R_UNDEFINED_GENERATOR}, - #else - {"UNDEFINED_GENERATOR", 16, 113}, - #endif - #ifdef EC_R_UNDEFINED_ORDER - {"UNDEFINED_ORDER", ERR_LIB_EC, EC_R_UNDEFINED_ORDER}, - #else - {"UNDEFINED_ORDER", 16, 128}, - #endif - #ifdef EC_R_UNKNOWN_COFACTOR - {"UNKNOWN_COFACTOR", ERR_LIB_EC, EC_R_UNKNOWN_COFACTOR}, - #else - {"UNKNOWN_COFACTOR", 16, 164}, - #endif - #ifdef EC_R_UNKNOWN_GROUP - {"UNKNOWN_GROUP", ERR_LIB_EC, EC_R_UNKNOWN_GROUP}, - #else - {"UNKNOWN_GROUP", 16, 129}, - #endif - #ifdef EC_R_UNKNOWN_ORDER - {"UNKNOWN_ORDER", ERR_LIB_EC, EC_R_UNKNOWN_ORDER}, - #else - {"UNKNOWN_ORDER", 16, 114}, - #endif - #ifdef EC_R_UNSUPPORTED_FIELD - {"UNSUPPORTED_FIELD", ERR_LIB_EC, EC_R_UNSUPPORTED_FIELD}, - #else - {"UNSUPPORTED_FIELD", 16, 131}, - #endif - #ifdef EC_R_WRONG_CURVE_PARAMETERS - {"WRONG_CURVE_PARAMETERS", ERR_LIB_EC, EC_R_WRONG_CURVE_PARAMETERS}, - #else - {"WRONG_CURVE_PARAMETERS", 16, 145}, - #endif - #ifdef EC_R_WRONG_ORDER - {"WRONG_ORDER", ERR_LIB_EC, EC_R_WRONG_ORDER}, - #else - {"WRONG_ORDER", 16, 130}, - #endif - #ifdef ENGINE_R_ALREADY_LOADED - {"ALREADY_LOADED", ERR_LIB_ENGINE, ENGINE_R_ALREADY_LOADED}, - #else - {"ALREADY_LOADED", 38, 100}, - #endif - #ifdef ENGINE_R_ARGUMENT_IS_NOT_A_NUMBER - {"ARGUMENT_IS_NOT_A_NUMBER", ERR_LIB_ENGINE, ENGINE_R_ARGUMENT_IS_NOT_A_NUMBER}, - #else - {"ARGUMENT_IS_NOT_A_NUMBER", 38, 133}, - #endif - #ifdef ENGINE_R_CMD_NOT_EXECUTABLE - {"CMD_NOT_EXECUTABLE", ERR_LIB_ENGINE, ENGINE_R_CMD_NOT_EXECUTABLE}, - #else - {"CMD_NOT_EXECUTABLE", 38, 134}, - #endif - #ifdef ENGINE_R_COMMAND_TAKES_INPUT - {"COMMAND_TAKES_INPUT", ERR_LIB_ENGINE, ENGINE_R_COMMAND_TAKES_INPUT}, - #else - {"COMMAND_TAKES_INPUT", 38, 135}, - #endif - #ifdef ENGINE_R_COMMAND_TAKES_NO_INPUT - {"COMMAND_TAKES_NO_INPUT", ERR_LIB_ENGINE, ENGINE_R_COMMAND_TAKES_NO_INPUT}, - #else - {"COMMAND_TAKES_NO_INPUT", 38, 136}, - #endif - #ifdef ENGINE_R_CONFLICTING_ENGINE_ID - {"CONFLICTING_ENGINE_ID", ERR_LIB_ENGINE, ENGINE_R_CONFLICTING_ENGINE_ID}, - #else - {"CONFLICTING_ENGINE_ID", 38, 103}, - #endif - #ifdef ENGINE_R_CTRL_COMMAND_NOT_IMPLEMENTED - {"CTRL_COMMAND_NOT_IMPLEMENTED", ERR_LIB_ENGINE, ENGINE_R_CTRL_COMMAND_NOT_IMPLEMENTED}, - #else - {"CTRL_COMMAND_NOT_IMPLEMENTED", 38, 119}, - #endif - #ifdef ENGINE_R_DSO_FAILURE - {"DSO_FAILURE", ERR_LIB_ENGINE, ENGINE_R_DSO_FAILURE}, - #else - {"DSO_FAILURE", 38, 104}, - #endif - #ifdef ENGINE_R_DSO_NOT_FOUND - {"DSO_NOT_FOUND", ERR_LIB_ENGINE, ENGINE_R_DSO_NOT_FOUND}, - #else - {"DSO_NOT_FOUND", 38, 132}, - #endif - #ifdef ENGINE_R_ENGINES_SECTION_ERROR - {"ENGINES_SECTION_ERROR", ERR_LIB_ENGINE, ENGINE_R_ENGINES_SECTION_ERROR}, - #else - {"ENGINES_SECTION_ERROR", 38, 148}, - #endif - #ifdef ENGINE_R_ENGINE_CONFIGURATION_ERROR - {"ENGINE_CONFIGURATION_ERROR", ERR_LIB_ENGINE, ENGINE_R_ENGINE_CONFIGURATION_ERROR}, - #else - {"ENGINE_CONFIGURATION_ERROR", 38, 102}, - #endif - #ifdef ENGINE_R_ENGINE_IS_NOT_IN_LIST - {"ENGINE_IS_NOT_IN_LIST", ERR_LIB_ENGINE, ENGINE_R_ENGINE_IS_NOT_IN_LIST}, - #else - {"ENGINE_IS_NOT_IN_LIST", 38, 105}, - #endif - #ifdef ENGINE_R_ENGINE_SECTION_ERROR - {"ENGINE_SECTION_ERROR", ERR_LIB_ENGINE, ENGINE_R_ENGINE_SECTION_ERROR}, - #else - {"ENGINE_SECTION_ERROR", 38, 149}, - #endif - #ifdef ENGINE_R_FAILED_LOADING_PRIVATE_KEY - {"FAILED_LOADING_PRIVATE_KEY", ERR_LIB_ENGINE, ENGINE_R_FAILED_LOADING_PRIVATE_KEY}, - #else - {"FAILED_LOADING_PRIVATE_KEY", 38, 128}, - #endif - #ifdef ENGINE_R_FAILED_LOADING_PUBLIC_KEY - {"FAILED_LOADING_PUBLIC_KEY", ERR_LIB_ENGINE, ENGINE_R_FAILED_LOADING_PUBLIC_KEY}, - #else - {"FAILED_LOADING_PUBLIC_KEY", 38, 129}, - #endif - #ifdef ENGINE_R_FINISH_FAILED - {"FINISH_FAILED", ERR_LIB_ENGINE, ENGINE_R_FINISH_FAILED}, - #else - {"FINISH_FAILED", 38, 106}, - #endif - #ifdef ENGINE_R_ID_OR_NAME_MISSING - {"ID_OR_NAME_MISSING", ERR_LIB_ENGINE, ENGINE_R_ID_OR_NAME_MISSING}, - #else - {"ID_OR_NAME_MISSING", 38, 108}, - #endif - #ifdef ENGINE_R_INIT_FAILED - {"INIT_FAILED", ERR_LIB_ENGINE, ENGINE_R_INIT_FAILED}, - #else - {"INIT_FAILED", 38, 109}, - #endif - #ifdef ENGINE_R_INTERNAL_LIST_ERROR - {"INTERNAL_LIST_ERROR", ERR_LIB_ENGINE, ENGINE_R_INTERNAL_LIST_ERROR}, - #else - {"INTERNAL_LIST_ERROR", 38, 110}, - #endif - #ifdef ENGINE_R_INVALID_ARGUMENT - {"INVALID_ARGUMENT", ERR_LIB_ENGINE, ENGINE_R_INVALID_ARGUMENT}, - #else - {"INVALID_ARGUMENT", 38, 143}, - #endif - #ifdef ENGINE_R_INVALID_CMD_NAME - {"INVALID_CMD_NAME", ERR_LIB_ENGINE, ENGINE_R_INVALID_CMD_NAME}, - #else - {"INVALID_CMD_NAME", 38, 137}, - #endif - #ifdef ENGINE_R_INVALID_CMD_NUMBER - {"INVALID_CMD_NUMBER", ERR_LIB_ENGINE, ENGINE_R_INVALID_CMD_NUMBER}, - #else - {"INVALID_CMD_NUMBER", 38, 138}, - #endif - #ifdef ENGINE_R_INVALID_INIT_VALUE - {"INVALID_INIT_VALUE", ERR_LIB_ENGINE, ENGINE_R_INVALID_INIT_VALUE}, - #else - {"INVALID_INIT_VALUE", 38, 151}, - #endif - #ifdef ENGINE_R_INVALID_STRING - {"INVALID_STRING", ERR_LIB_ENGINE, ENGINE_R_INVALID_STRING}, - #else - {"INVALID_STRING", 38, 150}, - #endif - #ifdef ENGINE_R_NOT_INITIALISED - {"NOT_INITIALISED", ERR_LIB_ENGINE, ENGINE_R_NOT_INITIALISED}, - #else - {"NOT_INITIALISED", 38, 117}, - #endif - #ifdef ENGINE_R_NOT_LOADED - {"NOT_LOADED", ERR_LIB_ENGINE, ENGINE_R_NOT_LOADED}, - #else - {"NOT_LOADED", 38, 112}, - #endif - #ifdef ENGINE_R_NO_CONTROL_FUNCTION - {"NO_CONTROL_FUNCTION", ERR_LIB_ENGINE, ENGINE_R_NO_CONTROL_FUNCTION}, - #else - {"NO_CONTROL_FUNCTION", 38, 120}, - #endif - #ifdef ENGINE_R_NO_INDEX - {"NO_INDEX", ERR_LIB_ENGINE, ENGINE_R_NO_INDEX}, - #else - {"NO_INDEX", 38, 144}, - #endif - #ifdef ENGINE_R_NO_LOAD_FUNCTION - {"NO_LOAD_FUNCTION", ERR_LIB_ENGINE, ENGINE_R_NO_LOAD_FUNCTION}, - #else - {"NO_LOAD_FUNCTION", 38, 125}, - #endif - #ifdef ENGINE_R_NO_REFERENCE - {"NO_REFERENCE", ERR_LIB_ENGINE, ENGINE_R_NO_REFERENCE}, - #else - {"NO_REFERENCE", 38, 130}, - #endif - #ifdef ENGINE_R_NO_SUCH_ENGINE - {"NO_SUCH_ENGINE", ERR_LIB_ENGINE, ENGINE_R_NO_SUCH_ENGINE}, - #else - {"NO_SUCH_ENGINE", 38, 116}, - #endif - #ifdef ENGINE_R_UNIMPLEMENTED_CIPHER - {"UNIMPLEMENTED_CIPHER", ERR_LIB_ENGINE, ENGINE_R_UNIMPLEMENTED_CIPHER}, - #else - {"UNIMPLEMENTED_CIPHER", 38, 146}, - #endif - #ifdef ENGINE_R_UNIMPLEMENTED_DIGEST - {"UNIMPLEMENTED_DIGEST", ERR_LIB_ENGINE, ENGINE_R_UNIMPLEMENTED_DIGEST}, - #else - {"UNIMPLEMENTED_DIGEST", 38, 147}, - #endif - #ifdef ENGINE_R_UNIMPLEMENTED_PUBLIC_KEY_METHOD - {"UNIMPLEMENTED_PUBLIC_KEY_METHOD", ERR_LIB_ENGINE, ENGINE_R_UNIMPLEMENTED_PUBLIC_KEY_METHOD}, - #else - {"UNIMPLEMENTED_PUBLIC_KEY_METHOD", 38, 101}, - #endif - #ifdef ENGINE_R_VERSION_INCOMPATIBILITY - {"VERSION_INCOMPATIBILITY", ERR_LIB_ENGINE, ENGINE_R_VERSION_INCOMPATIBILITY}, - #else - {"VERSION_INCOMPATIBILITY", 38, 145}, - #endif - #ifdef EVP_R_AES_KEY_SETUP_FAILED - {"AES_KEY_SETUP_FAILED", ERR_LIB_EVP, EVP_R_AES_KEY_SETUP_FAILED}, - #else - {"AES_KEY_SETUP_FAILED", 6, 143}, - #endif - #ifdef EVP_R_ARIA_KEY_SETUP_FAILED - {"ARIA_KEY_SETUP_FAILED", ERR_LIB_EVP, EVP_R_ARIA_KEY_SETUP_FAILED}, - #else - {"ARIA_KEY_SETUP_FAILED", 6, 176}, - #endif - #ifdef EVP_R_BAD_DECRYPT - {"BAD_DECRYPT", ERR_LIB_EVP, EVP_R_BAD_DECRYPT}, - #else - {"BAD_DECRYPT", 6, 100}, - #endif - #ifdef EVP_R_BAD_KEY_LENGTH - {"BAD_KEY_LENGTH", ERR_LIB_EVP, EVP_R_BAD_KEY_LENGTH}, - #else - {"BAD_KEY_LENGTH", 6, 195}, - #endif - #ifdef EVP_R_BUFFER_TOO_SMALL - {"BUFFER_TOO_SMALL", ERR_LIB_EVP, EVP_R_BUFFER_TOO_SMALL}, - #else - {"BUFFER_TOO_SMALL", 6, 155}, - #endif - #ifdef EVP_R_CAMELLIA_KEY_SETUP_FAILED - {"CAMELLIA_KEY_SETUP_FAILED", ERR_LIB_EVP, EVP_R_CAMELLIA_KEY_SETUP_FAILED}, - #else - {"CAMELLIA_KEY_SETUP_FAILED", 6, 157}, - #endif - #ifdef EVP_R_CIPHER_PARAMETER_ERROR - {"CIPHER_PARAMETER_ERROR", ERR_LIB_EVP, EVP_R_CIPHER_PARAMETER_ERROR}, - #else - {"CIPHER_PARAMETER_ERROR", 6, 122}, - #endif - #ifdef EVP_R_COMMAND_NOT_SUPPORTED - {"COMMAND_NOT_SUPPORTED", ERR_LIB_EVP, EVP_R_COMMAND_NOT_SUPPORTED}, - #else - {"COMMAND_NOT_SUPPORTED", 6, 147}, - #endif - #ifdef EVP_R_COPY_ERROR - {"COPY_ERROR", ERR_LIB_EVP, EVP_R_COPY_ERROR}, - #else - {"COPY_ERROR", 6, 173}, - #endif - #ifdef EVP_R_CTRL_NOT_IMPLEMENTED - {"CTRL_NOT_IMPLEMENTED", ERR_LIB_EVP, EVP_R_CTRL_NOT_IMPLEMENTED}, - #else - {"CTRL_NOT_IMPLEMENTED", 6, 132}, - #endif - #ifdef EVP_R_CTRL_OPERATION_NOT_IMPLEMENTED - {"CTRL_OPERATION_NOT_IMPLEMENTED", ERR_LIB_EVP, EVP_R_CTRL_OPERATION_NOT_IMPLEMENTED}, - #else - {"CTRL_OPERATION_NOT_IMPLEMENTED", 6, 133}, - #endif - #ifdef EVP_R_DATA_NOT_MULTIPLE_OF_BLOCK_LENGTH - {"DATA_NOT_MULTIPLE_OF_BLOCK_LENGTH", ERR_LIB_EVP, EVP_R_DATA_NOT_MULTIPLE_OF_BLOCK_LENGTH}, - #else - {"DATA_NOT_MULTIPLE_OF_BLOCK_LENGTH", 6, 138}, - #endif - #ifdef EVP_R_DECODE_ERROR - {"DECODE_ERROR", ERR_LIB_EVP, EVP_R_DECODE_ERROR}, - #else - {"DECODE_ERROR", 6, 114}, - #endif - #ifdef EVP_R_DIFFERENT_KEY_TYPES - {"DIFFERENT_KEY_TYPES", ERR_LIB_EVP, EVP_R_DIFFERENT_KEY_TYPES}, - #else - {"DIFFERENT_KEY_TYPES", 6, 101}, - #endif - #ifdef EVP_R_DIFFERENT_PARAMETERS - {"DIFFERENT_PARAMETERS", ERR_LIB_EVP, EVP_R_DIFFERENT_PARAMETERS}, - #else - {"DIFFERENT_PARAMETERS", 6, 153}, - #endif - #ifdef EVP_R_ERROR_LOADING_SECTION - {"ERROR_LOADING_SECTION", ERR_LIB_EVP, EVP_R_ERROR_LOADING_SECTION}, - #else - {"ERROR_LOADING_SECTION", 6, 165}, - #endif - #ifdef EVP_R_ERROR_SETTING_FIPS_MODE - {"ERROR_SETTING_FIPS_MODE", ERR_LIB_EVP, EVP_R_ERROR_SETTING_FIPS_MODE}, - #else - {"ERROR_SETTING_FIPS_MODE", 6, 166}, - #endif - #ifdef EVP_R_EXPECTING_AN_HMAC_KEY - {"EXPECTING_AN_HMAC_KEY", ERR_LIB_EVP, EVP_R_EXPECTING_AN_HMAC_KEY}, - #else - {"EXPECTING_AN_HMAC_KEY", 6, 174}, - #endif - #ifdef EVP_R_EXPECTING_AN_RSA_KEY - {"EXPECTING_AN_RSA_KEY", ERR_LIB_EVP, EVP_R_EXPECTING_AN_RSA_KEY}, - #else - {"EXPECTING_AN_RSA_KEY", 6, 127}, - #endif - #ifdef EVP_R_EXPECTING_A_DH_KEY - {"EXPECTING_A_DH_KEY", ERR_LIB_EVP, EVP_R_EXPECTING_A_DH_KEY}, - #else - {"EXPECTING_A_DH_KEY", 6, 128}, - #endif - #ifdef EVP_R_EXPECTING_A_DSA_KEY - {"EXPECTING_A_DSA_KEY", ERR_LIB_EVP, EVP_R_EXPECTING_A_DSA_KEY}, - #else - {"EXPECTING_A_DSA_KEY", 6, 129}, - #endif - #ifdef EVP_R_EXPECTING_A_EC_KEY - {"EXPECTING_A_EC_KEY", ERR_LIB_EVP, EVP_R_EXPECTING_A_EC_KEY}, - #else - {"EXPECTING_A_EC_KEY", 6, 142}, - #endif - #ifdef EVP_R_EXPECTING_A_POLY1305_KEY - {"EXPECTING_A_POLY1305_KEY", ERR_LIB_EVP, EVP_R_EXPECTING_A_POLY1305_KEY}, - #else - {"EXPECTING_A_POLY1305_KEY", 6, 164}, - #endif - #ifdef EVP_R_EXPECTING_A_SIPHASH_KEY - {"EXPECTING_A_SIPHASH_KEY", ERR_LIB_EVP, EVP_R_EXPECTING_A_SIPHASH_KEY}, - #else - {"EXPECTING_A_SIPHASH_KEY", 6, 175}, - #endif - #ifdef EVP_R_FIPS_MODE_NOT_SUPPORTED - {"FIPS_MODE_NOT_SUPPORTED", ERR_LIB_EVP, EVP_R_FIPS_MODE_NOT_SUPPORTED}, - #else - {"FIPS_MODE_NOT_SUPPORTED", 6, 167}, - #endif - #ifdef EVP_R_GET_RAW_KEY_FAILED - {"GET_RAW_KEY_FAILED", ERR_LIB_EVP, EVP_R_GET_RAW_KEY_FAILED}, - #else - {"GET_RAW_KEY_FAILED", 6, 182}, - #endif - #ifdef EVP_R_ILLEGAL_SCRYPT_PARAMETERS - {"ILLEGAL_SCRYPT_PARAMETERS", ERR_LIB_EVP, EVP_R_ILLEGAL_SCRYPT_PARAMETERS}, - #else - {"ILLEGAL_SCRYPT_PARAMETERS", 6, 171}, - #endif - #ifdef EVP_R_INITIALIZATION_ERROR - {"INITIALIZATION_ERROR", ERR_LIB_EVP, EVP_R_INITIALIZATION_ERROR}, - #else - {"INITIALIZATION_ERROR", 6, 134}, - #endif - #ifdef EVP_R_INPUT_NOT_INITIALIZED - {"INPUT_NOT_INITIALIZED", ERR_LIB_EVP, EVP_R_INPUT_NOT_INITIALIZED}, - #else - {"INPUT_NOT_INITIALIZED", 6, 111}, - #endif - #ifdef EVP_R_INVALID_DIGEST - {"INVALID_DIGEST", ERR_LIB_EVP, EVP_R_INVALID_DIGEST}, - #else - {"INVALID_DIGEST", 6, 152}, - #endif - #ifdef EVP_R_INVALID_FIPS_MODE - {"INVALID_FIPS_MODE", ERR_LIB_EVP, EVP_R_INVALID_FIPS_MODE}, - #else - {"INVALID_FIPS_MODE", 6, 168}, - #endif - #ifdef EVP_R_INVALID_IV_LENGTH - {"INVALID_IV_LENGTH", ERR_LIB_EVP, EVP_R_INVALID_IV_LENGTH}, - #else - {"INVALID_IV_LENGTH", 6, 194}, - #endif - #ifdef EVP_R_INVALID_KEY - {"INVALID_KEY", ERR_LIB_EVP, EVP_R_INVALID_KEY}, - #else - {"INVALID_KEY", 6, 163}, - #endif - #ifdef EVP_R_INVALID_KEY_LENGTH - {"INVALID_KEY_LENGTH", ERR_LIB_EVP, EVP_R_INVALID_KEY_LENGTH}, - #else - {"INVALID_KEY_LENGTH", 6, 130}, - #endif - #ifdef EVP_R_INVALID_OPERATION - {"INVALID_OPERATION", ERR_LIB_EVP, EVP_R_INVALID_OPERATION}, - #else - {"INVALID_OPERATION", 6, 148}, - #endif - #ifdef EVP_R_KEYGEN_FAILURE - {"KEYGEN_FAILURE", ERR_LIB_EVP, EVP_R_KEYGEN_FAILURE}, - #else - {"KEYGEN_FAILURE", 6, 120}, - #endif - #ifdef EVP_R_KEY_SETUP_FAILED - {"KEY_SETUP_FAILED", ERR_LIB_EVP, EVP_R_KEY_SETUP_FAILED}, - #else - {"KEY_SETUP_FAILED", 6, 180}, - #endif - #ifdef EVP_R_MEMORY_LIMIT_EXCEEDED - {"MEMORY_LIMIT_EXCEEDED", ERR_LIB_EVP, EVP_R_MEMORY_LIMIT_EXCEEDED}, - #else - {"MEMORY_LIMIT_EXCEEDED", 6, 172}, - #endif - #ifdef EVP_R_MESSAGE_DIGEST_IS_NULL - {"MESSAGE_DIGEST_IS_NULL", ERR_LIB_EVP, EVP_R_MESSAGE_DIGEST_IS_NULL}, - #else - {"MESSAGE_DIGEST_IS_NULL", 6, 159}, - #endif - #ifdef EVP_R_METHOD_NOT_SUPPORTED - {"METHOD_NOT_SUPPORTED", ERR_LIB_EVP, EVP_R_METHOD_NOT_SUPPORTED}, - #else - {"METHOD_NOT_SUPPORTED", 6, 144}, - #endif - #ifdef EVP_R_MISSING_PARAMETERS - {"MISSING_PARAMETERS", ERR_LIB_EVP, EVP_R_MISSING_PARAMETERS}, - #else - {"MISSING_PARAMETERS", 6, 103}, - #endif - #ifdef EVP_R_NOT_XOF_OR_INVALID_LENGTH - {"NOT_XOF_OR_INVALID_LENGTH", ERR_LIB_EVP, EVP_R_NOT_XOF_OR_INVALID_LENGTH}, - #else - {"NOT_XOF_OR_INVALID_LENGTH", 6, 178}, - #endif - #ifdef EVP_R_NO_CIPHER_SET - {"NO_CIPHER_SET", ERR_LIB_EVP, EVP_R_NO_CIPHER_SET}, - #else - {"NO_CIPHER_SET", 6, 131}, - #endif - #ifdef EVP_R_NO_DEFAULT_DIGEST - {"NO_DEFAULT_DIGEST", ERR_LIB_EVP, EVP_R_NO_DEFAULT_DIGEST}, - #else - {"NO_DEFAULT_DIGEST", 6, 158}, - #endif - #ifdef EVP_R_NO_DIGEST_SET - {"NO_DIGEST_SET", ERR_LIB_EVP, EVP_R_NO_DIGEST_SET}, - #else - {"NO_DIGEST_SET", 6, 139}, - #endif - #ifdef EVP_R_NO_KEY_SET - {"NO_KEY_SET", ERR_LIB_EVP, EVP_R_NO_KEY_SET}, - #else - {"NO_KEY_SET", 6, 154}, - #endif - #ifdef EVP_R_NO_OPERATION_SET - {"NO_OPERATION_SET", ERR_LIB_EVP, EVP_R_NO_OPERATION_SET}, - #else - {"NO_OPERATION_SET", 6, 149}, - #endif - #ifdef EVP_R_ONLY_ONESHOT_SUPPORTED - {"ONLY_ONESHOT_SUPPORTED", ERR_LIB_EVP, EVP_R_ONLY_ONESHOT_SUPPORTED}, - #else - {"ONLY_ONESHOT_SUPPORTED", 6, 177}, - #endif - #ifdef EVP_R_OPERATION_NOT_SUPPORTED_FOR_THIS_KEYTYPE - {"OPERATION_NOT_SUPPORTED_FOR_THIS_KEYTYPE", ERR_LIB_EVP, EVP_R_OPERATION_NOT_SUPPORTED_FOR_THIS_KEYTYPE}, - #else - {"OPERATION_NOT_SUPPORTED_FOR_THIS_KEYTYPE", 6, 150}, - #endif - #ifdef EVP_R_OPERATON_NOT_INITIALIZED - {"OPERATON_NOT_INITIALIZED", ERR_LIB_EVP, EVP_R_OPERATON_NOT_INITIALIZED}, - #else - {"OPERATON_NOT_INITIALIZED", 6, 151}, - #endif - #ifdef EVP_R_PARTIALLY_OVERLAPPING - {"PARTIALLY_OVERLAPPING", ERR_LIB_EVP, EVP_R_PARTIALLY_OVERLAPPING}, - #else - {"PARTIALLY_OVERLAPPING", 6, 162}, - #endif - #ifdef EVP_R_PBKDF2_ERROR - {"PBKDF2_ERROR", ERR_LIB_EVP, EVP_R_PBKDF2_ERROR}, - #else - {"PBKDF2_ERROR", 6, 181}, - #endif - #ifdef EVP_R_PKEY_APPLICATION_ASN1_METHOD_ALREADY_REGISTERED - {"PKEY_APPLICATION_ASN1_METHOD_ALREADY_REGISTERED", ERR_LIB_EVP, EVP_R_PKEY_APPLICATION_ASN1_METHOD_ALREADY_REGISTERED}, - #else - {"PKEY_APPLICATION_ASN1_METHOD_ALREADY_REGISTERED", 6, 179}, - #endif - #ifdef EVP_R_PRIVATE_KEY_DECODE_ERROR - {"PRIVATE_KEY_DECODE_ERROR", ERR_LIB_EVP, EVP_R_PRIVATE_KEY_DECODE_ERROR}, - #else - {"PRIVATE_KEY_DECODE_ERROR", 6, 145}, - #endif - #ifdef EVP_R_PRIVATE_KEY_ENCODE_ERROR - {"PRIVATE_KEY_ENCODE_ERROR", ERR_LIB_EVP, EVP_R_PRIVATE_KEY_ENCODE_ERROR}, - #else - {"PRIVATE_KEY_ENCODE_ERROR", 6, 146}, - #endif - #ifdef EVP_R_PUBLIC_KEY_NOT_RSA - {"PUBLIC_KEY_NOT_RSA", ERR_LIB_EVP, EVP_R_PUBLIC_KEY_NOT_RSA}, - #else - {"PUBLIC_KEY_NOT_RSA", 6, 106}, - #endif - #ifdef EVP_R_UNKNOWN_CIPHER - {"UNKNOWN_CIPHER", ERR_LIB_EVP, EVP_R_UNKNOWN_CIPHER}, - #else - {"UNKNOWN_CIPHER", 6, 160}, - #endif - #ifdef EVP_R_UNKNOWN_DIGEST - {"UNKNOWN_DIGEST", ERR_LIB_EVP, EVP_R_UNKNOWN_DIGEST}, - #else - {"UNKNOWN_DIGEST", 6, 161}, - #endif - #ifdef EVP_R_UNKNOWN_OPTION - {"UNKNOWN_OPTION", ERR_LIB_EVP, EVP_R_UNKNOWN_OPTION}, - #else - {"UNKNOWN_OPTION", 6, 169}, - #endif - #ifdef EVP_R_UNKNOWN_PBE_ALGORITHM - {"UNKNOWN_PBE_ALGORITHM", ERR_LIB_EVP, EVP_R_UNKNOWN_PBE_ALGORITHM}, - #else - {"UNKNOWN_PBE_ALGORITHM", 6, 121}, - #endif - #ifdef EVP_R_UNSUPPORTED_ALGORITHM - {"UNSUPPORTED_ALGORITHM", ERR_LIB_EVP, EVP_R_UNSUPPORTED_ALGORITHM}, - #else - {"UNSUPPORTED_ALGORITHM", 6, 156}, - #endif - #ifdef EVP_R_UNSUPPORTED_CIPHER - {"UNSUPPORTED_CIPHER", ERR_LIB_EVP, EVP_R_UNSUPPORTED_CIPHER}, - #else - {"UNSUPPORTED_CIPHER", 6, 107}, - #endif - #ifdef EVP_R_UNSUPPORTED_KEYLENGTH - {"UNSUPPORTED_KEYLENGTH", ERR_LIB_EVP, EVP_R_UNSUPPORTED_KEYLENGTH}, - #else - {"UNSUPPORTED_KEYLENGTH", 6, 123}, - #endif - #ifdef EVP_R_UNSUPPORTED_KEY_DERIVATION_FUNCTION - {"UNSUPPORTED_KEY_DERIVATION_FUNCTION", ERR_LIB_EVP, EVP_R_UNSUPPORTED_KEY_DERIVATION_FUNCTION}, - #else - {"UNSUPPORTED_KEY_DERIVATION_FUNCTION", 6, 124}, - #endif - #ifdef EVP_R_UNSUPPORTED_KEY_SIZE - {"UNSUPPORTED_KEY_SIZE", ERR_LIB_EVP, EVP_R_UNSUPPORTED_KEY_SIZE}, - #else - {"UNSUPPORTED_KEY_SIZE", 6, 108}, - #endif - #ifdef EVP_R_UNSUPPORTED_NUMBER_OF_ROUNDS - {"UNSUPPORTED_NUMBER_OF_ROUNDS", ERR_LIB_EVP, EVP_R_UNSUPPORTED_NUMBER_OF_ROUNDS}, - #else - {"UNSUPPORTED_NUMBER_OF_ROUNDS", 6, 135}, - #endif - #ifdef EVP_R_UNSUPPORTED_PRF - {"UNSUPPORTED_PRF", ERR_LIB_EVP, EVP_R_UNSUPPORTED_PRF}, - #else - {"UNSUPPORTED_PRF", 6, 125}, - #endif - #ifdef EVP_R_UNSUPPORTED_PRIVATE_KEY_ALGORITHM - {"UNSUPPORTED_PRIVATE_KEY_ALGORITHM", ERR_LIB_EVP, EVP_R_UNSUPPORTED_PRIVATE_KEY_ALGORITHM}, - #else - {"UNSUPPORTED_PRIVATE_KEY_ALGORITHM", 6, 118}, - #endif - #ifdef EVP_R_UNSUPPORTED_SALT_TYPE - {"UNSUPPORTED_SALT_TYPE", ERR_LIB_EVP, EVP_R_UNSUPPORTED_SALT_TYPE}, - #else - {"UNSUPPORTED_SALT_TYPE", 6, 126}, - #endif - #ifdef EVP_R_WRAP_MODE_NOT_ALLOWED - {"WRAP_MODE_NOT_ALLOWED", ERR_LIB_EVP, EVP_R_WRAP_MODE_NOT_ALLOWED}, - #else - {"WRAP_MODE_NOT_ALLOWED", 6, 170}, - #endif - #ifdef EVP_R_WRONG_FINAL_BLOCK_LENGTH - {"WRONG_FINAL_BLOCK_LENGTH", ERR_LIB_EVP, EVP_R_WRONG_FINAL_BLOCK_LENGTH}, - #else - {"WRONG_FINAL_BLOCK_LENGTH", 6, 109}, - #endif - #ifdef EVP_R_XTS_DUPLICATED_KEYS - {"XTS_DUPLICATED_KEYS", ERR_LIB_EVP, EVP_R_XTS_DUPLICATED_KEYS}, - #else - {"XTS_DUPLICATED_KEYS", 6, 183}, - #endif - #ifdef KDF_R_INVALID_DIGEST - {"INVALID_DIGEST", ERR_LIB_KDF, KDF_R_INVALID_DIGEST}, - #else - {"INVALID_DIGEST", 52, 100}, - #endif - #ifdef KDF_R_MISSING_ITERATION_COUNT - {"MISSING_ITERATION_COUNT", ERR_LIB_KDF, KDF_R_MISSING_ITERATION_COUNT}, - #else - {"MISSING_ITERATION_COUNT", 52, 109}, - #endif - #ifdef KDF_R_MISSING_KEY - {"MISSING_KEY", ERR_LIB_KDF, KDF_R_MISSING_KEY}, - #else - {"MISSING_KEY", 52, 104}, - #endif - #ifdef KDF_R_MISSING_MESSAGE_DIGEST - {"MISSING_MESSAGE_DIGEST", ERR_LIB_KDF, KDF_R_MISSING_MESSAGE_DIGEST}, - #else - {"MISSING_MESSAGE_DIGEST", 52, 105}, - #endif - #ifdef KDF_R_MISSING_PARAMETER - {"MISSING_PARAMETER", ERR_LIB_KDF, KDF_R_MISSING_PARAMETER}, - #else - {"MISSING_PARAMETER", 52, 101}, - #endif - #ifdef KDF_R_MISSING_PASS - {"MISSING_PASS", ERR_LIB_KDF, KDF_R_MISSING_PASS}, - #else - {"MISSING_PASS", 52, 110}, - #endif - #ifdef KDF_R_MISSING_SALT - {"MISSING_SALT", ERR_LIB_KDF, KDF_R_MISSING_SALT}, - #else - {"MISSING_SALT", 52, 111}, - #endif - #ifdef KDF_R_MISSING_SECRET - {"MISSING_SECRET", ERR_LIB_KDF, KDF_R_MISSING_SECRET}, - #else - {"MISSING_SECRET", 52, 107}, - #endif - #ifdef KDF_R_MISSING_SEED - {"MISSING_SEED", ERR_LIB_KDF, KDF_R_MISSING_SEED}, - #else - {"MISSING_SEED", 52, 106}, - #endif - #ifdef KDF_R_UNKNOWN_PARAMETER_TYPE - {"UNKNOWN_PARAMETER_TYPE", ERR_LIB_KDF, KDF_R_UNKNOWN_PARAMETER_TYPE}, - #else - {"UNKNOWN_PARAMETER_TYPE", 52, 103}, - #endif - #ifdef KDF_R_VALUE_ERROR - {"VALUE_ERROR", ERR_LIB_KDF, KDF_R_VALUE_ERROR}, - #else - {"VALUE_ERROR", 52, 108}, - #endif - #ifdef KDF_R_VALUE_MISSING - {"VALUE_MISSING", ERR_LIB_KDF, KDF_R_VALUE_MISSING}, - #else - {"VALUE_MISSING", 52, 102}, - #endif - #ifdef OCSP_R_CERTIFICATE_VERIFY_ERROR - {"CERTIFICATE_VERIFY_ERROR", ERR_LIB_OCSP, OCSP_R_CERTIFICATE_VERIFY_ERROR}, - #else - {"CERTIFICATE_VERIFY_ERROR", 39, 101}, - #endif - #ifdef OCSP_R_DIGEST_ERR - {"DIGEST_ERR", ERR_LIB_OCSP, OCSP_R_DIGEST_ERR}, - #else - {"DIGEST_ERR", 39, 102}, - #endif - #ifdef OCSP_R_ERROR_IN_NEXTUPDATE_FIELD - {"ERROR_IN_NEXTUPDATE_FIELD", ERR_LIB_OCSP, OCSP_R_ERROR_IN_NEXTUPDATE_FIELD}, - #else - {"ERROR_IN_NEXTUPDATE_FIELD", 39, 122}, - #endif - #ifdef OCSP_R_ERROR_IN_THISUPDATE_FIELD - {"ERROR_IN_THISUPDATE_FIELD", ERR_LIB_OCSP, OCSP_R_ERROR_IN_THISUPDATE_FIELD}, - #else - {"ERROR_IN_THISUPDATE_FIELD", 39, 123}, - #endif - #ifdef OCSP_R_ERROR_PARSING_URL - {"ERROR_PARSING_URL", ERR_LIB_OCSP, OCSP_R_ERROR_PARSING_URL}, - #else - {"ERROR_PARSING_URL", 39, 121}, - #endif - #ifdef OCSP_R_MISSING_OCSPSIGNING_USAGE - {"MISSING_OCSPSIGNING_USAGE", ERR_LIB_OCSP, OCSP_R_MISSING_OCSPSIGNING_USAGE}, - #else - {"MISSING_OCSPSIGNING_USAGE", 39, 103}, - #endif - #ifdef OCSP_R_NEXTUPDATE_BEFORE_THISUPDATE - {"NEXTUPDATE_BEFORE_THISUPDATE", ERR_LIB_OCSP, OCSP_R_NEXTUPDATE_BEFORE_THISUPDATE}, - #else - {"NEXTUPDATE_BEFORE_THISUPDATE", 39, 124}, - #endif - #ifdef OCSP_R_NOT_BASIC_RESPONSE - {"NOT_BASIC_RESPONSE", ERR_LIB_OCSP, OCSP_R_NOT_BASIC_RESPONSE}, - #else - {"NOT_BASIC_RESPONSE", 39, 104}, - #endif - #ifdef OCSP_R_NO_CERTIFICATES_IN_CHAIN - {"NO_CERTIFICATES_IN_CHAIN", ERR_LIB_OCSP, OCSP_R_NO_CERTIFICATES_IN_CHAIN}, - #else - {"NO_CERTIFICATES_IN_CHAIN", 39, 105}, - #endif - #ifdef OCSP_R_NO_RESPONSE_DATA - {"NO_RESPONSE_DATA", ERR_LIB_OCSP, OCSP_R_NO_RESPONSE_DATA}, - #else - {"NO_RESPONSE_DATA", 39, 108}, - #endif - #ifdef OCSP_R_NO_REVOKED_TIME - {"NO_REVOKED_TIME", ERR_LIB_OCSP, OCSP_R_NO_REVOKED_TIME}, - #else - {"NO_REVOKED_TIME", 39, 109}, - #endif - #ifdef OCSP_R_NO_SIGNER_KEY - {"NO_SIGNER_KEY", ERR_LIB_OCSP, OCSP_R_NO_SIGNER_KEY}, - #else - {"NO_SIGNER_KEY", 39, 130}, - #endif - #ifdef OCSP_R_PRIVATE_KEY_DOES_NOT_MATCH_CERTIFICATE - {"PRIVATE_KEY_DOES_NOT_MATCH_CERTIFICATE", ERR_LIB_OCSP, OCSP_R_PRIVATE_KEY_DOES_NOT_MATCH_CERTIFICATE}, - #else - {"PRIVATE_KEY_DOES_NOT_MATCH_CERTIFICATE", 39, 110}, - #endif - #ifdef OCSP_R_REQUEST_NOT_SIGNED - {"REQUEST_NOT_SIGNED", ERR_LIB_OCSP, OCSP_R_REQUEST_NOT_SIGNED}, - #else - {"REQUEST_NOT_SIGNED", 39, 128}, - #endif - #ifdef OCSP_R_RESPONSE_CONTAINS_NO_REVOCATION_DATA - {"RESPONSE_CONTAINS_NO_REVOCATION_DATA", ERR_LIB_OCSP, OCSP_R_RESPONSE_CONTAINS_NO_REVOCATION_DATA}, - #else - {"RESPONSE_CONTAINS_NO_REVOCATION_DATA", 39, 111}, - #endif - #ifdef OCSP_R_ROOT_CA_NOT_TRUSTED - {"ROOT_CA_NOT_TRUSTED", ERR_LIB_OCSP, OCSP_R_ROOT_CA_NOT_TRUSTED}, - #else - {"ROOT_CA_NOT_TRUSTED", 39, 112}, - #endif - #ifdef OCSP_R_SERVER_RESPONSE_ERROR - {"SERVER_RESPONSE_ERROR", ERR_LIB_OCSP, OCSP_R_SERVER_RESPONSE_ERROR}, - #else - {"SERVER_RESPONSE_ERROR", 39, 114}, - #endif - #ifdef OCSP_R_SERVER_RESPONSE_PARSE_ERROR - {"SERVER_RESPONSE_PARSE_ERROR", ERR_LIB_OCSP, OCSP_R_SERVER_RESPONSE_PARSE_ERROR}, - #else - {"SERVER_RESPONSE_PARSE_ERROR", 39, 115}, - #endif - #ifdef OCSP_R_SIGNATURE_FAILURE - {"SIGNATURE_FAILURE", ERR_LIB_OCSP, OCSP_R_SIGNATURE_FAILURE}, - #else - {"SIGNATURE_FAILURE", 39, 117}, - #endif - #ifdef OCSP_R_SIGNER_CERTIFICATE_NOT_FOUND - {"SIGNER_CERTIFICATE_NOT_FOUND", ERR_LIB_OCSP, OCSP_R_SIGNER_CERTIFICATE_NOT_FOUND}, - #else - {"SIGNER_CERTIFICATE_NOT_FOUND", 39, 118}, - #endif - #ifdef OCSP_R_STATUS_EXPIRED - {"STATUS_EXPIRED", ERR_LIB_OCSP, OCSP_R_STATUS_EXPIRED}, - #else - {"STATUS_EXPIRED", 39, 125}, - #endif - #ifdef OCSP_R_STATUS_NOT_YET_VALID - {"STATUS_NOT_YET_VALID", ERR_LIB_OCSP, OCSP_R_STATUS_NOT_YET_VALID}, - #else - {"STATUS_NOT_YET_VALID", 39, 126}, - #endif - #ifdef OCSP_R_STATUS_TOO_OLD - {"STATUS_TOO_OLD", ERR_LIB_OCSP, OCSP_R_STATUS_TOO_OLD}, - #else - {"STATUS_TOO_OLD", 39, 127}, - #endif - #ifdef OCSP_R_UNKNOWN_MESSAGE_DIGEST - {"UNKNOWN_MESSAGE_DIGEST", ERR_LIB_OCSP, OCSP_R_UNKNOWN_MESSAGE_DIGEST}, - #else - {"UNKNOWN_MESSAGE_DIGEST", 39, 119}, - #endif - #ifdef OCSP_R_UNKNOWN_NID - {"UNKNOWN_NID", ERR_LIB_OCSP, OCSP_R_UNKNOWN_NID}, - #else - {"UNKNOWN_NID", 39, 120}, - #endif - #ifdef OCSP_R_UNSUPPORTED_REQUESTORNAME_TYPE - {"UNSUPPORTED_REQUESTORNAME_TYPE", ERR_LIB_OCSP, OCSP_R_UNSUPPORTED_REQUESTORNAME_TYPE}, - #else - {"UNSUPPORTED_REQUESTORNAME_TYPE", 39, 129}, - #endif - #ifdef PEM_R_BAD_BASE64_DECODE - {"BAD_BASE64_DECODE", ERR_LIB_PEM, PEM_R_BAD_BASE64_DECODE}, - #else - {"BAD_BASE64_DECODE", 9, 100}, - #endif - #ifdef PEM_R_BAD_DECRYPT - {"BAD_DECRYPT", ERR_LIB_PEM, PEM_R_BAD_DECRYPT}, - #else - {"BAD_DECRYPT", 9, 101}, - #endif - #ifdef PEM_R_BAD_END_LINE - {"BAD_END_LINE", ERR_LIB_PEM, PEM_R_BAD_END_LINE}, - #else - {"BAD_END_LINE", 9, 102}, - #endif - #ifdef PEM_R_BAD_IV_CHARS - {"BAD_IV_CHARS", ERR_LIB_PEM, PEM_R_BAD_IV_CHARS}, - #else - {"BAD_IV_CHARS", 9, 103}, - #endif - #ifdef PEM_R_BAD_MAGIC_NUMBER - {"BAD_MAGIC_NUMBER", ERR_LIB_PEM, PEM_R_BAD_MAGIC_NUMBER}, - #else - {"BAD_MAGIC_NUMBER", 9, 116}, - #endif - #ifdef PEM_R_BAD_PASSWORD_READ - {"BAD_PASSWORD_READ", ERR_LIB_PEM, PEM_R_BAD_PASSWORD_READ}, - #else - {"BAD_PASSWORD_READ", 9, 104}, - #endif - #ifdef PEM_R_BAD_VERSION_NUMBER - {"BAD_VERSION_NUMBER", ERR_LIB_PEM, PEM_R_BAD_VERSION_NUMBER}, - #else - {"BAD_VERSION_NUMBER", 9, 117}, - #endif - #ifdef PEM_R_BIO_WRITE_FAILURE - {"BIO_WRITE_FAILURE", ERR_LIB_PEM, PEM_R_BIO_WRITE_FAILURE}, - #else - {"BIO_WRITE_FAILURE", 9, 118}, - #endif - #ifdef PEM_R_CIPHER_IS_NULL - {"CIPHER_IS_NULL", ERR_LIB_PEM, PEM_R_CIPHER_IS_NULL}, - #else - {"CIPHER_IS_NULL", 9, 127}, - #endif - #ifdef PEM_R_ERROR_CONVERTING_PRIVATE_KEY - {"ERROR_CONVERTING_PRIVATE_KEY", ERR_LIB_PEM, PEM_R_ERROR_CONVERTING_PRIVATE_KEY}, - #else - {"ERROR_CONVERTING_PRIVATE_KEY", 9, 115}, - #endif - #ifdef PEM_R_EXPECTING_PRIVATE_KEY_BLOB - {"EXPECTING_PRIVATE_KEY_BLOB", ERR_LIB_PEM, PEM_R_EXPECTING_PRIVATE_KEY_BLOB}, - #else - {"EXPECTING_PRIVATE_KEY_BLOB", 9, 119}, - #endif - #ifdef PEM_R_EXPECTING_PUBLIC_KEY_BLOB - {"EXPECTING_PUBLIC_KEY_BLOB", ERR_LIB_PEM, PEM_R_EXPECTING_PUBLIC_KEY_BLOB}, - #else - {"EXPECTING_PUBLIC_KEY_BLOB", 9, 120}, - #endif - #ifdef PEM_R_HEADER_TOO_LONG - {"HEADER_TOO_LONG", ERR_LIB_PEM, PEM_R_HEADER_TOO_LONG}, - #else - {"HEADER_TOO_LONG", 9, 128}, - #endif - #ifdef PEM_R_INCONSISTENT_HEADER - {"INCONSISTENT_HEADER", ERR_LIB_PEM, PEM_R_INCONSISTENT_HEADER}, - #else - {"INCONSISTENT_HEADER", 9, 121}, - #endif - #ifdef PEM_R_KEYBLOB_HEADER_PARSE_ERROR - {"KEYBLOB_HEADER_PARSE_ERROR", ERR_LIB_PEM, PEM_R_KEYBLOB_HEADER_PARSE_ERROR}, - #else - {"KEYBLOB_HEADER_PARSE_ERROR", 9, 122}, - #endif - #ifdef PEM_R_KEYBLOB_TOO_SHORT - {"KEYBLOB_TOO_SHORT", ERR_LIB_PEM, PEM_R_KEYBLOB_TOO_SHORT}, - #else - {"KEYBLOB_TOO_SHORT", 9, 123}, - #endif - #ifdef PEM_R_MISSING_DEK_IV - {"MISSING_DEK_IV", ERR_LIB_PEM, PEM_R_MISSING_DEK_IV}, - #else - {"MISSING_DEK_IV", 9, 129}, - #endif - #ifdef PEM_R_NOT_DEK_INFO - {"NOT_DEK_INFO", ERR_LIB_PEM, PEM_R_NOT_DEK_INFO}, - #else - {"NOT_DEK_INFO", 9, 105}, - #endif - #ifdef PEM_R_NOT_ENCRYPTED - {"NOT_ENCRYPTED", ERR_LIB_PEM, PEM_R_NOT_ENCRYPTED}, - #else - {"NOT_ENCRYPTED", 9, 106}, - #endif - #ifdef PEM_R_NOT_PROC_TYPE - {"NOT_PROC_TYPE", ERR_LIB_PEM, PEM_R_NOT_PROC_TYPE}, - #else - {"NOT_PROC_TYPE", 9, 107}, - #endif - #ifdef PEM_R_NO_START_LINE - {"NO_START_LINE", ERR_LIB_PEM, PEM_R_NO_START_LINE}, - #else - {"NO_START_LINE", 9, 108}, - #endif - #ifdef PEM_R_PROBLEMS_GETTING_PASSWORD - {"PROBLEMS_GETTING_PASSWORD", ERR_LIB_PEM, PEM_R_PROBLEMS_GETTING_PASSWORD}, - #else - {"PROBLEMS_GETTING_PASSWORD", 9, 109}, - #endif - #ifdef PEM_R_PUBLIC_KEY_NO_RSA - {"PUBLIC_KEY_NO_RSA", ERR_LIB_PEM, PEM_R_PUBLIC_KEY_NO_RSA}, - #else - {"PUBLIC_KEY_NO_RSA", 9, 110}, - #endif - #ifdef PEM_R_PVK_DATA_TOO_SHORT - {"PVK_DATA_TOO_SHORT", ERR_LIB_PEM, PEM_R_PVK_DATA_TOO_SHORT}, - #else - {"PVK_DATA_TOO_SHORT", 9, 124}, - #endif - #ifdef PEM_R_PVK_TOO_SHORT - {"PVK_TOO_SHORT", ERR_LIB_PEM, PEM_R_PVK_TOO_SHORT}, - #else - {"PVK_TOO_SHORT", 9, 125}, - #endif - #ifdef PEM_R_READ_KEY - {"READ_KEY", ERR_LIB_PEM, PEM_R_READ_KEY}, - #else - {"READ_KEY", 9, 111}, - #endif - #ifdef PEM_R_SHORT_HEADER - {"SHORT_HEADER", ERR_LIB_PEM, PEM_R_SHORT_HEADER}, - #else - {"SHORT_HEADER", 9, 112}, - #endif - #ifdef PEM_R_UNEXPECTED_DEK_IV - {"UNEXPECTED_DEK_IV", ERR_LIB_PEM, PEM_R_UNEXPECTED_DEK_IV}, - #else - {"UNEXPECTED_DEK_IV", 9, 130}, - #endif - #ifdef PEM_R_UNSUPPORTED_CIPHER - {"UNSUPPORTED_CIPHER", ERR_LIB_PEM, PEM_R_UNSUPPORTED_CIPHER}, - #else - {"UNSUPPORTED_CIPHER", 9, 113}, - #endif - #ifdef PEM_R_UNSUPPORTED_ENCRYPTION - {"UNSUPPORTED_ENCRYPTION", ERR_LIB_PEM, PEM_R_UNSUPPORTED_ENCRYPTION}, - #else - {"UNSUPPORTED_ENCRYPTION", 9, 114}, - #endif - #ifdef PEM_R_UNSUPPORTED_KEY_COMPONENTS - {"UNSUPPORTED_KEY_COMPONENTS", ERR_LIB_PEM, PEM_R_UNSUPPORTED_KEY_COMPONENTS}, - #else - {"UNSUPPORTED_KEY_COMPONENTS", 9, 126}, - #endif - #ifdef PKCS12_R_CANT_PACK_STRUCTURE - {"CANT_PACK_STRUCTURE", ERR_LIB_PKCS12, PKCS12_R_CANT_PACK_STRUCTURE}, - #else - {"CANT_PACK_STRUCTURE", 35, 100}, - #endif - #ifdef PKCS12_R_CONTENT_TYPE_NOT_DATA - {"CONTENT_TYPE_NOT_DATA", ERR_LIB_PKCS12, PKCS12_R_CONTENT_TYPE_NOT_DATA}, - #else - {"CONTENT_TYPE_NOT_DATA", 35, 121}, - #endif - #ifdef PKCS12_R_DECODE_ERROR - {"DECODE_ERROR", ERR_LIB_PKCS12, PKCS12_R_DECODE_ERROR}, - #else - {"DECODE_ERROR", 35, 101}, - #endif - #ifdef PKCS12_R_ENCODE_ERROR - {"ENCODE_ERROR", ERR_LIB_PKCS12, PKCS12_R_ENCODE_ERROR}, - #else - {"ENCODE_ERROR", 35, 102}, - #endif - #ifdef PKCS12_R_ENCRYPT_ERROR - {"ENCRYPT_ERROR", ERR_LIB_PKCS12, PKCS12_R_ENCRYPT_ERROR}, - #else - {"ENCRYPT_ERROR", 35, 103}, - #endif - #ifdef PKCS12_R_ERROR_SETTING_ENCRYPTED_DATA_TYPE - {"ERROR_SETTING_ENCRYPTED_DATA_TYPE", ERR_LIB_PKCS12, PKCS12_R_ERROR_SETTING_ENCRYPTED_DATA_TYPE}, - #else - {"ERROR_SETTING_ENCRYPTED_DATA_TYPE", 35, 120}, - #endif - #ifdef PKCS12_R_INVALID_NULL_ARGUMENT - {"INVALID_NULL_ARGUMENT", ERR_LIB_PKCS12, PKCS12_R_INVALID_NULL_ARGUMENT}, - #else - {"INVALID_NULL_ARGUMENT", 35, 104}, - #endif - #ifdef PKCS12_R_INVALID_NULL_PKCS12_POINTER - {"INVALID_NULL_PKCS12_POINTER", ERR_LIB_PKCS12, PKCS12_R_INVALID_NULL_PKCS12_POINTER}, - #else - {"INVALID_NULL_PKCS12_POINTER", 35, 105}, - #endif - #ifdef PKCS12_R_IV_GEN_ERROR - {"IV_GEN_ERROR", ERR_LIB_PKCS12, PKCS12_R_IV_GEN_ERROR}, - #else - {"IV_GEN_ERROR", 35, 106}, - #endif - #ifdef PKCS12_R_KEY_GEN_ERROR - {"KEY_GEN_ERROR", ERR_LIB_PKCS12, PKCS12_R_KEY_GEN_ERROR}, - #else - {"KEY_GEN_ERROR", 35, 107}, - #endif - #ifdef PKCS12_R_MAC_ABSENT - {"MAC_ABSENT", ERR_LIB_PKCS12, PKCS12_R_MAC_ABSENT}, - #else - {"MAC_ABSENT", 35, 108}, - #endif - #ifdef PKCS12_R_MAC_GENERATION_ERROR - {"MAC_GENERATION_ERROR", ERR_LIB_PKCS12, PKCS12_R_MAC_GENERATION_ERROR}, - #else - {"MAC_GENERATION_ERROR", 35, 109}, - #endif - #ifdef PKCS12_R_MAC_SETUP_ERROR - {"MAC_SETUP_ERROR", ERR_LIB_PKCS12, PKCS12_R_MAC_SETUP_ERROR}, - #else - {"MAC_SETUP_ERROR", 35, 110}, - #endif - #ifdef PKCS12_R_MAC_STRING_SET_ERROR - {"MAC_STRING_SET_ERROR", ERR_LIB_PKCS12, PKCS12_R_MAC_STRING_SET_ERROR}, - #else - {"MAC_STRING_SET_ERROR", 35, 111}, - #endif - #ifdef PKCS12_R_MAC_VERIFY_FAILURE - {"MAC_VERIFY_FAILURE", ERR_LIB_PKCS12, PKCS12_R_MAC_VERIFY_FAILURE}, - #else - {"MAC_VERIFY_FAILURE", 35, 113}, - #endif - #ifdef PKCS12_R_PARSE_ERROR - {"PARSE_ERROR", ERR_LIB_PKCS12, PKCS12_R_PARSE_ERROR}, - #else - {"PARSE_ERROR", 35, 114}, - #endif - #ifdef PKCS12_R_PKCS12_ALGOR_CIPHERINIT_ERROR - {"PKCS12_ALGOR_CIPHERINIT_ERROR", ERR_LIB_PKCS12, PKCS12_R_PKCS12_ALGOR_CIPHERINIT_ERROR}, - #else - {"PKCS12_ALGOR_CIPHERINIT_ERROR", 35, 115}, - #endif - #ifdef PKCS12_R_PKCS12_CIPHERFINAL_ERROR - {"PKCS12_CIPHERFINAL_ERROR", ERR_LIB_PKCS12, PKCS12_R_PKCS12_CIPHERFINAL_ERROR}, - #else - {"PKCS12_CIPHERFINAL_ERROR", 35, 116}, - #endif - #ifdef PKCS12_R_PKCS12_PBE_CRYPT_ERROR - {"PKCS12_PBE_CRYPT_ERROR", ERR_LIB_PKCS12, PKCS12_R_PKCS12_PBE_CRYPT_ERROR}, - #else - {"PKCS12_PBE_CRYPT_ERROR", 35, 117}, - #endif - #ifdef PKCS12_R_UNKNOWN_DIGEST_ALGORITHM - {"UNKNOWN_DIGEST_ALGORITHM", ERR_LIB_PKCS12, PKCS12_R_UNKNOWN_DIGEST_ALGORITHM}, - #else - {"UNKNOWN_DIGEST_ALGORITHM", 35, 118}, - #endif - #ifdef PKCS12_R_UNSUPPORTED_PKCS12_MODE - {"UNSUPPORTED_PKCS12_MODE", ERR_LIB_PKCS12, PKCS12_R_UNSUPPORTED_PKCS12_MODE}, - #else - {"UNSUPPORTED_PKCS12_MODE", 35, 119}, - #endif - #ifdef PKCS7_R_CERTIFICATE_VERIFY_ERROR - {"CERTIFICATE_VERIFY_ERROR", ERR_LIB_PKCS7, PKCS7_R_CERTIFICATE_VERIFY_ERROR}, - #else - {"CERTIFICATE_VERIFY_ERROR", 33, 117}, - #endif - #ifdef PKCS7_R_CIPHER_HAS_NO_OBJECT_IDENTIFIER - {"CIPHER_HAS_NO_OBJECT_IDENTIFIER", ERR_LIB_PKCS7, PKCS7_R_CIPHER_HAS_NO_OBJECT_IDENTIFIER}, - #else - {"CIPHER_HAS_NO_OBJECT_IDENTIFIER", 33, 144}, - #endif - #ifdef PKCS7_R_CIPHER_NOT_INITIALIZED - {"CIPHER_NOT_INITIALIZED", ERR_LIB_PKCS7, PKCS7_R_CIPHER_NOT_INITIALIZED}, - #else - {"CIPHER_NOT_INITIALIZED", 33, 116}, - #endif - #ifdef PKCS7_R_CONTENT_AND_DATA_PRESENT - {"CONTENT_AND_DATA_PRESENT", ERR_LIB_PKCS7, PKCS7_R_CONTENT_AND_DATA_PRESENT}, - #else - {"CONTENT_AND_DATA_PRESENT", 33, 118}, - #endif - #ifdef PKCS7_R_CTRL_ERROR - {"CTRL_ERROR", ERR_LIB_PKCS7, PKCS7_R_CTRL_ERROR}, - #else - {"CTRL_ERROR", 33, 152}, - #endif - #ifdef PKCS7_R_DECRYPT_ERROR - {"DECRYPT_ERROR", ERR_LIB_PKCS7, PKCS7_R_DECRYPT_ERROR}, - #else - {"DECRYPT_ERROR", 33, 119}, - #endif - #ifdef PKCS7_R_DIGEST_FAILURE - {"DIGEST_FAILURE", ERR_LIB_PKCS7, PKCS7_R_DIGEST_FAILURE}, - #else - {"DIGEST_FAILURE", 33, 101}, - #endif - #ifdef PKCS7_R_ENCRYPTION_CTRL_FAILURE - {"ENCRYPTION_CTRL_FAILURE", ERR_LIB_PKCS7, PKCS7_R_ENCRYPTION_CTRL_FAILURE}, - #else - {"ENCRYPTION_CTRL_FAILURE", 33, 149}, - #endif - #ifdef PKCS7_R_ENCRYPTION_NOT_SUPPORTED_FOR_THIS_KEY_TYPE - {"ENCRYPTION_NOT_SUPPORTED_FOR_THIS_KEY_TYPE", ERR_LIB_PKCS7, PKCS7_R_ENCRYPTION_NOT_SUPPORTED_FOR_THIS_KEY_TYPE}, - #else - {"ENCRYPTION_NOT_SUPPORTED_FOR_THIS_KEY_TYPE", 33, 150}, - #endif - #ifdef PKCS7_R_ERROR_ADDING_RECIPIENT - {"ERROR_ADDING_RECIPIENT", ERR_LIB_PKCS7, PKCS7_R_ERROR_ADDING_RECIPIENT}, - #else - {"ERROR_ADDING_RECIPIENT", 33, 120}, - #endif - #ifdef PKCS7_R_ERROR_SETTING_CIPHER - {"ERROR_SETTING_CIPHER", ERR_LIB_PKCS7, PKCS7_R_ERROR_SETTING_CIPHER}, - #else - {"ERROR_SETTING_CIPHER", 33, 121}, - #endif - #ifdef PKCS7_R_INVALID_NULL_POINTER - {"INVALID_NULL_POINTER", ERR_LIB_PKCS7, PKCS7_R_INVALID_NULL_POINTER}, - #else - {"INVALID_NULL_POINTER", 33, 143}, - #endif - #ifdef PKCS7_R_INVALID_SIGNED_DATA_TYPE - {"INVALID_SIGNED_DATA_TYPE", ERR_LIB_PKCS7, PKCS7_R_INVALID_SIGNED_DATA_TYPE}, - #else - {"INVALID_SIGNED_DATA_TYPE", 33, 155}, - #endif - #ifdef PKCS7_R_NO_CONTENT - {"NO_CONTENT", ERR_LIB_PKCS7, PKCS7_R_NO_CONTENT}, - #else - {"NO_CONTENT", 33, 122}, - #endif - #ifdef PKCS7_R_NO_DEFAULT_DIGEST - {"NO_DEFAULT_DIGEST", ERR_LIB_PKCS7, PKCS7_R_NO_DEFAULT_DIGEST}, - #else - {"NO_DEFAULT_DIGEST", 33, 151}, - #endif - #ifdef PKCS7_R_NO_MATCHING_DIGEST_TYPE_FOUND - {"NO_MATCHING_DIGEST_TYPE_FOUND", ERR_LIB_PKCS7, PKCS7_R_NO_MATCHING_DIGEST_TYPE_FOUND}, - #else - {"NO_MATCHING_DIGEST_TYPE_FOUND", 33, 154}, - #endif - #ifdef PKCS7_R_NO_RECIPIENT_MATCHES_CERTIFICATE - {"NO_RECIPIENT_MATCHES_CERTIFICATE", ERR_LIB_PKCS7, PKCS7_R_NO_RECIPIENT_MATCHES_CERTIFICATE}, - #else - {"NO_RECIPIENT_MATCHES_CERTIFICATE", 33, 115}, - #endif - #ifdef PKCS7_R_NO_SIGNATURES_ON_DATA - {"NO_SIGNATURES_ON_DATA", ERR_LIB_PKCS7, PKCS7_R_NO_SIGNATURES_ON_DATA}, - #else - {"NO_SIGNATURES_ON_DATA", 33, 123}, - #endif - #ifdef PKCS7_R_NO_SIGNERS - {"NO_SIGNERS", ERR_LIB_PKCS7, PKCS7_R_NO_SIGNERS}, - #else - {"NO_SIGNERS", 33, 142}, - #endif - #ifdef PKCS7_R_OPERATION_NOT_SUPPORTED_ON_THIS_TYPE - {"OPERATION_NOT_SUPPORTED_ON_THIS_TYPE", ERR_LIB_PKCS7, PKCS7_R_OPERATION_NOT_SUPPORTED_ON_THIS_TYPE}, - #else - {"OPERATION_NOT_SUPPORTED_ON_THIS_TYPE", 33, 104}, - #endif - #ifdef PKCS7_R_PKCS7_ADD_SIGNATURE_ERROR - {"PKCS7_ADD_SIGNATURE_ERROR", ERR_LIB_PKCS7, PKCS7_R_PKCS7_ADD_SIGNATURE_ERROR}, - #else - {"PKCS7_ADD_SIGNATURE_ERROR", 33, 124}, - #endif - #ifdef PKCS7_R_PKCS7_ADD_SIGNER_ERROR - {"PKCS7_ADD_SIGNER_ERROR", ERR_LIB_PKCS7, PKCS7_R_PKCS7_ADD_SIGNER_ERROR}, - #else - {"PKCS7_ADD_SIGNER_ERROR", 33, 153}, - #endif - #ifdef PKCS7_R_PKCS7_DATASIGN - {"PKCS7_DATASIGN", ERR_LIB_PKCS7, PKCS7_R_PKCS7_DATASIGN}, - #else - {"PKCS7_DATASIGN", 33, 145}, - #endif - #ifdef PKCS7_R_PRIVATE_KEY_DOES_NOT_MATCH_CERTIFICATE - {"PRIVATE_KEY_DOES_NOT_MATCH_CERTIFICATE", ERR_LIB_PKCS7, PKCS7_R_PRIVATE_KEY_DOES_NOT_MATCH_CERTIFICATE}, - #else - {"PRIVATE_KEY_DOES_NOT_MATCH_CERTIFICATE", 33, 127}, - #endif - #ifdef PKCS7_R_SIGNATURE_FAILURE - {"SIGNATURE_FAILURE", ERR_LIB_PKCS7, PKCS7_R_SIGNATURE_FAILURE}, - #else - {"SIGNATURE_FAILURE", 33, 105}, - #endif - #ifdef PKCS7_R_SIGNER_CERTIFICATE_NOT_FOUND - {"SIGNER_CERTIFICATE_NOT_FOUND", ERR_LIB_PKCS7, PKCS7_R_SIGNER_CERTIFICATE_NOT_FOUND}, - #else - {"SIGNER_CERTIFICATE_NOT_FOUND", 33, 128}, - #endif - #ifdef PKCS7_R_SIGNING_CTRL_FAILURE - {"SIGNING_CTRL_FAILURE", ERR_LIB_PKCS7, PKCS7_R_SIGNING_CTRL_FAILURE}, - #else - {"SIGNING_CTRL_FAILURE", 33, 147}, - #endif - #ifdef PKCS7_R_SIGNING_NOT_SUPPORTED_FOR_THIS_KEY_TYPE - {"SIGNING_NOT_SUPPORTED_FOR_THIS_KEY_TYPE", ERR_LIB_PKCS7, PKCS7_R_SIGNING_NOT_SUPPORTED_FOR_THIS_KEY_TYPE}, - #else - {"SIGNING_NOT_SUPPORTED_FOR_THIS_KEY_TYPE", 33, 148}, - #endif - #ifdef PKCS7_R_SMIME_TEXT_ERROR - {"SMIME_TEXT_ERROR", ERR_LIB_PKCS7, PKCS7_R_SMIME_TEXT_ERROR}, - #else - {"SMIME_TEXT_ERROR", 33, 129}, - #endif - #ifdef PKCS7_R_UNABLE_TO_FIND_CERTIFICATE - {"UNABLE_TO_FIND_CERTIFICATE", ERR_LIB_PKCS7, PKCS7_R_UNABLE_TO_FIND_CERTIFICATE}, - #else - {"UNABLE_TO_FIND_CERTIFICATE", 33, 106}, - #endif - #ifdef PKCS7_R_UNABLE_TO_FIND_MEM_BIO - {"UNABLE_TO_FIND_MEM_BIO", ERR_LIB_PKCS7, PKCS7_R_UNABLE_TO_FIND_MEM_BIO}, - #else - {"UNABLE_TO_FIND_MEM_BIO", 33, 107}, - #endif - #ifdef PKCS7_R_UNABLE_TO_FIND_MESSAGE_DIGEST - {"UNABLE_TO_FIND_MESSAGE_DIGEST", ERR_LIB_PKCS7, PKCS7_R_UNABLE_TO_FIND_MESSAGE_DIGEST}, - #else - {"UNABLE_TO_FIND_MESSAGE_DIGEST", 33, 108}, - #endif - #ifdef PKCS7_R_UNKNOWN_DIGEST_TYPE - {"UNKNOWN_DIGEST_TYPE", ERR_LIB_PKCS7, PKCS7_R_UNKNOWN_DIGEST_TYPE}, - #else - {"UNKNOWN_DIGEST_TYPE", 33, 109}, - #endif - #ifdef PKCS7_R_UNKNOWN_OPERATION - {"UNKNOWN_OPERATION", ERR_LIB_PKCS7, PKCS7_R_UNKNOWN_OPERATION}, - #else - {"UNKNOWN_OPERATION", 33, 110}, - #endif - #ifdef PKCS7_R_UNSUPPORTED_CIPHER_TYPE - {"UNSUPPORTED_CIPHER_TYPE", ERR_LIB_PKCS7, PKCS7_R_UNSUPPORTED_CIPHER_TYPE}, - #else - {"UNSUPPORTED_CIPHER_TYPE", 33, 111}, - #endif - #ifdef PKCS7_R_UNSUPPORTED_CONTENT_TYPE - {"UNSUPPORTED_CONTENT_TYPE", ERR_LIB_PKCS7, PKCS7_R_UNSUPPORTED_CONTENT_TYPE}, - #else - {"UNSUPPORTED_CONTENT_TYPE", 33, 112}, - #endif - #ifdef PKCS7_R_WRONG_CONTENT_TYPE - {"WRONG_CONTENT_TYPE", ERR_LIB_PKCS7, PKCS7_R_WRONG_CONTENT_TYPE}, - #else - {"WRONG_CONTENT_TYPE", 33, 113}, - #endif - #ifdef PKCS7_R_WRONG_PKCS7_TYPE - {"WRONG_PKCS7_TYPE", ERR_LIB_PKCS7, PKCS7_R_WRONG_PKCS7_TYPE}, - #else - {"WRONG_PKCS7_TYPE", 33, 114}, - #endif - #ifdef RAND_R_ADDITIONAL_INPUT_TOO_LONG - {"ADDITIONAL_INPUT_TOO_LONG", ERR_LIB_RAND, RAND_R_ADDITIONAL_INPUT_TOO_LONG}, - #else - {"ADDITIONAL_INPUT_TOO_LONG", 36, 102}, - #endif - #ifdef RAND_R_ALREADY_INSTANTIATED - {"ALREADY_INSTANTIATED", ERR_LIB_RAND, RAND_R_ALREADY_INSTANTIATED}, - #else - {"ALREADY_INSTANTIATED", 36, 103}, - #endif - #ifdef RAND_R_ARGUMENT_OUT_OF_RANGE - {"ARGUMENT_OUT_OF_RANGE", ERR_LIB_RAND, RAND_R_ARGUMENT_OUT_OF_RANGE}, - #else - {"ARGUMENT_OUT_OF_RANGE", 36, 105}, - #endif - #ifdef RAND_R_CANNOT_OPEN_FILE - {"CANNOT_OPEN_FILE", ERR_LIB_RAND, RAND_R_CANNOT_OPEN_FILE}, - #else - {"CANNOT_OPEN_FILE", 36, 121}, - #endif - #ifdef RAND_R_DRBG_ALREADY_INITIALIZED - {"DRBG_ALREADY_INITIALIZED", ERR_LIB_RAND, RAND_R_DRBG_ALREADY_INITIALIZED}, - #else - {"DRBG_ALREADY_INITIALIZED", 36, 129}, - #endif - #ifdef RAND_R_DRBG_NOT_INITIALISED - {"DRBG_NOT_INITIALISED", ERR_LIB_RAND, RAND_R_DRBG_NOT_INITIALISED}, - #else - {"DRBG_NOT_INITIALISED", 36, 104}, - #endif - #ifdef RAND_R_ENTROPY_INPUT_TOO_LONG - {"ENTROPY_INPUT_TOO_LONG", ERR_LIB_RAND, RAND_R_ENTROPY_INPUT_TOO_LONG}, - #else - {"ENTROPY_INPUT_TOO_LONG", 36, 106}, - #endif - #ifdef RAND_R_ENTROPY_OUT_OF_RANGE - {"ENTROPY_OUT_OF_RANGE", ERR_LIB_RAND, RAND_R_ENTROPY_OUT_OF_RANGE}, - #else - {"ENTROPY_OUT_OF_RANGE", 36, 124}, - #endif - #ifdef RAND_R_ERROR_ENTROPY_POOL_WAS_IGNORED - {"ERROR_ENTROPY_POOL_WAS_IGNORED", ERR_LIB_RAND, RAND_R_ERROR_ENTROPY_POOL_WAS_IGNORED}, - #else - {"ERROR_ENTROPY_POOL_WAS_IGNORED", 36, 127}, - #endif - #ifdef RAND_R_ERROR_INITIALISING_DRBG - {"ERROR_INITIALISING_DRBG", ERR_LIB_RAND, RAND_R_ERROR_INITIALISING_DRBG}, - #else - {"ERROR_INITIALISING_DRBG", 36, 107}, - #endif - #ifdef RAND_R_ERROR_INSTANTIATING_DRBG - {"ERROR_INSTANTIATING_DRBG", ERR_LIB_RAND, RAND_R_ERROR_INSTANTIATING_DRBG}, - #else - {"ERROR_INSTANTIATING_DRBG", 36, 108}, - #endif - #ifdef RAND_R_ERROR_RETRIEVING_ADDITIONAL_INPUT - {"ERROR_RETRIEVING_ADDITIONAL_INPUT", ERR_LIB_RAND, RAND_R_ERROR_RETRIEVING_ADDITIONAL_INPUT}, - #else - {"ERROR_RETRIEVING_ADDITIONAL_INPUT", 36, 109}, - #endif - #ifdef RAND_R_ERROR_RETRIEVING_ENTROPY - {"ERROR_RETRIEVING_ENTROPY", ERR_LIB_RAND, RAND_R_ERROR_RETRIEVING_ENTROPY}, - #else - {"ERROR_RETRIEVING_ENTROPY", 36, 110}, - #endif - #ifdef RAND_R_ERROR_RETRIEVING_NONCE - {"ERROR_RETRIEVING_NONCE", ERR_LIB_RAND, RAND_R_ERROR_RETRIEVING_NONCE}, - #else - {"ERROR_RETRIEVING_NONCE", 36, 111}, - #endif - #ifdef RAND_R_FAILED_TO_CREATE_LOCK - {"FAILED_TO_CREATE_LOCK", ERR_LIB_RAND, RAND_R_FAILED_TO_CREATE_LOCK}, - #else - {"FAILED_TO_CREATE_LOCK", 36, 126}, - #endif - #ifdef RAND_R_FUNC_NOT_IMPLEMENTED - {"FUNC_NOT_IMPLEMENTED", ERR_LIB_RAND, RAND_R_FUNC_NOT_IMPLEMENTED}, - #else - {"FUNC_NOT_IMPLEMENTED", 36, 101}, - #endif - #ifdef RAND_R_FWRITE_ERROR - {"FWRITE_ERROR", ERR_LIB_RAND, RAND_R_FWRITE_ERROR}, - #else - {"FWRITE_ERROR", 36, 123}, - #endif - #ifdef RAND_R_GENERATE_ERROR - {"GENERATE_ERROR", ERR_LIB_RAND, RAND_R_GENERATE_ERROR}, - #else - {"GENERATE_ERROR", 36, 112}, - #endif - #ifdef RAND_R_INTERNAL_ERROR - {"INTERNAL_ERROR", ERR_LIB_RAND, RAND_R_INTERNAL_ERROR}, - #else - {"INTERNAL_ERROR", 36, 113}, - #endif - #ifdef RAND_R_IN_ERROR_STATE - {"IN_ERROR_STATE", ERR_LIB_RAND, RAND_R_IN_ERROR_STATE}, - #else - {"IN_ERROR_STATE", 36, 114}, - #endif - #ifdef RAND_R_NOT_A_REGULAR_FILE - {"NOT_A_REGULAR_FILE", ERR_LIB_RAND, RAND_R_NOT_A_REGULAR_FILE}, - #else - {"NOT_A_REGULAR_FILE", 36, 122}, - #endif - #ifdef RAND_R_NOT_INSTANTIATED - {"NOT_INSTANTIATED", ERR_LIB_RAND, RAND_R_NOT_INSTANTIATED}, - #else - {"NOT_INSTANTIATED", 36, 115}, - #endif - #ifdef RAND_R_NO_DRBG_IMPLEMENTATION_SELECTED - {"NO_DRBG_IMPLEMENTATION_SELECTED", ERR_LIB_RAND, RAND_R_NO_DRBG_IMPLEMENTATION_SELECTED}, - #else - {"NO_DRBG_IMPLEMENTATION_SELECTED", 36, 128}, - #endif - #ifdef RAND_R_PARENT_LOCKING_NOT_ENABLED - {"PARENT_LOCKING_NOT_ENABLED", ERR_LIB_RAND, RAND_R_PARENT_LOCKING_NOT_ENABLED}, - #else - {"PARENT_LOCKING_NOT_ENABLED", 36, 130}, - #endif - #ifdef RAND_R_PARENT_STRENGTH_TOO_WEAK - {"PARENT_STRENGTH_TOO_WEAK", ERR_LIB_RAND, RAND_R_PARENT_STRENGTH_TOO_WEAK}, - #else - {"PARENT_STRENGTH_TOO_WEAK", 36, 131}, - #endif - #ifdef RAND_R_PERSONALISATION_STRING_TOO_LONG - {"PERSONALISATION_STRING_TOO_LONG", ERR_LIB_RAND, RAND_R_PERSONALISATION_STRING_TOO_LONG}, - #else - {"PERSONALISATION_STRING_TOO_LONG", 36, 116}, - #endif - #ifdef RAND_R_PREDICTION_RESISTANCE_NOT_SUPPORTED - {"PREDICTION_RESISTANCE_NOT_SUPPORTED", ERR_LIB_RAND, RAND_R_PREDICTION_RESISTANCE_NOT_SUPPORTED}, - #else - {"PREDICTION_RESISTANCE_NOT_SUPPORTED", 36, 133}, - #endif - #ifdef RAND_R_PRNG_NOT_SEEDED - {"PRNG_NOT_SEEDED", ERR_LIB_RAND, RAND_R_PRNG_NOT_SEEDED}, - #else - {"PRNG_NOT_SEEDED", 36, 100}, - #endif - #ifdef RAND_R_RANDOM_POOL_OVERFLOW - {"RANDOM_POOL_OVERFLOW", ERR_LIB_RAND, RAND_R_RANDOM_POOL_OVERFLOW}, - #else - {"RANDOM_POOL_OVERFLOW", 36, 125}, - #endif - #ifdef RAND_R_RANDOM_POOL_UNDERFLOW - {"RANDOM_POOL_UNDERFLOW", ERR_LIB_RAND, RAND_R_RANDOM_POOL_UNDERFLOW}, - #else - {"RANDOM_POOL_UNDERFLOW", 36, 134}, - #endif - #ifdef RAND_R_REQUEST_TOO_LARGE_FOR_DRBG - {"REQUEST_TOO_LARGE_FOR_DRBG", ERR_LIB_RAND, RAND_R_REQUEST_TOO_LARGE_FOR_DRBG}, - #else - {"REQUEST_TOO_LARGE_FOR_DRBG", 36, 117}, - #endif - #ifdef RAND_R_RESEED_ERROR - {"RESEED_ERROR", ERR_LIB_RAND, RAND_R_RESEED_ERROR}, - #else - {"RESEED_ERROR", 36, 118}, - #endif - #ifdef RAND_R_SELFTEST_FAILURE - {"SELFTEST_FAILURE", ERR_LIB_RAND, RAND_R_SELFTEST_FAILURE}, - #else - {"SELFTEST_FAILURE", 36, 119}, - #endif - #ifdef RAND_R_TOO_LITTLE_NONCE_REQUESTED - {"TOO_LITTLE_NONCE_REQUESTED", ERR_LIB_RAND, RAND_R_TOO_LITTLE_NONCE_REQUESTED}, - #else - {"TOO_LITTLE_NONCE_REQUESTED", 36, 135}, - #endif - #ifdef RAND_R_TOO_MUCH_NONCE_REQUESTED - {"TOO_MUCH_NONCE_REQUESTED", ERR_LIB_RAND, RAND_R_TOO_MUCH_NONCE_REQUESTED}, - #else - {"TOO_MUCH_NONCE_REQUESTED", 36, 136}, - #endif - #ifdef RAND_R_UNSUPPORTED_DRBG_FLAGS - {"UNSUPPORTED_DRBG_FLAGS", ERR_LIB_RAND, RAND_R_UNSUPPORTED_DRBG_FLAGS}, - #else - {"UNSUPPORTED_DRBG_FLAGS", 36, 132}, - #endif - #ifdef RAND_R_UNSUPPORTED_DRBG_TYPE - {"UNSUPPORTED_DRBG_TYPE", ERR_LIB_RAND, RAND_R_UNSUPPORTED_DRBG_TYPE}, - #else - {"UNSUPPORTED_DRBG_TYPE", 36, 120}, - #endif - #ifdef RSA_R_ALGORITHM_MISMATCH - {"ALGORITHM_MISMATCH", ERR_LIB_RSA, RSA_R_ALGORITHM_MISMATCH}, - #else - {"ALGORITHM_MISMATCH", 4, 100}, - #endif - #ifdef RSA_R_BAD_E_VALUE - {"BAD_E_VALUE", ERR_LIB_RSA, RSA_R_BAD_E_VALUE}, - #else - {"BAD_E_VALUE", 4, 101}, - #endif - #ifdef RSA_R_BAD_FIXED_HEADER_DECRYPT - {"BAD_FIXED_HEADER_DECRYPT", ERR_LIB_RSA, RSA_R_BAD_FIXED_HEADER_DECRYPT}, - #else - {"BAD_FIXED_HEADER_DECRYPT", 4, 102}, - #endif - #ifdef RSA_R_BAD_PAD_BYTE_COUNT - {"BAD_PAD_BYTE_COUNT", ERR_LIB_RSA, RSA_R_BAD_PAD_BYTE_COUNT}, - #else - {"BAD_PAD_BYTE_COUNT", 4, 103}, - #endif - #ifdef RSA_R_BAD_SIGNATURE - {"BAD_SIGNATURE", ERR_LIB_RSA, RSA_R_BAD_SIGNATURE}, - #else - {"BAD_SIGNATURE", 4, 104}, - #endif - #ifdef RSA_R_BLOCK_TYPE_IS_NOT_01 - {"BLOCK_TYPE_IS_NOT_01", ERR_LIB_RSA, RSA_R_BLOCK_TYPE_IS_NOT_01}, - #else - {"BLOCK_TYPE_IS_NOT_01", 4, 106}, - #endif - #ifdef RSA_R_BLOCK_TYPE_IS_NOT_02 - {"BLOCK_TYPE_IS_NOT_02", ERR_LIB_RSA, RSA_R_BLOCK_TYPE_IS_NOT_02}, - #else - {"BLOCK_TYPE_IS_NOT_02", 4, 107}, - #endif - #ifdef RSA_R_DATA_GREATER_THAN_MOD_LEN - {"DATA_GREATER_THAN_MOD_LEN", ERR_LIB_RSA, RSA_R_DATA_GREATER_THAN_MOD_LEN}, - #else - {"DATA_GREATER_THAN_MOD_LEN", 4, 108}, - #endif - #ifdef RSA_R_DATA_TOO_LARGE - {"DATA_TOO_LARGE", ERR_LIB_RSA, RSA_R_DATA_TOO_LARGE}, - #else - {"DATA_TOO_LARGE", 4, 109}, - #endif - #ifdef RSA_R_DATA_TOO_LARGE_FOR_KEY_SIZE - {"DATA_TOO_LARGE_FOR_KEY_SIZE", ERR_LIB_RSA, RSA_R_DATA_TOO_LARGE_FOR_KEY_SIZE}, - #else - {"DATA_TOO_LARGE_FOR_KEY_SIZE", 4, 110}, - #endif - #ifdef RSA_R_DATA_TOO_LARGE_FOR_MODULUS - {"DATA_TOO_LARGE_FOR_MODULUS", ERR_LIB_RSA, RSA_R_DATA_TOO_LARGE_FOR_MODULUS}, - #else - {"DATA_TOO_LARGE_FOR_MODULUS", 4, 132}, - #endif - #ifdef RSA_R_DATA_TOO_SMALL - {"DATA_TOO_SMALL", ERR_LIB_RSA, RSA_R_DATA_TOO_SMALL}, - #else - {"DATA_TOO_SMALL", 4, 111}, - #endif - #ifdef RSA_R_DATA_TOO_SMALL_FOR_KEY_SIZE - {"DATA_TOO_SMALL_FOR_KEY_SIZE", ERR_LIB_RSA, RSA_R_DATA_TOO_SMALL_FOR_KEY_SIZE}, - #else - {"DATA_TOO_SMALL_FOR_KEY_SIZE", 4, 122}, - #endif - #ifdef RSA_R_DIGEST_DOES_NOT_MATCH - {"DIGEST_DOES_NOT_MATCH", ERR_LIB_RSA, RSA_R_DIGEST_DOES_NOT_MATCH}, - #else - {"DIGEST_DOES_NOT_MATCH", 4, 158}, - #endif - #ifdef RSA_R_DIGEST_NOT_ALLOWED - {"DIGEST_NOT_ALLOWED", ERR_LIB_RSA, RSA_R_DIGEST_NOT_ALLOWED}, - #else - {"DIGEST_NOT_ALLOWED", 4, 145}, - #endif - #ifdef RSA_R_DIGEST_TOO_BIG_FOR_RSA_KEY - {"DIGEST_TOO_BIG_FOR_RSA_KEY", ERR_LIB_RSA, RSA_R_DIGEST_TOO_BIG_FOR_RSA_KEY}, - #else - {"DIGEST_TOO_BIG_FOR_RSA_KEY", 4, 112}, - #endif - #ifdef RSA_R_DMP1_NOT_CONGRUENT_TO_D - {"DMP1_NOT_CONGRUENT_TO_D", ERR_LIB_RSA, RSA_R_DMP1_NOT_CONGRUENT_TO_D}, - #else - {"DMP1_NOT_CONGRUENT_TO_D", 4, 124}, - #endif - #ifdef RSA_R_DMQ1_NOT_CONGRUENT_TO_D - {"DMQ1_NOT_CONGRUENT_TO_D", ERR_LIB_RSA, RSA_R_DMQ1_NOT_CONGRUENT_TO_D}, - #else - {"DMQ1_NOT_CONGRUENT_TO_D", 4, 125}, - #endif - #ifdef RSA_R_D_E_NOT_CONGRUENT_TO_1 - {"D_E_NOT_CONGRUENT_TO_1", ERR_LIB_RSA, RSA_R_D_E_NOT_CONGRUENT_TO_1}, - #else - {"D_E_NOT_CONGRUENT_TO_1", 4, 123}, - #endif - #ifdef RSA_R_FIRST_OCTET_INVALID - {"FIRST_OCTET_INVALID", ERR_LIB_RSA, RSA_R_FIRST_OCTET_INVALID}, - #else - {"FIRST_OCTET_INVALID", 4, 133}, - #endif - #ifdef RSA_R_ILLEGAL_OR_UNSUPPORTED_PADDING_MODE - {"ILLEGAL_OR_UNSUPPORTED_PADDING_MODE", ERR_LIB_RSA, RSA_R_ILLEGAL_OR_UNSUPPORTED_PADDING_MODE}, - #else - {"ILLEGAL_OR_UNSUPPORTED_PADDING_MODE", 4, 144}, - #endif - #ifdef RSA_R_INVALID_DIGEST - {"INVALID_DIGEST", ERR_LIB_RSA, RSA_R_INVALID_DIGEST}, - #else - {"INVALID_DIGEST", 4, 157}, - #endif - #ifdef RSA_R_INVALID_DIGEST_LENGTH - {"INVALID_DIGEST_LENGTH", ERR_LIB_RSA, RSA_R_INVALID_DIGEST_LENGTH}, - #else - {"INVALID_DIGEST_LENGTH", 4, 143}, - #endif - #ifdef RSA_R_INVALID_HEADER - {"INVALID_HEADER", ERR_LIB_RSA, RSA_R_INVALID_HEADER}, - #else - {"INVALID_HEADER", 4, 137}, - #endif - #ifdef RSA_R_INVALID_LABEL - {"INVALID_LABEL", ERR_LIB_RSA, RSA_R_INVALID_LABEL}, - #else - {"INVALID_LABEL", 4, 160}, - #endif - #ifdef RSA_R_INVALID_MESSAGE_LENGTH - {"INVALID_MESSAGE_LENGTH", ERR_LIB_RSA, RSA_R_INVALID_MESSAGE_LENGTH}, - #else - {"INVALID_MESSAGE_LENGTH", 4, 131}, - #endif - #ifdef RSA_R_INVALID_MGF1_MD - {"INVALID_MGF1_MD", ERR_LIB_RSA, RSA_R_INVALID_MGF1_MD}, - #else - {"INVALID_MGF1_MD", 4, 156}, - #endif - #ifdef RSA_R_INVALID_MULTI_PRIME_KEY - {"INVALID_MULTI_PRIME_KEY", ERR_LIB_RSA, RSA_R_INVALID_MULTI_PRIME_KEY}, - #else - {"INVALID_MULTI_PRIME_KEY", 4, 167}, - #endif - #ifdef RSA_R_INVALID_OAEP_PARAMETERS - {"INVALID_OAEP_PARAMETERS", ERR_LIB_RSA, RSA_R_INVALID_OAEP_PARAMETERS}, - #else - {"INVALID_OAEP_PARAMETERS", 4, 161}, - #endif - #ifdef RSA_R_INVALID_PADDING - {"INVALID_PADDING", ERR_LIB_RSA, RSA_R_INVALID_PADDING}, - #else - {"INVALID_PADDING", 4, 138}, - #endif - #ifdef RSA_R_INVALID_PADDING_MODE - {"INVALID_PADDING_MODE", ERR_LIB_RSA, RSA_R_INVALID_PADDING_MODE}, - #else - {"INVALID_PADDING_MODE", 4, 141}, - #endif - #ifdef RSA_R_INVALID_PSS_PARAMETERS - {"INVALID_PSS_PARAMETERS", ERR_LIB_RSA, RSA_R_INVALID_PSS_PARAMETERS}, - #else - {"INVALID_PSS_PARAMETERS", 4, 149}, - #endif - #ifdef RSA_R_INVALID_PSS_SALTLEN - {"INVALID_PSS_SALTLEN", ERR_LIB_RSA, RSA_R_INVALID_PSS_SALTLEN}, - #else - {"INVALID_PSS_SALTLEN", 4, 146}, - #endif - #ifdef RSA_R_INVALID_SALT_LENGTH - {"INVALID_SALT_LENGTH", ERR_LIB_RSA, RSA_R_INVALID_SALT_LENGTH}, - #else - {"INVALID_SALT_LENGTH", 4, 150}, - #endif - #ifdef RSA_R_INVALID_TRAILER - {"INVALID_TRAILER", ERR_LIB_RSA, RSA_R_INVALID_TRAILER}, - #else - {"INVALID_TRAILER", 4, 139}, - #endif - #ifdef RSA_R_INVALID_X931_DIGEST - {"INVALID_X931_DIGEST", ERR_LIB_RSA, RSA_R_INVALID_X931_DIGEST}, - #else - {"INVALID_X931_DIGEST", 4, 142}, - #endif - #ifdef RSA_R_IQMP_NOT_INVERSE_OF_Q - {"IQMP_NOT_INVERSE_OF_Q", ERR_LIB_RSA, RSA_R_IQMP_NOT_INVERSE_OF_Q}, - #else - {"IQMP_NOT_INVERSE_OF_Q", 4, 126}, - #endif - #ifdef RSA_R_KEY_PRIME_NUM_INVALID - {"KEY_PRIME_NUM_INVALID", ERR_LIB_RSA, RSA_R_KEY_PRIME_NUM_INVALID}, - #else - {"KEY_PRIME_NUM_INVALID", 4, 165}, - #endif - #ifdef RSA_R_KEY_SIZE_TOO_SMALL - {"KEY_SIZE_TOO_SMALL", ERR_LIB_RSA, RSA_R_KEY_SIZE_TOO_SMALL}, - #else - {"KEY_SIZE_TOO_SMALL", 4, 120}, - #endif - #ifdef RSA_R_LAST_OCTET_INVALID - {"LAST_OCTET_INVALID", ERR_LIB_RSA, RSA_R_LAST_OCTET_INVALID}, - #else - {"LAST_OCTET_INVALID", 4, 134}, - #endif - #ifdef RSA_R_MGF1_DIGEST_NOT_ALLOWED - {"MGF1_DIGEST_NOT_ALLOWED", ERR_LIB_RSA, RSA_R_MGF1_DIGEST_NOT_ALLOWED}, - #else - {"MGF1_DIGEST_NOT_ALLOWED", 4, 152}, - #endif - #ifdef RSA_R_MISSING_PRIVATE_KEY - {"MISSING_PRIVATE_KEY", ERR_LIB_RSA, RSA_R_MISSING_PRIVATE_KEY}, - #else - {"MISSING_PRIVATE_KEY", 4, 179}, - #endif - #ifdef RSA_R_MODULUS_TOO_LARGE - {"MODULUS_TOO_LARGE", ERR_LIB_RSA, RSA_R_MODULUS_TOO_LARGE}, - #else - {"MODULUS_TOO_LARGE", 4, 105}, - #endif - #ifdef RSA_R_MP_COEFFICIENT_NOT_INVERSE_OF_R - {"MP_COEFFICIENT_NOT_INVERSE_OF_R", ERR_LIB_RSA, RSA_R_MP_COEFFICIENT_NOT_INVERSE_OF_R}, - #else - {"MP_COEFFICIENT_NOT_INVERSE_OF_R", 4, 168}, - #endif - #ifdef RSA_R_MP_EXPONENT_NOT_CONGRUENT_TO_D - {"MP_EXPONENT_NOT_CONGRUENT_TO_D", ERR_LIB_RSA, RSA_R_MP_EXPONENT_NOT_CONGRUENT_TO_D}, - #else - {"MP_EXPONENT_NOT_CONGRUENT_TO_D", 4, 169}, - #endif - #ifdef RSA_R_MP_R_NOT_PRIME - {"MP_R_NOT_PRIME", ERR_LIB_RSA, RSA_R_MP_R_NOT_PRIME}, - #else - {"MP_R_NOT_PRIME", 4, 170}, - #endif - #ifdef RSA_R_NO_PUBLIC_EXPONENT - {"NO_PUBLIC_EXPONENT", ERR_LIB_RSA, RSA_R_NO_PUBLIC_EXPONENT}, - #else - {"NO_PUBLIC_EXPONENT", 4, 140}, - #endif - #ifdef RSA_R_NULL_BEFORE_BLOCK_MISSING - {"NULL_BEFORE_BLOCK_MISSING", ERR_LIB_RSA, RSA_R_NULL_BEFORE_BLOCK_MISSING}, - #else - {"NULL_BEFORE_BLOCK_MISSING", 4, 113}, - #endif - #ifdef RSA_R_N_DOES_NOT_EQUAL_PRODUCT_OF_PRIMES - {"N_DOES_NOT_EQUAL_PRODUCT_OF_PRIMES", ERR_LIB_RSA, RSA_R_N_DOES_NOT_EQUAL_PRODUCT_OF_PRIMES}, - #else - {"N_DOES_NOT_EQUAL_PRODUCT_OF_PRIMES", 4, 172}, - #endif - #ifdef RSA_R_N_DOES_NOT_EQUAL_P_Q - {"N_DOES_NOT_EQUAL_P_Q", ERR_LIB_RSA, RSA_R_N_DOES_NOT_EQUAL_P_Q}, - #else - {"N_DOES_NOT_EQUAL_P_Q", 4, 127}, - #endif - #ifdef RSA_R_OAEP_DECODING_ERROR - {"OAEP_DECODING_ERROR", ERR_LIB_RSA, RSA_R_OAEP_DECODING_ERROR}, - #else - {"OAEP_DECODING_ERROR", 4, 121}, - #endif - #ifdef RSA_R_OPERATION_NOT_SUPPORTED_FOR_THIS_KEYTYPE - {"OPERATION_NOT_SUPPORTED_FOR_THIS_KEYTYPE", ERR_LIB_RSA, RSA_R_OPERATION_NOT_SUPPORTED_FOR_THIS_KEYTYPE}, - #else - {"OPERATION_NOT_SUPPORTED_FOR_THIS_KEYTYPE", 4, 148}, - #endif - #ifdef RSA_R_PADDING_CHECK_FAILED - {"PADDING_CHECK_FAILED", ERR_LIB_RSA, RSA_R_PADDING_CHECK_FAILED}, - #else - {"PADDING_CHECK_FAILED", 4, 114}, - #endif - #ifdef RSA_R_PKCS_DECODING_ERROR - {"PKCS_DECODING_ERROR", ERR_LIB_RSA, RSA_R_PKCS_DECODING_ERROR}, - #else - {"PKCS_DECODING_ERROR", 4, 159}, - #endif - #ifdef RSA_R_PSS_SALTLEN_TOO_SMALL - {"PSS_SALTLEN_TOO_SMALL", ERR_LIB_RSA, RSA_R_PSS_SALTLEN_TOO_SMALL}, - #else - {"PSS_SALTLEN_TOO_SMALL", 4, 164}, - #endif - #ifdef RSA_R_P_NOT_PRIME - {"P_NOT_PRIME", ERR_LIB_RSA, RSA_R_P_NOT_PRIME}, - #else - {"P_NOT_PRIME", 4, 128}, - #endif - #ifdef RSA_R_Q_NOT_PRIME - {"Q_NOT_PRIME", ERR_LIB_RSA, RSA_R_Q_NOT_PRIME}, - #else - {"Q_NOT_PRIME", 4, 129}, - #endif - #ifdef RSA_R_RSA_OPERATIONS_NOT_SUPPORTED - {"RSA_OPERATIONS_NOT_SUPPORTED", ERR_LIB_RSA, RSA_R_RSA_OPERATIONS_NOT_SUPPORTED}, - #else - {"RSA_OPERATIONS_NOT_SUPPORTED", 4, 130}, - #endif - #ifdef RSA_R_SLEN_CHECK_FAILED - {"SLEN_CHECK_FAILED", ERR_LIB_RSA, RSA_R_SLEN_CHECK_FAILED}, - #else - {"SLEN_CHECK_FAILED", 4, 136}, - #endif - #ifdef RSA_R_SLEN_RECOVERY_FAILED - {"SLEN_RECOVERY_FAILED", ERR_LIB_RSA, RSA_R_SLEN_RECOVERY_FAILED}, - #else - {"SLEN_RECOVERY_FAILED", 4, 135}, - #endif - #ifdef RSA_R_SSLV3_ROLLBACK_ATTACK - {"SSLV3_ROLLBACK_ATTACK", ERR_LIB_RSA, RSA_R_SSLV3_ROLLBACK_ATTACK}, - #else - {"SSLV3_ROLLBACK_ATTACK", 4, 115}, - #endif - #ifdef RSA_R_THE_ASN1_OBJECT_IDENTIFIER_IS_NOT_KNOWN_FOR_THIS_MD - {"THE_ASN1_OBJECT_IDENTIFIER_IS_NOT_KNOWN_FOR_THIS_MD", ERR_LIB_RSA, RSA_R_THE_ASN1_OBJECT_IDENTIFIER_IS_NOT_KNOWN_FOR_THIS_MD}, - #else - {"THE_ASN1_OBJECT_IDENTIFIER_IS_NOT_KNOWN_FOR_THIS_MD", 4, 116}, - #endif - #ifdef RSA_R_UNKNOWN_ALGORITHM_TYPE - {"UNKNOWN_ALGORITHM_TYPE", ERR_LIB_RSA, RSA_R_UNKNOWN_ALGORITHM_TYPE}, - #else - {"UNKNOWN_ALGORITHM_TYPE", 4, 117}, - #endif - #ifdef RSA_R_UNKNOWN_DIGEST - {"UNKNOWN_DIGEST", ERR_LIB_RSA, RSA_R_UNKNOWN_DIGEST}, - #else - {"UNKNOWN_DIGEST", 4, 166}, - #endif - #ifdef RSA_R_UNKNOWN_MASK_DIGEST - {"UNKNOWN_MASK_DIGEST", ERR_LIB_RSA, RSA_R_UNKNOWN_MASK_DIGEST}, - #else - {"UNKNOWN_MASK_DIGEST", 4, 151}, - #endif - #ifdef RSA_R_UNKNOWN_PADDING_TYPE - {"UNKNOWN_PADDING_TYPE", ERR_LIB_RSA, RSA_R_UNKNOWN_PADDING_TYPE}, - #else - {"UNKNOWN_PADDING_TYPE", 4, 118}, - #endif - #ifdef RSA_R_UNSUPPORTED_ENCRYPTION_TYPE - {"UNSUPPORTED_ENCRYPTION_TYPE", ERR_LIB_RSA, RSA_R_UNSUPPORTED_ENCRYPTION_TYPE}, - #else - {"UNSUPPORTED_ENCRYPTION_TYPE", 4, 162}, - #endif - #ifdef RSA_R_UNSUPPORTED_LABEL_SOURCE - {"UNSUPPORTED_LABEL_SOURCE", ERR_LIB_RSA, RSA_R_UNSUPPORTED_LABEL_SOURCE}, - #else - {"UNSUPPORTED_LABEL_SOURCE", 4, 163}, - #endif - #ifdef RSA_R_UNSUPPORTED_MASK_ALGORITHM - {"UNSUPPORTED_MASK_ALGORITHM", ERR_LIB_RSA, RSA_R_UNSUPPORTED_MASK_ALGORITHM}, - #else - {"UNSUPPORTED_MASK_ALGORITHM", 4, 153}, - #endif - #ifdef RSA_R_UNSUPPORTED_MASK_PARAMETER - {"UNSUPPORTED_MASK_PARAMETER", ERR_LIB_RSA, RSA_R_UNSUPPORTED_MASK_PARAMETER}, - #else - {"UNSUPPORTED_MASK_PARAMETER", 4, 154}, - #endif - #ifdef RSA_R_UNSUPPORTED_SIGNATURE_TYPE - {"UNSUPPORTED_SIGNATURE_TYPE", ERR_LIB_RSA, RSA_R_UNSUPPORTED_SIGNATURE_TYPE}, - #else - {"UNSUPPORTED_SIGNATURE_TYPE", 4, 155}, - #endif - #ifdef RSA_R_VALUE_MISSING - {"VALUE_MISSING", ERR_LIB_RSA, RSA_R_VALUE_MISSING}, - #else - {"VALUE_MISSING", 4, 147}, - #endif - #ifdef RSA_R_WRONG_SIGNATURE_LENGTH - {"WRONG_SIGNATURE_LENGTH", ERR_LIB_RSA, RSA_R_WRONG_SIGNATURE_LENGTH}, - #else - {"WRONG_SIGNATURE_LENGTH", 4, 119}, - #endif - #ifdef SSL_R_APPLICATION_DATA_AFTER_CLOSE_NOTIFY - {"APPLICATION_DATA_AFTER_CLOSE_NOTIFY", ERR_LIB_SSL, SSL_R_APPLICATION_DATA_AFTER_CLOSE_NOTIFY}, - #else - {"APPLICATION_DATA_AFTER_CLOSE_NOTIFY", 20, 291}, - #endif - #ifdef SSL_R_APP_DATA_IN_HANDSHAKE - {"APP_DATA_IN_HANDSHAKE", ERR_LIB_SSL, SSL_R_APP_DATA_IN_HANDSHAKE}, - #else - {"APP_DATA_IN_HANDSHAKE", 20, 100}, - #endif - #ifdef SSL_R_ATTEMPT_TO_REUSE_SESSION_IN_DIFFERENT_CONTEXT - {"ATTEMPT_TO_REUSE_SESSION_IN_DIFFERENT_CONTEXT", ERR_LIB_SSL, SSL_R_ATTEMPT_TO_REUSE_SESSION_IN_DIFFERENT_CONTEXT}, - #else - {"ATTEMPT_TO_REUSE_SESSION_IN_DIFFERENT_CONTEXT", 20, 272}, - #endif - #ifdef SSL_R_AT_LEAST_TLS_1_0_NEEDED_IN_FIPS_MODE - {"AT_LEAST_TLS_1_0_NEEDED_IN_FIPS_MODE", ERR_LIB_SSL, SSL_R_AT_LEAST_TLS_1_0_NEEDED_IN_FIPS_MODE}, - #else - {"AT_LEAST_TLS_1_0_NEEDED_IN_FIPS_MODE", 20, 143}, - #endif - #ifdef SSL_R_AT_LEAST_TLS_1_2_NEEDED_IN_SUITEB_MODE - {"AT_LEAST_TLS_1_2_NEEDED_IN_SUITEB_MODE", ERR_LIB_SSL, SSL_R_AT_LEAST_TLS_1_2_NEEDED_IN_SUITEB_MODE}, - #else - {"AT_LEAST_TLS_1_2_NEEDED_IN_SUITEB_MODE", 20, 158}, - #endif - #ifdef SSL_R_BAD_CHANGE_CIPHER_SPEC - {"BAD_CHANGE_CIPHER_SPEC", ERR_LIB_SSL, SSL_R_BAD_CHANGE_CIPHER_SPEC}, - #else - {"BAD_CHANGE_CIPHER_SPEC", 20, 103}, - #endif - #ifdef SSL_R_BAD_CIPHER - {"BAD_CIPHER", ERR_LIB_SSL, SSL_R_BAD_CIPHER}, - #else - {"BAD_CIPHER", 20, 186}, - #endif - #ifdef SSL_R_BAD_DATA - {"BAD_DATA", ERR_LIB_SSL, SSL_R_BAD_DATA}, - #else - {"BAD_DATA", 20, 390}, - #endif - #ifdef SSL_R_BAD_DATA_RETURNED_BY_CALLBACK - {"BAD_DATA_RETURNED_BY_CALLBACK", ERR_LIB_SSL, SSL_R_BAD_DATA_RETURNED_BY_CALLBACK}, - #else - {"BAD_DATA_RETURNED_BY_CALLBACK", 20, 106}, - #endif - #ifdef SSL_R_BAD_DECOMPRESSION - {"BAD_DECOMPRESSION", ERR_LIB_SSL, SSL_R_BAD_DECOMPRESSION}, - #else - {"BAD_DECOMPRESSION", 20, 107}, - #endif - #ifdef SSL_R_BAD_DH_VALUE - {"BAD_DH_VALUE", ERR_LIB_SSL, SSL_R_BAD_DH_VALUE}, - #else - {"BAD_DH_VALUE", 20, 102}, - #endif - #ifdef SSL_R_BAD_DIGEST_LENGTH - {"BAD_DIGEST_LENGTH", ERR_LIB_SSL, SSL_R_BAD_DIGEST_LENGTH}, - #else - {"BAD_DIGEST_LENGTH", 20, 111}, - #endif - #ifdef SSL_R_BAD_EARLY_DATA - {"BAD_EARLY_DATA", ERR_LIB_SSL, SSL_R_BAD_EARLY_DATA}, - #else - {"BAD_EARLY_DATA", 20, 233}, - #endif - #ifdef SSL_R_BAD_ECC_CERT - {"BAD_ECC_CERT", ERR_LIB_SSL, SSL_R_BAD_ECC_CERT}, - #else - {"BAD_ECC_CERT", 20, 304}, - #endif - #ifdef SSL_R_BAD_ECDSA_SIGNATURE - {"BAD_ECDSA_SIGNATURE", ERR_LIB_SSL, SSL_R_BAD_ECDSA_SIGNATURE}, - #else - {"BAD_ECDSA_SIGNATURE", 20, 305}, - #endif - #ifdef SSL_R_BAD_ECPOINT - {"BAD_ECPOINT", ERR_LIB_SSL, SSL_R_BAD_ECPOINT}, - #else - {"BAD_ECPOINT", 20, 306}, - #endif - #ifdef SSL_R_BAD_EXTENSION - {"BAD_EXTENSION", ERR_LIB_SSL, SSL_R_BAD_EXTENSION}, - #else - {"BAD_EXTENSION", 20, 110}, - #endif - #ifdef SSL_R_BAD_HANDSHAKE_LENGTH - {"BAD_HANDSHAKE_LENGTH", ERR_LIB_SSL, SSL_R_BAD_HANDSHAKE_LENGTH}, - #else - {"BAD_HANDSHAKE_LENGTH", 20, 332}, - #endif - #ifdef SSL_R_BAD_HANDSHAKE_STATE - {"BAD_HANDSHAKE_STATE", ERR_LIB_SSL, SSL_R_BAD_HANDSHAKE_STATE}, - #else - {"BAD_HANDSHAKE_STATE", 20, 236}, - #endif - #ifdef SSL_R_BAD_HELLO_REQUEST - {"BAD_HELLO_REQUEST", ERR_LIB_SSL, SSL_R_BAD_HELLO_REQUEST}, - #else - {"BAD_HELLO_REQUEST", 20, 105}, - #endif - #ifdef SSL_R_BAD_HRR_VERSION - {"BAD_HRR_VERSION", ERR_LIB_SSL, SSL_R_BAD_HRR_VERSION}, - #else - {"BAD_HRR_VERSION", 20, 263}, - #endif - #ifdef SSL_R_BAD_KEY_SHARE - {"BAD_KEY_SHARE", ERR_LIB_SSL, SSL_R_BAD_KEY_SHARE}, - #else - {"BAD_KEY_SHARE", 20, 108}, - #endif - #ifdef SSL_R_BAD_KEY_UPDATE - {"BAD_KEY_UPDATE", ERR_LIB_SSL, SSL_R_BAD_KEY_UPDATE}, - #else - {"BAD_KEY_UPDATE", 20, 122}, - #endif - #ifdef SSL_R_BAD_LEGACY_VERSION - {"BAD_LEGACY_VERSION", ERR_LIB_SSL, SSL_R_BAD_LEGACY_VERSION}, - #else - {"BAD_LEGACY_VERSION", 20, 292}, - #endif - #ifdef SSL_R_BAD_LENGTH - {"BAD_LENGTH", ERR_LIB_SSL, SSL_R_BAD_LENGTH}, - #else - {"BAD_LENGTH", 20, 271}, - #endif - #ifdef SSL_R_BAD_MAC_LENGTH - {"BAD_MAC_LENGTH", ERR_LIB_SSL, SSL_R_BAD_MAC_LENGTH}, - #else - {"BAD_MAC_LENGTH", 20, 333}, - #endif - #ifdef SSL_R_BAD_PACKET - {"BAD_PACKET", ERR_LIB_SSL, SSL_R_BAD_PACKET}, - #else - {"BAD_PACKET", 20, 240}, - #endif - #ifdef SSL_R_BAD_PACKET_LENGTH - {"BAD_PACKET_LENGTH", ERR_LIB_SSL, SSL_R_BAD_PACKET_LENGTH}, - #else - {"BAD_PACKET_LENGTH", 20, 115}, - #endif - #ifdef SSL_R_BAD_PROTOCOL_VERSION_NUMBER - {"BAD_PROTOCOL_VERSION_NUMBER", ERR_LIB_SSL, SSL_R_BAD_PROTOCOL_VERSION_NUMBER}, - #else - {"BAD_PROTOCOL_VERSION_NUMBER", 20, 116}, - #endif - #ifdef SSL_R_BAD_PSK - {"BAD_PSK", ERR_LIB_SSL, SSL_R_BAD_PSK}, - #else - {"BAD_PSK", 20, 219}, - #endif - #ifdef SSL_R_BAD_PSK_IDENTITY - {"BAD_PSK_IDENTITY", ERR_LIB_SSL, SSL_R_BAD_PSK_IDENTITY}, - #else - {"BAD_PSK_IDENTITY", 20, 114}, - #endif - #ifdef SSL_R_BAD_PSK_IDENTITY_HINT_LENGTH - {"BAD_PSK_IDENTITY_HINT_LENGTH", ERR_LIB_SSL, SSL_R_BAD_PSK_IDENTITY_HINT_LENGTH}, - #else - {"BAD_PSK_IDENTITY_HINT_LENGTH", 20, 316}, - #endif - #ifdef SSL_R_BAD_RECORD_TYPE - {"BAD_RECORD_TYPE", ERR_LIB_SSL, SSL_R_BAD_RECORD_TYPE}, - #else - {"BAD_RECORD_TYPE", 20, 443}, - #endif - #ifdef SSL_R_BAD_RSA_ENCRYPT - {"BAD_RSA_ENCRYPT", ERR_LIB_SSL, SSL_R_BAD_RSA_ENCRYPT}, - #else - {"BAD_RSA_ENCRYPT", 20, 119}, - #endif - #ifdef SSL_R_BAD_SIGNATURE - {"BAD_SIGNATURE", ERR_LIB_SSL, SSL_R_BAD_SIGNATURE}, - #else - {"BAD_SIGNATURE", 20, 123}, - #endif - #ifdef SSL_R_BAD_SRP_A_LENGTH - {"BAD_SRP_A_LENGTH", ERR_LIB_SSL, SSL_R_BAD_SRP_A_LENGTH}, - #else - {"BAD_SRP_A_LENGTH", 20, 347}, - #endif - #ifdef SSL_R_BAD_SRP_B_LENGTH - {"BAD_SRP_B_LENGTH", ERR_LIB_SSL, SSL_R_BAD_SRP_B_LENGTH}, - #else - {"BAD_SRP_B_LENGTH", 20, 348}, - #endif - #ifdef SSL_R_BAD_SRP_G_LENGTH - {"BAD_SRP_G_LENGTH", ERR_LIB_SSL, SSL_R_BAD_SRP_G_LENGTH}, - #else - {"BAD_SRP_G_LENGTH", 20, 349}, - #endif - #ifdef SSL_R_BAD_SRP_N_LENGTH - {"BAD_SRP_N_LENGTH", ERR_LIB_SSL, SSL_R_BAD_SRP_N_LENGTH}, - #else - {"BAD_SRP_N_LENGTH", 20, 350}, - #endif - #ifdef SSL_R_BAD_SRP_PARAMETERS - {"BAD_SRP_PARAMETERS", ERR_LIB_SSL, SSL_R_BAD_SRP_PARAMETERS}, - #else - {"BAD_SRP_PARAMETERS", 20, 371}, - #endif - #ifdef SSL_R_BAD_SRP_S_LENGTH - {"BAD_SRP_S_LENGTH", ERR_LIB_SSL, SSL_R_BAD_SRP_S_LENGTH}, - #else - {"BAD_SRP_S_LENGTH", 20, 351}, - #endif - #ifdef SSL_R_BAD_SRTP_MKI_VALUE - {"BAD_SRTP_MKI_VALUE", ERR_LIB_SSL, SSL_R_BAD_SRTP_MKI_VALUE}, - #else - {"BAD_SRTP_MKI_VALUE", 20, 352}, - #endif - #ifdef SSL_R_BAD_SRTP_PROTECTION_PROFILE_LIST - {"BAD_SRTP_PROTECTION_PROFILE_LIST", ERR_LIB_SSL, SSL_R_BAD_SRTP_PROTECTION_PROFILE_LIST}, - #else - {"BAD_SRTP_PROTECTION_PROFILE_LIST", 20, 353}, - #endif - #ifdef SSL_R_BAD_SSL_FILETYPE - {"BAD_SSL_FILETYPE", ERR_LIB_SSL, SSL_R_BAD_SSL_FILETYPE}, - #else - {"BAD_SSL_FILETYPE", 20, 124}, - #endif - #ifdef SSL_R_BAD_VALUE - {"BAD_VALUE", ERR_LIB_SSL, SSL_R_BAD_VALUE}, - #else - {"BAD_VALUE", 20, 384}, - #endif - #ifdef SSL_R_BAD_WRITE_RETRY - {"BAD_WRITE_RETRY", ERR_LIB_SSL, SSL_R_BAD_WRITE_RETRY}, - #else - {"BAD_WRITE_RETRY", 20, 127}, - #endif - #ifdef SSL_R_BINDER_DOES_NOT_VERIFY - {"BINDER_DOES_NOT_VERIFY", ERR_LIB_SSL, SSL_R_BINDER_DOES_NOT_VERIFY}, - #else - {"BINDER_DOES_NOT_VERIFY", 20, 253}, - #endif - #ifdef SSL_R_BIO_NOT_SET - {"BIO_NOT_SET", ERR_LIB_SSL, SSL_R_BIO_NOT_SET}, - #else - {"BIO_NOT_SET", 20, 128}, - #endif - #ifdef SSL_R_BLOCK_CIPHER_PAD_IS_WRONG - {"BLOCK_CIPHER_PAD_IS_WRONG", ERR_LIB_SSL, SSL_R_BLOCK_CIPHER_PAD_IS_WRONG}, - #else - {"BLOCK_CIPHER_PAD_IS_WRONG", 20, 129}, - #endif - #ifdef SSL_R_BN_LIB - {"BN_LIB", ERR_LIB_SSL, SSL_R_BN_LIB}, - #else - {"BN_LIB", 20, 130}, - #endif - #ifdef SSL_R_CALLBACK_FAILED - {"CALLBACK_FAILED", ERR_LIB_SSL, SSL_R_CALLBACK_FAILED}, - #else - {"CALLBACK_FAILED", 20, 234}, - #endif - #ifdef SSL_R_CANNOT_CHANGE_CIPHER - {"CANNOT_CHANGE_CIPHER", ERR_LIB_SSL, SSL_R_CANNOT_CHANGE_CIPHER}, - #else - {"CANNOT_CHANGE_CIPHER", 20, 109}, - #endif - #ifdef SSL_R_CA_DN_LENGTH_MISMATCH - {"CA_DN_LENGTH_MISMATCH", ERR_LIB_SSL, SSL_R_CA_DN_LENGTH_MISMATCH}, - #else - {"CA_DN_LENGTH_MISMATCH", 20, 131}, - #endif - #ifdef SSL_R_CA_KEY_TOO_SMALL - {"CA_KEY_TOO_SMALL", ERR_LIB_SSL, SSL_R_CA_KEY_TOO_SMALL}, - #else - {"CA_KEY_TOO_SMALL", 20, 397}, - #endif - #ifdef SSL_R_CA_MD_TOO_WEAK - {"CA_MD_TOO_WEAK", ERR_LIB_SSL, SSL_R_CA_MD_TOO_WEAK}, - #else - {"CA_MD_TOO_WEAK", 20, 398}, - #endif - #ifdef SSL_R_CCS_RECEIVED_EARLY - {"CCS_RECEIVED_EARLY", ERR_LIB_SSL, SSL_R_CCS_RECEIVED_EARLY}, - #else - {"CCS_RECEIVED_EARLY", 20, 133}, - #endif - #ifdef SSL_R_CERTIFICATE_VERIFY_FAILED - {"CERTIFICATE_VERIFY_FAILED", ERR_LIB_SSL, SSL_R_CERTIFICATE_VERIFY_FAILED}, - #else - {"CERTIFICATE_VERIFY_FAILED", 20, 134}, - #endif - #ifdef SSL_R_CERT_CB_ERROR - {"CERT_CB_ERROR", ERR_LIB_SSL, SSL_R_CERT_CB_ERROR}, - #else - {"CERT_CB_ERROR", 20, 377}, - #endif - #ifdef SSL_R_CERT_LENGTH_MISMATCH - {"CERT_LENGTH_MISMATCH", ERR_LIB_SSL, SSL_R_CERT_LENGTH_MISMATCH}, - #else - {"CERT_LENGTH_MISMATCH", 20, 135}, - #endif - #ifdef SSL_R_CIPHERSUITE_DIGEST_HAS_CHANGED - {"CIPHERSUITE_DIGEST_HAS_CHANGED", ERR_LIB_SSL, SSL_R_CIPHERSUITE_DIGEST_HAS_CHANGED}, - #else - {"CIPHERSUITE_DIGEST_HAS_CHANGED", 20, 218}, - #endif - #ifdef SSL_R_CIPHER_CODE_WRONG_LENGTH - {"CIPHER_CODE_WRONG_LENGTH", ERR_LIB_SSL, SSL_R_CIPHER_CODE_WRONG_LENGTH}, - #else - {"CIPHER_CODE_WRONG_LENGTH", 20, 137}, - #endif - #ifdef SSL_R_CIPHER_OR_HASH_UNAVAILABLE - {"CIPHER_OR_HASH_UNAVAILABLE", ERR_LIB_SSL, SSL_R_CIPHER_OR_HASH_UNAVAILABLE}, - #else - {"CIPHER_OR_HASH_UNAVAILABLE", 20, 138}, - #endif - #ifdef SSL_R_CLIENTHELLO_TLSEXT - {"CLIENTHELLO_TLSEXT", ERR_LIB_SSL, SSL_R_CLIENTHELLO_TLSEXT}, - #else - {"CLIENTHELLO_TLSEXT", 20, 226}, - #endif - #ifdef SSL_R_COMPRESSED_LENGTH_TOO_LONG - {"COMPRESSED_LENGTH_TOO_LONG", ERR_LIB_SSL, SSL_R_COMPRESSED_LENGTH_TOO_LONG}, - #else - {"COMPRESSED_LENGTH_TOO_LONG", 20, 140}, - #endif - #ifdef SSL_R_COMPRESSION_DISABLED - {"COMPRESSION_DISABLED", ERR_LIB_SSL, SSL_R_COMPRESSION_DISABLED}, - #else - {"COMPRESSION_DISABLED", 20, 343}, - #endif - #ifdef SSL_R_COMPRESSION_FAILURE - {"COMPRESSION_FAILURE", ERR_LIB_SSL, SSL_R_COMPRESSION_FAILURE}, - #else - {"COMPRESSION_FAILURE", 20, 141}, - #endif - #ifdef SSL_R_COMPRESSION_ID_NOT_WITHIN_PRIVATE_RANGE - {"COMPRESSION_ID_NOT_WITHIN_PRIVATE_RANGE", ERR_LIB_SSL, SSL_R_COMPRESSION_ID_NOT_WITHIN_PRIVATE_RANGE}, - #else - {"COMPRESSION_ID_NOT_WITHIN_PRIVATE_RANGE", 20, 307}, - #endif - #ifdef SSL_R_COMPRESSION_LIBRARY_ERROR - {"COMPRESSION_LIBRARY_ERROR", ERR_LIB_SSL, SSL_R_COMPRESSION_LIBRARY_ERROR}, - #else - {"COMPRESSION_LIBRARY_ERROR", 20, 142}, - #endif - #ifdef SSL_R_CONNECTION_TYPE_NOT_SET - {"CONNECTION_TYPE_NOT_SET", ERR_LIB_SSL, SSL_R_CONNECTION_TYPE_NOT_SET}, - #else - {"CONNECTION_TYPE_NOT_SET", 20, 144}, - #endif - #ifdef SSL_R_CONTEXT_NOT_DANE_ENABLED - {"CONTEXT_NOT_DANE_ENABLED", ERR_LIB_SSL, SSL_R_CONTEXT_NOT_DANE_ENABLED}, - #else - {"CONTEXT_NOT_DANE_ENABLED", 20, 167}, - #endif - #ifdef SSL_R_COOKIE_GEN_CALLBACK_FAILURE - {"COOKIE_GEN_CALLBACK_FAILURE", ERR_LIB_SSL, SSL_R_COOKIE_GEN_CALLBACK_FAILURE}, - #else - {"COOKIE_GEN_CALLBACK_FAILURE", 20, 400}, - #endif - #ifdef SSL_R_COOKIE_MISMATCH - {"COOKIE_MISMATCH", ERR_LIB_SSL, SSL_R_COOKIE_MISMATCH}, - #else - {"COOKIE_MISMATCH", 20, 308}, - #endif - #ifdef SSL_R_CUSTOM_EXT_HANDLER_ALREADY_INSTALLED - {"CUSTOM_EXT_HANDLER_ALREADY_INSTALLED", ERR_LIB_SSL, SSL_R_CUSTOM_EXT_HANDLER_ALREADY_INSTALLED}, - #else - {"CUSTOM_EXT_HANDLER_ALREADY_INSTALLED", 20, 206}, - #endif - #ifdef SSL_R_DANE_ALREADY_ENABLED - {"DANE_ALREADY_ENABLED", ERR_LIB_SSL, SSL_R_DANE_ALREADY_ENABLED}, - #else - {"DANE_ALREADY_ENABLED", 20, 172}, - #endif - #ifdef SSL_R_DANE_CANNOT_OVERRIDE_MTYPE_FULL - {"DANE_CANNOT_OVERRIDE_MTYPE_FULL", ERR_LIB_SSL, SSL_R_DANE_CANNOT_OVERRIDE_MTYPE_FULL}, - #else - {"DANE_CANNOT_OVERRIDE_MTYPE_FULL", 20, 173}, - #endif - #ifdef SSL_R_DANE_NOT_ENABLED - {"DANE_NOT_ENABLED", ERR_LIB_SSL, SSL_R_DANE_NOT_ENABLED}, - #else - {"DANE_NOT_ENABLED", 20, 175}, - #endif - #ifdef SSL_R_DANE_TLSA_BAD_CERTIFICATE - {"DANE_TLSA_BAD_CERTIFICATE", ERR_LIB_SSL, SSL_R_DANE_TLSA_BAD_CERTIFICATE}, - #else - {"DANE_TLSA_BAD_CERTIFICATE", 20, 180}, - #endif - #ifdef SSL_R_DANE_TLSA_BAD_CERTIFICATE_USAGE - {"DANE_TLSA_BAD_CERTIFICATE_USAGE", ERR_LIB_SSL, SSL_R_DANE_TLSA_BAD_CERTIFICATE_USAGE}, - #else - {"DANE_TLSA_BAD_CERTIFICATE_USAGE", 20, 184}, - #endif - #ifdef SSL_R_DANE_TLSA_BAD_DATA_LENGTH - {"DANE_TLSA_BAD_DATA_LENGTH", ERR_LIB_SSL, SSL_R_DANE_TLSA_BAD_DATA_LENGTH}, - #else - {"DANE_TLSA_BAD_DATA_LENGTH", 20, 189}, - #endif - #ifdef SSL_R_DANE_TLSA_BAD_DIGEST_LENGTH - {"DANE_TLSA_BAD_DIGEST_LENGTH", ERR_LIB_SSL, SSL_R_DANE_TLSA_BAD_DIGEST_LENGTH}, - #else - {"DANE_TLSA_BAD_DIGEST_LENGTH", 20, 192}, - #endif - #ifdef SSL_R_DANE_TLSA_BAD_MATCHING_TYPE - {"DANE_TLSA_BAD_MATCHING_TYPE", ERR_LIB_SSL, SSL_R_DANE_TLSA_BAD_MATCHING_TYPE}, - #else - {"DANE_TLSA_BAD_MATCHING_TYPE", 20, 200}, - #endif - #ifdef SSL_R_DANE_TLSA_BAD_PUBLIC_KEY - {"DANE_TLSA_BAD_PUBLIC_KEY", ERR_LIB_SSL, SSL_R_DANE_TLSA_BAD_PUBLIC_KEY}, - #else - {"DANE_TLSA_BAD_PUBLIC_KEY", 20, 201}, - #endif - #ifdef SSL_R_DANE_TLSA_BAD_SELECTOR - {"DANE_TLSA_BAD_SELECTOR", ERR_LIB_SSL, SSL_R_DANE_TLSA_BAD_SELECTOR}, - #else - {"DANE_TLSA_BAD_SELECTOR", 20, 202}, - #endif - #ifdef SSL_R_DANE_TLSA_NULL_DATA - {"DANE_TLSA_NULL_DATA", ERR_LIB_SSL, SSL_R_DANE_TLSA_NULL_DATA}, - #else - {"DANE_TLSA_NULL_DATA", 20, 203}, - #endif - #ifdef SSL_R_DATA_BETWEEN_CCS_AND_FINISHED - {"DATA_BETWEEN_CCS_AND_FINISHED", ERR_LIB_SSL, SSL_R_DATA_BETWEEN_CCS_AND_FINISHED}, - #else - {"DATA_BETWEEN_CCS_AND_FINISHED", 20, 145}, - #endif - #ifdef SSL_R_DATA_LENGTH_TOO_LONG - {"DATA_LENGTH_TOO_LONG", ERR_LIB_SSL, SSL_R_DATA_LENGTH_TOO_LONG}, - #else - {"DATA_LENGTH_TOO_LONG", 20, 146}, - #endif - #ifdef SSL_R_DECRYPTION_FAILED - {"DECRYPTION_FAILED", ERR_LIB_SSL, SSL_R_DECRYPTION_FAILED}, - #else - {"DECRYPTION_FAILED", 20, 147}, - #endif - #ifdef SSL_R_DECRYPTION_FAILED_OR_BAD_RECORD_MAC - {"DECRYPTION_FAILED_OR_BAD_RECORD_MAC", ERR_LIB_SSL, SSL_R_DECRYPTION_FAILED_OR_BAD_RECORD_MAC}, - #else - {"DECRYPTION_FAILED_OR_BAD_RECORD_MAC", 20, 281}, - #endif - #ifdef SSL_R_DH_KEY_TOO_SMALL - {"DH_KEY_TOO_SMALL", ERR_LIB_SSL, SSL_R_DH_KEY_TOO_SMALL}, - #else - {"DH_KEY_TOO_SMALL", 20, 394}, - #endif - #ifdef SSL_R_DH_PUBLIC_VALUE_LENGTH_IS_WRONG - {"DH_PUBLIC_VALUE_LENGTH_IS_WRONG", ERR_LIB_SSL, SSL_R_DH_PUBLIC_VALUE_LENGTH_IS_WRONG}, - #else - {"DH_PUBLIC_VALUE_LENGTH_IS_WRONG", 20, 148}, - #endif - #ifdef SSL_R_DIGEST_CHECK_FAILED - {"DIGEST_CHECK_FAILED", ERR_LIB_SSL, SSL_R_DIGEST_CHECK_FAILED}, - #else - {"DIGEST_CHECK_FAILED", 20, 149}, - #endif - #ifdef SSL_R_DTLS_MESSAGE_TOO_BIG - {"DTLS_MESSAGE_TOO_BIG", ERR_LIB_SSL, SSL_R_DTLS_MESSAGE_TOO_BIG}, - #else - {"DTLS_MESSAGE_TOO_BIG", 20, 334}, - #endif - #ifdef SSL_R_DUPLICATE_COMPRESSION_ID - {"DUPLICATE_COMPRESSION_ID", ERR_LIB_SSL, SSL_R_DUPLICATE_COMPRESSION_ID}, - #else - {"DUPLICATE_COMPRESSION_ID", 20, 309}, - #endif - #ifdef SSL_R_ECC_CERT_NOT_FOR_KEY_AGREEMENT - {"ECC_CERT_NOT_FOR_KEY_AGREEMENT", ERR_LIB_SSL, SSL_R_ECC_CERT_NOT_FOR_KEY_AGREEMENT}, - #else - {"ECC_CERT_NOT_FOR_KEY_AGREEMENT", 20, 317}, - #endif - #ifdef SSL_R_ECC_CERT_NOT_FOR_SIGNING - {"ECC_CERT_NOT_FOR_SIGNING", ERR_LIB_SSL, SSL_R_ECC_CERT_NOT_FOR_SIGNING}, - #else - {"ECC_CERT_NOT_FOR_SIGNING", 20, 318}, - #endif - #ifdef SSL_R_ECC_CERT_SHOULD_HAVE_RSA_SIGNATURE - {"ECC_CERT_SHOULD_HAVE_RSA_SIGNATURE", ERR_LIB_SSL, SSL_R_ECC_CERT_SHOULD_HAVE_RSA_SIGNATURE}, - #else - {"ECC_CERT_SHOULD_HAVE_RSA_SIGNATURE", 20, 322}, - #endif - #ifdef SSL_R_ECC_CERT_SHOULD_HAVE_SHA1_SIGNATURE - {"ECC_CERT_SHOULD_HAVE_SHA1_SIGNATURE", ERR_LIB_SSL, SSL_R_ECC_CERT_SHOULD_HAVE_SHA1_SIGNATURE}, - #else - {"ECC_CERT_SHOULD_HAVE_SHA1_SIGNATURE", 20, 323}, - #endif - #ifdef SSL_R_ECDH_REQUIRED_FOR_SUITEB_MODE - {"ECDH_REQUIRED_FOR_SUITEB_MODE", ERR_LIB_SSL, SSL_R_ECDH_REQUIRED_FOR_SUITEB_MODE}, - #else - {"ECDH_REQUIRED_FOR_SUITEB_MODE", 20, 374}, - #endif - #ifdef SSL_R_ECGROUP_TOO_LARGE_FOR_CIPHER - {"ECGROUP_TOO_LARGE_FOR_CIPHER", ERR_LIB_SSL, SSL_R_ECGROUP_TOO_LARGE_FOR_CIPHER}, - #else - {"ECGROUP_TOO_LARGE_FOR_CIPHER", 20, 310}, - #endif - #ifdef SSL_R_EE_KEY_TOO_SMALL - {"EE_KEY_TOO_SMALL", ERR_LIB_SSL, SSL_R_EE_KEY_TOO_SMALL}, - #else - {"EE_KEY_TOO_SMALL", 20, 399}, - #endif - #ifdef SSL_R_EMPTY_SRTP_PROTECTION_PROFILE_LIST - {"EMPTY_SRTP_PROTECTION_PROFILE_LIST", ERR_LIB_SSL, SSL_R_EMPTY_SRTP_PROTECTION_PROFILE_LIST}, - #else - {"EMPTY_SRTP_PROTECTION_PROFILE_LIST", 20, 354}, - #endif - #ifdef SSL_R_ENCRYPTED_LENGTH_TOO_LONG - {"ENCRYPTED_LENGTH_TOO_LONG", ERR_LIB_SSL, SSL_R_ENCRYPTED_LENGTH_TOO_LONG}, - #else - {"ENCRYPTED_LENGTH_TOO_LONG", 20, 150}, - #endif - #ifdef SSL_R_ERROR_IN_RECEIVED_CIPHER_LIST - {"ERROR_IN_RECEIVED_CIPHER_LIST", ERR_LIB_SSL, SSL_R_ERROR_IN_RECEIVED_CIPHER_LIST}, - #else - {"ERROR_IN_RECEIVED_CIPHER_LIST", 20, 151}, - #endif - #ifdef SSL_R_ERROR_SETTING_TLSA_BASE_DOMAIN - {"ERROR_SETTING_TLSA_BASE_DOMAIN", ERR_LIB_SSL, SSL_R_ERROR_SETTING_TLSA_BASE_DOMAIN}, - #else - {"ERROR_SETTING_TLSA_BASE_DOMAIN", 20, 204}, - #endif - #ifdef SSL_R_EXCEEDS_MAX_FRAGMENT_SIZE - {"EXCEEDS_MAX_FRAGMENT_SIZE", ERR_LIB_SSL, SSL_R_EXCEEDS_MAX_FRAGMENT_SIZE}, - #else - {"EXCEEDS_MAX_FRAGMENT_SIZE", 20, 194}, - #endif - #ifdef SSL_R_EXCESSIVE_MESSAGE_SIZE - {"EXCESSIVE_MESSAGE_SIZE", ERR_LIB_SSL, SSL_R_EXCESSIVE_MESSAGE_SIZE}, - #else - {"EXCESSIVE_MESSAGE_SIZE", 20, 152}, - #endif - #ifdef SSL_R_EXTENSION_NOT_RECEIVED - {"EXTENSION_NOT_RECEIVED", ERR_LIB_SSL, SSL_R_EXTENSION_NOT_RECEIVED}, - #else - {"EXTENSION_NOT_RECEIVED", 20, 279}, - #endif - #ifdef SSL_R_EXTRA_DATA_IN_MESSAGE - {"EXTRA_DATA_IN_MESSAGE", ERR_LIB_SSL, SSL_R_EXTRA_DATA_IN_MESSAGE}, - #else - {"EXTRA_DATA_IN_MESSAGE", 20, 153}, - #endif - #ifdef SSL_R_EXT_LENGTH_MISMATCH - {"EXT_LENGTH_MISMATCH", ERR_LIB_SSL, SSL_R_EXT_LENGTH_MISMATCH}, - #else - {"EXT_LENGTH_MISMATCH", 20, 163}, - #endif - #ifdef SSL_R_FAILED_TO_INIT_ASYNC - {"FAILED_TO_INIT_ASYNC", ERR_LIB_SSL, SSL_R_FAILED_TO_INIT_ASYNC}, - #else - {"FAILED_TO_INIT_ASYNC", 20, 405}, - #endif - #ifdef SSL_R_FRAGMENTED_CLIENT_HELLO - {"FRAGMENTED_CLIENT_HELLO", ERR_LIB_SSL, SSL_R_FRAGMENTED_CLIENT_HELLO}, - #else - {"FRAGMENTED_CLIENT_HELLO", 20, 401}, - #endif - #ifdef SSL_R_GOT_A_FIN_BEFORE_A_CCS - {"GOT_A_FIN_BEFORE_A_CCS", ERR_LIB_SSL, SSL_R_GOT_A_FIN_BEFORE_A_CCS}, - #else - {"GOT_A_FIN_BEFORE_A_CCS", 20, 154}, - #endif - #ifdef SSL_R_GOT_NEXT_PROTO_BEFORE_A_CCS - {"GOT_NEXT_PROTO_BEFORE_A_CCS", ERR_LIB_SSL, SSL_R_GOT_NEXT_PROTO_BEFORE_A_CCS}, - #else - {"GOT_NEXT_PROTO_BEFORE_A_CCS", 20, 355}, - #endif - #ifdef SSL_R_GOT_NEXT_PROTO_WITHOUT_EXTENSION - {"GOT_NEXT_PROTO_WITHOUT_EXTENSION", ERR_LIB_SSL, SSL_R_GOT_NEXT_PROTO_WITHOUT_EXTENSION}, - #else - {"GOT_NEXT_PROTO_WITHOUT_EXTENSION", 20, 356}, - #endif - #ifdef SSL_R_HTTPS_PROXY_REQUEST - {"HTTPS_PROXY_REQUEST", ERR_LIB_SSL, SSL_R_HTTPS_PROXY_REQUEST}, - #else - {"HTTPS_PROXY_REQUEST", 20, 155}, - #endif - #ifdef SSL_R_HTTP_REQUEST - {"HTTP_REQUEST", ERR_LIB_SSL, SSL_R_HTTP_REQUEST}, - #else - {"HTTP_REQUEST", 20, 156}, - #endif - #ifdef SSL_R_ILLEGAL_POINT_COMPRESSION - {"ILLEGAL_POINT_COMPRESSION", ERR_LIB_SSL, SSL_R_ILLEGAL_POINT_COMPRESSION}, - #else - {"ILLEGAL_POINT_COMPRESSION", 20, 162}, - #endif - #ifdef SSL_R_ILLEGAL_SUITEB_DIGEST - {"ILLEGAL_SUITEB_DIGEST", ERR_LIB_SSL, SSL_R_ILLEGAL_SUITEB_DIGEST}, - #else - {"ILLEGAL_SUITEB_DIGEST", 20, 380}, - #endif - #ifdef SSL_R_INAPPROPRIATE_FALLBACK - {"INAPPROPRIATE_FALLBACK", ERR_LIB_SSL, SSL_R_INAPPROPRIATE_FALLBACK}, - #else - {"INAPPROPRIATE_FALLBACK", 20, 373}, - #endif - #ifdef SSL_R_INCONSISTENT_COMPRESSION - {"INCONSISTENT_COMPRESSION", ERR_LIB_SSL, SSL_R_INCONSISTENT_COMPRESSION}, - #else - {"INCONSISTENT_COMPRESSION", 20, 340}, - #endif - #ifdef SSL_R_INCONSISTENT_EARLY_DATA_ALPN - {"INCONSISTENT_EARLY_DATA_ALPN", ERR_LIB_SSL, SSL_R_INCONSISTENT_EARLY_DATA_ALPN}, - #else - {"INCONSISTENT_EARLY_DATA_ALPN", 20, 222}, - #endif - #ifdef SSL_R_INCONSISTENT_EARLY_DATA_SNI - {"INCONSISTENT_EARLY_DATA_SNI", ERR_LIB_SSL, SSL_R_INCONSISTENT_EARLY_DATA_SNI}, - #else - {"INCONSISTENT_EARLY_DATA_SNI", 20, 231}, - #endif - #ifdef SSL_R_INCONSISTENT_EXTMS - {"INCONSISTENT_EXTMS", ERR_LIB_SSL, SSL_R_INCONSISTENT_EXTMS}, - #else - {"INCONSISTENT_EXTMS", 20, 104}, - #endif - #ifdef SSL_R_INSUFFICIENT_SECURITY - {"INSUFFICIENT_SECURITY", ERR_LIB_SSL, SSL_R_INSUFFICIENT_SECURITY}, - #else - {"INSUFFICIENT_SECURITY", 20, 241}, - #endif - #ifdef SSL_R_INVALID_ALERT - {"INVALID_ALERT", ERR_LIB_SSL, SSL_R_INVALID_ALERT}, - #else - {"INVALID_ALERT", 20, 205}, - #endif - #ifdef SSL_R_INVALID_CCS_MESSAGE - {"INVALID_CCS_MESSAGE", ERR_LIB_SSL, SSL_R_INVALID_CCS_MESSAGE}, - #else - {"INVALID_CCS_MESSAGE", 20, 260}, - #endif - #ifdef SSL_R_INVALID_CERTIFICATE_OR_ALG - {"INVALID_CERTIFICATE_OR_ALG", ERR_LIB_SSL, SSL_R_INVALID_CERTIFICATE_OR_ALG}, - #else - {"INVALID_CERTIFICATE_OR_ALG", 20, 238}, - #endif - #ifdef SSL_R_INVALID_COMMAND - {"INVALID_COMMAND", ERR_LIB_SSL, SSL_R_INVALID_COMMAND}, - #else - {"INVALID_COMMAND", 20, 280}, - #endif - #ifdef SSL_R_INVALID_COMPRESSION_ALGORITHM - {"INVALID_COMPRESSION_ALGORITHM", ERR_LIB_SSL, SSL_R_INVALID_COMPRESSION_ALGORITHM}, - #else - {"INVALID_COMPRESSION_ALGORITHM", 20, 341}, - #endif - #ifdef SSL_R_INVALID_CONFIG - {"INVALID_CONFIG", ERR_LIB_SSL, SSL_R_INVALID_CONFIG}, - #else - {"INVALID_CONFIG", 20, 283}, - #endif - #ifdef SSL_R_INVALID_CONFIGURATION_NAME - {"INVALID_CONFIGURATION_NAME", ERR_LIB_SSL, SSL_R_INVALID_CONFIGURATION_NAME}, - #else - {"INVALID_CONFIGURATION_NAME", 20, 113}, - #endif - #ifdef SSL_R_INVALID_CONTEXT - {"INVALID_CONTEXT", ERR_LIB_SSL, SSL_R_INVALID_CONTEXT}, - #else - {"INVALID_CONTEXT", 20, 282}, - #endif - #ifdef SSL_R_INVALID_CT_VALIDATION_TYPE - {"INVALID_CT_VALIDATION_TYPE", ERR_LIB_SSL, SSL_R_INVALID_CT_VALIDATION_TYPE}, - #else - {"INVALID_CT_VALIDATION_TYPE", 20, 212}, - #endif - #ifdef SSL_R_INVALID_KEY_UPDATE_TYPE - {"INVALID_KEY_UPDATE_TYPE", ERR_LIB_SSL, SSL_R_INVALID_KEY_UPDATE_TYPE}, - #else - {"INVALID_KEY_UPDATE_TYPE", 20, 120}, - #endif - #ifdef SSL_R_INVALID_MAX_EARLY_DATA - {"INVALID_MAX_EARLY_DATA", ERR_LIB_SSL, SSL_R_INVALID_MAX_EARLY_DATA}, - #else - {"INVALID_MAX_EARLY_DATA", 20, 174}, - #endif - #ifdef SSL_R_INVALID_NULL_CMD_NAME - {"INVALID_NULL_CMD_NAME", ERR_LIB_SSL, SSL_R_INVALID_NULL_CMD_NAME}, - #else - {"INVALID_NULL_CMD_NAME", 20, 385}, - #endif - #ifdef SSL_R_INVALID_SEQUENCE_NUMBER - {"INVALID_SEQUENCE_NUMBER", ERR_LIB_SSL, SSL_R_INVALID_SEQUENCE_NUMBER}, - #else - {"INVALID_SEQUENCE_NUMBER", 20, 402}, - #endif - #ifdef SSL_R_INVALID_SERVERINFO_DATA - {"INVALID_SERVERINFO_DATA", ERR_LIB_SSL, SSL_R_INVALID_SERVERINFO_DATA}, - #else - {"INVALID_SERVERINFO_DATA", 20, 388}, - #endif - #ifdef SSL_R_INVALID_SESSION_ID - {"INVALID_SESSION_ID", ERR_LIB_SSL, SSL_R_INVALID_SESSION_ID}, - #else - {"INVALID_SESSION_ID", 20, 999}, - #endif - #ifdef SSL_R_INVALID_SRP_USERNAME - {"INVALID_SRP_USERNAME", ERR_LIB_SSL, SSL_R_INVALID_SRP_USERNAME}, - #else - {"INVALID_SRP_USERNAME", 20, 357}, - #endif - #ifdef SSL_R_INVALID_STATUS_RESPONSE - {"INVALID_STATUS_RESPONSE", ERR_LIB_SSL, SSL_R_INVALID_STATUS_RESPONSE}, - #else - {"INVALID_STATUS_RESPONSE", 20, 328}, - #endif - #ifdef SSL_R_INVALID_TICKET_KEYS_LENGTH - {"INVALID_TICKET_KEYS_LENGTH", ERR_LIB_SSL, SSL_R_INVALID_TICKET_KEYS_LENGTH}, - #else - {"INVALID_TICKET_KEYS_LENGTH", 20, 325}, - #endif - #ifdef SSL_R_KRB5_S_TKT_NYV - {"KRB5_S_TKT_NYV", ERR_LIB_SSL, SSL_R_KRB5_S_TKT_NYV}, - #else - {"KRB5_S_TKT_NYV", 20, 294}, - #endif - #ifdef SSL_R_KRB5_S_TKT_SKEW - {"KRB5_S_TKT_SKEW", ERR_LIB_SSL, SSL_R_KRB5_S_TKT_SKEW}, - #else - {"KRB5_S_TKT_SKEW", 20, 295}, - #endif - #ifdef SSL_R_LENGTH_MISMATCH - {"LENGTH_MISMATCH", ERR_LIB_SSL, SSL_R_LENGTH_MISMATCH}, - #else - {"LENGTH_MISMATCH", 20, 159}, - #endif - #ifdef SSL_R_LENGTH_TOO_LONG - {"LENGTH_TOO_LONG", ERR_LIB_SSL, SSL_R_LENGTH_TOO_LONG}, - #else - {"LENGTH_TOO_LONG", 20, 404}, - #endif - #ifdef SSL_R_LENGTH_TOO_SHORT - {"LENGTH_TOO_SHORT", ERR_LIB_SSL, SSL_R_LENGTH_TOO_SHORT}, - #else - {"LENGTH_TOO_SHORT", 20, 160}, - #endif - #ifdef SSL_R_LIBRARY_BUG - {"LIBRARY_BUG", ERR_LIB_SSL, SSL_R_LIBRARY_BUG}, - #else - {"LIBRARY_BUG", 20, 274}, - #endif - #ifdef SSL_R_LIBRARY_HAS_NO_CIPHERS - {"LIBRARY_HAS_NO_CIPHERS", ERR_LIB_SSL, SSL_R_LIBRARY_HAS_NO_CIPHERS}, - #else - {"LIBRARY_HAS_NO_CIPHERS", 20, 161}, - #endif - #ifdef SSL_R_MESSAGE_TOO_LONG - {"MESSAGE_TOO_LONG", ERR_LIB_SSL, SSL_R_MESSAGE_TOO_LONG}, - #else - {"MESSAGE_TOO_LONG", 20, 296}, - #endif - #ifdef SSL_R_MISSING_DSA_SIGNING_CERT - {"MISSING_DSA_SIGNING_CERT", ERR_LIB_SSL, SSL_R_MISSING_DSA_SIGNING_CERT}, - #else - {"MISSING_DSA_SIGNING_CERT", 20, 165}, - #endif - #ifdef SSL_R_MISSING_ECDH_CERT - {"MISSING_ECDH_CERT", ERR_LIB_SSL, SSL_R_MISSING_ECDH_CERT}, - #else - {"MISSING_ECDH_CERT", 20, 382}, - #endif - #ifdef SSL_R_MISSING_ECDSA_SIGNING_CERT - {"MISSING_ECDSA_SIGNING_CERT", ERR_LIB_SSL, SSL_R_MISSING_ECDSA_SIGNING_CERT}, - #else - {"MISSING_ECDSA_SIGNING_CERT", 20, 381}, - #endif - #ifdef SSL_R_MISSING_FATAL - {"MISSING_FATAL", ERR_LIB_SSL, SSL_R_MISSING_FATAL}, - #else - {"MISSING_FATAL", 20, 256}, - #endif - #ifdef SSL_R_MISSING_PARAMETERS - {"MISSING_PARAMETERS", ERR_LIB_SSL, SSL_R_MISSING_PARAMETERS}, - #else - {"MISSING_PARAMETERS", 20, 290}, - #endif - #ifdef SSL_R_MISSING_RSA_CERTIFICATE - {"MISSING_RSA_CERTIFICATE", ERR_LIB_SSL, SSL_R_MISSING_RSA_CERTIFICATE}, - #else - {"MISSING_RSA_CERTIFICATE", 20, 168}, - #endif - #ifdef SSL_R_MISSING_RSA_ENCRYPTING_CERT - {"MISSING_RSA_ENCRYPTING_CERT", ERR_LIB_SSL, SSL_R_MISSING_RSA_ENCRYPTING_CERT}, - #else - {"MISSING_RSA_ENCRYPTING_CERT", 20, 169}, - #endif - #ifdef SSL_R_MISSING_RSA_SIGNING_CERT - {"MISSING_RSA_SIGNING_CERT", ERR_LIB_SSL, SSL_R_MISSING_RSA_SIGNING_CERT}, - #else - {"MISSING_RSA_SIGNING_CERT", 20, 170}, - #endif - #ifdef SSL_R_MISSING_SIGALGS_EXTENSION - {"MISSING_SIGALGS_EXTENSION", ERR_LIB_SSL, SSL_R_MISSING_SIGALGS_EXTENSION}, - #else - {"MISSING_SIGALGS_EXTENSION", 20, 112}, - #endif - #ifdef SSL_R_MISSING_SIGNING_CERT - {"MISSING_SIGNING_CERT", ERR_LIB_SSL, SSL_R_MISSING_SIGNING_CERT}, - #else - {"MISSING_SIGNING_CERT", 20, 221}, - #endif - #ifdef SSL_R_MISSING_SRP_PARAM - {"MISSING_SRP_PARAM", ERR_LIB_SSL, SSL_R_MISSING_SRP_PARAM}, - #else - {"MISSING_SRP_PARAM", 20, 358}, - #endif - #ifdef SSL_R_MISSING_SUPPORTED_GROUPS_EXTENSION - {"MISSING_SUPPORTED_GROUPS_EXTENSION", ERR_LIB_SSL, SSL_R_MISSING_SUPPORTED_GROUPS_EXTENSION}, - #else - {"MISSING_SUPPORTED_GROUPS_EXTENSION", 20, 209}, - #endif - #ifdef SSL_R_MISSING_TMP_DH_KEY - {"MISSING_TMP_DH_KEY", ERR_LIB_SSL, SSL_R_MISSING_TMP_DH_KEY}, - #else - {"MISSING_TMP_DH_KEY", 20, 171}, - #endif - #ifdef SSL_R_MISSING_TMP_ECDH_KEY - {"MISSING_TMP_ECDH_KEY", ERR_LIB_SSL, SSL_R_MISSING_TMP_ECDH_KEY}, - #else - {"MISSING_TMP_ECDH_KEY", 20, 311}, - #endif - #ifdef SSL_R_MIXED_HANDSHAKE_AND_NON_HANDSHAKE_DATA - {"MIXED_HANDSHAKE_AND_NON_HANDSHAKE_DATA", ERR_LIB_SSL, SSL_R_MIXED_HANDSHAKE_AND_NON_HANDSHAKE_DATA}, - #else - {"MIXED_HANDSHAKE_AND_NON_HANDSHAKE_DATA", 20, 293}, - #endif - #ifdef SSL_R_MULTIPLE_SGC_RESTARTS - {"MULTIPLE_SGC_RESTARTS", ERR_LIB_SSL, SSL_R_MULTIPLE_SGC_RESTARTS}, - #else - {"MULTIPLE_SGC_RESTARTS", 20, 346}, - #endif - #ifdef SSL_R_NOT_ON_RECORD_BOUNDARY - {"NOT_ON_RECORD_BOUNDARY", ERR_LIB_SSL, SSL_R_NOT_ON_RECORD_BOUNDARY}, - #else - {"NOT_ON_RECORD_BOUNDARY", 20, 182}, - #endif - #ifdef SSL_R_NOT_REPLACING_CERTIFICATE - {"NOT_REPLACING_CERTIFICATE", ERR_LIB_SSL, SSL_R_NOT_REPLACING_CERTIFICATE}, - #else - {"NOT_REPLACING_CERTIFICATE", 20, 289}, - #endif - #ifdef SSL_R_NOT_SERVER - {"NOT_SERVER", ERR_LIB_SSL, SSL_R_NOT_SERVER}, - #else - {"NOT_SERVER", 20, 284}, - #endif - #ifdef SSL_R_NO_APPLICATION_PROTOCOL - {"NO_APPLICATION_PROTOCOL", ERR_LIB_SSL, SSL_R_NO_APPLICATION_PROTOCOL}, - #else - {"NO_APPLICATION_PROTOCOL", 20, 235}, - #endif - #ifdef SSL_R_NO_CERTIFICATES_RETURNED - {"NO_CERTIFICATES_RETURNED", ERR_LIB_SSL, SSL_R_NO_CERTIFICATES_RETURNED}, - #else - {"NO_CERTIFICATES_RETURNED", 20, 176}, - #endif - #ifdef SSL_R_NO_CERTIFICATE_ASSIGNED - {"NO_CERTIFICATE_ASSIGNED", ERR_LIB_SSL, SSL_R_NO_CERTIFICATE_ASSIGNED}, - #else - {"NO_CERTIFICATE_ASSIGNED", 20, 177}, - #endif - #ifdef SSL_R_NO_CERTIFICATE_SET - {"NO_CERTIFICATE_SET", ERR_LIB_SSL, SSL_R_NO_CERTIFICATE_SET}, - #else - {"NO_CERTIFICATE_SET", 20, 179}, - #endif - #ifdef SSL_R_NO_CHANGE_FOLLOWING_HRR - {"NO_CHANGE_FOLLOWING_HRR", ERR_LIB_SSL, SSL_R_NO_CHANGE_FOLLOWING_HRR}, - #else - {"NO_CHANGE_FOLLOWING_HRR", 20, 214}, - #endif - #ifdef SSL_R_NO_CIPHERS_AVAILABLE - {"NO_CIPHERS_AVAILABLE", ERR_LIB_SSL, SSL_R_NO_CIPHERS_AVAILABLE}, - #else - {"NO_CIPHERS_AVAILABLE", 20, 181}, - #endif - #ifdef SSL_R_NO_CIPHERS_SPECIFIED - {"NO_CIPHERS_SPECIFIED", ERR_LIB_SSL, SSL_R_NO_CIPHERS_SPECIFIED}, - #else - {"NO_CIPHERS_SPECIFIED", 20, 183}, - #endif - #ifdef SSL_R_NO_CIPHER_MATCH - {"NO_CIPHER_MATCH", ERR_LIB_SSL, SSL_R_NO_CIPHER_MATCH}, - #else - {"NO_CIPHER_MATCH", 20, 185}, - #endif - #ifdef SSL_R_NO_CLIENT_CERT_METHOD - {"NO_CLIENT_CERT_METHOD", ERR_LIB_SSL, SSL_R_NO_CLIENT_CERT_METHOD}, - #else - {"NO_CLIENT_CERT_METHOD", 20, 331}, - #endif - #ifdef SSL_R_NO_COMPRESSION_SPECIFIED - {"NO_COMPRESSION_SPECIFIED", ERR_LIB_SSL, SSL_R_NO_COMPRESSION_SPECIFIED}, - #else - {"NO_COMPRESSION_SPECIFIED", 20, 187}, - #endif - #ifdef SSL_R_NO_COOKIE_CALLBACK_SET - {"NO_COOKIE_CALLBACK_SET", ERR_LIB_SSL, SSL_R_NO_COOKIE_CALLBACK_SET}, - #else - {"NO_COOKIE_CALLBACK_SET", 20, 287}, - #endif - #ifdef SSL_R_NO_GOST_CERTIFICATE_SENT_BY_PEER - {"NO_GOST_CERTIFICATE_SENT_BY_PEER", ERR_LIB_SSL, SSL_R_NO_GOST_CERTIFICATE_SENT_BY_PEER}, - #else - {"NO_GOST_CERTIFICATE_SENT_BY_PEER", 20, 330}, - #endif - #ifdef SSL_R_NO_METHOD_SPECIFIED - {"NO_METHOD_SPECIFIED", ERR_LIB_SSL, SSL_R_NO_METHOD_SPECIFIED}, - #else - {"NO_METHOD_SPECIFIED", 20, 188}, - #endif - #ifdef SSL_R_NO_PEM_EXTENSIONS - {"NO_PEM_EXTENSIONS", ERR_LIB_SSL, SSL_R_NO_PEM_EXTENSIONS}, - #else - {"NO_PEM_EXTENSIONS", 20, 389}, - #endif - #ifdef SSL_R_NO_PRIVATE_KEY_ASSIGNED - {"NO_PRIVATE_KEY_ASSIGNED", ERR_LIB_SSL, SSL_R_NO_PRIVATE_KEY_ASSIGNED}, - #else - {"NO_PRIVATE_KEY_ASSIGNED", 20, 190}, - #endif - #ifdef SSL_R_NO_PROTOCOLS_AVAILABLE - {"NO_PROTOCOLS_AVAILABLE", ERR_LIB_SSL, SSL_R_NO_PROTOCOLS_AVAILABLE}, - #else - {"NO_PROTOCOLS_AVAILABLE", 20, 191}, - #endif - #ifdef SSL_R_NO_RENEGOTIATION - {"NO_RENEGOTIATION", ERR_LIB_SSL, SSL_R_NO_RENEGOTIATION}, - #else - {"NO_RENEGOTIATION", 20, 339}, - #endif - #ifdef SSL_R_NO_REQUIRED_DIGEST - {"NO_REQUIRED_DIGEST", ERR_LIB_SSL, SSL_R_NO_REQUIRED_DIGEST}, - #else - {"NO_REQUIRED_DIGEST", 20, 324}, - #endif - #ifdef SSL_R_NO_SHARED_CIPHER - {"NO_SHARED_CIPHER", ERR_LIB_SSL, SSL_R_NO_SHARED_CIPHER}, - #else - {"NO_SHARED_CIPHER", 20, 193}, - #endif - #ifdef SSL_R_NO_SHARED_GROUPS - {"NO_SHARED_GROUPS", ERR_LIB_SSL, SSL_R_NO_SHARED_GROUPS}, - #else - {"NO_SHARED_GROUPS", 20, 410}, - #endif - #ifdef SSL_R_NO_SHARED_SIGNATURE_ALGORITHMS - {"NO_SHARED_SIGNATURE_ALGORITHMS", ERR_LIB_SSL, SSL_R_NO_SHARED_SIGNATURE_ALGORITHMS}, - #else - {"NO_SHARED_SIGNATURE_ALGORITHMS", 20, 376}, - #endif - #ifdef SSL_R_NO_SRTP_PROFILES - {"NO_SRTP_PROFILES", ERR_LIB_SSL, SSL_R_NO_SRTP_PROFILES}, - #else - {"NO_SRTP_PROFILES", 20, 359}, - #endif - #ifdef SSL_R_NO_SUITABLE_KEY_SHARE - {"NO_SUITABLE_KEY_SHARE", ERR_LIB_SSL, SSL_R_NO_SUITABLE_KEY_SHARE}, - #else - {"NO_SUITABLE_KEY_SHARE", 20, 101}, - #endif - #ifdef SSL_R_NO_SUITABLE_SIGNATURE_ALGORITHM - {"NO_SUITABLE_SIGNATURE_ALGORITHM", ERR_LIB_SSL, SSL_R_NO_SUITABLE_SIGNATURE_ALGORITHM}, - #else - {"NO_SUITABLE_SIGNATURE_ALGORITHM", 20, 118}, - #endif - #ifdef SSL_R_NO_VALID_SCTS - {"NO_VALID_SCTS", ERR_LIB_SSL, SSL_R_NO_VALID_SCTS}, - #else - {"NO_VALID_SCTS", 20, 216}, - #endif - #ifdef SSL_R_NO_VERIFY_COOKIE_CALLBACK - {"NO_VERIFY_COOKIE_CALLBACK", ERR_LIB_SSL, SSL_R_NO_VERIFY_COOKIE_CALLBACK}, - #else - {"NO_VERIFY_COOKIE_CALLBACK", 20, 403}, - #endif - #ifdef SSL_R_NULL_SSL_CTX - {"NULL_SSL_CTX", ERR_LIB_SSL, SSL_R_NULL_SSL_CTX}, - #else - {"NULL_SSL_CTX", 20, 195}, - #endif - #ifdef SSL_R_NULL_SSL_METHOD_PASSED - {"NULL_SSL_METHOD_PASSED", ERR_LIB_SSL, SSL_R_NULL_SSL_METHOD_PASSED}, - #else - {"NULL_SSL_METHOD_PASSED", 20, 196}, - #endif - #ifdef SSL_R_OLD_SESSION_CIPHER_NOT_RETURNED - {"OLD_SESSION_CIPHER_NOT_RETURNED", ERR_LIB_SSL, SSL_R_OLD_SESSION_CIPHER_NOT_RETURNED}, - #else - {"OLD_SESSION_CIPHER_NOT_RETURNED", 20, 197}, - #endif - #ifdef SSL_R_OLD_SESSION_COMPRESSION_ALGORITHM_NOT_RETURNED - {"OLD_SESSION_COMPRESSION_ALGORITHM_NOT_RETURNED", ERR_LIB_SSL, SSL_R_OLD_SESSION_COMPRESSION_ALGORITHM_NOT_RETURNED}, - #else - {"OLD_SESSION_COMPRESSION_ALGORITHM_NOT_RETURNED", 20, 344}, - #endif - #ifdef SSL_R_ONLY_DTLS_1_2_ALLOWED_IN_SUITEB_MODE - {"ONLY_DTLS_1_2_ALLOWED_IN_SUITEB_MODE", ERR_LIB_SSL, SSL_R_ONLY_DTLS_1_2_ALLOWED_IN_SUITEB_MODE}, - #else - {"ONLY_DTLS_1_2_ALLOWED_IN_SUITEB_MODE", 20, 387}, - #endif - #ifdef SSL_R_ONLY_TLS_1_2_ALLOWED_IN_SUITEB_MODE - {"ONLY_TLS_1_2_ALLOWED_IN_SUITEB_MODE", ERR_LIB_SSL, SSL_R_ONLY_TLS_1_2_ALLOWED_IN_SUITEB_MODE}, - #else - {"ONLY_TLS_1_2_ALLOWED_IN_SUITEB_MODE", 20, 379}, - #endif - #ifdef SSL_R_ONLY_TLS_ALLOWED_IN_FIPS_MODE - {"ONLY_TLS_ALLOWED_IN_FIPS_MODE", ERR_LIB_SSL, SSL_R_ONLY_TLS_ALLOWED_IN_FIPS_MODE}, - #else - {"ONLY_TLS_ALLOWED_IN_FIPS_MODE", 20, 297}, - #endif - #ifdef SSL_R_OPAQUE_PRF_INPUT_TOO_LONG - {"OPAQUE_PRF_INPUT_TOO_LONG", ERR_LIB_SSL, SSL_R_OPAQUE_PRF_INPUT_TOO_LONG}, - #else - {"OPAQUE_PRF_INPUT_TOO_LONG", 20, 327}, - #endif - #ifdef SSL_R_OVERFLOW_ERROR - {"OVERFLOW_ERROR", ERR_LIB_SSL, SSL_R_OVERFLOW_ERROR}, - #else - {"OVERFLOW_ERROR", 20, 237}, - #endif - #ifdef SSL_R_PACKET_LENGTH_TOO_LONG - {"PACKET_LENGTH_TOO_LONG", ERR_LIB_SSL, SSL_R_PACKET_LENGTH_TOO_LONG}, - #else - {"PACKET_LENGTH_TOO_LONG", 20, 198}, - #endif - #ifdef SSL_R_PARSE_TLSEXT - {"PARSE_TLSEXT", ERR_LIB_SSL, SSL_R_PARSE_TLSEXT}, - #else - {"PARSE_TLSEXT", 20, 227}, - #endif - #ifdef SSL_R_PATH_TOO_LONG - {"PATH_TOO_LONG", ERR_LIB_SSL, SSL_R_PATH_TOO_LONG}, - #else - {"PATH_TOO_LONG", 20, 270}, - #endif - #ifdef SSL_R_PEER_DID_NOT_RETURN_A_CERTIFICATE - {"PEER_DID_NOT_RETURN_A_CERTIFICATE", ERR_LIB_SSL, SSL_R_PEER_DID_NOT_RETURN_A_CERTIFICATE}, - #else - {"PEER_DID_NOT_RETURN_A_CERTIFICATE", 20, 199}, - #endif - #ifdef SSL_R_PEM_NAME_BAD_PREFIX - {"PEM_NAME_BAD_PREFIX", ERR_LIB_SSL, SSL_R_PEM_NAME_BAD_PREFIX}, - #else - {"PEM_NAME_BAD_PREFIX", 20, 391}, - #endif - #ifdef SSL_R_PEM_NAME_TOO_SHORT - {"PEM_NAME_TOO_SHORT", ERR_LIB_SSL, SSL_R_PEM_NAME_TOO_SHORT}, - #else - {"PEM_NAME_TOO_SHORT", 20, 392}, - #endif - #ifdef SSL_R_PIPELINE_FAILURE - {"PIPELINE_FAILURE", ERR_LIB_SSL, SSL_R_PIPELINE_FAILURE}, - #else - {"PIPELINE_FAILURE", 20, 406}, - #endif - #ifdef SSL_R_POST_HANDSHAKE_AUTH_ENCODING_ERR - {"POST_HANDSHAKE_AUTH_ENCODING_ERR", ERR_LIB_SSL, SSL_R_POST_HANDSHAKE_AUTH_ENCODING_ERR}, - #else - {"POST_HANDSHAKE_AUTH_ENCODING_ERR", 20, 278}, - #endif - #ifdef SSL_R_PRIVATE_KEY_MISMATCH - {"PRIVATE_KEY_MISMATCH", ERR_LIB_SSL, SSL_R_PRIVATE_KEY_MISMATCH}, - #else - {"PRIVATE_KEY_MISMATCH", 20, 288}, - #endif - #ifdef SSL_R_PROTOCOL_IS_SHUTDOWN - {"PROTOCOL_IS_SHUTDOWN", ERR_LIB_SSL, SSL_R_PROTOCOL_IS_SHUTDOWN}, - #else - {"PROTOCOL_IS_SHUTDOWN", 20, 207}, - #endif - #ifdef SSL_R_PSK_IDENTITY_NOT_FOUND - {"PSK_IDENTITY_NOT_FOUND", ERR_LIB_SSL, SSL_R_PSK_IDENTITY_NOT_FOUND}, - #else - {"PSK_IDENTITY_NOT_FOUND", 20, 223}, - #endif - #ifdef SSL_R_PSK_NO_CLIENT_CB - {"PSK_NO_CLIENT_CB", ERR_LIB_SSL, SSL_R_PSK_NO_CLIENT_CB}, - #else - {"PSK_NO_CLIENT_CB", 20, 224}, - #endif - #ifdef SSL_R_PSK_NO_SERVER_CB - {"PSK_NO_SERVER_CB", ERR_LIB_SSL, SSL_R_PSK_NO_SERVER_CB}, - #else - {"PSK_NO_SERVER_CB", 20, 225}, - #endif - #ifdef SSL_R_READ_BIO_NOT_SET - {"READ_BIO_NOT_SET", ERR_LIB_SSL, SSL_R_READ_BIO_NOT_SET}, - #else - {"READ_BIO_NOT_SET", 20, 211}, - #endif - #ifdef SSL_R_READ_TIMEOUT_EXPIRED - {"READ_TIMEOUT_EXPIRED", ERR_LIB_SSL, SSL_R_READ_TIMEOUT_EXPIRED}, - #else - {"READ_TIMEOUT_EXPIRED", 20, 312}, - #endif - #ifdef SSL_R_RECORD_LENGTH_MISMATCH - {"RECORD_LENGTH_MISMATCH", ERR_LIB_SSL, SSL_R_RECORD_LENGTH_MISMATCH}, - #else - {"RECORD_LENGTH_MISMATCH", 20, 213}, - #endif - #ifdef SSL_R_RECORD_TOO_SMALL - {"RECORD_TOO_SMALL", ERR_LIB_SSL, SSL_R_RECORD_TOO_SMALL}, - #else - {"RECORD_TOO_SMALL", 20, 298}, - #endif - #ifdef SSL_R_RENEGOTIATE_EXT_TOO_LONG - {"RENEGOTIATE_EXT_TOO_LONG", ERR_LIB_SSL, SSL_R_RENEGOTIATE_EXT_TOO_LONG}, - #else - {"RENEGOTIATE_EXT_TOO_LONG", 20, 335}, - #endif - #ifdef SSL_R_RENEGOTIATION_ENCODING_ERR - {"RENEGOTIATION_ENCODING_ERR", ERR_LIB_SSL, SSL_R_RENEGOTIATION_ENCODING_ERR}, - #else - {"RENEGOTIATION_ENCODING_ERR", 20, 336}, - #endif - #ifdef SSL_R_RENEGOTIATION_MISMATCH - {"RENEGOTIATION_MISMATCH", ERR_LIB_SSL, SSL_R_RENEGOTIATION_MISMATCH}, - #else - {"RENEGOTIATION_MISMATCH", 20, 337}, - #endif - #ifdef SSL_R_REQUEST_PENDING - {"REQUEST_PENDING", ERR_LIB_SSL, SSL_R_REQUEST_PENDING}, - #else - {"REQUEST_PENDING", 20, 285}, - #endif - #ifdef SSL_R_REQUEST_SENT - {"REQUEST_SENT", ERR_LIB_SSL, SSL_R_REQUEST_SENT}, - #else - {"REQUEST_SENT", 20, 286}, - #endif - #ifdef SSL_R_REQUIRED_CIPHER_MISSING - {"REQUIRED_CIPHER_MISSING", ERR_LIB_SSL, SSL_R_REQUIRED_CIPHER_MISSING}, - #else - {"REQUIRED_CIPHER_MISSING", 20, 215}, - #endif - #ifdef SSL_R_REQUIRED_COMPRESSION_ALGORITHM_MISSING - {"REQUIRED_COMPRESSION_ALGORITHM_MISSING", ERR_LIB_SSL, SSL_R_REQUIRED_COMPRESSION_ALGORITHM_MISSING}, - #else - {"REQUIRED_COMPRESSION_ALGORITHM_MISSING", 20, 342}, - #endif - #ifdef SSL_R_SCSV_RECEIVED_WHEN_RENEGOTIATING - {"SCSV_RECEIVED_WHEN_RENEGOTIATING", ERR_LIB_SSL, SSL_R_SCSV_RECEIVED_WHEN_RENEGOTIATING}, - #else - {"SCSV_RECEIVED_WHEN_RENEGOTIATING", 20, 345}, - #endif - #ifdef SSL_R_SCT_VERIFICATION_FAILED - {"SCT_VERIFICATION_FAILED", ERR_LIB_SSL, SSL_R_SCT_VERIFICATION_FAILED}, - #else - {"SCT_VERIFICATION_FAILED", 20, 208}, - #endif - #ifdef SSL_R_SERVERHELLO_TLSEXT - {"SERVERHELLO_TLSEXT", ERR_LIB_SSL, SSL_R_SERVERHELLO_TLSEXT}, - #else - {"SERVERHELLO_TLSEXT", 20, 275}, - #endif - #ifdef SSL_R_SESSION_ID_CONTEXT_UNINITIALIZED - {"SESSION_ID_CONTEXT_UNINITIALIZED", ERR_LIB_SSL, SSL_R_SESSION_ID_CONTEXT_UNINITIALIZED}, - #else - {"SESSION_ID_CONTEXT_UNINITIALIZED", 20, 277}, - #endif - #ifdef SSL_R_SHUTDOWN_WHILE_IN_INIT - {"SHUTDOWN_WHILE_IN_INIT", ERR_LIB_SSL, SSL_R_SHUTDOWN_WHILE_IN_INIT}, - #else - {"SHUTDOWN_WHILE_IN_INIT", 20, 407}, - #endif - #ifdef SSL_R_SIGNATURE_ALGORITHMS_ERROR - {"SIGNATURE_ALGORITHMS_ERROR", ERR_LIB_SSL, SSL_R_SIGNATURE_ALGORITHMS_ERROR}, - #else - {"SIGNATURE_ALGORITHMS_ERROR", 20, 360}, - #endif - #ifdef SSL_R_SIGNATURE_FOR_NON_SIGNING_CERTIFICATE - {"SIGNATURE_FOR_NON_SIGNING_CERTIFICATE", ERR_LIB_SSL, SSL_R_SIGNATURE_FOR_NON_SIGNING_CERTIFICATE}, - #else - {"SIGNATURE_FOR_NON_SIGNING_CERTIFICATE", 20, 220}, - #endif - #ifdef SSL_R_SRP_A_CALC - {"SRP_A_CALC", ERR_LIB_SSL, SSL_R_SRP_A_CALC}, - #else - {"SRP_A_CALC", 20, 361}, - #endif - #ifdef SSL_R_SRTP_COULD_NOT_ALLOCATE_PROFILES - {"SRTP_COULD_NOT_ALLOCATE_PROFILES", ERR_LIB_SSL, SSL_R_SRTP_COULD_NOT_ALLOCATE_PROFILES}, - #else - {"SRTP_COULD_NOT_ALLOCATE_PROFILES", 20, 362}, - #endif - #ifdef SSL_R_SRTP_PROTECTION_PROFILE_LIST_TOO_LONG - {"SRTP_PROTECTION_PROFILE_LIST_TOO_LONG", ERR_LIB_SSL, SSL_R_SRTP_PROTECTION_PROFILE_LIST_TOO_LONG}, - #else - {"SRTP_PROTECTION_PROFILE_LIST_TOO_LONG", 20, 363}, - #endif - #ifdef SSL_R_SRTP_UNKNOWN_PROTECTION_PROFILE - {"SRTP_UNKNOWN_PROTECTION_PROFILE", ERR_LIB_SSL, SSL_R_SRTP_UNKNOWN_PROTECTION_PROFILE}, - #else - {"SRTP_UNKNOWN_PROTECTION_PROFILE", 20, 364}, - #endif - #ifdef SSL_R_SSL2_CONNECTION_ID_TOO_LONG - {"SSL2_CONNECTION_ID_TOO_LONG", ERR_LIB_SSL, SSL_R_SSL2_CONNECTION_ID_TOO_LONG}, - #else - {"SSL2_CONNECTION_ID_TOO_LONG", 20, 299}, - #endif - #ifdef SSL_R_SSL3_EXT_INVALID_ECPOINTFORMAT - {"SSL3_EXT_INVALID_ECPOINTFORMAT", ERR_LIB_SSL, SSL_R_SSL3_EXT_INVALID_ECPOINTFORMAT}, - #else - {"SSL3_EXT_INVALID_ECPOINTFORMAT", 20, 321}, - #endif - #ifdef SSL_R_SSL3_EXT_INVALID_MAX_FRAGMENT_LENGTH - {"SSL3_EXT_INVALID_MAX_FRAGMENT_LENGTH", ERR_LIB_SSL, SSL_R_SSL3_EXT_INVALID_MAX_FRAGMENT_LENGTH}, - #else - {"SSL3_EXT_INVALID_MAX_FRAGMENT_LENGTH", 20, 232}, - #endif - #ifdef SSL_R_SSL3_EXT_INVALID_SERVERNAME - {"SSL3_EXT_INVALID_SERVERNAME", ERR_LIB_SSL, SSL_R_SSL3_EXT_INVALID_SERVERNAME}, - #else - {"SSL3_EXT_INVALID_SERVERNAME", 20, 319}, - #endif - #ifdef SSL_R_SSL3_EXT_INVALID_SERVERNAME_TYPE - {"SSL3_EXT_INVALID_SERVERNAME_TYPE", ERR_LIB_SSL, SSL_R_SSL3_EXT_INVALID_SERVERNAME_TYPE}, - #else - {"SSL3_EXT_INVALID_SERVERNAME_TYPE", 20, 320}, - #endif - #ifdef SSL_R_SSL3_SESSION_ID_TOO_LONG - {"SSL3_SESSION_ID_TOO_LONG", ERR_LIB_SSL, SSL_R_SSL3_SESSION_ID_TOO_LONG}, - #else - {"SSL3_SESSION_ID_TOO_LONG", 20, 300}, - #endif - #ifdef SSL_R_SSLV3_ALERT_BAD_CERTIFICATE - {"SSLV3_ALERT_BAD_CERTIFICATE", ERR_LIB_SSL, SSL_R_SSLV3_ALERT_BAD_CERTIFICATE}, - #else - {"SSLV3_ALERT_BAD_CERTIFICATE", 20, 1042}, - #endif - #ifdef SSL_R_SSLV3_ALERT_BAD_RECORD_MAC - {"SSLV3_ALERT_BAD_RECORD_MAC", ERR_LIB_SSL, SSL_R_SSLV3_ALERT_BAD_RECORD_MAC}, - #else - {"SSLV3_ALERT_BAD_RECORD_MAC", 20, 1020}, - #endif - #ifdef SSL_R_SSLV3_ALERT_CERTIFICATE_EXPIRED - {"SSLV3_ALERT_CERTIFICATE_EXPIRED", ERR_LIB_SSL, SSL_R_SSLV3_ALERT_CERTIFICATE_EXPIRED}, - #else - {"SSLV3_ALERT_CERTIFICATE_EXPIRED", 20, 1045}, - #endif - #ifdef SSL_R_SSLV3_ALERT_CERTIFICATE_REVOKED - {"SSLV3_ALERT_CERTIFICATE_REVOKED", ERR_LIB_SSL, SSL_R_SSLV3_ALERT_CERTIFICATE_REVOKED}, - #else - {"SSLV3_ALERT_CERTIFICATE_REVOKED", 20, 1044}, - #endif - #ifdef SSL_R_SSLV3_ALERT_CERTIFICATE_UNKNOWN - {"SSLV3_ALERT_CERTIFICATE_UNKNOWN", ERR_LIB_SSL, SSL_R_SSLV3_ALERT_CERTIFICATE_UNKNOWN}, - #else - {"SSLV3_ALERT_CERTIFICATE_UNKNOWN", 20, 1046}, - #endif - #ifdef SSL_R_SSLV3_ALERT_DECOMPRESSION_FAILURE - {"SSLV3_ALERT_DECOMPRESSION_FAILURE", ERR_LIB_SSL, SSL_R_SSLV3_ALERT_DECOMPRESSION_FAILURE}, - #else - {"SSLV3_ALERT_DECOMPRESSION_FAILURE", 20, 1030}, - #endif - #ifdef SSL_R_SSLV3_ALERT_HANDSHAKE_FAILURE - {"SSLV3_ALERT_HANDSHAKE_FAILURE", ERR_LIB_SSL, SSL_R_SSLV3_ALERT_HANDSHAKE_FAILURE}, - #else - {"SSLV3_ALERT_HANDSHAKE_FAILURE", 20, 1040}, - #endif - #ifdef SSL_R_SSLV3_ALERT_ILLEGAL_PARAMETER - {"SSLV3_ALERT_ILLEGAL_PARAMETER", ERR_LIB_SSL, SSL_R_SSLV3_ALERT_ILLEGAL_PARAMETER}, - #else - {"SSLV3_ALERT_ILLEGAL_PARAMETER", 20, 1047}, - #endif - #ifdef SSL_R_SSLV3_ALERT_NO_CERTIFICATE - {"SSLV3_ALERT_NO_CERTIFICATE", ERR_LIB_SSL, SSL_R_SSLV3_ALERT_NO_CERTIFICATE}, - #else - {"SSLV3_ALERT_NO_CERTIFICATE", 20, 1041}, - #endif - #ifdef SSL_R_SSLV3_ALERT_UNEXPECTED_MESSAGE - {"SSLV3_ALERT_UNEXPECTED_MESSAGE", ERR_LIB_SSL, SSL_R_SSLV3_ALERT_UNEXPECTED_MESSAGE}, - #else - {"SSLV3_ALERT_UNEXPECTED_MESSAGE", 20, 1010}, - #endif - #ifdef SSL_R_SSLV3_ALERT_UNSUPPORTED_CERTIFICATE - {"SSLV3_ALERT_UNSUPPORTED_CERTIFICATE", ERR_LIB_SSL, SSL_R_SSLV3_ALERT_UNSUPPORTED_CERTIFICATE}, - #else - {"SSLV3_ALERT_UNSUPPORTED_CERTIFICATE", 20, 1043}, - #endif - #ifdef SSL_R_SSL_COMMAND_SECTION_EMPTY - {"SSL_COMMAND_SECTION_EMPTY", ERR_LIB_SSL, SSL_R_SSL_COMMAND_SECTION_EMPTY}, - #else - {"SSL_COMMAND_SECTION_EMPTY", 20, 117}, - #endif - #ifdef SSL_R_SSL_COMMAND_SECTION_NOT_FOUND - {"SSL_COMMAND_SECTION_NOT_FOUND", ERR_LIB_SSL, SSL_R_SSL_COMMAND_SECTION_NOT_FOUND}, - #else - {"SSL_COMMAND_SECTION_NOT_FOUND", 20, 125}, - #endif - #ifdef SSL_R_SSL_CTX_HAS_NO_DEFAULT_SSL_VERSION - {"SSL_CTX_HAS_NO_DEFAULT_SSL_VERSION", ERR_LIB_SSL, SSL_R_SSL_CTX_HAS_NO_DEFAULT_SSL_VERSION}, - #else - {"SSL_CTX_HAS_NO_DEFAULT_SSL_VERSION", 20, 228}, - #endif - #ifdef SSL_R_SSL_HANDSHAKE_FAILURE - {"SSL_HANDSHAKE_FAILURE", ERR_LIB_SSL, SSL_R_SSL_HANDSHAKE_FAILURE}, - #else - {"SSL_HANDSHAKE_FAILURE", 20, 229}, - #endif - #ifdef SSL_R_SSL_LIBRARY_HAS_NO_CIPHERS - {"SSL_LIBRARY_HAS_NO_CIPHERS", ERR_LIB_SSL, SSL_R_SSL_LIBRARY_HAS_NO_CIPHERS}, - #else - {"SSL_LIBRARY_HAS_NO_CIPHERS", 20, 230}, - #endif - #ifdef SSL_R_SSL_NEGATIVE_LENGTH - {"SSL_NEGATIVE_LENGTH", ERR_LIB_SSL, SSL_R_SSL_NEGATIVE_LENGTH}, - #else - {"SSL_NEGATIVE_LENGTH", 20, 372}, - #endif - #ifdef SSL_R_SSL_SECTION_EMPTY - {"SSL_SECTION_EMPTY", ERR_LIB_SSL, SSL_R_SSL_SECTION_EMPTY}, - #else - {"SSL_SECTION_EMPTY", 20, 126}, - #endif - #ifdef SSL_R_SSL_SECTION_NOT_FOUND - {"SSL_SECTION_NOT_FOUND", ERR_LIB_SSL, SSL_R_SSL_SECTION_NOT_FOUND}, - #else - {"SSL_SECTION_NOT_FOUND", 20, 136}, - #endif - #ifdef SSL_R_SSL_SESSION_ID_CALLBACK_FAILED - {"SSL_SESSION_ID_CALLBACK_FAILED", ERR_LIB_SSL, SSL_R_SSL_SESSION_ID_CALLBACK_FAILED}, - #else - {"SSL_SESSION_ID_CALLBACK_FAILED", 20, 301}, - #endif - #ifdef SSL_R_SSL_SESSION_ID_CONFLICT - {"SSL_SESSION_ID_CONFLICT", ERR_LIB_SSL, SSL_R_SSL_SESSION_ID_CONFLICT}, - #else - {"SSL_SESSION_ID_CONFLICT", 20, 302}, - #endif - #ifdef SSL_R_SSL_SESSION_ID_CONTEXT_TOO_LONG - {"SSL_SESSION_ID_CONTEXT_TOO_LONG", ERR_LIB_SSL, SSL_R_SSL_SESSION_ID_CONTEXT_TOO_LONG}, - #else - {"SSL_SESSION_ID_CONTEXT_TOO_LONG", 20, 273}, - #endif - #ifdef SSL_R_SSL_SESSION_ID_HAS_BAD_LENGTH - {"SSL_SESSION_ID_HAS_BAD_LENGTH", ERR_LIB_SSL, SSL_R_SSL_SESSION_ID_HAS_BAD_LENGTH}, - #else - {"SSL_SESSION_ID_HAS_BAD_LENGTH", 20, 303}, - #endif - #ifdef SSL_R_SSL_SESSION_ID_TOO_LONG - {"SSL_SESSION_ID_TOO_LONG", ERR_LIB_SSL, SSL_R_SSL_SESSION_ID_TOO_LONG}, - #else - {"SSL_SESSION_ID_TOO_LONG", 20, 408}, - #endif - #ifdef SSL_R_SSL_SESSION_VERSION_MISMATCH - {"SSL_SESSION_VERSION_MISMATCH", ERR_LIB_SSL, SSL_R_SSL_SESSION_VERSION_MISMATCH}, - #else - {"SSL_SESSION_VERSION_MISMATCH", 20, 210}, - #endif - #ifdef SSL_R_STILL_IN_INIT - {"STILL_IN_INIT", ERR_LIB_SSL, SSL_R_STILL_IN_INIT}, - #else - {"STILL_IN_INIT", 20, 121}, - #endif - #ifdef SSL_R_TLSV13_ALERT_CERTIFICATE_REQUIRED - {"TLSV13_ALERT_CERTIFICATE_REQUIRED", ERR_LIB_SSL, SSL_R_TLSV13_ALERT_CERTIFICATE_REQUIRED}, - #else - {"TLSV13_ALERT_CERTIFICATE_REQUIRED", 20, 1116}, - #endif - #ifdef SSL_R_TLSV13_ALERT_MISSING_EXTENSION - {"TLSV13_ALERT_MISSING_EXTENSION", ERR_LIB_SSL, SSL_R_TLSV13_ALERT_MISSING_EXTENSION}, - #else - {"TLSV13_ALERT_MISSING_EXTENSION", 20, 1109}, - #endif - #ifdef SSL_R_TLSV1_ALERT_ACCESS_DENIED - {"TLSV1_ALERT_ACCESS_DENIED", ERR_LIB_SSL, SSL_R_TLSV1_ALERT_ACCESS_DENIED}, - #else - {"TLSV1_ALERT_ACCESS_DENIED", 20, 1049}, - #endif - #ifdef SSL_R_TLSV1_ALERT_DECODE_ERROR - {"TLSV1_ALERT_DECODE_ERROR", ERR_LIB_SSL, SSL_R_TLSV1_ALERT_DECODE_ERROR}, - #else - {"TLSV1_ALERT_DECODE_ERROR", 20, 1050}, - #endif - #ifdef SSL_R_TLSV1_ALERT_DECRYPTION_FAILED - {"TLSV1_ALERT_DECRYPTION_FAILED", ERR_LIB_SSL, SSL_R_TLSV1_ALERT_DECRYPTION_FAILED}, - #else - {"TLSV1_ALERT_DECRYPTION_FAILED", 20, 1021}, - #endif - #ifdef SSL_R_TLSV1_ALERT_DECRYPT_ERROR - {"TLSV1_ALERT_DECRYPT_ERROR", ERR_LIB_SSL, SSL_R_TLSV1_ALERT_DECRYPT_ERROR}, - #else - {"TLSV1_ALERT_DECRYPT_ERROR", 20, 1051}, - #endif - #ifdef SSL_R_TLSV1_ALERT_EXPORT_RESTRICTION - {"TLSV1_ALERT_EXPORT_RESTRICTION", ERR_LIB_SSL, SSL_R_TLSV1_ALERT_EXPORT_RESTRICTION}, - #else - {"TLSV1_ALERT_EXPORT_RESTRICTION", 20, 1060}, - #endif - #ifdef SSL_R_TLSV1_ALERT_INAPPROPRIATE_FALLBACK - {"TLSV1_ALERT_INAPPROPRIATE_FALLBACK", ERR_LIB_SSL, SSL_R_TLSV1_ALERT_INAPPROPRIATE_FALLBACK}, - #else - {"TLSV1_ALERT_INAPPROPRIATE_FALLBACK", 20, 1086}, - #endif - #ifdef SSL_R_TLSV1_ALERT_INSUFFICIENT_SECURITY - {"TLSV1_ALERT_INSUFFICIENT_SECURITY", ERR_LIB_SSL, SSL_R_TLSV1_ALERT_INSUFFICIENT_SECURITY}, - #else - {"TLSV1_ALERT_INSUFFICIENT_SECURITY", 20, 1071}, - #endif - #ifdef SSL_R_TLSV1_ALERT_INTERNAL_ERROR - {"TLSV1_ALERT_INTERNAL_ERROR", ERR_LIB_SSL, SSL_R_TLSV1_ALERT_INTERNAL_ERROR}, - #else - {"TLSV1_ALERT_INTERNAL_ERROR", 20, 1080}, - #endif - #ifdef SSL_R_TLSV1_ALERT_NO_RENEGOTIATION - {"TLSV1_ALERT_NO_RENEGOTIATION", ERR_LIB_SSL, SSL_R_TLSV1_ALERT_NO_RENEGOTIATION}, - #else - {"TLSV1_ALERT_NO_RENEGOTIATION", 20, 1100}, - #endif - #ifdef SSL_R_TLSV1_ALERT_PROTOCOL_VERSION - {"TLSV1_ALERT_PROTOCOL_VERSION", ERR_LIB_SSL, SSL_R_TLSV1_ALERT_PROTOCOL_VERSION}, - #else - {"TLSV1_ALERT_PROTOCOL_VERSION", 20, 1070}, - #endif - #ifdef SSL_R_TLSV1_ALERT_RECORD_OVERFLOW - {"TLSV1_ALERT_RECORD_OVERFLOW", ERR_LIB_SSL, SSL_R_TLSV1_ALERT_RECORD_OVERFLOW}, - #else - {"TLSV1_ALERT_RECORD_OVERFLOW", 20, 1022}, - #endif - #ifdef SSL_R_TLSV1_ALERT_UNKNOWN_CA - {"TLSV1_ALERT_UNKNOWN_CA", ERR_LIB_SSL, SSL_R_TLSV1_ALERT_UNKNOWN_CA}, - #else - {"TLSV1_ALERT_UNKNOWN_CA", 20, 1048}, - #endif - #ifdef SSL_R_TLSV1_ALERT_USER_CANCELLED - {"TLSV1_ALERT_USER_CANCELLED", ERR_LIB_SSL, SSL_R_TLSV1_ALERT_USER_CANCELLED}, - #else - {"TLSV1_ALERT_USER_CANCELLED", 20, 1090}, - #endif - #ifdef SSL_R_TLSV1_BAD_CERTIFICATE_HASH_VALUE - {"TLSV1_BAD_CERTIFICATE_HASH_VALUE", ERR_LIB_SSL, SSL_R_TLSV1_BAD_CERTIFICATE_HASH_VALUE}, - #else - {"TLSV1_BAD_CERTIFICATE_HASH_VALUE", 20, 1114}, - #endif - #ifdef SSL_R_TLSV1_BAD_CERTIFICATE_STATUS_RESPONSE - {"TLSV1_BAD_CERTIFICATE_STATUS_RESPONSE", ERR_LIB_SSL, SSL_R_TLSV1_BAD_CERTIFICATE_STATUS_RESPONSE}, - #else - {"TLSV1_BAD_CERTIFICATE_STATUS_RESPONSE", 20, 1113}, - #endif - #ifdef SSL_R_TLSV1_CERTIFICATE_UNOBTAINABLE - {"TLSV1_CERTIFICATE_UNOBTAINABLE", ERR_LIB_SSL, SSL_R_TLSV1_CERTIFICATE_UNOBTAINABLE}, - #else - {"TLSV1_CERTIFICATE_UNOBTAINABLE", 20, 1111}, - #endif - #ifdef SSL_R_TLSV1_UNRECOGNIZED_NAME - {"TLSV1_UNRECOGNIZED_NAME", ERR_LIB_SSL, SSL_R_TLSV1_UNRECOGNIZED_NAME}, - #else - {"TLSV1_UNRECOGNIZED_NAME", 20, 1112}, - #endif - #ifdef SSL_R_TLSV1_UNSUPPORTED_EXTENSION - {"TLSV1_UNSUPPORTED_EXTENSION", ERR_LIB_SSL, SSL_R_TLSV1_UNSUPPORTED_EXTENSION}, - #else - {"TLSV1_UNSUPPORTED_EXTENSION", 20, 1110}, - #endif - #ifdef SSL_R_TLS_HEARTBEAT_PEER_DOESNT_ACCEPT - {"TLS_HEARTBEAT_PEER_DOESNT_ACCEPT", ERR_LIB_SSL, SSL_R_TLS_HEARTBEAT_PEER_DOESNT_ACCEPT}, - #else - {"TLS_HEARTBEAT_PEER_DOESNT_ACCEPT", 20, 365}, - #endif - #ifdef SSL_R_TLS_HEARTBEAT_PENDING - {"TLS_HEARTBEAT_PENDING", ERR_LIB_SSL, SSL_R_TLS_HEARTBEAT_PENDING}, - #else - {"TLS_HEARTBEAT_PENDING", 20, 366}, - #endif - #ifdef SSL_R_TLS_ILLEGAL_EXPORTER_LABEL - {"TLS_ILLEGAL_EXPORTER_LABEL", ERR_LIB_SSL, SSL_R_TLS_ILLEGAL_EXPORTER_LABEL}, - #else - {"TLS_ILLEGAL_EXPORTER_LABEL", 20, 367}, - #endif - #ifdef SSL_R_TLS_INVALID_ECPOINTFORMAT_LIST - {"TLS_INVALID_ECPOINTFORMAT_LIST", ERR_LIB_SSL, SSL_R_TLS_INVALID_ECPOINTFORMAT_LIST}, - #else - {"TLS_INVALID_ECPOINTFORMAT_LIST", 20, 157}, - #endif - #ifdef SSL_R_TOO_MANY_KEY_UPDATES - {"TOO_MANY_KEY_UPDATES", ERR_LIB_SSL, SSL_R_TOO_MANY_KEY_UPDATES}, - #else - {"TOO_MANY_KEY_UPDATES", 20, 132}, - #endif - #ifdef SSL_R_TOO_MANY_WARN_ALERTS - {"TOO_MANY_WARN_ALERTS", ERR_LIB_SSL, SSL_R_TOO_MANY_WARN_ALERTS}, - #else - {"TOO_MANY_WARN_ALERTS", 20, 409}, - #endif - #ifdef SSL_R_TOO_MUCH_EARLY_DATA - {"TOO_MUCH_EARLY_DATA", ERR_LIB_SSL, SSL_R_TOO_MUCH_EARLY_DATA}, - #else - {"TOO_MUCH_EARLY_DATA", 20, 164}, - #endif - #ifdef SSL_R_UNABLE_TO_DECODE_ECDH_CERTS - {"UNABLE_TO_DECODE_ECDH_CERTS", ERR_LIB_SSL, SSL_R_UNABLE_TO_DECODE_ECDH_CERTS}, - #else - {"UNABLE_TO_DECODE_ECDH_CERTS", 20, 313}, - #endif - #ifdef SSL_R_UNABLE_TO_FIND_ECDH_PARAMETERS - {"UNABLE_TO_FIND_ECDH_PARAMETERS", ERR_LIB_SSL, SSL_R_UNABLE_TO_FIND_ECDH_PARAMETERS}, - #else - {"UNABLE_TO_FIND_ECDH_PARAMETERS", 20, 314}, - #endif - #ifdef SSL_R_UNABLE_TO_FIND_PUBLIC_KEY_PARAMETERS - {"UNABLE_TO_FIND_PUBLIC_KEY_PARAMETERS", ERR_LIB_SSL, SSL_R_UNABLE_TO_FIND_PUBLIC_KEY_PARAMETERS}, - #else - {"UNABLE_TO_FIND_PUBLIC_KEY_PARAMETERS", 20, 239}, - #endif - #ifdef SSL_R_UNABLE_TO_LOAD_SSL3_MD5_ROUTINES - {"UNABLE_TO_LOAD_SSL3_MD5_ROUTINES", ERR_LIB_SSL, SSL_R_UNABLE_TO_LOAD_SSL3_MD5_ROUTINES}, - #else - {"UNABLE_TO_LOAD_SSL3_MD5_ROUTINES", 20, 242}, - #endif - #ifdef SSL_R_UNABLE_TO_LOAD_SSL3_SHA1_ROUTINES - {"UNABLE_TO_LOAD_SSL3_SHA1_ROUTINES", ERR_LIB_SSL, SSL_R_UNABLE_TO_LOAD_SSL3_SHA1_ROUTINES}, - #else - {"UNABLE_TO_LOAD_SSL3_SHA1_ROUTINES", 20, 243}, - #endif - #ifdef SSL_R_UNEXPECTED_CCS_MESSAGE - {"UNEXPECTED_CCS_MESSAGE", ERR_LIB_SSL, SSL_R_UNEXPECTED_CCS_MESSAGE}, - #else - {"UNEXPECTED_CCS_MESSAGE", 20, 262}, - #endif - #ifdef SSL_R_UNEXPECTED_END_OF_EARLY_DATA - {"UNEXPECTED_END_OF_EARLY_DATA", ERR_LIB_SSL, SSL_R_UNEXPECTED_END_OF_EARLY_DATA}, - #else - {"UNEXPECTED_END_OF_EARLY_DATA", 20, 178}, - #endif - #ifdef SSL_R_UNEXPECTED_MESSAGE - {"UNEXPECTED_MESSAGE", ERR_LIB_SSL, SSL_R_UNEXPECTED_MESSAGE}, - #else - {"UNEXPECTED_MESSAGE", 20, 244}, - #endif - #ifdef SSL_R_UNEXPECTED_RECORD - {"UNEXPECTED_RECORD", ERR_LIB_SSL, SSL_R_UNEXPECTED_RECORD}, - #else - {"UNEXPECTED_RECORD", 20, 245}, - #endif - #ifdef SSL_R_UNINITIALIZED - {"UNINITIALIZED", ERR_LIB_SSL, SSL_R_UNINITIALIZED}, - #else - {"UNINITIALIZED", 20, 276}, - #endif - #ifdef SSL_R_UNKNOWN_ALERT_TYPE - {"UNKNOWN_ALERT_TYPE", ERR_LIB_SSL, SSL_R_UNKNOWN_ALERT_TYPE}, - #else - {"UNKNOWN_ALERT_TYPE", 20, 246}, - #endif - #ifdef SSL_R_UNKNOWN_CERTIFICATE_TYPE - {"UNKNOWN_CERTIFICATE_TYPE", ERR_LIB_SSL, SSL_R_UNKNOWN_CERTIFICATE_TYPE}, - #else - {"UNKNOWN_CERTIFICATE_TYPE", 20, 247}, - #endif - #ifdef SSL_R_UNKNOWN_CIPHER_RETURNED - {"UNKNOWN_CIPHER_RETURNED", ERR_LIB_SSL, SSL_R_UNKNOWN_CIPHER_RETURNED}, - #else - {"UNKNOWN_CIPHER_RETURNED", 20, 248}, - #endif - #ifdef SSL_R_UNKNOWN_CIPHER_TYPE - {"UNKNOWN_CIPHER_TYPE", ERR_LIB_SSL, SSL_R_UNKNOWN_CIPHER_TYPE}, - #else - {"UNKNOWN_CIPHER_TYPE", 20, 249}, - #endif - #ifdef SSL_R_UNKNOWN_CMD_NAME - {"UNKNOWN_CMD_NAME", ERR_LIB_SSL, SSL_R_UNKNOWN_CMD_NAME}, - #else - {"UNKNOWN_CMD_NAME", 20, 386}, - #endif - #ifdef SSL_R_UNKNOWN_COMMAND - {"UNKNOWN_COMMAND", ERR_LIB_SSL, SSL_R_UNKNOWN_COMMAND}, - #else - {"UNKNOWN_COMMAND", 20, 139}, - #endif - #ifdef SSL_R_UNKNOWN_DIGEST - {"UNKNOWN_DIGEST", ERR_LIB_SSL, SSL_R_UNKNOWN_DIGEST}, - #else - {"UNKNOWN_DIGEST", 20, 368}, - #endif - #ifdef SSL_R_UNKNOWN_KEY_EXCHANGE_TYPE - {"UNKNOWN_KEY_EXCHANGE_TYPE", ERR_LIB_SSL, SSL_R_UNKNOWN_KEY_EXCHANGE_TYPE}, - #else - {"UNKNOWN_KEY_EXCHANGE_TYPE", 20, 250}, - #endif - #ifdef SSL_R_UNKNOWN_PKEY_TYPE - {"UNKNOWN_PKEY_TYPE", ERR_LIB_SSL, SSL_R_UNKNOWN_PKEY_TYPE}, - #else - {"UNKNOWN_PKEY_TYPE", 20, 251}, - #endif - #ifdef SSL_R_UNKNOWN_PROTOCOL - {"UNKNOWN_PROTOCOL", ERR_LIB_SSL, SSL_R_UNKNOWN_PROTOCOL}, - #else - {"UNKNOWN_PROTOCOL", 20, 252}, - #endif - #ifdef SSL_R_UNKNOWN_SSL_VERSION - {"UNKNOWN_SSL_VERSION", ERR_LIB_SSL, SSL_R_UNKNOWN_SSL_VERSION}, - #else - {"UNKNOWN_SSL_VERSION", 20, 254}, - #endif - #ifdef SSL_R_UNKNOWN_STATE - {"UNKNOWN_STATE", ERR_LIB_SSL, SSL_R_UNKNOWN_STATE}, - #else - {"UNKNOWN_STATE", 20, 255}, - #endif - #ifdef SSL_R_UNSAFE_LEGACY_RENEGOTIATION_DISABLED - {"UNSAFE_LEGACY_RENEGOTIATION_DISABLED", ERR_LIB_SSL, SSL_R_UNSAFE_LEGACY_RENEGOTIATION_DISABLED}, - #else - {"UNSAFE_LEGACY_RENEGOTIATION_DISABLED", 20, 338}, - #endif - #ifdef SSL_R_UNSOLICITED_EXTENSION - {"UNSOLICITED_EXTENSION", ERR_LIB_SSL, SSL_R_UNSOLICITED_EXTENSION}, - #else - {"UNSOLICITED_EXTENSION", 20, 217}, - #endif - #ifdef SSL_R_UNSUPPORTED_COMPRESSION_ALGORITHM - {"UNSUPPORTED_COMPRESSION_ALGORITHM", ERR_LIB_SSL, SSL_R_UNSUPPORTED_COMPRESSION_ALGORITHM}, - #else - {"UNSUPPORTED_COMPRESSION_ALGORITHM", 20, 257}, - #endif - #ifdef SSL_R_UNSUPPORTED_DIGEST_TYPE - {"UNSUPPORTED_DIGEST_TYPE", ERR_LIB_SSL, SSL_R_UNSUPPORTED_DIGEST_TYPE}, - #else - {"UNSUPPORTED_DIGEST_TYPE", 20, 326}, - #endif - #ifdef SSL_R_UNSUPPORTED_ELLIPTIC_CURVE - {"UNSUPPORTED_ELLIPTIC_CURVE", ERR_LIB_SSL, SSL_R_UNSUPPORTED_ELLIPTIC_CURVE}, - #else - {"UNSUPPORTED_ELLIPTIC_CURVE", 20, 315}, - #endif - #ifdef SSL_R_UNSUPPORTED_PROTOCOL - {"UNSUPPORTED_PROTOCOL", ERR_LIB_SSL, SSL_R_UNSUPPORTED_PROTOCOL}, - #else - {"UNSUPPORTED_PROTOCOL", 20, 258}, - #endif - #ifdef SSL_R_UNSUPPORTED_SSL_VERSION - {"UNSUPPORTED_SSL_VERSION", ERR_LIB_SSL, SSL_R_UNSUPPORTED_SSL_VERSION}, - #else - {"UNSUPPORTED_SSL_VERSION", 20, 259}, - #endif - #ifdef SSL_R_UNSUPPORTED_STATUS_TYPE - {"UNSUPPORTED_STATUS_TYPE", ERR_LIB_SSL, SSL_R_UNSUPPORTED_STATUS_TYPE}, - #else - {"UNSUPPORTED_STATUS_TYPE", 20, 329}, - #endif - #ifdef SSL_R_USE_SRTP_NOT_NEGOTIATED - {"USE_SRTP_NOT_NEGOTIATED", ERR_LIB_SSL, SSL_R_USE_SRTP_NOT_NEGOTIATED}, - #else - {"USE_SRTP_NOT_NEGOTIATED", 20, 369}, - #endif - #ifdef SSL_R_VERSION_TOO_HIGH - {"VERSION_TOO_HIGH", ERR_LIB_SSL, SSL_R_VERSION_TOO_HIGH}, - #else - {"VERSION_TOO_HIGH", 20, 166}, - #endif - #ifdef SSL_R_VERSION_TOO_LOW - {"VERSION_TOO_LOW", ERR_LIB_SSL, SSL_R_VERSION_TOO_LOW}, - #else - {"VERSION_TOO_LOW", 20, 396}, - #endif - #ifdef SSL_R_WRONG_CERTIFICATE_TYPE - {"WRONG_CERTIFICATE_TYPE", ERR_LIB_SSL, SSL_R_WRONG_CERTIFICATE_TYPE}, - #else - {"WRONG_CERTIFICATE_TYPE", 20, 383}, - #endif - #ifdef SSL_R_WRONG_CIPHER_RETURNED - {"WRONG_CIPHER_RETURNED", ERR_LIB_SSL, SSL_R_WRONG_CIPHER_RETURNED}, - #else - {"WRONG_CIPHER_RETURNED", 20, 261}, - #endif - #ifdef SSL_R_WRONG_CURVE - {"WRONG_CURVE", ERR_LIB_SSL, SSL_R_WRONG_CURVE}, - #else - {"WRONG_CURVE", 20, 378}, - #endif - #ifdef SSL_R_WRONG_SIGNATURE_LENGTH - {"WRONG_SIGNATURE_LENGTH", ERR_LIB_SSL, SSL_R_WRONG_SIGNATURE_LENGTH}, - #else - {"WRONG_SIGNATURE_LENGTH", 20, 264}, - #endif - #ifdef SSL_R_WRONG_SIGNATURE_SIZE - {"WRONG_SIGNATURE_SIZE", ERR_LIB_SSL, SSL_R_WRONG_SIGNATURE_SIZE}, - #else - {"WRONG_SIGNATURE_SIZE", 20, 265}, - #endif - #ifdef SSL_R_WRONG_SIGNATURE_TYPE - {"WRONG_SIGNATURE_TYPE", ERR_LIB_SSL, SSL_R_WRONG_SIGNATURE_TYPE}, - #else - {"WRONG_SIGNATURE_TYPE", 20, 370}, - #endif - #ifdef SSL_R_WRONG_SSL_VERSION - {"WRONG_SSL_VERSION", ERR_LIB_SSL, SSL_R_WRONG_SSL_VERSION}, - #else - {"WRONG_SSL_VERSION", 20, 266}, - #endif - #ifdef SSL_R_WRONG_VERSION_NUMBER - {"WRONG_VERSION_NUMBER", ERR_LIB_SSL, SSL_R_WRONG_VERSION_NUMBER}, - #else - {"WRONG_VERSION_NUMBER", 20, 267}, - #endif - #ifdef SSL_R_X509_LIB - {"X509_LIB", ERR_LIB_SSL, SSL_R_X509_LIB}, - #else - {"X509_LIB", 20, 268}, - #endif - #ifdef SSL_R_X509_VERIFICATION_SETUP_PROBLEMS - {"X509_VERIFICATION_SETUP_PROBLEMS", ERR_LIB_SSL, SSL_R_X509_VERIFICATION_SETUP_PROBLEMS}, - #else - {"X509_VERIFICATION_SETUP_PROBLEMS", 20, 269}, - #endif - #ifdef TS_R_BAD_PKCS7_TYPE - {"BAD_PKCS7_TYPE", ERR_LIB_TS, TS_R_BAD_PKCS7_TYPE}, - #else - {"BAD_PKCS7_TYPE", 47, 132}, - #endif - #ifdef TS_R_BAD_TYPE - {"BAD_TYPE", ERR_LIB_TS, TS_R_BAD_TYPE}, - #else - {"BAD_TYPE", 47, 133}, - #endif - #ifdef TS_R_CANNOT_LOAD_CERT - {"CANNOT_LOAD_CERT", ERR_LIB_TS, TS_R_CANNOT_LOAD_CERT}, - #else - {"CANNOT_LOAD_CERT", 47, 137}, - #endif - #ifdef TS_R_CANNOT_LOAD_KEY - {"CANNOT_LOAD_KEY", ERR_LIB_TS, TS_R_CANNOT_LOAD_KEY}, - #else - {"CANNOT_LOAD_KEY", 47, 138}, - #endif - #ifdef TS_R_CERTIFICATE_VERIFY_ERROR - {"CERTIFICATE_VERIFY_ERROR", ERR_LIB_TS, TS_R_CERTIFICATE_VERIFY_ERROR}, - #else - {"CERTIFICATE_VERIFY_ERROR", 47, 100}, - #endif - #ifdef TS_R_COULD_NOT_SET_ENGINE - {"COULD_NOT_SET_ENGINE", ERR_LIB_TS, TS_R_COULD_NOT_SET_ENGINE}, - #else - {"COULD_NOT_SET_ENGINE", 47, 127}, - #endif - #ifdef TS_R_COULD_NOT_SET_TIME - {"COULD_NOT_SET_TIME", ERR_LIB_TS, TS_R_COULD_NOT_SET_TIME}, - #else - {"COULD_NOT_SET_TIME", 47, 115}, - #endif - #ifdef TS_R_DETACHED_CONTENT - {"DETACHED_CONTENT", ERR_LIB_TS, TS_R_DETACHED_CONTENT}, - #else - {"DETACHED_CONTENT", 47, 134}, - #endif - #ifdef TS_R_ESS_ADD_SIGNING_CERT_ERROR - {"ESS_ADD_SIGNING_CERT_ERROR", ERR_LIB_TS, TS_R_ESS_ADD_SIGNING_CERT_ERROR}, - #else - {"ESS_ADD_SIGNING_CERT_ERROR", 47, 116}, - #endif - #ifdef TS_R_ESS_ADD_SIGNING_CERT_V2_ERROR - {"ESS_ADD_SIGNING_CERT_V2_ERROR", ERR_LIB_TS, TS_R_ESS_ADD_SIGNING_CERT_V2_ERROR}, - #else - {"ESS_ADD_SIGNING_CERT_V2_ERROR", 47, 139}, - #endif - #ifdef TS_R_ESS_SIGNING_CERTIFICATE_ERROR - {"ESS_SIGNING_CERTIFICATE_ERROR", ERR_LIB_TS, TS_R_ESS_SIGNING_CERTIFICATE_ERROR}, - #else - {"ESS_SIGNING_CERTIFICATE_ERROR", 47, 101}, - #endif - #ifdef TS_R_INVALID_NULL_POINTER - {"INVALID_NULL_POINTER", ERR_LIB_TS, TS_R_INVALID_NULL_POINTER}, - #else - {"INVALID_NULL_POINTER", 47, 102}, - #endif - #ifdef TS_R_INVALID_SIGNER_CERTIFICATE_PURPOSE - {"INVALID_SIGNER_CERTIFICATE_PURPOSE", ERR_LIB_TS, TS_R_INVALID_SIGNER_CERTIFICATE_PURPOSE}, - #else - {"INVALID_SIGNER_CERTIFICATE_PURPOSE", 47, 117}, - #endif - #ifdef TS_R_MESSAGE_IMPRINT_MISMATCH - {"MESSAGE_IMPRINT_MISMATCH", ERR_LIB_TS, TS_R_MESSAGE_IMPRINT_MISMATCH}, - #else - {"MESSAGE_IMPRINT_MISMATCH", 47, 103}, - #endif - #ifdef TS_R_NONCE_MISMATCH - {"NONCE_MISMATCH", ERR_LIB_TS, TS_R_NONCE_MISMATCH}, - #else - {"NONCE_MISMATCH", 47, 104}, - #endif - #ifdef TS_R_NONCE_NOT_RETURNED - {"NONCE_NOT_RETURNED", ERR_LIB_TS, TS_R_NONCE_NOT_RETURNED}, - #else - {"NONCE_NOT_RETURNED", 47, 105}, - #endif - #ifdef TS_R_NO_CONTENT - {"NO_CONTENT", ERR_LIB_TS, TS_R_NO_CONTENT}, - #else - {"NO_CONTENT", 47, 106}, - #endif - #ifdef TS_R_NO_TIME_STAMP_TOKEN - {"NO_TIME_STAMP_TOKEN", ERR_LIB_TS, TS_R_NO_TIME_STAMP_TOKEN}, - #else - {"NO_TIME_STAMP_TOKEN", 47, 107}, - #endif - #ifdef TS_R_PKCS7_ADD_SIGNATURE_ERROR - {"PKCS7_ADD_SIGNATURE_ERROR", ERR_LIB_TS, TS_R_PKCS7_ADD_SIGNATURE_ERROR}, - #else - {"PKCS7_ADD_SIGNATURE_ERROR", 47, 118}, - #endif - #ifdef TS_R_PKCS7_ADD_SIGNED_ATTR_ERROR - {"PKCS7_ADD_SIGNED_ATTR_ERROR", ERR_LIB_TS, TS_R_PKCS7_ADD_SIGNED_ATTR_ERROR}, - #else - {"PKCS7_ADD_SIGNED_ATTR_ERROR", 47, 119}, - #endif - #ifdef TS_R_PKCS7_TO_TS_TST_INFO_FAILED - {"PKCS7_TO_TS_TST_INFO_FAILED", ERR_LIB_TS, TS_R_PKCS7_TO_TS_TST_INFO_FAILED}, - #else - {"PKCS7_TO_TS_TST_INFO_FAILED", 47, 129}, - #endif - #ifdef TS_R_POLICY_MISMATCH - {"POLICY_MISMATCH", ERR_LIB_TS, TS_R_POLICY_MISMATCH}, - #else - {"POLICY_MISMATCH", 47, 108}, - #endif - #ifdef TS_R_PRIVATE_KEY_DOES_NOT_MATCH_CERTIFICATE - {"PRIVATE_KEY_DOES_NOT_MATCH_CERTIFICATE", ERR_LIB_TS, TS_R_PRIVATE_KEY_DOES_NOT_MATCH_CERTIFICATE}, - #else - {"PRIVATE_KEY_DOES_NOT_MATCH_CERTIFICATE", 47, 120}, - #endif - #ifdef TS_R_RESPONSE_SETUP_ERROR - {"RESPONSE_SETUP_ERROR", ERR_LIB_TS, TS_R_RESPONSE_SETUP_ERROR}, - #else - {"RESPONSE_SETUP_ERROR", 47, 121}, - #endif - #ifdef TS_R_SIGNATURE_FAILURE - {"SIGNATURE_FAILURE", ERR_LIB_TS, TS_R_SIGNATURE_FAILURE}, - #else - {"SIGNATURE_FAILURE", 47, 109}, - #endif - #ifdef TS_R_THERE_MUST_BE_ONE_SIGNER - {"THERE_MUST_BE_ONE_SIGNER", ERR_LIB_TS, TS_R_THERE_MUST_BE_ONE_SIGNER}, - #else - {"THERE_MUST_BE_ONE_SIGNER", 47, 110}, - #endif - #ifdef TS_R_TIME_SYSCALL_ERROR - {"TIME_SYSCALL_ERROR", ERR_LIB_TS, TS_R_TIME_SYSCALL_ERROR}, - #else - {"TIME_SYSCALL_ERROR", 47, 122}, - #endif - #ifdef TS_R_TOKEN_NOT_PRESENT - {"TOKEN_NOT_PRESENT", ERR_LIB_TS, TS_R_TOKEN_NOT_PRESENT}, - #else - {"TOKEN_NOT_PRESENT", 47, 130}, - #endif - #ifdef TS_R_TOKEN_PRESENT - {"TOKEN_PRESENT", ERR_LIB_TS, TS_R_TOKEN_PRESENT}, - #else - {"TOKEN_PRESENT", 47, 131}, - #endif - #ifdef TS_R_TSA_NAME_MISMATCH - {"TSA_NAME_MISMATCH", ERR_LIB_TS, TS_R_TSA_NAME_MISMATCH}, - #else - {"TSA_NAME_MISMATCH", 47, 111}, - #endif - #ifdef TS_R_TSA_UNTRUSTED - {"TSA_UNTRUSTED", ERR_LIB_TS, TS_R_TSA_UNTRUSTED}, - #else - {"TSA_UNTRUSTED", 47, 112}, - #endif - #ifdef TS_R_TST_INFO_SETUP_ERROR - {"TST_INFO_SETUP_ERROR", ERR_LIB_TS, TS_R_TST_INFO_SETUP_ERROR}, - #else - {"TST_INFO_SETUP_ERROR", 47, 123}, - #endif - #ifdef TS_R_TS_DATASIGN - {"TS_DATASIGN", ERR_LIB_TS, TS_R_TS_DATASIGN}, - #else - {"TS_DATASIGN", 47, 124}, - #endif - #ifdef TS_R_UNACCEPTABLE_POLICY - {"UNACCEPTABLE_POLICY", ERR_LIB_TS, TS_R_UNACCEPTABLE_POLICY}, - #else - {"UNACCEPTABLE_POLICY", 47, 125}, - #endif - #ifdef TS_R_UNSUPPORTED_MD_ALGORITHM - {"UNSUPPORTED_MD_ALGORITHM", ERR_LIB_TS, TS_R_UNSUPPORTED_MD_ALGORITHM}, - #else - {"UNSUPPORTED_MD_ALGORITHM", 47, 126}, - #endif - #ifdef TS_R_UNSUPPORTED_VERSION - {"UNSUPPORTED_VERSION", ERR_LIB_TS, TS_R_UNSUPPORTED_VERSION}, - #else - {"UNSUPPORTED_VERSION", 47, 113}, - #endif - #ifdef TS_R_VAR_BAD_VALUE - {"VAR_BAD_VALUE", ERR_LIB_TS, TS_R_VAR_BAD_VALUE}, - #else - {"VAR_BAD_VALUE", 47, 135}, - #endif - #ifdef TS_R_VAR_LOOKUP_FAILURE - {"VAR_LOOKUP_FAILURE", ERR_LIB_TS, TS_R_VAR_LOOKUP_FAILURE}, - #else - {"VAR_LOOKUP_FAILURE", 47, 136}, - #endif - #ifdef TS_R_WRONG_CONTENT_TYPE - {"WRONG_CONTENT_TYPE", ERR_LIB_TS, TS_R_WRONG_CONTENT_TYPE}, - #else - {"WRONG_CONTENT_TYPE", 47, 114}, - #endif - #ifdef UI_R_COMMON_OK_AND_CANCEL_CHARACTERS - {"COMMON_OK_AND_CANCEL_CHARACTERS", ERR_LIB_UI, UI_R_COMMON_OK_AND_CANCEL_CHARACTERS}, - #else - {"COMMON_OK_AND_CANCEL_CHARACTERS", 40, 104}, - #endif - #ifdef UI_R_INDEX_TOO_LARGE - {"INDEX_TOO_LARGE", ERR_LIB_UI, UI_R_INDEX_TOO_LARGE}, - #else - {"INDEX_TOO_LARGE", 40, 102}, - #endif - #ifdef UI_R_INDEX_TOO_SMALL - {"INDEX_TOO_SMALL", ERR_LIB_UI, UI_R_INDEX_TOO_SMALL}, - #else - {"INDEX_TOO_SMALL", 40, 103}, - #endif - #ifdef UI_R_NO_RESULT_BUFFER - {"NO_RESULT_BUFFER", ERR_LIB_UI, UI_R_NO_RESULT_BUFFER}, - #else - {"NO_RESULT_BUFFER", 40, 105}, - #endif - #ifdef UI_R_PROCESSING_ERROR - {"PROCESSING_ERROR", ERR_LIB_UI, UI_R_PROCESSING_ERROR}, - #else - {"PROCESSING_ERROR", 40, 107}, - #endif - #ifdef UI_R_RESULT_TOO_LARGE - {"RESULT_TOO_LARGE", ERR_LIB_UI, UI_R_RESULT_TOO_LARGE}, - #else - {"RESULT_TOO_LARGE", 40, 100}, - #endif - #ifdef UI_R_RESULT_TOO_SMALL - {"RESULT_TOO_SMALL", ERR_LIB_UI, UI_R_RESULT_TOO_SMALL}, - #else - {"RESULT_TOO_SMALL", 40, 101}, - #endif - #ifdef UI_R_SYSASSIGN_ERROR - {"SYSASSIGN_ERROR", ERR_LIB_UI, UI_R_SYSASSIGN_ERROR}, - #else - {"SYSASSIGN_ERROR", 40, 109}, - #endif - #ifdef UI_R_SYSDASSGN_ERROR - {"SYSDASSGN_ERROR", ERR_LIB_UI, UI_R_SYSDASSGN_ERROR}, - #else - {"SYSDASSGN_ERROR", 40, 110}, - #endif - #ifdef UI_R_SYSQIOW_ERROR - {"SYSQIOW_ERROR", ERR_LIB_UI, UI_R_SYSQIOW_ERROR}, - #else - {"SYSQIOW_ERROR", 40, 111}, - #endif - #ifdef UI_R_UNKNOWN_CONTROL_COMMAND - {"UNKNOWN_CONTROL_COMMAND", ERR_LIB_UI, UI_R_UNKNOWN_CONTROL_COMMAND}, - #else - {"UNKNOWN_CONTROL_COMMAND", 40, 106}, - #endif - #ifdef UI_R_UNKNOWN_TTYGET_ERRNO_VALUE - {"UNKNOWN_TTYGET_ERRNO_VALUE", ERR_LIB_UI, UI_R_UNKNOWN_TTYGET_ERRNO_VALUE}, - #else - {"UNKNOWN_TTYGET_ERRNO_VALUE", 40, 108}, - #endif - #ifdef UI_R_USER_DATA_DUPLICATION_UNSUPPORTED - {"USER_DATA_DUPLICATION_UNSUPPORTED", ERR_LIB_UI, UI_R_USER_DATA_DUPLICATION_UNSUPPORTED}, - #else - {"USER_DATA_DUPLICATION_UNSUPPORTED", 40, 112}, - #endif - #ifdef X509V3_R_BAD_IP_ADDRESS - {"BAD_IP_ADDRESS", ERR_LIB_X509V3, X509V3_R_BAD_IP_ADDRESS}, - #else - {"BAD_IP_ADDRESS", 34, 118}, - #endif - #ifdef X509V3_R_BAD_OBJECT - {"BAD_OBJECT", ERR_LIB_X509V3, X509V3_R_BAD_OBJECT}, - #else - {"BAD_OBJECT", 34, 119}, - #endif - #ifdef X509V3_R_BN_DEC2BN_ERROR - {"BN_DEC2BN_ERROR", ERR_LIB_X509V3, X509V3_R_BN_DEC2BN_ERROR}, - #else - {"BN_DEC2BN_ERROR", 34, 100}, - #endif - #ifdef X509V3_R_BN_TO_ASN1_INTEGER_ERROR - {"BN_TO_ASN1_INTEGER_ERROR", ERR_LIB_X509V3, X509V3_R_BN_TO_ASN1_INTEGER_ERROR}, - #else - {"BN_TO_ASN1_INTEGER_ERROR", 34, 101}, - #endif - #ifdef X509V3_R_DIRNAME_ERROR - {"DIRNAME_ERROR", ERR_LIB_X509V3, X509V3_R_DIRNAME_ERROR}, - #else - {"DIRNAME_ERROR", 34, 149}, - #endif - #ifdef X509V3_R_DISTPOINT_ALREADY_SET - {"DISTPOINT_ALREADY_SET", ERR_LIB_X509V3, X509V3_R_DISTPOINT_ALREADY_SET}, - #else - {"DISTPOINT_ALREADY_SET", 34, 160}, - #endif - #ifdef X509V3_R_DUPLICATE_ZONE_ID - {"DUPLICATE_ZONE_ID", ERR_LIB_X509V3, X509V3_R_DUPLICATE_ZONE_ID}, - #else - {"DUPLICATE_ZONE_ID", 34, 133}, - #endif - #ifdef X509V3_R_ERROR_CONVERTING_ZONE - {"ERROR_CONVERTING_ZONE", ERR_LIB_X509V3, X509V3_R_ERROR_CONVERTING_ZONE}, - #else - {"ERROR_CONVERTING_ZONE", 34, 131}, - #endif - #ifdef X509V3_R_ERROR_CREATING_EXTENSION - {"ERROR_CREATING_EXTENSION", ERR_LIB_X509V3, X509V3_R_ERROR_CREATING_EXTENSION}, - #else - {"ERROR_CREATING_EXTENSION", 34, 144}, - #endif - #ifdef X509V3_R_ERROR_IN_EXTENSION - {"ERROR_IN_EXTENSION", ERR_LIB_X509V3, X509V3_R_ERROR_IN_EXTENSION}, - #else - {"ERROR_IN_EXTENSION", 34, 128}, - #endif - #ifdef X509V3_R_EXPECTED_A_SECTION_NAME - {"EXPECTED_A_SECTION_NAME", ERR_LIB_X509V3, X509V3_R_EXPECTED_A_SECTION_NAME}, - #else - {"EXPECTED_A_SECTION_NAME", 34, 137}, - #endif - #ifdef X509V3_R_EXTENSION_EXISTS - {"EXTENSION_EXISTS", ERR_LIB_X509V3, X509V3_R_EXTENSION_EXISTS}, - #else - {"EXTENSION_EXISTS", 34, 145}, - #endif - #ifdef X509V3_R_EXTENSION_NAME_ERROR - {"EXTENSION_NAME_ERROR", ERR_LIB_X509V3, X509V3_R_EXTENSION_NAME_ERROR}, - #else - {"EXTENSION_NAME_ERROR", 34, 115}, - #endif - #ifdef X509V3_R_EXTENSION_NOT_FOUND - {"EXTENSION_NOT_FOUND", ERR_LIB_X509V3, X509V3_R_EXTENSION_NOT_FOUND}, - #else - {"EXTENSION_NOT_FOUND", 34, 102}, - #endif - #ifdef X509V3_R_EXTENSION_SETTING_NOT_SUPPORTED - {"EXTENSION_SETTING_NOT_SUPPORTED", ERR_LIB_X509V3, X509V3_R_EXTENSION_SETTING_NOT_SUPPORTED}, - #else - {"EXTENSION_SETTING_NOT_SUPPORTED", 34, 103}, - #endif - #ifdef X509V3_R_EXTENSION_VALUE_ERROR - {"EXTENSION_VALUE_ERROR", ERR_LIB_X509V3, X509V3_R_EXTENSION_VALUE_ERROR}, - #else - {"EXTENSION_VALUE_ERROR", 34, 116}, - #endif - #ifdef X509V3_R_ILLEGAL_EMPTY_EXTENSION - {"ILLEGAL_EMPTY_EXTENSION", ERR_LIB_X509V3, X509V3_R_ILLEGAL_EMPTY_EXTENSION}, - #else - {"ILLEGAL_EMPTY_EXTENSION", 34, 151}, - #endif - #ifdef X509V3_R_INCORRECT_POLICY_SYNTAX_TAG - {"INCORRECT_POLICY_SYNTAX_TAG", ERR_LIB_X509V3, X509V3_R_INCORRECT_POLICY_SYNTAX_TAG}, - #else - {"INCORRECT_POLICY_SYNTAX_TAG", 34, 152}, - #endif - #ifdef X509V3_R_INVALID_ASNUMBER - {"INVALID_ASNUMBER", ERR_LIB_X509V3, X509V3_R_INVALID_ASNUMBER}, - #else - {"INVALID_ASNUMBER", 34, 162}, - #endif - #ifdef X509V3_R_INVALID_ASRANGE - {"INVALID_ASRANGE", ERR_LIB_X509V3, X509V3_R_INVALID_ASRANGE}, - #else - {"INVALID_ASRANGE", 34, 163}, - #endif - #ifdef X509V3_R_INVALID_BOOLEAN_STRING - {"INVALID_BOOLEAN_STRING", ERR_LIB_X509V3, X509V3_R_INVALID_BOOLEAN_STRING}, - #else - {"INVALID_BOOLEAN_STRING", 34, 104}, - #endif - #ifdef X509V3_R_INVALID_EXTENSION_STRING - {"INVALID_EXTENSION_STRING", ERR_LIB_X509V3, X509V3_R_INVALID_EXTENSION_STRING}, - #else - {"INVALID_EXTENSION_STRING", 34, 105}, - #endif - #ifdef X509V3_R_INVALID_INHERITANCE - {"INVALID_INHERITANCE", ERR_LIB_X509V3, X509V3_R_INVALID_INHERITANCE}, - #else - {"INVALID_INHERITANCE", 34, 165}, - #endif - #ifdef X509V3_R_INVALID_IPADDRESS - {"INVALID_IPADDRESS", ERR_LIB_X509V3, X509V3_R_INVALID_IPADDRESS}, - #else - {"INVALID_IPADDRESS", 34, 166}, - #endif - #ifdef X509V3_R_INVALID_MULTIPLE_RDNS - {"INVALID_MULTIPLE_RDNS", ERR_LIB_X509V3, X509V3_R_INVALID_MULTIPLE_RDNS}, - #else - {"INVALID_MULTIPLE_RDNS", 34, 161}, - #endif - #ifdef X509V3_R_INVALID_NAME - {"INVALID_NAME", ERR_LIB_X509V3, X509V3_R_INVALID_NAME}, - #else - {"INVALID_NAME", 34, 106}, - #endif - #ifdef X509V3_R_INVALID_NULL_ARGUMENT - {"INVALID_NULL_ARGUMENT", ERR_LIB_X509V3, X509V3_R_INVALID_NULL_ARGUMENT}, - #else - {"INVALID_NULL_ARGUMENT", 34, 107}, - #endif - #ifdef X509V3_R_INVALID_NULL_NAME - {"INVALID_NULL_NAME", ERR_LIB_X509V3, X509V3_R_INVALID_NULL_NAME}, - #else - {"INVALID_NULL_NAME", 34, 108}, - #endif - #ifdef X509V3_R_INVALID_NULL_VALUE - {"INVALID_NULL_VALUE", ERR_LIB_X509V3, X509V3_R_INVALID_NULL_VALUE}, - #else - {"INVALID_NULL_VALUE", 34, 109}, - #endif - #ifdef X509V3_R_INVALID_NUMBER - {"INVALID_NUMBER", ERR_LIB_X509V3, X509V3_R_INVALID_NUMBER}, - #else - {"INVALID_NUMBER", 34, 140}, - #endif - #ifdef X509V3_R_INVALID_NUMBERS - {"INVALID_NUMBERS", ERR_LIB_X509V3, X509V3_R_INVALID_NUMBERS}, - #else - {"INVALID_NUMBERS", 34, 141}, - #endif - #ifdef X509V3_R_INVALID_OBJECT_IDENTIFIER - {"INVALID_OBJECT_IDENTIFIER", ERR_LIB_X509V3, X509V3_R_INVALID_OBJECT_IDENTIFIER}, - #else - {"INVALID_OBJECT_IDENTIFIER", 34, 110}, - #endif - #ifdef X509V3_R_INVALID_OPTION - {"INVALID_OPTION", ERR_LIB_X509V3, X509V3_R_INVALID_OPTION}, - #else - {"INVALID_OPTION", 34, 138}, - #endif - #ifdef X509V3_R_INVALID_POLICY_IDENTIFIER - {"INVALID_POLICY_IDENTIFIER", ERR_LIB_X509V3, X509V3_R_INVALID_POLICY_IDENTIFIER}, - #else - {"INVALID_POLICY_IDENTIFIER", 34, 134}, - #endif - #ifdef X509V3_R_INVALID_PROXY_POLICY_SETTING - {"INVALID_PROXY_POLICY_SETTING", ERR_LIB_X509V3, X509V3_R_INVALID_PROXY_POLICY_SETTING}, - #else - {"INVALID_PROXY_POLICY_SETTING", 34, 153}, - #endif - #ifdef X509V3_R_INVALID_PURPOSE - {"INVALID_PURPOSE", ERR_LIB_X509V3, X509V3_R_INVALID_PURPOSE}, - #else - {"INVALID_PURPOSE", 34, 146}, - #endif - #ifdef X509V3_R_INVALID_SAFI - {"INVALID_SAFI", ERR_LIB_X509V3, X509V3_R_INVALID_SAFI}, - #else - {"INVALID_SAFI", 34, 164}, - #endif - #ifdef X509V3_R_INVALID_SECTION - {"INVALID_SECTION", ERR_LIB_X509V3, X509V3_R_INVALID_SECTION}, - #else - {"INVALID_SECTION", 34, 135}, - #endif - #ifdef X509V3_R_INVALID_SYNTAX - {"INVALID_SYNTAX", ERR_LIB_X509V3, X509V3_R_INVALID_SYNTAX}, - #else - {"INVALID_SYNTAX", 34, 143}, - #endif - #ifdef X509V3_R_ISSUER_DECODE_ERROR - {"ISSUER_DECODE_ERROR", ERR_LIB_X509V3, X509V3_R_ISSUER_DECODE_ERROR}, - #else - {"ISSUER_DECODE_ERROR", 34, 126}, - #endif - #ifdef X509V3_R_MISSING_VALUE - {"MISSING_VALUE", ERR_LIB_X509V3, X509V3_R_MISSING_VALUE}, - #else - {"MISSING_VALUE", 34, 124}, - #endif - #ifdef X509V3_R_NEED_ORGANIZATION_AND_NUMBERS - {"NEED_ORGANIZATION_AND_NUMBERS", ERR_LIB_X509V3, X509V3_R_NEED_ORGANIZATION_AND_NUMBERS}, - #else - {"NEED_ORGANIZATION_AND_NUMBERS", 34, 142}, - #endif - #ifdef X509V3_R_NO_CONFIG_DATABASE - {"NO_CONFIG_DATABASE", ERR_LIB_X509V3, X509V3_R_NO_CONFIG_DATABASE}, - #else - {"NO_CONFIG_DATABASE", 34, 136}, - #endif - #ifdef X509V3_R_NO_ISSUER_CERTIFICATE - {"NO_ISSUER_CERTIFICATE", ERR_LIB_X509V3, X509V3_R_NO_ISSUER_CERTIFICATE}, - #else - {"NO_ISSUER_CERTIFICATE", 34, 121}, - #endif - #ifdef X509V3_R_NO_ISSUER_DETAILS - {"NO_ISSUER_DETAILS", ERR_LIB_X509V3, X509V3_R_NO_ISSUER_DETAILS}, - #else - {"NO_ISSUER_DETAILS", 34, 127}, - #endif - #ifdef X509V3_R_NO_POLICY_IDENTIFIER - {"NO_POLICY_IDENTIFIER", ERR_LIB_X509V3, X509V3_R_NO_POLICY_IDENTIFIER}, - #else - {"NO_POLICY_IDENTIFIER", 34, 139}, - #endif - #ifdef X509V3_R_NO_PROXY_CERT_POLICY_LANGUAGE_DEFINED - {"NO_PROXY_CERT_POLICY_LANGUAGE_DEFINED", ERR_LIB_X509V3, X509V3_R_NO_PROXY_CERT_POLICY_LANGUAGE_DEFINED}, - #else - {"NO_PROXY_CERT_POLICY_LANGUAGE_DEFINED", 34, 154}, - #endif - #ifdef X509V3_R_NO_PUBLIC_KEY - {"NO_PUBLIC_KEY", ERR_LIB_X509V3, X509V3_R_NO_PUBLIC_KEY}, - #else - {"NO_PUBLIC_KEY", 34, 114}, - #endif - #ifdef X509V3_R_NO_SUBJECT_DETAILS - {"NO_SUBJECT_DETAILS", ERR_LIB_X509V3, X509V3_R_NO_SUBJECT_DETAILS}, - #else - {"NO_SUBJECT_DETAILS", 34, 125}, - #endif - #ifdef X509V3_R_OPERATION_NOT_DEFINED - {"OPERATION_NOT_DEFINED", ERR_LIB_X509V3, X509V3_R_OPERATION_NOT_DEFINED}, - #else - {"OPERATION_NOT_DEFINED", 34, 148}, - #endif - #ifdef X509V3_R_OTHERNAME_ERROR - {"OTHERNAME_ERROR", ERR_LIB_X509V3, X509V3_R_OTHERNAME_ERROR}, - #else - {"OTHERNAME_ERROR", 34, 147}, - #endif - #ifdef X509V3_R_POLICY_LANGUAGE_ALREADY_DEFINED - {"POLICY_LANGUAGE_ALREADY_DEFINED", ERR_LIB_X509V3, X509V3_R_POLICY_LANGUAGE_ALREADY_DEFINED}, - #else - {"POLICY_LANGUAGE_ALREADY_DEFINED", 34, 155}, - #endif - #ifdef X509V3_R_POLICY_PATH_LENGTH - {"POLICY_PATH_LENGTH", ERR_LIB_X509V3, X509V3_R_POLICY_PATH_LENGTH}, - #else - {"POLICY_PATH_LENGTH", 34, 156}, - #endif - #ifdef X509V3_R_POLICY_PATH_LENGTH_ALREADY_DEFINED - {"POLICY_PATH_LENGTH_ALREADY_DEFINED", ERR_LIB_X509V3, X509V3_R_POLICY_PATH_LENGTH_ALREADY_DEFINED}, - #else - {"POLICY_PATH_LENGTH_ALREADY_DEFINED", 34, 157}, - #endif - #ifdef X509V3_R_POLICY_WHEN_PROXY_LANGUAGE_REQUIRES_NO_POLICY - {"POLICY_WHEN_PROXY_LANGUAGE_REQUIRES_NO_POLICY", ERR_LIB_X509V3, X509V3_R_POLICY_WHEN_PROXY_LANGUAGE_REQUIRES_NO_POLICY}, - #else - {"POLICY_WHEN_PROXY_LANGUAGE_REQUIRES_NO_POLICY", 34, 159}, - #endif - #ifdef X509V3_R_SECTION_NOT_FOUND - {"SECTION_NOT_FOUND", ERR_LIB_X509V3, X509V3_R_SECTION_NOT_FOUND}, - #else - {"SECTION_NOT_FOUND", 34, 150}, - #endif - #ifdef X509V3_R_UNABLE_TO_GET_ISSUER_DETAILS - {"UNABLE_TO_GET_ISSUER_DETAILS", ERR_LIB_X509V3, X509V3_R_UNABLE_TO_GET_ISSUER_DETAILS}, - #else - {"UNABLE_TO_GET_ISSUER_DETAILS", 34, 122}, - #endif - #ifdef X509V3_R_UNABLE_TO_GET_ISSUER_KEYID - {"UNABLE_TO_GET_ISSUER_KEYID", ERR_LIB_X509V3, X509V3_R_UNABLE_TO_GET_ISSUER_KEYID}, - #else - {"UNABLE_TO_GET_ISSUER_KEYID", 34, 123}, - #endif - #ifdef X509V3_R_UNKNOWN_BIT_STRING_ARGUMENT - {"UNKNOWN_BIT_STRING_ARGUMENT", ERR_LIB_X509V3, X509V3_R_UNKNOWN_BIT_STRING_ARGUMENT}, - #else - {"UNKNOWN_BIT_STRING_ARGUMENT", 34, 111}, - #endif - #ifdef X509V3_R_UNKNOWN_EXTENSION - {"UNKNOWN_EXTENSION", ERR_LIB_X509V3, X509V3_R_UNKNOWN_EXTENSION}, - #else - {"UNKNOWN_EXTENSION", 34, 129}, - #endif - #ifdef X509V3_R_UNKNOWN_EXTENSION_NAME - {"UNKNOWN_EXTENSION_NAME", ERR_LIB_X509V3, X509V3_R_UNKNOWN_EXTENSION_NAME}, - #else - {"UNKNOWN_EXTENSION_NAME", 34, 130}, - #endif - #ifdef X509V3_R_UNKNOWN_OPTION - {"UNKNOWN_OPTION", ERR_LIB_X509V3, X509V3_R_UNKNOWN_OPTION}, - #else - {"UNKNOWN_OPTION", 34, 120}, - #endif - #ifdef X509V3_R_UNSUPPORTED_OPTION - {"UNSUPPORTED_OPTION", ERR_LIB_X509V3, X509V3_R_UNSUPPORTED_OPTION}, - #else - {"UNSUPPORTED_OPTION", 34, 117}, - #endif - #ifdef X509V3_R_UNSUPPORTED_TYPE - {"UNSUPPORTED_TYPE", ERR_LIB_X509V3, X509V3_R_UNSUPPORTED_TYPE}, - #else - {"UNSUPPORTED_TYPE", 34, 167}, - #endif - #ifdef X509V3_R_USER_TOO_LONG - {"USER_TOO_LONG", ERR_LIB_X509V3, X509V3_R_USER_TOO_LONG}, - #else - {"USER_TOO_LONG", 34, 132}, - #endif - #ifdef X509_R_AKID_MISMATCH - {"AKID_MISMATCH", ERR_LIB_X509, X509_R_AKID_MISMATCH}, - #else - {"AKID_MISMATCH", 11, 110}, - #endif - #ifdef X509_R_BAD_SELECTOR - {"BAD_SELECTOR", ERR_LIB_X509, X509_R_BAD_SELECTOR}, - #else - {"BAD_SELECTOR", 11, 133}, - #endif - #ifdef X509_R_BAD_X509_FILETYPE - {"BAD_X509_FILETYPE", ERR_LIB_X509, X509_R_BAD_X509_FILETYPE}, - #else - {"BAD_X509_FILETYPE", 11, 100}, - #endif - #ifdef X509_R_BASE64_DECODE_ERROR - {"BASE64_DECODE_ERROR", ERR_LIB_X509, X509_R_BASE64_DECODE_ERROR}, - #else - {"BASE64_DECODE_ERROR", 11, 118}, - #endif - #ifdef X509_R_CANT_CHECK_DH_KEY - {"CANT_CHECK_DH_KEY", ERR_LIB_X509, X509_R_CANT_CHECK_DH_KEY}, - #else - {"CANT_CHECK_DH_KEY", 11, 114}, - #endif - #ifdef X509_R_CERT_ALREADY_IN_HASH_TABLE - {"CERT_ALREADY_IN_HASH_TABLE", ERR_LIB_X509, X509_R_CERT_ALREADY_IN_HASH_TABLE}, - #else - {"CERT_ALREADY_IN_HASH_TABLE", 11, 101}, - #endif - #ifdef X509_R_CRL_ALREADY_DELTA - {"CRL_ALREADY_DELTA", ERR_LIB_X509, X509_R_CRL_ALREADY_DELTA}, - #else - {"CRL_ALREADY_DELTA", 11, 127}, - #endif - #ifdef X509_R_CRL_VERIFY_FAILURE - {"CRL_VERIFY_FAILURE", ERR_LIB_X509, X509_R_CRL_VERIFY_FAILURE}, - #else - {"CRL_VERIFY_FAILURE", 11, 131}, - #endif - #ifdef X509_R_ERR_ASN1_LIB - {"ERR_ASN1_LIB", ERR_LIB_X509, X509_R_ERR_ASN1_LIB}, - #else - {"ERR_ASN1_LIB", 11, 102}, - #endif - #ifdef X509_R_IDP_MISMATCH - {"IDP_MISMATCH", ERR_LIB_X509, X509_R_IDP_MISMATCH}, - #else - {"IDP_MISMATCH", 11, 128}, - #endif - #ifdef X509_R_INVALID_ATTRIBUTES - {"INVALID_ATTRIBUTES", ERR_LIB_X509, X509_R_INVALID_ATTRIBUTES}, - #else - {"INVALID_ATTRIBUTES", 11, 138}, - #endif - #ifdef X509_R_INVALID_DIRECTORY - {"INVALID_DIRECTORY", ERR_LIB_X509, X509_R_INVALID_DIRECTORY}, - #else - {"INVALID_DIRECTORY", 11, 113}, - #endif - #ifdef X509_R_INVALID_FIELD_NAME - {"INVALID_FIELD_NAME", ERR_LIB_X509, X509_R_INVALID_FIELD_NAME}, - #else - {"INVALID_FIELD_NAME", 11, 119}, - #endif - #ifdef X509_R_INVALID_TRUST - {"INVALID_TRUST", ERR_LIB_X509, X509_R_INVALID_TRUST}, - #else - {"INVALID_TRUST", 11, 123}, - #endif - #ifdef X509_R_ISSUER_MISMATCH - {"ISSUER_MISMATCH", ERR_LIB_X509, X509_R_ISSUER_MISMATCH}, - #else - {"ISSUER_MISMATCH", 11, 129}, - #endif - #ifdef X509_R_KEY_TYPE_MISMATCH - {"KEY_TYPE_MISMATCH", ERR_LIB_X509, X509_R_KEY_TYPE_MISMATCH}, - #else - {"KEY_TYPE_MISMATCH", 11, 115}, - #endif - #ifdef X509_R_KEY_VALUES_MISMATCH - {"KEY_VALUES_MISMATCH", ERR_LIB_X509, X509_R_KEY_VALUES_MISMATCH}, - #else - {"KEY_VALUES_MISMATCH", 11, 116}, - #endif - #ifdef X509_R_LOADING_CERT_DIR - {"LOADING_CERT_DIR", ERR_LIB_X509, X509_R_LOADING_CERT_DIR}, - #else - {"LOADING_CERT_DIR", 11, 103}, - #endif - #ifdef X509_R_LOADING_DEFAULTS - {"LOADING_DEFAULTS", ERR_LIB_X509, X509_R_LOADING_DEFAULTS}, - #else - {"LOADING_DEFAULTS", 11, 104}, - #endif - #ifdef X509_R_METHOD_NOT_SUPPORTED - {"METHOD_NOT_SUPPORTED", ERR_LIB_X509, X509_R_METHOD_NOT_SUPPORTED}, - #else - {"METHOD_NOT_SUPPORTED", 11, 124}, - #endif - #ifdef X509_R_NAME_TOO_LONG - {"NAME_TOO_LONG", ERR_LIB_X509, X509_R_NAME_TOO_LONG}, - #else - {"NAME_TOO_LONG", 11, 134}, - #endif - #ifdef X509_R_NEWER_CRL_NOT_NEWER - {"NEWER_CRL_NOT_NEWER", ERR_LIB_X509, X509_R_NEWER_CRL_NOT_NEWER}, - #else - {"NEWER_CRL_NOT_NEWER", 11, 132}, - #endif - #ifdef X509_R_NO_CERTIFICATE_FOUND - {"NO_CERTIFICATE_FOUND", ERR_LIB_X509, X509_R_NO_CERTIFICATE_FOUND}, - #else - {"NO_CERTIFICATE_FOUND", 11, 135}, - #endif - #ifdef X509_R_NO_CERTIFICATE_OR_CRL_FOUND - {"NO_CERTIFICATE_OR_CRL_FOUND", ERR_LIB_X509, X509_R_NO_CERTIFICATE_OR_CRL_FOUND}, - #else - {"NO_CERTIFICATE_OR_CRL_FOUND", 11, 136}, - #endif - #ifdef X509_R_NO_CERT_SET_FOR_US_TO_VERIFY - {"NO_CERT_SET_FOR_US_TO_VERIFY", ERR_LIB_X509, X509_R_NO_CERT_SET_FOR_US_TO_VERIFY}, - #else - {"NO_CERT_SET_FOR_US_TO_VERIFY", 11, 105}, - #endif - #ifdef X509_R_NO_CRL_FOUND - {"NO_CRL_FOUND", ERR_LIB_X509, X509_R_NO_CRL_FOUND}, - #else - {"NO_CRL_FOUND", 11, 137}, - #endif - #ifdef X509_R_NO_CRL_NUMBER - {"NO_CRL_NUMBER", ERR_LIB_X509, X509_R_NO_CRL_NUMBER}, - #else - {"NO_CRL_NUMBER", 11, 130}, - #endif - #ifdef X509_R_PUBLIC_KEY_DECODE_ERROR - {"PUBLIC_KEY_DECODE_ERROR", ERR_LIB_X509, X509_R_PUBLIC_KEY_DECODE_ERROR}, - #else - {"PUBLIC_KEY_DECODE_ERROR", 11, 125}, - #endif - #ifdef X509_R_PUBLIC_KEY_ENCODE_ERROR - {"PUBLIC_KEY_ENCODE_ERROR", ERR_LIB_X509, X509_R_PUBLIC_KEY_ENCODE_ERROR}, - #else - {"PUBLIC_KEY_ENCODE_ERROR", 11, 126}, - #endif - #ifdef X509_R_SHOULD_RETRY - {"SHOULD_RETRY", ERR_LIB_X509, X509_R_SHOULD_RETRY}, - #else - {"SHOULD_RETRY", 11, 106}, - #endif - #ifdef X509_R_UNABLE_TO_FIND_PARAMETERS_IN_CHAIN - {"UNABLE_TO_FIND_PARAMETERS_IN_CHAIN", ERR_LIB_X509, X509_R_UNABLE_TO_FIND_PARAMETERS_IN_CHAIN}, - #else - {"UNABLE_TO_FIND_PARAMETERS_IN_CHAIN", 11, 107}, - #endif - #ifdef X509_R_UNABLE_TO_GET_CERTS_PUBLIC_KEY - {"UNABLE_TO_GET_CERTS_PUBLIC_KEY", ERR_LIB_X509, X509_R_UNABLE_TO_GET_CERTS_PUBLIC_KEY}, - #else - {"UNABLE_TO_GET_CERTS_PUBLIC_KEY", 11, 108}, - #endif - #ifdef X509_R_UNKNOWN_KEY_TYPE - {"UNKNOWN_KEY_TYPE", ERR_LIB_X509, X509_R_UNKNOWN_KEY_TYPE}, - #else - {"UNKNOWN_KEY_TYPE", 11, 117}, - #endif - #ifdef X509_R_UNKNOWN_NID - {"UNKNOWN_NID", ERR_LIB_X509, X509_R_UNKNOWN_NID}, - #else - {"UNKNOWN_NID", 11, 109}, - #endif - #ifdef X509_R_UNKNOWN_PURPOSE_ID - {"UNKNOWN_PURPOSE_ID", ERR_LIB_X509, X509_R_UNKNOWN_PURPOSE_ID}, - #else - {"UNKNOWN_PURPOSE_ID", 11, 121}, - #endif - #ifdef X509_R_UNKNOWN_TRUST_ID - {"UNKNOWN_TRUST_ID", ERR_LIB_X509, X509_R_UNKNOWN_TRUST_ID}, - #else - {"UNKNOWN_TRUST_ID", 11, 120}, - #endif - #ifdef X509_R_UNSUPPORTED_ALGORITHM - {"UNSUPPORTED_ALGORITHM", ERR_LIB_X509, X509_R_UNSUPPORTED_ALGORITHM}, - #else - {"UNSUPPORTED_ALGORITHM", 11, 111}, - #endif - #ifdef X509_R_WRONG_LOOKUP_TYPE - {"WRONG_LOOKUP_TYPE", ERR_LIB_X509, X509_R_WRONG_LOOKUP_TYPE}, - #else - {"WRONG_LOOKUP_TYPE", 11, 112}, - #endif - #ifdef X509_R_WRONG_TYPE - {"WRONG_TYPE", ERR_LIB_X509, X509_R_WRONG_TYPE}, - #else - {"WRONG_TYPE", 11, 122}, - #endif - { NULL } -}; diff --git a/Modules/_stat.c b/Modules/_stat.c index 80f8a92668976b..b43e79453f5b2f 100644 --- a/Modules/_stat.c +++ b/Modules/_stat.c @@ -202,6 +202,10 @@ typedef unsigned short mode_t; /* Names for file flags */ +#ifndef UF_SETTABLE +# define UF_SETTABLE 0x0000ffff +#endif + #ifndef UF_NODUMP # define UF_NODUMP 0x00000001 #endif @@ -226,10 +230,22 @@ typedef unsigned short mode_t; # define UF_COMPRESSED 0x00000020 #endif +#ifndef UF_TRACKED +# define UF_TRACKED 0x00000040 +#endif + +#ifndef UF_DATAVAULT +# define UF_DATAVAULT 0x00000080 +#endif + #ifndef UF_HIDDEN # define UF_HIDDEN 0x00008000 #endif +#ifndef SF_SETTABLE +# define SF_SETTABLE 0xffff0000 +#endif + #ifndef SF_ARCHIVED # define SF_ARCHIVED 0x00010000 #endif @@ -250,6 +266,30 @@ typedef unsigned short mode_t; # define SF_SNAPSHOT 0x00200000 #endif +#ifndef SF_FIRMLINK +# define SF_FIRMLINK 0x00800000 +#endif + +#ifndef SF_DATALESS +# define SF_DATALESS 0x40000000 +#endif + +#if defined(__APPLE__) && !defined(SF_SUPPORTED) + /* On older macOS versions the definition of SF_SUPPORTED is different + * from that on newer versions. + * + * Provide a consistent experience by redefining. + * + * None of bit bits set in the actual SF_SUPPORTED but not in this + * definition are defined on these versions of macOS. + */ +# undef SF_SETTABLE +# define SF_SUPPORTED 0x009f0000 +# define SF_SETTABLE 0x3fff0000 +# define SF_SYNTHETIC 0xc0000000 +#endif + + static mode_t _PyLong_AsMode_t(PyObject *op) { @@ -467,18 +507,29 @@ S_IWOTH: write by others\n\ S_IXOTH: execute by others\n\ \n" -"UF_NODUMP: do not dump file\n\ +"UF_SETTABLE: mask of owner changable flags\n\ +UF_NODUMP: do not dump file\n\ UF_IMMUTABLE: file may not be changed\n\ UF_APPEND: file may only be appended to\n\ UF_OPAQUE: directory is opaque when viewed through a union stack\n\ UF_NOUNLINK: file may not be renamed or deleted\n\ -UF_COMPRESSED: OS X: file is hfs-compressed\n\ -UF_HIDDEN: OS X: file should not be displayed\n\ +UF_COMPRESSED: macOS: file is hfs-compressed\n\ +UF_TRACKED: used for dealing with document IDs\n\ +UF_DATAVAULT: entitlement required for reading and writing\n\ +UF_HIDDEN: macOS: file should not be displayed\n\ +SF_SETTABLE: mask of super user changeable flags\n\ SF_ARCHIVED: file may be archived\n\ SF_IMMUTABLE: file may not be changed\n\ SF_APPEND: file may only be appended to\n\ +SF_RESTRICTED: entitlement required for writing\n\ SF_NOUNLINK: file may not be renamed or deleted\n\ SF_SNAPSHOT: file is a snapshot file\n\ +SF_FIRMLINK: file is a firmlink\n\ +SF_DATALESS: file is a dataless object\n\ +\n\ +On macOS:\n\ +SF_SUPPORTED: mask of super user supported flags\n\ +SF_SYNTHETIC: mask of read-only synthetic flags\n\ \n" "ST_MODE\n\ @@ -543,18 +594,32 @@ stat_exec(PyObject *module) ADD_INT_MACRO(module, S_IWOTH); ADD_INT_MACRO(module, S_IXOTH); + ADD_INT_MACRO(module, UF_SETTABLE); ADD_INT_MACRO(module, UF_NODUMP); ADD_INT_MACRO(module, UF_IMMUTABLE); ADD_INT_MACRO(module, UF_APPEND); ADD_INT_MACRO(module, UF_OPAQUE); ADD_INT_MACRO(module, UF_NOUNLINK); ADD_INT_MACRO(module, UF_COMPRESSED); + ADD_INT_MACRO(module, UF_TRACKED); + ADD_INT_MACRO(module, UF_DATAVAULT); ADD_INT_MACRO(module, UF_HIDDEN); + ADD_INT_MACRO(module, SF_SETTABLE); ADD_INT_MACRO(module, SF_ARCHIVED); ADD_INT_MACRO(module, SF_IMMUTABLE); ADD_INT_MACRO(module, SF_APPEND); ADD_INT_MACRO(module, SF_NOUNLINK); ADD_INT_MACRO(module, SF_SNAPSHOT); + ADD_INT_MACRO(module, SF_FIRMLINK); + ADD_INT_MACRO(module, SF_DATALESS); + +#ifdef SF_SUPPORTED + ADD_INT_MACRO(module, SF_SUPPORTED); +#endif +#ifdef SF_SYNTHETIC + ADD_INT_MACRO(module, SF_SYNTHETIC); +#endif + const char* st_constants[] = { "ST_MODE", diff --git a/Modules/_suggestions.c b/Modules/_suggestions.c new file mode 100644 index 00000000000000..30b524d70c1211 --- /dev/null +++ b/Modules/_suggestions.c @@ -0,0 +1,63 @@ +#include "Python.h" +#include "pycore_pyerrors.h" +#include "clinic/_suggestions.c.h" + +/*[clinic input] +module _suggestions +[clinic start generated code]*/ +/*[clinic end generated code: output=da39a3ee5e6b4b0d input=e58d81fafad5637b]*/ + +/*[clinic input] +_suggestions._generate_suggestions + candidates: object + item: unicode + / +Returns the candidate in candidates that's closest to item +[clinic start generated code]*/ + +static PyObject * +_suggestions__generate_suggestions_impl(PyObject *module, + PyObject *candidates, PyObject *item) +/*[clinic end generated code: output=79be7b653ae5e7ca input=ba2a8dddc654e33a]*/ +{ + // Check if dir is a list + if (!PyList_Check(candidates)) { + PyErr_SetString(PyExc_TypeError, "candidates must be a list"); + return NULL; + } + + // Check if all elements in the list are Unicode + Py_ssize_t size = PyList_Size(candidates); + for (Py_ssize_t i = 0; i < size; ++i) { + PyObject *elem = PyList_GetItem(candidates, i); + if (!PyUnicode_Check(elem)) { + PyErr_SetString(PyExc_TypeError, "all elements in 'candidates' must be strings"); + return NULL; + } + } + + PyObject* result = _Py_CalculateSuggestions(candidates, item); + if (!result && !PyErr_Occurred()) { + Py_RETURN_NONE; + } + return result; +} + + +static PyMethodDef module_methods[] = { + _SUGGESTIONS__GENERATE_SUGGESTIONS_METHODDEF + {NULL, NULL, 0, NULL} // Sentinel +}; + +static struct PyModuleDef suggestions_module = { + PyModuleDef_HEAD_INIT, + "_suggestions", + NULL, + -1, + module_methods +}; + +PyMODINIT_FUNC PyInit__suggestions(void) { + return PyModule_Create(&suggestions_module); +} + diff --git a/Modules/_testcapi/buffer.c b/Modules/_testcapi/buffer.c index 942774156c6c47..7e2f6e5e29482c 100644 --- a/Modules/_testcapi/buffer.c +++ b/Modules/_testcapi/buffer.c @@ -54,8 +54,10 @@ static int testbuf_getbuf(testBufObject *self, Py_buffer *view, int flags) { int buf = PyObject_GetBuffer(self->obj, view, flags); - Py_SETREF(view->obj, Py_NewRef(self)); - self->references++; + if (buf == 0) { + Py_SETREF(view->obj, Py_NewRef(self)); + self->references++; + } return buf; } diff --git a/Modules/_testcapi/dict.c b/Modules/_testcapi/dict.c index 42e056b7d07a31..fe03c24f75e196 100644 --- a/Modules/_testcapi/dict.c +++ b/Modules/_testcapi/dict.c @@ -225,6 +225,31 @@ dict_setdefault(PyObject *self, PyObject *args) return PyDict_SetDefault(mapping, key, defaultobj); } +static PyObject * +dict_setdefaultref(PyObject *self, PyObject *args) +{ + PyObject *obj, *key, *default_value, *result = UNINITIALIZED_PTR; + if (!PyArg_ParseTuple(args, "OOO", &obj, &key, &default_value)) { + return NULL; + } + NULLABLE(obj); + NULLABLE(key); + NULLABLE(default_value); + switch (PyDict_SetDefaultRef(obj, key, default_value, &result)) { + case -1: + assert(result == NULL); + return NULL; + case 0: + assert(result == default_value); + return result; + case 1: + return result; + default: + Py_FatalError("PyDict_SetDefaultRef() returned invalid code"); + Py_UNREACHABLE(); + } +} + static PyObject * dict_delitem(PyObject *self, PyObject *args) { @@ -433,6 +458,7 @@ static PyMethodDef test_methods[] = { {"dict_delitem", dict_delitem, METH_VARARGS}, {"dict_delitemstring", dict_delitemstring, METH_VARARGS}, {"dict_setdefault", dict_setdefault, METH_VARARGS}, + {"dict_setdefaultref", dict_setdefaultref, METH_VARARGS}, {"dict_keys", dict_keys, METH_O}, {"dict_values", dict_values, METH_O}, {"dict_items", dict_items, METH_O}, diff --git a/Modules/_testcapi/gc.c b/Modules/_testcapi/gc.c index 829200ad12cd3c..f4feaaafbdc6cc 100644 --- a/Modules/_testcapi/gc.c +++ b/Modules/_testcapi/gc.c @@ -126,9 +126,7 @@ slot_tp_del(PyObject *self) * never happened. */ { - Py_ssize_t refcnt = Py_REFCNT(self); - _Py_NewReferenceNoTotal(self); - Py_SET_REFCNT(self, refcnt); + _Py_ResurrectReference(self); } assert(!PyType_IS_GC(Py_TYPE(self)) || PyObject_GC_IsTracked(self)); } diff --git a/Modules/_testcapi/getargs.c b/Modules/_testcapi/getargs.c index 33e8af7d7bbb39..0d61d8c8969f82 100644 --- a/Modules/_testcapi/getargs.c +++ b/Modules/_testcapi/getargs.c @@ -56,9 +56,9 @@ parse_tuple_and_keywords(PyObject *self, PyObject *args) keywords[i] = PyBytes_AS_STRING(o); } else { - PyErr_Format(PyExc_ValueError, + PyErr_SetString(PyExc_ValueError, "parse_tuple_and_keywords: " - "keywords must be str or bytes", i); + "keywords must be str or bytes"); goto exit; } } diff --git a/Modules/_testcapi/list.c b/Modules/_testcapi/list.c index 10e18699f01bc1..2cb6499e28336d 100644 --- a/Modules/_testcapi/list.c +++ b/Modules/_testcapi/list.c @@ -59,6 +59,18 @@ list_get_item(PyObject *Py_UNUSED(module), PyObject *args) return Py_XNewRef(PyList_GET_ITEM(obj, i)); } +static PyObject * +list_get_item_ref(PyObject *Py_UNUSED(module), PyObject *args) +{ + PyObject *obj; + Py_ssize_t i; + if (!PyArg_ParseTuple(args, "On", &obj, &i)) { + return NULL; + } + NULLABLE(obj); + return PyList_GetItemRef(obj, i); +} + static PyObject * list_setitem(PyObject *Py_UNUSED(module), PyObject *args) { @@ -191,6 +203,7 @@ static PyMethodDef test_methods[] = { {"list_get_size", list_get_size, METH_O}, {"list_getitem", list_getitem, METH_VARARGS}, {"list_get_item", list_get_item, METH_VARARGS}, + {"list_get_item_ref", list_get_item_ref, METH_VARARGS}, {"list_setitem", list_setitem, METH_VARARGS}, {"list_set_item", list_set_item, METH_VARARGS}, {"list_insert", list_insert, METH_VARARGS}, diff --git a/Modules/_testcapi/vectorcall_limited.c b/Modules/_testcapi/vectorcall_limited.c index d7b8d33b7f7162..d7070d37bb9e9b 100644 --- a/Modules/_testcapi/vectorcall_limited.c +++ b/Modules/_testcapi/vectorcall_limited.c @@ -195,6 +195,6 @@ _PyTestCapi_Init_VectorcallLimited(PyObject *m) { if (PyModule_AddType(m, (PyTypeObject *)LimitedVectorCallClass) < 0) { return -1; } - + Py_DECREF(LimitedVectorCallClass); return 0; } diff --git a/Modules/_testcapi/watchers.c b/Modules/_testcapi/watchers.c index a763ff46a3c290..1eb0db2c2e6576 100644 --- a/Modules/_testcapi/watchers.c +++ b/Modules/_testcapi/watchers.c @@ -15,8 +15,8 @@ module _testcapi /*[clinic end generated code: output=da39a3ee5e6b4b0d input=6361033e795369fc]*/ // Test dict watching -static PyObject *g_dict_watch_events; -static int g_dict_watchers_installed; +static PyObject *g_dict_watch_events = NULL; +static int g_dict_watchers_installed = 0; static int dict_watch_callback(PyDict_WatchEvent event, diff --git a/Modules/_testcapi_feature_macros.inc b/Modules/_testcapi_feature_macros.inc index a076e714980074..f5f3524f2c0177 100644 --- a/Modules/_testcapi_feature_macros.inc +++ b/Modules/_testcapi_feature_macros.inc @@ -38,6 +38,15 @@ if (res) { Py_DECREF(result); return NULL; } +#ifdef Py_TRACE_REFS + res = PyDict_SetItemString(result, "Py_TRACE_REFS", Py_True); +#else + res = PyDict_SetItemString(result, "Py_TRACE_REFS", Py_False); +#endif +if (res) { + Py_DECREF(result); return NULL; +} + #ifdef USE_STACKCHECK res = PyDict_SetItemString(result, "USE_STACKCHECK", Py_True); #else diff --git a/Modules/_testcapimodule.c b/Modules/_testcapimodule.c index 3527dfa77279ac..e67de3eeb6e17e 100644 --- a/Modules/_testcapimodule.c +++ b/Modules/_testcapimodule.c @@ -33,15 +33,32 @@ // Forward declarations static struct PyModuleDef _testcapimodule; -static PyObject *TestError; /* set to exception object in init */ +// Module state +typedef struct { + PyObject *error; // _testcapi.error object +} testcapistate_t; + +static testcapistate_t* +get_testcapi_state(PyObject *module) +{ + void *state = PyModule_GetState(module); + assert(state != NULL); + return (testcapistate_t *)state; +} + +static PyObject * +get_testerror(PyObject *self) { + testcapistate_t *state = get_testcapi_state(self); + return state->error; +} -/* Raise TestError with test_name + ": " + msg, and return NULL. */ +/* Raise _testcapi.error with test_name + ": " + msg, and return NULL. */ static PyObject * -raiseTestError(const char* test_name, const char* msg) +raiseTestError(PyObject *self, const char* test_name, const char* msg) { - PyErr_Format(TestError, "%s: %s", test_name, msg); + PyErr_Format(get_testerror(self), "%s: %s", test_name, msg); return NULL; } @@ -52,10 +69,10 @@ raiseTestError(const char* test_name, const char* msg) platforms have these hardcoded. Better safe than sorry. */ static PyObject* -sizeof_error(const char* fatname, const char* typname, +sizeof_error(PyObject *self, const char* fatname, const char* typname, int expected, int got) { - PyErr_Format(TestError, + PyErr_Format(get_testerror(self), "%s #define == %d but sizeof(%s) == %d", fatname, expected, typname, got); return (PyObject*)NULL; @@ -66,7 +83,7 @@ test_config(PyObject *self, PyObject *Py_UNUSED(ignored)) { #define CHECK_SIZEOF(FATNAME, TYPE) \ if (FATNAME != sizeof(TYPE)) \ - return sizeof_error(#FATNAME, #TYPE, FATNAME, sizeof(TYPE)) + return sizeof_error(self, #FATNAME, #TYPE, FATNAME, sizeof(TYPE)) CHECK_SIZEOF(SIZEOF_SHORT, short); CHECK_SIZEOF(SIZEOF_INT, int); @@ -89,18 +106,18 @@ test_sizeof_c_types(PyObject *self, PyObject *Py_UNUSED(ignored)) #endif #define CHECK_SIZEOF(TYPE, EXPECTED) \ if (EXPECTED != sizeof(TYPE)) { \ - PyErr_Format(TestError, \ + PyErr_Format(get_testerror(self), \ "sizeof(%s) = %u instead of %u", \ #TYPE, sizeof(TYPE), EXPECTED); \ return (PyObject*)NULL; \ } #define IS_SIGNED(TYPE) (((TYPE)-1) < (TYPE)0) -#define CHECK_SIGNNESS(TYPE, SIGNED) \ - if (IS_SIGNED(TYPE) != SIGNED) { \ - PyErr_Format(TestError, \ - "%s signness is, instead of %i", \ - #TYPE, IS_SIGNED(TYPE), SIGNED); \ - return (PyObject*)NULL; \ +#define CHECK_SIGNNESS(TYPE, SIGNED) \ + if (IS_SIGNED(TYPE) != SIGNED) { \ + PyErr_Format(get_testerror(self), \ + "%s signness is %i, instead of %i", \ + #TYPE, IS_SIGNED(TYPE), SIGNED); \ + return (PyObject*)NULL; \ } /* integer types */ @@ -170,7 +187,7 @@ test_list_api(PyObject *self, PyObject *Py_UNUSED(ignored)) for (i = 0; i < NLIST; ++i) { PyObject* anint = PyList_GET_ITEM(list, i); if (PyLong_AS_LONG(anint) != NLIST-1-i) { - PyErr_SetString(TestError, + PyErr_SetString(get_testerror(self), "test_list_api: reverse screwed up"); Py_DECREF(list); return (PyObject*)NULL; @@ -183,7 +200,7 @@ test_list_api(PyObject *self, PyObject *Py_UNUSED(ignored)) } static int -test_dict_inner(int count) +test_dict_inner(PyObject *self, int count) { Py_ssize_t pos = 0, iterations = 0; int i; @@ -231,7 +248,7 @@ test_dict_inner(int count) if (iterations != count) { PyErr_SetString( - TestError, + get_testerror(self), "test_dict_iteration: dict iteration went wrong "); return -1; } else { @@ -250,7 +267,7 @@ test_dict_iteration(PyObject* self, PyObject *Py_UNUSED(ignored)) int i; for (i = 0; i < 200; i++) { - if (test_dict_inner(i) < 0) { + if (test_dict_inner(self, i) < 0) { return NULL; } } @@ -334,14 +351,14 @@ test_lazy_hash_inheritance(PyObject* self, PyObject *Py_UNUSED(ignored)) if (obj == NULL) { PyErr_Clear(); PyErr_SetString( - TestError, + get_testerror(self), "test_lazy_hash_inheritance: failed to create object"); return NULL; } if (type->tp_dict != NULL) { PyErr_SetString( - TestError, + get_testerror(self), "test_lazy_hash_inheritance: type initialised too soon"); Py_DECREF(obj); return NULL; @@ -351,7 +368,7 @@ test_lazy_hash_inheritance(PyObject* self, PyObject *Py_UNUSED(ignored)) if ((hash == -1) && PyErr_Occurred()) { PyErr_Clear(); PyErr_SetString( - TestError, + get_testerror(self), "test_lazy_hash_inheritance: could not hash object"); Py_DECREF(obj); return NULL; @@ -359,7 +376,7 @@ test_lazy_hash_inheritance(PyObject* self, PyObject *Py_UNUSED(ignored)) if (type->tp_dict == NULL) { PyErr_SetString( - TestError, + get_testerror(self), "test_lazy_hash_inheritance: type not initialised by hash()"); Py_DECREF(obj); return NULL; @@ -367,7 +384,7 @@ test_lazy_hash_inheritance(PyObject* self, PyObject *Py_UNUSED(ignored)) if (type->tp_hash != PyType_Type.tp_hash) { PyErr_SetString( - TestError, + get_testerror(self), "test_lazy_hash_inheritance: unexpected hash function"); Py_DECREF(obj); return NULL; @@ -427,7 +444,7 @@ py_buildvalue_ints(PyObject *self, PyObject *args) } static int -test_buildvalue_N_error(const char *fmt) +test_buildvalue_N_error(PyObject *self, const char *fmt) { PyObject *arg, *res; @@ -443,7 +460,7 @@ test_buildvalue_N_error(const char *fmt) } Py_DECREF(res); if (Py_REFCNT(arg) != 1) { - PyErr_Format(TestError, "test_buildvalue_N: " + PyErr_Format(get_testerror(self), "test_buildvalue_N: " "arg was not decrefed in successful " "Py_BuildValue(\"%s\")", fmt); return -1; @@ -452,13 +469,13 @@ test_buildvalue_N_error(const char *fmt) Py_INCREF(arg); res = Py_BuildValue(fmt, raise_error, NULL, arg); if (res != NULL || !PyErr_Occurred()) { - PyErr_Format(TestError, "test_buildvalue_N: " + PyErr_Format(get_testerror(self), "test_buildvalue_N: " "Py_BuildValue(\"%s\") didn't complain", fmt); return -1; } PyErr_Clear(); if (Py_REFCNT(arg) != 1) { - PyErr_Format(TestError, "test_buildvalue_N: " + PyErr_Format(get_testerror(self), "test_buildvalue_N: " "arg was not decrefed in failed " "Py_BuildValue(\"%s\")", fmt); return -1; @@ -482,25 +499,25 @@ test_buildvalue_N(PyObject *self, PyObject *Py_UNUSED(ignored)) return NULL; } if (res != arg) { - return raiseTestError("test_buildvalue_N", + return raiseTestError(self, "test_buildvalue_N", "Py_BuildValue(\"N\") returned wrong result"); } if (Py_REFCNT(arg) != 2) { - return raiseTestError("test_buildvalue_N", + return raiseTestError(self, "test_buildvalue_N", "arg was not decrefed in Py_BuildValue(\"N\")"); } Py_DECREF(res); Py_DECREF(arg); - if (test_buildvalue_N_error("O&N") < 0) + if (test_buildvalue_N_error(self, "O&N") < 0) return NULL; - if (test_buildvalue_N_error("(O&N)") < 0) + if (test_buildvalue_N_error(self, "(O&N)") < 0) return NULL; - if (test_buildvalue_N_error("[O&N]") < 0) + if (test_buildvalue_N_error(self, "[O&N]") < 0) return NULL; - if (test_buildvalue_N_error("{O&N}") < 0) + if (test_buildvalue_N_error(self, "{O&N}") < 0) return NULL; - if (test_buildvalue_N_error("{()O&(())N}") < 0) + if (test_buildvalue_N_error(self, "{()O&(())N}") < 0) return NULL; Py_RETURN_NONE; @@ -910,7 +927,7 @@ test_string_to_double(PyObject *self, PyObject *Py_UNUSED(ignored)) { Py_RETURN_NONE; fail: - return raiseTestError("test_string_to_double", msg); + return raiseTestError(self, "test_string_to_double", msg); #undef CHECK_STRING #undef CHECK_INVALID } @@ -1061,7 +1078,7 @@ test_capsule(PyObject *self, PyObject *Py_UNUSED(ignored)) exit: if (error) { - return raiseTestError("test_capsule", error); + return raiseTestError(self, "test_capsule", error); } Py_RETURN_NONE; #undef FAIL @@ -1244,6 +1261,26 @@ make_memoryview_from_NULL_pointer(PyObject *self, PyObject *Py_UNUSED(ignored)) return PyMemoryView_FromBuffer(&info); } +static PyObject * +buffer_fill_info(PyObject *self, PyObject *args) +{ + Py_buffer info; + const char *data; + Py_ssize_t size; + int readonly; + int flags; + + if (!PyArg_ParseTuple(args, "s#ii:buffer_fill_info", + &data, &size, &readonly, &flags)) { + return NULL; + } + + if (PyBuffer_FillInfo(&info, NULL, (void *)data, size, readonly, flags) < 0) { + return NULL; + } + return PyMemoryView_FromBuffer(&info); +} + static PyObject * test_from_contiguous(PyObject* self, PyObject *Py_UNUSED(ignored)) { @@ -1272,7 +1309,7 @@ test_from_contiguous(PyObject* self, PyObject *Py_UNUSED(ignored)) ptr = view.buf; for (i = 0; i < 5; i++) { if (ptr[2*i] != i) { - PyErr_SetString(TestError, + PyErr_SetString(get_testerror(self), "test_from_contiguous: incorrect result"); return NULL; } @@ -1285,7 +1322,7 @@ test_from_contiguous(PyObject* self, PyObject *Py_UNUSED(ignored)) ptr = view.buf; for (i = 0; i < 5; i++) { if (*(ptr-2*i) != i) { - PyErr_SetString(TestError, + PyErr_SetString(get_testerror(self), "test_from_contiguous: incorrect result"); return NULL; } @@ -1338,7 +1375,7 @@ test_pep3118_obsolete_write_locks(PyObject* self, PyObject *Py_UNUSED(ignored)) Py_RETURN_NONE; error: - PyErr_SetString(TestError, + PyErr_SetString(get_testerror(self), "test_pep3118_obsolete_write_locks: failure"); return NULL; } @@ -1959,7 +1996,7 @@ test_pythread_tss_key_state(PyObject *self, PyObject *args) { Py_tss_t tss_key = Py_tss_NEEDS_INIT; if (PyThread_tss_is_created(&tss_key)) { - return raiseTestError("test_pythread_tss_key_state", + return raiseTestError(self, "test_pythread_tss_key_state", "TSS key not in an uninitialized state at " "creation time"); } @@ -1968,19 +2005,19 @@ test_pythread_tss_key_state(PyObject *self, PyObject *args) return NULL; } if (!PyThread_tss_is_created(&tss_key)) { - return raiseTestError("test_pythread_tss_key_state", + return raiseTestError(self, "test_pythread_tss_key_state", "PyThread_tss_create succeeded, " "but with TSS key in an uninitialized state"); } if (PyThread_tss_create(&tss_key) != 0) { - return raiseTestError("test_pythread_tss_key_state", + return raiseTestError(self, "test_pythread_tss_key_state", "PyThread_tss_create unsuccessful with " "an already initialized key"); } #define CHECK_TSS_API(expr) \ (void)(expr); \ if (!PyThread_tss_is_created(&tss_key)) { \ - return raiseTestError("test_pythread_tss_key_state", \ + return raiseTestError(self, "test_pythread_tss_key_state", \ "TSS key initialization state was not " \ "preserved after calling " #expr); } CHECK_TSS_API(PyThread_tss_set(&tss_key, NULL)); @@ -1988,7 +2025,7 @@ test_pythread_tss_key_state(PyObject *self, PyObject *args) #undef CHECK_TSS_API PyThread_tss_delete(&tss_key); if (PyThread_tss_is_created(&tss_key)) { - return raiseTestError("test_pythread_tss_key_state", + return raiseTestError(self, "test_pythread_tss_key_state", "PyThread_tss_delete called, but did not " "set the key state to uninitialized"); } @@ -1999,7 +2036,7 @@ test_pythread_tss_key_state(PyObject *self, PyObject *args) return NULL; } if (PyThread_tss_is_created(ptr_key)) { - return raiseTestError("test_pythread_tss_key_state", + return raiseTestError(self, "test_pythread_tss_key_state", "TSS key not in an uninitialized state at " "allocation time"); } @@ -2392,6 +2429,32 @@ type_get_version(PyObject *self, PyObject *type) return res; } +static PyObject * +type_modified(PyObject *self, PyObject *type) +{ + if (!PyType_Check(type)) { + PyErr_SetString(PyExc_TypeError, "argument must be a type"); + return NULL; + } + PyType_Modified((PyTypeObject *)type); + Py_RETURN_NONE; +} + +// Circumvents standard version assignment machinery - use with caution and only on +// short-lived heap types +static PyObject * +type_assign_specific_version_unsafe(PyObject *self, PyObject *args) +{ + PyTypeObject *type; + unsigned int version; + if (!PyArg_ParseTuple(args, "Oi:type_assign_specific_version_unsafe", &type, &version)) { + return NULL; + } + assert(!PyType_HasFeature(type, Py_TPFLAGS_IMMUTABLETYPE)); + type->tp_version_tag = version; + type->tp_flags |= Py_TPFLAGS_VALID_VERSION_TAG; + Py_RETURN_NONE; +} static PyObject * type_assign_version(PyObject *self, PyObject *type) @@ -3271,6 +3334,7 @@ static PyMethodDef TestMethods[] = { {"eval_code_ex", eval_eval_code_ex, METH_VARARGS}, {"make_memoryview_from_NULL_pointer", make_memoryview_from_NULL_pointer, METH_NOARGS}, + {"buffer_fill_info", buffer_fill_info, METH_VARARGS}, {"crash_no_current_thread", crash_no_current_thread, METH_NOARGS}, {"test_current_tstate_matches", test_current_tstate_matches, METH_NOARGS}, {"run_in_subinterp", run_in_subinterp, METH_VARARGS}, @@ -3325,6 +3389,9 @@ static PyMethodDef TestMethods[] = { {"test_py_is_macros", test_py_is_macros, METH_NOARGS}, {"test_py_is_funcs", test_py_is_funcs, METH_NOARGS}, {"type_get_version", type_get_version, METH_O, PyDoc_STR("type->tp_version_tag")}, + {"type_modified", type_modified, METH_O, PyDoc_STR("PyType_Modified")}, + {"type_assign_specific_version_unsafe", type_assign_specific_version_unsafe, METH_VARARGS, + PyDoc_STR("forcefully assign type->tp_version_tag")}, {"type_assign_version", type_assign_version, METH_O, PyDoc_STR("PyUnstable_Type_AssignVersionTag")}, {"type_get_tp_bases", type_get_tp_bases, METH_O}, {"type_get_tp_mro", type_get_tp_mro, METH_O}, @@ -3831,14 +3898,9 @@ static PyTypeObject ContainerNoGC_type = { static struct PyModuleDef _testcapimodule = { PyModuleDef_HEAD_INIT, - "_testcapi", - NULL, - -1, - TestMethods, - NULL, - NULL, - NULL, - NULL + .m_name = "_testcapi", + .m_size = sizeof(testcapistate_t), + .m_methods = TestMethods, }; /* Per PEP 489, this module will not be converted to multi-phase initialization @@ -3933,9 +3995,9 @@ PyInit__testcapi(void) PyModule_AddIntConstant(m, "the_number_three", 3); PyModule_AddIntMacro(m, Py_C_RECURSION_LIMIT); - TestError = PyErr_NewException("_testcapi.error", NULL, NULL); - Py_INCREF(TestError); - PyModule_AddObject(m, "error", TestError); + testcapistate_t *state = get_testcapi_state(m); + state->error = PyErr_NewException("_testcapi.error", NULL, NULL); + PyModule_AddObject(m, "error", state->error); if (PyType_Ready(&ContainerNoGC_type) < 0) { return NULL; diff --git a/Modules/_testclinic.c b/Modules/_testclinic.c index 15e0093f15ba1e..fb0936bbccd318 100644 --- a/Modules/_testclinic.c +++ b/Modules/_testclinic.c @@ -1213,6 +1213,40 @@ clone_with_conv_f2_impl(PyObject *module, custom_t path) } +/*[clinic input] +class _testclinic.TestClass "PyObject *" "PyObject" +[clinic start generated code]*/ +/*[clinic end generated code: output=da39a3ee5e6b4b0d input=668a591c65bec947]*/ + +/*[clinic input] +_testclinic.TestClass.meth_method_no_params + cls: defining_class + / +[clinic start generated code]*/ + +static PyObject * +_testclinic_TestClass_meth_method_no_params_impl(PyObject *self, + PyTypeObject *cls) +/*[clinic end generated code: output=c140f100080c2fc8 input=6bd34503d11c63c1]*/ +{ + Py_RETURN_NONE; +} + +static struct PyMethodDef test_class_methods[] = { + _TESTCLINIC_TESTCLASS_METH_METHOD_NO_PARAMS_METHODDEF + {NULL, NULL} +}; + +static PyTypeObject TestClass = { + PyVarObject_HEAD_INIT(NULL, 0) + .tp_name = "_testclinic.TestClass", + .tp_basicsize = sizeof(PyObject), + .tp_flags = Py_TPFLAGS_DEFAULT, + .tp_new = PyType_GenericNew, + .tp_methods = test_class_methods, +}; + + /*[clinic input] output push destination deprstar new file '{dirname}/clinic/_testclinic_depr.c.h' @@ -1906,6 +1940,9 @@ PyInit__testclinic(void) if (m == NULL) { return NULL; } + if (PyModule_AddType(m, &TestClass) < 0) { + goto error; + } if (PyModule_AddType(m, &DeprStarNew) < 0) { goto error; } diff --git a/Modules/_testinternalcapi.c b/Modules/_testinternalcapi.c index 7d277df164d3ec..0bb739b5398b11 100644 --- a/Modules/_testinternalcapi.c +++ b/Modules/_testinternalcapi.c @@ -1635,6 +1635,21 @@ get_type_module_name(PyObject *self, PyObject *type) return _PyType_GetModuleName((PyTypeObject *)type); } +static PyObject * +get_rare_event_counters(PyObject *self, PyObject *type) +{ + PyInterpreterState *interp = PyInterpreterState_Get(); + + return Py_BuildValue( + "{sksksksksk}", + "set_class", (unsigned long)interp->rare_events.set_class, + "set_bases", (unsigned long)interp->rare_events.set_bases, + "set_eval_frame_func", (unsigned long)interp->rare_events.set_eval_frame_func, + "builtin_dict", (unsigned long)interp->rare_events.builtin_dict, + "func_modification", (unsigned long)interp->rare_events.func_modification + ); +} + #ifdef Py_GIL_DISABLED static PyObject * @@ -1711,6 +1726,7 @@ static PyMethodDef module_functions[] = { {"restore_crossinterp_data", restore_crossinterp_data, METH_VARARGS}, _TESTINTERNALCAPI_TEST_LONG_NUMBITS_METHODDEF {"get_type_module_name", get_type_module_name, METH_O}, + {"get_rare_event_counters", get_rare_event_counters, METH_NOARGS}, #ifdef Py_GIL_DISABLED {"py_thread_id", get_py_thread_id, METH_NOARGS}, #endif @@ -1736,8 +1752,18 @@ module_exec(PyObject *module) return 1; } + Py_ssize_t sizeof_gc_head = 0; +#ifndef Py_GIL_DISABLED + sizeof_gc_head = sizeof(PyGC_Head); +#endif + if (PyModule_Add(module, "SIZEOF_PYGC_HEAD", - PyLong_FromSsize_t(sizeof(PyGC_Head))) < 0) { + PyLong_FromSsize_t(sizeof_gc_head)) < 0) { + return 1; + } + + if (PyModule_Add(module, "SIZEOF_MANAGED_PRE_HEADER", + PyLong_FromSsize_t(2 * sizeof(PyObject*))) < 0) { return 1; } diff --git a/Modules/_threadmodule.c b/Modules/_threadmodule.c index afcf646e3bc19e..df02b023012fbd 100644 --- a/Modules/_threadmodule.c +++ b/Modules/_threadmodule.c @@ -5,6 +5,7 @@ #include "Python.h" #include "pycore_interp.h" // _PyInterpreterState.threads.count #include "pycore_moduleobject.h" // _PyModule_GetState() +#include "pycore_modsupport.h" // _PyArg_NoKeywords() #include "pycore_pylifecycle.h" #include "pycore_pystate.h" // _PyThreadState_SetCurrent() #include "pycore_sysmodule.h" // _PySys_GetAttr() @@ -43,6 +44,7 @@ get_thread_state(PyObject *module) typedef struct { PyObject_HEAD + struct llist_node node; // linked list node (see _pythread_runtime_state) PyThread_ident_t ident; PyThread_handle_t handle; char joinable; @@ -58,6 +60,11 @@ new_thread_handle(thread_module_state* state) self->ident = 0; self->handle = 0; self->joinable = 0; + + HEAD_LOCK(&_PyRuntime); + llist_insert_tail(&_PyRuntime.threads.handles, &self->node); + HEAD_UNLOCK(&_PyRuntime); + return self; } @@ -65,6 +72,14 @@ static void ThreadHandle_dealloc(ThreadHandleObject *self) { PyObject *tp = (PyObject *) Py_TYPE(self); + + // Remove ourself from the global list of handles + HEAD_LOCK(&_PyRuntime); + if (self->node.next != NULL) { + llist_remove(&self->node); + } + HEAD_UNLOCK(&_PyRuntime); + if (self->joinable) { int ret = PyThread_detach_thread(self->handle); if (ret) { @@ -76,6 +91,28 @@ ThreadHandle_dealloc(ThreadHandleObject *self) Py_DECREF(tp); } +void +_PyThread_AfterFork(struct _pythread_runtime_state *state) +{ + // gh-115035: We mark ThreadHandles as not joinable early in the child's + // after-fork handler. We do this before calling any Python code to ensure + // that it happens before any ThreadHandles are deallocated, such as by a + // GC cycle. + PyThread_ident_t current = PyThread_get_thread_ident_ex(); + + struct llist_node *node; + llist_for_each_safe(node, &state->handles) { + ThreadHandleObject *hobj = llist_data(node, ThreadHandleObject, node); + if (hobj->ident == current) { + continue; + } + + // Disallow calls to detach() and join() as they could crash. + hobj->joinable = 0; + llist_remove(node); + } +} + static PyObject * ThreadHandle_repr(ThreadHandleObject *self) { @@ -90,21 +127,6 @@ ThreadHandle_get_ident(ThreadHandleObject *self, void *ignored) } -static PyObject * -ThreadHandle_after_fork_alive(ThreadHandleObject *self, void* ignored) -{ - PyThread_update_thread_after_fork(&self->ident, &self->handle); - Py_RETURN_NONE; -} - -static PyObject * -ThreadHandle_after_fork_dead(ThreadHandleObject *self, void* ignored) -{ - // Disallow calls to detach() and join() as they could crash. - self->joinable = 0; - Py_RETURN_NONE; -} - static PyObject * ThreadHandle_detach(ThreadHandleObject *self, void* ignored) { @@ -156,8 +178,6 @@ static PyGetSetDef ThreadHandle_getsetlist[] = { static PyMethodDef ThreadHandle_methods[] = { - {"after_fork_alive", (PyCFunction)ThreadHandle_after_fork_alive, METH_NOARGS}, - {"after_fork_dead", (PyCFunction)ThreadHandle_after_fork_dead, METH_NOARGS}, {"detach", (PyCFunction)ThreadHandle_detach, METH_NOARGS}, {"join", (PyCFunction)ThreadHandle_join, METH_NOARGS}, {0, 0} @@ -349,6 +369,27 @@ lock__at_fork_reinit(lockobject *self, PyObject *Py_UNUSED(args)) } #endif /* HAVE_FORK */ +static lockobject *newlockobject(PyObject *module); + +static PyObject * +lock_new(PyTypeObject *type, PyObject *args, PyObject *kwargs) +{ + // convert to AC? + if (!_PyArg_NoKeywords("lock", kwargs)) { + goto error; + } + if (!_PyArg_CheckPositional("lock", PyTuple_GET_SIZE(args), 0, 0)) { + goto error; + } + + PyObject *module = PyType_GetModuleByDef(type, &thread_module); + assert(module != NULL); + return (PyObject *)newlockobject(module); + +error: + return NULL; +} + static PyMethodDef lock_methods[] = { {"acquire_lock", _PyCFunction_CAST(lock_PyThread_acquire_lock), @@ -398,6 +439,7 @@ static PyType_Slot lock_type_slots[] = { {Py_tp_methods, lock_methods}, {Py_tp_traverse, lock_traverse}, {Py_tp_members, lock_type_members}, + {Py_tp_new, lock_new}, {0, 0} }; @@ -405,7 +447,7 @@ static PyType_Spec lock_type_spec = { .name = "_thread.lock", .basicsize = sizeof(lockobject), .flags = (Py_TPFLAGS_DEFAULT | Py_TPFLAGS_HAVE_GC | - Py_TPFLAGS_DISALLOW_INSTANTIATION | Py_TPFLAGS_IMMUTABLETYPE), + Py_TPFLAGS_IMMUTABLETYPE), .slots = lock_type_slots, }; @@ -901,6 +943,7 @@ local_new(PyTypeObject *type, PyObject *args, PyObject *kw) } PyObject *module = PyType_GetModuleByDef(type, &thread_module); + assert(module != NULL); thread_module_state *state = get_thread_state(module); localobject *self = (localobject *)type->tp_alloc(type, 0); @@ -1042,6 +1085,7 @@ static int local_setattro(localobject *self, PyObject *name, PyObject *v) { PyObject *module = PyType_GetModuleByDef(Py_TYPE(self), &thread_module); + assert(module != NULL); thread_module_state *state = get_thread_state(module); PyObject *ldict = _ldict(self, state); @@ -1094,6 +1138,7 @@ static PyObject * local_getattro(localobject *self, PyObject *name) { PyObject *module = PyType_GetModuleByDef(Py_TYPE(self), &thread_module); + assert(module != NULL); thread_module_state *state = get_thread_state(module); PyObject *ldict = _ldict(self, state); @@ -1439,8 +1484,6 @@ A subthread can use this function to interrupt the main thread.\n\ Note: the default signal handler for SIGINT raises ``KeyboardInterrupt``." ); -static lockobject *newlockobject(PyObject *module); - static PyObject * thread_PyThread_allocate_lock(PyObject *module, PyObject *Py_UNUSED(ignored)) { @@ -1838,10 +1881,14 @@ thread_module_exec(PyObject *module) } // Lock - state->lock_type = (PyTypeObject *)PyType_FromSpec(&lock_type_spec); + state->lock_type = (PyTypeObject *)PyType_FromModuleAndSpec(module, &lock_type_spec, NULL); if (state->lock_type == NULL) { return -1; } + if (PyModule_AddType(module, state->lock_type) < 0) { + return -1; + } + // Old alias: lock -> LockType if (PyDict_SetItemString(d, "LockType", (PyObject *)state->lock_type) < 0) { return -1; } diff --git a/Modules/_tkinter.c b/Modules/_tkinter.c index 64e752c305aae1..f6181168a85ae1 100644 --- a/Modules/_tkinter.c +++ b/Modules/_tkinter.c @@ -735,8 +735,9 @@ newPyTclObject(Tcl_Obj *arg) } static void -PyTclObject_dealloc(PyTclObject *self) +PyTclObject_dealloc(PyObject *_self) { + PyTclObject *self = (PyTclObject *)_self; PyObject *tp = (PyObject *) Py_TYPE(self); Tcl_DecrRefCount(self->value); Py_XDECREF(self->string); @@ -749,8 +750,9 @@ PyDoc_STRVAR(PyTclObject_string__doc__, "the string representation of this object, either as str or bytes"); static PyObject * -PyTclObject_string(PyTclObject *self, void *ignored) +PyTclObject_string(PyObject *_self, void *ignored) { + PyTclObject *self = (PyTclObject *)_self; if (!self->string) { self->string = unicodeFromTclObj(self->value); if (!self->string) @@ -760,8 +762,9 @@ PyTclObject_string(PyTclObject *self, void *ignored) } static PyObject * -PyTclObject_str(PyTclObject *self) +PyTclObject_str(PyObject *_self) { + PyTclObject *self = (PyTclObject *)_self; if (self->string) { return Py_NewRef(self->string); } @@ -770,9 +773,10 @@ PyTclObject_str(PyTclObject *self) } static PyObject * -PyTclObject_repr(PyTclObject *self) +PyTclObject_repr(PyObject *_self) { - PyObject *repr, *str = PyTclObject_str(self); + PyTclObject *self = (PyTclObject *)_self; + PyObject *repr, *str = PyTclObject_str(_self); if (str == NULL) return NULL; repr = PyUnicode_FromFormat("<%s object: %R>", @@ -809,23 +813,24 @@ PyTclObject_richcompare(PyObject *self, PyObject *other, int op) PyDoc_STRVAR(get_typename__doc__, "name of the Tcl type"); static PyObject* -get_typename(PyTclObject* obj, void* ignored) +get_typename(PyObject *self, void* ignored) { + PyTclObject *obj = (PyTclObject *)self; return unicodeFromTclString(obj->value->typePtr->name); } static PyGetSetDef PyTclObject_getsetlist[] = { - {"typename", (getter)get_typename, NULL, get_typename__doc__}, - {"string", (getter)PyTclObject_string, NULL, + {"typename", get_typename, NULL, get_typename__doc__}, + {"string", PyTclObject_string, NULL, PyTclObject_string__doc__}, {0}, }; static PyType_Slot PyTclObject_Type_slots[] = { - {Py_tp_dealloc, (destructor)PyTclObject_dealloc}, - {Py_tp_repr, (reprfunc)PyTclObject_repr}, - {Py_tp_str, (reprfunc)PyTclObject_str}, + {Py_tp_dealloc, PyTclObject_dealloc}, + {Py_tp_repr, PyTclObject_repr}, + {Py_tp_str, PyTclObject_str}, {Py_tp_getattro, PyObject_GenericGetAttr}, {Py_tp_richcompare, PyTclObject_richcompare}, {Py_tp_getset, PyTclObject_getsetlist}, @@ -1306,8 +1311,9 @@ Tkapp_ObjectResult(TkappObject *self) hold the Python lock. */ static int -Tkapp_CallProc(Tkapp_CallEvent *e, int flags) +Tkapp_CallProc(Tcl_Event *evPtr, int flags) { + Tkapp_CallEvent *e = (Tkapp_CallEvent *)evPtr; Tcl_Obj *objStore[ARGSZ]; Tcl_Obj **objv; int objc; @@ -1385,7 +1391,7 @@ Tkapp_Call(PyObject *selfptr, PyObject *args) PyErr_NoMemory(); return NULL; } - ev->ev.proc = (Tcl_EventProc*)Tkapp_CallProc; + ev->ev.proc = Tkapp_CallProc; ev->self = self; ev->args = args; ev->res = &res; @@ -1624,8 +1630,9 @@ var_perform(VarEvent *ev) } static int -var_proc(VarEvent* ev, int flags) +var_proc(Tcl_Event *evPtr, int flags) { + VarEvent *ev = (VarEvent *)evPtr; ENTER_PYTHON var_perform(ev); Tcl_MutexLock(&var_mutex); @@ -1663,7 +1670,7 @@ var_invoke(EventFunc func, PyObject *selfptr, PyObject *args, int flags) ev->res = &res; ev->exc = &exc; ev->cond = &cond; - ev->ev.proc = (Tcl_EventProc*)var_proc; + ev->ev.proc = var_proc; Tkapp_ThreadSend(self, (Tcl_Event*)ev, &cond, &var_mutex); Tcl_ConditionFinalize(&cond); if (!res) { @@ -2236,8 +2243,9 @@ typedef struct CommandEvent{ } CommandEvent; static int -Tkapp_CommandProc(CommandEvent *ev, int flags) +Tkapp_CommandProc(Tcl_Event *evPtr, int flags) { + CommandEvent *ev = (CommandEvent *)evPtr; if (ev->create) *ev->status = Tcl_CreateObjCommand( ev->interp, ev->name, PythonCmd, @@ -2290,7 +2298,7 @@ _tkinter_tkapp_createcommand_impl(TkappObject *self, const char *name, PyMem_Free(data); return NULL; } - ev->ev.proc = (Tcl_EventProc*)Tkapp_CommandProc; + ev->ev.proc = Tkapp_CommandProc; ev->interp = self->interp; ev->create = 1; ev->name = name; @@ -2343,7 +2351,7 @@ _tkinter_tkapp_deletecommand_impl(TkappObject *self, const char *name) PyErr_NoMemory(); return NULL; } - ev->ev.proc = (Tcl_EventProc*)Tkapp_CommandProc; + ev->ev.proc = Tkapp_CommandProc; ev->interp = self->interp; ev->create = 0; ev->name = name; diff --git a/Modules/_winapi.c b/Modules/_winapi.c index 8c48b6f3ec6ef6..5e5eb123c4ccff 100644 --- a/Modules/_winapi.c +++ b/Modules/_winapi.c @@ -441,7 +441,7 @@ _winapi_ConnectNamedPipe_impl(PyObject *module, HANDLE handle, /*[clinic input] _winapi.CreateFile -> HANDLE - file_name: LPCTSTR + file_name: LPCWSTR desired_access: DWORD share_mode: DWORD security_attributes: LPSECURITY_ATTRIBUTES @@ -452,12 +452,12 @@ _winapi.CreateFile -> HANDLE [clinic start generated code]*/ static HANDLE -_winapi_CreateFile_impl(PyObject *module, LPCTSTR file_name, +_winapi_CreateFile_impl(PyObject *module, LPCWSTR file_name, DWORD desired_access, DWORD share_mode, LPSECURITY_ATTRIBUTES security_attributes, DWORD creation_disposition, DWORD flags_and_attributes, HANDLE template_file) -/*[clinic end generated code: output=417ddcebfc5a3d53 input=6423c3e40372dbd5]*/ +/*[clinic end generated code: output=818c811e5e04d550 input=1fa870ed1c2e3d69]*/ { HANDLE handle; @@ -468,14 +468,15 @@ _winapi_CreateFile_impl(PyObject *module, LPCTSTR file_name, } Py_BEGIN_ALLOW_THREADS - handle = CreateFile(file_name, desired_access, - share_mode, security_attributes, - creation_disposition, - flags_and_attributes, template_file); + handle = CreateFileW(file_name, desired_access, + share_mode, security_attributes, + creation_disposition, + flags_and_attributes, template_file); Py_END_ALLOW_THREADS - if (handle == INVALID_HANDLE_VALUE) + if (handle == INVALID_HANDLE_VALUE) { PyErr_SetFromWindowsErr(0); + } return handle; } @@ -532,7 +533,12 @@ _winapi_CreateJunction_impl(PyObject *module, LPCWSTR src_path, { /* Privilege adjustment */ HANDLE token = NULL; - TOKEN_PRIVILEGES tp; + struct { + TOKEN_PRIVILEGES base; + /* overallocate by a few array elements */ + LUID_AND_ATTRIBUTES privs[4]; + } tp, previousTp; + int previousTpSize = 0; /* Reparse data buffer */ const USHORT prefix_len = 4; @@ -556,17 +562,21 @@ _winapi_CreateJunction_impl(PyObject *module, LPCWSTR src_path, /* Adjust privileges to allow rewriting directory entry as a junction point. */ - if (!OpenProcessToken(GetCurrentProcess(), TOKEN_ADJUST_PRIVILEGES, &token)) + if (!OpenProcessToken(GetCurrentProcess(), + TOKEN_ADJUST_PRIVILEGES | TOKEN_QUERY, &token)) { goto cleanup; + } - if (!LookupPrivilegeValue(NULL, SE_RESTORE_NAME, &tp.Privileges[0].Luid)) + if (!LookupPrivilegeValue(NULL, SE_RESTORE_NAME, &tp.base.Privileges[0].Luid)) { goto cleanup; + } - tp.PrivilegeCount = 1; - tp.Privileges[0].Attributes = SE_PRIVILEGE_ENABLED; - if (!AdjustTokenPrivileges(token, FALSE, &tp, sizeof(TOKEN_PRIVILEGES), - NULL, NULL)) + tp.base.PrivilegeCount = 1; + tp.base.Privileges[0].Attributes = SE_PRIVILEGE_ENABLED; + if (!AdjustTokenPrivileges(token, FALSE, &tp.base, sizeof(previousTp), + &previousTp.base, &previousTpSize)) { goto cleanup; + } if (GetFileAttributesW(src_path) == INVALID_FILE_ATTRIBUTES) goto cleanup; @@ -647,6 +657,11 @@ _winapi_CreateJunction_impl(PyObject *module, LPCWSTR src_path, cleanup: ret = GetLastError(); + if (previousTpSize) { + AdjustTokenPrivileges(token, FALSE, &previousTp.base, previousTpSize, + NULL, NULL); + } + if (token != NULL) CloseHandle(token); if (junction != NULL) @@ -774,12 +789,157 @@ gethandle(PyObject* obj, const char* name) return ret; } +static PyObject * +sortenvironmentkey(PyObject *module, PyObject *item) +{ + return _winapi_LCMapStringEx_impl(NULL, LOCALE_NAME_INVARIANT, + LCMAP_UPPERCASE, item); +} + +static PyMethodDef sortenvironmentkey_def = { + "sortenvironmentkey", _PyCFunction_CAST(sortenvironmentkey), METH_O, "", +}; + +static int +sort_environment_keys(PyObject *keys) +{ + PyObject *keyfunc = PyCFunction_New(&sortenvironmentkey_def, NULL); + if (keyfunc == NULL) { + return -1; + } + PyObject *kwnames = Py_BuildValue("(s)", "key"); + if (kwnames == NULL) { + Py_DECREF(keyfunc); + return -1; + } + PyObject *args[] = { keys, keyfunc }; + PyObject *ret = PyObject_VectorcallMethod(&_Py_ID(sort), args, 1, kwnames); + Py_DECREF(keyfunc); + Py_DECREF(kwnames); + if (ret == NULL) { + return -1; + } + Py_DECREF(ret); + + return 0; +} + +static int +compare_string_ordinal(PyObject *str1, PyObject *str2, int *result) +{ + wchar_t *s1 = PyUnicode_AsWideCharString(str1, NULL); + if (s1 == NULL) { + return -1; + } + wchar_t *s2 = PyUnicode_AsWideCharString(str2, NULL); + if (s2 == NULL) { + PyMem_Free(s1); + return -1; + } + *result = CompareStringOrdinal(s1, -1, s2, -1, TRUE); + PyMem_Free(s1); + PyMem_Free(s2); + return 0; +} + +static PyObject * +dedup_environment_keys(PyObject *keys) +{ + PyObject *result = PyList_New(0); + if (result == NULL) { + return NULL; + } + + // Iterate over the pre-ordered keys, check whether the current key is equal + // to the next key (ignoring case), if different, insert the current value + // into the result list. If they are equal, do nothing because we always + // want to keep the last inserted one. + for (Py_ssize_t i = 0; i < PyList_GET_SIZE(keys); i++) { + PyObject *key = PyList_GET_ITEM(keys, i); + + // The last key will always be kept. + if (i + 1 == PyList_GET_SIZE(keys)) { + if (PyList_Append(result, key) < 0) { + Py_DECREF(result); + return NULL; + } + continue; + } + + PyObject *next_key = PyList_GET_ITEM(keys, i + 1); + int compare_result; + if (compare_string_ordinal(key, next_key, &compare_result) < 0) { + Py_DECREF(result); + return NULL; + } + if (compare_result == CSTR_EQUAL) { + continue; + } + if (PyList_Append(result, key) < 0) { + Py_DECREF(result); + return NULL; + } + } + + return result; +} + +static PyObject * +normalize_environment(PyObject *environment) +{ + PyObject *keys = PyMapping_Keys(environment); + if (keys == NULL) { + return NULL; + } + + if (sort_environment_keys(keys) < 0) { + Py_DECREF(keys); + return NULL; + } + + PyObject *normalized_keys = dedup_environment_keys(keys); + Py_DECREF(keys); + if (normalized_keys == NULL) { + return NULL; + } + + PyObject *result = PyDict_New(); + if (result == NULL) { + Py_DECREF(normalized_keys); + return NULL; + } + + for (int i = 0; i < PyList_GET_SIZE(normalized_keys); i++) { + PyObject *key = PyList_GET_ITEM(normalized_keys, i); + PyObject *value = PyObject_GetItem(environment, key); + if (value == NULL) { + Py_DECREF(normalized_keys); + Py_DECREF(result); + return NULL; + } + + int ret = PyObject_SetItem(result, key, value); + Py_DECREF(value); + if (ret < 0) { + Py_DECREF(normalized_keys); + Py_DECREF(result); + return NULL; + } + } + + Py_DECREF(normalized_keys); + + return result; +} + static wchar_t * getenvironment(PyObject* environment) { Py_ssize_t i, envsize, totalsize; wchar_t *buffer = NULL, *p, *end; - PyObject *keys, *values; + PyObject *normalized_environment = NULL; + PyObject *keys = NULL; + PyObject *values = NULL; /* convert environment dictionary to windows environment string */ if (! PyMapping_Check(environment)) { @@ -788,11 +948,16 @@ getenvironment(PyObject* environment) return NULL; } - keys = PyMapping_Keys(environment); - if (!keys) { + normalized_environment = normalize_environment(environment); + if (normalize_environment == NULL) { return NULL; } - values = PyMapping_Values(environment); + + keys = PyMapping_Keys(normalized_environment); + if (!keys) { + goto error; + } + values = PyMapping_Values(normalized_environment); if (!values) { goto error; } @@ -884,6 +1049,7 @@ getenvironment(PyObject* environment) cleanup: error: + Py_XDECREF(normalized_environment); Py_XDECREF(keys); Py_XDECREF(values); return buffer; diff --git a/Modules/_xxtestfuzz/fuzzer.c b/Modules/_xxtestfuzz/fuzzer.c index e133b4d3c44480..6ea9f64d628530 100644 --- a/Modules/_xxtestfuzz/fuzzer.c +++ b/Modules/_xxtestfuzz/fuzzer.c @@ -502,7 +502,6 @@ static int fuzz_elementtree_parsewhole(const char* data, size_t size) { } #define MAX_PYCOMPILE_TEST_SIZE 16384 -static char pycompile_scratch[MAX_PYCOMPILE_TEST_SIZE]; static const int start_vals[] = {Py_eval_input, Py_single_input, Py_file_input}; const size_t NUM_START_VALS = sizeof(start_vals) / sizeof(start_vals[0]); @@ -531,6 +530,8 @@ static int fuzz_pycompile(const char* data, size_t size) { unsigned char optimize_idx = (unsigned char) data[1]; int optimize = optimize_vals[optimize_idx % NUM_OPTIMIZE_VALS]; + char pycompile_scratch[MAX_PYCOMPILE_TEST_SIZE]; + // Create a NUL-terminated C string from the remaining input memcpy(pycompile_scratch, data + 2, size - 2); // Put a NUL terminator just after the copied data. (Space was reserved already.) @@ -549,7 +550,13 @@ static int fuzz_pycompile(const char* data, size_t size) { PyObject *result = Py_CompileStringExFlags(pycompile_scratch, "", start, flags, optimize); if (result == NULL) { - /* compilation failed, most likely from a syntax error */ + /* Compilation failed, most likely from a syntax error. If it was a + SystemError we abort. There's no non-bug reason to raise a + SystemError. */ + if (PyErr_Occurred() && PyErr_ExceptionMatches(PyExc_SystemError)) { + PyErr_Print(); + abort(); + } PyErr_Clear(); } else { Py_DECREF(result); diff --git a/Modules/_zoneinfo.c b/Modules/_zoneinfo.c index 77644c3155bc33..fcd4af64df0be9 100644 --- a/Modules/_zoneinfo.c +++ b/Modules/_zoneinfo.c @@ -853,28 +853,19 @@ load_timedelta(zoneinfo_state *state, long seconds) if (pyoffset == NULL) { return NULL; } - rv = PyDict_GetItemWithError(state->TIMEDELTA_CACHE, pyoffset); - if (rv == NULL) { - if (PyErr_Occurred()) { - goto error; - } + if (PyDict_GetItemRef(state->TIMEDELTA_CACHE, pyoffset, &rv) == 0) { PyObject *tmp = PyDateTimeAPI->Delta_FromDelta( 0, seconds, 0, 1, PyDateTimeAPI->DeltaType); - if (tmp == NULL) { - goto error; + if (tmp != NULL) { + rv = PyDict_SetDefault(state->TIMEDELTA_CACHE, pyoffset, tmp); + Py_XINCREF(rv); + Py_DECREF(tmp); } - - rv = PyDict_SetDefault(state->TIMEDELTA_CACHE, pyoffset, tmp); - Py_DECREF(tmp); } - Py_XINCREF(rv); Py_DECREF(pyoffset); return rv; -error: - Py_DECREF(pyoffset); - return NULL; } /* Constructor for _ttinfo object - this starts by initializing the _ttinfo diff --git a/Modules/cjkcodecs/clinic/multibytecodec.c.h b/Modules/cjkcodecs/clinic/multibytecodec.c.h index 305ade17b1f1aa..b5639d5cf10a22 100644 --- a/Modules/cjkcodecs/clinic/multibytecodec.c.h +++ b/Modules/cjkcodecs/clinic/multibytecodec.c.h @@ -668,7 +668,7 @@ _multibytecodec_MultibyteStreamWriter_reset_impl(MultibyteStreamWriterObject *se static PyObject * _multibytecodec_MultibyteStreamWriter_reset(MultibyteStreamWriterObject *self, PyTypeObject *cls, PyObject *const *args, Py_ssize_t nargs, PyObject *kwnames) { - if (nargs) { + if (nargs || (kwnames && PyTuple_GET_SIZE(kwnames))) { PyErr_SetString(PyExc_TypeError, "reset() takes no arguments"); return NULL; } @@ -682,4 +682,4 @@ PyDoc_STRVAR(_multibytecodec___create_codec__doc__, #define _MULTIBYTECODEC___CREATE_CODEC_METHODDEF \ {"__create_codec", (PyCFunction)_multibytecodec___create_codec, METH_O, _multibytecodec___create_codec__doc__}, -/*[clinic end generated code: output=219a363662d2fbff input=a9049054013a1b77]*/ +/*[clinic end generated code: output=ee767a6d93c7108a input=a9049054013a1b77]*/ diff --git a/Modules/clinic/_asynciomodule.c.h b/Modules/clinic/_asynciomodule.c.h index d941c280a4300b..6a9c8ff6d8fdd9 100644 --- a/Modules/clinic/_asynciomodule.c.h +++ b/Modules/clinic/_asynciomodule.c.h @@ -120,7 +120,7 @@ _asyncio_Future_exception_impl(FutureObj *self, PyTypeObject *cls); static PyObject * _asyncio_Future_exception(FutureObj *self, PyTypeObject *cls, PyObject *const *args, Py_ssize_t nargs, PyObject *kwnames) { - if (nargs) { + if (nargs || (kwnames && PyTuple_GET_SIZE(kwnames))) { PyErr_SetString(PyExc_TypeError, "exception() takes no arguments"); return NULL; } @@ -453,7 +453,7 @@ _asyncio_Future_get_loop_impl(FutureObj *self, PyTypeObject *cls); static PyObject * _asyncio_Future_get_loop(FutureObj *self, PyTypeObject *cls, PyObject *const *args, Py_ssize_t nargs, PyObject *kwnames) { - if (nargs) { + if (nargs || (kwnames && PyTuple_GET_SIZE(kwnames))) { PyErr_SetString(PyExc_TypeError, "get_loop() takes no arguments"); return NULL; } @@ -1487,4 +1487,4 @@ _asyncio_current_task(PyObject *module, PyObject *const *args, Py_ssize_t nargs, exit: return return_value; } -/*[clinic end generated code: output=f3864d8e2af7635f input=a9049054013a1b77]*/ +/*[clinic end generated code: output=b26155080c82c472 input=a9049054013a1b77]*/ diff --git a/Modules/clinic/_codecsmodule.c.h b/Modules/clinic/_codecsmodule.c.h index 12fea806ab5209..1c0f37442ab350 100644 --- a/Modules/clinic/_codecsmodule.c.h +++ b/Modules/clinic/_codecsmodule.c.h @@ -297,7 +297,9 @@ _codecs_escape_decode(PyObject *module, PyObject *const *args, Py_ssize_t nargs) if (ptr == NULL) { goto exit; } - PyBuffer_FillInfo(&data, args[0], (void *)ptr, len, 1, 0); + if (PyBuffer_FillInfo(&data, args[0], (void *)ptr, len, 1, PyBUF_SIMPLE) < 0) { + goto exit; + } } else { /* any bytes-like object */ if (PyObject_GetBuffer(args[0], &data, PyBUF_SIMPLE) != 0) { @@ -1099,7 +1101,9 @@ _codecs_unicode_escape_decode(PyObject *module, PyObject *const *args, Py_ssize_ if (ptr == NULL) { goto exit; } - PyBuffer_FillInfo(&data, args[0], (void *)ptr, len, 1, 0); + if (PyBuffer_FillInfo(&data, args[0], (void *)ptr, len, 1, PyBUF_SIMPLE) < 0) { + goto exit; + } } else { /* any bytes-like object */ if (PyObject_GetBuffer(args[0], &data, PyBUF_SIMPLE) != 0) { @@ -1175,7 +1179,9 @@ _codecs_raw_unicode_escape_decode(PyObject *module, PyObject *const *args, Py_ss if (ptr == NULL) { goto exit; } - PyBuffer_FillInfo(&data, args[0], (void *)ptr, len, 1, 0); + if (PyBuffer_FillInfo(&data, args[0], (void *)ptr, len, 1, PyBUF_SIMPLE) < 0) { + goto exit; + } } else { /* any bytes-like object */ if (PyObject_GetBuffer(args[0], &data, PyBUF_SIMPLE) != 0) { @@ -1644,7 +1650,9 @@ _codecs_readbuffer_encode(PyObject *module, PyObject *const *args, Py_ssize_t na if (ptr == NULL) { goto exit; } - PyBuffer_FillInfo(&data, args[0], (void *)ptr, len, 1, 0); + if (PyBuffer_FillInfo(&data, args[0], (void *)ptr, len, 1, PyBUF_SIMPLE) < 0) { + goto exit; + } } else { /* any bytes-like object */ if (PyObject_GetBuffer(args[0], &data, PyBUF_SIMPLE) != 0) { @@ -2738,4 +2746,4 @@ _codecs_lookup_error(PyObject *module, PyObject *arg) #ifndef _CODECS_CODE_PAGE_ENCODE_METHODDEF #define _CODECS_CODE_PAGE_ENCODE_METHODDEF #endif /* !defined(_CODECS_CODE_PAGE_ENCODE_METHODDEF) */ -/*[clinic end generated code: output=d8d9e372f7ccba35 input=a9049054013a1b77]*/ +/*[clinic end generated code: output=e50d5fdf65bd45fa input=a9049054013a1b77]*/ diff --git a/Modules/clinic/_collectionsmodule.c.h b/Modules/clinic/_collectionsmodule.c.h index 591ab50c76a8e8..60fb12a2231619 100644 --- a/Modules/clinic/_collectionsmodule.c.h +++ b/Modules/clinic/_collectionsmodule.c.h @@ -2,9 +2,425 @@ preserve [clinic start generated code]*/ +#if defined(Py_BUILD_CORE) && !defined(Py_BUILD_CORE_MODULE) +# include "pycore_gc.h" // PyGC_Head +# include "pycore_runtime.h" // _Py_ID() +#endif #include "pycore_abstract.h" // _PyNumber_Index() #include "pycore_modsupport.h" // _PyArg_CheckPositional() +PyDoc_STRVAR(deque_pop__doc__, +"pop($self, /)\n" +"--\n" +"\n" +"Remove and return the rightmost element."); + +#define DEQUE_POP_METHODDEF \ + {"pop", (PyCFunction)deque_pop, METH_NOARGS, deque_pop__doc__}, + +static PyObject * +deque_pop_impl(dequeobject *deque); + +static PyObject * +deque_pop(dequeobject *deque, PyObject *Py_UNUSED(ignored)) +{ + return deque_pop_impl(deque); +} + +PyDoc_STRVAR(deque_popleft__doc__, +"popleft($self, /)\n" +"--\n" +"\n" +"Remove and return the leftmost element."); + +#define DEQUE_POPLEFT_METHODDEF \ + {"popleft", (PyCFunction)deque_popleft, METH_NOARGS, deque_popleft__doc__}, + +static PyObject * +deque_popleft_impl(dequeobject *deque); + +static PyObject * +deque_popleft(dequeobject *deque, PyObject *Py_UNUSED(ignored)) +{ + return deque_popleft_impl(deque); +} + +PyDoc_STRVAR(deque_append__doc__, +"append($self, item, /)\n" +"--\n" +"\n" +"Add an element to the right side of the deque."); + +#define DEQUE_APPEND_METHODDEF \ + {"append", (PyCFunction)deque_append, METH_O, deque_append__doc__}, + +PyDoc_STRVAR(deque_appendleft__doc__, +"appendleft($self, item, /)\n" +"--\n" +"\n" +"Add an element to the left side of the deque."); + +#define DEQUE_APPENDLEFT_METHODDEF \ + {"appendleft", (PyCFunction)deque_appendleft, METH_O, deque_appendleft__doc__}, + +PyDoc_STRVAR(deque_extend__doc__, +"extend($self, iterable, /)\n" +"--\n" +"\n" +"Extend the right side of the deque with elements from the iterable."); + +#define DEQUE_EXTEND_METHODDEF \ + {"extend", (PyCFunction)deque_extend, METH_O, deque_extend__doc__}, + +PyDoc_STRVAR(deque_extendleft__doc__, +"extendleft($self, iterable, /)\n" +"--\n" +"\n" +"Extend the left side of the deque with elements from the iterable."); + +#define DEQUE_EXTENDLEFT_METHODDEF \ + {"extendleft", (PyCFunction)deque_extendleft, METH_O, deque_extendleft__doc__}, + +PyDoc_STRVAR(deque_copy__doc__, +"copy($self, /)\n" +"--\n" +"\n" +"Return a shallow copy of a deque."); + +#define DEQUE_COPY_METHODDEF \ + {"copy", (PyCFunction)deque_copy, METH_NOARGS, deque_copy__doc__}, + +static PyObject * +deque_copy_impl(dequeobject *deque); + +static PyObject * +deque_copy(dequeobject *deque, PyObject *Py_UNUSED(ignored)) +{ + return deque_copy_impl(deque); +} + +PyDoc_STRVAR(deque___copy____doc__, +"__copy__($self, /)\n" +"--\n" +"\n" +"Return a shallow copy of a deque."); + +#define DEQUE___COPY___METHODDEF \ + {"__copy__", (PyCFunction)deque___copy__, METH_NOARGS, deque___copy____doc__}, + +static PyObject * +deque___copy___impl(dequeobject *deque); + +static PyObject * +deque___copy__(dequeobject *deque, PyObject *Py_UNUSED(ignored)) +{ + return deque___copy___impl(deque); +} + +PyDoc_STRVAR(deque_clearmethod__doc__, +"clear($self, /)\n" +"--\n" +"\n" +"Remove all elements from the deque."); + +#define DEQUE_CLEARMETHOD_METHODDEF \ + {"clear", (PyCFunction)deque_clearmethod, METH_NOARGS, deque_clearmethod__doc__}, + +static PyObject * +deque_clearmethod_impl(dequeobject *deque); + +static PyObject * +deque_clearmethod(dequeobject *deque, PyObject *Py_UNUSED(ignored)) +{ + return deque_clearmethod_impl(deque); +} + +PyDoc_STRVAR(deque_rotate__doc__, +"rotate($self, n=1, /)\n" +"--\n" +"\n" +"Rotate the deque n steps to the right. If n is negative, rotates left."); + +#define DEQUE_ROTATE_METHODDEF \ + {"rotate", _PyCFunction_CAST(deque_rotate), METH_FASTCALL, deque_rotate__doc__}, + +static PyObject * +deque_rotate_impl(dequeobject *deque, Py_ssize_t n); + +static PyObject * +deque_rotate(dequeobject *deque, PyObject *const *args, Py_ssize_t nargs) +{ + PyObject *return_value = NULL; + Py_ssize_t n = 1; + + if (!_PyArg_CheckPositional("rotate", nargs, 0, 1)) { + goto exit; + } + if (nargs < 1) { + goto skip_optional; + } + { + Py_ssize_t ival = -1; + PyObject *iobj = _PyNumber_Index(args[0]); + if (iobj != NULL) { + ival = PyLong_AsSsize_t(iobj); + Py_DECREF(iobj); + } + if (ival == -1 && PyErr_Occurred()) { + goto exit; + } + n = ival; + } +skip_optional: + return_value = deque_rotate_impl(deque, n); + +exit: + return return_value; +} + +PyDoc_STRVAR(deque_reverse__doc__, +"reverse($self, /)\n" +"--\n" +"\n" +"Reverse *IN PLACE*."); + +#define DEQUE_REVERSE_METHODDEF \ + {"reverse", (PyCFunction)deque_reverse, METH_NOARGS, deque_reverse__doc__}, + +static PyObject * +deque_reverse_impl(dequeobject *deque); + +static PyObject * +deque_reverse(dequeobject *deque, PyObject *Py_UNUSED(ignored)) +{ + return deque_reverse_impl(deque); +} + +PyDoc_STRVAR(deque_count__doc__, +"count($self, value, /)\n" +"--\n" +"\n" +"Return number of occurrences of value."); + +#define DEQUE_COUNT_METHODDEF \ + {"count", (PyCFunction)deque_count, METH_O, deque_count__doc__}, + +PyDoc_STRVAR(deque_index__doc__, +"index($self, value, [start, [stop]])\n" +"--\n" +"\n" +"Return first index of value.\n" +"\n" +"Raises ValueError if the value is not present."); + +#define DEQUE_INDEX_METHODDEF \ + {"index", _PyCFunction_CAST(deque_index), METH_FASTCALL, deque_index__doc__}, + +static PyObject * +deque_index_impl(dequeobject *deque, PyObject *v, Py_ssize_t start, + Py_ssize_t stop); + +static PyObject * +deque_index(dequeobject *deque, PyObject *const *args, Py_ssize_t nargs) +{ + PyObject *return_value = NULL; + PyObject *v; + Py_ssize_t start = 0; + Py_ssize_t stop = Py_SIZE(deque); + + if (!_PyArg_CheckPositional("index", nargs, 1, 3)) { + goto exit; + } + v = args[0]; + if (nargs < 2) { + goto skip_optional; + } + if (!_PyEval_SliceIndexNotNone(args[1], &start)) { + goto exit; + } + if (nargs < 3) { + goto skip_optional; + } + if (!_PyEval_SliceIndexNotNone(args[2], &stop)) { + goto exit; + } +skip_optional: + return_value = deque_index_impl(deque, v, start, stop); + +exit: + return return_value; +} + +PyDoc_STRVAR(deque_insert__doc__, +"insert($self, index, value, /)\n" +"--\n" +"\n" +"Insert value before index."); + +#define DEQUE_INSERT_METHODDEF \ + {"insert", _PyCFunction_CAST(deque_insert), METH_FASTCALL, deque_insert__doc__}, + +static PyObject * +deque_insert_impl(dequeobject *deque, Py_ssize_t index, PyObject *value); + +static PyObject * +deque_insert(dequeobject *deque, PyObject *const *args, Py_ssize_t nargs) +{ + PyObject *return_value = NULL; + Py_ssize_t index; + PyObject *value; + + if (!_PyArg_CheckPositional("insert", nargs, 2, 2)) { + goto exit; + } + { + Py_ssize_t ival = -1; + PyObject *iobj = _PyNumber_Index(args[0]); + if (iobj != NULL) { + ival = PyLong_AsSsize_t(iobj); + Py_DECREF(iobj); + } + if (ival == -1 && PyErr_Occurred()) { + goto exit; + } + index = ival; + } + value = args[1]; + return_value = deque_insert_impl(deque, index, value); + +exit: + return return_value; +} + +PyDoc_STRVAR(deque_remove__doc__, +"remove($self, value, /)\n" +"--\n" +"\n" +"Remove first occurrence of value."); + +#define DEQUE_REMOVE_METHODDEF \ + {"remove", (PyCFunction)deque_remove, METH_O, deque_remove__doc__}, + +PyDoc_STRVAR(deque___reduce____doc__, +"__reduce__($self, /)\n" +"--\n" +"\n" +"Return state information for pickling."); + +#define DEQUE___REDUCE___METHODDEF \ + {"__reduce__", (PyCFunction)deque___reduce__, METH_NOARGS, deque___reduce____doc__}, + +static PyObject * +deque___reduce___impl(dequeobject *deque); + +static PyObject * +deque___reduce__(dequeobject *deque, PyObject *Py_UNUSED(ignored)) +{ + return deque___reduce___impl(deque); +} + +PyDoc_STRVAR(deque_init__doc__, +"deque([iterable[, maxlen]])\n" +"--\n" +"\n" +"A list-like sequence optimized for data accesses near its endpoints."); + +static int +deque_init_impl(dequeobject *deque, PyObject *iterable, PyObject *maxlenobj); + +static int +deque_init(PyObject *deque, PyObject *args, PyObject *kwargs) +{ + int return_value = -1; + #if defined(Py_BUILD_CORE) && !defined(Py_BUILD_CORE_MODULE) + + #define NUM_KEYWORDS 2 + static struct { + PyGC_Head _this_is_not_used; + PyObject_VAR_HEAD + PyObject *ob_item[NUM_KEYWORDS]; + } _kwtuple = { + .ob_base = PyVarObject_HEAD_INIT(&PyTuple_Type, NUM_KEYWORDS) + .ob_item = { &_Py_ID(iterable), &_Py_ID(maxlen), }, + }; + #undef NUM_KEYWORDS + #define KWTUPLE (&_kwtuple.ob_base.ob_base) + + #else // !Py_BUILD_CORE + # define KWTUPLE NULL + #endif // !Py_BUILD_CORE + + static const char * const _keywords[] = {"iterable", "maxlen", NULL}; + static _PyArg_Parser _parser = { + .keywords = _keywords, + .fname = "deque", + .kwtuple = KWTUPLE, + }; + #undef KWTUPLE + PyObject *argsbuf[2]; + PyObject * const *fastargs; + Py_ssize_t nargs = PyTuple_GET_SIZE(args); + Py_ssize_t noptargs = nargs + (kwargs ? PyDict_GET_SIZE(kwargs) : 0) - 0; + PyObject *iterable = NULL; + PyObject *maxlenobj = NULL; + + fastargs = _PyArg_UnpackKeywords(_PyTuple_CAST(args)->ob_item, nargs, kwargs, NULL, &_parser, 0, 2, 0, argsbuf); + if (!fastargs) { + goto exit; + } + if (!noptargs) { + goto skip_optional_pos; + } + if (fastargs[0]) { + iterable = fastargs[0]; + if (!--noptargs) { + goto skip_optional_pos; + } + } + maxlenobj = fastargs[1]; +skip_optional_pos: + return_value = deque_init_impl((dequeobject *)deque, iterable, maxlenobj); + +exit: + return return_value; +} + +PyDoc_STRVAR(deque___sizeof____doc__, +"__sizeof__($self, /)\n" +"--\n" +"\n" +"Return the size of the deque in memory, in bytes."); + +#define DEQUE___SIZEOF___METHODDEF \ + {"__sizeof__", (PyCFunction)deque___sizeof__, METH_NOARGS, deque___sizeof____doc__}, + +static PyObject * +deque___sizeof___impl(dequeobject *deque); + +static PyObject * +deque___sizeof__(dequeobject *deque, PyObject *Py_UNUSED(ignored)) +{ + return deque___sizeof___impl(deque); +} + +PyDoc_STRVAR(deque___reversed____doc__, +"__reversed__($self, /)\n" +"--\n" +"\n" +"Return a reverse iterator over the deque."); + +#define DEQUE___REVERSED___METHODDEF \ + {"__reversed__", (PyCFunction)deque___reversed__, METH_NOARGS, deque___reversed____doc__}, + +static PyObject * +deque___reversed___impl(dequeobject *deque); + +static PyObject * +deque___reversed__(dequeobject *deque, PyObject *Py_UNUSED(ignored)) +{ + return deque___reversed___impl(deque); +} + PyDoc_STRVAR(_collections__count_elements__doc__, "_count_elements($module, mapping, iterable, /)\n" "--\n" @@ -72,4 +488,4 @@ tuplegetter_new(PyTypeObject *type, PyObject *args, PyObject *kwargs) exit: return return_value; } -/*[clinic end generated code: output=c896a72f8c45930d input=a9049054013a1b77]*/ +/*[clinic end generated code: output=3633a5cbc23e8440 input=a9049054013a1b77]*/ diff --git a/Modules/clinic/_curses_panel.c.h b/Modules/clinic/_curses_panel.c.h index 7945d93b5433f7..457f71370afda9 100644 --- a/Modules/clinic/_curses_panel.c.h +++ b/Modules/clinic/_curses_panel.c.h @@ -19,7 +19,7 @@ _curses_panel_panel_bottom_impl(PyCursesPanelObject *self, PyTypeObject *cls); static PyObject * _curses_panel_panel_bottom(PyCursesPanelObject *self, PyTypeObject *cls, PyObject *const *args, Py_ssize_t nargs, PyObject *kwnames) { - if (nargs) { + if (nargs || (kwnames && PyTuple_GET_SIZE(kwnames))) { PyErr_SetString(PyExc_TypeError, "bottom() takes no arguments"); return NULL; } @@ -43,7 +43,7 @@ _curses_panel_panel_hide_impl(PyCursesPanelObject *self, PyTypeObject *cls); static PyObject * _curses_panel_panel_hide(PyCursesPanelObject *self, PyTypeObject *cls, PyObject *const *args, Py_ssize_t nargs, PyObject *kwnames) { - if (nargs) { + if (nargs || (kwnames && PyTuple_GET_SIZE(kwnames))) { PyErr_SetString(PyExc_TypeError, "hide() takes no arguments"); return NULL; } @@ -65,7 +65,7 @@ _curses_panel_panel_show_impl(PyCursesPanelObject *self, PyTypeObject *cls); static PyObject * _curses_panel_panel_show(PyCursesPanelObject *self, PyTypeObject *cls, PyObject *const *args, Py_ssize_t nargs, PyObject *kwnames) { - if (nargs) { + if (nargs || (kwnames && PyTuple_GET_SIZE(kwnames))) { PyErr_SetString(PyExc_TypeError, "show() takes no arguments"); return NULL; } @@ -87,7 +87,7 @@ _curses_panel_panel_top_impl(PyCursesPanelObject *self, PyTypeObject *cls); static PyObject * _curses_panel_panel_top(PyCursesPanelObject *self, PyTypeObject *cls, PyObject *const *args, Py_ssize_t nargs, PyObject *kwnames) { - if (nargs) { + if (nargs || (kwnames && PyTuple_GET_SIZE(kwnames))) { PyErr_SetString(PyExc_TypeError, "top() takes no arguments"); return NULL; } @@ -327,7 +327,7 @@ _curses_panel_panel_userptr_impl(PyCursesPanelObject *self, static PyObject * _curses_panel_panel_userptr(PyCursesPanelObject *self, PyTypeObject *cls, PyObject *const *args, Py_ssize_t nargs, PyObject *kwnames) { - if (nargs) { + if (nargs || (kwnames && PyTuple_GET_SIZE(kwnames))) { PyErr_SetString(PyExc_TypeError, "userptr() takes no arguments"); return NULL; } @@ -418,4 +418,4 @@ _curses_panel_update_panels(PyObject *module, PyObject *Py_UNUSED(ignored)) { return _curses_panel_update_panels_impl(module); } -/*[clinic end generated code: output=636beecf71d96ff1 input=a9049054013a1b77]*/ +/*[clinic end generated code: output=7bac14e9a1194c87 input=a9049054013a1b77]*/ diff --git a/Modules/clinic/_datetimemodule.c.h b/Modules/clinic/_datetimemodule.c.h index 1ee50fc2a13762..48499e0aaf7783 100644 --- a/Modules/clinic/_datetimemodule.c.h +++ b/Modules/clinic/_datetimemodule.c.h @@ -82,6 +82,207 @@ iso_calendar_date_new(PyTypeObject *type, PyObject *args, PyObject *kwargs) return return_value; } +PyDoc_STRVAR(datetime_date_replace__doc__, +"replace($self, /, year=unchanged, month=unchanged, day=unchanged)\n" +"--\n" +"\n" +"Return date with new specified fields."); + +#define DATETIME_DATE_REPLACE_METHODDEF \ + {"replace", _PyCFunction_CAST(datetime_date_replace), METH_FASTCALL|METH_KEYWORDS, datetime_date_replace__doc__}, + +static PyObject * +datetime_date_replace_impl(PyDateTime_Date *self, int year, int month, + int day); + +static PyObject * +datetime_date_replace(PyDateTime_Date *self, PyObject *const *args, Py_ssize_t nargs, PyObject *kwnames) +{ + PyObject *return_value = NULL; + #if defined(Py_BUILD_CORE) && !defined(Py_BUILD_CORE_MODULE) + + #define NUM_KEYWORDS 3 + static struct { + PyGC_Head _this_is_not_used; + PyObject_VAR_HEAD + PyObject *ob_item[NUM_KEYWORDS]; + } _kwtuple = { + .ob_base = PyVarObject_HEAD_INIT(&PyTuple_Type, NUM_KEYWORDS) + .ob_item = { &_Py_ID(year), &_Py_ID(month), &_Py_ID(day), }, + }; + #undef NUM_KEYWORDS + #define KWTUPLE (&_kwtuple.ob_base.ob_base) + + #else // !Py_BUILD_CORE + # define KWTUPLE NULL + #endif // !Py_BUILD_CORE + + static const char * const _keywords[] = {"year", "month", "day", NULL}; + static _PyArg_Parser _parser = { + .keywords = _keywords, + .fname = "replace", + .kwtuple = KWTUPLE, + }; + #undef KWTUPLE + PyObject *argsbuf[3]; + Py_ssize_t noptargs = nargs + (kwnames ? PyTuple_GET_SIZE(kwnames) : 0) - 0; + int year = GET_YEAR(self); + int month = GET_MONTH(self); + int day = GET_DAY(self); + + args = _PyArg_UnpackKeywords(args, nargs, NULL, kwnames, &_parser, 0, 3, 0, argsbuf); + if (!args) { + goto exit; + } + if (!noptargs) { + goto skip_optional_pos; + } + if (args[0]) { + year = PyLong_AsInt(args[0]); + if (year == -1 && PyErr_Occurred()) { + goto exit; + } + if (!--noptargs) { + goto skip_optional_pos; + } + } + if (args[1]) { + month = PyLong_AsInt(args[1]); + if (month == -1 && PyErr_Occurred()) { + goto exit; + } + if (!--noptargs) { + goto skip_optional_pos; + } + } + day = PyLong_AsInt(args[2]); + if (day == -1 && PyErr_Occurred()) { + goto exit; + } +skip_optional_pos: + return_value = datetime_date_replace_impl(self, year, month, day); + +exit: + return return_value; +} + +PyDoc_STRVAR(datetime_time_replace__doc__, +"replace($self, /, hour=unchanged, minute=unchanged, second=unchanged,\n" +" microsecond=unchanged, tzinfo=unchanged, *, fold=unchanged)\n" +"--\n" +"\n" +"Return time with new specified fields."); + +#define DATETIME_TIME_REPLACE_METHODDEF \ + {"replace", _PyCFunction_CAST(datetime_time_replace), METH_FASTCALL|METH_KEYWORDS, datetime_time_replace__doc__}, + +static PyObject * +datetime_time_replace_impl(PyDateTime_Time *self, int hour, int minute, + int second, int microsecond, PyObject *tzinfo, + int fold); + +static PyObject * +datetime_time_replace(PyDateTime_Time *self, PyObject *const *args, Py_ssize_t nargs, PyObject *kwnames) +{ + PyObject *return_value = NULL; + #if defined(Py_BUILD_CORE) && !defined(Py_BUILD_CORE_MODULE) + + #define NUM_KEYWORDS 6 + static struct { + PyGC_Head _this_is_not_used; + PyObject_VAR_HEAD + PyObject *ob_item[NUM_KEYWORDS]; + } _kwtuple = { + .ob_base = PyVarObject_HEAD_INIT(&PyTuple_Type, NUM_KEYWORDS) + .ob_item = { &_Py_ID(hour), &_Py_ID(minute), &_Py_ID(second), &_Py_ID(microsecond), &_Py_ID(tzinfo), &_Py_ID(fold), }, + }; + #undef NUM_KEYWORDS + #define KWTUPLE (&_kwtuple.ob_base.ob_base) + + #else // !Py_BUILD_CORE + # define KWTUPLE NULL + #endif // !Py_BUILD_CORE + + static const char * const _keywords[] = {"hour", "minute", "second", "microsecond", "tzinfo", "fold", NULL}; + static _PyArg_Parser _parser = { + .keywords = _keywords, + .fname = "replace", + .kwtuple = KWTUPLE, + }; + #undef KWTUPLE + PyObject *argsbuf[6]; + Py_ssize_t noptargs = nargs + (kwnames ? PyTuple_GET_SIZE(kwnames) : 0) - 0; + int hour = TIME_GET_HOUR(self); + int minute = TIME_GET_MINUTE(self); + int second = TIME_GET_SECOND(self); + int microsecond = TIME_GET_MICROSECOND(self); + PyObject *tzinfo = HASTZINFO(self) ? self->tzinfo : Py_None; + int fold = TIME_GET_FOLD(self); + + args = _PyArg_UnpackKeywords(args, nargs, NULL, kwnames, &_parser, 0, 5, 0, argsbuf); + if (!args) { + goto exit; + } + if (!noptargs) { + goto skip_optional_pos; + } + if (args[0]) { + hour = PyLong_AsInt(args[0]); + if (hour == -1 && PyErr_Occurred()) { + goto exit; + } + if (!--noptargs) { + goto skip_optional_pos; + } + } + if (args[1]) { + minute = PyLong_AsInt(args[1]); + if (minute == -1 && PyErr_Occurred()) { + goto exit; + } + if (!--noptargs) { + goto skip_optional_pos; + } + } + if (args[2]) { + second = PyLong_AsInt(args[2]); + if (second == -1 && PyErr_Occurred()) { + goto exit; + } + if (!--noptargs) { + goto skip_optional_pos; + } + } + if (args[3]) { + microsecond = PyLong_AsInt(args[3]); + if (microsecond == -1 && PyErr_Occurred()) { + goto exit; + } + if (!--noptargs) { + goto skip_optional_pos; + } + } + if (args[4]) { + tzinfo = args[4]; + if (!--noptargs) { + goto skip_optional_pos; + } + } +skip_optional_pos: + if (!noptargs) { + goto skip_optional_kwonly; + } + fold = PyLong_AsInt(args[5]); + if (fold == -1 && PyErr_Occurred()) { + goto exit; + } +skip_optional_kwonly: + return_value = datetime_time_replace_impl(self, hour, minute, second, microsecond, tzinfo, fold); + +exit: + return return_value; +} + PyDoc_STRVAR(datetime_datetime_now__doc__, "now($type, /, tz=None)\n" "--\n" @@ -146,4 +347,153 @@ datetime_datetime_now(PyTypeObject *type, PyObject *const *args, Py_ssize_t narg exit: return return_value; } -/*[clinic end generated code: output=562813dd3e164794 input=a9049054013a1b77]*/ + +PyDoc_STRVAR(datetime_datetime_replace__doc__, +"replace($self, /, year=unchanged, month=unchanged, day=unchanged,\n" +" hour=unchanged, minute=unchanged, second=unchanged,\n" +" microsecond=unchanged, tzinfo=unchanged, *, fold=unchanged)\n" +"--\n" +"\n" +"Return datetime with new specified fields."); + +#define DATETIME_DATETIME_REPLACE_METHODDEF \ + {"replace", _PyCFunction_CAST(datetime_datetime_replace), METH_FASTCALL|METH_KEYWORDS, datetime_datetime_replace__doc__}, + +static PyObject * +datetime_datetime_replace_impl(PyDateTime_DateTime *self, int year, + int month, int day, int hour, int minute, + int second, int microsecond, PyObject *tzinfo, + int fold); + +static PyObject * +datetime_datetime_replace(PyDateTime_DateTime *self, PyObject *const *args, Py_ssize_t nargs, PyObject *kwnames) +{ + PyObject *return_value = NULL; + #if defined(Py_BUILD_CORE) && !defined(Py_BUILD_CORE_MODULE) + + #define NUM_KEYWORDS 9 + static struct { + PyGC_Head _this_is_not_used; + PyObject_VAR_HEAD + PyObject *ob_item[NUM_KEYWORDS]; + } _kwtuple = { + .ob_base = PyVarObject_HEAD_INIT(&PyTuple_Type, NUM_KEYWORDS) + .ob_item = { &_Py_ID(year), &_Py_ID(month), &_Py_ID(day), &_Py_ID(hour), &_Py_ID(minute), &_Py_ID(second), &_Py_ID(microsecond), &_Py_ID(tzinfo), &_Py_ID(fold), }, + }; + #undef NUM_KEYWORDS + #define KWTUPLE (&_kwtuple.ob_base.ob_base) + + #else // !Py_BUILD_CORE + # define KWTUPLE NULL + #endif // !Py_BUILD_CORE + + static const char * const _keywords[] = {"year", "month", "day", "hour", "minute", "second", "microsecond", "tzinfo", "fold", NULL}; + static _PyArg_Parser _parser = { + .keywords = _keywords, + .fname = "replace", + .kwtuple = KWTUPLE, + }; + #undef KWTUPLE + PyObject *argsbuf[9]; + Py_ssize_t noptargs = nargs + (kwnames ? PyTuple_GET_SIZE(kwnames) : 0) - 0; + int year = GET_YEAR(self); + int month = GET_MONTH(self); + int day = GET_DAY(self); + int hour = DATE_GET_HOUR(self); + int minute = DATE_GET_MINUTE(self); + int second = DATE_GET_SECOND(self); + int microsecond = DATE_GET_MICROSECOND(self); + PyObject *tzinfo = HASTZINFO(self) ? self->tzinfo : Py_None; + int fold = DATE_GET_FOLD(self); + + args = _PyArg_UnpackKeywords(args, nargs, NULL, kwnames, &_parser, 0, 8, 0, argsbuf); + if (!args) { + goto exit; + } + if (!noptargs) { + goto skip_optional_pos; + } + if (args[0]) { + year = PyLong_AsInt(args[0]); + if (year == -1 && PyErr_Occurred()) { + goto exit; + } + if (!--noptargs) { + goto skip_optional_pos; + } + } + if (args[1]) { + month = PyLong_AsInt(args[1]); + if (month == -1 && PyErr_Occurred()) { + goto exit; + } + if (!--noptargs) { + goto skip_optional_pos; + } + } + if (args[2]) { + day = PyLong_AsInt(args[2]); + if (day == -1 && PyErr_Occurred()) { + goto exit; + } + if (!--noptargs) { + goto skip_optional_pos; + } + } + if (args[3]) { + hour = PyLong_AsInt(args[3]); + if (hour == -1 && PyErr_Occurred()) { + goto exit; + } + if (!--noptargs) { + goto skip_optional_pos; + } + } + if (args[4]) { + minute = PyLong_AsInt(args[4]); + if (minute == -1 && PyErr_Occurred()) { + goto exit; + } + if (!--noptargs) { + goto skip_optional_pos; + } + } + if (args[5]) { + second = PyLong_AsInt(args[5]); + if (second == -1 && PyErr_Occurred()) { + goto exit; + } + if (!--noptargs) { + goto skip_optional_pos; + } + } + if (args[6]) { + microsecond = PyLong_AsInt(args[6]); + if (microsecond == -1 && PyErr_Occurred()) { + goto exit; + } + if (!--noptargs) { + goto skip_optional_pos; + } + } + if (args[7]) { + tzinfo = args[7]; + if (!--noptargs) { + goto skip_optional_pos; + } + } +skip_optional_pos: + if (!noptargs) { + goto skip_optional_kwonly; + } + fold = PyLong_AsInt(args[8]); + if (fold == -1 && PyErr_Occurred()) { + goto exit; + } +skip_optional_kwonly: + return_value = datetime_datetime_replace_impl(self, year, month, day, hour, minute, second, microsecond, tzinfo, fold); + +exit: + return return_value; +} +/*[clinic end generated code: output=c7a04b865b1e0890 input=a9049054013a1b77]*/ diff --git a/Modules/clinic/_dbmmodule.c.h b/Modules/clinic/_dbmmodule.c.h index 5a4aba2825e03a..d06271e18a49b2 100644 --- a/Modules/clinic/_dbmmodule.c.h +++ b/Modules/clinic/_dbmmodule.c.h @@ -37,7 +37,7 @@ _dbm_dbm_keys_impl(dbmobject *self, PyTypeObject *cls); static PyObject * _dbm_dbm_keys(dbmobject *self, PyTypeObject *cls, PyObject *const *args, Py_ssize_t nargs, PyObject *kwnames) { - if (nargs) { + if (nargs || (kwnames && PyTuple_GET_SIZE(kwnames))) { PyErr_SetString(PyExc_TypeError, "keys() takes no arguments"); return NULL; } @@ -149,7 +149,7 @@ _dbm_dbm_clear_impl(dbmobject *self, PyTypeObject *cls); static PyObject * _dbm_dbm_clear(dbmobject *self, PyTypeObject *cls, PyObject *const *args, Py_ssize_t nargs, PyObject *kwnames) { - if (nargs) { + if (nargs || (kwnames && PyTuple_GET_SIZE(kwnames))) { PyErr_SetString(PyExc_TypeError, "clear() takes no arguments"); return NULL; } @@ -218,4 +218,4 @@ dbmopen(PyObject *module, PyObject *const *args, Py_ssize_t nargs) exit: return return_value; } -/*[clinic end generated code: output=96fdd4bd7bd256c5 input=a9049054013a1b77]*/ +/*[clinic end generated code: output=743ce0cea116747e input=a9049054013a1b77]*/ diff --git a/Modules/clinic/_elementtree.c.h b/Modules/clinic/_elementtree.c.h index 02375c8a61e73e..9622591a1aa855 100644 --- a/Modules/clinic/_elementtree.c.h +++ b/Modules/clinic/_elementtree.c.h @@ -87,7 +87,7 @@ _elementtree_Element___copy___impl(ElementObject *self, PyTypeObject *cls); static PyObject * _elementtree_Element___copy__(ElementObject *self, PyTypeObject *cls, PyObject *const *args, Py_ssize_t nargs, PyObject *kwnames) { - if (nargs) { + if (nargs || (kwnames && PyTuple_GET_SIZE(kwnames))) { PyErr_SetString(PyExc_TypeError, "__copy__() takes no arguments"); return NULL; } @@ -644,7 +644,7 @@ _elementtree_Element_itertext_impl(ElementObject *self, PyTypeObject *cls); static PyObject * _elementtree_Element_itertext(ElementObject *self, PyTypeObject *cls, PyObject *const *args, Py_ssize_t nargs, PyObject *kwnames) { - if (nargs) { + if (nargs || (kwnames && PyTuple_GET_SIZE(kwnames))) { PyErr_SetString(PyExc_TypeError, "itertext() takes no arguments"); return NULL; } @@ -1219,4 +1219,4 @@ _elementtree_XMLParser__setevents(XMLParserObject *self, PyObject *const *args, exit: return return_value; } -/*[clinic end generated code: output=8fdaa17d3262800a input=a9049054013a1b77]*/ +/*[clinic end generated code: output=218ec9e6a889f796 input=a9049054013a1b77]*/ diff --git a/Modules/clinic/_gdbmmodule.c.h b/Modules/clinic/_gdbmmodule.c.h index c7164e519d0e7d..626e4678809d4f 100644 --- a/Modules/clinic/_gdbmmodule.c.h +++ b/Modules/clinic/_gdbmmodule.c.h @@ -106,7 +106,7 @@ _gdbm_gdbm_keys_impl(gdbmobject *self, PyTypeObject *cls); static PyObject * _gdbm_gdbm_keys(gdbmobject *self, PyTypeObject *cls, PyObject *const *args, Py_ssize_t nargs, PyObject *kwnames) { - if (nargs) { + if (nargs || (kwnames && PyTuple_GET_SIZE(kwnames))) { PyErr_SetString(PyExc_TypeError, "keys() takes no arguments"); return NULL; } @@ -132,7 +132,7 @@ _gdbm_gdbm_firstkey_impl(gdbmobject *self, PyTypeObject *cls); static PyObject * _gdbm_gdbm_firstkey(gdbmobject *self, PyTypeObject *cls, PyObject *const *args, Py_ssize_t nargs, PyObject *kwnames) { - if (nargs) { + if (nargs || (kwnames && PyTuple_GET_SIZE(kwnames))) { PyErr_SetString(PyExc_TypeError, "firstkey() takes no arguments"); return NULL; } @@ -211,7 +211,7 @@ _gdbm_gdbm_reorganize_impl(gdbmobject *self, PyTypeObject *cls); static PyObject * _gdbm_gdbm_reorganize(gdbmobject *self, PyTypeObject *cls, PyObject *const *args, Py_ssize_t nargs, PyObject *kwnames) { - if (nargs) { + if (nargs || (kwnames && PyTuple_GET_SIZE(kwnames))) { PyErr_SetString(PyExc_TypeError, "reorganize() takes no arguments"); return NULL; } @@ -236,7 +236,7 @@ _gdbm_gdbm_sync_impl(gdbmobject *self, PyTypeObject *cls); static PyObject * _gdbm_gdbm_sync(gdbmobject *self, PyTypeObject *cls, PyObject *const *args, Py_ssize_t nargs, PyObject *kwnames) { - if (nargs) { + if (nargs || (kwnames && PyTuple_GET_SIZE(kwnames))) { PyErr_SetString(PyExc_TypeError, "sync() takes no arguments"); return NULL; } @@ -258,7 +258,7 @@ _gdbm_gdbm_clear_impl(gdbmobject *self, PyTypeObject *cls); static PyObject * _gdbm_gdbm_clear(gdbmobject *self, PyTypeObject *cls, PyObject *const *args, Py_ssize_t nargs, PyObject *kwnames) { - if (nargs) { + if (nargs || (kwnames && PyTuple_GET_SIZE(kwnames))) { PyErr_SetString(PyExc_TypeError, "clear() takes no arguments"); return NULL; } @@ -340,4 +340,4 @@ dbmopen(PyObject *module, PyObject *const *args, Py_ssize_t nargs) exit: return return_value; } -/*[clinic end generated code: output=c5ee922363d5a81f input=a9049054013a1b77]*/ +/*[clinic end generated code: output=6b4c19905ac9967d input=a9049054013a1b77]*/ diff --git a/Modules/clinic/_lsprof.c.h b/Modules/clinic/_lsprof.c.h index dfc003eb54774c..b3b7fda5660bfd 100644 --- a/Modules/clinic/_lsprof.c.h +++ b/Modules/clinic/_lsprof.c.h @@ -39,10 +39,10 @@ _lsprof_Profiler_getstats_impl(ProfilerObject *self, PyTypeObject *cls); static PyObject * _lsprof_Profiler_getstats(ProfilerObject *self, PyTypeObject *cls, PyObject *const *args, Py_ssize_t nargs, PyObject *kwnames) { - if (nargs) { + if (nargs || (kwnames && PyTuple_GET_SIZE(kwnames))) { PyErr_SetString(PyExc_TypeError, "getstats() takes no arguments"); return NULL; } return _lsprof_Profiler_getstats_impl(self, cls); } -/*[clinic end generated code: output=0615a53cce828f06 input=a9049054013a1b77]*/ +/*[clinic end generated code: output=5c9d87d89863dc83 input=a9049054013a1b77]*/ diff --git a/Modules/clinic/_pickle.c.h b/Modules/clinic/_pickle.c.h index 932ace190e6059..5a6ae7be6b6ea7 100644 --- a/Modules/clinic/_pickle.c.h +++ b/Modules/clinic/_pickle.c.h @@ -266,6 +266,49 @@ _pickle_PicklerMemoProxy___reduce__(PicklerMemoProxyObject *self, PyObject *Py_U return _pickle_PicklerMemoProxy___reduce___impl(self); } +PyDoc_STRVAR(_pickle_Unpickler_persistent_load__doc__, +"persistent_load($self, pid, /)\n" +"--\n" +"\n"); + +#define _PICKLE_UNPICKLER_PERSISTENT_LOAD_METHODDEF \ + {"persistent_load", _PyCFunction_CAST(_pickle_Unpickler_persistent_load), METH_METHOD|METH_FASTCALL|METH_KEYWORDS, _pickle_Unpickler_persistent_load__doc__}, + +static PyObject * +_pickle_Unpickler_persistent_load_impl(UnpicklerObject *self, + PyTypeObject *cls, PyObject *pid); + +static PyObject * +_pickle_Unpickler_persistent_load(UnpicklerObject *self, PyTypeObject *cls, PyObject *const *args, Py_ssize_t nargs, PyObject *kwnames) +{ + PyObject *return_value = NULL; + #if defined(Py_BUILD_CORE) && !defined(Py_BUILD_CORE_MODULE) + # define KWTUPLE (PyObject *)&_Py_SINGLETON(tuple_empty) + #else + # define KWTUPLE NULL + #endif + + static const char * const _keywords[] = {"", NULL}; + static _PyArg_Parser _parser = { + .keywords = _keywords, + .fname = "persistent_load", + .kwtuple = KWTUPLE, + }; + #undef KWTUPLE + PyObject *argsbuf[1]; + PyObject *pid; + + args = _PyArg_UnpackKeywords(args, nargs, NULL, kwnames, &_parser, 1, 1, 0, argsbuf); + if (!args) { + goto exit; + } + pid = args[0]; + return_value = _pickle_Unpickler_persistent_load_impl(self, cls, pid); + +exit: + return return_value; +} + PyDoc_STRVAR(_pickle_Unpickler_load__doc__, "load($self, /)\n" "--\n" @@ -285,7 +328,7 @@ _pickle_Unpickler_load_impl(UnpicklerObject *self, PyTypeObject *cls); static PyObject * _pickle_Unpickler_load(UnpicklerObject *self, PyTypeObject *cls, PyObject *const *args, Py_ssize_t nargs, PyObject *kwnames) { - if (nargs) { + if (nargs || (kwnames && PyTuple_GET_SIZE(kwnames))) { PyErr_SetString(PyExc_TypeError, "load() takes no arguments"); return NULL; } @@ -1034,4 +1077,4 @@ _pickle_loads(PyObject *module, PyObject *const *args, Py_ssize_t nargs, PyObjec exit: return return_value; } -/*[clinic end generated code: output=7f0564b5fb5410a8 input=a9049054013a1b77]*/ +/*[clinic end generated code: output=bd63c85a8737b0aa input=a9049054013a1b77]*/ diff --git a/Modules/clinic/_queuemodule.c.h b/Modules/clinic/_queuemodule.c.h index 8e2a430835e35f..6f4c715c722965 100644 --- a/Modules/clinic/_queuemodule.c.h +++ b/Modules/clinic/_queuemodule.c.h @@ -6,6 +6,7 @@ preserve # include "pycore_gc.h" // PyGC_Head # include "pycore_runtime.h" // _Py_ID() #endif +#include "pycore_critical_section.h"// Py_BEGIN_CRITICAL_SECTION() #include "pycore_modsupport.h" // _PyArg_NoKeywords() PyDoc_STRVAR(simplequeue_new__doc__, @@ -107,7 +108,9 @@ _queue_SimpleQueue_put(simplequeueobject *self, PyObject *const *args, Py_ssize_ } timeout = args[2]; skip_optional_pos: + Py_BEGIN_CRITICAL_SECTION(self); return_value = _queue_SimpleQueue_put_impl(self, item, block, timeout); + Py_END_CRITICAL_SECTION(); exit: return return_value; @@ -165,7 +168,9 @@ _queue_SimpleQueue_put_nowait(simplequeueobject *self, PyObject *const *args, Py goto exit; } item = args[0]; + Py_BEGIN_CRITICAL_SECTION(self); return_value = _queue_SimpleQueue_put_nowait_impl(self, item); + Py_END_CRITICAL_SECTION(); exit: return return_value; @@ -244,7 +249,9 @@ _queue_SimpleQueue_get(simplequeueobject *self, PyTypeObject *cls, PyObject *con } timeout_obj = args[1]; skip_optional_pos: + Py_BEGIN_CRITICAL_SECTION(self); return_value = _queue_SimpleQueue_get_impl(self, cls, block, timeout_obj); + Py_END_CRITICAL_SECTION(); exit: return return_value; @@ -269,11 +276,18 @@ _queue_SimpleQueue_get_nowait_impl(simplequeueobject *self, static PyObject * _queue_SimpleQueue_get_nowait(simplequeueobject *self, PyTypeObject *cls, PyObject *const *args, Py_ssize_t nargs, PyObject *kwnames) { - if (nargs) { + PyObject *return_value = NULL; + + if (nargs || (kwnames && PyTuple_GET_SIZE(kwnames))) { PyErr_SetString(PyExc_TypeError, "get_nowait() takes no arguments"); - return NULL; + goto exit; } - return _queue_SimpleQueue_get_nowait_impl(self, cls); + Py_BEGIN_CRITICAL_SECTION(self); + return_value = _queue_SimpleQueue_get_nowait_impl(self, cls); + Py_END_CRITICAL_SECTION(); + +exit: + return return_value; } PyDoc_STRVAR(_queue_SimpleQueue_empty__doc__, @@ -294,7 +308,9 @@ _queue_SimpleQueue_empty(simplequeueobject *self, PyObject *Py_UNUSED(ignored)) PyObject *return_value = NULL; int _return_value; + Py_BEGIN_CRITICAL_SECTION(self); _return_value = _queue_SimpleQueue_empty_impl(self); + Py_END_CRITICAL_SECTION(); if ((_return_value == -1) && PyErr_Occurred()) { goto exit; } @@ -322,7 +338,9 @@ _queue_SimpleQueue_qsize(simplequeueobject *self, PyObject *Py_UNUSED(ignored)) PyObject *return_value = NULL; Py_ssize_t _return_value; + Py_BEGIN_CRITICAL_SECTION(self); _return_value = _queue_SimpleQueue_qsize_impl(self); + Py_END_CRITICAL_SECTION(); if ((_return_value == -1) && PyErr_Occurred()) { goto exit; } @@ -331,4 +349,4 @@ _queue_SimpleQueue_qsize(simplequeueobject *self, PyObject *Py_UNUSED(ignored)) exit: return return_value; } -/*[clinic end generated code: output=457310b20cb61cf8 input=a9049054013a1b77]*/ +/*[clinic end generated code: output=44a718f40072018a input=a9049054013a1b77]*/ diff --git a/Modules/clinic/_ssl.c.h b/Modules/clinic/_ssl.c.h index 19c0f619b92f45..2940f16a2cb7f6 100644 --- a/Modules/clinic/_ssl.c.h +++ b/Modules/clinic/_ssl.c.h @@ -1297,7 +1297,9 @@ _ssl_RAND_add(PyObject *module, PyObject *const *args, Py_ssize_t nargs) if (ptr == NULL) { goto exit; } - PyBuffer_FillInfo(&view, args[0], (void *)ptr, len, 1, 0); + if (PyBuffer_FillInfo(&view, args[0], (void *)ptr, len, 1, PyBUF_SIMPLE) < 0) { + goto exit; + } } else { /* any bytes-like object */ if (PyObject_GetBuffer(args[0], &view, PyBUF_SIMPLE) != 0) { @@ -1662,4 +1664,4 @@ _ssl_enum_crls(PyObject *module, PyObject *const *args, Py_ssize_t nargs, PyObje #ifndef _SSL_ENUM_CRLS_METHODDEF #define _SSL_ENUM_CRLS_METHODDEF #endif /* !defined(_SSL_ENUM_CRLS_METHODDEF) */ -/*[clinic end generated code: output=6342ea0062ab16c7 input=a9049054013a1b77]*/ +/*[clinic end generated code: output=fd1c3378fbba5240 input=a9049054013a1b77]*/ diff --git a/Modules/clinic/_suggestions.c.h b/Modules/clinic/_suggestions.c.h new file mode 100644 index 00000000000000..51484b13d5af89 --- /dev/null +++ b/Modules/clinic/_suggestions.c.h @@ -0,0 +1,41 @@ +/*[clinic input] +preserve +[clinic start generated code]*/ + +#include "pycore_modsupport.h" // _PyArg_CheckPositional() + +PyDoc_STRVAR(_suggestions__generate_suggestions__doc__, +"_generate_suggestions($module, candidates, item, /)\n" +"--\n" +"\n" +"Returns the candidate in candidates that\'s closest to item"); + +#define _SUGGESTIONS__GENERATE_SUGGESTIONS_METHODDEF \ + {"_generate_suggestions", _PyCFunction_CAST(_suggestions__generate_suggestions), METH_FASTCALL, _suggestions__generate_suggestions__doc__}, + +static PyObject * +_suggestions__generate_suggestions_impl(PyObject *module, + PyObject *candidates, PyObject *item); + +static PyObject * +_suggestions__generate_suggestions(PyObject *module, PyObject *const *args, Py_ssize_t nargs) +{ + PyObject *return_value = NULL; + PyObject *candidates; + PyObject *item; + + if (!_PyArg_CheckPositional("_generate_suggestions", nargs, 2, 2)) { + goto exit; + } + candidates = args[0]; + if (!PyUnicode_Check(args[1])) { + _PyArg_BadArgument("_generate_suggestions", "argument 2", "str", args[1]); + goto exit; + } + item = args[1]; + return_value = _suggestions__generate_suggestions_impl(module, candidates, item); + +exit: + return return_value; +} +/*[clinic end generated code: output=1d8e963cdae30b13 input=a9049054013a1b77]*/ diff --git a/Modules/clinic/_testclinic.c.h b/Modules/clinic/_testclinic.c.h index fea30e778381de..bb516be37ec3f0 100644 --- a/Modules/clinic/_testclinic.c.h +++ b/Modules/clinic/_testclinic.c.h @@ -3141,4 +3141,26 @@ clone_with_conv_f2(PyObject *module, PyObject *const *args, Py_ssize_t nargs, Py exit: return return_value; } -/*[clinic end generated code: output=90743ac900d60f9f input=a9049054013a1b77]*/ + +PyDoc_STRVAR(_testclinic_TestClass_meth_method_no_params__doc__, +"meth_method_no_params($self, /)\n" +"--\n" +"\n"); + +#define _TESTCLINIC_TESTCLASS_METH_METHOD_NO_PARAMS_METHODDEF \ + {"meth_method_no_params", _PyCFunction_CAST(_testclinic_TestClass_meth_method_no_params), METH_METHOD|METH_FASTCALL|METH_KEYWORDS, _testclinic_TestClass_meth_method_no_params__doc__}, + +static PyObject * +_testclinic_TestClass_meth_method_no_params_impl(PyObject *self, + PyTypeObject *cls); + +static PyObject * +_testclinic_TestClass_meth_method_no_params(PyObject *self, PyTypeObject *cls, PyObject *const *args, Py_ssize_t nargs, PyObject *kwnames) +{ + if (nargs || (kwnames && PyTuple_GET_SIZE(kwnames))) { + PyErr_SetString(PyExc_TypeError, "meth_method_no_params() takes no arguments"); + return NULL; + } + return _testclinic_TestClass_meth_method_no_params_impl(self, cls); +} +/*[clinic end generated code: output=6520c1ca5392a3f0 input=a9049054013a1b77]*/ diff --git a/Modules/clinic/_testmultiphase.c.h b/Modules/clinic/_testmultiphase.c.h index c0a00954c16cbe..7ac361ece1acc3 100644 --- a/Modules/clinic/_testmultiphase.c.h +++ b/Modules/clinic/_testmultiphase.c.h @@ -27,7 +27,7 @@ _testmultiphase_StateAccessType_get_defining_module_impl(StateAccessTypeObject * static PyObject * _testmultiphase_StateAccessType_get_defining_module(StateAccessTypeObject *self, PyTypeObject *cls, PyObject *const *args, Py_ssize_t nargs, PyObject *kwnames) { - if (nargs) { + if (nargs || (kwnames && PyTuple_GET_SIZE(kwnames))) { PyErr_SetString(PyExc_TypeError, "get_defining_module() takes no arguments"); return NULL; } @@ -50,7 +50,7 @@ _testmultiphase_StateAccessType_getmodulebydef_bad_def_impl(StateAccessTypeObjec static PyObject * _testmultiphase_StateAccessType_getmodulebydef_bad_def(StateAccessTypeObject *self, PyTypeObject *cls, PyObject *const *args, Py_ssize_t nargs, PyObject *kwnames) { - if (nargs) { + if (nargs || (kwnames && PyTuple_GET_SIZE(kwnames))) { PyErr_SetString(PyExc_TypeError, "getmodulebydef_bad_def() takes no arguments"); return NULL; } @@ -156,10 +156,10 @@ _testmultiphase_StateAccessType_get_count_impl(StateAccessTypeObject *self, static PyObject * _testmultiphase_StateAccessType_get_count(StateAccessTypeObject *self, PyTypeObject *cls, PyObject *const *args, Py_ssize_t nargs, PyObject *kwnames) { - if (nargs) { + if (nargs || (kwnames && PyTuple_GET_SIZE(kwnames))) { PyErr_SetString(PyExc_TypeError, "get_count() takes no arguments"); return NULL; } return _testmultiphase_StateAccessType_get_count_impl(self, cls); } -/*[clinic end generated code: output=d8c262af27b3b98d input=a9049054013a1b77]*/ +/*[clinic end generated code: output=2c199bad52e9cda7 input=a9049054013a1b77]*/ diff --git a/Modules/clinic/_winapi.c.h b/Modules/clinic/_winapi.c.h index 3a3231c051ef71..d1052f38919dde 100644 --- a/Modules/clinic/_winapi.c.h +++ b/Modules/clinic/_winapi.c.h @@ -162,7 +162,7 @@ PyDoc_STRVAR(_winapi_CreateFile__doc__, {"CreateFile", _PyCFunction_CAST(_winapi_CreateFile), METH_FASTCALL, _winapi_CreateFile__doc__}, static HANDLE -_winapi_CreateFile_impl(PyObject *module, LPCTSTR file_name, +_winapi_CreateFile_impl(PyObject *module, LPCWSTR file_name, DWORD desired_access, DWORD share_mode, LPSECURITY_ATTRIBUTES security_attributes, DWORD creation_disposition, @@ -172,7 +172,7 @@ static PyObject * _winapi_CreateFile(PyObject *module, PyObject *const *args, Py_ssize_t nargs) { PyObject *return_value = NULL; - LPCTSTR file_name; + LPCWSTR file_name = NULL; DWORD desired_access; DWORD share_mode; LPSECURITY_ATTRIBUTES security_attributes; @@ -181,8 +181,8 @@ _winapi_CreateFile(PyObject *module, PyObject *const *args, Py_ssize_t nargs) HANDLE template_file; HANDLE _return_value; - if (!_PyArg_ParseStack(args, nargs, "skk" F_POINTER "kk" F_HANDLE ":CreateFile", - &file_name, &desired_access, &share_mode, &security_attributes, &creation_disposition, &flags_and_attributes, &template_file)) { + if (!_PyArg_ParseStack(args, nargs, "O&kk" F_POINTER "kk" F_HANDLE ":CreateFile", + _PyUnicode_WideCharString_Converter, &file_name, &desired_access, &share_mode, &security_attributes, &creation_disposition, &flags_and_attributes, &template_file)) { goto exit; } _return_value = _winapi_CreateFile_impl(module, file_name, desired_access, share_mode, security_attributes, creation_disposition, flags_and_attributes, template_file); @@ -195,6 +195,9 @@ _winapi_CreateFile(PyObject *module, PyObject *const *args, Py_ssize_t nargs) return_value = HANDLE_TO_PYNUM(_return_value); exit: + /* Cleanup for file_name */ + PyMem_Free((void *)file_name); + return return_value; } @@ -1479,4 +1482,4 @@ _winapi_CopyFile2(PyObject *module, PyObject *const *args, Py_ssize_t nargs, PyO return return_value; } -/*[clinic end generated code: output=e1a9908bb82a6379 input=a9049054013a1b77]*/ +/*[clinic end generated code: output=2350d4f2275d3a6f input=a9049054013a1b77]*/ diff --git a/Modules/clinic/arraymodule.c.h b/Modules/clinic/arraymodule.c.h index dbce0313541649..0b764e43e19437 100644 --- a/Modules/clinic/arraymodule.c.h +++ b/Modules/clinic/arraymodule.c.h @@ -652,7 +652,7 @@ array_arrayiterator___reduce___impl(arrayiterobject *self, PyTypeObject *cls); static PyObject * array_arrayiterator___reduce__(arrayiterobject *self, PyTypeObject *cls, PyObject *const *args, Py_ssize_t nargs, PyObject *kwnames) { - if (nargs) { + if (nargs || (kwnames && PyTuple_GET_SIZE(kwnames))) { PyErr_SetString(PyExc_TypeError, "__reduce__() takes no arguments"); return NULL; } @@ -667,4 +667,4 @@ PyDoc_STRVAR(array_arrayiterator___setstate____doc__, #define ARRAY_ARRAYITERATOR___SETSTATE___METHODDEF \ {"__setstate__", (PyCFunction)array_arrayiterator___setstate__, METH_O, array_arrayiterator___setstate____doc__}, -/*[clinic end generated code: output=bf086c01e7e482bf input=a9049054013a1b77]*/ +/*[clinic end generated code: output=3be987238a4bb431 input=a9049054013a1b77]*/ diff --git a/Modules/clinic/gcmodule.c.h b/Modules/clinic/gcmodule.c.h index ad4469350447cb..d50d170589a2cd 100644 --- a/Modules/clinic/gcmodule.c.h +++ b/Modules/clinic/gcmodule.c.h @@ -214,6 +214,58 @@ gc_get_debug(PyObject *module, PyObject *Py_UNUSED(ignored)) return return_value; } +PyDoc_STRVAR(gc_set_threshold__doc__, +"set_threshold(threshold0, [threshold1, [threshold2]])\n" +"Set the collection thresholds (the collection frequency).\n" +"\n" +"Setting \'threshold0\' to zero disables collection."); + +#define GC_SET_THRESHOLD_METHODDEF \ + {"set_threshold", (PyCFunction)gc_set_threshold, METH_VARARGS, gc_set_threshold__doc__}, + +static PyObject * +gc_set_threshold_impl(PyObject *module, int threshold0, int group_right_1, + int threshold1, int group_right_2, int threshold2); + +static PyObject * +gc_set_threshold(PyObject *module, PyObject *args) +{ + PyObject *return_value = NULL; + int threshold0; + int group_right_1 = 0; + int threshold1 = 0; + int group_right_2 = 0; + int threshold2 = 0; + + switch (PyTuple_GET_SIZE(args)) { + case 1: + if (!PyArg_ParseTuple(args, "i:set_threshold", &threshold0)) { + goto exit; + } + break; + case 2: + if (!PyArg_ParseTuple(args, "ii:set_threshold", &threshold0, &threshold1)) { + goto exit; + } + group_right_1 = 1; + break; + case 3: + if (!PyArg_ParseTuple(args, "iii:set_threshold", &threshold0, &threshold1, &threshold2)) { + goto exit; + } + group_right_1 = 1; + group_right_2 = 1; + break; + default: + PyErr_SetString(PyExc_TypeError, "gc.set_threshold requires 1 to 3 arguments"); + goto exit; + } + return_value = gc_set_threshold_impl(module, threshold0, group_right_1, threshold1, group_right_2, threshold2); + +exit: + return return_value; +} + PyDoc_STRVAR(gc_get_threshold__doc__, "get_threshold($module, /)\n" "--\n" @@ -250,6 +302,76 @@ gc_get_count(PyObject *module, PyObject *Py_UNUSED(ignored)) return gc_get_count_impl(module); } +PyDoc_STRVAR(gc_get_referrers__doc__, +"get_referrers($module, /, *objs)\n" +"--\n" +"\n" +"Return the list of objects that directly refer to any of \'objs\'."); + +#define GC_GET_REFERRERS_METHODDEF \ + {"get_referrers", _PyCFunction_CAST(gc_get_referrers), METH_FASTCALL, gc_get_referrers__doc__}, + +static PyObject * +gc_get_referrers_impl(PyObject *module, PyObject *args); + +static PyObject * +gc_get_referrers(PyObject *module, PyObject *const *args, Py_ssize_t nargs) +{ + PyObject *return_value = NULL; + PyObject *__clinic_args = NULL; + + if (!_PyArg_CheckPositional("get_referrers", nargs, 0, PY_SSIZE_T_MAX)) { + goto exit; + } + __clinic_args = PyTuple_New(nargs - 0); + if (!__clinic_args) { + goto exit; + } + for (Py_ssize_t i = 0; i < nargs - 0; ++i) { + PyTuple_SET_ITEM(__clinic_args, i, Py_NewRef(args[0 + i])); + } + return_value = gc_get_referrers_impl(module, __clinic_args); + +exit: + Py_XDECREF(__clinic_args); + return return_value; +} + +PyDoc_STRVAR(gc_get_referents__doc__, +"get_referents($module, /, *objs)\n" +"--\n" +"\n" +"Return the list of objects that are directly referred to by \'objs\'."); + +#define GC_GET_REFERENTS_METHODDEF \ + {"get_referents", _PyCFunction_CAST(gc_get_referents), METH_FASTCALL, gc_get_referents__doc__}, + +static PyObject * +gc_get_referents_impl(PyObject *module, PyObject *args); + +static PyObject * +gc_get_referents(PyObject *module, PyObject *const *args, Py_ssize_t nargs) +{ + PyObject *return_value = NULL; + PyObject *__clinic_args = NULL; + + if (!_PyArg_CheckPositional("get_referents", nargs, 0, PY_SSIZE_T_MAX)) { + goto exit; + } + __clinic_args = PyTuple_New(nargs - 0); + if (!__clinic_args) { + goto exit; + } + for (Py_ssize_t i = 0; i < nargs - 0; ++i) { + PyTuple_SET_ITEM(__clinic_args, i, Py_NewRef(args[0 + i])); + } + return_value = gc_get_referents_impl(module, __clinic_args); + +exit: + Py_XDECREF(__clinic_args); + return return_value; +} + PyDoc_STRVAR(gc_get_objects__doc__, "get_objects($module, /, generation=None)\n" "--\n" @@ -425,4 +547,4 @@ gc_get_freeze_count(PyObject *module, PyObject *Py_UNUSED(ignored)) exit: return return_value; } -/*[clinic end generated code: output=5c345e7b4ce6085a input=a9049054013a1b77]*/ +/*[clinic end generated code: output=258f92524c1141fc input=a9049054013a1b77]*/ diff --git a/Modules/clinic/md5module.c.h b/Modules/clinic/md5module.c.h index 7d4d3108dab9b6..ee7fb3d7c613f2 100644 --- a/Modules/clinic/md5module.c.h +++ b/Modules/clinic/md5module.c.h @@ -23,7 +23,7 @@ MD5Type_copy_impl(MD5object *self, PyTypeObject *cls); static PyObject * MD5Type_copy(MD5object *self, PyTypeObject *cls, PyObject *const *args, Py_ssize_t nargs, PyObject *kwnames) { - if (nargs) { + if (nargs || (kwnames && PyTuple_GET_SIZE(kwnames))) { PyErr_SetString(PyExc_TypeError, "copy() takes no arguments"); return NULL; } @@ -148,4 +148,4 @@ _md5_md5(PyObject *module, PyObject *const *args, Py_ssize_t nargs, PyObject *kw exit: return return_value; } -/*[clinic end generated code: output=bfadda44914804a8 input=a9049054013a1b77]*/ +/*[clinic end generated code: output=4dbca39332d3f52f input=a9049054013a1b77]*/ diff --git a/Modules/clinic/posixmodule.c.h b/Modules/clinic/posixmodule.c.h index f36872a1eb7a0f..b49d64d4281889 100644 --- a/Modules/clinic/posixmodule.c.h +++ b/Modules/clinic/posixmodule.c.h @@ -601,7 +601,7 @@ os_chmod(PyObject *module, PyObject *const *args, Py_ssize_t nargs, PyObject *kw return return_value; } -#if defined(HAVE_FCHMOD) +#if (defined(HAVE_FCHMOD) || defined(MS_WINDOWS)) PyDoc_STRVAR(os_fchmod__doc__, "fchmod($module, /, fd, mode)\n" @@ -676,7 +676,7 @@ os_fchmod(PyObject *module, PyObject *const *args, Py_ssize_t nargs, PyObject *k return return_value; } -#endif /* defined(HAVE_FCHMOD) */ +#endif /* (defined(HAVE_FCHMOD) || defined(MS_WINDOWS)) */ #if (defined(HAVE_LCHMOD) || defined(MS_WINDOWS)) @@ -4465,6 +4465,156 @@ os_sched_getaffinity(PyObject *module, PyObject *arg) #endif /* defined(HAVE_SCHED_H) && defined(HAVE_SCHED_SETAFFINITY) */ +#if defined(HAVE_POSIX_OPENPT) + +PyDoc_STRVAR(os_posix_openpt__doc__, +"posix_openpt($module, oflag, /)\n" +"--\n" +"\n" +"Open and return a file descriptor for a master pseudo-terminal device.\n" +"\n" +"Performs a posix_openpt() C function call. The oflag argument is used to\n" +"set file status flags and file access modes as specified in the manual page\n" +"of posix_openpt() of your system."); + +#define OS_POSIX_OPENPT_METHODDEF \ + {"posix_openpt", (PyCFunction)os_posix_openpt, METH_O, os_posix_openpt__doc__}, + +static int +os_posix_openpt_impl(PyObject *module, int oflag); + +static PyObject * +os_posix_openpt(PyObject *module, PyObject *arg) +{ + PyObject *return_value = NULL; + int oflag; + int _return_value; + + oflag = PyLong_AsInt(arg); + if (oflag == -1 && PyErr_Occurred()) { + goto exit; + } + _return_value = os_posix_openpt_impl(module, oflag); + if ((_return_value == -1) && PyErr_Occurred()) { + goto exit; + } + return_value = PyLong_FromLong((long)_return_value); + +exit: + return return_value; +} + +#endif /* defined(HAVE_POSIX_OPENPT) */ + +#if defined(HAVE_GRANTPT) + +PyDoc_STRVAR(os_grantpt__doc__, +"grantpt($module, fd, /)\n" +"--\n" +"\n" +"Grant access to the slave pseudo-terminal device.\n" +"\n" +" fd\n" +" File descriptor of a master pseudo-terminal device.\n" +"\n" +"Performs a grantpt() C function call."); + +#define OS_GRANTPT_METHODDEF \ + {"grantpt", (PyCFunction)os_grantpt, METH_O, os_grantpt__doc__}, + +static PyObject * +os_grantpt_impl(PyObject *module, int fd); + +static PyObject * +os_grantpt(PyObject *module, PyObject *arg) +{ + PyObject *return_value = NULL; + int fd; + + if (!_PyLong_FileDescriptor_Converter(arg, &fd)) { + goto exit; + } + return_value = os_grantpt_impl(module, fd); + +exit: + return return_value; +} + +#endif /* defined(HAVE_GRANTPT) */ + +#if defined(HAVE_UNLOCKPT) + +PyDoc_STRVAR(os_unlockpt__doc__, +"unlockpt($module, fd, /)\n" +"--\n" +"\n" +"Unlock a pseudo-terminal master/slave pair.\n" +"\n" +" fd\n" +" File descriptor of a master pseudo-terminal device.\n" +"\n" +"Performs an unlockpt() C function call."); + +#define OS_UNLOCKPT_METHODDEF \ + {"unlockpt", (PyCFunction)os_unlockpt, METH_O, os_unlockpt__doc__}, + +static PyObject * +os_unlockpt_impl(PyObject *module, int fd); + +static PyObject * +os_unlockpt(PyObject *module, PyObject *arg) +{ + PyObject *return_value = NULL; + int fd; + + if (!_PyLong_FileDescriptor_Converter(arg, &fd)) { + goto exit; + } + return_value = os_unlockpt_impl(module, fd); + +exit: + return return_value; +} + +#endif /* defined(HAVE_UNLOCKPT) */ + +#if (defined(HAVE_PTSNAME) || defined(HAVE_PTSNAME_R)) + +PyDoc_STRVAR(os_ptsname__doc__, +"ptsname($module, fd, /)\n" +"--\n" +"\n" +"Return the name of the slave pseudo-terminal device.\n" +"\n" +" fd\n" +" File descriptor of a master pseudo-terminal device.\n" +"\n" +"If the ptsname_r() C function is available, it is called;\n" +"otherwise, performs a ptsname() C function call."); + +#define OS_PTSNAME_METHODDEF \ + {"ptsname", (PyCFunction)os_ptsname, METH_O, os_ptsname__doc__}, + +static PyObject * +os_ptsname_impl(PyObject *module, int fd); + +static PyObject * +os_ptsname(PyObject *module, PyObject *arg) +{ + PyObject *return_value = NULL; + int fd; + + if (!_PyLong_FileDescriptor_Converter(arg, &fd)) { + goto exit; + } + return_value = os_ptsname_impl(module, fd); + +exit: + return return_value; +} + +#endif /* (defined(HAVE_PTSNAME) || defined(HAVE_PTSNAME_R)) */ + #if (defined(HAVE_OPENPTY) || defined(HAVE__GETPTY) || defined(HAVE_DEV_PTMX)) PyDoc_STRVAR(os_openpty__doc__, @@ -5467,7 +5617,7 @@ os_wait4(PyObject *module, PyObject *const *args, Py_ssize_t nargs, PyObject *kw #endif /* defined(HAVE_WAIT4) */ -#if (defined(HAVE_WAITID) && !defined(__APPLE__)) +#if defined(HAVE_WAITID) PyDoc_STRVAR(os_waitid__doc__, "waitid($module, idtype, id, options, /)\n" @@ -5510,7 +5660,7 @@ os_waitid(PyObject *module, PyObject *const *args, Py_ssize_t nargs) return return_value; } -#endif /* (defined(HAVE_WAITID) && !defined(__APPLE__)) */ +#endif /* defined(HAVE_WAITID) */ #if defined(HAVE_WAITPID) @@ -11062,7 +11212,7 @@ os_DirEntry_is_symlink(DirEntry *self, PyTypeObject *defining_class, PyObject *c PyObject *return_value = NULL; int _return_value; - if (nargs) { + if (nargs || (kwnames && PyTuple_GET_SIZE(kwnames))) { PyErr_SetString(PyExc_TypeError, "is_symlink() takes no arguments"); goto exit; } @@ -11991,6 +12141,22 @@ os__supports_virtual_terminal(PyObject *module, PyObject *Py_UNUSED(ignored)) #define OS_SCHED_GETAFFINITY_METHODDEF #endif /* !defined(OS_SCHED_GETAFFINITY_METHODDEF) */ +#ifndef OS_POSIX_OPENPT_METHODDEF + #define OS_POSIX_OPENPT_METHODDEF +#endif /* !defined(OS_POSIX_OPENPT_METHODDEF) */ + +#ifndef OS_GRANTPT_METHODDEF + #define OS_GRANTPT_METHODDEF +#endif /* !defined(OS_GRANTPT_METHODDEF) */ + +#ifndef OS_UNLOCKPT_METHODDEF + #define OS_UNLOCKPT_METHODDEF +#endif /* !defined(OS_UNLOCKPT_METHODDEF) */ + +#ifndef OS_PTSNAME_METHODDEF + #define OS_PTSNAME_METHODDEF +#endif /* !defined(OS_PTSNAME_METHODDEF) */ + #ifndef OS_OPENPTY_METHODDEF #define OS_OPENPTY_METHODDEF #endif /* !defined(OS_OPENPTY_METHODDEF) */ @@ -12422,4 +12588,4 @@ os__supports_virtual_terminal(PyObject *module, PyObject *Py_UNUSED(ignored)) #ifndef OS__SUPPORTS_VIRTUAL_TERMINAL_METHODDEF #define OS__SUPPORTS_VIRTUAL_TERMINAL_METHODDEF #endif /* !defined(OS__SUPPORTS_VIRTUAL_TERMINAL_METHODDEF) */ -/*[clinic end generated code: output=1be15e60a553b40d input=a9049054013a1b77]*/ +/*[clinic end generated code: output=268af5cbc8baa9d4 input=a9049054013a1b77]*/ diff --git a/Modules/clinic/sha1module.c.h b/Modules/clinic/sha1module.c.h index ee391656fb67c3..b89c7e505c788e 100644 --- a/Modules/clinic/sha1module.c.h +++ b/Modules/clinic/sha1module.c.h @@ -23,7 +23,7 @@ SHA1Type_copy_impl(SHA1object *self, PyTypeObject *cls); static PyObject * SHA1Type_copy(SHA1object *self, PyTypeObject *cls, PyObject *const *args, Py_ssize_t nargs, PyObject *kwnames) { - if (nargs) { + if (nargs || (kwnames && PyTuple_GET_SIZE(kwnames))) { PyErr_SetString(PyExc_TypeError, "copy() takes no arguments"); return NULL; } @@ -148,4 +148,4 @@ _sha1_sha1(PyObject *module, PyObject *const *args, Py_ssize_t nargs, PyObject * exit: return return_value; } -/*[clinic end generated code: output=41fc7579213b57b4 input=a9049054013a1b77]*/ +/*[clinic end generated code: output=af5a640df662066f input=a9049054013a1b77]*/ diff --git a/Modules/clinic/sha2module.c.h b/Modules/clinic/sha2module.c.h index ec31d5545be4c1..cf4b88d52856b8 100644 --- a/Modules/clinic/sha2module.c.h +++ b/Modules/clinic/sha2module.c.h @@ -23,7 +23,7 @@ SHA256Type_copy_impl(SHA256object *self, PyTypeObject *cls); static PyObject * SHA256Type_copy(SHA256object *self, PyTypeObject *cls, PyObject *const *args, Py_ssize_t nargs, PyObject *kwnames) { - if (nargs) { + if (nargs || (kwnames && PyTuple_GET_SIZE(kwnames))) { PyErr_SetString(PyExc_TypeError, "copy() takes no arguments"); return NULL; } @@ -45,7 +45,7 @@ SHA512Type_copy_impl(SHA512object *self, PyTypeObject *cls); static PyObject * SHA512Type_copy(SHA512object *self, PyTypeObject *cls, PyObject *const *args, Py_ssize_t nargs, PyObject *kwnames) { - if (nargs) { + if (nargs || (kwnames && PyTuple_GET_SIZE(kwnames))) { PyErr_SetString(PyExc_TypeError, "copy() takes no arguments"); return NULL; } @@ -437,4 +437,4 @@ _sha2_sha384(PyObject *module, PyObject *const *args, Py_ssize_t nargs, PyObject exit: return return_value; } -/*[clinic end generated code: output=1482d9de086e45c4 input=a9049054013a1b77]*/ +/*[clinic end generated code: output=b46da764024b1764 input=a9049054013a1b77]*/ diff --git a/Modules/clinic/zlibmodule.c.h b/Modules/clinic/zlibmodule.c.h index 6b09abe309bf48..7ff3edf5a557f8 100644 --- a/Modules/clinic/zlibmodule.c.h +++ b/Modules/clinic/zlibmodule.c.h @@ -637,7 +637,7 @@ zlib_Compress_copy_impl(compobject *self, PyTypeObject *cls); static PyObject * zlib_Compress_copy(compobject *self, PyTypeObject *cls, PyObject *const *args, Py_ssize_t nargs, PyObject *kwnames) { - if (nargs) { + if (nargs || (kwnames && PyTuple_GET_SIZE(kwnames))) { PyErr_SetString(PyExc_TypeError, "copy() takes no arguments"); return NULL; } @@ -662,7 +662,7 @@ zlib_Compress___copy___impl(compobject *self, PyTypeObject *cls); static PyObject * zlib_Compress___copy__(compobject *self, PyTypeObject *cls, PyObject *const *args, Py_ssize_t nargs, PyObject *kwnames) { - if (nargs) { + if (nargs || (kwnames && PyTuple_GET_SIZE(kwnames))) { PyErr_SetString(PyExc_TypeError, "__copy__() takes no arguments"); return NULL; } @@ -735,7 +735,7 @@ zlib_Decompress_copy_impl(compobject *self, PyTypeObject *cls); static PyObject * zlib_Decompress_copy(compobject *self, PyTypeObject *cls, PyObject *const *args, Py_ssize_t nargs, PyObject *kwnames) { - if (nargs) { + if (nargs || (kwnames && PyTuple_GET_SIZE(kwnames))) { PyErr_SetString(PyExc_TypeError, "copy() takes no arguments"); return NULL; } @@ -760,7 +760,7 @@ zlib_Decompress___copy___impl(compobject *self, PyTypeObject *cls); static PyObject * zlib_Decompress___copy__(compobject *self, PyTypeObject *cls, PyObject *const *args, Py_ssize_t nargs, PyObject *kwnames) { - if (nargs) { + if (nargs || (kwnames && PyTuple_GET_SIZE(kwnames))) { PyErr_SetString(PyExc_TypeError, "__copy__() takes no arguments"); return NULL; } @@ -1098,4 +1098,4 @@ zlib_crc32(PyObject *module, PyObject *const *args, Py_ssize_t nargs) #ifndef ZLIB_DECOMPRESS___DEEPCOPY___METHODDEF #define ZLIB_DECOMPRESS___DEEPCOPY___METHODDEF #endif /* !defined(ZLIB_DECOMPRESS___DEEPCOPY___METHODDEF) */ -/*[clinic end generated code: output=6dd97dc851c39031 input=a9049054013a1b77]*/ +/*[clinic end generated code: output=8bb840fb6af43dd4 input=a9049054013a1b77]*/ diff --git a/Modules/faulthandler.c b/Modules/faulthandler.c index a2e3c2300b3ce8..95d646c9c65b3c 100644 --- a/Modules/faulthandler.c +++ b/Modules/faulthandler.c @@ -119,6 +119,13 @@ faulthandler_get_fileno(PyObject **file_ptr) } } else if (PyLong_Check(file)) { + if (PyBool_Check(file)) { + if (PyErr_WarnEx(PyExc_RuntimeWarning, + "bool is used as a file descriptor", 1)) + { + return -1; + } + } fd = PyLong_AsInt(file); if (fd == -1 && PyErr_Occurred()) return -1; diff --git a/Modules/fcntlmodule.c b/Modules/fcntlmodule.c index fd03abf0561da6..0d16602692b62d 100644 --- a/Modules/fcntlmodule.c +++ b/Modules/fcntlmodule.c @@ -583,6 +583,30 @@ all_ins(PyObject* m) #ifdef FICLONERANGE if (PyModule_AddIntMacro(m, FICLONERANGE)) return -1; #endif +#ifdef F_GETOWN_EX + // since Linux 2.6.32 + if (PyModule_AddIntMacro(m, F_GETOWN_EX)) return -1; + if (PyModule_AddIntMacro(m, F_SETOWN_EX)) return -1; + if (PyModule_AddIntMacro(m, F_OWNER_TID)) return -1; + if (PyModule_AddIntMacro(m, F_OWNER_PID)) return -1; + if (PyModule_AddIntMacro(m, F_OWNER_PGRP)) return -1; +#endif +#ifdef F_GET_RW_HINT + // since Linux 4.13 + if (PyModule_AddIntMacro(m, F_GET_RW_HINT)) return -1; + if (PyModule_AddIntMacro(m, F_SET_RW_HINT)) return -1; + if (PyModule_AddIntMacro(m, F_GET_FILE_RW_HINT)) return -1; + if (PyModule_AddIntMacro(m, F_SET_FILE_RW_HINT)) return -1; +#ifndef RWH_WRITE_LIFE_NOT_SET // typo in Linux < 5.5 +# define RWH_WRITE_LIFE_NOT_SET RWF_WRITE_LIFE_NOT_SET +#endif + if (PyModule_AddIntMacro(m, RWH_WRITE_LIFE_NOT_SET)) return -1; + if (PyModule_AddIntMacro(m, RWH_WRITE_LIFE_NONE)) return -1; + if (PyModule_AddIntMacro(m, RWH_WRITE_LIFE_SHORT)) return -1; + if (PyModule_AddIntMacro(m, RWH_WRITE_LIFE_MEDIUM)) return -1; + if (PyModule_AddIntMacro(m, RWH_WRITE_LIFE_LONG)) return -1; + if (PyModule_AddIntMacro(m, RWH_WRITE_LIFE_EXTREME)) return -1; +#endif /* OS X specifics */ #ifdef F_FULLFSYNC @@ -599,6 +623,32 @@ all_ins(PyObject* m) #ifdef F_DUP2FD_CLOEXEC if (PyModule_AddIntMacro(m, F_DUP2FD_CLOEXEC)) return -1; #endif +#ifdef F_READAHEAD + if (PyModule_AddIntMacro(m, F_READAHEAD)) return -1; +#endif +#ifdef F_RDAHEAD + if (PyModule_AddIntMacro(m, F_RDAHEAD)) return -1; +#endif +#ifdef F_ISUNIONSTACK + if (PyModule_AddIntMacro(m, F_ISUNIONSTACK)) return -1; +#endif +#ifdef F_KINFO + if (PyModule_AddIntMacro(m, F_KINFO)) return -1; +#endif + +/* NetBSD specifics */ +#ifdef F_CLOSEM + if (PyModule_AddIntMacro(m, F_CLOSEM)) return -1; +#endif +#ifdef F_MAXFD + if (PyModule_AddIntMacro(m, F_MAXFD)) return -1; +#endif +#ifdef F_GETNOSIGPIPE + if (PyModule_AddIntMacro(m, F_GETNOSIGPIPE)) return -1; +#endif +#ifdef F_SETNOSIGPIPE + if (PyModule_AddIntMacro(m, F_SETNOSIGPIPE)) return -1; +#endif /* For F_{GET|SET}FL */ #ifdef FD_CLOEXEC @@ -673,6 +723,9 @@ all_ins(PyObject* m) if (PyModule_AddIntMacro(m, F_SEAL_SHRINK)) return -1; if (PyModule_AddIntMacro(m, F_SEAL_GROW)) return -1; if (PyModule_AddIntMacro(m, F_SEAL_WRITE)) return -1; +#ifdef F_SEAL_FUTURE_WRITE + if (PyModule_AddIntMacro(m, F_SEAL_FUTURE_WRITE)) return -1; +#endif #endif return 0; } diff --git a/Modules/gcmodule.c b/Modules/gcmodule.c index 2d1f381e622226..a2b66b9b78c169 100644 --- a/Modules/gcmodule.c +++ b/Modules/gcmodule.c @@ -1,1530 +1,27 @@ /* + * Python interface to the garbage collector. + * + * See Python/gc.c for the implementation of the garbage collector. + */ - Reference Cycle Garbage Collection - ================================== - - Neil Schemenauer - - Based on a post on the python-dev list. Ideas from Guido van Rossum, - Eric Tiedemann, and various others. - - http://www.arctrix.com/nas/python/gc/ - - The following mailing list threads provide a historical perspective on - the design of this module. Note that a fair amount of refinement has - occurred since those discussions. - - http://mail.python.org/pipermail/python-dev/2000-March/002385.html - http://mail.python.org/pipermail/python-dev/2000-March/002434.html - http://mail.python.org/pipermail/python-dev/2000-March/002497.html - - For a highlevel view of the collection process, read the collect - function. - -*/ - -#include "Python.h" -#include "pycore_ceval.h" // _Py_set_eval_breaker_bit() -#include "pycore_context.h" -#include "pycore_dict.h" // _PyDict_MaybeUntrack() -#include "pycore_initconfig.h" -#include "pycore_interp.h" // PyInterpreterState.gc -#include "pycore_object.h" -#include "pycore_pyerrors.h" -#include "pycore_pystate.h" // _PyThreadState_GET() -#include "pycore_weakref.h" // _PyWeakref_ClearRef() -#include "pydtrace.h" - -typedef struct _gc_runtime_state GCState; - -/*[clinic input] -module gc -[clinic start generated code]*/ -/*[clinic end generated code: output=da39a3ee5e6b4b0d input=b5c9690ecc842d79]*/ - - -#ifdef Py_DEBUG -# define GC_DEBUG -#endif - -#define GC_NEXT _PyGCHead_NEXT -#define GC_PREV _PyGCHead_PREV - -// update_refs() set this bit for all objects in current generation. -// subtract_refs() and move_unreachable() uses this to distinguish -// visited object is in GCing or not. -// -// move_unreachable() removes this flag from reachable objects. -// Only unreachable objects have this flag. -// -// No objects in interpreter have this flag after GC ends. -#define PREV_MASK_COLLECTING _PyGC_PREV_MASK_COLLECTING - -// Lowest bit of _gc_next is used for UNREACHABLE flag. -// -// This flag represents the object is in unreachable list in move_unreachable() -// -// Although this flag is used only in move_unreachable(), move_unreachable() -// doesn't clear this flag to skip unnecessary iteration. -// move_legacy_finalizers() removes this flag instead. -// Between them, unreachable list is not normal list and we can not use -// most gc_list_* functions for it. -#define NEXT_MASK_UNREACHABLE (1) - -#define AS_GC(op) _Py_AS_GC(op) -#define FROM_GC(gc) _Py_FROM_GC(gc) - -// Automatically choose the generation that needs collecting. -#define GENERATION_AUTO (-1) - -typedef enum { - // GC was triggered by heap allocation - _Py_GC_REASON_HEAP, - - // GC was called during shutdown - _Py_GC_REASON_SHUTDOWN, - - // GC was called by gc.collect() or PyGC_Collect() - _Py_GC_REASON_MANUAL -} _PyGC_Reason; - - -static inline int -gc_is_collecting(PyGC_Head *g) -{ - return (g->_gc_prev & PREV_MASK_COLLECTING) != 0; -} - -static inline void -gc_clear_collecting(PyGC_Head *g) -{ - g->_gc_prev &= ~PREV_MASK_COLLECTING; -} - -static inline Py_ssize_t -gc_get_refs(PyGC_Head *g) -{ - return (Py_ssize_t)(g->_gc_prev >> _PyGC_PREV_SHIFT); -} - -static inline void -gc_set_refs(PyGC_Head *g, Py_ssize_t refs) -{ - g->_gc_prev = (g->_gc_prev & ~_PyGC_PREV_MASK) - | ((uintptr_t)(refs) << _PyGC_PREV_SHIFT); -} - -static inline void -gc_reset_refs(PyGC_Head *g, Py_ssize_t refs) -{ - g->_gc_prev = (g->_gc_prev & _PyGC_PREV_MASK_FINALIZED) - | PREV_MASK_COLLECTING - | ((uintptr_t)(refs) << _PyGC_PREV_SHIFT); -} - -static inline void -gc_decref(PyGC_Head *g) -{ - _PyObject_ASSERT_WITH_MSG(FROM_GC(g), - gc_get_refs(g) > 0, - "refcount is too small"); - g->_gc_prev -= 1 << _PyGC_PREV_SHIFT; -} - -/* set for debugging information */ -#define DEBUG_STATS (1<<0) /* print collection statistics */ -#define DEBUG_COLLECTABLE (1<<1) /* print collectable objects */ -#define DEBUG_UNCOLLECTABLE (1<<2) /* print uncollectable objects */ -#define DEBUG_SAVEALL (1<<5) /* save all garbage in gc.garbage */ -#define DEBUG_LEAK DEBUG_COLLECTABLE | \ - DEBUG_UNCOLLECTABLE | \ - DEBUG_SAVEALL - -#define GEN_HEAD(gcstate, n) (&(gcstate)->generations[n].head) - - -static GCState * -get_gc_state(void) -{ - PyInterpreterState *interp = _PyInterpreterState_GET(); - return &interp->gc; -} - - -void -_PyGC_InitState(GCState *gcstate) -{ -#define INIT_HEAD(GEN) \ - do { \ - GEN.head._gc_next = (uintptr_t)&GEN.head; \ - GEN.head._gc_prev = (uintptr_t)&GEN.head; \ - } while (0) - - for (int i = 0; i < NUM_GENERATIONS; i++) { - assert(gcstate->generations[i].count == 0); - INIT_HEAD(gcstate->generations[i]); - }; - gcstate->generation0 = GEN_HEAD(gcstate, 0); - INIT_HEAD(gcstate->permanent_generation); - -#undef INIT_HEAD -} - - -PyStatus -_PyGC_Init(PyInterpreterState *interp) -{ - GCState *gcstate = &interp->gc; - - gcstate->garbage = PyList_New(0); - if (gcstate->garbage == NULL) { - return _PyStatus_NO_MEMORY(); - } - - gcstate->callbacks = PyList_New(0); - if (gcstate->callbacks == NULL) { - return _PyStatus_NO_MEMORY(); - } - - return _PyStatus_OK(); -} - - -/* -_gc_prev values ---------------- - -Between collections, _gc_prev is used for doubly linked list. - -Lowest two bits of _gc_prev are used for flags. -PREV_MASK_COLLECTING is used only while collecting and cleared before GC ends -or _PyObject_GC_UNTRACK() is called. - -During a collection, _gc_prev is temporary used for gc_refs, and the gc list -is singly linked until _gc_prev is restored. - -gc_refs - At the start of a collection, update_refs() copies the true refcount - to gc_refs, for each object in the generation being collected. - subtract_refs() then adjusts gc_refs so that it equals the number of - times an object is referenced directly from outside the generation - being collected. - -PREV_MASK_COLLECTING - Objects in generation being collected are marked PREV_MASK_COLLECTING in - update_refs(). - - -_gc_next values ---------------- - -_gc_next takes these values: - -0 - The object is not tracked - -!= 0 - Pointer to the next object in the GC list. - Additionally, lowest bit is used temporary for - NEXT_MASK_UNREACHABLE flag described below. - -NEXT_MASK_UNREACHABLE - move_unreachable() then moves objects not reachable (whether directly or - indirectly) from outside the generation into an "unreachable" set and - set this flag. - - Objects that are found to be reachable have gc_refs set to 1. - When this flag is set for the reachable object, the object must be in - "unreachable" set. - The flag is unset and the object is moved back to "reachable" set. - - move_legacy_finalizers() will remove this flag from "unreachable" set. -*/ - -/*** list functions ***/ - -static inline void -gc_list_init(PyGC_Head *list) -{ - // List header must not have flags. - // We can assign pointer by simple cast. - list->_gc_prev = (uintptr_t)list; - list->_gc_next = (uintptr_t)list; -} - -static inline int -gc_list_is_empty(PyGC_Head *list) -{ - return (list->_gc_next == (uintptr_t)list); -} - -/* Append `node` to `list`. */ -static inline void -gc_list_append(PyGC_Head *node, PyGC_Head *list) -{ - PyGC_Head *last = (PyGC_Head *)list->_gc_prev; - - // last <-> node - _PyGCHead_SET_PREV(node, last); - _PyGCHead_SET_NEXT(last, node); - - // node <-> list - _PyGCHead_SET_NEXT(node, list); - list->_gc_prev = (uintptr_t)node; -} - -/* Remove `node` from the gc list it's currently in. */ -static inline void -gc_list_remove(PyGC_Head *node) -{ - PyGC_Head *prev = GC_PREV(node); - PyGC_Head *next = GC_NEXT(node); - - _PyGCHead_SET_NEXT(prev, next); - _PyGCHead_SET_PREV(next, prev); - - node->_gc_next = 0; /* object is not currently tracked */ -} - -/* Move `node` from the gc list it's currently in (which is not explicitly - * named here) to the end of `list`. This is semantically the same as - * gc_list_remove(node) followed by gc_list_append(node, list). - */ -static void -gc_list_move(PyGC_Head *node, PyGC_Head *list) -{ - /* Unlink from current list. */ - PyGC_Head *from_prev = GC_PREV(node); - PyGC_Head *from_next = GC_NEXT(node); - _PyGCHead_SET_NEXT(from_prev, from_next); - _PyGCHead_SET_PREV(from_next, from_prev); - - /* Relink at end of new list. */ - // list must not have flags. So we can skip macros. - PyGC_Head *to_prev = (PyGC_Head*)list->_gc_prev; - _PyGCHead_SET_PREV(node, to_prev); - _PyGCHead_SET_NEXT(to_prev, node); - list->_gc_prev = (uintptr_t)node; - _PyGCHead_SET_NEXT(node, list); -} - -/* append list `from` onto list `to`; `from` becomes an empty list */ -static void -gc_list_merge(PyGC_Head *from, PyGC_Head *to) -{ - assert(from != to); - if (!gc_list_is_empty(from)) { - PyGC_Head *to_tail = GC_PREV(to); - PyGC_Head *from_head = GC_NEXT(from); - PyGC_Head *from_tail = GC_PREV(from); - assert(from_head != from); - assert(from_tail != from); - - _PyGCHead_SET_NEXT(to_tail, from_head); - _PyGCHead_SET_PREV(from_head, to_tail); - - _PyGCHead_SET_NEXT(from_tail, to); - _PyGCHead_SET_PREV(to, from_tail); - } - gc_list_init(from); -} - -static Py_ssize_t -gc_list_size(PyGC_Head *list) -{ - PyGC_Head *gc; - Py_ssize_t n = 0; - for (gc = GC_NEXT(list); gc != list; gc = GC_NEXT(gc)) { - n++; - } - return n; -} - -/* Walk the list and mark all objects as non-collecting */ -static inline void -gc_list_clear_collecting(PyGC_Head *collectable) -{ - PyGC_Head *gc; - for (gc = GC_NEXT(collectable); gc != collectable; gc = GC_NEXT(gc)) { - gc_clear_collecting(gc); - } -} - -/* Append objects in a GC list to a Python list. - * Return 0 if all OK, < 0 if error (out of memory for list) - */ -static int -append_objects(PyObject *py_list, PyGC_Head *gc_list) -{ - PyGC_Head *gc; - for (gc = GC_NEXT(gc_list); gc != gc_list; gc = GC_NEXT(gc)) { - PyObject *op = FROM_GC(gc); - if (op != py_list) { - if (PyList_Append(py_list, op)) { - return -1; /* exception */ - } - } - } - return 0; -} - -// Constants for validate_list's flags argument. -enum flagstates {collecting_clear_unreachable_clear, - collecting_clear_unreachable_set, - collecting_set_unreachable_clear, - collecting_set_unreachable_set}; - -#ifdef GC_DEBUG -// validate_list checks list consistency. And it works as document -// describing when flags are expected to be set / unset. -// `head` must be a doubly-linked gc list, although it's fine (expected!) if -// the prev and next pointers are "polluted" with flags. -// What's checked: -// - The `head` pointers are not polluted. -// - The objects' PREV_MASK_COLLECTING and NEXT_MASK_UNREACHABLE flags are all -// `set or clear, as specified by the 'flags' argument. -// - The prev and next pointers are mutually consistent. -static void -validate_list(PyGC_Head *head, enum flagstates flags) -{ - assert((head->_gc_prev & PREV_MASK_COLLECTING) == 0); - assert((head->_gc_next & NEXT_MASK_UNREACHABLE) == 0); - uintptr_t prev_value = 0, next_value = 0; - switch (flags) { - case collecting_clear_unreachable_clear: - break; - case collecting_set_unreachable_clear: - prev_value = PREV_MASK_COLLECTING; - break; - case collecting_clear_unreachable_set: - next_value = NEXT_MASK_UNREACHABLE; - break; - case collecting_set_unreachable_set: - prev_value = PREV_MASK_COLLECTING; - next_value = NEXT_MASK_UNREACHABLE; - break; - default: - assert(! "bad internal flags argument"); - } - PyGC_Head *prev = head; - PyGC_Head *gc = GC_NEXT(head); - while (gc != head) { - PyGC_Head *trueprev = GC_PREV(gc); - PyGC_Head *truenext = (PyGC_Head *)(gc->_gc_next & ~NEXT_MASK_UNREACHABLE); - assert(truenext != NULL); - assert(trueprev == prev); - assert((gc->_gc_prev & PREV_MASK_COLLECTING) == prev_value); - assert((gc->_gc_next & NEXT_MASK_UNREACHABLE) == next_value); - prev = gc; - gc = truenext; - } - assert(prev == GC_PREV(head)); -} -#else -#define validate_list(x, y) do{}while(0) -#endif - -/*** end of list stuff ***/ - - -/* Set all gc_refs = ob_refcnt. After this, gc_refs is > 0 and - * PREV_MASK_COLLECTING bit is set for all objects in containers. - */ -static void -update_refs(PyGC_Head *containers) -{ - PyGC_Head *next; - PyGC_Head *gc = GC_NEXT(containers); - - while (gc != containers) { - next = GC_NEXT(gc); - /* Move any object that might have become immortal to the - * permanent generation as the reference count is not accurately - * reflecting the actual number of live references to this object - */ - if (_Py_IsImmortal(FROM_GC(gc))) { - gc_list_move(gc, &get_gc_state()->permanent_generation.head); - gc = next; - continue; - } - gc_reset_refs(gc, Py_REFCNT(FROM_GC(gc))); - /* Python's cyclic gc should never see an incoming refcount - * of 0: if something decref'ed to 0, it should have been - * deallocated immediately at that time. - * Possible cause (if the assert triggers): a tp_dealloc - * routine left a gc-aware object tracked during its teardown - * phase, and did something-- or allowed something to happen -- - * that called back into Python. gc can trigger then, and may - * see the still-tracked dying object. Before this assert - * was added, such mistakes went on to allow gc to try to - * delete the object again. In a debug build, that caused - * a mysterious segfault, when _Py_ForgetReference tried - * to remove the object from the doubly-linked list of all - * objects a second time. In a release build, an actual - * double deallocation occurred, which leads to corruption - * of the allocator's internal bookkeeping pointers. That's - * so serious that maybe this should be a release-build - * check instead of an assert? - */ - _PyObject_ASSERT(FROM_GC(gc), gc_get_refs(gc) != 0); - gc = next; - } -} - -/* A traversal callback for subtract_refs. */ -static int -visit_decref(PyObject *op, void *parent) -{ - OBJECT_STAT_INC(object_visits); - _PyObject_ASSERT(_PyObject_CAST(parent), !_PyObject_IsFreed(op)); - - if (_PyObject_IS_GC(op)) { - PyGC_Head *gc = AS_GC(op); - /* We're only interested in gc_refs for objects in the - * generation being collected, which can be recognized - * because only they have positive gc_refs. - */ - if (gc_is_collecting(gc)) { - gc_decref(gc); - } - } - return 0; -} - -/* Subtract internal references from gc_refs. After this, gc_refs is >= 0 - * for all objects in containers, and is GC_REACHABLE for all tracked gc - * objects not in containers. The ones with gc_refs > 0 are directly - * reachable from outside containers, and so can't be collected. - */ -static void -subtract_refs(PyGC_Head *containers) -{ - traverseproc traverse; - PyGC_Head *gc = GC_NEXT(containers); - for (; gc != containers; gc = GC_NEXT(gc)) { - PyObject *op = FROM_GC(gc); - traverse = Py_TYPE(op)->tp_traverse; - (void) traverse(op, - visit_decref, - op); - } -} - -/* A traversal callback for move_unreachable. */ -static int -visit_reachable(PyObject *op, void *arg) -{ - PyGC_Head *reachable = arg; - OBJECT_STAT_INC(object_visits); - if (!_PyObject_IS_GC(op)) { - return 0; - } - - PyGC_Head *gc = AS_GC(op); - const Py_ssize_t gc_refs = gc_get_refs(gc); - - // Ignore objects in other generation. - // This also skips objects "to the left" of the current position in - // move_unreachable's scan of the 'young' list - they've already been - // traversed, and no longer have the PREV_MASK_COLLECTING flag. - if (! gc_is_collecting(gc)) { - return 0; - } - // It would be a logic error elsewhere if the collecting flag were set on - // an untracked object. - assert(gc->_gc_next != 0); - - if (gc->_gc_next & NEXT_MASK_UNREACHABLE) { - /* This had gc_refs = 0 when move_unreachable got - * to it, but turns out it's reachable after all. - * Move it back to move_unreachable's 'young' list, - * and move_unreachable will eventually get to it - * again. - */ - // Manually unlink gc from unreachable list because the list functions - // don't work right in the presence of NEXT_MASK_UNREACHABLE flags. - PyGC_Head *prev = GC_PREV(gc); - PyGC_Head *next = (PyGC_Head*)(gc->_gc_next & ~NEXT_MASK_UNREACHABLE); - _PyObject_ASSERT(FROM_GC(prev), - prev->_gc_next & NEXT_MASK_UNREACHABLE); - _PyObject_ASSERT(FROM_GC(next), - next->_gc_next & NEXT_MASK_UNREACHABLE); - prev->_gc_next = gc->_gc_next; // copy NEXT_MASK_UNREACHABLE - _PyGCHead_SET_PREV(next, prev); - - gc_list_append(gc, reachable); - gc_set_refs(gc, 1); - } - else if (gc_refs == 0) { - /* This is in move_unreachable's 'young' list, but - * the traversal hasn't yet gotten to it. All - * we need to do is tell move_unreachable that it's - * reachable. - */ - gc_set_refs(gc, 1); - } - /* Else there's nothing to do. - * If gc_refs > 0, it must be in move_unreachable's 'young' - * list, and move_unreachable will eventually get to it. - */ - else { - _PyObject_ASSERT_WITH_MSG(op, gc_refs > 0, "refcount is too small"); - } - return 0; -} - -/* Move the unreachable objects from young to unreachable. After this, - * all objects in young don't have PREV_MASK_COLLECTING flag and - * unreachable have the flag. - * All objects in young after this are directly or indirectly reachable - * from outside the original young; and all objects in unreachable are - * not. - * - * This function restores _gc_prev pointer. young and unreachable are - * doubly linked list after this function. - * But _gc_next in unreachable list has NEXT_MASK_UNREACHABLE flag. - * So we can not gc_list_* functions for unreachable until we remove the flag. - */ -static void -move_unreachable(PyGC_Head *young, PyGC_Head *unreachable) -{ - // previous elem in the young list, used for restore gc_prev. - PyGC_Head *prev = young; - PyGC_Head *gc = GC_NEXT(young); - - /* Invariants: all objects "to the left" of us in young are reachable - * (directly or indirectly) from outside the young list as it was at entry. - * - * All other objects from the original young "to the left" of us are in - * unreachable now, and have NEXT_MASK_UNREACHABLE. All objects to the - * left of us in 'young' now have been scanned, and no objects here - * or to the right have been scanned yet. - */ - - while (gc != young) { - if (gc_get_refs(gc)) { - /* gc is definitely reachable from outside the - * original 'young'. Mark it as such, and traverse - * its pointers to find any other objects that may - * be directly reachable from it. Note that the - * call to tp_traverse may append objects to young, - * so we have to wait until it returns to determine - * the next object to visit. - */ - PyObject *op = FROM_GC(gc); - traverseproc traverse = Py_TYPE(op)->tp_traverse; - _PyObject_ASSERT_WITH_MSG(op, gc_get_refs(gc) > 0, - "refcount is too small"); - // NOTE: visit_reachable may change gc->_gc_next when - // young->_gc_prev == gc. Don't do gc = GC_NEXT(gc) before! - (void) traverse(op, - visit_reachable, - (void *)young); - // relink gc_prev to prev element. - _PyGCHead_SET_PREV(gc, prev); - // gc is not COLLECTING state after here. - gc_clear_collecting(gc); - prev = gc; - } - else { - /* This *may* be unreachable. To make progress, - * assume it is. gc isn't directly reachable from - * any object we've already traversed, but may be - * reachable from an object we haven't gotten to yet. - * visit_reachable will eventually move gc back into - * young if that's so, and we'll see it again. - */ - // Move gc to unreachable. - // No need to gc->next->prev = prev because it is single linked. - prev->_gc_next = gc->_gc_next; - - // We can't use gc_list_append() here because we use - // NEXT_MASK_UNREACHABLE here. - PyGC_Head *last = GC_PREV(unreachable); - // NOTE: Since all objects in unreachable set has - // NEXT_MASK_UNREACHABLE flag, we set it unconditionally. - // But this may pollute the unreachable list head's 'next' pointer - // too. That's semantically senseless but expedient here - the - // damage is repaired when this function ends. - last->_gc_next = (NEXT_MASK_UNREACHABLE | (uintptr_t)gc); - _PyGCHead_SET_PREV(gc, last); - gc->_gc_next = (NEXT_MASK_UNREACHABLE | (uintptr_t)unreachable); - unreachable->_gc_prev = (uintptr_t)gc; - } - gc = (PyGC_Head*)prev->_gc_next; - } - // young->_gc_prev must be last element remained in the list. - young->_gc_prev = (uintptr_t)prev; - // don't let the pollution of the list head's next pointer leak - unreachable->_gc_next &= ~NEXT_MASK_UNREACHABLE; -} - -static void -untrack_tuples(PyGC_Head *head) -{ - PyGC_Head *next, *gc = GC_NEXT(head); - while (gc != head) { - PyObject *op = FROM_GC(gc); - next = GC_NEXT(gc); - if (PyTuple_CheckExact(op)) { - _PyTuple_MaybeUntrack(op); - } - gc = next; - } -} - -/* Try to untrack all currently tracked dictionaries */ -static void -untrack_dicts(PyGC_Head *head) -{ - PyGC_Head *next, *gc = GC_NEXT(head); - while (gc != head) { - PyObject *op = FROM_GC(gc); - next = GC_NEXT(gc); - if (PyDict_CheckExact(op)) { - _PyDict_MaybeUntrack(op); - } - gc = next; - } -} - -/* Return true if object has a pre-PEP 442 finalization method. */ -static int -has_legacy_finalizer(PyObject *op) -{ - return Py_TYPE(op)->tp_del != NULL; -} - -/* Move the objects in unreachable with tp_del slots into `finalizers`. - * - * This function also removes NEXT_MASK_UNREACHABLE flag - * from _gc_next in unreachable. - */ -static void -move_legacy_finalizers(PyGC_Head *unreachable, PyGC_Head *finalizers) -{ - PyGC_Head *gc, *next; - assert((unreachable->_gc_next & NEXT_MASK_UNREACHABLE) == 0); - - /* March over unreachable. Move objects with finalizers into - * `finalizers`. - */ - for (gc = GC_NEXT(unreachable); gc != unreachable; gc = next) { - PyObject *op = FROM_GC(gc); - - _PyObject_ASSERT(op, gc->_gc_next & NEXT_MASK_UNREACHABLE); - gc->_gc_next &= ~NEXT_MASK_UNREACHABLE; - next = (PyGC_Head*)gc->_gc_next; - - if (has_legacy_finalizer(op)) { - gc_clear_collecting(gc); - gc_list_move(gc, finalizers); - } - } -} - -static inline void -clear_unreachable_mask(PyGC_Head *unreachable) -{ - /* Check that the list head does not have the unreachable bit set */ - assert(((uintptr_t)unreachable & NEXT_MASK_UNREACHABLE) == 0); - - PyGC_Head *gc, *next; - assert((unreachable->_gc_next & NEXT_MASK_UNREACHABLE) == 0); - for (gc = GC_NEXT(unreachable); gc != unreachable; gc = next) { - _PyObject_ASSERT((PyObject*)FROM_GC(gc), gc->_gc_next & NEXT_MASK_UNREACHABLE); - gc->_gc_next &= ~NEXT_MASK_UNREACHABLE; - next = (PyGC_Head*)gc->_gc_next; - } - validate_list(unreachable, collecting_set_unreachable_clear); -} - -/* A traversal callback for move_legacy_finalizer_reachable. */ -static int -visit_move(PyObject *op, void *arg) -{ - PyGC_Head *tolist = arg; - OBJECT_STAT_INC(object_visits); - if (_PyObject_IS_GC(op)) { - PyGC_Head *gc = AS_GC(op); - if (gc_is_collecting(gc)) { - gc_list_move(gc, tolist); - gc_clear_collecting(gc); - } - } - return 0; -} - -/* Move objects that are reachable from finalizers, from the unreachable set - * into finalizers set. - */ -static void -move_legacy_finalizer_reachable(PyGC_Head *finalizers) -{ - traverseproc traverse; - PyGC_Head *gc = GC_NEXT(finalizers); - for (; gc != finalizers; gc = GC_NEXT(gc)) { - /* Note that the finalizers list may grow during this. */ - traverse = Py_TYPE(FROM_GC(gc))->tp_traverse; - (void) traverse(FROM_GC(gc), - visit_move, - (void *)finalizers); - } -} - -/* Clear all weakrefs to unreachable objects, and if such a weakref has a - * callback, invoke it if necessary. Note that it's possible for such - * weakrefs to be outside the unreachable set -- indeed, those are precisely - * the weakrefs whose callbacks must be invoked. See gc_weakref.txt for - * overview & some details. Some weakrefs with callbacks may be reclaimed - * directly by this routine; the number reclaimed is the return value. Other - * weakrefs with callbacks may be moved into the `old` generation. Objects - * moved into `old` have gc_refs set to GC_REACHABLE; the objects remaining in - * unreachable are left at GC_TENTATIVELY_UNREACHABLE. When this returns, - * no object in `unreachable` is weakly referenced anymore. - */ -static int -handle_weakrefs(PyGC_Head *unreachable, PyGC_Head *old) -{ - PyGC_Head *gc; - PyObject *op; /* generally FROM_GC(gc) */ - PyWeakReference *wr; /* generally a cast of op */ - PyGC_Head wrcb_to_call; /* weakrefs with callbacks to call */ - PyGC_Head *next; - int num_freed = 0; - - gc_list_init(&wrcb_to_call); - - /* Clear all weakrefs to the objects in unreachable. If such a weakref - * also has a callback, move it into `wrcb_to_call` if the callback - * needs to be invoked. Note that we cannot invoke any callbacks until - * all weakrefs to unreachable objects are cleared, lest the callback - * resurrect an unreachable object via a still-active weakref. We - * make another pass over wrcb_to_call, invoking callbacks, after this - * pass completes. - */ - for (gc = GC_NEXT(unreachable); gc != unreachable; gc = next) { - PyWeakReference **wrlist; - - op = FROM_GC(gc); - next = GC_NEXT(gc); - - if (PyWeakref_Check(op)) { - /* A weakref inside the unreachable set must be cleared. If we - * allow its callback to execute inside delete_garbage(), it - * could expose objects that have tp_clear already called on - * them. Or, it could resurrect unreachable objects. One way - * this can happen is if some container objects do not implement - * tp_traverse. Then, wr_object can be outside the unreachable - * set but can be deallocated as a result of breaking the - * reference cycle. If we don't clear the weakref, the callback - * will run and potentially cause a crash. See bpo-38006 for - * one example. - */ - _PyWeakref_ClearRef((PyWeakReference *)op); - } - - if (! _PyType_SUPPORTS_WEAKREFS(Py_TYPE(op))) - continue; - - /* It supports weakrefs. Does it have any? - * - * This is never triggered for static types so we can avoid the - * (slightly) more costly _PyObject_GET_WEAKREFS_LISTPTR(). - */ - wrlist = _PyObject_GET_WEAKREFS_LISTPTR_FROM_OFFSET(op); - - /* `op` may have some weakrefs. March over the list, clear - * all the weakrefs, and move the weakrefs with callbacks - * that must be called into wrcb_to_call. - */ - for (wr = *wrlist; wr != NULL; wr = *wrlist) { - PyGC_Head *wrasgc; /* AS_GC(wr) */ - - /* _PyWeakref_ClearRef clears the weakref but leaves - * the callback pointer intact. Obscure: it also - * changes *wrlist. - */ - _PyObject_ASSERT((PyObject *)wr, wr->wr_object == op); - _PyWeakref_ClearRef(wr); - _PyObject_ASSERT((PyObject *)wr, wr->wr_object == Py_None); - if (wr->wr_callback == NULL) { - /* no callback */ - continue; - } - - /* Headache time. `op` is going away, and is weakly referenced by - * `wr`, which has a callback. Should the callback be invoked? If wr - * is also trash, no: - * - * 1. There's no need to call it. The object and the weakref are - * both going away, so it's legitimate to pretend the weakref is - * going away first. The user has to ensure a weakref outlives its - * referent if they want a guarantee that the wr callback will get - * invoked. - * - * 2. It may be catastrophic to call it. If the callback is also in - * cyclic trash (CT), then although the CT is unreachable from - * outside the current generation, CT may be reachable from the - * callback. Then the callback could resurrect insane objects. - * - * Since the callback is never needed and may be unsafe in this case, - * wr is simply left in the unreachable set. Note that because we - * already called _PyWeakref_ClearRef(wr), its callback will never - * trigger. - * - * OTOH, if wr isn't part of CT, we should invoke the callback: the - * weakref outlived the trash. Note that since wr isn't CT in this - * case, its callback can't be CT either -- wr acted as an external - * root to this generation, and therefore its callback did too. So - * nothing in CT is reachable from the callback either, so it's hard - * to imagine how calling it later could create a problem for us. wr - * is moved to wrcb_to_call in this case. - */ - if (gc_is_collecting(AS_GC((PyObject *)wr))) { - /* it should already have been cleared above */ - assert(wr->wr_object == Py_None); - continue; - } - - /* Create a new reference so that wr can't go away - * before we can process it again. - */ - Py_INCREF(wr); - - /* Move wr to wrcb_to_call, for the next pass. */ - wrasgc = AS_GC((PyObject *)wr); - assert(wrasgc != next); /* wrasgc is reachable, but - next isn't, so they can't - be the same */ - gc_list_move(wrasgc, &wrcb_to_call); - } - } - - /* Invoke the callbacks we decided to honor. It's safe to invoke them - * because they can't reference unreachable objects. - */ - while (! gc_list_is_empty(&wrcb_to_call)) { - PyObject *temp; - PyObject *callback; - - gc = (PyGC_Head*)wrcb_to_call._gc_next; - op = FROM_GC(gc); - _PyObject_ASSERT(op, PyWeakref_Check(op)); - wr = (PyWeakReference *)op; - callback = wr->wr_callback; - _PyObject_ASSERT(op, callback != NULL); - - /* copy-paste of weakrefobject.c's handle_callback() */ - temp = PyObject_CallOneArg(callback, (PyObject *)wr); - if (temp == NULL) - PyErr_WriteUnraisable(callback); - else - Py_DECREF(temp); - - /* Give up the reference we created in the first pass. When - * op's refcount hits 0 (which it may or may not do right now), - * op's tp_dealloc will decref op->wr_callback too. Note - * that the refcount probably will hit 0 now, and because this - * weakref was reachable to begin with, gc didn't already - * add it to its count of freed objects. Example: a reachable - * weak value dict maps some key to this reachable weakref. - * The callback removes this key->weakref mapping from the - * dict, leaving no other references to the weakref (excepting - * ours). - */ - Py_DECREF(op); - if (wrcb_to_call._gc_next == (uintptr_t)gc) { - /* object is still alive -- move it */ - gc_list_move(gc, old); - } - else { - ++num_freed; - } - } - - return num_freed; -} - -static void -debug_cycle(const char *msg, PyObject *op) -{ - PySys_FormatStderr("gc: %s <%s %p>\n", - msg, Py_TYPE(op)->tp_name, op); -} - -/* Handle uncollectable garbage (cycles with tp_del slots, and stuff reachable - * only from such cycles). - * If DEBUG_SAVEALL, all objects in finalizers are appended to the module - * garbage list (a Python list), else only the objects in finalizers with - * __del__ methods are appended to garbage. All objects in finalizers are - * merged into the old list regardless. - */ -static void -handle_legacy_finalizers(PyThreadState *tstate, - GCState *gcstate, - PyGC_Head *finalizers, PyGC_Head *old) -{ - assert(!_PyErr_Occurred(tstate)); - assert(gcstate->garbage != NULL); - - PyGC_Head *gc = GC_NEXT(finalizers); - for (; gc != finalizers; gc = GC_NEXT(gc)) { - PyObject *op = FROM_GC(gc); - - if ((gcstate->debug & DEBUG_SAVEALL) || has_legacy_finalizer(op)) { - if (PyList_Append(gcstate->garbage, op) < 0) { - _PyErr_Clear(tstate); - break; - } - } - } - - gc_list_merge(finalizers, old); -} - -/* Run first-time finalizers (if any) on all the objects in collectable. - * Note that this may remove some (or even all) of the objects from the - * list, due to refcounts falling to 0. - */ -static void -finalize_garbage(PyThreadState *tstate, PyGC_Head *collectable) -{ - destructor finalize; - PyGC_Head seen; - - /* While we're going through the loop, `finalize(op)` may cause op, or - * other objects, to be reclaimed via refcounts falling to zero. So - * there's little we can rely on about the structure of the input - * `collectable` list across iterations. For safety, we always take the - * first object in that list and move it to a temporary `seen` list. - * If objects vanish from the `collectable` and `seen` lists we don't - * care. - */ - gc_list_init(&seen); - - while (!gc_list_is_empty(collectable)) { - PyGC_Head *gc = GC_NEXT(collectable); - PyObject *op = FROM_GC(gc); - gc_list_move(gc, &seen); - if (!_PyGCHead_FINALIZED(gc) && - (finalize = Py_TYPE(op)->tp_finalize) != NULL) { - _PyGCHead_SET_FINALIZED(gc); - Py_INCREF(op); - finalize(op); - assert(!_PyErr_Occurred(tstate)); - Py_DECREF(op); - } - } - gc_list_merge(&seen, collectable); -} - -/* Break reference cycles by clearing the containers involved. This is - * tricky business as the lists can be changing and we don't know which - * objects may be freed. It is possible I screwed something up here. - */ -static void -delete_garbage(PyThreadState *tstate, GCState *gcstate, - PyGC_Head *collectable, PyGC_Head *old) -{ - assert(!_PyErr_Occurred(tstate)); - - while (!gc_list_is_empty(collectable)) { - PyGC_Head *gc = GC_NEXT(collectable); - PyObject *op = FROM_GC(gc); - - _PyObject_ASSERT_WITH_MSG(op, Py_REFCNT(op) > 0, - "refcount is too small"); - - if (gcstate->debug & DEBUG_SAVEALL) { - assert(gcstate->garbage != NULL); - if (PyList_Append(gcstate->garbage, op) < 0) { - _PyErr_Clear(tstate); - } - } - else { - inquiry clear; - if ((clear = Py_TYPE(op)->tp_clear) != NULL) { - Py_INCREF(op); - (void) clear(op); - if (_PyErr_Occurred(tstate)) { - PyErr_FormatUnraisable("Exception ignored in tp_clear of %s", - Py_TYPE(op)->tp_name); - } - Py_DECREF(op); - } - } - if (GC_NEXT(collectable) == gc) { - /* object is still alive, move it, it may die later */ - gc_clear_collecting(gc); - gc_list_move(gc, old); - } - } -} - -/* Clear all free lists - * All free lists are cleared during the collection of the highest generation. - * Allocated items in the free list may keep a pymalloc arena occupied. - * Clearing the free lists may give back memory to the OS earlier. - */ -static void -clear_freelists(PyInterpreterState *interp) -{ - _PyTuple_ClearFreeList(interp); - _PyFloat_ClearFreeList(interp); - _PyList_ClearFreeList(interp); - _PyDict_ClearFreeList(interp); - _PyAsyncGen_ClearFreeLists(interp); - _PyContext_ClearFreeList(interp); -} - -// Show stats for objects in each generations -static void -show_stats_each_generations(GCState *gcstate) -{ - char buf[100]; - size_t pos = 0; - - for (int i = 0; i < NUM_GENERATIONS && pos < sizeof(buf); i++) { - pos += PyOS_snprintf(buf+pos, sizeof(buf)-pos, - " %zd", - gc_list_size(GEN_HEAD(gcstate, i))); - } - - PySys_FormatStderr( - "gc: objects in each generation:%s\n" - "gc: objects in permanent generation: %zd\n", - buf, gc_list_size(&gcstate->permanent_generation.head)); -} - -/* Deduce which objects among "base" are unreachable from outside the list - and move them to 'unreachable'. The process consist in the following steps: - -1. Copy all reference counts to a different field (gc_prev is used to hold - this copy to save memory). -2. Traverse all objects in "base" and visit all referred objects using - "tp_traverse" and for every visited object, subtract 1 to the reference - count (the one that we copied in the previous step). After this step, all - objects that can be reached directly from outside must have strictly positive - reference count, while all unreachable objects must have a count of exactly 0. -3. Identify all unreachable objects (the ones with 0 reference count) and move - them to the "unreachable" list. This step also needs to move back to "base" all - objects that were initially marked as unreachable but are referred transitively - by the reachable objects (the ones with strictly positive reference count). - -Contracts: - - * The "base" has to be a valid list with no mask set. - - * The "unreachable" list must be uninitialized (this function calls - gc_list_init over 'unreachable'). - -IMPORTANT: This function leaves 'unreachable' with the NEXT_MASK_UNREACHABLE -flag set but it does not clear it to skip unnecessary iteration. Before the -flag is cleared (for example, by using 'clear_unreachable_mask' function or -by a call to 'move_legacy_finalizers'), the 'unreachable' list is not a normal -list and we can not use most gc_list_* functions for it. */ -static inline void -deduce_unreachable(PyGC_Head *base, PyGC_Head *unreachable) { - validate_list(base, collecting_clear_unreachable_clear); - /* Using ob_refcnt and gc_refs, calculate which objects in the - * container set are reachable from outside the set (i.e., have a - * refcount greater than 0 when all the references within the - * set are taken into account). - */ - update_refs(base); // gc_prev is used for gc_refs - subtract_refs(base); - - /* Leave everything reachable from outside base in base, and move - * everything else (in base) to unreachable. - * - * NOTE: This used to move the reachable objects into a reachable - * set instead. But most things usually turn out to be reachable, - * so it's more efficient to move the unreachable things. It "sounds slick" - * to move the unreachable objects, until you think about it - the reason it - * pays isn't actually obvious. - * - * Suppose we create objects A, B, C in that order. They appear in the young - * generation in the same order. If B points to A, and C to B, and C is - * reachable from outside, then the adjusted refcounts will be 0, 0, and 1 - * respectively. - * - * When move_unreachable finds A, A is moved to the unreachable list. The - * same for B when it's first encountered. Then C is traversed, B is moved - * _back_ to the reachable list. B is eventually traversed, and then A is - * moved back to the reachable list. - * - * So instead of not moving at all, the reachable objects B and A are moved - * twice each. Why is this a win? A straightforward algorithm to move the - * reachable objects instead would move A, B, and C once each. - * - * The key is that this dance leaves the objects in order C, B, A - it's - * reversed from the original order. On all _subsequent_ scans, none of - * them will move. Since most objects aren't in cycles, this can save an - * unbounded number of moves across an unbounded number of later collections. - * It can cost more only the first time the chain is scanned. - * - * Drawback: move_unreachable is also used to find out what's still trash - * after finalizers may resurrect objects. In _that_ case most unreachable - * objects will remain unreachable, so it would be more efficient to move - * the reachable objects instead. But this is a one-time cost, probably not - * worth complicating the code to speed just a little. - */ - gc_list_init(unreachable); - move_unreachable(base, unreachable); // gc_prev is pointer again - validate_list(base, collecting_clear_unreachable_clear); - validate_list(unreachable, collecting_set_unreachable_set); -} - -/* Handle objects that may have resurrected after a call to 'finalize_garbage', moving - them to 'old_generation' and placing the rest on 'still_unreachable'. - - Contracts: - * After this function 'unreachable' must not be used anymore and 'still_unreachable' - will contain the objects that did not resurrect. - - * The "still_unreachable" list must be uninitialized (this function calls - gc_list_init over 'still_unreachable'). - -IMPORTANT: After a call to this function, the 'still_unreachable' set will have the -PREV_MARK_COLLECTING set, but the objects in this set are going to be removed so -we can skip the expense of clearing the flag to avoid extra iteration. */ -static inline void -handle_resurrected_objects(PyGC_Head *unreachable, PyGC_Head* still_unreachable, - PyGC_Head *old_generation) -{ - // Remove the PREV_MASK_COLLECTING from unreachable - // to prepare it for a new call to 'deduce_unreachable' - gc_list_clear_collecting(unreachable); - - // After the call to deduce_unreachable, the 'still_unreachable' set will - // have the PREV_MARK_COLLECTING set, but the objects are going to be - // removed so we can skip the expense of clearing the flag. - PyGC_Head* resurrected = unreachable; - deduce_unreachable(resurrected, still_unreachable); - clear_unreachable_mask(still_unreachable); - - // Move the resurrected objects to the old generation for future collection. - gc_list_merge(resurrected, old_generation); -} - - -/* Invoke progress callbacks to notify clients that garbage collection - * is starting or stopping - */ -static void -invoke_gc_callback(PyThreadState *tstate, const char *phase, - int generation, Py_ssize_t collected, - Py_ssize_t uncollectable) -{ - assert(!_PyErr_Occurred(tstate)); - - /* we may get called very early */ - GCState *gcstate = &tstate->interp->gc; - if (gcstate->callbacks == NULL) { - return; - } - - /* The local variable cannot be rebound, check it for sanity */ - assert(PyList_CheckExact(gcstate->callbacks)); - PyObject *info = NULL; - if (PyList_GET_SIZE(gcstate->callbacks) != 0) { - info = Py_BuildValue("{sisnsn}", - "generation", generation, - "collected", collected, - "uncollectable", uncollectable); - if (info == NULL) { - PyErr_FormatUnraisable("Exception ignored on invoking gc callbacks"); - return; - } - } - - PyObject *phase_obj = PyUnicode_FromString(phase); - if (phase_obj == NULL) { - Py_XDECREF(info); - PyErr_FormatUnraisable("Exception ignored on invoking gc callbacks"); - return; - } - - PyObject *stack[] = {phase_obj, info}; - for (Py_ssize_t i=0; icallbacks); i++) { - PyObject *r, *cb = PyList_GET_ITEM(gcstate->callbacks, i); - Py_INCREF(cb); /* make sure cb doesn't go away */ - r = PyObject_Vectorcall(cb, stack, 2, NULL); - if (r == NULL) { - PyErr_WriteUnraisable(cb); - } - else { - Py_DECREF(r); - } - Py_DECREF(cb); - } - Py_DECREF(phase_obj); - Py_XDECREF(info); - assert(!_PyErr_Occurred(tstate)); -} - - -/* Find the oldest generation (highest numbered) where the count - * exceeds the threshold. Objects in the that generation and - * generations younger than it will be collected. */ -static int -gc_select_generation(GCState *gcstate) -{ - for (int i = NUM_GENERATIONS-1; i >= 0; i--) { - if (gcstate->generations[i].count > gcstate->generations[i].threshold) { - /* Avoid quadratic performance degradation in number - of tracked objects (see also issue #4074): - - To limit the cost of garbage collection, there are two strategies; - - make each collection faster, e.g. by scanning fewer objects - - do less collections - This heuristic is about the latter strategy. - - In addition to the various configurable thresholds, we only trigger a - full collection if the ratio - - long_lived_pending / long_lived_total - - is above a given value (hardwired to 25%). - - The reason is that, while "non-full" collections (i.e., collections of - the young and middle generations) will always examine roughly the same - number of objects -- determined by the aforementioned thresholds --, - the cost of a full collection is proportional to the total number of - long-lived objects, which is virtually unbounded. - - Indeed, it has been remarked that doing a full collection every - of object creations entails a dramatic performance - degradation in workloads which consist in creating and storing lots of - long-lived objects (e.g. building a large list of GC-tracked objects would - show quadratic performance, instead of linear as expected: see issue #4074). - - Using the above ratio, instead, yields amortized linear performance in - the total number of objects (the effect of which can be summarized - thusly: "each full garbage collection is more and more costly as the - number of objects grows, but we do fewer and fewer of them"). - - This heuristic was suggested by Martin von Löwis on python-dev in - June 2008. His original analysis and proposal can be found at: - http://mail.python.org/pipermail/python-dev/2008-June/080579.html - */ - if (i == NUM_GENERATIONS - 1 - && gcstate->long_lived_pending < gcstate->long_lived_total / 4) - continue; - return i; - } - } - return -1; -} - - -/* This is the main function. Read this to understand how the - * collection process works. */ -static Py_ssize_t -gc_collect_main(PyThreadState *tstate, int generation, _PyGC_Reason reason) -{ - int i; - Py_ssize_t m = 0; /* # objects collected */ - Py_ssize_t n = 0; /* # unreachable objects that couldn't be collected */ - PyGC_Head *young; /* the generation we are examining */ - PyGC_Head *old; /* next older generation */ - PyGC_Head unreachable; /* non-problematic unreachable trash */ - PyGC_Head finalizers; /* objects with, & reachable from, __del__ */ - PyGC_Head *gc; - _PyTime_t t1 = 0; /* initialize to prevent a compiler warning */ - GCState *gcstate = &tstate->interp->gc; - - // gc_collect_main() must not be called before _PyGC_Init - // or after _PyGC_Fini() - assert(gcstate->garbage != NULL); - assert(!_PyErr_Occurred(tstate)); - - int expected = 0; - if (!_Py_atomic_compare_exchange_int(&gcstate->collecting, &expected, 1)) { - // Don't start a garbage collection if one is already in progress. - return 0; - } - - if (generation == GENERATION_AUTO) { - // Select the oldest generation that needs collecting. We will collect - // objects from that generation and all generations younger than it. - generation = gc_select_generation(gcstate); - if (generation < 0) { - // No generation needs to be collected. - _Py_atomic_store_int(&gcstate->collecting, 0); - return 0; - } - } - - assert(generation >= 0 && generation < NUM_GENERATIONS); - -#ifdef Py_STATS - if (_Py_stats) { - _Py_stats->object_stats.object_visits = 0; - } -#endif - GC_STAT_ADD(generation, collections, 1); - - if (reason != _Py_GC_REASON_SHUTDOWN) { - invoke_gc_callback(tstate, "start", generation, 0, 0); - } - - if (gcstate->debug & DEBUG_STATS) { - PySys_WriteStderr("gc: collecting generation %d...\n", generation); - show_stats_each_generations(gcstate); - t1 = _PyTime_GetPerfCounter(); - } - - if (PyDTrace_GC_START_ENABLED()) - PyDTrace_GC_START(generation); - - /* update collection and allocation counters */ - if (generation+1 < NUM_GENERATIONS) - gcstate->generations[generation+1].count += 1; - for (i = 0; i <= generation; i++) - gcstate->generations[i].count = 0; - - /* merge younger generations with one we are currently collecting */ - for (i = 0; i < generation; i++) { - gc_list_merge(GEN_HEAD(gcstate, i), GEN_HEAD(gcstate, generation)); - } - - /* handy references */ - young = GEN_HEAD(gcstate, generation); - if (generation < NUM_GENERATIONS-1) - old = GEN_HEAD(gcstate, generation+1); - else - old = young; - validate_list(old, collecting_clear_unreachable_clear); - - deduce_unreachable(young, &unreachable); - - untrack_tuples(young); - /* Move reachable objects to next generation. */ - if (young != old) { - if (generation == NUM_GENERATIONS - 2) { - gcstate->long_lived_pending += gc_list_size(young); - } - gc_list_merge(young, old); - } - else { - /* We only un-track dicts in full collections, to avoid quadratic - dict build-up. See issue #14775. */ - untrack_dicts(young); - gcstate->long_lived_pending = 0; - gcstate->long_lived_total = gc_list_size(young); - } - - /* All objects in unreachable are trash, but objects reachable from - * legacy finalizers (e.g. tp_del) can't safely be deleted. - */ - gc_list_init(&finalizers); - // NEXT_MASK_UNREACHABLE is cleared here. - // After move_legacy_finalizers(), unreachable is normal list. - move_legacy_finalizers(&unreachable, &finalizers); - /* finalizers contains the unreachable objects with a legacy finalizer; - * unreachable objects reachable *from* those are also uncollectable, - * and we move those into the finalizers list too. - */ - move_legacy_finalizer_reachable(&finalizers); - - validate_list(&finalizers, collecting_clear_unreachable_clear); - validate_list(&unreachable, collecting_set_unreachable_clear); - - /* Print debugging information. */ - if (gcstate->debug & DEBUG_COLLECTABLE) { - for (gc = GC_NEXT(&unreachable); gc != &unreachable; gc = GC_NEXT(gc)) { - debug_cycle("collectable", FROM_GC(gc)); - } - } - - /* Clear weakrefs and invoke callbacks as necessary. */ - m += handle_weakrefs(&unreachable, old); - - validate_list(old, collecting_clear_unreachable_clear); - validate_list(&unreachable, collecting_set_unreachable_clear); - - /* Call tp_finalize on objects which have one. */ - finalize_garbage(tstate, &unreachable); - - /* Handle any objects that may have resurrected after the call - * to 'finalize_garbage' and continue the collection with the - * objects that are still unreachable */ - PyGC_Head final_unreachable; - handle_resurrected_objects(&unreachable, &final_unreachable, old); - - /* Call tp_clear on objects in the final_unreachable set. This will cause - * the reference cycles to be broken. It may also cause some objects - * in finalizers to be freed. - */ - m += gc_list_size(&final_unreachable); - delete_garbage(tstate, gcstate, &final_unreachable, old); - - /* Collect statistics on uncollectable objects found and print - * debugging information. */ - for (gc = GC_NEXT(&finalizers); gc != &finalizers; gc = GC_NEXT(gc)) { - n++; - if (gcstate->debug & DEBUG_UNCOLLECTABLE) - debug_cycle("uncollectable", FROM_GC(gc)); - } - if (gcstate->debug & DEBUG_STATS) { - double d = _PyTime_AsSecondsDouble(_PyTime_GetPerfCounter() - t1); - PySys_WriteStderr( - "gc: done, %zd unreachable, %zd uncollectable, %.4fs elapsed\n", - n+m, n, d); - } - - /* Append instances in the uncollectable set to a Python - * reachable list of garbage. The programmer has to deal with - * this if they insist on creating this type of structure. - */ - handle_legacy_finalizers(tstate, gcstate, &finalizers, old); - validate_list(old, collecting_clear_unreachable_clear); - - /* Clear free list only during the collection of the highest - * generation */ - if (generation == NUM_GENERATIONS-1) { - clear_freelists(tstate->interp); - } - - if (_PyErr_Occurred(tstate)) { - if (reason == _Py_GC_REASON_SHUTDOWN) { - _PyErr_Clear(tstate); - } - else { - PyErr_FormatUnraisable("Exception ignored in garbage collection"); - } - } - - /* Update stats */ - struct gc_generation_stats *stats = &gcstate->generation_stats[generation]; - stats->collections++; - stats->collected += m; - stats->uncollectable += n; - - GC_STAT_ADD(generation, objects_collected, m); -#ifdef Py_STATS - if (_Py_stats) { - GC_STAT_ADD(generation, object_visits, - _Py_stats->object_stats.object_visits); - _Py_stats->object_stats.object_visits = 0; - } -#endif - - if (PyDTrace_GC_DONE_ENABLED()) { - PyDTrace_GC_DONE(n + m); - } +#include "Python.h" +#include "pycore_gc.h" +#include "pycore_object.h" // _PyObject_IS_GC() +#include "pycore_pystate.h" // _PyInterpreterState_GET() - if (reason != _Py_GC_REASON_SHUTDOWN) { - invoke_gc_callback(tstate, "stop", generation, m, n); - } +typedef struct _gc_runtime_state GCState; - assert(!_PyErr_Occurred(tstate)); - _Py_atomic_store_int(&gcstate->collecting, 0); - return n + m; +static GCState * +get_gc_state(void) +{ + PyInterpreterState *interp = _PyInterpreterState_GET(); + return &interp->gc; } +/*[clinic input] +module gc +[clinic start generated code]*/ +/*[clinic end generated code: output=da39a3ee5e6b4b0d input=b5c9690ecc842d79]*/ #include "clinic/gcmodule.c.h" /*[clinic input] @@ -1593,7 +90,7 @@ gc_collect_impl(PyObject *module, int generation) return -1; } - return gc_collect_main(tstate, generation, _Py_GC_REASON_MANUAL); + return _PyGC_Collect(tstate, generation, _Py_GC_REASON_MANUAL); } /*[clinic input] @@ -1637,24 +134,41 @@ gc_get_debug_impl(PyObject *module) return gcstate->debug; } -PyDoc_STRVAR(gc_set_thresh__doc__, -"set_threshold(threshold0, [threshold1, threshold2]) -> None\n" -"\n" -"Sets the collection thresholds. Setting threshold0 to zero disables\n" -"collection.\n"); +/*[clinic input] +gc.set_threshold + + threshold0: int + [ + threshold1: int + [ + threshold2: int + ] + ] + / + +Set the collection thresholds (the collection frequency). + +Setting 'threshold0' to zero disables collection. +[clinic start generated code]*/ static PyObject * -gc_set_threshold(PyObject *self, PyObject *args) +gc_set_threshold_impl(PyObject *module, int threshold0, int group_right_1, + int threshold1, int group_right_2, int threshold2) +/*[clinic end generated code: output=2e3c7c7dd59060f3 input=0d9612db50984eec]*/ { GCState *gcstate = get_gc_state(); - if (!PyArg_ParseTuple(args, "i|ii:set_threshold", - &gcstate->generations[0].threshold, - &gcstate->generations[1].threshold, - &gcstate->generations[2].threshold)) - return NULL; - for (int i = 3; i < NUM_GENERATIONS; i++) { + + gcstate->generations[0].threshold = threshold0; + if (group_right_1) { + gcstate->generations[1].threshold = threshold1; + } + if (group_right_2) { + gcstate->generations[2].threshold = threshold2; + /* generations higher than 2 get the same threshold */ - gcstate->generations[i].threshold = gcstate->generations[2].threshold; + for (int i = 3; i < NUM_GENERATIONS; i++) { + gcstate->generations[i].threshold = gcstate->generations[2].threshold; + } } Py_RETURN_NONE; } @@ -1693,60 +207,24 @@ gc_get_count_impl(PyObject *module) gcstate->generations[2].count); } -static int -referrersvisit(PyObject* obj, void *arg) -{ - PyObject *objs = arg; - Py_ssize_t i; - for (i = 0; i < PyTuple_GET_SIZE(objs); i++) - if (PyTuple_GET_ITEM(objs, i) == obj) - return 1; - return 0; -} +/*[clinic input] +gc.get_referrers -static int -gc_referrers_for(PyObject *objs, PyGC_Head *list, PyObject *resultlist) -{ - PyGC_Head *gc; - PyObject *obj; - traverseproc traverse; - for (gc = GC_NEXT(list); gc != list; gc = GC_NEXT(gc)) { - obj = FROM_GC(gc); - traverse = Py_TYPE(obj)->tp_traverse; - if (obj == objs || obj == resultlist) - continue; - if (traverse(obj, referrersvisit, objs)) { - if (PyList_Append(resultlist, obj) < 0) - return 0; /* error */ - } - } - return 1; /* no error */ -} + *objs as args: object -PyDoc_STRVAR(gc_get_referrers__doc__, -"get_referrers(*objs) -> list\n\ -Return the list of objects that directly refer to any of objs."); +Return the list of objects that directly refer to any of 'objs'. +[clinic start generated code]*/ static PyObject * -gc_get_referrers(PyObject *self, PyObject *args) +gc_get_referrers_impl(PyObject *module, PyObject *args) +/*[clinic end generated code: output=296a09587f6a86b5 input=bae96961b14a0922]*/ { if (PySys_Audit("gc.get_referrers", "(O)", args) < 0) { return NULL; } - PyObject *result = PyList_New(0); - if (!result) { - return NULL; - } - - GCState *gcstate = get_gc_state(); - for (int i = 0; i < NUM_GENERATIONS; i++) { - if (!(gc_referrers_for(args, GEN_HEAD(gcstate, i), result))) { - Py_DECREF(result); - return NULL; - } - } - return result; + PyInterpreterState *interp = _PyInterpreterState_GET(); + return _PyGC_GetReferrers(interp, args); } /* Append obj to list; return true if error (out of memory), false if OK. */ @@ -1757,36 +235,56 @@ referentsvisit(PyObject *obj, void *arg) return PyList_Append(list, obj) < 0; } -PyDoc_STRVAR(gc_get_referents__doc__, -"get_referents(*objs) -> list\n\ -Return the list of objects that are directly referred to by objs."); +static int +append_referrents(PyObject *result, PyObject *args) +{ + for (Py_ssize_t i = 0; i < PyTuple_GET_SIZE(args); i++) { + PyObject *obj = PyTuple_GET_ITEM(args, i); + if (!_PyObject_IS_GC(obj)) { + continue; + } + + traverseproc traverse = Py_TYPE(obj)->tp_traverse; + if (!traverse) { + continue; + } + if (traverse(obj, referentsvisit, result)) { + return -1; + } + } + return 0; +} + +/*[clinic input] +gc.get_referents + + *objs as args: object + +Return the list of objects that are directly referred to by 'objs'. +[clinic start generated code]*/ static PyObject * -gc_get_referents(PyObject *self, PyObject *args) +gc_get_referents_impl(PyObject *module, PyObject *args) +/*[clinic end generated code: output=d47dc02cefd06fe8 input=b3ceab0c34038cbf]*/ { - Py_ssize_t i; if (PySys_Audit("gc.get_referents", "(O)", args) < 0) { return NULL; } + PyInterpreterState *interp = _PyInterpreterState_GET(); PyObject *result = PyList_New(0); if (result == NULL) return NULL; - for (i = 0; i < PyTuple_GET_SIZE(args); i++) { - traverseproc traverse; - PyObject *obj = PyTuple_GET_ITEM(args, i); + // NOTE: stop the world is a no-op in default build + _PyEval_StopTheWorld(interp); + int err = append_referrents(result, args); + _PyEval_StartTheWorld(interp); - if (!_PyObject_IS_GC(obj)) - continue; - traverse = Py_TYPE(obj)->tp_traverse; - if (! traverse) - continue; - if (traverse(obj, referentsvisit, result)) { - Py_DECREF(result); - return NULL; - } + if (err < 0) { + Py_CLEAR(result); } + return result; } @@ -1805,54 +303,25 @@ static PyObject * gc_get_objects_impl(PyObject *module, Py_ssize_t generation) /*[clinic end generated code: output=48b35fea4ba6cb0e input=ef7da9df9806754c]*/ { - PyThreadState *tstate = _PyThreadState_GET(); - int i; - PyObject* result; - GCState *gcstate = &tstate->interp->gc; - if (PySys_Audit("gc.get_objects", "n", generation) < 0) { return NULL; } - result = PyList_New(0); - if (result == NULL) { - return NULL; - } - - /* If generation is passed, we extract only that generation */ - if (generation != -1) { - if (generation >= NUM_GENERATIONS) { - _PyErr_Format(tstate, PyExc_ValueError, - "generation parameter must be less than the number of " - "available generations (%i)", - NUM_GENERATIONS); - goto error; - } - - if (generation < 0) { - _PyErr_SetString(tstate, PyExc_ValueError, - "generation parameter cannot be negative"); - goto error; - } - - if (append_objects(result, GEN_HEAD(gcstate, generation))) { - goto error; - } - - return result; + if (generation >= NUM_GENERATIONS) { + return PyErr_Format(PyExc_ValueError, + "generation parameter must be less than the number of " + "available generations (%i)", + NUM_GENERATIONS); } - /* If generation is not passed or None, get all objects from all generations */ - for (i = 0; i < NUM_GENERATIONS; i++) { - if (append_objects(result, GEN_HEAD(gcstate, i))) { - goto error; - } + if (generation < -1) { + PyErr_SetString(PyExc_ValueError, + "generation parameter cannot be negative"); + return NULL; } - return result; -error: - Py_DECREF(result); - return NULL; + PyInterpreterState *interp = _PyInterpreterState_GET(); + return _PyGC_GetObjects(interp, generation); } /*[clinic input] @@ -1960,11 +429,8 @@ static PyObject * gc_freeze_impl(PyObject *module) /*[clinic end generated code: output=502159d9cdc4c139 input=b602b16ac5febbe5]*/ { - GCState *gcstate = get_gc_state(); - for (int i = 0; i < NUM_GENERATIONS; ++i) { - gc_list_merge(GEN_HEAD(gcstate, i), &gcstate->permanent_generation.head); - gcstate->generations[i].count = 0; - } + PyInterpreterState *interp = _PyInterpreterState_GET(); + _PyGC_Freeze(interp); Py_RETURN_NONE; } @@ -1980,9 +446,8 @@ static PyObject * gc_unfreeze_impl(PyObject *module) /*[clinic end generated code: output=1c15f2043b25e169 input=2dd52b170f4cef6c]*/ { - GCState *gcstate = get_gc_state(); - gc_list_merge(&gcstate->permanent_generation.head, - GEN_HEAD(gcstate, NUM_GENERATIONS-1)); + PyInterpreterState *interp = _PyInterpreterState_GET(); + _PyGC_Unfreeze(interp); Py_RETURN_NONE; } @@ -1996,8 +461,8 @@ static Py_ssize_t gc_get_freeze_count_impl(PyObject *module) /*[clinic end generated code: output=61cbd9f43aa032e1 input=45ffbc65cfe2a6ed]*/ { - GCState *gcstate = get_gc_state(); - return gc_list_size(&gcstate->permanent_generation.head); + PyInterpreterState *interp = _PyInterpreterState_GET(); + return _PyGC_GetFreezeCount(interp); } @@ -2030,17 +495,15 @@ static PyMethodDef GcMethods[] = { GC_SET_DEBUG_METHODDEF GC_GET_DEBUG_METHODDEF GC_GET_COUNT_METHODDEF - {"set_threshold", gc_set_threshold, METH_VARARGS, gc_set_thresh__doc__}, + GC_SET_THRESHOLD_METHODDEF GC_GET_THRESHOLD_METHODDEF GC_COLLECT_METHODDEF GC_GET_OBJECTS_METHODDEF GC_GET_STATS_METHODDEF GC_IS_TRACKED_METHODDEF GC_IS_FINALIZED_METHODDEF - {"get_referrers", gc_get_referrers, METH_VARARGS, - gc_get_referrers__doc__}, - {"get_referents", gc_get_referents, METH_VARARGS, - gc_get_referents__doc__}, + GC_GET_REFERRERS_METHODDEF + GC_GET_REFERENTS_METHODDEF GC_FREEZE_METHODDEF GC_UNFREEZE_METHODDEF GC_GET_FREEZE_COUNT_METHODDEF @@ -2063,7 +526,7 @@ gcmodule_exec(PyObject *module) return -1; } -#define ADD_INT(NAME) if (PyModule_AddIntConstant(module, #NAME, NAME) < 0) { return -1; } +#define ADD_INT(NAME) if (PyModule_AddIntConstant(module, #NAME, _PyGC_ ## NAME) < 0) { return -1; } ADD_INT(DEBUG_STATS); ADD_INT(DEBUG_COLLECTABLE); ADD_INT(DEBUG_UNCOLLECTABLE); @@ -2093,353 +556,3 @@ PyInit_gc(void) { return PyModuleDef_Init(&gcmodule); } - -/* C API for controlling the state of the garbage collector */ -int -PyGC_Enable(void) -{ - GCState *gcstate = get_gc_state(); - int old_state = gcstate->enabled; - gcstate->enabled = 1; - return old_state; -} - -int -PyGC_Disable(void) -{ - GCState *gcstate = get_gc_state(); - int old_state = gcstate->enabled; - gcstate->enabled = 0; - return old_state; -} - -int -PyGC_IsEnabled(void) -{ - GCState *gcstate = get_gc_state(); - return gcstate->enabled; -} - -/* Public API to invoke gc.collect() from C */ -Py_ssize_t -PyGC_Collect(void) -{ - PyThreadState *tstate = _PyThreadState_GET(); - GCState *gcstate = &tstate->interp->gc; - - if (!gcstate->enabled) { - return 0; - } - - Py_ssize_t n; - PyObject *exc = _PyErr_GetRaisedException(tstate); - n = gc_collect_main(tstate, NUM_GENERATIONS - 1, _Py_GC_REASON_MANUAL); - _PyErr_SetRaisedException(tstate, exc); - - return n; -} - -Py_ssize_t -_PyGC_CollectNoFail(PyThreadState *tstate) -{ - /* Ideally, this function is only called on interpreter shutdown, - and therefore not recursively. Unfortunately, when there are daemon - threads, a daemon thread can start a cyclic garbage collection - during interpreter shutdown (and then never finish it). - See http://bugs.python.org/issue8713#msg195178 for an example. - */ - return gc_collect_main(tstate, NUM_GENERATIONS - 1, _Py_GC_REASON_SHUTDOWN); -} - -void -_PyGC_DumpShutdownStats(PyInterpreterState *interp) -{ - GCState *gcstate = &interp->gc; - if (!(gcstate->debug & DEBUG_SAVEALL) - && gcstate->garbage != NULL && PyList_GET_SIZE(gcstate->garbage) > 0) { - const char *message; - if (gcstate->debug & DEBUG_UNCOLLECTABLE) - message = "gc: %zd uncollectable objects at " \ - "shutdown"; - else - message = "gc: %zd uncollectable objects at " \ - "shutdown; use gc.set_debug(gc.DEBUG_UNCOLLECTABLE) to list them"; - /* PyErr_WarnFormat does too many things and we are at shutdown, - the warnings module's dependencies (e.g. linecache) may be gone - already. */ - if (PyErr_WarnExplicitFormat(PyExc_ResourceWarning, "gc", 0, - "gc", NULL, message, - PyList_GET_SIZE(gcstate->garbage))) - PyErr_WriteUnraisable(NULL); - if (gcstate->debug & DEBUG_UNCOLLECTABLE) { - PyObject *repr = NULL, *bytes = NULL; - repr = PyObject_Repr(gcstate->garbage); - if (!repr || !(bytes = PyUnicode_EncodeFSDefault(repr))) - PyErr_WriteUnraisable(gcstate->garbage); - else { - PySys_WriteStderr( - " %s\n", - PyBytes_AS_STRING(bytes) - ); - } - Py_XDECREF(repr); - Py_XDECREF(bytes); - } - } -} - - -void -_PyGC_Fini(PyInterpreterState *interp) -{ - GCState *gcstate = &interp->gc; - Py_CLEAR(gcstate->garbage); - Py_CLEAR(gcstate->callbacks); - - /* We expect that none of this interpreters objects are shared - with other interpreters. - See https://github.com/python/cpython/issues/90228. */ -} - -/* for debugging */ -void -_PyGC_Dump(PyGC_Head *g) -{ - _PyObject_Dump(FROM_GC(g)); -} - - -#ifdef Py_DEBUG -static int -visit_validate(PyObject *op, void *parent_raw) -{ - PyObject *parent = _PyObject_CAST(parent_raw); - if (_PyObject_IsFreed(op)) { - _PyObject_ASSERT_FAILED_MSG(parent, - "PyObject_GC_Track() object is not valid"); - } - return 0; -} -#endif - - -/* extension modules might be compiled with GC support so these - functions must always be available */ - -void -PyObject_GC_Track(void *op_raw) -{ - PyObject *op = _PyObject_CAST(op_raw); - if (_PyObject_GC_IS_TRACKED(op)) { - _PyObject_ASSERT_FAILED_MSG(op, - "object already tracked " - "by the garbage collector"); - } - _PyObject_GC_TRACK(op); - -#ifdef Py_DEBUG - /* Check that the object is valid: validate objects traversed - by tp_traverse() */ - traverseproc traverse = Py_TYPE(op)->tp_traverse; - (void)traverse(op, visit_validate, op); -#endif -} - -void -PyObject_GC_UnTrack(void *op_raw) -{ - PyObject *op = _PyObject_CAST(op_raw); - /* Obscure: the Py_TRASHCAN mechanism requires that we be able to - * call PyObject_GC_UnTrack twice on an object. - */ - if (_PyObject_GC_IS_TRACKED(op)) { - _PyObject_GC_UNTRACK(op); - } -} - -int -PyObject_IS_GC(PyObject *obj) -{ - return _PyObject_IS_GC(obj); -} - -void -_Py_ScheduleGC(PyInterpreterState *interp) -{ - _Py_set_eval_breaker_bit(interp, _PY_GC_SCHEDULED_BIT, 1); -} - -void -_PyObject_GC_Link(PyObject *op) -{ - PyGC_Head *g = AS_GC(op); - assert(((uintptr_t)g & (sizeof(uintptr_t)-1)) == 0); // g must be correctly aligned - - PyThreadState *tstate = _PyThreadState_GET(); - GCState *gcstate = &tstate->interp->gc; - g->_gc_next = 0; - g->_gc_prev = 0; - gcstate->generations[0].count++; /* number of allocated GC objects */ - if (gcstate->generations[0].count > gcstate->generations[0].threshold && - gcstate->enabled && - gcstate->generations[0].threshold && - !_Py_atomic_load_int_relaxed(&gcstate->collecting) && - !_PyErr_Occurred(tstate)) - { - _Py_ScheduleGC(tstate->interp); - } -} - -void -_Py_RunGC(PyThreadState *tstate) -{ - gc_collect_main(tstate, GENERATION_AUTO, _Py_GC_REASON_HEAP); -} - -static PyObject * -gc_alloc(size_t basicsize, size_t presize) -{ - PyThreadState *tstate = _PyThreadState_GET(); - if (basicsize > PY_SSIZE_T_MAX - presize) { - return _PyErr_NoMemory(tstate); - } - size_t size = presize + basicsize; - char *mem = PyObject_Malloc(size); - if (mem == NULL) { - return _PyErr_NoMemory(tstate); - } - ((PyObject **)mem)[0] = NULL; - ((PyObject **)mem)[1] = NULL; - PyObject *op = (PyObject *)(mem + presize); - _PyObject_GC_Link(op); - return op; -} - -PyObject * -_PyObject_GC_New(PyTypeObject *tp) -{ - size_t presize = _PyType_PreHeaderSize(tp); - PyObject *op = gc_alloc(_PyObject_SIZE(tp), presize); - if (op == NULL) { - return NULL; - } - _PyObject_Init(op, tp); - return op; -} - -PyVarObject * -_PyObject_GC_NewVar(PyTypeObject *tp, Py_ssize_t nitems) -{ - PyVarObject *op; - - if (nitems < 0) { - PyErr_BadInternalCall(); - return NULL; - } - size_t presize = _PyType_PreHeaderSize(tp); - size_t size = _PyObject_VAR_SIZE(tp, nitems); - op = (PyVarObject *)gc_alloc(size, presize); - if (op == NULL) { - return NULL; - } - _PyObject_InitVar(op, tp, nitems); - return op; -} - -PyObject * -PyUnstable_Object_GC_NewWithExtraData(PyTypeObject *tp, size_t extra_size) -{ - size_t presize = _PyType_PreHeaderSize(tp); - PyObject *op = gc_alloc(_PyObject_SIZE(tp) + extra_size, presize); - if (op == NULL) { - return NULL; - } - memset(op, 0, _PyObject_SIZE(tp) + extra_size); - _PyObject_Init(op, tp); - return op; -} - -PyVarObject * -_PyObject_GC_Resize(PyVarObject *op, Py_ssize_t nitems) -{ - const size_t basicsize = _PyObject_VAR_SIZE(Py_TYPE(op), nitems); - const size_t presize = _PyType_PreHeaderSize(((PyObject *)op)->ob_type); - _PyObject_ASSERT((PyObject *)op, !_PyObject_GC_IS_TRACKED(op)); - if (basicsize > (size_t)PY_SSIZE_T_MAX - presize) { - return (PyVarObject *)PyErr_NoMemory(); - } - char *mem = (char *)op - presize; - mem = (char *)PyObject_Realloc(mem, presize + basicsize); - if (mem == NULL) { - return (PyVarObject *)PyErr_NoMemory(); - } - op = (PyVarObject *) (mem + presize); - Py_SET_SIZE(op, nitems); - return op; -} - -void -PyObject_GC_Del(void *op) -{ - size_t presize = _PyType_PreHeaderSize(((PyObject *)op)->ob_type); - PyGC_Head *g = AS_GC(op); - if (_PyObject_GC_IS_TRACKED(op)) { - gc_list_remove(g); -#ifdef Py_DEBUG - PyObject *exc = PyErr_GetRaisedException(); - if (PyErr_WarnExplicitFormat(PyExc_ResourceWarning, "gc", 0, - "gc", NULL, "Object of type %s is not untracked before destruction", - ((PyObject*)op)->ob_type->tp_name)) { - PyErr_WriteUnraisable(NULL); - } - PyErr_SetRaisedException(exc); -#endif - } - GCState *gcstate = get_gc_state(); - if (gcstate->generations[0].count > 0) { - gcstate->generations[0].count--; - } - PyObject_Free(((char *)op)-presize); -} - -int -PyObject_GC_IsTracked(PyObject* obj) -{ - if (_PyObject_IS_GC(obj) && _PyObject_GC_IS_TRACKED(obj)) { - return 1; - } - return 0; -} - -int -PyObject_GC_IsFinalized(PyObject *obj) -{ - if (_PyObject_IS_GC(obj) && _PyGC_FINALIZED(obj)) { - return 1; - } - return 0; -} - -void -PyUnstable_GC_VisitObjects(gcvisitobjects_t callback, void *arg) -{ - size_t i; - GCState *gcstate = get_gc_state(); - int origenstate = gcstate->enabled; - gcstate->enabled = 0; - for (i = 0; i < NUM_GENERATIONS; i++) { - PyGC_Head *gc_list, *gc; - gc_list = GEN_HEAD(gcstate, i); - for (gc = GC_NEXT(gc_list); gc != gc_list; gc = GC_NEXT(gc)) { - PyObject *op = FROM_GC(gc); - Py_INCREF(op); - int res = callback(op, arg); - Py_DECREF(op); - if (!res) { - goto done; - } - } - } -done: - gcstate->enabled = origenstate; -} diff --git a/Modules/getpath.c b/Modules/getpath.c index a3c8fc269d1c3c..abed139028244a 100644 --- a/Modules/getpath.c +++ b/Modules/getpath.c @@ -262,6 +262,10 @@ getpath_joinpath(PyObject *Py_UNUSED(self), PyObject *args) } /* Convert all parts to wchar and accumulate max final length */ wchar_t **parts = (wchar_t **)PyMem_Malloc(n * sizeof(wchar_t *)); + if (parts == NULL) { + PyErr_NoMemory(); + return NULL; + } memset(parts, 0, n * sizeof(wchar_t *)); Py_ssize_t cchFinal = 0; Py_ssize_t first = 0; diff --git a/Modules/mathmodule.c b/Modules/mathmodule.c index 6cd61e9ab75424..a877bfcd6afb68 100644 --- a/Modules/mathmodule.c +++ b/Modules/mathmodule.c @@ -759,13 +759,17 @@ m_log10(double x) static PyObject * math_gcd(PyObject *module, PyObject * const *args, Py_ssize_t nargs) { - PyObject *res, *x; - Py_ssize_t i; + // Fast-path for the common case: gcd(int, int) + if (nargs == 2 && PyLong_CheckExact(args[0]) && PyLong_CheckExact(args[1])) + { + return _PyLong_GCD(args[0], args[1]); + } if (nargs == 0) { return PyLong_FromLong(0); } - res = PyNumber_Index(args[0]); + + PyObject *res = PyNumber_Index(args[0]); if (res == NULL) { return NULL; } @@ -775,8 +779,8 @@ math_gcd(PyObject *module, PyObject * const *args, Py_ssize_t nargs) } PyObject *one = _PyLong_GetOne(); // borrowed ref - for (i = 1; i < nargs; i++) { - x = _PyNumber_Index(args[i]); + for (Py_ssize_t i = 1; i < nargs; i++) { + PyObject *x = _PyNumber_Index(args[i]); if (x == NULL) { Py_DECREF(res); return NULL; @@ -2070,11 +2074,6 @@ math_trunc(PyObject *module, PyObject *x) return PyFloat_Type.tp_as_number->nb_int(x); } - if (!_PyType_IsReady(Py_TYPE(x))) { - if (PyType_Ready(Py_TYPE(x)) < 0) - return NULL; - } - math_module_state *state = get_math_module_state(module); trunc = _PyObject_LookupSpecial(x, state->str___trunc__); if (trunc == NULL) { @@ -2566,7 +2565,7 @@ math_dist_impl(PyObject *module, PyObject *p, PyObject *q) goto error_exit; } if (n > NUM_STACK_ELEMS) { - diffs = (double *) PyObject_Malloc(n * sizeof(double)); + diffs = (double *) PyMem_Malloc(n * sizeof(double)); if (diffs == NULL) { PyErr_NoMemory(); goto error_exit; @@ -2586,7 +2585,7 @@ math_dist_impl(PyObject *module, PyObject *p, PyObject *q) } result = vector_norm(n, diffs, max, found_nan); if (diffs != diffs_on_stack) { - PyObject_Free(diffs); + PyMem_Free(diffs); } if (p_allocated) { Py_DECREF(p); @@ -2598,7 +2597,7 @@ math_dist_impl(PyObject *module, PyObject *p, PyObject *q) error_exit: if (diffs != diffs_on_stack) { - PyObject_Free(diffs); + PyMem_Free(diffs); } if (p_allocated) { Py_DECREF(p); @@ -2622,7 +2621,7 @@ math_hypot(PyObject *self, PyObject *const *args, Py_ssize_t nargs) double *coordinates = coord_on_stack; if (nargs > NUM_STACK_ELEMS) { - coordinates = (double *) PyObject_Malloc(nargs * sizeof(double)); + coordinates = (double *) PyMem_Malloc(nargs * sizeof(double)); if (coordinates == NULL) { return PyErr_NoMemory(); } @@ -2639,13 +2638,13 @@ math_hypot(PyObject *self, PyObject *const *args, Py_ssize_t nargs) } result = vector_norm(nargs, coordinates, max, found_nan); if (coordinates != coord_on_stack) { - PyObject_Free(coordinates); + PyMem_Free(coordinates); } return PyFloat_FromDouble(result); error_exit: if (coordinates != coord_on_stack) { - PyObject_Free(coordinates); + PyMem_Free(coordinates); } return NULL; } diff --git a/Modules/mmapmodule.c b/Modules/mmapmodule.c index 66ed0b8efb775c..0cce7c27f9b16a 100644 --- a/Modules/mmapmodule.c +++ b/Modules/mmapmodule.c @@ -32,10 +32,6 @@ # include // close() #endif -// to support MS_WINDOWS_SYSTEM OpenFileMappingA / CreateFileMappingA -// need to be replaced with OpenFileMappingW / CreateFileMappingW -#if !defined(MS_WINDOWS) || defined(MS_WINDOWS_DESKTOP) || defined(MS_WINDOWS_GAMES) - #ifndef MS_WINDOWS #define UNIX # ifdef HAVE_FCNTL_H @@ -116,11 +112,12 @@ typedef struct { #ifdef MS_WINDOWS HANDLE map_handle; HANDLE file_handle; - char * tagname; + wchar_t * tagname; #endif #ifdef UNIX int fd; + _Bool trackfd; #endif PyObject *weakreflist; @@ -397,6 +394,13 @@ is_resizeable(mmap_object *self) "mmap can't resize with extant buffers exported."); return 0; } +#ifdef UNIX + if (!self->trackfd) { + PyErr_SetString(PyExc_ValueError, + "mmap can't resize with trackfd=False."); + return 0; + } +#endif if ((self->access == ACCESS_WRITE) || (self->access == ACCESS_DEFAULT)) return 1; PyErr_Format(PyExc_TypeError, @@ -534,7 +538,7 @@ mmap_resize_method(mmap_object *self, CloseHandle(self->map_handle); /* if the file mapping still exists, it cannot be resized. */ if (self->tagname) { - self->map_handle = OpenFileMappingA(FILE_MAP_WRITE, FALSE, + self->map_handle = OpenFileMappingW(FILE_MAP_WRITE, FALSE, self->tagname); if (self->map_handle) { PyErr_SetFromWindowsErr(ERROR_USER_MAPPED_FILE); @@ -563,7 +567,7 @@ mmap_resize_method(mmap_object *self, /* create a new file mapping and map a new view */ /* FIXME: call CreateFileMappingW with wchar_t tagname */ - self->map_handle = CreateFileMappingA( + self->map_handle = CreateFileMappingW( self->file_handle, NULL, PAGE_READWRITE, @@ -845,7 +849,7 @@ mmap__sizeof__method(mmap_object *self, void *Py_UNUSED(ignored)) { size_t res = _PyObject_SIZE(Py_TYPE(self)); if (self->tagname) { - res += strlen(self->tagname) + 1; + res += (wcslen(self->tagname) + 1) * sizeof(self->tagname[0]); } return PyLong_FromSize_t(res); } @@ -1158,7 +1162,7 @@ is 0, the maximum length of the map is the current size of the file,\n\ except that if the file is empty Windows raises an exception (you cannot\n\ create an empty mapping on Windows).\n\ \n\ -Unix: mmap(fileno, length[, flags[, prot[, access[, offset]]]])\n\ +Unix: mmap(fileno, length[, flags[, prot[, access[, offset[, trackfd]]]]])\n\ \n\ Maps length bytes from the file specified by the file descriptor fileno,\n\ and returns a mmap object. If length is 0, the maximum length of the map\n\ @@ -1225,15 +1229,17 @@ new_mmap_object(PyTypeObject *type, PyObject *args, PyObject *kwdict) off_t offset = 0; int fd, flags = MAP_SHARED, prot = PROT_WRITE | PROT_READ; int devzero = -1; - int access = (int)ACCESS_DEFAULT; + int access = (int)ACCESS_DEFAULT, trackfd = 1; static char *keywords[] = {"fileno", "length", "flags", "prot", - "access", "offset", NULL}; + "access", "offset", "trackfd", NULL}; - if (!PyArg_ParseTupleAndKeywords(args, kwdict, "in|iii" _Py_PARSE_OFF_T, keywords, + if (!PyArg_ParseTupleAndKeywords(args, kwdict, + "in|iii" _Py_PARSE_OFF_T "$p", keywords, &fd, &map_size, &flags, &prot, - &access, &offset)) + &access, &offset, &trackfd)) { return NULL; + } if (map_size < 0) { PyErr_SetString(PyExc_OverflowError, "memory mapped length must be positive"); @@ -1329,6 +1335,7 @@ new_mmap_object(PyTypeObject *type, PyObject *args, PyObject *kwdict) m_obj->weakreflist = NULL; m_obj->exports = 0; m_obj->offset = offset; + m_obj->trackfd = trackfd; if (fd == -1) { m_obj->fd = -1; /* Assume the caller wants to map anonymous memory. @@ -1354,13 +1361,16 @@ new_mmap_object(PyTypeObject *type, PyObject *args, PyObject *kwdict) } #endif } - else { + else if (trackfd) { m_obj->fd = _Py_dup(fd); if (m_obj->fd == -1) { Py_DECREF(m_obj); return NULL; } } + else { + m_obj->fd = -1; + } Py_BEGIN_ALLOW_THREADS m_obj->data = mmap(NULL, map_size, prot, flags, fd, offset); @@ -1400,7 +1410,7 @@ new_mmap_object(PyTypeObject *type, PyObject *args, PyObject *kwdict) DWORD off_lo; /* lower 32 bits of offset */ DWORD size_hi; /* upper 32 bits of size */ DWORD size_lo; /* lower 32 bits of size */ - const char *tagname = ""; + PyObject *tagname = Py_None; DWORD dwErr = 0; int fileno; HANDLE fh = 0; @@ -1410,7 +1420,7 @@ new_mmap_object(PyTypeObject *type, PyObject *args, PyObject *kwdict) "tagname", "access", "offset", NULL }; - if (!PyArg_ParseTupleAndKeywords(args, kwdict, "in|ziL", keywords, + if (!PyArg_ParseTupleAndKeywords(args, kwdict, "in|OiL", keywords, &fileno, &map_size, &tagname, &access, &offset)) { return NULL; @@ -1543,17 +1553,19 @@ new_mmap_object(PyTypeObject *type, PyObject *args, PyObject *kwdict) m_obj->weakreflist = NULL; m_obj->exports = 0; /* set the tag name */ - if (tagname != NULL && *tagname != '\0') { - m_obj->tagname = PyMem_Malloc(strlen(tagname)+1); + if (!Py_IsNone(tagname)) { + if (!PyUnicode_Check(tagname)) { + Py_DECREF(m_obj); + return PyErr_Format(PyExc_TypeError, "expected str or None for " + "'tagname', not %.200s", + Py_TYPE(tagname)->tp_name); + } + m_obj->tagname = PyUnicode_AsWideCharString(tagname, NULL); if (m_obj->tagname == NULL) { - PyErr_NoMemory(); Py_DECREF(m_obj); return NULL; } - strcpy(m_obj->tagname, tagname); } - else - m_obj->tagname = NULL; m_obj->access = (access_mode)access; size_hi = (DWORD)(size >> 32); @@ -1562,7 +1574,7 @@ new_mmap_object(PyTypeObject *type, PyObject *args, PyObject *kwdict) off_lo = (DWORD)(offset & 0xFFFFFFFF); /* For files, it would be sufficient to pass 0 as size. For anonymous maps, we have to pass the size explicitly. */ - m_obj->map_handle = CreateFileMappingA(m_obj->file_handle, + m_obj->map_handle = CreateFileMappingW(m_obj->file_handle, NULL, flProtect, size_hi, @@ -1655,6 +1667,39 @@ mmap_exec(PyObject *module) #endif #ifdef MAP_CONCEAL ADD_INT_MACRO(module, MAP_CONCEAL); +#endif +#ifdef MAP_NORESERVE + ADD_INT_MACRO(module, MAP_NORESERVE); +#endif +#ifdef MAP_NOEXTEND + ADD_INT_MACRO(module, MAP_NOEXTEND); +#endif +#ifdef MAP_HASSEMAPHORE + ADD_INT_MACRO(module, MAP_HASSEMAPHORE); +#endif +#ifdef MAP_NOCACHE + ADD_INT_MACRO(module, MAP_NOCACHE); +#endif +#ifdef MAP_JIT + ADD_INT_MACRO(module, MAP_JIT); +#endif +#ifdef MAP_RESILIENT_CODESIGN + ADD_INT_MACRO(module, MAP_RESILIENT_CODESIGN); +#endif +#ifdef MAP_RESILIENT_MEDIA + ADD_INT_MACRO(module, MAP_RESILIENT_MEDIA); +#endif +#ifdef MAP_32BIT + ADD_INT_MACRO(module, MAP_32BIT); +#endif +#ifdef MAP_TRANSLATED_ALLOW_EXECUTE + ADD_INT_MACRO(module, MAP_TRANSLATED_ALLOW_EXECUTE); +#endif +#ifdef MAP_UNIX03 + ADD_INT_MACRO(module, MAP_UNIX03); +#endif +#ifdef MAP_TPRO + ADD_INT_MACRO(module, MAP_TPRO); #endif if (PyModule_AddIntConstant(module, "PAGESIZE", (long)my_getpagesize()) < 0 ) { return -1; @@ -1771,5 +1816,3 @@ PyInit_mmap(void) { return PyModuleDef_Init(&mmapmodule); } - -#endif /* !MS_WINDOWS || MS_WINDOWS_DESKTOP || MS_WINDOWS_GAMES */ diff --git a/Modules/posixmodule.c b/Modules/posixmodule.c index 4708c4d8706dc1..d05b4ba723ce8c 100644 --- a/Modules/posixmodule.c +++ b/Modules/posixmodule.c @@ -242,15 +242,16 @@ corresponding Unix manual entries for more information on calls."); # include #endif +#ifdef HAVE_SYS_TYPES_H +/* Should be included before on HP-UX v3 */ +# include +#endif /* HAVE_SYS_TYPES_H */ + #ifdef HAVE_SYS_SYSMACROS_H /* GNU C Library: major(), minor(), makedev() */ # include #endif -#ifdef HAVE_SYS_TYPES_H -# include -#endif /* HAVE_SYS_TYPES_H */ - #ifdef HAVE_SYS_STAT_H # include #endif /* HAVE_SYS_STAT_H */ @@ -642,6 +643,10 @@ PyOS_AfterFork_Child(void) tstate->native_thread_id = PyThread_get_thread_native_id(); #endif +#ifdef Py_GIL_DISABLED + _Py_brc_after_fork(tstate->interp); +#endif + status = _PyEval_ReInitThreads(tstate); if (_PyStatus_EXCEPTION(status)) { goto fatal_error; @@ -974,6 +979,13 @@ _fd_converter(PyObject *o, int *p) int overflow; long long_value; + if (PyBool_Check(o)) { + if (PyErr_WarnEx(PyExc_RuntimeWarning, + "bool is used as a file descriptor", 1)) + { + return 0; + } + } PyObject *index = _PyNumber_Index(o); if (index == NULL) { return 0; @@ -1029,7 +1041,7 @@ typedef struct { PyObject *TerminalSizeType; PyObject *TimesResultType; PyObject *UnameResultType; -#if defined(HAVE_WAITID) && !defined(__APPLE__) +#if defined(HAVE_WAITID) PyObject *WaitidResultType; #endif #if defined(HAVE_WAIT3) || defined(HAVE_WAIT4) @@ -1625,7 +1637,7 @@ convertenviron(void) Py_DECREF(d); return NULL; } - if (PyDict_SetDefault(d, k, v) == NULL) { + if (PyDict_SetDefaultRef(d, k, v, NULL) < 0) { Py_DECREF(v); Py_DECREF(k); Py_DECREF(d); @@ -1891,8 +1903,9 @@ win32_xstat_slow_impl(const wchar_t *path, struct _Py_stat_struct *result, HANDLE hFile; BY_HANDLE_FILE_INFORMATION fileInfo; FILE_BASIC_INFO basicInfo; + FILE_BASIC_INFO *pBasicInfo = NULL; FILE_ID_INFO idInfo; - FILE_ID_INFO *pIdInfo = &idInfo; + FILE_ID_INFO *pIdInfo = NULL; FILE_ATTRIBUTE_TAG_INFO tagInfo = { 0 }; DWORD fileType, error; BOOL isUnhandledTag = FALSE; @@ -2043,14 +2056,17 @@ win32_xstat_slow_impl(const wchar_t *path, struct _Py_stat_struct *result, retval = -1; goto cleanup; } - } - if (!GetFileInformationByHandleEx(hFile, FileIdInfo, &idInfo, sizeof(idInfo))) { - /* Failed to get FileIdInfo, so do not pass it along */ - pIdInfo = NULL; + /* Successfully got FileBasicInfo, so we'll pass it along */ + pBasicInfo = &basicInfo; + + if (GetFileInformationByHandleEx(hFile, FileIdInfo, &idInfo, sizeof(idInfo))) { + /* Successfully got FileIdInfo, so pass it along */ + pIdInfo = &idInfo; + } } - _Py_attribute_data_to_stat(&fileInfo, tagInfo.ReparseTag, &basicInfo, pIdInfo, result); + _Py_attribute_data_to_stat(&fileInfo, tagInfo.ReparseTag, pBasicInfo, pIdInfo, result); update_st_mode_from_path(path, fileInfo.dwFileAttributes, result); cleanup: @@ -2297,7 +2313,7 @@ static PyStructSequence_Desc statvfs_result_desc = { 10 }; -#if defined(HAVE_WAITID) && !defined(__APPLE__) +#if defined(HAVE_WAITID) PyDoc_STRVAR(waitid_result__doc__, "waitid_result: Result from waitid.\n\n\ This object may be accessed either as a tuple of\n\ @@ -2372,7 +2388,7 @@ _posix_clear(PyObject *module) Py_CLEAR(state->TerminalSizeType); Py_CLEAR(state->TimesResultType); Py_CLEAR(state->UnameResultType); -#if defined(HAVE_WAITID) && !defined(__APPLE__) +#if defined(HAVE_WAITID) Py_CLEAR(state->WaitidResultType); #endif #if defined(HAVE_WAIT3) || defined(HAVE_WAIT4) @@ -2397,7 +2413,7 @@ _posix_traverse(PyObject *module, visitproc visit, void *arg) Py_VISIT(state->TerminalSizeType); Py_VISIT(state->TimesResultType); Py_VISIT(state->UnameResultType); -#if defined(HAVE_WAITID) && !defined(__APPLE__) +#if defined(HAVE_WAITID) Py_VISIT(state->WaitidResultType); #endif #if defined(HAVE_WAIT3) || defined(HAVE_WAIT4) @@ -2861,6 +2877,8 @@ FTRUNCATE #ifdef MS_WINDOWS #undef PATH_HAVE_FTRUNCATE #define PATH_HAVE_FTRUNCATE 1 + #undef PATH_HAVE_FCHMOD + #define PATH_HAVE_FCHMOD 1 #endif /*[python input] @@ -3338,7 +3356,38 @@ win32_lchmod(LPCWSTR path, int mode) } return SetFileAttributesW(path, attr); } -#endif + +static int +win32_hchmod(HANDLE hfile, int mode) +{ + FILE_BASIC_INFO info; + if (!GetFileInformationByHandleEx(hfile, FileBasicInfo, + &info, sizeof(info))) + { + return 0; + } + if (mode & _S_IWRITE) { + info.FileAttributes &= ~FILE_ATTRIBUTE_READONLY; + } + else { + info.FileAttributes |= FILE_ATTRIBUTE_READONLY; + } + return SetFileInformationByHandle(hfile, FileBasicInfo, + &info, sizeof(info)); +} + +static int +win32_fchmod(int fd, int mode) +{ + HANDLE hfile = _Py_get_osfhandle_noraise(fd); + if (hfile == INVALID_HANDLE_VALUE) { + SetLastError(ERROR_INVALID_HANDLE); + return 0; + } + return win32_hchmod(hfile, mode); +} + +#endif /* MS_WINDOWS */ /*[clinic input] os.chmod @@ -3401,27 +3450,16 @@ os_chmod_impl(PyObject *module, path_t *path, int mode, int dir_fd, #ifdef MS_WINDOWS result = 0; Py_BEGIN_ALLOW_THREADS - if (follow_symlinks) { - HANDLE hfile; - FILE_BASIC_INFO info; - - hfile = CreateFileW(path->wide, - FILE_READ_ATTRIBUTES|FILE_WRITE_ATTRIBUTES, - 0, NULL, - OPEN_EXISTING, FILE_FLAG_BACKUP_SEMANTICS, NULL); + if (path->fd != -1) { + result = win32_fchmod(path->fd, mode); + } + else if (follow_symlinks) { + HANDLE hfile = CreateFileW(path->wide, + FILE_READ_ATTRIBUTES|FILE_WRITE_ATTRIBUTES, + 0, NULL, + OPEN_EXISTING, FILE_FLAG_BACKUP_SEMANTICS, NULL); if (hfile != INVALID_HANDLE_VALUE) { - if (GetFileInformationByHandleEx(hfile, FileBasicInfo, - &info, sizeof(info))) - { - if (mode & _S_IWRITE) { - info.FileAttributes &= ~FILE_ATTRIBUTE_READONLY; - } - else { - info.FileAttributes |= FILE_ATTRIBUTE_READONLY; - } - result = SetFileInformationByHandle(hfile, FileBasicInfo, - &info, sizeof(info)); - } + result = win32_hchmod(hfile, mode); (void)CloseHandle(hfile); } } @@ -3517,7 +3555,7 @@ os_chmod_impl(PyObject *module, path_t *path, int mode, int dir_fd, } -#ifdef HAVE_FCHMOD +#if defined(HAVE_FCHMOD) || defined(MS_WINDOWS) /*[clinic input] os.fchmod @@ -3539,12 +3577,21 @@ os_fchmod_impl(PyObject *module, int fd, int mode) /*[clinic end generated code: output=afd9bc05b4e426b3 input=b5594618bbbc22df]*/ { int res; - int async_err = 0; if (PySys_Audit("os.chmod", "iii", fd, mode, -1) < 0) { return NULL; } +#ifdef MS_WINDOWS + res = 0; + Py_BEGIN_ALLOW_THREADS + res = win32_fchmod(fd, mode); + Py_END_ALLOW_THREADS + if (!res) { + return PyErr_SetFromWindowsErr(0); + } +#else /* MS_WINDOWS */ + int async_err = 0; do { Py_BEGIN_ALLOW_THREADS res = fchmod(fd, mode); @@ -3552,10 +3599,11 @@ os_fchmod_impl(PyObject *module, int fd, int mode) } while (res != 0 && errno == EINTR && !(async_err = PyErr_CheckSignals())); if (res != 0) return (!async_err) ? posix_error() : NULL; +#endif /* MS_WINDOWS */ Py_RETURN_NONE; } -#endif /* HAVE_FCHMOD */ +#endif /* HAVE_FCHMOD || MS_WINDOWS */ #if defined(HAVE_LCHMOD) || defined(MS_WINDOWS) @@ -8327,6 +8375,149 @@ os_sched_getaffinity_impl(PyObject *module, pid_t pid) #endif /* HAVE_SCHED_H */ +#ifdef HAVE_POSIX_OPENPT +/*[clinic input] +os.posix_openpt -> int + + oflag: int + / + +Open and return a file descriptor for a master pseudo-terminal device. + +Performs a posix_openpt() C function call. The oflag argument is used to +set file status flags and file access modes as specified in the manual page +of posix_openpt() of your system. +[clinic start generated code]*/ + +static int +os_posix_openpt_impl(PyObject *module, int oflag) +/*[clinic end generated code: output=ee0bc2624305fc79 input=0de33d0e29693caa]*/ +{ + int fd; + +#if defined(O_CLOEXEC) + oflag |= O_CLOEXEC; +#endif + + fd = posix_openpt(oflag); + if (fd == -1) { + posix_error(); + return -1; + } + + // Just in case, likely a no-op given O_CLOEXEC above. + if (_Py_set_inheritable(fd, 0, NULL) < 0) { + close(fd); + return -1; + } + + return fd; +} +#endif /* HAVE_POSIX_OPENPT */ + +#ifdef HAVE_GRANTPT +/*[clinic input] +os.grantpt + + fd: fildes + File descriptor of a master pseudo-terminal device. + / + +Grant access to the slave pseudo-terminal device. + +Performs a grantpt() C function call. +[clinic start generated code]*/ + +static PyObject * +os_grantpt_impl(PyObject *module, int fd) +/*[clinic end generated code: output=dfd580015cf548ab input=0668e3b96760e849]*/ +{ + int ret; + int saved_errno; + PyOS_sighandler_t sig_saved; + + sig_saved = PyOS_setsig(SIGCHLD, SIG_DFL); + + ret = grantpt(fd); + if (ret == -1) + saved_errno = errno; + + PyOS_setsig(SIGCHLD, sig_saved); + + if (ret == -1) { + errno = saved_errno; + return posix_error(); + } + + Py_RETURN_NONE; +} +#endif /* HAVE_GRANTPT */ + +#ifdef HAVE_UNLOCKPT +/*[clinic input] +os.unlockpt + + fd: fildes + File descriptor of a master pseudo-terminal device. + / + +Unlock a pseudo-terminal master/slave pair. + +Performs an unlockpt() C function call. +[clinic start generated code]*/ + +static PyObject * +os_unlockpt_impl(PyObject *module, int fd) +/*[clinic end generated code: output=e08d354dec12d30c input=de7ab1f59f69a2b4]*/ +{ + if (unlockpt(fd) == -1) + return posix_error(); + + Py_RETURN_NONE; +} +#endif /* HAVE_UNLOCKPT */ + +#if defined(HAVE_PTSNAME) || defined(HAVE_PTSNAME_R) +/*[clinic input] +os.ptsname + + fd: fildes + File descriptor of a master pseudo-terminal device. + / + +Return the name of the slave pseudo-terminal device. + +If the ptsname_r() C function is available, it is called; +otherwise, performs a ptsname() C function call. +[clinic start generated code]*/ + +static PyObject * +os_ptsname_impl(PyObject *module, int fd) +/*[clinic end generated code: output=ef300fadc5675872 input=1369ccc0546f3130]*/ +{ +#ifdef HAVE_PTSNAME_R + int ret; + char name[MAXPATHLEN+1]; + + ret = ptsname_r(fd, name, sizeof(name)); + if (ret != 0) { + errno = ret; + return posix_error(); + } +#else + char *name; + + name = ptsname(fd); + /* POSIX manpage: Upon failure, ptsname() shall return a null pointer and may set errno. + *MAY* set errno? Hmm... */ + if (name == NULL) + return posix_error(); +#endif /* HAVE_PTSNAME_R */ + + return PyUnicode_DecodeFSDefault(name); +} +#endif /* defined(HAVE_PTSNAME) || defined(HAVE_PTSNAME_R) */ + /* AIX uses /dev/ptc but is otherwise the same as /dev/ptmx */ #if defined(HAVE_DEV_PTC) && !defined(HAVE_DEV_PTMX) # define DEV_PTY_FILE "/dev/ptc" @@ -9491,7 +9682,7 @@ os_wait4_impl(PyObject *module, pid_t pid, int options) #endif /* HAVE_WAIT4 */ -#if defined(HAVE_WAITID) && !defined(__APPLE__) +#if defined(HAVE_WAITID) /*[clinic input] os.waitid @@ -9548,7 +9739,7 @@ os_waitid_impl(PyObject *module, idtype_t idtype, id_t id, int options) return result; } -#endif /* defined(HAVE_WAITID) && !defined(__APPLE__) */ +#endif /* defined(HAVE_WAITID) */ #if defined(HAVE_WAITPID) @@ -11551,8 +11742,8 @@ os_pipe_impl(PyObject *module) Py_BEGIN_ALLOW_THREADS ok = CreatePipe(&read, &write, &attr, 0); if (ok) { - fds[0] = _Py_open_osfhandle_noraise(read, _O_RDONLY); - fds[1] = _Py_open_osfhandle_noraise(write, _O_WRONLY); + fds[0] = _Py_open_osfhandle_noraise(read, _O_RDONLY | _O_NOINHERIT); + fds[1] = _Py_open_osfhandle_noraise(write, _O_WRONLY | _O_NOINHERIT); if (fds[0] == -1 || fds[1] == -1) { CloseHandle(read); CloseHandle(write); @@ -16339,6 +16530,10 @@ static PyMethodDef posix_methods[] = { OS_SCHED_YIELD_METHODDEF OS_SCHED_SETAFFINITY_METHODDEF OS_SCHED_GETAFFINITY_METHODDEF + OS_POSIX_OPENPT_METHODDEF + OS_GRANTPT_METHODDEF + OS_UNLOCKPT_METHODDEF + OS_PTSNAME_METHODDEF OS_OPENPTY_METHODDEF OS_LOGIN_TTY_METHODDEF OS_FORKPTY_METHODDEF @@ -17377,7 +17572,7 @@ posixmodule_exec(PyObject *m) return -1; } -#if defined(HAVE_WAITID) && !defined(__APPLE__) +#if defined(HAVE_WAITID) waitid_result_desc.name = MODNAME ".waitid_result"; state->WaitidResultType = (PyObject *)PyStructSequence_NewType(&waitid_result_desc); if (PyModule_AddObjectRef(m, "waitid_result", state->WaitidResultType) < 0) { diff --git a/Modules/pyexpat.c b/Modules/pyexpat.c index 9d95309dbb7aa6..62cd262a7885e9 100644 --- a/Modules/pyexpat.c +++ b/Modules/pyexpat.c @@ -21,7 +21,7 @@ module pyexpat #define XML_COMBINED_VERSION (10000*XML_MAJOR_VERSION+100*XML_MINOR_VERSION+XML_MICRO_VERSION) static XML_Memory_Handling_Suite ExpatMemoryHandler = { - PyObject_Malloc, PyObject_Realloc, PyObject_Free}; + PyMem_Malloc, PyMem_Realloc, PyMem_Free}; enum HandlerTypes { StartElement, @@ -1615,7 +1615,8 @@ static int init_handler_descrs(pyexpat_state *state) if (descr == NULL) return -1; - if (PyDict_SetDefault(state->xml_parse_type->tp_dict, PyDescr_NAME(descr), descr) == NULL) { + if (PyDict_SetDefaultRef(state->xml_parse_type->tp_dict, + PyDescr_NAME(descr), descr, NULL) < 0) { Py_DECREF(descr); return -1; } @@ -2062,9 +2063,7 @@ pyexpat_free(void *module) static PyModuleDef_Slot pyexpat_slots[] = { {Py_mod_exec, pyexpat_exec}, - // XXX gh-103092: fix isolation. - {Py_mod_multiple_interpreters, Py_MOD_MULTIPLE_INTERPRETERS_NOT_SUPPORTED}, - //{Py_mod_multiple_interpreters, Py_MOD_PER_INTERPRETER_GIL_SUPPORTED}, + {Py_mod_multiple_interpreters, Py_MOD_PER_INTERPRETER_GIL_SUPPORTED}, {0, NULL} }; diff --git a/Modules/termios.c b/Modules/termios.c index c4f0fd9d50044a..69dbd88be5fcc2 100644 --- a/Modules/termios.c +++ b/Modules/termios.c @@ -98,6 +98,8 @@ termios_tcgetattr_impl(PyObject *module, int fd) struct termios mode; int r; + /* Alpine Linux can leave some fields uninitialized. */ + memset(&mode, 0, sizeof(mode)); Py_BEGIN_ALLOW_THREADS r = tcgetattr(fd, &mode); Py_END_ALLOW_THREADS diff --git a/Modules/timemodule.c b/Modules/timemodule.c index b3fe175d9b184a..2b0d3900dbddd6 100644 --- a/Modules/timemodule.c +++ b/Modules/timemodule.c @@ -1993,20 +1993,16 @@ time_exec(PyObject *module) return -1; } #endif - #ifdef CLOCK_MONOTONIC - if (PyModule_AddIntMacro(module, CLOCK_MONOTONIC) < 0) { return -1; } - #endif #ifdef CLOCK_MONOTONIC_RAW if (PyModule_AddIntMacro(module, CLOCK_MONOTONIC_RAW) < 0) { return -1; } #endif - #ifdef CLOCK_HIGHRES if (PyModule_AddIntMacro(module, CLOCK_HIGHRES) < 0) { return -1; @@ -2017,7 +2013,6 @@ time_exec(PyObject *module) return -1; } #endif - #ifdef CLOCK_THREAD_CPUTIME_ID if (PyModule_AddIntMacro(module, CLOCK_THREAD_CPUTIME_ID) < 0) { return -1; @@ -2044,10 +2039,19 @@ time_exec(PyObject *module) } #endif #ifdef CLOCK_UPTIME_RAW - if (PyModule_AddIntMacro(module, CLOCK_UPTIME_RAW) < 0) { return -1; } +#endif +#ifdef CLOCK_MONOTONIC_RAW_APPROX + if (PyModule_AddIntMacro(module, CLOCK_MONOTONIC_RAW_APPROX) < 0) { + return -1; + } +#endif +#ifdef CLOCK_UPTIME_RAW_APPROX + if (PyModule_AddIntMacro(module, CLOCK_UPTIME_RAW_APPROX) < 0) { + return -1; + } #endif } diff --git a/Objects/abstract.c b/Objects/abstract.c index 1ec5c5b8c3dc2f..07d4b89fe188c8 100644 --- a/Objects/abstract.c +++ b/Objects/abstract.c @@ -425,6 +425,12 @@ PyObject_AsWriteBuffer(PyObject *obj, int PyObject_GetBuffer(PyObject *obj, Py_buffer *view, int flags) { + if (flags != PyBUF_SIMPLE) { /* fast path */ + if (flags == PyBUF_READ || flags == PyBUF_WRITE) { + PyErr_BadInternalCall(); + return -1; + } + } PyBufferProcs *pb = Py_TYPE(obj)->tp_as_buffer; if (pb == NULL || pb->bf_getbuffer == NULL) { @@ -761,11 +767,17 @@ PyBuffer_FillInfo(Py_buffer *view, PyObject *obj, void *buf, Py_ssize_t len, return -1; } - if (((flags & PyBUF_WRITABLE) == PyBUF_WRITABLE) && - (readonly == 1)) { - PyErr_SetString(PyExc_BufferError, - "Object is not writable."); - return -1; + if (flags != PyBUF_SIMPLE) { /* fast path */ + if (flags == PyBUF_READ || flags == PyBUF_WRITE) { + PyErr_BadInternalCall(); + return -1; + } + if (((flags & PyBUF_WRITABLE) == PyBUF_WRITABLE) && + (readonly == 1)) { + PyErr_SetString(PyExc_BufferError, + "Object is not writable."); + return -1; + } } view->obj = Py_XNewRef(obj); diff --git a/Objects/bytearrayobject.c b/Objects/bytearrayobject.c index 659de7d3dd5a99..acc59b926448ca 100644 --- a/Objects/bytearrayobject.c +++ b/Objects/bytearrayobject.c @@ -132,7 +132,7 @@ PyByteArray_FromStringAndSize(const char *bytes, Py_ssize_t size) } else { alloc = size + 1; - new->ob_bytes = PyObject_Malloc(alloc); + new->ob_bytes = PyMem_Malloc(alloc); if (new->ob_bytes == NULL) { Py_DECREF(new); return PyErr_NoMemory(); @@ -221,17 +221,17 @@ PyByteArray_Resize(PyObject *self, Py_ssize_t requested_size) } if (logical_offset > 0) { - sval = PyObject_Malloc(alloc); + sval = PyMem_Malloc(alloc); if (sval == NULL) { PyErr_NoMemory(); return -1; } memcpy(sval, PyByteArray_AS_STRING(self), Py_MIN((size_t)requested_size, (size_t)Py_SIZE(self))); - PyObject_Free(obj->ob_bytes); + PyMem_Free(obj->ob_bytes); } else { - sval = PyObject_Realloc(obj->ob_bytes, alloc); + sval = PyMem_Realloc(obj->ob_bytes, alloc); if (sval == NULL) { PyErr_NoMemory(); return -1; @@ -951,7 +951,7 @@ bytearray_repr(PyByteArrayObject *self) } newsize += 6 + length * 4; - buffer = PyObject_Malloc(newsize); + buffer = PyMem_Malloc(newsize); if (buffer == NULL) { PyErr_NoMemory(); return NULL; @@ -1008,7 +1008,7 @@ bytearray_repr(PyByteArrayObject *self) } v = PyUnicode_FromStringAndSize(buffer, p - buffer); - PyObject_Free(buffer); + PyMem_Free(buffer); return v; } @@ -1088,7 +1088,7 @@ bytearray_dealloc(PyByteArrayObject *self) PyErr_Print(); } if (self->ob_bytes != 0) { - PyObject_Free(self->ob_bytes); + PyMem_Free(self->ob_bytes); } Py_TYPE(self)->tp_free((PyObject *)self); } diff --git a/Objects/clinic/dictobject.c.h b/Objects/clinic/dictobject.c.h index 641514235c2341..daaef211b1db49 100644 --- a/Objects/clinic/dictobject.c.h +++ b/Objects/clinic/dictobject.c.h @@ -2,6 +2,7 @@ preserve [clinic start generated code]*/ +#include "pycore_critical_section.h"// Py_BEGIN_CRITICAL_SECTION() #include "pycore_modsupport.h" // _PyArg_CheckPositional() PyDoc_STRVAR(dict_fromkeys__doc__, @@ -38,6 +39,24 @@ dict_fromkeys(PyTypeObject *type, PyObject *const *args, Py_ssize_t nargs) return return_value; } +PyDoc_STRVAR(dict_copy__doc__, +"copy($self, /)\n" +"--\n" +"\n" +"Return a shallow copy of the dict."); + +#define DICT_COPY_METHODDEF \ + {"copy", (PyCFunction)dict_copy, METH_NOARGS, dict_copy__doc__}, + +static PyObject * +dict_copy_impl(PyDictObject *self); + +static PyObject * +dict_copy(PyDictObject *self, PyObject *Py_UNUSED(ignored)) +{ + return dict_copy_impl(self); +} + PyDoc_STRVAR(dict___contains____doc__, "__contains__($self, key, /)\n" "--\n" @@ -47,6 +66,21 @@ PyDoc_STRVAR(dict___contains____doc__, #define DICT___CONTAINS___METHODDEF \ {"__contains__", (PyCFunction)dict___contains__, METH_O|METH_COEXIST, dict___contains____doc__}, +static PyObject * +dict___contains___impl(PyDictObject *self, PyObject *key); + +static PyObject * +dict___contains__(PyDictObject *self, PyObject *key) +{ + PyObject *return_value = NULL; + + Py_BEGIN_CRITICAL_SECTION(self); + return_value = dict___contains___impl(self, key); + Py_END_CRITICAL_SECTION(); + + return return_value; +} + PyDoc_STRVAR(dict_get__doc__, "get($self, key, default=None, /)\n" "--\n" @@ -75,7 +109,9 @@ dict_get(PyDictObject *self, PyObject *const *args, Py_ssize_t nargs) } default_value = args[1]; skip_optional: + Py_BEGIN_CRITICAL_SECTION(self); return_value = dict_get_impl(self, key, default_value); + Py_END_CRITICAL_SECTION(); exit: return return_value; @@ -112,12 +148,32 @@ dict_setdefault(PyDictObject *self, PyObject *const *args, Py_ssize_t nargs) } default_value = args[1]; skip_optional: + Py_BEGIN_CRITICAL_SECTION(self); return_value = dict_setdefault_impl(self, key, default_value); + Py_END_CRITICAL_SECTION(); exit: return return_value; } +PyDoc_STRVAR(dict_clear__doc__, +"clear($self, /)\n" +"--\n" +"\n" +"Remove all items from the dict."); + +#define DICT_CLEAR_METHODDEF \ + {"clear", (PyCFunction)dict_clear, METH_NOARGS, dict_clear__doc__}, + +static PyObject * +dict_clear_impl(PyDictObject *self); + +static PyObject * +dict_clear(PyDictObject *self, PyObject *Py_UNUSED(ignored)) +{ + return dict_clear_impl(self); +} + PyDoc_STRVAR(dict_pop__doc__, "pop($self, key, default=, /)\n" "--\n" @@ -173,7 +229,31 @@ dict_popitem_impl(PyDictObject *self); static PyObject * dict_popitem(PyDictObject *self, PyObject *Py_UNUSED(ignored)) { - return dict_popitem_impl(self); + PyObject *return_value = NULL; + + Py_BEGIN_CRITICAL_SECTION(self); + return_value = dict_popitem_impl(self); + Py_END_CRITICAL_SECTION(); + + return return_value; +} + +PyDoc_STRVAR(dict___sizeof____doc__, +"__sizeof__($self, /)\n" +"--\n" +"\n" +"Return the size of the dict in memory, in bytes."); + +#define DICT___SIZEOF___METHODDEF \ + {"__sizeof__", (PyCFunction)dict___sizeof__, METH_NOARGS, dict___sizeof____doc__}, + +static PyObject * +dict___sizeof___impl(PyDictObject *self); + +static PyObject * +dict___sizeof__(PyDictObject *self, PyObject *Py_UNUSED(ignored)) +{ + return dict___sizeof___impl(self); } PyDoc_STRVAR(dict___reversed____doc__, @@ -193,4 +273,58 @@ dict___reversed__(PyDictObject *self, PyObject *Py_UNUSED(ignored)) { return dict___reversed___impl(self); } -/*[clinic end generated code: output=17c3c4cf9a9b95a7 input=a9049054013a1b77]*/ + +PyDoc_STRVAR(dict_keys__doc__, +"keys($self, /)\n" +"--\n" +"\n" +"Return a set-like object providing a view on the dict\'s keys."); + +#define DICT_KEYS_METHODDEF \ + {"keys", (PyCFunction)dict_keys, METH_NOARGS, dict_keys__doc__}, + +static PyObject * +dict_keys_impl(PyDictObject *self); + +static PyObject * +dict_keys(PyDictObject *self, PyObject *Py_UNUSED(ignored)) +{ + return dict_keys_impl(self); +} + +PyDoc_STRVAR(dict_items__doc__, +"items($self, /)\n" +"--\n" +"\n" +"Return a set-like object providing a view on the dict\'s items."); + +#define DICT_ITEMS_METHODDEF \ + {"items", (PyCFunction)dict_items, METH_NOARGS, dict_items__doc__}, + +static PyObject * +dict_items_impl(PyDictObject *self); + +static PyObject * +dict_items(PyDictObject *self, PyObject *Py_UNUSED(ignored)) +{ + return dict_items_impl(self); +} + +PyDoc_STRVAR(dict_values__doc__, +"values($self, /)\n" +"--\n" +"\n" +"Return an object providing a view on the dict\'s values."); + +#define DICT_VALUES_METHODDEF \ + {"values", (PyCFunction)dict_values, METH_NOARGS, dict_values__doc__}, + +static PyObject * +dict_values_impl(PyDictObject *self); + +static PyObject * +dict_values(PyDictObject *self, PyObject *Py_UNUSED(ignored)) +{ + return dict_values_impl(self); +} +/*[clinic end generated code: output=c8fda06bac5b05f3 input=a9049054013a1b77]*/ diff --git a/Objects/clinic/funcobject.c.h b/Objects/clinic/funcobject.c.h index 138f87716acbf7..8f20bda26438cf 100644 --- a/Objects/clinic/funcobject.c.h +++ b/Objects/clinic/funcobject.c.h @@ -9,7 +9,8 @@ preserve #include "pycore_modsupport.h" // _PyArg_UnpackKeywords() PyDoc_STRVAR(func_new__doc__, -"function(code, globals, name=None, argdefs=None, closure=None)\n" +"function(code, globals, name=None, argdefs=None, closure=None,\n" +" kwdefaults=None)\n" "--\n" "\n" "Create a function object.\n" @@ -23,11 +24,14 @@ PyDoc_STRVAR(func_new__doc__, " argdefs\n" " a tuple that specifies the default argument values\n" " closure\n" -" a tuple that supplies the bindings for free variables"); +" a tuple that supplies the bindings for free variables\n" +" kwdefaults\n" +" a dictionary that specifies the default keyword argument values"); static PyObject * func_new_impl(PyTypeObject *type, PyCodeObject *code, PyObject *globals, - PyObject *name, PyObject *defaults, PyObject *closure); + PyObject *name, PyObject *defaults, PyObject *closure, + PyObject *kwdefaults); static PyObject * func_new(PyTypeObject *type, PyObject *args, PyObject *kwargs) @@ -35,14 +39,14 @@ func_new(PyTypeObject *type, PyObject *args, PyObject *kwargs) PyObject *return_value = NULL; #if defined(Py_BUILD_CORE) && !defined(Py_BUILD_CORE_MODULE) - #define NUM_KEYWORDS 5 + #define NUM_KEYWORDS 6 static struct { PyGC_Head _this_is_not_used; PyObject_VAR_HEAD PyObject *ob_item[NUM_KEYWORDS]; } _kwtuple = { .ob_base = PyVarObject_HEAD_INIT(&PyTuple_Type, NUM_KEYWORDS) - .ob_item = { &_Py_ID(code), &_Py_ID(globals), &_Py_ID(name), &_Py_ID(argdefs), &_Py_ID(closure), }, + .ob_item = { &_Py_ID(code), &_Py_ID(globals), &_Py_ID(name), &_Py_ID(argdefs), &_Py_ID(closure), &_Py_ID(kwdefaults), }, }; #undef NUM_KEYWORDS #define KWTUPLE (&_kwtuple.ob_base.ob_base) @@ -51,14 +55,14 @@ func_new(PyTypeObject *type, PyObject *args, PyObject *kwargs) # define KWTUPLE NULL #endif // !Py_BUILD_CORE - static const char * const _keywords[] = {"code", "globals", "name", "argdefs", "closure", NULL}; + static const char * const _keywords[] = {"code", "globals", "name", "argdefs", "closure", "kwdefaults", NULL}; static _PyArg_Parser _parser = { .keywords = _keywords, .fname = "function", .kwtuple = KWTUPLE, }; #undef KWTUPLE - PyObject *argsbuf[5]; + PyObject *argsbuf[6]; PyObject * const *fastargs; Py_ssize_t nargs = PyTuple_GET_SIZE(args); Py_ssize_t noptargs = nargs + (kwargs ? PyDict_GET_SIZE(kwargs) : 0) - 2; @@ -67,8 +71,9 @@ func_new(PyTypeObject *type, PyObject *args, PyObject *kwargs) PyObject *name = Py_None; PyObject *defaults = Py_None; PyObject *closure = Py_None; + PyObject *kwdefaults = Py_None; - fastargs = _PyArg_UnpackKeywords(_PyTuple_CAST(args)->ob_item, nargs, kwargs, NULL, &_parser, 2, 5, 0, argsbuf); + fastargs = _PyArg_UnpackKeywords(_PyTuple_CAST(args)->ob_item, nargs, kwargs, NULL, &_parser, 2, 6, 0, argsbuf); if (!fastargs) { goto exit; } @@ -97,11 +102,17 @@ func_new(PyTypeObject *type, PyObject *args, PyObject *kwargs) goto skip_optional_pos; } } - closure = fastargs[4]; + if (fastargs[4]) { + closure = fastargs[4]; + if (!--noptargs) { + goto skip_optional_pos; + } + } + kwdefaults = fastargs[5]; skip_optional_pos: - return_value = func_new_impl(type, code, globals, name, defaults, closure); + return_value = func_new_impl(type, code, globals, name, defaults, closure, kwdefaults); exit: return return_value; } -/*[clinic end generated code: output=ff7b995500d2bee6 input=a9049054013a1b77]*/ +/*[clinic end generated code: output=10947342188f38a9 input=a9049054013a1b77]*/ diff --git a/Objects/clinic/listobject.c.h b/Objects/clinic/listobject.c.h index 54e6060451f3ff..a61550a49b66fc 100644 --- a/Objects/clinic/listobject.c.h +++ b/Objects/clinic/listobject.c.h @@ -7,6 +7,7 @@ preserve # include "pycore_runtime.h" // _Py_ID() #endif #include "pycore_abstract.h" // _PyNumber_Index() +#include "pycore_critical_section.h"// Py_BEGIN_CRITICAL_SECTION() #include "pycore_modsupport.h" // _PyArg_CheckPositional() PyDoc_STRVAR(list_insert__doc__, @@ -44,7 +45,9 @@ list_insert(PyListObject *self, PyObject *const *args, Py_ssize_t nargs) index = ival; } object = args[1]; + Py_BEGIN_CRITICAL_SECTION(self); return_value = list_insert_impl(self, index, object); + Py_END_CRITICAL_SECTION(); exit: return return_value; @@ -65,7 +68,13 @@ py_list_clear_impl(PyListObject *self); static PyObject * py_list_clear(PyListObject *self, PyObject *Py_UNUSED(ignored)) { - return py_list_clear_impl(self); + PyObject *return_value = NULL; + + Py_BEGIN_CRITICAL_SECTION(self); + return_value = py_list_clear_impl(self); + Py_END_CRITICAL_SECTION(); + + return return_value; } PyDoc_STRVAR(list_copy__doc__, @@ -83,7 +92,13 @@ list_copy_impl(PyListObject *self); static PyObject * list_copy(PyListObject *self, PyObject *Py_UNUSED(ignored)) { - return list_copy_impl(self); + PyObject *return_value = NULL; + + Py_BEGIN_CRITICAL_SECTION(self); + return_value = list_copy_impl(self); + Py_END_CRITICAL_SECTION(); + + return return_value; } PyDoc_STRVAR(list_append__doc__, @@ -95,6 +110,21 @@ PyDoc_STRVAR(list_append__doc__, #define LIST_APPEND_METHODDEF \ {"append", (PyCFunction)list_append, METH_O, list_append__doc__}, +static PyObject * +list_append_impl(PyListObject *self, PyObject *object); + +static PyObject * +list_append(PyListObject *self, PyObject *object) +{ + PyObject *return_value = NULL; + + Py_BEGIN_CRITICAL_SECTION(self); + return_value = list_append_impl(self, object); + Py_END_CRITICAL_SECTION(); + + return return_value; +} + PyDoc_STRVAR(py_list_extend__doc__, "extend($self, iterable, /)\n" "--\n" @@ -104,6 +134,21 @@ PyDoc_STRVAR(py_list_extend__doc__, #define PY_LIST_EXTEND_METHODDEF \ {"extend", (PyCFunction)py_list_extend, METH_O, py_list_extend__doc__}, +static PyObject * +py_list_extend_impl(PyListObject *self, PyObject *iterable); + +static PyObject * +py_list_extend(PyListObject *self, PyObject *iterable) +{ + PyObject *return_value = NULL; + + Py_BEGIN_CRITICAL_SECTION2(self, iterable); + return_value = py_list_extend_impl(self, iterable); + Py_END_CRITICAL_SECTION2(); + + return return_value; +} + PyDoc_STRVAR(list_pop__doc__, "pop($self, index=-1, /)\n" "--\n" @@ -143,7 +188,9 @@ list_pop(PyListObject *self, PyObject *const *args, Py_ssize_t nargs) index = ival; } skip_optional: + Py_BEGIN_CRITICAL_SECTION(self); return_value = list_pop_impl(self, index); + Py_END_CRITICAL_SECTION(); exit: return return_value; @@ -242,7 +289,13 @@ list_reverse_impl(PyListObject *self); static PyObject * list_reverse(PyListObject *self, PyObject *Py_UNUSED(ignored)) { - return list_reverse_impl(self); + PyObject *return_value = NULL; + + Py_BEGIN_CRITICAL_SECTION(self); + return_value = list_reverse_impl(self); + Py_END_CRITICAL_SECTION(); + + return return_value; } PyDoc_STRVAR(list_index__doc__, @@ -311,6 +364,21 @@ PyDoc_STRVAR(list_remove__doc__, #define LIST_REMOVE_METHODDEF \ {"remove", (PyCFunction)list_remove, METH_O, list_remove__doc__}, +static PyObject * +list_remove_impl(PyListObject *self, PyObject *value); + +static PyObject * +list_remove(PyListObject *self, PyObject *value) +{ + PyObject *return_value = NULL; + + Py_BEGIN_CRITICAL_SECTION(self); + return_value = list_remove_impl(self, value); + Py_END_CRITICAL_SECTION(); + + return return_value; +} + PyDoc_STRVAR(list___init____doc__, "list(iterable=(), /)\n" "--\n" @@ -384,4 +452,4 @@ list___reversed__(PyListObject *self, PyObject *Py_UNUSED(ignored)) { return list___reversed___impl(self); } -/*[clinic end generated code: output=f2d7b63119464ff4 input=a9049054013a1b77]*/ +/*[clinic end generated code: output=26dfb2c9846348f9 input=a9049054013a1b77]*/ diff --git a/Objects/clinic/setobject.c.h b/Objects/clinic/setobject.c.h new file mode 100644 index 00000000000000..f3c96995ede60d --- /dev/null +++ b/Objects/clinic/setobject.c.h @@ -0,0 +1,414 @@ +/*[clinic input] +preserve +[clinic start generated code]*/ + +#include "pycore_modsupport.h" // _PyArg_CheckPositional() + +PyDoc_STRVAR(set_pop__doc__, +"pop($self, /)\n" +"--\n" +"\n" +"Remove and return an arbitrary set element.\n" +"\n" +"Raises KeyError if the set is empty."); + +#define SET_POP_METHODDEF \ + {"pop", (PyCFunction)set_pop, METH_NOARGS, set_pop__doc__}, + +static PyObject * +set_pop_impl(PySetObject *so); + +static PyObject * +set_pop(PySetObject *so, PyObject *Py_UNUSED(ignored)) +{ + return set_pop_impl(so); +} + +PyDoc_STRVAR(set_update__doc__, +"update($self, /, *others)\n" +"--\n" +"\n" +"Update the set, adding elements from all others."); + +#define SET_UPDATE_METHODDEF \ + {"update", _PyCFunction_CAST(set_update), METH_FASTCALL, set_update__doc__}, + +static PyObject * +set_update_impl(PySetObject *so, PyObject *args); + +static PyObject * +set_update(PySetObject *so, PyObject *const *args, Py_ssize_t nargs) +{ + PyObject *return_value = NULL; + PyObject *__clinic_args = NULL; + + if (!_PyArg_CheckPositional("update", nargs, 0, PY_SSIZE_T_MAX)) { + goto exit; + } + __clinic_args = PyTuple_New(nargs - 0); + if (!__clinic_args) { + goto exit; + } + for (Py_ssize_t i = 0; i < nargs - 0; ++i) { + PyTuple_SET_ITEM(__clinic_args, i, Py_NewRef(args[0 + i])); + } + return_value = set_update_impl(so, __clinic_args); + +exit: + Py_XDECREF(__clinic_args); + return return_value; +} + +PyDoc_STRVAR(set_copy__doc__, +"copy($self, /)\n" +"--\n" +"\n" +"Return a shallow copy of a set."); + +#define SET_COPY_METHODDEF \ + {"copy", (PyCFunction)set_copy, METH_NOARGS, set_copy__doc__}, + +static PyObject * +set_copy_impl(PySetObject *so); + +static PyObject * +set_copy(PySetObject *so, PyObject *Py_UNUSED(ignored)) +{ + return set_copy_impl(so); +} + +PyDoc_STRVAR(frozenset_copy__doc__, +"copy($self, /)\n" +"--\n" +"\n" +"Return a shallow copy of a set."); + +#define FROZENSET_COPY_METHODDEF \ + {"copy", (PyCFunction)frozenset_copy, METH_NOARGS, frozenset_copy__doc__}, + +static PyObject * +frozenset_copy_impl(PySetObject *so); + +static PyObject * +frozenset_copy(PySetObject *so, PyObject *Py_UNUSED(ignored)) +{ + return frozenset_copy_impl(so); +} + +PyDoc_STRVAR(set_clear__doc__, +"clear($self, /)\n" +"--\n" +"\n" +"Remove all elements from this set."); + +#define SET_CLEAR_METHODDEF \ + {"clear", (PyCFunction)set_clear, METH_NOARGS, set_clear__doc__}, + +static PyObject * +set_clear_impl(PySetObject *so); + +static PyObject * +set_clear(PySetObject *so, PyObject *Py_UNUSED(ignored)) +{ + return set_clear_impl(so); +} + +PyDoc_STRVAR(set_union__doc__, +"union($self, /, *others)\n" +"--\n" +"\n" +"Return a new set with elements from the set and all others."); + +#define SET_UNION_METHODDEF \ + {"union", _PyCFunction_CAST(set_union), METH_FASTCALL, set_union__doc__}, + +static PyObject * +set_union_impl(PySetObject *so, PyObject *args); + +static PyObject * +set_union(PySetObject *so, PyObject *const *args, Py_ssize_t nargs) +{ + PyObject *return_value = NULL; + PyObject *__clinic_args = NULL; + + if (!_PyArg_CheckPositional("union", nargs, 0, PY_SSIZE_T_MAX)) { + goto exit; + } + __clinic_args = PyTuple_New(nargs - 0); + if (!__clinic_args) { + goto exit; + } + for (Py_ssize_t i = 0; i < nargs - 0; ++i) { + PyTuple_SET_ITEM(__clinic_args, i, Py_NewRef(args[0 + i])); + } + return_value = set_union_impl(so, __clinic_args); + +exit: + Py_XDECREF(__clinic_args); + return return_value; +} + +PyDoc_STRVAR(set_intersection_multi__doc__, +"intersection($self, /, *others)\n" +"--\n" +"\n" +"Return a new set with elements common to the set and all others."); + +#define SET_INTERSECTION_MULTI_METHODDEF \ + {"intersection", _PyCFunction_CAST(set_intersection_multi), METH_FASTCALL, set_intersection_multi__doc__}, + +static PyObject * +set_intersection_multi_impl(PySetObject *so, PyObject *args); + +static PyObject * +set_intersection_multi(PySetObject *so, PyObject *const *args, Py_ssize_t nargs) +{ + PyObject *return_value = NULL; + PyObject *__clinic_args = NULL; + + if (!_PyArg_CheckPositional("intersection", nargs, 0, PY_SSIZE_T_MAX)) { + goto exit; + } + __clinic_args = PyTuple_New(nargs - 0); + if (!__clinic_args) { + goto exit; + } + for (Py_ssize_t i = 0; i < nargs - 0; ++i) { + PyTuple_SET_ITEM(__clinic_args, i, Py_NewRef(args[0 + i])); + } + return_value = set_intersection_multi_impl(so, __clinic_args); + +exit: + Py_XDECREF(__clinic_args); + return return_value; +} + +PyDoc_STRVAR(set_intersection_update_multi__doc__, +"intersection_update($self, /, *others)\n" +"--\n" +"\n" +"Update the set, keeping only elements found in it and all others."); + +#define SET_INTERSECTION_UPDATE_MULTI_METHODDEF \ + {"intersection_update", _PyCFunction_CAST(set_intersection_update_multi), METH_FASTCALL, set_intersection_update_multi__doc__}, + +static PyObject * +set_intersection_update_multi_impl(PySetObject *so, PyObject *args); + +static PyObject * +set_intersection_update_multi(PySetObject *so, PyObject *const *args, Py_ssize_t nargs) +{ + PyObject *return_value = NULL; + PyObject *__clinic_args = NULL; + + if (!_PyArg_CheckPositional("intersection_update", nargs, 0, PY_SSIZE_T_MAX)) { + goto exit; + } + __clinic_args = PyTuple_New(nargs - 0); + if (!__clinic_args) { + goto exit; + } + for (Py_ssize_t i = 0; i < nargs - 0; ++i) { + PyTuple_SET_ITEM(__clinic_args, i, Py_NewRef(args[0 + i])); + } + return_value = set_intersection_update_multi_impl(so, __clinic_args); + +exit: + Py_XDECREF(__clinic_args); + return return_value; +} + +PyDoc_STRVAR(set_isdisjoint__doc__, +"isdisjoint($self, other, /)\n" +"--\n" +"\n" +"Return True if two sets have a null intersection."); + +#define SET_ISDISJOINT_METHODDEF \ + {"isdisjoint", (PyCFunction)set_isdisjoint, METH_O, set_isdisjoint__doc__}, + +PyDoc_STRVAR(set_difference_update__doc__, +"difference_update($self, /, *others)\n" +"--\n" +"\n" +"Update the set, removing elements found in others."); + +#define SET_DIFFERENCE_UPDATE_METHODDEF \ + {"difference_update", _PyCFunction_CAST(set_difference_update), METH_FASTCALL, set_difference_update__doc__}, + +static PyObject * +set_difference_update_impl(PySetObject *so, PyObject *args); + +static PyObject * +set_difference_update(PySetObject *so, PyObject *const *args, Py_ssize_t nargs) +{ + PyObject *return_value = NULL; + PyObject *__clinic_args = NULL; + + if (!_PyArg_CheckPositional("difference_update", nargs, 0, PY_SSIZE_T_MAX)) { + goto exit; + } + __clinic_args = PyTuple_New(nargs - 0); + if (!__clinic_args) { + goto exit; + } + for (Py_ssize_t i = 0; i < nargs - 0; ++i) { + PyTuple_SET_ITEM(__clinic_args, i, Py_NewRef(args[0 + i])); + } + return_value = set_difference_update_impl(so, __clinic_args); + +exit: + Py_XDECREF(__clinic_args); + return return_value; +} + +PyDoc_STRVAR(set_difference_multi__doc__, +"difference($self, /, *others)\n" +"--\n" +"\n" +"Return a new set with elements in the set that are not in the others."); + +#define SET_DIFFERENCE_MULTI_METHODDEF \ + {"difference", _PyCFunction_CAST(set_difference_multi), METH_FASTCALL, set_difference_multi__doc__}, + +static PyObject * +set_difference_multi_impl(PySetObject *so, PyObject *args); + +static PyObject * +set_difference_multi(PySetObject *so, PyObject *const *args, Py_ssize_t nargs) +{ + PyObject *return_value = NULL; + PyObject *__clinic_args = NULL; + + if (!_PyArg_CheckPositional("difference", nargs, 0, PY_SSIZE_T_MAX)) { + goto exit; + } + __clinic_args = PyTuple_New(nargs - 0); + if (!__clinic_args) { + goto exit; + } + for (Py_ssize_t i = 0; i < nargs - 0; ++i) { + PyTuple_SET_ITEM(__clinic_args, i, Py_NewRef(args[0 + i])); + } + return_value = set_difference_multi_impl(so, __clinic_args); + +exit: + Py_XDECREF(__clinic_args); + return return_value; +} + +PyDoc_STRVAR(set_symmetric_difference_update__doc__, +"symmetric_difference_update($self, other, /)\n" +"--\n" +"\n" +"Update the set, keeping only elements found in either set, but not in both."); + +#define SET_SYMMETRIC_DIFFERENCE_UPDATE_METHODDEF \ + {"symmetric_difference_update", (PyCFunction)set_symmetric_difference_update, METH_O, set_symmetric_difference_update__doc__}, + +PyDoc_STRVAR(set_symmetric_difference__doc__, +"symmetric_difference($self, other, /)\n" +"--\n" +"\n" +"Return a new set with elements in either the set or other but not both."); + +#define SET_SYMMETRIC_DIFFERENCE_METHODDEF \ + {"symmetric_difference", (PyCFunction)set_symmetric_difference, METH_O, set_symmetric_difference__doc__}, + +PyDoc_STRVAR(set_issubset__doc__, +"issubset($self, other, /)\n" +"--\n" +"\n" +"Report whether another set contains this set."); + +#define SET_ISSUBSET_METHODDEF \ + {"issubset", (PyCFunction)set_issubset, METH_O, set_issubset__doc__}, + +PyDoc_STRVAR(set_issuperset__doc__, +"issuperset($self, other, /)\n" +"--\n" +"\n" +"Report whether this set contains another set."); + +#define SET_ISSUPERSET_METHODDEF \ + {"issuperset", (PyCFunction)set_issuperset, METH_O, set_issuperset__doc__}, + +PyDoc_STRVAR(set_add__doc__, +"add($self, object, /)\n" +"--\n" +"\n" +"Add an element to a set.\n" +"\n" +"This has no effect if the element is already present."); + +#define SET_ADD_METHODDEF \ + {"add", (PyCFunction)set_add, METH_O, set_add__doc__}, + +PyDoc_STRVAR(set___contains____doc__, +"__contains__($self, object, /)\n" +"--\n" +"\n" +"x.__contains__(y) <==> y in x."); + +#define SET___CONTAINS___METHODDEF \ + {"__contains__", (PyCFunction)set___contains__, METH_O|METH_COEXIST, set___contains____doc__}, + +PyDoc_STRVAR(set_remove__doc__, +"remove($self, object, /)\n" +"--\n" +"\n" +"Remove an element from a set; it must be a member.\n" +"\n" +"If the element is not a member, raise a KeyError."); + +#define SET_REMOVE_METHODDEF \ + {"remove", (PyCFunction)set_remove, METH_O, set_remove__doc__}, + +PyDoc_STRVAR(set_discard__doc__, +"discard($self, object, /)\n" +"--\n" +"\n" +"Remove an element from a set if it is a member.\n" +"\n" +"Unlike set.remove(), the discard() method does not raise\n" +"an exception when an element is missing from the set."); + +#define SET_DISCARD_METHODDEF \ + {"discard", (PyCFunction)set_discard, METH_O, set_discard__doc__}, + +PyDoc_STRVAR(set___reduce____doc__, +"__reduce__($self, /)\n" +"--\n" +"\n" +"Return state information for pickling."); + +#define SET___REDUCE___METHODDEF \ + {"__reduce__", (PyCFunction)set___reduce__, METH_NOARGS, set___reduce____doc__}, + +static PyObject * +set___reduce___impl(PySetObject *so); + +static PyObject * +set___reduce__(PySetObject *so, PyObject *Py_UNUSED(ignored)) +{ + return set___reduce___impl(so); +} + +PyDoc_STRVAR(set___sizeof____doc__, +"__sizeof__($self, /)\n" +"--\n" +"\n" +"S.__sizeof__() -> size of S in memory, in bytes."); + +#define SET___SIZEOF___METHODDEF \ + {"__sizeof__", (PyCFunction)set___sizeof__, METH_NOARGS, set___sizeof____doc__}, + +static PyObject * +set___sizeof___impl(PySetObject *so); + +static PyObject * +set___sizeof__(PySetObject *so, PyObject *Py_UNUSED(ignored)) +{ + return set___sizeof___impl(so); +} +/*[clinic end generated code: output=34a30591148da884 input=a9049054013a1b77]*/ diff --git a/Objects/complexobject.c b/Objects/complexobject.c index 0e96f54584677c..d8b0e84da5df4a 100644 --- a/Objects/complexobject.c +++ b/Objects/complexobject.c @@ -256,26 +256,51 @@ PyComplex_FromDoubles(double real, double imag) return PyComplex_FromCComplex(c); } +static PyObject * try_complex_special_method(PyObject *); + double PyComplex_RealAsDouble(PyObject *op) { + double real = -1.0; + if (PyComplex_Check(op)) { - return ((PyComplexObject *)op)->cval.real; + real = ((PyComplexObject *)op)->cval.real; } else { - return PyFloat_AsDouble(op); + PyObject* newop = try_complex_special_method(op); + if (newop) { + real = ((PyComplexObject *)newop)->cval.real; + Py_DECREF(newop); + } else if (!PyErr_Occurred()) { + real = PyFloat_AsDouble(op); + } } + + return real; } double PyComplex_ImagAsDouble(PyObject *op) { + double imag = -1.0; + if (PyComplex_Check(op)) { - return ((PyComplexObject *)op)->cval.imag; + imag = ((PyComplexObject *)op)->cval.imag; } else { - return 0.0; + PyObject* newop = try_complex_special_method(op); + if (newop) { + imag = ((PyComplexObject *)newop)->cval.imag; + Py_DECREF(newop); + } else if (!PyErr_Occurred()) { + PyFloat_AsDouble(op); + if (!PyErr_Occurred()) { + imag = 0.0; + } + } } + + return imag; } static PyObject * diff --git a/Objects/descrobject.c b/Objects/descrobject.c index 57921b110591e5..8d771adf307dc4 100644 --- a/Objects/descrobject.c +++ b/Objects/descrobject.c @@ -19,8 +19,9 @@ class property "propertyobject *" "&PyProperty_Type" /*[clinic end generated code: output=da39a3ee5e6b4b0d input=556352653fd4c02e]*/ static void -descr_dealloc(PyDescrObject *descr) +descr_dealloc(PyObject *self) { + PyDescrObject *descr = (PyDescrObject *)self; _PyObject_GC_UNTRACK(descr); Py_XDECREF(descr->d_type); Py_XDECREF(descr->d_name); @@ -47,28 +48,28 @@ descr_repr(PyDescrObject *descr, const char *format) } static PyObject * -method_repr(PyMethodDescrObject *descr) +method_repr(PyObject *descr) { return descr_repr((PyDescrObject *)descr, ""); } static PyObject * -member_repr(PyMemberDescrObject *descr) +member_repr(PyObject *descr) { return descr_repr((PyDescrObject *)descr, ""); } static PyObject * -getset_repr(PyGetSetDescrObject *descr) +getset_repr(PyObject *descr) { return descr_repr((PyDescrObject *)descr, ""); } static PyObject * -wrapperdescr_repr(PyWrapperDescrObject *descr) +wrapperdescr_repr(PyObject *descr) { return descr_repr((PyDescrObject *)descr, ""); @@ -90,8 +91,9 @@ descr_check(PyDescrObject *descr, PyObject *obj) } static PyObject * -classmethod_get(PyMethodDescrObject *descr, PyObject *obj, PyObject *type) +classmethod_get(PyObject *self, PyObject *obj, PyObject *type) { + PyMethodDescrObject *descr = (PyMethodDescrObject *)self; /* Ensure a valid type. Class methods ignore obj. */ if (type == NULL) { if (obj != NULL) @@ -132,8 +134,9 @@ classmethod_get(PyMethodDescrObject *descr, PyObject *obj, PyObject *type) } static PyObject * -method_get(PyMethodDescrObject *descr, PyObject *obj, PyObject *type) +method_get(PyObject *self, PyObject *obj, PyObject *type) { + PyMethodDescrObject *descr = (PyMethodDescrObject *)self; if (obj == NULL) { return Py_NewRef(descr); } @@ -156,8 +159,9 @@ method_get(PyMethodDescrObject *descr, PyObject *obj, PyObject *type) } static PyObject * -member_get(PyMemberDescrObject *descr, PyObject *obj, PyObject *type) +member_get(PyObject *self, PyObject *obj, PyObject *type) { + PyMemberDescrObject *descr = (PyMemberDescrObject *)self; if (obj == NULL) { return Py_NewRef(descr); } @@ -176,8 +180,9 @@ member_get(PyMemberDescrObject *descr, PyObject *obj, PyObject *type) } static PyObject * -getset_get(PyGetSetDescrObject *descr, PyObject *obj, PyObject *type) +getset_get(PyObject *self, PyObject *obj, PyObject *type) { + PyGetSetDescrObject *descr = (PyGetSetDescrObject *)self; if (obj == NULL) { return Py_NewRef(descr); } @@ -195,8 +200,9 @@ getset_get(PyGetSetDescrObject *descr, PyObject *obj, PyObject *type) } static PyObject * -wrapperdescr_get(PyWrapperDescrObject *descr, PyObject *obj, PyObject *type) +wrapperdescr_get(PyObject *self, PyObject *obj, PyObject *type) { + PyWrapperDescrObject *descr = (PyWrapperDescrObject *)self; if (obj == NULL) { return Py_NewRef(descr); } @@ -223,8 +229,9 @@ descr_setcheck(PyDescrObject *descr, PyObject *obj, PyObject *value) } static int -member_set(PyMemberDescrObject *descr, PyObject *obj, PyObject *value) +member_set(PyObject *self, PyObject *obj, PyObject *value) { + PyMemberDescrObject *descr = (PyMemberDescrObject *)self; if (descr_setcheck((PyDescrObject *)descr, obj, value) < 0) { return -1; } @@ -232,8 +239,9 @@ member_set(PyMemberDescrObject *descr, PyObject *obj, PyObject *value) } static int -getset_set(PyGetSetDescrObject *descr, PyObject *obj, PyObject *value) +getset_set(PyObject *self, PyObject *obj, PyObject *value) { + PyGetSetDescrObject *descr = (PyGetSetDescrObject *)self; if (descr_setcheck((PyDescrObject *)descr, obj, value) < 0) { return -1; } @@ -479,9 +487,10 @@ method_vectorcall_O( we implement this simply by calling __get__ and then calling the result. */ static PyObject * -classmethoddescr_call(PyMethodDescrObject *descr, PyObject *args, +classmethoddescr_call(PyObject *_descr, PyObject *args, PyObject *kwds) { + PyMethodDescrObject *descr = (PyMethodDescrObject *)_descr; Py_ssize_t argc = PyTuple_GET_SIZE(args); if (argc < 1) { PyErr_Format(PyExc_TypeError, @@ -492,7 +501,7 @@ classmethoddescr_call(PyMethodDescrObject *descr, PyObject *args, return NULL; } PyObject *self = PyTuple_GET_ITEM(args, 0); - PyObject *bound = classmethod_get(descr, NULL, self); + PyObject *bound = classmethod_get((PyObject *)descr, NULL, self); if (bound == NULL) { return NULL; } @@ -523,8 +532,9 @@ wrapperdescr_raw_call(PyWrapperDescrObject *descr, PyObject *self, } static PyObject * -wrapperdescr_call(PyWrapperDescrObject *descr, PyObject *args, PyObject *kwds) +wrapperdescr_call(PyObject *_descr, PyObject *args, PyObject *kwds) { + PyWrapperDescrObject *descr = (PyWrapperDescrObject *)_descr; Py_ssize_t argc; PyObject *self, *result; @@ -563,14 +573,16 @@ wrapperdescr_call(PyWrapperDescrObject *descr, PyObject *args, PyObject *kwds) static PyObject * -method_get_doc(PyMethodDescrObject *descr, void *closure) +method_get_doc(PyObject *_descr, void *closure) { + PyMethodDescrObject *descr = (PyMethodDescrObject *)_descr; return _PyType_GetDocFromInternalDoc(descr->d_method->ml_name, descr->d_method->ml_doc); } static PyObject * -method_get_text_signature(PyMethodDescrObject *descr, void *closure) +method_get_text_signature(PyObject *_descr, void *closure) { + PyMethodDescrObject *descr = (PyMethodDescrObject *)_descr; return _PyType_GetTextSignatureFromInternalDoc(descr->d_method->ml_name, descr->d_method->ml_doc, descr->d_method->ml_flags); @@ -605,22 +617,24 @@ calculate_qualname(PyDescrObject *descr) } static PyObject * -descr_get_qualname(PyDescrObject *descr, void *Py_UNUSED(ignored)) +descr_get_qualname(PyObject *self, void *Py_UNUSED(ignored)) { + PyDescrObject *descr = (PyDescrObject *)self; if (descr->d_qualname == NULL) descr->d_qualname = calculate_qualname(descr); return Py_XNewRef(descr->d_qualname); } static PyObject * -descr_reduce(PyDescrObject *descr, PyObject *Py_UNUSED(ignored)) +descr_reduce(PyObject *self, PyObject *Py_UNUSED(ignored)) { + PyDescrObject *descr = (PyDescrObject *)self; return Py_BuildValue("N(OO)", _PyEval_GetBuiltin(&_Py_ID(getattr)), PyDescr_TYPE(descr), PyDescr_NAME(descr)); } static PyMethodDef descr_methods[] = { - {"__reduce__", (PyCFunction)descr_reduce, METH_NOARGS, NULL}, + {"__reduce__", descr_reduce, METH_NOARGS, NULL}, {NULL, NULL} }; @@ -631,15 +645,16 @@ static PyMemberDef descr_members[] = { }; static PyGetSetDef method_getset[] = { - {"__doc__", (getter)method_get_doc}, - {"__qualname__", (getter)descr_get_qualname}, - {"__text_signature__", (getter)method_get_text_signature}, + {"__doc__", method_get_doc}, + {"__qualname__", descr_get_qualname}, + {"__text_signature__", method_get_text_signature}, {0} }; static PyObject * -member_get_doc(PyMemberDescrObject *descr, void *closure) +member_get_doc(PyObject *_descr, void *closure) { + PyMemberDescrObject *descr = (PyMemberDescrObject *)_descr; if (descr->d_member->doc == NULL) { Py_RETURN_NONE; } @@ -647,14 +662,15 @@ member_get_doc(PyMemberDescrObject *descr, void *closure) } static PyGetSetDef member_getset[] = { - {"__doc__", (getter)member_get_doc}, - {"__qualname__", (getter)descr_get_qualname}, + {"__doc__", member_get_doc}, + {"__qualname__", descr_get_qualname}, {0} }; static PyObject * -getset_get_doc(PyGetSetDescrObject *descr, void *closure) +getset_get_doc(PyObject *self, void *closure) { + PyGetSetDescrObject *descr = (PyGetSetDescrObject *)self; if (descr->d_getset->doc == NULL) { Py_RETURN_NONE; } @@ -662,28 +678,30 @@ getset_get_doc(PyGetSetDescrObject *descr, void *closure) } static PyGetSetDef getset_getset[] = { - {"__doc__", (getter)getset_get_doc}, - {"__qualname__", (getter)descr_get_qualname}, + {"__doc__", getset_get_doc}, + {"__qualname__", descr_get_qualname}, {0} }; static PyObject * -wrapperdescr_get_doc(PyWrapperDescrObject *descr, void *closure) +wrapperdescr_get_doc(PyObject *self, void *closure) { + PyWrapperDescrObject *descr = (PyWrapperDescrObject *)self; return _PyType_GetDocFromInternalDoc(descr->d_base->name, descr->d_base->doc); } static PyObject * -wrapperdescr_get_text_signature(PyWrapperDescrObject *descr, void *closure) +wrapperdescr_get_text_signature(PyObject *self, void *closure) { + PyWrapperDescrObject *descr = (PyWrapperDescrObject *)self; return _PyType_GetTextSignatureFromInternalDoc(descr->d_base->name, descr->d_base->doc, 0); } static PyGetSetDef wrapperdescr_getset[] = { - {"__doc__", (getter)wrapperdescr_get_doc}, - {"__qualname__", (getter)descr_get_qualname}, - {"__text_signature__", (getter)wrapperdescr_get_text_signature}, + {"__doc__", wrapperdescr_get_doc}, + {"__qualname__", descr_get_qualname}, + {"__text_signature__", wrapperdescr_get_text_signature}, {0} }; @@ -700,12 +718,12 @@ PyTypeObject PyMethodDescr_Type = { "method_descriptor", sizeof(PyMethodDescrObject), 0, - (destructor)descr_dealloc, /* tp_dealloc */ + descr_dealloc, /* tp_dealloc */ offsetof(PyMethodDescrObject, vectorcall), /* tp_vectorcall_offset */ 0, /* tp_getattr */ 0, /* tp_setattr */ 0, /* tp_as_async */ - (reprfunc)method_repr, /* tp_repr */ + method_repr, /* tp_repr */ 0, /* tp_as_number */ 0, /* tp_as_sequence */ 0, /* tp_as_mapping */ @@ -730,7 +748,7 @@ PyTypeObject PyMethodDescr_Type = { method_getset, /* tp_getset */ 0, /* tp_base */ 0, /* tp_dict */ - (descrgetfunc)method_get, /* tp_descr_get */ + method_get, /* tp_descr_get */ 0, /* tp_descr_set */ }; @@ -740,17 +758,17 @@ PyTypeObject PyClassMethodDescr_Type = { "classmethod_descriptor", sizeof(PyMethodDescrObject), 0, - (destructor)descr_dealloc, /* tp_dealloc */ + descr_dealloc, /* tp_dealloc */ 0, /* tp_vectorcall_offset */ 0, /* tp_getattr */ 0, /* tp_setattr */ 0, /* tp_as_async */ - (reprfunc)method_repr, /* tp_repr */ + method_repr, /* tp_repr */ 0, /* tp_as_number */ 0, /* tp_as_sequence */ 0, /* tp_as_mapping */ 0, /* tp_hash */ - (ternaryfunc)classmethoddescr_call, /* tp_call */ + classmethoddescr_call, /* tp_call */ 0, /* tp_str */ PyObject_GenericGetAttr, /* tp_getattro */ 0, /* tp_setattro */ @@ -768,7 +786,7 @@ PyTypeObject PyClassMethodDescr_Type = { method_getset, /* tp_getset */ 0, /* tp_base */ 0, /* tp_dict */ - (descrgetfunc)classmethod_get, /* tp_descr_get */ + classmethod_get, /* tp_descr_get */ 0, /* tp_descr_set */ }; @@ -777,12 +795,12 @@ PyTypeObject PyMemberDescr_Type = { "member_descriptor", sizeof(PyMemberDescrObject), 0, - (destructor)descr_dealloc, /* tp_dealloc */ + descr_dealloc, /* tp_dealloc */ 0, /* tp_vectorcall_offset */ 0, /* tp_getattr */ 0, /* tp_setattr */ 0, /* tp_as_async */ - (reprfunc)member_repr, /* tp_repr */ + member_repr, /* tp_repr */ 0, /* tp_as_number */ 0, /* tp_as_sequence */ 0, /* tp_as_mapping */ @@ -805,8 +823,8 @@ PyTypeObject PyMemberDescr_Type = { member_getset, /* tp_getset */ 0, /* tp_base */ 0, /* tp_dict */ - (descrgetfunc)member_get, /* tp_descr_get */ - (descrsetfunc)member_set, /* tp_descr_set */ + member_get, /* tp_descr_get */ + member_set, /* tp_descr_set */ }; PyTypeObject PyGetSetDescr_Type = { @@ -814,12 +832,12 @@ PyTypeObject PyGetSetDescr_Type = { "getset_descriptor", sizeof(PyGetSetDescrObject), 0, - (destructor)descr_dealloc, /* tp_dealloc */ + descr_dealloc, /* tp_dealloc */ 0, /* tp_vectorcall_offset */ 0, /* tp_getattr */ 0, /* tp_setattr */ 0, /* tp_as_async */ - (reprfunc)getset_repr, /* tp_repr */ + getset_repr, /* tp_repr */ 0, /* tp_as_number */ 0, /* tp_as_sequence */ 0, /* tp_as_mapping */ @@ -842,8 +860,8 @@ PyTypeObject PyGetSetDescr_Type = { getset_getset, /* tp_getset */ 0, /* tp_base */ 0, /* tp_dict */ - (descrgetfunc)getset_get, /* tp_descr_get */ - (descrsetfunc)getset_set, /* tp_descr_set */ + getset_get, /* tp_descr_get */ + getset_set, /* tp_descr_set */ }; PyTypeObject PyWrapperDescr_Type = { @@ -851,17 +869,17 @@ PyTypeObject PyWrapperDescr_Type = { "wrapper_descriptor", sizeof(PyWrapperDescrObject), 0, - (destructor)descr_dealloc, /* tp_dealloc */ + descr_dealloc, /* tp_dealloc */ 0, /* tp_vectorcall_offset */ 0, /* tp_getattr */ 0, /* tp_setattr */ 0, /* tp_as_async */ - (reprfunc)wrapperdescr_repr, /* tp_repr */ + wrapperdescr_repr, /* tp_repr */ 0, /* tp_as_number */ 0, /* tp_as_sequence */ 0, /* tp_as_mapping */ 0, /* tp_hash */ - (ternaryfunc)wrapperdescr_call, /* tp_call */ + wrapperdescr_call, /* tp_call */ 0, /* tp_str */ PyObject_GenericGetAttr, /* tp_getattro */ 0, /* tp_setattro */ @@ -880,7 +898,7 @@ PyTypeObject PyWrapperDescr_Type = { wrapperdescr_getset, /* tp_getset */ 0, /* tp_base */ 0, /* tp_dict */ - (descrgetfunc)wrapperdescr_get, /* tp_descr_get */ + wrapperdescr_get, /* tp_descr_get */ 0, /* tp_descr_set */ }; @@ -1022,20 +1040,22 @@ typedef struct { } mappingproxyobject; static Py_ssize_t -mappingproxy_len(mappingproxyobject *pp) +mappingproxy_len(PyObject *self) { + mappingproxyobject *pp = (mappingproxyobject *)self; return PyObject_Size(pp->mapping); } static PyObject * -mappingproxy_getitem(mappingproxyobject *pp, PyObject *key) +mappingproxy_getitem(PyObject *self, PyObject *key) { + mappingproxyobject *pp = (mappingproxyobject *)self; return PyObject_GetItem(pp->mapping, key); } static PyMappingMethods mappingproxy_as_mapping = { - (lenfunc)mappingproxy_len, /* mp_length */ - (binaryfunc)mappingproxy_getitem, /* mp_subscript */ + mappingproxy_len, /* mp_length */ + mappingproxy_getitem, /* mp_subscript */ 0, /* mp_ass_subscript */ }; @@ -1064,8 +1084,9 @@ static PyNumberMethods mappingproxy_as_number = { }; static int -mappingproxy_contains(mappingproxyobject *pp, PyObject *key) +mappingproxy_contains(PyObject *self, PyObject *key) { + mappingproxyobject *pp = (mappingproxyobject *)self; if (PyDict_CheckExact(pp->mapping)) return PyDict_Contains(pp->mapping, key); else @@ -1080,14 +1101,15 @@ static PySequenceMethods mappingproxy_as_sequence = { 0, /* sq_slice */ 0, /* sq_ass_item */ 0, /* sq_ass_slice */ - (objobjproc)mappingproxy_contains, /* sq_contains */ + mappingproxy_contains, /* sq_contains */ 0, /* sq_inplace_concat */ 0, /* sq_inplace_repeat */ }; static PyObject * -mappingproxy_get(mappingproxyobject *pp, PyObject *const *args, Py_ssize_t nargs) +mappingproxy_get(PyObject *self, PyObject *const *args, Py_ssize_t nargs) { + mappingproxyobject *pp = (mappingproxyobject *)self; /* newargs: mapping, key, default=None */ PyObject *newargs[3]; newargs[0] = pp->mapping; @@ -1104,32 +1126,37 @@ mappingproxy_get(mappingproxyobject *pp, PyObject *const *args, Py_ssize_t nargs } static PyObject * -mappingproxy_keys(mappingproxyobject *pp, PyObject *Py_UNUSED(ignored)) +mappingproxy_keys(PyObject *self, PyObject *Py_UNUSED(ignored)) { + mappingproxyobject *pp = (mappingproxyobject *)self; return PyObject_CallMethodNoArgs(pp->mapping, &_Py_ID(keys)); } static PyObject * -mappingproxy_values(mappingproxyobject *pp, PyObject *Py_UNUSED(ignored)) +mappingproxy_values(PyObject *self, PyObject *Py_UNUSED(ignored)) { + mappingproxyobject *pp = (mappingproxyobject *)self; return PyObject_CallMethodNoArgs(pp->mapping, &_Py_ID(values)); } static PyObject * -mappingproxy_items(mappingproxyobject *pp, PyObject *Py_UNUSED(ignored)) +mappingproxy_items(PyObject *self, PyObject *Py_UNUSED(ignored)) { + mappingproxyobject *pp = (mappingproxyobject *)self; return PyObject_CallMethodNoArgs(pp->mapping, &_Py_ID(items)); } static PyObject * -mappingproxy_copy(mappingproxyobject *pp, PyObject *Py_UNUSED(ignored)) +mappingproxy_copy(PyObject *self, PyObject *Py_UNUSED(ignored)) { + mappingproxyobject *pp = (mappingproxyobject *)self; return PyObject_CallMethodNoArgs(pp->mapping, &_Py_ID(copy)); } static PyObject * -mappingproxy_reversed(mappingproxyobject *pp, PyObject *Py_UNUSED(ignored)) +mappingproxy_reversed(PyObject *self, PyObject *Py_UNUSED(ignored)) { + mappingproxyobject *pp = (mappingproxyobject *)self; return PyObject_CallMethodNoArgs(pp->mapping, &_Py_ID(__reversed__)); } @@ -1140,50 +1167,55 @@ static PyMethodDef mappingproxy_methods[] = { {"get", _PyCFunction_CAST(mappingproxy_get), METH_FASTCALL, PyDoc_STR("D.get(k[,d]) -> D[k] if k in D, else d." " d defaults to None.")}, - {"keys", (PyCFunction)mappingproxy_keys, METH_NOARGS, + {"keys", mappingproxy_keys, METH_NOARGS, PyDoc_STR("D.keys() -> a set-like object providing a view on D's keys")}, - {"values", (PyCFunction)mappingproxy_values, METH_NOARGS, + {"values", mappingproxy_values, METH_NOARGS, PyDoc_STR("D.values() -> an object providing a view on D's values")}, - {"items", (PyCFunction)mappingproxy_items, METH_NOARGS, + {"items", mappingproxy_items, METH_NOARGS, PyDoc_STR("D.items() -> a set-like object providing a view on D's items")}, - {"copy", (PyCFunction)mappingproxy_copy, METH_NOARGS, + {"copy", mappingproxy_copy, METH_NOARGS, PyDoc_STR("D.copy() -> a shallow copy of D")}, {"__class_getitem__", Py_GenericAlias, METH_O|METH_CLASS, PyDoc_STR("See PEP 585")}, - {"__reversed__", (PyCFunction)mappingproxy_reversed, METH_NOARGS, + {"__reversed__", mappingproxy_reversed, METH_NOARGS, PyDoc_STR("D.__reversed__() -> reverse iterator")}, {0} }; static void -mappingproxy_dealloc(mappingproxyobject *pp) +mappingproxy_dealloc(PyObject *self) { + mappingproxyobject *pp = (mappingproxyobject *)self; _PyObject_GC_UNTRACK(pp); Py_DECREF(pp->mapping); PyObject_GC_Del(pp); } static PyObject * -mappingproxy_getiter(mappingproxyobject *pp) +mappingproxy_getiter(PyObject *self) { + mappingproxyobject *pp = (mappingproxyobject *)self; return PyObject_GetIter(pp->mapping); } static Py_hash_t -mappingproxy_hash(mappingproxyobject *pp) +mappingproxy_hash(PyObject *self) { + mappingproxyobject *pp = (mappingproxyobject *)self; return PyObject_Hash(pp->mapping); } static PyObject * -mappingproxy_str(mappingproxyobject *pp) +mappingproxy_str(PyObject *self) { + mappingproxyobject *pp = (mappingproxyobject *)self; return PyObject_Str(pp->mapping); } static PyObject * -mappingproxy_repr(mappingproxyobject *pp) +mappingproxy_repr(PyObject *self) { + mappingproxyobject *pp = (mappingproxyobject *)self; return PyUnicode_FromFormat("mappingproxy(%R)", pp->mapping); } @@ -1196,8 +1228,9 @@ mappingproxy_traverse(PyObject *self, visitproc visit, void *arg) } static PyObject * -mappingproxy_richcompare(mappingproxyobject *v, PyObject *w, int op) +mappingproxy_richcompare(PyObject *self, PyObject *w, int op) { + mappingproxyobject *v = (mappingproxyobject *)self; return PyObject_RichCompare(v->mapping, w, op); } @@ -1271,8 +1304,9 @@ typedef struct { #define Wrapper_Check(v) Py_IS_TYPE(v, &_PyMethodWrapper_Type) static void -wrapper_dealloc(wrapperobject *wp) +wrapper_dealloc(PyObject *self) { + wrapperobject *wp = (wrapperobject *)self; PyObject_GC_UnTrack(wp); Py_TRASHCAN_BEGIN(wp, wrapper_dealloc) Py_XDECREF(wp->descr); @@ -1308,8 +1342,9 @@ wrapper_richcompare(PyObject *a, PyObject *b, int op) } static Py_hash_t -wrapper_hash(wrapperobject *wp) +wrapper_hash(PyObject *self) { + wrapperobject *wp = (wrapperobject *)self; Py_hash_t x, y; x = _Py_HashPointer(wp->self); y = _Py_HashPointer(wp->descr); @@ -1320,8 +1355,9 @@ wrapper_hash(wrapperobject *wp) } static PyObject * -wrapper_repr(wrapperobject *wp) +wrapper_repr(PyObject *self) { + wrapperobject *wp = (wrapperobject *)self; return PyUnicode_FromFormat("", wp->descr->d_base->name, Py_TYPE(wp->self)->tp_name, @@ -1329,14 +1365,15 @@ wrapper_repr(wrapperobject *wp) } static PyObject * -wrapper_reduce(wrapperobject *wp, PyObject *Py_UNUSED(ignored)) +wrapper_reduce(PyObject *self, PyObject *Py_UNUSED(ignored)) { + wrapperobject *wp = (wrapperobject *)self; return Py_BuildValue("N(OO)", _PyEval_GetBuiltin(&_Py_ID(getattr)), wp->self, PyDescr_NAME(wp->descr)); } static PyMethodDef wrapper_methods[] = { - {"__reduce__", (PyCFunction)wrapper_reduce, METH_NOARGS, NULL}, + {"__reduce__", wrapper_reduce, METH_NOARGS, NULL}, {NULL, NULL} }; @@ -1346,52 +1383,56 @@ static PyMemberDef wrapper_members[] = { }; static PyObject * -wrapper_objclass(wrapperobject *wp, void *Py_UNUSED(ignored)) +wrapper_objclass(PyObject *wp, void *Py_UNUSED(ignored)) { - PyObject *c = (PyObject *)PyDescr_TYPE(wp->descr); + PyObject *c = (PyObject *)PyDescr_TYPE(((wrapperobject *)wp)->descr); return Py_NewRef(c); } static PyObject * -wrapper_name(wrapperobject *wp, void *Py_UNUSED(ignored)) +wrapper_name(PyObject *wp, void *Py_UNUSED(ignored)) { - const char *s = wp->descr->d_base->name; + const char *s = ((wrapperobject *)wp)->descr->d_base->name; return PyUnicode_FromString(s); } static PyObject * -wrapper_doc(wrapperobject *wp, void *Py_UNUSED(ignored)) +wrapper_doc(PyObject *self, void *Py_UNUSED(ignored)) { + wrapperobject *wp = (wrapperobject *)self; return _PyType_GetDocFromInternalDoc(wp->descr->d_base->name, wp->descr->d_base->doc); } static PyObject * -wrapper_text_signature(wrapperobject *wp, void *Py_UNUSED(ignored)) +wrapper_text_signature(PyObject *self, void *Py_UNUSED(ignored)) { + wrapperobject *wp = (wrapperobject *)self; return _PyType_GetTextSignatureFromInternalDoc(wp->descr->d_base->name, wp->descr->d_base->doc, 0); } static PyObject * -wrapper_qualname(wrapperobject *wp, void *Py_UNUSED(ignored)) +wrapper_qualname(PyObject *self, void *Py_UNUSED(ignored)) { - return descr_get_qualname((PyDescrObject *)wp->descr, NULL); + wrapperobject *wp = (wrapperobject *)self; + return descr_get_qualname((PyObject *)wp->descr, NULL); } static PyGetSetDef wrapper_getsets[] = { - {"__objclass__", (getter)wrapper_objclass}, - {"__name__", (getter)wrapper_name}, - {"__qualname__", (getter)wrapper_qualname}, - {"__doc__", (getter)wrapper_doc}, - {"__text_signature__", (getter)wrapper_text_signature}, + {"__objclass__", wrapper_objclass}, + {"__name__", wrapper_name}, + {"__qualname__", wrapper_qualname}, + {"__doc__", wrapper_doc}, + {"__text_signature__", wrapper_text_signature}, {0} }; static PyObject * -wrapper_call(wrapperobject *wp, PyObject *args, PyObject *kwds) +wrapper_call(PyObject *self, PyObject *args, PyObject *kwds) { + wrapperobject *wp = (wrapperobject *)self; return wrapperdescr_raw_call(wp->descr, wp->self, args, kwds); } @@ -1410,17 +1451,17 @@ PyTypeObject _PyMethodWrapper_Type = { sizeof(wrapperobject), /* tp_basicsize */ 0, /* tp_itemsize */ /* methods */ - (destructor)wrapper_dealloc, /* tp_dealloc */ + wrapper_dealloc, /* tp_dealloc */ 0, /* tp_vectorcall_offset */ 0, /* tp_getattr */ 0, /* tp_setattr */ 0, /* tp_as_async */ - (reprfunc)wrapper_repr, /* tp_repr */ + wrapper_repr, /* tp_repr */ 0, /* tp_as_number */ 0, /* tp_as_sequence */ 0, /* tp_as_mapping */ - (hashfunc)wrapper_hash, /* tp_hash */ - (ternaryfunc)wrapper_call, /* tp_call */ + wrapper_hash, /* tp_hash */ + wrapper_call, /* tp_call */ 0, /* tp_str */ PyObject_GenericGetAttr, /* tp_getattro */ 0, /* tp_setattro */ @@ -1910,18 +1951,18 @@ PyTypeObject PyDictProxy_Type = { sizeof(mappingproxyobject), /* tp_basicsize */ 0, /* tp_itemsize */ /* methods */ - (destructor)mappingproxy_dealloc, /* tp_dealloc */ + mappingproxy_dealloc, /* tp_dealloc */ 0, /* tp_vectorcall_offset */ 0, /* tp_getattr */ 0, /* tp_setattr */ 0, /* tp_as_async */ - (reprfunc)mappingproxy_repr, /* tp_repr */ + mappingproxy_repr, /* tp_repr */ &mappingproxy_as_number, /* tp_as_number */ &mappingproxy_as_sequence, /* tp_as_sequence */ &mappingproxy_as_mapping, /* tp_as_mapping */ - (hashfunc)mappingproxy_hash, /* tp_hash */ + mappingproxy_hash, /* tp_hash */ 0, /* tp_call */ - (reprfunc)mappingproxy_str, /* tp_str */ + mappingproxy_str, /* tp_str */ PyObject_GenericGetAttr, /* tp_getattro */ 0, /* tp_setattro */ 0, /* tp_as_buffer */ @@ -1930,9 +1971,9 @@ PyTypeObject PyDictProxy_Type = { 0, /* tp_doc */ mappingproxy_traverse, /* tp_traverse */ 0, /* tp_clear */ - (richcmpfunc)mappingproxy_richcompare, /* tp_richcompare */ + mappingproxy_richcompare, /* tp_richcompare */ 0, /* tp_weaklistoffset */ - (getiterfunc)mappingproxy_getiter, /* tp_iter */ + mappingproxy_getiter, /* tp_iter */ 0, /* tp_iternext */ mappingproxy_methods, /* tp_methods */ 0, /* tp_members */ @@ -1972,7 +2013,7 @@ PyTypeObject PyProperty_Type = { Py_TPFLAGS_BASETYPE, /* tp_flags */ property_init__doc__, /* tp_doc */ property_traverse, /* tp_traverse */ - (inquiry)property_clear, /* tp_clear */ + property_clear, /* tp_clear */ 0, /* tp_richcompare */ 0, /* tp_weaklistoffset */ 0, /* tp_iter */ diff --git a/Objects/dictobject.c b/Objects/dictobject.c index 70f424e07ece9a..9b1defa5cbc609 100644 --- a/Objects/dictobject.c +++ b/Objects/dictobject.c @@ -113,17 +113,19 @@ As a consequence of this, split keys have a maximum size of 16. #define PyDict_MINSIZE 8 #include "Python.h" -#include "pycore_bitutils.h" // _Py_bit_length -#include "pycore_call.h" // _PyObject_CallNoArgs() -#include "pycore_ceval.h" // _PyEval_GetBuiltin() -#include "pycore_code.h" // stats -#include "pycore_dict.h" // export _PyDict_SizeOf() -#include "pycore_gc.h" // _PyObject_GC_IS_TRACKED() -#include "pycore_object.h" // _PyObject_GC_TRACK(), _PyDebugAllocatorStats() -#include "pycore_pyerrors.h" // _PyErr_GetRaisedException() -#include "pycore_pystate.h" // _PyThreadState_GET() -#include "pycore_setobject.h" // _PySet_NextEntry() -#include "stringlib/eq.h" // unicode_eq() +#include "pycore_bitutils.h" // _Py_bit_length +#include "pycore_call.h" // _PyObject_CallNoArgs() +#include "pycore_ceval.h" // _PyEval_GetBuiltin() +#include "pycore_code.h" // stats +#include "pycore_critical_section.h" // Py_BEGIN_CRITICAL_SECTION, Py_END_CRITICAL_SECTION +#include "pycore_dict.h" // export _PyDict_SizeOf() +#include "pycore_freelist.h" // _PyFreeListState_GET() +#include "pycore_gc.h" // _PyObject_GC_IS_TRACKED() +#include "pycore_object.h" // _PyObject_GC_TRACK(), _PyDebugAllocatorStats() +#include "pycore_pyerrors.h" // _PyErr_GetRaisedException() +#include "pycore_pystate.h" // _PyThreadState_GET() +#include "pycore_setobject.h" // _PySet_NextEntry() +#include "stringlib/eq.h" // unicode_eq() #include @@ -140,6 +142,21 @@ To avoid slowing down lookups on a near-full table, we resize the table when it's USABLE_FRACTION (currently two-thirds) full. */ +#ifdef Py_GIL_DISABLED + +static inline void +ASSERT_DICT_LOCKED(PyObject *op) +{ + _Py_CRITICAL_SECTION_ASSERT_OBJECT_LOCKED(op); +} +#define ASSERT_DICT_LOCKED(op) ASSERT_DICT_LOCKED(_Py_CAST(PyObject*, op)) + +#else + +#define ASSERT_DICT_LOCKED(op) + +#endif + #define PERTURB_SHIFT 5 /* @@ -237,45 +254,59 @@ equally good collision statistics, needed less code & used less memory. static int dictresize(PyInterpreterState *interp, PyDictObject *mp, uint8_t log_newsize, int unicode); -static PyObject* dict_iter(PyDictObject *dict); +static PyObject* dict_iter(PyObject *dict); + +static int +contains_lock_held(PyDictObject *mp, PyObject *key); +static int +contains_known_hash_lock_held(PyDictObject *mp, PyObject *key, Py_ssize_t hash); +static int +setitem_lock_held(PyDictObject *mp, PyObject *key, PyObject *value); +static int +dict_setdefault_ref_lock_held(PyObject *d, PyObject *key, PyObject *default_value, + PyObject **result, int incref_result); #include "clinic/dictobject.c.h" -#if PyDict_MAXFREELIST > 0 -static struct _Py_dict_state * -get_dict_state(PyInterpreterState *interp) +#ifdef WITH_FREELISTS +static struct _Py_dict_freelist * +get_dict_state(void) { - return &interp->dict_state; + _PyFreeListState *state = _PyFreeListState_GET(); + return &state->dicts; } #endif void -_PyDict_ClearFreeList(PyInterpreterState *interp) +_PyDict_ClearFreeList(_PyFreeListState *freelist_state, int is_finalization) { -#if PyDict_MAXFREELIST > 0 - struct _Py_dict_state *state = &interp->dict_state; - while (state->numfree) { +#ifdef WITH_FREELISTS + struct _Py_dict_freelist *state = &freelist_state->dicts; + while (state->numfree > 0) { PyDictObject *op = state->free_list[--state->numfree]; assert(PyDict_CheckExact(op)); PyObject_GC_Del(op); } - while (state->keys_numfree) { - PyObject_Free(state->keys_free_list[--state->keys_numfree]); + while (state->keys_numfree > 0) { + PyMem_Free(state->keys_free_list[--state->keys_numfree]); + } + if (is_finalization) { + state->numfree = -1; + state->keys_numfree = -1; } #endif } - void -_PyDict_Fini(PyInterpreterState *interp) +_PyDict_Fini(PyInterpreterState *Py_UNUSED(interp)) { - _PyDict_ClearFreeList(interp); -#if defined(Py_DEBUG) && PyDict_MAXFREELIST > 0 - struct _Py_dict_state *state = &interp->dict_state; - state->numfree = -1; - state->keys_numfree = -1; + // With Py_GIL_DISABLED: + // the freelists for the current thread state have already been cleared. +#ifndef Py_GIL_DISABLED + _PyFreeListState *state = _PyFreeListState_GET(); + _PyDict_ClearFreeList(state, 1); #endif } @@ -290,9 +321,8 @@ unicode_get_hash(PyObject *o) void _PyDict_DebugMallocStats(FILE *out) { -#if PyDict_MAXFREELIST > 0 - PyInterpreterState *interp = _PyInterpreterState_GET(); - struct _Py_dict_state *state = get_dict_state(interp); +#ifdef WITH_FREELISTS + struct _Py_dict_freelist *state = get_dict_state(); _PyDebugAllocatorStats(out, "free PyDictObject", state->numfree, sizeof(PyDictObject)); #endif @@ -300,7 +330,7 @@ _PyDict_DebugMallocStats(FILE *out) #define DK_MASK(dk) (DK_SIZE(dk)-1) -static void free_keys_object(PyInterpreterState *interp, PyDictKeysObject *keys); +static void free_keys_object(PyDictKeysObject *keys); /* PyDictKeysObject has refcounts like PyObject does, so we have the following two functions to mirror what Py_INCREF() and Py_DECREF() do. @@ -332,7 +362,23 @@ dictkeys_decref(PyInterpreterState *interp, PyDictKeysObject *dk) _Py_DecRefTotal(_PyInterpreterState_GET()); #endif if (--dk->dk_refcnt == 0) { - free_keys_object(interp, dk); + if (DK_IS_UNICODE(dk)) { + PyDictUnicodeEntry *entries = DK_UNICODE_ENTRIES(dk); + Py_ssize_t i, n; + for (i = 0, n = dk->dk_nentries; i < n; i++) { + Py_XDECREF(entries[i].me_key); + Py_XDECREF(entries[i].me_value); + } + } + else { + PyDictKeyEntry *entries = DK_ENTRIES(dk); + Py_ssize_t i, n; + for (i = 0, n = dk->dk_nentries; i < n; i++) { + Py_XDECREF(entries[i].me_key); + Py_XDECREF(entries[i].me_value); + } + } + free_keys_object(dk); } } @@ -627,12 +673,8 @@ new_keys_object(PyInterpreterState *interp, uint8_t log2_size, bool unicode) log2_bytes = log2_size + 2; } -#if PyDict_MAXFREELIST > 0 - struct _Py_dict_state *state = get_dict_state(interp); -#ifdef Py_DEBUG - // new_keys_object() must not be called after _PyDict_Fini() - assert(state->keys_numfree != -1); -#endif +#ifdef WITH_FREELISTS + struct _Py_dict_freelist *state = get_dict_state(); if (log2_size == PyDict_LOG_MINSIZE && unicode && state->keys_numfree > 0) { dk = state->keys_free_list[--state->keys_numfree]; OBJECT_STAT_INC(from_freelist); @@ -640,9 +682,9 @@ new_keys_object(PyInterpreterState *interp, uint8_t log2_size, bool unicode) else #endif { - dk = PyObject_Malloc(sizeof(PyDictKeysObject) - + ((size_t)1 << log2_bytes) - + entry_size * usable); + dk = PyMem_Malloc(sizeof(PyDictKeysObject) + + ((size_t)1 << log2_bytes) + + entry_size * usable); if (dk == NULL) { PyErr_NoMemory(); return NULL; @@ -664,40 +706,20 @@ new_keys_object(PyInterpreterState *interp, uint8_t log2_size, bool unicode) } static void -free_keys_object(PyInterpreterState *interp, PyDictKeysObject *keys) +free_keys_object(PyDictKeysObject *keys) { - assert(keys != Py_EMPTY_KEYS); - if (DK_IS_UNICODE(keys)) { - PyDictUnicodeEntry *entries = DK_UNICODE_ENTRIES(keys); - Py_ssize_t i, n; - for (i = 0, n = keys->dk_nentries; i < n; i++) { - Py_XDECREF(entries[i].me_key); - Py_XDECREF(entries[i].me_value); - } - } - else { - PyDictKeyEntry *entries = DK_ENTRIES(keys); - Py_ssize_t i, n; - for (i = 0, n = keys->dk_nentries; i < n; i++) { - Py_XDECREF(entries[i].me_key); - Py_XDECREF(entries[i].me_value); - } - } -#if PyDict_MAXFREELIST > 0 - struct _Py_dict_state *state = get_dict_state(interp); -#ifdef Py_DEBUG - // free_keys_object() must not be called after _PyDict_Fini() - assert(state->keys_numfree != -1); -#endif +#ifdef WITH_FREELISTS + struct _Py_dict_freelist *state = get_dict_state(); if (DK_LOG_SIZE(keys) == PyDict_LOG_MINSIZE && state->keys_numfree < PyDict_MAXFREELIST + && state->keys_numfree >= 0 && DK_IS_UNICODE(keys)) { state->keys_free_list[state->keys_numfree++] = keys; OBJECT_STAT_INC(to_freelist); return; } #endif - PyObject_Free(keys); + PyMem_Free(keys); } static inline PyDictValues* @@ -731,13 +753,9 @@ new_dict(PyInterpreterState *interp, { PyDictObject *mp; assert(keys != NULL); -#if PyDict_MAXFREELIST > 0 - struct _Py_dict_state *state = get_dict_state(interp); -#ifdef Py_DEBUG - // new_dict() must not be called after _PyDict_Fini() - assert(state->numfree != -1); -#endif - if (state->numfree) { +#ifdef WITH_FREELISTS + struct _Py_dict_freelist *state = get_dict_state(); + if (state->numfree > 0) { mp = state->free_list[--state->numfree]; assert (mp != NULL); assert (Py_IS_TYPE(mp, &PyDict_Type)); @@ -792,13 +810,15 @@ static PyDictKeysObject * clone_combined_dict_keys(PyDictObject *orig) { assert(PyDict_Check(orig)); - assert(Py_TYPE(orig)->tp_iter == (getiterfunc)dict_iter); + assert(Py_TYPE(orig)->tp_iter == dict_iter); assert(orig->ma_values == NULL); assert(orig->ma_keys != Py_EMPTY_KEYS); assert(orig->ma_keys->dk_refcnt == 1); + ASSERT_DICT_LOCKED(orig); + size_t keys_size = _PyDict_KeysSize(orig->ma_keys); - PyDictKeysObject *keys = PyObject_Malloc(keys_size); + PyDictKeysObject *keys = PyMem_Malloc(keys_size); if (keys == NULL) { PyErr_NoMemory(); return NULL; @@ -875,11 +895,11 @@ lookdict_index(PyDictKeysObject *k, Py_hash_t hash, Py_ssize_t index) Py_UNREACHABLE(); } -// Search non-Unicode key from Unicode table -static Py_ssize_t -unicodekeys_lookup_generic(PyDictObject *mp, PyDictKeysObject* dk, PyObject *key, Py_hash_t hash) +static inline Py_ALWAYS_INLINE Py_ssize_t +do_lookup(PyDictObject *mp, PyDictKeysObject *dk, PyObject *key, Py_hash_t hash, + Py_ssize_t (*check_lookup)(PyDictObject *, PyDictKeysObject *, void *, Py_ssize_t ix, PyObject *key, Py_hash_t)) { - PyDictUnicodeEntry *ep0 = DK_UNICODE_ENTRIES(dk); + void *ep0 = _DK_ENTRIES(dk); size_t mask = DK_MASK(dk); size_t perturb = hash; size_t i = (size_t)hash & mask; @@ -887,73 +907,26 @@ unicodekeys_lookup_generic(PyDictObject *mp, PyDictKeysObject* dk, PyObject *key for (;;) { ix = dictkeys_get_index(dk, i); if (ix >= 0) { - PyDictUnicodeEntry *ep = &ep0[ix]; - assert(ep->me_key != NULL); - assert(PyUnicode_CheckExact(ep->me_key)); - if (ep->me_key == key) { + Py_ssize_t cmp = check_lookup(mp, dk, ep0, ix, key, hash); + if (cmp < 0) { + return cmp; + } else if (cmp) { return ix; } - if (unicode_get_hash(ep->me_key) == hash) { - PyObject *startkey = ep->me_key; - Py_INCREF(startkey); - int cmp = PyObject_RichCompareBool(startkey, key, Py_EQ); - Py_DECREF(startkey); - if (cmp < 0) { - return DKIX_ERROR; - } - if (dk == mp->ma_keys && ep->me_key == startkey) { - if (cmp > 0) { - return ix; - } - } - else { - /* The dict was mutated, restart */ - return DKIX_KEY_CHANGED; - } - } } else if (ix == DKIX_EMPTY) { return DKIX_EMPTY; } perturb >>= PERTURB_SHIFT; i = mask & (i*5 + perturb + 1); - } - Py_UNREACHABLE(); -} -// Search Unicode key from Unicode table. -static Py_ssize_t _Py_HOT_FUNCTION -unicodekeys_lookup_unicode(PyDictKeysObject* dk, PyObject *key, Py_hash_t hash) -{ - PyDictUnicodeEntry *ep0 = DK_UNICODE_ENTRIES(dk); - size_t mask = DK_MASK(dk); - size_t perturb = hash; - size_t i = (size_t)hash & mask; - Py_ssize_t ix; - for (;;) { - ix = dictkeys_get_index(dk, i); - if (ix >= 0) { - PyDictUnicodeEntry *ep = &ep0[ix]; - assert(ep->me_key != NULL); - assert(PyUnicode_CheckExact(ep->me_key)); - if (ep->me_key == key || - (unicode_get_hash(ep->me_key) == hash && unicode_eq(ep->me_key, key))) { - return ix; - } - } - else if (ix == DKIX_EMPTY) { - return DKIX_EMPTY; - } - perturb >>= PERTURB_SHIFT; - i = mask & (i*5 + perturb + 1); // Manual loop unrolling ix = dictkeys_get_index(dk, i); if (ix >= 0) { - PyDictUnicodeEntry *ep = &ep0[ix]; - assert(ep->me_key != NULL); - assert(PyUnicode_CheckExact(ep->me_key)); - if (ep->me_key == key || - (unicode_get_hash(ep->me_key) == hash && unicode_eq(ep->me_key, key))) { + Py_ssize_t cmp = check_lookup(mp, dk, ep0, ix, key, hash); + if (cmp < 0) { + return cmp; + } else if (cmp) { return ix; } } @@ -966,49 +939,94 @@ unicodekeys_lookup_unicode(PyDictKeysObject* dk, PyObject *key, Py_hash_t hash) Py_UNREACHABLE(); } -// Search key from Generic table. +static inline Py_ALWAYS_INLINE Py_ssize_t +compare_unicode_generic(PyDictObject *mp, PyDictKeysObject *dk, + void *ep0, Py_ssize_t ix, PyObject *key, Py_hash_t hash) +{ + PyDictUnicodeEntry *ep = &((PyDictUnicodeEntry *)ep0)[ix]; + assert(ep->me_key != NULL); + assert(PyUnicode_CheckExact(ep->me_key)); + assert(!PyUnicode_CheckExact(key)); + // TODO: Thread safety + + if (unicode_get_hash(ep->me_key) == hash) { + PyObject *startkey = ep->me_key; + Py_INCREF(startkey); + int cmp = PyObject_RichCompareBool(startkey, key, Py_EQ); + Py_DECREF(startkey); + if (cmp < 0) { + return DKIX_ERROR; + } + if (dk == mp->ma_keys && ep->me_key == startkey) { + return cmp; + } + else { + /* The dict was mutated, restart */ + return DKIX_KEY_CHANGED; + } + } + return 0; +} + +// Search non-Unicode key from Unicode table static Py_ssize_t -dictkeys_generic_lookup(PyDictObject *mp, PyDictKeysObject* dk, PyObject *key, Py_hash_t hash) +unicodekeys_lookup_generic(PyDictObject *mp, PyDictKeysObject* dk, PyObject *key, Py_hash_t hash) { - PyDictKeyEntry *ep0 = DK_ENTRIES(dk); - size_t mask = DK_MASK(dk); - size_t perturb = hash; - size_t i = (size_t)hash & mask; - Py_ssize_t ix; - for (;;) { - ix = dictkeys_get_index(dk, i); - if (ix >= 0) { - PyDictKeyEntry *ep = &ep0[ix]; - assert(ep->me_key != NULL); - if (ep->me_key == key) { - return ix; - } - if (ep->me_hash == hash) { - PyObject *startkey = ep->me_key; - Py_INCREF(startkey); - int cmp = PyObject_RichCompareBool(startkey, key, Py_EQ); - Py_DECREF(startkey); - if (cmp < 0) { - return DKIX_ERROR; - } - if (dk == mp->ma_keys && ep->me_key == startkey) { - if (cmp > 0) { - return ix; - } - } - else { - /* The dict was mutated, restart */ - return DKIX_KEY_CHANGED; - } - } + return do_lookup(mp, dk, key, hash, compare_unicode_generic); +} + +static inline Py_ALWAYS_INLINE Py_ssize_t +compare_unicode_unicode(PyDictObject *mp, PyDictKeysObject *dk, + void *ep0, Py_ssize_t ix, PyObject *key, Py_hash_t hash) +{ + PyDictUnicodeEntry *ep = &((PyDictUnicodeEntry *)ep0)[ix]; + assert(ep->me_key != NULL); + assert(PyUnicode_CheckExact(ep->me_key)); + if (ep->me_key == key || + (unicode_get_hash(ep->me_key) == hash && unicode_eq(ep->me_key, key))) { + return 1; + } + return 0; +} + +static Py_ssize_t _Py_HOT_FUNCTION +unicodekeys_lookup_unicode(PyDictKeysObject* dk, PyObject *key, Py_hash_t hash) +{ + return do_lookup(NULL, dk, key, hash, compare_unicode_unicode); +} + +static inline Py_ALWAYS_INLINE Py_ssize_t +compare_generic(PyDictObject *mp, PyDictKeysObject *dk, + void *ep0, Py_ssize_t ix, PyObject *key, Py_hash_t hash) +{ + PyDictKeyEntry *ep = &((PyDictKeyEntry *)ep0)[ix]; + assert(ep->me_key != NULL); + if (ep->me_key == key) { + return 1; + } + if (ep->me_hash == hash) { + PyObject *startkey = ep->me_key; + Py_INCREF(startkey); + int cmp = PyObject_RichCompareBool(startkey, key, Py_EQ); + Py_DECREF(startkey); + if (cmp < 0) { + return DKIX_ERROR; } - else if (ix == DKIX_EMPTY) { - return DKIX_EMPTY; + if (dk == mp->ma_keys && ep->me_key == startkey) { + return cmp; + } + else { + /* The dict was mutated, restart */ + return DKIX_KEY_CHANGED; } - perturb >>= PERTURB_SHIFT; - i = mask & (i*5 + perturb + 1); } - Py_UNREACHABLE(); + return 0; +} + +static Py_ssize_t +dictkeys_generic_lookup(PyDictObject *mp, PyDictKeysObject* dk, PyObject *key, Py_hash_t hash) +{ + return do_lookup(mp, dk, key, hash, compare_generic); } /* Lookup a string in a (all unicode) dict keys. @@ -1240,6 +1258,8 @@ insertdict(PyInterpreterState *interp, PyDictObject *mp, { PyObject *old_value; + ASSERT_DICT_LOCKED(mp); + if (DK_IS_UNICODE(mp->ma_keys) && !PyUnicode_CheckExact(key)) { if (insertion_resize(interp, mp, 0) < 0) goto Fail; @@ -1336,6 +1356,7 @@ insert_to_emptydict(PyInterpreterState *interp, PyDictObject *mp, PyObject *key, Py_hash_t hash, PyObject *value) { assert(mp->ma_keys == Py_EMPTY_KEYS); + ASSERT_DICT_LOCKED(mp); uint64_t new_version = _PyDict_NotifyEvent( interp, PyDict_EVENT_ADDED, mp, key, value); @@ -1429,6 +1450,8 @@ dictresize(PyInterpreterState *interp, PyDictObject *mp, PyDictKeysObject *oldkeys; PyDictValues *oldvalues; + ASSERT_DICT_LOCKED(mp); + if (log2_newsize >= SIZEOF_SIZE_T*8) { PyErr_NoMemory(); return -1; @@ -1544,32 +1567,13 @@ dictresize(PyInterpreterState *interp, PyDictObject *mp, } } - // We can not use free_keys_object here because key's reference - // are moved already. if (oldkeys != Py_EMPTY_KEYS) { #ifdef Py_REF_DEBUG _Py_DecRefTotal(_PyInterpreterState_GET()); #endif assert(oldkeys->dk_kind != DICT_KEYS_SPLIT); assert(oldkeys->dk_refcnt == 1); -#if PyDict_MAXFREELIST > 0 - struct _Py_dict_state *state = get_dict_state(interp); -#ifdef Py_DEBUG - // dictresize() must not be called after _PyDict_Fini() - assert(state->keys_numfree != -1); -#endif - if (DK_LOG_SIZE(oldkeys) == PyDict_LOG_MINSIZE && - DK_IS_UNICODE(oldkeys) && - state->keys_numfree < PyDict_MAXFREELIST) - { - state->keys_free_list[state->keys_numfree++] = oldkeys; - OBJECT_STAT_INC(to_freelist); - } - else -#endif - { - PyObject_Free(oldkeys); - } + free_keys_object(oldkeys); } } @@ -1642,7 +1646,7 @@ _PyDict_FromItems(PyObject *const *keys, Py_ssize_t keys_offset, for (Py_ssize_t i = 0; i < length; i++) { PyObject *key = *ks; PyObject *value = *vs; - if (PyDict_SetItem(dict, key, value) < 0) { + if (setitem_lock_held((PyDictObject *)dict, key, value) < 0) { Py_DECREF(dict); return NULL; } @@ -1717,6 +1721,7 @@ PyDict_GetItem(PyObject *op, PyObject *key) Py_ssize_t _PyDict_LookupIndex(PyDictObject *mp, PyObject *key) { + // TODO: Thread safety PyObject *value; assert(PyDict_CheckExact((PyObject*)mp)); assert(PyUnicode_CheckExact(key)); @@ -1893,9 +1898,11 @@ _PyDict_LoadGlobal(PyDictObject *globals, PyDictObject *builtins, PyObject *key) } /* Consumes references to key and value */ -int -_PyDict_SetItem_Take2(PyDictObject *mp, PyObject *key, PyObject *value) +static int +setitem_take2_lock_held(PyDictObject *mp, PyObject *key, PyObject *value) { + ASSERT_DICT_LOCKED(mp); + assert(key); assert(value); assert(PyDict_Check(mp)); @@ -1908,7 +1915,9 @@ _PyDict_SetItem_Take2(PyDictObject *mp, PyObject *key, PyObject *value) return -1; } } + PyInterpreterState *interp = _PyInterpreterState_GET(); + if (mp->ma_keys == Py_EMPTY_KEYS) { return insert_to_emptydict(interp, mp, key, hash, value); } @@ -1916,6 +1925,16 @@ _PyDict_SetItem_Take2(PyDictObject *mp, PyObject *key, PyObject *value) return insertdict(interp, mp, key, hash, value); } +int +_PyDict_SetItem_Take2(PyDictObject *mp, PyObject *key, PyObject *value) +{ + int res; + Py_BEGIN_CRITICAL_SECTION(mp); + res = setitem_take2_lock_held(mp, key, value); + Py_END_CRITICAL_SECTION(); + return res; +} + /* CAUTION: PyDict_SetItem() must guarantee that it won't resize the * dictionary if it's merely replacing the value for an existing key. * This means that it's safe to loop over a dictionary with PyDict_Next() @@ -1935,6 +1954,16 @@ PyDict_SetItem(PyObject *op, PyObject *key, PyObject *value) Py_NewRef(key), Py_NewRef(value)); } +static int +setitem_lock_held(PyDictObject *mp, PyObject *key, PyObject *value) +{ + assert(key); + assert(value); + return setitem_take2_lock_held(mp, + Py_NewRef(key), Py_NewRef(value)); +} + + int _PyDict_SetItem_KnownHash(PyObject *op, PyObject *key, PyObject *value, Py_hash_t hash) @@ -1950,12 +1979,21 @@ _PyDict_SetItem_KnownHash(PyObject *op, PyObject *key, PyObject *value, assert(hash != -1); mp = (PyDictObject *)op; + int res; PyInterpreterState *interp = _PyInterpreterState_GET(); + + Py_BEGIN_CRITICAL_SECTION(mp); + if (mp->ma_keys == Py_EMPTY_KEYS) { - return insert_to_emptydict(interp, mp, Py_NewRef(key), hash, Py_NewRef(value)); + res = insert_to_emptydict(interp, mp, Py_NewRef(key), hash, Py_NewRef(value)); } - /* insertdict() handles any resizing that might be necessary */ - return insertdict(interp, mp, Py_NewRef(key), hash, Py_NewRef(value)); + else { + /* insertdict() handles any resizing that might be necessary */ + res = insertdict(interp, mp, Py_NewRef(key), hash, Py_NewRef(value)); + } + + Py_END_CRITICAL_SECTION(); + return res; } static void @@ -1980,6 +2018,8 @@ delitem_common(PyDictObject *mp, Py_hash_t hash, Py_ssize_t ix, { PyObject *old_key; + ASSERT_DICT_LOCKED(mp); + Py_ssize_t hashpos = lookdict_index(mp->ma_keys, hash, ix); assert(hashpos >= 0); @@ -2031,8 +2071,8 @@ PyDict_DelItem(PyObject *op, PyObject *key) return _PyDict_DelItem_KnownHash(op, key, hash); } -int -_PyDict_DelItem_KnownHash(PyObject *op, PyObject *key, Py_hash_t hash) +static int +delitem_knownhash_lock_held(PyObject *op, PyObject *key, Py_hash_t hash) { Py_ssize_t ix; PyDictObject *mp; @@ -2042,6 +2082,9 @@ _PyDict_DelItem_KnownHash(PyObject *op, PyObject *key, Py_hash_t hash) PyErr_BadInternalCall(); return -1; } + + ASSERT_DICT_LOCKED(op); + assert(key); assert(hash != -1); mp = (PyDictObject *)op; @@ -2059,13 +2102,19 @@ _PyDict_DelItem_KnownHash(PyObject *op, PyObject *key, Py_hash_t hash) return delitem_common(mp, hash, ix, old_value, new_version); } -/* This function promises that the predicate -> deletion sequence is atomic - * (i.e. protected by the GIL), assuming the predicate itself doesn't - * release the GIL. - */ int -_PyDict_DelItemIf(PyObject *op, PyObject *key, - int (*predicate)(PyObject *value)) +_PyDict_DelItem_KnownHash(PyObject *op, PyObject *key, Py_hash_t hash) +{ + int res; + Py_BEGIN_CRITICAL_SECTION(op); + res = delitem_knownhash_lock_held(op, key, hash); + Py_END_CRITICAL_SECTION(); + return res; +} + +static int +delitemif_lock_held(PyObject *op, PyObject *key, + int (*predicate)(PyObject *value)) { Py_ssize_t hashpos, ix; PyDictObject *mp; @@ -2073,6 +2122,8 @@ _PyDict_DelItemIf(PyObject *op, PyObject *key, PyObject *old_value; int res; + ASSERT_DICT_LOCKED(op); + if (!PyDict_Check(op)) { PyErr_BadInternalCall(); return -1; @@ -2106,16 +2157,32 @@ _PyDict_DelItemIf(PyObject *op, PyObject *key, return 0; } } +/* This function promises that the predicate -> deletion sequence is atomic + * (i.e. protected by the GIL or the per-dict mutex in free threaded builds), + * assuming the predicate itself doesn't release the GIL (or cause re-entrancy + * which would release the per-dict mutex) + */ +int +_PyDict_DelItemIf(PyObject *op, PyObject *key, + int (*predicate)(PyObject *value)) +{ + int res; + Py_BEGIN_CRITICAL_SECTION(op); + res = delitemif_lock_held(op, key, predicate); + Py_END_CRITICAL_SECTION(); + return res; +} - -void -PyDict_Clear(PyObject *op) +static void +clear_lock_held(PyObject *op) { PyDictObject *mp; PyDictKeysObject *oldkeys; PyDictValues *oldvalues; Py_ssize_t i, n; + ASSERT_DICT_LOCKED(op); + if (!PyDict_Check(op)) return; mp = ((PyDictObject *)op); @@ -2148,6 +2215,14 @@ PyDict_Clear(PyObject *op) ASSERT_CONSISTENT(mp); } +void +PyDict_Clear(PyObject *op) +{ + Py_BEGIN_CRITICAL_SECTION(op); + clear_lock_held(op); + Py_END_CRITICAL_SECTION(); +} + /* Internal version of PyDict_Next that returns a hash value in addition * to the key and value. * Return 1 on success, return 0 when the reached the end of the dictionary @@ -2164,6 +2239,9 @@ _PyDict_Next(PyObject *op, Py_ssize_t *ppos, PyObject **pkey, if (!PyDict_Check(op)) return 0; + + ASSERT_DICT_LOCKED(op); + mp = (PyDictObject *)op; i = *ppos; if (mp->ma_values) { @@ -2237,7 +2315,11 @@ _PyDict_Next(PyObject *op, Py_ssize_t *ppos, PyObject **pkey, int PyDict_Next(PyObject *op, Py_ssize_t *ppos, PyObject **pkey, PyObject **pvalue) { - return _PyDict_Next(op, ppos, pkey, pvalue, NULL); + int res; + Py_BEGIN_CRITICAL_SECTION(op); + res = _PyDict_Next(op, ppos, pkey, pvalue, NULL); + Py_END_CRITICAL_SECTION(); + return res; } @@ -2248,6 +2330,8 @@ _PyDict_Pop_KnownHash(PyDictObject *mp, PyObject *key, Py_hash_t hash, { assert(PyDict_Check(mp)); + ASSERT_DICT_LOCKED(mp); + if (mp->ma_used == 0) { if (result) { *result = NULL; @@ -2287,10 +2371,11 @@ _PyDict_Pop_KnownHash(PyDictObject *mp, PyObject *key, Py_hash_t hash, return 1; } - -int -PyDict_Pop(PyObject *op, PyObject *key, PyObject **result) +static int +pop_lock_held(PyObject *op, PyObject *key, PyObject **result) { + ASSERT_DICT_LOCKED(op); + if (!PyDict_Check(op)) { if (result) { *result = NULL; @@ -2320,6 +2405,17 @@ PyDict_Pop(PyObject *op, PyObject *key, PyObject **result) return _PyDict_Pop_KnownHash(dict, key, hash, result); } +int +PyDict_Pop(PyObject *op, PyObject *key, PyObject **result) +{ + int err; + Py_BEGIN_CRITICAL_SECTION(op); + err = pop_lock_held(op, key, result); + Py_END_CRITICAL_SECTION(); + + return err; +} + int PyDict_PopString(PyObject *op, const char *key, PyObject **result) @@ -2352,6 +2448,55 @@ _PyDict_Pop(PyObject *dict, PyObject *key, PyObject *default_value) return result; } +static PyDictObject * +dict_dict_fromkeys(PyInterpreterState *interp, PyDictObject *mp, + PyObject *iterable, PyObject *value) +{ + PyObject *oldvalue; + Py_ssize_t pos = 0; + PyObject *key; + Py_hash_t hash; + int unicode = DK_IS_UNICODE(((PyDictObject*)iterable)->ma_keys); + uint8_t new_size = Py_MAX( + estimate_log2_keysize(PyDict_GET_SIZE(iterable)), + DK_LOG_SIZE(mp->ma_keys)); + if (dictresize(interp, mp, new_size, unicode)) { + Py_DECREF(mp); + return NULL; + } + + while (_PyDict_Next(iterable, &pos, &key, &oldvalue, &hash)) { + if (insertdict(interp, mp, + Py_NewRef(key), hash, Py_NewRef(value))) { + Py_DECREF(mp); + return NULL; + } + } + return mp; +} + +static PyDictObject * +dict_set_fromkeys(PyInterpreterState *interp, PyDictObject *mp, + PyObject *iterable, PyObject *value) +{ + Py_ssize_t pos = 0; + PyObject *key; + Py_hash_t hash; + + if (dictresize(interp, mp, + estimate_log2_keysize(PySet_GET_SIZE(iterable)), 0)) { + Py_DECREF(mp); + return NULL; + } + + while (_PySet_NextEntry(iterable, &pos, &key, &hash)) { + if (insertdict(interp, mp, Py_NewRef(key), hash, Py_NewRef(value))) { + Py_DECREF(mp); + return NULL; + } + } + return mp; +} /* Internal version of dict.from_keys(). It is subclass-friendly. */ PyObject * @@ -2367,49 +2512,22 @@ _PyDict_FromKeys(PyObject *cls, PyObject *iterable, PyObject *value) if (d == NULL) return NULL; - if (PyDict_CheckExact(d) && ((PyDictObject *)d)->ma_used == 0) { + + if (PyDict_CheckExact(d)) { if (PyDict_CheckExact(iterable)) { PyDictObject *mp = (PyDictObject *)d; - PyObject *oldvalue; - Py_ssize_t pos = 0; - PyObject *key; - Py_hash_t hash; - - int unicode = DK_IS_UNICODE(((PyDictObject*)iterable)->ma_keys); - if (dictresize(interp, mp, - estimate_log2_keysize(PyDict_GET_SIZE(iterable)), - unicode)) { - Py_DECREF(d); - return NULL; - } - while (_PyDict_Next(iterable, &pos, &key, &oldvalue, &hash)) { - if (insertdict(interp, mp, - Py_NewRef(key), hash, Py_NewRef(value))) { - Py_DECREF(d); - return NULL; - } - } + Py_BEGIN_CRITICAL_SECTION2(d, iterable); + d = (PyObject *)dict_dict_fromkeys(interp, mp, iterable, value); + Py_END_CRITICAL_SECTION2(); return d; } - if (PyAnySet_CheckExact(iterable)) { + else if (PyAnySet_CheckExact(iterable)) { PyDictObject *mp = (PyDictObject *)d; - Py_ssize_t pos = 0; - PyObject *key; - Py_hash_t hash; - if (dictresize(interp, mp, - estimate_log2_keysize(PySet_GET_SIZE(iterable)), 0)) { - Py_DECREF(d); - return NULL; - } - - while (_PySet_NextEntry(iterable, &pos, &key, &hash)) { - if (insertdict(interp, mp, Py_NewRef(key), hash, Py_NewRef(value))) { - Py_DECREF(d); - return NULL; - } - } + Py_BEGIN_CRITICAL_SECTION2(d, iterable); + d = (PyObject *)dict_set_fromkeys(interp, mp, iterable, value); + Py_END_CRITICAL_SECTION2(); return d; } } @@ -2421,12 +2539,17 @@ _PyDict_FromKeys(PyObject *cls, PyObject *iterable, PyObject *value) } if (PyDict_CheckExact(d)) { + Py_BEGIN_CRITICAL_SECTION(d); while ((key = PyIter_Next(it)) != NULL) { - status = PyDict_SetItem(d, key, value); + status = setitem_lock_held((PyDictObject *)d, key, value); Py_DECREF(key); - if (status < 0) - goto Fail; + if (status < 0) { + assert(PyErr_Occurred()); + goto dict_iter_exit; + } } +dict_iter_exit: + Py_END_CRITICAL_SECTION(); } else { while ((key = PyIter_Next(it)) != NULL) { status = PyObject_SetItem(d, key, value); @@ -2450,8 +2573,9 @@ _PyDict_FromKeys(PyObject *cls, PyObject *iterable, PyObject *value) /* Methods */ static void -dict_dealloc(PyDictObject *mp) +dict_dealloc(PyObject *self) { + PyDictObject *mp = (PyDictObject *)self; PyInterpreterState *interp = _PyInterpreterState_GET(); assert(Py_REFCNT(mp) == 0); Py_SET_REFCNT(mp, 1); @@ -2479,13 +2603,10 @@ dict_dealloc(PyDictObject *mp) assert(keys->dk_refcnt == 1 || keys == Py_EMPTY_KEYS); dictkeys_decref(interp, keys); } -#if PyDict_MAXFREELIST > 0 - struct _Py_dict_state *state = get_dict_state(interp); -#ifdef Py_DEBUG - // new_dict() must not be called after _PyDict_Fini() - assert(state->numfree != -1); -#endif - if (state->numfree < PyDict_MAXFREELIST && Py_IS_TYPE(mp, &PyDict_Type)) { +#ifdef WITH_FREELISTS + struct _Py_dict_freelist *state = get_dict_state(); + if (state->numfree < PyDict_MAXFREELIST && state->numfree >=0 && + Py_IS_TYPE(mp, &PyDict_Type)) { state->free_list[state->numfree++] = mp; OBJECT_STAT_INC(to_freelist); } @@ -2499,8 +2620,9 @@ dict_dealloc(PyDictObject *mp) static PyObject * -dict_repr(PyDictObject *mp) +dict_repr_lock_held(PyObject *self) { + PyDictObject *mp = (PyDictObject *)self; Py_ssize_t i; PyObject *key = NULL, *value = NULL; _PyUnicodeWriter writer; @@ -2528,7 +2650,7 @@ dict_repr(PyDictObject *mp) Note that repr may mutate the dict. */ i = 0; first = 1; - while (PyDict_Next((PyObject *)mp, &i, &key, &value)) { + while (_PyDict_Next((PyObject *)mp, &i, &key, &value, NULL)) { PyObject *s; int res; @@ -2581,15 +2703,27 @@ dict_repr(PyDictObject *mp) return NULL; } +static PyObject * +dict_repr(PyObject *self) +{ + PyObject *res; + Py_BEGIN_CRITICAL_SECTION(self); + res = dict_repr_lock_held(self); + Py_END_CRITICAL_SECTION(); + return res; +} + static Py_ssize_t -dict_length(PyDictObject *mp) +dict_length(PyObject *self) { - return mp->ma_used; + PyDictObject *mp = (PyDictObject *)self; + return _Py_atomic_load_ssize_relaxed(&mp->ma_used); } static PyObject * -dict_subscript(PyDictObject *mp, PyObject *key) +dict_subscript_lock_held(PyObject *self, PyObject *key) { + PyDictObject *mp = (PyDictObject *)self; Py_ssize_t ix; Py_hash_t hash; PyObject *value; @@ -2622,24 +2756,41 @@ dict_subscript(PyDictObject *mp, PyObject *key) return Py_NewRef(value); } +static PyObject * +dict_subscript(PyObject *self, PyObject *key) +{ + PyObject *res; + Py_BEGIN_CRITICAL_SECTION(self); + res = dict_subscript_lock_held(self, key); + Py_END_CRITICAL_SECTION(); + return res; +} + static int -dict_ass_sub(PyDictObject *mp, PyObject *v, PyObject *w) +dict_ass_sub(PyObject *mp, PyObject *v, PyObject *w) { if (w == NULL) - return PyDict_DelItem((PyObject *)mp, v); + return PyDict_DelItem(mp, v); else - return PyDict_SetItem((PyObject *)mp, v, w); + return PyDict_SetItem(mp, v, w); } static PyMappingMethods dict_as_mapping = { - (lenfunc)dict_length, /*mp_length*/ - (binaryfunc)dict_subscript, /*mp_subscript*/ - (objobjargproc)dict_ass_sub, /*mp_ass_subscript*/ + dict_length, /*mp_length*/ + dict_subscript, /*mp_subscript*/ + dict_ass_sub, /*mp_ass_subscript*/ }; static PyObject * -dict_keys(PyDictObject *mp) +keys_lock_held(PyObject *dict) { + ASSERT_DICT_LOCKED(dict); + + if (dict == NULL || !PyDict_Check(dict)) { + PyErr_BadInternalCall(); + return NULL; + } + PyDictObject *mp = (PyDictObject *)dict; PyObject *v; Py_ssize_t n; @@ -2668,9 +2819,27 @@ dict_keys(PyDictObject *mp) return v; } +PyObject * +PyDict_Keys(PyObject *dict) +{ + PyObject *res; + Py_BEGIN_CRITICAL_SECTION(dict); + res = keys_lock_held(dict); + Py_END_CRITICAL_SECTION(); + + return res; +} + static PyObject * -dict_values(PyDictObject *mp) +values_lock_held(PyObject *dict) { + ASSERT_DICT_LOCKED(dict); + + if (dict == NULL || !PyDict_Check(dict)) { + PyErr_BadInternalCall(); + return NULL; + } + PyDictObject *mp = (PyDictObject *)dict; PyObject *v; Py_ssize_t n; @@ -2699,9 +2868,26 @@ dict_values(PyDictObject *mp) return v; } +PyObject * +PyDict_Values(PyObject *dict) +{ + PyObject *res; + Py_BEGIN_CRITICAL_SECTION(dict); + res = values_lock_held(dict); + Py_END_CRITICAL_SECTION(); + return res; +} + static PyObject * -dict_items(PyDictObject *mp) +items_lock_held(PyObject *dict) { + ASSERT_DICT_LOCKED(dict); + + if (dict == NULL || !PyDict_Check(dict)) { + PyErr_BadInternalCall(); + return NULL; + } + PyDictObject *mp = (PyDictObject *)dict; PyObject *v; Py_ssize_t i, n; PyObject *item; @@ -2745,6 +2931,17 @@ dict_items(PyDictObject *mp) return v; } +PyObject * +PyDict_Items(PyObject *dict) +{ + PyObject *res; + Py_BEGIN_CRITICAL_SECTION(dict); + res = items_lock_held(dict); + Py_END_CRITICAL_SECTION(); + + return res; +} + /*[clinic input] @classmethod dict.fromkeys @@ -2823,8 +3020,8 @@ dict_update(PyObject *self, PyObject *args, PyObject *kwds) producing iterable objects of length 2. */ -int -PyDict_MergeFromSeq2(PyObject *d, PyObject *seq2, int override) +static int +merge_from_seq2_lock_held(PyObject *d, PyObject *seq2, int override) { PyObject *it; /* iter(seq2) */ Py_ssize_t i; /* index into seq2 of current element */ @@ -2876,14 +3073,14 @@ PyDict_MergeFromSeq2(PyObject *d, PyObject *seq2, int override) Py_INCREF(key); Py_INCREF(value); if (override) { - if (PyDict_SetItem(d, key, value) < 0) { + if (setitem_lock_held((PyDictObject *)d, key, value) < 0) { Py_DECREF(key); Py_DECREF(value); goto Fail; } } else { - if (PyDict_SetDefault(d, key, value) == NULL) { + if (dict_setdefault_ref_lock_held(d, key, value, NULL, 0) < 0) { Py_DECREF(key); Py_DECREF(value); goto Fail; @@ -2908,6 +3105,117 @@ PyDict_MergeFromSeq2(PyObject *d, PyObject *seq2, int override) return Py_SAFE_DOWNCAST(i, Py_ssize_t, int); } +int +PyDict_MergeFromSeq2(PyObject *d, PyObject *seq2, int override) +{ + int res; + Py_BEGIN_CRITICAL_SECTION(d); + res = merge_from_seq2_lock_held(d, seq2, override); + Py_END_CRITICAL_SECTION(); + + return res; +} + +static int +dict_dict_merge(PyInterpreterState *interp, PyDictObject *mp, PyDictObject *other, int override) +{ + if (other == mp || other->ma_used == 0) + /* a.update(a) or a.update({}); nothing to do */ + return 0; + if (mp->ma_used == 0) { + /* Since the target dict is empty, PyDict_GetItem() + * always returns NULL. Setting override to 1 + * skips the unnecessary test. + */ + override = 1; + PyDictKeysObject *okeys = other->ma_keys; + + // If other is clean, combined, and just allocated, just clone it. + if (other->ma_values == NULL && + other->ma_used == okeys->dk_nentries && + (DK_LOG_SIZE(okeys) == PyDict_LOG_MINSIZE || + USABLE_FRACTION(DK_SIZE(okeys)/2) < other->ma_used)) { + uint64_t new_version = _PyDict_NotifyEvent( + interp, PyDict_EVENT_CLONED, mp, (PyObject *)other, NULL); + PyDictKeysObject *keys = clone_combined_dict_keys(other); + if (keys == NULL) + return -1; + + dictkeys_decref(interp, mp->ma_keys); + mp->ma_keys = keys; + if (mp->ma_values != NULL) { + free_values(mp->ma_values); + mp->ma_values = NULL; + } + + mp->ma_used = other->ma_used; + mp->ma_version_tag = new_version; + ASSERT_CONSISTENT(mp); + + if (_PyObject_GC_IS_TRACKED(other) && !_PyObject_GC_IS_TRACKED(mp)) { + /* Maintain tracking. */ + _PyObject_GC_TRACK(mp); + } + + return 0; + } + } + /* Do one big resize at the start, rather than + * incrementally resizing as we insert new items. Expect + * that there will be no (or few) overlapping keys. + */ + if (USABLE_FRACTION(DK_SIZE(mp->ma_keys)) < other->ma_used) { + int unicode = DK_IS_UNICODE(other->ma_keys); + if (dictresize(interp, mp, + estimate_log2_keysize(mp->ma_used + other->ma_used), + unicode)) { + return -1; + } + } + + Py_ssize_t orig_size = other->ma_keys->dk_nentries; + Py_ssize_t pos = 0; + Py_hash_t hash; + PyObject *key, *value; + + while (_PyDict_Next((PyObject*)other, &pos, &key, &value, &hash)) { + int err = 0; + Py_INCREF(key); + Py_INCREF(value); + if (override == 1) { + err = insertdict(interp, mp, + Py_NewRef(key), hash, Py_NewRef(value)); + } + else { + err = contains_known_hash_lock_held(mp, key, hash); + if (err == 0) { + err = insertdict(interp, mp, + Py_NewRef(key), hash, Py_NewRef(value)); + } + else if (err > 0) { + if (override != 0) { + _PyErr_SetKeyError(key); + Py_DECREF(value); + Py_DECREF(key); + return -1; + } + err = 0; + } + } + Py_DECREF(value); + Py_DECREF(key); + if (err != 0) + return -1; + + if (orig_size != other->ma_keys->dk_nentries) { + PyErr_SetString(PyExc_RuntimeError, + "dict mutated during update"); + return -1; + } + } + return 0; +} + static int dict_merge(PyInterpreterState *interp, PyObject *a, PyObject *b, int override) { @@ -2925,127 +3233,44 @@ dict_merge(PyInterpreterState *interp, PyObject *a, PyObject *b, int override) return -1; } mp = (PyDictObject*)a; - if (PyDict_Check(b) && (Py_TYPE(b)->tp_iter == (getiterfunc)dict_iter)) { + int res = 0; + if (PyDict_Check(b) && (Py_TYPE(b)->tp_iter == dict_iter)) { other = (PyDictObject*)b; - if (other == mp || other->ma_used == 0) - /* a.update(a) or a.update({}); nothing to do */ - return 0; - if (mp->ma_used == 0) { - /* Since the target dict is empty, PyDict_GetItem() - * always returns NULL. Setting override to 1 - * skips the unnecessary test. - */ - override = 1; - PyDictKeysObject *okeys = other->ma_keys; - - // If other is clean, combined, and just allocated, just clone it. - if (other->ma_values == NULL && - other->ma_used == okeys->dk_nentries && - (DK_LOG_SIZE(okeys) == PyDict_LOG_MINSIZE || - USABLE_FRACTION(DK_SIZE(okeys)/2) < other->ma_used)) { - uint64_t new_version = _PyDict_NotifyEvent( - interp, PyDict_EVENT_CLONED, mp, b, NULL); - PyDictKeysObject *keys = clone_combined_dict_keys(other); - if (keys == NULL) { - return -1; - } - - dictkeys_decref(interp, mp->ma_keys); - mp->ma_keys = keys; - if (mp->ma_values != NULL) { - free_values(mp->ma_values); - mp->ma_values = NULL; - } - - mp->ma_used = other->ma_used; - mp->ma_version_tag = new_version; - ASSERT_CONSISTENT(mp); - - if (_PyObject_GC_IS_TRACKED(other) && !_PyObject_GC_IS_TRACKED(mp)) { - /* Maintain tracking. */ - _PyObject_GC_TRACK(mp); - } - - return 0; - } - } - /* Do one big resize at the start, rather than - * incrementally resizing as we insert new items. Expect - * that there will be no (or few) overlapping keys. - */ - if (USABLE_FRACTION(DK_SIZE(mp->ma_keys)) < other->ma_used) { - int unicode = DK_IS_UNICODE(other->ma_keys); - if (dictresize(interp, mp, - estimate_log2_keysize(mp->ma_used + other->ma_used), - unicode)) { - return -1; - } - } - - Py_ssize_t orig_size = other->ma_keys->dk_nentries; - Py_ssize_t pos = 0; - Py_hash_t hash; - PyObject *key, *value; - - while (_PyDict_Next((PyObject*)other, &pos, &key, &value, &hash)) { - int err = 0; - Py_INCREF(key); - Py_INCREF(value); - if (override == 1) { - err = insertdict(interp, mp, - Py_NewRef(key), hash, Py_NewRef(value)); - } - else { - err = _PyDict_Contains_KnownHash(a, key, hash); - if (err == 0) { - err = insertdict(interp, mp, - Py_NewRef(key), hash, Py_NewRef(value)); - } - else if (err > 0) { - if (override != 0) { - _PyErr_SetKeyError(key); - Py_DECREF(value); - Py_DECREF(key); - return -1; - } - err = 0; - } - } - Py_DECREF(value); - Py_DECREF(key); - if (err != 0) - return -1; - - if (orig_size != other->ma_keys->dk_nentries) { - PyErr_SetString(PyExc_RuntimeError, - "dict mutated during update"); - return -1; - } - } + int res; + Py_BEGIN_CRITICAL_SECTION2(a, b); + res = dict_dict_merge(interp, (PyDictObject *)a, other, override); + ASSERT_CONSISTENT(a); + Py_END_CRITICAL_SECTION2(); + return res; } else { /* Do it the generic, slower way */ + Py_BEGIN_CRITICAL_SECTION(a); PyObject *keys = PyMapping_Keys(b); PyObject *iter; PyObject *key, *value; int status; - if (keys == NULL) + if (keys == NULL) { /* Docstring says this is equivalent to E.keys() so * if E doesn't have a .keys() method we want * AttributeError to percolate up. Might as well * do the same for any other error. */ - return -1; + res = -1; + goto slow_exit; + } iter = PyObject_GetIter(keys); Py_DECREF(keys); - if (iter == NULL) - return -1; + if (iter == NULL) { + res = -1; + goto slow_exit; + } for (key = PyIter_Next(iter); key; key = PyIter_Next(iter)) { if (override != 1) { - status = PyDict_Contains(a, key); + status = contains_lock_held(mp, key); if (status != 0) { if (status > 0) { if (override == 0) { @@ -3056,30 +3281,39 @@ dict_merge(PyInterpreterState *interp, PyObject *a, PyObject *b, int override) } Py_DECREF(key); Py_DECREF(iter); - return -1; + res = -1; + goto slow_exit; } } value = PyObject_GetItem(b, key); if (value == NULL) { Py_DECREF(iter); Py_DECREF(key); - return -1; + res = -1; + goto slow_exit; } - status = PyDict_SetItem(a, key, value); + status = setitem_lock_held(mp, key, value); Py_DECREF(key); Py_DECREF(value); if (status < 0) { Py_DECREF(iter); + res = -1; + goto slow_exit; return -1; } } Py_DECREF(iter); - if (PyErr_Occurred()) + if (PyErr_Occurred()) { /* Iterator completed, via error */ - return -1; + res = -1; + goto slow_exit; + } + +slow_exit: + ASSERT_CONSISTENT(a); + Py_END_CRITICAL_SECTION(); + return res; } - ASSERT_CONSISTENT(a); - return 0; } int @@ -3104,23 +3338,27 @@ _PyDict_MergeEx(PyObject *a, PyObject *b, int override) return dict_merge(interp, a, b, override); } +/*[clinic input] +dict.copy + +Return a shallow copy of the dict. +[clinic start generated code]*/ + static PyObject * -dict_copy(PyDictObject *mp, PyObject *Py_UNUSED(ignored)) +dict_copy_impl(PyDictObject *self) +/*[clinic end generated code: output=ffb782cf970a5c39 input=73935f042b639de4]*/ { - return PyDict_Copy((PyObject*)mp); + return PyDict_Copy((PyObject *)self); } -PyObject * -PyDict_Copy(PyObject *o) +static PyObject * +copy_lock_held(PyObject *o) { PyObject *copy; PyDictObject *mp; PyInterpreterState *interp = _PyInterpreterState_GET(); - if (o == NULL || !PyDict_Check(o)) { - PyErr_BadInternalCall(); - return NULL; - } + ASSERT_DICT_LOCKED(o); mp = (PyDictObject *)o; if (mp->ma_used == 0) { @@ -3155,7 +3393,7 @@ PyDict_Copy(PyObject *o) return (PyObject *)split_copy; } - if (Py_TYPE(mp)->tp_iter == (getiterfunc)dict_iter && + if (Py_TYPE(mp)->tp_iter == dict_iter && mp->ma_values == NULL && (mp->ma_used >= (mp->ma_keys->dk_nentries * 2) / 3)) { @@ -3203,44 +3441,31 @@ PyDict_Copy(PyObject *o) return NULL; } -Py_ssize_t -PyDict_Size(PyObject *mp) -{ - if (mp == NULL || !PyDict_Check(mp)) { - PyErr_BadInternalCall(); - return -1; - } - return ((PyDictObject *)mp)->ma_used; -} - PyObject * -PyDict_Keys(PyObject *mp) +PyDict_Copy(PyObject *o) { - if (mp == NULL || !PyDict_Check(mp)) { + if (o == NULL || !PyDict_Check(o)) { PyErr_BadInternalCall(); return NULL; } - return dict_keys((PyDictObject *)mp); -} -PyObject * -PyDict_Values(PyObject *mp) -{ - if (mp == NULL || !PyDict_Check(mp)) { - PyErr_BadInternalCall(); - return NULL; - } - return dict_values((PyDictObject *)mp); + PyObject *res; + Py_BEGIN_CRITICAL_SECTION(o); + + res = copy_lock_held(o); + + Py_END_CRITICAL_SECTION(); + return res; } -PyObject * -PyDict_Items(PyObject *mp) +Py_ssize_t +PyDict_Size(PyObject *mp) { if (mp == NULL || !PyDict_Check(mp)) { PyErr_BadInternalCall(); - return NULL; + return -1; } - return dict_items((PyDictObject *)mp); + return ((PyDictObject *)mp)->ma_used; } /* Return 1 if dicts equal, 0 if not, -1 if error. @@ -3248,10 +3473,13 @@ PyDict_Items(PyObject *mp) * Uses only Py_EQ comparison. */ static int -dict_equal(PyDictObject *a, PyDictObject *b) +dict_equal_lock_held(PyDictObject *a, PyDictObject *b) { Py_ssize_t i; + ASSERT_DICT_LOCKED(a); + ASSERT_DICT_LOCKED(b); + if (a->ma_used != b->ma_used) /* can't be equal if # of entries differ */ return 0; @@ -3306,6 +3534,17 @@ dict_equal(PyDictObject *a, PyDictObject *b) return 1; } +static int +dict_equal(PyDictObject *a, PyDictObject *b) +{ + int res; + Py_BEGIN_CRITICAL_SECTION2(a, b); + res = dict_equal_lock_held(a, b); + Py_END_CRITICAL_SECTION2(); + + return res; +} + static PyObject * dict_richcompare(PyObject *v, PyObject *w, int op) { @@ -3329,6 +3568,7 @@ dict_richcompare(PyObject *v, PyObject *w, int op) /*[clinic input] @coexist +@critical_section dict.__contains__ key: object @@ -3338,8 +3578,8 @@ True if the dictionary has the specified key, else False. [clinic start generated code]*/ static PyObject * -dict___contains__(PyDictObject *self, PyObject *key) -/*[clinic end generated code: output=a3d03db709ed6e6b input=fe1cb42ad831e820]*/ +dict___contains___impl(PyDictObject *self, PyObject *key) +/*[clinic end generated code: output=1b314e6da7687dae input=bc76ec9c157cb81b]*/ { register PyDictObject *mp = self; Py_hash_t hash; @@ -3360,6 +3600,7 @@ dict___contains__(PyDictObject *self, PyObject *key) } /*[clinic input] +@critical_section dict.get key: object @@ -3371,7 +3612,7 @@ Return the value for key if key is in the dictionary, else default. static PyObject * dict_get_impl(PyDictObject *self, PyObject *key, PyObject *default_value) -/*[clinic end generated code: output=bba707729dee05bf input=279ddb5790b6b107]*/ +/*[clinic end generated code: output=bba707729dee05bf input=a631d3f18f584c60]*/ { PyObject *val = NULL; Py_hash_t hash; @@ -3391,51 +3632,77 @@ dict_get_impl(PyDictObject *self, PyObject *key, PyObject *default_value) return Py_NewRef(val); } -PyObject * -PyDict_SetDefault(PyObject *d, PyObject *key, PyObject *defaultobj) +static int +dict_setdefault_ref_lock_held(PyObject *d, PyObject *key, PyObject *default_value, + PyObject **result, int incref_result) { PyDictObject *mp = (PyDictObject *)d; PyObject *value; Py_hash_t hash; PyInterpreterState *interp = _PyInterpreterState_GET(); + ASSERT_DICT_LOCKED(d); + if (!PyDict_Check(d)) { PyErr_BadInternalCall(); - return NULL; + if (result) { + *result = NULL; + } + return -1; } if (!PyUnicode_CheckExact(key) || (hash = unicode_get_hash(key)) == -1) { hash = PyObject_Hash(key); - if (hash == -1) - return NULL; + if (hash == -1) { + if (result) { + *result = NULL; + } + return -1; + } } if (mp->ma_keys == Py_EMPTY_KEYS) { if (insert_to_emptydict(interp, mp, Py_NewRef(key), hash, - Py_NewRef(defaultobj)) < 0) { - return NULL; + Py_NewRef(default_value)) < 0) { + if (result) { + *result = NULL; + } + return -1; + } + if (result) { + *result = incref_result ? Py_NewRef(default_value) : default_value; } - return defaultobj; + return 0; } if (!PyUnicode_CheckExact(key) && DK_IS_UNICODE(mp->ma_keys)) { if (insertion_resize(interp, mp, 0) < 0) { - return NULL; + if (result) { + *result = NULL; + } + return -1; } } Py_ssize_t ix = _Py_dict_lookup(mp, key, hash, &value); - if (ix == DKIX_ERROR) - return NULL; + if (ix == DKIX_ERROR) { + if (result) { + *result = NULL; + } + return -1; + } if (ix == DKIX_EMPTY) { uint64_t new_version = _PyDict_NotifyEvent( - interp, PyDict_EVENT_ADDED, mp, key, defaultobj); + interp, PyDict_EVENT_ADDED, mp, key, default_value); mp->ma_keys->dk_version = 0; - value = defaultobj; + value = default_value; if (mp->ma_keys->dk_usable <= 0) { if (insertion_resize(interp, mp, 1) < 0) { - return NULL; + if (result) { + *result = NULL; + } + return -1; } } Py_ssize_t hashpos = find_empty_slot(mp->ma_keys, hash); @@ -3467,11 +3734,16 @@ PyDict_SetDefault(PyObject *d, PyObject *key, PyObject *defaultobj) mp->ma_keys->dk_usable--; mp->ma_keys->dk_nentries++; assert(mp->ma_keys->dk_usable >= 0); + ASSERT_CONSISTENT(mp); + if (result) { + *result = incref_result ? Py_NewRef(value) : value; + } + return 0; } else if (value == NULL) { uint64_t new_version = _PyDict_NotifyEvent( - interp, PyDict_EVENT_ADDED, mp, key, defaultobj); - value = defaultobj; + interp, PyDict_EVENT_ADDED, mp, key, default_value); + value = default_value; assert(_PyDict_HasSplitTable(mp)); assert(mp->ma_values->values[ix] == NULL); MAINTAIN_TRACKING(mp, key, value); @@ -3479,13 +3751,43 @@ PyDict_SetDefault(PyObject *d, PyObject *key, PyObject *defaultobj) _PyDictValues_AddToInsertionOrder(mp->ma_values, ix); mp->ma_used++; mp->ma_version_tag = new_version; + ASSERT_CONSISTENT(mp); + if (result) { + *result = incref_result ? Py_NewRef(value) : value; + } + return 0; } ASSERT_CONSISTENT(mp); - return value; + if (result) { + *result = incref_result ? Py_NewRef(value) : value; + } + return 1; +} + +int +PyDict_SetDefaultRef(PyObject *d, PyObject *key, PyObject *default_value, + PyObject **result) +{ + int res; + Py_BEGIN_CRITICAL_SECTION(d); + res = dict_setdefault_ref_lock_held(d, key, default_value, result, 1); + Py_END_CRITICAL_SECTION(); + return res; +} + +PyObject * +PyDict_SetDefault(PyObject *d, PyObject *key, PyObject *defaultobj) +{ + PyObject *result; + Py_BEGIN_CRITICAL_SECTION(d); + dict_setdefault_ref_lock_held(d, key, defaultobj, &result, 0); + Py_END_CRITICAL_SECTION(); + return result; } /*[clinic input] +@critical_section dict.setdefault key: object @@ -3500,18 +3802,25 @@ Return the value for key if key is in the dictionary, else default. static PyObject * dict_setdefault_impl(PyDictObject *self, PyObject *key, PyObject *default_value) -/*[clinic end generated code: output=f8c1101ebf69e220 input=0f063756e815fd9d]*/ +/*[clinic end generated code: output=f8c1101ebf69e220 input=9237af9a0a224302]*/ { PyObject *val; - - val = PyDict_SetDefault((PyObject *)self, key, default_value); - return Py_XNewRef(val); + dict_setdefault_ref_lock_held((PyObject *)self, key, default_value, &val, 1); + return val; } + +/*[clinic input] +dict.clear + +Remove all items from the dict. +[clinic start generated code]*/ + static PyObject * -dict_clear(PyDictObject *mp, PyObject *Py_UNUSED(ignored)) +dict_clear_impl(PyDictObject *self) +/*[clinic end generated code: output=5139a830df00830a input=0bf729baba97a4c2]*/ { - PyDict_Clear((PyObject *)mp); + PyDict_Clear((PyObject *)self); Py_RETURN_NONE; } @@ -3536,6 +3845,7 @@ dict_pop_impl(PyDictObject *self, PyObject *key, PyObject *default_value) } /*[clinic input] +@critical_section dict.popitem Remove and return a (key, value) pair as a 2-tuple. @@ -3546,7 +3856,7 @@ Raises KeyError if the dict is empty. static PyObject * dict_popitem_impl(PyDictObject *self) -/*[clinic end generated code: output=e65fcb04420d230d input=1c38a49f21f64941]*/ +/*[clinic end generated code: output=e65fcb04420d230d input=ef28b4da5f0f762e]*/ { Py_ssize_t i, j; PyObject *res; @@ -3672,8 +3982,8 @@ dict_tp_clear(PyObject *op) static PyObject *dictiter_new(PyDictObject *, PyTypeObject *); -Py_ssize_t -_PyDict_SizeOf(PyDictObject *mp) +static Py_ssize_t +sizeof_lock_held(PyDictObject *mp) { size_t res = _PyObject_SIZE(Py_TYPE(mp)); if (mp->ma_values) { @@ -3688,6 +3998,17 @@ _PyDict_SizeOf(PyDictObject *mp) return (Py_ssize_t)res; } +Py_ssize_t +_PyDict_SizeOf(PyDictObject *mp) +{ + Py_ssize_t res; + Py_BEGIN_CRITICAL_SECTION(mp); + res = sizeof_lock_held(mp); + Py_END_CRITICAL_SECTION(); + + return res; +} + size_t _PyDict_KeysSize(PyDictKeysObject *keys) { @@ -3699,10 +4020,17 @@ _PyDict_KeysSize(PyDictKeysObject *keys) return size; } +/*[clinic input] +dict.__sizeof__ + +Return the size of the dict in memory, in bytes. +[clinic start generated code]*/ + static PyObject * -dict_sizeof(PyDictObject *mp, PyObject *Py_UNUSED(ignored)) +dict___sizeof___impl(PyDictObject *self) +/*[clinic end generated code: output=44279379b3824bda input=4fec4ddfc44a4d1a]*/ { - return PyLong_FromSsize_t(_PyDict_SizeOf(mp)); + return PyLong_FromSsize_t(_PyDict_SizeOf(self)); } static PyObject * @@ -3734,70 +4062,59 @@ dict_ior(PyObject *self, PyObject *other) PyDoc_STRVAR(getitem__doc__, "__getitem__($self, key, /)\n--\n\nReturn self[key]."); -PyDoc_STRVAR(sizeof__doc__, -"D.__sizeof__() -> size of D in memory, in bytes"); - PyDoc_STRVAR(update__doc__, "D.update([E, ]**F) -> None. Update D from dict/iterable E and F.\n\ If E is present and has a .keys() method, then does: for k in E: D[k] = E[k]\n\ If E is present and lacks a .keys() method, then does: for k, v in E: D[k] = v\n\ In either case, this is followed by: for k in F: D[k] = F[k]"); -PyDoc_STRVAR(clear__doc__, -"D.clear() -> None. Remove all items from D."); - -PyDoc_STRVAR(copy__doc__, -"D.copy() -> a shallow copy of D"); - /* Forward */ -static PyObject *dictkeys_new(PyObject *, PyObject *); -static PyObject *dictitems_new(PyObject *, PyObject *); -static PyObject *dictvalues_new(PyObject *, PyObject *); - -PyDoc_STRVAR(keys__doc__, - "D.keys() -> a set-like object providing a view on D's keys"); -PyDoc_STRVAR(items__doc__, - "D.items() -> a set-like object providing a view on D's items"); -PyDoc_STRVAR(values__doc__, - "D.values() -> an object providing a view on D's values"); static PyMethodDef mapp_methods[] = { DICT___CONTAINS___METHODDEF - {"__getitem__", _PyCFunction_CAST(dict_subscript), METH_O | METH_COEXIST, + {"__getitem__", dict_subscript, METH_O | METH_COEXIST, getitem__doc__}, - {"__sizeof__", _PyCFunction_CAST(dict_sizeof), METH_NOARGS, - sizeof__doc__}, + DICT___SIZEOF___METHODDEF DICT_GET_METHODDEF DICT_SETDEFAULT_METHODDEF DICT_POP_METHODDEF DICT_POPITEM_METHODDEF - {"keys", dictkeys_new, METH_NOARGS, - keys__doc__}, - {"items", dictitems_new, METH_NOARGS, - items__doc__}, - {"values", dictvalues_new, METH_NOARGS, - values__doc__}, + DICT_KEYS_METHODDEF + DICT_ITEMS_METHODDEF + DICT_VALUES_METHODDEF {"update", _PyCFunction_CAST(dict_update), METH_VARARGS | METH_KEYWORDS, update__doc__}, DICT_FROMKEYS_METHODDEF - {"clear", (PyCFunction)dict_clear, METH_NOARGS, - clear__doc__}, - {"copy", (PyCFunction)dict_copy, METH_NOARGS, - copy__doc__}, + DICT_CLEAR_METHODDEF + DICT_COPY_METHODDEF DICT___REVERSED___METHODDEF {"__class_getitem__", Py_GenericAlias, METH_O|METH_CLASS, PyDoc_STR("See PEP 585")}, {NULL, NULL} /* sentinel */ }; -/* Return 1 if `key` is in dict `op`, 0 if not, and -1 on error. */ -int -PyDict_Contains(PyObject *op, PyObject *key) +static int +contains_known_hash_lock_held(PyDictObject *mp, PyObject *key, Py_ssize_t hash) +{ + Py_ssize_t ix; + PyObject *value; + + ASSERT_DICT_LOCKED(mp); + + ix = _Py_dict_lookup(mp, key, hash, &value); + if (ix == DKIX_ERROR) + return -1; + return (ix != DKIX_EMPTY && value != NULL); +} + +static int +contains_lock_held(PyDictObject *mp, PyObject *key) { Py_hash_t hash; Py_ssize_t ix; - PyDictObject *mp = (PyDictObject *)op; PyObject *value; + ASSERT_DICT_LOCKED(mp); + if (!PyUnicode_CheckExact(key) || (hash = unicode_get_hash(key)) == -1) { hash = PyObject_Hash(key); if (hash == -1) @@ -3809,6 +4126,17 @@ PyDict_Contains(PyObject *op, PyObject *key) return (ix != DKIX_EMPTY && value != NULL); } +/* Return 1 if `key` is in dict `op`, 0 if not, and -1 on error. */ +int +PyDict_Contains(PyObject *op, PyObject *key) +{ + int res; + Py_BEGIN_CRITICAL_SECTION(op); + res = contains_lock_held((PyDictObject *)op, key); + Py_END_CRITICAL_SECTION(); + return res; +} + int PyDict_ContainsString(PyObject *op, const char *key) { @@ -3937,8 +4265,9 @@ dict_vectorcall(PyObject *type, PyObject * const*args, } static PyObject * -dict_iter(PyDictObject *dict) +dict_iter(PyObject *self) { + PyDictObject *dict = (PyDictObject *)self; return dictiter_new(dict, &PyDictIterKey_Type); } @@ -3958,12 +4287,12 @@ PyTypeObject PyDict_Type = { "dict", sizeof(PyDictObject), 0, - (destructor)dict_dealloc, /* tp_dealloc */ + dict_dealloc, /* tp_dealloc */ 0, /* tp_vectorcall_offset */ 0, /* tp_getattr */ 0, /* tp_setattr */ 0, /* tp_as_async */ - (reprfunc)dict_repr, /* tp_repr */ + dict_repr, /* tp_repr */ &dict_as_number, /* tp_as_number */ &dict_as_sequence, /* tp_as_sequence */ &dict_as_mapping, /* tp_as_mapping */ @@ -3981,7 +4310,7 @@ PyTypeObject PyDict_Type = { dict_tp_clear, /* tp_clear */ dict_richcompare, /* tp_richcompare */ 0, /* tp_weaklistoffset */ - (getiterfunc)dict_iter, /* tp_iter */ + dict_iter, /* tp_iter */ 0, /* tp_iternext */ mapp_methods, /* tp_methods */ 0, /* tp_members */ @@ -4128,8 +4457,9 @@ dictiter_new(PyDictObject *dict, PyTypeObject *itertype) } static void -dictiter_dealloc(dictiterobject *di) +dictiter_dealloc(PyObject *self) { + dictiterobject *di = (dictiterobject *)self; /* bpo-31095: UnTrack is needed before calling any callbacks */ _PyObject_GC_UNTRACK(di); Py_XDECREF(di->di_dict); @@ -4138,16 +4468,18 @@ dictiter_dealloc(dictiterobject *di) } static int -dictiter_traverse(dictiterobject *di, visitproc visit, void *arg) +dictiter_traverse(PyObject *self, visitproc visit, void *arg) { + dictiterobject *di = (dictiterobject *)self; Py_VISIT(di->di_dict); Py_VISIT(di->di_result); return 0; } static PyObject * -dictiter_len(dictiterobject *di, PyObject *Py_UNUSED(ignored)) +dictiter_len(PyObject *self, PyObject *Py_UNUSED(ignored)) { + dictiterobject *di = (dictiterobject *)self; Py_ssize_t len = 0; if (di->di_dict != NULL && di->di_used == di->di_dict->ma_used) len = di->len; @@ -4158,29 +4490,28 @@ PyDoc_STRVAR(length_hint_doc, "Private method returning an estimate of len(list(it))."); static PyObject * -dictiter_reduce(dictiterobject *di, PyObject *Py_UNUSED(ignored)); +dictiter_reduce(PyObject *di, PyObject *Py_UNUSED(ignored)); PyDoc_STRVAR(reduce_doc, "Return state information for pickling."); static PyMethodDef dictiter_methods[] = { - {"__length_hint__", _PyCFunction_CAST(dictiter_len), METH_NOARGS, + {"__length_hint__", dictiter_len, METH_NOARGS, length_hint_doc}, - {"__reduce__", _PyCFunction_CAST(dictiter_reduce), METH_NOARGS, + {"__reduce__", dictiter_reduce, METH_NOARGS, reduce_doc}, {NULL, NULL} /* sentinel */ }; static PyObject* -dictiter_iternextkey(dictiterobject *di) +dictiter_iternextkey_lock_held(PyDictObject *d, PyObject *self) { + dictiterobject *di = (dictiterobject *)self; PyObject *key; Py_ssize_t i; PyDictKeysObject *k; - PyDictObject *d = di->di_dict; - if (d == NULL) - return NULL; assert (PyDict_Check(d)); + ASSERT_DICT_LOCKED(d); if (di->di_used != d->ma_used) { PyErr_SetString(PyExc_RuntimeError, @@ -4238,13 +4569,30 @@ dictiter_iternextkey(dictiterobject *di) return NULL; } +static PyObject* +dictiter_iternextkey(PyObject *self) +{ + dictiterobject *di = (dictiterobject *)self; + PyDictObject *d = di->di_dict; + + if (d == NULL) + return NULL; + + PyObject *value; + Py_BEGIN_CRITICAL_SECTION(d); + value = dictiter_iternextkey_lock_held(d, self); + Py_END_CRITICAL_SECTION(); + + return value; +} + PyTypeObject PyDictIterKey_Type = { PyVarObject_HEAD_INIT(&PyType_Type, 0) "dict_keyiterator", /* tp_name */ sizeof(dictiterobject), /* tp_basicsize */ 0, /* tp_itemsize */ /* methods */ - (destructor)dictiter_dealloc, /* tp_dealloc */ + dictiter_dealloc, /* tp_dealloc */ 0, /* tp_vectorcall_offset */ 0, /* tp_getattr */ 0, /* tp_setattr */ @@ -4261,26 +4609,25 @@ PyTypeObject PyDictIterKey_Type = { 0, /* tp_as_buffer */ Py_TPFLAGS_DEFAULT | Py_TPFLAGS_HAVE_GC,/* tp_flags */ 0, /* tp_doc */ - (traverseproc)dictiter_traverse, /* tp_traverse */ + dictiter_traverse, /* tp_traverse */ 0, /* tp_clear */ 0, /* tp_richcompare */ 0, /* tp_weaklistoffset */ PyObject_SelfIter, /* tp_iter */ - (iternextfunc)dictiter_iternextkey, /* tp_iternext */ + dictiter_iternextkey, /* tp_iternext */ dictiter_methods, /* tp_methods */ 0, }; static PyObject * -dictiter_iternextvalue(dictiterobject *di) +dictiter_iternextvalue_lock_held(PyDictObject *d, PyObject *self) { + dictiterobject *di = (dictiterobject *)self; PyObject *value; Py_ssize_t i; - PyDictObject *d = di->di_dict; - if (d == NULL) - return NULL; assert (PyDict_Check(d)); + ASSERT_DICT_LOCKED(d); if (di->di_used != d->ma_used) { PyErr_SetString(PyExc_RuntimeError, @@ -4337,13 +4684,30 @@ dictiter_iternextvalue(dictiterobject *di) return NULL; } +static PyObject * +dictiter_iternextvalue(PyObject *self) +{ + dictiterobject *di = (dictiterobject *)self; + PyDictObject *d = di->di_dict; + + if (d == NULL) + return NULL; + + PyObject *value; + Py_BEGIN_CRITICAL_SECTION(d); + value = dictiter_iternextvalue_lock_held(d, self); + Py_END_CRITICAL_SECTION(); + + return value; +} + PyTypeObject PyDictIterValue_Type = { PyVarObject_HEAD_INIT(&PyType_Type, 0) "dict_valueiterator", /* tp_name */ sizeof(dictiterobject), /* tp_basicsize */ 0, /* tp_itemsize */ /* methods */ - (destructor)dictiter_dealloc, /* tp_dealloc */ + dictiter_dealloc, /* tp_dealloc */ 0, /* tp_vectorcall_offset */ 0, /* tp_getattr */ 0, /* tp_setattr */ @@ -4360,25 +4724,23 @@ PyTypeObject PyDictIterValue_Type = { 0, /* tp_as_buffer */ Py_TPFLAGS_DEFAULT | Py_TPFLAGS_HAVE_GC, /* tp_flags */ 0, /* tp_doc */ - (traverseproc)dictiter_traverse, /* tp_traverse */ + dictiter_traverse, /* tp_traverse */ 0, /* tp_clear */ 0, /* tp_richcompare */ 0, /* tp_weaklistoffset */ PyObject_SelfIter, /* tp_iter */ - (iternextfunc)dictiter_iternextvalue, /* tp_iternext */ + dictiter_iternextvalue, /* tp_iternext */ dictiter_methods, /* tp_methods */ 0, }; static PyObject * -dictiter_iternextitem(dictiterobject *di) +dictiter_iternextitem_lock_held(PyDictObject *d, PyObject *self) { + dictiterobject *di = (dictiterobject *)self; PyObject *key, *value, *result; Py_ssize_t i; - PyDictObject *d = di->di_dict; - if (d == NULL) - return NULL; assert (PyDict_Check(d)); if (di->di_used != d->ma_used) { @@ -4461,13 +4823,29 @@ dictiter_iternextitem(dictiterobject *di) return NULL; } +static PyObject * +dictiter_iternextitem(PyObject *self) +{ + dictiterobject *di = (dictiterobject *)self; + PyDictObject *d = di->di_dict; + + if (d == NULL) + return NULL; + + PyObject *item; + Py_BEGIN_CRITICAL_SECTION(d); + item = dictiter_iternextitem_lock_held(d, self); + Py_END_CRITICAL_SECTION(); + return item; +} + PyTypeObject PyDictIterItem_Type = { PyVarObject_HEAD_INIT(&PyType_Type, 0) "dict_itemiterator", /* tp_name */ sizeof(dictiterobject), /* tp_basicsize */ 0, /* tp_itemsize */ /* methods */ - (destructor)dictiter_dealloc, /* tp_dealloc */ + dictiter_dealloc, /* tp_dealloc */ 0, /* tp_vectorcall_offset */ 0, /* tp_getattr */ 0, /* tp_setattr */ @@ -4484,12 +4862,12 @@ PyTypeObject PyDictIterItem_Type = { 0, /* tp_as_buffer */ Py_TPFLAGS_DEFAULT | Py_TPFLAGS_HAVE_GC,/* tp_flags */ 0, /* tp_doc */ - (traverseproc)dictiter_traverse, /* tp_traverse */ + dictiter_traverse, /* tp_traverse */ 0, /* tp_clear */ 0, /* tp_richcompare */ 0, /* tp_weaklistoffset */ PyObject_SelfIter, /* tp_iter */ - (iternextfunc)dictiter_iternextitem, /* tp_iternext */ + dictiter_iternextitem, /* tp_iternext */ dictiter_methods, /* tp_methods */ 0, }; @@ -4498,14 +4876,12 @@ PyTypeObject PyDictIterItem_Type = { /* dictreviter */ static PyObject * -dictreviter_iternext(dictiterobject *di) +dictreviter_iter_PyDict_Next(PyDictObject *d, PyObject *self) { - PyDictObject *d = di->di_dict; + dictiterobject *di = (dictiterobject *)self; - if (d == NULL) { - return NULL; - } assert (PyDict_Check(d)); + ASSERT_DICT_LOCKED(d); if (di->di_used != d->ma_used) { PyErr_SetString(PyExc_RuntimeError, @@ -4596,15 +4972,32 @@ dictreviter_iternext(dictiterobject *di) return NULL; } +static PyObject * +dictreviter_iternext(PyObject *self) +{ + dictiterobject *di = (dictiterobject *)self; + PyDictObject *d = di->di_dict; + + if (d == NULL) + return NULL; + + PyObject *value; + Py_BEGIN_CRITICAL_SECTION(d); + value = dictreviter_iter_PyDict_Next(d, self); + Py_END_CRITICAL_SECTION(); + + return value; +} + PyTypeObject PyDictRevIterKey_Type = { PyVarObject_HEAD_INIT(&PyType_Type, 0) "dict_reversekeyiterator", sizeof(dictiterobject), - .tp_dealloc = (destructor)dictiter_dealloc, + .tp_dealloc = dictiter_dealloc, .tp_flags = Py_TPFLAGS_DEFAULT | Py_TPFLAGS_HAVE_GC, - .tp_traverse = (traverseproc)dictiter_traverse, + .tp_traverse = dictiter_traverse, .tp_iter = PyObject_SelfIter, - .tp_iternext = (iternextfunc)dictreviter_iternext, + .tp_iternext = dictreviter_iternext, .tp_methods = dictiter_methods }; @@ -4624,8 +5017,9 @@ dict___reversed___impl(PyDictObject *self) } static PyObject * -dictiter_reduce(dictiterobject *di, PyObject *Py_UNUSED(ignored)) +dictiter_reduce(PyObject *self, PyObject *Py_UNUSED(ignored)) { + dictiterobject *di = (dictiterobject *)self; /* copy the iterator state */ dictiterobject tmp = *di; Py_XINCREF(tmp.di_dict); @@ -4641,11 +5035,11 @@ PyTypeObject PyDictRevIterItem_Type = { PyVarObject_HEAD_INIT(&PyType_Type, 0) "dict_reverseitemiterator", sizeof(dictiterobject), - .tp_dealloc = (destructor)dictiter_dealloc, + .tp_dealloc = dictiter_dealloc, .tp_flags = Py_TPFLAGS_DEFAULT | Py_TPFLAGS_HAVE_GC, - .tp_traverse = (traverseproc)dictiter_traverse, + .tp_traverse = dictiter_traverse, .tp_iter = PyObject_SelfIter, - .tp_iternext = (iternextfunc)dictreviter_iternext, + .tp_iternext = dictreviter_iternext, .tp_methods = dictiter_methods }; @@ -4653,11 +5047,11 @@ PyTypeObject PyDictRevIterValue_Type = { PyVarObject_HEAD_INIT(&PyType_Type, 0) "dict_reversevalueiterator", sizeof(dictiterobject), - .tp_dealloc = (destructor)dictiter_dealloc, + .tp_dealloc = dictiter_dealloc, .tp_flags = Py_TPFLAGS_DEFAULT | Py_TPFLAGS_HAVE_GC, - .tp_traverse = (traverseproc)dictiter_traverse, + .tp_traverse = dictiter_traverse, .tp_iter = PyObject_SelfIter, - .tp_iternext = (iternextfunc)dictreviter_iternext, + .tp_iternext = dictreviter_iternext, .tp_methods = dictiter_methods }; @@ -4668,8 +5062,9 @@ PyTypeObject PyDictRevIterValue_Type = { /* The instance lay-out is the same for all three; but the type differs. */ static void -dictview_dealloc(_PyDictViewObject *dv) +dictview_dealloc(PyObject *self) { + _PyDictViewObject *dv = (_PyDictViewObject *)self; /* bpo-31095: UnTrack is needed before calling any callbacks */ _PyObject_GC_UNTRACK(dv); Py_XDECREF(dv->dv_dict); @@ -4677,15 +5072,17 @@ dictview_dealloc(_PyDictViewObject *dv) } static int -dictview_traverse(_PyDictViewObject *dv, visitproc visit, void *arg) +dictview_traverse(PyObject *self, visitproc visit, void *arg) { + _PyDictViewObject *dv = (_PyDictViewObject *)self; Py_VISIT(dv->dv_dict); return 0; } static Py_ssize_t -dictview_len(_PyDictViewObject *dv) +dictview_len(PyObject *self) { + _PyDictViewObject *dv = (_PyDictViewObject *)self; Py_ssize_t len = 0; if (dv->dv_dict != NULL) len = dv->dv_dict->ma_used; @@ -4825,8 +5222,9 @@ dictview_richcompare(PyObject *self, PyObject *other, int op) } static PyObject * -dictview_repr(_PyDictViewObject *dv) +dictview_repr(PyObject *self) { + _PyDictViewObject *dv = (_PyDictViewObject *)self; PyObject *seq; PyObject *result = NULL; Py_ssize_t rc; @@ -4850,8 +5248,9 @@ dictview_repr(_PyDictViewObject *dv) /*** dict_keys ***/ static PyObject * -dictkeys_iter(_PyDictViewObject *dv) +dictkeys_iter(PyObject *self) { + _PyDictViewObject *dv = (_PyDictViewObject *)self; if (dv->dv_dict == NULL) { Py_RETURN_NONE; } @@ -4859,22 +5258,23 @@ dictkeys_iter(_PyDictViewObject *dv) } static int -dictkeys_contains(_PyDictViewObject *dv, PyObject *obj) +dictkeys_contains(PyObject *self, PyObject *obj) { + _PyDictViewObject *dv = (_PyDictViewObject *)self; if (dv->dv_dict == NULL) return 0; return PyDict_Contains((PyObject *)dv->dv_dict, obj); } static PySequenceMethods dictkeys_as_sequence = { - (lenfunc)dictview_len, /* sq_length */ + dictview_len, /* sq_length */ 0, /* sq_concat */ 0, /* sq_repeat */ 0, /* sq_item */ 0, /* sq_slice */ 0, /* sq_ass_item */ 0, /* sq_ass_slice */ - (objobjproc)dictkeys_contains, /* sq_contains */ + dictkeys_contains, /* sq_contains */ }; // Create a set object from dictviews object. @@ -4914,7 +5314,7 @@ dictviews_sub(PyObject *self, PyObject *other) } static int -dictitems_contains(_PyDictViewObject *dv, PyObject *obj); +dictitems_contains(PyObject *dv, PyObject *obj); PyObject * _PyDictView_Intersect(PyObject* self, PyObject *other) @@ -4924,7 +5324,7 @@ _PyDictView_Intersect(PyObject* self, PyObject *other) PyObject *key; Py_ssize_t len_self; int rv; - int (*dict_contains)(_PyDictViewObject *, PyObject *); + objobjproc dict_contains; /* Python interpreter swaps parameters when dict view is on right side of & */ @@ -4934,7 +5334,7 @@ _PyDictView_Intersect(PyObject* self, PyObject *other) self = tmp; } - len_self = dictview_len((_PyDictViewObject *)self); + len_self = dictview_len(self); /* if other is a set and self is smaller than other, reuse set intersection logic */ @@ -4946,7 +5346,7 @@ _PyDictView_Intersect(PyObject* self, PyObject *other) /* if other is another dict view, and it is bigger than self, swap them */ if (PyDictViewSet_Check(other)) { - Py_ssize_t len_other = dictview_len((_PyDictViewObject *)other); + Py_ssize_t len_other = dictview_len(other); if (len_other > len_self) { PyObject *tmp = other; other = self; @@ -4976,7 +5376,7 @@ _PyDictView_Intersect(PyObject* self, PyObject *other) } while ((key = PyIter_Next(it)) != NULL) { - rv = dict_contains((_PyDictViewObject *)self, key); + rv = dict_contains(self, key); if (rv < 0) { goto error; } @@ -5017,14 +5417,12 @@ dictviews_or(PyObject* self, PyObject *other) } static PyObject * -dictitems_xor(PyObject *self, PyObject *other) +dictitems_xor_lock_held(PyObject *d1, PyObject *d2) { - assert(PyDictItems_Check(self)); - assert(PyDictItems_Check(other)); - PyObject *d1 = (PyObject *)((_PyDictViewObject *)self)->dv_dict; - PyObject *d2 = (PyObject *)((_PyDictViewObject *)other)->dv_dict; + ASSERT_DICT_LOCKED(d1); + ASSERT_DICT_LOCKED(d2); - PyObject *temp_dict = PyDict_Copy(d1); + PyObject *temp_dict = copy_lock_held(d1); if (temp_dict == NULL) { return NULL; } @@ -5102,6 +5500,22 @@ dictitems_xor(PyObject *self, PyObject *other) return NULL; } +static PyObject * +dictitems_xor(PyObject *self, PyObject *other) +{ + assert(PyDictItems_Check(self)); + assert(PyDictItems_Check(other)); + PyObject *d1 = (PyObject *)((_PyDictViewObject *)self)->dv_dict; + PyObject *d2 = (PyObject *)((_PyDictViewObject *)other)->dv_dict; + + PyObject *res; + Py_BEGIN_CRITICAL_SECTION2(d1, d2); + res = dictitems_xor_lock_held(d1, d2); + Py_END_CRITICAL_SECTION2(); + + return res; +} + static PyObject* dictviews_xor(PyObject* self, PyObject *other) { @@ -5150,7 +5564,7 @@ dictviews_isdisjoint(PyObject *self, PyObject *other) PyObject *item = NULL; if (self == other) { - if (dictview_len((_PyDictViewObject *)self) == 0) + if (dictview_len(self) == 0) Py_RETURN_TRUE; else Py_RETURN_FALSE; @@ -5159,7 +5573,7 @@ dictviews_isdisjoint(PyObject *self, PyObject *other) /* Iterate over the shorter object (only if other is a set, * because PySequence_Contains may be expensive otherwise): */ if (PyAnySet_Check(other) || PyDictViewSet_Check(other)) { - Py_ssize_t len_self = dictview_len((_PyDictViewObject *)self); + Py_ssize_t len_self = dictview_len(self); Py_ssize_t len_other = PyObject_Size(other); if (len_other == -1) return NULL; @@ -5197,15 +5611,15 @@ dictviews_isdisjoint(PyObject *self, PyObject *other) PyDoc_STRVAR(isdisjoint_doc, "Return True if the view and the given iterable have a null intersection."); -static PyObject* dictkeys_reversed(_PyDictViewObject *dv, PyObject *Py_UNUSED(ignored)); +static PyObject* dictkeys_reversed(PyObject *dv, PyObject *Py_UNUSED(ignored)); PyDoc_STRVAR(reversed_keys_doc, "Return a reverse iterator over the dict keys."); static PyMethodDef dictkeys_methods[] = { - {"isdisjoint", (PyCFunction)dictviews_isdisjoint, METH_O, + {"isdisjoint", dictviews_isdisjoint, METH_O, isdisjoint_doc}, - {"__reversed__", _PyCFunction_CAST(dictkeys_reversed), METH_NOARGS, + {"__reversed__", dictkeys_reversed, METH_NOARGS, reversed_keys_doc}, {NULL, NULL} /* sentinel */ }; @@ -5216,12 +5630,12 @@ PyTypeObject PyDictKeys_Type = { sizeof(_PyDictViewObject), /* tp_basicsize */ 0, /* tp_itemsize */ /* methods */ - (destructor)dictview_dealloc, /* tp_dealloc */ + dictview_dealloc, /* tp_dealloc */ 0, /* tp_vectorcall_offset */ 0, /* tp_getattr */ 0, /* tp_setattr */ 0, /* tp_as_async */ - (reprfunc)dictview_repr, /* tp_repr */ + dictview_repr, /* tp_repr */ &dictviews_as_number, /* tp_as_number */ &dictkeys_as_sequence, /* tp_as_sequence */ 0, /* tp_as_mapping */ @@ -5233,25 +5647,33 @@ PyTypeObject PyDictKeys_Type = { 0, /* tp_as_buffer */ Py_TPFLAGS_DEFAULT | Py_TPFLAGS_HAVE_GC,/* tp_flags */ 0, /* tp_doc */ - (traverseproc)dictview_traverse, /* tp_traverse */ + dictview_traverse, /* tp_traverse */ 0, /* tp_clear */ dictview_richcompare, /* tp_richcompare */ 0, /* tp_weaklistoffset */ - (getiterfunc)dictkeys_iter, /* tp_iter */ + dictkeys_iter, /* tp_iter */ 0, /* tp_iternext */ dictkeys_methods, /* tp_methods */ .tp_getset = dictview_getset, }; +/*[clinic input] +dict.keys + +Return a set-like object providing a view on the dict's keys. +[clinic start generated code]*/ + static PyObject * -dictkeys_new(PyObject *dict, PyObject *Py_UNUSED(ignored)) +dict_keys_impl(PyDictObject *self) +/*[clinic end generated code: output=aac2830c62990358 input=42f48a7a771212a7]*/ { - return _PyDictView_New(dict, &PyDictKeys_Type); + return _PyDictView_New((PyObject *)self, &PyDictKeys_Type); } static PyObject * -dictkeys_reversed(_PyDictViewObject *dv, PyObject *Py_UNUSED(ignored)) +dictkeys_reversed(PyObject *self, PyObject *Py_UNUSED(ignored)) { + _PyDictViewObject *dv = (_PyDictViewObject *)self; if (dv->dv_dict == NULL) { Py_RETURN_NONE; } @@ -5261,8 +5683,9 @@ dictkeys_reversed(_PyDictViewObject *dv, PyObject *Py_UNUSED(ignored)) /*** dict_items ***/ static PyObject * -dictitems_iter(_PyDictViewObject *dv) +dictitems_iter(PyObject *self) { + _PyDictViewObject *dv = (_PyDictViewObject *)self; if (dv->dv_dict == NULL) { Py_RETURN_NONE; } @@ -5270,8 +5693,9 @@ dictitems_iter(_PyDictViewObject *dv) } static int -dictitems_contains(_PyDictViewObject *dv, PyObject *obj) +dictitems_contains(PyObject *self, PyObject *obj) { + _PyDictViewObject *dv = (_PyDictViewObject *)self; int result; PyObject *key, *value, *found; if (dv->dv_dict == NULL) @@ -5289,25 +5713,25 @@ dictitems_contains(_PyDictViewObject *dv, PyObject *obj) } static PySequenceMethods dictitems_as_sequence = { - (lenfunc)dictview_len, /* sq_length */ + dictview_len, /* sq_length */ 0, /* sq_concat */ 0, /* sq_repeat */ 0, /* sq_item */ 0, /* sq_slice */ 0, /* sq_ass_item */ 0, /* sq_ass_slice */ - (objobjproc)dictitems_contains, /* sq_contains */ + dictitems_contains, /* sq_contains */ }; -static PyObject* dictitems_reversed(_PyDictViewObject *dv, PyObject *Py_UNUSED(ignored)); +static PyObject* dictitems_reversed(PyObject *dv, PyObject *Py_UNUSED(ignored)); PyDoc_STRVAR(reversed_items_doc, "Return a reverse iterator over the dict items."); static PyMethodDef dictitems_methods[] = { - {"isdisjoint", (PyCFunction)dictviews_isdisjoint, METH_O, + {"isdisjoint", dictviews_isdisjoint, METH_O, isdisjoint_doc}, - {"__reversed__", (PyCFunction)dictitems_reversed, METH_NOARGS, + {"__reversed__", dictitems_reversed, METH_NOARGS, reversed_items_doc}, {NULL, NULL} /* sentinel */ }; @@ -5318,12 +5742,12 @@ PyTypeObject PyDictItems_Type = { sizeof(_PyDictViewObject), /* tp_basicsize */ 0, /* tp_itemsize */ /* methods */ - (destructor)dictview_dealloc, /* tp_dealloc */ + dictview_dealloc, /* tp_dealloc */ 0, /* tp_vectorcall_offset */ 0, /* tp_getattr */ 0, /* tp_setattr */ 0, /* tp_as_async */ - (reprfunc)dictview_repr, /* tp_repr */ + dictview_repr, /* tp_repr */ &dictviews_as_number, /* tp_as_number */ &dictitems_as_sequence, /* tp_as_sequence */ 0, /* tp_as_mapping */ @@ -5335,25 +5759,33 @@ PyTypeObject PyDictItems_Type = { 0, /* tp_as_buffer */ Py_TPFLAGS_DEFAULT | Py_TPFLAGS_HAVE_GC,/* tp_flags */ 0, /* tp_doc */ - (traverseproc)dictview_traverse, /* tp_traverse */ + dictview_traverse, /* tp_traverse */ 0, /* tp_clear */ dictview_richcompare, /* tp_richcompare */ 0, /* tp_weaklistoffset */ - (getiterfunc)dictitems_iter, /* tp_iter */ + dictitems_iter, /* tp_iter */ 0, /* tp_iternext */ dictitems_methods, /* tp_methods */ .tp_getset = dictview_getset, }; +/*[clinic input] +dict.items + +Return a set-like object providing a view on the dict's items. +[clinic start generated code]*/ + static PyObject * -dictitems_new(PyObject *dict, PyObject *Py_UNUSED(ignored)) +dict_items_impl(PyDictObject *self) +/*[clinic end generated code: output=88c7db7150c7909a input=87c822872eb71f5a]*/ { - return _PyDictView_New(dict, &PyDictItems_Type); + return _PyDictView_New((PyObject *)self, &PyDictItems_Type); } static PyObject * -dictitems_reversed(_PyDictViewObject *dv, PyObject *Py_UNUSED(ignored)) +dictitems_reversed(PyObject *self, PyObject *Py_UNUSED(ignored)) { + _PyDictViewObject *dv = (_PyDictViewObject *)self; if (dv->dv_dict == NULL) { Py_RETURN_NONE; } @@ -5363,8 +5795,9 @@ dictitems_reversed(_PyDictViewObject *dv, PyObject *Py_UNUSED(ignored)) /*** dict_values ***/ static PyObject * -dictvalues_iter(_PyDictViewObject *dv) +dictvalues_iter(PyObject *self) { + _PyDictViewObject *dv = (_PyDictViewObject *)self; if (dv->dv_dict == NULL) { Py_RETURN_NONE; } @@ -5372,7 +5805,7 @@ dictvalues_iter(_PyDictViewObject *dv) } static PySequenceMethods dictvalues_as_sequence = { - (lenfunc)dictview_len, /* sq_length */ + dictview_len, /* sq_length */ 0, /* sq_concat */ 0, /* sq_repeat */ 0, /* sq_item */ @@ -5382,13 +5815,13 @@ static PySequenceMethods dictvalues_as_sequence = { (objobjproc)0, /* sq_contains */ }; -static PyObject* dictvalues_reversed(_PyDictViewObject *dv, PyObject *Py_UNUSED(ignored)); +static PyObject* dictvalues_reversed(PyObject *dv, PyObject *Py_UNUSED(ignored)); PyDoc_STRVAR(reversed_values_doc, "Return a reverse iterator over the dict values."); static PyMethodDef dictvalues_methods[] = { - {"__reversed__", (PyCFunction)dictvalues_reversed, METH_NOARGS, + {"__reversed__", dictvalues_reversed, METH_NOARGS, reversed_values_doc}, {NULL, NULL} /* sentinel */ }; @@ -5399,12 +5832,12 @@ PyTypeObject PyDictValues_Type = { sizeof(_PyDictViewObject), /* tp_basicsize */ 0, /* tp_itemsize */ /* methods */ - (destructor)dictview_dealloc, /* tp_dealloc */ + dictview_dealloc, /* tp_dealloc */ 0, /* tp_vectorcall_offset */ 0, /* tp_getattr */ 0, /* tp_setattr */ 0, /* tp_as_async */ - (reprfunc)dictview_repr, /* tp_repr */ + dictview_repr, /* tp_repr */ 0, /* tp_as_number */ &dictvalues_as_sequence, /* tp_as_sequence */ 0, /* tp_as_mapping */ @@ -5416,25 +5849,33 @@ PyTypeObject PyDictValues_Type = { 0, /* tp_as_buffer */ Py_TPFLAGS_DEFAULT | Py_TPFLAGS_HAVE_GC,/* tp_flags */ 0, /* tp_doc */ - (traverseproc)dictview_traverse, /* tp_traverse */ + dictview_traverse, /* tp_traverse */ 0, /* tp_clear */ 0, /* tp_richcompare */ 0, /* tp_weaklistoffset */ - (getiterfunc)dictvalues_iter, /* tp_iter */ + dictvalues_iter, /* tp_iter */ 0, /* tp_iternext */ dictvalues_methods, /* tp_methods */ .tp_getset = dictview_getset, }; +/*[clinic input] +dict.values + +Return an object providing a view on the dict's values. +[clinic start generated code]*/ + static PyObject * -dictvalues_new(PyObject *dict, PyObject *Py_UNUSED(ignored)) +dict_values_impl(PyDictObject *self) +/*[clinic end generated code: output=ce9f2e9e8a959dd4 input=b46944f85493b230]*/ { - return _PyDictView_New(dict, &PyDictValues_Type); + return _PyDictView_New((PyObject *)self, &PyDictValues_Type); } static PyObject * -dictvalues_reversed(_PyDictViewObject *dv, PyObject *Py_UNUSED(ignored)) +dictvalues_reversed(PyObject *self, PyObject *Py_UNUSED(ignored)) { + _PyDictViewObject *dv = (_PyDictViewObject *)self; if (dv->dv_dict == NULL) { Py_RETURN_NONE; } @@ -5548,6 +5989,18 @@ _PyObject_MakeDictFromInstanceAttributes(PyObject *obj, PyDictValues *values) return make_dict_from_instance_attributes(interp, keys, values); } +static bool +has_unique_reference(PyObject *op) +{ +#ifdef Py_GIL_DISABLED + return (_Py_IsOwnedByCurrentThread(op) && + op->ob_ref_local == 1 && + _Py_atomic_load_ssize_relaxed(&op->ob_ref_shared) == 0); +#else + return Py_REFCNT(op) == 1; +#endif +} + // Return true if the dict was dematerialized, false otherwise. bool _PyObject_MakeInstanceAttributesFromDict(PyObject *obj, PyDictOrValues *dorv) @@ -5564,7 +6017,9 @@ _PyObject_MakeInstanceAttributesFromDict(PyObject *obj, PyDictOrValues *dorv) return false; } assert(_PyType_HasFeature(Py_TYPE(obj), Py_TPFLAGS_HEAPTYPE)); - if (dict->ma_keys != CACHED_KEYS(Py_TYPE(obj)) || Py_REFCNT(dict) != 1) { + if (dict->ma_keys != CACHED_KEYS(Py_TYPE(obj)) || + !has_unique_reference((PyObject *)dict)) + { return false; } assert(dict->ma_values); @@ -5947,7 +6402,8 @@ PyDict_AddWatcher(PyDict_WatchCallback callback) { PyInterpreterState *interp = _PyInterpreterState_GET(); - for (int i = 0; i < DICT_MAX_WATCHERS; i++) { + /* Start at 2, as 0 and 1 are reserved for CPython */ + for (int i = 2; i < DICT_MAX_WATCHERS; i++) { if (!interp->dict_state.watchers[i]) { interp->dict_state.watchers[i] = callback; return i; diff --git a/Objects/exceptions.c b/Objects/exceptions.c index a685ed803cd02d..cff55d05163b6b 100644 --- a/Objects/exceptions.c +++ b/Objects/exceptions.c @@ -2566,6 +2566,11 @@ MiddlingExtendsException(PyExc_SyntaxError, IndentationError, SyntaxError, MiddlingExtendsException(PyExc_IndentationError, TabError, SyntaxError, "Improper mixture of spaces and tabs."); +/* + * IncompleteInputError extends SyntaxError + */ +MiddlingExtendsException(PyExc_SyntaxError, IncompleteInputError, SyntaxError, + "incomplete input."); /* * LookupError extends Exception @@ -3635,6 +3640,7 @@ static struct static_exception static_exceptions[] = { // Level 4: Other subclasses ITEM(IndentationError), // base: SyntaxError(Exception) + ITEM(IncompleteInputError), // base: SyntaxError(Exception) ITEM(IndexError), // base: LookupError(Exception) ITEM(KeyError), // base: LookupError(Exception) ITEM(ModuleNotFoundError), // base: ImportError(Exception) diff --git a/Objects/fileobject.c b/Objects/fileobject.c index 5522eba34eace9..e30ab952dff571 100644 --- a/Objects/fileobject.c +++ b/Objects/fileobject.c @@ -174,6 +174,13 @@ PyObject_AsFileDescriptor(PyObject *o) PyObject *meth; if (PyLong_Check(o)) { + if (PyBool_Check(o)) { + if (PyErr_WarnEx(PyExc_RuntimeWarning, + "bool is used as a file descriptor", 1)) + { + return -1; + } + } fd = PyLong_AsInt(o); } else if (PyObject_GetOptionalAttr(o, &_Py_ID(fileno), &meth) < 0) { diff --git a/Objects/floatobject.c b/Objects/floatobject.c index 364cf1553bb5d4..9b322c52d4daea 100644 --- a/Objects/floatobject.c +++ b/Objects/floatobject.c @@ -26,17 +26,13 @@ class float "PyObject *" "&PyFloat_Type" #include "clinic/floatobject.c.h" -#ifndef PyFloat_MAXFREELIST -# define PyFloat_MAXFREELIST 100 -#endif - - -#if PyFloat_MAXFREELIST > 0 +#ifdef WITH_FREELISTS static struct _Py_float_state * get_float_state(void) { - PyInterpreterState *interp = _PyInterpreterState_GET(); - return &interp->float_state; + _PyFreeListState *state = _PyFreeListState_GET(); + assert(state != NULL); + return &state->floats; } #endif @@ -132,14 +128,10 @@ PyObject * PyFloat_FromDouble(double fval) { PyFloatObject *op; -#if PyFloat_MAXFREELIST > 0 +#ifdef WITH_FREELISTS struct _Py_float_state *state = get_float_state(); op = state->free_list; if (op != NULL) { -#ifdef Py_DEBUG - // PyFloat_FromDouble() must not be called after _PyFloat_Fini() - assert(state->numfree != -1); -#endif state->free_list = (PyFloatObject *) Py_TYPE(op); state->numfree--; OBJECT_STAT_INC(from_freelist); @@ -252,13 +244,9 @@ _PyFloat_ExactDealloc(PyObject *obj) { assert(PyFloat_CheckExact(obj)); PyFloatObject *op = (PyFloatObject *)obj; -#if PyFloat_MAXFREELIST > 0 +#ifdef WITH_FREELISTS struct _Py_float_state *state = get_float_state(); -#ifdef Py_DEBUG - // float_dealloc() must not be called after _PyFloat_Fini() - assert(state->numfree != -1); -#endif - if (state->numfree >= PyFloat_MAXFREELIST) { + if (state->numfree >= PyFloat_MAXFREELIST || state->numfree < 0) { PyObject_Free(op); return; } @@ -275,7 +263,7 @@ static void float_dealloc(PyObject *op) { assert(PyFloat_Check(op)); -#if PyFloat_MAXFREELIST > 0 +#ifdef WITH_FREELISTS if (PyFloat_CheckExact(op)) { _PyFloat_ExactDealloc(op); } @@ -2002,10 +1990,10 @@ _PyFloat_InitTypes(PyInterpreterState *interp) } void -_PyFloat_ClearFreeList(PyInterpreterState *interp) +_PyFloat_ClearFreeList(_PyFreeListState *freelist_state, int is_finalization) { -#if PyFloat_MAXFREELIST > 0 - struct _Py_float_state *state = &interp->float_state; +#ifdef WITH_FREELISTS + struct _Py_float_state *state = &freelist_state->floats; PyFloatObject *f = state->free_list; while (f != NULL) { PyFloatObject *next = (PyFloatObject*) Py_TYPE(f); @@ -2013,17 +2001,12 @@ _PyFloat_ClearFreeList(PyInterpreterState *interp) f = next; } state->free_list = NULL; - state->numfree = 0; -#endif -} - -void -_PyFloat_Fini(PyInterpreterState *interp) -{ - _PyFloat_ClearFreeList(interp); -#if defined(Py_DEBUG) && PyFloat_MAXFREELIST > 0 - struct _Py_float_state *state = &interp->float_state; - state->numfree = -1; + if (is_finalization) { + state->numfree = -1; + } + else { + state->numfree = 0; + } #endif } @@ -2037,7 +2020,7 @@ _PyFloat_FiniType(PyInterpreterState *interp) void _PyFloat_DebugMallocStats(FILE *out) { -#if PyFloat_MAXFREELIST > 0 +#ifdef WITH_FREELISTS struct _Py_float_state *state = get_float_state(); _PyDebugAllocatorStats(out, "free PyFloatObject", diff --git a/Objects/frameobject.c b/Objects/frameobject.c index cafe4ef6141d9a..a914c61aac2fd5 100644 --- a/Objects/frameobject.c +++ b/Objects/frameobject.c @@ -926,6 +926,7 @@ frame_tp_clear(PyFrameObject *f) Py_CLEAR(locals[i]); } f->f_frame->stacktop = 0; + Py_CLEAR(f->f_frame->f_locals); return 0; } diff --git a/Objects/funcobject.c b/Objects/funcobject.c index 4d88dd2229295d..08b2823d8cf024 100644 --- a/Objects/funcobject.c +++ b/Objects/funcobject.c @@ -53,6 +53,15 @@ handle_func_event(PyFunction_WatchEvent event, PyFunctionObject *func, if (interp->active_func_watchers) { notify_func_watchers(interp, event, func, new_value); } + switch (event) { + case PyFunction_EVENT_MODIFY_CODE: + case PyFunction_EVENT_MODIFY_DEFAULTS: + case PyFunction_EVENT_MODIFY_KWDEFAULTS: + RARE_EVENT_INTERP_INC(interp, func_modification); + break; + default: + break; + } } int @@ -809,14 +818,17 @@ function.__new__ as func_new a tuple that specifies the default argument values closure: object = None a tuple that supplies the bindings for free variables + kwdefaults: object = None + a dictionary that specifies the default keyword argument values Create a function object. [clinic start generated code]*/ static PyObject * func_new_impl(PyTypeObject *type, PyCodeObject *code, PyObject *globals, - PyObject *name, PyObject *defaults, PyObject *closure) -/*[clinic end generated code: output=99c6d9da3a24e3be input=93611752fc2daf11]*/ + PyObject *name, PyObject *defaults, PyObject *closure, + PyObject *kwdefaults) +/*[clinic end generated code: output=de72f4c22ac57144 input=20c9c9f04ad2d3f2]*/ { PyFunctionObject *newfunc; Py_ssize_t nclosure; @@ -843,6 +855,11 @@ func_new_impl(PyTypeObject *type, PyCodeObject *code, PyObject *globals, return NULL; } } + if (kwdefaults != Py_None && !PyDict_Check(kwdefaults)) { + PyErr_SetString(PyExc_TypeError, + "arg 6 (kwdefaults) must be None or dict"); + return NULL; + } /* check that the closure is well-formed */ nclosure = closure == Py_None ? 0 : PyTuple_GET_SIZE(closure); @@ -879,6 +896,9 @@ func_new_impl(PyTypeObject *type, PyCodeObject *code, PyObject *globals, if (closure != Py_None) { newfunc->func_closure = Py_NewRef(closure); } + if (kwdefaults != Py_None) { + newfunc->func_kwdefaults = Py_NewRef(kwdefaults); + } return (PyObject *)newfunc; } diff --git a/Objects/genobject.c b/Objects/genobject.c index 9614713883741c..59ab7abf6180bd 100644 --- a/Objects/genobject.c +++ b/Objects/genobject.c @@ -6,6 +6,7 @@ #include "pycore_call.h" // _PyObject_CallNoArgs() #include "pycore_ceval.h" // _PyEval_EvalFrame() #include "pycore_frame.h" // _PyInterpreterFrame +#include "pycore_gc.h" // _PyGC_CLEAR_FINALIZED() #include "pycore_genobject.h" // struct _Py_async_gen_state #include "pycore_modsupport.h" // _PyArg_CheckPositional() #include "pycore_object.h" // _PyObject_GC_UNTRACK() @@ -1627,12 +1628,12 @@ PyTypeObject PyAsyncGen_Type = { }; -#if _PyAsyncGen_MAXFREELIST > 0 +#ifdef WITH_FREELISTS static struct _Py_async_gen_state * get_async_gen_state(void) { - PyInterpreterState *interp = _PyInterpreterState_GET(); - return &interp->async_gen; + _PyFreeListState *state = _PyFreeListState_GET(); + return &state->async_gens; } #endif @@ -1655,39 +1656,32 @@ PyAsyncGen_New(PyFrameObject *f, PyObject *name, PyObject *qualname) void -_PyAsyncGen_ClearFreeLists(PyInterpreterState *interp) +_PyAsyncGen_ClearFreeLists(_PyFreeListState *freelist_state, int is_finalization) { -#if _PyAsyncGen_MAXFREELIST > 0 - struct _Py_async_gen_state *state = &interp->async_gen; +#ifdef WITH_FREELISTS + struct _Py_async_gen_state *state = &freelist_state->async_gens; - while (state->value_numfree) { + while (state->value_numfree > 0) { _PyAsyncGenWrappedValue *o; o = state->value_freelist[--state->value_numfree]; assert(_PyAsyncGenWrappedValue_CheckExact(o)); PyObject_GC_Del(o); } - while (state->asend_numfree) { + while (state->asend_numfree > 0) { PyAsyncGenASend *o; o = state->asend_freelist[--state->asend_numfree]; assert(Py_IS_TYPE(o, &_PyAsyncGenASend_Type)); PyObject_GC_Del(o); } -#endif -} -void -_PyAsyncGen_Fini(PyInterpreterState *interp) -{ - _PyAsyncGen_ClearFreeLists(interp); -#if defined(Py_DEBUG) && _PyAsyncGen_MAXFREELIST > 0 - struct _Py_async_gen_state *state = &interp->async_gen; - state->value_numfree = -1; - state->asend_numfree = -1; + if (is_finalization) { + state->value_numfree = -1; + state->asend_numfree = -1; + } #endif } - static PyObject * async_gen_unwrap_value(PyAsyncGenObject *gen, PyObject *result) { @@ -1731,14 +1725,11 @@ async_gen_asend_dealloc(PyAsyncGenASend *o) _PyObject_GC_UNTRACK((PyObject *)o); Py_CLEAR(o->ags_gen); Py_CLEAR(o->ags_sendval); -#if _PyAsyncGen_MAXFREELIST > 0 +#ifdef WITH_FREELISTS struct _Py_async_gen_state *state = get_async_gen_state(); -#ifdef Py_DEBUG - // async_gen_asend_dealloc() must not be called after _PyAsyncGen_Fini() - assert(state->asend_numfree != -1); -#endif - if (state->asend_numfree < _PyAsyncGen_MAXFREELIST) { + if (state->asend_numfree >= 0 && state->asend_numfree < _PyAsyncGen_MAXFREELIST) { assert(PyAsyncGenASend_CheckExact(o)); + _PyGC_CLEAR_FINALIZED((PyObject *)o); state->asend_freelist[state->asend_numfree++] = o; } else @@ -1904,13 +1895,9 @@ static PyObject * async_gen_asend_new(PyAsyncGenObject *gen, PyObject *sendval) { PyAsyncGenASend *o; -#if _PyAsyncGen_MAXFREELIST > 0 +#ifdef WITH_FREELISTS struct _Py_async_gen_state *state = get_async_gen_state(); -#ifdef Py_DEBUG - // async_gen_asend_new() must not be called after _PyAsyncGen_Fini() - assert(state->asend_numfree != -1); -#endif - if (state->asend_numfree) { + if (state->asend_numfree > 0) { state->asend_numfree--; o = state->asend_freelist[state->asend_numfree]; _Py_NewReference((PyObject *)o); @@ -1943,13 +1930,9 @@ async_gen_wrapped_val_dealloc(_PyAsyncGenWrappedValue *o) { _PyObject_GC_UNTRACK((PyObject *)o); Py_CLEAR(o->agw_val); -#if _PyAsyncGen_MAXFREELIST > 0 +#ifdef WITH_FREELISTS struct _Py_async_gen_state *state = get_async_gen_state(); -#ifdef Py_DEBUG - // async_gen_wrapped_val_dealloc() must not be called after _PyAsyncGen_Fini() - assert(state->value_numfree != -1); -#endif - if (state->value_numfree < _PyAsyncGen_MAXFREELIST) { + if (state->value_numfree >= 0 && state->value_numfree < _PyAsyncGen_MAXFREELIST) { assert(_PyAsyncGenWrappedValue_CheckExact(o)); state->value_freelist[state->value_numfree++] = o; OBJECT_STAT_INC(to_freelist); @@ -2020,13 +2003,9 @@ _PyAsyncGenValueWrapperNew(PyThreadState *tstate, PyObject *val) _PyAsyncGenWrappedValue *o; assert(val); -#if _PyAsyncGen_MAXFREELIST > 0 - struct _Py_async_gen_state *state = &tstate->interp->async_gen; -#ifdef Py_DEBUG - // _PyAsyncGenValueWrapperNew() must not be called after _PyAsyncGen_Fini() - assert(state->value_numfree != -1); -#endif - if (state->value_numfree) { +#ifdef WITH_FREELISTS + struct _Py_async_gen_state *state = get_async_gen_state(); + if (state->value_numfree > 0) { state->value_numfree--; o = state->value_freelist[state->value_numfree]; OBJECT_STAT_INC(from_freelist); diff --git a/Objects/listobject.c b/Objects/listobject.c index 2d04218439bd20..7fdb91eab890b5 100644 --- a/Objects/listobject.c +++ b/Objects/listobject.c @@ -20,16 +20,16 @@ class list "PyListObject *" "&PyList_Type" _Py_DECLARE_STR(list_err, "list index out of range"); -#if PyList_MAXFREELIST > 0 +#ifdef WITH_FREELISTS static struct _Py_list_state * get_list_state(void) { - PyInterpreterState *interp = _PyInterpreterState_GET(); - return &interp->list; + _PyFreeListState *state = _PyFreeListState_GET(); + assert(state != NULL); + return &state->lists; } #endif - /* Ensure ob_item has room for at least newsize elements, and set * ob_size to newsize. If newsize > ob_size on entry, the content * of the new slots at exit is undefined heap trash; it's the caller's @@ -120,25 +120,18 @@ list_preallocate_exact(PyListObject *self, Py_ssize_t size) } void -_PyList_ClearFreeList(PyInterpreterState *interp) +_PyList_ClearFreeList(_PyFreeListState *freelist_state, int is_finalization) { -#if PyList_MAXFREELIST > 0 - struct _Py_list_state *state = &interp->list; - while (state->numfree) { +#ifdef WITH_FREELISTS + struct _Py_list_state *state = &freelist_state->lists; + while (state->numfree > 0) { PyListObject *op = state->free_list[--state->numfree]; assert(PyList_CheckExact(op)); PyObject_GC_Del(op); } -#endif -} - -void -_PyList_Fini(PyInterpreterState *interp) -{ - _PyList_ClearFreeList(interp); -#if defined(Py_DEBUG) && PyList_MAXFREELIST > 0 - struct _Py_list_state *state = &interp->list; - state->numfree = -1; + if (is_finalization) { + state->numfree = -1; + } #endif } @@ -146,7 +139,7 @@ _PyList_Fini(PyInterpreterState *interp) void _PyList_DebugMallocStats(FILE *out) { -#if PyList_MAXFREELIST > 0 +#ifdef WITH_FREELISTS struct _Py_list_state *state = get_list_state(); _PyDebugAllocatorStats(out, "free PyListObject", @@ -164,13 +157,9 @@ PyList_New(Py_ssize_t size) return NULL; } -#if PyList_MAXFREELIST > 0 +#ifdef WITH_FREELISTS struct _Py_list_state *state = get_list_state(); -#ifdef Py_DEBUG - // PyList_New() must not be called after _PyList_Fini() - assert(state->numfree != -1); -#endif - if (PyList_MAXFREELIST && state->numfree) { + if (PyList_MAXFREELIST && state->numfree > 0) { state->numfree--; op = state->free_list[state->numfree]; OBJECT_STAT_INC(from_freelist); @@ -225,8 +214,9 @@ PyList_Size(PyObject *op) PyErr_BadInternalCall(); return -1; } - else - return Py_SIZE(op); + else { + return PyList_GET_SIZE(op); + } } static inline int @@ -257,6 +247,30 @@ PyList_GetItem(PyObject *op, Py_ssize_t i) return ((PyListObject *)op) -> ob_item[i]; } +PyObject * +PyList_GetItemRef(PyObject *op, Py_ssize_t i) +{ + if (!PyList_Check(op)) { + PyErr_SetString(PyExc_TypeError, "expected a list"); + return NULL; + } + if (!valid_index(i, Py_SIZE(op))) { + _Py_DECLARE_STR(list_err, "list index out of range"); + PyErr_SetObject(PyExc_IndexError, &_Py_STR(list_err)); + return NULL; + } + return Py_NewRef(PyList_GET_ITEM(op, i)); +} + +static inline PyObject* +list_get_item_ref(PyListObject *op, Py_ssize_t i) +{ + if (!valid_index(i, Py_SIZE(op))) { + return NULL; + } + return Py_NewRef(PyList_GET_ITEM(op, i)); +} + int PyList_SetItem(PyObject *op, Py_ssize_t i, PyObject *newitem) @@ -267,15 +281,22 @@ PyList_SetItem(PyObject *op, Py_ssize_t i, PyErr_BadInternalCall(); return -1; } - if (!valid_index(i, Py_SIZE(op))) { + int ret; + PyListObject *self = ((PyListObject *)op); + Py_BEGIN_CRITICAL_SECTION(self); + if (!valid_index(i, Py_SIZE(self))) { Py_XDECREF(newitem); PyErr_SetString(PyExc_IndexError, "list assignment index out of range"); - return -1; + ret = -1; + goto end; } - p = ((PyListObject *)op) -> ob_item + i; + p = self->ob_item + i; Py_XSETREF(*p, newitem); - return 0; + ret = 0; +end: + Py_END_CRITICAL_SECTION(); + return ret; } static int @@ -313,14 +334,19 @@ PyList_Insert(PyObject *op, Py_ssize_t where, PyObject *newitem) PyErr_BadInternalCall(); return -1; } - return ins1((PyListObject *)op, where, newitem); + PyListObject *self = (PyListObject *)op; + int err; + Py_BEGIN_CRITICAL_SECTION(self); + err = ins1(self, where, newitem); + Py_END_CRITICAL_SECTION(); + return err; } /* internal, used by _PyList_AppendTakeRef */ int _PyList_AppendTakeRefListResize(PyListObject *self, PyObject *newitem) { - Py_ssize_t len = PyList_GET_SIZE(self); + Py_ssize_t len = Py_SIZE(self); assert(self->allocated == -1 || self->allocated == len); if (list_resize(self, len + 1) < 0) { Py_DECREF(newitem); @@ -334,7 +360,11 @@ int PyList_Append(PyObject *op, PyObject *newitem) { if (PyList_Check(op) && (newitem != NULL)) { - return _PyList_AppendTakeRef((PyListObject *)op, Py_NewRef(newitem)); + int ret; + Py_BEGIN_CRITICAL_SECTION(op); + ret = _PyList_AppendTakeRef((PyListObject *)op, Py_NewRef(newitem)); + Py_END_CRITICAL_SECTION(); + return ret; } PyErr_BadInternalCall(); return -1; @@ -343,8 +373,9 @@ PyList_Append(PyObject *op, PyObject *newitem) /* Methods */ static void -list_dealloc(PyListObject *op) +list_dealloc(PyObject *self) { + PyListObject *op = (PyListObject *)self; Py_ssize_t i; PyObject_GC_UnTrack(op); Py_TRASHCAN_BEGIN(op, list_dealloc) @@ -359,13 +390,9 @@ list_dealloc(PyListObject *op) } PyMem_Free(op->ob_item); } -#if PyList_MAXFREELIST > 0 +#ifdef WITH_FREELISTS struct _Py_list_state *state = get_list_state(); -#ifdef Py_DEBUG - // list_dealloc() must not be called after _PyList_Fini() - assert(state->numfree != -1); -#endif - if (state->numfree < PyList_MAXFREELIST && PyList_CheckExact(op)) { + if (state->numfree < PyList_MAXFREELIST && state->numfree >= 0 && PyList_CheckExact(op)) { state->free_list[state->numfree++] = op; OBJECT_STAT_INC(to_freelist); } @@ -378,17 +405,11 @@ list_dealloc(PyListObject *op) } static PyObject * -list_repr(PyListObject *v) +list_repr_impl(PyListObject *v) { - Py_ssize_t i; PyObject *s; _PyUnicodeWriter writer; - - if (Py_SIZE(v) == 0) { - return PyUnicode_FromString("[]"); - } - - i = Py_ReprEnter((PyObject*)v); + Py_ssize_t i = Py_ReprEnter((PyObject*)v); if (i != 0) { return i > 0 ? PyUnicode_FromString("[...]") : NULL; } @@ -433,32 +454,50 @@ list_repr(PyListObject *v) return NULL; } +static PyObject * +list_repr(PyObject *self) +{ + if (PyList_GET_SIZE(self) == 0) { + return PyUnicode_FromString("[]"); + } + PyListObject *v = (PyListObject *)self; + PyObject *ret = NULL; + Py_BEGIN_CRITICAL_SECTION(v); + ret = list_repr_impl(v); + Py_END_CRITICAL_SECTION(); + return ret; +} + static Py_ssize_t -list_length(PyListObject *a) +list_length(PyObject *a) { - return Py_SIZE(a); + return PyList_GET_SIZE(a); } static int -list_contains(PyListObject *a, PyObject *el) +list_contains(PyObject *aa, PyObject *el) { - PyObject *item; - Py_ssize_t i; - int cmp; - for (i = 0, cmp = 0 ; cmp == 0 && i < Py_SIZE(a); ++i) { - item = PyList_GET_ITEM(a, i); - Py_INCREF(item); - cmp = PyObject_RichCompareBool(item, el, Py_EQ); + for (Py_ssize_t i = 0; ; i++) { + PyObject *item = list_get_item_ref((PyListObject *)aa, i); + if (item == NULL) { + // out-of-bounds + return 0; + } + int cmp = PyObject_RichCompareBool(item, el, Py_EQ); Py_DECREF(item); + if (cmp != 0) { + return cmp; + } } - return cmp; + return 0; } static PyObject * -list_item(PyListObject *a, Py_ssize_t i) +list_item(PyObject *aa, Py_ssize_t i) { - if (!valid_index(i, Py_SIZE(a))) { + PyListObject *a = (PyListObject *)aa; + if (!valid_index(i, PyList_GET_SIZE(a))) { PyErr_SetObject(PyExc_IndexError, &_Py_STR(list_err)); return NULL; } @@ -496,6 +535,8 @@ PyList_GetSlice(PyObject *a, Py_ssize_t ilow, Py_ssize_t ihigh) PyErr_BadInternalCall(); return NULL; } + PyObject *ret; + Py_BEGIN_CRITICAL_SECTION(a); if (ilow < 0) { ilow = 0; } @@ -508,12 +549,15 @@ PyList_GetSlice(PyObject *a, Py_ssize_t ilow, Py_ssize_t ihigh) else if (ihigh > Py_SIZE(a)) { ihigh = Py_SIZE(a); } - return list_slice((PyListObject *)a, ilow, ihigh); + ret = list_slice((PyListObject *)a, ilow, ihigh); + Py_END_CRITICAL_SECTION(); + return ret; } static PyObject * -list_concat(PyListObject *a, PyObject *bb) +list_concat(PyObject *aa, PyObject *bb) { + PyListObject *a = (PyListObject *)aa; Py_ssize_t size; Py_ssize_t i; PyObject **src, **dest; @@ -552,8 +596,9 @@ list_concat(PyListObject *a, PyObject *bb) } static PyObject * -list_repeat(PyListObject *a, Py_ssize_t n) +list_repeat(PyObject *aa, Py_ssize_t n) { + PyListObject *a = (PyListObject *)aa; const Py_ssize_t input_size = Py_SIZE(a); if (input_size == 0 || n <= 0) return PyList_New(0); @@ -616,9 +661,9 @@ list_clear(PyListObject *a) } static int -list_clear_slot(PyListObject *self) +list_clear_slot(PyObject *self) { - list_clear(self); + list_clear((PyListObject *)self); return 0; } @@ -745,8 +790,9 @@ PyList_SetSlice(PyObject *a, Py_ssize_t ilow, Py_ssize_t ihigh, PyObject *v) } static PyObject * -list_inplace_repeat(PyListObject *self, Py_ssize_t n) +list_inplace_repeat(PyObject *_self, Py_ssize_t n) { + PyListObject *self = (PyListObject *)_self; Py_ssize_t input_size = PyList_GET_SIZE(self); if (input_size == 0 || n == 1) { return Py_NewRef(self); @@ -776,8 +822,9 @@ list_inplace_repeat(PyListObject *self, Py_ssize_t n) } static int -list_ass_item(PyListObject *a, Py_ssize_t i, PyObject *v) +list_ass_item(PyObject *aa, Py_ssize_t i, PyObject *v) { + PyListObject *a = (PyListObject *)aa; if (!valid_index(i, Py_SIZE(a))) { PyErr_SetString(PyExc_IndexError, "list assignment index out of range"); @@ -790,6 +837,7 @@ list_ass_item(PyListObject *a, Py_ssize_t i, PyObject *v) } /*[clinic input] +@critical_section list.insert index: Py_ssize_t @@ -801,14 +849,16 @@ Insert object before index. static PyObject * list_insert_impl(PyListObject *self, Py_ssize_t index, PyObject *object) -/*[clinic end generated code: output=7f35e32f60c8cb78 input=858514cf894c7eab]*/ +/*[clinic end generated code: output=7f35e32f60c8cb78 input=b1987ca998a4ae2d]*/ { - if (ins1(self, index, object) == 0) + if (ins1(self, index, object) == 0) { Py_RETURN_NONE; + } return NULL; } /*[clinic input] +@critical_section list.clear as py_list_clear Remove all items from list. @@ -816,13 +866,14 @@ Remove all items from list. static PyObject * py_list_clear_impl(PyListObject *self) -/*[clinic end generated code: output=83726743807e3518 input=378711e10f545c53]*/ +/*[clinic end generated code: output=83726743807e3518 input=e285b7f09051a9ba]*/ { list_clear(self); Py_RETURN_NONE; } /*[clinic input] +@critical_section list.copy Return a shallow copy of the list. @@ -830,12 +881,13 @@ Return a shallow copy of the list. static PyObject * list_copy_impl(PyListObject *self) -/*[clinic end generated code: output=ec6b72d6209d418e input=6453ab159e84771f]*/ +/*[clinic end generated code: output=ec6b72d6209d418e input=81c54b0c7bb4f73d]*/ { return list_slice(self, 0, Py_SIZE(self)); } /*[clinic input] +@critical_section list.append object: object @@ -845,8 +897,8 @@ Append object to the end of the list. [clinic start generated code]*/ static PyObject * -list_append(PyListObject *self, PyObject *object) -/*[clinic end generated code: output=7c096003a29c0eae input=43a3fe48a7066e91]*/ +list_append_impl(PyListObject *self, PyObject *object) +/*[clinic end generated code: output=78423561d92ed405 input=122b0853de54004f]*/ { if (_PyList_AppendTakeRef(self, Py_NewRef(object)) < 0) { return NULL; @@ -1004,6 +1056,7 @@ _PyList_Extend(PyListObject *self, PyObject *iterable) /*[clinic input] +@critical_section self iterable list.extend as py_list_extend iterable: object @@ -1013,8 +1066,8 @@ Extend list by appending elements from the iterable. [clinic start generated code]*/ static PyObject * -py_list_extend(PyListObject *self, PyObject *iterable) -/*[clinic end generated code: output=b8e0bff0ceae2abd input=9a8376a8633ed3ba]*/ +py_list_extend_impl(PyListObject *self, PyObject *iterable) +/*[clinic end generated code: output=a2f115ceace2c845 input=1d42175414e1a5f3]*/ { return _PyList_Extend(self, iterable); } @@ -1044,8 +1097,9 @@ PyList_Clear(PyObject *self) static PyObject * -list_inplace_concat(PyListObject *self, PyObject *other) +list_inplace_concat(PyObject *_self, PyObject *other) { + PyListObject *self = (PyListObject *)_self; if (list_extend(self, other) < 0) { return NULL; } @@ -1053,6 +1107,7 @@ list_inplace_concat(PyListObject *self, PyObject *other) } /*[clinic input] +@critical_section list.pop index: Py_ssize_t = -1 @@ -1065,7 +1120,7 @@ Raises IndexError if list is empty or index is out of range. static PyObject * list_pop_impl(PyListObject *self, Py_ssize_t index) -/*[clinic end generated code: output=6bd69dcb3f17eca8 input=b83675976f329e6f]*/ +/*[clinic end generated code: output=6bd69dcb3f17eca8 input=c269141068ae4b8f]*/ { PyObject *v; int status; @@ -2584,6 +2639,7 @@ PyList_Sort(PyObject *v) } /*[clinic input] +@critical_section list.reverse Reverse *IN PLACE*. @@ -2591,7 +2647,7 @@ Reverse *IN PLACE*. static PyObject * list_reverse_impl(PyListObject *self) -/*[clinic end generated code: output=482544fc451abea9 input=eefd4c3ae1bc9887]*/ +/*[clinic end generated code: output=482544fc451abea9 input=04ac8e0c6a66e4d9]*/ { if (Py_SIZE(self) > 1) reverse_slice(self->ob_item, self->ob_item + Py_SIZE(self)); @@ -2607,8 +2663,11 @@ PyList_Reverse(PyObject *v) PyErr_BadInternalCall(); return -1; } - if (Py_SIZE(self) > 1) + Py_BEGIN_CRITICAL_SECTION(self); + if (Py_SIZE(self) > 1) { reverse_slice(self->ob_item, self->ob_item + Py_SIZE(self)); + } + Py_END_CRITICAL_SECTION() return 0; } @@ -2619,7 +2678,12 @@ PyList_AsTuple(PyObject *v) PyErr_BadInternalCall(); return NULL; } - return _PyTuple_FromArray(((PyListObject *)v)->ob_item, Py_SIZE(v)); + PyObject *ret; + PyListObject *self = (PyListObject *)v; + Py_BEGIN_CRITICAL_SECTION(self); + ret = _PyTuple_FromArray(self->ob_item, Py_SIZE(v)); + Py_END_CRITICAL_SECTION(); + return ret; } PyObject * @@ -2661,8 +2725,6 @@ list_index_impl(PyListObject *self, PyObject *value, Py_ssize_t start, Py_ssize_t stop) /*[clinic end generated code: output=ec51b88787e4e481 input=40ec5826303a0eb1]*/ { - Py_ssize_t i; - if (start < 0) { start += Py_SIZE(self); if (start < 0) @@ -2673,9 +2735,12 @@ list_index_impl(PyListObject *self, PyObject *value, Py_ssize_t start, if (stop < 0) stop = 0; } - for (i = start; i < stop && i < Py_SIZE(self); i++) { - PyObject *obj = self->ob_item[i]; - Py_INCREF(obj); + for (Py_ssize_t i = start; i < stop; i++) { + PyObject *obj = list_get_item_ref(self, i); + if (obj == NULL) { + // out-of-bounds + break; + } int cmp = PyObject_RichCompareBool(obj, value, Py_EQ); Py_DECREF(obj); if (cmp > 0) @@ -2701,15 +2766,17 @@ list_count(PyListObject *self, PyObject *value) /*[clinic end generated code: output=b1f5d284205ae714 input=3bdc3a5e6f749565]*/ { Py_ssize_t count = 0; - Py_ssize_t i; - - for (i = 0; i < Py_SIZE(self); i++) { - PyObject *obj = self->ob_item[i]; + for (Py_ssize_t i = 0; ; i++) { + PyObject *obj = list_get_item_ref(self, i); + if (obj == NULL) { + // out-of-bounds + break; + } if (obj == value) { count++; + Py_DECREF(obj); continue; } - Py_INCREF(obj); int cmp = PyObject_RichCompareBool(obj, value, Py_EQ); Py_DECREF(obj); if (cmp > 0) @@ -2721,6 +2788,7 @@ list_count(PyListObject *self, PyObject *value) } /*[clinic input] +@critical_section list.remove value: object @@ -2732,8 +2800,8 @@ Raises ValueError if the value is not present. [clinic start generated code]*/ static PyObject * -list_remove(PyListObject *self, PyObject *value) -/*[clinic end generated code: output=f087e1951a5e30d1 input=2dc2ba5bb2fb1f82]*/ +list_remove_impl(PyListObject *self, PyObject *value) +/*[clinic end generated code: output=b9b76a6633b18778 input=26c813dbb95aa93b]*/ { Py_ssize_t i; @@ -2756,8 +2824,9 @@ list_remove(PyListObject *self, PyObject *value) } static int -list_traverse(PyListObject *o, visitproc visit, void *arg) +list_traverse(PyObject *self, visitproc visit, void *arg) { + PyListObject *o = (PyListObject *)self; Py_ssize_t i; for (i = Py_SIZE(o); --i >= 0; ) @@ -2766,7 +2835,7 @@ list_traverse(PyListObject *o, visitproc visit, void *arg) } static PyObject * -list_richcompare(PyObject *v, PyObject *w, int op) +list_richcompare_impl(PyObject *v, PyObject *w, int op) { PyListObject *vl, *wl; Py_ssize_t i; @@ -2821,6 +2890,16 @@ list_richcompare(PyObject *v, PyObject *w, int op) return PyObject_RichCompare(vl->ob_item[i], wl->ob_item[i], op); } +static PyObject * +list_richcompare(PyObject *v, PyObject *w, int op) +{ + PyObject *ret; + Py_BEGIN_CRITICAL_SECTION2(v, w); + ret = list_richcompare_impl(v, w, op); + Py_END_CRITICAL_SECTION2() + return ret; +} + /*[clinic input] list.__init__ @@ -2897,10 +2976,10 @@ list___sizeof___impl(PyListObject *self) } static PyObject *list_iter(PyObject *seq); -static PyObject *list_subscript(PyListObject*, PyObject*); +static PyObject *list_subscript(PyObject*, PyObject*); static PyMethodDef list_methods[] = { - {"__getitem__", (PyCFunction)list_subscript, METH_O|METH_COEXIST, + {"__getitem__", list_subscript, METH_O|METH_COEXIST, PyDoc_STR("__getitem__($self, index, /)\n--\n\nReturn self[index].")}, LIST___REVERSED___METHODDEF LIST___SIZEOF___METHODDEF @@ -2920,21 +2999,22 @@ static PyMethodDef list_methods[] = { }; static PySequenceMethods list_as_sequence = { - (lenfunc)list_length, /* sq_length */ - (binaryfunc)list_concat, /* sq_concat */ - (ssizeargfunc)list_repeat, /* sq_repeat */ - (ssizeargfunc)list_item, /* sq_item */ + list_length, /* sq_length */ + list_concat, /* sq_concat */ + list_repeat, /* sq_repeat */ + list_item, /* sq_item */ 0, /* sq_slice */ - (ssizeobjargproc)list_ass_item, /* sq_ass_item */ + list_ass_item, /* sq_ass_item */ 0, /* sq_ass_slice */ - (objobjproc)list_contains, /* sq_contains */ - (binaryfunc)list_inplace_concat, /* sq_inplace_concat */ - (ssizeargfunc)list_inplace_repeat, /* sq_inplace_repeat */ + list_contains, /* sq_contains */ + list_inplace_concat, /* sq_inplace_concat */ + list_inplace_repeat, /* sq_inplace_repeat */ }; static PyObject * -list_subscript(PyListObject* self, PyObject* item) +list_subscript(PyObject* _self, PyObject* item) { + PyListObject* self = (PyListObject*)_self; if (_PyIndex_Check(item)) { Py_ssize_t i; i = PyNumber_AsSsize_t(item, PyExc_IndexError); @@ -2942,7 +3022,7 @@ list_subscript(PyListObject* self, PyObject* item) return NULL; if (i < 0) i += PyList_GET_SIZE(self); - return list_item(self, i); + return list_item((PyObject *)self, i); } else if (PySlice_Check(item)) { Py_ssize_t start, stop, step, slicelength, i; @@ -2987,15 +3067,16 @@ list_subscript(PyListObject* self, PyObject* item) } static int -list_ass_subscript(PyListObject* self, PyObject* item, PyObject* value) +list_ass_subscript(PyObject* _self, PyObject* item, PyObject* value) { + PyListObject *self = (PyListObject *)_self; if (_PyIndex_Check(item)) { Py_ssize_t i = PyNumber_AsSsize_t(item, PyExc_IndexError); if (i == -1 && PyErr_Occurred()) return -1; if (i < 0) i += PyList_GET_SIZE(self); - return list_ass_item(self, i, value); + return list_ass_item((PyObject *)self, i, value); } else if (PySlice_Check(item)) { Py_ssize_t start, stop, step, slicelength; @@ -3149,9 +3230,9 @@ list_ass_subscript(PyListObject* self, PyObject* item, PyObject* value) } static PyMappingMethods list_as_mapping = { - (lenfunc)list_length, - (binaryfunc)list_subscript, - (objobjargproc)list_ass_subscript + list_length, + list_subscript, + list_ass_subscript }; PyTypeObject PyList_Type = { @@ -3159,12 +3240,12 @@ PyTypeObject PyList_Type = { "list", sizeof(PyListObject), 0, - (destructor)list_dealloc, /* tp_dealloc */ + list_dealloc, /* tp_dealloc */ 0, /* tp_vectorcall_offset */ 0, /* tp_getattr */ 0, /* tp_setattr */ 0, /* tp_as_async */ - (reprfunc)list_repr, /* tp_repr */ + list_repr, /* tp_repr */ 0, /* tp_as_number */ &list_as_sequence, /* tp_as_sequence */ &list_as_mapping, /* tp_as_mapping */ @@ -3178,8 +3259,8 @@ PyTypeObject PyList_Type = { Py_TPFLAGS_BASETYPE | Py_TPFLAGS_LIST_SUBCLASS | _Py_TPFLAGS_MATCH_SELF | Py_TPFLAGS_SEQUENCE, /* tp_flags */ list___init____doc__, /* tp_doc */ - (traverseproc)list_traverse, /* tp_traverse */ - (inquiry)list_clear_slot, /* tp_clear */ + list_traverse, /* tp_traverse */ + list_clear_slot, /* tp_clear */ list_richcompare, /* tp_richcompare */ 0, /* tp_weaklistoffset */ list_iter, /* tp_iter */ @@ -3201,22 +3282,22 @@ PyTypeObject PyList_Type = { /*********************** List Iterator **************************/ -static void listiter_dealloc(_PyListIterObject *); -static int listiter_traverse(_PyListIterObject *, visitproc, void *); -static PyObject *listiter_next(_PyListIterObject *); -static PyObject *listiter_len(_PyListIterObject *, PyObject *); +static void listiter_dealloc(PyObject *); +static int listiter_traverse(PyObject *, visitproc, void *); +static PyObject *listiter_next(PyObject *); +static PyObject *listiter_len(PyObject *, PyObject *); static PyObject *listiter_reduce_general(void *_it, int forward); -static PyObject *listiter_reduce(_PyListIterObject *, PyObject *); -static PyObject *listiter_setstate(_PyListIterObject *, PyObject *state); +static PyObject *listiter_reduce(PyObject *, PyObject *); +static PyObject *listiter_setstate(PyObject *, PyObject *state); PyDoc_STRVAR(length_hint_doc, "Private method returning an estimate of len(list(it))."); PyDoc_STRVAR(reduce_doc, "Return state information for pickling."); PyDoc_STRVAR(setstate_doc, "Set state information for unpickling."); static PyMethodDef listiter_methods[] = { - {"__length_hint__", (PyCFunction)listiter_len, METH_NOARGS, length_hint_doc}, - {"__reduce__", (PyCFunction)listiter_reduce, METH_NOARGS, reduce_doc}, - {"__setstate__", (PyCFunction)listiter_setstate, METH_O, setstate_doc}, + {"__length_hint__", listiter_len, METH_NOARGS, length_hint_doc}, + {"__reduce__", listiter_reduce, METH_NOARGS, reduce_doc}, + {"__setstate__", listiter_setstate, METH_O, setstate_doc}, {NULL, NULL} /* sentinel */ }; @@ -3226,7 +3307,7 @@ PyTypeObject PyListIter_Type = { sizeof(_PyListIterObject), /* tp_basicsize */ 0, /* tp_itemsize */ /* methods */ - (destructor)listiter_dealloc, /* tp_dealloc */ + listiter_dealloc, /* tp_dealloc */ 0, /* tp_vectorcall_offset */ 0, /* tp_getattr */ 0, /* tp_setattr */ @@ -3243,12 +3324,12 @@ PyTypeObject PyListIter_Type = { 0, /* tp_as_buffer */ Py_TPFLAGS_DEFAULT | Py_TPFLAGS_HAVE_GC,/* tp_flags */ 0, /* tp_doc */ - (traverseproc)listiter_traverse, /* tp_traverse */ + listiter_traverse, /* tp_traverse */ 0, /* tp_clear */ 0, /* tp_richcompare */ 0, /* tp_weaklistoffset */ PyObject_SelfIter, /* tp_iter */ - (iternextfunc)listiter_next, /* tp_iternext */ + listiter_next, /* tp_iternext */ listiter_methods, /* tp_methods */ 0, /* tp_members */ }; @@ -3273,23 +3354,25 @@ list_iter(PyObject *seq) } static void -listiter_dealloc(_PyListIterObject *it) +listiter_dealloc(PyObject *self) { + _PyListIterObject *it = (_PyListIterObject *)self; _PyObject_GC_UNTRACK(it); Py_XDECREF(it->it_seq); PyObject_GC_Del(it); } static int -listiter_traverse(_PyListIterObject *it, visitproc visit, void *arg) +listiter_traverse(PyObject *it, visitproc visit, void *arg) { - Py_VISIT(it->it_seq); + Py_VISIT(((_PyListIterObject *)it)->it_seq); return 0; } static PyObject * -listiter_next(_PyListIterObject *it) +listiter_next(PyObject *self) { + _PyListIterObject *it = (_PyListIterObject *)self; PyListObject *seq; PyObject *item; @@ -3311,8 +3394,9 @@ listiter_next(_PyListIterObject *it) } static PyObject * -listiter_len(_PyListIterObject *it, PyObject *Py_UNUSED(ignored)) +listiter_len(PyObject *self, PyObject *Py_UNUSED(ignored)) { + _PyListIterObject *it = (_PyListIterObject *)self; Py_ssize_t len; if (it->it_seq) { len = PyList_GET_SIZE(it->it_seq) - it->it_index; @@ -3323,14 +3407,15 @@ listiter_len(_PyListIterObject *it, PyObject *Py_UNUSED(ignored)) } static PyObject * -listiter_reduce(_PyListIterObject *it, PyObject *Py_UNUSED(ignored)) +listiter_reduce(PyObject *it, PyObject *Py_UNUSED(ignored)) { return listiter_reduce_general(it, 1); } static PyObject * -listiter_setstate(_PyListIterObject *it, PyObject *state) +listiter_setstate(PyObject *self, PyObject *state) { + _PyListIterObject *it = (_PyListIterObject *)self; Py_ssize_t index = PyLong_AsSsize_t(state); if (index == -1 && PyErr_Occurred()) return NULL; @@ -3352,17 +3437,17 @@ typedef struct { PyListObject *it_seq; /* Set to NULL when iterator is exhausted */ } listreviterobject; -static void listreviter_dealloc(listreviterobject *); -static int listreviter_traverse(listreviterobject *, visitproc, void *); -static PyObject *listreviter_next(listreviterobject *); -static PyObject *listreviter_len(listreviterobject *, PyObject *); -static PyObject *listreviter_reduce(listreviterobject *, PyObject *); -static PyObject *listreviter_setstate(listreviterobject *, PyObject *); +static void listreviter_dealloc(PyObject *); +static int listreviter_traverse(PyObject *, visitproc, void *); +static PyObject *listreviter_next(PyObject *); +static PyObject *listreviter_len(PyObject *, PyObject *); +static PyObject *listreviter_reduce(PyObject *, PyObject *); +static PyObject *listreviter_setstate(PyObject *, PyObject *); static PyMethodDef listreviter_methods[] = { - {"__length_hint__", (PyCFunction)listreviter_len, METH_NOARGS, length_hint_doc}, - {"__reduce__", (PyCFunction)listreviter_reduce, METH_NOARGS, reduce_doc}, - {"__setstate__", (PyCFunction)listreviter_setstate, METH_O, setstate_doc}, + {"__length_hint__", listreviter_len, METH_NOARGS, length_hint_doc}, + {"__reduce__", listreviter_reduce, METH_NOARGS, reduce_doc}, + {"__setstate__", listreviter_setstate, METH_O, setstate_doc}, {NULL, NULL} /* sentinel */ }; @@ -3372,7 +3457,7 @@ PyTypeObject PyListRevIter_Type = { sizeof(listreviterobject), /* tp_basicsize */ 0, /* tp_itemsize */ /* methods */ - (destructor)listreviter_dealloc, /* tp_dealloc */ + listreviter_dealloc, /* tp_dealloc */ 0, /* tp_vectorcall_offset */ 0, /* tp_getattr */ 0, /* tp_setattr */ @@ -3389,12 +3474,12 @@ PyTypeObject PyListRevIter_Type = { 0, /* tp_as_buffer */ Py_TPFLAGS_DEFAULT | Py_TPFLAGS_HAVE_GC,/* tp_flags */ 0, /* tp_doc */ - (traverseproc)listreviter_traverse, /* tp_traverse */ + listreviter_traverse, /* tp_traverse */ 0, /* tp_clear */ 0, /* tp_richcompare */ 0, /* tp_weaklistoffset */ PyObject_SelfIter, /* tp_iter */ - (iternextfunc)listreviter_next, /* tp_iternext */ + listreviter_next, /* tp_iternext */ listreviter_methods, /* tp_methods */ 0, }; @@ -3422,23 +3507,25 @@ list___reversed___impl(PyListObject *self) } static void -listreviter_dealloc(listreviterobject *it) +listreviter_dealloc(PyObject *self) { + listreviterobject *it = (listreviterobject *)self; PyObject_GC_UnTrack(it); Py_XDECREF(it->it_seq); PyObject_GC_Del(it); } static int -listreviter_traverse(listreviterobject *it, visitproc visit, void *arg) +listreviter_traverse(PyObject *it, visitproc visit, void *arg) { - Py_VISIT(it->it_seq); + Py_VISIT(((listreviterobject *)it)->it_seq); return 0; } static PyObject * -listreviter_next(listreviterobject *it) +listreviter_next(PyObject *self) { + listreviterobject *it = (listreviterobject *)self; PyObject *item; Py_ssize_t index; PyListObject *seq; @@ -3463,8 +3550,9 @@ listreviter_next(listreviterobject *it) } static PyObject * -listreviter_len(listreviterobject *it, PyObject *Py_UNUSED(ignored)) +listreviter_len(PyObject *self, PyObject *Py_UNUSED(ignored)) { + listreviterobject *it = (listreviterobject *)self; Py_ssize_t len = it->it_index + 1; if (it->it_seq == NULL || PyList_GET_SIZE(it->it_seq) < len) len = 0; @@ -3472,14 +3560,15 @@ listreviter_len(listreviterobject *it, PyObject *Py_UNUSED(ignored)) } static PyObject * -listreviter_reduce(listreviterobject *it, PyObject *Py_UNUSED(ignored)) +listreviter_reduce(PyObject *it, PyObject *Py_UNUSED(ignored)) { return listiter_reduce_general(it, 0); } static PyObject * -listreviter_setstate(listreviterobject *it, PyObject *state) +listreviter_setstate(PyObject *self, PyObject *state) { + listreviterobject *it = (listreviterobject *)self; Py_ssize_t index = PyLong_AsSsize_t(state); if (index == -1 && PyErr_Occurred()) return NULL; diff --git a/Objects/lnotab_notes.txt b/Objects/lnotab_notes.txt index d45d09d4ab9a50..0f3599340318f0 100644 --- a/Objects/lnotab_notes.txt +++ b/Objects/lnotab_notes.txt @@ -60,7 +60,7 @@ Final form: Iterating over the table. ------------------------- -For the `co_lines` attribute we want to emit the full form, omitting the (350, 360, No line number) and empty entries. +For the `co_lines` method we want to emit the full form, omitting the (350, 360, No line number) and empty entries. The code is as follows: diff --git a/Objects/longobject.c b/Objects/longobject.c index fae70dd13bb18a..e655ba19e8f1c1 100644 --- a/Objects/longobject.c +++ b/Objects/longobject.c @@ -6171,7 +6171,7 @@ long_vectorcall(PyObject *type, PyObject * const*args, return long_new_impl(_PyType_CAST(type), args[0], args[1]); default: return PyErr_Format(PyExc_TypeError, - "int expected at most 2 argument%s, got %zd", + "int expected at most 2 arguments, got %zd", nargs); } } diff --git a/Objects/mimalloc/heap.c b/Objects/mimalloc/heap.c index 4eb622ed4bad76..154dad0b128480 100644 --- a/Objects/mimalloc/heap.c +++ b/Objects/mimalloc/heap.c @@ -26,7 +26,7 @@ typedef bool (heap_page_visitor_fun)(mi_heap_t* heap, mi_page_queue_t* pq, mi_pa // Visit all pages in a heap; returns `false` if break was called. static bool mi_heap_visit_pages(mi_heap_t* heap, heap_page_visitor_fun* fn, void* arg1, void* arg2) { - if (heap==NULL || heap->page_count==0) return 0; + if (heap==NULL || heap->page_count==0) return true; // visit all pages #if MI_DEBUG>1 @@ -123,6 +123,9 @@ static void mi_heap_collect_ex(mi_heap_t* heap, mi_collect_t collect) const bool force = collect >= MI_FORCE; _mi_deferred_free(heap, force); + // gh-112532: we may be called from a thread that is not the owner of the heap + bool is_main_thread = _mi_is_main_thread() && heap->thread_id == _mi_thread_id(); + // note: never reclaim on collect but leave it to threads that need storage to reclaim const bool force_main = #ifdef NDEBUG @@ -130,7 +133,7 @@ static void mi_heap_collect_ex(mi_heap_t* heap, mi_collect_t collect) #else collect >= MI_FORCE #endif - && _mi_is_main_thread() && mi_heap_is_backing(heap) && !heap->no_reclaim; + && is_main_thread && mi_heap_is_backing(heap) && !heap->no_reclaim; if (force_main) { // the main thread is abandoned (end-of-program), try to reclaim all abandoned segments. @@ -164,7 +167,7 @@ static void mi_heap_collect_ex(mi_heap_t* heap, mi_collect_t collect) } // collect regions on program-exit (or shared library unload) - if (force && _mi_is_main_thread() && mi_heap_is_backing(heap)) { + if (force && is_main_thread && mi_heap_is_backing(heap)) { _mi_thread_data_collect(); // collect thread data cache _mi_arena_collect(true /* force purge */, &heap->tld->stats); } @@ -206,22 +209,34 @@ mi_heap_t* mi_heap_get_backing(void) { return bheap; } -mi_decl_nodiscard mi_heap_t* mi_heap_new_in_arena(mi_arena_id_t arena_id) { - mi_heap_t* bheap = mi_heap_get_backing(); - mi_heap_t* heap = mi_heap_malloc_tp(bheap, mi_heap_t); // todo: OS allocate in secure mode? - if (heap == NULL) return NULL; +void _mi_heap_init_ex(mi_heap_t* heap, mi_tld_t* tld, mi_arena_id_t arena_id, bool no_reclaim, uint8_t tag) +{ _mi_memcpy_aligned(heap, &_mi_heap_empty, sizeof(mi_heap_t)); - heap->tld = bheap->tld; + heap->tld = tld; heap->thread_id = _mi_thread_id(); heap->arena_id = arena_id; - _mi_random_split(&bheap->random, &heap->random); + if (heap == tld->heap_backing) { + _mi_random_init(&heap->random); + } + else { + _mi_random_split(&tld->heap_backing->random, &heap->random); + } heap->cookie = _mi_heap_random_next(heap) | 1; heap->keys[0] = _mi_heap_random_next(heap); heap->keys[1] = _mi_heap_random_next(heap); - heap->no_reclaim = true; // don't reclaim abandoned pages or otherwise destroy is unsafe + heap->no_reclaim = no_reclaim; + heap->tag = tag; // push on the thread local heaps list heap->next = heap->tld->heaps; heap->tld->heaps = heap; +} + +mi_decl_nodiscard mi_heap_t* mi_heap_new_in_arena(mi_arena_id_t arena_id) { + mi_heap_t* bheap = mi_heap_get_backing(); + mi_heap_t* heap = mi_heap_malloc_tp(bheap, mi_heap_t); // todo: OS allocate in secure mode? + if (heap == NULL) return NULL; + // don't reclaim abandoned pages or otherwise destroy is unsafe + _mi_heap_init_ex(heap, bheap->tld, arena_id, true, 0); return heap; } @@ -506,15 +521,23 @@ typedef struct mi_heap_area_ex_s { mi_page_t* page; } mi_heap_area_ex_t; -static bool mi_heap_area_visit_blocks(const mi_heap_area_ex_t* xarea, mi_block_visit_fun* visitor, void* arg) { - mi_assert(xarea != NULL); - if (xarea==NULL) return true; - const mi_heap_area_t* area = &xarea->area; - mi_page_t* page = xarea->page; +static void mi_fast_divisor(size_t divisor, size_t* magic, size_t* shift) { + mi_assert_internal(divisor > 0 && divisor <= UINT32_MAX); + *shift = MI_INTPTR_BITS - mi_clz(divisor - 1); + *magic = (size_t)(((1ULL << 32) * ((1ULL << *shift) - divisor)) / divisor + 1); +} + +static size_t mi_fast_divide(size_t n, size_t magic, size_t shift) { + mi_assert_internal(n <= UINT32_MAX); + return ((((uint64_t) n * magic) >> 32) + n) >> shift; +} + +bool _mi_heap_area_visit_blocks(const mi_heap_area_t* area, mi_page_t *page, mi_block_visit_fun* visitor, void* arg) { + mi_assert(area != NULL); + if (area==NULL) return true; mi_assert(page != NULL); if (page == NULL) return true; - _mi_page_free_collect(page,true); mi_assert_internal(page->local_free == NULL); if (page->used == 0) return true; @@ -522,17 +545,39 @@ static bool mi_heap_area_visit_blocks(const mi_heap_area_ex_t* xarea, mi_block_v const size_t ubsize = mi_page_usable_block_size(page); // without padding size_t psize; uint8_t* pstart = _mi_page_start(_mi_page_segment(page), page, &psize); + mi_heap_t* heap = mi_page_heap(page); if (page->capacity == 1) { // optimize page with one block mi_assert_internal(page->used == 1 && page->free == NULL); - return visitor(mi_page_heap(page), area, pstart, ubsize, arg); + return visitor(heap, area, pstart, ubsize, arg); + } + + if (page->used == page->capacity) { + // optimize full pages + uint8_t* block = pstart; + for (size_t i = 0; i < page->capacity; i++) { + if (!visitor(heap, area, block, ubsize, arg)) return false; + block += bsize; + } + return true; } // create a bitmap of free blocks. #define MI_MAX_BLOCKS (MI_SMALL_PAGE_SIZE / sizeof(void*)) - uintptr_t free_map[MI_MAX_BLOCKS / sizeof(uintptr_t)]; - memset(free_map, 0, sizeof(free_map)); + uintptr_t free_map[MI_MAX_BLOCKS / MI_INTPTR_BITS]; + size_t bmapsize = (page->capacity + MI_INTPTR_BITS - 1) / MI_INTPTR_BITS; + memset(free_map, 0, bmapsize * sizeof(uintptr_t)); + + if (page->capacity % MI_INTPTR_BITS != 0) { + size_t shift = (page->capacity % MI_INTPTR_BITS); + uintptr_t mask = (UINTPTR_MAX << shift); + free_map[bmapsize-1] = mask; + } + + // fast repeated division by the block size + size_t magic, shift; + mi_fast_divisor(bsize, &magic, &shift); #if MI_DEBUG>1 size_t free_count = 0; @@ -544,10 +589,11 @@ static bool mi_heap_area_visit_blocks(const mi_heap_area_ex_t* xarea, mi_block_v mi_assert_internal((uint8_t*)block >= pstart && (uint8_t*)block < (pstart + psize)); size_t offset = (uint8_t*)block - pstart; mi_assert_internal(offset % bsize == 0); - size_t blockidx = offset / bsize; // Todo: avoid division? - mi_assert_internal( blockidx < MI_MAX_BLOCKS); - size_t bitidx = (blockidx / sizeof(uintptr_t)); - size_t bit = blockidx - (bitidx * sizeof(uintptr_t)); + size_t blockidx = mi_fast_divide(offset, magic, shift); + mi_assert_internal(blockidx == offset / bsize); + mi_assert_internal(blockidx < MI_MAX_BLOCKS); + size_t bitidx = (blockidx / MI_INTPTR_BITS); + size_t bit = blockidx - (bitidx * MI_INTPTR_BITS); free_map[bitidx] |= ((uintptr_t)1 << bit); } mi_assert_internal(page->capacity == (free_count + page->used)); @@ -556,19 +602,29 @@ static bool mi_heap_area_visit_blocks(const mi_heap_area_ex_t* xarea, mi_block_v #if MI_DEBUG>1 size_t used_count = 0; #endif - for (size_t i = 0; i < page->capacity; i++) { - size_t bitidx = (i / sizeof(uintptr_t)); - size_t bit = i - (bitidx * sizeof(uintptr_t)); - uintptr_t m = free_map[bitidx]; - if (bit == 0 && m == UINTPTR_MAX) { - i += (sizeof(uintptr_t) - 1); // skip a run of free blocks + uint8_t* block = pstart; + for (size_t i = 0; i < bmapsize; i++) { + if (free_map[i] == 0) { + // every block is in use + for (size_t j = 0; j < MI_INTPTR_BITS; j++) { + #if MI_DEBUG>1 + used_count++; + #endif + if (!visitor(heap, area, block, ubsize, arg)) return false; + block += bsize; + } } - else if ((m & ((uintptr_t)1 << bit)) == 0) { - #if MI_DEBUG>1 - used_count++; - #endif - uint8_t* block = pstart + (i * bsize); - if (!visitor(mi_page_heap(page), area, block, ubsize, arg)) return false; + else { + uintptr_t m = ~free_map[i]; + while (m) { + #if MI_DEBUG>1 + used_count++; + #endif + size_t bitidx = mi_ctz(m); + if (!visitor(heap, area, block + (bitidx * bsize), ubsize, arg)) return false; + m &= m - 1; + } + block += bsize * MI_INTPTR_BITS; } } mi_assert_internal(page->used == used_count); @@ -577,21 +633,25 @@ static bool mi_heap_area_visit_blocks(const mi_heap_area_ex_t* xarea, mi_block_v typedef bool (mi_heap_area_visit_fun)(const mi_heap_t* heap, const mi_heap_area_ex_t* area, void* arg); +void _mi_heap_area_init(mi_heap_area_t* area, mi_page_t* page) { + _mi_page_free_collect(page,true); + const size_t bsize = mi_page_block_size(page); + const size_t ubsize = mi_page_usable_block_size(page); + area->reserved = page->reserved * bsize; + area->committed = page->capacity * bsize; + area->blocks = _mi_page_start(_mi_page_segment(page), page, NULL); + area->used = page->used; // number of blocks in use (#553) + area->block_size = ubsize; + area->full_block_size = bsize; +} static bool mi_heap_visit_areas_page(mi_heap_t* heap, mi_page_queue_t* pq, mi_page_t* page, void* vfun, void* arg) { MI_UNUSED(heap); MI_UNUSED(pq); mi_heap_area_visit_fun* fun = (mi_heap_area_visit_fun*)vfun; mi_heap_area_ex_t xarea; - const size_t bsize = mi_page_block_size(page); - const size_t ubsize = mi_page_usable_block_size(page); xarea.page = page; - xarea.area.reserved = page->reserved * bsize; - xarea.area.committed = page->capacity * bsize; - xarea.area.blocks = _mi_page_start(_mi_page_segment(page), page, NULL); - xarea.area.used = page->used; // number of blocks in use (#553) - xarea.area.block_size = ubsize; - xarea.area.full_block_size = bsize; + _mi_heap_area_init(&xarea.area, page); return fun(heap, &xarea, arg); } @@ -612,7 +672,7 @@ static bool mi_heap_area_visitor(const mi_heap_t* heap, const mi_heap_area_ex_t* mi_visit_blocks_args_t* args = (mi_visit_blocks_args_t*)arg; if (!args->visitor(heap, &xarea->area, NULL, xarea->area.block_size, args->arg)) return false; if (args->visit_blocks) { - return mi_heap_area_visit_blocks(xarea, args->visitor, args->arg); + return _mi_heap_area_visit_blocks(&xarea->area, xarea->page, args->visitor, args->arg); } else { return true; @@ -622,5 +682,6 @@ static bool mi_heap_area_visitor(const mi_heap_t* heap, const mi_heap_area_ex_t* // Visit all blocks in a heap bool mi_heap_visit_blocks(const mi_heap_t* heap, bool visit_blocks, mi_block_visit_fun* visitor, void* arg) { mi_visit_blocks_args_t args = { visit_blocks, visitor, arg }; + _mi_heap_delayed_free_partial((mi_heap_t *)heap); return mi_heap_visit_areas(heap, &mi_heap_area_visitor, &args); } diff --git a/Objects/mimalloc/init.c b/Objects/mimalloc/init.c index 7dfa7657737117..5897f0512f8ef9 100644 --- a/Objects/mimalloc/init.c +++ b/Objects/mimalloc/init.c @@ -14,7 +14,7 @@ terms of the MIT license. A copy of the license can be found in the file // Empty page used to initialize the small free pages array const mi_page_t _mi_page_empty = { - 0, false, false, false, + 0, false, false, false, 0, 0, // capacity 0, // reserved capacity { 0 }, // flags @@ -121,7 +121,8 @@ mi_decl_cache_align const mi_heap_t _mi_heap_empty = { 0, // page count MI_BIN_FULL, 0, // page retired min/max NULL, // next - false + false, + 0 }; #define tld_empty_stats ((mi_stats_t*)((uint8_t*)&tld_empty + offsetof(mi_tld_t,stats))) @@ -131,7 +132,7 @@ mi_decl_cache_align static const mi_tld_t tld_empty = { 0, false, NULL, NULL, - { MI_SEGMENT_SPAN_QUEUES_EMPTY, 0, 0, 0, 0, tld_empty_stats, tld_empty_os }, // segments + { MI_SEGMENT_SPAN_QUEUES_EMPTY, 0, 0, 0, 0, tld_empty_stats, tld_empty_os, &_mi_abandoned_default }, // segments { 0, tld_empty_stats }, // os { MI_STATS_NULL } // stats }; @@ -148,7 +149,7 @@ extern mi_heap_t _mi_heap_main; static mi_tld_t tld_main = { 0, false, &_mi_heap_main, & _mi_heap_main, - { MI_SEGMENT_SPAN_QUEUES_EMPTY, 0, 0, 0, 0, &tld_main.stats, &tld_main.os }, // segments + { MI_SEGMENT_SPAN_QUEUES_EMPTY, 0, 0, 0, 0, &tld_main.stats, &tld_main.os, &_mi_abandoned_default }, // segments { 0, &tld_main.stats }, // os { MI_STATS_NULL } // stats }; @@ -297,24 +298,20 @@ static bool _mi_heap_init(void) { mi_thread_data_t* td = mi_thread_data_zalloc(); if (td == NULL) return false; - mi_tld_t* tld = &td->tld; - mi_heap_t* heap = &td->heap; + _mi_tld_init(&td->tld, &td->heap); + _mi_heap_init_ex(&td->heap, &td->tld, _mi_arena_id_none(), false, 0); + _mi_heap_set_default_direct(&td->heap); + } + return false; +} + +void _mi_tld_init(mi_tld_t* tld, mi_heap_t* bheap) { _mi_memcpy_aligned(tld, &tld_empty, sizeof(*tld)); - _mi_memcpy_aligned(heap, &_mi_heap_empty, sizeof(*heap)); - heap->thread_id = _mi_thread_id(); - _mi_random_init(&heap->random); - heap->cookie = _mi_heap_random_next(heap) | 1; - heap->keys[0] = _mi_heap_random_next(heap); - heap->keys[1] = _mi_heap_random_next(heap); - heap->tld = tld; - tld->heap_backing = heap; - tld->heaps = heap; tld->segments.stats = &tld->stats; tld->segments.os = &tld->os; + tld->segments.abandoned = &_mi_abandoned_default; tld->os.stats = &tld->stats; - _mi_heap_set_default_direct(heap); - } - return false; + tld->heap_backing = bheap; } // Free the thread local default heap (called from `mi_thread_done`) diff --git a/Objects/mimalloc/page.c b/Objects/mimalloc/page.c index 4610cf27afff75..8f0ce920156e04 100644 --- a/Objects/mimalloc/page.c +++ b/Objects/mimalloc/page.c @@ -660,6 +660,7 @@ static void mi_page_init(mi_heap_t* heap, mi_page_t* page, size_t block_size, mi mi_assert_internal(block_size > 0); // set fields mi_page_set_heap(page, heap); + page->tag = heap->tag; page->xblock_size = (block_size < MI_HUGE_BLOCK_SIZE ? (uint32_t)block_size : MI_HUGE_BLOCK_SIZE); // initialize before _mi_segment_page_start size_t page_size; const void* page_start = _mi_segment_page_start(segment, page, &page_size); diff --git a/Objects/mimalloc/prim/unix/prim.c b/Objects/mimalloc/prim/unix/prim.c index 2152017e01fb43..ec8447ab40d70c 100644 --- a/Objects/mimalloc/prim/unix/prim.c +++ b/Objects/mimalloc/prim/unix/prim.c @@ -50,7 +50,7 @@ terms of the MIT license. A copy of the license can be found in the file #include #endif -#if !defined(__HAIKU__) && !defined(__APPLE__) && !defined(__CYGWIN__) && !defined(_AIX) && !defined(__FreeBSD__) +#if !defined(__HAIKU__) && !defined(__APPLE__) && !defined(__CYGWIN__) && !defined(_AIX) && !defined(__FreeBSD__) && !defined(__sun) #define MI_HAS_SYSCALL_H #include #endif @@ -76,7 +76,7 @@ static int mi_prim_access(const char *fpath, int mode) { return syscall(SYS_access,fpath,mode); } -#elif !defined(__APPLE__) && !defined(_AIX) && !defined(__FreeBSD__) // avoid unused warnings +#elif !defined(__APPLE__) && !defined(_AIX) && !defined(__FreeBSD__) && !defined(__sun) // avoid unused warnings static int mi_prim_open(const char* fpath, int open_flags) { return open(fpath,open_flags); @@ -310,7 +310,7 @@ static void* unix_mmap(void* addr, size_t size, size_t try_alignment, int protec #elif defined(__sun) if (allow_large && _mi_os_use_large_page(size, try_alignment)) { struct memcntl_mha cmd = {0}; - cmd.mha_pagesize = large_os_page_size; + cmd.mha_pagesize = 2*MI_MiB; cmd.mha_cmd = MHA_MAPSIZE_VA; if (memcntl((caddr_t)p, size, MC_HAT_ADVISE, (caddr_t)&cmd, 0, 0) == 0) { *is_large = true; diff --git a/Objects/mimalloc/segment.c b/Objects/mimalloc/segment.c index 033e0f97c36c14..584233b8b57bb4 100644 --- a/Objects/mimalloc/segment.c +++ b/Objects/mimalloc/segment.c @@ -395,7 +395,7 @@ static void mi_segment_os_free(mi_segment_t* segment, mi_segments_tld_t* tld) { const size_t size = mi_segment_size(segment); const size_t csize = _mi_commit_mask_committed_size(&segment->commit_mask, size); - _mi_abandoned_await_readers(); // wait until safe to free + _mi_abandoned_await_readers(tld->abandoned); // wait until safe to free _mi_arena_free(segment, mi_segment_size(segment), csize, segment->memid, tld->stats); } @@ -1059,7 +1059,6 @@ would be spread among all other segments in the arenas. // Use the bottom 20-bits (on 64-bit) of the aligned segment pointers // to put in a tag that increments on update to avoid the A-B-A problem. #define MI_TAGGED_MASK MI_SEGMENT_MASK -typedef uintptr_t mi_tagged_segment_t; static mi_segment_t* mi_tagged_segment_ptr(mi_tagged_segment_t ts) { return (mi_segment_t*)(ts & ~MI_TAGGED_MASK); @@ -1071,55 +1070,40 @@ static mi_tagged_segment_t mi_tagged_segment(mi_segment_t* segment, mi_tagged_se return ((uintptr_t)segment | tag); } -// This is a list of visited abandoned pages that were full at the time. -// this list migrates to `abandoned` when that becomes NULL. The use of -// this list reduces contention and the rate at which segments are visited. -static mi_decl_cache_align _Atomic(mi_segment_t*) abandoned_visited; // = NULL - -// The abandoned page list (tagged as it supports pop) -static mi_decl_cache_align _Atomic(mi_tagged_segment_t) abandoned; // = NULL - -// Maintain these for debug purposes (these counts may be a bit off) -static mi_decl_cache_align _Atomic(size_t) abandoned_count; -static mi_decl_cache_align _Atomic(size_t) abandoned_visited_count; - -// We also maintain a count of current readers of the abandoned list -// in order to prevent resetting/decommitting segment memory if it might -// still be read. -static mi_decl_cache_align _Atomic(size_t) abandoned_readers; // = 0 +mi_abandoned_pool_t _mi_abandoned_default; // Push on the visited list -static void mi_abandoned_visited_push(mi_segment_t* segment) { +static void mi_abandoned_visited_push(mi_abandoned_pool_t *pool, mi_segment_t* segment) { mi_assert_internal(segment->thread_id == 0); mi_assert_internal(mi_atomic_load_ptr_relaxed(mi_segment_t,&segment->abandoned_next) == NULL); mi_assert_internal(segment->next == NULL); mi_assert_internal(segment->used > 0); - mi_segment_t* anext = mi_atomic_load_ptr_relaxed(mi_segment_t, &abandoned_visited); + mi_segment_t* anext = mi_atomic_load_ptr_relaxed(mi_segment_t, &pool->abandoned_visited); do { mi_atomic_store_ptr_release(mi_segment_t, &segment->abandoned_next, anext); - } while (!mi_atomic_cas_ptr_weak_release(mi_segment_t, &abandoned_visited, &anext, segment)); - mi_atomic_increment_relaxed(&abandoned_visited_count); + } while (!mi_atomic_cas_ptr_weak_release(mi_segment_t, &pool->abandoned_visited, &anext, segment)); + mi_atomic_increment_relaxed(&pool->abandoned_visited_count); } // Move the visited list to the abandoned list. -static bool mi_abandoned_visited_revisit(void) +static bool mi_abandoned_visited_revisit(mi_abandoned_pool_t *pool) { // quick check if the visited list is empty - if (mi_atomic_load_ptr_relaxed(mi_segment_t, &abandoned_visited) == NULL) return false; + if (mi_atomic_load_ptr_relaxed(mi_segment_t, &pool->abandoned_visited) == NULL) return false; // grab the whole visited list - mi_segment_t* first = mi_atomic_exchange_ptr_acq_rel(mi_segment_t, &abandoned_visited, NULL); + mi_segment_t* first = mi_atomic_exchange_ptr_acq_rel(mi_segment_t, &pool->abandoned_visited, NULL); if (first == NULL) return false; // first try to swap directly if the abandoned list happens to be NULL mi_tagged_segment_t afirst; - mi_tagged_segment_t ts = mi_atomic_load_relaxed(&abandoned); + mi_tagged_segment_t ts = mi_atomic_load_relaxed(&pool->abandoned); if (mi_tagged_segment_ptr(ts)==NULL) { - size_t count = mi_atomic_load_relaxed(&abandoned_visited_count); + size_t count = mi_atomic_load_relaxed(&pool->abandoned_visited_count); afirst = mi_tagged_segment(first, ts); - if (mi_atomic_cas_strong_acq_rel(&abandoned, &ts, afirst)) { - mi_atomic_add_relaxed(&abandoned_count, count); - mi_atomic_sub_relaxed(&abandoned_visited_count, count); + if (mi_atomic_cas_strong_acq_rel(&pool->abandoned, &ts, afirst)) { + mi_atomic_add_relaxed(&pool->abandoned_count, count); + mi_atomic_sub_relaxed(&pool->abandoned_visited_count, count); return true; } } @@ -1133,51 +1117,51 @@ static bool mi_abandoned_visited_revisit(void) // and atomically prepend to the abandoned list // (no need to increase the readers as we don't access the abandoned segments) - mi_tagged_segment_t anext = mi_atomic_load_relaxed(&abandoned); + mi_tagged_segment_t anext = mi_atomic_load_relaxed(&pool->abandoned); size_t count; do { - count = mi_atomic_load_relaxed(&abandoned_visited_count); + count = mi_atomic_load_relaxed(&pool->abandoned_visited_count); mi_atomic_store_ptr_release(mi_segment_t, &last->abandoned_next, mi_tagged_segment_ptr(anext)); afirst = mi_tagged_segment(first, anext); - } while (!mi_atomic_cas_weak_release(&abandoned, &anext, afirst)); - mi_atomic_add_relaxed(&abandoned_count, count); - mi_atomic_sub_relaxed(&abandoned_visited_count, count); + } while (!mi_atomic_cas_weak_release(&pool->abandoned, &anext, afirst)); + mi_atomic_add_relaxed(&pool->abandoned_count, count); + mi_atomic_sub_relaxed(&pool->abandoned_visited_count, count); return true; } // Push on the abandoned list. -static void mi_abandoned_push(mi_segment_t* segment) { +static void mi_abandoned_push(mi_abandoned_pool_t* pool, mi_segment_t* segment) { mi_assert_internal(segment->thread_id == 0); mi_assert_internal(mi_atomic_load_ptr_relaxed(mi_segment_t, &segment->abandoned_next) == NULL); mi_assert_internal(segment->next == NULL); mi_assert_internal(segment->used > 0); mi_tagged_segment_t next; - mi_tagged_segment_t ts = mi_atomic_load_relaxed(&abandoned); + mi_tagged_segment_t ts = mi_atomic_load_relaxed(&pool->abandoned); do { mi_atomic_store_ptr_release(mi_segment_t, &segment->abandoned_next, mi_tagged_segment_ptr(ts)); next = mi_tagged_segment(segment, ts); - } while (!mi_atomic_cas_weak_release(&abandoned, &ts, next)); - mi_atomic_increment_relaxed(&abandoned_count); + } while (!mi_atomic_cas_weak_release(&pool->abandoned, &ts, next)); + mi_atomic_increment_relaxed(&pool->abandoned_count); } // Wait until there are no more pending reads on segments that used to be in the abandoned list // called for example from `arena.c` before decommitting -void _mi_abandoned_await_readers(void) { +void _mi_abandoned_await_readers(mi_abandoned_pool_t* pool) { size_t n; do { - n = mi_atomic_load_acquire(&abandoned_readers); + n = mi_atomic_load_acquire(&pool->abandoned_readers); if (n != 0) mi_atomic_yield(); } while (n != 0); } // Pop from the abandoned list -static mi_segment_t* mi_abandoned_pop(void) { +static mi_segment_t* mi_abandoned_pop(mi_abandoned_pool_t* pool) { mi_segment_t* segment; // Check efficiently if it is empty (or if the visited list needs to be moved) - mi_tagged_segment_t ts = mi_atomic_load_relaxed(&abandoned); + mi_tagged_segment_t ts = mi_atomic_load_relaxed(&pool->abandoned); segment = mi_tagged_segment_ptr(ts); if mi_likely(segment == NULL) { - if mi_likely(!mi_abandoned_visited_revisit()) { // try to swap in the visited list on NULL + if mi_likely(!mi_abandoned_visited_revisit(pool)) { // try to swap in the visited list on NULL return NULL; } } @@ -1186,20 +1170,20 @@ static mi_segment_t* mi_abandoned_pop(void) { // a segment to be decommitted while a read is still pending, // and a tagged pointer to prevent A-B-A link corruption. // (this is called from `region.c:_mi_mem_free` for example) - mi_atomic_increment_relaxed(&abandoned_readers); // ensure no segment gets decommitted + mi_atomic_increment_relaxed(&pool->abandoned_readers); // ensure no segment gets decommitted mi_tagged_segment_t next = 0; - ts = mi_atomic_load_acquire(&abandoned); + ts = mi_atomic_load_acquire(&pool->abandoned); do { segment = mi_tagged_segment_ptr(ts); if (segment != NULL) { mi_segment_t* anext = mi_atomic_load_ptr_relaxed(mi_segment_t, &segment->abandoned_next); next = mi_tagged_segment(anext, ts); // note: reads the segment's `abandoned_next` field so should not be decommitted } - } while (segment != NULL && !mi_atomic_cas_weak_acq_rel(&abandoned, &ts, next)); - mi_atomic_decrement_relaxed(&abandoned_readers); // release reader lock + } while (segment != NULL && !mi_atomic_cas_weak_acq_rel(&pool->abandoned, &ts, next)); + mi_atomic_decrement_relaxed(&pool->abandoned_readers); // release reader lock if (segment != NULL) { mi_atomic_store_ptr_release(mi_segment_t, &segment->abandoned_next, NULL); - mi_atomic_decrement_relaxed(&abandoned_count); + mi_atomic_decrement_relaxed(&pool->abandoned_count); } return segment; } @@ -1237,7 +1221,7 @@ static void mi_segment_abandon(mi_segment_t* segment, mi_segments_tld_t* tld) { segment->thread_id = 0; mi_atomic_store_ptr_release(mi_segment_t, &segment->abandoned_next, NULL); segment->abandoned_visits = 1; // from 0 to 1 to signify it is abandoned - mi_abandoned_push(segment); + mi_abandoned_push(tld->abandoned, segment); } void _mi_segment_page_abandon(mi_page_t* page, mi_segments_tld_t* tld) { @@ -1315,6 +1299,18 @@ static bool mi_segment_check_free(mi_segment_t* segment, size_t slices_needed, s return has_page; } +static mi_heap_t* mi_heap_by_tag(mi_heap_t* heap, uint8_t tag) { + if (heap->tag == tag) { + return heap; + } + for (mi_heap_t *curr = heap->tld->heaps; curr != NULL; curr = curr->next) { + if (curr->tag == tag) { + return curr; + } + } + return NULL; +} + // Reclaim an abandoned segment; returns NULL if the segment was freed // set `right_page_reclaimed` to `true` if it reclaimed a page of the right `block_size` that was not full. static mi_segment_t* mi_segment_reclaim(mi_segment_t* segment, mi_heap_t* heap, size_t requested_block_size, bool* right_page_reclaimed, mi_segments_tld_t* tld) { @@ -1337,6 +1333,7 @@ static mi_segment_t* mi_segment_reclaim(mi_segment_t* segment, mi_heap_t* heap, if (mi_slice_is_used(slice)) { // in use: reclaim the page in our heap mi_page_t* page = mi_slice_to_page(slice); + mi_heap_t* target_heap = mi_heap_by_tag(heap, page->tag); mi_assert_internal(page->is_committed); mi_assert_internal(mi_page_thread_free_flag(page)==MI_NEVER_DELAYED_FREE); mi_assert_internal(mi_page_heap(page) == NULL); @@ -1344,7 +1341,7 @@ static mi_segment_t* mi_segment_reclaim(mi_segment_t* segment, mi_heap_t* heap, _mi_stat_decrease(&tld->stats->pages_abandoned, 1); segment->abandoned--; // set the heap again and allow delayed free again - mi_page_set_heap(page, heap); + mi_page_set_heap(page, target_heap); _mi_page_use_delayed_free(page, MI_USE_DELAYED_FREE, true); // override never (after heap is set) _mi_page_free_collect(page, false); // ensure used count is up to date if (mi_page_all_free(page)) { @@ -1353,8 +1350,9 @@ static mi_segment_t* mi_segment_reclaim(mi_segment_t* segment, mi_heap_t* heap, } else { // otherwise reclaim it into the heap - _mi_page_reclaim(heap, page); - if (requested_block_size == page->xblock_size && mi_page_has_any_available(page)) { + _mi_page_reclaim(target_heap, page); + if (requested_block_size == page->xblock_size && mi_page_has_any_available(page) && + heap == target_heap) { if (right_page_reclaimed != NULL) { *right_page_reclaimed = true; } } } @@ -1381,7 +1379,7 @@ static mi_segment_t* mi_segment_reclaim(mi_segment_t* segment, mi_heap_t* heap, void _mi_abandoned_reclaim_all(mi_heap_t* heap, mi_segments_tld_t* tld) { mi_segment_t* segment; - while ((segment = mi_abandoned_pop()) != NULL) { + while ((segment = mi_abandoned_pop(tld->abandoned)) != NULL) { mi_segment_reclaim(segment, heap, 0, NULL, tld); } } @@ -1391,7 +1389,7 @@ static mi_segment_t* mi_segment_try_reclaim(mi_heap_t* heap, size_t needed_slice *reclaimed = false; mi_segment_t* segment; long max_tries = mi_option_get_clamp(mi_option_max_segment_reclaim, 8, 1024); // limit the work to bound allocation times - while ((max_tries-- > 0) && ((segment = mi_abandoned_pop()) != NULL)) { + while ((max_tries-- > 0) && ((segment = mi_abandoned_pop(tld->abandoned)) != NULL)) { segment->abandoned_visits++; // todo: an arena exclusive heap will potentially visit many abandoned unsuitable segments // and push them into the visited list and use many tries. Perhaps we can skip non-suitable ones in a better way? @@ -1418,7 +1416,7 @@ static mi_segment_t* mi_segment_try_reclaim(mi_heap_t* heap, size_t needed_slice else { // otherwise, push on the visited list so it gets not looked at too quickly again mi_segment_try_purge(segment, true /* force? */, tld->stats); // force purge if needed as we may not visit soon again - mi_abandoned_visited_push(segment); + mi_abandoned_visited_push(tld->abandoned, segment); } } return NULL; @@ -1428,11 +1426,12 @@ static mi_segment_t* mi_segment_try_reclaim(mi_heap_t* heap, size_t needed_slice void _mi_abandoned_collect(mi_heap_t* heap, bool force, mi_segments_tld_t* tld) { mi_segment_t* segment; + mi_abandoned_pool_t* pool = tld->abandoned; int max_tries = (force ? 16*1024 : 1024); // limit latency if (force) { - mi_abandoned_visited_revisit(); + mi_abandoned_visited_revisit(pool); } - while ((max_tries-- > 0) && ((segment = mi_abandoned_pop()) != NULL)) { + while ((max_tries-- > 0) && ((segment = mi_abandoned_pop(pool)) != NULL)) { mi_segment_check_free(segment,0,0,tld); // try to free up pages (due to concurrent frees) if (segment->used == 0) { // free the segment (by forced reclaim) to make it available to other threads. @@ -1444,7 +1443,7 @@ void _mi_abandoned_collect(mi_heap_t* heap, bool force, mi_segments_tld_t* tld) // otherwise, purge if needed and push on the visited list // note: forced purge can be expensive if many threads are destroyed/created as in mstress. mi_segment_try_purge(segment, force, tld->stats); - mi_abandoned_visited_push(segment); + mi_abandoned_visited_push(pool, segment); } } } @@ -1615,3 +1614,53 @@ mi_page_t* _mi_segment_page_alloc(mi_heap_t* heap, size_t block_size, size_t pag mi_assert_expensive(page == NULL || mi_segment_is_valid(_mi_page_segment(page),tld)); return page; } + +/* ----------------------------------------------------------- + Visit blocks in abandoned segments +----------------------------------------------------------- */ + +static bool mi_segment_visit_page(mi_segment_t* segment, mi_page_t* page, bool visit_blocks, mi_block_visit_fun* visitor, void* arg) +{ + mi_heap_area_t area; + _mi_heap_area_init(&area, page); + if (!visitor(NULL, &area, NULL, area.block_size, arg)) return false; + if (visit_blocks) { + return _mi_heap_area_visit_blocks(&area, page, visitor, arg); + } + else { + return true; + } +} + +static bool mi_segment_visit_pages(mi_segment_t* segment, uint8_t page_tag, bool visit_blocks, mi_block_visit_fun* visitor, void* arg) { + const mi_slice_t* end; + mi_slice_t* slice = mi_slices_start_iterate(segment, &end); + while (slice < end) { + if (mi_slice_is_used(slice)) { + mi_page_t* const page = mi_slice_to_page(slice); + if (page->tag == page_tag) { + if (!mi_segment_visit_page(segment, page, visit_blocks, visitor, arg)) return false; + } + } + slice = slice + slice->slice_count; + } + return true; +} + +// Visit all blocks in a abandoned segments +bool _mi_abandoned_pool_visit_blocks(mi_abandoned_pool_t* pool, uint8_t page_tag, bool visit_blocks, mi_block_visit_fun* visitor, void* arg) { + // Note: this is not safe in any other thread is abandoning or claiming segments from the pool + mi_segment_t* segment = mi_tagged_segment_ptr(pool->abandoned); + while (segment != NULL) { + if (!mi_segment_visit_pages(segment, page_tag, visit_blocks, visitor, arg)) return false; + segment = segment->abandoned_next; + } + + segment = pool->abandoned_visited; + while (segment != NULL) { + if (!mi_segment_visit_pages(segment, page_tag, visit_blocks, visitor, arg)) return false; + segment = segment->abandoned_next; + } + + return true; +} diff --git a/Objects/object.c b/Objects/object.c index d970a26756173b..275aa6713c8c21 100644 --- a/Objects/object.c +++ b/Objects/object.c @@ -2,6 +2,7 @@ /* Generic object operations; and implementation of None */ #include "Python.h" +#include "pycore_brc.h" // _Py_brc_queue_object() #include "pycore_call.h" // _PyObject_CallNoArgs() #include "pycore_ceval.h" // _Py_EnterRecursiveCallTstate() #include "pycore_context.h" // _PyContextTokenMissing_Type @@ -344,12 +345,10 @@ _Py_DecRefSharedDebug(PyObject *o, const char *filename, int lineno) &shared, new_shared)); if (should_queue) { - // TODO: the inter-thread queue is not yet implemented. For now, - // we just merge the refcount here. - Py_ssize_t refcount = _Py_ExplicitMergeRefcount(o, -1); - if (refcount == 0) { - _Py_Dealloc(o); - } +#ifdef Py_REF_DEBUG + _Py_IncRefTotal(_PyInterpreterState_GET()); +#endif + _Py_brc_queue_object(o); } else if (new_shared == _Py_REF_MERGED) { // refcount is zero AND merged @@ -399,10 +398,6 @@ _Py_ExplicitMergeRefcount(PyObject *op, Py_ssize_t extra) Py_ssize_t shared = _Py_atomic_load_ssize_relaxed(&op->ob_ref_shared); do { refcnt = Py_ARITHMETIC_RIGHT_SHIFT(Py_ssize_t, shared, _Py_REF_SHARED_SHIFT); - if (_Py_REF_IS_MERGED(shared)) { - return refcnt; - } - refcnt += (Py_ssize_t)op->ob_ref_local; refcnt += extra; @@ -410,6 +405,10 @@ _Py_ExplicitMergeRefcount(PyObject *op, Py_ssize_t extra) } while (!_Py_atomic_compare_exchange_ssize(&op->ob_ref_shared, &shared, new_shared)); +#ifdef Py_REF_DEBUG + _Py_AddRefTotal(_PyInterpreterState_GET(), extra); +#endif + _Py_atomic_store_uint32_relaxed(&op->ob_ref_local, 0); _Py_atomic_store_uintptr_relaxed(&op->ob_tid, 0); return refcnt; @@ -509,9 +508,7 @@ PyObject_CallFinalizerFromDealloc(PyObject *self) /* tp_finalize resurrected it! Make it look like the original Py_DECREF * never happened. */ - Py_ssize_t refcnt = Py_REFCNT(self); - _Py_NewReferenceNoTotal(self); - Py_SET_REFCNT(self, refcnt); + _Py_ResurrectReference(self); _PyObject_ASSERT(self, (!_PyType_IS_GC(Py_TYPE(self)) @@ -796,6 +793,21 @@ PyObject_Bytes(PyObject *v) return PyBytes_FromObject(v); } +void +_PyObject_ClearFreeLists(_PyFreeListState *state, int is_finalization) +{ + // In the free-threaded build, freelists are per-PyThreadState and cleared in PyThreadState_Clear() + // In the default build, freelists are per-interpreter and cleared in finalize_interp_types() + _PyFloat_ClearFreeList(state, is_finalization); + _PyTuple_ClearFreeList(state, is_finalization); + _PyList_ClearFreeList(state, is_finalization); + _PyDict_ClearFreeList(state, is_finalization); + _PyContext_ClearFreeList(state, is_finalization); + _PyAsyncGen_ClearFreeLists(state, is_finalization); + // Only be cleared if is_finalization is true. + _PyObjectStackChunk_ClearFreeList(state, is_finalization); + _PySlice_ClearFreeList(state, is_finalization); +} /* def _PyObject_FunctionStr(x): @@ -2389,6 +2401,17 @@ _Py_NewReferenceNoTotal(PyObject *op) new_reference(op); } +void +_Py_ResurrectReference(PyObject *op) +{ + if (_PyRuntime.tracemalloc.config.tracing) { + _PyTraceMalloc_NewReference(op); + } +#ifdef Py_TRACE_REFS + _Py_AddToAllObjects(op); +#endif +} + #ifdef Py_TRACE_REFS void @@ -2662,7 +2685,12 @@ _PyTrash_thread_deposit_object(struct _py_trashcan *trash, PyObject *op) _PyObject_ASSERT(op, _PyObject_IS_GC(op)); _PyObject_ASSERT(op, !_PyObject_GC_IS_TRACKED(op)); _PyObject_ASSERT(op, Py_REFCNT(op) == 0); +#ifdef Py_GIL_DISABLED + _PyObject_ASSERT(op, op->ob_tid == 0); + op->ob_tid = (uintptr_t)trash->delete_later; +#else _PyGCHead_SET_PREV(_Py_AS_GC(op), (PyGC_Head*)trash->delete_later); +#endif trash->delete_later = op; } @@ -2688,8 +2716,12 @@ _PyTrash_thread_destroy_chain(struct _py_trashcan *trash) PyObject *op = trash->delete_later; destructor dealloc = Py_TYPE(op)->tp_dealloc; - trash->delete_later = - (PyObject*) _PyGCHead_PREV(_Py_AS_GC(op)); +#ifdef Py_GIL_DISABLED + trash->delete_later = (PyObject*) op->ob_tid; + op->ob_tid = 0; +#else + trash->delete_later = (PyObject*) _PyGCHead_PREV(_Py_AS_GC(op)); +#endif /* Call the deallocator directly. This used to try to * fool Py_DECREF into calling it indirectly, but diff --git a/Objects/object_layout.md b/Objects/object_layout.md index 3f7d72eb22f224..4f379bed8d77e2 100644 --- a/Objects/object_layout.md +++ b/Objects/object_layout.md @@ -7,7 +7,7 @@ Each Python object starts with two fields: * ob_refcnt * ob_type -which the form the header common to all Python objects, for all versions, +which form the header common to all Python objects, for all versions, and hold the reference count and class of the object, respectively. ## Pre-header @@ -36,7 +36,7 @@ and the ``dict`` field points to the dictionary. ## 3.12 pre-header -In 3.12 the pointer to the list of weak references is added to the +In 3.12, the pointer to the list of weak references is added to the pre-header. In order to make space for it, the ``dict`` and ``values`` pointers are combined into a single tagged pointer: @@ -62,7 +62,7 @@ the values pointer, to enable the (legacy) C-API function * ob_refcnt * ob_type -For a "normal" Python object, that is one that doesn't inherit from a builtin +For a "normal" Python object, one that doesn't inherit from a builtin class or have slots, the header and pre-header form the entire object. ![Layout of "normal" object in 3.12](./object_layout_312.png) diff --git a/Objects/obmalloc.c b/Objects/obmalloc.c index 99c95d90658b08..6a12c3dca38b36 100644 --- a/Objects/obmalloc.c +++ b/Objects/obmalloc.c @@ -7,6 +7,7 @@ #include "pycore_pyerrors.h" // _Py_FatalErrorFormat() #include "pycore_pymem.h" #include "pycore_pystate.h" // _PyInterpreterState_GET +#include "pycore_obmalloc_init.h" #include // malloc() #include @@ -88,19 +89,37 @@ _PyMem_RawFree(void *Py_UNUSED(ctx), void *ptr) void * _PyMem_MiMalloc(void *ctx, size_t size) { +#ifdef Py_GIL_DISABLED + _PyThreadStateImpl *tstate = (_PyThreadStateImpl *)_PyThreadState_GET(); + mi_heap_t *heap = &tstate->mimalloc.heaps[_Py_MIMALLOC_HEAP_MEM]; + return mi_heap_malloc(heap, size); +#else return mi_malloc(size); +#endif } void * _PyMem_MiCalloc(void *ctx, size_t nelem, size_t elsize) { +#ifdef Py_GIL_DISABLED + _PyThreadStateImpl *tstate = (_PyThreadStateImpl *)_PyThreadState_GET(); + mi_heap_t *heap = &tstate->mimalloc.heaps[_Py_MIMALLOC_HEAP_MEM]; + return mi_heap_calloc(heap, nelem, elsize); +#else return mi_calloc(nelem, elsize); +#endif } void * _PyMem_MiRealloc(void *ctx, void *ptr, size_t size) { +#ifdef Py_GIL_DISABLED + _PyThreadStateImpl *tstate = (_PyThreadStateImpl *)_PyThreadState_GET(); + mi_heap_t *heap = &tstate->mimalloc.heaps[_Py_MIMALLOC_HEAP_MEM]; + return mi_heap_realloc(heap, ptr, size); +#else return mi_realloc(ptr, size); +#endif } void @@ -112,20 +131,38 @@ _PyMem_MiFree(void *ctx, void *ptr) void * _PyObject_MiMalloc(void *ctx, size_t nbytes) { +#ifdef Py_GIL_DISABLED + _PyThreadStateImpl *tstate = (_PyThreadStateImpl *)_PyThreadState_GET(); + mi_heap_t *heap = tstate->mimalloc.current_object_heap; + return mi_heap_malloc(heap, nbytes); +#else return mi_malloc(nbytes); +#endif } void * _PyObject_MiCalloc(void *ctx, size_t nelem, size_t elsize) { +#ifdef Py_GIL_DISABLED + _PyThreadStateImpl *tstate = (_PyThreadStateImpl *)_PyThreadState_GET(); + mi_heap_t *heap = tstate->mimalloc.current_object_heap; + return mi_heap_calloc(heap, nelem, elsize); +#else return mi_calloc(nelem, elsize); +#endif } void * _PyObject_MiRealloc(void *ctx, void *ptr, size_t nbytes) { +#ifdef Py_GIL_DISABLED + _PyThreadStateImpl *tstate = (_PyThreadStateImpl *)_PyThreadState_GET(); + mi_heap_t *heap = tstate->mimalloc.current_object_heap; + return mi_heap_realloc(heap, ptr, nbytes); +#else return mi_realloc(ptr, nbytes); +#endif } void @@ -403,12 +440,14 @@ set_up_allocators_unlocked(PyMemAllocatorName allocator) (void)set_default_allocator_unlocked(PYMEM_DOMAIN_RAW, pydebug, NULL); (void)set_default_allocator_unlocked(PYMEM_DOMAIN_MEM, pydebug, NULL); (void)set_default_allocator_unlocked(PYMEM_DOMAIN_OBJ, pydebug, NULL); + _PyRuntime.allocators.is_debug_enabled = pydebug; break; case PYMEM_ALLOCATOR_DEBUG: (void)set_default_allocator_unlocked(PYMEM_DOMAIN_RAW, 1, NULL); (void)set_default_allocator_unlocked(PYMEM_DOMAIN_MEM, 1, NULL); (void)set_default_allocator_unlocked(PYMEM_DOMAIN_OBJ, 1, NULL); + _PyRuntime.allocators.is_debug_enabled = 1; break; #ifdef WITH_PYMALLOC @@ -422,7 +461,9 @@ set_up_allocators_unlocked(PyMemAllocatorName allocator) set_allocator_unlocked(PYMEM_DOMAIN_MEM, &pymalloc); set_allocator_unlocked(PYMEM_DOMAIN_OBJ, &pymalloc); - if (allocator == PYMEM_ALLOCATOR_PYMALLOC_DEBUG) { + int is_debug = (allocator == PYMEM_ALLOCATOR_PYMALLOC_DEBUG); + _PyRuntime.allocators.is_debug_enabled = is_debug; + if (is_debug) { set_up_debug_hooks_unlocked(); } break; @@ -441,7 +482,9 @@ set_up_allocators_unlocked(PyMemAllocatorName allocator) PyMemAllocatorEx objmalloc = MIMALLOC_OBJALLOC; set_allocator_unlocked(PYMEM_DOMAIN_OBJ, &objmalloc); - if (allocator == PYMEM_ALLOCATOR_MIMALLOC_DEBUG) { + int is_debug = (allocator == PYMEM_ALLOCATOR_MIMALLOC_DEBUG); + _PyRuntime.allocators.is_debug_enabled = is_debug; + if (is_debug) { set_up_debug_hooks_unlocked(); } @@ -457,7 +500,9 @@ set_up_allocators_unlocked(PyMemAllocatorName allocator) set_allocator_unlocked(PYMEM_DOMAIN_MEM, &malloc_alloc); set_allocator_unlocked(PYMEM_DOMAIN_OBJ, &malloc_alloc); - if (allocator == PYMEM_ALLOCATOR_MALLOC_DEBUG) { + int is_debug = (allocator == PYMEM_ALLOCATOR_MALLOC_DEBUG); + _PyRuntime.allocators.is_debug_enabled = is_debug; + if (is_debug) { set_up_debug_hooks_unlocked(); } break; @@ -568,13 +613,13 @@ _PyMem_GetCurrentAllocatorName(void) } -#ifdef WITH_PYMALLOC -static int +int _PyMem_DebugEnabled(void) { - return (_PyObject.malloc == _PyMem_DebugMalloc); + return _PyRuntime.allocators.is_debug_enabled; } +#ifdef WITH_PYMALLOC static int _PyMem_PymallocEnabled(void) { @@ -590,12 +635,16 @@ _PyMem_PymallocEnabled(void) static int _PyMem_MimallocEnabled(void) { +#ifdef Py_GIL_DISABLED + return 1; +#else if (_PyMem_DebugEnabled()) { return (_PyMem_Debug.obj.alloc.malloc == _PyObject_MiMalloc); } else { return (_PyObject.malloc == _PyObject_MiMalloc); } +#endif } #endif // WITH_MIMALLOC @@ -655,6 +704,7 @@ set_up_debug_hooks_unlocked(void) set_up_debug_hooks_domain_unlocked(PYMEM_DOMAIN_RAW); set_up_debug_hooks_domain_unlocked(PYMEM_DOMAIN_MEM); set_up_debug_hooks_domain_unlocked(PYMEM_DOMAIN_OBJ); + _PyRuntime.allocators.is_debug_enabled = 1; } void @@ -967,6 +1017,13 @@ static int running_on_valgrind = -1; typedef struct _obmalloc_state OMState; +/* obmalloc state for main interpreter and shared by all interpreters without + * their own obmalloc state. By not explicitly initalizing this structure, it + * will be allocated in the BSS which is a small performance win. The radix + * tree arrays are fairly large but are sparsely used. */ +static struct _obmalloc_state obmalloc_state_main; +static bool obmalloc_state_initialized; + static inline int has_own_state(PyInterpreterState *interp) { @@ -979,10 +1036,8 @@ static inline OMState * get_state(void) { PyInterpreterState *interp = _PyInterpreterState_GET(); - if (!has_own_state(interp)) { - interp = _PyInterpreterState_Main(); - } - return &interp->obmalloc; + assert(interp->obmalloc != NULL); // otherwise not initialized or freed + return interp->obmalloc; } // These macros all rely on a local "state" variable. @@ -1005,20 +1060,40 @@ static bool count_blocks( *(size_t *)allocated_blocks += area->used; return 1; } + +static Py_ssize_t +get_mimalloc_allocated_blocks(PyInterpreterState *interp) +{ + size_t allocated_blocks = 0; +#ifdef Py_GIL_DISABLED + for (PyThreadState *t = interp->threads.head; t != NULL; t = t->next) { + _PyThreadStateImpl *tstate = (_PyThreadStateImpl *)t; + for (int i = 0; i < _Py_MIMALLOC_HEAP_COUNT; i++) { + mi_heap_t *heap = &tstate->mimalloc.heaps[i]; + mi_heap_visit_blocks(heap, false, &count_blocks, &allocated_blocks); + } + } + + mi_abandoned_pool_t *pool = &interp->mimalloc.abandoned_pool; + for (uint8_t tag = 0; tag < _Py_MIMALLOC_HEAP_COUNT; tag++) { + _mi_abandoned_pool_visit_blocks(pool, tag, false, &count_blocks, + &allocated_blocks); + } +#else + // TODO(sgross): this only counts the current thread's blocks. + mi_heap_t *heap = mi_heap_get_default(); + mi_heap_visit_blocks(heap, false, &count_blocks, &allocated_blocks); +#endif + return allocated_blocks; +} #endif Py_ssize_t _PyInterpreterState_GetAllocatedBlocks(PyInterpreterState *interp) { #ifdef WITH_MIMALLOC - // TODO(sgross): this only counts the current thread's blocks. if (_PyMem_MimallocEnabled()) { - size_t allocated_blocks = 0; - - mi_heap_t *heap = mi_heap_get_default(); - mi_heap_visit_blocks(heap, false, &count_blocks, &allocated_blocks); - - return allocated_blocks; + return get_mimalloc_allocated_blocks(interp); } #endif @@ -1030,7 +1105,11 @@ _PyInterpreterState_GetAllocatedBlocks(PyInterpreterState *interp) "the interpreter doesn't have its own allocator"); } #endif - OMState *state = &interp->obmalloc; + OMState *state = interp->obmalloc; + + if (state == NULL) { + return 0; + } Py_ssize_t n = raw_allocated_blocks; /* add up allocated blocks for used pools */ @@ -1052,6 +1131,8 @@ _PyInterpreterState_GetAllocatedBlocks(PyInterpreterState *interp) return n; } +static void free_obmalloc_arenas(PyInterpreterState *interp); + void _PyInterpreterState_FinalizeAllocatedBlocks(PyInterpreterState *interp) { @@ -1060,16 +1141,26 @@ _PyInterpreterState_FinalizeAllocatedBlocks(PyInterpreterState *interp) return; } #endif - if (has_own_state(interp)) { + if (has_own_state(interp) && interp->obmalloc != NULL) { Py_ssize_t leaked = _PyInterpreterState_GetAllocatedBlocks(interp); assert(has_own_state(interp) || leaked == 0); interp->runtime->obmalloc.interpreter_leaks += leaked; + if (_PyMem_obmalloc_state_on_heap(interp) && leaked == 0) { + // free the obmalloc arenas and radix tree nodes. If leaked > 0 + // then some of the memory allocated by obmalloc has not been + // freed. It might be safe to free the arenas in that case but + // it's possible that extension modules are still using that + // memory. So, it is safer to not free and to leak. Perhaps there + // should be warning when this happens. It should be possible to + // use a tool like "-fsanitize=address" to track down these leaks. + free_obmalloc_arenas(interp); + } } } static Py_ssize_t get_num_global_allocated_blocks(_PyRuntimeState *); -/* We preserve the number of blockss leaked during runtime finalization, +/* We preserve the number of blocks leaked during runtime finalization, so they can be reported if the runtime is initialized again. */ // XXX We don't lose any information by dropping this, // so we should consider doing so. @@ -1085,16 +1176,6 @@ _Py_FinalizeAllocatedBlocks(_PyRuntimeState *runtime) static Py_ssize_t get_num_global_allocated_blocks(_PyRuntimeState *runtime) { -#ifdef WITH_MIMALLOC - if (_PyMem_MimallocEnabled()) { - size_t allocated_blocks = 0; - - mi_heap_t *heap = mi_heap_get_default(); - mi_heap_visit_blocks(heap, false, &count_blocks, &allocated_blocks); - - return allocated_blocks; - } -#endif Py_ssize_t total = 0; if (_PyRuntimeState_GetFinalizing(runtime) != NULL) { PyInterpreterState *interp = _PyInterpreterState_Main(); @@ -1113,6 +1194,7 @@ get_num_global_allocated_blocks(_PyRuntimeState *runtime) } } else { + _PyEval_StopTheWorldAll(&_PyRuntime); HEAD_LOCK(runtime); PyInterpreterState *interp = PyInterpreterState_Head(); assert(interp != NULL); @@ -1132,6 +1214,7 @@ get_num_global_allocated_blocks(_PyRuntimeState *runtime) } } HEAD_UNLOCK(runtime); + _PyEval_StartTheWorldAll(&_PyRuntime); #ifdef Py_DEBUG assert(got_main); #endif @@ -2663,9 +2746,96 @@ _PyDebugAllocatorStats(FILE *out, (void)printone(out, buf2, num_blocks * sizeof_block); } +// Return true if the obmalloc state structure is heap allocated, +// by PyMem_RawCalloc(). For the main interpreter, this structure +// allocated in the BSS. Allocating that way gives some memory savings +// and a small performance win (at least on a demand paged OS). On +// 64-bit platforms, the obmalloc structure is 256 kB. Most of that +// memory is for the arena_map_top array. Since normally only one entry +// of that array is used, only one page of resident memory is actually +// used, rather than the full 256 kB. +bool _PyMem_obmalloc_state_on_heap(PyInterpreterState *interp) +{ +#if WITH_PYMALLOC + return interp->obmalloc && interp->obmalloc != &obmalloc_state_main; +#else + return false; +#endif +} + +#ifdef WITH_PYMALLOC +static void +init_obmalloc_pools(PyInterpreterState *interp) +{ + // initialize the obmalloc->pools structure. This must be done + // before the obmalloc alloc/free functions can be called. + poolp temp[OBMALLOC_USED_POOLS_SIZE] = + _obmalloc_pools_INIT(interp->obmalloc->pools); + memcpy(&interp->obmalloc->pools.used, temp, sizeof(temp)); +} +#endif /* WITH_PYMALLOC */ + +int _PyMem_init_obmalloc(PyInterpreterState *interp) +{ +#ifdef WITH_PYMALLOC + /* Initialize obmalloc, but only for subinterpreters, + since the main interpreter is initialized statically. */ + if (_Py_IsMainInterpreter(interp) + || _PyInterpreterState_HasFeature(interp, + Py_RTFLAGS_USE_MAIN_OBMALLOC)) { + interp->obmalloc = &obmalloc_state_main; + if (!obmalloc_state_initialized) { + init_obmalloc_pools(interp); + obmalloc_state_initialized = true; + } + } else { + interp->obmalloc = PyMem_RawCalloc(1, sizeof(struct _obmalloc_state)); + if (interp->obmalloc == NULL) { + return -1; + } + init_obmalloc_pools(interp); + } +#endif /* WITH_PYMALLOC */ + return 0; // success +} + #ifdef WITH_PYMALLOC +static void +free_obmalloc_arenas(PyInterpreterState *interp) +{ + OMState *state = interp->obmalloc; + for (uint i = 0; i < maxarenas; ++i) { + // free each obmalloc memory arena + struct arena_object *ao = &allarenas[i]; + _PyObject_Arena.free(_PyObject_Arena.ctx, + (void *)ao->address, ARENA_SIZE); + } + // free the array containing pointers to all arenas + PyMem_RawFree(allarenas); +#if WITH_PYMALLOC_RADIX_TREE +#ifdef USE_INTERIOR_NODES + // Free the middle and bottom nodes of the radix tree. These are allocated + // by arena_map_mark_used() but not freed when arenas are freed. + for (int i1 = 0; i1 < MAP_TOP_LENGTH; i1++) { + arena_map_mid_t *mid = arena_map_root.ptrs[i1]; + if (mid == NULL) { + continue; + } + for (int i2 = 0; i2 < MAP_MID_LENGTH; i2++) { + arena_map_bot_t *bot = arena_map_root.ptrs[i1]->ptrs[i2]; + if (bot == NULL) { + continue; + } + PyMem_RawFree(bot); + } + PyMem_RawFree(mid); + } +#endif +#endif +} + #ifdef Py_DEBUG /* Is target in the list? The list is traversed via the nextpool pointers. * The list may be NULL-terminated, or circular. Return 1 if target is in diff --git a/Objects/odictobject.c b/Objects/odictobject.c index b5280c39e1be54..421bc52992d735 100644 --- a/Objects/odictobject.c +++ b/Objects/odictobject.c @@ -465,12 +465,13 @@ Potential Optimizations */ #include "Python.h" -#include "pycore_call.h" // _PyObject_CallNoArgs() -#include "pycore_ceval.h" // _PyEval_GetBuiltin() -#include "pycore_dict.h" // _Py_dict_lookup() -#include "pycore_object.h" // _PyObject_GC_UNTRACK() -#include "pycore_pyerrors.h" // _PyErr_ChainExceptions1() -#include // offsetof() +#include "pycore_call.h" // _PyObject_CallNoArgs() +#include "pycore_ceval.h" // _PyEval_GetBuiltin() +#include "pycore_critical_section.h" //_Py_BEGIN_CRITICAL_SECTION +#include "pycore_dict.h" // _Py_dict_lookup() +#include "pycore_object.h" // _PyObject_GC_UNTRACK() +#include "pycore_pyerrors.h" // _PyErr_ChainExceptions1() +#include // offsetof() #include "clinic/odictobject.c.h" @@ -1039,6 +1040,8 @@ _odict_popkey_hash(PyObject *od, PyObject *key, PyObject *failobj, { PyObject *value = NULL; + Py_BEGIN_CRITICAL_SECTION(od); + _ODictNode *node = _odict_find_node_hash((PyODictObject *)od, key, hash); if (node != NULL) { /* Pop the node first to avoid a possible dict resize (due to @@ -1046,7 +1049,7 @@ _odict_popkey_hash(PyObject *od, PyObject *key, PyObject *failobj, resolution. */ int res = _odict_clear_node((PyODictObject *)od, node, key, hash); if (res < 0) { - return NULL; + goto done; } /* Now delete the value from the dict. */ if (_PyDict_Pop_KnownHash((PyDictObject *)od, key, hash, @@ -1063,6 +1066,8 @@ _odict_popkey_hash(PyObject *od, PyObject *key, PyObject *failobj, PyErr_SetObject(PyExc_KeyError, key); } } + Py_END_CRITICAL_SECTION(); +done: return value; } diff --git a/Objects/setobject.c b/Objects/setobject.c index 88d20019bfb4a7..6a4c8c45f0836d 100644 --- a/Objects/setobject.c +++ b/Objects/setobject.c @@ -32,13 +32,27 @@ */ #include "Python.h" -#include "pycore_ceval.h" // _PyEval_GetBuiltin() -#include "pycore_dict.h" // _PyDict_Contains_KnownHash() -#include "pycore_modsupport.h" // _PyArg_NoKwnames() -#include "pycore_object.h" // _PyObject_GC_UNTRACK() -#include "pycore_pyerrors.h" // _PyErr_SetKeyError() -#include "pycore_setobject.h" // _PySet_NextEntry() definition -#include // offsetof() +#include "pycore_ceval.h" // _PyEval_GetBuiltin() +#include "pycore_critical_section.h" // Py_BEGIN_CRITICAL_SECTION, Py_END_CRITICAL_SECTION +#include "pycore_dict.h" // _PyDict_Contains_KnownHash() +#include "pycore_modsupport.h" // _PyArg_NoKwnames() +#include "pycore_object.h" // _PyObject_GC_UNTRACK() +#include "pycore_pyerrors.h" // _PyErr_SetKeyError() +#include "pycore_setobject.h" // _PySet_NextEntry() definition +#include // offsetof() +#include "clinic/setobject.c.h" + +/*[clinic input] +class set "PySetObject *" "&PySet_Type" +class frozenset "PySetObject *" "&PyFrozenSet_Type" +[clinic start generated code]*/ +/*[clinic end generated code: output=da39a3ee5e6b4b0d input=97ad1d3e9f117079]*/ + +/*[python input] +class setobject_converter(self_converter): + type = "PySetObject *" +[python start generated code]*/ +/*[python end generated code: output=da39a3ee5e6b4b0d input=33a44506d4d57793]*/ /* Object used as dummy key to fill deleted entries */ static PyObject _dummy_struct; @@ -630,8 +644,18 @@ set_merge(PySetObject *so, PyObject *otherset) return 0; } +/*[clinic input] +set.pop + so: setobject + +Remove and return an arbitrary set element. + +Raises KeyError if the set is empty. +[clinic start generated code]*/ + static PyObject * -set_pop(PySetObject *so, PyObject *Py_UNUSED(ignored)) +set_pop_impl(PySetObject *so) +/*[clinic end generated code: output=4d65180f1271871b input=4a3f5552e660a260]*/ { /* Make sure the search finger is in bounds */ setentry *entry = so->table + (so->finger & so->mask); @@ -655,9 +679,6 @@ set_pop(PySetObject *so, PyObject *Py_UNUSED(ignored)) return key; } -PyDoc_STRVAR(pop_doc, "Remove and return an arbitrary set element.\n\ -Raises KeyError if the set is empty."); - static int set_traverse(PySetObject *so, visitproc visit, void *arg) { @@ -903,11 +924,17 @@ set_update_internal(PySetObject *so, PyObject *other) if (set_table_resize(so, (so->used + dictsize)*2) != 0) return -1; } + int err = 0; + Py_BEGIN_CRITICAL_SECTION(other); while (_PyDict_Next(other, &pos, &key, &value, &hash)) { - if (set_add_entry(so, key, hash)) - return -1; + if (set_add_entry(so, key, hash)) { + err = -1; + goto exit; + } } - return 0; +exit: + Py_END_CRITICAL_SECTION(); + return err; } it = PyObject_GetIter(other); @@ -928,8 +955,18 @@ set_update_internal(PySetObject *so, PyObject *other) return 0; } +/*[clinic input] +set.update + so: setobject + *others as args: object + / + +Update the set, adding elements from all others. +[clinic start generated code]*/ + static PyObject * -set_update(PySetObject *so, PyObject *args) +set_update_impl(PySetObject *so, PyObject *args) +/*[clinic end generated code: output=34f6371704974c8a input=eb47c4fbaeb3286e]*/ { Py_ssize_t i; @@ -941,12 +978,6 @@ set_update(PySetObject *so, PyObject *args) Py_RETURN_NONE; } -PyDoc_STRVAR(update_doc, -"update($self, /, *others)\n\ ---\n\ -\n\ -Update the set, adding elements from all others."); - /* XXX Todo: If aligned memory allocations become available, make the set object 64 byte aligned so that most of the fields @@ -1094,14 +1125,30 @@ set_swap_bodies(PySetObject *a, PySetObject *b) } } +/*[clinic input] +set.copy + so: setobject + +Return a shallow copy of a set. +[clinic start generated code]*/ + static PyObject * -set_copy(PySetObject *so, PyObject *Py_UNUSED(ignored)) +set_copy_impl(PySetObject *so) +/*[clinic end generated code: output=c9223a1e1cc6b041 input=2b80b288d47b8cf1]*/ { return make_new_set_basetype(Py_TYPE(so), (PyObject *)so); } +/*[clinic input] +frozenset.copy + so: setobject + +Return a shallow copy of a set. +[clinic start generated code]*/ + static PyObject * -frozenset_copy(PySetObject *so, PyObject *Py_UNUSED(ignored)) +frozenset_copy_impl(PySetObject *so) +/*[clinic end generated code: output=b356263526af9e70 input=3dc65577d344eff7]*/ { if (PyFrozenSet_CheckExact(so)) { return Py_NewRef(so); @@ -1109,19 +1156,33 @@ frozenset_copy(PySetObject *so, PyObject *Py_UNUSED(ignored)) return set_copy(so, NULL); } -PyDoc_STRVAR(copy_doc, "Return a shallow copy of a set."); +/*[clinic input] +set.clear + so: setobject + +Remove all elements from this set. +[clinic start generated code]*/ static PyObject * -set_clear(PySetObject *so, PyObject *Py_UNUSED(ignored)) +set_clear_impl(PySetObject *so) +/*[clinic end generated code: output=4e71d5a83904161a input=74ac19794da81a39]*/ { set_clear_internal(so); Py_RETURN_NONE; } -PyDoc_STRVAR(clear_doc, "Remove all elements from this set."); +/*[clinic input] +set.union + so: setobject + *others as args: object + / + +Return a new set with elements from the set and all others. +[clinic start generated code]*/ static PyObject * -set_union(PySetObject *so, PyObject *args) +set_union_impl(PySetObject *so, PyObject *args) +/*[clinic end generated code: output=2c83d05a446a1477 input=2e2024fa1e40ac84]*/ { PySetObject *result; PyObject *other; @@ -1143,12 +1204,6 @@ set_union(PySetObject *so, PyObject *args) return (PyObject *)result; } -PyDoc_STRVAR(union_doc, -"union($self, /, *others)\n\ ---\n\ -\n\ -Return a new set with elements from the set and all others."); - static PyObject * set_or(PySetObject *so, PyObject *other) { @@ -1263,8 +1318,18 @@ set_intersection(PySetObject *so, PyObject *other) return NULL; } +/*[clinic input] +set.intersection as set_intersection_multi + so: setobject + *others as args: object + / + +Return a new set with elements common to the set and all others. +[clinic start generated code]*/ + static PyObject * -set_intersection_multi(PySetObject *so, PyObject *args) +set_intersection_multi_impl(PySetObject *so, PyObject *args) +/*[clinic end generated code: output=2406ef3387adbe2f input=04108ea6d7f0532b]*/ { Py_ssize_t i; @@ -1284,12 +1349,6 @@ set_intersection_multi(PySetObject *so, PyObject *args) return result; } -PyDoc_STRVAR(intersection_doc, -"intersection($self, /, *others)\n\ ---\n\ -\n\ -Return a new set with elements common to the set and all others."); - static PyObject * set_intersection_update(PySetObject *so, PyObject *other) { @@ -1303,12 +1362,22 @@ set_intersection_update(PySetObject *so, PyObject *other) Py_RETURN_NONE; } +/*[clinic input] +set.intersection_update as set_intersection_update_multi + so: setobject + *others as args: object + / + +Update the set, keeping only elements found in it and all others. +[clinic start generated code]*/ + static PyObject * -set_intersection_update_multi(PySetObject *so, PyObject *args) +set_intersection_update_multi_impl(PySetObject *so, PyObject *args) +/*[clinic end generated code: output=251c1f729063609d input=ff8f119f97458d16]*/ { PyObject *tmp; - tmp = set_intersection_multi(so, args); + tmp = set_intersection_multi_impl(so, args); if (tmp == NULL) return NULL; set_swap_bodies(so, (PySetObject *)tmp); @@ -1316,12 +1385,6 @@ set_intersection_update_multi(PySetObject *so, PyObject *args) Py_RETURN_NONE; } -PyDoc_STRVAR(intersection_update_doc, -"intersection_update($self, /, *others)\n\ ---\n\ -\n\ -Update the set, keeping only elements found in it and all others."); - static PyObject * set_and(PySetObject *so, PyObject *other) { @@ -1344,8 +1407,18 @@ set_iand(PySetObject *so, PyObject *other) return Py_NewRef(so); } +/*[clinic input] +set.isdisjoint + so: setobject + other: object + / + +Return True if two sets have a null intersection. +[clinic start generated code]*/ + static PyObject * set_isdisjoint(PySetObject *so, PyObject *other) +/*[clinic end generated code: output=a92bbf9a2db6a3da input=c254ddec8a2326e3]*/ { PyObject *key, *it, *tmp; int rv; @@ -1403,9 +1476,6 @@ set_isdisjoint(PySetObject *so, PyObject *other) Py_RETURN_TRUE; } -PyDoc_STRVAR(isdisjoint_doc, -"Return True if two sets have a null intersection."); - static int set_difference_update_internal(PySetObject *so, PyObject *other) { @@ -1464,8 +1534,18 @@ set_difference_update_internal(PySetObject *so, PyObject *other) return set_table_resize(so, so->used>50000 ? so->used*2 : so->used*4); } +/*[clinic input] +set.difference_update + so: setobject + *others as args: object + / + +Update the set, removing elements found in others. +[clinic start generated code]*/ + static PyObject * -set_difference_update(PySetObject *so, PyObject *args) +set_difference_update_impl(PySetObject *so, PyObject *args) +/*[clinic end generated code: output=28685b2fc63e41c4 input=e7abb43c9f2c5a73]*/ { Py_ssize_t i; @@ -1477,12 +1557,6 @@ set_difference_update(PySetObject *so, PyObject *args) Py_RETURN_NONE; } -PyDoc_STRVAR(difference_update_doc, -"difference_update($self, /, *others)\n\ ---\n\ -\n\ -Update the set, removing elements found in others."); - static PyObject * set_copy_and_difference(PySetObject *so, PyObject *other) { @@ -1573,8 +1647,18 @@ set_difference(PySetObject *so, PyObject *other) return result; } +/*[clinic input] +set.difference as set_difference_multi + so: setobject + *others as args: object + / + +Return a new set with elements in the set that are not in the others. +[clinic start generated code]*/ + static PyObject * -set_difference_multi(PySetObject *so, PyObject *args) +set_difference_multi_impl(PySetObject *so, PyObject *args) +/*[clinic end generated code: output=3130c3bb3cac873d input=d8ae9bb6d518ab95]*/ { Py_ssize_t i; PyObject *result, *other; @@ -1597,11 +1681,6 @@ set_difference_multi(PySetObject *so, PyObject *args) return result; } -PyDoc_STRVAR(difference_doc, -"difference($self, /, *others)\n\ ---\n\ -\n\ -Return a new set with elements in the set that are not in the others."); static PyObject * set_sub(PySetObject *so, PyObject *other) { @@ -1620,8 +1699,45 @@ set_isub(PySetObject *so, PyObject *other) return Py_NewRef(so); } +static PyObject * +set_symmetric_difference_update_dict(PySetObject *so, PyObject *other) +{ + PyObject *key; + Py_ssize_t pos = 0; + Py_hash_t hash; + PyObject *value; + int rv; + + while (_PyDict_Next(other, &pos, &key, &value, &hash)) { + Py_INCREF(key); + rv = set_discard_entry(so, key, hash); + if (rv < 0) { + Py_DECREF(key); + return NULL; + } + if (rv == DISCARD_NOTFOUND) { + if (set_add_entry(so, key, hash)) { + Py_DECREF(key); + return NULL; + } + } + Py_DECREF(key); + } + Py_RETURN_NONE; +} + +/*[clinic input] +set.symmetric_difference_update + so: setobject + other: object + / + +Update the set, keeping only elements found in either set, but not in both. +[clinic start generated code]*/ + static PyObject * set_symmetric_difference_update(PySetObject *so, PyObject *other) +/*[clinic end generated code: output=fbb049c0806028de input=a50acf0365e1f0a5]*/ { PySetObject *otherset; PyObject *key; @@ -1634,23 +1750,13 @@ set_symmetric_difference_update(PySetObject *so, PyObject *other) return set_clear(so, NULL); if (PyDict_CheckExact(other)) { - PyObject *value; - while (_PyDict_Next(other, &pos, &key, &value, &hash)) { - Py_INCREF(key); - rv = set_discard_entry(so, key, hash); - if (rv < 0) { - Py_DECREF(key); - return NULL; - } - if (rv == DISCARD_NOTFOUND) { - if (set_add_entry(so, key, hash)) { - Py_DECREF(key); - return NULL; - } - } - Py_DECREF(key); - } - Py_RETURN_NONE; + PyObject *res; + + Py_BEGIN_CRITICAL_SECTION(other); + res = set_symmetric_difference_update_dict(so, other); + Py_END_CRITICAL_SECTION(); + + return res; } if (PyAnySet_Check(other)) { @@ -1684,14 +1790,18 @@ set_symmetric_difference_update(PySetObject *so, PyObject *other) Py_RETURN_NONE; } -PyDoc_STRVAR(symmetric_difference_update_doc, -"symmetric_difference_update($self, other, /)\n\ ---\n\ -\n\ -Update the set, keeping only elements found in either set, but not in both."); +/*[clinic input] +set.symmetric_difference + so: setobject + other: object + / + +Return a new set with elements in either the set or other but not both. +[clinic start generated code]*/ static PyObject * set_symmetric_difference(PySetObject *so, PyObject *other) +/*[clinic end generated code: output=f95364211b88775a input=f18af370ad72ebac]*/ { PyObject *rv; PySetObject *otherset; @@ -1708,12 +1818,6 @@ set_symmetric_difference(PySetObject *so, PyObject *other) return (PyObject *)otherset; } -PyDoc_STRVAR(symmetric_difference_doc, -"symmetric_difference($self, other, /)\n\ ---\n\ -\n\ -Return a new set with elements in either the set or other but not both."); - static PyObject * set_xor(PySetObject *so, PyObject *other) { @@ -1736,8 +1840,18 @@ set_ixor(PySetObject *so, PyObject *other) return Py_NewRef(so); } +/*[clinic input] +set.issubset + so: setobject + other: object + / + +Report whether another set contains this set. +[clinic start generated code]*/ + static PyObject * set_issubset(PySetObject *so, PyObject *other) +/*[clinic end generated code: output=78aef1f377aedef1 input=37fbc579b609db0c]*/ { setentry *entry; Py_ssize_t pos = 0; @@ -1770,10 +1884,18 @@ set_issubset(PySetObject *so, PyObject *other) Py_RETURN_TRUE; } -PyDoc_STRVAR(issubset_doc, "Report whether another set contains this set."); +/*[clinic input] +set.issuperset + so: setobject + other: object + / + +Report whether this set contains another set. +[clinic start generated code]*/ static PyObject * set_issuperset(PySetObject *so, PyObject *other) +/*[clinic end generated code: output=7d2b71dd714a7ec7 input=fd5dab052f2e9bb3]*/ { if (PyAnySet_Check(other)) { return set_issubset((PySetObject *)other, (PyObject *)so); @@ -1802,8 +1924,6 @@ set_issuperset(PySetObject *so, PyObject *other) Py_RETURN_TRUE; } -PyDoc_STRVAR(issuperset_doc, "Report whether this set contains another set."); - static PyObject * set_richcompare(PySetObject *v, PyObject *w, int op) { @@ -1847,19 +1967,26 @@ set_richcompare(PySetObject *v, PyObject *w, int op) Py_RETURN_NOTIMPLEMENTED; } +/*[clinic input] +set.add + so: setobject + object as key: object + / + +Add an element to a set. + +This has no effect if the element is already present. +[clinic start generated code]*/ + static PyObject * set_add(PySetObject *so, PyObject *key) +/*[clinic end generated code: output=cd9c2d5c2069c2ba input=96f1efe029e47972]*/ { if (set_add_key(so, key)) return NULL; Py_RETURN_NONE; } -PyDoc_STRVAR(add_doc, -"Add an element to a set.\n\ -\n\ -This has no effect if the element is already present."); - static int set_contains(PySetObject *so, PyObject *key) { @@ -1880,8 +2007,19 @@ set_contains(PySetObject *so, PyObject *key) return rv; } +/*[clinic input] +@coexist +set.__contains__ + so: setobject + object as key: object + / + +x.__contains__(y) <==> y in x. +[clinic start generated code]*/ + static PyObject * -set_direct_contains(PySetObject *so, PyObject *key) +set___contains__(PySetObject *so, PyObject *key) +/*[clinic end generated code: output=b5948bc5c590d3ca input=cf4c72db704e4cf0]*/ { long result; @@ -1891,10 +2029,20 @@ set_direct_contains(PySetObject *so, PyObject *key) return PyBool_FromLong(result); } -PyDoc_STRVAR(contains_doc, "x.__contains__(y) <==> y in x."); +/*[clinic input] +set.remove + so: setobject + object as key: object + / + +Remove an element from a set; it must be a member. + +If the element is not a member, raise a KeyError. +[clinic start generated code]*/ static PyObject * set_remove(PySetObject *so, PyObject *key) +/*[clinic end generated code: output=08ae496d0cd2b8c1 input=10132515dfe8ebd7]*/ { PyObject *tmpkey; int rv; @@ -1920,13 +2068,21 @@ set_remove(PySetObject *so, PyObject *key) Py_RETURN_NONE; } -PyDoc_STRVAR(remove_doc, -"Remove an element from a set; it must be a member.\n\ -\n\ -If the element is not a member, raise a KeyError."); +/*[clinic input] +set.discard + so: setobject + object as key: object + / + +Remove an element from a set if it is a member. + +Unlike set.remove(), the discard() method does not raise +an exception when an element is missing from the set. +[clinic start generated code]*/ static PyObject * set_discard(PySetObject *so, PyObject *key) +/*[clinic end generated code: output=9181b60d7bb7d480 input=82a689eba94d5ad9]*/ { PyObject *tmpkey; int rv; @@ -1947,14 +2103,16 @@ set_discard(PySetObject *so, PyObject *key) Py_RETURN_NONE; } -PyDoc_STRVAR(discard_doc, -"Remove an element from a set if it is a member.\n\ -\n\ -Unlike set.remove(), the discard() method does not raise\n\ -an exception when an element is missing from the set."); +/*[clinic input] +set.__reduce__ + so: setobject + +Return state information for pickling. +[clinic start generated code]*/ static PyObject * -set_reduce(PySetObject *so, PyObject *Py_UNUSED(ignored)) +set___reduce___impl(PySetObject *so) +/*[clinic end generated code: output=9af7d0e029df87ee input=531375e87a24a449]*/ { PyObject *keys=NULL, *args=NULL, *result=NULL, *state=NULL; @@ -1975,8 +2133,16 @@ set_reduce(PySetObject *so, PyObject *Py_UNUSED(ignored)) return result; } +/*[clinic input] +set.__sizeof__ + so: setobject + +S.__sizeof__() -> size of S in memory, in bytes. +[clinic start generated code]*/ + static PyObject * -set_sizeof(PySetObject *so, PyObject *Py_UNUSED(ignored)) +set___sizeof___impl(PySetObject *so) +/*[clinic end generated code: output=4bfa3df7bd38ed88 input=0f214fc2225319fc]*/ { size_t res = _PyObject_SIZE(Py_TYPE(so)); if (so->table != so->smalltable) { @@ -1985,7 +2151,6 @@ set_sizeof(PySetObject *so, PyObject *Py_UNUSED(ignored)) return PyLong_FromSize_t(res); } -PyDoc_STRVAR(sizeof_doc, "S.__sizeof__() -> size of S in memory, in bytes"); static int set_init(PySetObject *self, PyObject *args, PyObject *kwds) { @@ -2039,46 +2204,26 @@ static PySequenceMethods set_as_sequence = { /* set object ********************************************************/ static PyMethodDef set_methods[] = { - {"add", (PyCFunction)set_add, METH_O, - add_doc}, - {"clear", (PyCFunction)set_clear, METH_NOARGS, - clear_doc}, - {"__contains__",(PyCFunction)set_direct_contains, METH_O | METH_COEXIST, - contains_doc}, - {"copy", (PyCFunction)set_copy, METH_NOARGS, - copy_doc}, - {"discard", (PyCFunction)set_discard, METH_O, - discard_doc}, - {"difference", (PyCFunction)set_difference_multi, METH_VARARGS, - difference_doc}, - {"difference_update", (PyCFunction)set_difference_update, METH_VARARGS, - difference_update_doc}, - {"intersection",(PyCFunction)set_intersection_multi, METH_VARARGS, - intersection_doc}, - {"intersection_update",(PyCFunction)set_intersection_update_multi, METH_VARARGS, - intersection_update_doc}, - {"isdisjoint", (PyCFunction)set_isdisjoint, METH_O, - isdisjoint_doc}, - {"issubset", (PyCFunction)set_issubset, METH_O, - issubset_doc}, - {"issuperset", (PyCFunction)set_issuperset, METH_O, - issuperset_doc}, - {"pop", (PyCFunction)set_pop, METH_NOARGS, - pop_doc}, - {"__reduce__", (PyCFunction)set_reduce, METH_NOARGS, - reduce_doc}, - {"remove", (PyCFunction)set_remove, METH_O, - remove_doc}, - {"__sizeof__", (PyCFunction)set_sizeof, METH_NOARGS, - sizeof_doc}, - {"symmetric_difference",(PyCFunction)set_symmetric_difference, METH_O, - symmetric_difference_doc}, - {"symmetric_difference_update",(PyCFunction)set_symmetric_difference_update, METH_O, - symmetric_difference_update_doc}, - {"union", (PyCFunction)set_union, METH_VARARGS, - union_doc}, - {"update", (PyCFunction)set_update, METH_VARARGS, - update_doc}, + SET_ADD_METHODDEF + SET_CLEAR_METHODDEF + SET___CONTAINS___METHODDEF + SET_COPY_METHODDEF + SET_DISCARD_METHODDEF + SET_DIFFERENCE_MULTI_METHODDEF + SET_DIFFERENCE_UPDATE_METHODDEF + SET_INTERSECTION_MULTI_METHODDEF + SET_INTERSECTION_UPDATE_MULTI_METHODDEF + SET_ISDISJOINT_METHODDEF + SET_ISSUBSET_METHODDEF + SET_ISSUPERSET_METHODDEF + SET_POP_METHODDEF + SET___REDUCE___METHODDEF + SET_REMOVE_METHODDEF + SET___SIZEOF___METHODDEF + SET_SYMMETRIC_DIFFERENCE_METHODDEF + SET_SYMMETRIC_DIFFERENCE_UPDATE_METHODDEF + SET_UNION_METHODDEF + SET_UPDATE_METHODDEF {"__class_getitem__", Py_GenericAlias, METH_O|METH_CLASS, PyDoc_STR("See PEP 585")}, {NULL, NULL} /* sentinel */ }; @@ -2171,28 +2316,17 @@ PyTypeObject PySet_Type = { static PyMethodDef frozenset_methods[] = { - {"__contains__",(PyCFunction)set_direct_contains, METH_O | METH_COEXIST, - contains_doc}, - {"copy", (PyCFunction)frozenset_copy, METH_NOARGS, - copy_doc}, - {"difference", (PyCFunction)set_difference_multi, METH_VARARGS, - difference_doc}, - {"intersection", (PyCFunction)set_intersection_multi, METH_VARARGS, - intersection_doc}, - {"isdisjoint", (PyCFunction)set_isdisjoint, METH_O, - isdisjoint_doc}, - {"issubset", (PyCFunction)set_issubset, METH_O, - issubset_doc}, - {"issuperset", (PyCFunction)set_issuperset, METH_O, - issuperset_doc}, - {"__reduce__", (PyCFunction)set_reduce, METH_NOARGS, - reduce_doc}, - {"__sizeof__", (PyCFunction)set_sizeof, METH_NOARGS, - sizeof_doc}, - {"symmetric_difference",(PyCFunction)set_symmetric_difference, METH_O, - symmetric_difference_doc}, - {"union", (PyCFunction)set_union, METH_VARARGS, - union_doc}, + SET___CONTAINS___METHODDEF + FROZENSET_COPY_METHODDEF + SET_DIFFERENCE_MULTI_METHODDEF + SET_INTERSECTION_MULTI_METHODDEF + SET_ISDISJOINT_METHODDEF + SET_ISSUBSET_METHODDEF + SET_ISSUPERSET_METHODDEF + SET___REDUCE___METHODDEF + SET___SIZEOF___METHODDEF + SET_SYMMETRIC_DIFFERENCE_METHODDEF + SET_UNION_METHODDEF {"__class_getitem__", Py_GenericAlias, METH_O|METH_CLASS, PyDoc_STR("See PEP 585")}, {NULL, NULL} /* sentinel */ }; diff --git a/Objects/sliceobject.c b/Objects/sliceobject.c index a3ed0c096d84ed..9880c123c80f95 100644 --- a/Objects/sliceobject.c +++ b/Objects/sliceobject.c @@ -103,14 +103,18 @@ PyObject _Py_EllipsisObject = _PyObject_HEAD_INIT(&PyEllipsis_Type); /* Slice object implementation */ - -void _PySlice_Fini(PyInterpreterState *interp) +void _PySlice_ClearFreeList(_PyFreeListState *state, int is_finalization) { - PySliceObject *obj = interp->slice_cache; + if (!is_finalization) { + return; + } +#ifdef WITH_FREELISTS + PySliceObject *obj = state->slices.slice_cache; if (obj != NULL) { - interp->slice_cache = NULL; + state->slices.slice_cache = NULL; PyObject_GC_Del(obj); } +#endif } /* start, stop, and step are python objects with None indicating no @@ -121,15 +125,17 @@ static PySliceObject * _PyBuildSlice_Consume2(PyObject *start, PyObject *stop, PyObject *step) { assert(start != NULL && stop != NULL && step != NULL); - - PyInterpreterState *interp = _PyInterpreterState_GET(); PySliceObject *obj; - if (interp->slice_cache != NULL) { - obj = interp->slice_cache; - interp->slice_cache = NULL; +#ifdef WITH_FREELISTS + _PyFreeListState *state = _PyFreeListState_GET(); + if (state->slices.slice_cache != NULL) { + obj = state->slices.slice_cache; + state->slices.slice_cache = NULL; _Py_NewReference((PyObject *)obj); } - else { + else +#endif + { obj = PyObject_GC_New(PySliceObject, &PySlice_Type); if (obj == NULL) { goto error; @@ -354,15 +360,18 @@ Create a slice object. This is used for extended slicing (e.g. a[0:10:2])."); static void slice_dealloc(PySliceObject *r) { - PyInterpreterState *interp = _PyInterpreterState_GET(); _PyObject_GC_UNTRACK(r); Py_DECREF(r->step); Py_DECREF(r->start); Py_DECREF(r->stop); - if (interp->slice_cache == NULL) { - interp->slice_cache = r; +#ifdef WITH_FREELISTS + _PyFreeListState *state = _PyFreeListState_GET(); + if (state->slices.slice_cache == NULL) { + state->slices.slice_cache = r; } - else { + else +#endif + { PyObject_GC_Del(r); } } diff --git a/Objects/tupleobject.c b/Objects/tupleobject.c index d567839c5e3a0b..7d73c3fb0f7f2c 100644 --- a/Objects/tupleobject.c +++ b/Objects/tupleobject.c @@ -962,18 +962,13 @@ _PyTuple_Resize(PyObject **pv, Py_ssize_t newsize) } -static void maybe_freelist_clear(PyInterpreterState *, int); +static void maybe_freelist_clear(_PyFreeListState *, int); -void -_PyTuple_Fini(PyInterpreterState *interp) -{ - maybe_freelist_clear(interp, 1); -} void -_PyTuple_ClearFreeList(PyInterpreterState *interp) +_PyTuple_ClearFreeList(_PyFreeListState *state, int is_finalization) { - maybe_freelist_clear(interp, 0); + maybe_freelist_clear(state, is_finalization); } /*********************** Tuple Iterator **************************/ @@ -1125,18 +1120,14 @@ tuple_iter(PyObject *seq) * freelists * *************/ -#define STATE (interp->tuple) +#define STATE (state->tuples) #define FREELIST_FINALIZED (STATE.numfree[0] < 0) static inline PyTupleObject * maybe_freelist_pop(Py_ssize_t size) { -#if PyTuple_NFREELISTS > 0 - PyInterpreterState *interp = _PyInterpreterState_GET(); -#ifdef Py_DEBUG - /* maybe_freelist_pop() must not be called after maybe_freelist_fini(). */ - assert(!FREELIST_FINALIZED); -#endif +#ifdef WITH_FREELISTS + _PyFreeListState *state = _PyFreeListState_GET(); if (size == 0) { return NULL; } @@ -1169,18 +1160,15 @@ maybe_freelist_pop(Py_ssize_t size) static inline int maybe_freelist_push(PyTupleObject *op) { -#if PyTuple_NFREELISTS > 0 - PyInterpreterState *interp = _PyInterpreterState_GET(); -#ifdef Py_DEBUG - /* maybe_freelist_push() must not be called after maybe_freelist_fini(). */ - assert(!FREELIST_FINALIZED); -#endif +#ifdef WITH_FREELISTS + _PyFreeListState *state = _PyFreeListState_GET(); if (Py_SIZE(op) == 0) { return 0; } Py_ssize_t index = Py_SIZE(op) - 1; if (index < PyTuple_NFREELISTS && STATE.numfree[index] < PyTuple_MAXFREELIST + && STATE.numfree[index] >= 0 && Py_IS_TYPE(op, &PyTuple_Type)) { /* op is the head of a linked list, with the first item @@ -1196,9 +1184,9 @@ maybe_freelist_push(PyTupleObject *op) } static void -maybe_freelist_clear(PyInterpreterState *interp, int fini) +maybe_freelist_clear(_PyFreeListState *state, int fini) { -#if PyTuple_NFREELISTS > 0 +#ifdef WITH_FREELISTS for (Py_ssize_t i = 0; i < PyTuple_NFREELISTS; i++) { PyTupleObject *p = STATE.free_list[i]; STATE.free_list[i] = NULL; @@ -1216,8 +1204,8 @@ maybe_freelist_clear(PyInterpreterState *interp, int fini) void _PyTuple_DebugMallocStats(FILE *out) { -#if PyTuple_NFREELISTS > 0 - PyInterpreterState *interp = _PyInterpreterState_GET(); +#ifdef WITH_FREELISTS + _PyFreeListState *state = _PyFreeListState_GET(); for (int i = 0; i < PyTuple_NFREELISTS; i++) { int len = i + 1; char buf[128]; diff --git a/Objects/typeobject.c b/Objects/typeobject.c index 5261ef92413615..c65d0ec2acae52 100644 --- a/Objects/typeobject.c +++ b/Objects/typeobject.c @@ -11,6 +11,7 @@ #include "pycore_modsupport.h" // _PyArg_NoKwnames() #include "pycore_moduleobject.h" // _PyModule_GetDef() #include "pycore_object.h" // _PyType_HasFeature() +#include "pycore_object_alloc.h" // _PyObject_MallocWithType() #include "pycore_pyerrors.h" // _PyErr_Occurred() #include "pycore_pystate.h" // _PyThreadState_GET() #include "pycore_symtable.h" // _Py_Mangle() @@ -907,6 +908,8 @@ type_mro_modified(PyTypeObject *type, PyObject *bases) { } } +#define MAX_VERSIONS_PER_CLASS 1000 + static int assign_version_tag(PyInterpreterState *interp, PyTypeObject *type) { @@ -921,7 +924,10 @@ assign_version_tag(PyInterpreterState *interp, PyTypeObject *type) if (!_PyType_HasFeature(type, Py_TPFLAGS_READY)) { return 0; } - + if (type->tp_versions_used >= MAX_VERSIONS_PER_CLASS) { + return 0; + } + type->tp_versions_used++; if (type->tp_flags & Py_TPFLAGS_IMMUTABLETYPE) { /* static types */ if (NEXT_GLOBAL_VERSION_TAG > _Py_MAX_GLOBAL_TYPE_VERSION_TAG) { @@ -1370,6 +1376,7 @@ type_set_bases(PyTypeObject *type, PyObject *new_bases, void *context) res = 0; } + RARE_EVENT_INC(set_bases); Py_DECREF(old_bases); Py_DECREF(old_base); @@ -1729,7 +1736,7 @@ _PyType_AllocNoTrack(PyTypeObject *type, Py_ssize_t nitems) const size_t size = _PyObject_VAR_SIZE(type, nitems+1); const size_t presize = _PyType_PreHeaderSize(type); - char *alloc = PyObject_Malloc(size + presize); + char *alloc = _PyObject_MallocWithType(type, size + presize); if (alloc == NULL) { return PyErr_NoMemory(); } @@ -3491,7 +3498,7 @@ type_new_set_doc(PyTypeObject *type) // Silently truncate the docstring if it contains a null byte Py_ssize_t size = strlen(doc_str) + 1; - char *tp_doc = (char *)PyObject_Malloc(size); + char *tp_doc = (char *)PyMem_Malloc(size); if (tp_doc == NULL) { PyErr_NoMemory(); return -1; @@ -3826,6 +3833,17 @@ type_new_impl(type_new_ctx *ctx) // Put the proper slots in place fixup_slot_dispatchers(type); + if (!_PyDict_HasOnlyStringKeys(type->tp_dict)) { + if (PyErr_WarnFormat( + PyExc_RuntimeWarning, + 1, + "non-string key in the __dict__ of class %.200s", + type->tp_name) == -1) + { + goto error; + } + } + if (type_new_set_names(type) < 0) { goto error; } @@ -4164,12 +4182,12 @@ _PyType_FromMetaclass_impl( goto finally; } if (slot->pfunc == NULL) { - PyObject_Free(tp_doc); + PyMem_Free(tp_doc); tp_doc = NULL; } else { size_t len = strlen(slot->pfunc)+1; - tp_doc = PyObject_Malloc(len); + tp_doc = PyMem_Malloc(len); if (tp_doc == NULL) { PyErr_NoMemory(); goto finally; @@ -4499,7 +4517,7 @@ _PyType_FromMetaclass_impl( Py_CLEAR(res); } Py_XDECREF(bases); - PyObject_Free(tp_doc); + PyMem_Free(tp_doc); Py_XDECREF(ht_name); PyMem_Free(_ht_tpname); return (PyObject*)res; @@ -5097,7 +5115,7 @@ type_dealloc(PyObject *self) /* A type's tp_doc is heap allocated, unlike the tp_doc slots * of most other objects. It's okay to cast it to char *. */ - PyObject_Free((char *)type->tp_doc); + PyMem_Free((char *)type->tp_doc); PyHeapTypeObject *et = (PyHeapTypeObject *)type; Py_XDECREF(et->ht_name); @@ -5841,6 +5859,8 @@ object_set_class(PyObject *self, PyObject *value, void *closure) Py_SET_TYPE(self, newto); if (oldto->tp_flags & Py_TPFLAGS_HEAPTYPE) Py_DECREF(oldto); + + RARE_EVENT_INC(set_class); return 0; } else { @@ -6663,7 +6683,7 @@ type_add_method(PyTypeObject *type, PyMethodDef *meth) int err; PyObject *dict = lookup_tp_dict(type); if (!(meth->ml_flags & METH_COEXIST)) { - err = PyDict_SetDefault(dict, name, descr) == NULL; + err = PyDict_SetDefaultRef(dict, name, descr, NULL) < 0; } else { err = PyDict_SetItem(dict, name, descr) < 0; @@ -6711,7 +6731,7 @@ type_add_members(PyTypeObject *type) if (descr == NULL) return -1; - if (PyDict_SetDefault(dict, PyDescr_NAME(descr), descr) == NULL) { + if (PyDict_SetDefaultRef(dict, PyDescr_NAME(descr), descr, NULL) < 0) { Py_DECREF(descr); return -1; } @@ -6736,7 +6756,7 @@ type_add_getset(PyTypeObject *type) return -1; } - if (PyDict_SetDefault(dict, PyDescr_NAME(descr), descr) == NULL) { + if (PyDict_SetDefaultRef(dict, PyDescr_NAME(descr), descr, NULL) < 0) { Py_DECREF(descr); return -1; } @@ -10404,9 +10424,22 @@ supercheck(PyTypeObject *type, PyObject *obj) Py_XDECREF(class_attr); } - PyErr_SetString(PyExc_TypeError, - "super(type, obj): " - "obj must be an instance or subtype of type"); + const char *type_or_instance, *obj_str; + + if (PyType_Check(obj)) { + type_or_instance = "type"; + obj_str = ((PyTypeObject*)obj)->tp_name; + } + else { + type_or_instance = "instance of"; + obj_str = Py_TYPE(obj)->tp_name; + } + + PyErr_Format(PyExc_TypeError, + "super(type, obj): obj (%s %.200s) is not " + "an instance or subtype of type (%.200s).", + type_or_instance, obj_str, type->tp_name); + return NULL; } diff --git a/Objects/unicodeobject.c b/Objects/unicodeobject.c index ad87206b2a8200..0a569a950e88e2 100644 --- a/Objects/unicodeobject.c +++ b/Objects/unicodeobject.c @@ -996,7 +996,7 @@ resize_compact(PyObject *unicode, Py_ssize_t length) new_size = (struct_size + (length + 1) * char_size); if (_PyUnicode_HAS_UTF8_MEMORY(unicode)) { - PyObject_Free(_PyUnicode_UTF8(unicode)); + PyMem_Free(_PyUnicode_UTF8(unicode)); _PyUnicode_UTF8(unicode) = NULL; _PyUnicode_UTF8_LENGTH(unicode) = 0; } @@ -1049,7 +1049,7 @@ resize_inplace(PyObject *unicode, Py_ssize_t length) if (!share_utf8 && _PyUnicode_HAS_UTF8_MEMORY(unicode)) { - PyObject_Free(_PyUnicode_UTF8(unicode)); + PyMem_Free(_PyUnicode_UTF8(unicode)); _PyUnicode_UTF8(unicode) = NULL; _PyUnicode_UTF8_LENGTH(unicode) = 0; } @@ -1590,10 +1590,10 @@ unicode_dealloc(PyObject *unicode) return; } if (_PyUnicode_HAS_UTF8_MEMORY(unicode)) { - PyObject_Free(_PyUnicode_UTF8(unicode)); + PyMem_Free(_PyUnicode_UTF8(unicode)); } if (!PyUnicode_IS_COMPACT(unicode) && _PyUnicode_DATA_ANY(unicode)) { - PyObject_Free(_PyUnicode_DATA_ANY(unicode)); + PyMem_Free(_PyUnicode_DATA_ANY(unicode)); } Py_TYPE(unicode)->tp_free(unicode); @@ -1897,6 +1897,7 @@ PyUnicode_FromString(const char *u) PyObject * _PyUnicode_FromId(_Py_Identifier *id) { + PyMutex_Lock((PyMutex *)&id->mutex); PyInterpreterState *interp = _PyInterpreterState_GET(); struct _Py_unicode_ids *ids = &interp->unicode.ids; @@ -1923,14 +1924,14 @@ _PyUnicode_FromId(_Py_Identifier *id) obj = ids->array[index]; if (obj) { // Return a borrowed reference - return obj; + goto end; } } obj = PyUnicode_DecodeUTF8Stateful(id->string, strlen(id->string), NULL, NULL); if (!obj) { - return NULL; + goto end; } PyUnicode_InternInPlace(&obj); @@ -1941,7 +1942,8 @@ _PyUnicode_FromId(_Py_Identifier *id) PyObject **new_array = PyMem_Realloc(ids->array, new_size * item_size); if (new_array == NULL) { PyErr_NoMemory(); - return NULL; + obj = NULL; + goto end; } memset(&new_array[ids->size], 0, (new_size - ids->size) * item_size); ids->array = new_array; @@ -1951,6 +1953,8 @@ _PyUnicode_FromId(_Py_Identifier *id) // The array stores a strong reference ids->array[index] = obj; +end: + PyMutex_Unlock((PyMutex *)&id->mutex); // Return a borrowed reference return obj; } @@ -5199,7 +5203,7 @@ unicode_fill_utf8(PyObject *unicode) PyBytes_AS_STRING(writer.buffer); Py_ssize_t len = end - start; - char *cache = PyObject_Malloc(len + 1); + char *cache = PyMem_Malloc(len + 1); if (cache == NULL) { _PyBytesWriter_Dealloc(&writer); PyErr_NoMemory(); @@ -14670,7 +14674,7 @@ unicode_subtype_new(PyTypeObject *type, PyObject *unicode) PyErr_NoMemory(); goto onError; } - data = PyObject_Malloc((length + 1) * char_size); + data = PyMem_Malloc((length + 1) * char_size); if (data == NULL) { PyErr_NoMemory(); goto onError; @@ -14890,16 +14894,18 @@ _PyUnicode_InternInPlace(PyInterpreterState *interp, PyObject **p) PyObject *interned = get_interned_dict(interp); assert(interned != NULL); - PyObject *t = PyDict_SetDefault(interned, s, s); - if (t == NULL) { + PyObject *t; + int res = PyDict_SetDefaultRef(interned, s, s, &t); + if (res < 0) { PyErr_Clear(); return; } - - if (t != s) { - Py_SETREF(*p, Py_NewRef(t)); + else if (res == 1) { + // value was already present (not inserted) + Py_SETREF(*p, t); return; } + Py_DECREF(t); if (_Py_IsImmortal(s)) { // XXX Restrict this to the main interpreter? diff --git a/PC/config.c b/PC/config.c index f754ce6d3b057b..5eff2f5b2310bb 100644 --- a/PC/config.c +++ b/PC/config.c @@ -44,9 +44,7 @@ extern PyObject* PyInit__collections(void); extern PyObject* PyInit__heapq(void); extern PyObject* PyInit__bisect(void); extern PyObject* PyInit__symtable(void); -#if defined(MS_WINDOWS_DESKTOP) || defined(MS_WINDOWS_GAMES) extern PyObject* PyInit_mmap(void); -#endif extern PyObject* PyInit__csv(void); extern PyObject* PyInit__sre(void); #if defined(MS_WINDOWS_DESKTOP) || defined(MS_WINDOWS_SYSTEM) || defined(MS_WINDOWS_GAMES) diff --git a/PC/launcher2.c b/PC/launcher2.c index 2a8f8a101fc8a6..e426eccd700044 100644 --- a/PC/launcher2.c +++ b/PC/launcher2.c @@ -572,6 +572,21 @@ findArgv0End(const wchar_t *buffer, int bufferLength) *** COMMAND-LINE PARSING *** \******************************************************************************/ +// Adapted from https://stackoverflow.com/a/65583702 +typedef struct AppExecLinkFile { // For tag IO_REPARSE_TAG_APPEXECLINK + DWORD reparseTag; + WORD reparseDataLength; + WORD reserved; + ULONG version; + wchar_t stringList[MAX_PATH * 4]; // Multistring (Consecutive UTF-16 strings each ending with a NUL) + /* There are normally 4 strings here. Ex: + Package ID: L"Microsoft.DesktopAppInstaller_8wekyb3d8bbwe" + Entry Point: L"Microsoft.DesktopAppInstaller_8wekyb3d8bbwe!PythonRedirector" + Executable: L"C:\Program Files\WindowsApps\Microsoft.DesktopAppInstaller_1.17.106910_x64__8wekyb3d8bbwe\AppInstallerPythonRedirector.exe" + Applic. Type: L"0" // Integer as ASCII. "0" = Desktop bridge application; Else sandboxed UWP application + */ +} AppExecLinkFile; + int parseCommandLine(SearchInfo *search) @@ -763,6 +778,55 @@ _shebangStartsWith(const wchar_t *buffer, int bufferLength, const wchar_t *prefi } +int +ensure_no_redirector_stub(wchar_t* filename, wchar_t* buffer) +{ + // Make sure we didn't find a reparse point that will open the Microsoft Store + // If we did, pretend there was no shebang and let normal handling take over + WIN32_FIND_DATAW findData; + HANDLE hFind = FindFirstFileW(buffer, &findData); + if (!hFind) { + // Let normal handling take over + debug(L"# Did not find %s on PATH\n", filename); + return RC_NO_SHEBANG; + } + + FindClose(hFind); + + if (!(findData.dwFileAttributes & FILE_ATTRIBUTE_REPARSE_POINT && + findData.dwReserved0 & IO_REPARSE_TAG_APPEXECLINK)) { + return 0; + } + + HANDLE hReparsePoint = CreateFileW(buffer, 0, FILE_SHARE_READ, NULL, OPEN_EXISTING, FILE_FLAG_OPEN_REPARSE_POINT, NULL); + if (!hReparsePoint) { + // Let normal handling take over + debug(L"# Did not find %s on PATH\n", filename); + return RC_NO_SHEBANG; + } + + AppExecLinkFile appExecLink; + + if (!DeviceIoControl(hReparsePoint, FSCTL_GET_REPARSE_POINT, NULL, 0, &appExecLink, sizeof(appExecLink), NULL, NULL)) { + // Let normal handling take over + debug(L"# Did not find %s on PATH\n", filename); + CloseHandle(hReparsePoint); + return RC_NO_SHEBANG; + } + + CloseHandle(hReparsePoint); + + const wchar_t* redirectorPackageId = L"Microsoft.DesktopAppInstaller_8wekyb3d8bbwe"; + + if (0 == wcscmp(appExecLink.stringList, redirectorPackageId)) { + debug(L"# ignoring redirector that would launch store\n"); + return RC_NO_SHEBANG; + } + + return 0; +} + + int searchPath(SearchInfo *search, const wchar_t *shebang, int shebangLength) { @@ -826,6 +890,11 @@ searchPath(SearchInfo *search, const wchar_t *shebang, int shebangLength) return RC_BAD_VIRTUAL_PATH; } + int result = ensure_no_redirector_stub(filename, buffer); + if (result) { + return result; + } + // Check that we aren't going to call ourselves again // If we are, pretend there was no shebang and let normal handling take over if (GetModuleFileNameW(NULL, filename, MAXLEN) && diff --git a/PC/layout/main.py b/PC/layout/main.py index accfd51dd978fb..d176b272f1c19d 100644 --- a/PC/layout/main.py +++ b/PC/layout/main.py @@ -41,7 +41,7 @@ VENV_DIRS_ONLY = FileNameSet("venv", "ensurepip") -EXCLUDE_FROM_PYDS = FileStemSet("python*", "pyshellext", "vcruntime*") +EXCLUDE_FROM_DLLS = FileStemSet("python*", "pyshellext", "vcruntime*") EXCLUDE_FROM_LIB = FileNameSet("*.pyc", "__pycache__", "*.pickle") EXCLUDE_FROM_PACKAGED_LIB = FileNameSet("readme.txt") EXCLUDE_FROM_COMPILE = FileNameSet("badsyntax_*", "bad_*") @@ -126,9 +126,9 @@ def in_build(f, dest="", new_name=None): n = new_name or n src = ns.build / f if ns.debug and src not in REQUIRED_DLLS: - if not src.stem.endswith("_d"): + if not "_d." in src.name: src = src.parent / (src.stem + "_d" + src.suffix) - if not n.endswith("_d"): + if "_d." not in f: n += "_d" f = n + "." + x yield dest + n + "." + x, src @@ -141,17 +141,45 @@ def in_build(f, dest="", new_name=None): if lib.is_file(): yield "libs/" + n + ".lib", lib + source = "python.exe" + sourcew = "pythonw.exe" + alias = [ + "python", + "python{}".format(VER_MAJOR) if ns.include_alias3 else "", + "python{}".format(VER_DOT) if ns.include_alias3x else "", + ] + aliasw = [ + "pythonw", + "pythonw{}".format(VER_MAJOR) if ns.include_alias3 else "", + "pythonw{}".format(VER_DOT) if ns.include_alias3x else "", + ] if ns.include_appxmanifest: - yield from in_build("python_uwp.exe", new_name="python{}".format(VER_DOT)) - yield from in_build("pythonw_uwp.exe", new_name="pythonw{}".format(VER_DOT)) - # For backwards compatibility, but we don't reference these ourselves. - yield from in_build("python_uwp.exe", new_name="python") - yield from in_build("pythonw_uwp.exe", new_name="pythonw") + source = "python_uwp.exe" + sourcew = "pythonw_uwp.exe" + elif ns.include_freethreaded: + source = "python{}t.exe".format(VER_DOT) + sourcew = "pythonw{}t.exe".format(VER_DOT) + if not ns.include_alias: + alias = [] + aliasw = [] + alias.extend([ + "python{}t".format(VER_DOT), + "python{}t".format(VER_MAJOR) if ns.include_alias3 else None, + ]) + aliasw.extend([ + "pythonw{}t".format(VER_DOT), + "pythonw{}t".format(VER_MAJOR) if ns.include_alias3 else None, + ]) + + for a in filter(None, alias): + yield from in_build(source, new_name=a) + for a in filter(None, aliasw): + yield from in_build(sourcew, new_name=a) + + if ns.include_freethreaded: + yield from in_build(FREETHREADED_PYTHON_DLL_NAME) else: - yield from in_build("python.exe", new_name="python") - yield from in_build("pythonw.exe", new_name="pythonw") - - yield from in_build(PYTHON_DLL_NAME) + yield from in_build(PYTHON_DLL_NAME) if ns.include_launchers and ns.include_appxmanifest: if ns.include_pip: @@ -160,7 +188,10 @@ def in_build(f, dest="", new_name=None): yield from in_build("pythonw_uwp.exe", new_name="idle{}".format(VER_DOT)) if ns.include_stable: - yield from in_build(PYTHON_STABLE_DLL_NAME) + if ns.include_freethreaded: + yield from in_build(FREETHREADED_PYTHON_STABLE_DLL_NAME) + else: + yield from in_build(PYTHON_STABLE_DLL_NAME) found_any = False for dest, src in rglob(ns.build, "vcruntime*.dll"): @@ -171,16 +202,28 @@ def in_build(f, dest="", new_name=None): yield "LICENSE.txt", ns.build / "LICENSE.txt" - for dest, src in rglob(ns.build, ("*.pyd", "*.dll")): - if src.stem.endswith("_d") != bool(ns.debug) and src not in REQUIRED_DLLS: - continue - if src in EXCLUDE_FROM_PYDS: - continue + for dest, src in rglob(ns.build, "*.pyd"): + if ns.include_freethreaded: + if not src.match("*.cp*t-win*.pyd"): + continue + if bool(src.match("*_d.cp*.pyd")) != bool(ns.debug): + continue + else: + if src.match("*.cp*t-win*.pyd"): + continue + if bool(src.match("*_d.pyd")) != bool(ns.debug): + continue if src in TEST_PYDS_ONLY and not ns.include_tests: continue if src in TCLTK_PYDS_ONLY and not ns.include_tcltk: continue + yield from in_build(src.name, dest="" if ns.flat_dlls else "DLLs/") + for dest, src in rglob(ns.build, "*.dll"): + if src.stem.endswith("_d") != bool(ns.debug) and src not in REQUIRED_DLLS: + continue + if src in EXCLUDE_FROM_DLLS: + continue yield from in_build(src.name, dest="" if ns.flat_dlls else "DLLs/") if ns.zip_lib: @@ -191,8 +234,12 @@ def in_build(f, dest="", new_name=None): yield "Lib/{}".format(dest), src if ns.include_venv: - yield from in_build("venvlauncher.exe", "Lib/venv/scripts/nt/", "python") - yield from in_build("venvwlauncher.exe", "Lib/venv/scripts/nt/", "pythonw") + if ns.include_freethreaded: + yield from in_build("venvlaunchert.exe", "Lib/venv/scripts/nt/") + yield from in_build("venvwlaunchert.exe", "Lib/venv/scripts/nt/") + else: + yield from in_build("venvlauncher.exe", "Lib/venv/scripts/nt/") + yield from in_build("venvwlauncher.exe", "Lib/venv/scripts/nt/") if ns.include_tools: @@ -208,7 +255,6 @@ def _c(d): yield PYTHON_PTH_NAME, ns.temp / PYTHON_PTH_NAME if ns.include_dev: - for dest, src in rglob(ns.source / "Include", "**/*.h"): yield "include/{}".format(dest), src yield "include/pyconfig.h", ns.build / "pyconfig.h" @@ -552,7 +598,6 @@ def main(): ns.source = ns.source or (Path(__file__).resolve().parent.parent.parent) ns.build = ns.build or Path(sys.executable).parent - ns.temp = ns.temp or Path(tempfile.mkdtemp()) ns.doc_build = ns.doc_build or (ns.source / "Doc" / "build") if not ns.source.is_absolute(): ns.source = (Path.cwd() / ns.source).resolve() @@ -565,7 +610,12 @@ def main(): if ns.include_cat and not ns.include_cat.is_absolute(): ns.include_cat = (Path.cwd() / ns.include_cat).resolve() if not ns.arch: - ns.arch = "amd64" if sys.maxsize > 2 ** 32 else "win32" + if sys.winver.endswith("-arm64"): + ns.arch = "arm64" + elif sys.winver.endswith("-32"): + ns.arch = "win32" + else: + ns.arch = "amd64" if ns.copy and not ns.copy.is_absolute(): ns.copy = (Path.cwd() / ns.copy).resolve() @@ -574,6 +624,14 @@ def main(): if ns.catalog and not ns.catalog.is_absolute(): ns.catalog = (Path.cwd() / ns.catalog).resolve() + if not ns.temp: + # Put temp on a Dev Drive for speed if we're copying to one. + # If not, the regular temp dir will have to do. + if ns.copy and getattr(os.path, "isdevdrive", lambda d: False)(ns.copy): + ns.temp = ns.copy.with_name(ns.copy.name + "_temp") + else: + ns.temp = Path(tempfile.mkdtemp()) + configure_logger(ns) log_info( @@ -602,6 +660,12 @@ def main(): log_warning("Assuming --include-tcltk to support --include-idle") ns.include_tcltk = True + if not (ns.include_alias or ns.include_alias3 or ns.include_alias3x): + if ns.include_freethreaded: + ns.include_alias3x = True + else: + ns.include_alias = True + try: generate_source_files(ns) files = list(get_layout(ns)) diff --git a/PC/layout/support/constants.py b/PC/layout/support/constants.py index 8195c3dc30cdc7..ae22aa16ebfa5d 100644 --- a/PC/layout/support/constants.py +++ b/PC/layout/support/constants.py @@ -39,3 +39,6 @@ def _get_suffix(field4): PYTHON_CHM_NAME = "python{}{}{}{}.chm".format( VER_MAJOR, VER_MINOR, VER_MICRO, VER_SUFFIX ) + +FREETHREADED_PYTHON_DLL_NAME = "python{}{}t.dll".format(VER_MAJOR, VER_MINOR) +FREETHREADED_PYTHON_STABLE_DLL_NAME = "python{}t.dll".format(VER_MAJOR) diff --git a/PC/layout/support/nuspec.py b/PC/layout/support/nuspec.py index dbcb713ef9d0c0..a87e0bea049427 100644 --- a/PC/layout/support/nuspec.py +++ b/PC/layout/support/nuspec.py @@ -24,6 +24,10 @@ amd64=("64-bit", "python", "Python"), arm32=("ARM", "pythonarm", "Python (ARM)"), arm64=("ARM64", "pythonarm64", "Python (ARM64)"), + win32t=("32-bit free-threaded", "pythonx86-freethreaded", "Python (32-bit, free-threaded)"), + amd64t=("64-bit free-threaded", "python-freethreaded", "Python (free-threaded)"), + arm32t=("ARM free-threaded", "pythonarm-freethreaded", "Python (ARM, free-threaded)"), + arm64t=("ARM64 free-threaded", "pythonarm64-freethreaded", "Python (ARM64, free-threaded)"), ) if not NUSPEC_DATA["PYTHON_VERSION"]: @@ -58,7 +62,10 @@ def _get_nuspec_data_overrides(ns): - for k, v in zip(NUSPEC_PLATFORM_DATA["_keys"], NUSPEC_PLATFORM_DATA[ns.arch]): + arch = ns.arch + if ns.include_freethreaded: + arch += "t" + for k, v in zip(NUSPEC_PLATFORM_DATA["_keys"], NUSPEC_PLATFORM_DATA[arch]): ev = os.getenv("PYTHON_NUSPEC_" + k) if ev: yield k, ev diff --git a/PC/layout/support/options.py b/PC/layout/support/options.py index 60256fb32fe329..f1a8eb0b317744 100644 --- a/PC/layout/support/options.py +++ b/PC/layout/support/options.py @@ -32,6 +32,10 @@ def public(f): "nuspec": {"help": "a python.nuspec file"}, "chm": {"help": "the CHM documentation"}, "html-doc": {"help": "the HTML documentation"}, + "freethreaded": {"help": "freethreaded binaries", "not-in-all": True}, + "alias": {"help": "aliased python.exe entry-point binaries"}, + "alias3": {"help": "aliased python3.exe entry-point binaries"}, + "alias3x": {"help": "aliased python3.x.exe entry-point binaries"}, } @@ -47,6 +51,8 @@ def public(f): "dev", "launchers", "appxmanifest", + "alias", + "alias3x", # XXX: Disabled for now "precompile", ], }, @@ -59,9 +65,10 @@ def public(f): "venv", "props", "nuspec", + "alias", ], }, - "iot": {"help": "Windows IoT Core", "options": ["stable", "pip"]}, + "iot": {"help": "Windows IoT Core", "options": ["alias", "stable", "pip"]}, "default": { "help": "development kit package", "options": [ @@ -74,11 +81,19 @@ def public(f): "dev", "symbols", "html-doc", + "alias", ], }, "embed": { "help": "embeddable package", - "options": ["stable", "zip-lib", "flat-dlls", "underpth", "precompile"], + "options": [ + "alias", + "stable", + "zip-lib", + "flat-dlls", + "underpth", + "precompile", + ], }, } diff --git a/PC/pyconfig.h.in b/PC/pyconfig.h.in index d8f0a6be69c21a..8bbf877a5bb5ed 100644 --- a/PC/pyconfig.h.in +++ b/PC/pyconfig.h.in @@ -94,6 +94,9 @@ WIN32 is still required for the locale module. #endif #endif /* Py_BUILD_CORE || Py_BUILD_CORE_BUILTIN || Py_BUILD_CORE_MODULE */ +/* Define to 1 if you want to disable the GIL */ +#undef Py_GIL_DISABLED + /* Compiler specific defines */ /* ------------------------------------------------------------------------*/ @@ -305,8 +308,16 @@ Py_NO_ENABLE_SHARED to find out. Also support MS_NO_COREDLL for b/w compat */ /* not building the core - must be an ext */ # if defined(_MSC_VER) /* So MSVC users need not specify the .lib - file in their Makefile (other compilers are - generally taken care of by distutils.) */ + file in their Makefile */ +# if defined(Py_GIL_DISABLED) +# if defined(_DEBUG) +# pragma comment(lib,"python313t_d.lib") +# elif defined(Py_LIMITED_API) +# pragma comment(lib,"python3t.lib") +# else +# pragma comment(lib,"python313t.lib") +# endif /* _DEBUG */ +# else /* Py_GIL_DISABLED */ # if defined(_DEBUG) # pragma comment(lib,"python313_d.lib") # elif defined(Py_LIMITED_API) @@ -314,6 +325,7 @@ Py_NO_ENABLE_SHARED to find out. Also support MS_NO_COREDLL for b/w compat */ # else # pragma comment(lib,"python313.lib") # endif /* _DEBUG */ +# endif /* Py_GIL_DISABLED */ # endif /* _MSC_VER */ # endif /* Py_BUILD_CORE */ #endif /* MS_COREDLL */ @@ -739,7 +751,4 @@ Py_NO_ENABLE_SHARED to find out. Also support MS_NO_COREDLL for b/w compat */ /* Define if libssl has X509_VERIFY_PARAM_set1_host and related function */ #define HAVE_X509_VERIFY_PARAM_SET1_HOST 1 -/* Define if you want to disable the GIL */ -#undef Py_GIL_DISABLED - #endif /* !Py_CONFIG_H */ diff --git a/PC/python3dll.c b/PC/python3dll.c index 07aa84c91f9fc7..aa6bfe2c4022db 100755 --- a/PC/python3dll.c +++ b/PC/python3dll.c @@ -324,6 +324,7 @@ EXPORT_FUNC(PyIter_Send) EXPORT_FUNC(PyList_Append) EXPORT_FUNC(PyList_AsTuple) EXPORT_FUNC(PyList_GetItem) +EXPORT_FUNC(PyList_GetItemRef) EXPORT_FUNC(PyList_GetSlice) EXPORT_FUNC(PyList_Insert) EXPORT_FUNC(PyList_New) @@ -830,6 +831,7 @@ EXPORT_DATA(PyExc_FutureWarning) EXPORT_DATA(PyExc_GeneratorExit) EXPORT_DATA(PyExc_ImportError) EXPORT_DATA(PyExc_ImportWarning) +EXPORT_DATA(PyExc_IncompleteInputError) EXPORT_DATA(PyExc_IndentationError) EXPORT_DATA(PyExc_IndexError) EXPORT_DATA(PyExc_InterruptedError) diff --git a/PC/python_ver_rc.h b/PC/python_ver_rc.h index 5b55b810cd2152..08509f96ed1db8 100644 --- a/PC/python_ver_rc.h +++ b/PC/python_ver_rc.h @@ -5,7 +5,7 @@ #include "winver.h" #define PYTHON_COMPANY "Python Software Foundation" -#define PYTHON_COPYRIGHT "Copyright \xA9 2001-2023 Python Software Foundation. Copyright \xA9 2000 BeOpen.com. Copyright \xA9 1995-2001 CNRI. Copyright \xA9 1991-1995 SMC." +#define PYTHON_COPYRIGHT "Copyright \xA9 2001-2024 Python Software Foundation. Copyright \xA9 2000 BeOpen.com. Copyright \xA9 1995-2001 CNRI. Copyright \xA9 1991-1995 SMC." #define MS_WINDOWS #include "modsupport.h" diff --git a/PC/venvlauncher.c b/PC/venvlauncher.c new file mode 100644 index 00000000000000..fe97d32e93b5f6 --- /dev/null +++ b/PC/venvlauncher.c @@ -0,0 +1,510 @@ +/* + * venv redirector for Windows + * + * This launcher looks for a nearby pyvenv.cfg to find the correct home + * directory, and then launches the original Python executable from it. + * The name of this executable is passed as argv[0]. + */ + +#define __STDC_WANT_LIB_EXT1__ 1 + +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#define MS_WINDOWS +#include "patchlevel.h" + +#define MAXLEN PATHCCH_MAX_CCH +#define MSGSIZE 1024 + +#define RC_NO_STD_HANDLES 100 +#define RC_CREATE_PROCESS 101 +#define RC_NO_PYTHON 103 +#define RC_NO_MEMORY 104 +#define RC_NO_VENV_CFG 106 +#define RC_BAD_VENV_CFG 107 +#define RC_NO_COMMANDLINE 108 +#define RC_INTERNAL_ERROR 109 + +// This should always be defined when we build for real, +// but it's handy to have a definition for quick testing +#ifndef EXENAME +#define EXENAME L"python.exe" +#endif + +#ifndef CFGNAME +#define CFGNAME L"pyvenv.cfg" +#endif + +static FILE * log_fp = NULL; + +void +debug(wchar_t * format, ...) +{ + va_list va; + + if (log_fp != NULL) { + wchar_t buffer[MAXLEN]; + int r = 0; + va_start(va, format); + r = vswprintf_s(buffer, MAXLEN, format, va); + va_end(va); + + if (r <= 0) { + return; + } + fwprintf(log_fp, L"%ls\n", buffer); + while (r && isspace(buffer[r])) { + buffer[r--] = L'\0'; + } + if (buffer[0]) { + OutputDebugStringW(buffer); + } + } +} + + +void +formatWinerror(int rc, wchar_t * message, int size) +{ + FormatMessageW( + FORMAT_MESSAGE_FROM_SYSTEM | FORMAT_MESSAGE_IGNORE_INSERTS, + NULL, rc, MAKELANGID(LANG_NEUTRAL, SUBLANG_DEFAULT), + message, size, NULL); +} + + +void +winerror(int err, wchar_t * format, ... ) +{ + va_list va; + wchar_t message[MSGSIZE]; + wchar_t win_message[MSGSIZE]; + int len; + + if (err == 0) { + err = GetLastError(); + } + + va_start(va, format); + len = _vsnwprintf_s(message, MSGSIZE, _TRUNCATE, format, va); + va_end(va); + + formatWinerror(err, win_message, MSGSIZE); + if (len >= 0) { + _snwprintf_s(&message[len], MSGSIZE - len, _TRUNCATE, L": %ls", + win_message); + } + +#if !defined(_WINDOWS) + fwprintf(stderr, L"%ls\n", message); +#else + MessageBoxW(NULL, message, L"Python venv launcher is sorry to say ...", + MB_OK); +#endif +} + + +void +error(wchar_t * format, ... ) +{ + va_list va; + wchar_t message[MSGSIZE]; + + va_start(va, format); + _vsnwprintf_s(message, MSGSIZE, _TRUNCATE, format, va); + va_end(va); + +#if !defined(_WINDOWS) + fwprintf(stderr, L"%ls\n", message); +#else + MessageBoxW(NULL, message, L"Python venv launcher is sorry to say ...", + MB_OK); +#endif +} + + +bool +isEnvVarSet(const wchar_t *name) +{ + /* only looking for non-empty, which means at least one character + and the null terminator */ + return GetEnvironmentVariableW(name, NULL, 0) >= 2; +} + + +bool +join(wchar_t *buffer, size_t bufferLength, const wchar_t *fragment) +{ + if (SUCCEEDED(PathCchCombineEx(buffer, bufferLength, buffer, fragment, PATHCCH_ALLOW_LONG_PATHS))) { + return true; + } + return false; +} + + +bool +split_parent(wchar_t *buffer, size_t bufferLength) +{ + return SUCCEEDED(PathCchRemoveFileSpec(buffer, bufferLength)); +} + + +/* + * Path calculation + */ + +int +calculate_pyvenvcfg_path(wchar_t *pyvenvcfg_path, size_t maxlen) +{ + if (!pyvenvcfg_path) { + error(L"invalid buffer provided"); + return RC_INTERNAL_ERROR; + } + if ((DWORD)maxlen != maxlen) { + error(L"path buffer is too large"); + return RC_INTERNAL_ERROR; + } + if (!GetModuleFileNameW(NULL, pyvenvcfg_path, (DWORD)maxlen)) { + winerror(GetLastError(), L"failed to read executable directory"); + return RC_NO_COMMANDLINE; + } + // Remove 'python.exe' from our path + if (!split_parent(pyvenvcfg_path, maxlen)) { + error(L"failed to remove segment from '%ls'", pyvenvcfg_path); + return RC_NO_COMMANDLINE; + } + // Replace with 'pyvenv.cfg' + if (!join(pyvenvcfg_path, maxlen, CFGNAME)) { + error(L"failed to append '%ls' to '%ls'", CFGNAME, pyvenvcfg_path); + return RC_NO_MEMORY; + } + // If it exists, return + if (GetFileAttributesW(pyvenvcfg_path) != INVALID_FILE_ATTRIBUTES) { + return 0; + } + // Otherwise, remove 'pyvenv.cfg' and (probably) 'Scripts' + if (!split_parent(pyvenvcfg_path, maxlen) || + !split_parent(pyvenvcfg_path, maxlen)) { + error(L"failed to remove segments from '%ls'", pyvenvcfg_path); + return RC_NO_COMMANDLINE; + } + // Replace 'pyvenv.cfg' + if (!join(pyvenvcfg_path, maxlen, CFGNAME)) { + error(L"failed to append '%ls' to '%ls'", CFGNAME, pyvenvcfg_path); + return RC_NO_MEMORY; + } + // If it exists, return + if (GetFileAttributesW(pyvenvcfg_path) != INVALID_FILE_ATTRIBUTES) { + return 0; + } + // Otherwise, we fail + winerror(GetLastError(), L"failed to locate %ls", CFGNAME); + return RC_NO_VENV_CFG; +} + + +/* + * pyvenv.cfg parsing + */ + +static int +find_home_value(const char *buffer, DWORD maxlen, const char **start, DWORD *length) +{ + if (!buffer || !start || !length) { + error(L"invalid find_home_value parameters()"); + return 0; + } + for (const char *s = strstr(buffer, "home"); + s && ((ptrdiff_t)s - (ptrdiff_t)buffer) < maxlen; + s = strstr(s + 1, "\nhome") + ) { + if (*s == '\n') { + ++s; + } + for (int i = 4; i > 0 && *s; --i, ++s); + + while (*s && iswspace(*s)) { + ++s; + } + if (*s != L'=') { + continue; + } + + do { + ++s; + } while (*s && iswspace(*s)); + + *start = s; + char *nl = strchr(s, '\n'); + if (nl) { + while (nl != s && iswspace(nl[-1])) { + --nl; + } + *length = (DWORD)((ptrdiff_t)nl - (ptrdiff_t)s); + } else { + *length = (DWORD)strlen(s); + } + return 1; + } + return 0; +} + + +int +read_home(const wchar_t *pyvenv_cfg, wchar_t *home_path, size_t maxlen) +{ + HANDLE hFile = CreateFileW(pyvenv_cfg, GENERIC_READ, + FILE_SHARE_READ | FILE_SHARE_WRITE | FILE_SHARE_DELETE, + NULL, OPEN_EXISTING, 0, NULL); + + if (hFile == INVALID_HANDLE_VALUE) { + winerror(GetLastError(), L"failed to open '%ls'", pyvenv_cfg); + return RC_BAD_VENV_CFG; + } + + // 8192 characters ought to be enough for anyone + // (doubled compared to the old implementation!) + char buffer[8192]; + DWORD len; + if (!ReadFile(hFile, buffer, sizeof(buffer) - 1, &len, NULL)) { + winerror(GetLastError(), L"failed to read '%ls'", pyvenv_cfg); + CloseHandle(hFile); + return RC_BAD_VENV_CFG; + } + CloseHandle(hFile); + // Ensure null termination + buffer[len] = '\0'; + + char *home; + DWORD home_len; + if (!find_home_value(buffer, sizeof(buffer), &home, &home_len)) { + error(L"no home= specified in '%ls'", pyvenv_cfg); + return RC_BAD_VENV_CFG; + } + + if ((DWORD)maxlen != maxlen) { + maxlen = 8192; + } + len = MultiByteToWideChar(CP_UTF8, 0, home, home_len, home_path, (DWORD)maxlen); + if (!len) { + winerror(GetLastError(), L"failed to decode home setting in '%ls'", pyvenv_cfg); + return RC_BAD_VENV_CFG; + } + home_path[len] = L'\0'; + + return 0; +} + + +int +locate_python(wchar_t *path, size_t maxlen) +{ + if (!join(path, maxlen, EXENAME)) { + error(L"failed to append %ls to '%ls'", EXENAME, path); + return RC_NO_MEMORY; + } + + if (GetFileAttributesW(path) == INVALID_FILE_ATTRIBUTES) { + winerror(GetLastError(), L"did not find executable at '%ls'", path); + return RC_NO_PYTHON; + } + + return 0; +} + + +int +smuggle_path() +{ + wchar_t buffer[MAXLEN]; + // We could use argv[0], but that may be wrong in certain rare cases (if the + // user is doing something weird like symlinks to venv redirectors), and + // what we _really_ want is the directory of the venv. We always copy the + // redirectors, so if we've made the venv, this will be correct. + DWORD len = GetModuleFileNameW(NULL, buffer, MAXLEN); + if (!len) { + winerror(GetLastError(), L"Failed to get own executable path"); + return RC_INTERNAL_ERROR; + } + buffer[len] = L'\0'; + debug(L"Setting __PYVENV_LAUNCHER__ = '%s'", buffer); + + if (!SetEnvironmentVariableW(L"__PYVENV_LAUNCHER__", buffer)) { + winerror(GetLastError(), L"Failed to set launcher environment"); + return RC_INTERNAL_ERROR; + } + + return 0; +} + +/* + * Process creation + */ + +static BOOL +safe_duplicate_handle(HANDLE in, HANDLE * pout, const wchar_t *name) +{ + BOOL ok; + HANDLE process = GetCurrentProcess(); + DWORD rc; + + *pout = NULL; + ok = DuplicateHandle(process, in, process, pout, 0, TRUE, + DUPLICATE_SAME_ACCESS); + if (!ok) { + rc = GetLastError(); + if (rc == ERROR_INVALID_HANDLE) { + debug(L"DuplicateHandle(%ls) returned ERROR_INVALID_HANDLE\n", name); + ok = TRUE; + } + else { + debug(L"DuplicateHandle(%ls) returned %d\n", name, rc); + } + } + return ok; +} + +static BOOL WINAPI +ctrl_c_handler(DWORD code) +{ + return TRUE; /* We just ignore all control events. */ +} + +static int +launch(const wchar_t *executable, wchar_t *cmdline) +{ + HANDLE job; + JOBOBJECT_EXTENDED_LIMIT_INFORMATION info; + DWORD rc; + BOOL ok; + STARTUPINFOW si; + PROCESS_INFORMATION pi; + +#if defined(_WINDOWS) + /* + When explorer launches a Windows (GUI) application, it displays + the "app starting" (the "pointer + hourglass") cursor for a number + of seconds, or until the app does something UI-ish (eg, creating a + window, or fetching a message). As this launcher doesn't do this + directly, that cursor remains even after the child process does these + things. We avoid that by doing a simple post+get message. + See http://bugs.python.org/issue17290 + */ + MSG msg; + + PostMessage(0, 0, 0, 0); + GetMessage(&msg, 0, 0, 0); +#endif + + debug(L"run_child: about to run '%ls' with '%ls'\n", executable, cmdline); + job = CreateJobObject(NULL, NULL); + ok = QueryInformationJobObject(job, JobObjectExtendedLimitInformation, + &info, sizeof(info), &rc); + if (!ok || (rc != sizeof(info)) || !job) { + winerror(GetLastError(), L"Job information querying failed"); + return RC_CREATE_PROCESS; + } + info.BasicLimitInformation.LimitFlags |= JOB_OBJECT_LIMIT_KILL_ON_JOB_CLOSE | + JOB_OBJECT_LIMIT_SILENT_BREAKAWAY_OK; + ok = SetInformationJobObject(job, JobObjectExtendedLimitInformation, &info, + sizeof(info)); + if (!ok) { + winerror(GetLastError(), L"Job information setting failed"); + return RC_CREATE_PROCESS; + } + memset(&si, 0, sizeof(si)); + GetStartupInfoW(&si); + ok = safe_duplicate_handle(GetStdHandle(STD_INPUT_HANDLE), &si.hStdInput, L"stdin"); + if (!ok) { + return RC_NO_STD_HANDLES; + } + ok = safe_duplicate_handle(GetStdHandle(STD_OUTPUT_HANDLE), &si.hStdOutput, L"stdout"); + if (!ok) { + return RC_NO_STD_HANDLES; + } + ok = safe_duplicate_handle(GetStdHandle(STD_ERROR_HANDLE), &si.hStdError, L"stderr"); + if (!ok) { + return RC_NO_STD_HANDLES; + } + + ok = SetConsoleCtrlHandler(ctrl_c_handler, TRUE); + if (!ok) { + winerror(GetLastError(), L"control handler setting failed"); + return RC_CREATE_PROCESS; + } + + si.dwFlags = STARTF_USESTDHANDLES; + ok = CreateProcessW(executable, cmdline, NULL, NULL, TRUE, + 0, NULL, NULL, &si, &pi); + if (!ok) { + winerror(GetLastError(), L"Unable to create process using '%ls'", cmdline); + return RC_CREATE_PROCESS; + } + AssignProcessToJobObject(job, pi.hProcess); + CloseHandle(pi.hThread); + WaitForSingleObjectEx(pi.hProcess, INFINITE, FALSE); + ok = GetExitCodeProcess(pi.hProcess, &rc); + if (!ok) { + winerror(GetLastError(), L"Failed to get exit code of process"); + return RC_CREATE_PROCESS; + } + debug(L"child process exit code: %d", rc); + return rc; +} + + +int +process(int argc, wchar_t ** argv) +{ + int exitCode; + wchar_t pyvenvcfg_path[MAXLEN]; + wchar_t home_path[MAXLEN]; + + if (isEnvVarSet(L"PYLAUNCHER_DEBUG")) { + setvbuf(stderr, (char *)NULL, _IONBF, 0); + log_fp = stderr; + } + + exitCode = calculate_pyvenvcfg_path(pyvenvcfg_path, MAXLEN); + if (exitCode) return exitCode; + + exitCode = read_home(pyvenvcfg_path, home_path, MAXLEN); + if (exitCode) return exitCode; + + exitCode = locate_python(home_path, MAXLEN); + if (exitCode) return exitCode; + + // We do not update argv[0] to point at the target runtime, and so we do not + // pass through our original argv[0] in an environment variable. + //exitCode = smuggle_path(); + //if (exitCode) return exitCode; + + exitCode = launch(home_path, GetCommandLineW()); + return exitCode; +} + + +#if defined(_WINDOWS) + +int WINAPI wWinMain(HINSTANCE hInstance, HINSTANCE hPrevInstance, + LPWSTR lpstrCmd, int nShow) +{ + return process(__argc, __wargv); +} + +#else + +int cdecl wmain(int argc, wchar_t ** argv) +{ + return process(argc, argv); +} + +#endif diff --git a/PCbuild/_asyncio.vcxproj b/PCbuild/_asyncio.vcxproj index ed1e1bc0a420dc..76b0ffd660dba0 100644 --- a/PCbuild/_asyncio.vcxproj +++ b/PCbuild/_asyncio.vcxproj @@ -79,7 +79,7 @@ - .pyd + $(PyStdlibPydExt) diff --git a/PCbuild/_bz2.vcxproj b/PCbuild/_bz2.vcxproj index 3fe95fbf83993a..e0dc6ec187a08d 100644 --- a/PCbuild/_bz2.vcxproj +++ b/PCbuild/_bz2.vcxproj @@ -80,7 +80,7 @@ - .pyd + $(PyStdlibPydExt) diff --git a/PCbuild/_ctypes.vcxproj b/PCbuild/_ctypes.vcxproj index 253da31e9ce182..63d5fa49cd4e17 100644 --- a/PCbuild/_ctypes.vcxproj +++ b/PCbuild/_ctypes.vcxproj @@ -79,7 +79,7 @@ - .pyd + $(PyStdlibPydExt) diff --git a/PCbuild/_ctypes_test.vcxproj b/PCbuild/_ctypes_test.vcxproj index 8a01e743a4d86f..97354739c09834 100644 --- a/PCbuild/_ctypes_test.vcxproj +++ b/PCbuild/_ctypes_test.vcxproj @@ -80,7 +80,7 @@ - .pyd + $(PyStdlibPydExt) diff --git a/PCbuild/_decimal.vcxproj b/PCbuild/_decimal.vcxproj index 0916f1a2d37887..490d7df87eb1c6 100644 --- a/PCbuild/_decimal.vcxproj +++ b/PCbuild/_decimal.vcxproj @@ -79,7 +79,7 @@ - .pyd + $(PyStdlibPydExt) diff --git a/PCbuild/_elementtree.vcxproj b/PCbuild/_elementtree.vcxproj index 8da5244bac0cb6..8c9c0e42f7fe3e 100644 --- a/PCbuild/_elementtree.vcxproj +++ b/PCbuild/_elementtree.vcxproj @@ -79,7 +79,7 @@ - .pyd + $(PyStdlibPydExt) diff --git a/PCbuild/_freeze_module.vcxproj b/PCbuild/_freeze_module.vcxproj index f8c5fafa561efa..49f529ebbc2f9b 100644 --- a/PCbuild/_freeze_module.vcxproj +++ b/PCbuild/_freeze_module.vcxproj @@ -191,6 +191,7 @@ + @@ -207,6 +208,9 @@ + + + @@ -221,12 +225,14 @@ + + @@ -266,117 +272,117 @@ importlib._bootstrap $(IntDir)importlib._bootstrap.g.h - $(PySourcePath)Python\frozen_modules\importlib._bootstrap.h + $(GeneratedFrozenModulesDir)Python\frozen_modules\importlib._bootstrap.h importlib._bootstrap_external $(IntDir)importlib._bootstrap_external.g.h - $(PySourcePath)Python\frozen_modules\importlib._bootstrap_external.h + $(GeneratedFrozenModulesDir)Python\frozen_modules\importlib._bootstrap_external.h zipimport $(IntDir)zipimport.g.h - $(PySourcePath)Python\frozen_modules\zipimport.h + $(GeneratedFrozenModulesDir)Python\frozen_modules\zipimport.h abc $(IntDir)abc.g.h - $(PySourcePath)Python\frozen_modules\abc.h + $(GeneratedFrozenModulesDir)Python\frozen_modules\abc.h codecs $(IntDir)codecs.g.h - $(PySourcePath)Python\frozen_modules\codecs.h + $(GeneratedFrozenModulesDir)Python\frozen_modules\codecs.h io $(IntDir)io.g.h - $(PySourcePath)Python\frozen_modules\io.h + $(GeneratedFrozenModulesDir)Python\frozen_modules\io.h _collections_abc $(IntDir)_collections_abc.g.h - $(PySourcePath)Python\frozen_modules\_collections_abc.h + $(GeneratedFrozenModulesDir)Python\frozen_modules\_collections_abc.h _sitebuiltins $(IntDir)_sitebuiltins.g.h - $(PySourcePath)Python\frozen_modules\_sitebuiltins.h + $(GeneratedFrozenModulesDir)Python\frozen_modules\_sitebuiltins.h genericpath $(IntDir)genericpath.g.h - $(PySourcePath)Python\frozen_modules\genericpath.h + $(GeneratedFrozenModulesDir)Python\frozen_modules\genericpath.h ntpath $(IntDir)ntpath.g.h - $(PySourcePath)Python\frozen_modules\ntpath.h + $(GeneratedFrozenModulesDir)Python\frozen_modules\ntpath.h posixpath $(IntDir)posixpath.g.h - $(PySourcePath)Python\frozen_modules\posixpath.h + $(GeneratedFrozenModulesDir)Python\frozen_modules\posixpath.h os $(IntDir)os.g.h - $(PySourcePath)Python\frozen_modules\os.h + $(GeneratedFrozenModulesDir)Python\frozen_modules\os.h site $(IntDir)site.g.h - $(PySourcePath)Python\frozen_modules\site.h + $(GeneratedFrozenModulesDir)Python\frozen_modules\site.h stat $(IntDir)stat.g.h - $(PySourcePath)Python\frozen_modules\stat.h + $(GeneratedFrozenModulesDir)Python\frozen_modules\stat.h importlib.util $(IntDir)importlib.util.g.h - $(PySourcePath)Python\frozen_modules\importlib.util.h + $(GeneratedFrozenModulesDir)Python\frozen_modules\importlib.util.h importlib.machinery $(IntDir)importlib.machinery.g.h - $(PySourcePath)Python\frozen_modules\importlib.machinery.h + $(GeneratedFrozenModulesDir)Python\frozen_modules\importlib.machinery.h runpy $(IntDir)runpy.g.h - $(PySourcePath)Python\frozen_modules\runpy.h + $(GeneratedFrozenModulesDir)Python\frozen_modules\runpy.h __hello__ $(IntDir)__hello__.g.h - $(PySourcePath)Python\frozen_modules\__hello__.h + $(GeneratedFrozenModulesDir)Python\frozen_modules\__hello__.h __phello__ $(IntDir)__phello__.g.h - $(PySourcePath)Python\frozen_modules\__phello__.h + $(GeneratedFrozenModulesDir)Python\frozen_modules\__phello__.h __phello__.ham $(IntDir)__phello__.ham.g.h - $(PySourcePath)Python\frozen_modules\__phello__.ham.h + $(GeneratedFrozenModulesDir)Python\frozen_modules\__phello__.ham.h __phello__.ham.eggs $(IntDir)__phello__.ham.eggs.g.h - $(PySourcePath)Python\frozen_modules\__phello__.ham.eggs.h + $(GeneratedFrozenModulesDir)Python\frozen_modules\__phello__.ham.eggs.h __phello__.spam $(IntDir)__phello__.spam.g.h - $(PySourcePath)Python\frozen_modules\__phello__.spam.h + $(GeneratedFrozenModulesDir)Python\frozen_modules\__phello__.spam.h frozen_only $(IntDir)frozen_only.g.h - $(PySourcePath)Python\frozen_modules\frozen_only.h + $(GeneratedFrozenModulesDir)Python\frozen_modules\frozen_only.h @@ -385,34 +391,34 @@ getpath $(IntDir)getpath.g.h - $(PySourcePath)Python\frozen_modules\getpath.h + $(GeneratedFrozenModulesDir)Python\frozen_modules\getpath.h - - - - - - - - - - - - - - - - - - - - - - - + + + + + + + + + + + + + + + + + + + + + + + @@ -484,7 +490,7 @@ $(IntDir)\deepfreeze_mappings.txt Overwrite="true" Lines="@(FrozenModule->'%(FullPath):%(FrozenId)')" /> - + @@ -493,7 +499,7 @@ $(IntDir)\deepfreeze_mappings.txt - + diff --git a/PCbuild/_freeze_module.vcxproj.filters b/PCbuild/_freeze_module.vcxproj.filters index 1c5a6d623f4dad..5b1bd7552b4cd9 100644 --- a/PCbuild/_freeze_module.vcxproj.filters +++ b/PCbuild/_freeze_module.vcxproj.filters @@ -46,6 +46,9 @@ Source Files + + Python + Source Files @@ -166,6 +169,15 @@ Source Files + + Source Files + + + Source Files + + + Source Files + Source Files @@ -241,6 +253,9 @@ Source Files + + Source Files + Source Files @@ -280,6 +295,9 @@ Source Files + + Source Files + Source Files diff --git a/PCbuild/_hashlib.vcxproj b/PCbuild/_hashlib.vcxproj index 6dad8183c57ae3..2cd205224bc089 100644 --- a/PCbuild/_hashlib.vcxproj +++ b/PCbuild/_hashlib.vcxproj @@ -79,7 +79,7 @@ - .pyd + $(PyStdlibPydExt) diff --git a/PCbuild/_lzma.vcxproj b/PCbuild/_lzma.vcxproj index fe076a6fc57168..40107d4b76cd53 100644 --- a/PCbuild/_lzma.vcxproj +++ b/PCbuild/_lzma.vcxproj @@ -79,7 +79,7 @@ - .pyd + $(PyStdlibPydExt) diff --git a/PCbuild/_multiprocessing.vcxproj b/PCbuild/_multiprocessing.vcxproj index 77b6bfc8e1e483..a65397f532aa86 100644 --- a/PCbuild/_multiprocessing.vcxproj +++ b/PCbuild/_multiprocessing.vcxproj @@ -79,7 +79,7 @@ - .pyd + $(PyStdlibPydExt) diff --git a/PCbuild/_overlapped.vcxproj b/PCbuild/_overlapped.vcxproj index 9e60d3b5db336c..224bf05d5303a0 100644 --- a/PCbuild/_overlapped.vcxproj +++ b/PCbuild/_overlapped.vcxproj @@ -79,7 +79,7 @@ - .pyd + $(PyStdlibPydExt) diff --git a/PCbuild/_queue.vcxproj b/PCbuild/_queue.vcxproj index 8065b235851686..80a1c3c6a4ad3e 100644 --- a/PCbuild/_queue.vcxproj +++ b/PCbuild/_queue.vcxproj @@ -79,7 +79,7 @@ - .pyd + $(PyStdlibPydExt) diff --git a/PCbuild/_socket.vcxproj b/PCbuild/_socket.vcxproj index 78fa4d6729abb9..41af0895921bbb 100644 --- a/PCbuild/_socket.vcxproj +++ b/PCbuild/_socket.vcxproj @@ -79,7 +79,7 @@ - .pyd + $(PyStdlibPydExt) diff --git a/PCbuild/_sqlite3.vcxproj b/PCbuild/_sqlite3.vcxproj index 57c7413671e54e..9ae0a0fc3a009d 100644 --- a/PCbuild/_sqlite3.vcxproj +++ b/PCbuild/_sqlite3.vcxproj @@ -79,7 +79,7 @@ - .pyd + $(PyStdlibPydExt) diff --git a/PCbuild/_ssl.vcxproj b/PCbuild/_ssl.vcxproj index 226ff506f8c62b..d4e1affab031d7 100644 --- a/PCbuild/_ssl.vcxproj +++ b/PCbuild/_ssl.vcxproj @@ -79,7 +79,7 @@ - .pyd + $(PyStdlibPydExt) diff --git a/PCbuild/_testbuffer.vcxproj b/PCbuild/_testbuffer.vcxproj index 917d7ae50feb14..4e721e8ce09f0c 100644 --- a/PCbuild/_testbuffer.vcxproj +++ b/PCbuild/_testbuffer.vcxproj @@ -80,7 +80,7 @@ - .pyd + $(PyStdlibPydExt) diff --git a/PCbuild/_testcapi.vcxproj b/PCbuild/_testcapi.vcxproj index 1c15541d3ec735..6911aacab29b97 100644 --- a/PCbuild/_testcapi.vcxproj +++ b/PCbuild/_testcapi.vcxproj @@ -80,7 +80,7 @@ - .pyd + $(PyStdlibPydExt) diff --git a/PCbuild/_testclinic.vcxproj b/PCbuild/_testclinic.vcxproj index e319b3c0f42e0f..ef981332c6ab03 100644 --- a/PCbuild/_testclinic.vcxproj +++ b/PCbuild/_testclinic.vcxproj @@ -80,7 +80,7 @@ - .pyd + $(PyStdlibPydExt) @@ -107,4 +107,4 @@ - \ No newline at end of file + diff --git a/PCbuild/_testclinic_limited.vcxproj b/PCbuild/_testclinic_limited.vcxproj index b00b2be491b423..183a55080e8693 100644 --- a/PCbuild/_testclinic_limited.vcxproj +++ b/PCbuild/_testclinic_limited.vcxproj @@ -79,7 +79,7 @@ - .pyd + $(PyStdlibPydExt) diff --git a/PCbuild/_testconsole.vcxproj b/PCbuild/_testconsole.vcxproj index 5d7e14eff10294..69d312b17a5a62 100644 --- a/PCbuild/_testconsole.vcxproj +++ b/PCbuild/_testconsole.vcxproj @@ -80,7 +80,7 @@ - .pyd + $(PyStdlibPydExt) diff --git a/PCbuild/_testimportmultiple.vcxproj b/PCbuild/_testimportmultiple.vcxproj index 6d80d5779f24d8..c35ac83c1c739f 100644 --- a/PCbuild/_testimportmultiple.vcxproj +++ b/PCbuild/_testimportmultiple.vcxproj @@ -80,7 +80,7 @@ - .pyd + $(PyStdlibPydExt) diff --git a/PCbuild/_testinternalcapi.vcxproj b/PCbuild/_testinternalcapi.vcxproj index 558f66ca95cd33..a825cac9138674 100644 --- a/PCbuild/_testinternalcapi.vcxproj +++ b/PCbuild/_testinternalcapi.vcxproj @@ -80,7 +80,7 @@ - .pyd + $(PyStdlibPydExt) diff --git a/PCbuild/_testmultiphase.vcxproj b/PCbuild/_testmultiphase.vcxproj index 430eb528cc3927..e730fe308ab835 100644 --- a/PCbuild/_testmultiphase.vcxproj +++ b/PCbuild/_testmultiphase.vcxproj @@ -80,7 +80,7 @@ - .pyd + $(PyStdlibPydExt) diff --git a/PCbuild/_testsinglephase.vcxproj b/PCbuild/_testsinglephase.vcxproj index fb4bcd953923f8..bf4dabf66c1040 100644 --- a/PCbuild/_testsinglephase.vcxproj +++ b/PCbuild/_testsinglephase.vcxproj @@ -80,7 +80,7 @@ - .pyd + $(PyStdlibPydExt) diff --git a/PCbuild/_tkinter.vcxproj b/PCbuild/_tkinter.vcxproj index 30cedcbb43de76..117488a01621cc 100644 --- a/PCbuild/_tkinter.vcxproj +++ b/PCbuild/_tkinter.vcxproj @@ -79,7 +79,7 @@ - .pyd + $(PyStdlibPydExt) diff --git a/PCbuild/_uuid.vcxproj b/PCbuild/_uuid.vcxproj index 2437b7eb2d9399..50d81cc7916dbd 100644 --- a/PCbuild/_uuid.vcxproj +++ b/PCbuild/_uuid.vcxproj @@ -80,7 +80,7 @@ - .pyd + $(PyStdlibPydExt) diff --git a/PCbuild/_wmi.vcxproj b/PCbuild/_wmi.vcxproj index c1914a3fa5a1bf..22fa8960982528 100644 --- a/PCbuild/_wmi.vcxproj +++ b/PCbuild/_wmi.vcxproj @@ -80,7 +80,7 @@ - .pyd + $(PyStdlibPydExt) diff --git a/PCbuild/_zoneinfo.vcxproj b/PCbuild/_zoneinfo.vcxproj index 6e6389c3773397..47b5bfa5b8815a 100644 --- a/PCbuild/_zoneinfo.vcxproj +++ b/PCbuild/_zoneinfo.vcxproj @@ -79,7 +79,7 @@ - .pyd + $(PyStdlibPydExt) diff --git a/PCbuild/build.bat b/PCbuild/build.bat index e61267b5852a8f..83b50db4467033 100644 --- a/PCbuild/build.bat +++ b/PCbuild/build.bat @@ -36,6 +36,7 @@ echo. overrides -c and -d echo. --disable-gil Enable experimental support for running without the GIL. echo. --test-marker Enable the test marker within the build. echo. --regen Regenerate all opcodes, grammar and tokens. +echo. --experimental-jit Enable the experimental just-in-time compiler. echo. echo.Available flags to avoid building certain modules. echo.These flags have no effect if '-e' is not given: @@ -85,6 +86,7 @@ if "%~1"=="--disable-gil" (set UseDisableGil=true) & shift & goto CheckOpts if "%~1"=="--test-marker" (set UseTestMarker=true) & shift & goto CheckOpts if "%~1"=="-V" shift & goto Version if "%~1"=="--regen" (set Regen=true) & shift & goto CheckOpts +if "%~1"=="--experimental-jit" (set UseJIT=true) & shift & goto CheckOpts rem These use the actual property names used by MSBuild. We could just let rem them in through the environment, but we specify them on the command line rem anyway for visibility so set defaults after this @@ -176,6 +178,7 @@ echo on /p:IncludeSSL=%IncludeSSL% /p:IncludeTkinter=%IncludeTkinter%^ /p:DisableGil=%UseDisableGil%^ /p:UseTestMarker=%UseTestMarker% %GITProperty%^ + /p:UseJIT=%UseJIT%^ %1 %2 %3 %4 %5 %6 %7 %8 %9 @echo off diff --git a/PCbuild/get_externals.bat b/PCbuild/get_externals.bat index 3919c0592ec00d..60ce12b725e233 100644 --- a/PCbuild/get_externals.bat +++ b/PCbuild/get_externals.bat @@ -53,12 +53,12 @@ echo.Fetching external libraries... set libraries= set libraries=%libraries% bzip2-1.0.8 if NOT "%IncludeLibffiSrc%"=="false" set libraries=%libraries% libffi-3.4.4 -if NOT "%IncludeSSLSrc%"=="false" set libraries=%libraries% openssl-3.0.11 -set libraries=%libraries% sqlite-3.44.2.0 +if NOT "%IncludeSSLSrc%"=="false" set libraries=%libraries% openssl-3.0.13 +set libraries=%libraries% sqlite-3.45.1.0 if NOT "%IncludeTkinterSrc%"=="false" set libraries=%libraries% tcl-core-8.6.13.1 if NOT "%IncludeTkinterSrc%"=="false" set libraries=%libraries% tk-8.6.13.1 set libraries=%libraries% xz-5.2.5 -set libraries=%libraries% zlib-1.2.13 +set libraries=%libraries% zlib-1.3.1 for %%e in (%libraries%) do ( if exist "%EXTERNALS_DIR%\%%e" ( @@ -76,7 +76,7 @@ echo.Fetching external binaries... set binaries= if NOT "%IncludeLibffi%"=="false" set binaries=%binaries% libffi-3.4.4 -if NOT "%IncludeSSL%"=="false" set binaries=%binaries% openssl-bin-3.0.11 +if NOT "%IncludeSSL%"=="false" set binaries=%binaries% openssl-bin-3.0.13 if NOT "%IncludeTkinter%"=="false" set binaries=%binaries% tcltk-8.6.13.1 if NOT "%IncludeSSLSrc%"=="false" set binaries=%binaries% nasm-2.11.06 diff --git a/PCbuild/pyexpat.vcxproj b/PCbuild/pyexpat.vcxproj index 001f8afd89b9e9..dc9161a8b290f9 100644 --- a/PCbuild/pyexpat.vcxproj +++ b/PCbuild/pyexpat.vcxproj @@ -79,7 +79,7 @@ - .pyd + $(PyStdlibPydExt) diff --git a/PCbuild/pyproject.props b/PCbuild/pyproject.props index 68c0550f7603b7..9c85e5efa4af4a 100644 --- a/PCbuild/pyproject.props +++ b/PCbuild/pyproject.props @@ -12,6 +12,7 @@ $(IntDir.Replace(`\\`, `\`)) $(Py_IntDir)\$(MajorVersionNumber)$(MinorVersionNumber)$(ArchName)_$(Configuration)\pythoncore\ + $(Py_IntDir)\$(MajorVersionNumber)$(MinorVersionNumber)_frozen\ $(ProjectName) $(TargetName)$(PyDebugExt) false @@ -23,11 +24,19 @@ false + + $(TargetName)$(TargetExt) + <_TargetNameSep>$(TargetNameExt.LastIndexOf(`.`)) + $(TargetNameExt.Substring(0, $(_TargetNameSep))) + $(TargetNameExt.Substring($(_TargetNameSep))) + + <_VCToolsVersion>$([System.Version]::Parse(`$(VCToolsVersion)`).Major).$([System.Version]::Parse(`$(VCToolsVersion)`).Minor) true + true @@ -36,7 +45,7 @@ <_PlatformPreprocessorDefinition>_WIN32; <_PlatformPreprocessorDefinition Condition="$(Platform) == 'x64'">_WIN64; <_PlatformPreprocessorDefinition Condition="$(Platform) == 'x64' and $(PlatformToolset) != 'ClangCL'">_M_X64;$(_PlatformPreprocessorDefinition) - <_Py3NamePreprocessorDefinition>PY3_DLLNAME=L"$(Py3DllName)"; + <_Py3NamePreprocessorDefinition>PY3_DLLNAME=L"$(Py3DllName)$(PyDebugExt)"; @@ -61,6 +70,7 @@ -Wno-deprecated-non-prototype -Wno-unused-label -Wno-pointer-sign -Wno-incompatible-pointer-types-discards-qualifiers -Wno-unused-function %(AdditionalOptions) -flto %(AdditionalOptions) -d2pattern-opt-disable:-932189325 %(AdditionalOptions) + -d2ssa-patterns-all- %(AdditionalOptions) /sourceDependencies "$(IntDir.Trim(`\`))" %(AdditionalOptions) @@ -155,8 +165,8 @@ public override bool Execute() { - - + + $([System.IO.Path]::GetFullPath($(MSBuildThisFileDirectory)\..\)) $(PySourcePath)\ - + $(PySourcePath)PCbuild\win32\ $(Py_OutDir)\win32\ @@ -52,7 +52,7 @@ $(PySourcePath)PCbuild\$(ArchName)\ $(BuildPath)\ $(BuildPath)instrumented\ - + ..\\.. ..\\..\\.. @@ -68,38 +68,35 @@ - $(ExternalsDir)sqlite-3.44.2.0\ + $(ExternalsDir)sqlite-3.45.1.0\ $(ExternalsDir)bzip2-1.0.8\ $(ExternalsDir)xz-5.2.5\ $(ExternalsDir)libffi-3.4.4\ $(libffiDir)$(ArchName)\ $(libffiOutDir)include - $(ExternalsDir)openssl-3.0.11\ - $(ExternalsDir)openssl-bin-3.0.11\$(ArchName)\ + $(ExternalsDir)openssl-3.0.13\ + $(ExternalsDir)openssl-bin-3.0.13\$(ArchName)\ $(opensslOutDir)include $(ExternalsDir)\nasm-2.11.06\ - $(ExternalsDir)\zlib-1.2.13\ + $(ExternalsDir)\zlib-1.3.1\ _d - + -test - + -32 -arm32 -arm64 - - - $(BuildPath)python$(PyDebugExt).exe true - + true @@ -141,7 +138,7 @@ @@ -223,22 +220,55 @@ )) )) $([msbuild]::Add($(Field3Value), 9000)) - + + + python$(MajorVersionNumber).$(MinorVersionNumber)t + python + $(BuildPath)$(PyExeName)$(PyDebugExt).exe + pythonw$(MajorVersionNumber).$(MinorVersionNumber)t + pythonw + - python$(MajorVersionNumber)$(MinorVersionNumber)$(PyDebugExt) + python$(MajorVersionNumber)$(MinorVersionNumber)t$(PyDebugExt) + python$(MajorVersionNumber)$(MinorVersionNumber)$(PyDebugExt) - python3$(PyDebugExt) + python3t + python3 .cp$(MajorVersionNumber)$(MinorVersionNumber)-win32 .cp$(MajorVersionNumber)$(MinorVersionNumber)-win_arm32 .cp$(MajorVersionNumber)$(MinorVersionNumber)-win_arm64 .cp$(MajorVersionNumber)$(MinorVersionNumber)-win_amd64 - + $(MajorVersionNumber).$(MinorVersionNumber)$(PyArchExt)$(PyTestExt) + + + .cp$(MajorVersionNumber)$(MinorVersionNumber)t-win32 + .cp$(MajorVersionNumber)$(MinorVersionNumber)t-win_arm32 + .cp$(MajorVersionNumber)$(MinorVersionNumber)t-win_arm64 + .cp$(MajorVersionNumber)$(MinorVersionNumber)t-win_amd64 + + + $(MajorVersionNumber).$(MinorVersionNumber)t$(PyArchExt)$(PyTestExt) + + + + + .pyd - + + + $(FreethreadedPydTag) + + + $(PydTag).pyd + + + $(FreethreadedSysWinVer) + + diff --git a/PCbuild/python.vcxproj b/PCbuild/python.vcxproj index 8b733865962373..4a99ffc677c287 100644 --- a/PCbuild/python.vcxproj +++ b/PCbuild/python.vcxproj @@ -72,6 +72,7 @@ + $(PyExeName) Application false MultiByte @@ -94,8 +95,11 @@ Console - 2000000 + 2000000 12000000 + 12000000 + + 3000000 @@ -129,7 +133,7 @@ +"$(OutDir)$(PyExeName)$(PyDebugExt).exe" "$(PySourcePath)PC\validate_ucrtbase.py" $(UcrtName)' ContinueOnError="true" /> @@ -142,7 +146,7 @@ set PYTHONPATH=$(PySourcePath)Lib @echo Running $(Configuration)^|$(Platform) interpreter... @setlocal @set PYTHONHOME=$(PySourcePath) -@"$(OutDir)python$(PyDebugExt).exe" %* +@"$(OutDir)$(PyExeName)$(PyDebugExt).exe" %* <_ExistingContent Condition="Exists('$(PySourcePath)python.bat')">$([System.IO.File]::ReadAllText('$(PySourcePath)python.bat')) diff --git a/PCbuild/python3dll.vcxproj b/PCbuild/python3dll.vcxproj index ec22e6fc76e584..235ea1cf9d33fb 100644 --- a/PCbuild/python3dll.vcxproj +++ b/PCbuild/python3dll.vcxproj @@ -70,12 +70,12 @@ {885D4898-D08D-4091-9C40-C700CFE3FC5A} python3dll Win32Proj - python3 false + $(Py3DllName) DynamicLibrary diff --git a/PCbuild/pythoncore.vcxproj b/PCbuild/pythoncore.vcxproj index c90ad1a3592f67..4cc0ca4b9af8de 100644 --- a/PCbuild/pythoncore.vcxproj +++ b/PCbuild/pythoncore.vcxproj @@ -104,6 +104,7 @@ $(zlibDir);%(AdditionalIncludeDirectories) _USRDLL;Py_BUILD_CORE;Py_BUILD_CORE_BUILTIN;Py_ENABLE_SHARED;MS_DLL_ID="$(SysWinVer)";%(PreprocessorDefinitions) _Py_HAVE_ZLIB;%(PreprocessorDefinitions) + _Py_JIT;%(PreprocessorDefinitions) version.lib;ws2_32.lib;pathcch.lib;bcrypt.lib;%(AdditionalDependencies) @@ -111,6 +112,7 @@ + $(GeneratedFrozenModulesDir);%(AdditionalIncludeDirectories) PREFIX=NULL; EXEC_PREFIX=NULL; @@ -120,7 +122,6 @@ PLATLIBDIR="DLLs"; %(PreprocessorDefinitions) - $(PySourcePath);%(AdditionalIncludeDirectories) @@ -205,6 +206,7 @@ + @@ -231,6 +233,7 @@ + @@ -246,6 +249,7 @@ + @@ -254,6 +258,7 @@ + @@ -294,7 +299,6 @@ - @@ -424,6 +428,7 @@ + @@ -549,6 +554,7 @@ + @@ -562,8 +568,13 @@ - + + $(GeneratedFrozenModulesDir)Python;%(AdditionalIncludeDirectories) + + + + @@ -578,12 +589,14 @@ + + @@ -617,7 +630,7 @@ - + diff --git a/PCbuild/pythoncore.vcxproj.filters b/PCbuild/pythoncore.vcxproj.filters index a96ca24cf08b66..ceaa21217267cf 100644 --- a/PCbuild/pythoncore.vcxproj.filters +++ b/PCbuild/pythoncore.vcxproj.filters @@ -546,6 +546,9 @@ Include\internal + + Include\internal + Include\internal @@ -618,6 +621,12 @@ Include\internal + + Include\internal + + + Include\internal + Include\internal @@ -663,6 +672,9 @@ Include\cpython + + Include\internal + Include\internal @@ -687,6 +699,9 @@ Include\internal + + Include\internal + Include\internal @@ -798,9 +813,6 @@ Include\internal - - Include\internal - Include\internal\mimalloc @@ -932,6 +944,9 @@ Modules + + Modules + Modules @@ -1241,6 +1256,9 @@ Python + + Python + Python @@ -1280,6 +1298,15 @@ Python + + Python + + + Python + + + Python + Python @@ -1319,6 +1346,9 @@ Source Files + + Python + Source Files @@ -1337,6 +1367,9 @@ Python + + Python + Python diff --git a/PCbuild/pythonw.vcxproj b/PCbuild/pythonw.vcxproj index e23635e5ea9411..d08c210ef8a1dc 100644 --- a/PCbuild/pythonw.vcxproj +++ b/PCbuild/pythonw.vcxproj @@ -73,6 +73,7 @@ + $(PyWExeName) Application false @@ -89,8 +90,11 @@ - 2000000 - 8000000 + 2000000 + 12000000 + 12000000 + + 3000000 diff --git a/PCbuild/readme.txt b/PCbuild/readme.txt index b9d76515c383f7..387565515fa0b0 100644 --- a/PCbuild/readme.txt +++ b/PCbuild/readme.txt @@ -189,7 +189,7 @@ _ssl again when building. _sqlite3 - Wraps SQLite 3.44.2, which is itself built by sqlite3.vcxproj + Wraps SQLite 3.45.1, which is itself built by sqlite3.vcxproj Homepage: https://www.sqlite.org/ _tkinter diff --git a/PCbuild/regen.targets b/PCbuild/regen.targets index cc9469c7ddd726..a90620d6ca8b7d 100644 --- a/PCbuild/regen.targets +++ b/PCbuild/regen.targets @@ -28,6 +28,9 @@ <_KeywordSources Include="$(PySourcePath)Grammar\python.gram;$(PySourcePath)Grammar\Tokens" /> <_KeywordOutputs Include="$(PySourcePath)Lib\keyword.py" /> + + <_JITSources Include="$(PySourcePath)Python\executor_cases.c.h;$(GeneratedPyConfigDir)pyconfig.h;$(PySourcePath)Tools\jit\**"/> + <_JITOutputs Include="$(GeneratedPyConfigDir)jit_stencils.h"/> @@ -76,10 +79,28 @@ + + + + aarch64-pc-windows-msvc + i686-pc-windows-msvc + x86_64-pc-windows-msvc + $(JITArgs) --debug + + + - + + + diff --git a/PCbuild/rt.bat b/PCbuild/rt.bat index 332ba5edcf4082..293f99ae135faa 100644 --- a/PCbuild/rt.bat +++ b/PCbuild/rt.bat @@ -9,6 +9,7 @@ rem -q runs the tests just once, and without deleting .pyc files. rem -p or -win32, -x64, -arm32, -arm64 rem Run the specified architecture of python (or python_d if -d rem was specified). If omitted, uses %PREFIX% if set or 64-bit. +rem --disable-gil Run free-threaded build. rem All leading instances of these switches are shifted off, and rem whatever remains (up to 9 arguments) is passed to regrtest.py. rem For example, @@ -29,6 +30,7 @@ rem rt -u "network,largefile" setlocal set pcbuild=%~dp0 +set pyname=python set suffix= set qmode= set dashO= @@ -39,15 +41,18 @@ set exe= if "%1"=="-O" (set dashO=-O) & shift & goto CheckOpts if "%1"=="-q" (set qmode=yes) & shift & goto CheckOpts if "%1"=="-d" (set suffix=_d) & shift & goto CheckOpts +rem HACK: Need some way to infer the version number in this script +if "%1"=="--disable-gil" (set pyname=python3.13t) & shift & goto CheckOpts if "%1"=="-win32" (set prefix=%pcbuild%win32) & shift & goto CheckOpts if "%1"=="-x64" (set prefix=%pcbuild%amd64) & shift & goto CheckOpts +if "%1"=="-amd64" (set prefix=%pcbuild%amd64) & shift & goto CheckOpts if "%1"=="-arm64" (set prefix=%pcbuild%arm64) & shift & goto CheckOpts if "%1"=="-arm32" (set prefix=%pcbuild%arm32) & shift & goto CheckOpts if "%1"=="-p" (call :SetPlatform %~2) & shift & shift & goto CheckOpts if NOT "%1"=="" (set regrtestargs=%regrtestargs% %1) & shift & goto CheckOpts if not defined prefix set prefix=%pcbuild%amd64 -set exe=%prefix%\python%suffix%.exe +set exe=%prefix%\%pyname%%suffix%.exe set cmd="%exe%" %dashO% -m test %regrtestargs% if defined qmode goto Qmode diff --git a/PCbuild/select.vcxproj b/PCbuild/select.vcxproj index 750a713949919a..d7448fd4d72380 100644 --- a/PCbuild/select.vcxproj +++ b/PCbuild/select.vcxproj @@ -78,7 +78,7 @@ - .pyd + $(PyStdlibPydExt) diff --git a/PCbuild/sqlite3.vcxproj b/PCbuild/sqlite3.vcxproj index c502d51833b91a..6bcc4e913c8e77 100644 --- a/PCbuild/sqlite3.vcxproj +++ b/PCbuild/sqlite3.vcxproj @@ -69,12 +69,12 @@ {A1A295E5-463C-437F-81CA-1F32367685DA} sqlite3 - .pyd false + $(PyStdlibPydExt) DynamicLibrary NotSet diff --git a/PCbuild/unicodedata.vcxproj b/PCbuild/unicodedata.vcxproj index addef753359ed6..781f938e2ab78e 100644 --- a/PCbuild/unicodedata.vcxproj +++ b/PCbuild/unicodedata.vcxproj @@ -79,7 +79,7 @@ - .pyd + $(PyStdlibPydExt) diff --git a/PCbuild/venvlauncher.vcxproj b/PCbuild/venvlauncher.vcxproj index 123e84ec4e3682..1193e032245c94 100644 --- a/PCbuild/venvlauncher.vcxproj +++ b/PCbuild/venvlauncher.vcxproj @@ -69,12 +69,13 @@ {494BAC80-A60C-43A9-99E7-ACB691CE2C4D} venvlauncher - venvlauncher false + venvlauncher + $(TargetName)t Application MultiByte @@ -91,19 +92,19 @@ - _CONSOLE;VENV_REDIRECT;%(PreprocessorDefinitions) + EXENAME=L"$(PyExeName)$(PyDebugExt).exe";_CONSOLE;%(PreprocessorDefinitions) MultiThreaded PY_ICON;%(PreprocessorDefinitions) - version.lib;%(AdditionalDependencies) + pathcch.lib;%(AdditionalDependencies) Console - + diff --git a/PCbuild/venvlauncher.vcxproj.filters b/PCbuild/venvlauncher.vcxproj.filters index ec13936bf6cb7e..56a0f005a3fa2a 100644 --- a/PCbuild/venvlauncher.vcxproj.filters +++ b/PCbuild/venvlauncher.vcxproj.filters @@ -19,7 +19,7 @@ - + Source Files diff --git a/PCbuild/venvwlauncher.vcxproj b/PCbuild/venvwlauncher.vcxproj index b8504d5d08e52f..1b61718201367f 100644 --- a/PCbuild/venvwlauncher.vcxproj +++ b/PCbuild/venvwlauncher.vcxproj @@ -69,12 +69,13 @@ {FDB84CBB-2FB6-47C8-A2D6-091E0833239D} venvwlauncher - venvwlauncher false + venvwlauncher + $(TargetName)t Application MultiByte @@ -91,19 +92,19 @@ - _WINDOWS;VENV_REDIRECT;%(PreprocessorDefinitions) + EXENAME=L"$(PyExeName)$(PyDebugExt).exe";_WINDOWS;%(PreprocessorDefinitions) MultiThreaded PYW_ICON;%(PreprocessorDefinitions) - version.lib;%(AdditionalDependencies) + pathcch.lib;%(AdditionalDependencies) Windows - + diff --git a/PCbuild/venvwlauncher.vcxproj.filters b/PCbuild/venvwlauncher.vcxproj.filters index 8addc13e977e7a..61a514395e82dc 100644 --- a/PCbuild/venvwlauncher.vcxproj.filters +++ b/PCbuild/venvwlauncher.vcxproj.filters @@ -9,7 +9,7 @@ - + Source Files diff --git a/PCbuild/winsound.vcxproj b/PCbuild/winsound.vcxproj index 32cedc9b444902..c26029b15a339f 100644 --- a/PCbuild/winsound.vcxproj +++ b/PCbuild/winsound.vcxproj @@ -80,7 +80,7 @@ - .pyd + $(PyStdlibPydExt) diff --git a/PCbuild/xxlimited.vcxproj b/PCbuild/xxlimited.vcxproj index 1c776fb0da3e72..093e6920c0b76c 100644 --- a/PCbuild/xxlimited.vcxproj +++ b/PCbuild/xxlimited.vcxproj @@ -80,7 +80,7 @@ - .pyd + $(PyStdlibPydExt) diff --git a/PCbuild/xxlimited_35.vcxproj b/PCbuild/xxlimited_35.vcxproj index dd830b3b6aaa91..3f4d4463f24af0 100644 --- a/PCbuild/xxlimited_35.vcxproj +++ b/PCbuild/xxlimited_35.vcxproj @@ -80,7 +80,7 @@ - .pyd + $(PyStdlibPydExt) diff --git a/Parser/asdl_c.py b/Parser/asdl_c.py index c9bf08ed2e0f6d..ce92672bf00776 100755 --- a/Parser/asdl_c.py +++ b/Parser/asdl_c.py @@ -731,7 +731,7 @@ def emit_sequence_constructor(self, name, type): class PyTypesDeclareVisitor(PickleVisitor): def visitProduct(self, prod, name): - self.emit("static PyObject* ast2obj_%s(struct ast_state *state, void*);" % name, 0) + self.emit("static PyObject* ast2obj_%s(struct ast_state *state, struct validator *vstate, void*);" % name, 0) if prod.attributes: self.emit("static const char * const %s_attributes[] = {" % name, 0) for a in prod.attributes: @@ -752,7 +752,7 @@ def visitSum(self, sum, name): ptype = "void*" if is_simple(sum): ptype = get_c_type(name) - self.emit("static PyObject* ast2obj_%s(struct ast_state *state, %s);" % (name, ptype), 0) + self.emit("static PyObject* ast2obj_%s(struct ast_state *state, struct validator *vstate, %s);" % (name, ptype), 0) for t in sum.types: self.visitConstructor(t, name) @@ -984,7 +984,8 @@ def visitModule(self, mod): /* Conversion AST -> Python */ -static PyObject* ast2obj_list(struct ast_state *state, asdl_seq *seq, PyObject* (*func)(struct ast_state *state, void*)) +static PyObject* ast2obj_list(struct ast_state *state, struct validator *vstate, asdl_seq *seq, + PyObject* (*func)(struct ast_state *state, struct validator *vstate, void*)) { Py_ssize_t i, n = asdl_seq_LEN(seq); PyObject *result = PyList_New(n); @@ -992,7 +993,7 @@ def visitModule(self, mod): if (!result) return NULL; for (i = 0; i < n; i++) { - value = func(state, asdl_seq_GET_UNTYPED(seq, i)); + value = func(state, vstate, asdl_seq_GET_UNTYPED(seq, i)); if (!value) { Py_DECREF(result); return NULL; @@ -1002,7 +1003,7 @@ def visitModule(self, mod): return result; } -static PyObject* ast2obj_object(struct ast_state *Py_UNUSED(state), void *o) +static PyObject* ast2obj_object(struct ast_state *Py_UNUSED(state), struct validator *Py_UNUSED(vstate), void *o) { PyObject *op = (PyObject*)o; if (!op) { @@ -1014,7 +1015,7 @@ def visitModule(self, mod): #define ast2obj_identifier ast2obj_object #define ast2obj_string ast2obj_object -static PyObject* ast2obj_int(struct ast_state *Py_UNUSED(state), long b) +static PyObject* ast2obj_int(struct ast_state *Py_UNUSED(state), struct validator *Py_UNUSED(vstate), long b) { return PyLong_FromLong(b); } @@ -1116,8 +1117,6 @@ def visitModule(self, mod): for dfn in mod.dfns: self.visit(dfn) self.file.write(textwrap.dedent(''' - state->recursion_depth = 0; - state->recursion_limit = 0; return 0; } ''')) @@ -1260,7 +1259,7 @@ class ObjVisitor(PickleVisitor): def func_begin(self, name): ctype = get_c_type(name) self.emit("PyObject*", 0) - self.emit("ast2obj_%s(struct ast_state *state, void* _o)" % (name), 0) + self.emit("ast2obj_%s(struct ast_state *state, struct validator *vstate, void* _o)" % (name), 0) self.emit("{", 0) self.emit("%s o = (%s)_o;" % (ctype, ctype), 1) self.emit("PyObject *result = NULL, *value = NULL;", 1) @@ -1268,17 +1267,17 @@ def func_begin(self, name): self.emit('if (!o) {', 1) self.emit("Py_RETURN_NONE;", 2) self.emit("}", 1) - self.emit("if (++state->recursion_depth > state->recursion_limit) {", 1) + self.emit("if (++vstate->recursion_depth > vstate->recursion_limit) {", 1) self.emit("PyErr_SetString(PyExc_RecursionError,", 2) self.emit('"maximum recursion depth exceeded during ast construction");', 3) self.emit("return NULL;", 2) self.emit("}", 1) def func_end(self): - self.emit("state->recursion_depth--;", 1) + self.emit("vstate->recursion_depth--;", 1) self.emit("return result;", 1) self.emit("failed:", 0) - self.emit("state->recursion_depth--;", 1) + self.emit("vstate->recursion_depth--;", 1) self.emit("Py_XDECREF(value);", 1) self.emit("Py_XDECREF(result);", 1) self.emit("return NULL;", 1) @@ -1296,7 +1295,7 @@ def visitSum(self, sum, name): self.visitConstructor(t, i + 1, name) self.emit("}", 1) for a in sum.attributes: - self.emit("value = ast2obj_%s(state, o->%s);" % (a.type, a.name), 1) + self.emit("value = ast2obj_%s(state, vstate, o->%s);" % (a.type, a.name), 1) self.emit("if (!value) goto failed;", 1) self.emit('if (PyObject_SetAttr(result, state->%s, value) < 0)' % a.name, 1) self.emit('goto failed;', 2) @@ -1304,7 +1303,7 @@ def visitSum(self, sum, name): self.func_end() def simpleSum(self, sum, name): - self.emit("PyObject* ast2obj_%s(struct ast_state *state, %s_ty o)" % (name, name), 0) + self.emit("PyObject* ast2obj_%s(struct ast_state *state, struct validator *vstate, %s_ty o)" % (name, name), 0) self.emit("{", 0) self.emit("switch(o) {", 1) for t in sum.types: @@ -1322,7 +1321,7 @@ def visitProduct(self, prod, name): for field in prod.fields: self.visitField(field, name, 1, True) for a in prod.attributes: - self.emit("value = ast2obj_%s(state, o->%s);" % (a.type, a.name), 1) + self.emit("value = ast2obj_%s(state, vstate, o->%s);" % (a.type, a.name), 1) self.emit("if (!value) goto failed;", 1) self.emit("if (PyObject_SetAttr(result, state->%s, value) < 0)" % a.name, 1) self.emit('goto failed;', 2) @@ -1363,7 +1362,7 @@ def set(self, field, value, depth): self.emit("for(i = 0; i < n; i++)", depth+1) # This cannot fail, so no need for error handling self.emit( - "PyList_SET_ITEM(value, i, ast2obj_{0}(state, ({0}_ty)asdl_seq_GET({1}, i)));".format( + "PyList_SET_ITEM(value, i, ast2obj_{0}(state, vstate, ({0}_ty)asdl_seq_GET({1}, i)));".format( field.type, value ), @@ -1372,9 +1371,9 @@ def set(self, field, value, depth): ) self.emit("}", depth) else: - self.emit("value = ast2obj_list(state, (asdl_seq*)%s, ast2obj_%s);" % (value, field.type), depth) + self.emit("value = ast2obj_list(state, vstate, (asdl_seq*)%s, ast2obj_%s);" % (value, field.type), depth) else: - self.emit("value = ast2obj_%s(state, %s);" % (field.type, value), depth, reflow=False) + self.emit("value = ast2obj_%s(state, vstate, %s);" % (field.type, value), depth, reflow=False) class PartingShots(StaticVisitor): @@ -1389,23 +1388,23 @@ class PartingShots(StaticVisitor): int starting_recursion_depth; /* Be careful here to prevent overflow. */ - int COMPILER_STACK_FRAME_SCALE = 2; PyThreadState *tstate = _PyThreadState_GET(); if (!tstate) { return NULL; } - state->recursion_limit = Py_C_RECURSION_LIMIT * COMPILER_STACK_FRAME_SCALE; + struct validator vstate; + vstate.recursion_limit = Py_C_RECURSION_LIMIT; int recursion_depth = Py_C_RECURSION_LIMIT - tstate->c_recursion_remaining; - starting_recursion_depth = recursion_depth * COMPILER_STACK_FRAME_SCALE; - state->recursion_depth = starting_recursion_depth; + starting_recursion_depth = recursion_depth; + vstate.recursion_depth = starting_recursion_depth; - PyObject *result = ast2obj_mod(state, t); + PyObject *result = ast2obj_mod(state, &vstate, t); /* Check that the recursion depth counting balanced correctly */ - if (result && state->recursion_depth != starting_recursion_depth) { + if (result && vstate.recursion_depth != starting_recursion_depth) { PyErr_Format(PyExc_SystemError, "AST constructor recursion depth mismatch (before=%d, after=%d)", - starting_recursion_depth, state->recursion_depth); + starting_recursion_depth, vstate.recursion_depth); return NULL; } return result; @@ -1475,8 +1474,6 @@ def generate_ast_state(module_state, f): f.write('struct ast_state {\n') f.write(' _PyOnceFlag once;\n') f.write(' int finalized;\n') - f.write(' int recursion_depth;\n') - f.write(' int recursion_limit;\n') for s in module_state: f.write(' PyObject *' + s + ';\n') f.write('};') @@ -1539,6 +1536,11 @@ def generate_module_def(mod, metadata, f, internal_h): #include "pycore_pystate.h" // _PyInterpreterState_GET() #include + struct validator { + int recursion_depth; /* current recursion depth */ + int recursion_limit; /* recursion limit */ + }; + // Forward declaration static int init_types(struct ast_state *state); diff --git a/Parser/lexer/lexer.c b/Parser/lexer/lexer.c index ea4bdf7ce4a24c..82b0e4ee352d62 100644 --- a/Parser/lexer/lexer.c +++ b/Parser/lexer/lexer.c @@ -129,7 +129,7 @@ set_fstring_expr(struct tok_state* tok, struct token *token, char c) { if (hash_detected) { Py_ssize_t input_length = tok_mode->last_expr_size - tok_mode->last_expr_end; - char *result = (char *)PyObject_Malloc((input_length + 1) * sizeof(char)); + char *result = (char *)PyMem_Malloc((input_length + 1) * sizeof(char)); if (!result) { return -1; } @@ -154,7 +154,7 @@ set_fstring_expr(struct tok_state* tok, struct token *token, char c) { result[j] = '\0'; // Null-terminate the result string res = PyUnicode_DecodeUTF8(result, j, NULL); - PyObject_Free(result); + PyMem_Free(result); } else { res = PyUnicode_DecodeUTF8( tok_mode->last_expr_buffer, @@ -1355,9 +1355,13 @@ tok_get_fstring_mode(struct tok_state *tok, tokenizer_mode* current_tok, struct tok->lineno = the_current_tok->f_string_line_start; if (current_tok->f_string_quote_size == 3) { - return MAKE_TOKEN(_PyTokenizer_syntaxerror(tok, + _PyTokenizer_syntaxerror(tok, "unterminated triple-quoted f-string literal" - " (detected at line %d)", start)); + " (detected at line %d)", start); + if (c != '\n') { + tok->done = E_EOFS; + } + return MAKE_TOKEN(ERRORTOKEN); } else { return MAKE_TOKEN(_PyTokenizer_syntaxerror(tok, diff --git a/Parser/parser.c b/Parser/parser.c index 9006efb09a59fa..779b18e9650e9f 100644 --- a/Parser/parser.c +++ b/Parser/parser.c @@ -21,54 +21,54 @@ static KeywordToken *reserved_keywords[] = { (KeywordToken[]) {{NULL, -1}}, (KeywordToken[]) {{NULL, -1}}, (KeywordToken[]) { - {"if", 658}, - {"as", 656}, - {"in", 669}, + {"if", 661}, + {"as", 659}, + {"in", 672}, {"or", 581}, {"is", 589}, {NULL, -1}, }, (KeywordToken[]) { - {"del", 613}, - {"def", 671}, - {"for", 668}, - {"try", 640}, + {"del", 616}, + {"def", 674}, + {"for", 671}, + {"try", 643}, {"and", 582}, {"not", 588}, {NULL, -1}, }, (KeywordToken[]) { - {"from", 618}, + {"from", 621}, {"pass", 504}, - {"with", 631}, - {"elif", 660}, - {"else", 661}, - {"None", 611}, - {"True", 610}, + {"with", 634}, + {"elif", 663}, + {"else", 664}, + {"None", 614}, + {"True", 613}, {NULL, -1}, }, (KeywordToken[]) { {"raise", 525}, {"yield", 580}, {"break", 508}, - {"async", 670}, - {"class", 673}, - {"while", 663}, - {"False", 612}, + {"async", 673}, + {"class", 676}, + {"while", 666}, + {"False", 615}, {"await", 590}, {NULL, -1}, }, (KeywordToken[]) { {"return", 522}, - {"import", 617}, + {"import", 620}, {"assert", 529}, {"global", 526}, - {"except", 653}, - {"lambda", 609}, + {"except", 656}, + {"lambda", 612}, {NULL, -1}, }, (KeywordToken[]) { - {"finally", 649}, + {"finally", 652}, {NULL, -1}, }, (KeywordToken[]) { @@ -451,33 +451,33 @@ static char *soft_keywords[] = { #define _loop0_120_type 1364 #define _loop0_121_type 1365 #define _tmp_122_type 1366 -#define _loop0_124_type 1367 -#define _gather_123_type 1368 -#define _tmp_125_type 1369 -#define _loop0_127_type 1370 -#define _gather_126_type 1371 -#define _loop0_129_type 1372 -#define _gather_128_type 1373 -#define _loop0_131_type 1374 -#define _gather_130_type 1375 -#define _loop0_133_type 1376 -#define _gather_132_type 1377 -#define _loop0_134_type 1378 -#define _loop0_136_type 1379 -#define _gather_135_type 1380 -#define _loop1_137_type 1381 -#define _tmp_138_type 1382 -#define _loop0_140_type 1383 -#define _gather_139_type 1384 -#define _loop0_142_type 1385 -#define _gather_141_type 1386 -#define _loop0_144_type 1387 -#define _gather_143_type 1388 -#define _loop0_146_type 1389 -#define _gather_145_type 1390 -#define _loop0_148_type 1391 -#define _gather_147_type 1392 -#define _tmp_149_type 1393 +#define _tmp_123_type 1367 +#define _loop0_125_type 1368 +#define _gather_124_type 1369 +#define _tmp_126_type 1370 +#define _loop0_128_type 1371 +#define _gather_127_type 1372 +#define _loop0_130_type 1373 +#define _gather_129_type 1374 +#define _loop0_132_type 1375 +#define _gather_131_type 1376 +#define _loop0_134_type 1377 +#define _gather_133_type 1378 +#define _loop0_135_type 1379 +#define _loop0_137_type 1380 +#define _gather_136_type 1381 +#define _loop1_138_type 1382 +#define _tmp_139_type 1383 +#define _loop0_141_type 1384 +#define _gather_140_type 1385 +#define _loop0_143_type 1386 +#define _gather_142_type 1387 +#define _loop0_145_type 1388 +#define _gather_144_type 1389 +#define _loop0_147_type 1390 +#define _gather_146_type 1391 +#define _loop0_149_type 1392 +#define _gather_148_type 1393 #define _tmp_150_type 1394 #define _tmp_151_type 1395 #define _tmp_152_type 1396 @@ -489,65 +489,65 @@ static char *soft_keywords[] = { #define _tmp_158_type 1402 #define _tmp_159_type 1403 #define _tmp_160_type 1404 -#define _loop0_161_type 1405 +#define _tmp_161_type 1405 #define _loop0_162_type 1406 #define _loop0_163_type 1407 -#define _tmp_164_type 1408 +#define _loop0_164_type 1408 #define _tmp_165_type 1409 #define _tmp_166_type 1410 #define _tmp_167_type 1411 #define _tmp_168_type 1412 -#define _loop0_169_type 1413 +#define _tmp_169_type 1413 #define _loop0_170_type 1414 #define _loop0_171_type 1415 -#define _loop1_172_type 1416 -#define _tmp_173_type 1417 -#define _loop0_174_type 1418 -#define _tmp_175_type 1419 -#define _loop0_176_type 1420 -#define _loop1_177_type 1421 -#define _tmp_178_type 1422 +#define _loop0_172_type 1416 +#define _loop1_173_type 1417 +#define _tmp_174_type 1418 +#define _loop0_175_type 1419 +#define _tmp_176_type 1420 +#define _loop0_177_type 1421 +#define _loop1_178_type 1422 #define _tmp_179_type 1423 #define _tmp_180_type 1424 -#define _loop0_181_type 1425 -#define _tmp_182_type 1426 +#define _tmp_181_type 1425 +#define _loop0_182_type 1426 #define _tmp_183_type 1427 -#define _loop1_184_type 1428 -#define _tmp_185_type 1429 -#define _loop0_186_type 1430 +#define _tmp_184_type 1428 +#define _loop1_185_type 1429 +#define _tmp_186_type 1430 #define _loop0_187_type 1431 #define _loop0_188_type 1432 -#define _loop0_190_type 1433 -#define _gather_189_type 1434 -#define _tmp_191_type 1435 -#define _loop0_192_type 1436 -#define _tmp_193_type 1437 -#define _loop0_194_type 1438 -#define _loop1_195_type 1439 +#define _loop0_189_type 1433 +#define _loop0_191_type 1434 +#define _gather_190_type 1435 +#define _tmp_192_type 1436 +#define _loop0_193_type 1437 +#define _tmp_194_type 1438 +#define _loop0_195_type 1439 #define _loop1_196_type 1440 -#define _tmp_197_type 1441 +#define _loop1_197_type 1441 #define _tmp_198_type 1442 -#define _loop0_199_type 1443 -#define _tmp_200_type 1444 +#define _tmp_199_type 1443 +#define _loop0_200_type 1444 #define _tmp_201_type 1445 #define _tmp_202_type 1446 -#define _loop0_204_type 1447 -#define _gather_203_type 1448 -#define _loop0_206_type 1449 -#define _gather_205_type 1450 -#define _loop0_208_type 1451 -#define _gather_207_type 1452 -#define _loop0_210_type 1453 -#define _gather_209_type 1454 -#define _loop0_212_type 1455 -#define _gather_211_type 1456 -#define _tmp_213_type 1457 -#define _loop0_214_type 1458 -#define _loop1_215_type 1459 -#define _tmp_216_type 1460 -#define _loop0_217_type 1461 -#define _loop1_218_type 1462 -#define _tmp_219_type 1463 +#define _tmp_203_type 1447 +#define _loop0_205_type 1448 +#define _gather_204_type 1449 +#define _loop0_207_type 1450 +#define _gather_206_type 1451 +#define _loop0_209_type 1452 +#define _gather_208_type 1453 +#define _loop0_211_type 1454 +#define _gather_210_type 1455 +#define _loop0_213_type 1456 +#define _gather_212_type 1457 +#define _tmp_214_type 1458 +#define _loop0_215_type 1459 +#define _loop1_216_type 1460 +#define _tmp_217_type 1461 +#define _loop0_218_type 1462 +#define _loop1_219_type 1463 #define _tmp_220_type 1464 #define _tmp_221_type 1465 #define _tmp_222_type 1466 @@ -557,9 +557,9 @@ static char *soft_keywords[] = { #define _tmp_226_type 1470 #define _tmp_227_type 1471 #define _tmp_228_type 1472 -#define _loop0_230_type 1473 -#define _gather_229_type 1474 -#define _tmp_231_type 1475 +#define _tmp_229_type 1473 +#define _loop0_231_type 1474 +#define _gather_230_type 1475 #define _tmp_232_type 1476 #define _tmp_233_type 1477 #define _tmp_234_type 1478 @@ -572,8 +572,8 @@ static char *soft_keywords[] = { #define _tmp_241_type 1485 #define _tmp_242_type 1486 #define _tmp_243_type 1487 -#define _loop0_244_type 1488 -#define _tmp_245_type 1489 +#define _tmp_244_type 1488 +#define _loop0_245_type 1489 #define _tmp_246_type 1490 #define _tmp_247_type 1491 #define _tmp_248_type 1492 @@ -589,7 +589,7 @@ static char *soft_keywords[] = { #define _tmp_258_type 1502 #define _tmp_259_type 1503 #define _tmp_260_type 1504 -#define _tmp_261_type 1505 +#define _loop0_261_type 1505 #define _tmp_262_type 1506 #define _tmp_263_type 1507 #define _tmp_264_type 1508 @@ -602,14 +602,17 @@ static char *soft_keywords[] = { #define _tmp_271_type 1515 #define _tmp_272_type 1516 #define _tmp_273_type 1517 -#define _loop0_275_type 1518 -#define _gather_274_type 1519 +#define _tmp_274_type 1518 +#define _tmp_275_type 1519 #define _tmp_276_type 1520 -#define _tmp_277_type 1521 -#define _tmp_278_type 1522 +#define _loop0_278_type 1521 +#define _gather_277_type 1522 #define _tmp_279_type 1523 #define _tmp_280_type 1524 #define _tmp_281_type 1525 +#define _tmp_282_type 1526 +#define _tmp_283_type 1527 +#define _tmp_284_type 1528 static mod_ty file_rule(Parser *p); static mod_ty interactive_rule(Parser *p); @@ -978,33 +981,33 @@ static asdl_seq *_loop1_119_rule(Parser *p); static asdl_seq *_loop0_120_rule(Parser *p); static asdl_seq *_loop0_121_rule(Parser *p); static void *_tmp_122_rule(Parser *p); -static asdl_seq *_loop0_124_rule(Parser *p); -static asdl_seq *_gather_123_rule(Parser *p); -static void *_tmp_125_rule(Parser *p); -static asdl_seq *_loop0_127_rule(Parser *p); -static asdl_seq *_gather_126_rule(Parser *p); -static asdl_seq *_loop0_129_rule(Parser *p); -static asdl_seq *_gather_128_rule(Parser *p); -static asdl_seq *_loop0_131_rule(Parser *p); -static asdl_seq *_gather_130_rule(Parser *p); -static asdl_seq *_loop0_133_rule(Parser *p); -static asdl_seq *_gather_132_rule(Parser *p); +static void *_tmp_123_rule(Parser *p); +static asdl_seq *_loop0_125_rule(Parser *p); +static asdl_seq *_gather_124_rule(Parser *p); +static void *_tmp_126_rule(Parser *p); +static asdl_seq *_loop0_128_rule(Parser *p); +static asdl_seq *_gather_127_rule(Parser *p); +static asdl_seq *_loop0_130_rule(Parser *p); +static asdl_seq *_gather_129_rule(Parser *p); +static asdl_seq *_loop0_132_rule(Parser *p); +static asdl_seq *_gather_131_rule(Parser *p); static asdl_seq *_loop0_134_rule(Parser *p); -static asdl_seq *_loop0_136_rule(Parser *p); -static asdl_seq *_gather_135_rule(Parser *p); -static asdl_seq *_loop1_137_rule(Parser *p); -static void *_tmp_138_rule(Parser *p); -static asdl_seq *_loop0_140_rule(Parser *p); -static asdl_seq *_gather_139_rule(Parser *p); -static asdl_seq *_loop0_142_rule(Parser *p); -static asdl_seq *_gather_141_rule(Parser *p); -static asdl_seq *_loop0_144_rule(Parser *p); -static asdl_seq *_gather_143_rule(Parser *p); -static asdl_seq *_loop0_146_rule(Parser *p); -static asdl_seq *_gather_145_rule(Parser *p); -static asdl_seq *_loop0_148_rule(Parser *p); -static asdl_seq *_gather_147_rule(Parser *p); -static void *_tmp_149_rule(Parser *p); +static asdl_seq *_gather_133_rule(Parser *p); +static asdl_seq *_loop0_135_rule(Parser *p); +static asdl_seq *_loop0_137_rule(Parser *p); +static asdl_seq *_gather_136_rule(Parser *p); +static asdl_seq *_loop1_138_rule(Parser *p); +static void *_tmp_139_rule(Parser *p); +static asdl_seq *_loop0_141_rule(Parser *p); +static asdl_seq *_gather_140_rule(Parser *p); +static asdl_seq *_loop0_143_rule(Parser *p); +static asdl_seq *_gather_142_rule(Parser *p); +static asdl_seq *_loop0_145_rule(Parser *p); +static asdl_seq *_gather_144_rule(Parser *p); +static asdl_seq *_loop0_147_rule(Parser *p); +static asdl_seq *_gather_146_rule(Parser *p); +static asdl_seq *_loop0_149_rule(Parser *p); +static asdl_seq *_gather_148_rule(Parser *p); static void *_tmp_150_rule(Parser *p); static void *_tmp_151_rule(Parser *p); static void *_tmp_152_rule(Parser *p); @@ -1016,65 +1019,65 @@ static void *_tmp_157_rule(Parser *p); static void *_tmp_158_rule(Parser *p); static void *_tmp_159_rule(Parser *p); static void *_tmp_160_rule(Parser *p); -static asdl_seq *_loop0_161_rule(Parser *p); +static void *_tmp_161_rule(Parser *p); static asdl_seq *_loop0_162_rule(Parser *p); static asdl_seq *_loop0_163_rule(Parser *p); -static void *_tmp_164_rule(Parser *p); +static asdl_seq *_loop0_164_rule(Parser *p); static void *_tmp_165_rule(Parser *p); static void *_tmp_166_rule(Parser *p); static void *_tmp_167_rule(Parser *p); static void *_tmp_168_rule(Parser *p); -static asdl_seq *_loop0_169_rule(Parser *p); +static void *_tmp_169_rule(Parser *p); static asdl_seq *_loop0_170_rule(Parser *p); static asdl_seq *_loop0_171_rule(Parser *p); -static asdl_seq *_loop1_172_rule(Parser *p); -static void *_tmp_173_rule(Parser *p); -static asdl_seq *_loop0_174_rule(Parser *p); -static void *_tmp_175_rule(Parser *p); -static asdl_seq *_loop0_176_rule(Parser *p); -static asdl_seq *_loop1_177_rule(Parser *p); -static void *_tmp_178_rule(Parser *p); +static asdl_seq *_loop0_172_rule(Parser *p); +static asdl_seq *_loop1_173_rule(Parser *p); +static void *_tmp_174_rule(Parser *p); +static asdl_seq *_loop0_175_rule(Parser *p); +static void *_tmp_176_rule(Parser *p); +static asdl_seq *_loop0_177_rule(Parser *p); +static asdl_seq *_loop1_178_rule(Parser *p); static void *_tmp_179_rule(Parser *p); static void *_tmp_180_rule(Parser *p); -static asdl_seq *_loop0_181_rule(Parser *p); -static void *_tmp_182_rule(Parser *p); +static void *_tmp_181_rule(Parser *p); +static asdl_seq *_loop0_182_rule(Parser *p); static void *_tmp_183_rule(Parser *p); -static asdl_seq *_loop1_184_rule(Parser *p); -static void *_tmp_185_rule(Parser *p); -static asdl_seq *_loop0_186_rule(Parser *p); +static void *_tmp_184_rule(Parser *p); +static asdl_seq *_loop1_185_rule(Parser *p); +static void *_tmp_186_rule(Parser *p); static asdl_seq *_loop0_187_rule(Parser *p); static asdl_seq *_loop0_188_rule(Parser *p); -static asdl_seq *_loop0_190_rule(Parser *p); -static asdl_seq *_gather_189_rule(Parser *p); -static void *_tmp_191_rule(Parser *p); -static asdl_seq *_loop0_192_rule(Parser *p); -static void *_tmp_193_rule(Parser *p); -static asdl_seq *_loop0_194_rule(Parser *p); -static asdl_seq *_loop1_195_rule(Parser *p); +static asdl_seq *_loop0_189_rule(Parser *p); +static asdl_seq *_loop0_191_rule(Parser *p); +static asdl_seq *_gather_190_rule(Parser *p); +static void *_tmp_192_rule(Parser *p); +static asdl_seq *_loop0_193_rule(Parser *p); +static void *_tmp_194_rule(Parser *p); +static asdl_seq *_loop0_195_rule(Parser *p); static asdl_seq *_loop1_196_rule(Parser *p); -static void *_tmp_197_rule(Parser *p); +static asdl_seq *_loop1_197_rule(Parser *p); static void *_tmp_198_rule(Parser *p); -static asdl_seq *_loop0_199_rule(Parser *p); -static void *_tmp_200_rule(Parser *p); +static void *_tmp_199_rule(Parser *p); +static asdl_seq *_loop0_200_rule(Parser *p); static void *_tmp_201_rule(Parser *p); static void *_tmp_202_rule(Parser *p); -static asdl_seq *_loop0_204_rule(Parser *p); -static asdl_seq *_gather_203_rule(Parser *p); -static asdl_seq *_loop0_206_rule(Parser *p); -static asdl_seq *_gather_205_rule(Parser *p); -static asdl_seq *_loop0_208_rule(Parser *p); -static asdl_seq *_gather_207_rule(Parser *p); -static asdl_seq *_loop0_210_rule(Parser *p); -static asdl_seq *_gather_209_rule(Parser *p); -static asdl_seq *_loop0_212_rule(Parser *p); -static asdl_seq *_gather_211_rule(Parser *p); -static void *_tmp_213_rule(Parser *p); -static asdl_seq *_loop0_214_rule(Parser *p); -static asdl_seq *_loop1_215_rule(Parser *p); -static void *_tmp_216_rule(Parser *p); -static asdl_seq *_loop0_217_rule(Parser *p); -static asdl_seq *_loop1_218_rule(Parser *p); -static void *_tmp_219_rule(Parser *p); +static void *_tmp_203_rule(Parser *p); +static asdl_seq *_loop0_205_rule(Parser *p); +static asdl_seq *_gather_204_rule(Parser *p); +static asdl_seq *_loop0_207_rule(Parser *p); +static asdl_seq *_gather_206_rule(Parser *p); +static asdl_seq *_loop0_209_rule(Parser *p); +static asdl_seq *_gather_208_rule(Parser *p); +static asdl_seq *_loop0_211_rule(Parser *p); +static asdl_seq *_gather_210_rule(Parser *p); +static asdl_seq *_loop0_213_rule(Parser *p); +static asdl_seq *_gather_212_rule(Parser *p); +static void *_tmp_214_rule(Parser *p); +static asdl_seq *_loop0_215_rule(Parser *p); +static asdl_seq *_loop1_216_rule(Parser *p); +static void *_tmp_217_rule(Parser *p); +static asdl_seq *_loop0_218_rule(Parser *p); +static asdl_seq *_loop1_219_rule(Parser *p); static void *_tmp_220_rule(Parser *p); static void *_tmp_221_rule(Parser *p); static void *_tmp_222_rule(Parser *p); @@ -1084,9 +1087,9 @@ static void *_tmp_225_rule(Parser *p); static void *_tmp_226_rule(Parser *p); static void *_tmp_227_rule(Parser *p); static void *_tmp_228_rule(Parser *p); -static asdl_seq *_loop0_230_rule(Parser *p); -static asdl_seq *_gather_229_rule(Parser *p); -static void *_tmp_231_rule(Parser *p); +static void *_tmp_229_rule(Parser *p); +static asdl_seq *_loop0_231_rule(Parser *p); +static asdl_seq *_gather_230_rule(Parser *p); static void *_tmp_232_rule(Parser *p); static void *_tmp_233_rule(Parser *p); static void *_tmp_234_rule(Parser *p); @@ -1099,8 +1102,8 @@ static void *_tmp_240_rule(Parser *p); static void *_tmp_241_rule(Parser *p); static void *_tmp_242_rule(Parser *p); static void *_tmp_243_rule(Parser *p); -static asdl_seq *_loop0_244_rule(Parser *p); -static void *_tmp_245_rule(Parser *p); +static void *_tmp_244_rule(Parser *p); +static asdl_seq *_loop0_245_rule(Parser *p); static void *_tmp_246_rule(Parser *p); static void *_tmp_247_rule(Parser *p); static void *_tmp_248_rule(Parser *p); @@ -1116,7 +1119,7 @@ static void *_tmp_257_rule(Parser *p); static void *_tmp_258_rule(Parser *p); static void *_tmp_259_rule(Parser *p); static void *_tmp_260_rule(Parser *p); -static void *_tmp_261_rule(Parser *p); +static asdl_seq *_loop0_261_rule(Parser *p); static void *_tmp_262_rule(Parser *p); static void *_tmp_263_rule(Parser *p); static void *_tmp_264_rule(Parser *p); @@ -1129,14 +1132,17 @@ static void *_tmp_270_rule(Parser *p); static void *_tmp_271_rule(Parser *p); static void *_tmp_272_rule(Parser *p); static void *_tmp_273_rule(Parser *p); -static asdl_seq *_loop0_275_rule(Parser *p); -static asdl_seq *_gather_274_rule(Parser *p); +static void *_tmp_274_rule(Parser *p); +static void *_tmp_275_rule(Parser *p); static void *_tmp_276_rule(Parser *p); -static void *_tmp_277_rule(Parser *p); -static void *_tmp_278_rule(Parser *p); +static asdl_seq *_loop0_278_rule(Parser *p); +static asdl_seq *_gather_277_rule(Parser *p); static void *_tmp_279_rule(Parser *p); static void *_tmp_280_rule(Parser *p); static void *_tmp_281_rule(Parser *p); +static void *_tmp_282_rule(Parser *p); +static void *_tmp_283_rule(Parser *p); +static void *_tmp_284_rule(Parser *p); // file: statements? $ @@ -1875,7 +1881,7 @@ simple_stmt_rule(Parser *p) D(fprintf(stderr, "%*c> simple_stmt[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "&'del' del_stmt")); stmt_ty del_stmt_var; if ( - _PyPegen_lookahead_with_int(1, _PyPegen_expect_token, p, 613) // token='del' + _PyPegen_lookahead_with_int(1, _PyPegen_expect_token, p, 616) // token='del' && (del_stmt_var = del_stmt_rule(p)) // del_stmt ) @@ -2115,7 +2121,7 @@ compound_stmt_rule(Parser *p) D(fprintf(stderr, "%*c> compound_stmt[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "&'if' if_stmt")); stmt_ty if_stmt_var; if ( - _PyPegen_lookahead_with_int(1, _PyPegen_expect_token, p, 658) // token='if' + _PyPegen_lookahead_with_int(1, _PyPegen_expect_token, p, 661) // token='if' && (if_stmt_var = if_stmt_rule(p)) // if_stmt ) @@ -2199,7 +2205,7 @@ compound_stmt_rule(Parser *p) D(fprintf(stderr, "%*c> compound_stmt[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "&'try' try_stmt")); stmt_ty try_stmt_var; if ( - _PyPegen_lookahead_with_int(1, _PyPegen_expect_token, p, 640) // token='try' + _PyPegen_lookahead_with_int(1, _PyPegen_expect_token, p, 643) // token='try' && (try_stmt_var = try_stmt_rule(p)) // try_stmt ) @@ -2220,7 +2226,7 @@ compound_stmt_rule(Parser *p) D(fprintf(stderr, "%*c> compound_stmt[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "&'while' while_stmt")); stmt_ty while_stmt_var; if ( - _PyPegen_lookahead_with_int(1, _PyPegen_expect_token, p, 663) // token='while' + _PyPegen_lookahead_with_int(1, _PyPegen_expect_token, p, 666) // token='while' && (while_stmt_var = while_stmt_rule(p)) // while_stmt ) @@ -3205,7 +3211,7 @@ del_stmt_rule(Parser *p) Token * _keyword; asdl_expr_seq* a; if ( - (_keyword = _PyPegen_expect_token(p, 613)) // token='del' + (_keyword = _PyPegen_expect_token(p, 616)) // token='del' && (a = del_targets_rule(p)) // del_targets && @@ -3494,7 +3500,7 @@ import_name_rule(Parser *p) Token * _keyword; asdl_alias_seq* a; if ( - (_keyword = _PyPegen_expect_token(p, 617)) // token='import' + (_keyword = _PyPegen_expect_token(p, 620)) // token='import' && (a = dotted_as_names_rule(p)) // dotted_as_names ) @@ -3563,13 +3569,13 @@ import_from_rule(Parser *p) expr_ty b; asdl_alias_seq* c; if ( - (_keyword = _PyPegen_expect_token(p, 618)) // token='from' + (_keyword = _PyPegen_expect_token(p, 621)) // token='from' && (a = _loop0_24_rule(p)) // (('.' | '...'))* && (b = dotted_name_rule(p)) // dotted_name && - (_keyword_1 = _PyPegen_expect_token(p, 617)) // token='import' + (_keyword_1 = _PyPegen_expect_token(p, 620)) // token='import' && (c = import_from_targets_rule(p)) // import_from_targets ) @@ -3607,11 +3613,11 @@ import_from_rule(Parser *p) asdl_seq * a; asdl_alias_seq* b; if ( - (_keyword = _PyPegen_expect_token(p, 618)) // token='from' + (_keyword = _PyPegen_expect_token(p, 621)) // token='from' && (a = _loop1_25_rule(p)) // (('.' | '...'))+ && - (_keyword_1 = _PyPegen_expect_token(p, 617)) // token='import' + (_keyword_1 = _PyPegen_expect_token(p, 620)) // token='import' && (b = import_from_targets_rule(p)) // import_from_targets ) @@ -4360,7 +4366,7 @@ class_def_raw_rule(Parser *p) asdl_stmt_seq* c; void *t; if ( - (_keyword = _PyPegen_expect_token(p, 673)) // token='class' + (_keyword = _PyPegen_expect_token(p, 676)) // token='class' && (a = _PyPegen_name_token(p)) // NAME && @@ -4527,7 +4533,7 @@ function_def_raw_rule(Parser *p) void *t; void *tc; if ( - (_keyword = _PyPegen_expect_token(p, 671)) // token='def' + (_keyword = _PyPegen_expect_token(p, 674)) // token='def' && (n = _PyPegen_name_token(p)) // NAME && @@ -4588,9 +4594,9 @@ function_def_raw_rule(Parser *p) void *t; void *tc; if ( - (_keyword = _PyPegen_expect_token(p, 670)) // token='async' + (_keyword = _PyPegen_expect_token(p, 673)) // token='async' && - (_keyword_1 = _PyPegen_expect_token(p, 671)) // token='def' + (_keyword_1 = _PyPegen_expect_token(p, 674)) // token='def' && (n = _PyPegen_name_token(p)) // NAME && @@ -5928,7 +5934,7 @@ if_stmt_rule(Parser *p) asdl_stmt_seq* b; stmt_ty c; if ( - (_keyword = _PyPegen_expect_token(p, 658)) // token='if' + (_keyword = _PyPegen_expect_token(p, 661)) // token='if' && (a = named_expression_rule(p)) // named_expression && @@ -5973,7 +5979,7 @@ if_stmt_rule(Parser *p) asdl_stmt_seq* b; void *c; if ( - (_keyword = _PyPegen_expect_token(p, 658)) // token='if' + (_keyword = _PyPegen_expect_token(p, 661)) // token='if' && (a = named_expression_rule(p)) // named_expression && @@ -6068,7 +6074,7 @@ elif_stmt_rule(Parser *p) asdl_stmt_seq* b; stmt_ty c; if ( - (_keyword = _PyPegen_expect_token(p, 660)) // token='elif' + (_keyword = _PyPegen_expect_token(p, 663)) // token='elif' && (a = named_expression_rule(p)) // named_expression && @@ -6113,7 +6119,7 @@ elif_stmt_rule(Parser *p) asdl_stmt_seq* b; void *c; if ( - (_keyword = _PyPegen_expect_token(p, 660)) // token='elif' + (_keyword = _PyPegen_expect_token(p, 663)) // token='elif' && (a = named_expression_rule(p)) // named_expression && @@ -6194,7 +6200,7 @@ else_block_rule(Parser *p) Token * _literal; asdl_stmt_seq* b; if ( - (_keyword = _PyPegen_expect_token(p, 661)) // token='else' + (_keyword = _PyPegen_expect_token(p, 664)) // token='else' && (_literal = _PyPegen_expect_forced_token(p, 11, ":")) // forced_token=':' && @@ -6273,7 +6279,7 @@ while_stmt_rule(Parser *p) asdl_stmt_seq* b; void *c; if ( - (_keyword = _PyPegen_expect_token(p, 663)) // token='while' + (_keyword = _PyPegen_expect_token(p, 666)) // token='while' && (a = named_expression_rule(p)) // named_expression && @@ -6373,11 +6379,11 @@ for_stmt_rule(Parser *p) expr_ty t; void *tc; if ( - (_keyword = _PyPegen_expect_token(p, 668)) // token='for' + (_keyword = _PyPegen_expect_token(p, 671)) // token='for' && (t = star_targets_rule(p)) // star_targets && - (_keyword_1 = _PyPegen_expect_token(p, 669)) // token='in' + (_keyword_1 = _PyPegen_expect_token(p, 672)) // token='in' && (_cut_var = 1) && @@ -6435,13 +6441,13 @@ for_stmt_rule(Parser *p) expr_ty t; void *tc; if ( - (_keyword = _PyPegen_expect_token(p, 670)) // token='async' + (_keyword = _PyPegen_expect_token(p, 673)) // token='async' && - (_keyword_1 = _PyPegen_expect_token(p, 668)) // token='for' + (_keyword_1 = _PyPegen_expect_token(p, 671)) // token='for' && (t = star_targets_rule(p)) // star_targets && - (_keyword_2 = _PyPegen_expect_token(p, 669)) // token='in' + (_keyword_2 = _PyPegen_expect_token(p, 672)) // token='in' && (_cut_var = 1) && @@ -6570,7 +6576,7 @@ with_stmt_rule(Parser *p) asdl_stmt_seq* b; void *tc; if ( - (_keyword = _PyPegen_expect_token(p, 631)) // token='with' + (_keyword = _PyPegen_expect_token(p, 634)) // token='with' && (_literal = _PyPegen_expect_token(p, 7)) // token='(' && @@ -6621,7 +6627,7 @@ with_stmt_rule(Parser *p) asdl_stmt_seq* b; void *tc; if ( - (_keyword = _PyPegen_expect_token(p, 631)) // token='with' + (_keyword = _PyPegen_expect_token(p, 634)) // token='with' && (a = (asdl_withitem_seq*)_gather_53_rule(p)) // ','.with_item+ && @@ -6670,9 +6676,9 @@ with_stmt_rule(Parser *p) asdl_withitem_seq* a; asdl_stmt_seq* b; if ( - (_keyword = _PyPegen_expect_token(p, 670)) // token='async' + (_keyword = _PyPegen_expect_token(p, 673)) // token='async' && - (_keyword_1 = _PyPegen_expect_token(p, 631)) // token='with' + (_keyword_1 = _PyPegen_expect_token(p, 634)) // token='with' && (_literal = _PyPegen_expect_token(p, 7)) // token='(' && @@ -6722,9 +6728,9 @@ with_stmt_rule(Parser *p) asdl_stmt_seq* b; void *tc; if ( - (_keyword = _PyPegen_expect_token(p, 670)) // token='async' + (_keyword = _PyPegen_expect_token(p, 673)) // token='async' && - (_keyword_1 = _PyPegen_expect_token(p, 631)) // token='with' + (_keyword_1 = _PyPegen_expect_token(p, 634)) // token='with' && (a = (asdl_withitem_seq*)_gather_57_rule(p)) // ','.with_item+ && @@ -6810,7 +6816,7 @@ with_item_rule(Parser *p) if ( (e = expression_rule(p)) // expression && - (_keyword = _PyPegen_expect_token(p, 656)) // token='as' + (_keyword = _PyPegen_expect_token(p, 659)) // token='as' && (t = star_target_rule(p)) // star_target && @@ -6935,7 +6941,7 @@ try_stmt_rule(Parser *p) asdl_stmt_seq* b; asdl_stmt_seq* f; if ( - (_keyword = _PyPegen_expect_token(p, 640)) // token='try' + (_keyword = _PyPegen_expect_token(p, 643)) // token='try' && (_literal = _PyPegen_expect_forced_token(p, 11, ":")) // forced_token=':' && @@ -6979,7 +6985,7 @@ try_stmt_rule(Parser *p) asdl_excepthandler_seq* ex; void *f; if ( - (_keyword = _PyPegen_expect_token(p, 640)) // token='try' + (_keyword = _PyPegen_expect_token(p, 643)) // token='try' && (_literal = _PyPegen_expect_forced_token(p, 11, ":")) // forced_token=':' && @@ -7027,7 +7033,7 @@ try_stmt_rule(Parser *p) asdl_excepthandler_seq* ex; void *f; if ( - (_keyword = _PyPegen_expect_token(p, 640)) // token='try' + (_keyword = _PyPegen_expect_token(p, 643)) // token='try' && (_literal = _PyPegen_expect_forced_token(p, 11, ":")) // forced_token=':' && @@ -7125,7 +7131,7 @@ except_block_rule(Parser *p) expr_ty e; void *t; if ( - (_keyword = _PyPegen_expect_token(p, 653)) // token='except' + (_keyword = _PyPegen_expect_token(p, 656)) // token='except' && (e = expression_rule(p)) // expression && @@ -7168,7 +7174,7 @@ except_block_rule(Parser *p) Token * _literal; asdl_stmt_seq* b; if ( - (_keyword = _PyPegen_expect_token(p, 653)) // token='except' + (_keyword = _PyPegen_expect_token(p, 656)) // token='except' && (_literal = _PyPegen_expect_token(p, 11)) // token=':' && @@ -7279,7 +7285,7 @@ except_star_block_rule(Parser *p) expr_ty e; void *t; if ( - (_keyword = _PyPegen_expect_token(p, 653)) // token='except' + (_keyword = _PyPegen_expect_token(p, 656)) // token='except' && (_literal = _PyPegen_expect_token(p, 16)) // token='*' && @@ -7381,7 +7387,7 @@ finally_block_rule(Parser *p) Token * _literal; asdl_stmt_seq* a; if ( - (_keyword = _PyPegen_expect_token(p, 649)) // token='finally' + (_keyword = _PyPegen_expect_token(p, 652)) // token='finally' && (_literal = _PyPegen_expect_forced_token(p, 11, ":")) // forced_token=':' && @@ -7689,7 +7695,7 @@ guard_rule(Parser *p) Token * _keyword; expr_ty guard; if ( - (_keyword = _PyPegen_expect_token(p, 658)) // token='if' + (_keyword = _PyPegen_expect_token(p, 661)) // token='if' && (guard = named_expression_rule(p)) // named_expression ) @@ -7884,7 +7890,7 @@ as_pattern_rule(Parser *p) if ( (pattern = or_pattern_rule(p)) // or_pattern && - (_keyword = _PyPegen_expect_token(p, 656)) // token='as' + (_keyword = _PyPegen_expect_token(p, 659)) // token='as' && (target = pattern_capture_target_rule(p)) // pattern_capture_target ) @@ -8318,7 +8324,7 @@ literal_pattern_rule(Parser *p) D(fprintf(stderr, "%*c> literal_pattern[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "'None'")); Token * _keyword; if ( - (_keyword = _PyPegen_expect_token(p, 611)) // token='None' + (_keyword = _PyPegen_expect_token(p, 614)) // token='None' ) { D(fprintf(stderr, "%*c+ literal_pattern[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "'None'")); @@ -8351,7 +8357,7 @@ literal_pattern_rule(Parser *p) D(fprintf(stderr, "%*c> literal_pattern[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "'True'")); Token * _keyword; if ( - (_keyword = _PyPegen_expect_token(p, 610)) // token='True' + (_keyword = _PyPegen_expect_token(p, 613)) // token='True' ) { D(fprintf(stderr, "%*c+ literal_pattern[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "'True'")); @@ -8384,7 +8390,7 @@ literal_pattern_rule(Parser *p) D(fprintf(stderr, "%*c> literal_pattern[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "'False'")); Token * _keyword; if ( - (_keyword = _PyPegen_expect_token(p, 612)) // token='False' + (_keyword = _PyPegen_expect_token(p, 615)) // token='False' ) { D(fprintf(stderr, "%*c+ literal_pattern[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "'False'")); @@ -8510,7 +8516,7 @@ literal_expr_rule(Parser *p) D(fprintf(stderr, "%*c> literal_expr[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "'None'")); Token * _keyword; if ( - (_keyword = _PyPegen_expect_token(p, 611)) // token='None' + (_keyword = _PyPegen_expect_token(p, 614)) // token='None' ) { D(fprintf(stderr, "%*c+ literal_expr[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "'None'")); @@ -8543,7 +8549,7 @@ literal_expr_rule(Parser *p) D(fprintf(stderr, "%*c> literal_expr[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "'True'")); Token * _keyword; if ( - (_keyword = _PyPegen_expect_token(p, 610)) // token='True' + (_keyword = _PyPegen_expect_token(p, 613)) // token='True' ) { D(fprintf(stderr, "%*c+ literal_expr[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "'True'")); @@ -8576,7 +8582,7 @@ literal_expr_rule(Parser *p) D(fprintf(stderr, "%*c> literal_expr[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "'False'")); Token * _keyword; if ( - (_keyword = _PyPegen_expect_token(p, 612)) // token='False' + (_keyword = _PyPegen_expect_token(p, 615)) // token='False' ) { D(fprintf(stderr, "%*c+ literal_expr[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "'False'")); @@ -11107,11 +11113,11 @@ expression_rule(Parser *p) if ( (a = disjunction_rule(p)) // disjunction && - (_keyword = _PyPegen_expect_token(p, 658)) // token='if' + (_keyword = _PyPegen_expect_token(p, 661)) // token='if' && (b = disjunction_rule(p)) // disjunction && - (_keyword_1 = _PyPegen_expect_token(p, 661)) // token='else' + (_keyword_1 = _PyPegen_expect_token(p, 664)) // token='else' && (c = expression_rule(p)) // expression ) @@ -11217,7 +11223,7 @@ yield_expr_rule(Parser *p) if ( (_keyword = _PyPegen_expect_token(p, 580)) // token='yield' && - (_keyword_1 = _PyPegen_expect_token(p, 618)) // token='from' + (_keyword_1 = _PyPegen_expect_token(p, 621)) // token='from' && (a = expression_rule(p)) // expression ) @@ -12649,7 +12655,7 @@ notin_bitwise_or_rule(Parser *p) if ( (_keyword = _PyPegen_expect_token(p, 588)) // token='not' && - (_keyword_1 = _PyPegen_expect_token(p, 669)) // token='in' + (_keyword_1 = _PyPegen_expect_token(p, 672)) // token='in' && (a = bitwise_or_rule(p)) // bitwise_or ) @@ -12695,7 +12701,7 @@ in_bitwise_or_rule(Parser *p) Token * _keyword; expr_ty a; if ( - (_keyword = _PyPegen_expect_token(p, 669)) // token='in' + (_keyword = _PyPegen_expect_token(p, 672)) // token='in' && (a = bitwise_or_rule(p)) // bitwise_or ) @@ -14611,7 +14617,7 @@ atom_rule(Parser *p) D(fprintf(stderr, "%*c> atom[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "'True'")); Token * _keyword; if ( - (_keyword = _PyPegen_expect_token(p, 610)) // token='True' + (_keyword = _PyPegen_expect_token(p, 613)) // token='True' ) { D(fprintf(stderr, "%*c+ atom[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "'True'")); @@ -14644,7 +14650,7 @@ atom_rule(Parser *p) D(fprintf(stderr, "%*c> atom[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "'False'")); Token * _keyword; if ( - (_keyword = _PyPegen_expect_token(p, 612)) // token='False' + (_keyword = _PyPegen_expect_token(p, 615)) // token='False' ) { D(fprintf(stderr, "%*c+ atom[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "'False'")); @@ -14677,7 +14683,7 @@ atom_rule(Parser *p) D(fprintf(stderr, "%*c> atom[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "'None'")); Token * _keyword; if ( - (_keyword = _PyPegen_expect_token(p, 611)) // token='None' + (_keyword = _PyPegen_expect_token(p, 614)) // token='None' ) { D(fprintf(stderr, "%*c+ atom[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "'None'")); @@ -14945,7 +14951,7 @@ lambdef_rule(Parser *p) void *a; expr_ty b; if ( - (_keyword = _PyPegen_expect_token(p, 609)) // token='lambda' + (_keyword = _PyPegen_expect_token(p, 612)) // token='lambda' && (a = lambda_params_rule(p), !p->error_indicator) // lambda_params? && @@ -16848,6 +16854,7 @@ for_if_clauses_rule(Parser *p) // for_if_clause: // | 'async' 'for' star_targets 'in' ~ disjunction (('if' disjunction))* // | 'for' star_targets 'in' ~ disjunction (('if' disjunction))* +// | 'async'? 'for' (bitwise_or ((',' bitwise_or))* ','?) !'in' // | invalid_for_target static comprehension_ty for_if_clause_rule(Parser *p) @@ -16875,13 +16882,13 @@ for_if_clause_rule(Parser *p) expr_ty b; asdl_expr_seq* c; if ( - (_keyword = _PyPegen_expect_token(p, 670)) // token='async' + (_keyword = _PyPegen_expect_token(p, 673)) // token='async' && - (_keyword_1 = _PyPegen_expect_token(p, 668)) // token='for' + (_keyword_1 = _PyPegen_expect_token(p, 671)) // token='for' && (a = star_targets_rule(p)) // star_targets && - (_keyword_2 = _PyPegen_expect_token(p, 669)) // token='in' + (_keyword_2 = _PyPegen_expect_token(p, 672)) // token='in' && (_cut_var = 1) && @@ -16920,11 +16927,11 @@ for_if_clause_rule(Parser *p) expr_ty b; asdl_expr_seq* c; if ( - (_keyword = _PyPegen_expect_token(p, 668)) // token='for' + (_keyword = _PyPegen_expect_token(p, 671)) // token='for' && (a = star_targets_rule(p)) // star_targets && - (_keyword_1 = _PyPegen_expect_token(p, 669)) // token='in' + (_keyword_1 = _PyPegen_expect_token(p, 672)) // token='in' && (_cut_var = 1) && @@ -16950,6 +16957,39 @@ for_if_clause_rule(Parser *p) return NULL; } } + { // 'async'? 'for' (bitwise_or ((',' bitwise_or))* ','?) !'in' + if (p->error_indicator) { + p->level--; + return NULL; + } + D(fprintf(stderr, "%*c> for_if_clause[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "'async'? 'for' (bitwise_or ((',' bitwise_or))* ','?) !'in'")); + Token * _keyword; + void *_opt_var; + UNUSED(_opt_var); // Silence compiler warnings + void *_tmp_122_var; + if ( + (_opt_var = _PyPegen_expect_token(p, 673), !p->error_indicator) // 'async'? + && + (_keyword = _PyPegen_expect_token(p, 671)) // token='for' + && + (_tmp_122_var = _tmp_122_rule(p)) // bitwise_or ((',' bitwise_or))* ','? + && + _PyPegen_lookahead_with_int(0, _PyPegen_expect_token, p, 672) // token='in' + ) + { + D(fprintf(stderr, "%*c+ for_if_clause[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "'async'? 'for' (bitwise_or ((',' bitwise_or))* ','?) !'in'")); + _res = RAISE_SYNTAX_ERROR ( "'in' expected after for-loop variables" ); + if (_res == NULL && PyErr_Occurred()) { + p->error_indicator = 1; + p->level--; + return NULL; + } + goto done; + } + p->mark = _mark; + D(fprintf(stderr, "%*c%s for_if_clause[%d-%d]: %s failed!\n", p->level, ' ', + p->error_indicator ? "ERROR!" : "-", _mark, p->mark, "'async'? 'for' (bitwise_or ((',' bitwise_or))* ','?) !'in'")); + } if (p->call_invalid_rules) { // invalid_for_target if (p->error_indicator) { p->level--; @@ -17190,7 +17230,7 @@ genexp_rule(Parser *p) if ( (_literal = _PyPegen_expect_token(p, 7)) // token='(' && - (a = _tmp_122_rule(p)) // assignment_expression | expression !':=' + (a = _tmp_123_rule(p)) // assignment_expression | expression !':=' && (b = for_if_clauses_rule(p)) // for_if_clauses && @@ -17439,9 +17479,9 @@ args_rule(Parser *p) asdl_expr_seq* a; void *b; if ( - (a = (asdl_expr_seq*)_gather_123_rule(p)) // ','.(starred_expression | (assignment_expression | expression !':=') !'=')+ + (a = (asdl_expr_seq*)_gather_124_rule(p)) // ','.(starred_expression | (assignment_expression | expression !':=') !'=')+ && - (b = _tmp_125_rule(p), !p->error_indicator) // [',' kwargs] + (b = _tmp_126_rule(p), !p->error_indicator) // [',' kwargs] ) { D(fprintf(stderr, "%*c+ args[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "','.(starred_expression | (assignment_expression | expression !':=') !'=')+ [',' kwargs]")); @@ -17531,11 +17571,11 @@ kwargs_rule(Parser *p) asdl_seq * a; asdl_seq * b; if ( - (a = _gather_126_rule(p)) // ','.kwarg_or_starred+ + (a = _gather_127_rule(p)) // ','.kwarg_or_starred+ && (_literal = _PyPegen_expect_token(p, 12)) // token=',' && - (b = _gather_128_rule(p)) // ','.kwarg_or_double_starred+ + (b = _gather_129_rule(p)) // ','.kwarg_or_double_starred+ ) { D(fprintf(stderr, "%*c+ kwargs[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "','.kwarg_or_starred+ ',' ','.kwarg_or_double_starred+")); @@ -17557,13 +17597,13 @@ kwargs_rule(Parser *p) return NULL; } D(fprintf(stderr, "%*c> kwargs[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "','.kwarg_or_starred+")); - asdl_seq * _gather_130_var; + asdl_seq * _gather_131_var; if ( - (_gather_130_var = _gather_130_rule(p)) // ','.kwarg_or_starred+ + (_gather_131_var = _gather_131_rule(p)) // ','.kwarg_or_starred+ ) { D(fprintf(stderr, "%*c+ kwargs[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "','.kwarg_or_starred+")); - _res = _gather_130_var; + _res = _gather_131_var; goto done; } p->mark = _mark; @@ -17576,13 +17616,13 @@ kwargs_rule(Parser *p) return NULL; } D(fprintf(stderr, "%*c> kwargs[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "','.kwarg_or_double_starred+")); - asdl_seq * _gather_132_var; + asdl_seq * _gather_133_var; if ( - (_gather_132_var = _gather_132_rule(p)) // ','.kwarg_or_double_starred+ + (_gather_133_var = _gather_133_rule(p)) // ','.kwarg_or_double_starred+ ) { D(fprintf(stderr, "%*c+ kwargs[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "','.kwarg_or_double_starred+")); - _res = _gather_132_var; + _res = _gather_133_var; goto done; } p->mark = _mark; @@ -17971,7 +18011,7 @@ star_targets_rule(Parser *p) if ( (a = star_target_rule(p)) // star_target && - (b = _loop0_134_rule(p)) // ((',' star_target))* + (b = _loop0_135_rule(p)) // ((',' star_target))* && (_opt_var = _PyPegen_expect_token(p, 12), !p->error_indicator) // ','? ) @@ -18027,7 +18067,7 @@ star_targets_list_seq_rule(Parser *p) UNUSED(_opt_var); // Silence compiler warnings asdl_expr_seq* a; if ( - (a = (asdl_expr_seq*)_gather_135_rule(p)) // ','.star_target+ + (a = (asdl_expr_seq*)_gather_136_rule(p)) // ','.star_target+ && (_opt_var = _PyPegen_expect_token(p, 12), !p->error_indicator) // ','? ) @@ -18077,7 +18117,7 @@ star_targets_tuple_seq_rule(Parser *p) if ( (a = star_target_rule(p)) // star_target && - (b = _loop1_137_rule(p)) // ((',' star_target))+ + (b = _loop1_138_rule(p)) // ((',' star_target))+ && (_opt_var = _PyPegen_expect_token(p, 12), !p->error_indicator) // ','? ) @@ -18165,7 +18205,7 @@ star_target_rule(Parser *p) if ( (_literal = _PyPegen_expect_token(p, 16)) // token='*' && - (a = _tmp_138_rule(p)) // !'*' star_target + (a = _tmp_139_rule(p)) // !'*' star_target ) { D(fprintf(stderr, "%*c+ star_target[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "'*' (!'*' star_target)")); @@ -19088,7 +19128,7 @@ del_targets_rule(Parser *p) UNUSED(_opt_var); // Silence compiler warnings asdl_expr_seq* a; if ( - (a = (asdl_expr_seq*)_gather_139_rule(p)) // ','.del_target+ + (a = (asdl_expr_seq*)_gather_140_rule(p)) // ','.del_target+ && (_opt_var = _PyPegen_expect_token(p, 12), !p->error_indicator) // ','? ) @@ -19446,7 +19486,7 @@ type_expressions_rule(Parser *p) expr_ty b; expr_ty c; if ( - (a = _gather_141_rule(p)) // ','.expression+ + (a = _gather_142_rule(p)) // ','.expression+ && (_literal = _PyPegen_expect_token(p, 12)) // token=',' && @@ -19485,7 +19525,7 @@ type_expressions_rule(Parser *p) asdl_seq * a; expr_ty b; if ( - (a = _gather_143_rule(p)) // ','.expression+ + (a = _gather_144_rule(p)) // ','.expression+ && (_literal = _PyPegen_expect_token(p, 12)) // token=',' && @@ -19518,7 +19558,7 @@ type_expressions_rule(Parser *p) asdl_seq * a; expr_ty b; if ( - (a = _gather_145_rule(p)) // ','.expression+ + (a = _gather_146_rule(p)) // ','.expression+ && (_literal = _PyPegen_expect_token(p, 12)) // token=',' && @@ -19638,7 +19678,7 @@ type_expressions_rule(Parser *p) D(fprintf(stderr, "%*c> type_expressions[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "','.expression+")); asdl_expr_seq* a; if ( - (a = (asdl_expr_seq*)_gather_147_rule(p)) // ','.expression+ + (a = (asdl_expr_seq*)_gather_148_rule(p)) // ','.expression+ ) { D(fprintf(stderr, "%*c+ type_expressions[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "','.expression+")); @@ -19689,7 +19729,7 @@ func_type_comment_rule(Parser *p) && (t = _PyPegen_expect_token(p, TYPE_COMMENT)) // token='TYPE_COMMENT' && - _PyPegen_lookahead(1, _tmp_149_rule, p) + _PyPegen_lookahead(1, _tmp_150_rule, p) ) { D(fprintf(stderr, "%*c+ func_type_comment[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "NEWLINE TYPE_COMMENT &(NEWLINE INDENT)")); @@ -19776,10 +19816,10 @@ invalid_arguments_rule(Parser *p) } D(fprintf(stderr, "%*c> invalid_arguments[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "((','.(starred_expression | (assignment_expression | expression !':=') !'=')+ ',' kwargs) | kwargs) ',' '*'")); Token * _literal; - void *_tmp_150_var; + void *_tmp_151_var; Token * b; if ( - (_tmp_150_var = _tmp_150_rule(p)) // (','.(starred_expression | (assignment_expression | expression !':=') !'=')+ ',' kwargs) | kwargs + (_tmp_151_var = _tmp_151_rule(p)) // (','.(starred_expression | (assignment_expression | expression !':=') !'=')+ ',' kwargs) | kwargs && (_literal = _PyPegen_expect_token(p, 12)) // token=',' && @@ -19817,7 +19857,7 @@ invalid_arguments_rule(Parser *p) && (_literal = _PyPegen_expect_token(p, 12)) // token=',' && - (_opt_var = _tmp_151_rule(p), !p->error_indicator) // [args | expression for_if_clauses] + (_opt_var = _tmp_152_rule(p), !p->error_indicator) // [args | expression for_if_clauses] ) { D(fprintf(stderr, "%*c+ invalid_arguments[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "expression for_if_clauses ',' [args | expression for_if_clauses]")); @@ -19877,13 +19917,13 @@ invalid_arguments_rule(Parser *p) expr_ty a; Token * b; if ( - (_opt_var = _tmp_152_rule(p), !p->error_indicator) // [(args ',')] + (_opt_var = _tmp_153_rule(p), !p->error_indicator) // [(args ',')] && (a = _PyPegen_name_token(p)) // NAME && (b = _PyPegen_expect_token(p, 22)) // token='=' && - _PyPegen_lookahead(1, _tmp_153_rule, p) + _PyPegen_lookahead(1, _tmp_154_rule, p) ) { D(fprintf(stderr, "%*c+ invalid_arguments[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "[(args ',')] NAME '=' &(',' | ')')")); @@ -20021,7 +20061,7 @@ invalid_kwarg_rule(Parser *p) Token* a; Token * b; if ( - (a = (Token*)_tmp_154_rule(p)) // 'True' | 'False' | 'None' + (a = (Token*)_tmp_155_rule(p)) // 'True' | 'False' | 'None' && (b = _PyPegen_expect_token(p, 22)) // token='=' ) @@ -20081,7 +20121,7 @@ invalid_kwarg_rule(Parser *p) expr_ty a; Token * b; if ( - _PyPegen_lookahead(0, _tmp_155_rule, p) + _PyPegen_lookahead(0, _tmp_156_rule, p) && (a = expression_rule(p)) // expression && @@ -20184,11 +20224,11 @@ expression_without_invalid_rule(Parser *p) if ( (a = disjunction_rule(p)) // disjunction && - (_keyword = _PyPegen_expect_token(p, 658)) // token='if' + (_keyword = _PyPegen_expect_token(p, 661)) // token='if' && (b = disjunction_rule(p)) // disjunction && - (_keyword_1 = _PyPegen_expect_token(p, 661)) // token='else' + (_keyword_1 = _PyPegen_expect_token(p, 664)) // token='else' && (c = expression_rule(p)) // expression ) @@ -20337,7 +20377,7 @@ invalid_expression_rule(Parser *p) expr_ty a; expr_ty b; if ( - _PyPegen_lookahead(0, _tmp_156_rule, p) + _PyPegen_lookahead(0, _tmp_157_rule, p) && (a = disjunction_rule(p)) // disjunction && @@ -20369,11 +20409,11 @@ invalid_expression_rule(Parser *p) if ( (a = disjunction_rule(p)) // disjunction && - (_keyword = _PyPegen_expect_token(p, 658)) // token='if' + (_keyword = _PyPegen_expect_token(p, 661)) // token='if' && (b = disjunction_rule(p)) // disjunction && - _PyPegen_lookahead(0, _tmp_157_rule, p) + _PyPegen_lookahead(0, _tmp_158_rule, p) ) { D(fprintf(stderr, "%*c+ invalid_expression[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "disjunction 'if' disjunction !('else' | ':')")); @@ -20400,7 +20440,7 @@ invalid_expression_rule(Parser *p) Token * a; Token * b; if ( - (a = _PyPegen_expect_token(p, 609)) // token='lambda' + (a = _PyPegen_expect_token(p, 612)) // token='lambda' && (_opt_var = lambda_params_rule(p), !p->error_indicator) // lambda_params? && @@ -20494,7 +20534,7 @@ invalid_named_expression_rule(Parser *p) && (b = bitwise_or_rule(p)) // bitwise_or && - _PyPegen_lookahead(0, _tmp_158_rule, p) + _PyPegen_lookahead(0, _tmp_159_rule, p) ) { D(fprintf(stderr, "%*c+ invalid_named_expression[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "NAME '=' bitwise_or !('=' | ':=')")); @@ -20520,7 +20560,7 @@ invalid_named_expression_rule(Parser *p) Token * b; expr_ty bitwise_or_var; if ( - _PyPegen_lookahead(0, _tmp_159_rule, p) + _PyPegen_lookahead(0, _tmp_160_rule, p) && (a = bitwise_or_rule(p)) // bitwise_or && @@ -20528,7 +20568,7 @@ invalid_named_expression_rule(Parser *p) && (bitwise_or_var = bitwise_or_rule(p)) // bitwise_or && - _PyPegen_lookahead(0, _tmp_160_rule, p) + _PyPegen_lookahead(0, _tmp_161_rule, p) ) { D(fprintf(stderr, "%*c+ invalid_named_expression[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "!(list | tuple | genexp | 'True' | 'None' | 'False') bitwise_or '=' bitwise_or !('=' | ':=')")); @@ -20608,7 +20648,7 @@ invalid_assignment_rule(Parser *p) D(fprintf(stderr, "%*c> invalid_assignment[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "star_named_expression ',' star_named_expressions* ':' expression")); Token * _literal; Token * _literal_1; - asdl_seq * _loop0_161_var; + asdl_seq * _loop0_162_var; expr_ty a; expr_ty expression_var; if ( @@ -20616,7 +20656,7 @@ invalid_assignment_rule(Parser *p) && (_literal = _PyPegen_expect_token(p, 12)) // token=',' && - (_loop0_161_var = _loop0_161_rule(p)) // star_named_expressions* + (_loop0_162_var = _loop0_162_rule(p)) // star_named_expressions* && (_literal_1 = _PyPegen_expect_token(p, 11)) // token=':' && @@ -20673,10 +20713,10 @@ invalid_assignment_rule(Parser *p) } D(fprintf(stderr, "%*c> invalid_assignment[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "((star_targets '='))* star_expressions '='")); Token * _literal; - asdl_seq * _loop0_162_var; + asdl_seq * _loop0_163_var; expr_ty a; if ( - (_loop0_162_var = _loop0_162_rule(p)) // ((star_targets '='))* + (_loop0_163_var = _loop0_163_rule(p)) // ((star_targets '='))* && (a = star_expressions_rule(p)) // star_expressions && @@ -20703,10 +20743,10 @@ invalid_assignment_rule(Parser *p) } D(fprintf(stderr, "%*c> invalid_assignment[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "((star_targets '='))* yield_expr '='")); Token * _literal; - asdl_seq * _loop0_163_var; + asdl_seq * _loop0_164_var; expr_ty a; if ( - (_loop0_163_var = _loop0_163_rule(p)) // ((star_targets '='))* + (_loop0_164_var = _loop0_164_rule(p)) // ((star_targets '='))* && (a = yield_expr_rule(p)) // yield_expr && @@ -20732,7 +20772,7 @@ invalid_assignment_rule(Parser *p) return NULL; } D(fprintf(stderr, "%*c> invalid_assignment[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "star_expressions augassign (yield_expr | star_expressions)")); - void *_tmp_164_var; + void *_tmp_165_var; expr_ty a; AugOperator* augassign_var; if ( @@ -20740,7 +20780,7 @@ invalid_assignment_rule(Parser *p) && (augassign_var = augassign_rule(p)) // augassign && - (_tmp_164_var = _tmp_164_rule(p)) // yield_expr | star_expressions + (_tmp_165_var = _tmp_165_rule(p)) // yield_expr | star_expressions ) { D(fprintf(stderr, "%*c+ invalid_assignment[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "star_expressions augassign (yield_expr | star_expressions)")); @@ -20871,7 +20911,7 @@ invalid_del_stmt_rule(Parser *p) Token * _keyword; expr_ty a; if ( - (_keyword = _PyPegen_expect_token(p, 613)) // token='del' + (_keyword = _PyPegen_expect_token(p, 616)) // token='del' && (a = star_expressions_rule(p)) // star_expressions ) @@ -20962,11 +21002,11 @@ invalid_comprehension_rule(Parser *p) return NULL; } D(fprintf(stderr, "%*c> invalid_comprehension[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "('[' | '(' | '{') starred_expression for_if_clauses")); - void *_tmp_165_var; + void *_tmp_166_var; expr_ty a; asdl_comprehension_seq* for_if_clauses_var; if ( - (_tmp_165_var = _tmp_165_rule(p)) // '[' | '(' | '{' + (_tmp_166_var = _tmp_166_rule(p)) // '[' | '(' | '{' && (a = starred_expression_rule(p)) // starred_expression && @@ -20993,12 +21033,12 @@ invalid_comprehension_rule(Parser *p) } D(fprintf(stderr, "%*c> invalid_comprehension[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "('[' | '{') star_named_expression ',' star_named_expressions for_if_clauses")); Token * _literal; - void *_tmp_166_var; + void *_tmp_167_var; expr_ty a; asdl_expr_seq* b; asdl_comprehension_seq* for_if_clauses_var; if ( - (_tmp_166_var = _tmp_166_rule(p)) // '[' | '{' + (_tmp_167_var = _tmp_167_rule(p)) // '[' | '{' && (a = star_named_expression_rule(p)) // star_named_expression && @@ -21028,12 +21068,12 @@ invalid_comprehension_rule(Parser *p) return NULL; } D(fprintf(stderr, "%*c> invalid_comprehension[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "('[' | '{') star_named_expression ',' for_if_clauses")); - void *_tmp_167_var; + void *_tmp_168_var; expr_ty a; Token * b; asdl_comprehension_seq* for_if_clauses_var; if ( - (_tmp_167_var = _tmp_167_rule(p)) // '[' | '{' + (_tmp_168_var = _tmp_168_rule(p)) // '[' | '{' && (a = star_named_expression_rule(p)) // star_named_expression && @@ -21168,13 +21208,13 @@ invalid_parameters_rule(Parser *p) return NULL; } D(fprintf(stderr, "%*c> invalid_parameters[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "(slash_no_default | slash_with_default) param_maybe_default* '/'")); - asdl_seq * _loop0_169_var; - void *_tmp_168_var; + asdl_seq * _loop0_170_var; + void *_tmp_169_var; Token * a; if ( - (_tmp_168_var = _tmp_168_rule(p)) // slash_no_default | slash_with_default + (_tmp_169_var = _tmp_169_rule(p)) // slash_no_default | slash_with_default && - (_loop0_169_var = _loop0_169_rule(p)) // param_maybe_default* + (_loop0_170_var = _loop0_170_rule(p)) // param_maybe_default* && (a = _PyPegen_expect_token(p, 17)) // token='/' ) @@ -21198,7 +21238,7 @@ invalid_parameters_rule(Parser *p) return NULL; } D(fprintf(stderr, "%*c> invalid_parameters[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "slash_no_default? param_no_default* invalid_parameters_helper param_no_default")); - asdl_seq * _loop0_170_var; + asdl_seq * _loop0_171_var; void *_opt_var; UNUSED(_opt_var); // Silence compiler warnings arg_ty a; @@ -21206,7 +21246,7 @@ invalid_parameters_rule(Parser *p) if ( (_opt_var = slash_no_default_rule(p), !p->error_indicator) // slash_no_default? && - (_loop0_170_var = _loop0_170_rule(p)) // param_no_default* + (_loop0_171_var = _loop0_171_rule(p)) // param_no_default* && (invalid_parameters_helper_var = invalid_parameters_helper_rule(p)) // invalid_parameters_helper && @@ -21232,18 +21272,18 @@ invalid_parameters_rule(Parser *p) return NULL; } D(fprintf(stderr, "%*c> invalid_parameters[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "param_no_default* '(' param_no_default+ ','? ')'")); - asdl_seq * _loop0_171_var; - asdl_seq * _loop1_172_var; + asdl_seq * _loop0_172_var; + asdl_seq * _loop1_173_var; void *_opt_var; UNUSED(_opt_var); // Silence compiler warnings Token * a; Token * b; if ( - (_loop0_171_var = _loop0_171_rule(p)) // param_no_default* + (_loop0_172_var = _loop0_172_rule(p)) // param_no_default* && (a = _PyPegen_expect_token(p, 7)) // token='(' && - (_loop1_172_var = _loop1_172_rule(p)) // param_no_default+ + (_loop1_173_var = _loop1_173_rule(p)) // param_no_default+ && (_opt_var = _PyPegen_expect_token(p, 12), !p->error_indicator) // ','? && @@ -21270,22 +21310,22 @@ invalid_parameters_rule(Parser *p) } D(fprintf(stderr, "%*c> invalid_parameters[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "[(slash_no_default | slash_with_default)] param_maybe_default* '*' (',' | param_no_default) param_maybe_default* '/'")); Token * _literal; - asdl_seq * _loop0_174_var; - asdl_seq * _loop0_176_var; + asdl_seq * _loop0_175_var; + asdl_seq * _loop0_177_var; void *_opt_var; UNUSED(_opt_var); // Silence compiler warnings - void *_tmp_175_var; + void *_tmp_176_var; Token * a; if ( - (_opt_var = _tmp_173_rule(p), !p->error_indicator) // [(slash_no_default | slash_with_default)] + (_opt_var = _tmp_174_rule(p), !p->error_indicator) // [(slash_no_default | slash_with_default)] && - (_loop0_174_var = _loop0_174_rule(p)) // param_maybe_default* + (_loop0_175_var = _loop0_175_rule(p)) // param_maybe_default* && (_literal = _PyPegen_expect_token(p, 16)) // token='*' && - (_tmp_175_var = _tmp_175_rule(p)) // ',' | param_no_default + (_tmp_176_var = _tmp_176_rule(p)) // ',' | param_no_default && - (_loop0_176_var = _loop0_176_rule(p)) // param_maybe_default* + (_loop0_177_var = _loop0_177_rule(p)) // param_maybe_default* && (a = _PyPegen_expect_token(p, 17)) // token='/' ) @@ -21310,10 +21350,10 @@ invalid_parameters_rule(Parser *p) } D(fprintf(stderr, "%*c> invalid_parameters[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "param_maybe_default+ '/' '*'")); Token * _literal; - asdl_seq * _loop1_177_var; + asdl_seq * _loop1_178_var; Token * a; if ( - (_loop1_177_var = _loop1_177_rule(p)) // param_maybe_default+ + (_loop1_178_var = _loop1_178_rule(p)) // param_maybe_default+ && (_literal = _PyPegen_expect_token(p, 17)) // token='/' && @@ -21362,7 +21402,7 @@ invalid_default_rule(Parser *p) if ( (a = _PyPegen_expect_token(p, 22)) // token='=' && - _PyPegen_lookahead(1, _tmp_178_rule, p) + _PyPegen_lookahead(1, _tmp_179_rule, p) ) { D(fprintf(stderr, "%*c+ invalid_default[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "'=' &(')' | ',')")); @@ -21407,12 +21447,12 @@ invalid_star_etc_rule(Parser *p) return NULL; } D(fprintf(stderr, "%*c> invalid_star_etc[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "'*' (')' | ',' (')' | '**'))")); - void *_tmp_179_var; + void *_tmp_180_var; Token * a; if ( (a = _PyPegen_expect_token(p, 16)) // token='*' && - (_tmp_179_var = _tmp_179_rule(p)) // ')' | ',' (')' | '**') + (_tmp_180_var = _tmp_180_rule(p)) // ')' | ',' (')' | '**') ) { D(fprintf(stderr, "%*c+ invalid_star_etc[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "'*' (')' | ',' (')' | '**'))")); @@ -21495,20 +21535,20 @@ invalid_star_etc_rule(Parser *p) } D(fprintf(stderr, "%*c> invalid_star_etc[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "'*' (param_no_default | ',') param_maybe_default* '*' (param_no_default | ',')")); Token * _literal; - asdl_seq * _loop0_181_var; - void *_tmp_180_var; - void *_tmp_182_var; + asdl_seq * _loop0_182_var; + void *_tmp_181_var; + void *_tmp_183_var; Token * a; if ( (_literal = _PyPegen_expect_token(p, 16)) // token='*' && - (_tmp_180_var = _tmp_180_rule(p)) // param_no_default | ',' + (_tmp_181_var = _tmp_181_rule(p)) // param_no_default | ',' && - (_loop0_181_var = _loop0_181_rule(p)) // param_maybe_default* + (_loop0_182_var = _loop0_182_rule(p)) // param_maybe_default* && (a = _PyPegen_expect_token(p, 16)) // token='*' && - (_tmp_182_var = _tmp_182_rule(p)) // param_no_default | ',' + (_tmp_183_var = _tmp_183_rule(p)) // param_no_default | ',' ) { D(fprintf(stderr, "%*c+ invalid_star_etc[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "'*' (param_no_default | ',') param_maybe_default* '*' (param_no_default | ',')")); @@ -21623,7 +21663,7 @@ invalid_kwds_rule(Parser *p) && (_literal_1 = _PyPegen_expect_token(p, 12)) // token=',' && - (a = (Token*)_tmp_183_rule(p)) // '*' | '**' | '/' + (a = (Token*)_tmp_184_rule(p)) // '*' | '**' | '/' ) { D(fprintf(stderr, "%*c+ invalid_kwds[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "'**' param ',' ('*' | '**' | '/')")); @@ -21688,13 +21728,13 @@ invalid_parameters_helper_rule(Parser *p) return NULL; } D(fprintf(stderr, "%*c> invalid_parameters_helper[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "param_with_default+")); - asdl_seq * _loop1_184_var; + asdl_seq * _loop1_185_var; if ( - (_loop1_184_var = _loop1_184_rule(p)) // param_with_default+ + (_loop1_185_var = _loop1_185_rule(p)) // param_with_default+ ) { D(fprintf(stderr, "%*c+ invalid_parameters_helper[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "param_with_default+")); - _res = _loop1_184_var; + _res = _loop1_185_var; goto done; } p->mark = _mark; @@ -21759,13 +21799,13 @@ invalid_lambda_parameters_rule(Parser *p) return NULL; } D(fprintf(stderr, "%*c> invalid_lambda_parameters[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "(lambda_slash_no_default | lambda_slash_with_default) lambda_param_maybe_default* '/'")); - asdl_seq * _loop0_186_var; - void *_tmp_185_var; + asdl_seq * _loop0_187_var; + void *_tmp_186_var; Token * a; if ( - (_tmp_185_var = _tmp_185_rule(p)) // lambda_slash_no_default | lambda_slash_with_default + (_tmp_186_var = _tmp_186_rule(p)) // lambda_slash_no_default | lambda_slash_with_default && - (_loop0_186_var = _loop0_186_rule(p)) // lambda_param_maybe_default* + (_loop0_187_var = _loop0_187_rule(p)) // lambda_param_maybe_default* && (a = _PyPegen_expect_token(p, 17)) // token='/' ) @@ -21789,7 +21829,7 @@ invalid_lambda_parameters_rule(Parser *p) return NULL; } D(fprintf(stderr, "%*c> invalid_lambda_parameters[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "lambda_slash_no_default? lambda_param_no_default* invalid_lambda_parameters_helper lambda_param_no_default")); - asdl_seq * _loop0_187_var; + asdl_seq * _loop0_188_var; void *_opt_var; UNUSED(_opt_var); // Silence compiler warnings arg_ty a; @@ -21797,7 +21837,7 @@ invalid_lambda_parameters_rule(Parser *p) if ( (_opt_var = lambda_slash_no_default_rule(p), !p->error_indicator) // lambda_slash_no_default? && - (_loop0_187_var = _loop0_187_rule(p)) // lambda_param_no_default* + (_loop0_188_var = _loop0_188_rule(p)) // lambda_param_no_default* && (invalid_lambda_parameters_helper_var = invalid_lambda_parameters_helper_rule(p)) // invalid_lambda_parameters_helper && @@ -21823,18 +21863,18 @@ invalid_lambda_parameters_rule(Parser *p) return NULL; } D(fprintf(stderr, "%*c> invalid_lambda_parameters[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "lambda_param_no_default* '(' ','.lambda_param+ ','? ')'")); - asdl_seq * _gather_189_var; - asdl_seq * _loop0_188_var; + asdl_seq * _gather_190_var; + asdl_seq * _loop0_189_var; void *_opt_var; UNUSED(_opt_var); // Silence compiler warnings Token * a; Token * b; if ( - (_loop0_188_var = _loop0_188_rule(p)) // lambda_param_no_default* + (_loop0_189_var = _loop0_189_rule(p)) // lambda_param_no_default* && (a = _PyPegen_expect_token(p, 7)) // token='(' && - (_gather_189_var = _gather_189_rule(p)) // ','.lambda_param+ + (_gather_190_var = _gather_190_rule(p)) // ','.lambda_param+ && (_opt_var = _PyPegen_expect_token(p, 12), !p->error_indicator) // ','? && @@ -21861,22 +21901,22 @@ invalid_lambda_parameters_rule(Parser *p) } D(fprintf(stderr, "%*c> invalid_lambda_parameters[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "[(lambda_slash_no_default | lambda_slash_with_default)] lambda_param_maybe_default* '*' (',' | lambda_param_no_default) lambda_param_maybe_default* '/'")); Token * _literal; - asdl_seq * _loop0_192_var; - asdl_seq * _loop0_194_var; + asdl_seq * _loop0_193_var; + asdl_seq * _loop0_195_var; void *_opt_var; UNUSED(_opt_var); // Silence compiler warnings - void *_tmp_193_var; + void *_tmp_194_var; Token * a; if ( - (_opt_var = _tmp_191_rule(p), !p->error_indicator) // [(lambda_slash_no_default | lambda_slash_with_default)] + (_opt_var = _tmp_192_rule(p), !p->error_indicator) // [(lambda_slash_no_default | lambda_slash_with_default)] && - (_loop0_192_var = _loop0_192_rule(p)) // lambda_param_maybe_default* + (_loop0_193_var = _loop0_193_rule(p)) // lambda_param_maybe_default* && (_literal = _PyPegen_expect_token(p, 16)) // token='*' && - (_tmp_193_var = _tmp_193_rule(p)) // ',' | lambda_param_no_default + (_tmp_194_var = _tmp_194_rule(p)) // ',' | lambda_param_no_default && - (_loop0_194_var = _loop0_194_rule(p)) // lambda_param_maybe_default* + (_loop0_195_var = _loop0_195_rule(p)) // lambda_param_maybe_default* && (a = _PyPegen_expect_token(p, 17)) // token='/' ) @@ -21901,10 +21941,10 @@ invalid_lambda_parameters_rule(Parser *p) } D(fprintf(stderr, "%*c> invalid_lambda_parameters[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "lambda_param_maybe_default+ '/' '*'")); Token * _literal; - asdl_seq * _loop1_195_var; + asdl_seq * _loop1_196_var; Token * a; if ( - (_loop1_195_var = _loop1_195_rule(p)) // lambda_param_maybe_default+ + (_loop1_196_var = _loop1_196_rule(p)) // lambda_param_maybe_default+ && (_literal = _PyPegen_expect_token(p, 17)) // token='/' && @@ -21975,13 +22015,13 @@ invalid_lambda_parameters_helper_rule(Parser *p) return NULL; } D(fprintf(stderr, "%*c> invalid_lambda_parameters_helper[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "lambda_param_with_default+")); - asdl_seq * _loop1_196_var; + asdl_seq * _loop1_197_var; if ( - (_loop1_196_var = _loop1_196_rule(p)) // lambda_param_with_default+ + (_loop1_197_var = _loop1_197_rule(p)) // lambda_param_with_default+ ) { D(fprintf(stderr, "%*c+ invalid_lambda_parameters_helper[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "lambda_param_with_default+")); - _res = _loop1_196_var; + _res = _loop1_197_var; goto done; } p->mark = _mark; @@ -22017,11 +22057,11 @@ invalid_lambda_star_etc_rule(Parser *p) } D(fprintf(stderr, "%*c> invalid_lambda_star_etc[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "'*' (':' | ',' (':' | '**'))")); Token * _literal; - void *_tmp_197_var; + void *_tmp_198_var; if ( (_literal = _PyPegen_expect_token(p, 16)) // token='*' && - (_tmp_197_var = _tmp_197_rule(p)) // ':' | ',' (':' | '**') + (_tmp_198_var = _tmp_198_rule(p)) // ':' | ',' (':' | '**') ) { D(fprintf(stderr, "%*c+ invalid_lambda_star_etc[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "'*' (':' | ',' (':' | '**'))")); @@ -22074,20 +22114,20 @@ invalid_lambda_star_etc_rule(Parser *p) } D(fprintf(stderr, "%*c> invalid_lambda_star_etc[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "'*' (lambda_param_no_default | ',') lambda_param_maybe_default* '*' (lambda_param_no_default | ',')")); Token * _literal; - asdl_seq * _loop0_199_var; - void *_tmp_198_var; - void *_tmp_200_var; + asdl_seq * _loop0_200_var; + void *_tmp_199_var; + void *_tmp_201_var; Token * a; if ( (_literal = _PyPegen_expect_token(p, 16)) // token='*' && - (_tmp_198_var = _tmp_198_rule(p)) // lambda_param_no_default | ',' + (_tmp_199_var = _tmp_199_rule(p)) // lambda_param_no_default | ',' && - (_loop0_199_var = _loop0_199_rule(p)) // lambda_param_maybe_default* + (_loop0_200_var = _loop0_200_rule(p)) // lambda_param_maybe_default* && (a = _PyPegen_expect_token(p, 16)) // token='*' && - (_tmp_200_var = _tmp_200_rule(p)) // lambda_param_no_default | ',' + (_tmp_201_var = _tmp_201_rule(p)) // lambda_param_no_default | ',' ) { D(fprintf(stderr, "%*c+ invalid_lambda_star_etc[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "'*' (lambda_param_no_default | ',') lambda_param_maybe_default* '*' (lambda_param_no_default | ',')")); @@ -22205,7 +22245,7 @@ invalid_lambda_kwds_rule(Parser *p) && (_literal_1 = _PyPegen_expect_token(p, 12)) // token=',' && - (a = (Token*)_tmp_201_rule(p)) // '*' | '**' | '/' + (a = (Token*)_tmp_202_rule(p)) // '*' | '**' | '/' ) { D(fprintf(stderr, "%*c+ invalid_lambda_kwds[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "'**' lambda_param ',' ('*' | '**' | '/')")); @@ -22307,11 +22347,11 @@ invalid_with_item_rule(Parser *p) if ( (expression_var = expression_rule(p)) // expression && - (_keyword = _PyPegen_expect_token(p, 656)) // token='as' + (_keyword = _PyPegen_expect_token(p, 659)) // token='as' && (a = expression_rule(p)) // expression && - _PyPegen_lookahead(1, _tmp_202_rule, p) + _PyPegen_lookahead(1, _tmp_203_rule, p) ) { D(fprintf(stderr, "%*c+ invalid_with_item[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "expression 'as' expression &(',' | ')' | ':')")); @@ -22357,9 +22397,9 @@ invalid_for_target_rule(Parser *p) UNUSED(_opt_var); // Silence compiler warnings expr_ty a; if ( - (_opt_var = _PyPegen_expect_token(p, 670), !p->error_indicator) // 'async'? + (_opt_var = _PyPegen_expect_token(p, 673), !p->error_indicator) // 'async'? && - (_keyword = _PyPegen_expect_token(p, 668)) // token='for' + (_keyword = _PyPegen_expect_token(p, 671)) // token='for' && (a = star_expressions_rule(p)) // star_expressions ) @@ -22484,16 +22524,16 @@ invalid_import_rule(Parser *p) return NULL; } D(fprintf(stderr, "%*c> invalid_import[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "'import' ','.dotted_name+ 'from' dotted_name")); - asdl_seq * _gather_203_var; + asdl_seq * _gather_204_var; Token * _keyword; Token * a; expr_ty dotted_name_var; if ( - (a = _PyPegen_expect_token(p, 617)) // token='import' + (a = _PyPegen_expect_token(p, 620)) // token='import' && - (_gather_203_var = _gather_203_rule(p)) // ','.dotted_name+ + (_gather_204_var = _gather_204_rule(p)) // ','.dotted_name+ && - (_keyword = _PyPegen_expect_token(p, 618)) // token='from' + (_keyword = _PyPegen_expect_token(p, 621)) // token='from' && (dotted_name_var = dotted_name_rule(p)) // dotted_name ) @@ -22589,7 +22629,7 @@ invalid_compound_stmt_rule(Parser *p) Token * a; expr_ty named_expression_var; if ( - (a = _PyPegen_expect_token(p, 660)) // token='elif' + (a = _PyPegen_expect_token(p, 663)) // token='elif' && (named_expression_var = named_expression_rule(p)) // named_expression && @@ -22618,7 +22658,7 @@ invalid_compound_stmt_rule(Parser *p) Token * _literal; Token * a; if ( - (a = _PyPegen_expect_token(p, 661)) // token='else' + (a = _PyPegen_expect_token(p, 664)) // token='else' && (_literal = _PyPegen_expect_token(p, 11)) // token=':' ) @@ -22663,17 +22703,17 @@ invalid_with_stmt_rule(Parser *p) return NULL; } D(fprintf(stderr, "%*c> invalid_with_stmt[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "'async'? 'with' ','.(expression ['as' star_target])+ NEWLINE")); - asdl_seq * _gather_205_var; + asdl_seq * _gather_206_var; Token * _keyword; void *_opt_var; UNUSED(_opt_var); // Silence compiler warnings Token * newline_var; if ( - (_opt_var = _PyPegen_expect_token(p, 670), !p->error_indicator) // 'async'? + (_opt_var = _PyPegen_expect_token(p, 673), !p->error_indicator) // 'async'? && - (_keyword = _PyPegen_expect_token(p, 631)) // token='with' + (_keyword = _PyPegen_expect_token(p, 634)) // token='with' && - (_gather_205_var = _gather_205_rule(p)) // ','.(expression ['as' star_target])+ + (_gather_206_var = _gather_206_rule(p)) // ','.(expression ['as' star_target])+ && (newline_var = _PyPegen_expect_token(p, NEWLINE)) // token='NEWLINE' ) @@ -22697,7 +22737,7 @@ invalid_with_stmt_rule(Parser *p) return NULL; } D(fprintf(stderr, "%*c> invalid_with_stmt[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "'async'? 'with' '(' ','.(expressions ['as' star_target])+ ','? ')' NEWLINE")); - asdl_seq * _gather_207_var; + asdl_seq * _gather_208_var; Token * _keyword; Token * _literal; Token * _literal_1; @@ -22707,13 +22747,13 @@ invalid_with_stmt_rule(Parser *p) UNUSED(_opt_var_1); // Silence compiler warnings Token * newline_var; if ( - (_opt_var = _PyPegen_expect_token(p, 670), !p->error_indicator) // 'async'? + (_opt_var = _PyPegen_expect_token(p, 673), !p->error_indicator) // 'async'? && - (_keyword = _PyPegen_expect_token(p, 631)) // token='with' + (_keyword = _PyPegen_expect_token(p, 634)) // token='with' && (_literal = _PyPegen_expect_token(p, 7)) // token='(' && - (_gather_207_var = _gather_207_rule(p)) // ','.(expressions ['as' star_target])+ + (_gather_208_var = _gather_208_rule(p)) // ','.(expressions ['as' star_target])+ && (_opt_var_1 = _PyPegen_expect_token(p, 12), !p->error_indicator) // ','? && @@ -22762,18 +22802,18 @@ invalid_with_stmt_indent_rule(Parser *p) return NULL; } D(fprintf(stderr, "%*c> invalid_with_stmt_indent[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "'async'? 'with' ','.(expression ['as' star_target])+ ':' NEWLINE !INDENT")); - asdl_seq * _gather_209_var; + asdl_seq * _gather_210_var; Token * _literal; void *_opt_var; UNUSED(_opt_var); // Silence compiler warnings Token * a; Token * newline_var; if ( - (_opt_var = _PyPegen_expect_token(p, 670), !p->error_indicator) // 'async'? + (_opt_var = _PyPegen_expect_token(p, 673), !p->error_indicator) // 'async'? && - (a = _PyPegen_expect_token(p, 631)) // token='with' + (a = _PyPegen_expect_token(p, 634)) // token='with' && - (_gather_209_var = _gather_209_rule(p)) // ','.(expression ['as' star_target])+ + (_gather_210_var = _gather_210_rule(p)) // ','.(expression ['as' star_target])+ && (_literal = _PyPegen_expect_token(p, 11)) // token=':' && @@ -22801,7 +22841,7 @@ invalid_with_stmt_indent_rule(Parser *p) return NULL; } D(fprintf(stderr, "%*c> invalid_with_stmt_indent[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "'async'? 'with' '(' ','.(expressions ['as' star_target])+ ','? ')' ':' NEWLINE !INDENT")); - asdl_seq * _gather_211_var; + asdl_seq * _gather_212_var; Token * _literal; Token * _literal_1; Token * _literal_2; @@ -22812,13 +22852,13 @@ invalid_with_stmt_indent_rule(Parser *p) Token * a; Token * newline_var; if ( - (_opt_var = _PyPegen_expect_token(p, 670), !p->error_indicator) // 'async'? + (_opt_var = _PyPegen_expect_token(p, 673), !p->error_indicator) // 'async'? && - (a = _PyPegen_expect_token(p, 631)) // token='with' + (a = _PyPegen_expect_token(p, 634)) // token='with' && (_literal = _PyPegen_expect_token(p, 7)) // token='(' && - (_gather_211_var = _gather_211_rule(p)) // ','.(expressions ['as' star_target])+ + (_gather_212_var = _gather_212_rule(p)) // ','.(expressions ['as' star_target])+ && (_opt_var_1 = _PyPegen_expect_token(p, 12), !p->error_indicator) // ','? && @@ -22877,7 +22917,7 @@ invalid_try_stmt_rule(Parser *p) Token * a; Token * newline_var; if ( - (a = _PyPegen_expect_token(p, 640)) // token='try' + (a = _PyPegen_expect_token(p, 643)) // token='try' && (_literal = _PyPegen_expect_token(p, 11)) // token=':' && @@ -22909,13 +22949,13 @@ invalid_try_stmt_rule(Parser *p) Token * _literal; asdl_stmt_seq* block_var; if ( - (_keyword = _PyPegen_expect_token(p, 640)) // token='try' + (_keyword = _PyPegen_expect_token(p, 643)) // token='try' && (_literal = _PyPegen_expect_token(p, 11)) // token=':' && (block_var = block_rule(p)) // block && - _PyPegen_lookahead(0, _tmp_213_rule, p) + _PyPegen_lookahead(0, _tmp_214_rule, p) ) { D(fprintf(stderr, "%*c+ invalid_try_stmt[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "'try' ':' block !('except' | 'finally')")); @@ -22940,29 +22980,29 @@ invalid_try_stmt_rule(Parser *p) Token * _keyword; Token * _literal; Token * _literal_1; - asdl_seq * _loop0_214_var; - asdl_seq * _loop1_215_var; + asdl_seq * _loop0_215_var; + asdl_seq * _loop1_216_var; void *_opt_var; UNUSED(_opt_var); // Silence compiler warnings Token * a; Token * b; expr_ty expression_var; if ( - (_keyword = _PyPegen_expect_token(p, 640)) // token='try' + (_keyword = _PyPegen_expect_token(p, 643)) // token='try' && (_literal = _PyPegen_expect_token(p, 11)) // token=':' && - (_loop0_214_var = _loop0_214_rule(p)) // block* + (_loop0_215_var = _loop0_215_rule(p)) // block* && - (_loop1_215_var = _loop1_215_rule(p)) // except_block+ + (_loop1_216_var = _loop1_216_rule(p)) // except_block+ && - (a = _PyPegen_expect_token(p, 653)) // token='except' + (a = _PyPegen_expect_token(p, 656)) // token='except' && (b = _PyPegen_expect_token(p, 16)) // token='*' && (expression_var = expression_rule(p)) // expression && - (_opt_var = _tmp_216_rule(p), !p->error_indicator) // ['as' NAME] + (_opt_var = _tmp_217_rule(p), !p->error_indicator) // ['as' NAME] && (_literal_1 = _PyPegen_expect_token(p, 11)) // token=':' ) @@ -22989,23 +23029,23 @@ invalid_try_stmt_rule(Parser *p) Token * _keyword; Token * _literal; Token * _literal_1; - asdl_seq * _loop0_217_var; - asdl_seq * _loop1_218_var; + asdl_seq * _loop0_218_var; + asdl_seq * _loop1_219_var; void *_opt_var; UNUSED(_opt_var); // Silence compiler warnings Token * a; if ( - (_keyword = _PyPegen_expect_token(p, 640)) // token='try' + (_keyword = _PyPegen_expect_token(p, 643)) // token='try' && (_literal = _PyPegen_expect_token(p, 11)) // token=':' && - (_loop0_217_var = _loop0_217_rule(p)) // block* + (_loop0_218_var = _loop0_218_rule(p)) // block* && - (_loop1_218_var = _loop1_218_rule(p)) // except_star_block+ + (_loop1_219_var = _loop1_219_rule(p)) // except_star_block+ && - (a = _PyPegen_expect_token(p, 653)) // token='except' + (a = _PyPegen_expect_token(p, 656)) // token='except' && - (_opt_var = _tmp_219_rule(p), !p->error_indicator) // [expression ['as' NAME]] + (_opt_var = _tmp_220_rule(p), !p->error_indicator) // [expression ['as' NAME]] && (_literal_1 = _PyPegen_expect_token(p, 11)) // token=':' ) @@ -23062,7 +23102,7 @@ invalid_except_stmt_rule(Parser *p) expr_ty a; expr_ty expressions_var; if ( - (_keyword = _PyPegen_expect_token(p, 653)) // token='except' + (_keyword = _PyPegen_expect_token(p, 656)) // token='except' && (_opt_var = _PyPegen_expect_token(p, 16), !p->error_indicator) // '*'? && @@ -23072,7 +23112,7 @@ invalid_except_stmt_rule(Parser *p) && (expressions_var = expressions_rule(p)) // expressions && - (_opt_var_1 = _tmp_220_rule(p), !p->error_indicator) // ['as' NAME] + (_opt_var_1 = _tmp_221_rule(p), !p->error_indicator) // ['as' NAME] && (_literal_1 = _PyPegen_expect_token(p, 11)) // token=':' ) @@ -23104,13 +23144,13 @@ invalid_except_stmt_rule(Parser *p) expr_ty expression_var; Token * newline_var; if ( - (a = _PyPegen_expect_token(p, 653)) // token='except' + (a = _PyPegen_expect_token(p, 656)) // token='except' && (_opt_var = _PyPegen_expect_token(p, 16), !p->error_indicator) // '*'? && (expression_var = expression_rule(p)) // expression && - (_opt_var_1 = _tmp_221_rule(p), !p->error_indicator) // ['as' NAME] + (_opt_var_1 = _tmp_222_rule(p), !p->error_indicator) // ['as' NAME] && (newline_var = _PyPegen_expect_token(p, NEWLINE)) // token='NEWLINE' ) @@ -23137,7 +23177,7 @@ invalid_except_stmt_rule(Parser *p) Token * a; Token * newline_var; if ( - (a = _PyPegen_expect_token(p, 653)) // token='except' + (a = _PyPegen_expect_token(p, 656)) // token='except' && (newline_var = _PyPegen_expect_token(p, NEWLINE)) // token='NEWLINE' ) @@ -23162,14 +23202,14 @@ invalid_except_stmt_rule(Parser *p) } D(fprintf(stderr, "%*c> invalid_except_stmt[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "'except' '*' (NEWLINE | ':')")); Token * _literal; - void *_tmp_222_var; + void *_tmp_223_var; Token * a; if ( - (a = _PyPegen_expect_token(p, 653)) // token='except' + (a = _PyPegen_expect_token(p, 656)) // token='except' && (_literal = _PyPegen_expect_token(p, 16)) // token='*' && - (_tmp_222_var = _tmp_222_rule(p)) // NEWLINE | ':' + (_tmp_223_var = _tmp_223_rule(p)) // NEWLINE | ':' ) { D(fprintf(stderr, "%*c+ invalid_except_stmt[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "'except' '*' (NEWLINE | ':')")); @@ -23214,7 +23254,7 @@ invalid_finally_stmt_rule(Parser *p) Token * a; Token * newline_var; if ( - (a = _PyPegen_expect_token(p, 649)) // token='finally' + (a = _PyPegen_expect_token(p, 652)) // token='finally' && (_literal = _PyPegen_expect_token(p, 11)) // token=':' && @@ -23270,11 +23310,11 @@ invalid_except_stmt_indent_rule(Parser *p) expr_ty expression_var; Token * newline_var; if ( - (a = _PyPegen_expect_token(p, 653)) // token='except' + (a = _PyPegen_expect_token(p, 656)) // token='except' && (expression_var = expression_rule(p)) // expression && - (_opt_var = _tmp_223_rule(p), !p->error_indicator) // ['as' NAME] + (_opt_var = _tmp_224_rule(p), !p->error_indicator) // ['as' NAME] && (_literal = _PyPegen_expect_token(p, 11)) // token=':' && @@ -23306,7 +23346,7 @@ invalid_except_stmt_indent_rule(Parser *p) Token * a; Token * newline_var; if ( - (a = _PyPegen_expect_token(p, 653)) // token='except' + (a = _PyPegen_expect_token(p, 656)) // token='except' && (_literal = _PyPegen_expect_token(p, 11)) // token=':' && @@ -23362,13 +23402,13 @@ invalid_except_star_stmt_indent_rule(Parser *p) expr_ty expression_var; Token * newline_var; if ( - (a = _PyPegen_expect_token(p, 653)) // token='except' + (a = _PyPegen_expect_token(p, 656)) // token='except' && (_literal = _PyPegen_expect_token(p, 16)) // token='*' && (expression_var = expression_rule(p)) // expression && - (_opt_var = _tmp_224_rule(p), !p->error_indicator) // ['as' NAME] + (_opt_var = _tmp_225_rule(p), !p->error_indicator) // ['as' NAME] && (_literal_1 = _PyPegen_expect_token(p, 11)) // token=':' && @@ -23601,7 +23641,7 @@ invalid_as_pattern_rule(Parser *p) if ( (or_pattern_var = or_pattern_rule(p)) // or_pattern && - (_keyword = _PyPegen_expect_token(p, 656)) // token='as' + (_keyword = _PyPegen_expect_token(p, 659)) // token='as' && (a = _PyPegen_expect_soft_keyword(p, "_")) // soft_keyword='"_"' ) @@ -23631,7 +23671,7 @@ invalid_as_pattern_rule(Parser *p) if ( (or_pattern_var = or_pattern_rule(p)) // or_pattern && - (_keyword = _PyPegen_expect_token(p, 656)) // token='as' + (_keyword = _PyPegen_expect_token(p, 659)) // token='as' && _PyPegen_lookahead_with_name(0, _PyPegen_name_token, p) && @@ -23732,7 +23772,7 @@ invalid_class_argument_pattern_rule(Parser *p) asdl_pattern_seq* a; asdl_seq* keyword_patterns_var; if ( - (_opt_var = _tmp_225_rule(p), !p->error_indicator) // [positional_patterns ','] + (_opt_var = _tmp_226_rule(p), !p->error_indicator) // [positional_patterns ','] && (keyword_patterns_var = keyword_patterns_rule(p)) // keyword_patterns && @@ -23785,7 +23825,7 @@ invalid_if_stmt_rule(Parser *p) expr_ty named_expression_var; Token * newline_var; if ( - (_keyword = _PyPegen_expect_token(p, 658)) // token='if' + (_keyword = _PyPegen_expect_token(p, 661)) // token='if' && (named_expression_var = named_expression_rule(p)) // named_expression && @@ -23816,7 +23856,7 @@ invalid_if_stmt_rule(Parser *p) expr_ty a_1; Token * newline_var; if ( - (a = _PyPegen_expect_token(p, 658)) // token='if' + (a = _PyPegen_expect_token(p, 661)) // token='if' && (a_1 = named_expression_rule(p)) // named_expression && @@ -23871,7 +23911,7 @@ invalid_elif_stmt_rule(Parser *p) expr_ty named_expression_var; Token * newline_var; if ( - (_keyword = _PyPegen_expect_token(p, 660)) // token='elif' + (_keyword = _PyPegen_expect_token(p, 663)) // token='elif' && (named_expression_var = named_expression_rule(p)) // named_expression && @@ -23902,7 +23942,7 @@ invalid_elif_stmt_rule(Parser *p) expr_ty named_expression_var; Token * newline_var; if ( - (a = _PyPegen_expect_token(p, 660)) // token='elif' + (a = _PyPegen_expect_token(p, 663)) // token='elif' && (named_expression_var = named_expression_rule(p)) // named_expression && @@ -23955,7 +23995,7 @@ invalid_else_stmt_rule(Parser *p) Token * a; Token * newline_var; if ( - (a = _PyPegen_expect_token(p, 661)) // token='else' + (a = _PyPegen_expect_token(p, 664)) // token='else' && (_literal = _PyPegen_expect_token(p, 11)) // token=':' && @@ -24008,7 +24048,7 @@ invalid_while_stmt_rule(Parser *p) expr_ty named_expression_var; Token * newline_var; if ( - (_keyword = _PyPegen_expect_token(p, 663)) // token='while' + (_keyword = _PyPegen_expect_token(p, 666)) // token='while' && (named_expression_var = named_expression_rule(p)) // named_expression && @@ -24039,7 +24079,7 @@ invalid_while_stmt_rule(Parser *p) expr_ty named_expression_var; Token * newline_var; if ( - (a = _PyPegen_expect_token(p, 663)) // token='while' + (a = _PyPegen_expect_token(p, 666)) // token='while' && (named_expression_var = named_expression_rule(p)) // named_expression && @@ -24098,13 +24138,13 @@ invalid_for_stmt_rule(Parser *p) expr_ty star_expressions_var; expr_ty star_targets_var; if ( - (_opt_var = _PyPegen_expect_token(p, 670), !p->error_indicator) // 'async'? + (_opt_var = _PyPegen_expect_token(p, 673), !p->error_indicator) // 'async'? && - (_keyword = _PyPegen_expect_token(p, 668)) // token='for' + (_keyword = _PyPegen_expect_token(p, 671)) // token='for' && (star_targets_var = star_targets_rule(p)) // star_targets && - (_keyword_1 = _PyPegen_expect_token(p, 669)) // token='in' + (_keyword_1 = _PyPegen_expect_token(p, 672)) // token='in' && (star_expressions_var = star_expressions_rule(p)) // star_expressions && @@ -24139,13 +24179,13 @@ invalid_for_stmt_rule(Parser *p) expr_ty star_expressions_var; expr_ty star_targets_var; if ( - (_opt_var = _PyPegen_expect_token(p, 670), !p->error_indicator) // 'async'? + (_opt_var = _PyPegen_expect_token(p, 673), !p->error_indicator) // 'async'? && - (a = _PyPegen_expect_token(p, 668)) // token='for' + (a = _PyPegen_expect_token(p, 671)) // token='for' && (star_targets_var = star_targets_rule(p)) // star_targets && - (_keyword = _PyPegen_expect_token(p, 669)) // token='in' + (_keyword = _PyPegen_expect_token(p, 672)) // token='in' && (star_expressions_var = star_expressions_rule(p)) // star_expressions && @@ -24210,9 +24250,9 @@ invalid_def_raw_rule(Parser *p) expr_ty name_var; Token * newline_var; if ( - (_opt_var = _PyPegen_expect_token(p, 670), !p->error_indicator) // 'async'? + (_opt_var = _PyPegen_expect_token(p, 673), !p->error_indicator) // 'async'? && - (a = _PyPegen_expect_token(p, 671)) // token='def' + (a = _PyPegen_expect_token(p, 674)) // token='def' && (name_var = _PyPegen_name_token(p)) // NAME && @@ -24224,7 +24264,7 @@ invalid_def_raw_rule(Parser *p) && (_literal_1 = _PyPegen_expect_token(p, 8)) // token=')' && - (_opt_var_3 = _tmp_226_rule(p), !p->error_indicator) // ['->' expression] + (_opt_var_3 = _tmp_227_rule(p), !p->error_indicator) // ['->' expression] && (_literal_2 = _PyPegen_expect_token(p, 11)) // token=':' && @@ -24281,13 +24321,13 @@ invalid_class_def_raw_rule(Parser *p) expr_ty name_var; Token * newline_var; if ( - (_keyword = _PyPegen_expect_token(p, 673)) // token='class' + (_keyword = _PyPegen_expect_token(p, 676)) // token='class' && (name_var = _PyPegen_name_token(p)) // NAME && (_opt_var = type_params_rule(p), !p->error_indicator) // type_params? && - (_opt_var_1 = _tmp_227_rule(p), !p->error_indicator) // ['(' arguments? ')'] + (_opt_var_1 = _tmp_228_rule(p), !p->error_indicator) // ['(' arguments? ')'] && (newline_var = _PyPegen_expect_token(p, NEWLINE)) // token='NEWLINE' ) @@ -24320,13 +24360,13 @@ invalid_class_def_raw_rule(Parser *p) expr_ty name_var; Token * newline_var; if ( - (a = _PyPegen_expect_token(p, 673)) // token='class' + (a = _PyPegen_expect_token(p, 676)) // token='class' && (name_var = _PyPegen_name_token(p)) // NAME && (_opt_var = type_params_rule(p), !p->error_indicator) // type_params? && - (_opt_var_1 = _tmp_228_rule(p), !p->error_indicator) // ['(' arguments? ')'] + (_opt_var_1 = _tmp_229_rule(p), !p->error_indicator) // ['(' arguments? ')'] && (_literal = _PyPegen_expect_token(p, 11)) // token=':' && @@ -24376,11 +24416,11 @@ invalid_double_starred_kvpairs_rule(Parser *p) return NULL; } D(fprintf(stderr, "%*c> invalid_double_starred_kvpairs[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "','.double_starred_kvpair+ ',' invalid_kvpair")); - asdl_seq * _gather_229_var; + asdl_seq * _gather_230_var; Token * _literal; void *invalid_kvpair_var; if ( - (_gather_229_var = _gather_229_rule(p)) // ','.double_starred_kvpair+ + (_gather_230_var = _gather_230_rule(p)) // ','.double_starred_kvpair+ && (_literal = _PyPegen_expect_token(p, 12)) // token=',' && @@ -24388,7 +24428,7 @@ invalid_double_starred_kvpairs_rule(Parser *p) ) { D(fprintf(stderr, "%*c+ invalid_double_starred_kvpairs[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "','.double_starred_kvpair+ ',' invalid_kvpair")); - _res = _PyPegen_dummy_name(p, _gather_229_var, _literal, invalid_kvpair_var); + _res = _PyPegen_dummy_name(p, _gather_230_var, _literal, invalid_kvpair_var); goto done; } p->mark = _mark; @@ -24441,7 +24481,7 @@ invalid_double_starred_kvpairs_rule(Parser *p) && (a = _PyPegen_expect_token(p, 11)) // token=':' && - _PyPegen_lookahead(1, _tmp_231_rule, p) + _PyPegen_lookahead(1, _tmp_232_rule, p) ) { D(fprintf(stderr, "%*c+ invalid_double_starred_kvpairs[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "expression ':' &('}' | ',')")); @@ -24551,7 +24591,7 @@ invalid_kvpair_rule(Parser *p) && (a = _PyPegen_expect_token(p, 11)) // token=':' && - _PyPegen_lookahead(1, _tmp_232_rule, p) + _PyPegen_lookahead(1, _tmp_233_rule, p) ) { D(fprintf(stderr, "%*c+ invalid_kvpair[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "expression ':' &('}' | ',')")); @@ -24767,7 +24807,7 @@ invalid_replacement_field_rule(Parser *p) if ( (_literal = _PyPegen_expect_token(p, 25)) // token='{' && - _PyPegen_lookahead(0, _tmp_233_rule, p) + _PyPegen_lookahead(0, _tmp_234_rule, p) ) { D(fprintf(stderr, "%*c+ invalid_replacement_field[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "'{' !(yield_expr | star_expressions)")); @@ -24790,13 +24830,13 @@ invalid_replacement_field_rule(Parser *p) } D(fprintf(stderr, "%*c> invalid_replacement_field[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "'{' (yield_expr | star_expressions) !('=' | '!' | ':' | '}')")); Token * _literal; - void *_tmp_234_var; + void *_tmp_235_var; if ( (_literal = _PyPegen_expect_token(p, 25)) // token='{' && - (_tmp_234_var = _tmp_234_rule(p)) // yield_expr | star_expressions + (_tmp_235_var = _tmp_235_rule(p)) // yield_expr | star_expressions && - _PyPegen_lookahead(0, _tmp_235_rule, p) + _PyPegen_lookahead(0, _tmp_236_rule, p) ) { D(fprintf(stderr, "%*c+ invalid_replacement_field[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "'{' (yield_expr | star_expressions) !('=' | '!' | ':' | '}')")); @@ -24820,15 +24860,15 @@ invalid_replacement_field_rule(Parser *p) D(fprintf(stderr, "%*c> invalid_replacement_field[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "'{' (yield_expr | star_expressions) '=' !('!' | ':' | '}')")); Token * _literal; Token * _literal_1; - void *_tmp_236_var; + void *_tmp_237_var; if ( (_literal = _PyPegen_expect_token(p, 25)) // token='{' && - (_tmp_236_var = _tmp_236_rule(p)) // yield_expr | star_expressions + (_tmp_237_var = _tmp_237_rule(p)) // yield_expr | star_expressions && (_literal_1 = _PyPegen_expect_token(p, 22)) // token='=' && - _PyPegen_lookahead(0, _tmp_237_rule, p) + _PyPegen_lookahead(0, _tmp_238_rule, p) ) { D(fprintf(stderr, "%*c+ invalid_replacement_field[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "'{' (yield_expr | star_expressions) '=' !('!' | ':' | '}')")); @@ -24853,12 +24893,12 @@ invalid_replacement_field_rule(Parser *p) Token * _literal; void *_opt_var; UNUSED(_opt_var); // Silence compiler warnings - void *_tmp_238_var; + void *_tmp_239_var; void *invalid_conversion_character_var; if ( (_literal = _PyPegen_expect_token(p, 25)) // token='{' && - (_tmp_238_var = _tmp_238_rule(p)) // yield_expr | star_expressions + (_tmp_239_var = _tmp_239_rule(p)) // yield_expr | star_expressions && (_opt_var = _PyPegen_expect_token(p, 22), !p->error_indicator) // '='? && @@ -24866,7 +24906,7 @@ invalid_replacement_field_rule(Parser *p) ) { D(fprintf(stderr, "%*c+ invalid_replacement_field[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "'{' (yield_expr | star_expressions) '='? invalid_conversion_character")); - _res = _PyPegen_dummy_name(p, _literal, _tmp_238_var, _opt_var, invalid_conversion_character_var); + _res = _PyPegen_dummy_name(p, _literal, _tmp_239_var, _opt_var, invalid_conversion_character_var); goto done; } p->mark = _mark; @@ -24884,17 +24924,17 @@ invalid_replacement_field_rule(Parser *p) UNUSED(_opt_var); // Silence compiler warnings void *_opt_var_1; UNUSED(_opt_var_1); // Silence compiler warnings - void *_tmp_239_var; + void *_tmp_240_var; if ( (_literal = _PyPegen_expect_token(p, 25)) // token='{' && - (_tmp_239_var = _tmp_239_rule(p)) // yield_expr | star_expressions + (_tmp_240_var = _tmp_240_rule(p)) // yield_expr | star_expressions && (_opt_var = _PyPegen_expect_token(p, 22), !p->error_indicator) // '='? && - (_opt_var_1 = _tmp_240_rule(p), !p->error_indicator) // ['!' NAME] + (_opt_var_1 = _tmp_241_rule(p), !p->error_indicator) // ['!' NAME] && - _PyPegen_lookahead(0, _tmp_241_rule, p) + _PyPegen_lookahead(0, _tmp_242_rule, p) ) { D(fprintf(stderr, "%*c+ invalid_replacement_field[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "'{' (yield_expr | star_expressions) '='? ['!' NAME] !(':' | '}')")); @@ -24918,24 +24958,24 @@ invalid_replacement_field_rule(Parser *p) D(fprintf(stderr, "%*c> invalid_replacement_field[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "'{' (yield_expr | star_expressions) '='? ['!' NAME] ':' fstring_format_spec* !'}'")); Token * _literal; Token * _literal_1; - asdl_seq * _loop0_244_var; + asdl_seq * _loop0_245_var; void *_opt_var; UNUSED(_opt_var); // Silence compiler warnings void *_opt_var_1; UNUSED(_opt_var_1); // Silence compiler warnings - void *_tmp_242_var; + void *_tmp_243_var; if ( (_literal = _PyPegen_expect_token(p, 25)) // token='{' && - (_tmp_242_var = _tmp_242_rule(p)) // yield_expr | star_expressions + (_tmp_243_var = _tmp_243_rule(p)) // yield_expr | star_expressions && (_opt_var = _PyPegen_expect_token(p, 22), !p->error_indicator) // '='? && - (_opt_var_1 = _tmp_243_rule(p), !p->error_indicator) // ['!' NAME] + (_opt_var_1 = _tmp_244_rule(p), !p->error_indicator) // ['!' NAME] && (_literal_1 = _PyPegen_expect_token(p, 11)) // token=':' && - (_loop0_244_var = _loop0_244_rule(p)) // fstring_format_spec* + (_loop0_245_var = _loop0_245_rule(p)) // fstring_format_spec* && _PyPegen_lookahead_with_int(0, _PyPegen_expect_token, p, 26) // token='}' ) @@ -24964,15 +25004,15 @@ invalid_replacement_field_rule(Parser *p) UNUSED(_opt_var); // Silence compiler warnings void *_opt_var_1; UNUSED(_opt_var_1); // Silence compiler warnings - void *_tmp_245_var; + void *_tmp_246_var; if ( (_literal = _PyPegen_expect_token(p, 25)) // token='{' && - (_tmp_245_var = _tmp_245_rule(p)) // yield_expr | star_expressions + (_tmp_246_var = _tmp_246_rule(p)) // yield_expr | star_expressions && (_opt_var = _PyPegen_expect_token(p, 22), !p->error_indicator) // '='? && - (_opt_var_1 = _tmp_246_rule(p), !p->error_indicator) // ['!' NAME] + (_opt_var_1 = _tmp_247_rule(p), !p->error_indicator) // ['!' NAME] && _PyPegen_lookahead_with_int(0, _PyPegen_expect_token, p, 26) // token='}' ) @@ -25019,7 +25059,7 @@ invalid_conversion_character_rule(Parser *p) if ( (_literal = _PyPegen_expect_token(p, 54)) // token='!' && - _PyPegen_lookahead(1, _tmp_247_rule, p) + _PyPegen_lookahead(1, _tmp_248_rule, p) ) { D(fprintf(stderr, "%*c+ invalid_conversion_character[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "'!' &(':' | '}')")); @@ -25411,7 +25451,7 @@ _tmp_6_rule(Parser *p) D(fprintf(stderr, "%*c> _tmp_6[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "'import'")); Token * _keyword; if ( - (_keyword = _PyPegen_expect_token(p, 617)) // token='import' + (_keyword = _PyPegen_expect_token(p, 620)) // token='import' ) { D(fprintf(stderr, "%*c+ _tmp_6[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "'import'")); @@ -25430,7 +25470,7 @@ _tmp_6_rule(Parser *p) D(fprintf(stderr, "%*c> _tmp_6[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "'from'")); Token * _keyword; if ( - (_keyword = _PyPegen_expect_token(p, 618)) // token='from' + (_keyword = _PyPegen_expect_token(p, 621)) // token='from' ) { D(fprintf(stderr, "%*c+ _tmp_6[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "'from'")); @@ -25468,7 +25508,7 @@ _tmp_7_rule(Parser *p) D(fprintf(stderr, "%*c> _tmp_7[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "'def'")); Token * _keyword; if ( - (_keyword = _PyPegen_expect_token(p, 671)) // token='def' + (_keyword = _PyPegen_expect_token(p, 674)) // token='def' ) { D(fprintf(stderr, "%*c+ _tmp_7[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "'def'")); @@ -25506,7 +25546,7 @@ _tmp_7_rule(Parser *p) D(fprintf(stderr, "%*c> _tmp_7[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "'async'")); Token * _keyword; if ( - (_keyword = _PyPegen_expect_token(p, 670)) // token='async' + (_keyword = _PyPegen_expect_token(p, 673)) // token='async' ) { D(fprintf(stderr, "%*c+ _tmp_7[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "'async'")); @@ -25544,7 +25584,7 @@ _tmp_8_rule(Parser *p) D(fprintf(stderr, "%*c> _tmp_8[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "'class'")); Token * _keyword; if ( - (_keyword = _PyPegen_expect_token(p, 673)) // token='class' + (_keyword = _PyPegen_expect_token(p, 676)) // token='class' ) { D(fprintf(stderr, "%*c+ _tmp_8[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "'class'")); @@ -25601,7 +25641,7 @@ _tmp_9_rule(Parser *p) D(fprintf(stderr, "%*c> _tmp_9[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "'with'")); Token * _keyword; if ( - (_keyword = _PyPegen_expect_token(p, 631)) // token='with' + (_keyword = _PyPegen_expect_token(p, 634)) // token='with' ) { D(fprintf(stderr, "%*c+ _tmp_9[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "'with'")); @@ -25620,7 +25660,7 @@ _tmp_9_rule(Parser *p) D(fprintf(stderr, "%*c> _tmp_9[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "'async'")); Token * _keyword; if ( - (_keyword = _PyPegen_expect_token(p, 670)) // token='async' + (_keyword = _PyPegen_expect_token(p, 673)) // token='async' ) { D(fprintf(stderr, "%*c+ _tmp_9[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "'async'")); @@ -25658,7 +25698,7 @@ _tmp_10_rule(Parser *p) D(fprintf(stderr, "%*c> _tmp_10[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "'for'")); Token * _keyword; if ( - (_keyword = _PyPegen_expect_token(p, 668)) // token='for' + (_keyword = _PyPegen_expect_token(p, 671)) // token='for' ) { D(fprintf(stderr, "%*c+ _tmp_10[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "'for'")); @@ -25677,7 +25717,7 @@ _tmp_10_rule(Parser *p) D(fprintf(stderr, "%*c> _tmp_10[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "'async'")); Token * _keyword; if ( - (_keyword = _PyPegen_expect_token(p, 670)) // token='async' + (_keyword = _PyPegen_expect_token(p, 673)) // token='async' ) { D(fprintf(stderr, "%*c+ _tmp_10[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "'async'")); @@ -25882,12 +25922,12 @@ _loop1_14_rule(Parser *p) return NULL; } D(fprintf(stderr, "%*c> _loop1_14[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "(star_targets '=')")); - void *_tmp_248_var; + void *_tmp_249_var; while ( - (_tmp_248_var = _tmp_248_rule(p)) // star_targets '=' + (_tmp_249_var = _tmp_249_rule(p)) // star_targets '=' ) { - _res = _tmp_248_var; + _res = _tmp_249_var; if (_n == _children_capacity) { _children_capacity *= 2; void **_new_children = PyMem_Realloc(_children, _children_capacity*sizeof(void *)); @@ -26062,7 +26102,7 @@ _tmp_17_rule(Parser *p) Token * _keyword; expr_ty z; if ( - (_keyword = _PyPegen_expect_token(p, 618)) // token='from' + (_keyword = _PyPegen_expect_token(p, 621)) // token='from' && (z = expression_rule(p)) // expression ) @@ -26451,12 +26491,12 @@ _loop0_24_rule(Parser *p) return NULL; } D(fprintf(stderr, "%*c> _loop0_24[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "('.' | '...')")); - void *_tmp_249_var; + void *_tmp_250_var; while ( - (_tmp_249_var = _tmp_249_rule(p)) // '.' | '...' + (_tmp_250_var = _tmp_250_rule(p)) // '.' | '...' ) { - _res = _tmp_249_var; + _res = _tmp_250_var; if (_n == _children_capacity) { _children_capacity *= 2; void **_new_children = PyMem_Realloc(_children, _children_capacity*sizeof(void *)); @@ -26518,12 +26558,12 @@ _loop1_25_rule(Parser *p) return NULL; } D(fprintf(stderr, "%*c> _loop1_25[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "('.' | '...')")); - void *_tmp_250_var; + void *_tmp_251_var; while ( - (_tmp_250_var = _tmp_250_rule(p)) // '.' | '...' + (_tmp_251_var = _tmp_251_rule(p)) // '.' | '...' ) { - _res = _tmp_250_var; + _res = _tmp_251_var; if (_n == _children_capacity) { _children_capacity *= 2; void **_new_children = PyMem_Realloc(_children, _children_capacity*sizeof(void *)); @@ -26701,7 +26741,7 @@ _tmp_28_rule(Parser *p) Token * _keyword; expr_ty z; if ( - (_keyword = _PyPegen_expect_token(p, 656)) // token='as' + (_keyword = _PyPegen_expect_token(p, 659)) // token='as' && (z = _PyPegen_name_token(p)) // NAME ) @@ -26864,7 +26904,7 @@ _tmp_31_rule(Parser *p) Token * _keyword; expr_ty z; if ( - (_keyword = _PyPegen_expect_token(p, 656)) // token='as' + (_keyword = _PyPegen_expect_token(p, 659)) // token='as' && (z = _PyPegen_name_token(p)) // NAME ) @@ -26916,12 +26956,12 @@ _loop1_32_rule(Parser *p) return NULL; } D(fprintf(stderr, "%*c> _loop1_32[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "('@' named_expression NEWLINE)")); - void *_tmp_251_var; + void *_tmp_252_var; while ( - (_tmp_251_var = _tmp_251_rule(p)) // '@' named_expression NEWLINE + (_tmp_252_var = _tmp_252_rule(p)) // '@' named_expression NEWLINE ) { - _res = _tmp_251_var; + _res = _tmp_252_var; if (_n == _children_capacity) { _children_capacity *= 2; void **_new_children = PyMem_Realloc(_children, _children_capacity*sizeof(void *)); @@ -28851,7 +28891,7 @@ _tmp_62_rule(Parser *p) Token * _keyword; expr_ty z; if ( - (_keyword = _PyPegen_expect_token(p, 656)) // token='as' + (_keyword = _PyPegen_expect_token(p, 659)) // token='as' && (z = _PyPegen_name_token(p)) // NAME ) @@ -28897,7 +28937,7 @@ _tmp_63_rule(Parser *p) Token * _keyword; expr_ty z; if ( - (_keyword = _PyPegen_expect_token(p, 656)) // token='as' + (_keyword = _PyPegen_expect_token(p, 659)) // token='as' && (z = _PyPegen_name_token(p)) // NAME ) @@ -30046,12 +30086,12 @@ _loop1_82_rule(Parser *p) return NULL; } D(fprintf(stderr, "%*c> _loop1_82[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "(',' expression)")); - void *_tmp_252_var; + void *_tmp_253_var; while ( - (_tmp_252_var = _tmp_252_rule(p)) // ',' expression + (_tmp_253_var = _tmp_253_rule(p)) // ',' expression ) { - _res = _tmp_252_var; + _res = _tmp_253_var; if (_n == _children_capacity) { _children_capacity *= 2; void **_new_children = PyMem_Realloc(_children, _children_capacity*sizeof(void *)); @@ -30118,12 +30158,12 @@ _loop1_83_rule(Parser *p) return NULL; } D(fprintf(stderr, "%*c> _loop1_83[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "(',' star_expression)")); - void *_tmp_253_var; + void *_tmp_254_var; while ( - (_tmp_253_var = _tmp_253_rule(p)) // ',' star_expression + (_tmp_254_var = _tmp_254_rule(p)) // ',' star_expression ) { - _res = _tmp_253_var; + _res = _tmp_254_var; if (_n == _children_capacity) { _children_capacity *= 2; void **_new_children = PyMem_Realloc(_children, _children_capacity*sizeof(void *)); @@ -30307,12 +30347,12 @@ _loop1_86_rule(Parser *p) return NULL; } D(fprintf(stderr, "%*c> _loop1_86[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "('or' conjunction)")); - void *_tmp_254_var; + void *_tmp_255_var; while ( - (_tmp_254_var = _tmp_254_rule(p)) // 'or' conjunction + (_tmp_255_var = _tmp_255_rule(p)) // 'or' conjunction ) { - _res = _tmp_254_var; + _res = _tmp_255_var; if (_n == _children_capacity) { _children_capacity *= 2; void **_new_children = PyMem_Realloc(_children, _children_capacity*sizeof(void *)); @@ -30379,12 +30419,12 @@ _loop1_87_rule(Parser *p) return NULL; } D(fprintf(stderr, "%*c> _loop1_87[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "('and' inversion)")); - void *_tmp_255_var; + void *_tmp_256_var; while ( - (_tmp_255_var = _tmp_255_rule(p)) // 'and' inversion + (_tmp_256_var = _tmp_256_rule(p)) // 'and' inversion ) { - _res = _tmp_255_var; + _res = _tmp_256_var; if (_n == _children_capacity) { _children_capacity *= 2; void **_new_children = PyMem_Realloc(_children, _children_capacity*sizeof(void *)); @@ -30571,7 +30611,7 @@ _loop0_91_rule(Parser *p) while ( (_literal = _PyPegen_expect_token(p, 12)) // token=',' && - (elem = _tmp_256_rule(p)) // slice | starred_expression + (elem = _tmp_257_rule(p)) // slice | starred_expression ) { _res = elem; @@ -30636,7 +30676,7 @@ _gather_90_rule(Parser *p) void *elem; asdl_seq * seq; if ( - (elem = _tmp_256_rule(p)) // slice | starred_expression + (elem = _tmp_257_rule(p)) // slice | starred_expression && (seq = _loop0_91_rule(p)) // _loop0_91 ) @@ -32235,12 +32275,12 @@ _loop1_115_rule(Parser *p) return NULL; } D(fprintf(stderr, "%*c> _loop1_115[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "(fstring | string)")); - void *_tmp_257_var; + void *_tmp_258_var; while ( - (_tmp_257_var = _tmp_257_rule(p)) // fstring | string + (_tmp_258_var = _tmp_258_rule(p)) // fstring | string ) { - _res = _tmp_257_var; + _res = _tmp_258_var; if (_n == _children_capacity) { _children_capacity *= 2; void **_new_children = PyMem_Realloc(_children, _children_capacity*sizeof(void *)); @@ -32545,12 +32585,12 @@ _loop0_120_rule(Parser *p) return NULL; } D(fprintf(stderr, "%*c> _loop0_120[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "('if' disjunction)")); - void *_tmp_258_var; + void *_tmp_259_var; while ( - (_tmp_258_var = _tmp_258_rule(p)) // 'if' disjunction + (_tmp_259_var = _tmp_259_rule(p)) // 'if' disjunction ) { - _res = _tmp_258_var; + _res = _tmp_259_var; if (_n == _children_capacity) { _children_capacity *= 2; void **_new_children = PyMem_Realloc(_children, _children_capacity*sizeof(void *)); @@ -32612,12 +32652,12 @@ _loop0_121_rule(Parser *p) return NULL; } D(fprintf(stderr, "%*c> _loop0_121[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "('if' disjunction)")); - void *_tmp_259_var; + void *_tmp_260_var; while ( - (_tmp_259_var = _tmp_259_rule(p)) // 'if' disjunction + (_tmp_260_var = _tmp_260_rule(p)) // 'if' disjunction ) { - _res = _tmp_259_var; + _res = _tmp_260_var; if (_n == _children_capacity) { _children_capacity *= 2; void **_new_children = PyMem_Realloc(_children, _children_capacity*sizeof(void *)); @@ -32651,9 +32691,54 @@ _loop0_121_rule(Parser *p) return _seq; } -// _tmp_122: assignment_expression | expression !':=' +// _tmp_122: bitwise_or ((',' bitwise_or))* ','? static void * _tmp_122_rule(Parser *p) +{ + if (p->level++ == MAXSTACK) { + _Pypegen_stack_overflow(p); + } + if (p->error_indicator) { + p->level--; + return NULL; + } + void * _res = NULL; + int _mark = p->mark; + { // bitwise_or ((',' bitwise_or))* ','? + if (p->error_indicator) { + p->level--; + return NULL; + } + D(fprintf(stderr, "%*c> _tmp_122[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "bitwise_or ((',' bitwise_or))* ','?")); + asdl_seq * _loop0_261_var; + void *_opt_var; + UNUSED(_opt_var); // Silence compiler warnings + expr_ty bitwise_or_var; + if ( + (bitwise_or_var = bitwise_or_rule(p)) // bitwise_or + && + (_loop0_261_var = _loop0_261_rule(p)) // ((',' bitwise_or))* + && + (_opt_var = _PyPegen_expect_token(p, 12), !p->error_indicator) // ','? + ) + { + D(fprintf(stderr, "%*c+ _tmp_122[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "bitwise_or ((',' bitwise_or))* ','?")); + _res = _PyPegen_dummy_name(p, bitwise_or_var, _loop0_261_var, _opt_var); + goto done; + } + p->mark = _mark; + D(fprintf(stderr, "%*c%s _tmp_122[%d-%d]: %s failed!\n", p->level, ' ', + p->error_indicator ? "ERROR!" : "-", _mark, p->mark, "bitwise_or ((',' bitwise_or))* ','?")); + } + _res = NULL; + done: + p->level--; + return _res; +} + +// _tmp_123: assignment_expression | expression !':=' +static void * +_tmp_123_rule(Parser *p) { if (p->level++ == MAXSTACK) { _Pypegen_stack_overflow(p); @@ -32669,18 +32754,18 @@ _tmp_122_rule(Parser *p) p->level--; return NULL; } - D(fprintf(stderr, "%*c> _tmp_122[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "assignment_expression")); + D(fprintf(stderr, "%*c> _tmp_123[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "assignment_expression")); expr_ty assignment_expression_var; if ( (assignment_expression_var = assignment_expression_rule(p)) // assignment_expression ) { - D(fprintf(stderr, "%*c+ _tmp_122[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "assignment_expression")); + D(fprintf(stderr, "%*c+ _tmp_123[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "assignment_expression")); _res = assignment_expression_var; goto done; } p->mark = _mark; - D(fprintf(stderr, "%*c%s _tmp_122[%d-%d]: %s failed!\n", p->level, ' ', + D(fprintf(stderr, "%*c%s _tmp_123[%d-%d]: %s failed!\n", p->level, ' ', p->error_indicator ? "ERROR!" : "-", _mark, p->mark, "assignment_expression")); } { // expression !':=' @@ -32688,7 +32773,7 @@ _tmp_122_rule(Parser *p) p->level--; return NULL; } - D(fprintf(stderr, "%*c> _tmp_122[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "expression !':='")); + D(fprintf(stderr, "%*c> _tmp_123[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "expression !':='")); expr_ty expression_var; if ( (expression_var = expression_rule(p)) // expression @@ -32696,12 +32781,12 @@ _tmp_122_rule(Parser *p) _PyPegen_lookahead_with_int(0, _PyPegen_expect_token, p, 53) // token=':=' ) { - D(fprintf(stderr, "%*c+ _tmp_122[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "expression !':='")); + D(fprintf(stderr, "%*c+ _tmp_123[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "expression !':='")); _res = expression_var; goto done; } p->mark = _mark; - D(fprintf(stderr, "%*c%s _tmp_122[%d-%d]: %s failed!\n", p->level, ' ', + D(fprintf(stderr, "%*c%s _tmp_123[%d-%d]: %s failed!\n", p->level, ' ', p->error_indicator ? "ERROR!" : "-", _mark, p->mark, "expression !':='")); } _res = NULL; @@ -32710,9 +32795,9 @@ _tmp_122_rule(Parser *p) return _res; } -// _loop0_124: ',' (starred_expression | (assignment_expression | expression !':=') !'=') +// _loop0_125: ',' (starred_expression | (assignment_expression | expression !':=') !'=') static asdl_seq * -_loop0_124_rule(Parser *p) +_loop0_125_rule(Parser *p) { if (p->level++ == MAXSTACK) { _Pypegen_stack_overflow(p); @@ -32737,13 +32822,13 @@ _loop0_124_rule(Parser *p) p->level--; return NULL; } - D(fprintf(stderr, "%*c> _loop0_124[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "',' (starred_expression | (assignment_expression | expression !':=') !'=')")); + D(fprintf(stderr, "%*c> _loop0_125[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "',' (starred_expression | (assignment_expression | expression !':=') !'=')")); Token * _literal; void *elem; while ( (_literal = _PyPegen_expect_token(p, 12)) // token=',' && - (elem = _tmp_260_rule(p)) // starred_expression | (assignment_expression | expression !':=') !'=' + (elem = _tmp_262_rule(p)) // starred_expression | (assignment_expression | expression !':=') !'=' ) { _res = elem; @@ -32769,7 +32854,7 @@ _loop0_124_rule(Parser *p) _mark = p->mark; } p->mark = _mark; - D(fprintf(stderr, "%*c%s _loop0_124[%d-%d]: %s failed!\n", p->level, ' ', + D(fprintf(stderr, "%*c%s _loop0_125[%d-%d]: %s failed!\n", p->level, ' ', p->error_indicator ? "ERROR!" : "-", _mark, p->mark, "',' (starred_expression | (assignment_expression | expression !':=') !'=')")); } asdl_seq *_seq = (asdl_seq*)_Py_asdl_generic_seq_new(_n, p->arena); @@ -32786,10 +32871,10 @@ _loop0_124_rule(Parser *p) return _seq; } -// _gather_123: -// | (starred_expression | (assignment_expression | expression !':=') !'=') _loop0_124 +// _gather_124: +// | (starred_expression | (assignment_expression | expression !':=') !'=') _loop0_125 static asdl_seq * -_gather_123_rule(Parser *p) +_gather_124_rule(Parser *p) { if (p->level++ == MAXSTACK) { _Pypegen_stack_overflow(p); @@ -32800,27 +32885,27 @@ _gather_123_rule(Parser *p) } asdl_seq * _res = NULL; int _mark = p->mark; - { // (starred_expression | (assignment_expression | expression !':=') !'=') _loop0_124 + { // (starred_expression | (assignment_expression | expression !':=') !'=') _loop0_125 if (p->error_indicator) { p->level--; return NULL; } - D(fprintf(stderr, "%*c> _gather_123[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "(starred_expression | (assignment_expression | expression !':=') !'=') _loop0_124")); + D(fprintf(stderr, "%*c> _gather_124[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "(starred_expression | (assignment_expression | expression !':=') !'=') _loop0_125")); void *elem; asdl_seq * seq; if ( - (elem = _tmp_260_rule(p)) // starred_expression | (assignment_expression | expression !':=') !'=' + (elem = _tmp_262_rule(p)) // starred_expression | (assignment_expression | expression !':=') !'=' && - (seq = _loop0_124_rule(p)) // _loop0_124 + (seq = _loop0_125_rule(p)) // _loop0_125 ) { - D(fprintf(stderr, "%*c+ _gather_123[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "(starred_expression | (assignment_expression | expression !':=') !'=') _loop0_124")); + D(fprintf(stderr, "%*c+ _gather_124[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "(starred_expression | (assignment_expression | expression !':=') !'=') _loop0_125")); _res = _PyPegen_seq_insert_in_front(p, elem, seq); goto done; } p->mark = _mark; - D(fprintf(stderr, "%*c%s _gather_123[%d-%d]: %s failed!\n", p->level, ' ', - p->error_indicator ? "ERROR!" : "-", _mark, p->mark, "(starred_expression | (assignment_expression | expression !':=') !'=') _loop0_124")); + D(fprintf(stderr, "%*c%s _gather_124[%d-%d]: %s failed!\n", p->level, ' ', + p->error_indicator ? "ERROR!" : "-", _mark, p->mark, "(starred_expression | (assignment_expression | expression !':=') !'=') _loop0_125")); } _res = NULL; done: @@ -32828,9 +32913,9 @@ _gather_123_rule(Parser *p) return _res; } -// _tmp_125: ',' kwargs +// _tmp_126: ',' kwargs static void * -_tmp_125_rule(Parser *p) +_tmp_126_rule(Parser *p) { if (p->level++ == MAXSTACK) { _Pypegen_stack_overflow(p); @@ -32846,7 +32931,7 @@ _tmp_125_rule(Parser *p) p->level--; return NULL; } - D(fprintf(stderr, "%*c> _tmp_125[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "',' kwargs")); + D(fprintf(stderr, "%*c> _tmp_126[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "',' kwargs")); Token * _literal; asdl_seq* k; if ( @@ -32855,7 +32940,7 @@ _tmp_125_rule(Parser *p) (k = kwargs_rule(p)) // kwargs ) { - D(fprintf(stderr, "%*c+ _tmp_125[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "',' kwargs")); + D(fprintf(stderr, "%*c+ _tmp_126[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "',' kwargs")); _res = k; if (_res == NULL && PyErr_Occurred()) { p->error_indicator = 1; @@ -32865,7 +32950,7 @@ _tmp_125_rule(Parser *p) goto done; } p->mark = _mark; - D(fprintf(stderr, "%*c%s _tmp_125[%d-%d]: %s failed!\n", p->level, ' ', + D(fprintf(stderr, "%*c%s _tmp_126[%d-%d]: %s failed!\n", p->level, ' ', p->error_indicator ? "ERROR!" : "-", _mark, p->mark, "',' kwargs")); } _res = NULL; @@ -32874,9 +32959,9 @@ _tmp_125_rule(Parser *p) return _res; } -// _loop0_127: ',' kwarg_or_starred +// _loop0_128: ',' kwarg_or_starred static asdl_seq * -_loop0_127_rule(Parser *p) +_loop0_128_rule(Parser *p) { if (p->level++ == MAXSTACK) { _Pypegen_stack_overflow(p); @@ -32901,7 +32986,7 @@ _loop0_127_rule(Parser *p) p->level--; return NULL; } - D(fprintf(stderr, "%*c> _loop0_127[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "',' kwarg_or_starred")); + D(fprintf(stderr, "%*c> _loop0_128[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "',' kwarg_or_starred")); Token * _literal; KeywordOrStarred* elem; while ( @@ -32933,7 +33018,7 @@ _loop0_127_rule(Parser *p) _mark = p->mark; } p->mark = _mark; - D(fprintf(stderr, "%*c%s _loop0_127[%d-%d]: %s failed!\n", p->level, ' ', + D(fprintf(stderr, "%*c%s _loop0_128[%d-%d]: %s failed!\n", p->level, ' ', p->error_indicator ? "ERROR!" : "-", _mark, p->mark, "',' kwarg_or_starred")); } asdl_seq *_seq = (asdl_seq*)_Py_asdl_generic_seq_new(_n, p->arena); @@ -32950,9 +33035,9 @@ _loop0_127_rule(Parser *p) return _seq; } -// _gather_126: kwarg_or_starred _loop0_127 +// _gather_127: kwarg_or_starred _loop0_128 static asdl_seq * -_gather_126_rule(Parser *p) +_gather_127_rule(Parser *p) { if (p->level++ == MAXSTACK) { _Pypegen_stack_overflow(p); @@ -32963,27 +33048,27 @@ _gather_126_rule(Parser *p) } asdl_seq * _res = NULL; int _mark = p->mark; - { // kwarg_or_starred _loop0_127 + { // kwarg_or_starred _loop0_128 if (p->error_indicator) { p->level--; return NULL; } - D(fprintf(stderr, "%*c> _gather_126[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "kwarg_or_starred _loop0_127")); + D(fprintf(stderr, "%*c> _gather_127[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "kwarg_or_starred _loop0_128")); KeywordOrStarred* elem; asdl_seq * seq; if ( (elem = kwarg_or_starred_rule(p)) // kwarg_or_starred && - (seq = _loop0_127_rule(p)) // _loop0_127 + (seq = _loop0_128_rule(p)) // _loop0_128 ) { - D(fprintf(stderr, "%*c+ _gather_126[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "kwarg_or_starred _loop0_127")); + D(fprintf(stderr, "%*c+ _gather_127[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "kwarg_or_starred _loop0_128")); _res = _PyPegen_seq_insert_in_front(p, elem, seq); goto done; } p->mark = _mark; - D(fprintf(stderr, "%*c%s _gather_126[%d-%d]: %s failed!\n", p->level, ' ', - p->error_indicator ? "ERROR!" : "-", _mark, p->mark, "kwarg_or_starred _loop0_127")); + D(fprintf(stderr, "%*c%s _gather_127[%d-%d]: %s failed!\n", p->level, ' ', + p->error_indicator ? "ERROR!" : "-", _mark, p->mark, "kwarg_or_starred _loop0_128")); } _res = NULL; done: @@ -32991,9 +33076,9 @@ _gather_126_rule(Parser *p) return _res; } -// _loop0_129: ',' kwarg_or_double_starred +// _loop0_130: ',' kwarg_or_double_starred static asdl_seq * -_loop0_129_rule(Parser *p) +_loop0_130_rule(Parser *p) { if (p->level++ == MAXSTACK) { _Pypegen_stack_overflow(p); @@ -33018,7 +33103,7 @@ _loop0_129_rule(Parser *p) p->level--; return NULL; } - D(fprintf(stderr, "%*c> _loop0_129[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "',' kwarg_or_double_starred")); + D(fprintf(stderr, "%*c> _loop0_130[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "',' kwarg_or_double_starred")); Token * _literal; KeywordOrStarred* elem; while ( @@ -33050,7 +33135,7 @@ _loop0_129_rule(Parser *p) _mark = p->mark; } p->mark = _mark; - D(fprintf(stderr, "%*c%s _loop0_129[%d-%d]: %s failed!\n", p->level, ' ', + D(fprintf(stderr, "%*c%s _loop0_130[%d-%d]: %s failed!\n", p->level, ' ', p->error_indicator ? "ERROR!" : "-", _mark, p->mark, "',' kwarg_or_double_starred")); } asdl_seq *_seq = (asdl_seq*)_Py_asdl_generic_seq_new(_n, p->arena); @@ -33067,9 +33152,9 @@ _loop0_129_rule(Parser *p) return _seq; } -// _gather_128: kwarg_or_double_starred _loop0_129 +// _gather_129: kwarg_or_double_starred _loop0_130 static asdl_seq * -_gather_128_rule(Parser *p) +_gather_129_rule(Parser *p) { if (p->level++ == MAXSTACK) { _Pypegen_stack_overflow(p); @@ -33080,27 +33165,27 @@ _gather_128_rule(Parser *p) } asdl_seq * _res = NULL; int _mark = p->mark; - { // kwarg_or_double_starred _loop0_129 + { // kwarg_or_double_starred _loop0_130 if (p->error_indicator) { p->level--; return NULL; } - D(fprintf(stderr, "%*c> _gather_128[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "kwarg_or_double_starred _loop0_129")); + D(fprintf(stderr, "%*c> _gather_129[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "kwarg_or_double_starred _loop0_130")); KeywordOrStarred* elem; asdl_seq * seq; if ( (elem = kwarg_or_double_starred_rule(p)) // kwarg_or_double_starred && - (seq = _loop0_129_rule(p)) // _loop0_129 + (seq = _loop0_130_rule(p)) // _loop0_130 ) { - D(fprintf(stderr, "%*c+ _gather_128[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "kwarg_or_double_starred _loop0_129")); + D(fprintf(stderr, "%*c+ _gather_129[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "kwarg_or_double_starred _loop0_130")); _res = _PyPegen_seq_insert_in_front(p, elem, seq); goto done; } p->mark = _mark; - D(fprintf(stderr, "%*c%s _gather_128[%d-%d]: %s failed!\n", p->level, ' ', - p->error_indicator ? "ERROR!" : "-", _mark, p->mark, "kwarg_or_double_starred _loop0_129")); + D(fprintf(stderr, "%*c%s _gather_129[%d-%d]: %s failed!\n", p->level, ' ', + p->error_indicator ? "ERROR!" : "-", _mark, p->mark, "kwarg_or_double_starred _loop0_130")); } _res = NULL; done: @@ -33108,9 +33193,9 @@ _gather_128_rule(Parser *p) return _res; } -// _loop0_131: ',' kwarg_or_starred +// _loop0_132: ',' kwarg_or_starred static asdl_seq * -_loop0_131_rule(Parser *p) +_loop0_132_rule(Parser *p) { if (p->level++ == MAXSTACK) { _Pypegen_stack_overflow(p); @@ -33135,7 +33220,7 @@ _loop0_131_rule(Parser *p) p->level--; return NULL; } - D(fprintf(stderr, "%*c> _loop0_131[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "',' kwarg_or_starred")); + D(fprintf(stderr, "%*c> _loop0_132[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "',' kwarg_or_starred")); Token * _literal; KeywordOrStarred* elem; while ( @@ -33167,7 +33252,7 @@ _loop0_131_rule(Parser *p) _mark = p->mark; } p->mark = _mark; - D(fprintf(stderr, "%*c%s _loop0_131[%d-%d]: %s failed!\n", p->level, ' ', + D(fprintf(stderr, "%*c%s _loop0_132[%d-%d]: %s failed!\n", p->level, ' ', p->error_indicator ? "ERROR!" : "-", _mark, p->mark, "',' kwarg_or_starred")); } asdl_seq *_seq = (asdl_seq*)_Py_asdl_generic_seq_new(_n, p->arena); @@ -33184,9 +33269,9 @@ _loop0_131_rule(Parser *p) return _seq; } -// _gather_130: kwarg_or_starred _loop0_131 +// _gather_131: kwarg_or_starred _loop0_132 static asdl_seq * -_gather_130_rule(Parser *p) +_gather_131_rule(Parser *p) { if (p->level++ == MAXSTACK) { _Pypegen_stack_overflow(p); @@ -33197,27 +33282,27 @@ _gather_130_rule(Parser *p) } asdl_seq * _res = NULL; int _mark = p->mark; - { // kwarg_or_starred _loop0_131 + { // kwarg_or_starred _loop0_132 if (p->error_indicator) { p->level--; return NULL; } - D(fprintf(stderr, "%*c> _gather_130[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "kwarg_or_starred _loop0_131")); + D(fprintf(stderr, "%*c> _gather_131[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "kwarg_or_starred _loop0_132")); KeywordOrStarred* elem; asdl_seq * seq; if ( (elem = kwarg_or_starred_rule(p)) // kwarg_or_starred && - (seq = _loop0_131_rule(p)) // _loop0_131 + (seq = _loop0_132_rule(p)) // _loop0_132 ) { - D(fprintf(stderr, "%*c+ _gather_130[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "kwarg_or_starred _loop0_131")); + D(fprintf(stderr, "%*c+ _gather_131[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "kwarg_or_starred _loop0_132")); _res = _PyPegen_seq_insert_in_front(p, elem, seq); goto done; } p->mark = _mark; - D(fprintf(stderr, "%*c%s _gather_130[%d-%d]: %s failed!\n", p->level, ' ', - p->error_indicator ? "ERROR!" : "-", _mark, p->mark, "kwarg_or_starred _loop0_131")); + D(fprintf(stderr, "%*c%s _gather_131[%d-%d]: %s failed!\n", p->level, ' ', + p->error_indicator ? "ERROR!" : "-", _mark, p->mark, "kwarg_or_starred _loop0_132")); } _res = NULL; done: @@ -33225,9 +33310,9 @@ _gather_130_rule(Parser *p) return _res; } -// _loop0_133: ',' kwarg_or_double_starred +// _loop0_134: ',' kwarg_or_double_starred static asdl_seq * -_loop0_133_rule(Parser *p) +_loop0_134_rule(Parser *p) { if (p->level++ == MAXSTACK) { _Pypegen_stack_overflow(p); @@ -33252,7 +33337,7 @@ _loop0_133_rule(Parser *p) p->level--; return NULL; } - D(fprintf(stderr, "%*c> _loop0_133[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "',' kwarg_or_double_starred")); + D(fprintf(stderr, "%*c> _loop0_134[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "',' kwarg_or_double_starred")); Token * _literal; KeywordOrStarred* elem; while ( @@ -33284,7 +33369,7 @@ _loop0_133_rule(Parser *p) _mark = p->mark; } p->mark = _mark; - D(fprintf(stderr, "%*c%s _loop0_133[%d-%d]: %s failed!\n", p->level, ' ', + D(fprintf(stderr, "%*c%s _loop0_134[%d-%d]: %s failed!\n", p->level, ' ', p->error_indicator ? "ERROR!" : "-", _mark, p->mark, "',' kwarg_or_double_starred")); } asdl_seq *_seq = (asdl_seq*)_Py_asdl_generic_seq_new(_n, p->arena); @@ -33301,9 +33386,9 @@ _loop0_133_rule(Parser *p) return _seq; } -// _gather_132: kwarg_or_double_starred _loop0_133 +// _gather_133: kwarg_or_double_starred _loop0_134 static asdl_seq * -_gather_132_rule(Parser *p) +_gather_133_rule(Parser *p) { if (p->level++ == MAXSTACK) { _Pypegen_stack_overflow(p); @@ -33314,27 +33399,27 @@ _gather_132_rule(Parser *p) } asdl_seq * _res = NULL; int _mark = p->mark; - { // kwarg_or_double_starred _loop0_133 + { // kwarg_or_double_starred _loop0_134 if (p->error_indicator) { p->level--; return NULL; } - D(fprintf(stderr, "%*c> _gather_132[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "kwarg_or_double_starred _loop0_133")); + D(fprintf(stderr, "%*c> _gather_133[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "kwarg_or_double_starred _loop0_134")); KeywordOrStarred* elem; asdl_seq * seq; if ( (elem = kwarg_or_double_starred_rule(p)) // kwarg_or_double_starred && - (seq = _loop0_133_rule(p)) // _loop0_133 + (seq = _loop0_134_rule(p)) // _loop0_134 ) { - D(fprintf(stderr, "%*c+ _gather_132[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "kwarg_or_double_starred _loop0_133")); + D(fprintf(stderr, "%*c+ _gather_133[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "kwarg_or_double_starred _loop0_134")); _res = _PyPegen_seq_insert_in_front(p, elem, seq); goto done; } p->mark = _mark; - D(fprintf(stderr, "%*c%s _gather_132[%d-%d]: %s failed!\n", p->level, ' ', - p->error_indicator ? "ERROR!" : "-", _mark, p->mark, "kwarg_or_double_starred _loop0_133")); + D(fprintf(stderr, "%*c%s _gather_133[%d-%d]: %s failed!\n", p->level, ' ', + p->error_indicator ? "ERROR!" : "-", _mark, p->mark, "kwarg_or_double_starred _loop0_134")); } _res = NULL; done: @@ -33342,9 +33427,9 @@ _gather_132_rule(Parser *p) return _res; } -// _loop0_134: (',' star_target) +// _loop0_135: (',' star_target) static asdl_seq * -_loop0_134_rule(Parser *p) +_loop0_135_rule(Parser *p) { if (p->level++ == MAXSTACK) { _Pypegen_stack_overflow(p); @@ -33369,13 +33454,13 @@ _loop0_134_rule(Parser *p) p->level--; return NULL; } - D(fprintf(stderr, "%*c> _loop0_134[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "(',' star_target)")); - void *_tmp_261_var; + D(fprintf(stderr, "%*c> _loop0_135[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "(',' star_target)")); + void *_tmp_263_var; while ( - (_tmp_261_var = _tmp_261_rule(p)) // ',' star_target + (_tmp_263_var = _tmp_263_rule(p)) // ',' star_target ) { - _res = _tmp_261_var; + _res = _tmp_263_var; if (_n == _children_capacity) { _children_capacity *= 2; void **_new_children = PyMem_Realloc(_children, _children_capacity*sizeof(void *)); @@ -33392,7 +33477,7 @@ _loop0_134_rule(Parser *p) _mark = p->mark; } p->mark = _mark; - D(fprintf(stderr, "%*c%s _loop0_134[%d-%d]: %s failed!\n", p->level, ' ', + D(fprintf(stderr, "%*c%s _loop0_135[%d-%d]: %s failed!\n", p->level, ' ', p->error_indicator ? "ERROR!" : "-", _mark, p->mark, "(',' star_target)")); } asdl_seq *_seq = (asdl_seq*)_Py_asdl_generic_seq_new(_n, p->arena); @@ -33409,9 +33494,9 @@ _loop0_134_rule(Parser *p) return _seq; } -// _loop0_136: ',' star_target +// _loop0_137: ',' star_target static asdl_seq * -_loop0_136_rule(Parser *p) +_loop0_137_rule(Parser *p) { if (p->level++ == MAXSTACK) { _Pypegen_stack_overflow(p); @@ -33436,7 +33521,7 @@ _loop0_136_rule(Parser *p) p->level--; return NULL; } - D(fprintf(stderr, "%*c> _loop0_136[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "',' star_target")); + D(fprintf(stderr, "%*c> _loop0_137[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "',' star_target")); Token * _literal; expr_ty elem; while ( @@ -33468,7 +33553,7 @@ _loop0_136_rule(Parser *p) _mark = p->mark; } p->mark = _mark; - D(fprintf(stderr, "%*c%s _loop0_136[%d-%d]: %s failed!\n", p->level, ' ', + D(fprintf(stderr, "%*c%s _loop0_137[%d-%d]: %s failed!\n", p->level, ' ', p->error_indicator ? "ERROR!" : "-", _mark, p->mark, "',' star_target")); } asdl_seq *_seq = (asdl_seq*)_Py_asdl_generic_seq_new(_n, p->arena); @@ -33485,9 +33570,9 @@ _loop0_136_rule(Parser *p) return _seq; } -// _gather_135: star_target _loop0_136 +// _gather_136: star_target _loop0_137 static asdl_seq * -_gather_135_rule(Parser *p) +_gather_136_rule(Parser *p) { if (p->level++ == MAXSTACK) { _Pypegen_stack_overflow(p); @@ -33498,27 +33583,27 @@ _gather_135_rule(Parser *p) } asdl_seq * _res = NULL; int _mark = p->mark; - { // star_target _loop0_136 + { // star_target _loop0_137 if (p->error_indicator) { p->level--; return NULL; } - D(fprintf(stderr, "%*c> _gather_135[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "star_target _loop0_136")); + D(fprintf(stderr, "%*c> _gather_136[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "star_target _loop0_137")); expr_ty elem; asdl_seq * seq; if ( (elem = star_target_rule(p)) // star_target && - (seq = _loop0_136_rule(p)) // _loop0_136 + (seq = _loop0_137_rule(p)) // _loop0_137 ) { - D(fprintf(stderr, "%*c+ _gather_135[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "star_target _loop0_136")); + D(fprintf(stderr, "%*c+ _gather_136[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "star_target _loop0_137")); _res = _PyPegen_seq_insert_in_front(p, elem, seq); goto done; } p->mark = _mark; - D(fprintf(stderr, "%*c%s _gather_135[%d-%d]: %s failed!\n", p->level, ' ', - p->error_indicator ? "ERROR!" : "-", _mark, p->mark, "star_target _loop0_136")); + D(fprintf(stderr, "%*c%s _gather_136[%d-%d]: %s failed!\n", p->level, ' ', + p->error_indicator ? "ERROR!" : "-", _mark, p->mark, "star_target _loop0_137")); } _res = NULL; done: @@ -33526,9 +33611,9 @@ _gather_135_rule(Parser *p) return _res; } -// _loop1_137: (',' star_target) +// _loop1_138: (',' star_target) static asdl_seq * -_loop1_137_rule(Parser *p) +_loop1_138_rule(Parser *p) { if (p->level++ == MAXSTACK) { _Pypegen_stack_overflow(p); @@ -33553,13 +33638,13 @@ _loop1_137_rule(Parser *p) p->level--; return NULL; } - D(fprintf(stderr, "%*c> _loop1_137[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "(',' star_target)")); - void *_tmp_262_var; + D(fprintf(stderr, "%*c> _loop1_138[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "(',' star_target)")); + void *_tmp_264_var; while ( - (_tmp_262_var = _tmp_262_rule(p)) // ',' star_target + (_tmp_264_var = _tmp_264_rule(p)) // ',' star_target ) { - _res = _tmp_262_var; + _res = _tmp_264_var; if (_n == _children_capacity) { _children_capacity *= 2; void **_new_children = PyMem_Realloc(_children, _children_capacity*sizeof(void *)); @@ -33576,7 +33661,7 @@ _loop1_137_rule(Parser *p) _mark = p->mark; } p->mark = _mark; - D(fprintf(stderr, "%*c%s _loop1_137[%d-%d]: %s failed!\n", p->level, ' ', + D(fprintf(stderr, "%*c%s _loop1_138[%d-%d]: %s failed!\n", p->level, ' ', p->error_indicator ? "ERROR!" : "-", _mark, p->mark, "(',' star_target)")); } if (_n == 0 || p->error_indicator) { @@ -33598,9 +33683,9 @@ _loop1_137_rule(Parser *p) return _seq; } -// _tmp_138: !'*' star_target +// _tmp_139: !'*' star_target static void * -_tmp_138_rule(Parser *p) +_tmp_139_rule(Parser *p) { if (p->level++ == MAXSTACK) { _Pypegen_stack_overflow(p); @@ -33616,7 +33701,7 @@ _tmp_138_rule(Parser *p) p->level--; return NULL; } - D(fprintf(stderr, "%*c> _tmp_138[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "!'*' star_target")); + D(fprintf(stderr, "%*c> _tmp_139[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "!'*' star_target")); expr_ty star_target_var; if ( _PyPegen_lookahead_with_int(0, _PyPegen_expect_token, p, 16) // token='*' @@ -33624,12 +33709,12 @@ _tmp_138_rule(Parser *p) (star_target_var = star_target_rule(p)) // star_target ) { - D(fprintf(stderr, "%*c+ _tmp_138[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "!'*' star_target")); + D(fprintf(stderr, "%*c+ _tmp_139[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "!'*' star_target")); _res = star_target_var; goto done; } p->mark = _mark; - D(fprintf(stderr, "%*c%s _tmp_138[%d-%d]: %s failed!\n", p->level, ' ', + D(fprintf(stderr, "%*c%s _tmp_139[%d-%d]: %s failed!\n", p->level, ' ', p->error_indicator ? "ERROR!" : "-", _mark, p->mark, "!'*' star_target")); } _res = NULL; @@ -33638,9 +33723,9 @@ _tmp_138_rule(Parser *p) return _res; } -// _loop0_140: ',' del_target +// _loop0_141: ',' del_target static asdl_seq * -_loop0_140_rule(Parser *p) +_loop0_141_rule(Parser *p) { if (p->level++ == MAXSTACK) { _Pypegen_stack_overflow(p); @@ -33665,7 +33750,7 @@ _loop0_140_rule(Parser *p) p->level--; return NULL; } - D(fprintf(stderr, "%*c> _loop0_140[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "',' del_target")); + D(fprintf(stderr, "%*c> _loop0_141[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "',' del_target")); Token * _literal; expr_ty elem; while ( @@ -33697,7 +33782,7 @@ _loop0_140_rule(Parser *p) _mark = p->mark; } p->mark = _mark; - D(fprintf(stderr, "%*c%s _loop0_140[%d-%d]: %s failed!\n", p->level, ' ', + D(fprintf(stderr, "%*c%s _loop0_141[%d-%d]: %s failed!\n", p->level, ' ', p->error_indicator ? "ERROR!" : "-", _mark, p->mark, "',' del_target")); } asdl_seq *_seq = (asdl_seq*)_Py_asdl_generic_seq_new(_n, p->arena); @@ -33714,9 +33799,9 @@ _loop0_140_rule(Parser *p) return _seq; } -// _gather_139: del_target _loop0_140 +// _gather_140: del_target _loop0_141 static asdl_seq * -_gather_139_rule(Parser *p) +_gather_140_rule(Parser *p) { if (p->level++ == MAXSTACK) { _Pypegen_stack_overflow(p); @@ -33727,27 +33812,27 @@ _gather_139_rule(Parser *p) } asdl_seq * _res = NULL; int _mark = p->mark; - { // del_target _loop0_140 + { // del_target _loop0_141 if (p->error_indicator) { p->level--; return NULL; } - D(fprintf(stderr, "%*c> _gather_139[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "del_target _loop0_140")); + D(fprintf(stderr, "%*c> _gather_140[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "del_target _loop0_141")); expr_ty elem; asdl_seq * seq; if ( (elem = del_target_rule(p)) // del_target && - (seq = _loop0_140_rule(p)) // _loop0_140 + (seq = _loop0_141_rule(p)) // _loop0_141 ) { - D(fprintf(stderr, "%*c+ _gather_139[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "del_target _loop0_140")); + D(fprintf(stderr, "%*c+ _gather_140[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "del_target _loop0_141")); _res = _PyPegen_seq_insert_in_front(p, elem, seq); goto done; } p->mark = _mark; - D(fprintf(stderr, "%*c%s _gather_139[%d-%d]: %s failed!\n", p->level, ' ', - p->error_indicator ? "ERROR!" : "-", _mark, p->mark, "del_target _loop0_140")); + D(fprintf(stderr, "%*c%s _gather_140[%d-%d]: %s failed!\n", p->level, ' ', + p->error_indicator ? "ERROR!" : "-", _mark, p->mark, "del_target _loop0_141")); } _res = NULL; done: @@ -33755,9 +33840,9 @@ _gather_139_rule(Parser *p) return _res; } -// _loop0_142: ',' expression +// _loop0_143: ',' expression static asdl_seq * -_loop0_142_rule(Parser *p) +_loop0_143_rule(Parser *p) { if (p->level++ == MAXSTACK) { _Pypegen_stack_overflow(p); @@ -33782,7 +33867,7 @@ _loop0_142_rule(Parser *p) p->level--; return NULL; } - D(fprintf(stderr, "%*c> _loop0_142[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "',' expression")); + D(fprintf(stderr, "%*c> _loop0_143[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "',' expression")); Token * _literal; expr_ty elem; while ( @@ -33814,7 +33899,7 @@ _loop0_142_rule(Parser *p) _mark = p->mark; } p->mark = _mark; - D(fprintf(stderr, "%*c%s _loop0_142[%d-%d]: %s failed!\n", p->level, ' ', + D(fprintf(stderr, "%*c%s _loop0_143[%d-%d]: %s failed!\n", p->level, ' ', p->error_indicator ? "ERROR!" : "-", _mark, p->mark, "',' expression")); } asdl_seq *_seq = (asdl_seq*)_Py_asdl_generic_seq_new(_n, p->arena); @@ -33831,9 +33916,9 @@ _loop0_142_rule(Parser *p) return _seq; } -// _gather_141: expression _loop0_142 +// _gather_142: expression _loop0_143 static asdl_seq * -_gather_141_rule(Parser *p) +_gather_142_rule(Parser *p) { if (p->level++ == MAXSTACK) { _Pypegen_stack_overflow(p); @@ -33844,27 +33929,27 @@ _gather_141_rule(Parser *p) } asdl_seq * _res = NULL; int _mark = p->mark; - { // expression _loop0_142 + { // expression _loop0_143 if (p->error_indicator) { p->level--; return NULL; } - D(fprintf(stderr, "%*c> _gather_141[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "expression _loop0_142")); + D(fprintf(stderr, "%*c> _gather_142[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "expression _loop0_143")); expr_ty elem; asdl_seq * seq; if ( (elem = expression_rule(p)) // expression && - (seq = _loop0_142_rule(p)) // _loop0_142 + (seq = _loop0_143_rule(p)) // _loop0_143 ) { - D(fprintf(stderr, "%*c+ _gather_141[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "expression _loop0_142")); + D(fprintf(stderr, "%*c+ _gather_142[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "expression _loop0_143")); _res = _PyPegen_seq_insert_in_front(p, elem, seq); goto done; } p->mark = _mark; - D(fprintf(stderr, "%*c%s _gather_141[%d-%d]: %s failed!\n", p->level, ' ', - p->error_indicator ? "ERROR!" : "-", _mark, p->mark, "expression _loop0_142")); + D(fprintf(stderr, "%*c%s _gather_142[%d-%d]: %s failed!\n", p->level, ' ', + p->error_indicator ? "ERROR!" : "-", _mark, p->mark, "expression _loop0_143")); } _res = NULL; done: @@ -33872,9 +33957,9 @@ _gather_141_rule(Parser *p) return _res; } -// _loop0_144: ',' expression +// _loop0_145: ',' expression static asdl_seq * -_loop0_144_rule(Parser *p) +_loop0_145_rule(Parser *p) { if (p->level++ == MAXSTACK) { _Pypegen_stack_overflow(p); @@ -33899,7 +33984,7 @@ _loop0_144_rule(Parser *p) p->level--; return NULL; } - D(fprintf(stderr, "%*c> _loop0_144[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "',' expression")); + D(fprintf(stderr, "%*c> _loop0_145[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "',' expression")); Token * _literal; expr_ty elem; while ( @@ -33931,7 +34016,7 @@ _loop0_144_rule(Parser *p) _mark = p->mark; } p->mark = _mark; - D(fprintf(stderr, "%*c%s _loop0_144[%d-%d]: %s failed!\n", p->level, ' ', + D(fprintf(stderr, "%*c%s _loop0_145[%d-%d]: %s failed!\n", p->level, ' ', p->error_indicator ? "ERROR!" : "-", _mark, p->mark, "',' expression")); } asdl_seq *_seq = (asdl_seq*)_Py_asdl_generic_seq_new(_n, p->arena); @@ -33948,9 +34033,9 @@ _loop0_144_rule(Parser *p) return _seq; } -// _gather_143: expression _loop0_144 +// _gather_144: expression _loop0_145 static asdl_seq * -_gather_143_rule(Parser *p) +_gather_144_rule(Parser *p) { if (p->level++ == MAXSTACK) { _Pypegen_stack_overflow(p); @@ -33961,27 +34046,27 @@ _gather_143_rule(Parser *p) } asdl_seq * _res = NULL; int _mark = p->mark; - { // expression _loop0_144 + { // expression _loop0_145 if (p->error_indicator) { p->level--; return NULL; } - D(fprintf(stderr, "%*c> _gather_143[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "expression _loop0_144")); + D(fprintf(stderr, "%*c> _gather_144[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "expression _loop0_145")); expr_ty elem; asdl_seq * seq; if ( (elem = expression_rule(p)) // expression && - (seq = _loop0_144_rule(p)) // _loop0_144 + (seq = _loop0_145_rule(p)) // _loop0_145 ) { - D(fprintf(stderr, "%*c+ _gather_143[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "expression _loop0_144")); + D(fprintf(stderr, "%*c+ _gather_144[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "expression _loop0_145")); _res = _PyPegen_seq_insert_in_front(p, elem, seq); goto done; } p->mark = _mark; - D(fprintf(stderr, "%*c%s _gather_143[%d-%d]: %s failed!\n", p->level, ' ', - p->error_indicator ? "ERROR!" : "-", _mark, p->mark, "expression _loop0_144")); + D(fprintf(stderr, "%*c%s _gather_144[%d-%d]: %s failed!\n", p->level, ' ', + p->error_indicator ? "ERROR!" : "-", _mark, p->mark, "expression _loop0_145")); } _res = NULL; done: @@ -33989,9 +34074,9 @@ _gather_143_rule(Parser *p) return _res; } -// _loop0_146: ',' expression +// _loop0_147: ',' expression static asdl_seq * -_loop0_146_rule(Parser *p) +_loop0_147_rule(Parser *p) { if (p->level++ == MAXSTACK) { _Pypegen_stack_overflow(p); @@ -34016,7 +34101,7 @@ _loop0_146_rule(Parser *p) p->level--; return NULL; } - D(fprintf(stderr, "%*c> _loop0_146[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "',' expression")); + D(fprintf(stderr, "%*c> _loop0_147[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "',' expression")); Token * _literal; expr_ty elem; while ( @@ -34048,7 +34133,7 @@ _loop0_146_rule(Parser *p) _mark = p->mark; } p->mark = _mark; - D(fprintf(stderr, "%*c%s _loop0_146[%d-%d]: %s failed!\n", p->level, ' ', + D(fprintf(stderr, "%*c%s _loop0_147[%d-%d]: %s failed!\n", p->level, ' ', p->error_indicator ? "ERROR!" : "-", _mark, p->mark, "',' expression")); } asdl_seq *_seq = (asdl_seq*)_Py_asdl_generic_seq_new(_n, p->arena); @@ -34065,9 +34150,9 @@ _loop0_146_rule(Parser *p) return _seq; } -// _gather_145: expression _loop0_146 +// _gather_146: expression _loop0_147 static asdl_seq * -_gather_145_rule(Parser *p) +_gather_146_rule(Parser *p) { if (p->level++ == MAXSTACK) { _Pypegen_stack_overflow(p); @@ -34078,27 +34163,27 @@ _gather_145_rule(Parser *p) } asdl_seq * _res = NULL; int _mark = p->mark; - { // expression _loop0_146 + { // expression _loop0_147 if (p->error_indicator) { p->level--; return NULL; } - D(fprintf(stderr, "%*c> _gather_145[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "expression _loop0_146")); + D(fprintf(stderr, "%*c> _gather_146[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "expression _loop0_147")); expr_ty elem; asdl_seq * seq; if ( (elem = expression_rule(p)) // expression && - (seq = _loop0_146_rule(p)) // _loop0_146 + (seq = _loop0_147_rule(p)) // _loop0_147 ) { - D(fprintf(stderr, "%*c+ _gather_145[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "expression _loop0_146")); + D(fprintf(stderr, "%*c+ _gather_146[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "expression _loop0_147")); _res = _PyPegen_seq_insert_in_front(p, elem, seq); goto done; } p->mark = _mark; - D(fprintf(stderr, "%*c%s _gather_145[%d-%d]: %s failed!\n", p->level, ' ', - p->error_indicator ? "ERROR!" : "-", _mark, p->mark, "expression _loop0_146")); + D(fprintf(stderr, "%*c%s _gather_146[%d-%d]: %s failed!\n", p->level, ' ', + p->error_indicator ? "ERROR!" : "-", _mark, p->mark, "expression _loop0_147")); } _res = NULL; done: @@ -34106,9 +34191,9 @@ _gather_145_rule(Parser *p) return _res; } -// _loop0_148: ',' expression +// _loop0_149: ',' expression static asdl_seq * -_loop0_148_rule(Parser *p) +_loop0_149_rule(Parser *p) { if (p->level++ == MAXSTACK) { _Pypegen_stack_overflow(p); @@ -34133,7 +34218,7 @@ _loop0_148_rule(Parser *p) p->level--; return NULL; } - D(fprintf(stderr, "%*c> _loop0_148[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "',' expression")); + D(fprintf(stderr, "%*c> _loop0_149[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "',' expression")); Token * _literal; expr_ty elem; while ( @@ -34165,7 +34250,7 @@ _loop0_148_rule(Parser *p) _mark = p->mark; } p->mark = _mark; - D(fprintf(stderr, "%*c%s _loop0_148[%d-%d]: %s failed!\n", p->level, ' ', + D(fprintf(stderr, "%*c%s _loop0_149[%d-%d]: %s failed!\n", p->level, ' ', p->error_indicator ? "ERROR!" : "-", _mark, p->mark, "',' expression")); } asdl_seq *_seq = (asdl_seq*)_Py_asdl_generic_seq_new(_n, p->arena); @@ -34182,9 +34267,9 @@ _loop0_148_rule(Parser *p) return _seq; } -// _gather_147: expression _loop0_148 +// _gather_148: expression _loop0_149 static asdl_seq * -_gather_147_rule(Parser *p) +_gather_148_rule(Parser *p) { if (p->level++ == MAXSTACK) { _Pypegen_stack_overflow(p); @@ -34195,27 +34280,27 @@ _gather_147_rule(Parser *p) } asdl_seq * _res = NULL; int _mark = p->mark; - { // expression _loop0_148 + { // expression _loop0_149 if (p->error_indicator) { p->level--; return NULL; } - D(fprintf(stderr, "%*c> _gather_147[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "expression _loop0_148")); + D(fprintf(stderr, "%*c> _gather_148[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "expression _loop0_149")); expr_ty elem; asdl_seq * seq; if ( (elem = expression_rule(p)) // expression && - (seq = _loop0_148_rule(p)) // _loop0_148 + (seq = _loop0_149_rule(p)) // _loop0_149 ) { - D(fprintf(stderr, "%*c+ _gather_147[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "expression _loop0_148")); + D(fprintf(stderr, "%*c+ _gather_148[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "expression _loop0_149")); _res = _PyPegen_seq_insert_in_front(p, elem, seq); goto done; } p->mark = _mark; - D(fprintf(stderr, "%*c%s _gather_147[%d-%d]: %s failed!\n", p->level, ' ', - p->error_indicator ? "ERROR!" : "-", _mark, p->mark, "expression _loop0_148")); + D(fprintf(stderr, "%*c%s _gather_148[%d-%d]: %s failed!\n", p->level, ' ', + p->error_indicator ? "ERROR!" : "-", _mark, p->mark, "expression _loop0_149")); } _res = NULL; done: @@ -34223,9 +34308,9 @@ _gather_147_rule(Parser *p) return _res; } -// _tmp_149: NEWLINE INDENT +// _tmp_150: NEWLINE INDENT static void * -_tmp_149_rule(Parser *p) +_tmp_150_rule(Parser *p) { if (p->level++ == MAXSTACK) { _Pypegen_stack_overflow(p); @@ -34241,7 +34326,7 @@ _tmp_149_rule(Parser *p) p->level--; return NULL; } - D(fprintf(stderr, "%*c> _tmp_149[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "NEWLINE INDENT")); + D(fprintf(stderr, "%*c> _tmp_150[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "NEWLINE INDENT")); Token * indent_var; Token * newline_var; if ( @@ -34250,12 +34335,12 @@ _tmp_149_rule(Parser *p) (indent_var = _PyPegen_expect_token(p, INDENT)) // token='INDENT' ) { - D(fprintf(stderr, "%*c+ _tmp_149[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "NEWLINE INDENT")); + D(fprintf(stderr, "%*c+ _tmp_150[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "NEWLINE INDENT")); _res = _PyPegen_dummy_name(p, newline_var, indent_var); goto done; } p->mark = _mark; - D(fprintf(stderr, "%*c%s _tmp_149[%d-%d]: %s failed!\n", p->level, ' ', + D(fprintf(stderr, "%*c%s _tmp_150[%d-%d]: %s failed!\n", p->level, ' ', p->error_indicator ? "ERROR!" : "-", _mark, p->mark, "NEWLINE INDENT")); } _res = NULL; @@ -34264,11 +34349,11 @@ _tmp_149_rule(Parser *p) return _res; } -// _tmp_150: +// _tmp_151: // | (','.(starred_expression | (assignment_expression | expression !':=') !'=')+ ',' kwargs) // | kwargs static void * -_tmp_150_rule(Parser *p) +_tmp_151_rule(Parser *p) { if (p->level++ == MAXSTACK) { _Pypegen_stack_overflow(p); @@ -34284,18 +34369,18 @@ _tmp_150_rule(Parser *p) p->level--; return NULL; } - D(fprintf(stderr, "%*c> _tmp_150[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "(','.(starred_expression | (assignment_expression | expression !':=') !'=')+ ',' kwargs)")); - void *_tmp_263_var; + D(fprintf(stderr, "%*c> _tmp_151[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "(','.(starred_expression | (assignment_expression | expression !':=') !'=')+ ',' kwargs)")); + void *_tmp_265_var; if ( - (_tmp_263_var = _tmp_263_rule(p)) // ','.(starred_expression | (assignment_expression | expression !':=') !'=')+ ',' kwargs + (_tmp_265_var = _tmp_265_rule(p)) // ','.(starred_expression | (assignment_expression | expression !':=') !'=')+ ',' kwargs ) { - D(fprintf(stderr, "%*c+ _tmp_150[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "(','.(starred_expression | (assignment_expression | expression !':=') !'=')+ ',' kwargs)")); - _res = _tmp_263_var; + D(fprintf(stderr, "%*c+ _tmp_151[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "(','.(starred_expression | (assignment_expression | expression !':=') !'=')+ ',' kwargs)")); + _res = _tmp_265_var; goto done; } p->mark = _mark; - D(fprintf(stderr, "%*c%s _tmp_150[%d-%d]: %s failed!\n", p->level, ' ', + D(fprintf(stderr, "%*c%s _tmp_151[%d-%d]: %s failed!\n", p->level, ' ', p->error_indicator ? "ERROR!" : "-", _mark, p->mark, "(','.(starred_expression | (assignment_expression | expression !':=') !'=')+ ',' kwargs)")); } { // kwargs @@ -34303,18 +34388,18 @@ _tmp_150_rule(Parser *p) p->level--; return NULL; } - D(fprintf(stderr, "%*c> _tmp_150[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "kwargs")); + D(fprintf(stderr, "%*c> _tmp_151[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "kwargs")); asdl_seq* kwargs_var; if ( (kwargs_var = kwargs_rule(p)) // kwargs ) { - D(fprintf(stderr, "%*c+ _tmp_150[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "kwargs")); + D(fprintf(stderr, "%*c+ _tmp_151[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "kwargs")); _res = kwargs_var; goto done; } p->mark = _mark; - D(fprintf(stderr, "%*c%s _tmp_150[%d-%d]: %s failed!\n", p->level, ' ', + D(fprintf(stderr, "%*c%s _tmp_151[%d-%d]: %s failed!\n", p->level, ' ', p->error_indicator ? "ERROR!" : "-", _mark, p->mark, "kwargs")); } _res = NULL; @@ -34323,9 +34408,9 @@ _tmp_150_rule(Parser *p) return _res; } -// _tmp_151: args | expression for_if_clauses +// _tmp_152: args | expression for_if_clauses static void * -_tmp_151_rule(Parser *p) +_tmp_152_rule(Parser *p) { if (p->level++ == MAXSTACK) { _Pypegen_stack_overflow(p); @@ -34341,18 +34426,18 @@ _tmp_151_rule(Parser *p) p->level--; return NULL; } - D(fprintf(stderr, "%*c> _tmp_151[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "args")); + D(fprintf(stderr, "%*c> _tmp_152[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "args")); expr_ty args_var; if ( (args_var = args_rule(p)) // args ) { - D(fprintf(stderr, "%*c+ _tmp_151[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "args")); + D(fprintf(stderr, "%*c+ _tmp_152[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "args")); _res = args_var; goto done; } p->mark = _mark; - D(fprintf(stderr, "%*c%s _tmp_151[%d-%d]: %s failed!\n", p->level, ' ', + D(fprintf(stderr, "%*c%s _tmp_152[%d-%d]: %s failed!\n", p->level, ' ', p->error_indicator ? "ERROR!" : "-", _mark, p->mark, "args")); } { // expression for_if_clauses @@ -34360,7 +34445,7 @@ _tmp_151_rule(Parser *p) p->level--; return NULL; } - D(fprintf(stderr, "%*c> _tmp_151[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "expression for_if_clauses")); + D(fprintf(stderr, "%*c> _tmp_152[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "expression for_if_clauses")); expr_ty expression_var; asdl_comprehension_seq* for_if_clauses_var; if ( @@ -34369,12 +34454,12 @@ _tmp_151_rule(Parser *p) (for_if_clauses_var = for_if_clauses_rule(p)) // for_if_clauses ) { - D(fprintf(stderr, "%*c+ _tmp_151[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "expression for_if_clauses")); + D(fprintf(stderr, "%*c+ _tmp_152[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "expression for_if_clauses")); _res = _PyPegen_dummy_name(p, expression_var, for_if_clauses_var); goto done; } p->mark = _mark; - D(fprintf(stderr, "%*c%s _tmp_151[%d-%d]: %s failed!\n", p->level, ' ', + D(fprintf(stderr, "%*c%s _tmp_152[%d-%d]: %s failed!\n", p->level, ' ', p->error_indicator ? "ERROR!" : "-", _mark, p->mark, "expression for_if_clauses")); } _res = NULL; @@ -34383,9 +34468,9 @@ _tmp_151_rule(Parser *p) return _res; } -// _tmp_152: args ',' +// _tmp_153: args ',' static void * -_tmp_152_rule(Parser *p) +_tmp_153_rule(Parser *p) { if (p->level++ == MAXSTACK) { _Pypegen_stack_overflow(p); @@ -34401,7 +34486,7 @@ _tmp_152_rule(Parser *p) p->level--; return NULL; } - D(fprintf(stderr, "%*c> _tmp_152[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "args ','")); + D(fprintf(stderr, "%*c> _tmp_153[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "args ','")); Token * _literal; expr_ty args_var; if ( @@ -34410,12 +34495,12 @@ _tmp_152_rule(Parser *p) (_literal = _PyPegen_expect_token(p, 12)) // token=',' ) { - D(fprintf(stderr, "%*c+ _tmp_152[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "args ','")); + D(fprintf(stderr, "%*c+ _tmp_153[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "args ','")); _res = _PyPegen_dummy_name(p, args_var, _literal); goto done; } p->mark = _mark; - D(fprintf(stderr, "%*c%s _tmp_152[%d-%d]: %s failed!\n", p->level, ' ', + D(fprintf(stderr, "%*c%s _tmp_153[%d-%d]: %s failed!\n", p->level, ' ', p->error_indicator ? "ERROR!" : "-", _mark, p->mark, "args ','")); } _res = NULL; @@ -34424,9 +34509,9 @@ _tmp_152_rule(Parser *p) return _res; } -// _tmp_153: ',' | ')' +// _tmp_154: ',' | ')' static void * -_tmp_153_rule(Parser *p) +_tmp_154_rule(Parser *p) { if (p->level++ == MAXSTACK) { _Pypegen_stack_overflow(p); @@ -34442,18 +34527,18 @@ _tmp_153_rule(Parser *p) p->level--; return NULL; } - D(fprintf(stderr, "%*c> _tmp_153[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "','")); + D(fprintf(stderr, "%*c> _tmp_154[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "','")); Token * _literal; if ( (_literal = _PyPegen_expect_token(p, 12)) // token=',' ) { - D(fprintf(stderr, "%*c+ _tmp_153[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "','")); + D(fprintf(stderr, "%*c+ _tmp_154[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "','")); _res = _literal; goto done; } p->mark = _mark; - D(fprintf(stderr, "%*c%s _tmp_153[%d-%d]: %s failed!\n", p->level, ' ', + D(fprintf(stderr, "%*c%s _tmp_154[%d-%d]: %s failed!\n", p->level, ' ', p->error_indicator ? "ERROR!" : "-", _mark, p->mark, "','")); } { // ')' @@ -34461,18 +34546,18 @@ _tmp_153_rule(Parser *p) p->level--; return NULL; } - D(fprintf(stderr, "%*c> _tmp_153[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "')'")); + D(fprintf(stderr, "%*c> _tmp_154[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "')'")); Token * _literal; if ( (_literal = _PyPegen_expect_token(p, 8)) // token=')' ) { - D(fprintf(stderr, "%*c+ _tmp_153[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "')'")); + D(fprintf(stderr, "%*c+ _tmp_154[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "')'")); _res = _literal; goto done; } p->mark = _mark; - D(fprintf(stderr, "%*c%s _tmp_153[%d-%d]: %s failed!\n", p->level, ' ', + D(fprintf(stderr, "%*c%s _tmp_154[%d-%d]: %s failed!\n", p->level, ' ', p->error_indicator ? "ERROR!" : "-", _mark, p->mark, "')'")); } _res = NULL; @@ -34481,9 +34566,9 @@ _tmp_153_rule(Parser *p) return _res; } -// _tmp_154: 'True' | 'False' | 'None' +// _tmp_155: 'True' | 'False' | 'None' static void * -_tmp_154_rule(Parser *p) +_tmp_155_rule(Parser *p) { if (p->level++ == MAXSTACK) { _Pypegen_stack_overflow(p); @@ -34499,18 +34584,18 @@ _tmp_154_rule(Parser *p) p->level--; return NULL; } - D(fprintf(stderr, "%*c> _tmp_154[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "'True'")); + D(fprintf(stderr, "%*c> _tmp_155[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "'True'")); Token * _keyword; if ( - (_keyword = _PyPegen_expect_token(p, 610)) // token='True' + (_keyword = _PyPegen_expect_token(p, 613)) // token='True' ) { - D(fprintf(stderr, "%*c+ _tmp_154[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "'True'")); + D(fprintf(stderr, "%*c+ _tmp_155[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "'True'")); _res = _keyword; goto done; } p->mark = _mark; - D(fprintf(stderr, "%*c%s _tmp_154[%d-%d]: %s failed!\n", p->level, ' ', + D(fprintf(stderr, "%*c%s _tmp_155[%d-%d]: %s failed!\n", p->level, ' ', p->error_indicator ? "ERROR!" : "-", _mark, p->mark, "'True'")); } { // 'False' @@ -34518,18 +34603,18 @@ _tmp_154_rule(Parser *p) p->level--; return NULL; } - D(fprintf(stderr, "%*c> _tmp_154[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "'False'")); + D(fprintf(stderr, "%*c> _tmp_155[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "'False'")); Token * _keyword; if ( - (_keyword = _PyPegen_expect_token(p, 612)) // token='False' + (_keyword = _PyPegen_expect_token(p, 615)) // token='False' ) { - D(fprintf(stderr, "%*c+ _tmp_154[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "'False'")); + D(fprintf(stderr, "%*c+ _tmp_155[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "'False'")); _res = _keyword; goto done; } p->mark = _mark; - D(fprintf(stderr, "%*c%s _tmp_154[%d-%d]: %s failed!\n", p->level, ' ', + D(fprintf(stderr, "%*c%s _tmp_155[%d-%d]: %s failed!\n", p->level, ' ', p->error_indicator ? "ERROR!" : "-", _mark, p->mark, "'False'")); } { // 'None' @@ -34537,18 +34622,18 @@ _tmp_154_rule(Parser *p) p->level--; return NULL; } - D(fprintf(stderr, "%*c> _tmp_154[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "'None'")); + D(fprintf(stderr, "%*c> _tmp_155[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "'None'")); Token * _keyword; if ( - (_keyword = _PyPegen_expect_token(p, 611)) // token='None' + (_keyword = _PyPegen_expect_token(p, 614)) // token='None' ) { - D(fprintf(stderr, "%*c+ _tmp_154[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "'None'")); + D(fprintf(stderr, "%*c+ _tmp_155[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "'None'")); _res = _keyword; goto done; } p->mark = _mark; - D(fprintf(stderr, "%*c%s _tmp_154[%d-%d]: %s failed!\n", p->level, ' ', + D(fprintf(stderr, "%*c%s _tmp_155[%d-%d]: %s failed!\n", p->level, ' ', p->error_indicator ? "ERROR!" : "-", _mark, p->mark, "'None'")); } _res = NULL; @@ -34557,9 +34642,9 @@ _tmp_154_rule(Parser *p) return _res; } -// _tmp_155: NAME '=' +// _tmp_156: NAME '=' static void * -_tmp_155_rule(Parser *p) +_tmp_156_rule(Parser *p) { if (p->level++ == MAXSTACK) { _Pypegen_stack_overflow(p); @@ -34575,7 +34660,7 @@ _tmp_155_rule(Parser *p) p->level--; return NULL; } - D(fprintf(stderr, "%*c> _tmp_155[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "NAME '='")); + D(fprintf(stderr, "%*c> _tmp_156[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "NAME '='")); Token * _literal; expr_ty name_var; if ( @@ -34584,12 +34669,12 @@ _tmp_155_rule(Parser *p) (_literal = _PyPegen_expect_token(p, 22)) // token='=' ) { - D(fprintf(stderr, "%*c+ _tmp_155[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "NAME '='")); + D(fprintf(stderr, "%*c+ _tmp_156[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "NAME '='")); _res = _PyPegen_dummy_name(p, name_var, _literal); goto done; } p->mark = _mark; - D(fprintf(stderr, "%*c%s _tmp_155[%d-%d]: %s failed!\n", p->level, ' ', + D(fprintf(stderr, "%*c%s _tmp_156[%d-%d]: %s failed!\n", p->level, ' ', p->error_indicator ? "ERROR!" : "-", _mark, p->mark, "NAME '='")); } _res = NULL; @@ -34598,9 +34683,9 @@ _tmp_155_rule(Parser *p) return _res; } -// _tmp_156: NAME STRING | SOFT_KEYWORD +// _tmp_157: NAME STRING | SOFT_KEYWORD static void * -_tmp_156_rule(Parser *p) +_tmp_157_rule(Parser *p) { if (p->level++ == MAXSTACK) { _Pypegen_stack_overflow(p); @@ -34616,7 +34701,7 @@ _tmp_156_rule(Parser *p) p->level--; return NULL; } - D(fprintf(stderr, "%*c> _tmp_156[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "NAME STRING")); + D(fprintf(stderr, "%*c> _tmp_157[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "NAME STRING")); expr_ty name_var; expr_ty string_var; if ( @@ -34625,12 +34710,12 @@ _tmp_156_rule(Parser *p) (string_var = _PyPegen_string_token(p)) // STRING ) { - D(fprintf(stderr, "%*c+ _tmp_156[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "NAME STRING")); + D(fprintf(stderr, "%*c+ _tmp_157[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "NAME STRING")); _res = _PyPegen_dummy_name(p, name_var, string_var); goto done; } p->mark = _mark; - D(fprintf(stderr, "%*c%s _tmp_156[%d-%d]: %s failed!\n", p->level, ' ', + D(fprintf(stderr, "%*c%s _tmp_157[%d-%d]: %s failed!\n", p->level, ' ', p->error_indicator ? "ERROR!" : "-", _mark, p->mark, "NAME STRING")); } { // SOFT_KEYWORD @@ -34638,18 +34723,18 @@ _tmp_156_rule(Parser *p) p->level--; return NULL; } - D(fprintf(stderr, "%*c> _tmp_156[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "SOFT_KEYWORD")); + D(fprintf(stderr, "%*c> _tmp_157[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "SOFT_KEYWORD")); expr_ty soft_keyword_var; if ( (soft_keyword_var = _PyPegen_soft_keyword_token(p)) // SOFT_KEYWORD ) { - D(fprintf(stderr, "%*c+ _tmp_156[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "SOFT_KEYWORD")); + D(fprintf(stderr, "%*c+ _tmp_157[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "SOFT_KEYWORD")); _res = soft_keyword_var; goto done; } p->mark = _mark; - D(fprintf(stderr, "%*c%s _tmp_156[%d-%d]: %s failed!\n", p->level, ' ', + D(fprintf(stderr, "%*c%s _tmp_157[%d-%d]: %s failed!\n", p->level, ' ', p->error_indicator ? "ERROR!" : "-", _mark, p->mark, "SOFT_KEYWORD")); } _res = NULL; @@ -34658,9 +34743,9 @@ _tmp_156_rule(Parser *p) return _res; } -// _tmp_157: 'else' | ':' +// _tmp_158: 'else' | ':' static void * -_tmp_157_rule(Parser *p) +_tmp_158_rule(Parser *p) { if (p->level++ == MAXSTACK) { _Pypegen_stack_overflow(p); @@ -34676,18 +34761,18 @@ _tmp_157_rule(Parser *p) p->level--; return NULL; } - D(fprintf(stderr, "%*c> _tmp_157[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "'else'")); + D(fprintf(stderr, "%*c> _tmp_158[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "'else'")); Token * _keyword; if ( - (_keyword = _PyPegen_expect_token(p, 661)) // token='else' + (_keyword = _PyPegen_expect_token(p, 664)) // token='else' ) { - D(fprintf(stderr, "%*c+ _tmp_157[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "'else'")); + D(fprintf(stderr, "%*c+ _tmp_158[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "'else'")); _res = _keyword; goto done; } p->mark = _mark; - D(fprintf(stderr, "%*c%s _tmp_157[%d-%d]: %s failed!\n", p->level, ' ', + D(fprintf(stderr, "%*c%s _tmp_158[%d-%d]: %s failed!\n", p->level, ' ', p->error_indicator ? "ERROR!" : "-", _mark, p->mark, "'else'")); } { // ':' @@ -34695,18 +34780,18 @@ _tmp_157_rule(Parser *p) p->level--; return NULL; } - D(fprintf(stderr, "%*c> _tmp_157[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "':'")); + D(fprintf(stderr, "%*c> _tmp_158[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "':'")); Token * _literal; if ( (_literal = _PyPegen_expect_token(p, 11)) // token=':' ) { - D(fprintf(stderr, "%*c+ _tmp_157[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "':'")); + D(fprintf(stderr, "%*c+ _tmp_158[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "':'")); _res = _literal; goto done; } p->mark = _mark; - D(fprintf(stderr, "%*c%s _tmp_157[%d-%d]: %s failed!\n", p->level, ' ', + D(fprintf(stderr, "%*c%s _tmp_158[%d-%d]: %s failed!\n", p->level, ' ', p->error_indicator ? "ERROR!" : "-", _mark, p->mark, "':'")); } _res = NULL; @@ -34715,9 +34800,9 @@ _tmp_157_rule(Parser *p) return _res; } -// _tmp_158: '=' | ':=' +// _tmp_159: '=' | ':=' static void * -_tmp_158_rule(Parser *p) +_tmp_159_rule(Parser *p) { if (p->level++ == MAXSTACK) { _Pypegen_stack_overflow(p); @@ -34733,18 +34818,18 @@ _tmp_158_rule(Parser *p) p->level--; return NULL; } - D(fprintf(stderr, "%*c> _tmp_158[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "'='")); + D(fprintf(stderr, "%*c> _tmp_159[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "'='")); Token * _literal; if ( (_literal = _PyPegen_expect_token(p, 22)) // token='=' ) { - D(fprintf(stderr, "%*c+ _tmp_158[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "'='")); + D(fprintf(stderr, "%*c+ _tmp_159[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "'='")); _res = _literal; goto done; } p->mark = _mark; - D(fprintf(stderr, "%*c%s _tmp_158[%d-%d]: %s failed!\n", p->level, ' ', + D(fprintf(stderr, "%*c%s _tmp_159[%d-%d]: %s failed!\n", p->level, ' ', p->error_indicator ? "ERROR!" : "-", _mark, p->mark, "'='")); } { // ':=' @@ -34752,18 +34837,18 @@ _tmp_158_rule(Parser *p) p->level--; return NULL; } - D(fprintf(stderr, "%*c> _tmp_158[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "':='")); + D(fprintf(stderr, "%*c> _tmp_159[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "':='")); Token * _literal; if ( (_literal = _PyPegen_expect_token(p, 53)) // token=':=' ) { - D(fprintf(stderr, "%*c+ _tmp_158[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "':='")); + D(fprintf(stderr, "%*c+ _tmp_159[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "':='")); _res = _literal; goto done; } p->mark = _mark; - D(fprintf(stderr, "%*c%s _tmp_158[%d-%d]: %s failed!\n", p->level, ' ', + D(fprintf(stderr, "%*c%s _tmp_159[%d-%d]: %s failed!\n", p->level, ' ', p->error_indicator ? "ERROR!" : "-", _mark, p->mark, "':='")); } _res = NULL; @@ -34772,9 +34857,9 @@ _tmp_158_rule(Parser *p) return _res; } -// _tmp_159: list | tuple | genexp | 'True' | 'None' | 'False' +// _tmp_160: list | tuple | genexp | 'True' | 'None' | 'False' static void * -_tmp_159_rule(Parser *p) +_tmp_160_rule(Parser *p) { if (p->level++ == MAXSTACK) { _Pypegen_stack_overflow(p); @@ -34790,18 +34875,18 @@ _tmp_159_rule(Parser *p) p->level--; return NULL; } - D(fprintf(stderr, "%*c> _tmp_159[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "list")); + D(fprintf(stderr, "%*c> _tmp_160[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "list")); expr_ty list_var; if ( (list_var = list_rule(p)) // list ) { - D(fprintf(stderr, "%*c+ _tmp_159[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "list")); + D(fprintf(stderr, "%*c+ _tmp_160[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "list")); _res = list_var; goto done; } p->mark = _mark; - D(fprintf(stderr, "%*c%s _tmp_159[%d-%d]: %s failed!\n", p->level, ' ', + D(fprintf(stderr, "%*c%s _tmp_160[%d-%d]: %s failed!\n", p->level, ' ', p->error_indicator ? "ERROR!" : "-", _mark, p->mark, "list")); } { // tuple @@ -34809,18 +34894,18 @@ _tmp_159_rule(Parser *p) p->level--; return NULL; } - D(fprintf(stderr, "%*c> _tmp_159[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "tuple")); + D(fprintf(stderr, "%*c> _tmp_160[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "tuple")); expr_ty tuple_var; if ( (tuple_var = tuple_rule(p)) // tuple ) { - D(fprintf(stderr, "%*c+ _tmp_159[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "tuple")); + D(fprintf(stderr, "%*c+ _tmp_160[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "tuple")); _res = tuple_var; goto done; } p->mark = _mark; - D(fprintf(stderr, "%*c%s _tmp_159[%d-%d]: %s failed!\n", p->level, ' ', + D(fprintf(stderr, "%*c%s _tmp_160[%d-%d]: %s failed!\n", p->level, ' ', p->error_indicator ? "ERROR!" : "-", _mark, p->mark, "tuple")); } { // genexp @@ -34828,18 +34913,18 @@ _tmp_159_rule(Parser *p) p->level--; return NULL; } - D(fprintf(stderr, "%*c> _tmp_159[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "genexp")); + D(fprintf(stderr, "%*c> _tmp_160[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "genexp")); expr_ty genexp_var; if ( (genexp_var = genexp_rule(p)) // genexp ) { - D(fprintf(stderr, "%*c+ _tmp_159[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "genexp")); + D(fprintf(stderr, "%*c+ _tmp_160[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "genexp")); _res = genexp_var; goto done; } p->mark = _mark; - D(fprintf(stderr, "%*c%s _tmp_159[%d-%d]: %s failed!\n", p->level, ' ', + D(fprintf(stderr, "%*c%s _tmp_160[%d-%d]: %s failed!\n", p->level, ' ', p->error_indicator ? "ERROR!" : "-", _mark, p->mark, "genexp")); } { // 'True' @@ -34847,18 +34932,18 @@ _tmp_159_rule(Parser *p) p->level--; return NULL; } - D(fprintf(stderr, "%*c> _tmp_159[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "'True'")); + D(fprintf(stderr, "%*c> _tmp_160[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "'True'")); Token * _keyword; if ( - (_keyword = _PyPegen_expect_token(p, 610)) // token='True' + (_keyword = _PyPegen_expect_token(p, 613)) // token='True' ) { - D(fprintf(stderr, "%*c+ _tmp_159[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "'True'")); + D(fprintf(stderr, "%*c+ _tmp_160[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "'True'")); _res = _keyword; goto done; } p->mark = _mark; - D(fprintf(stderr, "%*c%s _tmp_159[%d-%d]: %s failed!\n", p->level, ' ', + D(fprintf(stderr, "%*c%s _tmp_160[%d-%d]: %s failed!\n", p->level, ' ', p->error_indicator ? "ERROR!" : "-", _mark, p->mark, "'True'")); } { // 'None' @@ -34866,18 +34951,18 @@ _tmp_159_rule(Parser *p) p->level--; return NULL; } - D(fprintf(stderr, "%*c> _tmp_159[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "'None'")); + D(fprintf(stderr, "%*c> _tmp_160[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "'None'")); Token * _keyword; if ( - (_keyword = _PyPegen_expect_token(p, 611)) // token='None' + (_keyword = _PyPegen_expect_token(p, 614)) // token='None' ) { - D(fprintf(stderr, "%*c+ _tmp_159[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "'None'")); + D(fprintf(stderr, "%*c+ _tmp_160[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "'None'")); _res = _keyword; goto done; } p->mark = _mark; - D(fprintf(stderr, "%*c%s _tmp_159[%d-%d]: %s failed!\n", p->level, ' ', + D(fprintf(stderr, "%*c%s _tmp_160[%d-%d]: %s failed!\n", p->level, ' ', p->error_indicator ? "ERROR!" : "-", _mark, p->mark, "'None'")); } { // 'False' @@ -34885,18 +34970,18 @@ _tmp_159_rule(Parser *p) p->level--; return NULL; } - D(fprintf(stderr, "%*c> _tmp_159[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "'False'")); + D(fprintf(stderr, "%*c> _tmp_160[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "'False'")); Token * _keyword; if ( - (_keyword = _PyPegen_expect_token(p, 612)) // token='False' + (_keyword = _PyPegen_expect_token(p, 615)) // token='False' ) { - D(fprintf(stderr, "%*c+ _tmp_159[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "'False'")); + D(fprintf(stderr, "%*c+ _tmp_160[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "'False'")); _res = _keyword; goto done; } p->mark = _mark; - D(fprintf(stderr, "%*c%s _tmp_159[%d-%d]: %s failed!\n", p->level, ' ', + D(fprintf(stderr, "%*c%s _tmp_160[%d-%d]: %s failed!\n", p->level, ' ', p->error_indicator ? "ERROR!" : "-", _mark, p->mark, "'False'")); } _res = NULL; @@ -34905,9 +34990,9 @@ _tmp_159_rule(Parser *p) return _res; } -// _tmp_160: '=' | ':=' +// _tmp_161: '=' | ':=' static void * -_tmp_160_rule(Parser *p) +_tmp_161_rule(Parser *p) { if (p->level++ == MAXSTACK) { _Pypegen_stack_overflow(p); @@ -34923,18 +35008,18 @@ _tmp_160_rule(Parser *p) p->level--; return NULL; } - D(fprintf(stderr, "%*c> _tmp_160[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "'='")); + D(fprintf(stderr, "%*c> _tmp_161[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "'='")); Token * _literal; if ( (_literal = _PyPegen_expect_token(p, 22)) // token='=' ) { - D(fprintf(stderr, "%*c+ _tmp_160[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "'='")); + D(fprintf(stderr, "%*c+ _tmp_161[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "'='")); _res = _literal; goto done; } p->mark = _mark; - D(fprintf(stderr, "%*c%s _tmp_160[%d-%d]: %s failed!\n", p->level, ' ', + D(fprintf(stderr, "%*c%s _tmp_161[%d-%d]: %s failed!\n", p->level, ' ', p->error_indicator ? "ERROR!" : "-", _mark, p->mark, "'='")); } { // ':=' @@ -34942,18 +35027,18 @@ _tmp_160_rule(Parser *p) p->level--; return NULL; } - D(fprintf(stderr, "%*c> _tmp_160[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "':='")); + D(fprintf(stderr, "%*c> _tmp_161[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "':='")); Token * _literal; if ( (_literal = _PyPegen_expect_token(p, 53)) // token=':=' ) { - D(fprintf(stderr, "%*c+ _tmp_160[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "':='")); + D(fprintf(stderr, "%*c+ _tmp_161[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "':='")); _res = _literal; goto done; } p->mark = _mark; - D(fprintf(stderr, "%*c%s _tmp_160[%d-%d]: %s failed!\n", p->level, ' ', + D(fprintf(stderr, "%*c%s _tmp_161[%d-%d]: %s failed!\n", p->level, ' ', p->error_indicator ? "ERROR!" : "-", _mark, p->mark, "':='")); } _res = NULL; @@ -34962,9 +35047,9 @@ _tmp_160_rule(Parser *p) return _res; } -// _loop0_161: star_named_expressions +// _loop0_162: star_named_expressions static asdl_seq * -_loop0_161_rule(Parser *p) +_loop0_162_rule(Parser *p) { if (p->level++ == MAXSTACK) { _Pypegen_stack_overflow(p); @@ -34989,7 +35074,7 @@ _loop0_161_rule(Parser *p) p->level--; return NULL; } - D(fprintf(stderr, "%*c> _loop0_161[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "star_named_expressions")); + D(fprintf(stderr, "%*c> _loop0_162[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "star_named_expressions")); asdl_expr_seq* star_named_expressions_var; while ( (star_named_expressions_var = star_named_expressions_rule(p)) // star_named_expressions @@ -35012,7 +35097,7 @@ _loop0_161_rule(Parser *p) _mark = p->mark; } p->mark = _mark; - D(fprintf(stderr, "%*c%s _loop0_161[%d-%d]: %s failed!\n", p->level, ' ', + D(fprintf(stderr, "%*c%s _loop0_162[%d-%d]: %s failed!\n", p->level, ' ', p->error_indicator ? "ERROR!" : "-", _mark, p->mark, "star_named_expressions")); } asdl_seq *_seq = (asdl_seq*)_Py_asdl_generic_seq_new(_n, p->arena); @@ -35029,9 +35114,9 @@ _loop0_161_rule(Parser *p) return _seq; } -// _loop0_162: (star_targets '=') +// _loop0_163: (star_targets '=') static asdl_seq * -_loop0_162_rule(Parser *p) +_loop0_163_rule(Parser *p) { if (p->level++ == MAXSTACK) { _Pypegen_stack_overflow(p); @@ -35056,13 +35141,13 @@ _loop0_162_rule(Parser *p) p->level--; return NULL; } - D(fprintf(stderr, "%*c> _loop0_162[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "(star_targets '=')")); - void *_tmp_264_var; + D(fprintf(stderr, "%*c> _loop0_163[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "(star_targets '=')")); + void *_tmp_266_var; while ( - (_tmp_264_var = _tmp_264_rule(p)) // star_targets '=' + (_tmp_266_var = _tmp_266_rule(p)) // star_targets '=' ) { - _res = _tmp_264_var; + _res = _tmp_266_var; if (_n == _children_capacity) { _children_capacity *= 2; void **_new_children = PyMem_Realloc(_children, _children_capacity*sizeof(void *)); @@ -35079,7 +35164,7 @@ _loop0_162_rule(Parser *p) _mark = p->mark; } p->mark = _mark; - D(fprintf(stderr, "%*c%s _loop0_162[%d-%d]: %s failed!\n", p->level, ' ', + D(fprintf(stderr, "%*c%s _loop0_163[%d-%d]: %s failed!\n", p->level, ' ', p->error_indicator ? "ERROR!" : "-", _mark, p->mark, "(star_targets '=')")); } asdl_seq *_seq = (asdl_seq*)_Py_asdl_generic_seq_new(_n, p->arena); @@ -35096,9 +35181,9 @@ _loop0_162_rule(Parser *p) return _seq; } -// _loop0_163: (star_targets '=') +// _loop0_164: (star_targets '=') static asdl_seq * -_loop0_163_rule(Parser *p) +_loop0_164_rule(Parser *p) { if (p->level++ == MAXSTACK) { _Pypegen_stack_overflow(p); @@ -35123,13 +35208,13 @@ _loop0_163_rule(Parser *p) p->level--; return NULL; } - D(fprintf(stderr, "%*c> _loop0_163[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "(star_targets '=')")); - void *_tmp_265_var; + D(fprintf(stderr, "%*c> _loop0_164[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "(star_targets '=')")); + void *_tmp_267_var; while ( - (_tmp_265_var = _tmp_265_rule(p)) // star_targets '=' + (_tmp_267_var = _tmp_267_rule(p)) // star_targets '=' ) { - _res = _tmp_265_var; + _res = _tmp_267_var; if (_n == _children_capacity) { _children_capacity *= 2; void **_new_children = PyMem_Realloc(_children, _children_capacity*sizeof(void *)); @@ -35146,7 +35231,7 @@ _loop0_163_rule(Parser *p) _mark = p->mark; } p->mark = _mark; - D(fprintf(stderr, "%*c%s _loop0_163[%d-%d]: %s failed!\n", p->level, ' ', + D(fprintf(stderr, "%*c%s _loop0_164[%d-%d]: %s failed!\n", p->level, ' ', p->error_indicator ? "ERROR!" : "-", _mark, p->mark, "(star_targets '=')")); } asdl_seq *_seq = (asdl_seq*)_Py_asdl_generic_seq_new(_n, p->arena); @@ -35163,9 +35248,9 @@ _loop0_163_rule(Parser *p) return _seq; } -// _tmp_164: yield_expr | star_expressions +// _tmp_165: yield_expr | star_expressions static void * -_tmp_164_rule(Parser *p) +_tmp_165_rule(Parser *p) { if (p->level++ == MAXSTACK) { _Pypegen_stack_overflow(p); @@ -35181,18 +35266,18 @@ _tmp_164_rule(Parser *p) p->level--; return NULL; } - D(fprintf(stderr, "%*c> _tmp_164[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "yield_expr")); + D(fprintf(stderr, "%*c> _tmp_165[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "yield_expr")); expr_ty yield_expr_var; if ( (yield_expr_var = yield_expr_rule(p)) // yield_expr ) { - D(fprintf(stderr, "%*c+ _tmp_164[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "yield_expr")); + D(fprintf(stderr, "%*c+ _tmp_165[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "yield_expr")); _res = yield_expr_var; goto done; } p->mark = _mark; - D(fprintf(stderr, "%*c%s _tmp_164[%d-%d]: %s failed!\n", p->level, ' ', + D(fprintf(stderr, "%*c%s _tmp_165[%d-%d]: %s failed!\n", p->level, ' ', p->error_indicator ? "ERROR!" : "-", _mark, p->mark, "yield_expr")); } { // star_expressions @@ -35200,18 +35285,18 @@ _tmp_164_rule(Parser *p) p->level--; return NULL; } - D(fprintf(stderr, "%*c> _tmp_164[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "star_expressions")); + D(fprintf(stderr, "%*c> _tmp_165[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "star_expressions")); expr_ty star_expressions_var; if ( (star_expressions_var = star_expressions_rule(p)) // star_expressions ) { - D(fprintf(stderr, "%*c+ _tmp_164[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "star_expressions")); + D(fprintf(stderr, "%*c+ _tmp_165[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "star_expressions")); _res = star_expressions_var; goto done; } p->mark = _mark; - D(fprintf(stderr, "%*c%s _tmp_164[%d-%d]: %s failed!\n", p->level, ' ', + D(fprintf(stderr, "%*c%s _tmp_165[%d-%d]: %s failed!\n", p->level, ' ', p->error_indicator ? "ERROR!" : "-", _mark, p->mark, "star_expressions")); } _res = NULL; @@ -35220,9 +35305,9 @@ _tmp_164_rule(Parser *p) return _res; } -// _tmp_165: '[' | '(' | '{' +// _tmp_166: '[' | '(' | '{' static void * -_tmp_165_rule(Parser *p) +_tmp_166_rule(Parser *p) { if (p->level++ == MAXSTACK) { _Pypegen_stack_overflow(p); @@ -35238,18 +35323,18 @@ _tmp_165_rule(Parser *p) p->level--; return NULL; } - D(fprintf(stderr, "%*c> _tmp_165[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "'['")); + D(fprintf(stderr, "%*c> _tmp_166[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "'['")); Token * _literal; if ( (_literal = _PyPegen_expect_token(p, 9)) // token='[' ) { - D(fprintf(stderr, "%*c+ _tmp_165[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "'['")); + D(fprintf(stderr, "%*c+ _tmp_166[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "'['")); _res = _literal; goto done; } p->mark = _mark; - D(fprintf(stderr, "%*c%s _tmp_165[%d-%d]: %s failed!\n", p->level, ' ', + D(fprintf(stderr, "%*c%s _tmp_166[%d-%d]: %s failed!\n", p->level, ' ', p->error_indicator ? "ERROR!" : "-", _mark, p->mark, "'['")); } { // '(' @@ -35257,18 +35342,18 @@ _tmp_165_rule(Parser *p) p->level--; return NULL; } - D(fprintf(stderr, "%*c> _tmp_165[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "'('")); + D(fprintf(stderr, "%*c> _tmp_166[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "'('")); Token * _literal; if ( (_literal = _PyPegen_expect_token(p, 7)) // token='(' ) { - D(fprintf(stderr, "%*c+ _tmp_165[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "'('")); + D(fprintf(stderr, "%*c+ _tmp_166[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "'('")); _res = _literal; goto done; } p->mark = _mark; - D(fprintf(stderr, "%*c%s _tmp_165[%d-%d]: %s failed!\n", p->level, ' ', + D(fprintf(stderr, "%*c%s _tmp_166[%d-%d]: %s failed!\n", p->level, ' ', p->error_indicator ? "ERROR!" : "-", _mark, p->mark, "'('")); } { // '{' @@ -35276,18 +35361,18 @@ _tmp_165_rule(Parser *p) p->level--; return NULL; } - D(fprintf(stderr, "%*c> _tmp_165[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "'{'")); + D(fprintf(stderr, "%*c> _tmp_166[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "'{'")); Token * _literal; if ( (_literal = _PyPegen_expect_token(p, 25)) // token='{' ) { - D(fprintf(stderr, "%*c+ _tmp_165[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "'{'")); + D(fprintf(stderr, "%*c+ _tmp_166[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "'{'")); _res = _literal; goto done; } p->mark = _mark; - D(fprintf(stderr, "%*c%s _tmp_165[%d-%d]: %s failed!\n", p->level, ' ', + D(fprintf(stderr, "%*c%s _tmp_166[%d-%d]: %s failed!\n", p->level, ' ', p->error_indicator ? "ERROR!" : "-", _mark, p->mark, "'{'")); } _res = NULL; @@ -35296,9 +35381,9 @@ _tmp_165_rule(Parser *p) return _res; } -// _tmp_166: '[' | '{' +// _tmp_167: '[' | '{' static void * -_tmp_166_rule(Parser *p) +_tmp_167_rule(Parser *p) { if (p->level++ == MAXSTACK) { _Pypegen_stack_overflow(p); @@ -35314,18 +35399,18 @@ _tmp_166_rule(Parser *p) p->level--; return NULL; } - D(fprintf(stderr, "%*c> _tmp_166[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "'['")); + D(fprintf(stderr, "%*c> _tmp_167[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "'['")); Token * _literal; if ( (_literal = _PyPegen_expect_token(p, 9)) // token='[' ) { - D(fprintf(stderr, "%*c+ _tmp_166[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "'['")); + D(fprintf(stderr, "%*c+ _tmp_167[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "'['")); _res = _literal; goto done; } p->mark = _mark; - D(fprintf(stderr, "%*c%s _tmp_166[%d-%d]: %s failed!\n", p->level, ' ', + D(fprintf(stderr, "%*c%s _tmp_167[%d-%d]: %s failed!\n", p->level, ' ', p->error_indicator ? "ERROR!" : "-", _mark, p->mark, "'['")); } { // '{' @@ -35333,18 +35418,18 @@ _tmp_166_rule(Parser *p) p->level--; return NULL; } - D(fprintf(stderr, "%*c> _tmp_166[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "'{'")); + D(fprintf(stderr, "%*c> _tmp_167[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "'{'")); Token * _literal; if ( (_literal = _PyPegen_expect_token(p, 25)) // token='{' ) { - D(fprintf(stderr, "%*c+ _tmp_166[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "'{'")); + D(fprintf(stderr, "%*c+ _tmp_167[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "'{'")); _res = _literal; goto done; } p->mark = _mark; - D(fprintf(stderr, "%*c%s _tmp_166[%d-%d]: %s failed!\n", p->level, ' ', + D(fprintf(stderr, "%*c%s _tmp_167[%d-%d]: %s failed!\n", p->level, ' ', p->error_indicator ? "ERROR!" : "-", _mark, p->mark, "'{'")); } _res = NULL; @@ -35353,9 +35438,9 @@ _tmp_166_rule(Parser *p) return _res; } -// _tmp_167: '[' | '{' +// _tmp_168: '[' | '{' static void * -_tmp_167_rule(Parser *p) +_tmp_168_rule(Parser *p) { if (p->level++ == MAXSTACK) { _Pypegen_stack_overflow(p); @@ -35371,18 +35456,18 @@ _tmp_167_rule(Parser *p) p->level--; return NULL; } - D(fprintf(stderr, "%*c> _tmp_167[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "'['")); + D(fprintf(stderr, "%*c> _tmp_168[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "'['")); Token * _literal; if ( (_literal = _PyPegen_expect_token(p, 9)) // token='[' ) { - D(fprintf(stderr, "%*c+ _tmp_167[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "'['")); + D(fprintf(stderr, "%*c+ _tmp_168[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "'['")); _res = _literal; goto done; } p->mark = _mark; - D(fprintf(stderr, "%*c%s _tmp_167[%d-%d]: %s failed!\n", p->level, ' ', + D(fprintf(stderr, "%*c%s _tmp_168[%d-%d]: %s failed!\n", p->level, ' ', p->error_indicator ? "ERROR!" : "-", _mark, p->mark, "'['")); } { // '{' @@ -35390,18 +35475,18 @@ _tmp_167_rule(Parser *p) p->level--; return NULL; } - D(fprintf(stderr, "%*c> _tmp_167[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "'{'")); + D(fprintf(stderr, "%*c> _tmp_168[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "'{'")); Token * _literal; if ( (_literal = _PyPegen_expect_token(p, 25)) // token='{' ) { - D(fprintf(stderr, "%*c+ _tmp_167[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "'{'")); + D(fprintf(stderr, "%*c+ _tmp_168[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "'{'")); _res = _literal; goto done; } p->mark = _mark; - D(fprintf(stderr, "%*c%s _tmp_167[%d-%d]: %s failed!\n", p->level, ' ', + D(fprintf(stderr, "%*c%s _tmp_168[%d-%d]: %s failed!\n", p->level, ' ', p->error_indicator ? "ERROR!" : "-", _mark, p->mark, "'{'")); } _res = NULL; @@ -35410,9 +35495,9 @@ _tmp_167_rule(Parser *p) return _res; } -// _tmp_168: slash_no_default | slash_with_default +// _tmp_169: slash_no_default | slash_with_default static void * -_tmp_168_rule(Parser *p) +_tmp_169_rule(Parser *p) { if (p->level++ == MAXSTACK) { _Pypegen_stack_overflow(p); @@ -35428,18 +35513,18 @@ _tmp_168_rule(Parser *p) p->level--; return NULL; } - D(fprintf(stderr, "%*c> _tmp_168[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "slash_no_default")); + D(fprintf(stderr, "%*c> _tmp_169[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "slash_no_default")); asdl_arg_seq* slash_no_default_var; if ( (slash_no_default_var = slash_no_default_rule(p)) // slash_no_default ) { - D(fprintf(stderr, "%*c+ _tmp_168[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "slash_no_default")); + D(fprintf(stderr, "%*c+ _tmp_169[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "slash_no_default")); _res = slash_no_default_var; goto done; } p->mark = _mark; - D(fprintf(stderr, "%*c%s _tmp_168[%d-%d]: %s failed!\n", p->level, ' ', + D(fprintf(stderr, "%*c%s _tmp_169[%d-%d]: %s failed!\n", p->level, ' ', p->error_indicator ? "ERROR!" : "-", _mark, p->mark, "slash_no_default")); } { // slash_with_default @@ -35447,18 +35532,18 @@ _tmp_168_rule(Parser *p) p->level--; return NULL; } - D(fprintf(stderr, "%*c> _tmp_168[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "slash_with_default")); + D(fprintf(stderr, "%*c> _tmp_169[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "slash_with_default")); SlashWithDefault* slash_with_default_var; if ( (slash_with_default_var = slash_with_default_rule(p)) // slash_with_default ) { - D(fprintf(stderr, "%*c+ _tmp_168[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "slash_with_default")); + D(fprintf(stderr, "%*c+ _tmp_169[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "slash_with_default")); _res = slash_with_default_var; goto done; } p->mark = _mark; - D(fprintf(stderr, "%*c%s _tmp_168[%d-%d]: %s failed!\n", p->level, ' ', + D(fprintf(stderr, "%*c%s _tmp_169[%d-%d]: %s failed!\n", p->level, ' ', p->error_indicator ? "ERROR!" : "-", _mark, p->mark, "slash_with_default")); } _res = NULL; @@ -35467,9 +35552,9 @@ _tmp_168_rule(Parser *p) return _res; } -// _loop0_169: param_maybe_default +// _loop0_170: param_maybe_default static asdl_seq * -_loop0_169_rule(Parser *p) +_loop0_170_rule(Parser *p) { if (p->level++ == MAXSTACK) { _Pypegen_stack_overflow(p); @@ -35494,7 +35579,7 @@ _loop0_169_rule(Parser *p) p->level--; return NULL; } - D(fprintf(stderr, "%*c> _loop0_169[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "param_maybe_default")); + D(fprintf(stderr, "%*c> _loop0_170[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "param_maybe_default")); NameDefaultPair* param_maybe_default_var; while ( (param_maybe_default_var = param_maybe_default_rule(p)) // param_maybe_default @@ -35517,7 +35602,7 @@ _loop0_169_rule(Parser *p) _mark = p->mark; } p->mark = _mark; - D(fprintf(stderr, "%*c%s _loop0_169[%d-%d]: %s failed!\n", p->level, ' ', + D(fprintf(stderr, "%*c%s _loop0_170[%d-%d]: %s failed!\n", p->level, ' ', p->error_indicator ? "ERROR!" : "-", _mark, p->mark, "param_maybe_default")); } asdl_seq *_seq = (asdl_seq*)_Py_asdl_generic_seq_new(_n, p->arena); @@ -35534,9 +35619,9 @@ _loop0_169_rule(Parser *p) return _seq; } -// _loop0_170: param_no_default +// _loop0_171: param_no_default static asdl_seq * -_loop0_170_rule(Parser *p) +_loop0_171_rule(Parser *p) { if (p->level++ == MAXSTACK) { _Pypegen_stack_overflow(p); @@ -35561,7 +35646,7 @@ _loop0_170_rule(Parser *p) p->level--; return NULL; } - D(fprintf(stderr, "%*c> _loop0_170[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "param_no_default")); + D(fprintf(stderr, "%*c> _loop0_171[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "param_no_default")); arg_ty param_no_default_var; while ( (param_no_default_var = param_no_default_rule(p)) // param_no_default @@ -35584,7 +35669,7 @@ _loop0_170_rule(Parser *p) _mark = p->mark; } p->mark = _mark; - D(fprintf(stderr, "%*c%s _loop0_170[%d-%d]: %s failed!\n", p->level, ' ', + D(fprintf(stderr, "%*c%s _loop0_171[%d-%d]: %s failed!\n", p->level, ' ', p->error_indicator ? "ERROR!" : "-", _mark, p->mark, "param_no_default")); } asdl_seq *_seq = (asdl_seq*)_Py_asdl_generic_seq_new(_n, p->arena); @@ -35601,9 +35686,9 @@ _loop0_170_rule(Parser *p) return _seq; } -// _loop0_171: param_no_default +// _loop0_172: param_no_default static asdl_seq * -_loop0_171_rule(Parser *p) +_loop0_172_rule(Parser *p) { if (p->level++ == MAXSTACK) { _Pypegen_stack_overflow(p); @@ -35628,7 +35713,7 @@ _loop0_171_rule(Parser *p) p->level--; return NULL; } - D(fprintf(stderr, "%*c> _loop0_171[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "param_no_default")); + D(fprintf(stderr, "%*c> _loop0_172[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "param_no_default")); arg_ty param_no_default_var; while ( (param_no_default_var = param_no_default_rule(p)) // param_no_default @@ -35651,7 +35736,7 @@ _loop0_171_rule(Parser *p) _mark = p->mark; } p->mark = _mark; - D(fprintf(stderr, "%*c%s _loop0_171[%d-%d]: %s failed!\n", p->level, ' ', + D(fprintf(stderr, "%*c%s _loop0_172[%d-%d]: %s failed!\n", p->level, ' ', p->error_indicator ? "ERROR!" : "-", _mark, p->mark, "param_no_default")); } asdl_seq *_seq = (asdl_seq*)_Py_asdl_generic_seq_new(_n, p->arena); @@ -35668,9 +35753,9 @@ _loop0_171_rule(Parser *p) return _seq; } -// _loop1_172: param_no_default +// _loop1_173: param_no_default static asdl_seq * -_loop1_172_rule(Parser *p) +_loop1_173_rule(Parser *p) { if (p->level++ == MAXSTACK) { _Pypegen_stack_overflow(p); @@ -35695,7 +35780,7 @@ _loop1_172_rule(Parser *p) p->level--; return NULL; } - D(fprintf(stderr, "%*c> _loop1_172[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "param_no_default")); + D(fprintf(stderr, "%*c> _loop1_173[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "param_no_default")); arg_ty param_no_default_var; while ( (param_no_default_var = param_no_default_rule(p)) // param_no_default @@ -35718,7 +35803,7 @@ _loop1_172_rule(Parser *p) _mark = p->mark; } p->mark = _mark; - D(fprintf(stderr, "%*c%s _loop1_172[%d-%d]: %s failed!\n", p->level, ' ', + D(fprintf(stderr, "%*c%s _loop1_173[%d-%d]: %s failed!\n", p->level, ' ', p->error_indicator ? "ERROR!" : "-", _mark, p->mark, "param_no_default")); } if (_n == 0 || p->error_indicator) { @@ -35740,9 +35825,9 @@ _loop1_172_rule(Parser *p) return _seq; } -// _tmp_173: slash_no_default | slash_with_default +// _tmp_174: slash_no_default | slash_with_default static void * -_tmp_173_rule(Parser *p) +_tmp_174_rule(Parser *p) { if (p->level++ == MAXSTACK) { _Pypegen_stack_overflow(p); @@ -35758,18 +35843,18 @@ _tmp_173_rule(Parser *p) p->level--; return NULL; } - D(fprintf(stderr, "%*c> _tmp_173[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "slash_no_default")); + D(fprintf(stderr, "%*c> _tmp_174[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "slash_no_default")); asdl_arg_seq* slash_no_default_var; if ( (slash_no_default_var = slash_no_default_rule(p)) // slash_no_default ) { - D(fprintf(stderr, "%*c+ _tmp_173[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "slash_no_default")); + D(fprintf(stderr, "%*c+ _tmp_174[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "slash_no_default")); _res = slash_no_default_var; goto done; } p->mark = _mark; - D(fprintf(stderr, "%*c%s _tmp_173[%d-%d]: %s failed!\n", p->level, ' ', + D(fprintf(stderr, "%*c%s _tmp_174[%d-%d]: %s failed!\n", p->level, ' ', p->error_indicator ? "ERROR!" : "-", _mark, p->mark, "slash_no_default")); } { // slash_with_default @@ -35777,18 +35862,18 @@ _tmp_173_rule(Parser *p) p->level--; return NULL; } - D(fprintf(stderr, "%*c> _tmp_173[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "slash_with_default")); + D(fprintf(stderr, "%*c> _tmp_174[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "slash_with_default")); SlashWithDefault* slash_with_default_var; if ( (slash_with_default_var = slash_with_default_rule(p)) // slash_with_default ) { - D(fprintf(stderr, "%*c+ _tmp_173[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "slash_with_default")); + D(fprintf(stderr, "%*c+ _tmp_174[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "slash_with_default")); _res = slash_with_default_var; goto done; } p->mark = _mark; - D(fprintf(stderr, "%*c%s _tmp_173[%d-%d]: %s failed!\n", p->level, ' ', + D(fprintf(stderr, "%*c%s _tmp_174[%d-%d]: %s failed!\n", p->level, ' ', p->error_indicator ? "ERROR!" : "-", _mark, p->mark, "slash_with_default")); } _res = NULL; @@ -35797,9 +35882,9 @@ _tmp_173_rule(Parser *p) return _res; } -// _loop0_174: param_maybe_default +// _loop0_175: param_maybe_default static asdl_seq * -_loop0_174_rule(Parser *p) +_loop0_175_rule(Parser *p) { if (p->level++ == MAXSTACK) { _Pypegen_stack_overflow(p); @@ -35824,7 +35909,7 @@ _loop0_174_rule(Parser *p) p->level--; return NULL; } - D(fprintf(stderr, "%*c> _loop0_174[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "param_maybe_default")); + D(fprintf(stderr, "%*c> _loop0_175[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "param_maybe_default")); NameDefaultPair* param_maybe_default_var; while ( (param_maybe_default_var = param_maybe_default_rule(p)) // param_maybe_default @@ -35847,7 +35932,7 @@ _loop0_174_rule(Parser *p) _mark = p->mark; } p->mark = _mark; - D(fprintf(stderr, "%*c%s _loop0_174[%d-%d]: %s failed!\n", p->level, ' ', + D(fprintf(stderr, "%*c%s _loop0_175[%d-%d]: %s failed!\n", p->level, ' ', p->error_indicator ? "ERROR!" : "-", _mark, p->mark, "param_maybe_default")); } asdl_seq *_seq = (asdl_seq*)_Py_asdl_generic_seq_new(_n, p->arena); @@ -35864,9 +35949,9 @@ _loop0_174_rule(Parser *p) return _seq; } -// _tmp_175: ',' | param_no_default +// _tmp_176: ',' | param_no_default static void * -_tmp_175_rule(Parser *p) +_tmp_176_rule(Parser *p) { if (p->level++ == MAXSTACK) { _Pypegen_stack_overflow(p); @@ -35882,18 +35967,18 @@ _tmp_175_rule(Parser *p) p->level--; return NULL; } - D(fprintf(stderr, "%*c> _tmp_175[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "','")); + D(fprintf(stderr, "%*c> _tmp_176[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "','")); Token * _literal; if ( (_literal = _PyPegen_expect_token(p, 12)) // token=',' ) { - D(fprintf(stderr, "%*c+ _tmp_175[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "','")); + D(fprintf(stderr, "%*c+ _tmp_176[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "','")); _res = _literal; goto done; } p->mark = _mark; - D(fprintf(stderr, "%*c%s _tmp_175[%d-%d]: %s failed!\n", p->level, ' ', + D(fprintf(stderr, "%*c%s _tmp_176[%d-%d]: %s failed!\n", p->level, ' ', p->error_indicator ? "ERROR!" : "-", _mark, p->mark, "','")); } { // param_no_default @@ -35901,18 +35986,18 @@ _tmp_175_rule(Parser *p) p->level--; return NULL; } - D(fprintf(stderr, "%*c> _tmp_175[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "param_no_default")); + D(fprintf(stderr, "%*c> _tmp_176[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "param_no_default")); arg_ty param_no_default_var; if ( (param_no_default_var = param_no_default_rule(p)) // param_no_default ) { - D(fprintf(stderr, "%*c+ _tmp_175[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "param_no_default")); + D(fprintf(stderr, "%*c+ _tmp_176[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "param_no_default")); _res = param_no_default_var; goto done; } p->mark = _mark; - D(fprintf(stderr, "%*c%s _tmp_175[%d-%d]: %s failed!\n", p->level, ' ', + D(fprintf(stderr, "%*c%s _tmp_176[%d-%d]: %s failed!\n", p->level, ' ', p->error_indicator ? "ERROR!" : "-", _mark, p->mark, "param_no_default")); } _res = NULL; @@ -35921,9 +36006,9 @@ _tmp_175_rule(Parser *p) return _res; } -// _loop0_176: param_maybe_default +// _loop0_177: param_maybe_default static asdl_seq * -_loop0_176_rule(Parser *p) +_loop0_177_rule(Parser *p) { if (p->level++ == MAXSTACK) { _Pypegen_stack_overflow(p); @@ -35948,7 +36033,7 @@ _loop0_176_rule(Parser *p) p->level--; return NULL; } - D(fprintf(stderr, "%*c> _loop0_176[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "param_maybe_default")); + D(fprintf(stderr, "%*c> _loop0_177[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "param_maybe_default")); NameDefaultPair* param_maybe_default_var; while ( (param_maybe_default_var = param_maybe_default_rule(p)) // param_maybe_default @@ -35971,7 +36056,7 @@ _loop0_176_rule(Parser *p) _mark = p->mark; } p->mark = _mark; - D(fprintf(stderr, "%*c%s _loop0_176[%d-%d]: %s failed!\n", p->level, ' ', + D(fprintf(stderr, "%*c%s _loop0_177[%d-%d]: %s failed!\n", p->level, ' ', p->error_indicator ? "ERROR!" : "-", _mark, p->mark, "param_maybe_default")); } asdl_seq *_seq = (asdl_seq*)_Py_asdl_generic_seq_new(_n, p->arena); @@ -35988,9 +36073,9 @@ _loop0_176_rule(Parser *p) return _seq; } -// _loop1_177: param_maybe_default +// _loop1_178: param_maybe_default static asdl_seq * -_loop1_177_rule(Parser *p) +_loop1_178_rule(Parser *p) { if (p->level++ == MAXSTACK) { _Pypegen_stack_overflow(p); @@ -36015,7 +36100,7 @@ _loop1_177_rule(Parser *p) p->level--; return NULL; } - D(fprintf(stderr, "%*c> _loop1_177[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "param_maybe_default")); + D(fprintf(stderr, "%*c> _loop1_178[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "param_maybe_default")); NameDefaultPair* param_maybe_default_var; while ( (param_maybe_default_var = param_maybe_default_rule(p)) // param_maybe_default @@ -36038,7 +36123,7 @@ _loop1_177_rule(Parser *p) _mark = p->mark; } p->mark = _mark; - D(fprintf(stderr, "%*c%s _loop1_177[%d-%d]: %s failed!\n", p->level, ' ', + D(fprintf(stderr, "%*c%s _loop1_178[%d-%d]: %s failed!\n", p->level, ' ', p->error_indicator ? "ERROR!" : "-", _mark, p->mark, "param_maybe_default")); } if (_n == 0 || p->error_indicator) { @@ -36060,9 +36145,9 @@ _loop1_177_rule(Parser *p) return _seq; } -// _tmp_178: ')' | ',' +// _tmp_179: ')' | ',' static void * -_tmp_178_rule(Parser *p) +_tmp_179_rule(Parser *p) { if (p->level++ == MAXSTACK) { _Pypegen_stack_overflow(p); @@ -36078,18 +36163,18 @@ _tmp_178_rule(Parser *p) p->level--; return NULL; } - D(fprintf(stderr, "%*c> _tmp_178[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "')'")); + D(fprintf(stderr, "%*c> _tmp_179[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "')'")); Token * _literal; if ( (_literal = _PyPegen_expect_token(p, 8)) // token=')' ) { - D(fprintf(stderr, "%*c+ _tmp_178[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "')'")); + D(fprintf(stderr, "%*c+ _tmp_179[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "')'")); _res = _literal; goto done; } p->mark = _mark; - D(fprintf(stderr, "%*c%s _tmp_178[%d-%d]: %s failed!\n", p->level, ' ', + D(fprintf(stderr, "%*c%s _tmp_179[%d-%d]: %s failed!\n", p->level, ' ', p->error_indicator ? "ERROR!" : "-", _mark, p->mark, "')'")); } { // ',' @@ -36097,18 +36182,18 @@ _tmp_178_rule(Parser *p) p->level--; return NULL; } - D(fprintf(stderr, "%*c> _tmp_178[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "','")); + D(fprintf(stderr, "%*c> _tmp_179[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "','")); Token * _literal; if ( (_literal = _PyPegen_expect_token(p, 12)) // token=',' ) { - D(fprintf(stderr, "%*c+ _tmp_178[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "','")); + D(fprintf(stderr, "%*c+ _tmp_179[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "','")); _res = _literal; goto done; } p->mark = _mark; - D(fprintf(stderr, "%*c%s _tmp_178[%d-%d]: %s failed!\n", p->level, ' ', + D(fprintf(stderr, "%*c%s _tmp_179[%d-%d]: %s failed!\n", p->level, ' ', p->error_indicator ? "ERROR!" : "-", _mark, p->mark, "','")); } _res = NULL; @@ -36117,9 +36202,9 @@ _tmp_178_rule(Parser *p) return _res; } -// _tmp_179: ')' | ',' (')' | '**') +// _tmp_180: ')' | ',' (')' | '**') static void * -_tmp_179_rule(Parser *p) +_tmp_180_rule(Parser *p) { if (p->level++ == MAXSTACK) { _Pypegen_stack_overflow(p); @@ -36135,18 +36220,18 @@ _tmp_179_rule(Parser *p) p->level--; return NULL; } - D(fprintf(stderr, "%*c> _tmp_179[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "')'")); + D(fprintf(stderr, "%*c> _tmp_180[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "')'")); Token * _literal; if ( (_literal = _PyPegen_expect_token(p, 8)) // token=')' ) { - D(fprintf(stderr, "%*c+ _tmp_179[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "')'")); + D(fprintf(stderr, "%*c+ _tmp_180[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "')'")); _res = _literal; goto done; } p->mark = _mark; - D(fprintf(stderr, "%*c%s _tmp_179[%d-%d]: %s failed!\n", p->level, ' ', + D(fprintf(stderr, "%*c%s _tmp_180[%d-%d]: %s failed!\n", p->level, ' ', p->error_indicator ? "ERROR!" : "-", _mark, p->mark, "')'")); } { // ',' (')' | '**') @@ -36154,21 +36239,21 @@ _tmp_179_rule(Parser *p) p->level--; return NULL; } - D(fprintf(stderr, "%*c> _tmp_179[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "',' (')' | '**')")); + D(fprintf(stderr, "%*c> _tmp_180[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "',' (')' | '**')")); Token * _literal; - void *_tmp_266_var; + void *_tmp_268_var; if ( (_literal = _PyPegen_expect_token(p, 12)) // token=',' && - (_tmp_266_var = _tmp_266_rule(p)) // ')' | '**' + (_tmp_268_var = _tmp_268_rule(p)) // ')' | '**' ) { - D(fprintf(stderr, "%*c+ _tmp_179[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "',' (')' | '**')")); - _res = _PyPegen_dummy_name(p, _literal, _tmp_266_var); + D(fprintf(stderr, "%*c+ _tmp_180[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "',' (')' | '**')")); + _res = _PyPegen_dummy_name(p, _literal, _tmp_268_var); goto done; } p->mark = _mark; - D(fprintf(stderr, "%*c%s _tmp_179[%d-%d]: %s failed!\n", p->level, ' ', + D(fprintf(stderr, "%*c%s _tmp_180[%d-%d]: %s failed!\n", p->level, ' ', p->error_indicator ? "ERROR!" : "-", _mark, p->mark, "',' (')' | '**')")); } _res = NULL; @@ -36177,9 +36262,9 @@ _tmp_179_rule(Parser *p) return _res; } -// _tmp_180: param_no_default | ',' +// _tmp_181: param_no_default | ',' static void * -_tmp_180_rule(Parser *p) +_tmp_181_rule(Parser *p) { if (p->level++ == MAXSTACK) { _Pypegen_stack_overflow(p); @@ -36195,18 +36280,18 @@ _tmp_180_rule(Parser *p) p->level--; return NULL; } - D(fprintf(stderr, "%*c> _tmp_180[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "param_no_default")); + D(fprintf(stderr, "%*c> _tmp_181[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "param_no_default")); arg_ty param_no_default_var; if ( (param_no_default_var = param_no_default_rule(p)) // param_no_default ) { - D(fprintf(stderr, "%*c+ _tmp_180[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "param_no_default")); + D(fprintf(stderr, "%*c+ _tmp_181[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "param_no_default")); _res = param_no_default_var; goto done; } p->mark = _mark; - D(fprintf(stderr, "%*c%s _tmp_180[%d-%d]: %s failed!\n", p->level, ' ', + D(fprintf(stderr, "%*c%s _tmp_181[%d-%d]: %s failed!\n", p->level, ' ', p->error_indicator ? "ERROR!" : "-", _mark, p->mark, "param_no_default")); } { // ',' @@ -36214,18 +36299,18 @@ _tmp_180_rule(Parser *p) p->level--; return NULL; } - D(fprintf(stderr, "%*c> _tmp_180[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "','")); + D(fprintf(stderr, "%*c> _tmp_181[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "','")); Token * _literal; if ( (_literal = _PyPegen_expect_token(p, 12)) // token=',' ) { - D(fprintf(stderr, "%*c+ _tmp_180[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "','")); + D(fprintf(stderr, "%*c+ _tmp_181[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "','")); _res = _literal; goto done; } p->mark = _mark; - D(fprintf(stderr, "%*c%s _tmp_180[%d-%d]: %s failed!\n", p->level, ' ', + D(fprintf(stderr, "%*c%s _tmp_181[%d-%d]: %s failed!\n", p->level, ' ', p->error_indicator ? "ERROR!" : "-", _mark, p->mark, "','")); } _res = NULL; @@ -36234,9 +36319,9 @@ _tmp_180_rule(Parser *p) return _res; } -// _loop0_181: param_maybe_default +// _loop0_182: param_maybe_default static asdl_seq * -_loop0_181_rule(Parser *p) +_loop0_182_rule(Parser *p) { if (p->level++ == MAXSTACK) { _Pypegen_stack_overflow(p); @@ -36261,7 +36346,7 @@ _loop0_181_rule(Parser *p) p->level--; return NULL; } - D(fprintf(stderr, "%*c> _loop0_181[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "param_maybe_default")); + D(fprintf(stderr, "%*c> _loop0_182[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "param_maybe_default")); NameDefaultPair* param_maybe_default_var; while ( (param_maybe_default_var = param_maybe_default_rule(p)) // param_maybe_default @@ -36284,7 +36369,7 @@ _loop0_181_rule(Parser *p) _mark = p->mark; } p->mark = _mark; - D(fprintf(stderr, "%*c%s _loop0_181[%d-%d]: %s failed!\n", p->level, ' ', + D(fprintf(stderr, "%*c%s _loop0_182[%d-%d]: %s failed!\n", p->level, ' ', p->error_indicator ? "ERROR!" : "-", _mark, p->mark, "param_maybe_default")); } asdl_seq *_seq = (asdl_seq*)_Py_asdl_generic_seq_new(_n, p->arena); @@ -36301,9 +36386,9 @@ _loop0_181_rule(Parser *p) return _seq; } -// _tmp_182: param_no_default | ',' +// _tmp_183: param_no_default | ',' static void * -_tmp_182_rule(Parser *p) +_tmp_183_rule(Parser *p) { if (p->level++ == MAXSTACK) { _Pypegen_stack_overflow(p); @@ -36319,18 +36404,18 @@ _tmp_182_rule(Parser *p) p->level--; return NULL; } - D(fprintf(stderr, "%*c> _tmp_182[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "param_no_default")); + D(fprintf(stderr, "%*c> _tmp_183[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "param_no_default")); arg_ty param_no_default_var; if ( (param_no_default_var = param_no_default_rule(p)) // param_no_default ) { - D(fprintf(stderr, "%*c+ _tmp_182[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "param_no_default")); + D(fprintf(stderr, "%*c+ _tmp_183[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "param_no_default")); _res = param_no_default_var; goto done; } p->mark = _mark; - D(fprintf(stderr, "%*c%s _tmp_182[%d-%d]: %s failed!\n", p->level, ' ', + D(fprintf(stderr, "%*c%s _tmp_183[%d-%d]: %s failed!\n", p->level, ' ', p->error_indicator ? "ERROR!" : "-", _mark, p->mark, "param_no_default")); } { // ',' @@ -36338,18 +36423,18 @@ _tmp_182_rule(Parser *p) p->level--; return NULL; } - D(fprintf(stderr, "%*c> _tmp_182[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "','")); + D(fprintf(stderr, "%*c> _tmp_183[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "','")); Token * _literal; if ( (_literal = _PyPegen_expect_token(p, 12)) // token=',' ) { - D(fprintf(stderr, "%*c+ _tmp_182[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "','")); + D(fprintf(stderr, "%*c+ _tmp_183[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "','")); _res = _literal; goto done; } p->mark = _mark; - D(fprintf(stderr, "%*c%s _tmp_182[%d-%d]: %s failed!\n", p->level, ' ', + D(fprintf(stderr, "%*c%s _tmp_183[%d-%d]: %s failed!\n", p->level, ' ', p->error_indicator ? "ERROR!" : "-", _mark, p->mark, "','")); } _res = NULL; @@ -36358,9 +36443,9 @@ _tmp_182_rule(Parser *p) return _res; } -// _tmp_183: '*' | '**' | '/' +// _tmp_184: '*' | '**' | '/' static void * -_tmp_183_rule(Parser *p) +_tmp_184_rule(Parser *p) { if (p->level++ == MAXSTACK) { _Pypegen_stack_overflow(p); @@ -36376,18 +36461,18 @@ _tmp_183_rule(Parser *p) p->level--; return NULL; } - D(fprintf(stderr, "%*c> _tmp_183[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "'*'")); + D(fprintf(stderr, "%*c> _tmp_184[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "'*'")); Token * _literal; if ( (_literal = _PyPegen_expect_token(p, 16)) // token='*' ) { - D(fprintf(stderr, "%*c+ _tmp_183[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "'*'")); + D(fprintf(stderr, "%*c+ _tmp_184[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "'*'")); _res = _literal; goto done; } p->mark = _mark; - D(fprintf(stderr, "%*c%s _tmp_183[%d-%d]: %s failed!\n", p->level, ' ', + D(fprintf(stderr, "%*c%s _tmp_184[%d-%d]: %s failed!\n", p->level, ' ', p->error_indicator ? "ERROR!" : "-", _mark, p->mark, "'*'")); } { // '**' @@ -36395,18 +36480,18 @@ _tmp_183_rule(Parser *p) p->level--; return NULL; } - D(fprintf(stderr, "%*c> _tmp_183[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "'**'")); + D(fprintf(stderr, "%*c> _tmp_184[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "'**'")); Token * _literal; if ( (_literal = _PyPegen_expect_token(p, 35)) // token='**' ) { - D(fprintf(stderr, "%*c+ _tmp_183[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "'**'")); + D(fprintf(stderr, "%*c+ _tmp_184[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "'**'")); _res = _literal; goto done; } p->mark = _mark; - D(fprintf(stderr, "%*c%s _tmp_183[%d-%d]: %s failed!\n", p->level, ' ', + D(fprintf(stderr, "%*c%s _tmp_184[%d-%d]: %s failed!\n", p->level, ' ', p->error_indicator ? "ERROR!" : "-", _mark, p->mark, "'**'")); } { // '/' @@ -36414,18 +36499,18 @@ _tmp_183_rule(Parser *p) p->level--; return NULL; } - D(fprintf(stderr, "%*c> _tmp_183[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "'/'")); + D(fprintf(stderr, "%*c> _tmp_184[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "'/'")); Token * _literal; if ( (_literal = _PyPegen_expect_token(p, 17)) // token='/' ) { - D(fprintf(stderr, "%*c+ _tmp_183[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "'/'")); + D(fprintf(stderr, "%*c+ _tmp_184[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "'/'")); _res = _literal; goto done; } p->mark = _mark; - D(fprintf(stderr, "%*c%s _tmp_183[%d-%d]: %s failed!\n", p->level, ' ', + D(fprintf(stderr, "%*c%s _tmp_184[%d-%d]: %s failed!\n", p->level, ' ', p->error_indicator ? "ERROR!" : "-", _mark, p->mark, "'/'")); } _res = NULL; @@ -36434,9 +36519,9 @@ _tmp_183_rule(Parser *p) return _res; } -// _loop1_184: param_with_default +// _loop1_185: param_with_default static asdl_seq * -_loop1_184_rule(Parser *p) +_loop1_185_rule(Parser *p) { if (p->level++ == MAXSTACK) { _Pypegen_stack_overflow(p); @@ -36461,7 +36546,7 @@ _loop1_184_rule(Parser *p) p->level--; return NULL; } - D(fprintf(stderr, "%*c> _loop1_184[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "param_with_default")); + D(fprintf(stderr, "%*c> _loop1_185[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "param_with_default")); NameDefaultPair* param_with_default_var; while ( (param_with_default_var = param_with_default_rule(p)) // param_with_default @@ -36484,7 +36569,7 @@ _loop1_184_rule(Parser *p) _mark = p->mark; } p->mark = _mark; - D(fprintf(stderr, "%*c%s _loop1_184[%d-%d]: %s failed!\n", p->level, ' ', + D(fprintf(stderr, "%*c%s _loop1_185[%d-%d]: %s failed!\n", p->level, ' ', p->error_indicator ? "ERROR!" : "-", _mark, p->mark, "param_with_default")); } if (_n == 0 || p->error_indicator) { @@ -36506,9 +36591,9 @@ _loop1_184_rule(Parser *p) return _seq; } -// _tmp_185: lambda_slash_no_default | lambda_slash_with_default +// _tmp_186: lambda_slash_no_default | lambda_slash_with_default static void * -_tmp_185_rule(Parser *p) +_tmp_186_rule(Parser *p) { if (p->level++ == MAXSTACK) { _Pypegen_stack_overflow(p); @@ -36524,18 +36609,18 @@ _tmp_185_rule(Parser *p) p->level--; return NULL; } - D(fprintf(stderr, "%*c> _tmp_185[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "lambda_slash_no_default")); + D(fprintf(stderr, "%*c> _tmp_186[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "lambda_slash_no_default")); asdl_arg_seq* lambda_slash_no_default_var; if ( (lambda_slash_no_default_var = lambda_slash_no_default_rule(p)) // lambda_slash_no_default ) { - D(fprintf(stderr, "%*c+ _tmp_185[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "lambda_slash_no_default")); + D(fprintf(stderr, "%*c+ _tmp_186[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "lambda_slash_no_default")); _res = lambda_slash_no_default_var; goto done; } p->mark = _mark; - D(fprintf(stderr, "%*c%s _tmp_185[%d-%d]: %s failed!\n", p->level, ' ', + D(fprintf(stderr, "%*c%s _tmp_186[%d-%d]: %s failed!\n", p->level, ' ', p->error_indicator ? "ERROR!" : "-", _mark, p->mark, "lambda_slash_no_default")); } { // lambda_slash_with_default @@ -36543,18 +36628,18 @@ _tmp_185_rule(Parser *p) p->level--; return NULL; } - D(fprintf(stderr, "%*c> _tmp_185[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "lambda_slash_with_default")); + D(fprintf(stderr, "%*c> _tmp_186[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "lambda_slash_with_default")); SlashWithDefault* lambda_slash_with_default_var; if ( (lambda_slash_with_default_var = lambda_slash_with_default_rule(p)) // lambda_slash_with_default ) { - D(fprintf(stderr, "%*c+ _tmp_185[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "lambda_slash_with_default")); + D(fprintf(stderr, "%*c+ _tmp_186[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "lambda_slash_with_default")); _res = lambda_slash_with_default_var; goto done; } p->mark = _mark; - D(fprintf(stderr, "%*c%s _tmp_185[%d-%d]: %s failed!\n", p->level, ' ', + D(fprintf(stderr, "%*c%s _tmp_186[%d-%d]: %s failed!\n", p->level, ' ', p->error_indicator ? "ERROR!" : "-", _mark, p->mark, "lambda_slash_with_default")); } _res = NULL; @@ -36563,9 +36648,9 @@ _tmp_185_rule(Parser *p) return _res; } -// _loop0_186: lambda_param_maybe_default +// _loop0_187: lambda_param_maybe_default static asdl_seq * -_loop0_186_rule(Parser *p) +_loop0_187_rule(Parser *p) { if (p->level++ == MAXSTACK) { _Pypegen_stack_overflow(p); @@ -36590,7 +36675,7 @@ _loop0_186_rule(Parser *p) p->level--; return NULL; } - D(fprintf(stderr, "%*c> _loop0_186[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "lambda_param_maybe_default")); + D(fprintf(stderr, "%*c> _loop0_187[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "lambda_param_maybe_default")); NameDefaultPair* lambda_param_maybe_default_var; while ( (lambda_param_maybe_default_var = lambda_param_maybe_default_rule(p)) // lambda_param_maybe_default @@ -36613,7 +36698,7 @@ _loop0_186_rule(Parser *p) _mark = p->mark; } p->mark = _mark; - D(fprintf(stderr, "%*c%s _loop0_186[%d-%d]: %s failed!\n", p->level, ' ', + D(fprintf(stderr, "%*c%s _loop0_187[%d-%d]: %s failed!\n", p->level, ' ', p->error_indicator ? "ERROR!" : "-", _mark, p->mark, "lambda_param_maybe_default")); } asdl_seq *_seq = (asdl_seq*)_Py_asdl_generic_seq_new(_n, p->arena); @@ -36630,9 +36715,9 @@ _loop0_186_rule(Parser *p) return _seq; } -// _loop0_187: lambda_param_no_default +// _loop0_188: lambda_param_no_default static asdl_seq * -_loop0_187_rule(Parser *p) +_loop0_188_rule(Parser *p) { if (p->level++ == MAXSTACK) { _Pypegen_stack_overflow(p); @@ -36657,7 +36742,7 @@ _loop0_187_rule(Parser *p) p->level--; return NULL; } - D(fprintf(stderr, "%*c> _loop0_187[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "lambda_param_no_default")); + D(fprintf(stderr, "%*c> _loop0_188[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "lambda_param_no_default")); arg_ty lambda_param_no_default_var; while ( (lambda_param_no_default_var = lambda_param_no_default_rule(p)) // lambda_param_no_default @@ -36680,7 +36765,7 @@ _loop0_187_rule(Parser *p) _mark = p->mark; } p->mark = _mark; - D(fprintf(stderr, "%*c%s _loop0_187[%d-%d]: %s failed!\n", p->level, ' ', + D(fprintf(stderr, "%*c%s _loop0_188[%d-%d]: %s failed!\n", p->level, ' ', p->error_indicator ? "ERROR!" : "-", _mark, p->mark, "lambda_param_no_default")); } asdl_seq *_seq = (asdl_seq*)_Py_asdl_generic_seq_new(_n, p->arena); @@ -36697,9 +36782,9 @@ _loop0_187_rule(Parser *p) return _seq; } -// _loop0_188: lambda_param_no_default +// _loop0_189: lambda_param_no_default static asdl_seq * -_loop0_188_rule(Parser *p) +_loop0_189_rule(Parser *p) { if (p->level++ == MAXSTACK) { _Pypegen_stack_overflow(p); @@ -36724,7 +36809,7 @@ _loop0_188_rule(Parser *p) p->level--; return NULL; } - D(fprintf(stderr, "%*c> _loop0_188[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "lambda_param_no_default")); + D(fprintf(stderr, "%*c> _loop0_189[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "lambda_param_no_default")); arg_ty lambda_param_no_default_var; while ( (lambda_param_no_default_var = lambda_param_no_default_rule(p)) // lambda_param_no_default @@ -36747,7 +36832,7 @@ _loop0_188_rule(Parser *p) _mark = p->mark; } p->mark = _mark; - D(fprintf(stderr, "%*c%s _loop0_188[%d-%d]: %s failed!\n", p->level, ' ', + D(fprintf(stderr, "%*c%s _loop0_189[%d-%d]: %s failed!\n", p->level, ' ', p->error_indicator ? "ERROR!" : "-", _mark, p->mark, "lambda_param_no_default")); } asdl_seq *_seq = (asdl_seq*)_Py_asdl_generic_seq_new(_n, p->arena); @@ -36764,9 +36849,9 @@ _loop0_188_rule(Parser *p) return _seq; } -// _loop0_190: ',' lambda_param +// _loop0_191: ',' lambda_param static asdl_seq * -_loop0_190_rule(Parser *p) +_loop0_191_rule(Parser *p) { if (p->level++ == MAXSTACK) { _Pypegen_stack_overflow(p); @@ -36791,7 +36876,7 @@ _loop0_190_rule(Parser *p) p->level--; return NULL; } - D(fprintf(stderr, "%*c> _loop0_190[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "',' lambda_param")); + D(fprintf(stderr, "%*c> _loop0_191[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "',' lambda_param")); Token * _literal; arg_ty elem; while ( @@ -36823,7 +36908,7 @@ _loop0_190_rule(Parser *p) _mark = p->mark; } p->mark = _mark; - D(fprintf(stderr, "%*c%s _loop0_190[%d-%d]: %s failed!\n", p->level, ' ', + D(fprintf(stderr, "%*c%s _loop0_191[%d-%d]: %s failed!\n", p->level, ' ', p->error_indicator ? "ERROR!" : "-", _mark, p->mark, "',' lambda_param")); } asdl_seq *_seq = (asdl_seq*)_Py_asdl_generic_seq_new(_n, p->arena); @@ -36840,9 +36925,9 @@ _loop0_190_rule(Parser *p) return _seq; } -// _gather_189: lambda_param _loop0_190 +// _gather_190: lambda_param _loop0_191 static asdl_seq * -_gather_189_rule(Parser *p) +_gather_190_rule(Parser *p) { if (p->level++ == MAXSTACK) { _Pypegen_stack_overflow(p); @@ -36853,27 +36938,27 @@ _gather_189_rule(Parser *p) } asdl_seq * _res = NULL; int _mark = p->mark; - { // lambda_param _loop0_190 + { // lambda_param _loop0_191 if (p->error_indicator) { p->level--; return NULL; } - D(fprintf(stderr, "%*c> _gather_189[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "lambda_param _loop0_190")); + D(fprintf(stderr, "%*c> _gather_190[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "lambda_param _loop0_191")); arg_ty elem; asdl_seq * seq; if ( (elem = lambda_param_rule(p)) // lambda_param && - (seq = _loop0_190_rule(p)) // _loop0_190 + (seq = _loop0_191_rule(p)) // _loop0_191 ) { - D(fprintf(stderr, "%*c+ _gather_189[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "lambda_param _loop0_190")); + D(fprintf(stderr, "%*c+ _gather_190[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "lambda_param _loop0_191")); _res = _PyPegen_seq_insert_in_front(p, elem, seq); goto done; } p->mark = _mark; - D(fprintf(stderr, "%*c%s _gather_189[%d-%d]: %s failed!\n", p->level, ' ', - p->error_indicator ? "ERROR!" : "-", _mark, p->mark, "lambda_param _loop0_190")); + D(fprintf(stderr, "%*c%s _gather_190[%d-%d]: %s failed!\n", p->level, ' ', + p->error_indicator ? "ERROR!" : "-", _mark, p->mark, "lambda_param _loop0_191")); } _res = NULL; done: @@ -36881,9 +36966,9 @@ _gather_189_rule(Parser *p) return _res; } -// _tmp_191: lambda_slash_no_default | lambda_slash_with_default +// _tmp_192: lambda_slash_no_default | lambda_slash_with_default static void * -_tmp_191_rule(Parser *p) +_tmp_192_rule(Parser *p) { if (p->level++ == MAXSTACK) { _Pypegen_stack_overflow(p); @@ -36899,18 +36984,18 @@ _tmp_191_rule(Parser *p) p->level--; return NULL; } - D(fprintf(stderr, "%*c> _tmp_191[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "lambda_slash_no_default")); + D(fprintf(stderr, "%*c> _tmp_192[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "lambda_slash_no_default")); asdl_arg_seq* lambda_slash_no_default_var; if ( (lambda_slash_no_default_var = lambda_slash_no_default_rule(p)) // lambda_slash_no_default ) { - D(fprintf(stderr, "%*c+ _tmp_191[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "lambda_slash_no_default")); + D(fprintf(stderr, "%*c+ _tmp_192[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "lambda_slash_no_default")); _res = lambda_slash_no_default_var; goto done; } p->mark = _mark; - D(fprintf(stderr, "%*c%s _tmp_191[%d-%d]: %s failed!\n", p->level, ' ', + D(fprintf(stderr, "%*c%s _tmp_192[%d-%d]: %s failed!\n", p->level, ' ', p->error_indicator ? "ERROR!" : "-", _mark, p->mark, "lambda_slash_no_default")); } { // lambda_slash_with_default @@ -36918,18 +37003,18 @@ _tmp_191_rule(Parser *p) p->level--; return NULL; } - D(fprintf(stderr, "%*c> _tmp_191[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "lambda_slash_with_default")); + D(fprintf(stderr, "%*c> _tmp_192[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "lambda_slash_with_default")); SlashWithDefault* lambda_slash_with_default_var; if ( (lambda_slash_with_default_var = lambda_slash_with_default_rule(p)) // lambda_slash_with_default ) { - D(fprintf(stderr, "%*c+ _tmp_191[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "lambda_slash_with_default")); + D(fprintf(stderr, "%*c+ _tmp_192[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "lambda_slash_with_default")); _res = lambda_slash_with_default_var; goto done; } p->mark = _mark; - D(fprintf(stderr, "%*c%s _tmp_191[%d-%d]: %s failed!\n", p->level, ' ', + D(fprintf(stderr, "%*c%s _tmp_192[%d-%d]: %s failed!\n", p->level, ' ', p->error_indicator ? "ERROR!" : "-", _mark, p->mark, "lambda_slash_with_default")); } _res = NULL; @@ -36938,9 +37023,9 @@ _tmp_191_rule(Parser *p) return _res; } -// _loop0_192: lambda_param_maybe_default +// _loop0_193: lambda_param_maybe_default static asdl_seq * -_loop0_192_rule(Parser *p) +_loop0_193_rule(Parser *p) { if (p->level++ == MAXSTACK) { _Pypegen_stack_overflow(p); @@ -36965,7 +37050,7 @@ _loop0_192_rule(Parser *p) p->level--; return NULL; } - D(fprintf(stderr, "%*c> _loop0_192[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "lambda_param_maybe_default")); + D(fprintf(stderr, "%*c> _loop0_193[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "lambda_param_maybe_default")); NameDefaultPair* lambda_param_maybe_default_var; while ( (lambda_param_maybe_default_var = lambda_param_maybe_default_rule(p)) // lambda_param_maybe_default @@ -36988,7 +37073,7 @@ _loop0_192_rule(Parser *p) _mark = p->mark; } p->mark = _mark; - D(fprintf(stderr, "%*c%s _loop0_192[%d-%d]: %s failed!\n", p->level, ' ', + D(fprintf(stderr, "%*c%s _loop0_193[%d-%d]: %s failed!\n", p->level, ' ', p->error_indicator ? "ERROR!" : "-", _mark, p->mark, "lambda_param_maybe_default")); } asdl_seq *_seq = (asdl_seq*)_Py_asdl_generic_seq_new(_n, p->arena); @@ -37005,9 +37090,9 @@ _loop0_192_rule(Parser *p) return _seq; } -// _tmp_193: ',' | lambda_param_no_default +// _tmp_194: ',' | lambda_param_no_default static void * -_tmp_193_rule(Parser *p) +_tmp_194_rule(Parser *p) { if (p->level++ == MAXSTACK) { _Pypegen_stack_overflow(p); @@ -37023,18 +37108,18 @@ _tmp_193_rule(Parser *p) p->level--; return NULL; } - D(fprintf(stderr, "%*c> _tmp_193[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "','")); + D(fprintf(stderr, "%*c> _tmp_194[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "','")); Token * _literal; if ( (_literal = _PyPegen_expect_token(p, 12)) // token=',' ) { - D(fprintf(stderr, "%*c+ _tmp_193[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "','")); + D(fprintf(stderr, "%*c+ _tmp_194[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "','")); _res = _literal; goto done; } p->mark = _mark; - D(fprintf(stderr, "%*c%s _tmp_193[%d-%d]: %s failed!\n", p->level, ' ', + D(fprintf(stderr, "%*c%s _tmp_194[%d-%d]: %s failed!\n", p->level, ' ', p->error_indicator ? "ERROR!" : "-", _mark, p->mark, "','")); } { // lambda_param_no_default @@ -37042,18 +37127,18 @@ _tmp_193_rule(Parser *p) p->level--; return NULL; } - D(fprintf(stderr, "%*c> _tmp_193[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "lambda_param_no_default")); + D(fprintf(stderr, "%*c> _tmp_194[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "lambda_param_no_default")); arg_ty lambda_param_no_default_var; if ( (lambda_param_no_default_var = lambda_param_no_default_rule(p)) // lambda_param_no_default ) { - D(fprintf(stderr, "%*c+ _tmp_193[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "lambda_param_no_default")); + D(fprintf(stderr, "%*c+ _tmp_194[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "lambda_param_no_default")); _res = lambda_param_no_default_var; goto done; } p->mark = _mark; - D(fprintf(stderr, "%*c%s _tmp_193[%d-%d]: %s failed!\n", p->level, ' ', + D(fprintf(stderr, "%*c%s _tmp_194[%d-%d]: %s failed!\n", p->level, ' ', p->error_indicator ? "ERROR!" : "-", _mark, p->mark, "lambda_param_no_default")); } _res = NULL; @@ -37062,9 +37147,9 @@ _tmp_193_rule(Parser *p) return _res; } -// _loop0_194: lambda_param_maybe_default +// _loop0_195: lambda_param_maybe_default static asdl_seq * -_loop0_194_rule(Parser *p) +_loop0_195_rule(Parser *p) { if (p->level++ == MAXSTACK) { _Pypegen_stack_overflow(p); @@ -37089,7 +37174,7 @@ _loop0_194_rule(Parser *p) p->level--; return NULL; } - D(fprintf(stderr, "%*c> _loop0_194[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "lambda_param_maybe_default")); + D(fprintf(stderr, "%*c> _loop0_195[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "lambda_param_maybe_default")); NameDefaultPair* lambda_param_maybe_default_var; while ( (lambda_param_maybe_default_var = lambda_param_maybe_default_rule(p)) // lambda_param_maybe_default @@ -37112,7 +37197,7 @@ _loop0_194_rule(Parser *p) _mark = p->mark; } p->mark = _mark; - D(fprintf(stderr, "%*c%s _loop0_194[%d-%d]: %s failed!\n", p->level, ' ', + D(fprintf(stderr, "%*c%s _loop0_195[%d-%d]: %s failed!\n", p->level, ' ', p->error_indicator ? "ERROR!" : "-", _mark, p->mark, "lambda_param_maybe_default")); } asdl_seq *_seq = (asdl_seq*)_Py_asdl_generic_seq_new(_n, p->arena); @@ -37129,9 +37214,9 @@ _loop0_194_rule(Parser *p) return _seq; } -// _loop1_195: lambda_param_maybe_default +// _loop1_196: lambda_param_maybe_default static asdl_seq * -_loop1_195_rule(Parser *p) +_loop1_196_rule(Parser *p) { if (p->level++ == MAXSTACK) { _Pypegen_stack_overflow(p); @@ -37156,7 +37241,7 @@ _loop1_195_rule(Parser *p) p->level--; return NULL; } - D(fprintf(stderr, "%*c> _loop1_195[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "lambda_param_maybe_default")); + D(fprintf(stderr, "%*c> _loop1_196[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "lambda_param_maybe_default")); NameDefaultPair* lambda_param_maybe_default_var; while ( (lambda_param_maybe_default_var = lambda_param_maybe_default_rule(p)) // lambda_param_maybe_default @@ -37179,7 +37264,7 @@ _loop1_195_rule(Parser *p) _mark = p->mark; } p->mark = _mark; - D(fprintf(stderr, "%*c%s _loop1_195[%d-%d]: %s failed!\n", p->level, ' ', + D(fprintf(stderr, "%*c%s _loop1_196[%d-%d]: %s failed!\n", p->level, ' ', p->error_indicator ? "ERROR!" : "-", _mark, p->mark, "lambda_param_maybe_default")); } if (_n == 0 || p->error_indicator) { @@ -37201,9 +37286,9 @@ _loop1_195_rule(Parser *p) return _seq; } -// _loop1_196: lambda_param_with_default +// _loop1_197: lambda_param_with_default static asdl_seq * -_loop1_196_rule(Parser *p) +_loop1_197_rule(Parser *p) { if (p->level++ == MAXSTACK) { _Pypegen_stack_overflow(p); @@ -37228,7 +37313,7 @@ _loop1_196_rule(Parser *p) p->level--; return NULL; } - D(fprintf(stderr, "%*c> _loop1_196[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "lambda_param_with_default")); + D(fprintf(stderr, "%*c> _loop1_197[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "lambda_param_with_default")); NameDefaultPair* lambda_param_with_default_var; while ( (lambda_param_with_default_var = lambda_param_with_default_rule(p)) // lambda_param_with_default @@ -37251,7 +37336,7 @@ _loop1_196_rule(Parser *p) _mark = p->mark; } p->mark = _mark; - D(fprintf(stderr, "%*c%s _loop1_196[%d-%d]: %s failed!\n", p->level, ' ', + D(fprintf(stderr, "%*c%s _loop1_197[%d-%d]: %s failed!\n", p->level, ' ', p->error_indicator ? "ERROR!" : "-", _mark, p->mark, "lambda_param_with_default")); } if (_n == 0 || p->error_indicator) { @@ -37273,9 +37358,9 @@ _loop1_196_rule(Parser *p) return _seq; } -// _tmp_197: ':' | ',' (':' | '**') +// _tmp_198: ':' | ',' (':' | '**') static void * -_tmp_197_rule(Parser *p) +_tmp_198_rule(Parser *p) { if (p->level++ == MAXSTACK) { _Pypegen_stack_overflow(p); @@ -37291,18 +37376,18 @@ _tmp_197_rule(Parser *p) p->level--; return NULL; } - D(fprintf(stderr, "%*c> _tmp_197[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "':'")); + D(fprintf(stderr, "%*c> _tmp_198[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "':'")); Token * _literal; if ( (_literal = _PyPegen_expect_token(p, 11)) // token=':' ) { - D(fprintf(stderr, "%*c+ _tmp_197[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "':'")); + D(fprintf(stderr, "%*c+ _tmp_198[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "':'")); _res = _literal; goto done; } p->mark = _mark; - D(fprintf(stderr, "%*c%s _tmp_197[%d-%d]: %s failed!\n", p->level, ' ', + D(fprintf(stderr, "%*c%s _tmp_198[%d-%d]: %s failed!\n", p->level, ' ', p->error_indicator ? "ERROR!" : "-", _mark, p->mark, "':'")); } { // ',' (':' | '**') @@ -37310,21 +37395,21 @@ _tmp_197_rule(Parser *p) p->level--; return NULL; } - D(fprintf(stderr, "%*c> _tmp_197[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "',' (':' | '**')")); + D(fprintf(stderr, "%*c> _tmp_198[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "',' (':' | '**')")); Token * _literal; - void *_tmp_267_var; + void *_tmp_269_var; if ( (_literal = _PyPegen_expect_token(p, 12)) // token=',' && - (_tmp_267_var = _tmp_267_rule(p)) // ':' | '**' + (_tmp_269_var = _tmp_269_rule(p)) // ':' | '**' ) { - D(fprintf(stderr, "%*c+ _tmp_197[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "',' (':' | '**')")); - _res = _PyPegen_dummy_name(p, _literal, _tmp_267_var); + D(fprintf(stderr, "%*c+ _tmp_198[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "',' (':' | '**')")); + _res = _PyPegen_dummy_name(p, _literal, _tmp_269_var); goto done; } p->mark = _mark; - D(fprintf(stderr, "%*c%s _tmp_197[%d-%d]: %s failed!\n", p->level, ' ', + D(fprintf(stderr, "%*c%s _tmp_198[%d-%d]: %s failed!\n", p->level, ' ', p->error_indicator ? "ERROR!" : "-", _mark, p->mark, "',' (':' | '**')")); } _res = NULL; @@ -37333,9 +37418,9 @@ _tmp_197_rule(Parser *p) return _res; } -// _tmp_198: lambda_param_no_default | ',' +// _tmp_199: lambda_param_no_default | ',' static void * -_tmp_198_rule(Parser *p) +_tmp_199_rule(Parser *p) { if (p->level++ == MAXSTACK) { _Pypegen_stack_overflow(p); @@ -37351,18 +37436,18 @@ _tmp_198_rule(Parser *p) p->level--; return NULL; } - D(fprintf(stderr, "%*c> _tmp_198[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "lambda_param_no_default")); + D(fprintf(stderr, "%*c> _tmp_199[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "lambda_param_no_default")); arg_ty lambda_param_no_default_var; if ( (lambda_param_no_default_var = lambda_param_no_default_rule(p)) // lambda_param_no_default ) { - D(fprintf(stderr, "%*c+ _tmp_198[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "lambda_param_no_default")); + D(fprintf(stderr, "%*c+ _tmp_199[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "lambda_param_no_default")); _res = lambda_param_no_default_var; goto done; } p->mark = _mark; - D(fprintf(stderr, "%*c%s _tmp_198[%d-%d]: %s failed!\n", p->level, ' ', + D(fprintf(stderr, "%*c%s _tmp_199[%d-%d]: %s failed!\n", p->level, ' ', p->error_indicator ? "ERROR!" : "-", _mark, p->mark, "lambda_param_no_default")); } { // ',' @@ -37370,18 +37455,18 @@ _tmp_198_rule(Parser *p) p->level--; return NULL; } - D(fprintf(stderr, "%*c> _tmp_198[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "','")); + D(fprintf(stderr, "%*c> _tmp_199[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "','")); Token * _literal; if ( (_literal = _PyPegen_expect_token(p, 12)) // token=',' ) { - D(fprintf(stderr, "%*c+ _tmp_198[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "','")); + D(fprintf(stderr, "%*c+ _tmp_199[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "','")); _res = _literal; goto done; } p->mark = _mark; - D(fprintf(stderr, "%*c%s _tmp_198[%d-%d]: %s failed!\n", p->level, ' ', + D(fprintf(stderr, "%*c%s _tmp_199[%d-%d]: %s failed!\n", p->level, ' ', p->error_indicator ? "ERROR!" : "-", _mark, p->mark, "','")); } _res = NULL; @@ -37390,9 +37475,9 @@ _tmp_198_rule(Parser *p) return _res; } -// _loop0_199: lambda_param_maybe_default +// _loop0_200: lambda_param_maybe_default static asdl_seq * -_loop0_199_rule(Parser *p) +_loop0_200_rule(Parser *p) { if (p->level++ == MAXSTACK) { _Pypegen_stack_overflow(p); @@ -37417,7 +37502,7 @@ _loop0_199_rule(Parser *p) p->level--; return NULL; } - D(fprintf(stderr, "%*c> _loop0_199[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "lambda_param_maybe_default")); + D(fprintf(stderr, "%*c> _loop0_200[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "lambda_param_maybe_default")); NameDefaultPair* lambda_param_maybe_default_var; while ( (lambda_param_maybe_default_var = lambda_param_maybe_default_rule(p)) // lambda_param_maybe_default @@ -37440,7 +37525,7 @@ _loop0_199_rule(Parser *p) _mark = p->mark; } p->mark = _mark; - D(fprintf(stderr, "%*c%s _loop0_199[%d-%d]: %s failed!\n", p->level, ' ', + D(fprintf(stderr, "%*c%s _loop0_200[%d-%d]: %s failed!\n", p->level, ' ', p->error_indicator ? "ERROR!" : "-", _mark, p->mark, "lambda_param_maybe_default")); } asdl_seq *_seq = (asdl_seq*)_Py_asdl_generic_seq_new(_n, p->arena); @@ -37457,9 +37542,9 @@ _loop0_199_rule(Parser *p) return _seq; } -// _tmp_200: lambda_param_no_default | ',' +// _tmp_201: lambda_param_no_default | ',' static void * -_tmp_200_rule(Parser *p) +_tmp_201_rule(Parser *p) { if (p->level++ == MAXSTACK) { _Pypegen_stack_overflow(p); @@ -37475,18 +37560,18 @@ _tmp_200_rule(Parser *p) p->level--; return NULL; } - D(fprintf(stderr, "%*c> _tmp_200[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "lambda_param_no_default")); + D(fprintf(stderr, "%*c> _tmp_201[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "lambda_param_no_default")); arg_ty lambda_param_no_default_var; if ( (lambda_param_no_default_var = lambda_param_no_default_rule(p)) // lambda_param_no_default ) { - D(fprintf(stderr, "%*c+ _tmp_200[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "lambda_param_no_default")); + D(fprintf(stderr, "%*c+ _tmp_201[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "lambda_param_no_default")); _res = lambda_param_no_default_var; goto done; } p->mark = _mark; - D(fprintf(stderr, "%*c%s _tmp_200[%d-%d]: %s failed!\n", p->level, ' ', + D(fprintf(stderr, "%*c%s _tmp_201[%d-%d]: %s failed!\n", p->level, ' ', p->error_indicator ? "ERROR!" : "-", _mark, p->mark, "lambda_param_no_default")); } { // ',' @@ -37494,18 +37579,18 @@ _tmp_200_rule(Parser *p) p->level--; return NULL; } - D(fprintf(stderr, "%*c> _tmp_200[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "','")); + D(fprintf(stderr, "%*c> _tmp_201[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "','")); Token * _literal; if ( (_literal = _PyPegen_expect_token(p, 12)) // token=',' ) { - D(fprintf(stderr, "%*c+ _tmp_200[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "','")); + D(fprintf(stderr, "%*c+ _tmp_201[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "','")); _res = _literal; goto done; } p->mark = _mark; - D(fprintf(stderr, "%*c%s _tmp_200[%d-%d]: %s failed!\n", p->level, ' ', + D(fprintf(stderr, "%*c%s _tmp_201[%d-%d]: %s failed!\n", p->level, ' ', p->error_indicator ? "ERROR!" : "-", _mark, p->mark, "','")); } _res = NULL; @@ -37514,9 +37599,9 @@ _tmp_200_rule(Parser *p) return _res; } -// _tmp_201: '*' | '**' | '/' +// _tmp_202: '*' | '**' | '/' static void * -_tmp_201_rule(Parser *p) +_tmp_202_rule(Parser *p) { if (p->level++ == MAXSTACK) { _Pypegen_stack_overflow(p); @@ -37532,18 +37617,18 @@ _tmp_201_rule(Parser *p) p->level--; return NULL; } - D(fprintf(stderr, "%*c> _tmp_201[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "'*'")); + D(fprintf(stderr, "%*c> _tmp_202[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "'*'")); Token * _literal; if ( (_literal = _PyPegen_expect_token(p, 16)) // token='*' ) { - D(fprintf(stderr, "%*c+ _tmp_201[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "'*'")); + D(fprintf(stderr, "%*c+ _tmp_202[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "'*'")); _res = _literal; goto done; } p->mark = _mark; - D(fprintf(stderr, "%*c%s _tmp_201[%d-%d]: %s failed!\n", p->level, ' ', + D(fprintf(stderr, "%*c%s _tmp_202[%d-%d]: %s failed!\n", p->level, ' ', p->error_indicator ? "ERROR!" : "-", _mark, p->mark, "'*'")); } { // '**' @@ -37551,18 +37636,18 @@ _tmp_201_rule(Parser *p) p->level--; return NULL; } - D(fprintf(stderr, "%*c> _tmp_201[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "'**'")); + D(fprintf(stderr, "%*c> _tmp_202[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "'**'")); Token * _literal; if ( (_literal = _PyPegen_expect_token(p, 35)) // token='**' ) { - D(fprintf(stderr, "%*c+ _tmp_201[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "'**'")); + D(fprintf(stderr, "%*c+ _tmp_202[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "'**'")); _res = _literal; goto done; } p->mark = _mark; - D(fprintf(stderr, "%*c%s _tmp_201[%d-%d]: %s failed!\n", p->level, ' ', + D(fprintf(stderr, "%*c%s _tmp_202[%d-%d]: %s failed!\n", p->level, ' ', p->error_indicator ? "ERROR!" : "-", _mark, p->mark, "'**'")); } { // '/' @@ -37570,18 +37655,18 @@ _tmp_201_rule(Parser *p) p->level--; return NULL; } - D(fprintf(stderr, "%*c> _tmp_201[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "'/'")); + D(fprintf(stderr, "%*c> _tmp_202[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "'/'")); Token * _literal; if ( (_literal = _PyPegen_expect_token(p, 17)) // token='/' ) { - D(fprintf(stderr, "%*c+ _tmp_201[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "'/'")); + D(fprintf(stderr, "%*c+ _tmp_202[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "'/'")); _res = _literal; goto done; } p->mark = _mark; - D(fprintf(stderr, "%*c%s _tmp_201[%d-%d]: %s failed!\n", p->level, ' ', + D(fprintf(stderr, "%*c%s _tmp_202[%d-%d]: %s failed!\n", p->level, ' ', p->error_indicator ? "ERROR!" : "-", _mark, p->mark, "'/'")); } _res = NULL; @@ -37590,9 +37675,9 @@ _tmp_201_rule(Parser *p) return _res; } -// _tmp_202: ',' | ')' | ':' +// _tmp_203: ',' | ')' | ':' static void * -_tmp_202_rule(Parser *p) +_tmp_203_rule(Parser *p) { if (p->level++ == MAXSTACK) { _Pypegen_stack_overflow(p); @@ -37608,18 +37693,18 @@ _tmp_202_rule(Parser *p) p->level--; return NULL; } - D(fprintf(stderr, "%*c> _tmp_202[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "','")); + D(fprintf(stderr, "%*c> _tmp_203[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "','")); Token * _literal; if ( (_literal = _PyPegen_expect_token(p, 12)) // token=',' ) { - D(fprintf(stderr, "%*c+ _tmp_202[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "','")); + D(fprintf(stderr, "%*c+ _tmp_203[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "','")); _res = _literal; goto done; } p->mark = _mark; - D(fprintf(stderr, "%*c%s _tmp_202[%d-%d]: %s failed!\n", p->level, ' ', + D(fprintf(stderr, "%*c%s _tmp_203[%d-%d]: %s failed!\n", p->level, ' ', p->error_indicator ? "ERROR!" : "-", _mark, p->mark, "','")); } { // ')' @@ -37627,18 +37712,18 @@ _tmp_202_rule(Parser *p) p->level--; return NULL; } - D(fprintf(stderr, "%*c> _tmp_202[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "')'")); + D(fprintf(stderr, "%*c> _tmp_203[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "')'")); Token * _literal; if ( (_literal = _PyPegen_expect_token(p, 8)) // token=')' ) { - D(fprintf(stderr, "%*c+ _tmp_202[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "')'")); + D(fprintf(stderr, "%*c+ _tmp_203[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "')'")); _res = _literal; goto done; } p->mark = _mark; - D(fprintf(stderr, "%*c%s _tmp_202[%d-%d]: %s failed!\n", p->level, ' ', + D(fprintf(stderr, "%*c%s _tmp_203[%d-%d]: %s failed!\n", p->level, ' ', p->error_indicator ? "ERROR!" : "-", _mark, p->mark, "')'")); } { // ':' @@ -37646,18 +37731,18 @@ _tmp_202_rule(Parser *p) p->level--; return NULL; } - D(fprintf(stderr, "%*c> _tmp_202[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "':'")); + D(fprintf(stderr, "%*c> _tmp_203[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "':'")); Token * _literal; if ( (_literal = _PyPegen_expect_token(p, 11)) // token=':' ) { - D(fprintf(stderr, "%*c+ _tmp_202[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "':'")); + D(fprintf(stderr, "%*c+ _tmp_203[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "':'")); _res = _literal; goto done; } p->mark = _mark; - D(fprintf(stderr, "%*c%s _tmp_202[%d-%d]: %s failed!\n", p->level, ' ', + D(fprintf(stderr, "%*c%s _tmp_203[%d-%d]: %s failed!\n", p->level, ' ', p->error_indicator ? "ERROR!" : "-", _mark, p->mark, "':'")); } _res = NULL; @@ -37666,9 +37751,9 @@ _tmp_202_rule(Parser *p) return _res; } -// _loop0_204: ',' dotted_name +// _loop0_205: ',' dotted_name static asdl_seq * -_loop0_204_rule(Parser *p) +_loop0_205_rule(Parser *p) { if (p->level++ == MAXSTACK) { _Pypegen_stack_overflow(p); @@ -37693,7 +37778,7 @@ _loop0_204_rule(Parser *p) p->level--; return NULL; } - D(fprintf(stderr, "%*c> _loop0_204[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "',' dotted_name")); + D(fprintf(stderr, "%*c> _loop0_205[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "',' dotted_name")); Token * _literal; expr_ty elem; while ( @@ -37725,7 +37810,7 @@ _loop0_204_rule(Parser *p) _mark = p->mark; } p->mark = _mark; - D(fprintf(stderr, "%*c%s _loop0_204[%d-%d]: %s failed!\n", p->level, ' ', + D(fprintf(stderr, "%*c%s _loop0_205[%d-%d]: %s failed!\n", p->level, ' ', p->error_indicator ? "ERROR!" : "-", _mark, p->mark, "',' dotted_name")); } asdl_seq *_seq = (asdl_seq*)_Py_asdl_generic_seq_new(_n, p->arena); @@ -37742,9 +37827,9 @@ _loop0_204_rule(Parser *p) return _seq; } -// _gather_203: dotted_name _loop0_204 +// _gather_204: dotted_name _loop0_205 static asdl_seq * -_gather_203_rule(Parser *p) +_gather_204_rule(Parser *p) { if (p->level++ == MAXSTACK) { _Pypegen_stack_overflow(p); @@ -37755,27 +37840,27 @@ _gather_203_rule(Parser *p) } asdl_seq * _res = NULL; int _mark = p->mark; - { // dotted_name _loop0_204 + { // dotted_name _loop0_205 if (p->error_indicator) { p->level--; return NULL; } - D(fprintf(stderr, "%*c> _gather_203[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "dotted_name _loop0_204")); + D(fprintf(stderr, "%*c> _gather_204[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "dotted_name _loop0_205")); expr_ty elem; asdl_seq * seq; if ( (elem = dotted_name_rule(p)) // dotted_name && - (seq = _loop0_204_rule(p)) // _loop0_204 + (seq = _loop0_205_rule(p)) // _loop0_205 ) { - D(fprintf(stderr, "%*c+ _gather_203[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "dotted_name _loop0_204")); + D(fprintf(stderr, "%*c+ _gather_204[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "dotted_name _loop0_205")); _res = _PyPegen_seq_insert_in_front(p, elem, seq); goto done; } p->mark = _mark; - D(fprintf(stderr, "%*c%s _gather_203[%d-%d]: %s failed!\n", p->level, ' ', - p->error_indicator ? "ERROR!" : "-", _mark, p->mark, "dotted_name _loop0_204")); + D(fprintf(stderr, "%*c%s _gather_204[%d-%d]: %s failed!\n", p->level, ' ', + p->error_indicator ? "ERROR!" : "-", _mark, p->mark, "dotted_name _loop0_205")); } _res = NULL; done: @@ -37783,9 +37868,9 @@ _gather_203_rule(Parser *p) return _res; } -// _loop0_206: ',' (expression ['as' star_target]) +// _loop0_207: ',' (expression ['as' star_target]) static asdl_seq * -_loop0_206_rule(Parser *p) +_loop0_207_rule(Parser *p) { if (p->level++ == MAXSTACK) { _Pypegen_stack_overflow(p); @@ -37810,13 +37895,13 @@ _loop0_206_rule(Parser *p) p->level--; return NULL; } - D(fprintf(stderr, "%*c> _loop0_206[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "',' (expression ['as' star_target])")); + D(fprintf(stderr, "%*c> _loop0_207[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "',' (expression ['as' star_target])")); Token * _literal; void *elem; while ( (_literal = _PyPegen_expect_token(p, 12)) // token=',' && - (elem = _tmp_268_rule(p)) // expression ['as' star_target] + (elem = _tmp_270_rule(p)) // expression ['as' star_target] ) { _res = elem; @@ -37842,7 +37927,7 @@ _loop0_206_rule(Parser *p) _mark = p->mark; } p->mark = _mark; - D(fprintf(stderr, "%*c%s _loop0_206[%d-%d]: %s failed!\n", p->level, ' ', + D(fprintf(stderr, "%*c%s _loop0_207[%d-%d]: %s failed!\n", p->level, ' ', p->error_indicator ? "ERROR!" : "-", _mark, p->mark, "',' (expression ['as' star_target])")); } asdl_seq *_seq = (asdl_seq*)_Py_asdl_generic_seq_new(_n, p->arena); @@ -37859,9 +37944,9 @@ _loop0_206_rule(Parser *p) return _seq; } -// _gather_205: (expression ['as' star_target]) _loop0_206 +// _gather_206: (expression ['as' star_target]) _loop0_207 static asdl_seq * -_gather_205_rule(Parser *p) +_gather_206_rule(Parser *p) { if (p->level++ == MAXSTACK) { _Pypegen_stack_overflow(p); @@ -37872,27 +37957,27 @@ _gather_205_rule(Parser *p) } asdl_seq * _res = NULL; int _mark = p->mark; - { // (expression ['as' star_target]) _loop0_206 + { // (expression ['as' star_target]) _loop0_207 if (p->error_indicator) { p->level--; return NULL; } - D(fprintf(stderr, "%*c> _gather_205[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "(expression ['as' star_target]) _loop0_206")); + D(fprintf(stderr, "%*c> _gather_206[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "(expression ['as' star_target]) _loop0_207")); void *elem; asdl_seq * seq; if ( - (elem = _tmp_268_rule(p)) // expression ['as' star_target] + (elem = _tmp_270_rule(p)) // expression ['as' star_target] && - (seq = _loop0_206_rule(p)) // _loop0_206 + (seq = _loop0_207_rule(p)) // _loop0_207 ) { - D(fprintf(stderr, "%*c+ _gather_205[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "(expression ['as' star_target]) _loop0_206")); + D(fprintf(stderr, "%*c+ _gather_206[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "(expression ['as' star_target]) _loop0_207")); _res = _PyPegen_seq_insert_in_front(p, elem, seq); goto done; } p->mark = _mark; - D(fprintf(stderr, "%*c%s _gather_205[%d-%d]: %s failed!\n", p->level, ' ', - p->error_indicator ? "ERROR!" : "-", _mark, p->mark, "(expression ['as' star_target]) _loop0_206")); + D(fprintf(stderr, "%*c%s _gather_206[%d-%d]: %s failed!\n", p->level, ' ', + p->error_indicator ? "ERROR!" : "-", _mark, p->mark, "(expression ['as' star_target]) _loop0_207")); } _res = NULL; done: @@ -37900,9 +37985,9 @@ _gather_205_rule(Parser *p) return _res; } -// _loop0_208: ',' (expressions ['as' star_target]) +// _loop0_209: ',' (expressions ['as' star_target]) static asdl_seq * -_loop0_208_rule(Parser *p) +_loop0_209_rule(Parser *p) { if (p->level++ == MAXSTACK) { _Pypegen_stack_overflow(p); @@ -37927,13 +38012,13 @@ _loop0_208_rule(Parser *p) p->level--; return NULL; } - D(fprintf(stderr, "%*c> _loop0_208[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "',' (expressions ['as' star_target])")); + D(fprintf(stderr, "%*c> _loop0_209[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "',' (expressions ['as' star_target])")); Token * _literal; void *elem; while ( (_literal = _PyPegen_expect_token(p, 12)) // token=',' && - (elem = _tmp_269_rule(p)) // expressions ['as' star_target] + (elem = _tmp_271_rule(p)) // expressions ['as' star_target] ) { _res = elem; @@ -37959,7 +38044,7 @@ _loop0_208_rule(Parser *p) _mark = p->mark; } p->mark = _mark; - D(fprintf(stderr, "%*c%s _loop0_208[%d-%d]: %s failed!\n", p->level, ' ', + D(fprintf(stderr, "%*c%s _loop0_209[%d-%d]: %s failed!\n", p->level, ' ', p->error_indicator ? "ERROR!" : "-", _mark, p->mark, "',' (expressions ['as' star_target])")); } asdl_seq *_seq = (asdl_seq*)_Py_asdl_generic_seq_new(_n, p->arena); @@ -37976,9 +38061,9 @@ _loop0_208_rule(Parser *p) return _seq; } -// _gather_207: (expressions ['as' star_target]) _loop0_208 +// _gather_208: (expressions ['as' star_target]) _loop0_209 static asdl_seq * -_gather_207_rule(Parser *p) +_gather_208_rule(Parser *p) { if (p->level++ == MAXSTACK) { _Pypegen_stack_overflow(p); @@ -37989,27 +38074,27 @@ _gather_207_rule(Parser *p) } asdl_seq * _res = NULL; int _mark = p->mark; - { // (expressions ['as' star_target]) _loop0_208 + { // (expressions ['as' star_target]) _loop0_209 if (p->error_indicator) { p->level--; return NULL; } - D(fprintf(stderr, "%*c> _gather_207[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "(expressions ['as' star_target]) _loop0_208")); + D(fprintf(stderr, "%*c> _gather_208[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "(expressions ['as' star_target]) _loop0_209")); void *elem; asdl_seq * seq; if ( - (elem = _tmp_269_rule(p)) // expressions ['as' star_target] + (elem = _tmp_271_rule(p)) // expressions ['as' star_target] && - (seq = _loop0_208_rule(p)) // _loop0_208 + (seq = _loop0_209_rule(p)) // _loop0_209 ) { - D(fprintf(stderr, "%*c+ _gather_207[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "(expressions ['as' star_target]) _loop0_208")); + D(fprintf(stderr, "%*c+ _gather_208[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "(expressions ['as' star_target]) _loop0_209")); _res = _PyPegen_seq_insert_in_front(p, elem, seq); goto done; } p->mark = _mark; - D(fprintf(stderr, "%*c%s _gather_207[%d-%d]: %s failed!\n", p->level, ' ', - p->error_indicator ? "ERROR!" : "-", _mark, p->mark, "(expressions ['as' star_target]) _loop0_208")); + D(fprintf(stderr, "%*c%s _gather_208[%d-%d]: %s failed!\n", p->level, ' ', + p->error_indicator ? "ERROR!" : "-", _mark, p->mark, "(expressions ['as' star_target]) _loop0_209")); } _res = NULL; done: @@ -38017,9 +38102,9 @@ _gather_207_rule(Parser *p) return _res; } -// _loop0_210: ',' (expression ['as' star_target]) +// _loop0_211: ',' (expression ['as' star_target]) static asdl_seq * -_loop0_210_rule(Parser *p) +_loop0_211_rule(Parser *p) { if (p->level++ == MAXSTACK) { _Pypegen_stack_overflow(p); @@ -38044,13 +38129,13 @@ _loop0_210_rule(Parser *p) p->level--; return NULL; } - D(fprintf(stderr, "%*c> _loop0_210[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "',' (expression ['as' star_target])")); + D(fprintf(stderr, "%*c> _loop0_211[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "',' (expression ['as' star_target])")); Token * _literal; void *elem; while ( (_literal = _PyPegen_expect_token(p, 12)) // token=',' && - (elem = _tmp_270_rule(p)) // expression ['as' star_target] + (elem = _tmp_272_rule(p)) // expression ['as' star_target] ) { _res = elem; @@ -38076,7 +38161,7 @@ _loop0_210_rule(Parser *p) _mark = p->mark; } p->mark = _mark; - D(fprintf(stderr, "%*c%s _loop0_210[%d-%d]: %s failed!\n", p->level, ' ', + D(fprintf(stderr, "%*c%s _loop0_211[%d-%d]: %s failed!\n", p->level, ' ', p->error_indicator ? "ERROR!" : "-", _mark, p->mark, "',' (expression ['as' star_target])")); } asdl_seq *_seq = (asdl_seq*)_Py_asdl_generic_seq_new(_n, p->arena); @@ -38093,9 +38178,9 @@ _loop0_210_rule(Parser *p) return _seq; } -// _gather_209: (expression ['as' star_target]) _loop0_210 +// _gather_210: (expression ['as' star_target]) _loop0_211 static asdl_seq * -_gather_209_rule(Parser *p) +_gather_210_rule(Parser *p) { if (p->level++ == MAXSTACK) { _Pypegen_stack_overflow(p); @@ -38106,27 +38191,27 @@ _gather_209_rule(Parser *p) } asdl_seq * _res = NULL; int _mark = p->mark; - { // (expression ['as' star_target]) _loop0_210 + { // (expression ['as' star_target]) _loop0_211 if (p->error_indicator) { p->level--; return NULL; } - D(fprintf(stderr, "%*c> _gather_209[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "(expression ['as' star_target]) _loop0_210")); + D(fprintf(stderr, "%*c> _gather_210[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "(expression ['as' star_target]) _loop0_211")); void *elem; asdl_seq * seq; if ( - (elem = _tmp_270_rule(p)) // expression ['as' star_target] + (elem = _tmp_272_rule(p)) // expression ['as' star_target] && - (seq = _loop0_210_rule(p)) // _loop0_210 + (seq = _loop0_211_rule(p)) // _loop0_211 ) { - D(fprintf(stderr, "%*c+ _gather_209[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "(expression ['as' star_target]) _loop0_210")); + D(fprintf(stderr, "%*c+ _gather_210[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "(expression ['as' star_target]) _loop0_211")); _res = _PyPegen_seq_insert_in_front(p, elem, seq); goto done; } p->mark = _mark; - D(fprintf(stderr, "%*c%s _gather_209[%d-%d]: %s failed!\n", p->level, ' ', - p->error_indicator ? "ERROR!" : "-", _mark, p->mark, "(expression ['as' star_target]) _loop0_210")); + D(fprintf(stderr, "%*c%s _gather_210[%d-%d]: %s failed!\n", p->level, ' ', + p->error_indicator ? "ERROR!" : "-", _mark, p->mark, "(expression ['as' star_target]) _loop0_211")); } _res = NULL; done: @@ -38134,9 +38219,9 @@ _gather_209_rule(Parser *p) return _res; } -// _loop0_212: ',' (expressions ['as' star_target]) +// _loop0_213: ',' (expressions ['as' star_target]) static asdl_seq * -_loop0_212_rule(Parser *p) +_loop0_213_rule(Parser *p) { if (p->level++ == MAXSTACK) { _Pypegen_stack_overflow(p); @@ -38161,13 +38246,13 @@ _loop0_212_rule(Parser *p) p->level--; return NULL; } - D(fprintf(stderr, "%*c> _loop0_212[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "',' (expressions ['as' star_target])")); + D(fprintf(stderr, "%*c> _loop0_213[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "',' (expressions ['as' star_target])")); Token * _literal; void *elem; while ( (_literal = _PyPegen_expect_token(p, 12)) // token=',' && - (elem = _tmp_271_rule(p)) // expressions ['as' star_target] + (elem = _tmp_273_rule(p)) // expressions ['as' star_target] ) { _res = elem; @@ -38193,7 +38278,7 @@ _loop0_212_rule(Parser *p) _mark = p->mark; } p->mark = _mark; - D(fprintf(stderr, "%*c%s _loop0_212[%d-%d]: %s failed!\n", p->level, ' ', + D(fprintf(stderr, "%*c%s _loop0_213[%d-%d]: %s failed!\n", p->level, ' ', p->error_indicator ? "ERROR!" : "-", _mark, p->mark, "',' (expressions ['as' star_target])")); } asdl_seq *_seq = (asdl_seq*)_Py_asdl_generic_seq_new(_n, p->arena); @@ -38210,9 +38295,9 @@ _loop0_212_rule(Parser *p) return _seq; } -// _gather_211: (expressions ['as' star_target]) _loop0_212 +// _gather_212: (expressions ['as' star_target]) _loop0_213 static asdl_seq * -_gather_211_rule(Parser *p) +_gather_212_rule(Parser *p) { if (p->level++ == MAXSTACK) { _Pypegen_stack_overflow(p); @@ -38223,27 +38308,27 @@ _gather_211_rule(Parser *p) } asdl_seq * _res = NULL; int _mark = p->mark; - { // (expressions ['as' star_target]) _loop0_212 + { // (expressions ['as' star_target]) _loop0_213 if (p->error_indicator) { p->level--; return NULL; } - D(fprintf(stderr, "%*c> _gather_211[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "(expressions ['as' star_target]) _loop0_212")); + D(fprintf(stderr, "%*c> _gather_212[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "(expressions ['as' star_target]) _loop0_213")); void *elem; asdl_seq * seq; if ( - (elem = _tmp_271_rule(p)) // expressions ['as' star_target] + (elem = _tmp_273_rule(p)) // expressions ['as' star_target] && - (seq = _loop0_212_rule(p)) // _loop0_212 + (seq = _loop0_213_rule(p)) // _loop0_213 ) { - D(fprintf(stderr, "%*c+ _gather_211[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "(expressions ['as' star_target]) _loop0_212")); + D(fprintf(stderr, "%*c+ _gather_212[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "(expressions ['as' star_target]) _loop0_213")); _res = _PyPegen_seq_insert_in_front(p, elem, seq); goto done; } p->mark = _mark; - D(fprintf(stderr, "%*c%s _gather_211[%d-%d]: %s failed!\n", p->level, ' ', - p->error_indicator ? "ERROR!" : "-", _mark, p->mark, "(expressions ['as' star_target]) _loop0_212")); + D(fprintf(stderr, "%*c%s _gather_212[%d-%d]: %s failed!\n", p->level, ' ', + p->error_indicator ? "ERROR!" : "-", _mark, p->mark, "(expressions ['as' star_target]) _loop0_213")); } _res = NULL; done: @@ -38251,9 +38336,9 @@ _gather_211_rule(Parser *p) return _res; } -// _tmp_213: 'except' | 'finally' +// _tmp_214: 'except' | 'finally' static void * -_tmp_213_rule(Parser *p) +_tmp_214_rule(Parser *p) { if (p->level++ == MAXSTACK) { _Pypegen_stack_overflow(p); @@ -38269,18 +38354,18 @@ _tmp_213_rule(Parser *p) p->level--; return NULL; } - D(fprintf(stderr, "%*c> _tmp_213[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "'except'")); + D(fprintf(stderr, "%*c> _tmp_214[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "'except'")); Token * _keyword; if ( - (_keyword = _PyPegen_expect_token(p, 653)) // token='except' + (_keyword = _PyPegen_expect_token(p, 656)) // token='except' ) { - D(fprintf(stderr, "%*c+ _tmp_213[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "'except'")); + D(fprintf(stderr, "%*c+ _tmp_214[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "'except'")); _res = _keyword; goto done; } p->mark = _mark; - D(fprintf(stderr, "%*c%s _tmp_213[%d-%d]: %s failed!\n", p->level, ' ', + D(fprintf(stderr, "%*c%s _tmp_214[%d-%d]: %s failed!\n", p->level, ' ', p->error_indicator ? "ERROR!" : "-", _mark, p->mark, "'except'")); } { // 'finally' @@ -38288,18 +38373,18 @@ _tmp_213_rule(Parser *p) p->level--; return NULL; } - D(fprintf(stderr, "%*c> _tmp_213[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "'finally'")); + D(fprintf(stderr, "%*c> _tmp_214[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "'finally'")); Token * _keyword; if ( - (_keyword = _PyPegen_expect_token(p, 649)) // token='finally' + (_keyword = _PyPegen_expect_token(p, 652)) // token='finally' ) { - D(fprintf(stderr, "%*c+ _tmp_213[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "'finally'")); + D(fprintf(stderr, "%*c+ _tmp_214[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "'finally'")); _res = _keyword; goto done; } p->mark = _mark; - D(fprintf(stderr, "%*c%s _tmp_213[%d-%d]: %s failed!\n", p->level, ' ', + D(fprintf(stderr, "%*c%s _tmp_214[%d-%d]: %s failed!\n", p->level, ' ', p->error_indicator ? "ERROR!" : "-", _mark, p->mark, "'finally'")); } _res = NULL; @@ -38308,9 +38393,9 @@ _tmp_213_rule(Parser *p) return _res; } -// _loop0_214: block +// _loop0_215: block static asdl_seq * -_loop0_214_rule(Parser *p) +_loop0_215_rule(Parser *p) { if (p->level++ == MAXSTACK) { _Pypegen_stack_overflow(p); @@ -38335,7 +38420,7 @@ _loop0_214_rule(Parser *p) p->level--; return NULL; } - D(fprintf(stderr, "%*c> _loop0_214[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "block")); + D(fprintf(stderr, "%*c> _loop0_215[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "block")); asdl_stmt_seq* block_var; while ( (block_var = block_rule(p)) // block @@ -38358,7 +38443,7 @@ _loop0_214_rule(Parser *p) _mark = p->mark; } p->mark = _mark; - D(fprintf(stderr, "%*c%s _loop0_214[%d-%d]: %s failed!\n", p->level, ' ', + D(fprintf(stderr, "%*c%s _loop0_215[%d-%d]: %s failed!\n", p->level, ' ', p->error_indicator ? "ERROR!" : "-", _mark, p->mark, "block")); } asdl_seq *_seq = (asdl_seq*)_Py_asdl_generic_seq_new(_n, p->arena); @@ -38375,9 +38460,9 @@ _loop0_214_rule(Parser *p) return _seq; } -// _loop1_215: except_block +// _loop1_216: except_block static asdl_seq * -_loop1_215_rule(Parser *p) +_loop1_216_rule(Parser *p) { if (p->level++ == MAXSTACK) { _Pypegen_stack_overflow(p); @@ -38402,7 +38487,7 @@ _loop1_215_rule(Parser *p) p->level--; return NULL; } - D(fprintf(stderr, "%*c> _loop1_215[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "except_block")); + D(fprintf(stderr, "%*c> _loop1_216[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "except_block")); excepthandler_ty except_block_var; while ( (except_block_var = except_block_rule(p)) // except_block @@ -38425,7 +38510,7 @@ _loop1_215_rule(Parser *p) _mark = p->mark; } p->mark = _mark; - D(fprintf(stderr, "%*c%s _loop1_215[%d-%d]: %s failed!\n", p->level, ' ', + D(fprintf(stderr, "%*c%s _loop1_216[%d-%d]: %s failed!\n", p->level, ' ', p->error_indicator ? "ERROR!" : "-", _mark, p->mark, "except_block")); } if (_n == 0 || p->error_indicator) { @@ -38447,9 +38532,9 @@ _loop1_215_rule(Parser *p) return _seq; } -// _tmp_216: 'as' NAME +// _tmp_217: 'as' NAME static void * -_tmp_216_rule(Parser *p) +_tmp_217_rule(Parser *p) { if (p->level++ == MAXSTACK) { _Pypegen_stack_overflow(p); @@ -38465,21 +38550,21 @@ _tmp_216_rule(Parser *p) p->level--; return NULL; } - D(fprintf(stderr, "%*c> _tmp_216[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "'as' NAME")); + D(fprintf(stderr, "%*c> _tmp_217[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "'as' NAME")); Token * _keyword; expr_ty name_var; if ( - (_keyword = _PyPegen_expect_token(p, 656)) // token='as' + (_keyword = _PyPegen_expect_token(p, 659)) // token='as' && (name_var = _PyPegen_name_token(p)) // NAME ) { - D(fprintf(stderr, "%*c+ _tmp_216[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "'as' NAME")); + D(fprintf(stderr, "%*c+ _tmp_217[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "'as' NAME")); _res = _PyPegen_dummy_name(p, _keyword, name_var); goto done; } p->mark = _mark; - D(fprintf(stderr, "%*c%s _tmp_216[%d-%d]: %s failed!\n", p->level, ' ', + D(fprintf(stderr, "%*c%s _tmp_217[%d-%d]: %s failed!\n", p->level, ' ', p->error_indicator ? "ERROR!" : "-", _mark, p->mark, "'as' NAME")); } _res = NULL; @@ -38488,9 +38573,9 @@ _tmp_216_rule(Parser *p) return _res; } -// _loop0_217: block +// _loop0_218: block static asdl_seq * -_loop0_217_rule(Parser *p) +_loop0_218_rule(Parser *p) { if (p->level++ == MAXSTACK) { _Pypegen_stack_overflow(p); @@ -38515,7 +38600,7 @@ _loop0_217_rule(Parser *p) p->level--; return NULL; } - D(fprintf(stderr, "%*c> _loop0_217[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "block")); + D(fprintf(stderr, "%*c> _loop0_218[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "block")); asdl_stmt_seq* block_var; while ( (block_var = block_rule(p)) // block @@ -38538,7 +38623,7 @@ _loop0_217_rule(Parser *p) _mark = p->mark; } p->mark = _mark; - D(fprintf(stderr, "%*c%s _loop0_217[%d-%d]: %s failed!\n", p->level, ' ', + D(fprintf(stderr, "%*c%s _loop0_218[%d-%d]: %s failed!\n", p->level, ' ', p->error_indicator ? "ERROR!" : "-", _mark, p->mark, "block")); } asdl_seq *_seq = (asdl_seq*)_Py_asdl_generic_seq_new(_n, p->arena); @@ -38555,9 +38640,9 @@ _loop0_217_rule(Parser *p) return _seq; } -// _loop1_218: except_star_block +// _loop1_219: except_star_block static asdl_seq * -_loop1_218_rule(Parser *p) +_loop1_219_rule(Parser *p) { if (p->level++ == MAXSTACK) { _Pypegen_stack_overflow(p); @@ -38582,7 +38667,7 @@ _loop1_218_rule(Parser *p) p->level--; return NULL; } - D(fprintf(stderr, "%*c> _loop1_218[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "except_star_block")); + D(fprintf(stderr, "%*c> _loop1_219[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "except_star_block")); excepthandler_ty except_star_block_var; while ( (except_star_block_var = except_star_block_rule(p)) // except_star_block @@ -38605,7 +38690,7 @@ _loop1_218_rule(Parser *p) _mark = p->mark; } p->mark = _mark; - D(fprintf(stderr, "%*c%s _loop1_218[%d-%d]: %s failed!\n", p->level, ' ', + D(fprintf(stderr, "%*c%s _loop1_219[%d-%d]: %s failed!\n", p->level, ' ', p->error_indicator ? "ERROR!" : "-", _mark, p->mark, "except_star_block")); } if (_n == 0 || p->error_indicator) { @@ -38627,9 +38712,9 @@ _loop1_218_rule(Parser *p) return _seq; } -// _tmp_219: expression ['as' NAME] +// _tmp_220: expression ['as' NAME] static void * -_tmp_219_rule(Parser *p) +_tmp_220_rule(Parser *p) { if (p->level++ == MAXSTACK) { _Pypegen_stack_overflow(p); @@ -38645,22 +38730,22 @@ _tmp_219_rule(Parser *p) p->level--; return NULL; } - D(fprintf(stderr, "%*c> _tmp_219[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "expression ['as' NAME]")); + D(fprintf(stderr, "%*c> _tmp_220[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "expression ['as' NAME]")); void *_opt_var; UNUSED(_opt_var); // Silence compiler warnings expr_ty expression_var; if ( (expression_var = expression_rule(p)) // expression && - (_opt_var = _tmp_272_rule(p), !p->error_indicator) // ['as' NAME] + (_opt_var = _tmp_274_rule(p), !p->error_indicator) // ['as' NAME] ) { - D(fprintf(stderr, "%*c+ _tmp_219[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "expression ['as' NAME]")); + D(fprintf(stderr, "%*c+ _tmp_220[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "expression ['as' NAME]")); _res = _PyPegen_dummy_name(p, expression_var, _opt_var); goto done; } p->mark = _mark; - D(fprintf(stderr, "%*c%s _tmp_219[%d-%d]: %s failed!\n", p->level, ' ', + D(fprintf(stderr, "%*c%s _tmp_220[%d-%d]: %s failed!\n", p->level, ' ', p->error_indicator ? "ERROR!" : "-", _mark, p->mark, "expression ['as' NAME]")); } _res = NULL; @@ -38669,9 +38754,9 @@ _tmp_219_rule(Parser *p) return _res; } -// _tmp_220: 'as' NAME +// _tmp_221: 'as' NAME static void * -_tmp_220_rule(Parser *p) +_tmp_221_rule(Parser *p) { if (p->level++ == MAXSTACK) { _Pypegen_stack_overflow(p); @@ -38687,21 +38772,21 @@ _tmp_220_rule(Parser *p) p->level--; return NULL; } - D(fprintf(stderr, "%*c> _tmp_220[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "'as' NAME")); + D(fprintf(stderr, "%*c> _tmp_221[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "'as' NAME")); Token * _keyword; expr_ty name_var; if ( - (_keyword = _PyPegen_expect_token(p, 656)) // token='as' + (_keyword = _PyPegen_expect_token(p, 659)) // token='as' && (name_var = _PyPegen_name_token(p)) // NAME ) { - D(fprintf(stderr, "%*c+ _tmp_220[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "'as' NAME")); + D(fprintf(stderr, "%*c+ _tmp_221[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "'as' NAME")); _res = _PyPegen_dummy_name(p, _keyword, name_var); goto done; } p->mark = _mark; - D(fprintf(stderr, "%*c%s _tmp_220[%d-%d]: %s failed!\n", p->level, ' ', + D(fprintf(stderr, "%*c%s _tmp_221[%d-%d]: %s failed!\n", p->level, ' ', p->error_indicator ? "ERROR!" : "-", _mark, p->mark, "'as' NAME")); } _res = NULL; @@ -38710,9 +38795,9 @@ _tmp_220_rule(Parser *p) return _res; } -// _tmp_221: 'as' NAME +// _tmp_222: 'as' NAME static void * -_tmp_221_rule(Parser *p) +_tmp_222_rule(Parser *p) { if (p->level++ == MAXSTACK) { _Pypegen_stack_overflow(p); @@ -38728,21 +38813,21 @@ _tmp_221_rule(Parser *p) p->level--; return NULL; } - D(fprintf(stderr, "%*c> _tmp_221[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "'as' NAME")); + D(fprintf(stderr, "%*c> _tmp_222[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "'as' NAME")); Token * _keyword; expr_ty name_var; if ( - (_keyword = _PyPegen_expect_token(p, 656)) // token='as' + (_keyword = _PyPegen_expect_token(p, 659)) // token='as' && (name_var = _PyPegen_name_token(p)) // NAME ) { - D(fprintf(stderr, "%*c+ _tmp_221[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "'as' NAME")); + D(fprintf(stderr, "%*c+ _tmp_222[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "'as' NAME")); _res = _PyPegen_dummy_name(p, _keyword, name_var); goto done; } p->mark = _mark; - D(fprintf(stderr, "%*c%s _tmp_221[%d-%d]: %s failed!\n", p->level, ' ', + D(fprintf(stderr, "%*c%s _tmp_222[%d-%d]: %s failed!\n", p->level, ' ', p->error_indicator ? "ERROR!" : "-", _mark, p->mark, "'as' NAME")); } _res = NULL; @@ -38751,9 +38836,9 @@ _tmp_221_rule(Parser *p) return _res; } -// _tmp_222: NEWLINE | ':' +// _tmp_223: NEWLINE | ':' static void * -_tmp_222_rule(Parser *p) +_tmp_223_rule(Parser *p) { if (p->level++ == MAXSTACK) { _Pypegen_stack_overflow(p); @@ -38769,18 +38854,18 @@ _tmp_222_rule(Parser *p) p->level--; return NULL; } - D(fprintf(stderr, "%*c> _tmp_222[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "NEWLINE")); + D(fprintf(stderr, "%*c> _tmp_223[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "NEWLINE")); Token * newline_var; if ( (newline_var = _PyPegen_expect_token(p, NEWLINE)) // token='NEWLINE' ) { - D(fprintf(stderr, "%*c+ _tmp_222[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "NEWLINE")); + D(fprintf(stderr, "%*c+ _tmp_223[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "NEWLINE")); _res = newline_var; goto done; } p->mark = _mark; - D(fprintf(stderr, "%*c%s _tmp_222[%d-%d]: %s failed!\n", p->level, ' ', + D(fprintf(stderr, "%*c%s _tmp_223[%d-%d]: %s failed!\n", p->level, ' ', p->error_indicator ? "ERROR!" : "-", _mark, p->mark, "NEWLINE")); } { // ':' @@ -38788,18 +38873,18 @@ _tmp_222_rule(Parser *p) p->level--; return NULL; } - D(fprintf(stderr, "%*c> _tmp_222[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "':'")); + D(fprintf(stderr, "%*c> _tmp_223[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "':'")); Token * _literal; if ( (_literal = _PyPegen_expect_token(p, 11)) // token=':' ) { - D(fprintf(stderr, "%*c+ _tmp_222[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "':'")); + D(fprintf(stderr, "%*c+ _tmp_223[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "':'")); _res = _literal; goto done; } p->mark = _mark; - D(fprintf(stderr, "%*c%s _tmp_222[%d-%d]: %s failed!\n", p->level, ' ', + D(fprintf(stderr, "%*c%s _tmp_223[%d-%d]: %s failed!\n", p->level, ' ', p->error_indicator ? "ERROR!" : "-", _mark, p->mark, "':'")); } _res = NULL; @@ -38808,9 +38893,9 @@ _tmp_222_rule(Parser *p) return _res; } -// _tmp_223: 'as' NAME +// _tmp_224: 'as' NAME static void * -_tmp_223_rule(Parser *p) +_tmp_224_rule(Parser *p) { if (p->level++ == MAXSTACK) { _Pypegen_stack_overflow(p); @@ -38826,21 +38911,21 @@ _tmp_223_rule(Parser *p) p->level--; return NULL; } - D(fprintf(stderr, "%*c> _tmp_223[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "'as' NAME")); + D(fprintf(stderr, "%*c> _tmp_224[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "'as' NAME")); Token * _keyword; expr_ty name_var; if ( - (_keyword = _PyPegen_expect_token(p, 656)) // token='as' + (_keyword = _PyPegen_expect_token(p, 659)) // token='as' && (name_var = _PyPegen_name_token(p)) // NAME ) { - D(fprintf(stderr, "%*c+ _tmp_223[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "'as' NAME")); + D(fprintf(stderr, "%*c+ _tmp_224[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "'as' NAME")); _res = _PyPegen_dummy_name(p, _keyword, name_var); goto done; } p->mark = _mark; - D(fprintf(stderr, "%*c%s _tmp_223[%d-%d]: %s failed!\n", p->level, ' ', + D(fprintf(stderr, "%*c%s _tmp_224[%d-%d]: %s failed!\n", p->level, ' ', p->error_indicator ? "ERROR!" : "-", _mark, p->mark, "'as' NAME")); } _res = NULL; @@ -38849,9 +38934,9 @@ _tmp_223_rule(Parser *p) return _res; } -// _tmp_224: 'as' NAME +// _tmp_225: 'as' NAME static void * -_tmp_224_rule(Parser *p) +_tmp_225_rule(Parser *p) { if (p->level++ == MAXSTACK) { _Pypegen_stack_overflow(p); @@ -38867,21 +38952,21 @@ _tmp_224_rule(Parser *p) p->level--; return NULL; } - D(fprintf(stderr, "%*c> _tmp_224[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "'as' NAME")); + D(fprintf(stderr, "%*c> _tmp_225[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "'as' NAME")); Token * _keyword; expr_ty name_var; if ( - (_keyword = _PyPegen_expect_token(p, 656)) // token='as' + (_keyword = _PyPegen_expect_token(p, 659)) // token='as' && (name_var = _PyPegen_name_token(p)) // NAME ) { - D(fprintf(stderr, "%*c+ _tmp_224[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "'as' NAME")); + D(fprintf(stderr, "%*c+ _tmp_225[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "'as' NAME")); _res = _PyPegen_dummy_name(p, _keyword, name_var); goto done; } p->mark = _mark; - D(fprintf(stderr, "%*c%s _tmp_224[%d-%d]: %s failed!\n", p->level, ' ', + D(fprintf(stderr, "%*c%s _tmp_225[%d-%d]: %s failed!\n", p->level, ' ', p->error_indicator ? "ERROR!" : "-", _mark, p->mark, "'as' NAME")); } _res = NULL; @@ -38890,9 +38975,9 @@ _tmp_224_rule(Parser *p) return _res; } -// _tmp_225: positional_patterns ',' +// _tmp_226: positional_patterns ',' static void * -_tmp_225_rule(Parser *p) +_tmp_226_rule(Parser *p) { if (p->level++ == MAXSTACK) { _Pypegen_stack_overflow(p); @@ -38908,7 +38993,7 @@ _tmp_225_rule(Parser *p) p->level--; return NULL; } - D(fprintf(stderr, "%*c> _tmp_225[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "positional_patterns ','")); + D(fprintf(stderr, "%*c> _tmp_226[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "positional_patterns ','")); Token * _literal; asdl_pattern_seq* positional_patterns_var; if ( @@ -38917,12 +39002,12 @@ _tmp_225_rule(Parser *p) (_literal = _PyPegen_expect_token(p, 12)) // token=',' ) { - D(fprintf(stderr, "%*c+ _tmp_225[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "positional_patterns ','")); + D(fprintf(stderr, "%*c+ _tmp_226[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "positional_patterns ','")); _res = _PyPegen_dummy_name(p, positional_patterns_var, _literal); goto done; } p->mark = _mark; - D(fprintf(stderr, "%*c%s _tmp_225[%d-%d]: %s failed!\n", p->level, ' ', + D(fprintf(stderr, "%*c%s _tmp_226[%d-%d]: %s failed!\n", p->level, ' ', p->error_indicator ? "ERROR!" : "-", _mark, p->mark, "positional_patterns ','")); } _res = NULL; @@ -38931,9 +39016,9 @@ _tmp_225_rule(Parser *p) return _res; } -// _tmp_226: '->' expression +// _tmp_227: '->' expression static void * -_tmp_226_rule(Parser *p) +_tmp_227_rule(Parser *p) { if (p->level++ == MAXSTACK) { _Pypegen_stack_overflow(p); @@ -38949,7 +39034,7 @@ _tmp_226_rule(Parser *p) p->level--; return NULL; } - D(fprintf(stderr, "%*c> _tmp_226[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "'->' expression")); + D(fprintf(stderr, "%*c> _tmp_227[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "'->' expression")); Token * _literal; expr_ty expression_var; if ( @@ -38958,12 +39043,12 @@ _tmp_226_rule(Parser *p) (expression_var = expression_rule(p)) // expression ) { - D(fprintf(stderr, "%*c+ _tmp_226[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "'->' expression")); + D(fprintf(stderr, "%*c+ _tmp_227[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "'->' expression")); _res = _PyPegen_dummy_name(p, _literal, expression_var); goto done; } p->mark = _mark; - D(fprintf(stderr, "%*c%s _tmp_226[%d-%d]: %s failed!\n", p->level, ' ', + D(fprintf(stderr, "%*c%s _tmp_227[%d-%d]: %s failed!\n", p->level, ' ', p->error_indicator ? "ERROR!" : "-", _mark, p->mark, "'->' expression")); } _res = NULL; @@ -38972,9 +39057,9 @@ _tmp_226_rule(Parser *p) return _res; } -// _tmp_227: '(' arguments? ')' +// _tmp_228: '(' arguments? ')' static void * -_tmp_227_rule(Parser *p) +_tmp_228_rule(Parser *p) { if (p->level++ == MAXSTACK) { _Pypegen_stack_overflow(p); @@ -38990,7 +39075,7 @@ _tmp_227_rule(Parser *p) p->level--; return NULL; } - D(fprintf(stderr, "%*c> _tmp_227[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "'(' arguments? ')'")); + D(fprintf(stderr, "%*c> _tmp_228[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "'(' arguments? ')'")); Token * _literal; Token * _literal_1; void *_opt_var; @@ -39003,12 +39088,12 @@ _tmp_227_rule(Parser *p) (_literal_1 = _PyPegen_expect_token(p, 8)) // token=')' ) { - D(fprintf(stderr, "%*c+ _tmp_227[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "'(' arguments? ')'")); + D(fprintf(stderr, "%*c+ _tmp_228[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "'(' arguments? ')'")); _res = _PyPegen_dummy_name(p, _literal, _opt_var, _literal_1); goto done; } p->mark = _mark; - D(fprintf(stderr, "%*c%s _tmp_227[%d-%d]: %s failed!\n", p->level, ' ', + D(fprintf(stderr, "%*c%s _tmp_228[%d-%d]: %s failed!\n", p->level, ' ', p->error_indicator ? "ERROR!" : "-", _mark, p->mark, "'(' arguments? ')'")); } _res = NULL; @@ -39017,9 +39102,9 @@ _tmp_227_rule(Parser *p) return _res; } -// _tmp_228: '(' arguments? ')' +// _tmp_229: '(' arguments? ')' static void * -_tmp_228_rule(Parser *p) +_tmp_229_rule(Parser *p) { if (p->level++ == MAXSTACK) { _Pypegen_stack_overflow(p); @@ -39035,7 +39120,7 @@ _tmp_228_rule(Parser *p) p->level--; return NULL; } - D(fprintf(stderr, "%*c> _tmp_228[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "'(' arguments? ')'")); + D(fprintf(stderr, "%*c> _tmp_229[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "'(' arguments? ')'")); Token * _literal; Token * _literal_1; void *_opt_var; @@ -39048,12 +39133,12 @@ _tmp_228_rule(Parser *p) (_literal_1 = _PyPegen_expect_token(p, 8)) // token=')' ) { - D(fprintf(stderr, "%*c+ _tmp_228[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "'(' arguments? ')'")); + D(fprintf(stderr, "%*c+ _tmp_229[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "'(' arguments? ')'")); _res = _PyPegen_dummy_name(p, _literal, _opt_var, _literal_1); goto done; } p->mark = _mark; - D(fprintf(stderr, "%*c%s _tmp_228[%d-%d]: %s failed!\n", p->level, ' ', + D(fprintf(stderr, "%*c%s _tmp_229[%d-%d]: %s failed!\n", p->level, ' ', p->error_indicator ? "ERROR!" : "-", _mark, p->mark, "'(' arguments? ')'")); } _res = NULL; @@ -39062,9 +39147,9 @@ _tmp_228_rule(Parser *p) return _res; } -// _loop0_230: ',' double_starred_kvpair +// _loop0_231: ',' double_starred_kvpair static asdl_seq * -_loop0_230_rule(Parser *p) +_loop0_231_rule(Parser *p) { if (p->level++ == MAXSTACK) { _Pypegen_stack_overflow(p); @@ -39089,7 +39174,7 @@ _loop0_230_rule(Parser *p) p->level--; return NULL; } - D(fprintf(stderr, "%*c> _loop0_230[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "',' double_starred_kvpair")); + D(fprintf(stderr, "%*c> _loop0_231[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "',' double_starred_kvpair")); Token * _literal; KeyValuePair* elem; while ( @@ -39121,7 +39206,7 @@ _loop0_230_rule(Parser *p) _mark = p->mark; } p->mark = _mark; - D(fprintf(stderr, "%*c%s _loop0_230[%d-%d]: %s failed!\n", p->level, ' ', + D(fprintf(stderr, "%*c%s _loop0_231[%d-%d]: %s failed!\n", p->level, ' ', p->error_indicator ? "ERROR!" : "-", _mark, p->mark, "',' double_starred_kvpair")); } asdl_seq *_seq = (asdl_seq*)_Py_asdl_generic_seq_new(_n, p->arena); @@ -39138,9 +39223,9 @@ _loop0_230_rule(Parser *p) return _seq; } -// _gather_229: double_starred_kvpair _loop0_230 +// _gather_230: double_starred_kvpair _loop0_231 static asdl_seq * -_gather_229_rule(Parser *p) +_gather_230_rule(Parser *p) { if (p->level++ == MAXSTACK) { _Pypegen_stack_overflow(p); @@ -39151,27 +39236,27 @@ _gather_229_rule(Parser *p) } asdl_seq * _res = NULL; int _mark = p->mark; - { // double_starred_kvpair _loop0_230 + { // double_starred_kvpair _loop0_231 if (p->error_indicator) { p->level--; return NULL; } - D(fprintf(stderr, "%*c> _gather_229[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "double_starred_kvpair _loop0_230")); + D(fprintf(stderr, "%*c> _gather_230[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "double_starred_kvpair _loop0_231")); KeyValuePair* elem; asdl_seq * seq; if ( (elem = double_starred_kvpair_rule(p)) // double_starred_kvpair && - (seq = _loop0_230_rule(p)) // _loop0_230 + (seq = _loop0_231_rule(p)) // _loop0_231 ) { - D(fprintf(stderr, "%*c+ _gather_229[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "double_starred_kvpair _loop0_230")); + D(fprintf(stderr, "%*c+ _gather_230[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "double_starred_kvpair _loop0_231")); _res = _PyPegen_seq_insert_in_front(p, elem, seq); goto done; } p->mark = _mark; - D(fprintf(stderr, "%*c%s _gather_229[%d-%d]: %s failed!\n", p->level, ' ', - p->error_indicator ? "ERROR!" : "-", _mark, p->mark, "double_starred_kvpair _loop0_230")); + D(fprintf(stderr, "%*c%s _gather_230[%d-%d]: %s failed!\n", p->level, ' ', + p->error_indicator ? "ERROR!" : "-", _mark, p->mark, "double_starred_kvpair _loop0_231")); } _res = NULL; done: @@ -39179,9 +39264,9 @@ _gather_229_rule(Parser *p) return _res; } -// _tmp_231: '}' | ',' +// _tmp_232: '}' | ',' static void * -_tmp_231_rule(Parser *p) +_tmp_232_rule(Parser *p) { if (p->level++ == MAXSTACK) { _Pypegen_stack_overflow(p); @@ -39197,18 +39282,18 @@ _tmp_231_rule(Parser *p) p->level--; return NULL; } - D(fprintf(stderr, "%*c> _tmp_231[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "'}'")); + D(fprintf(stderr, "%*c> _tmp_232[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "'}'")); Token * _literal; if ( (_literal = _PyPegen_expect_token(p, 26)) // token='}' ) { - D(fprintf(stderr, "%*c+ _tmp_231[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "'}'")); + D(fprintf(stderr, "%*c+ _tmp_232[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "'}'")); _res = _literal; goto done; } p->mark = _mark; - D(fprintf(stderr, "%*c%s _tmp_231[%d-%d]: %s failed!\n", p->level, ' ', + D(fprintf(stderr, "%*c%s _tmp_232[%d-%d]: %s failed!\n", p->level, ' ', p->error_indicator ? "ERROR!" : "-", _mark, p->mark, "'}'")); } { // ',' @@ -39216,18 +39301,18 @@ _tmp_231_rule(Parser *p) p->level--; return NULL; } - D(fprintf(stderr, "%*c> _tmp_231[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "','")); + D(fprintf(stderr, "%*c> _tmp_232[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "','")); Token * _literal; if ( (_literal = _PyPegen_expect_token(p, 12)) // token=',' ) { - D(fprintf(stderr, "%*c+ _tmp_231[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "','")); + D(fprintf(stderr, "%*c+ _tmp_232[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "','")); _res = _literal; goto done; } p->mark = _mark; - D(fprintf(stderr, "%*c%s _tmp_231[%d-%d]: %s failed!\n", p->level, ' ', + D(fprintf(stderr, "%*c%s _tmp_232[%d-%d]: %s failed!\n", p->level, ' ', p->error_indicator ? "ERROR!" : "-", _mark, p->mark, "','")); } _res = NULL; @@ -39236,9 +39321,9 @@ _tmp_231_rule(Parser *p) return _res; } -// _tmp_232: '}' | ',' +// _tmp_233: '}' | ',' static void * -_tmp_232_rule(Parser *p) +_tmp_233_rule(Parser *p) { if (p->level++ == MAXSTACK) { _Pypegen_stack_overflow(p); @@ -39254,18 +39339,18 @@ _tmp_232_rule(Parser *p) p->level--; return NULL; } - D(fprintf(stderr, "%*c> _tmp_232[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "'}'")); + D(fprintf(stderr, "%*c> _tmp_233[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "'}'")); Token * _literal; if ( (_literal = _PyPegen_expect_token(p, 26)) // token='}' ) { - D(fprintf(stderr, "%*c+ _tmp_232[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "'}'")); + D(fprintf(stderr, "%*c+ _tmp_233[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "'}'")); _res = _literal; goto done; } p->mark = _mark; - D(fprintf(stderr, "%*c%s _tmp_232[%d-%d]: %s failed!\n", p->level, ' ', + D(fprintf(stderr, "%*c%s _tmp_233[%d-%d]: %s failed!\n", p->level, ' ', p->error_indicator ? "ERROR!" : "-", _mark, p->mark, "'}'")); } { // ',' @@ -39273,18 +39358,18 @@ _tmp_232_rule(Parser *p) p->level--; return NULL; } - D(fprintf(stderr, "%*c> _tmp_232[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "','")); + D(fprintf(stderr, "%*c> _tmp_233[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "','")); Token * _literal; if ( (_literal = _PyPegen_expect_token(p, 12)) // token=',' ) { - D(fprintf(stderr, "%*c+ _tmp_232[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "','")); + D(fprintf(stderr, "%*c+ _tmp_233[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "','")); _res = _literal; goto done; } p->mark = _mark; - D(fprintf(stderr, "%*c%s _tmp_232[%d-%d]: %s failed!\n", p->level, ' ', + D(fprintf(stderr, "%*c%s _tmp_233[%d-%d]: %s failed!\n", p->level, ' ', p->error_indicator ? "ERROR!" : "-", _mark, p->mark, "','")); } _res = NULL; @@ -39293,9 +39378,9 @@ _tmp_232_rule(Parser *p) return _res; } -// _tmp_233: yield_expr | star_expressions +// _tmp_234: yield_expr | star_expressions static void * -_tmp_233_rule(Parser *p) +_tmp_234_rule(Parser *p) { if (p->level++ == MAXSTACK) { _Pypegen_stack_overflow(p); @@ -39311,18 +39396,18 @@ _tmp_233_rule(Parser *p) p->level--; return NULL; } - D(fprintf(stderr, "%*c> _tmp_233[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "yield_expr")); + D(fprintf(stderr, "%*c> _tmp_234[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "yield_expr")); expr_ty yield_expr_var; if ( (yield_expr_var = yield_expr_rule(p)) // yield_expr ) { - D(fprintf(stderr, "%*c+ _tmp_233[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "yield_expr")); + D(fprintf(stderr, "%*c+ _tmp_234[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "yield_expr")); _res = yield_expr_var; goto done; } p->mark = _mark; - D(fprintf(stderr, "%*c%s _tmp_233[%d-%d]: %s failed!\n", p->level, ' ', + D(fprintf(stderr, "%*c%s _tmp_234[%d-%d]: %s failed!\n", p->level, ' ', p->error_indicator ? "ERROR!" : "-", _mark, p->mark, "yield_expr")); } { // star_expressions @@ -39330,18 +39415,18 @@ _tmp_233_rule(Parser *p) p->level--; return NULL; } - D(fprintf(stderr, "%*c> _tmp_233[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "star_expressions")); + D(fprintf(stderr, "%*c> _tmp_234[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "star_expressions")); expr_ty star_expressions_var; if ( (star_expressions_var = star_expressions_rule(p)) // star_expressions ) { - D(fprintf(stderr, "%*c+ _tmp_233[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "star_expressions")); + D(fprintf(stderr, "%*c+ _tmp_234[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "star_expressions")); _res = star_expressions_var; goto done; } p->mark = _mark; - D(fprintf(stderr, "%*c%s _tmp_233[%d-%d]: %s failed!\n", p->level, ' ', + D(fprintf(stderr, "%*c%s _tmp_234[%d-%d]: %s failed!\n", p->level, ' ', p->error_indicator ? "ERROR!" : "-", _mark, p->mark, "star_expressions")); } _res = NULL; @@ -39350,9 +39435,9 @@ _tmp_233_rule(Parser *p) return _res; } -// _tmp_234: yield_expr | star_expressions +// _tmp_235: yield_expr | star_expressions static void * -_tmp_234_rule(Parser *p) +_tmp_235_rule(Parser *p) { if (p->level++ == MAXSTACK) { _Pypegen_stack_overflow(p); @@ -39368,18 +39453,18 @@ _tmp_234_rule(Parser *p) p->level--; return NULL; } - D(fprintf(stderr, "%*c> _tmp_234[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "yield_expr")); + D(fprintf(stderr, "%*c> _tmp_235[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "yield_expr")); expr_ty yield_expr_var; if ( (yield_expr_var = yield_expr_rule(p)) // yield_expr ) { - D(fprintf(stderr, "%*c+ _tmp_234[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "yield_expr")); + D(fprintf(stderr, "%*c+ _tmp_235[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "yield_expr")); _res = yield_expr_var; goto done; } p->mark = _mark; - D(fprintf(stderr, "%*c%s _tmp_234[%d-%d]: %s failed!\n", p->level, ' ', + D(fprintf(stderr, "%*c%s _tmp_235[%d-%d]: %s failed!\n", p->level, ' ', p->error_indicator ? "ERROR!" : "-", _mark, p->mark, "yield_expr")); } { // star_expressions @@ -39387,18 +39472,18 @@ _tmp_234_rule(Parser *p) p->level--; return NULL; } - D(fprintf(stderr, "%*c> _tmp_234[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "star_expressions")); + D(fprintf(stderr, "%*c> _tmp_235[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "star_expressions")); expr_ty star_expressions_var; if ( (star_expressions_var = star_expressions_rule(p)) // star_expressions ) { - D(fprintf(stderr, "%*c+ _tmp_234[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "star_expressions")); + D(fprintf(stderr, "%*c+ _tmp_235[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "star_expressions")); _res = star_expressions_var; goto done; } p->mark = _mark; - D(fprintf(stderr, "%*c%s _tmp_234[%d-%d]: %s failed!\n", p->level, ' ', + D(fprintf(stderr, "%*c%s _tmp_235[%d-%d]: %s failed!\n", p->level, ' ', p->error_indicator ? "ERROR!" : "-", _mark, p->mark, "star_expressions")); } _res = NULL; @@ -39407,9 +39492,9 @@ _tmp_234_rule(Parser *p) return _res; } -// _tmp_235: '=' | '!' | ':' | '}' +// _tmp_236: '=' | '!' | ':' | '}' static void * -_tmp_235_rule(Parser *p) +_tmp_236_rule(Parser *p) { if (p->level++ == MAXSTACK) { _Pypegen_stack_overflow(p); @@ -39425,18 +39510,18 @@ _tmp_235_rule(Parser *p) p->level--; return NULL; } - D(fprintf(stderr, "%*c> _tmp_235[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "'='")); + D(fprintf(stderr, "%*c> _tmp_236[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "'='")); Token * _literal; if ( (_literal = _PyPegen_expect_token(p, 22)) // token='=' ) { - D(fprintf(stderr, "%*c+ _tmp_235[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "'='")); + D(fprintf(stderr, "%*c+ _tmp_236[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "'='")); _res = _literal; goto done; } p->mark = _mark; - D(fprintf(stderr, "%*c%s _tmp_235[%d-%d]: %s failed!\n", p->level, ' ', + D(fprintf(stderr, "%*c%s _tmp_236[%d-%d]: %s failed!\n", p->level, ' ', p->error_indicator ? "ERROR!" : "-", _mark, p->mark, "'='")); } { // '!' @@ -39444,18 +39529,18 @@ _tmp_235_rule(Parser *p) p->level--; return NULL; } - D(fprintf(stderr, "%*c> _tmp_235[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "'!'")); + D(fprintf(stderr, "%*c> _tmp_236[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "'!'")); Token * _literal; if ( (_literal = _PyPegen_expect_token(p, 54)) // token='!' ) { - D(fprintf(stderr, "%*c+ _tmp_235[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "'!'")); + D(fprintf(stderr, "%*c+ _tmp_236[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "'!'")); _res = _literal; goto done; } p->mark = _mark; - D(fprintf(stderr, "%*c%s _tmp_235[%d-%d]: %s failed!\n", p->level, ' ', + D(fprintf(stderr, "%*c%s _tmp_236[%d-%d]: %s failed!\n", p->level, ' ', p->error_indicator ? "ERROR!" : "-", _mark, p->mark, "'!'")); } { // ':' @@ -39463,18 +39548,18 @@ _tmp_235_rule(Parser *p) p->level--; return NULL; } - D(fprintf(stderr, "%*c> _tmp_235[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "':'")); + D(fprintf(stderr, "%*c> _tmp_236[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "':'")); Token * _literal; if ( (_literal = _PyPegen_expect_token(p, 11)) // token=':' ) { - D(fprintf(stderr, "%*c+ _tmp_235[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "':'")); + D(fprintf(stderr, "%*c+ _tmp_236[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "':'")); _res = _literal; goto done; } p->mark = _mark; - D(fprintf(stderr, "%*c%s _tmp_235[%d-%d]: %s failed!\n", p->level, ' ', + D(fprintf(stderr, "%*c%s _tmp_236[%d-%d]: %s failed!\n", p->level, ' ', p->error_indicator ? "ERROR!" : "-", _mark, p->mark, "':'")); } { // '}' @@ -39482,18 +39567,18 @@ _tmp_235_rule(Parser *p) p->level--; return NULL; } - D(fprintf(stderr, "%*c> _tmp_235[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "'}'")); + D(fprintf(stderr, "%*c> _tmp_236[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "'}'")); Token * _literal; if ( (_literal = _PyPegen_expect_token(p, 26)) // token='}' ) { - D(fprintf(stderr, "%*c+ _tmp_235[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "'}'")); + D(fprintf(stderr, "%*c+ _tmp_236[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "'}'")); _res = _literal; goto done; } p->mark = _mark; - D(fprintf(stderr, "%*c%s _tmp_235[%d-%d]: %s failed!\n", p->level, ' ', + D(fprintf(stderr, "%*c%s _tmp_236[%d-%d]: %s failed!\n", p->level, ' ', p->error_indicator ? "ERROR!" : "-", _mark, p->mark, "'}'")); } _res = NULL; @@ -39502,9 +39587,9 @@ _tmp_235_rule(Parser *p) return _res; } -// _tmp_236: yield_expr | star_expressions +// _tmp_237: yield_expr | star_expressions static void * -_tmp_236_rule(Parser *p) +_tmp_237_rule(Parser *p) { if (p->level++ == MAXSTACK) { _Pypegen_stack_overflow(p); @@ -39520,18 +39605,18 @@ _tmp_236_rule(Parser *p) p->level--; return NULL; } - D(fprintf(stderr, "%*c> _tmp_236[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "yield_expr")); + D(fprintf(stderr, "%*c> _tmp_237[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "yield_expr")); expr_ty yield_expr_var; if ( (yield_expr_var = yield_expr_rule(p)) // yield_expr ) { - D(fprintf(stderr, "%*c+ _tmp_236[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "yield_expr")); + D(fprintf(stderr, "%*c+ _tmp_237[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "yield_expr")); _res = yield_expr_var; goto done; } p->mark = _mark; - D(fprintf(stderr, "%*c%s _tmp_236[%d-%d]: %s failed!\n", p->level, ' ', + D(fprintf(stderr, "%*c%s _tmp_237[%d-%d]: %s failed!\n", p->level, ' ', p->error_indicator ? "ERROR!" : "-", _mark, p->mark, "yield_expr")); } { // star_expressions @@ -39539,18 +39624,18 @@ _tmp_236_rule(Parser *p) p->level--; return NULL; } - D(fprintf(stderr, "%*c> _tmp_236[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "star_expressions")); + D(fprintf(stderr, "%*c> _tmp_237[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "star_expressions")); expr_ty star_expressions_var; if ( (star_expressions_var = star_expressions_rule(p)) // star_expressions ) { - D(fprintf(stderr, "%*c+ _tmp_236[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "star_expressions")); + D(fprintf(stderr, "%*c+ _tmp_237[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "star_expressions")); _res = star_expressions_var; goto done; } p->mark = _mark; - D(fprintf(stderr, "%*c%s _tmp_236[%d-%d]: %s failed!\n", p->level, ' ', + D(fprintf(stderr, "%*c%s _tmp_237[%d-%d]: %s failed!\n", p->level, ' ', p->error_indicator ? "ERROR!" : "-", _mark, p->mark, "star_expressions")); } _res = NULL; @@ -39559,9 +39644,9 @@ _tmp_236_rule(Parser *p) return _res; } -// _tmp_237: '!' | ':' | '}' +// _tmp_238: '!' | ':' | '}' static void * -_tmp_237_rule(Parser *p) +_tmp_238_rule(Parser *p) { if (p->level++ == MAXSTACK) { _Pypegen_stack_overflow(p); @@ -39577,18 +39662,18 @@ _tmp_237_rule(Parser *p) p->level--; return NULL; } - D(fprintf(stderr, "%*c> _tmp_237[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "'!'")); + D(fprintf(stderr, "%*c> _tmp_238[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "'!'")); Token * _literal; if ( (_literal = _PyPegen_expect_token(p, 54)) // token='!' ) { - D(fprintf(stderr, "%*c+ _tmp_237[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "'!'")); + D(fprintf(stderr, "%*c+ _tmp_238[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "'!'")); _res = _literal; goto done; } p->mark = _mark; - D(fprintf(stderr, "%*c%s _tmp_237[%d-%d]: %s failed!\n", p->level, ' ', + D(fprintf(stderr, "%*c%s _tmp_238[%d-%d]: %s failed!\n", p->level, ' ', p->error_indicator ? "ERROR!" : "-", _mark, p->mark, "'!'")); } { // ':' @@ -39596,18 +39681,18 @@ _tmp_237_rule(Parser *p) p->level--; return NULL; } - D(fprintf(stderr, "%*c> _tmp_237[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "':'")); + D(fprintf(stderr, "%*c> _tmp_238[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "':'")); Token * _literal; if ( (_literal = _PyPegen_expect_token(p, 11)) // token=':' ) { - D(fprintf(stderr, "%*c+ _tmp_237[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "':'")); + D(fprintf(stderr, "%*c+ _tmp_238[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "':'")); _res = _literal; goto done; } p->mark = _mark; - D(fprintf(stderr, "%*c%s _tmp_237[%d-%d]: %s failed!\n", p->level, ' ', + D(fprintf(stderr, "%*c%s _tmp_238[%d-%d]: %s failed!\n", p->level, ' ', p->error_indicator ? "ERROR!" : "-", _mark, p->mark, "':'")); } { // '}' @@ -39615,18 +39700,18 @@ _tmp_237_rule(Parser *p) p->level--; return NULL; } - D(fprintf(stderr, "%*c> _tmp_237[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "'}'")); + D(fprintf(stderr, "%*c> _tmp_238[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "'}'")); Token * _literal; if ( (_literal = _PyPegen_expect_token(p, 26)) // token='}' ) { - D(fprintf(stderr, "%*c+ _tmp_237[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "'}'")); + D(fprintf(stderr, "%*c+ _tmp_238[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "'}'")); _res = _literal; goto done; } p->mark = _mark; - D(fprintf(stderr, "%*c%s _tmp_237[%d-%d]: %s failed!\n", p->level, ' ', + D(fprintf(stderr, "%*c%s _tmp_238[%d-%d]: %s failed!\n", p->level, ' ', p->error_indicator ? "ERROR!" : "-", _mark, p->mark, "'}'")); } _res = NULL; @@ -39635,9 +39720,9 @@ _tmp_237_rule(Parser *p) return _res; } -// _tmp_238: yield_expr | star_expressions +// _tmp_239: yield_expr | star_expressions static void * -_tmp_238_rule(Parser *p) +_tmp_239_rule(Parser *p) { if (p->level++ == MAXSTACK) { _Pypegen_stack_overflow(p); @@ -39653,18 +39738,18 @@ _tmp_238_rule(Parser *p) p->level--; return NULL; } - D(fprintf(stderr, "%*c> _tmp_238[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "yield_expr")); + D(fprintf(stderr, "%*c> _tmp_239[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "yield_expr")); expr_ty yield_expr_var; if ( (yield_expr_var = yield_expr_rule(p)) // yield_expr ) { - D(fprintf(stderr, "%*c+ _tmp_238[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "yield_expr")); + D(fprintf(stderr, "%*c+ _tmp_239[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "yield_expr")); _res = yield_expr_var; goto done; } p->mark = _mark; - D(fprintf(stderr, "%*c%s _tmp_238[%d-%d]: %s failed!\n", p->level, ' ', + D(fprintf(stderr, "%*c%s _tmp_239[%d-%d]: %s failed!\n", p->level, ' ', p->error_indicator ? "ERROR!" : "-", _mark, p->mark, "yield_expr")); } { // star_expressions @@ -39672,18 +39757,18 @@ _tmp_238_rule(Parser *p) p->level--; return NULL; } - D(fprintf(stderr, "%*c> _tmp_238[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "star_expressions")); + D(fprintf(stderr, "%*c> _tmp_239[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "star_expressions")); expr_ty star_expressions_var; if ( (star_expressions_var = star_expressions_rule(p)) // star_expressions ) { - D(fprintf(stderr, "%*c+ _tmp_238[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "star_expressions")); + D(fprintf(stderr, "%*c+ _tmp_239[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "star_expressions")); _res = star_expressions_var; goto done; } p->mark = _mark; - D(fprintf(stderr, "%*c%s _tmp_238[%d-%d]: %s failed!\n", p->level, ' ', + D(fprintf(stderr, "%*c%s _tmp_239[%d-%d]: %s failed!\n", p->level, ' ', p->error_indicator ? "ERROR!" : "-", _mark, p->mark, "star_expressions")); } _res = NULL; @@ -39692,9 +39777,9 @@ _tmp_238_rule(Parser *p) return _res; } -// _tmp_239: yield_expr | star_expressions +// _tmp_240: yield_expr | star_expressions static void * -_tmp_239_rule(Parser *p) +_tmp_240_rule(Parser *p) { if (p->level++ == MAXSTACK) { _Pypegen_stack_overflow(p); @@ -39710,18 +39795,18 @@ _tmp_239_rule(Parser *p) p->level--; return NULL; } - D(fprintf(stderr, "%*c> _tmp_239[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "yield_expr")); + D(fprintf(stderr, "%*c> _tmp_240[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "yield_expr")); expr_ty yield_expr_var; if ( (yield_expr_var = yield_expr_rule(p)) // yield_expr ) { - D(fprintf(stderr, "%*c+ _tmp_239[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "yield_expr")); + D(fprintf(stderr, "%*c+ _tmp_240[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "yield_expr")); _res = yield_expr_var; goto done; } p->mark = _mark; - D(fprintf(stderr, "%*c%s _tmp_239[%d-%d]: %s failed!\n", p->level, ' ', + D(fprintf(stderr, "%*c%s _tmp_240[%d-%d]: %s failed!\n", p->level, ' ', p->error_indicator ? "ERROR!" : "-", _mark, p->mark, "yield_expr")); } { // star_expressions @@ -39729,18 +39814,18 @@ _tmp_239_rule(Parser *p) p->level--; return NULL; } - D(fprintf(stderr, "%*c> _tmp_239[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "star_expressions")); + D(fprintf(stderr, "%*c> _tmp_240[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "star_expressions")); expr_ty star_expressions_var; if ( (star_expressions_var = star_expressions_rule(p)) // star_expressions ) { - D(fprintf(stderr, "%*c+ _tmp_239[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "star_expressions")); + D(fprintf(stderr, "%*c+ _tmp_240[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "star_expressions")); _res = star_expressions_var; goto done; } p->mark = _mark; - D(fprintf(stderr, "%*c%s _tmp_239[%d-%d]: %s failed!\n", p->level, ' ', + D(fprintf(stderr, "%*c%s _tmp_240[%d-%d]: %s failed!\n", p->level, ' ', p->error_indicator ? "ERROR!" : "-", _mark, p->mark, "star_expressions")); } _res = NULL; @@ -39749,9 +39834,9 @@ _tmp_239_rule(Parser *p) return _res; } -// _tmp_240: '!' NAME +// _tmp_241: '!' NAME static void * -_tmp_240_rule(Parser *p) +_tmp_241_rule(Parser *p) { if (p->level++ == MAXSTACK) { _Pypegen_stack_overflow(p); @@ -39767,7 +39852,7 @@ _tmp_240_rule(Parser *p) p->level--; return NULL; } - D(fprintf(stderr, "%*c> _tmp_240[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "'!' NAME")); + D(fprintf(stderr, "%*c> _tmp_241[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "'!' NAME")); Token * _literal; expr_ty name_var; if ( @@ -39776,12 +39861,12 @@ _tmp_240_rule(Parser *p) (name_var = _PyPegen_name_token(p)) // NAME ) { - D(fprintf(stderr, "%*c+ _tmp_240[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "'!' NAME")); + D(fprintf(stderr, "%*c+ _tmp_241[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "'!' NAME")); _res = _PyPegen_dummy_name(p, _literal, name_var); goto done; } p->mark = _mark; - D(fprintf(stderr, "%*c%s _tmp_240[%d-%d]: %s failed!\n", p->level, ' ', + D(fprintf(stderr, "%*c%s _tmp_241[%d-%d]: %s failed!\n", p->level, ' ', p->error_indicator ? "ERROR!" : "-", _mark, p->mark, "'!' NAME")); } _res = NULL; @@ -39790,9 +39875,9 @@ _tmp_240_rule(Parser *p) return _res; } -// _tmp_241: ':' | '}' +// _tmp_242: ':' | '}' static void * -_tmp_241_rule(Parser *p) +_tmp_242_rule(Parser *p) { if (p->level++ == MAXSTACK) { _Pypegen_stack_overflow(p); @@ -39808,18 +39893,18 @@ _tmp_241_rule(Parser *p) p->level--; return NULL; } - D(fprintf(stderr, "%*c> _tmp_241[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "':'")); + D(fprintf(stderr, "%*c> _tmp_242[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "':'")); Token * _literal; if ( (_literal = _PyPegen_expect_token(p, 11)) // token=':' ) { - D(fprintf(stderr, "%*c+ _tmp_241[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "':'")); + D(fprintf(stderr, "%*c+ _tmp_242[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "':'")); _res = _literal; goto done; } p->mark = _mark; - D(fprintf(stderr, "%*c%s _tmp_241[%d-%d]: %s failed!\n", p->level, ' ', + D(fprintf(stderr, "%*c%s _tmp_242[%d-%d]: %s failed!\n", p->level, ' ', p->error_indicator ? "ERROR!" : "-", _mark, p->mark, "':'")); } { // '}' @@ -39827,18 +39912,18 @@ _tmp_241_rule(Parser *p) p->level--; return NULL; } - D(fprintf(stderr, "%*c> _tmp_241[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "'}'")); + D(fprintf(stderr, "%*c> _tmp_242[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "'}'")); Token * _literal; if ( (_literal = _PyPegen_expect_token(p, 26)) // token='}' ) { - D(fprintf(stderr, "%*c+ _tmp_241[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "'}'")); + D(fprintf(stderr, "%*c+ _tmp_242[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "'}'")); _res = _literal; goto done; } p->mark = _mark; - D(fprintf(stderr, "%*c%s _tmp_241[%d-%d]: %s failed!\n", p->level, ' ', + D(fprintf(stderr, "%*c%s _tmp_242[%d-%d]: %s failed!\n", p->level, ' ', p->error_indicator ? "ERROR!" : "-", _mark, p->mark, "'}'")); } _res = NULL; @@ -39847,9 +39932,9 @@ _tmp_241_rule(Parser *p) return _res; } -// _tmp_242: yield_expr | star_expressions +// _tmp_243: yield_expr | star_expressions static void * -_tmp_242_rule(Parser *p) +_tmp_243_rule(Parser *p) { if (p->level++ == MAXSTACK) { _Pypegen_stack_overflow(p); @@ -39865,18 +39950,18 @@ _tmp_242_rule(Parser *p) p->level--; return NULL; } - D(fprintf(stderr, "%*c> _tmp_242[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "yield_expr")); + D(fprintf(stderr, "%*c> _tmp_243[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "yield_expr")); expr_ty yield_expr_var; if ( (yield_expr_var = yield_expr_rule(p)) // yield_expr ) { - D(fprintf(stderr, "%*c+ _tmp_242[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "yield_expr")); + D(fprintf(stderr, "%*c+ _tmp_243[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "yield_expr")); _res = yield_expr_var; goto done; } p->mark = _mark; - D(fprintf(stderr, "%*c%s _tmp_242[%d-%d]: %s failed!\n", p->level, ' ', + D(fprintf(stderr, "%*c%s _tmp_243[%d-%d]: %s failed!\n", p->level, ' ', p->error_indicator ? "ERROR!" : "-", _mark, p->mark, "yield_expr")); } { // star_expressions @@ -39884,18 +39969,18 @@ _tmp_242_rule(Parser *p) p->level--; return NULL; } - D(fprintf(stderr, "%*c> _tmp_242[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "star_expressions")); + D(fprintf(stderr, "%*c> _tmp_243[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "star_expressions")); expr_ty star_expressions_var; if ( (star_expressions_var = star_expressions_rule(p)) // star_expressions ) { - D(fprintf(stderr, "%*c+ _tmp_242[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "star_expressions")); + D(fprintf(stderr, "%*c+ _tmp_243[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "star_expressions")); _res = star_expressions_var; goto done; } p->mark = _mark; - D(fprintf(stderr, "%*c%s _tmp_242[%d-%d]: %s failed!\n", p->level, ' ', + D(fprintf(stderr, "%*c%s _tmp_243[%d-%d]: %s failed!\n", p->level, ' ', p->error_indicator ? "ERROR!" : "-", _mark, p->mark, "star_expressions")); } _res = NULL; @@ -39904,9 +39989,9 @@ _tmp_242_rule(Parser *p) return _res; } -// _tmp_243: '!' NAME +// _tmp_244: '!' NAME static void * -_tmp_243_rule(Parser *p) +_tmp_244_rule(Parser *p) { if (p->level++ == MAXSTACK) { _Pypegen_stack_overflow(p); @@ -39922,7 +40007,7 @@ _tmp_243_rule(Parser *p) p->level--; return NULL; } - D(fprintf(stderr, "%*c> _tmp_243[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "'!' NAME")); + D(fprintf(stderr, "%*c> _tmp_244[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "'!' NAME")); Token * _literal; expr_ty name_var; if ( @@ -39931,12 +40016,12 @@ _tmp_243_rule(Parser *p) (name_var = _PyPegen_name_token(p)) // NAME ) { - D(fprintf(stderr, "%*c+ _tmp_243[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "'!' NAME")); + D(fprintf(stderr, "%*c+ _tmp_244[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "'!' NAME")); _res = _PyPegen_dummy_name(p, _literal, name_var); goto done; } p->mark = _mark; - D(fprintf(stderr, "%*c%s _tmp_243[%d-%d]: %s failed!\n", p->level, ' ', + D(fprintf(stderr, "%*c%s _tmp_244[%d-%d]: %s failed!\n", p->level, ' ', p->error_indicator ? "ERROR!" : "-", _mark, p->mark, "'!' NAME")); } _res = NULL; @@ -39945,9 +40030,9 @@ _tmp_243_rule(Parser *p) return _res; } -// _loop0_244: fstring_format_spec +// _loop0_245: fstring_format_spec static asdl_seq * -_loop0_244_rule(Parser *p) +_loop0_245_rule(Parser *p) { if (p->level++ == MAXSTACK) { _Pypegen_stack_overflow(p); @@ -39972,7 +40057,7 @@ _loop0_244_rule(Parser *p) p->level--; return NULL; } - D(fprintf(stderr, "%*c> _loop0_244[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "fstring_format_spec")); + D(fprintf(stderr, "%*c> _loop0_245[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "fstring_format_spec")); expr_ty fstring_format_spec_var; while ( (fstring_format_spec_var = fstring_format_spec_rule(p)) // fstring_format_spec @@ -39995,7 +40080,7 @@ _loop0_244_rule(Parser *p) _mark = p->mark; } p->mark = _mark; - D(fprintf(stderr, "%*c%s _loop0_244[%d-%d]: %s failed!\n", p->level, ' ', + D(fprintf(stderr, "%*c%s _loop0_245[%d-%d]: %s failed!\n", p->level, ' ', p->error_indicator ? "ERROR!" : "-", _mark, p->mark, "fstring_format_spec")); } asdl_seq *_seq = (asdl_seq*)_Py_asdl_generic_seq_new(_n, p->arena); @@ -40012,9 +40097,9 @@ _loop0_244_rule(Parser *p) return _seq; } -// _tmp_245: yield_expr | star_expressions +// _tmp_246: yield_expr | star_expressions static void * -_tmp_245_rule(Parser *p) +_tmp_246_rule(Parser *p) { if (p->level++ == MAXSTACK) { _Pypegen_stack_overflow(p); @@ -40030,18 +40115,18 @@ _tmp_245_rule(Parser *p) p->level--; return NULL; } - D(fprintf(stderr, "%*c> _tmp_245[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "yield_expr")); + D(fprintf(stderr, "%*c> _tmp_246[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "yield_expr")); expr_ty yield_expr_var; if ( (yield_expr_var = yield_expr_rule(p)) // yield_expr ) { - D(fprintf(stderr, "%*c+ _tmp_245[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "yield_expr")); + D(fprintf(stderr, "%*c+ _tmp_246[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "yield_expr")); _res = yield_expr_var; goto done; } p->mark = _mark; - D(fprintf(stderr, "%*c%s _tmp_245[%d-%d]: %s failed!\n", p->level, ' ', + D(fprintf(stderr, "%*c%s _tmp_246[%d-%d]: %s failed!\n", p->level, ' ', p->error_indicator ? "ERROR!" : "-", _mark, p->mark, "yield_expr")); } { // star_expressions @@ -40049,18 +40134,18 @@ _tmp_245_rule(Parser *p) p->level--; return NULL; } - D(fprintf(stderr, "%*c> _tmp_245[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "star_expressions")); + D(fprintf(stderr, "%*c> _tmp_246[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "star_expressions")); expr_ty star_expressions_var; if ( (star_expressions_var = star_expressions_rule(p)) // star_expressions ) { - D(fprintf(stderr, "%*c+ _tmp_245[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "star_expressions")); + D(fprintf(stderr, "%*c+ _tmp_246[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "star_expressions")); _res = star_expressions_var; goto done; } p->mark = _mark; - D(fprintf(stderr, "%*c%s _tmp_245[%d-%d]: %s failed!\n", p->level, ' ', + D(fprintf(stderr, "%*c%s _tmp_246[%d-%d]: %s failed!\n", p->level, ' ', p->error_indicator ? "ERROR!" : "-", _mark, p->mark, "star_expressions")); } _res = NULL; @@ -40069,9 +40154,9 @@ _tmp_245_rule(Parser *p) return _res; } -// _tmp_246: '!' NAME +// _tmp_247: '!' NAME static void * -_tmp_246_rule(Parser *p) +_tmp_247_rule(Parser *p) { if (p->level++ == MAXSTACK) { _Pypegen_stack_overflow(p); @@ -40087,7 +40172,7 @@ _tmp_246_rule(Parser *p) p->level--; return NULL; } - D(fprintf(stderr, "%*c> _tmp_246[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "'!' NAME")); + D(fprintf(stderr, "%*c> _tmp_247[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "'!' NAME")); Token * _literal; expr_ty name_var; if ( @@ -40096,12 +40181,12 @@ _tmp_246_rule(Parser *p) (name_var = _PyPegen_name_token(p)) // NAME ) { - D(fprintf(stderr, "%*c+ _tmp_246[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "'!' NAME")); + D(fprintf(stderr, "%*c+ _tmp_247[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "'!' NAME")); _res = _PyPegen_dummy_name(p, _literal, name_var); goto done; } p->mark = _mark; - D(fprintf(stderr, "%*c%s _tmp_246[%d-%d]: %s failed!\n", p->level, ' ', + D(fprintf(stderr, "%*c%s _tmp_247[%d-%d]: %s failed!\n", p->level, ' ', p->error_indicator ? "ERROR!" : "-", _mark, p->mark, "'!' NAME")); } _res = NULL; @@ -40110,9 +40195,9 @@ _tmp_246_rule(Parser *p) return _res; } -// _tmp_247: ':' | '}' +// _tmp_248: ':' | '}' static void * -_tmp_247_rule(Parser *p) +_tmp_248_rule(Parser *p) { if (p->level++ == MAXSTACK) { _Pypegen_stack_overflow(p); @@ -40128,18 +40213,18 @@ _tmp_247_rule(Parser *p) p->level--; return NULL; } - D(fprintf(stderr, "%*c> _tmp_247[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "':'")); + D(fprintf(stderr, "%*c> _tmp_248[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "':'")); Token * _literal; if ( (_literal = _PyPegen_expect_token(p, 11)) // token=':' ) { - D(fprintf(stderr, "%*c+ _tmp_247[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "':'")); + D(fprintf(stderr, "%*c+ _tmp_248[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "':'")); _res = _literal; goto done; } p->mark = _mark; - D(fprintf(stderr, "%*c%s _tmp_247[%d-%d]: %s failed!\n", p->level, ' ', + D(fprintf(stderr, "%*c%s _tmp_248[%d-%d]: %s failed!\n", p->level, ' ', p->error_indicator ? "ERROR!" : "-", _mark, p->mark, "':'")); } { // '}' @@ -40147,18 +40232,18 @@ _tmp_247_rule(Parser *p) p->level--; return NULL; } - D(fprintf(stderr, "%*c> _tmp_247[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "'}'")); + D(fprintf(stderr, "%*c> _tmp_248[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "'}'")); Token * _literal; if ( (_literal = _PyPegen_expect_token(p, 26)) // token='}' ) { - D(fprintf(stderr, "%*c+ _tmp_247[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "'}'")); + D(fprintf(stderr, "%*c+ _tmp_248[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "'}'")); _res = _literal; goto done; } p->mark = _mark; - D(fprintf(stderr, "%*c%s _tmp_247[%d-%d]: %s failed!\n", p->level, ' ', + D(fprintf(stderr, "%*c%s _tmp_248[%d-%d]: %s failed!\n", p->level, ' ', p->error_indicator ? "ERROR!" : "-", _mark, p->mark, "'}'")); } _res = NULL; @@ -40167,9 +40252,9 @@ _tmp_247_rule(Parser *p) return _res; } -// _tmp_248: star_targets '=' +// _tmp_249: star_targets '=' static void * -_tmp_248_rule(Parser *p) +_tmp_249_rule(Parser *p) { if (p->level++ == MAXSTACK) { _Pypegen_stack_overflow(p); @@ -40185,7 +40270,7 @@ _tmp_248_rule(Parser *p) p->level--; return NULL; } - D(fprintf(stderr, "%*c> _tmp_248[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "star_targets '='")); + D(fprintf(stderr, "%*c> _tmp_249[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "star_targets '='")); Token * _literal; expr_ty z; if ( @@ -40194,7 +40279,7 @@ _tmp_248_rule(Parser *p) (_literal = _PyPegen_expect_token(p, 22)) // token='=' ) { - D(fprintf(stderr, "%*c+ _tmp_248[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "star_targets '='")); + D(fprintf(stderr, "%*c+ _tmp_249[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "star_targets '='")); _res = z; if (_res == NULL && PyErr_Occurred()) { p->error_indicator = 1; @@ -40204,7 +40289,7 @@ _tmp_248_rule(Parser *p) goto done; } p->mark = _mark; - D(fprintf(stderr, "%*c%s _tmp_248[%d-%d]: %s failed!\n", p->level, ' ', + D(fprintf(stderr, "%*c%s _tmp_249[%d-%d]: %s failed!\n", p->level, ' ', p->error_indicator ? "ERROR!" : "-", _mark, p->mark, "star_targets '='")); } _res = NULL; @@ -40213,9 +40298,9 @@ _tmp_248_rule(Parser *p) return _res; } -// _tmp_249: '.' | '...' +// _tmp_250: '.' | '...' static void * -_tmp_249_rule(Parser *p) +_tmp_250_rule(Parser *p) { if (p->level++ == MAXSTACK) { _Pypegen_stack_overflow(p); @@ -40231,18 +40316,18 @@ _tmp_249_rule(Parser *p) p->level--; return NULL; } - D(fprintf(stderr, "%*c> _tmp_249[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "'.'")); + D(fprintf(stderr, "%*c> _tmp_250[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "'.'")); Token * _literal; if ( (_literal = _PyPegen_expect_token(p, 23)) // token='.' ) { - D(fprintf(stderr, "%*c+ _tmp_249[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "'.'")); + D(fprintf(stderr, "%*c+ _tmp_250[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "'.'")); _res = _literal; goto done; } p->mark = _mark; - D(fprintf(stderr, "%*c%s _tmp_249[%d-%d]: %s failed!\n", p->level, ' ', + D(fprintf(stderr, "%*c%s _tmp_250[%d-%d]: %s failed!\n", p->level, ' ', p->error_indicator ? "ERROR!" : "-", _mark, p->mark, "'.'")); } { // '...' @@ -40250,18 +40335,18 @@ _tmp_249_rule(Parser *p) p->level--; return NULL; } - D(fprintf(stderr, "%*c> _tmp_249[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "'...'")); + D(fprintf(stderr, "%*c> _tmp_250[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "'...'")); Token * _literal; if ( (_literal = _PyPegen_expect_token(p, 52)) // token='...' ) { - D(fprintf(stderr, "%*c+ _tmp_249[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "'...'")); + D(fprintf(stderr, "%*c+ _tmp_250[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "'...'")); _res = _literal; goto done; } p->mark = _mark; - D(fprintf(stderr, "%*c%s _tmp_249[%d-%d]: %s failed!\n", p->level, ' ', + D(fprintf(stderr, "%*c%s _tmp_250[%d-%d]: %s failed!\n", p->level, ' ', p->error_indicator ? "ERROR!" : "-", _mark, p->mark, "'...'")); } _res = NULL; @@ -40270,9 +40355,9 @@ _tmp_249_rule(Parser *p) return _res; } -// _tmp_250: '.' | '...' +// _tmp_251: '.' | '...' static void * -_tmp_250_rule(Parser *p) +_tmp_251_rule(Parser *p) { if (p->level++ == MAXSTACK) { _Pypegen_stack_overflow(p); @@ -40288,18 +40373,18 @@ _tmp_250_rule(Parser *p) p->level--; return NULL; } - D(fprintf(stderr, "%*c> _tmp_250[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "'.'")); + D(fprintf(stderr, "%*c> _tmp_251[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "'.'")); Token * _literal; if ( (_literal = _PyPegen_expect_token(p, 23)) // token='.' ) { - D(fprintf(stderr, "%*c+ _tmp_250[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "'.'")); + D(fprintf(stderr, "%*c+ _tmp_251[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "'.'")); _res = _literal; goto done; } p->mark = _mark; - D(fprintf(stderr, "%*c%s _tmp_250[%d-%d]: %s failed!\n", p->level, ' ', + D(fprintf(stderr, "%*c%s _tmp_251[%d-%d]: %s failed!\n", p->level, ' ', p->error_indicator ? "ERROR!" : "-", _mark, p->mark, "'.'")); } { // '...' @@ -40307,18 +40392,18 @@ _tmp_250_rule(Parser *p) p->level--; return NULL; } - D(fprintf(stderr, "%*c> _tmp_250[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "'...'")); + D(fprintf(stderr, "%*c> _tmp_251[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "'...'")); Token * _literal; if ( (_literal = _PyPegen_expect_token(p, 52)) // token='...' ) { - D(fprintf(stderr, "%*c+ _tmp_250[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "'...'")); + D(fprintf(stderr, "%*c+ _tmp_251[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "'...'")); _res = _literal; goto done; } p->mark = _mark; - D(fprintf(stderr, "%*c%s _tmp_250[%d-%d]: %s failed!\n", p->level, ' ', + D(fprintf(stderr, "%*c%s _tmp_251[%d-%d]: %s failed!\n", p->level, ' ', p->error_indicator ? "ERROR!" : "-", _mark, p->mark, "'...'")); } _res = NULL; @@ -40327,9 +40412,9 @@ _tmp_250_rule(Parser *p) return _res; } -// _tmp_251: '@' named_expression NEWLINE +// _tmp_252: '@' named_expression NEWLINE static void * -_tmp_251_rule(Parser *p) +_tmp_252_rule(Parser *p) { if (p->level++ == MAXSTACK) { _Pypegen_stack_overflow(p); @@ -40345,7 +40430,7 @@ _tmp_251_rule(Parser *p) p->level--; return NULL; } - D(fprintf(stderr, "%*c> _tmp_251[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "'@' named_expression NEWLINE")); + D(fprintf(stderr, "%*c> _tmp_252[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "'@' named_expression NEWLINE")); Token * _literal; expr_ty f; Token * newline_var; @@ -40357,7 +40442,7 @@ _tmp_251_rule(Parser *p) (newline_var = _PyPegen_expect_token(p, NEWLINE)) // token='NEWLINE' ) { - D(fprintf(stderr, "%*c+ _tmp_251[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "'@' named_expression NEWLINE")); + D(fprintf(stderr, "%*c+ _tmp_252[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "'@' named_expression NEWLINE")); _res = f; if (_res == NULL && PyErr_Occurred()) { p->error_indicator = 1; @@ -40367,7 +40452,7 @@ _tmp_251_rule(Parser *p) goto done; } p->mark = _mark; - D(fprintf(stderr, "%*c%s _tmp_251[%d-%d]: %s failed!\n", p->level, ' ', + D(fprintf(stderr, "%*c%s _tmp_252[%d-%d]: %s failed!\n", p->level, ' ', p->error_indicator ? "ERROR!" : "-", _mark, p->mark, "'@' named_expression NEWLINE")); } _res = NULL; @@ -40376,9 +40461,9 @@ _tmp_251_rule(Parser *p) return _res; } -// _tmp_252: ',' expression +// _tmp_253: ',' expression static void * -_tmp_252_rule(Parser *p) +_tmp_253_rule(Parser *p) { if (p->level++ == MAXSTACK) { _Pypegen_stack_overflow(p); @@ -40394,7 +40479,7 @@ _tmp_252_rule(Parser *p) p->level--; return NULL; } - D(fprintf(stderr, "%*c> _tmp_252[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "',' expression")); + D(fprintf(stderr, "%*c> _tmp_253[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "',' expression")); Token * _literal; expr_ty c; if ( @@ -40403,7 +40488,7 @@ _tmp_252_rule(Parser *p) (c = expression_rule(p)) // expression ) { - D(fprintf(stderr, "%*c+ _tmp_252[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "',' expression")); + D(fprintf(stderr, "%*c+ _tmp_253[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "',' expression")); _res = c; if (_res == NULL && PyErr_Occurred()) { p->error_indicator = 1; @@ -40413,7 +40498,7 @@ _tmp_252_rule(Parser *p) goto done; } p->mark = _mark; - D(fprintf(stderr, "%*c%s _tmp_252[%d-%d]: %s failed!\n", p->level, ' ', + D(fprintf(stderr, "%*c%s _tmp_253[%d-%d]: %s failed!\n", p->level, ' ', p->error_indicator ? "ERROR!" : "-", _mark, p->mark, "',' expression")); } _res = NULL; @@ -40422,9 +40507,9 @@ _tmp_252_rule(Parser *p) return _res; } -// _tmp_253: ',' star_expression +// _tmp_254: ',' star_expression static void * -_tmp_253_rule(Parser *p) +_tmp_254_rule(Parser *p) { if (p->level++ == MAXSTACK) { _Pypegen_stack_overflow(p); @@ -40440,7 +40525,7 @@ _tmp_253_rule(Parser *p) p->level--; return NULL; } - D(fprintf(stderr, "%*c> _tmp_253[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "',' star_expression")); + D(fprintf(stderr, "%*c> _tmp_254[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "',' star_expression")); Token * _literal; expr_ty c; if ( @@ -40449,7 +40534,7 @@ _tmp_253_rule(Parser *p) (c = star_expression_rule(p)) // star_expression ) { - D(fprintf(stderr, "%*c+ _tmp_253[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "',' star_expression")); + D(fprintf(stderr, "%*c+ _tmp_254[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "',' star_expression")); _res = c; if (_res == NULL && PyErr_Occurred()) { p->error_indicator = 1; @@ -40459,7 +40544,7 @@ _tmp_253_rule(Parser *p) goto done; } p->mark = _mark; - D(fprintf(stderr, "%*c%s _tmp_253[%d-%d]: %s failed!\n", p->level, ' ', + D(fprintf(stderr, "%*c%s _tmp_254[%d-%d]: %s failed!\n", p->level, ' ', p->error_indicator ? "ERROR!" : "-", _mark, p->mark, "',' star_expression")); } _res = NULL; @@ -40468,9 +40553,9 @@ _tmp_253_rule(Parser *p) return _res; } -// _tmp_254: 'or' conjunction +// _tmp_255: 'or' conjunction static void * -_tmp_254_rule(Parser *p) +_tmp_255_rule(Parser *p) { if (p->level++ == MAXSTACK) { _Pypegen_stack_overflow(p); @@ -40486,7 +40571,7 @@ _tmp_254_rule(Parser *p) p->level--; return NULL; } - D(fprintf(stderr, "%*c> _tmp_254[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "'or' conjunction")); + D(fprintf(stderr, "%*c> _tmp_255[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "'or' conjunction")); Token * _keyword; expr_ty c; if ( @@ -40495,7 +40580,7 @@ _tmp_254_rule(Parser *p) (c = conjunction_rule(p)) // conjunction ) { - D(fprintf(stderr, "%*c+ _tmp_254[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "'or' conjunction")); + D(fprintf(stderr, "%*c+ _tmp_255[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "'or' conjunction")); _res = c; if (_res == NULL && PyErr_Occurred()) { p->error_indicator = 1; @@ -40505,7 +40590,7 @@ _tmp_254_rule(Parser *p) goto done; } p->mark = _mark; - D(fprintf(stderr, "%*c%s _tmp_254[%d-%d]: %s failed!\n", p->level, ' ', + D(fprintf(stderr, "%*c%s _tmp_255[%d-%d]: %s failed!\n", p->level, ' ', p->error_indicator ? "ERROR!" : "-", _mark, p->mark, "'or' conjunction")); } _res = NULL; @@ -40514,9 +40599,9 @@ _tmp_254_rule(Parser *p) return _res; } -// _tmp_255: 'and' inversion +// _tmp_256: 'and' inversion static void * -_tmp_255_rule(Parser *p) +_tmp_256_rule(Parser *p) { if (p->level++ == MAXSTACK) { _Pypegen_stack_overflow(p); @@ -40532,7 +40617,7 @@ _tmp_255_rule(Parser *p) p->level--; return NULL; } - D(fprintf(stderr, "%*c> _tmp_255[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "'and' inversion")); + D(fprintf(stderr, "%*c> _tmp_256[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "'and' inversion")); Token * _keyword; expr_ty c; if ( @@ -40541,7 +40626,7 @@ _tmp_255_rule(Parser *p) (c = inversion_rule(p)) // inversion ) { - D(fprintf(stderr, "%*c+ _tmp_255[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "'and' inversion")); + D(fprintf(stderr, "%*c+ _tmp_256[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "'and' inversion")); _res = c; if (_res == NULL && PyErr_Occurred()) { p->error_indicator = 1; @@ -40551,7 +40636,7 @@ _tmp_255_rule(Parser *p) goto done; } p->mark = _mark; - D(fprintf(stderr, "%*c%s _tmp_255[%d-%d]: %s failed!\n", p->level, ' ', + D(fprintf(stderr, "%*c%s _tmp_256[%d-%d]: %s failed!\n", p->level, ' ', p->error_indicator ? "ERROR!" : "-", _mark, p->mark, "'and' inversion")); } _res = NULL; @@ -40560,9 +40645,9 @@ _tmp_255_rule(Parser *p) return _res; } -// _tmp_256: slice | starred_expression +// _tmp_257: slice | starred_expression static void * -_tmp_256_rule(Parser *p) +_tmp_257_rule(Parser *p) { if (p->level++ == MAXSTACK) { _Pypegen_stack_overflow(p); @@ -40578,18 +40663,18 @@ _tmp_256_rule(Parser *p) p->level--; return NULL; } - D(fprintf(stderr, "%*c> _tmp_256[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "slice")); + D(fprintf(stderr, "%*c> _tmp_257[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "slice")); expr_ty slice_var; if ( (slice_var = slice_rule(p)) // slice ) { - D(fprintf(stderr, "%*c+ _tmp_256[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "slice")); + D(fprintf(stderr, "%*c+ _tmp_257[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "slice")); _res = slice_var; goto done; } p->mark = _mark; - D(fprintf(stderr, "%*c%s _tmp_256[%d-%d]: %s failed!\n", p->level, ' ', + D(fprintf(stderr, "%*c%s _tmp_257[%d-%d]: %s failed!\n", p->level, ' ', p->error_indicator ? "ERROR!" : "-", _mark, p->mark, "slice")); } { // starred_expression @@ -40597,18 +40682,18 @@ _tmp_256_rule(Parser *p) p->level--; return NULL; } - D(fprintf(stderr, "%*c> _tmp_256[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "starred_expression")); + D(fprintf(stderr, "%*c> _tmp_257[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "starred_expression")); expr_ty starred_expression_var; if ( (starred_expression_var = starred_expression_rule(p)) // starred_expression ) { - D(fprintf(stderr, "%*c+ _tmp_256[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "starred_expression")); + D(fprintf(stderr, "%*c+ _tmp_257[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "starred_expression")); _res = starred_expression_var; goto done; } p->mark = _mark; - D(fprintf(stderr, "%*c%s _tmp_256[%d-%d]: %s failed!\n", p->level, ' ', + D(fprintf(stderr, "%*c%s _tmp_257[%d-%d]: %s failed!\n", p->level, ' ', p->error_indicator ? "ERROR!" : "-", _mark, p->mark, "starred_expression")); } _res = NULL; @@ -40617,9 +40702,9 @@ _tmp_256_rule(Parser *p) return _res; } -// _tmp_257: fstring | string +// _tmp_258: fstring | string static void * -_tmp_257_rule(Parser *p) +_tmp_258_rule(Parser *p) { if (p->level++ == MAXSTACK) { _Pypegen_stack_overflow(p); @@ -40635,18 +40720,18 @@ _tmp_257_rule(Parser *p) p->level--; return NULL; } - D(fprintf(stderr, "%*c> _tmp_257[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "fstring")); + D(fprintf(stderr, "%*c> _tmp_258[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "fstring")); expr_ty fstring_var; if ( (fstring_var = fstring_rule(p)) // fstring ) { - D(fprintf(stderr, "%*c+ _tmp_257[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "fstring")); + D(fprintf(stderr, "%*c+ _tmp_258[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "fstring")); _res = fstring_var; goto done; } p->mark = _mark; - D(fprintf(stderr, "%*c%s _tmp_257[%d-%d]: %s failed!\n", p->level, ' ', + D(fprintf(stderr, "%*c%s _tmp_258[%d-%d]: %s failed!\n", p->level, ' ', p->error_indicator ? "ERROR!" : "-", _mark, p->mark, "fstring")); } { // string @@ -40654,18 +40739,18 @@ _tmp_257_rule(Parser *p) p->level--; return NULL; } - D(fprintf(stderr, "%*c> _tmp_257[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "string")); + D(fprintf(stderr, "%*c> _tmp_258[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "string")); expr_ty string_var; if ( (string_var = string_rule(p)) // string ) { - D(fprintf(stderr, "%*c+ _tmp_257[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "string")); + D(fprintf(stderr, "%*c+ _tmp_258[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "string")); _res = string_var; goto done; } p->mark = _mark; - D(fprintf(stderr, "%*c%s _tmp_257[%d-%d]: %s failed!\n", p->level, ' ', + D(fprintf(stderr, "%*c%s _tmp_258[%d-%d]: %s failed!\n", p->level, ' ', p->error_indicator ? "ERROR!" : "-", _mark, p->mark, "string")); } _res = NULL; @@ -40674,9 +40759,9 @@ _tmp_257_rule(Parser *p) return _res; } -// _tmp_258: 'if' disjunction +// _tmp_259: 'if' disjunction static void * -_tmp_258_rule(Parser *p) +_tmp_259_rule(Parser *p) { if (p->level++ == MAXSTACK) { _Pypegen_stack_overflow(p); @@ -40692,16 +40777,16 @@ _tmp_258_rule(Parser *p) p->level--; return NULL; } - D(fprintf(stderr, "%*c> _tmp_258[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "'if' disjunction")); + D(fprintf(stderr, "%*c> _tmp_259[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "'if' disjunction")); Token * _keyword; expr_ty z; if ( - (_keyword = _PyPegen_expect_token(p, 658)) // token='if' + (_keyword = _PyPegen_expect_token(p, 661)) // token='if' && (z = disjunction_rule(p)) // disjunction ) { - D(fprintf(stderr, "%*c+ _tmp_258[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "'if' disjunction")); + D(fprintf(stderr, "%*c+ _tmp_259[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "'if' disjunction")); _res = z; if (_res == NULL && PyErr_Occurred()) { p->error_indicator = 1; @@ -40711,7 +40796,7 @@ _tmp_258_rule(Parser *p) goto done; } p->mark = _mark; - D(fprintf(stderr, "%*c%s _tmp_258[%d-%d]: %s failed!\n", p->level, ' ', + D(fprintf(stderr, "%*c%s _tmp_259[%d-%d]: %s failed!\n", p->level, ' ', p->error_indicator ? "ERROR!" : "-", _mark, p->mark, "'if' disjunction")); } _res = NULL; @@ -40720,9 +40805,9 @@ _tmp_258_rule(Parser *p) return _res; } -// _tmp_259: 'if' disjunction +// _tmp_260: 'if' disjunction static void * -_tmp_259_rule(Parser *p) +_tmp_260_rule(Parser *p) { if (p->level++ == MAXSTACK) { _Pypegen_stack_overflow(p); @@ -40738,16 +40823,16 @@ _tmp_259_rule(Parser *p) p->level--; return NULL; } - D(fprintf(stderr, "%*c> _tmp_259[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "'if' disjunction")); + D(fprintf(stderr, "%*c> _tmp_260[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "'if' disjunction")); Token * _keyword; expr_ty z; if ( - (_keyword = _PyPegen_expect_token(p, 658)) // token='if' + (_keyword = _PyPegen_expect_token(p, 661)) // token='if' && (z = disjunction_rule(p)) // disjunction ) { - D(fprintf(stderr, "%*c+ _tmp_259[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "'if' disjunction")); + D(fprintf(stderr, "%*c+ _tmp_260[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "'if' disjunction")); _res = z; if (_res == NULL && PyErr_Occurred()) { p->error_indicator = 1; @@ -40757,7 +40842,7 @@ _tmp_259_rule(Parser *p) goto done; } p->mark = _mark; - D(fprintf(stderr, "%*c%s _tmp_259[%d-%d]: %s failed!\n", p->level, ' ', + D(fprintf(stderr, "%*c%s _tmp_260[%d-%d]: %s failed!\n", p->level, ' ', p->error_indicator ? "ERROR!" : "-", _mark, p->mark, "'if' disjunction")); } _res = NULL; @@ -40766,9 +40851,76 @@ _tmp_259_rule(Parser *p) return _res; } -// _tmp_260: starred_expression | (assignment_expression | expression !':=') !'=' +// _loop0_261: (',' bitwise_or) +static asdl_seq * +_loop0_261_rule(Parser *p) +{ + if (p->level++ == MAXSTACK) { + _Pypegen_stack_overflow(p); + } + if (p->error_indicator) { + p->level--; + return NULL; + } + void *_res = NULL; + int _mark = p->mark; + void **_children = PyMem_Malloc(sizeof(void *)); + if (!_children) { + p->error_indicator = 1; + PyErr_NoMemory(); + p->level--; + return NULL; + } + Py_ssize_t _children_capacity = 1; + Py_ssize_t _n = 0; + { // (',' bitwise_or) + if (p->error_indicator) { + p->level--; + return NULL; + } + D(fprintf(stderr, "%*c> _loop0_261[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "(',' bitwise_or)")); + void *_tmp_275_var; + while ( + (_tmp_275_var = _tmp_275_rule(p)) // ',' bitwise_or + ) + { + _res = _tmp_275_var; + if (_n == _children_capacity) { + _children_capacity *= 2; + void **_new_children = PyMem_Realloc(_children, _children_capacity*sizeof(void *)); + if (!_new_children) { + PyMem_Free(_children); + p->error_indicator = 1; + PyErr_NoMemory(); + p->level--; + return NULL; + } + _children = _new_children; + } + _children[_n++] = _res; + _mark = p->mark; + } + p->mark = _mark; + D(fprintf(stderr, "%*c%s _loop0_261[%d-%d]: %s failed!\n", p->level, ' ', + p->error_indicator ? "ERROR!" : "-", _mark, p->mark, "(',' bitwise_or)")); + } + asdl_seq *_seq = (asdl_seq*)_Py_asdl_generic_seq_new(_n, p->arena); + if (!_seq) { + PyMem_Free(_children); + p->error_indicator = 1; + PyErr_NoMemory(); + p->level--; + return NULL; + } + for (int i = 0; i < _n; i++) asdl_seq_SET_UNTYPED(_seq, i, _children[i]); + PyMem_Free(_children); + p->level--; + return _seq; +} + +// _tmp_262: starred_expression | (assignment_expression | expression !':=') !'=' static void * -_tmp_260_rule(Parser *p) +_tmp_262_rule(Parser *p) { if (p->level++ == MAXSTACK) { _Pypegen_stack_overflow(p); @@ -40784,18 +40936,18 @@ _tmp_260_rule(Parser *p) p->level--; return NULL; } - D(fprintf(stderr, "%*c> _tmp_260[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "starred_expression")); + D(fprintf(stderr, "%*c> _tmp_262[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "starred_expression")); expr_ty starred_expression_var; if ( (starred_expression_var = starred_expression_rule(p)) // starred_expression ) { - D(fprintf(stderr, "%*c+ _tmp_260[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "starred_expression")); + D(fprintf(stderr, "%*c+ _tmp_262[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "starred_expression")); _res = starred_expression_var; goto done; } p->mark = _mark; - D(fprintf(stderr, "%*c%s _tmp_260[%d-%d]: %s failed!\n", p->level, ' ', + D(fprintf(stderr, "%*c%s _tmp_262[%d-%d]: %s failed!\n", p->level, ' ', p->error_indicator ? "ERROR!" : "-", _mark, p->mark, "starred_expression")); } { // (assignment_expression | expression !':=') !'=' @@ -40803,20 +40955,20 @@ _tmp_260_rule(Parser *p) p->level--; return NULL; } - D(fprintf(stderr, "%*c> _tmp_260[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "(assignment_expression | expression !':=') !'='")); - void *_tmp_273_var; + D(fprintf(stderr, "%*c> _tmp_262[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "(assignment_expression | expression !':=') !'='")); + void *_tmp_276_var; if ( - (_tmp_273_var = _tmp_273_rule(p)) // assignment_expression | expression !':=' + (_tmp_276_var = _tmp_276_rule(p)) // assignment_expression | expression !':=' && _PyPegen_lookahead_with_int(0, _PyPegen_expect_token, p, 22) // token='=' ) { - D(fprintf(stderr, "%*c+ _tmp_260[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "(assignment_expression | expression !':=') !'='")); - _res = _tmp_273_var; + D(fprintf(stderr, "%*c+ _tmp_262[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "(assignment_expression | expression !':=') !'='")); + _res = _tmp_276_var; goto done; } p->mark = _mark; - D(fprintf(stderr, "%*c%s _tmp_260[%d-%d]: %s failed!\n", p->level, ' ', + D(fprintf(stderr, "%*c%s _tmp_262[%d-%d]: %s failed!\n", p->level, ' ', p->error_indicator ? "ERROR!" : "-", _mark, p->mark, "(assignment_expression | expression !':=') !'='")); } _res = NULL; @@ -40825,9 +40977,9 @@ _tmp_260_rule(Parser *p) return _res; } -// _tmp_261: ',' star_target +// _tmp_263: ',' star_target static void * -_tmp_261_rule(Parser *p) +_tmp_263_rule(Parser *p) { if (p->level++ == MAXSTACK) { _Pypegen_stack_overflow(p); @@ -40843,7 +40995,7 @@ _tmp_261_rule(Parser *p) p->level--; return NULL; } - D(fprintf(stderr, "%*c> _tmp_261[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "',' star_target")); + D(fprintf(stderr, "%*c> _tmp_263[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "',' star_target")); Token * _literal; expr_ty c; if ( @@ -40852,7 +41004,7 @@ _tmp_261_rule(Parser *p) (c = star_target_rule(p)) // star_target ) { - D(fprintf(stderr, "%*c+ _tmp_261[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "',' star_target")); + D(fprintf(stderr, "%*c+ _tmp_263[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "',' star_target")); _res = c; if (_res == NULL && PyErr_Occurred()) { p->error_indicator = 1; @@ -40862,7 +41014,7 @@ _tmp_261_rule(Parser *p) goto done; } p->mark = _mark; - D(fprintf(stderr, "%*c%s _tmp_261[%d-%d]: %s failed!\n", p->level, ' ', + D(fprintf(stderr, "%*c%s _tmp_263[%d-%d]: %s failed!\n", p->level, ' ', p->error_indicator ? "ERROR!" : "-", _mark, p->mark, "',' star_target")); } _res = NULL; @@ -40871,9 +41023,9 @@ _tmp_261_rule(Parser *p) return _res; } -// _tmp_262: ',' star_target +// _tmp_264: ',' star_target static void * -_tmp_262_rule(Parser *p) +_tmp_264_rule(Parser *p) { if (p->level++ == MAXSTACK) { _Pypegen_stack_overflow(p); @@ -40889,7 +41041,7 @@ _tmp_262_rule(Parser *p) p->level--; return NULL; } - D(fprintf(stderr, "%*c> _tmp_262[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "',' star_target")); + D(fprintf(stderr, "%*c> _tmp_264[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "',' star_target")); Token * _literal; expr_ty c; if ( @@ -40898,7 +41050,7 @@ _tmp_262_rule(Parser *p) (c = star_target_rule(p)) // star_target ) { - D(fprintf(stderr, "%*c+ _tmp_262[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "',' star_target")); + D(fprintf(stderr, "%*c+ _tmp_264[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "',' star_target")); _res = c; if (_res == NULL && PyErr_Occurred()) { p->error_indicator = 1; @@ -40908,7 +41060,7 @@ _tmp_262_rule(Parser *p) goto done; } p->mark = _mark; - D(fprintf(stderr, "%*c%s _tmp_262[%d-%d]: %s failed!\n", p->level, ' ', + D(fprintf(stderr, "%*c%s _tmp_264[%d-%d]: %s failed!\n", p->level, ' ', p->error_indicator ? "ERROR!" : "-", _mark, p->mark, "',' star_target")); } _res = NULL; @@ -40917,10 +41069,10 @@ _tmp_262_rule(Parser *p) return _res; } -// _tmp_263: +// _tmp_265: // | ','.(starred_expression | (assignment_expression | expression !':=') !'=')+ ',' kwargs static void * -_tmp_263_rule(Parser *p) +_tmp_265_rule(Parser *p) { if (p->level++ == MAXSTACK) { _Pypegen_stack_overflow(p); @@ -40936,24 +41088,24 @@ _tmp_263_rule(Parser *p) p->level--; return NULL; } - D(fprintf(stderr, "%*c> _tmp_263[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "','.(starred_expression | (assignment_expression | expression !':=') !'=')+ ',' kwargs")); - asdl_seq * _gather_274_var; + D(fprintf(stderr, "%*c> _tmp_265[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "','.(starred_expression | (assignment_expression | expression !':=') !'=')+ ',' kwargs")); + asdl_seq * _gather_277_var; Token * _literal; asdl_seq* kwargs_var; if ( - (_gather_274_var = _gather_274_rule(p)) // ','.(starred_expression | (assignment_expression | expression !':=') !'=')+ + (_gather_277_var = _gather_277_rule(p)) // ','.(starred_expression | (assignment_expression | expression !':=') !'=')+ && (_literal = _PyPegen_expect_token(p, 12)) // token=',' && (kwargs_var = kwargs_rule(p)) // kwargs ) { - D(fprintf(stderr, "%*c+ _tmp_263[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "','.(starred_expression | (assignment_expression | expression !':=') !'=')+ ',' kwargs")); - _res = _PyPegen_dummy_name(p, _gather_274_var, _literal, kwargs_var); + D(fprintf(stderr, "%*c+ _tmp_265[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "','.(starred_expression | (assignment_expression | expression !':=') !'=')+ ',' kwargs")); + _res = _PyPegen_dummy_name(p, _gather_277_var, _literal, kwargs_var); goto done; } p->mark = _mark; - D(fprintf(stderr, "%*c%s _tmp_263[%d-%d]: %s failed!\n", p->level, ' ', + D(fprintf(stderr, "%*c%s _tmp_265[%d-%d]: %s failed!\n", p->level, ' ', p->error_indicator ? "ERROR!" : "-", _mark, p->mark, "','.(starred_expression | (assignment_expression | expression !':=') !'=')+ ',' kwargs")); } _res = NULL; @@ -40962,9 +41114,9 @@ _tmp_263_rule(Parser *p) return _res; } -// _tmp_264: star_targets '=' +// _tmp_266: star_targets '=' static void * -_tmp_264_rule(Parser *p) +_tmp_266_rule(Parser *p) { if (p->level++ == MAXSTACK) { _Pypegen_stack_overflow(p); @@ -40980,7 +41132,7 @@ _tmp_264_rule(Parser *p) p->level--; return NULL; } - D(fprintf(stderr, "%*c> _tmp_264[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "star_targets '='")); + D(fprintf(stderr, "%*c> _tmp_266[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "star_targets '='")); Token * _literal; expr_ty star_targets_var; if ( @@ -40989,12 +41141,12 @@ _tmp_264_rule(Parser *p) (_literal = _PyPegen_expect_token(p, 22)) // token='=' ) { - D(fprintf(stderr, "%*c+ _tmp_264[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "star_targets '='")); + D(fprintf(stderr, "%*c+ _tmp_266[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "star_targets '='")); _res = _PyPegen_dummy_name(p, star_targets_var, _literal); goto done; } p->mark = _mark; - D(fprintf(stderr, "%*c%s _tmp_264[%d-%d]: %s failed!\n", p->level, ' ', + D(fprintf(stderr, "%*c%s _tmp_266[%d-%d]: %s failed!\n", p->level, ' ', p->error_indicator ? "ERROR!" : "-", _mark, p->mark, "star_targets '='")); } _res = NULL; @@ -41003,9 +41155,9 @@ _tmp_264_rule(Parser *p) return _res; } -// _tmp_265: star_targets '=' +// _tmp_267: star_targets '=' static void * -_tmp_265_rule(Parser *p) +_tmp_267_rule(Parser *p) { if (p->level++ == MAXSTACK) { _Pypegen_stack_overflow(p); @@ -41021,7 +41173,7 @@ _tmp_265_rule(Parser *p) p->level--; return NULL; } - D(fprintf(stderr, "%*c> _tmp_265[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "star_targets '='")); + D(fprintf(stderr, "%*c> _tmp_267[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "star_targets '='")); Token * _literal; expr_ty star_targets_var; if ( @@ -41030,12 +41182,12 @@ _tmp_265_rule(Parser *p) (_literal = _PyPegen_expect_token(p, 22)) // token='=' ) { - D(fprintf(stderr, "%*c+ _tmp_265[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "star_targets '='")); + D(fprintf(stderr, "%*c+ _tmp_267[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "star_targets '='")); _res = _PyPegen_dummy_name(p, star_targets_var, _literal); goto done; } p->mark = _mark; - D(fprintf(stderr, "%*c%s _tmp_265[%d-%d]: %s failed!\n", p->level, ' ', + D(fprintf(stderr, "%*c%s _tmp_267[%d-%d]: %s failed!\n", p->level, ' ', p->error_indicator ? "ERROR!" : "-", _mark, p->mark, "star_targets '='")); } _res = NULL; @@ -41044,9 +41196,9 @@ _tmp_265_rule(Parser *p) return _res; } -// _tmp_266: ')' | '**' +// _tmp_268: ')' | '**' static void * -_tmp_266_rule(Parser *p) +_tmp_268_rule(Parser *p) { if (p->level++ == MAXSTACK) { _Pypegen_stack_overflow(p); @@ -41062,18 +41214,18 @@ _tmp_266_rule(Parser *p) p->level--; return NULL; } - D(fprintf(stderr, "%*c> _tmp_266[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "')'")); + D(fprintf(stderr, "%*c> _tmp_268[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "')'")); Token * _literal; if ( (_literal = _PyPegen_expect_token(p, 8)) // token=')' ) { - D(fprintf(stderr, "%*c+ _tmp_266[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "')'")); + D(fprintf(stderr, "%*c+ _tmp_268[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "')'")); _res = _literal; goto done; } p->mark = _mark; - D(fprintf(stderr, "%*c%s _tmp_266[%d-%d]: %s failed!\n", p->level, ' ', + D(fprintf(stderr, "%*c%s _tmp_268[%d-%d]: %s failed!\n", p->level, ' ', p->error_indicator ? "ERROR!" : "-", _mark, p->mark, "')'")); } { // '**' @@ -41081,18 +41233,18 @@ _tmp_266_rule(Parser *p) p->level--; return NULL; } - D(fprintf(stderr, "%*c> _tmp_266[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "'**'")); + D(fprintf(stderr, "%*c> _tmp_268[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "'**'")); Token * _literal; if ( (_literal = _PyPegen_expect_token(p, 35)) // token='**' ) { - D(fprintf(stderr, "%*c+ _tmp_266[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "'**'")); + D(fprintf(stderr, "%*c+ _tmp_268[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "'**'")); _res = _literal; goto done; } p->mark = _mark; - D(fprintf(stderr, "%*c%s _tmp_266[%d-%d]: %s failed!\n", p->level, ' ', + D(fprintf(stderr, "%*c%s _tmp_268[%d-%d]: %s failed!\n", p->level, ' ', p->error_indicator ? "ERROR!" : "-", _mark, p->mark, "'**'")); } _res = NULL; @@ -41101,9 +41253,9 @@ _tmp_266_rule(Parser *p) return _res; } -// _tmp_267: ':' | '**' +// _tmp_269: ':' | '**' static void * -_tmp_267_rule(Parser *p) +_tmp_269_rule(Parser *p) { if (p->level++ == MAXSTACK) { _Pypegen_stack_overflow(p); @@ -41119,18 +41271,18 @@ _tmp_267_rule(Parser *p) p->level--; return NULL; } - D(fprintf(stderr, "%*c> _tmp_267[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "':'")); + D(fprintf(stderr, "%*c> _tmp_269[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "':'")); Token * _literal; if ( (_literal = _PyPegen_expect_token(p, 11)) // token=':' ) { - D(fprintf(stderr, "%*c+ _tmp_267[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "':'")); + D(fprintf(stderr, "%*c+ _tmp_269[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "':'")); _res = _literal; goto done; } p->mark = _mark; - D(fprintf(stderr, "%*c%s _tmp_267[%d-%d]: %s failed!\n", p->level, ' ', + D(fprintf(stderr, "%*c%s _tmp_269[%d-%d]: %s failed!\n", p->level, ' ', p->error_indicator ? "ERROR!" : "-", _mark, p->mark, "':'")); } { // '**' @@ -41138,18 +41290,18 @@ _tmp_267_rule(Parser *p) p->level--; return NULL; } - D(fprintf(stderr, "%*c> _tmp_267[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "'**'")); + D(fprintf(stderr, "%*c> _tmp_269[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "'**'")); Token * _literal; if ( (_literal = _PyPegen_expect_token(p, 35)) // token='**' ) { - D(fprintf(stderr, "%*c+ _tmp_267[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "'**'")); + D(fprintf(stderr, "%*c+ _tmp_269[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "'**'")); _res = _literal; goto done; } p->mark = _mark; - D(fprintf(stderr, "%*c%s _tmp_267[%d-%d]: %s failed!\n", p->level, ' ', + D(fprintf(stderr, "%*c%s _tmp_269[%d-%d]: %s failed!\n", p->level, ' ', p->error_indicator ? "ERROR!" : "-", _mark, p->mark, "'**'")); } _res = NULL; @@ -41158,9 +41310,9 @@ _tmp_267_rule(Parser *p) return _res; } -// _tmp_268: expression ['as' star_target] +// _tmp_270: expression ['as' star_target] static void * -_tmp_268_rule(Parser *p) +_tmp_270_rule(Parser *p) { if (p->level++ == MAXSTACK) { _Pypegen_stack_overflow(p); @@ -41176,22 +41328,22 @@ _tmp_268_rule(Parser *p) p->level--; return NULL; } - D(fprintf(stderr, "%*c> _tmp_268[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "expression ['as' star_target]")); + D(fprintf(stderr, "%*c> _tmp_270[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "expression ['as' star_target]")); void *_opt_var; UNUSED(_opt_var); // Silence compiler warnings expr_ty expression_var; if ( (expression_var = expression_rule(p)) // expression && - (_opt_var = _tmp_276_rule(p), !p->error_indicator) // ['as' star_target] + (_opt_var = _tmp_279_rule(p), !p->error_indicator) // ['as' star_target] ) { - D(fprintf(stderr, "%*c+ _tmp_268[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "expression ['as' star_target]")); + D(fprintf(stderr, "%*c+ _tmp_270[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "expression ['as' star_target]")); _res = _PyPegen_dummy_name(p, expression_var, _opt_var); goto done; } p->mark = _mark; - D(fprintf(stderr, "%*c%s _tmp_268[%d-%d]: %s failed!\n", p->level, ' ', + D(fprintf(stderr, "%*c%s _tmp_270[%d-%d]: %s failed!\n", p->level, ' ', p->error_indicator ? "ERROR!" : "-", _mark, p->mark, "expression ['as' star_target]")); } _res = NULL; @@ -41200,9 +41352,9 @@ _tmp_268_rule(Parser *p) return _res; } -// _tmp_269: expressions ['as' star_target] +// _tmp_271: expressions ['as' star_target] static void * -_tmp_269_rule(Parser *p) +_tmp_271_rule(Parser *p) { if (p->level++ == MAXSTACK) { _Pypegen_stack_overflow(p); @@ -41218,22 +41370,22 @@ _tmp_269_rule(Parser *p) p->level--; return NULL; } - D(fprintf(stderr, "%*c> _tmp_269[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "expressions ['as' star_target]")); + D(fprintf(stderr, "%*c> _tmp_271[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "expressions ['as' star_target]")); void *_opt_var; UNUSED(_opt_var); // Silence compiler warnings expr_ty expressions_var; if ( (expressions_var = expressions_rule(p)) // expressions && - (_opt_var = _tmp_277_rule(p), !p->error_indicator) // ['as' star_target] + (_opt_var = _tmp_280_rule(p), !p->error_indicator) // ['as' star_target] ) { - D(fprintf(stderr, "%*c+ _tmp_269[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "expressions ['as' star_target]")); + D(fprintf(stderr, "%*c+ _tmp_271[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "expressions ['as' star_target]")); _res = _PyPegen_dummy_name(p, expressions_var, _opt_var); goto done; } p->mark = _mark; - D(fprintf(stderr, "%*c%s _tmp_269[%d-%d]: %s failed!\n", p->level, ' ', + D(fprintf(stderr, "%*c%s _tmp_271[%d-%d]: %s failed!\n", p->level, ' ', p->error_indicator ? "ERROR!" : "-", _mark, p->mark, "expressions ['as' star_target]")); } _res = NULL; @@ -41242,9 +41394,9 @@ _tmp_269_rule(Parser *p) return _res; } -// _tmp_270: expression ['as' star_target] +// _tmp_272: expression ['as' star_target] static void * -_tmp_270_rule(Parser *p) +_tmp_272_rule(Parser *p) { if (p->level++ == MAXSTACK) { _Pypegen_stack_overflow(p); @@ -41260,22 +41412,22 @@ _tmp_270_rule(Parser *p) p->level--; return NULL; } - D(fprintf(stderr, "%*c> _tmp_270[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "expression ['as' star_target]")); + D(fprintf(stderr, "%*c> _tmp_272[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "expression ['as' star_target]")); void *_opt_var; UNUSED(_opt_var); // Silence compiler warnings expr_ty expression_var; if ( (expression_var = expression_rule(p)) // expression && - (_opt_var = _tmp_278_rule(p), !p->error_indicator) // ['as' star_target] + (_opt_var = _tmp_281_rule(p), !p->error_indicator) // ['as' star_target] ) { - D(fprintf(stderr, "%*c+ _tmp_270[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "expression ['as' star_target]")); + D(fprintf(stderr, "%*c+ _tmp_272[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "expression ['as' star_target]")); _res = _PyPegen_dummy_name(p, expression_var, _opt_var); goto done; } p->mark = _mark; - D(fprintf(stderr, "%*c%s _tmp_270[%d-%d]: %s failed!\n", p->level, ' ', + D(fprintf(stderr, "%*c%s _tmp_272[%d-%d]: %s failed!\n", p->level, ' ', p->error_indicator ? "ERROR!" : "-", _mark, p->mark, "expression ['as' star_target]")); } _res = NULL; @@ -41284,9 +41436,9 @@ _tmp_270_rule(Parser *p) return _res; } -// _tmp_271: expressions ['as' star_target] +// _tmp_273: expressions ['as' star_target] static void * -_tmp_271_rule(Parser *p) +_tmp_273_rule(Parser *p) { if (p->level++ == MAXSTACK) { _Pypegen_stack_overflow(p); @@ -41302,22 +41454,22 @@ _tmp_271_rule(Parser *p) p->level--; return NULL; } - D(fprintf(stderr, "%*c> _tmp_271[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "expressions ['as' star_target]")); + D(fprintf(stderr, "%*c> _tmp_273[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "expressions ['as' star_target]")); void *_opt_var; UNUSED(_opt_var); // Silence compiler warnings expr_ty expressions_var; if ( (expressions_var = expressions_rule(p)) // expressions && - (_opt_var = _tmp_279_rule(p), !p->error_indicator) // ['as' star_target] + (_opt_var = _tmp_282_rule(p), !p->error_indicator) // ['as' star_target] ) { - D(fprintf(stderr, "%*c+ _tmp_271[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "expressions ['as' star_target]")); + D(fprintf(stderr, "%*c+ _tmp_273[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "expressions ['as' star_target]")); _res = _PyPegen_dummy_name(p, expressions_var, _opt_var); goto done; } p->mark = _mark; - D(fprintf(stderr, "%*c%s _tmp_271[%d-%d]: %s failed!\n", p->level, ' ', + D(fprintf(stderr, "%*c%s _tmp_273[%d-%d]: %s failed!\n", p->level, ' ', p->error_indicator ? "ERROR!" : "-", _mark, p->mark, "expressions ['as' star_target]")); } _res = NULL; @@ -41326,9 +41478,9 @@ _tmp_271_rule(Parser *p) return _res; } -// _tmp_272: 'as' NAME +// _tmp_274: 'as' NAME static void * -_tmp_272_rule(Parser *p) +_tmp_274_rule(Parser *p) { if (p->level++ == MAXSTACK) { _Pypegen_stack_overflow(p); @@ -41344,21 +41496,21 @@ _tmp_272_rule(Parser *p) p->level--; return NULL; } - D(fprintf(stderr, "%*c> _tmp_272[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "'as' NAME")); + D(fprintf(stderr, "%*c> _tmp_274[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "'as' NAME")); Token * _keyword; expr_ty name_var; if ( - (_keyword = _PyPegen_expect_token(p, 656)) // token='as' + (_keyword = _PyPegen_expect_token(p, 659)) // token='as' && (name_var = _PyPegen_name_token(p)) // NAME ) { - D(fprintf(stderr, "%*c+ _tmp_272[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "'as' NAME")); + D(fprintf(stderr, "%*c+ _tmp_274[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "'as' NAME")); _res = _PyPegen_dummy_name(p, _keyword, name_var); goto done; } p->mark = _mark; - D(fprintf(stderr, "%*c%s _tmp_272[%d-%d]: %s failed!\n", p->level, ' ', + D(fprintf(stderr, "%*c%s _tmp_274[%d-%d]: %s failed!\n", p->level, ' ', p->error_indicator ? "ERROR!" : "-", _mark, p->mark, "'as' NAME")); } _res = NULL; @@ -41367,9 +41519,50 @@ _tmp_272_rule(Parser *p) return _res; } -// _tmp_273: assignment_expression | expression !':=' +// _tmp_275: ',' bitwise_or static void * -_tmp_273_rule(Parser *p) +_tmp_275_rule(Parser *p) +{ + if (p->level++ == MAXSTACK) { + _Pypegen_stack_overflow(p); + } + if (p->error_indicator) { + p->level--; + return NULL; + } + void * _res = NULL; + int _mark = p->mark; + { // ',' bitwise_or + if (p->error_indicator) { + p->level--; + return NULL; + } + D(fprintf(stderr, "%*c> _tmp_275[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "',' bitwise_or")); + Token * _literal; + expr_ty bitwise_or_var; + if ( + (_literal = _PyPegen_expect_token(p, 12)) // token=',' + && + (bitwise_or_var = bitwise_or_rule(p)) // bitwise_or + ) + { + D(fprintf(stderr, "%*c+ _tmp_275[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "',' bitwise_or")); + _res = _PyPegen_dummy_name(p, _literal, bitwise_or_var); + goto done; + } + p->mark = _mark; + D(fprintf(stderr, "%*c%s _tmp_275[%d-%d]: %s failed!\n", p->level, ' ', + p->error_indicator ? "ERROR!" : "-", _mark, p->mark, "',' bitwise_or")); + } + _res = NULL; + done: + p->level--; + return _res; +} + +// _tmp_276: assignment_expression | expression !':=' +static void * +_tmp_276_rule(Parser *p) { if (p->level++ == MAXSTACK) { _Pypegen_stack_overflow(p); @@ -41385,18 +41578,18 @@ _tmp_273_rule(Parser *p) p->level--; return NULL; } - D(fprintf(stderr, "%*c> _tmp_273[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "assignment_expression")); + D(fprintf(stderr, "%*c> _tmp_276[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "assignment_expression")); expr_ty assignment_expression_var; if ( (assignment_expression_var = assignment_expression_rule(p)) // assignment_expression ) { - D(fprintf(stderr, "%*c+ _tmp_273[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "assignment_expression")); + D(fprintf(stderr, "%*c+ _tmp_276[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "assignment_expression")); _res = assignment_expression_var; goto done; } p->mark = _mark; - D(fprintf(stderr, "%*c%s _tmp_273[%d-%d]: %s failed!\n", p->level, ' ', + D(fprintf(stderr, "%*c%s _tmp_276[%d-%d]: %s failed!\n", p->level, ' ', p->error_indicator ? "ERROR!" : "-", _mark, p->mark, "assignment_expression")); } { // expression !':=' @@ -41404,7 +41597,7 @@ _tmp_273_rule(Parser *p) p->level--; return NULL; } - D(fprintf(stderr, "%*c> _tmp_273[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "expression !':='")); + D(fprintf(stderr, "%*c> _tmp_276[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "expression !':='")); expr_ty expression_var; if ( (expression_var = expression_rule(p)) // expression @@ -41412,12 +41605,12 @@ _tmp_273_rule(Parser *p) _PyPegen_lookahead_with_int(0, _PyPegen_expect_token, p, 53) // token=':=' ) { - D(fprintf(stderr, "%*c+ _tmp_273[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "expression !':='")); + D(fprintf(stderr, "%*c+ _tmp_276[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "expression !':='")); _res = expression_var; goto done; } p->mark = _mark; - D(fprintf(stderr, "%*c%s _tmp_273[%d-%d]: %s failed!\n", p->level, ' ', + D(fprintf(stderr, "%*c%s _tmp_276[%d-%d]: %s failed!\n", p->level, ' ', p->error_indicator ? "ERROR!" : "-", _mark, p->mark, "expression !':='")); } _res = NULL; @@ -41426,9 +41619,9 @@ _tmp_273_rule(Parser *p) return _res; } -// _loop0_275: ',' (starred_expression | (assignment_expression | expression !':=') !'=') +// _loop0_278: ',' (starred_expression | (assignment_expression | expression !':=') !'=') static asdl_seq * -_loop0_275_rule(Parser *p) +_loop0_278_rule(Parser *p) { if (p->level++ == MAXSTACK) { _Pypegen_stack_overflow(p); @@ -41453,13 +41646,13 @@ _loop0_275_rule(Parser *p) p->level--; return NULL; } - D(fprintf(stderr, "%*c> _loop0_275[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "',' (starred_expression | (assignment_expression | expression !':=') !'=')")); + D(fprintf(stderr, "%*c> _loop0_278[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "',' (starred_expression | (assignment_expression | expression !':=') !'=')")); Token * _literal; void *elem; while ( (_literal = _PyPegen_expect_token(p, 12)) // token=',' && - (elem = _tmp_280_rule(p)) // starred_expression | (assignment_expression | expression !':=') !'=' + (elem = _tmp_283_rule(p)) // starred_expression | (assignment_expression | expression !':=') !'=' ) { _res = elem; @@ -41485,7 +41678,7 @@ _loop0_275_rule(Parser *p) _mark = p->mark; } p->mark = _mark; - D(fprintf(stderr, "%*c%s _loop0_275[%d-%d]: %s failed!\n", p->level, ' ', + D(fprintf(stderr, "%*c%s _loop0_278[%d-%d]: %s failed!\n", p->level, ' ', p->error_indicator ? "ERROR!" : "-", _mark, p->mark, "',' (starred_expression | (assignment_expression | expression !':=') !'=')")); } asdl_seq *_seq = (asdl_seq*)_Py_asdl_generic_seq_new(_n, p->arena); @@ -41502,10 +41695,10 @@ _loop0_275_rule(Parser *p) return _seq; } -// _gather_274: -// | (starred_expression | (assignment_expression | expression !':=') !'=') _loop0_275 +// _gather_277: +// | (starred_expression | (assignment_expression | expression !':=') !'=') _loop0_278 static asdl_seq * -_gather_274_rule(Parser *p) +_gather_277_rule(Parser *p) { if (p->level++ == MAXSTACK) { _Pypegen_stack_overflow(p); @@ -41516,27 +41709,27 @@ _gather_274_rule(Parser *p) } asdl_seq * _res = NULL; int _mark = p->mark; - { // (starred_expression | (assignment_expression | expression !':=') !'=') _loop0_275 + { // (starred_expression | (assignment_expression | expression !':=') !'=') _loop0_278 if (p->error_indicator) { p->level--; return NULL; } - D(fprintf(stderr, "%*c> _gather_274[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "(starred_expression | (assignment_expression | expression !':=') !'=') _loop0_275")); + D(fprintf(stderr, "%*c> _gather_277[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "(starred_expression | (assignment_expression | expression !':=') !'=') _loop0_278")); void *elem; asdl_seq * seq; if ( - (elem = _tmp_280_rule(p)) // starred_expression | (assignment_expression | expression !':=') !'=' + (elem = _tmp_283_rule(p)) // starred_expression | (assignment_expression | expression !':=') !'=' && - (seq = _loop0_275_rule(p)) // _loop0_275 + (seq = _loop0_278_rule(p)) // _loop0_278 ) { - D(fprintf(stderr, "%*c+ _gather_274[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "(starred_expression | (assignment_expression | expression !':=') !'=') _loop0_275")); + D(fprintf(stderr, "%*c+ _gather_277[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "(starred_expression | (assignment_expression | expression !':=') !'=') _loop0_278")); _res = _PyPegen_seq_insert_in_front(p, elem, seq); goto done; } p->mark = _mark; - D(fprintf(stderr, "%*c%s _gather_274[%d-%d]: %s failed!\n", p->level, ' ', - p->error_indicator ? "ERROR!" : "-", _mark, p->mark, "(starred_expression | (assignment_expression | expression !':=') !'=') _loop0_275")); + D(fprintf(stderr, "%*c%s _gather_277[%d-%d]: %s failed!\n", p->level, ' ', + p->error_indicator ? "ERROR!" : "-", _mark, p->mark, "(starred_expression | (assignment_expression | expression !':=') !'=') _loop0_278")); } _res = NULL; done: @@ -41544,9 +41737,9 @@ _gather_274_rule(Parser *p) return _res; } -// _tmp_276: 'as' star_target +// _tmp_279: 'as' star_target static void * -_tmp_276_rule(Parser *p) +_tmp_279_rule(Parser *p) { if (p->level++ == MAXSTACK) { _Pypegen_stack_overflow(p); @@ -41562,21 +41755,21 @@ _tmp_276_rule(Parser *p) p->level--; return NULL; } - D(fprintf(stderr, "%*c> _tmp_276[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "'as' star_target")); + D(fprintf(stderr, "%*c> _tmp_279[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "'as' star_target")); Token * _keyword; expr_ty star_target_var; if ( - (_keyword = _PyPegen_expect_token(p, 656)) // token='as' + (_keyword = _PyPegen_expect_token(p, 659)) // token='as' && (star_target_var = star_target_rule(p)) // star_target ) { - D(fprintf(stderr, "%*c+ _tmp_276[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "'as' star_target")); + D(fprintf(stderr, "%*c+ _tmp_279[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "'as' star_target")); _res = _PyPegen_dummy_name(p, _keyword, star_target_var); goto done; } p->mark = _mark; - D(fprintf(stderr, "%*c%s _tmp_276[%d-%d]: %s failed!\n", p->level, ' ', + D(fprintf(stderr, "%*c%s _tmp_279[%d-%d]: %s failed!\n", p->level, ' ', p->error_indicator ? "ERROR!" : "-", _mark, p->mark, "'as' star_target")); } _res = NULL; @@ -41585,9 +41778,9 @@ _tmp_276_rule(Parser *p) return _res; } -// _tmp_277: 'as' star_target +// _tmp_280: 'as' star_target static void * -_tmp_277_rule(Parser *p) +_tmp_280_rule(Parser *p) { if (p->level++ == MAXSTACK) { _Pypegen_stack_overflow(p); @@ -41603,21 +41796,21 @@ _tmp_277_rule(Parser *p) p->level--; return NULL; } - D(fprintf(stderr, "%*c> _tmp_277[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "'as' star_target")); + D(fprintf(stderr, "%*c> _tmp_280[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "'as' star_target")); Token * _keyword; expr_ty star_target_var; if ( - (_keyword = _PyPegen_expect_token(p, 656)) // token='as' + (_keyword = _PyPegen_expect_token(p, 659)) // token='as' && (star_target_var = star_target_rule(p)) // star_target ) { - D(fprintf(stderr, "%*c+ _tmp_277[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "'as' star_target")); + D(fprintf(stderr, "%*c+ _tmp_280[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "'as' star_target")); _res = _PyPegen_dummy_name(p, _keyword, star_target_var); goto done; } p->mark = _mark; - D(fprintf(stderr, "%*c%s _tmp_277[%d-%d]: %s failed!\n", p->level, ' ', + D(fprintf(stderr, "%*c%s _tmp_280[%d-%d]: %s failed!\n", p->level, ' ', p->error_indicator ? "ERROR!" : "-", _mark, p->mark, "'as' star_target")); } _res = NULL; @@ -41626,9 +41819,9 @@ _tmp_277_rule(Parser *p) return _res; } -// _tmp_278: 'as' star_target +// _tmp_281: 'as' star_target static void * -_tmp_278_rule(Parser *p) +_tmp_281_rule(Parser *p) { if (p->level++ == MAXSTACK) { _Pypegen_stack_overflow(p); @@ -41644,21 +41837,21 @@ _tmp_278_rule(Parser *p) p->level--; return NULL; } - D(fprintf(stderr, "%*c> _tmp_278[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "'as' star_target")); + D(fprintf(stderr, "%*c> _tmp_281[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "'as' star_target")); Token * _keyword; expr_ty star_target_var; if ( - (_keyword = _PyPegen_expect_token(p, 656)) // token='as' + (_keyword = _PyPegen_expect_token(p, 659)) // token='as' && (star_target_var = star_target_rule(p)) // star_target ) { - D(fprintf(stderr, "%*c+ _tmp_278[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "'as' star_target")); + D(fprintf(stderr, "%*c+ _tmp_281[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "'as' star_target")); _res = _PyPegen_dummy_name(p, _keyword, star_target_var); goto done; } p->mark = _mark; - D(fprintf(stderr, "%*c%s _tmp_278[%d-%d]: %s failed!\n", p->level, ' ', + D(fprintf(stderr, "%*c%s _tmp_281[%d-%d]: %s failed!\n", p->level, ' ', p->error_indicator ? "ERROR!" : "-", _mark, p->mark, "'as' star_target")); } _res = NULL; @@ -41667,9 +41860,9 @@ _tmp_278_rule(Parser *p) return _res; } -// _tmp_279: 'as' star_target +// _tmp_282: 'as' star_target static void * -_tmp_279_rule(Parser *p) +_tmp_282_rule(Parser *p) { if (p->level++ == MAXSTACK) { _Pypegen_stack_overflow(p); @@ -41685,21 +41878,21 @@ _tmp_279_rule(Parser *p) p->level--; return NULL; } - D(fprintf(stderr, "%*c> _tmp_279[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "'as' star_target")); + D(fprintf(stderr, "%*c> _tmp_282[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "'as' star_target")); Token * _keyword; expr_ty star_target_var; if ( - (_keyword = _PyPegen_expect_token(p, 656)) // token='as' + (_keyword = _PyPegen_expect_token(p, 659)) // token='as' && (star_target_var = star_target_rule(p)) // star_target ) { - D(fprintf(stderr, "%*c+ _tmp_279[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "'as' star_target")); + D(fprintf(stderr, "%*c+ _tmp_282[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "'as' star_target")); _res = _PyPegen_dummy_name(p, _keyword, star_target_var); goto done; } p->mark = _mark; - D(fprintf(stderr, "%*c%s _tmp_279[%d-%d]: %s failed!\n", p->level, ' ', + D(fprintf(stderr, "%*c%s _tmp_282[%d-%d]: %s failed!\n", p->level, ' ', p->error_indicator ? "ERROR!" : "-", _mark, p->mark, "'as' star_target")); } _res = NULL; @@ -41708,9 +41901,9 @@ _tmp_279_rule(Parser *p) return _res; } -// _tmp_280: starred_expression | (assignment_expression | expression !':=') !'=' +// _tmp_283: starred_expression | (assignment_expression | expression !':=') !'=' static void * -_tmp_280_rule(Parser *p) +_tmp_283_rule(Parser *p) { if (p->level++ == MAXSTACK) { _Pypegen_stack_overflow(p); @@ -41726,18 +41919,18 @@ _tmp_280_rule(Parser *p) p->level--; return NULL; } - D(fprintf(stderr, "%*c> _tmp_280[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "starred_expression")); + D(fprintf(stderr, "%*c> _tmp_283[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "starred_expression")); expr_ty starred_expression_var; if ( (starred_expression_var = starred_expression_rule(p)) // starred_expression ) { - D(fprintf(stderr, "%*c+ _tmp_280[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "starred_expression")); + D(fprintf(stderr, "%*c+ _tmp_283[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "starred_expression")); _res = starred_expression_var; goto done; } p->mark = _mark; - D(fprintf(stderr, "%*c%s _tmp_280[%d-%d]: %s failed!\n", p->level, ' ', + D(fprintf(stderr, "%*c%s _tmp_283[%d-%d]: %s failed!\n", p->level, ' ', p->error_indicator ? "ERROR!" : "-", _mark, p->mark, "starred_expression")); } { // (assignment_expression | expression !':=') !'=' @@ -41745,20 +41938,20 @@ _tmp_280_rule(Parser *p) p->level--; return NULL; } - D(fprintf(stderr, "%*c> _tmp_280[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "(assignment_expression | expression !':=') !'='")); - void *_tmp_281_var; + D(fprintf(stderr, "%*c> _tmp_283[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "(assignment_expression | expression !':=') !'='")); + void *_tmp_284_var; if ( - (_tmp_281_var = _tmp_281_rule(p)) // assignment_expression | expression !':=' + (_tmp_284_var = _tmp_284_rule(p)) // assignment_expression | expression !':=' && _PyPegen_lookahead_with_int(0, _PyPegen_expect_token, p, 22) // token='=' ) { - D(fprintf(stderr, "%*c+ _tmp_280[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "(assignment_expression | expression !':=') !'='")); - _res = _tmp_281_var; + D(fprintf(stderr, "%*c+ _tmp_283[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "(assignment_expression | expression !':=') !'='")); + _res = _tmp_284_var; goto done; } p->mark = _mark; - D(fprintf(stderr, "%*c%s _tmp_280[%d-%d]: %s failed!\n", p->level, ' ', + D(fprintf(stderr, "%*c%s _tmp_283[%d-%d]: %s failed!\n", p->level, ' ', p->error_indicator ? "ERROR!" : "-", _mark, p->mark, "(assignment_expression | expression !':=') !'='")); } _res = NULL; @@ -41767,9 +41960,9 @@ _tmp_280_rule(Parser *p) return _res; } -// _tmp_281: assignment_expression | expression !':=' +// _tmp_284: assignment_expression | expression !':=' static void * -_tmp_281_rule(Parser *p) +_tmp_284_rule(Parser *p) { if (p->level++ == MAXSTACK) { _Pypegen_stack_overflow(p); @@ -41785,18 +41978,18 @@ _tmp_281_rule(Parser *p) p->level--; return NULL; } - D(fprintf(stderr, "%*c> _tmp_281[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "assignment_expression")); + D(fprintf(stderr, "%*c> _tmp_284[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "assignment_expression")); expr_ty assignment_expression_var; if ( (assignment_expression_var = assignment_expression_rule(p)) // assignment_expression ) { - D(fprintf(stderr, "%*c+ _tmp_281[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "assignment_expression")); + D(fprintf(stderr, "%*c+ _tmp_284[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "assignment_expression")); _res = assignment_expression_var; goto done; } p->mark = _mark; - D(fprintf(stderr, "%*c%s _tmp_281[%d-%d]: %s failed!\n", p->level, ' ', + D(fprintf(stderr, "%*c%s _tmp_284[%d-%d]: %s failed!\n", p->level, ' ', p->error_indicator ? "ERROR!" : "-", _mark, p->mark, "assignment_expression")); } { // expression !':=' @@ -41804,7 +41997,7 @@ _tmp_281_rule(Parser *p) p->level--; return NULL; } - D(fprintf(stderr, "%*c> _tmp_281[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "expression !':='")); + D(fprintf(stderr, "%*c> _tmp_284[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "expression !':='")); expr_ty expression_var; if ( (expression_var = expression_rule(p)) // expression @@ -41812,12 +42005,12 @@ _tmp_281_rule(Parser *p) _PyPegen_lookahead_with_int(0, _PyPegen_expect_token, p, 53) // token=':=' ) { - D(fprintf(stderr, "%*c+ _tmp_281[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "expression !':='")); + D(fprintf(stderr, "%*c+ _tmp_284[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "expression !':='")); _res = expression_var; goto done; } p->mark = _mark; - D(fprintf(stderr, "%*c%s _tmp_281[%d-%d]: %s failed!\n", p->level, ' ', + D(fprintf(stderr, "%*c%s _tmp_284[%d-%d]: %s failed!\n", p->level, ' ', p->error_indicator ? "ERROR!" : "-", _mark, p->mark, "expression !':='")); } _res = NULL; diff --git a/Parser/pegen.c b/Parser/pegen.c index 7766253a76066f..3d3e64559403b1 100644 --- a/Parser/pegen.c +++ b/Parser/pegen.c @@ -844,7 +844,7 @@ _PyPegen_run_parser(Parser *p) if (res == NULL) { if ((p->flags & PyPARSE_ALLOW_INCOMPLETE_INPUT) && _is_end_of_source(p)) { PyErr_Clear(); - return RAISE_SYNTAX_ERROR("incomplete input"); + return _PyPegen_raise_error(p, PyExc_IncompleteInputError, 0, "incomplete input"); } if (PyErr_Occurred() && !PyErr_ExceptionMatches(PyExc_SyntaxError)) { return NULL; diff --git a/Parser/pegen_errors.c b/Parser/pegen_errors.c index 8a02aab1f4e504..e15673d02dd3b0 100644 --- a/Parser/pegen_errors.c +++ b/Parser/pegen_errors.c @@ -311,6 +311,10 @@ _PyPegen_raise_error_known_location(Parser *p, PyObject *errtype, Py_ssize_t end_lineno, Py_ssize_t end_col_offset, const char *errmsg, va_list va) { + // Bail out if we already have an error set. + if (p->error_indicator && PyErr_Occurred()) { + return NULL; + } PyObject *value = NULL; PyObject *errstr = NULL; PyObject *error_line = NULL; diff --git a/Programs/test_frozenmain.h b/Programs/test_frozenmain.h index 4fb78cf632d70e..657e9345cf5ab7 100644 --- a/Programs/test_frozenmain.h +++ b/Programs/test_frozenmain.h @@ -1,7 +1,7 @@ // Auto-generated by Programs/freeze_test_frozenmain.py unsigned char M_test_frozenmain[] = { 227,0,0,0,0,0,0,0,0,0,0,0,0,8,0,0, - 0,0,0,0,0,243,164,0,0,0,149,0,83,0,83,1, + 0,0,0,0,0,243,166,0,0,0,149,0,83,0,83,1, 75,0,114,0,83,0,83,1,75,1,114,1,92,2,34,0, 83,2,53,1,0,0,0,0,0,0,32,0,92,2,34,0, 83,3,92,0,82,6,0,0,0,0,0,0,0,0,0,0, @@ -11,28 +11,28 @@ unsigned char M_test_frozenmain[] = { 0,0,83,4,5,0,0,0,114,5,83,5,19,0,72,20, 0,0,114,6,92,2,34,0,83,6,92,6,14,0,83,7, 92,5,92,6,5,0,0,0,14,0,51,4,53,1,0,0, - 0,0,0,0,32,0,77,22,0,0,11,0,103,1,41,8, - 233,0,0,0,0,78,122,18,70,114,111,122,101,110,32,72, - 101,108,108,111,32,87,111,114,108,100,122,8,115,121,115,46, - 97,114,103,118,218,6,99,111,110,102,105,103,41,5,218,12, - 112,114,111,103,114,97,109,95,110,97,109,101,218,10,101,120, - 101,99,117,116,97,98,108,101,218,15,117,115,101,95,101,110, - 118,105,114,111,110,109,101,110,116,218,17,99,111,110,102,105, - 103,117,114,101,95,99,95,115,116,100,105,111,218,14,98,117, - 102,102,101,114,101,100,95,115,116,100,105,111,122,7,99,111, - 110,102,105,103,32,122,2,58,32,41,7,218,3,115,121,115, - 218,17,95,116,101,115,116,105,110,116,101,114,110,97,108,99, - 97,112,105,218,5,112,114,105,110,116,218,4,97,114,103,118, - 218,11,103,101,116,95,99,111,110,102,105,103,115,114,3,0, - 0,0,218,3,107,101,121,169,0,243,0,0,0,0,250,18, - 116,101,115,116,95,102,114,111,122,101,110,109,97,105,110,46, - 112,121,250,8,60,109,111,100,117,108,101,62,114,18,0,0, - 0,1,0,0,0,115,99,0,0,0,240,3,1,1,1,243, - 8,0,1,11,219,0,24,225,0,5,208,6,26,212,0,27, - 217,0,5,128,106,144,35,151,40,145,40,212,0,27,216,9, - 26,215,9,38,210,9,38,211,9,40,168,24,209,9,50,128, - 6,240,2,6,12,2,242,0,7,1,42,128,67,241,14,0, - 5,10,136,71,144,67,144,53,152,2,152,54,160,35,153,59, - 152,45,208,10,40,214,4,41,241,15,7,1,42,114,16,0, - 0,0, + 0,0,0,0,32,0,77,22,0,0,11,0,32,0,103,1, + 41,8,233,0,0,0,0,78,122,18,70,114,111,122,101,110, + 32,72,101,108,108,111,32,87,111,114,108,100,122,8,115,121, + 115,46,97,114,103,118,218,6,99,111,110,102,105,103,41,5, + 218,12,112,114,111,103,114,97,109,95,110,97,109,101,218,10, + 101,120,101,99,117,116,97,98,108,101,218,15,117,115,101,95, + 101,110,118,105,114,111,110,109,101,110,116,218,17,99,111,110, + 102,105,103,117,114,101,95,99,95,115,116,100,105,111,218,14, + 98,117,102,102,101,114,101,100,95,115,116,100,105,111,122,7, + 99,111,110,102,105,103,32,122,2,58,32,41,7,218,3,115, + 121,115,218,17,95,116,101,115,116,105,110,116,101,114,110,97, + 108,99,97,112,105,218,5,112,114,105,110,116,218,4,97,114, + 103,118,218,11,103,101,116,95,99,111,110,102,105,103,115,114, + 3,0,0,0,218,3,107,101,121,169,0,243,0,0,0,0, + 250,18,116,101,115,116,95,102,114,111,122,101,110,109,97,105, + 110,46,112,121,250,8,60,109,111,100,117,108,101,62,114,18, + 0,0,0,1,0,0,0,115,99,0,0,0,240,3,1,1, + 1,243,8,0,1,11,219,0,24,225,0,5,208,6,26,212, + 0,27,217,0,5,128,106,144,35,151,40,145,40,212,0,27, + 216,9,26,215,9,38,210,9,38,211,9,40,168,24,209,9, + 50,128,6,240,2,6,12,2,242,0,7,1,42,128,67,241, + 14,0,5,10,136,71,144,67,144,53,152,2,152,54,160,35, + 153,59,152,45,208,10,40,214,4,41,242,15,7,1,42,114, + 16,0,0,0, }; diff --git a/Python/Python-ast.c b/Python/Python-ast.c index 75dc3e156aa945..d77e986ba067a3 100644 --- a/Python/Python-ast.c +++ b/Python/Python-ast.c @@ -9,6 +9,11 @@ #include "pycore_pystate.h" // _PyInterpreterState_GET() #include +struct validator { + int recursion_depth; /* current recursion depth */ + int recursion_limit; /* recursion limit */ +}; + // Forward declaration static int init_types(struct ast_state *state); @@ -383,7 +388,8 @@ GENERATE_ASDL_SEQ_CONSTRUCTOR(pattern, pattern_ty) GENERATE_ASDL_SEQ_CONSTRUCTOR(type_ignore, type_ignore_ty) GENERATE_ASDL_SEQ_CONSTRUCTOR(type_param, type_param_ty) -static PyObject* ast2obj_mod(struct ast_state *state, void*); +static PyObject* ast2obj_mod(struct ast_state *state, struct validator *vstate, + void*); static const char * const Module_fields[]={ "body", "type_ignores", @@ -404,7 +410,8 @@ static const char * const stmt_attributes[] = { "end_lineno", "end_col_offset", }; -static PyObject* ast2obj_stmt(struct ast_state *state, void*); +static PyObject* ast2obj_stmt(struct ast_state *state, struct validator + *vstate, void*); static const char * const FunctionDef_fields[]={ "name", "args", @@ -539,7 +546,8 @@ static const char * const expr_attributes[] = { "end_lineno", "end_col_offset", }; -static PyObject* ast2obj_expr(struct ast_state *state, void*); +static PyObject* ast2obj_expr(struct ast_state *state, struct validator + *vstate, void*); static const char * const BoolOp_fields[]={ "op", "values", @@ -652,12 +660,18 @@ static const char * const Slice_fields[]={ "upper", "step", }; -static PyObject* ast2obj_expr_context(struct ast_state *state, expr_context_ty); -static PyObject* ast2obj_boolop(struct ast_state *state, boolop_ty); -static PyObject* ast2obj_operator(struct ast_state *state, operator_ty); -static PyObject* ast2obj_unaryop(struct ast_state *state, unaryop_ty); -static PyObject* ast2obj_cmpop(struct ast_state *state, cmpop_ty); -static PyObject* ast2obj_comprehension(struct ast_state *state, void*); +static PyObject* ast2obj_expr_context(struct ast_state *state, struct validator + *vstate, expr_context_ty); +static PyObject* ast2obj_boolop(struct ast_state *state, struct validator + *vstate, boolop_ty); +static PyObject* ast2obj_operator(struct ast_state *state, struct validator + *vstate, operator_ty); +static PyObject* ast2obj_unaryop(struct ast_state *state, struct validator + *vstate, unaryop_ty); +static PyObject* ast2obj_cmpop(struct ast_state *state, struct validator + *vstate, cmpop_ty); +static PyObject* ast2obj_comprehension(struct ast_state *state, struct + validator *vstate, void*); static const char * const comprehension_fields[]={ "target", "iter", @@ -670,13 +684,15 @@ static const char * const excepthandler_attributes[] = { "end_lineno", "end_col_offset", }; -static PyObject* ast2obj_excepthandler(struct ast_state *state, void*); +static PyObject* ast2obj_excepthandler(struct ast_state *state, struct + validator *vstate, void*); static const char * const ExceptHandler_fields[]={ "type", "name", "body", }; -static PyObject* ast2obj_arguments(struct ast_state *state, void*); +static PyObject* ast2obj_arguments(struct ast_state *state, struct validator + *vstate, void*); static const char * const arguments_fields[]={ "posonlyargs", "args", @@ -686,7 +702,8 @@ static const char * const arguments_fields[]={ "kwarg", "defaults", }; -static PyObject* ast2obj_arg(struct ast_state *state, void*); +static PyObject* ast2obj_arg(struct ast_state *state, struct validator *vstate, + void*); static const char * const arg_attributes[] = { "lineno", "col_offset", @@ -698,7 +715,8 @@ static const char * const arg_fields[]={ "annotation", "type_comment", }; -static PyObject* ast2obj_keyword(struct ast_state *state, void*); +static PyObject* ast2obj_keyword(struct ast_state *state, struct validator + *vstate, void*); static const char * const keyword_attributes[] = { "lineno", "col_offset", @@ -709,7 +727,8 @@ static const char * const keyword_fields[]={ "arg", "value", }; -static PyObject* ast2obj_alias(struct ast_state *state, void*); +static PyObject* ast2obj_alias(struct ast_state *state, struct validator + *vstate, void*); static const char * const alias_attributes[] = { "lineno", "col_offset", @@ -720,12 +739,14 @@ static const char * const alias_fields[]={ "name", "asname", }; -static PyObject* ast2obj_withitem(struct ast_state *state, void*); +static PyObject* ast2obj_withitem(struct ast_state *state, struct validator + *vstate, void*); static const char * const withitem_fields[]={ "context_expr", "optional_vars", }; -static PyObject* ast2obj_match_case(struct ast_state *state, void*); +static PyObject* ast2obj_match_case(struct ast_state *state, struct validator + *vstate, void*); static const char * const match_case_fields[]={ "pattern", "guard", @@ -737,7 +758,8 @@ static const char * const pattern_attributes[] = { "end_lineno", "end_col_offset", }; -static PyObject* ast2obj_pattern(struct ast_state *state, void*); +static PyObject* ast2obj_pattern(struct ast_state *state, struct validator + *vstate, void*); static const char * const MatchValue_fields[]={ "value", }; @@ -768,7 +790,8 @@ static const char * const MatchAs_fields[]={ static const char * const MatchOr_fields[]={ "patterns", }; -static PyObject* ast2obj_type_ignore(struct ast_state *state, void*); +static PyObject* ast2obj_type_ignore(struct ast_state *state, struct validator + *vstate, void*); static const char * const TypeIgnore_fields[]={ "lineno", "tag", @@ -779,7 +802,8 @@ static const char * const type_param_attributes[] = { "end_lineno", "end_col_offset", }; -static PyObject* ast2obj_type_param(struct ast_state *state, void*); +static PyObject* ast2obj_type_param(struct ast_state *state, struct validator + *vstate, void*); static const char * const TypeVar_fields[]={ "name", "bound", @@ -1008,7 +1032,8 @@ add_attributes(struct ast_state *state, PyObject *type, const char * const *attr /* Conversion AST -> Python */ -static PyObject* ast2obj_list(struct ast_state *state, asdl_seq *seq, PyObject* (*func)(struct ast_state *state, void*)) +static PyObject* ast2obj_list(struct ast_state *state, struct validator *vstate, asdl_seq *seq, + PyObject* (*func)(struct ast_state *state, struct validator *vstate, void*)) { Py_ssize_t i, n = asdl_seq_LEN(seq); PyObject *result = PyList_New(n); @@ -1016,7 +1041,7 @@ static PyObject* ast2obj_list(struct ast_state *state, asdl_seq *seq, PyObject* if (!result) return NULL; for (i = 0; i < n; i++) { - value = func(state, asdl_seq_GET_UNTYPED(seq, i)); + value = func(state, vstate, asdl_seq_GET_UNTYPED(seq, i)); if (!value) { Py_DECREF(result); return NULL; @@ -1026,7 +1051,7 @@ static PyObject* ast2obj_list(struct ast_state *state, asdl_seq *seq, PyObject* return result; } -static PyObject* ast2obj_object(struct ast_state *Py_UNUSED(state), void *o) +static PyObject* ast2obj_object(struct ast_state *Py_UNUSED(state), struct validator *Py_UNUSED(vstate), void *o) { PyObject *op = (PyObject*)o; if (!op) { @@ -1038,7 +1063,7 @@ static PyObject* ast2obj_object(struct ast_state *Py_UNUSED(state), void *o) #define ast2obj_identifier ast2obj_object #define ast2obj_string ast2obj_object -static PyObject* ast2obj_int(struct ast_state *Py_UNUSED(state), long b) +static PyObject* ast2obj_int(struct ast_state *Py_UNUSED(state), struct validator *Py_UNUSED(vstate), long b) { return PyLong_FromLong(b); } @@ -1914,8 +1939,6 @@ init_types(struct ast_state *state) "TypeVarTuple(identifier name)"); if (!state->TypeVarTuple_type) return -1; - state->recursion_depth = 0; - state->recursion_limit = 0; return 0; } @@ -3770,7 +3793,7 @@ _PyAST_TypeVarTuple(identifier name, int lineno, int col_offset, int PyObject* -ast2obj_mod(struct ast_state *state, void* _o) +ast2obj_mod(struct ast_state *state, struct validator *vstate, void* _o) { mod_ty o = (mod_ty)_o; PyObject *result = NULL, *value = NULL; @@ -3778,7 +3801,7 @@ ast2obj_mod(struct ast_state *state, void* _o) if (!o) { Py_RETURN_NONE; } - if (++state->recursion_depth > state->recursion_limit) { + if (++vstate->recursion_depth > vstate->recursion_limit) { PyErr_SetString(PyExc_RecursionError, "maximum recursion depth exceeded during ast construction"); return NULL; @@ -3788,12 +3811,14 @@ ast2obj_mod(struct ast_state *state, void* _o) tp = (PyTypeObject *)state->Module_type; result = PyType_GenericNew(tp, NULL, NULL); if (!result) goto failed; - value = ast2obj_list(state, (asdl_seq*)o->v.Module.body, ast2obj_stmt); + value = ast2obj_list(state, vstate, (asdl_seq*)o->v.Module.body, + ast2obj_stmt); if (!value) goto failed; if (PyObject_SetAttr(result, state->body, value) == -1) goto failed; Py_DECREF(value); - value = ast2obj_list(state, (asdl_seq*)o->v.Module.type_ignores, + value = ast2obj_list(state, vstate, + (asdl_seq*)o->v.Module.type_ignores, ast2obj_type_ignore); if (!value) goto failed; if (PyObject_SetAttr(result, state->type_ignores, value) == -1) @@ -3804,7 +3829,7 @@ ast2obj_mod(struct ast_state *state, void* _o) tp = (PyTypeObject *)state->Interactive_type; result = PyType_GenericNew(tp, NULL, NULL); if (!result) goto failed; - value = ast2obj_list(state, (asdl_seq*)o->v.Interactive.body, + value = ast2obj_list(state, vstate, (asdl_seq*)o->v.Interactive.body, ast2obj_stmt); if (!value) goto failed; if (PyObject_SetAttr(result, state->body, value) == -1) @@ -3815,7 +3840,7 @@ ast2obj_mod(struct ast_state *state, void* _o) tp = (PyTypeObject *)state->Expression_type; result = PyType_GenericNew(tp, NULL, NULL); if (!result) goto failed; - value = ast2obj_expr(state, o->v.Expression.body); + value = ast2obj_expr(state, vstate, o->v.Expression.body); if (!value) goto failed; if (PyObject_SetAttr(result, state->body, value) == -1) goto failed; @@ -3825,30 +3850,31 @@ ast2obj_mod(struct ast_state *state, void* _o) tp = (PyTypeObject *)state->FunctionType_type; result = PyType_GenericNew(tp, NULL, NULL); if (!result) goto failed; - value = ast2obj_list(state, (asdl_seq*)o->v.FunctionType.argtypes, + value = ast2obj_list(state, vstate, + (asdl_seq*)o->v.FunctionType.argtypes, ast2obj_expr); if (!value) goto failed; if (PyObject_SetAttr(result, state->argtypes, value) == -1) goto failed; Py_DECREF(value); - value = ast2obj_expr(state, o->v.FunctionType.returns); + value = ast2obj_expr(state, vstate, o->v.FunctionType.returns); if (!value) goto failed; if (PyObject_SetAttr(result, state->returns, value) == -1) goto failed; Py_DECREF(value); break; } - state->recursion_depth--; + vstate->recursion_depth--; return result; failed: - state->recursion_depth--; + vstate->recursion_depth--; Py_XDECREF(value); Py_XDECREF(result); return NULL; } PyObject* -ast2obj_stmt(struct ast_state *state, void* _o) +ast2obj_stmt(struct ast_state *state, struct validator *vstate, void* _o) { stmt_ty o = (stmt_ty)_o; PyObject *result = NULL, *value = NULL; @@ -3856,7 +3882,7 @@ ast2obj_stmt(struct ast_state *state, void* _o) if (!o) { Py_RETURN_NONE; } - if (++state->recursion_depth > state->recursion_limit) { + if (++vstate->recursion_depth > vstate->recursion_limit) { PyErr_SetString(PyExc_RecursionError, "maximum recursion depth exceeded during ast construction"); return NULL; @@ -3866,39 +3892,41 @@ ast2obj_stmt(struct ast_state *state, void* _o) tp = (PyTypeObject *)state->FunctionDef_type; result = PyType_GenericNew(tp, NULL, NULL); if (!result) goto failed; - value = ast2obj_identifier(state, o->v.FunctionDef.name); + value = ast2obj_identifier(state, vstate, o->v.FunctionDef.name); if (!value) goto failed; if (PyObject_SetAttr(result, state->name, value) == -1) goto failed; Py_DECREF(value); - value = ast2obj_arguments(state, o->v.FunctionDef.args); + value = ast2obj_arguments(state, vstate, o->v.FunctionDef.args); if (!value) goto failed; if (PyObject_SetAttr(result, state->args, value) == -1) goto failed; Py_DECREF(value); - value = ast2obj_list(state, (asdl_seq*)o->v.FunctionDef.body, + value = ast2obj_list(state, vstate, (asdl_seq*)o->v.FunctionDef.body, ast2obj_stmt); if (!value) goto failed; if (PyObject_SetAttr(result, state->body, value) == -1) goto failed; Py_DECREF(value); - value = ast2obj_list(state, (asdl_seq*)o->v.FunctionDef.decorator_list, + value = ast2obj_list(state, vstate, + (asdl_seq*)o->v.FunctionDef.decorator_list, ast2obj_expr); if (!value) goto failed; if (PyObject_SetAttr(result, state->decorator_list, value) == -1) goto failed; Py_DECREF(value); - value = ast2obj_expr(state, o->v.FunctionDef.returns); + value = ast2obj_expr(state, vstate, o->v.FunctionDef.returns); if (!value) goto failed; if (PyObject_SetAttr(result, state->returns, value) == -1) goto failed; Py_DECREF(value); - value = ast2obj_string(state, o->v.FunctionDef.type_comment); + value = ast2obj_string(state, vstate, o->v.FunctionDef.type_comment); if (!value) goto failed; if (PyObject_SetAttr(result, state->type_comment, value) == -1) goto failed; Py_DECREF(value); - value = ast2obj_list(state, (asdl_seq*)o->v.FunctionDef.type_params, + value = ast2obj_list(state, vstate, + (asdl_seq*)o->v.FunctionDef.type_params, ast2obj_type_param); if (!value) goto failed; if (PyObject_SetAttr(result, state->type_params, value) == -1) @@ -3909,40 +3937,41 @@ ast2obj_stmt(struct ast_state *state, void* _o) tp = (PyTypeObject *)state->AsyncFunctionDef_type; result = PyType_GenericNew(tp, NULL, NULL); if (!result) goto failed; - value = ast2obj_identifier(state, o->v.AsyncFunctionDef.name); + value = ast2obj_identifier(state, vstate, o->v.AsyncFunctionDef.name); if (!value) goto failed; if (PyObject_SetAttr(result, state->name, value) == -1) goto failed; Py_DECREF(value); - value = ast2obj_arguments(state, o->v.AsyncFunctionDef.args); + value = ast2obj_arguments(state, vstate, o->v.AsyncFunctionDef.args); if (!value) goto failed; if (PyObject_SetAttr(result, state->args, value) == -1) goto failed; Py_DECREF(value); - value = ast2obj_list(state, (asdl_seq*)o->v.AsyncFunctionDef.body, + value = ast2obj_list(state, vstate, + (asdl_seq*)o->v.AsyncFunctionDef.body, ast2obj_stmt); if (!value) goto failed; if (PyObject_SetAttr(result, state->body, value) == -1) goto failed; Py_DECREF(value); - value = ast2obj_list(state, + value = ast2obj_list(state, vstate, (asdl_seq*)o->v.AsyncFunctionDef.decorator_list, ast2obj_expr); if (!value) goto failed; if (PyObject_SetAttr(result, state->decorator_list, value) == -1) goto failed; Py_DECREF(value); - value = ast2obj_expr(state, o->v.AsyncFunctionDef.returns); + value = ast2obj_expr(state, vstate, o->v.AsyncFunctionDef.returns); if (!value) goto failed; if (PyObject_SetAttr(result, state->returns, value) == -1) goto failed; Py_DECREF(value); - value = ast2obj_string(state, o->v.AsyncFunctionDef.type_comment); + value = ast2obj_string(state, vstate, o->v.AsyncFunctionDef.type_comment); if (!value) goto failed; if (PyObject_SetAttr(result, state->type_comment, value) == -1) goto failed; Py_DECREF(value); - value = ast2obj_list(state, + value = ast2obj_list(state, vstate, (asdl_seq*)o->v.AsyncFunctionDef.type_params, ast2obj_type_param); if (!value) goto failed; @@ -3954,36 +3983,38 @@ ast2obj_stmt(struct ast_state *state, void* _o) tp = (PyTypeObject *)state->ClassDef_type; result = PyType_GenericNew(tp, NULL, NULL); if (!result) goto failed; - value = ast2obj_identifier(state, o->v.ClassDef.name); + value = ast2obj_identifier(state, vstate, o->v.ClassDef.name); if (!value) goto failed; if (PyObject_SetAttr(result, state->name, value) == -1) goto failed; Py_DECREF(value); - value = ast2obj_list(state, (asdl_seq*)o->v.ClassDef.bases, + value = ast2obj_list(state, vstate, (asdl_seq*)o->v.ClassDef.bases, ast2obj_expr); if (!value) goto failed; if (PyObject_SetAttr(result, state->bases, value) == -1) goto failed; Py_DECREF(value); - value = ast2obj_list(state, (asdl_seq*)o->v.ClassDef.keywords, + value = ast2obj_list(state, vstate, (asdl_seq*)o->v.ClassDef.keywords, ast2obj_keyword); if (!value) goto failed; if (PyObject_SetAttr(result, state->keywords, value) == -1) goto failed; Py_DECREF(value); - value = ast2obj_list(state, (asdl_seq*)o->v.ClassDef.body, + value = ast2obj_list(state, vstate, (asdl_seq*)o->v.ClassDef.body, ast2obj_stmt); if (!value) goto failed; if (PyObject_SetAttr(result, state->body, value) == -1) goto failed; Py_DECREF(value); - value = ast2obj_list(state, (asdl_seq*)o->v.ClassDef.decorator_list, + value = ast2obj_list(state, vstate, + (asdl_seq*)o->v.ClassDef.decorator_list, ast2obj_expr); if (!value) goto failed; if (PyObject_SetAttr(result, state->decorator_list, value) == -1) goto failed; Py_DECREF(value); - value = ast2obj_list(state, (asdl_seq*)o->v.ClassDef.type_params, + value = ast2obj_list(state, vstate, + (asdl_seq*)o->v.ClassDef.type_params, ast2obj_type_param); if (!value) goto failed; if (PyObject_SetAttr(result, state->type_params, value) == -1) @@ -3994,7 +4025,7 @@ ast2obj_stmt(struct ast_state *state, void* _o) tp = (PyTypeObject *)state->Return_type; result = PyType_GenericNew(tp, NULL, NULL); if (!result) goto failed; - value = ast2obj_expr(state, o->v.Return.value); + value = ast2obj_expr(state, vstate, o->v.Return.value); if (!value) goto failed; if (PyObject_SetAttr(result, state->value, value) == -1) goto failed; @@ -4004,7 +4035,7 @@ ast2obj_stmt(struct ast_state *state, void* _o) tp = (PyTypeObject *)state->Delete_type; result = PyType_GenericNew(tp, NULL, NULL); if (!result) goto failed; - value = ast2obj_list(state, (asdl_seq*)o->v.Delete.targets, + value = ast2obj_list(state, vstate, (asdl_seq*)o->v.Delete.targets, ast2obj_expr); if (!value) goto failed; if (PyObject_SetAttr(result, state->targets, value) == -1) @@ -4015,18 +4046,18 @@ ast2obj_stmt(struct ast_state *state, void* _o) tp = (PyTypeObject *)state->Assign_type; result = PyType_GenericNew(tp, NULL, NULL); if (!result) goto failed; - value = ast2obj_list(state, (asdl_seq*)o->v.Assign.targets, + value = ast2obj_list(state, vstate, (asdl_seq*)o->v.Assign.targets, ast2obj_expr); if (!value) goto failed; if (PyObject_SetAttr(result, state->targets, value) == -1) goto failed; Py_DECREF(value); - value = ast2obj_expr(state, o->v.Assign.value); + value = ast2obj_expr(state, vstate, o->v.Assign.value); if (!value) goto failed; if (PyObject_SetAttr(result, state->value, value) == -1) goto failed; Py_DECREF(value); - value = ast2obj_string(state, o->v.Assign.type_comment); + value = ast2obj_string(state, vstate, o->v.Assign.type_comment); if (!value) goto failed; if (PyObject_SetAttr(result, state->type_comment, value) == -1) goto failed; @@ -4036,18 +4067,19 @@ ast2obj_stmt(struct ast_state *state, void* _o) tp = (PyTypeObject *)state->TypeAlias_type; result = PyType_GenericNew(tp, NULL, NULL); if (!result) goto failed; - value = ast2obj_expr(state, o->v.TypeAlias.name); + value = ast2obj_expr(state, vstate, o->v.TypeAlias.name); if (!value) goto failed; if (PyObject_SetAttr(result, state->name, value) == -1) goto failed; Py_DECREF(value); - value = ast2obj_list(state, (asdl_seq*)o->v.TypeAlias.type_params, + value = ast2obj_list(state, vstate, + (asdl_seq*)o->v.TypeAlias.type_params, ast2obj_type_param); if (!value) goto failed; if (PyObject_SetAttr(result, state->type_params, value) == -1) goto failed; Py_DECREF(value); - value = ast2obj_expr(state, o->v.TypeAlias.value); + value = ast2obj_expr(state, vstate, o->v.TypeAlias.value); if (!value) goto failed; if (PyObject_SetAttr(result, state->value, value) == -1) goto failed; @@ -4057,17 +4089,17 @@ ast2obj_stmt(struct ast_state *state, void* _o) tp = (PyTypeObject *)state->AugAssign_type; result = PyType_GenericNew(tp, NULL, NULL); if (!result) goto failed; - value = ast2obj_expr(state, o->v.AugAssign.target); + value = ast2obj_expr(state, vstate, o->v.AugAssign.target); if (!value) goto failed; if (PyObject_SetAttr(result, state->target, value) == -1) goto failed; Py_DECREF(value); - value = ast2obj_operator(state, o->v.AugAssign.op); + value = ast2obj_operator(state, vstate, o->v.AugAssign.op); if (!value) goto failed; if (PyObject_SetAttr(result, state->op, value) == -1) goto failed; Py_DECREF(value); - value = ast2obj_expr(state, o->v.AugAssign.value); + value = ast2obj_expr(state, vstate, o->v.AugAssign.value); if (!value) goto failed; if (PyObject_SetAttr(result, state->value, value) == -1) goto failed; @@ -4077,22 +4109,22 @@ ast2obj_stmt(struct ast_state *state, void* _o) tp = (PyTypeObject *)state->AnnAssign_type; result = PyType_GenericNew(tp, NULL, NULL); if (!result) goto failed; - value = ast2obj_expr(state, o->v.AnnAssign.target); + value = ast2obj_expr(state, vstate, o->v.AnnAssign.target); if (!value) goto failed; if (PyObject_SetAttr(result, state->target, value) == -1) goto failed; Py_DECREF(value); - value = ast2obj_expr(state, o->v.AnnAssign.annotation); + value = ast2obj_expr(state, vstate, o->v.AnnAssign.annotation); if (!value) goto failed; if (PyObject_SetAttr(result, state->annotation, value) == -1) goto failed; Py_DECREF(value); - value = ast2obj_expr(state, o->v.AnnAssign.value); + value = ast2obj_expr(state, vstate, o->v.AnnAssign.value); if (!value) goto failed; if (PyObject_SetAttr(result, state->value, value) == -1) goto failed; Py_DECREF(value); - value = ast2obj_int(state, o->v.AnnAssign.simple); + value = ast2obj_int(state, vstate, o->v.AnnAssign.simple); if (!value) goto failed; if (PyObject_SetAttr(result, state->simple, value) == -1) goto failed; @@ -4102,27 +4134,29 @@ ast2obj_stmt(struct ast_state *state, void* _o) tp = (PyTypeObject *)state->For_type; result = PyType_GenericNew(tp, NULL, NULL); if (!result) goto failed; - value = ast2obj_expr(state, o->v.For.target); + value = ast2obj_expr(state, vstate, o->v.For.target); if (!value) goto failed; if (PyObject_SetAttr(result, state->target, value) == -1) goto failed; Py_DECREF(value); - value = ast2obj_expr(state, o->v.For.iter); + value = ast2obj_expr(state, vstate, o->v.For.iter); if (!value) goto failed; if (PyObject_SetAttr(result, state->iter, value) == -1) goto failed; Py_DECREF(value); - value = ast2obj_list(state, (asdl_seq*)o->v.For.body, ast2obj_stmt); + value = ast2obj_list(state, vstate, (asdl_seq*)o->v.For.body, + ast2obj_stmt); if (!value) goto failed; if (PyObject_SetAttr(result, state->body, value) == -1) goto failed; Py_DECREF(value); - value = ast2obj_list(state, (asdl_seq*)o->v.For.orelse, ast2obj_stmt); + value = ast2obj_list(state, vstate, (asdl_seq*)o->v.For.orelse, + ast2obj_stmt); if (!value) goto failed; if (PyObject_SetAttr(result, state->orelse, value) == -1) goto failed; Py_DECREF(value); - value = ast2obj_string(state, o->v.For.type_comment); + value = ast2obj_string(state, vstate, o->v.For.type_comment); if (!value) goto failed; if (PyObject_SetAttr(result, state->type_comment, value) == -1) goto failed; @@ -4132,29 +4166,29 @@ ast2obj_stmt(struct ast_state *state, void* _o) tp = (PyTypeObject *)state->AsyncFor_type; result = PyType_GenericNew(tp, NULL, NULL); if (!result) goto failed; - value = ast2obj_expr(state, o->v.AsyncFor.target); + value = ast2obj_expr(state, vstate, o->v.AsyncFor.target); if (!value) goto failed; if (PyObject_SetAttr(result, state->target, value) == -1) goto failed; Py_DECREF(value); - value = ast2obj_expr(state, o->v.AsyncFor.iter); + value = ast2obj_expr(state, vstate, o->v.AsyncFor.iter); if (!value) goto failed; if (PyObject_SetAttr(result, state->iter, value) == -1) goto failed; Py_DECREF(value); - value = ast2obj_list(state, (asdl_seq*)o->v.AsyncFor.body, + value = ast2obj_list(state, vstate, (asdl_seq*)o->v.AsyncFor.body, ast2obj_stmt); if (!value) goto failed; if (PyObject_SetAttr(result, state->body, value) == -1) goto failed; Py_DECREF(value); - value = ast2obj_list(state, (asdl_seq*)o->v.AsyncFor.orelse, + value = ast2obj_list(state, vstate, (asdl_seq*)o->v.AsyncFor.orelse, ast2obj_stmt); if (!value) goto failed; if (PyObject_SetAttr(result, state->orelse, value) == -1) goto failed; Py_DECREF(value); - value = ast2obj_string(state, o->v.AsyncFor.type_comment); + value = ast2obj_string(state, vstate, o->v.AsyncFor.type_comment); if (!value) goto failed; if (PyObject_SetAttr(result, state->type_comment, value) == -1) goto failed; @@ -4164,17 +4198,19 @@ ast2obj_stmt(struct ast_state *state, void* _o) tp = (PyTypeObject *)state->While_type; result = PyType_GenericNew(tp, NULL, NULL); if (!result) goto failed; - value = ast2obj_expr(state, o->v.While.test); + value = ast2obj_expr(state, vstate, o->v.While.test); if (!value) goto failed; if (PyObject_SetAttr(result, state->test, value) == -1) goto failed; Py_DECREF(value); - value = ast2obj_list(state, (asdl_seq*)o->v.While.body, ast2obj_stmt); + value = ast2obj_list(state, vstate, (asdl_seq*)o->v.While.body, + ast2obj_stmt); if (!value) goto failed; if (PyObject_SetAttr(result, state->body, value) == -1) goto failed; Py_DECREF(value); - value = ast2obj_list(state, (asdl_seq*)o->v.While.orelse, ast2obj_stmt); + value = ast2obj_list(state, vstate, (asdl_seq*)o->v.While.orelse, + ast2obj_stmt); if (!value) goto failed; if (PyObject_SetAttr(result, state->orelse, value) == -1) goto failed; @@ -4184,17 +4220,19 @@ ast2obj_stmt(struct ast_state *state, void* _o) tp = (PyTypeObject *)state->If_type; result = PyType_GenericNew(tp, NULL, NULL); if (!result) goto failed; - value = ast2obj_expr(state, o->v.If.test); + value = ast2obj_expr(state, vstate, o->v.If.test); if (!value) goto failed; if (PyObject_SetAttr(result, state->test, value) == -1) goto failed; Py_DECREF(value); - value = ast2obj_list(state, (asdl_seq*)o->v.If.body, ast2obj_stmt); + value = ast2obj_list(state, vstate, (asdl_seq*)o->v.If.body, + ast2obj_stmt); if (!value) goto failed; if (PyObject_SetAttr(result, state->body, value) == -1) goto failed; Py_DECREF(value); - value = ast2obj_list(state, (asdl_seq*)o->v.If.orelse, ast2obj_stmt); + value = ast2obj_list(state, vstate, (asdl_seq*)o->v.If.orelse, + ast2obj_stmt); if (!value) goto failed; if (PyObject_SetAttr(result, state->orelse, value) == -1) goto failed; @@ -4204,18 +4242,19 @@ ast2obj_stmt(struct ast_state *state, void* _o) tp = (PyTypeObject *)state->With_type; result = PyType_GenericNew(tp, NULL, NULL); if (!result) goto failed; - value = ast2obj_list(state, (asdl_seq*)o->v.With.items, + value = ast2obj_list(state, vstate, (asdl_seq*)o->v.With.items, ast2obj_withitem); if (!value) goto failed; if (PyObject_SetAttr(result, state->items, value) == -1) goto failed; Py_DECREF(value); - value = ast2obj_list(state, (asdl_seq*)o->v.With.body, ast2obj_stmt); + value = ast2obj_list(state, vstate, (asdl_seq*)o->v.With.body, + ast2obj_stmt); if (!value) goto failed; if (PyObject_SetAttr(result, state->body, value) == -1) goto failed; Py_DECREF(value); - value = ast2obj_string(state, o->v.With.type_comment); + value = ast2obj_string(state, vstate, o->v.With.type_comment); if (!value) goto failed; if (PyObject_SetAttr(result, state->type_comment, value) == -1) goto failed; @@ -4225,19 +4264,19 @@ ast2obj_stmt(struct ast_state *state, void* _o) tp = (PyTypeObject *)state->AsyncWith_type; result = PyType_GenericNew(tp, NULL, NULL); if (!result) goto failed; - value = ast2obj_list(state, (asdl_seq*)o->v.AsyncWith.items, + value = ast2obj_list(state, vstate, (asdl_seq*)o->v.AsyncWith.items, ast2obj_withitem); if (!value) goto failed; if (PyObject_SetAttr(result, state->items, value) == -1) goto failed; Py_DECREF(value); - value = ast2obj_list(state, (asdl_seq*)o->v.AsyncWith.body, + value = ast2obj_list(state, vstate, (asdl_seq*)o->v.AsyncWith.body, ast2obj_stmt); if (!value) goto failed; if (PyObject_SetAttr(result, state->body, value) == -1) goto failed; Py_DECREF(value); - value = ast2obj_string(state, o->v.AsyncWith.type_comment); + value = ast2obj_string(state, vstate, o->v.AsyncWith.type_comment); if (!value) goto failed; if (PyObject_SetAttr(result, state->type_comment, value) == -1) goto failed; @@ -4247,12 +4286,12 @@ ast2obj_stmt(struct ast_state *state, void* _o) tp = (PyTypeObject *)state->Match_type; result = PyType_GenericNew(tp, NULL, NULL); if (!result) goto failed; - value = ast2obj_expr(state, o->v.Match.subject); + value = ast2obj_expr(state, vstate, o->v.Match.subject); if (!value) goto failed; if (PyObject_SetAttr(result, state->subject, value) == -1) goto failed; Py_DECREF(value); - value = ast2obj_list(state, (asdl_seq*)o->v.Match.cases, + value = ast2obj_list(state, vstate, (asdl_seq*)o->v.Match.cases, ast2obj_match_case); if (!value) goto failed; if (PyObject_SetAttr(result, state->cases, value) == -1) @@ -4263,12 +4302,12 @@ ast2obj_stmt(struct ast_state *state, void* _o) tp = (PyTypeObject *)state->Raise_type; result = PyType_GenericNew(tp, NULL, NULL); if (!result) goto failed; - value = ast2obj_expr(state, o->v.Raise.exc); + value = ast2obj_expr(state, vstate, o->v.Raise.exc); if (!value) goto failed; if (PyObject_SetAttr(result, state->exc, value) == -1) goto failed; Py_DECREF(value); - value = ast2obj_expr(state, o->v.Raise.cause); + value = ast2obj_expr(state, vstate, o->v.Raise.cause); if (!value) goto failed; if (PyObject_SetAttr(result, state->cause, value) == -1) goto failed; @@ -4278,23 +4317,25 @@ ast2obj_stmt(struct ast_state *state, void* _o) tp = (PyTypeObject *)state->Try_type; result = PyType_GenericNew(tp, NULL, NULL); if (!result) goto failed; - value = ast2obj_list(state, (asdl_seq*)o->v.Try.body, ast2obj_stmt); + value = ast2obj_list(state, vstate, (asdl_seq*)o->v.Try.body, + ast2obj_stmt); if (!value) goto failed; if (PyObject_SetAttr(result, state->body, value) == -1) goto failed; Py_DECREF(value); - value = ast2obj_list(state, (asdl_seq*)o->v.Try.handlers, + value = ast2obj_list(state, vstate, (asdl_seq*)o->v.Try.handlers, ast2obj_excepthandler); if (!value) goto failed; if (PyObject_SetAttr(result, state->handlers, value) == -1) goto failed; Py_DECREF(value); - value = ast2obj_list(state, (asdl_seq*)o->v.Try.orelse, ast2obj_stmt); + value = ast2obj_list(state, vstate, (asdl_seq*)o->v.Try.orelse, + ast2obj_stmt); if (!value) goto failed; if (PyObject_SetAttr(result, state->orelse, value) == -1) goto failed; Py_DECREF(value); - value = ast2obj_list(state, (asdl_seq*)o->v.Try.finalbody, + value = ast2obj_list(state, vstate, (asdl_seq*)o->v.Try.finalbody, ast2obj_stmt); if (!value) goto failed; if (PyObject_SetAttr(result, state->finalbody, value) == -1) @@ -4305,24 +4346,25 @@ ast2obj_stmt(struct ast_state *state, void* _o) tp = (PyTypeObject *)state->TryStar_type; result = PyType_GenericNew(tp, NULL, NULL); if (!result) goto failed; - value = ast2obj_list(state, (asdl_seq*)o->v.TryStar.body, ast2obj_stmt); + value = ast2obj_list(state, vstate, (asdl_seq*)o->v.TryStar.body, + ast2obj_stmt); if (!value) goto failed; if (PyObject_SetAttr(result, state->body, value) == -1) goto failed; Py_DECREF(value); - value = ast2obj_list(state, (asdl_seq*)o->v.TryStar.handlers, + value = ast2obj_list(state, vstate, (asdl_seq*)o->v.TryStar.handlers, ast2obj_excepthandler); if (!value) goto failed; if (PyObject_SetAttr(result, state->handlers, value) == -1) goto failed; Py_DECREF(value); - value = ast2obj_list(state, (asdl_seq*)o->v.TryStar.orelse, + value = ast2obj_list(state, vstate, (asdl_seq*)o->v.TryStar.orelse, ast2obj_stmt); if (!value) goto failed; if (PyObject_SetAttr(result, state->orelse, value) == -1) goto failed; Py_DECREF(value); - value = ast2obj_list(state, (asdl_seq*)o->v.TryStar.finalbody, + value = ast2obj_list(state, vstate, (asdl_seq*)o->v.TryStar.finalbody, ast2obj_stmt); if (!value) goto failed; if (PyObject_SetAttr(result, state->finalbody, value) == -1) @@ -4333,12 +4375,12 @@ ast2obj_stmt(struct ast_state *state, void* _o) tp = (PyTypeObject *)state->Assert_type; result = PyType_GenericNew(tp, NULL, NULL); if (!result) goto failed; - value = ast2obj_expr(state, o->v.Assert.test); + value = ast2obj_expr(state, vstate, o->v.Assert.test); if (!value) goto failed; if (PyObject_SetAttr(result, state->test, value) == -1) goto failed; Py_DECREF(value); - value = ast2obj_expr(state, o->v.Assert.msg); + value = ast2obj_expr(state, vstate, o->v.Assert.msg); if (!value) goto failed; if (PyObject_SetAttr(result, state->msg, value) == -1) goto failed; @@ -4348,7 +4390,7 @@ ast2obj_stmt(struct ast_state *state, void* _o) tp = (PyTypeObject *)state->Import_type; result = PyType_GenericNew(tp, NULL, NULL); if (!result) goto failed; - value = ast2obj_list(state, (asdl_seq*)o->v.Import.names, + value = ast2obj_list(state, vstate, (asdl_seq*)o->v.Import.names, ast2obj_alias); if (!value) goto failed; if (PyObject_SetAttr(result, state->names, value) == -1) @@ -4359,18 +4401,18 @@ ast2obj_stmt(struct ast_state *state, void* _o) tp = (PyTypeObject *)state->ImportFrom_type; result = PyType_GenericNew(tp, NULL, NULL); if (!result) goto failed; - value = ast2obj_identifier(state, o->v.ImportFrom.module); + value = ast2obj_identifier(state, vstate, o->v.ImportFrom.module); if (!value) goto failed; if (PyObject_SetAttr(result, state->module, value) == -1) goto failed; Py_DECREF(value); - value = ast2obj_list(state, (asdl_seq*)o->v.ImportFrom.names, + value = ast2obj_list(state, vstate, (asdl_seq*)o->v.ImportFrom.names, ast2obj_alias); if (!value) goto failed; if (PyObject_SetAttr(result, state->names, value) == -1) goto failed; Py_DECREF(value); - value = ast2obj_int(state, o->v.ImportFrom.level); + value = ast2obj_int(state, vstate, o->v.ImportFrom.level); if (!value) goto failed; if (PyObject_SetAttr(result, state->level, value) == -1) goto failed; @@ -4380,7 +4422,7 @@ ast2obj_stmt(struct ast_state *state, void* _o) tp = (PyTypeObject *)state->Global_type; result = PyType_GenericNew(tp, NULL, NULL); if (!result) goto failed; - value = ast2obj_list(state, (asdl_seq*)o->v.Global.names, + value = ast2obj_list(state, vstate, (asdl_seq*)o->v.Global.names, ast2obj_identifier); if (!value) goto failed; if (PyObject_SetAttr(result, state->names, value) == -1) @@ -4391,7 +4433,7 @@ ast2obj_stmt(struct ast_state *state, void* _o) tp = (PyTypeObject *)state->Nonlocal_type; result = PyType_GenericNew(tp, NULL, NULL); if (!result) goto failed; - value = ast2obj_list(state, (asdl_seq*)o->v.Nonlocal.names, + value = ast2obj_list(state, vstate, (asdl_seq*)o->v.Nonlocal.names, ast2obj_identifier); if (!value) goto failed; if (PyObject_SetAttr(result, state->names, value) == -1) @@ -4402,7 +4444,7 @@ ast2obj_stmt(struct ast_state *state, void* _o) tp = (PyTypeObject *)state->Expr_type; result = PyType_GenericNew(tp, NULL, NULL); if (!result) goto failed; - value = ast2obj_expr(state, o->v.Expr.value); + value = ast2obj_expr(state, vstate, o->v.Expr.value); if (!value) goto failed; if (PyObject_SetAttr(result, state->value, value) == -1) goto failed; @@ -4424,37 +4466,37 @@ ast2obj_stmt(struct ast_state *state, void* _o) if (!result) goto failed; break; } - value = ast2obj_int(state, o->lineno); + value = ast2obj_int(state, vstate, o->lineno); if (!value) goto failed; if (PyObject_SetAttr(result, state->lineno, value) < 0) goto failed; Py_DECREF(value); - value = ast2obj_int(state, o->col_offset); + value = ast2obj_int(state, vstate, o->col_offset); if (!value) goto failed; if (PyObject_SetAttr(result, state->col_offset, value) < 0) goto failed; Py_DECREF(value); - value = ast2obj_int(state, o->end_lineno); + value = ast2obj_int(state, vstate, o->end_lineno); if (!value) goto failed; if (PyObject_SetAttr(result, state->end_lineno, value) < 0) goto failed; Py_DECREF(value); - value = ast2obj_int(state, o->end_col_offset); + value = ast2obj_int(state, vstate, o->end_col_offset); if (!value) goto failed; if (PyObject_SetAttr(result, state->end_col_offset, value) < 0) goto failed; Py_DECREF(value); - state->recursion_depth--; + vstate->recursion_depth--; return result; failed: - state->recursion_depth--; + vstate->recursion_depth--; Py_XDECREF(value); Py_XDECREF(result); return NULL; } PyObject* -ast2obj_expr(struct ast_state *state, void* _o) +ast2obj_expr(struct ast_state *state, struct validator *vstate, void* _o) { expr_ty o = (expr_ty)_o; PyObject *result = NULL, *value = NULL; @@ -4462,7 +4504,7 @@ ast2obj_expr(struct ast_state *state, void* _o) if (!o) { Py_RETURN_NONE; } - if (++state->recursion_depth > state->recursion_limit) { + if (++vstate->recursion_depth > vstate->recursion_limit) { PyErr_SetString(PyExc_RecursionError, "maximum recursion depth exceeded during ast construction"); return NULL; @@ -4472,12 +4514,12 @@ ast2obj_expr(struct ast_state *state, void* _o) tp = (PyTypeObject *)state->BoolOp_type; result = PyType_GenericNew(tp, NULL, NULL); if (!result) goto failed; - value = ast2obj_boolop(state, o->v.BoolOp.op); + value = ast2obj_boolop(state, vstate, o->v.BoolOp.op); if (!value) goto failed; if (PyObject_SetAttr(result, state->op, value) == -1) goto failed; Py_DECREF(value); - value = ast2obj_list(state, (asdl_seq*)o->v.BoolOp.values, + value = ast2obj_list(state, vstate, (asdl_seq*)o->v.BoolOp.values, ast2obj_expr); if (!value) goto failed; if (PyObject_SetAttr(result, state->values, value) == -1) @@ -4488,12 +4530,12 @@ ast2obj_expr(struct ast_state *state, void* _o) tp = (PyTypeObject *)state->NamedExpr_type; result = PyType_GenericNew(tp, NULL, NULL); if (!result) goto failed; - value = ast2obj_expr(state, o->v.NamedExpr.target); + value = ast2obj_expr(state, vstate, o->v.NamedExpr.target); if (!value) goto failed; if (PyObject_SetAttr(result, state->target, value) == -1) goto failed; Py_DECREF(value); - value = ast2obj_expr(state, o->v.NamedExpr.value); + value = ast2obj_expr(state, vstate, o->v.NamedExpr.value); if (!value) goto failed; if (PyObject_SetAttr(result, state->value, value) == -1) goto failed; @@ -4503,17 +4545,17 @@ ast2obj_expr(struct ast_state *state, void* _o) tp = (PyTypeObject *)state->BinOp_type; result = PyType_GenericNew(tp, NULL, NULL); if (!result) goto failed; - value = ast2obj_expr(state, o->v.BinOp.left); + value = ast2obj_expr(state, vstate, o->v.BinOp.left); if (!value) goto failed; if (PyObject_SetAttr(result, state->left, value) == -1) goto failed; Py_DECREF(value); - value = ast2obj_operator(state, o->v.BinOp.op); + value = ast2obj_operator(state, vstate, o->v.BinOp.op); if (!value) goto failed; if (PyObject_SetAttr(result, state->op, value) == -1) goto failed; Py_DECREF(value); - value = ast2obj_expr(state, o->v.BinOp.right); + value = ast2obj_expr(state, vstate, o->v.BinOp.right); if (!value) goto failed; if (PyObject_SetAttr(result, state->right, value) == -1) goto failed; @@ -4523,12 +4565,12 @@ ast2obj_expr(struct ast_state *state, void* _o) tp = (PyTypeObject *)state->UnaryOp_type; result = PyType_GenericNew(tp, NULL, NULL); if (!result) goto failed; - value = ast2obj_unaryop(state, o->v.UnaryOp.op); + value = ast2obj_unaryop(state, vstate, o->v.UnaryOp.op); if (!value) goto failed; if (PyObject_SetAttr(result, state->op, value) == -1) goto failed; Py_DECREF(value); - value = ast2obj_expr(state, o->v.UnaryOp.operand); + value = ast2obj_expr(state, vstate, o->v.UnaryOp.operand); if (!value) goto failed; if (PyObject_SetAttr(result, state->operand, value) == -1) goto failed; @@ -4538,12 +4580,12 @@ ast2obj_expr(struct ast_state *state, void* _o) tp = (PyTypeObject *)state->Lambda_type; result = PyType_GenericNew(tp, NULL, NULL); if (!result) goto failed; - value = ast2obj_arguments(state, o->v.Lambda.args); + value = ast2obj_arguments(state, vstate, o->v.Lambda.args); if (!value) goto failed; if (PyObject_SetAttr(result, state->args, value) == -1) goto failed; Py_DECREF(value); - value = ast2obj_expr(state, o->v.Lambda.body); + value = ast2obj_expr(state, vstate, o->v.Lambda.body); if (!value) goto failed; if (PyObject_SetAttr(result, state->body, value) == -1) goto failed; @@ -4553,17 +4595,17 @@ ast2obj_expr(struct ast_state *state, void* _o) tp = (PyTypeObject *)state->IfExp_type; result = PyType_GenericNew(tp, NULL, NULL); if (!result) goto failed; - value = ast2obj_expr(state, o->v.IfExp.test); + value = ast2obj_expr(state, vstate, o->v.IfExp.test); if (!value) goto failed; if (PyObject_SetAttr(result, state->test, value) == -1) goto failed; Py_DECREF(value); - value = ast2obj_expr(state, o->v.IfExp.body); + value = ast2obj_expr(state, vstate, o->v.IfExp.body); if (!value) goto failed; if (PyObject_SetAttr(result, state->body, value) == -1) goto failed; Py_DECREF(value); - value = ast2obj_expr(state, o->v.IfExp.orelse); + value = ast2obj_expr(state, vstate, o->v.IfExp.orelse); if (!value) goto failed; if (PyObject_SetAttr(result, state->orelse, value) == -1) goto failed; @@ -4573,12 +4615,14 @@ ast2obj_expr(struct ast_state *state, void* _o) tp = (PyTypeObject *)state->Dict_type; result = PyType_GenericNew(tp, NULL, NULL); if (!result) goto failed; - value = ast2obj_list(state, (asdl_seq*)o->v.Dict.keys, ast2obj_expr); + value = ast2obj_list(state, vstate, (asdl_seq*)o->v.Dict.keys, + ast2obj_expr); if (!value) goto failed; if (PyObject_SetAttr(result, state->keys, value) == -1) goto failed; Py_DECREF(value); - value = ast2obj_list(state, (asdl_seq*)o->v.Dict.values, ast2obj_expr); + value = ast2obj_list(state, vstate, (asdl_seq*)o->v.Dict.values, + ast2obj_expr); if (!value) goto failed; if (PyObject_SetAttr(result, state->values, value) == -1) goto failed; @@ -4588,7 +4632,8 @@ ast2obj_expr(struct ast_state *state, void* _o) tp = (PyTypeObject *)state->Set_type; result = PyType_GenericNew(tp, NULL, NULL); if (!result) goto failed; - value = ast2obj_list(state, (asdl_seq*)o->v.Set.elts, ast2obj_expr); + value = ast2obj_list(state, vstate, (asdl_seq*)o->v.Set.elts, + ast2obj_expr); if (!value) goto failed; if (PyObject_SetAttr(result, state->elts, value) == -1) goto failed; @@ -4598,12 +4643,13 @@ ast2obj_expr(struct ast_state *state, void* _o) tp = (PyTypeObject *)state->ListComp_type; result = PyType_GenericNew(tp, NULL, NULL); if (!result) goto failed; - value = ast2obj_expr(state, o->v.ListComp.elt); + value = ast2obj_expr(state, vstate, o->v.ListComp.elt); if (!value) goto failed; if (PyObject_SetAttr(result, state->elt, value) == -1) goto failed; Py_DECREF(value); - value = ast2obj_list(state, (asdl_seq*)o->v.ListComp.generators, + value = ast2obj_list(state, vstate, + (asdl_seq*)o->v.ListComp.generators, ast2obj_comprehension); if (!value) goto failed; if (PyObject_SetAttr(result, state->generators, value) == -1) @@ -4614,12 +4660,12 @@ ast2obj_expr(struct ast_state *state, void* _o) tp = (PyTypeObject *)state->SetComp_type; result = PyType_GenericNew(tp, NULL, NULL); if (!result) goto failed; - value = ast2obj_expr(state, o->v.SetComp.elt); + value = ast2obj_expr(state, vstate, o->v.SetComp.elt); if (!value) goto failed; if (PyObject_SetAttr(result, state->elt, value) == -1) goto failed; Py_DECREF(value); - value = ast2obj_list(state, (asdl_seq*)o->v.SetComp.generators, + value = ast2obj_list(state, vstate, (asdl_seq*)o->v.SetComp.generators, ast2obj_comprehension); if (!value) goto failed; if (PyObject_SetAttr(result, state->generators, value) == -1) @@ -4630,17 +4676,18 @@ ast2obj_expr(struct ast_state *state, void* _o) tp = (PyTypeObject *)state->DictComp_type; result = PyType_GenericNew(tp, NULL, NULL); if (!result) goto failed; - value = ast2obj_expr(state, o->v.DictComp.key); + value = ast2obj_expr(state, vstate, o->v.DictComp.key); if (!value) goto failed; if (PyObject_SetAttr(result, state->key, value) == -1) goto failed; Py_DECREF(value); - value = ast2obj_expr(state, o->v.DictComp.value); + value = ast2obj_expr(state, vstate, o->v.DictComp.value); if (!value) goto failed; if (PyObject_SetAttr(result, state->value, value) == -1) goto failed; Py_DECREF(value); - value = ast2obj_list(state, (asdl_seq*)o->v.DictComp.generators, + value = ast2obj_list(state, vstate, + (asdl_seq*)o->v.DictComp.generators, ast2obj_comprehension); if (!value) goto failed; if (PyObject_SetAttr(result, state->generators, value) == -1) @@ -4651,12 +4698,13 @@ ast2obj_expr(struct ast_state *state, void* _o) tp = (PyTypeObject *)state->GeneratorExp_type; result = PyType_GenericNew(tp, NULL, NULL); if (!result) goto failed; - value = ast2obj_expr(state, o->v.GeneratorExp.elt); + value = ast2obj_expr(state, vstate, o->v.GeneratorExp.elt); if (!value) goto failed; if (PyObject_SetAttr(result, state->elt, value) == -1) goto failed; Py_DECREF(value); - value = ast2obj_list(state, (asdl_seq*)o->v.GeneratorExp.generators, + value = ast2obj_list(state, vstate, + (asdl_seq*)o->v.GeneratorExp.generators, ast2obj_comprehension); if (!value) goto failed; if (PyObject_SetAttr(result, state->generators, value) == -1) @@ -4667,7 +4715,7 @@ ast2obj_expr(struct ast_state *state, void* _o) tp = (PyTypeObject *)state->Await_type; result = PyType_GenericNew(tp, NULL, NULL); if (!result) goto failed; - value = ast2obj_expr(state, o->v.Await.value); + value = ast2obj_expr(state, vstate, o->v.Await.value); if (!value) goto failed; if (PyObject_SetAttr(result, state->value, value) == -1) goto failed; @@ -4677,7 +4725,7 @@ ast2obj_expr(struct ast_state *state, void* _o) tp = (PyTypeObject *)state->Yield_type; result = PyType_GenericNew(tp, NULL, NULL); if (!result) goto failed; - value = ast2obj_expr(state, o->v.Yield.value); + value = ast2obj_expr(state, vstate, o->v.Yield.value); if (!value) goto failed; if (PyObject_SetAttr(result, state->value, value) == -1) goto failed; @@ -4687,7 +4735,7 @@ ast2obj_expr(struct ast_state *state, void* _o) tp = (PyTypeObject *)state->YieldFrom_type; result = PyType_GenericNew(tp, NULL, NULL); if (!result) goto failed; - value = ast2obj_expr(state, o->v.YieldFrom.value); + value = ast2obj_expr(state, vstate, o->v.YieldFrom.value); if (!value) goto failed; if (PyObject_SetAttr(result, state->value, value) == -1) goto failed; @@ -4697,7 +4745,7 @@ ast2obj_expr(struct ast_state *state, void* _o) tp = (PyTypeObject *)state->Compare_type; result = PyType_GenericNew(tp, NULL, NULL); if (!result) goto failed; - value = ast2obj_expr(state, o->v.Compare.left); + value = ast2obj_expr(state, vstate, o->v.Compare.left); if (!value) goto failed; if (PyObject_SetAttr(result, state->left, value) == -1) goto failed; @@ -4707,14 +4755,14 @@ ast2obj_expr(struct ast_state *state, void* _o) value = PyList_New(n); if (!value) goto failed; for(i = 0; i < n; i++) - PyList_SET_ITEM(value, i, ast2obj_cmpop(state, (cmpop_ty)asdl_seq_GET(o->v.Compare.ops, i))); + PyList_SET_ITEM(value, i, ast2obj_cmpop(state, vstate, (cmpop_ty)asdl_seq_GET(o->v.Compare.ops, i))); } if (!value) goto failed; if (PyObject_SetAttr(result, state->ops, value) == -1) goto failed; Py_DECREF(value); - value = ast2obj_list(state, (asdl_seq*)o->v.Compare.comparators, - ast2obj_expr); + value = ast2obj_list(state, vstate, + (asdl_seq*)o->v.Compare.comparators, ast2obj_expr); if (!value) goto failed; if (PyObject_SetAttr(result, state->comparators, value) == -1) goto failed; @@ -4724,17 +4772,18 @@ ast2obj_expr(struct ast_state *state, void* _o) tp = (PyTypeObject *)state->Call_type; result = PyType_GenericNew(tp, NULL, NULL); if (!result) goto failed; - value = ast2obj_expr(state, o->v.Call.func); + value = ast2obj_expr(state, vstate, o->v.Call.func); if (!value) goto failed; if (PyObject_SetAttr(result, state->func, value) == -1) goto failed; Py_DECREF(value); - value = ast2obj_list(state, (asdl_seq*)o->v.Call.args, ast2obj_expr); + value = ast2obj_list(state, vstate, (asdl_seq*)o->v.Call.args, + ast2obj_expr); if (!value) goto failed; if (PyObject_SetAttr(result, state->args, value) == -1) goto failed; Py_DECREF(value); - value = ast2obj_list(state, (asdl_seq*)o->v.Call.keywords, + value = ast2obj_list(state, vstate, (asdl_seq*)o->v.Call.keywords, ast2obj_keyword); if (!value) goto failed; if (PyObject_SetAttr(result, state->keywords, value) == -1) @@ -4745,17 +4794,17 @@ ast2obj_expr(struct ast_state *state, void* _o) tp = (PyTypeObject *)state->FormattedValue_type; result = PyType_GenericNew(tp, NULL, NULL); if (!result) goto failed; - value = ast2obj_expr(state, o->v.FormattedValue.value); + value = ast2obj_expr(state, vstate, o->v.FormattedValue.value); if (!value) goto failed; if (PyObject_SetAttr(result, state->value, value) == -1) goto failed; Py_DECREF(value); - value = ast2obj_int(state, o->v.FormattedValue.conversion); + value = ast2obj_int(state, vstate, o->v.FormattedValue.conversion); if (!value) goto failed; if (PyObject_SetAttr(result, state->conversion, value) == -1) goto failed; Py_DECREF(value); - value = ast2obj_expr(state, o->v.FormattedValue.format_spec); + value = ast2obj_expr(state, vstate, o->v.FormattedValue.format_spec); if (!value) goto failed; if (PyObject_SetAttr(result, state->format_spec, value) == -1) goto failed; @@ -4765,7 +4814,7 @@ ast2obj_expr(struct ast_state *state, void* _o) tp = (PyTypeObject *)state->JoinedStr_type; result = PyType_GenericNew(tp, NULL, NULL); if (!result) goto failed; - value = ast2obj_list(state, (asdl_seq*)o->v.JoinedStr.values, + value = ast2obj_list(state, vstate, (asdl_seq*)o->v.JoinedStr.values, ast2obj_expr); if (!value) goto failed; if (PyObject_SetAttr(result, state->values, value) == -1) @@ -4776,12 +4825,12 @@ ast2obj_expr(struct ast_state *state, void* _o) tp = (PyTypeObject *)state->Constant_type; result = PyType_GenericNew(tp, NULL, NULL); if (!result) goto failed; - value = ast2obj_constant(state, o->v.Constant.value); + value = ast2obj_constant(state, vstate, o->v.Constant.value); if (!value) goto failed; if (PyObject_SetAttr(result, state->value, value) == -1) goto failed; Py_DECREF(value); - value = ast2obj_string(state, o->v.Constant.kind); + value = ast2obj_string(state, vstate, o->v.Constant.kind); if (!value) goto failed; if (PyObject_SetAttr(result, state->kind, value) == -1) goto failed; @@ -4791,17 +4840,17 @@ ast2obj_expr(struct ast_state *state, void* _o) tp = (PyTypeObject *)state->Attribute_type; result = PyType_GenericNew(tp, NULL, NULL); if (!result) goto failed; - value = ast2obj_expr(state, o->v.Attribute.value); + value = ast2obj_expr(state, vstate, o->v.Attribute.value); if (!value) goto failed; if (PyObject_SetAttr(result, state->value, value) == -1) goto failed; Py_DECREF(value); - value = ast2obj_identifier(state, o->v.Attribute.attr); + value = ast2obj_identifier(state, vstate, o->v.Attribute.attr); if (!value) goto failed; if (PyObject_SetAttr(result, state->attr, value) == -1) goto failed; Py_DECREF(value); - value = ast2obj_expr_context(state, o->v.Attribute.ctx); + value = ast2obj_expr_context(state, vstate, o->v.Attribute.ctx); if (!value) goto failed; if (PyObject_SetAttr(result, state->ctx, value) == -1) goto failed; @@ -4811,17 +4860,17 @@ ast2obj_expr(struct ast_state *state, void* _o) tp = (PyTypeObject *)state->Subscript_type; result = PyType_GenericNew(tp, NULL, NULL); if (!result) goto failed; - value = ast2obj_expr(state, o->v.Subscript.value); + value = ast2obj_expr(state, vstate, o->v.Subscript.value); if (!value) goto failed; if (PyObject_SetAttr(result, state->value, value) == -1) goto failed; Py_DECREF(value); - value = ast2obj_expr(state, o->v.Subscript.slice); + value = ast2obj_expr(state, vstate, o->v.Subscript.slice); if (!value) goto failed; if (PyObject_SetAttr(result, state->slice, value) == -1) goto failed; Py_DECREF(value); - value = ast2obj_expr_context(state, o->v.Subscript.ctx); + value = ast2obj_expr_context(state, vstate, o->v.Subscript.ctx); if (!value) goto failed; if (PyObject_SetAttr(result, state->ctx, value) == -1) goto failed; @@ -4831,12 +4880,12 @@ ast2obj_expr(struct ast_state *state, void* _o) tp = (PyTypeObject *)state->Starred_type; result = PyType_GenericNew(tp, NULL, NULL); if (!result) goto failed; - value = ast2obj_expr(state, o->v.Starred.value); + value = ast2obj_expr(state, vstate, o->v.Starred.value); if (!value) goto failed; if (PyObject_SetAttr(result, state->value, value) == -1) goto failed; Py_DECREF(value); - value = ast2obj_expr_context(state, o->v.Starred.ctx); + value = ast2obj_expr_context(state, vstate, o->v.Starred.ctx); if (!value) goto failed; if (PyObject_SetAttr(result, state->ctx, value) == -1) goto failed; @@ -4846,12 +4895,12 @@ ast2obj_expr(struct ast_state *state, void* _o) tp = (PyTypeObject *)state->Name_type; result = PyType_GenericNew(tp, NULL, NULL); if (!result) goto failed; - value = ast2obj_identifier(state, o->v.Name.id); + value = ast2obj_identifier(state, vstate, o->v.Name.id); if (!value) goto failed; if (PyObject_SetAttr(result, state->id, value) == -1) goto failed; Py_DECREF(value); - value = ast2obj_expr_context(state, o->v.Name.ctx); + value = ast2obj_expr_context(state, vstate, o->v.Name.ctx); if (!value) goto failed; if (PyObject_SetAttr(result, state->ctx, value) == -1) goto failed; @@ -4861,12 +4910,13 @@ ast2obj_expr(struct ast_state *state, void* _o) tp = (PyTypeObject *)state->List_type; result = PyType_GenericNew(tp, NULL, NULL); if (!result) goto failed; - value = ast2obj_list(state, (asdl_seq*)o->v.List.elts, ast2obj_expr); + value = ast2obj_list(state, vstate, (asdl_seq*)o->v.List.elts, + ast2obj_expr); if (!value) goto failed; if (PyObject_SetAttr(result, state->elts, value) == -1) goto failed; Py_DECREF(value); - value = ast2obj_expr_context(state, o->v.List.ctx); + value = ast2obj_expr_context(state, vstate, o->v.List.ctx); if (!value) goto failed; if (PyObject_SetAttr(result, state->ctx, value) == -1) goto failed; @@ -4876,12 +4926,13 @@ ast2obj_expr(struct ast_state *state, void* _o) tp = (PyTypeObject *)state->Tuple_type; result = PyType_GenericNew(tp, NULL, NULL); if (!result) goto failed; - value = ast2obj_list(state, (asdl_seq*)o->v.Tuple.elts, ast2obj_expr); + value = ast2obj_list(state, vstate, (asdl_seq*)o->v.Tuple.elts, + ast2obj_expr); if (!value) goto failed; if (PyObject_SetAttr(result, state->elts, value) == -1) goto failed; Py_DECREF(value); - value = ast2obj_expr_context(state, o->v.Tuple.ctx); + value = ast2obj_expr_context(state, vstate, o->v.Tuple.ctx); if (!value) goto failed; if (PyObject_SetAttr(result, state->ctx, value) == -1) goto failed; @@ -4891,53 +4942,54 @@ ast2obj_expr(struct ast_state *state, void* _o) tp = (PyTypeObject *)state->Slice_type; result = PyType_GenericNew(tp, NULL, NULL); if (!result) goto failed; - value = ast2obj_expr(state, o->v.Slice.lower); + value = ast2obj_expr(state, vstate, o->v.Slice.lower); if (!value) goto failed; if (PyObject_SetAttr(result, state->lower, value) == -1) goto failed; Py_DECREF(value); - value = ast2obj_expr(state, o->v.Slice.upper); + value = ast2obj_expr(state, vstate, o->v.Slice.upper); if (!value) goto failed; if (PyObject_SetAttr(result, state->upper, value) == -1) goto failed; Py_DECREF(value); - value = ast2obj_expr(state, o->v.Slice.step); + value = ast2obj_expr(state, vstate, o->v.Slice.step); if (!value) goto failed; if (PyObject_SetAttr(result, state->step, value) == -1) goto failed; Py_DECREF(value); break; } - value = ast2obj_int(state, o->lineno); + value = ast2obj_int(state, vstate, o->lineno); if (!value) goto failed; if (PyObject_SetAttr(result, state->lineno, value) < 0) goto failed; Py_DECREF(value); - value = ast2obj_int(state, o->col_offset); + value = ast2obj_int(state, vstate, o->col_offset); if (!value) goto failed; if (PyObject_SetAttr(result, state->col_offset, value) < 0) goto failed; Py_DECREF(value); - value = ast2obj_int(state, o->end_lineno); + value = ast2obj_int(state, vstate, o->end_lineno); if (!value) goto failed; if (PyObject_SetAttr(result, state->end_lineno, value) < 0) goto failed; Py_DECREF(value); - value = ast2obj_int(state, o->end_col_offset); + value = ast2obj_int(state, vstate, o->end_col_offset); if (!value) goto failed; if (PyObject_SetAttr(result, state->end_col_offset, value) < 0) goto failed; Py_DECREF(value); - state->recursion_depth--; + vstate->recursion_depth--; return result; failed: - state->recursion_depth--; + vstate->recursion_depth--; Py_XDECREF(value); Py_XDECREF(result); return NULL; } -PyObject* ast2obj_expr_context(struct ast_state *state, expr_context_ty o) +PyObject* ast2obj_expr_context(struct ast_state *state, struct validator + *vstate, expr_context_ty o) { switch(o) { case Load: @@ -4949,7 +5001,8 @@ PyObject* ast2obj_expr_context(struct ast_state *state, expr_context_ty o) } Py_UNREACHABLE(); } -PyObject* ast2obj_boolop(struct ast_state *state, boolop_ty o) +PyObject* ast2obj_boolop(struct ast_state *state, struct validator *vstate, + boolop_ty o) { switch(o) { case And: @@ -4959,7 +5012,8 @@ PyObject* ast2obj_boolop(struct ast_state *state, boolop_ty o) } Py_UNREACHABLE(); } -PyObject* ast2obj_operator(struct ast_state *state, operator_ty o) +PyObject* ast2obj_operator(struct ast_state *state, struct validator *vstate, + operator_ty o) { switch(o) { case Add: @@ -4991,7 +5045,8 @@ PyObject* ast2obj_operator(struct ast_state *state, operator_ty o) } Py_UNREACHABLE(); } -PyObject* ast2obj_unaryop(struct ast_state *state, unaryop_ty o) +PyObject* ast2obj_unaryop(struct ast_state *state, struct validator *vstate, + unaryop_ty o) { switch(o) { case Invert: @@ -5005,7 +5060,8 @@ PyObject* ast2obj_unaryop(struct ast_state *state, unaryop_ty o) } Py_UNREACHABLE(); } -PyObject* ast2obj_cmpop(struct ast_state *state, cmpop_ty o) +PyObject* ast2obj_cmpop(struct ast_state *state, struct validator *vstate, + cmpop_ty o) { switch(o) { case Eq: @@ -5032,7 +5088,8 @@ PyObject* ast2obj_cmpop(struct ast_state *state, cmpop_ty o) Py_UNREACHABLE(); } PyObject* -ast2obj_comprehension(struct ast_state *state, void* _o) +ast2obj_comprehension(struct ast_state *state, struct validator *vstate, void* + _o) { comprehension_ty o = (comprehension_ty)_o; PyObject *result = NULL, *value = NULL; @@ -5040,7 +5097,7 @@ ast2obj_comprehension(struct ast_state *state, void* _o) if (!o) { Py_RETURN_NONE; } - if (++state->recursion_depth > state->recursion_limit) { + if (++vstate->recursion_depth > vstate->recursion_limit) { PyErr_SetString(PyExc_RecursionError, "maximum recursion depth exceeded during ast construction"); return NULL; @@ -5048,37 +5105,38 @@ ast2obj_comprehension(struct ast_state *state, void* _o) tp = (PyTypeObject *)state->comprehension_type; result = PyType_GenericNew(tp, NULL, NULL); if (!result) return NULL; - value = ast2obj_expr(state, o->target); + value = ast2obj_expr(state, vstate, o->target); if (!value) goto failed; if (PyObject_SetAttr(result, state->target, value) == -1) goto failed; Py_DECREF(value); - value = ast2obj_expr(state, o->iter); + value = ast2obj_expr(state, vstate, o->iter); if (!value) goto failed; if (PyObject_SetAttr(result, state->iter, value) == -1) goto failed; Py_DECREF(value); - value = ast2obj_list(state, (asdl_seq*)o->ifs, ast2obj_expr); + value = ast2obj_list(state, vstate, (asdl_seq*)o->ifs, ast2obj_expr); if (!value) goto failed; if (PyObject_SetAttr(result, state->ifs, value) == -1) goto failed; Py_DECREF(value); - value = ast2obj_int(state, o->is_async); + value = ast2obj_int(state, vstate, o->is_async); if (!value) goto failed; if (PyObject_SetAttr(result, state->is_async, value) == -1) goto failed; Py_DECREF(value); - state->recursion_depth--; + vstate->recursion_depth--; return result; failed: - state->recursion_depth--; + vstate->recursion_depth--; Py_XDECREF(value); Py_XDECREF(result); return NULL; } PyObject* -ast2obj_excepthandler(struct ast_state *state, void* _o) +ast2obj_excepthandler(struct ast_state *state, struct validator *vstate, void* + _o) { excepthandler_ty o = (excepthandler_ty)_o; PyObject *result = NULL, *value = NULL; @@ -5086,7 +5144,7 @@ ast2obj_excepthandler(struct ast_state *state, void* _o) if (!o) { Py_RETURN_NONE; } - if (++state->recursion_depth > state->recursion_limit) { + if (++vstate->recursion_depth > vstate->recursion_limit) { PyErr_SetString(PyExc_RecursionError, "maximum recursion depth exceeded during ast construction"); return NULL; @@ -5096,17 +5154,17 @@ ast2obj_excepthandler(struct ast_state *state, void* _o) tp = (PyTypeObject *)state->ExceptHandler_type; result = PyType_GenericNew(tp, NULL, NULL); if (!result) goto failed; - value = ast2obj_expr(state, o->v.ExceptHandler.type); + value = ast2obj_expr(state, vstate, o->v.ExceptHandler.type); if (!value) goto failed; if (PyObject_SetAttr(result, state->type, value) == -1) goto failed; Py_DECREF(value); - value = ast2obj_identifier(state, o->v.ExceptHandler.name); + value = ast2obj_identifier(state, vstate, o->v.ExceptHandler.name); if (!value) goto failed; if (PyObject_SetAttr(result, state->name, value) == -1) goto failed; Py_DECREF(value); - value = ast2obj_list(state, (asdl_seq*)o->v.ExceptHandler.body, + value = ast2obj_list(state, vstate, (asdl_seq*)o->v.ExceptHandler.body, ast2obj_stmt); if (!value) goto failed; if (PyObject_SetAttr(result, state->body, value) == -1) @@ -5114,37 +5172,37 @@ ast2obj_excepthandler(struct ast_state *state, void* _o) Py_DECREF(value); break; } - value = ast2obj_int(state, o->lineno); + value = ast2obj_int(state, vstate, o->lineno); if (!value) goto failed; if (PyObject_SetAttr(result, state->lineno, value) < 0) goto failed; Py_DECREF(value); - value = ast2obj_int(state, o->col_offset); + value = ast2obj_int(state, vstate, o->col_offset); if (!value) goto failed; if (PyObject_SetAttr(result, state->col_offset, value) < 0) goto failed; Py_DECREF(value); - value = ast2obj_int(state, o->end_lineno); + value = ast2obj_int(state, vstate, o->end_lineno); if (!value) goto failed; if (PyObject_SetAttr(result, state->end_lineno, value) < 0) goto failed; Py_DECREF(value); - value = ast2obj_int(state, o->end_col_offset); + value = ast2obj_int(state, vstate, o->end_col_offset); if (!value) goto failed; if (PyObject_SetAttr(result, state->end_col_offset, value) < 0) goto failed; Py_DECREF(value); - state->recursion_depth--; + vstate->recursion_depth--; return result; failed: - state->recursion_depth--; + vstate->recursion_depth--; Py_XDECREF(value); Py_XDECREF(result); return NULL; } PyObject* -ast2obj_arguments(struct ast_state *state, void* _o) +ast2obj_arguments(struct ast_state *state, struct validator *vstate, void* _o) { arguments_ty o = (arguments_ty)_o; PyObject *result = NULL, *value = NULL; @@ -5152,7 +5210,7 @@ ast2obj_arguments(struct ast_state *state, void* _o) if (!o) { Py_RETURN_NONE; } - if (++state->recursion_depth > state->recursion_limit) { + if (++vstate->recursion_depth > vstate->recursion_limit) { PyErr_SetString(PyExc_RecursionError, "maximum recursion depth exceeded during ast construction"); return NULL; @@ -5160,52 +5218,53 @@ ast2obj_arguments(struct ast_state *state, void* _o) tp = (PyTypeObject *)state->arguments_type; result = PyType_GenericNew(tp, NULL, NULL); if (!result) return NULL; - value = ast2obj_list(state, (asdl_seq*)o->posonlyargs, ast2obj_arg); + value = ast2obj_list(state, vstate, (asdl_seq*)o->posonlyargs, ast2obj_arg); if (!value) goto failed; if (PyObject_SetAttr(result, state->posonlyargs, value) == -1) goto failed; Py_DECREF(value); - value = ast2obj_list(state, (asdl_seq*)o->args, ast2obj_arg); + value = ast2obj_list(state, vstate, (asdl_seq*)o->args, ast2obj_arg); if (!value) goto failed; if (PyObject_SetAttr(result, state->args, value) == -1) goto failed; Py_DECREF(value); - value = ast2obj_arg(state, o->vararg); + value = ast2obj_arg(state, vstate, o->vararg); if (!value) goto failed; if (PyObject_SetAttr(result, state->vararg, value) == -1) goto failed; Py_DECREF(value); - value = ast2obj_list(state, (asdl_seq*)o->kwonlyargs, ast2obj_arg); + value = ast2obj_list(state, vstate, (asdl_seq*)o->kwonlyargs, ast2obj_arg); if (!value) goto failed; if (PyObject_SetAttr(result, state->kwonlyargs, value) == -1) goto failed; Py_DECREF(value); - value = ast2obj_list(state, (asdl_seq*)o->kw_defaults, ast2obj_expr); + value = ast2obj_list(state, vstate, (asdl_seq*)o->kw_defaults, + ast2obj_expr); if (!value) goto failed; if (PyObject_SetAttr(result, state->kw_defaults, value) == -1) goto failed; Py_DECREF(value); - value = ast2obj_arg(state, o->kwarg); + value = ast2obj_arg(state, vstate, o->kwarg); if (!value) goto failed; if (PyObject_SetAttr(result, state->kwarg, value) == -1) goto failed; Py_DECREF(value); - value = ast2obj_list(state, (asdl_seq*)o->defaults, ast2obj_expr); + value = ast2obj_list(state, vstate, (asdl_seq*)o->defaults, ast2obj_expr); if (!value) goto failed; if (PyObject_SetAttr(result, state->defaults, value) == -1) goto failed; Py_DECREF(value); - state->recursion_depth--; + vstate->recursion_depth--; return result; failed: - state->recursion_depth--; + vstate->recursion_depth--; Py_XDECREF(value); Py_XDECREF(result); return NULL; } PyObject* -ast2obj_arg(struct ast_state *state, void* _o) +ast2obj_arg(struct ast_state *state, struct validator *vstate, void* _o) { arg_ty o = (arg_ty)_o; PyObject *result = NULL, *value = NULL; @@ -5213,7 +5272,7 @@ ast2obj_arg(struct ast_state *state, void* _o) if (!o) { Py_RETURN_NONE; } - if (++state->recursion_depth > state->recursion_limit) { + if (++vstate->recursion_depth > vstate->recursion_limit) { PyErr_SetString(PyExc_RecursionError, "maximum recursion depth exceeded during ast construction"); return NULL; @@ -5221,52 +5280,52 @@ ast2obj_arg(struct ast_state *state, void* _o) tp = (PyTypeObject *)state->arg_type; result = PyType_GenericNew(tp, NULL, NULL); if (!result) return NULL; - value = ast2obj_identifier(state, o->arg); + value = ast2obj_identifier(state, vstate, o->arg); if (!value) goto failed; if (PyObject_SetAttr(result, state->arg, value) == -1) goto failed; Py_DECREF(value); - value = ast2obj_expr(state, o->annotation); + value = ast2obj_expr(state, vstate, o->annotation); if (!value) goto failed; if (PyObject_SetAttr(result, state->annotation, value) == -1) goto failed; Py_DECREF(value); - value = ast2obj_string(state, o->type_comment); + value = ast2obj_string(state, vstate, o->type_comment); if (!value) goto failed; if (PyObject_SetAttr(result, state->type_comment, value) == -1) goto failed; Py_DECREF(value); - value = ast2obj_int(state, o->lineno); + value = ast2obj_int(state, vstate, o->lineno); if (!value) goto failed; if (PyObject_SetAttr(result, state->lineno, value) < 0) goto failed; Py_DECREF(value); - value = ast2obj_int(state, o->col_offset); + value = ast2obj_int(state, vstate, o->col_offset); if (!value) goto failed; if (PyObject_SetAttr(result, state->col_offset, value) < 0) goto failed; Py_DECREF(value); - value = ast2obj_int(state, o->end_lineno); + value = ast2obj_int(state, vstate, o->end_lineno); if (!value) goto failed; if (PyObject_SetAttr(result, state->end_lineno, value) < 0) goto failed; Py_DECREF(value); - value = ast2obj_int(state, o->end_col_offset); + value = ast2obj_int(state, vstate, o->end_col_offset); if (!value) goto failed; if (PyObject_SetAttr(result, state->end_col_offset, value) < 0) goto failed; Py_DECREF(value); - state->recursion_depth--; + vstate->recursion_depth--; return result; failed: - state->recursion_depth--; + vstate->recursion_depth--; Py_XDECREF(value); Py_XDECREF(result); return NULL; } PyObject* -ast2obj_keyword(struct ast_state *state, void* _o) +ast2obj_keyword(struct ast_state *state, struct validator *vstate, void* _o) { keyword_ty o = (keyword_ty)_o; PyObject *result = NULL, *value = NULL; @@ -5274,7 +5333,7 @@ ast2obj_keyword(struct ast_state *state, void* _o) if (!o) { Py_RETURN_NONE; } - if (++state->recursion_depth > state->recursion_limit) { + if (++vstate->recursion_depth > vstate->recursion_limit) { PyErr_SetString(PyExc_RecursionError, "maximum recursion depth exceeded during ast construction"); return NULL; @@ -5282,47 +5341,47 @@ ast2obj_keyword(struct ast_state *state, void* _o) tp = (PyTypeObject *)state->keyword_type; result = PyType_GenericNew(tp, NULL, NULL); if (!result) return NULL; - value = ast2obj_identifier(state, o->arg); + value = ast2obj_identifier(state, vstate, o->arg); if (!value) goto failed; if (PyObject_SetAttr(result, state->arg, value) == -1) goto failed; Py_DECREF(value); - value = ast2obj_expr(state, o->value); + value = ast2obj_expr(state, vstate, o->value); if (!value) goto failed; if (PyObject_SetAttr(result, state->value, value) == -1) goto failed; Py_DECREF(value); - value = ast2obj_int(state, o->lineno); + value = ast2obj_int(state, vstate, o->lineno); if (!value) goto failed; if (PyObject_SetAttr(result, state->lineno, value) < 0) goto failed; Py_DECREF(value); - value = ast2obj_int(state, o->col_offset); + value = ast2obj_int(state, vstate, o->col_offset); if (!value) goto failed; if (PyObject_SetAttr(result, state->col_offset, value) < 0) goto failed; Py_DECREF(value); - value = ast2obj_int(state, o->end_lineno); + value = ast2obj_int(state, vstate, o->end_lineno); if (!value) goto failed; if (PyObject_SetAttr(result, state->end_lineno, value) < 0) goto failed; Py_DECREF(value); - value = ast2obj_int(state, o->end_col_offset); + value = ast2obj_int(state, vstate, o->end_col_offset); if (!value) goto failed; if (PyObject_SetAttr(result, state->end_col_offset, value) < 0) goto failed; Py_DECREF(value); - state->recursion_depth--; + vstate->recursion_depth--; return result; failed: - state->recursion_depth--; + vstate->recursion_depth--; Py_XDECREF(value); Py_XDECREF(result); return NULL; } PyObject* -ast2obj_alias(struct ast_state *state, void* _o) +ast2obj_alias(struct ast_state *state, struct validator *vstate, void* _o) { alias_ty o = (alias_ty)_o; PyObject *result = NULL, *value = NULL; @@ -5330,7 +5389,7 @@ ast2obj_alias(struct ast_state *state, void* _o) if (!o) { Py_RETURN_NONE; } - if (++state->recursion_depth > state->recursion_limit) { + if (++vstate->recursion_depth > vstate->recursion_limit) { PyErr_SetString(PyExc_RecursionError, "maximum recursion depth exceeded during ast construction"); return NULL; @@ -5338,47 +5397,47 @@ ast2obj_alias(struct ast_state *state, void* _o) tp = (PyTypeObject *)state->alias_type; result = PyType_GenericNew(tp, NULL, NULL); if (!result) return NULL; - value = ast2obj_identifier(state, o->name); + value = ast2obj_identifier(state, vstate, o->name); if (!value) goto failed; if (PyObject_SetAttr(result, state->name, value) == -1) goto failed; Py_DECREF(value); - value = ast2obj_identifier(state, o->asname); + value = ast2obj_identifier(state, vstate, o->asname); if (!value) goto failed; if (PyObject_SetAttr(result, state->asname, value) == -1) goto failed; Py_DECREF(value); - value = ast2obj_int(state, o->lineno); + value = ast2obj_int(state, vstate, o->lineno); if (!value) goto failed; if (PyObject_SetAttr(result, state->lineno, value) < 0) goto failed; Py_DECREF(value); - value = ast2obj_int(state, o->col_offset); + value = ast2obj_int(state, vstate, o->col_offset); if (!value) goto failed; if (PyObject_SetAttr(result, state->col_offset, value) < 0) goto failed; Py_DECREF(value); - value = ast2obj_int(state, o->end_lineno); + value = ast2obj_int(state, vstate, o->end_lineno); if (!value) goto failed; if (PyObject_SetAttr(result, state->end_lineno, value) < 0) goto failed; Py_DECREF(value); - value = ast2obj_int(state, o->end_col_offset); + value = ast2obj_int(state, vstate, o->end_col_offset); if (!value) goto failed; if (PyObject_SetAttr(result, state->end_col_offset, value) < 0) goto failed; Py_DECREF(value); - state->recursion_depth--; + vstate->recursion_depth--; return result; failed: - state->recursion_depth--; + vstate->recursion_depth--; Py_XDECREF(value); Py_XDECREF(result); return NULL; } PyObject* -ast2obj_withitem(struct ast_state *state, void* _o) +ast2obj_withitem(struct ast_state *state, struct validator *vstate, void* _o) { withitem_ty o = (withitem_ty)_o; PyObject *result = NULL, *value = NULL; @@ -5386,7 +5445,7 @@ ast2obj_withitem(struct ast_state *state, void* _o) if (!o) { Py_RETURN_NONE; } - if (++state->recursion_depth > state->recursion_limit) { + if (++vstate->recursion_depth > vstate->recursion_limit) { PyErr_SetString(PyExc_RecursionError, "maximum recursion depth exceeded during ast construction"); return NULL; @@ -5394,27 +5453,27 @@ ast2obj_withitem(struct ast_state *state, void* _o) tp = (PyTypeObject *)state->withitem_type; result = PyType_GenericNew(tp, NULL, NULL); if (!result) return NULL; - value = ast2obj_expr(state, o->context_expr); + value = ast2obj_expr(state, vstate, o->context_expr); if (!value) goto failed; if (PyObject_SetAttr(result, state->context_expr, value) == -1) goto failed; Py_DECREF(value); - value = ast2obj_expr(state, o->optional_vars); + value = ast2obj_expr(state, vstate, o->optional_vars); if (!value) goto failed; if (PyObject_SetAttr(result, state->optional_vars, value) == -1) goto failed; Py_DECREF(value); - state->recursion_depth--; + vstate->recursion_depth--; return result; failed: - state->recursion_depth--; + vstate->recursion_depth--; Py_XDECREF(value); Py_XDECREF(result); return NULL; } PyObject* -ast2obj_match_case(struct ast_state *state, void* _o) +ast2obj_match_case(struct ast_state *state, struct validator *vstate, void* _o) { match_case_ty o = (match_case_ty)_o; PyObject *result = NULL, *value = NULL; @@ -5422,7 +5481,7 @@ ast2obj_match_case(struct ast_state *state, void* _o) if (!o) { Py_RETURN_NONE; } - if (++state->recursion_depth > state->recursion_limit) { + if (++vstate->recursion_depth > vstate->recursion_limit) { PyErr_SetString(PyExc_RecursionError, "maximum recursion depth exceeded during ast construction"); return NULL; @@ -5430,32 +5489,32 @@ ast2obj_match_case(struct ast_state *state, void* _o) tp = (PyTypeObject *)state->match_case_type; result = PyType_GenericNew(tp, NULL, NULL); if (!result) return NULL; - value = ast2obj_pattern(state, o->pattern); + value = ast2obj_pattern(state, vstate, o->pattern); if (!value) goto failed; if (PyObject_SetAttr(result, state->pattern, value) == -1) goto failed; Py_DECREF(value); - value = ast2obj_expr(state, o->guard); + value = ast2obj_expr(state, vstate, o->guard); if (!value) goto failed; if (PyObject_SetAttr(result, state->guard, value) == -1) goto failed; Py_DECREF(value); - value = ast2obj_list(state, (asdl_seq*)o->body, ast2obj_stmt); + value = ast2obj_list(state, vstate, (asdl_seq*)o->body, ast2obj_stmt); if (!value) goto failed; if (PyObject_SetAttr(result, state->body, value) == -1) goto failed; Py_DECREF(value); - state->recursion_depth--; + vstate->recursion_depth--; return result; failed: - state->recursion_depth--; + vstate->recursion_depth--; Py_XDECREF(value); Py_XDECREF(result); return NULL; } PyObject* -ast2obj_pattern(struct ast_state *state, void* _o) +ast2obj_pattern(struct ast_state *state, struct validator *vstate, void* _o) { pattern_ty o = (pattern_ty)_o; PyObject *result = NULL, *value = NULL; @@ -5463,7 +5522,7 @@ ast2obj_pattern(struct ast_state *state, void* _o) if (!o) { Py_RETURN_NONE; } - if (++state->recursion_depth > state->recursion_limit) { + if (++vstate->recursion_depth > vstate->recursion_limit) { PyErr_SetString(PyExc_RecursionError, "maximum recursion depth exceeded during ast construction"); return NULL; @@ -5473,7 +5532,7 @@ ast2obj_pattern(struct ast_state *state, void* _o) tp = (PyTypeObject *)state->MatchValue_type; result = PyType_GenericNew(tp, NULL, NULL); if (!result) goto failed; - value = ast2obj_expr(state, o->v.MatchValue.value); + value = ast2obj_expr(state, vstate, o->v.MatchValue.value); if (!value) goto failed; if (PyObject_SetAttr(result, state->value, value) == -1) goto failed; @@ -5483,7 +5542,7 @@ ast2obj_pattern(struct ast_state *state, void* _o) tp = (PyTypeObject *)state->MatchSingleton_type; result = PyType_GenericNew(tp, NULL, NULL); if (!result) goto failed; - value = ast2obj_constant(state, o->v.MatchSingleton.value); + value = ast2obj_constant(state, vstate, o->v.MatchSingleton.value); if (!value) goto failed; if (PyObject_SetAttr(result, state->value, value) == -1) goto failed; @@ -5493,7 +5552,8 @@ ast2obj_pattern(struct ast_state *state, void* _o) tp = (PyTypeObject *)state->MatchSequence_type; result = PyType_GenericNew(tp, NULL, NULL); if (!result) goto failed; - value = ast2obj_list(state, (asdl_seq*)o->v.MatchSequence.patterns, + value = ast2obj_list(state, vstate, + (asdl_seq*)o->v.MatchSequence.patterns, ast2obj_pattern); if (!value) goto failed; if (PyObject_SetAttr(result, state->patterns, value) == -1) @@ -5504,19 +5564,20 @@ ast2obj_pattern(struct ast_state *state, void* _o) tp = (PyTypeObject *)state->MatchMapping_type; result = PyType_GenericNew(tp, NULL, NULL); if (!result) goto failed; - value = ast2obj_list(state, (asdl_seq*)o->v.MatchMapping.keys, + value = ast2obj_list(state, vstate, (asdl_seq*)o->v.MatchMapping.keys, ast2obj_expr); if (!value) goto failed; if (PyObject_SetAttr(result, state->keys, value) == -1) goto failed; Py_DECREF(value); - value = ast2obj_list(state, (asdl_seq*)o->v.MatchMapping.patterns, + value = ast2obj_list(state, vstate, + (asdl_seq*)o->v.MatchMapping.patterns, ast2obj_pattern); if (!value) goto failed; if (PyObject_SetAttr(result, state->patterns, value) == -1) goto failed; Py_DECREF(value); - value = ast2obj_identifier(state, o->v.MatchMapping.rest); + value = ast2obj_identifier(state, vstate, o->v.MatchMapping.rest); if (!value) goto failed; if (PyObject_SetAttr(result, state->rest, value) == -1) goto failed; @@ -5526,24 +5587,27 @@ ast2obj_pattern(struct ast_state *state, void* _o) tp = (PyTypeObject *)state->MatchClass_type; result = PyType_GenericNew(tp, NULL, NULL); if (!result) goto failed; - value = ast2obj_expr(state, o->v.MatchClass.cls); + value = ast2obj_expr(state, vstate, o->v.MatchClass.cls); if (!value) goto failed; if (PyObject_SetAttr(result, state->cls, value) == -1) goto failed; Py_DECREF(value); - value = ast2obj_list(state, (asdl_seq*)o->v.MatchClass.patterns, + value = ast2obj_list(state, vstate, + (asdl_seq*)o->v.MatchClass.patterns, ast2obj_pattern); if (!value) goto failed; if (PyObject_SetAttr(result, state->patterns, value) == -1) goto failed; Py_DECREF(value); - value = ast2obj_list(state, (asdl_seq*)o->v.MatchClass.kwd_attrs, + value = ast2obj_list(state, vstate, + (asdl_seq*)o->v.MatchClass.kwd_attrs, ast2obj_identifier); if (!value) goto failed; if (PyObject_SetAttr(result, state->kwd_attrs, value) == -1) goto failed; Py_DECREF(value); - value = ast2obj_list(state, (asdl_seq*)o->v.MatchClass.kwd_patterns, + value = ast2obj_list(state, vstate, + (asdl_seq*)o->v.MatchClass.kwd_patterns, ast2obj_pattern); if (!value) goto failed; if (PyObject_SetAttr(result, state->kwd_patterns, value) == -1) @@ -5554,7 +5618,7 @@ ast2obj_pattern(struct ast_state *state, void* _o) tp = (PyTypeObject *)state->MatchStar_type; result = PyType_GenericNew(tp, NULL, NULL); if (!result) goto failed; - value = ast2obj_identifier(state, o->v.MatchStar.name); + value = ast2obj_identifier(state, vstate, o->v.MatchStar.name); if (!value) goto failed; if (PyObject_SetAttr(result, state->name, value) == -1) goto failed; @@ -5564,12 +5628,12 @@ ast2obj_pattern(struct ast_state *state, void* _o) tp = (PyTypeObject *)state->MatchAs_type; result = PyType_GenericNew(tp, NULL, NULL); if (!result) goto failed; - value = ast2obj_pattern(state, o->v.MatchAs.pattern); + value = ast2obj_pattern(state, vstate, o->v.MatchAs.pattern); if (!value) goto failed; if (PyObject_SetAttr(result, state->pattern, value) == -1) goto failed; Py_DECREF(value); - value = ast2obj_identifier(state, o->v.MatchAs.name); + value = ast2obj_identifier(state, vstate, o->v.MatchAs.name); if (!value) goto failed; if (PyObject_SetAttr(result, state->name, value) == -1) goto failed; @@ -5579,7 +5643,7 @@ ast2obj_pattern(struct ast_state *state, void* _o) tp = (PyTypeObject *)state->MatchOr_type; result = PyType_GenericNew(tp, NULL, NULL); if (!result) goto failed; - value = ast2obj_list(state, (asdl_seq*)o->v.MatchOr.patterns, + value = ast2obj_list(state, vstate, (asdl_seq*)o->v.MatchOr.patterns, ast2obj_pattern); if (!value) goto failed; if (PyObject_SetAttr(result, state->patterns, value) == -1) @@ -5587,37 +5651,37 @@ ast2obj_pattern(struct ast_state *state, void* _o) Py_DECREF(value); break; } - value = ast2obj_int(state, o->lineno); + value = ast2obj_int(state, vstate, o->lineno); if (!value) goto failed; if (PyObject_SetAttr(result, state->lineno, value) < 0) goto failed; Py_DECREF(value); - value = ast2obj_int(state, o->col_offset); + value = ast2obj_int(state, vstate, o->col_offset); if (!value) goto failed; if (PyObject_SetAttr(result, state->col_offset, value) < 0) goto failed; Py_DECREF(value); - value = ast2obj_int(state, o->end_lineno); + value = ast2obj_int(state, vstate, o->end_lineno); if (!value) goto failed; if (PyObject_SetAttr(result, state->end_lineno, value) < 0) goto failed; Py_DECREF(value); - value = ast2obj_int(state, o->end_col_offset); + value = ast2obj_int(state, vstate, o->end_col_offset); if (!value) goto failed; if (PyObject_SetAttr(result, state->end_col_offset, value) < 0) goto failed; Py_DECREF(value); - state->recursion_depth--; + vstate->recursion_depth--; return result; failed: - state->recursion_depth--; + vstate->recursion_depth--; Py_XDECREF(value); Py_XDECREF(result); return NULL; } PyObject* -ast2obj_type_ignore(struct ast_state *state, void* _o) +ast2obj_type_ignore(struct ast_state *state, struct validator *vstate, void* _o) { type_ignore_ty o = (type_ignore_ty)_o; PyObject *result = NULL, *value = NULL; @@ -5625,7 +5689,7 @@ ast2obj_type_ignore(struct ast_state *state, void* _o) if (!o) { Py_RETURN_NONE; } - if (++state->recursion_depth > state->recursion_limit) { + if (++vstate->recursion_depth > vstate->recursion_limit) { PyErr_SetString(PyExc_RecursionError, "maximum recursion depth exceeded during ast construction"); return NULL; @@ -5635,29 +5699,29 @@ ast2obj_type_ignore(struct ast_state *state, void* _o) tp = (PyTypeObject *)state->TypeIgnore_type; result = PyType_GenericNew(tp, NULL, NULL); if (!result) goto failed; - value = ast2obj_int(state, o->v.TypeIgnore.lineno); + value = ast2obj_int(state, vstate, o->v.TypeIgnore.lineno); if (!value) goto failed; if (PyObject_SetAttr(result, state->lineno, value) == -1) goto failed; Py_DECREF(value); - value = ast2obj_string(state, o->v.TypeIgnore.tag); + value = ast2obj_string(state, vstate, o->v.TypeIgnore.tag); if (!value) goto failed; if (PyObject_SetAttr(result, state->tag, value) == -1) goto failed; Py_DECREF(value); break; } - state->recursion_depth--; + vstate->recursion_depth--; return result; failed: - state->recursion_depth--; + vstate->recursion_depth--; Py_XDECREF(value); Py_XDECREF(result); return NULL; } PyObject* -ast2obj_type_param(struct ast_state *state, void* _o) +ast2obj_type_param(struct ast_state *state, struct validator *vstate, void* _o) { type_param_ty o = (type_param_ty)_o; PyObject *result = NULL, *value = NULL; @@ -5665,7 +5729,7 @@ ast2obj_type_param(struct ast_state *state, void* _o) if (!o) { Py_RETURN_NONE; } - if (++state->recursion_depth > state->recursion_limit) { + if (++vstate->recursion_depth > vstate->recursion_limit) { PyErr_SetString(PyExc_RecursionError, "maximum recursion depth exceeded during ast construction"); return NULL; @@ -5675,12 +5739,12 @@ ast2obj_type_param(struct ast_state *state, void* _o) tp = (PyTypeObject *)state->TypeVar_type; result = PyType_GenericNew(tp, NULL, NULL); if (!result) goto failed; - value = ast2obj_identifier(state, o->v.TypeVar.name); + value = ast2obj_identifier(state, vstate, o->v.TypeVar.name); if (!value) goto failed; if (PyObject_SetAttr(result, state->name, value) == -1) goto failed; Py_DECREF(value); - value = ast2obj_expr(state, o->v.TypeVar.bound); + value = ast2obj_expr(state, vstate, o->v.TypeVar.bound); if (!value) goto failed; if (PyObject_SetAttr(result, state->bound, value) == -1) goto failed; @@ -5690,7 +5754,7 @@ ast2obj_type_param(struct ast_state *state, void* _o) tp = (PyTypeObject *)state->ParamSpec_type; result = PyType_GenericNew(tp, NULL, NULL); if (!result) goto failed; - value = ast2obj_identifier(state, o->v.ParamSpec.name); + value = ast2obj_identifier(state, vstate, o->v.ParamSpec.name); if (!value) goto failed; if (PyObject_SetAttr(result, state->name, value) == -1) goto failed; @@ -5700,37 +5764,37 @@ ast2obj_type_param(struct ast_state *state, void* _o) tp = (PyTypeObject *)state->TypeVarTuple_type; result = PyType_GenericNew(tp, NULL, NULL); if (!result) goto failed; - value = ast2obj_identifier(state, o->v.TypeVarTuple.name); + value = ast2obj_identifier(state, vstate, o->v.TypeVarTuple.name); if (!value) goto failed; if (PyObject_SetAttr(result, state->name, value) == -1) goto failed; Py_DECREF(value); break; } - value = ast2obj_int(state, o->lineno); + value = ast2obj_int(state, vstate, o->lineno); if (!value) goto failed; if (PyObject_SetAttr(result, state->lineno, value) < 0) goto failed; Py_DECREF(value); - value = ast2obj_int(state, o->col_offset); + value = ast2obj_int(state, vstate, o->col_offset); if (!value) goto failed; if (PyObject_SetAttr(result, state->col_offset, value) < 0) goto failed; Py_DECREF(value); - value = ast2obj_int(state, o->end_lineno); + value = ast2obj_int(state, vstate, o->end_lineno); if (!value) goto failed; if (PyObject_SetAttr(result, state->end_lineno, value) < 0) goto failed; Py_DECREF(value); - value = ast2obj_int(state, o->end_col_offset); + value = ast2obj_int(state, vstate, o->end_col_offset); if (!value) goto failed; if (PyObject_SetAttr(result, state->end_col_offset, value) < 0) goto failed; Py_DECREF(value); - state->recursion_depth--; + vstate->recursion_depth--; return result; failed: - state->recursion_depth--; + vstate->recursion_depth--; Py_XDECREF(value); Py_XDECREF(result); return NULL; @@ -13085,23 +13149,23 @@ PyObject* PyAST_mod2obj(mod_ty t) int starting_recursion_depth; /* Be careful here to prevent overflow. */ - int COMPILER_STACK_FRAME_SCALE = 2; PyThreadState *tstate = _PyThreadState_GET(); if (!tstate) { return NULL; } - state->recursion_limit = Py_C_RECURSION_LIMIT * COMPILER_STACK_FRAME_SCALE; + struct validator vstate; + vstate.recursion_limit = Py_C_RECURSION_LIMIT; int recursion_depth = Py_C_RECURSION_LIMIT - tstate->c_recursion_remaining; - starting_recursion_depth = recursion_depth * COMPILER_STACK_FRAME_SCALE; - state->recursion_depth = starting_recursion_depth; + starting_recursion_depth = recursion_depth; + vstate.recursion_depth = starting_recursion_depth; - PyObject *result = ast2obj_mod(state, t); + PyObject *result = ast2obj_mod(state, &vstate, t); /* Check that the recursion depth counting balanced correctly */ - if (result && state->recursion_depth != starting_recursion_depth) { + if (result && vstate.recursion_depth != starting_recursion_depth) { PyErr_Format(PyExc_SystemError, "AST constructor recursion depth mismatch (before=%d, after=%d)", - starting_recursion_depth, state->recursion_depth); + starting_recursion_depth, vstate.recursion_depth); return NULL; } return result; diff --git a/Python/ast.c b/Python/ast.c index 5f46d4149c2ed0..71b09d889f17c1 100644 --- a/Python/ast.c +++ b/Python/ast.c @@ -1037,10 +1037,6 @@ validate_type_params(struct validator *state, asdl_type_param_seq *tps) return 1; } - -/* See comments in symtable.c. */ -#define COMPILER_STACK_FRAME_SCALE 2 - int _PyAST_Validate(mod_ty mod) { @@ -1057,9 +1053,9 @@ _PyAST_Validate(mod_ty mod) } /* Be careful here to prevent overflow. */ int recursion_depth = Py_C_RECURSION_LIMIT - tstate->c_recursion_remaining; - starting_recursion_depth = recursion_depth * COMPILER_STACK_FRAME_SCALE; + starting_recursion_depth = recursion_depth; state.recursion_depth = starting_recursion_depth; - state.recursion_limit = Py_C_RECURSION_LIMIT * COMPILER_STACK_FRAME_SCALE; + state.recursion_limit = Py_C_RECURSION_LIMIT; switch (mod->kind) { case Module_kind: diff --git a/Python/ast_opt.c b/Python/ast_opt.c index 04d7ae6eaafbc0..41e906c66e8eec 100644 --- a/Python/ast_opt.c +++ b/Python/ast_opt.c @@ -1100,9 +1100,6 @@ astfold_type_param(type_param_ty node_, PyArena *ctx_, _PyASTOptimizeState *stat #undef CALL_OPT #undef CALL_SEQ -/* See comments in symtable.c. */ -#define COMPILER_STACK_FRAME_SCALE 2 - int _PyAST_Optimize(mod_ty mod, PyArena *arena, int optimize, int ff_features) { @@ -1120,9 +1117,9 @@ _PyAST_Optimize(mod_ty mod, PyArena *arena, int optimize, int ff_features) } /* Be careful here to prevent overflow. */ int recursion_depth = Py_C_RECURSION_LIMIT - tstate->c_recursion_remaining; - starting_recursion_depth = recursion_depth * COMPILER_STACK_FRAME_SCALE; + starting_recursion_depth = recursion_depth; state.recursion_depth = starting_recursion_depth; - state.recursion_limit = Py_C_RECURSION_LIMIT * COMPILER_STACK_FRAME_SCALE; + state.recursion_limit = Py_C_RECURSION_LIMIT; int ret = astfold_mod(mod, arena, &state); assert(ret || PyErr_Occurred()); diff --git a/Python/bltinmodule.c b/Python/bltinmodule.c index e54d5cbacdc96f..31c1bf07e8fb91 100644 --- a/Python/bltinmodule.c +++ b/Python/bltinmodule.c @@ -2382,11 +2382,6 @@ builtin_round_impl(PyObject *module, PyObject *number, PyObject *ndigits) { PyObject *round, *result; - if (!_PyType_IsReady(Py_TYPE(number))) { - if (PyType_Ready(Py_TYPE(number)) < 0) - return NULL; - } - round = _PyObject_LookupSpecial(number, &_Py_ID(__round__)); if (round == NULL) { if (!PyErr_Occurred()) diff --git a/Python/brc.c b/Python/brc.c new file mode 100644 index 00000000000000..f1fd57a2964cf5 --- /dev/null +++ b/Python/brc.c @@ -0,0 +1,198 @@ +// Implementation of biased reference counting inter-thread queue. +// +// Biased reference counting maintains two refcount fields in each object: +// ob_ref_local and ob_ref_shared. The true refcount is the sum of these two +// fields. In some cases, when refcounting operations are split across threads, +// the ob_ref_shared field can be negative (although the total refcount must +// be at least zero). In this case, the thread that decremented the refcount +// requests that the owning thread give up ownership and merge the refcount +// fields. This file implements the mechanism for doing so. +// +// Each thread state maintains a queue of objects whose refcounts it should +// merge. The thread states are stored in a per-interpreter hash table by +// thread id. The hash table has a fixed size and uses a linked list to store +// thread states within each bucket. +// +// The queueing thread uses the eval breaker mechanism to notify the owning +// thread that it has objects to merge. Additionaly, all queued objects are +// merged during GC. +#include "Python.h" +#include "pycore_object.h" // _Py_ExplicitMergeRefcount +#include "pycore_brc.h" // struct _brc_thread_state +#include "pycore_ceval.h" // _Py_set_eval_breaker_bit +#include "pycore_llist.h" // struct llist_node +#include "pycore_pystate.h" // _PyThreadStateImpl + +#ifdef Py_GIL_DISABLED + +// Get the hashtable bucket for a given thread id. +static struct _brc_bucket * +get_bucket(PyInterpreterState *interp, uintptr_t tid) +{ + return &interp->brc.table[tid % _Py_BRC_NUM_BUCKETS]; +} + +// Find the thread state in a hash table bucket by thread id. +static _PyThreadStateImpl * +find_thread_state(struct _brc_bucket *bucket, uintptr_t thread_id) +{ + struct llist_node *node; + llist_for_each(node, &bucket->root) { + // Get the containing _PyThreadStateImpl from the linked-list node. + _PyThreadStateImpl *ts = llist_data(node, _PyThreadStateImpl, + brc.bucket_node); + if (ts->brc.tid == thread_id) { + return ts; + } + } + return NULL; +} + +// Enqueue an object to be merged by the owning thread. This steals a +// reference to the object. +void +_Py_brc_queue_object(PyObject *ob) +{ + PyInterpreterState *interp = _PyInterpreterState_GET(); + + uintptr_t ob_tid = _Py_atomic_load_uintptr(&ob->ob_tid); + if (ob_tid == 0) { + // The owning thread may have concurrently decided to merge the + // refcount fields. + Py_DECREF(ob); + return; + } + + struct _brc_bucket *bucket = get_bucket(interp, ob_tid); + PyMutex_Lock(&bucket->mutex); + _PyThreadStateImpl *tstate = find_thread_state(bucket, ob_tid); + if (tstate == NULL) { + // If we didn't find the owning thread then it must have already exited. + // It's safe (and necessary) to merge the refcount. Subtract one when + // merging because we've stolen a reference. + Py_ssize_t refcount = _Py_ExplicitMergeRefcount(ob, -1); + PyMutex_Unlock(&bucket->mutex); + if (refcount == 0) { + _Py_Dealloc(ob); + } + return; + } + + if (_PyObjectStack_Push(&tstate->brc.objects_to_merge, ob) < 0) { + PyMutex_Unlock(&bucket->mutex); + + // Fall back to stopping all threads and manually merging the refcount + // if we can't enqueue the object to be merged. + _PyEval_StopTheWorld(interp); + Py_ssize_t refcount = _Py_ExplicitMergeRefcount(ob, -1); + _PyEval_StartTheWorld(interp); + + if (refcount == 0) { + _Py_Dealloc(ob); + } + return; + } + + // Notify owning thread + _Py_set_eval_breaker_bit(interp, _PY_EVAL_EXPLICIT_MERGE_BIT, 1); + + PyMutex_Unlock(&bucket->mutex); +} + +static void +merge_queued_objects(_PyObjectStack *to_merge) +{ + PyObject *ob; + while ((ob = _PyObjectStack_Pop(to_merge)) != NULL) { + // Subtract one when merging because the queue had a reference. + Py_ssize_t refcount = _Py_ExplicitMergeRefcount(ob, -1); + if (refcount == 0) { + _Py_Dealloc(ob); + } + } +} + +// Process this thread's queue of objects to merge. +void +_Py_brc_merge_refcounts(PyThreadState *tstate) +{ + struct _brc_thread_state *brc = &((_PyThreadStateImpl *)tstate)->brc; + struct _brc_bucket *bucket = get_bucket(tstate->interp, brc->tid); + + // Append all objects into a local stack. We don't want to hold the lock + // while calling destructors. + PyMutex_Lock(&bucket->mutex); + _PyObjectStack_Merge(&brc->local_objects_to_merge, &brc->objects_to_merge); + PyMutex_Unlock(&bucket->mutex); + + // Process the local stack until it's empty + merge_queued_objects(&brc->local_objects_to_merge); +} + +void +_Py_brc_init_state(PyInterpreterState *interp) +{ + struct _brc_state *brc = &interp->brc; + for (Py_ssize_t i = 0; i < _Py_BRC_NUM_BUCKETS; i++) { + llist_init(&brc->table[i].root); + } +} + +void +_Py_brc_init_thread(PyThreadState *tstate) +{ + struct _brc_thread_state *brc = &((_PyThreadStateImpl *)tstate)->brc; + brc->tid = _Py_ThreadId(); + + // Add ourself to the hashtable + struct _brc_bucket *bucket = get_bucket(tstate->interp, brc->tid); + PyMutex_Lock(&bucket->mutex); + llist_insert_tail(&bucket->root, &brc->bucket_node); + PyMutex_Unlock(&bucket->mutex); +} + +void +_Py_brc_remove_thread(PyThreadState *tstate) +{ + struct _brc_thread_state *brc = &((_PyThreadStateImpl *)tstate)->brc; + struct _brc_bucket *bucket = get_bucket(tstate->interp, brc->tid); + + // We need to fully process any objects to merge before removing ourself + // from the hashtable. It is not safe to perform any refcount operations + // after we are removed. After that point, other threads treat our objects + // as abandoned and may merge the objects' refcounts directly. + bool empty = false; + while (!empty) { + // Process the local stack until it's empty + merge_queued_objects(&brc->local_objects_to_merge); + + PyMutex_Lock(&bucket->mutex); + empty = (brc->objects_to_merge.head == NULL); + if (empty) { + llist_remove(&brc->bucket_node); + } + else { + _PyObjectStack_Merge(&brc->local_objects_to_merge, + &brc->objects_to_merge); + } + PyMutex_Unlock(&bucket->mutex); + } + + assert(brc->local_objects_to_merge.head == NULL); + assert(brc->objects_to_merge.head == NULL); +} + +void +_Py_brc_after_fork(PyInterpreterState *interp) +{ + // Unlock all bucket mutexes. Some of the buckets may be locked because + // locks can be handed off to a parked thread (see lock.c). We don't have + // to worry about consistency here, becuase no thread can be actively + // modifying a bucket, but it might be paused (not yet woken up) on a + // PyMutex_Lock while holding that lock. + for (Py_ssize_t i = 0; i < _Py_BRC_NUM_BUCKETS; i++) { + _PyMutex_at_fork_reinit(&interp->brc.table[i].mutex); + } +} + +#endif /* Py_GIL_DISABLED */ diff --git a/Python/bytecodes.c b/Python/bytecodes.c index 29e1dab184ef4e..6fb4d719e43991 100644 --- a/Python/bytecodes.c +++ b/Python/bytecodes.c @@ -1,6 +1,6 @@ // This file contains instruction definitions. -// It is read by Tools/cases_generator/generate_cases.py -// to generate Python/generated_cases.c.h. +// It is read by generators stored in Tools/cases_generator/ +// to generate Python/generated_cases.c.h and others. // Note that there is some dummy C code at the top and bottom of the file // to fool text editors like VS Code into believing this is valid C code. // The actual instruction definitions start at // BEGIN BYTECODES //. @@ -68,7 +68,7 @@ static size_t jump; static uint16_t invert, counter, index, hint; #define unused 0 // Used in a macro def, can't be static static uint32_t type_version; -static _PyUOpExecutorObject *current_executor; +static _PyExecutorObject *current_executor; static PyObject * dummy_func( @@ -192,7 +192,7 @@ dummy_func( ERROR_IF(err, error); if (frame->instr_ptr != this_instr) { /* Instrumentation has jumped */ - next_instr = this_instr; + next_instr = frame->instr_ptr; DISPATCH(); } } @@ -208,7 +208,7 @@ dummy_func( Py_INCREF(value); } - inst(LOAD_FAST, (-- value)) { + pure inst(LOAD_FAST, (-- value)) { value = GETLOCAL(oparg); assert(value != NULL); Py_INCREF(value); @@ -229,7 +229,7 @@ dummy_func( Py_INCREF(value2); } - inst(LOAD_CONST, (-- value)) { + pure inst(LOAD_CONST, (-- value)) { value = GETITEM(FRAME_CO_CONSTS, oparg); Py_INCREF(value); } @@ -257,17 +257,17 @@ dummy_func( SETLOCAL(oparg2, value2); } - inst(POP_TOP, (value --)) { + pure inst(POP_TOP, (value --)) { DECREF_INPUTS(); } - inst(PUSH_NULL, (-- res)) { + pure inst(PUSH_NULL, (-- res)) { res = NULL; } - macro(END_FOR) = POP_TOP + POP_TOP; + macro(END_FOR) = POP_TOP; - inst(INSTRUMENTED_END_FOR, (receiver, value --)) { + inst(INSTRUMENTED_END_FOR, (receiver, value -- receiver)) { TIER_ONE_ONLY /* Need to create a fake StopIteration error here, * to conform to PEP 380 */ @@ -281,7 +281,7 @@ dummy_func( DECREF_INPUTS(); } - inst(END_SEND, (receiver, value -- value)) { + pure inst(END_SEND, (receiver, value -- value)) { Py_DECREF(receiver); } @@ -303,7 +303,7 @@ dummy_func( ERROR_IF(res == NULL, error); } - inst(UNARY_NOT, (value -- res)) { + pure inst(UNARY_NOT, (value -- res)) { assert(PyBool_Check(value)); res = Py_IsFalse(value) ? Py_True : Py_False; } @@ -411,12 +411,12 @@ dummy_func( // BINARY_OP_INPLACE_ADD_UNICODE, // See comments at that opcode. }; - op(_GUARD_BOTH_INT, (left, right -- left, right)) { + op(_GUARD_BOTH_INT, (left, right -- left: &PYLONG_TYPE, right: &PYLONG_TYPE)) { DEOPT_IF(!PyLong_CheckExact(left)); DEOPT_IF(!PyLong_CheckExact(right)); } - op(_BINARY_OP_MULTIPLY_INT, (left, right -- res)) { + pure op(_BINARY_OP_MULTIPLY_INT, (left, right -- res: &PYLONG_TYPE)) { STAT_INC(BINARY_OP, hit); res = _PyLong_Multiply((PyLongObject *)left, (PyLongObject *)right); _Py_DECREF_SPECIALIZED(right, (destructor)PyObject_Free); @@ -424,7 +424,7 @@ dummy_func( ERROR_IF(res == NULL, error); } - op(_BINARY_OP_ADD_INT, (left, right -- res)) { + pure op(_BINARY_OP_ADD_INT, (left, right -- res: &PYLONG_TYPE)) { STAT_INC(BINARY_OP, hit); res = _PyLong_Add((PyLongObject *)left, (PyLongObject *)right); _Py_DECREF_SPECIALIZED(right, (destructor)PyObject_Free); @@ -432,7 +432,7 @@ dummy_func( ERROR_IF(res == NULL, error); } - op(_BINARY_OP_SUBTRACT_INT, (left, right -- res)) { + pure op(_BINARY_OP_SUBTRACT_INT, (left, right -- res: &PYLONG_TYPE)) { STAT_INC(BINARY_OP, hit); res = _PyLong_Subtract((PyLongObject *)left, (PyLongObject *)right); _Py_DECREF_SPECIALIZED(right, (destructor)PyObject_Free); @@ -447,12 +447,12 @@ dummy_func( macro(BINARY_OP_SUBTRACT_INT) = _GUARD_BOTH_INT + unused/1 + _BINARY_OP_SUBTRACT_INT; - op(_GUARD_BOTH_FLOAT, (left, right -- left, right)) { + op(_GUARD_BOTH_FLOAT, (left, right -- left: &PYFLOAT_TYPE, right: &PYFLOAT_TYPE)) { DEOPT_IF(!PyFloat_CheckExact(left)); DEOPT_IF(!PyFloat_CheckExact(right)); } - op(_BINARY_OP_MULTIPLY_FLOAT, (left, right -- res)) { + pure op(_BINARY_OP_MULTIPLY_FLOAT, (left, right -- res: &PYFLOAT_TYPE)) { STAT_INC(BINARY_OP, hit); double dres = ((PyFloatObject *)left)->ob_fval * @@ -460,7 +460,7 @@ dummy_func( DECREF_INPUTS_AND_REUSE_FLOAT(left, right, dres, res); } - op(_BINARY_OP_ADD_FLOAT, (left, right -- res)) { + pure op(_BINARY_OP_ADD_FLOAT, (left, right -- res: &PYFLOAT_TYPE)) { STAT_INC(BINARY_OP, hit); double dres = ((PyFloatObject *)left)->ob_fval + @@ -468,7 +468,7 @@ dummy_func( DECREF_INPUTS_AND_REUSE_FLOAT(left, right, dres, res); } - op(_BINARY_OP_SUBTRACT_FLOAT, (left, right -- res)) { + pure op(_BINARY_OP_SUBTRACT_FLOAT, (left, right -- res: &PYFLOAT_TYPE)) { STAT_INC(BINARY_OP, hit); double dres = ((PyFloatObject *)left)->ob_fval - @@ -483,12 +483,12 @@ dummy_func( macro(BINARY_OP_SUBTRACT_FLOAT) = _GUARD_BOTH_FLOAT + unused/1 + _BINARY_OP_SUBTRACT_FLOAT; - op(_GUARD_BOTH_UNICODE, (left, right -- left, right)) { + op(_GUARD_BOTH_UNICODE, (left, right -- left: &PYUNICODE_TYPE, right: &PYUNICODE_TYPE)) { DEOPT_IF(!PyUnicode_CheckExact(left)); DEOPT_IF(!PyUnicode_CheckExact(right)); } - op(_BINARY_OP_ADD_UNICODE, (left, right -- res)) { + pure op(_BINARY_OP_ADD_UNICODE, (left, right -- res: &PYUNICODE_TYPE)) { STAT_INC(BINARY_OP, hit); res = PyUnicode_Concat(left, right); _Py_DECREF_SPECIALIZED(left, _PyUnicode_ExactDealloc); @@ -805,7 +805,8 @@ dummy_func( #if TIER_ONE assert(frame != &entry_frame); #endif - STORE_SP(); + SYNC_SP(); + _PyFrame_SetStackPointer(frame, stack_pointer); assert(EMPTY()); _Py_LeaveRecursiveCallPy(tstate); // GH-99729: We need to unlink the frame *before* clearing it: @@ -1900,7 +1901,7 @@ dummy_func( LOAD_ATTR, }; - op(_GUARD_TYPE_VERSION, (type_version/2, owner -- owner)) { + op(_GUARD_TYPE_VERSION, (type_version/2, owner -- owner: &(GUARD_TYPE_VERSION_TYPE + type_version))) { PyTypeObject *tp = Py_TYPE(owner); assert(type_version != 0); DEOPT_IF(tp->tp_version_tag != type_version); @@ -2081,7 +2082,7 @@ dummy_func( DISPATCH_INLINED(new_frame); } - op(_GUARD_DORV_VALUES, (owner -- owner)) { + op(_GUARD_DORV_VALUES, (owner -- owner: &GUARD_DORV_VALUES_TYPE)) { assert(Py_TYPE(owner)->tp_flags & Py_TPFLAGS_MANAGED_DICT); PyDictOrValues dorv = *_PyObject_DictOrValuesPointer(owner); DEOPT_IF(!_PyDictOrValues_IsValues(dorv)); @@ -2325,12 +2326,18 @@ dummy_func( // Double-check that the opcode isn't instrumented or something: if (ucounter > threshold && this_instr->op.code == JUMP_BACKWARD) { OPT_STAT_INC(attempts); - int optimized = _PyOptimizer_BackEdge(frame, this_instr, next_instr, stack_pointer); + _Py_CODEUNIT *start = this_instr; + /* Back up over EXTENDED_ARGs so optimizer sees the whole instruction */ + while (oparg > 255) { + oparg >>= 8; + start--; + } + int optimized = _PyOptimizer_Optimize(frame, start, stack_pointer); ERROR_IF(optimized < 0, error); if (optimized) { // Rewind and enter the executor: - assert(this_instr->op.code == ENTER_EXECUTOR); - next_instr = this_instr; + assert(start->op.code == ENTER_EXECUTOR); + next_instr = start; this_instr[1].cache &= ((1 << OPTIMIZER_BITS_IN_COUNTER) - 1); } else { @@ -2363,18 +2370,23 @@ dummy_func( CHECK_EVAL_BREAKER(); PyCodeObject *code = _PyFrame_GetCode(frame); - _PyExecutorObject *executor = (_PyExecutorObject *)code->co_executors->executors[oparg&255]; - Py_INCREF(executor); - if (executor->execute == _PyUOpExecute) { - current_executor = (_PyUOpExecutorObject *)executor; + _PyExecutorObject *executor = code->co_executors->executors[oparg & 255]; + if (executor->vm_data.valid) { + Py_INCREF(executor); + current_executor = executor; GOTO_TIER_TWO(); } - next_instr = executor->execute(executor, frame, stack_pointer); - frame = tstate->current_frame; - if (next_instr == NULL) { - goto resume_with_error; + else { + /* ENTER_EXECUTOR will be the first code unit of the instruction */ + assert(oparg < 256); + code->co_executors->executors[oparg] = NULL; + opcode = this_instr->op.code = executor->vm_data.opcode; + this_instr->op.arg = executor->vm_data.oparg; + oparg = executor->vm_data.oparg; + Py_DECREF(executor); + next_instr = this_instr; + DISPATCH_GOTO(); } - stack_pointer = _PyFrame_GetStackPointer(frame); } replaced op(_POP_JUMP_IF_FALSE, (cond -- )) { @@ -2538,8 +2550,8 @@ dummy_func( next_instr[oparg].op.code == INSTRUMENTED_END_FOR); Py_DECREF(iter); STACK_SHRINK(1); - /* Jump forward oparg, then skip following END_FOR instruction */ - JUMPBY(oparg + 1); + /* Jump forward oparg, then skip following END_FOR and POP_TOP instruction */ + JUMPBY(oparg + 2); DISPATCH(); } // Common case: no jump, leave it to the code generator @@ -2587,8 +2599,8 @@ dummy_func( next_instr[oparg].op.code == INSTRUMENTED_END_FOR); STACK_SHRINK(1); Py_DECREF(iter); - /* Skip END_FOR */ - target = next_instr + oparg + 1; + /* Skip END_FOR and POP_TOP */ + target = next_instr + oparg + 2; } INSTRUMENTED_JUMP(this_instr, target, PY_MONITORING_EVENT_BRANCH); } @@ -2609,8 +2621,8 @@ dummy_func( } Py_DECREF(iter); STACK_SHRINK(1); - /* Jump forward oparg, then skip following END_FOR instruction */ - JUMPBY(oparg + 1); + /* Jump forward oparg, then skip following END_FOR and POP_TOP instructions */ + JUMPBY(oparg + 2); DISPATCH(); } } @@ -2655,8 +2667,8 @@ dummy_func( } Py_DECREF(iter); STACK_SHRINK(1); - /* Jump forward oparg, then skip following END_FOR instruction */ - JUMPBY(oparg + 1); + /* Jump forward oparg, then skip following END_FOR and POP_TOP instructions */ + JUMPBY(oparg + 2); DISPATCH(); } } @@ -2697,8 +2709,8 @@ dummy_func( if (r->len <= 0) { STACK_SHRINK(1); Py_DECREF(r); - // Jump over END_FOR instruction. - JUMPBY(oparg + 1); + // Jump over END_FOR and POP_TOP instructions. + JUMPBY(oparg + 2); DISPATCH(); } } @@ -2710,7 +2722,7 @@ dummy_func( DEOPT_IF(r->len <= 0); } - op(_ITER_NEXT_RANGE, (iter -- iter, next)) { + op(_ITER_NEXT_RANGE, (iter -- iter, next: &PYLONG_TYPE)) { _PyRangeIterObject *r = (_PyRangeIterObject *)iter; assert(Py_TYPE(r) == &PyRangeIter_Type); assert(r->len > 0); @@ -2868,13 +2880,13 @@ dummy_func( exc_info->exc_value = Py_NewRef(new_exc); } - op(_GUARD_DORV_VALUES_INST_ATTR_FROM_DICT, (owner -- owner)) { + op(_GUARD_DORV_VALUES_INST_ATTR_FROM_DICT, (owner -- owner: &GUARD_DORV_VALUES_INST_ATTR_FROM_DICT_TYPE)) { assert(Py_TYPE(owner)->tp_flags & Py_TPFLAGS_MANAGED_DICT); PyDictOrValues *dorv = _PyObject_DictOrValuesPointer(owner); DEOPT_IF(!_PyDictOrValues_IsValues(*dorv) && !_PyObject_MakeInstanceAttributesFromDict(owner, dorv)); } - op(_GUARD_KEYS_VERSION, (keys_version/2, owner -- owner)) { + op(_GUARD_KEYS_VERSION, (keys_version/2, owner -- owner: &(GUARD_KEYS_VERSION_TYPE + keys_version))) { PyTypeObject *owner_cls = Py_TYPE(owner); PyHeapTypeObject *owner_heap_type = (PyHeapTypeObject *)owner_cls; DEOPT_IF(owner_heap_type->ht_cached_keys->dk_version != keys_version); @@ -3089,7 +3101,7 @@ dummy_func( macro(CALL) = _SPECIALIZE_CALL + unused/2 + _CALL; - op(_CHECK_CALL_BOUND_METHOD_EXACT_ARGS, (callable, null, unused[oparg] -- callable, null, unused[oparg])) { + op(_CHECK_CALL_BOUND_METHOD_EXACT_ARGS, (callable, null, unused[oparg] -- callable: &PYMETHOD_TYPE, null: &NULL_TYPE, unused[oparg])) { DEOPT_IF(null != NULL); DEOPT_IF(Py_TYPE(callable) != &PyMethod_Type); } @@ -3107,7 +3119,7 @@ dummy_func( DEOPT_IF(tstate->interp->eval_frame); } - op(_CHECK_FUNCTION_EXACT_ARGS, (func_version/2, callable, self_or_null, unused[oparg] -- callable, self_or_null, unused[oparg])) { + op(_CHECK_FUNCTION_EXACT_ARGS, (func_version/2, callable, self_or_null, unused[oparg] -- callable: &(PYFUNCTION_TYPE_VERSION_TYPE + func_version), self_or_null, unused[oparg])) { DEOPT_IF(!PyFunction_Check(callable)); PyFunctionObject *func = (PyFunctionObject *)callable; DEOPT_IF(func->func_version != func_version); @@ -3122,7 +3134,7 @@ dummy_func( DEOPT_IF(tstate->py_recursion_remaining <= 1); } - op(_INIT_CALL_PY_EXACT_ARGS, (callable, self_or_null, args[oparg] -- new_frame: _PyInterpreterFrame*)) { + pure op(_INIT_CALL_PY_EXACT_ARGS, (callable, self_or_null, args[oparg] -- new_frame: _PyInterpreterFrame*)) { int argcount = oparg; if (self_or_null != NULL) { args--; @@ -3143,7 +3155,8 @@ dummy_func( // Write it out explicitly because it's subtly different. // Eventually this should be the only occurrence of this code. assert(tstate->interp->eval_frame == NULL); - STORE_SP(); + SYNC_SP(); + _PyFrame_SetStackPointer(frame, stack_pointer); new_frame->previous = frame; CALL_STAT_INC(inlined_py_calls); frame = tstate->current_frame = new_frame; @@ -3866,7 +3879,7 @@ dummy_func( ERROR_IF(res == NULL, error); } - inst(COPY, (bottom, unused[oparg-1] -- bottom, unused[oparg-1], top)) { + pure inst(COPY, (bottom, unused[oparg-1] -- bottom, unused[oparg-1], top)) { assert(oparg > 0); top = Py_NewRef(bottom); } @@ -3895,7 +3908,7 @@ dummy_func( macro(BINARY_OP) = _SPECIALIZE_BINARY_OP + _BINARY_OP; - inst(SWAP, (bottom, unused[oparg-2], top -- + pure inst(SWAP, (bottom, unused[oparg-2], top -- top, unused[oparg-2], bottom)) { assert(oparg >= 2); } @@ -4002,20 +4015,27 @@ dummy_func( ///////// Tier-2 only opcodes ///////// op (_GUARD_IS_TRUE_POP, (flag -- )) { - DEOPT_IF(Py_IsFalse(flag)); + SYNC_SP(); + DEOPT_IF(!Py_IsTrue(flag)); assert(Py_IsTrue(flag)); } op (_GUARD_IS_FALSE_POP, (flag -- )) { - DEOPT_IF(Py_IsTrue(flag)); + SYNC_SP(); + DEOPT_IF(!Py_IsFalse(flag)); assert(Py_IsFalse(flag)); } op (_GUARD_IS_NONE_POP, (val -- )) { - DEOPT_IF(!Py_IsNone(val)); + SYNC_SP(); + if (!Py_IsNone(val)) { + Py_DECREF(val); + DEOPT_IF(1); + } } op (_GUARD_IS_NOT_NONE_POP, (val -- )) { + SYNC_SP(); DEOPT_IF(Py_IsNone(val)); Py_DECREF(val); } @@ -4045,14 +4065,47 @@ dummy_func( DEOPT_IF(1); } - op(_INSERT, (unused[oparg], top -- top, unused[oparg])) { - // Inserts TOS at position specified by oparg; - memmove(&stack_pointer[-1 - oparg], &stack_pointer[-oparg], oparg * sizeof(stack_pointer[0])); + op(_CHECK_VALIDITY, (--)) { + TIER_TWO_ONLY + DEOPT_IF(!current_executor->vm_data.valid); + } + + op(_LOAD_CONST_INLINE, (ptr/4 -- value)) { + TIER_TWO_ONLY + value = Py_NewRef(ptr); } - op(_CHECK_VALIDITY, (--)) { + op(_LOAD_CONST_INLINE_BORROW, (ptr/4 -- value)) { + TIER_TWO_ONLY + value = ptr; + } + + op(_LOAD_CONST_INLINE_WITH_NULL, (ptr/4 -- value, null)) { + TIER_TWO_ONLY + value = Py_NewRef(ptr); + null = NULL; + } + + op(_LOAD_CONST_INLINE_BORROW_WITH_NULL, (ptr/4 -- value, null)) { + TIER_TWO_ONLY + value = ptr; + null = NULL; + } + + op(_CHECK_GLOBALS, (dict/4 -- )) { TIER_TWO_ONLY - DEOPT_IF(!current_executor->base.vm_data.valid); + DEOPT_IF(GLOBALS() != dict); + } + + op(_CHECK_BUILTINS, (dict/4 -- )) { + TIER_TWO_ONLY + DEOPT_IF(BUILTINS() != dict); + } + + /* Internal -- for testing executors */ + op(_INTERNAL_INCREMENT_OPT_COUNTER, (opt --)) { + _PyCounterOptimizerObject *exe = (_PyCounterOptimizerObject *)opt; + exe->count++; } diff --git a/Python/ceval.c b/Python/ceval.c index 1fea9747488102..4f208009086191 100644 --- a/Python/ceval.c +++ b/Python/ceval.c @@ -11,6 +11,7 @@ #include "pycore_function.h" #include "pycore_instruments.h" #include "pycore_intrinsics.h" +#include "pycore_jit.h" #include "pycore_long.h" // _PyLong_GetZero() #include "pycore_moduleobject.h" // PyModuleObject #include "pycore_object.h" // _PyObject_GC_TRACK() @@ -25,7 +26,6 @@ #include "pycore_tuple.h" // _PyTuple_ITEMS() #include "pycore_typeobject.h" // _PySuper_Lookup() #include "pycore_uop_ids.h" // Uops -#include "pycore_uops.h" // _PyUOpExecutorObject #include "pycore_pyerrors.h" #include "pycore_dict.h" @@ -739,7 +739,7 @@ _PyEval_EvalFrameDefault(PyThreadState *tstate, _PyInterpreterFrame *frame, int } /* State shared between Tier 1 and Tier 2 interpreter */ - _PyUOpExecutorObject *current_executor = NULL; + _PyExecutorObject *current_executor = NULL; /* Local "register" variables. * These are cached values from the frame and code object. */ @@ -956,9 +956,24 @@ _PyEval_EvalFrameDefault(PyThreadState *tstate, _PyInterpreterFrame *frame, int -// The Tier 2 interpreter is also here! +// Tier 2 is also here! enter_tier_two: +#ifdef _Py_JIT + + ; // ;) + jit_func jitted = current_executor->jit_code; + next_instr = jitted(frame, stack_pointer, tstate); + frame = tstate->current_frame; + Py_DECREF(current_executor); + if (next_instr == NULL) { + goto resume_with_error; + } + stack_pointer = _PyFrame_GetStackPointer(frame); + DISPATCH(); + +#else + #undef LOAD_IP #define LOAD_IP(UNUSED) (void)0 @@ -1074,6 +1089,8 @@ _PyEval_EvalFrameDefault(PyThreadState *tstate, _PyInterpreterFrame *frame, int Py_DECREF(current_executor); DISPATCH(); +#endif // _Py_JIT + } #if defined(__GNUC__) # pragma GCC diagnostic pop @@ -2111,6 +2128,9 @@ do_monitor_exc(PyThreadState *tstate, _PyInterpreterFrame *frame, _Py_CODEUNIT *instr, int event) { assert(event < _PY_MONITORING_UNGROUPED_EVENTS); + if (_PyFrame_GetCode(frame)->co_flags & CO_NO_MONITORING_EVENTS) { + return 0; + } PyObject *exc = PyErr_GetRaisedException(); assert(exc != NULL); int err = _Py_call_instrumentation_arg(tstate, event, frame, instr, exc); diff --git a/Python/ceval_gil.c b/Python/ceval_gil.c index d70abbc27606b4..deb9741291fca7 100644 --- a/Python/ceval_gil.c +++ b/Python/ceval_gil.c @@ -610,8 +610,16 @@ PyEval_SaveThread(void) void PyEval_RestoreThread(PyThreadState *tstate) { +#ifdef MS_WINDOWS + int err = GetLastError(); +#endif + _Py_EnsureTstateNotNULL(tstate); _PyThreadState_Attach(tstate); + +#ifdef MS_WINDOWS + SetLastError(err); +#endif } @@ -949,6 +957,15 @@ _Py_HandlePending(PyThreadState *tstate) { PyInterpreterState *interp = tstate->interp; + /* Stop-the-world */ + if (_Py_eval_breaker_bit_is_set(interp, _PY_EVAL_PLEASE_STOP_BIT)) { + _Py_set_eval_breaker_bit(interp, _PY_EVAL_PLEASE_STOP_BIT, 0); + _PyThreadState_Suspend(tstate); + + /* The attach blocks until the stop-the-world event is complete. */ + _PyThreadState_Attach(tstate); + } + /* Pending signals */ if (_Py_eval_breaker_bit_is_set(interp, _PY_SIGNALS_PENDING_BIT)) { if (handle_signals(tstate) != 0) { @@ -963,6 +980,14 @@ _Py_HandlePending(PyThreadState *tstate) } } +#ifdef Py_GIL_DISABLED + /* Objects with refcounts to merge */ + if (_Py_eval_breaker_bit_is_set(interp, _PY_EVAL_EXPLICIT_MERGE_BIT)) { + _Py_set_eval_breaker_bit(interp, _PY_EVAL_EXPLICIT_MERGE_BIT, 0); + _Py_brc_merge_refcounts(tstate); + } +#endif + /* GC scheduled to run */ if (_Py_eval_breaker_bit_is_set(interp, _PY_GC_SCHEDULED_BIT)) { _Py_set_eval_breaker_bit(interp, _PY_GC_SCHEDULED_BIT, 0); diff --git a/Python/ceval_macros.h b/Python/ceval_macros.h index a3606b17b71c62..c2550f53ad6eaa 100644 --- a/Python/ceval_macros.h +++ b/Python/ceval_macros.h @@ -382,9 +382,6 @@ static inline void _Py_LeaveRecursiveCallPy(PyThreadState *tstate) { /* There's no STORE_IP(), it's inlined by the code generator. */ -#define STORE_SP() \ -_PyFrame_SetStackPointer(frame, stack_pointer) - #define LOAD_SP() \ stack_pointer = _PyFrame_GetStackPointer(frame); diff --git a/Python/clinic/marshal.c.h b/Python/clinic/marshal.c.h index e6b0f1999a41c5..c19a3ed5050ed3 100644 --- a/Python/clinic/marshal.c.h +++ b/Python/clinic/marshal.c.h @@ -2,10 +2,14 @@ preserve [clinic start generated code]*/ -#include "pycore_modsupport.h" // _PyArg_CheckPositional() +#if defined(Py_BUILD_CORE) && !defined(Py_BUILD_CORE_MODULE) +# include "pycore_gc.h" // PyGC_Head +# include "pycore_runtime.h" // _Py_ID() +#endif +#include "pycore_modsupport.h" // _PyArg_UnpackKeywords() PyDoc_STRVAR(marshal_dump__doc__, -"dump($module, value, file, version=version, /)\n" +"dump($module, value, file, version=version, /, *, allow_code=True)\n" "--\n" "\n" "Write the value on the open file.\n" @@ -16,53 +20,95 @@ PyDoc_STRVAR(marshal_dump__doc__, " Must be a writeable binary file.\n" " version\n" " Indicates the data format that dump should use.\n" +" allow_code\n" +" Allow to write code objects.\n" "\n" "If the value has (or contains an object that has) an unsupported type, a\n" "ValueError exception is raised - but garbage data will also be written\n" "to the file. The object will not be properly read back by load()."); #define MARSHAL_DUMP_METHODDEF \ - {"dump", _PyCFunction_CAST(marshal_dump), METH_FASTCALL, marshal_dump__doc__}, + {"dump", _PyCFunction_CAST(marshal_dump), METH_FASTCALL|METH_KEYWORDS, marshal_dump__doc__}, static PyObject * marshal_dump_impl(PyObject *module, PyObject *value, PyObject *file, - int version); + int version, int allow_code); static PyObject * -marshal_dump(PyObject *module, PyObject *const *args, Py_ssize_t nargs) +marshal_dump(PyObject *module, PyObject *const *args, Py_ssize_t nargs, PyObject *kwnames) { PyObject *return_value = NULL; + #if defined(Py_BUILD_CORE) && !defined(Py_BUILD_CORE_MODULE) + + #define NUM_KEYWORDS 1 + static struct { + PyGC_Head _this_is_not_used; + PyObject_VAR_HEAD + PyObject *ob_item[NUM_KEYWORDS]; + } _kwtuple = { + .ob_base = PyVarObject_HEAD_INIT(&PyTuple_Type, NUM_KEYWORDS) + .ob_item = { &_Py_ID(allow_code), }, + }; + #undef NUM_KEYWORDS + #define KWTUPLE (&_kwtuple.ob_base.ob_base) + + #else // !Py_BUILD_CORE + # define KWTUPLE NULL + #endif // !Py_BUILD_CORE + + static const char * const _keywords[] = {"", "", "", "allow_code", NULL}; + static _PyArg_Parser _parser = { + .keywords = _keywords, + .fname = "dump", + .kwtuple = KWTUPLE, + }; + #undef KWTUPLE + PyObject *argsbuf[4]; + Py_ssize_t noptargs = nargs + (kwnames ? PyTuple_GET_SIZE(kwnames) : 0) - 2; PyObject *value; PyObject *file; int version = Py_MARSHAL_VERSION; + int allow_code = 1; - if (!_PyArg_CheckPositional("dump", nargs, 2, 3)) { + args = _PyArg_UnpackKeywords(args, nargs, NULL, kwnames, &_parser, 2, 3, 0, argsbuf); + if (!args) { goto exit; } value = args[0]; file = args[1]; if (nargs < 3) { - goto skip_optional; + goto skip_optional_posonly; } + noptargs--; version = PyLong_AsInt(args[2]); if (version == -1 && PyErr_Occurred()) { goto exit; } -skip_optional: - return_value = marshal_dump_impl(module, value, file, version); +skip_optional_posonly: + if (!noptargs) { + goto skip_optional_kwonly; + } + allow_code = PyObject_IsTrue(args[3]); + if (allow_code < 0) { + goto exit; + } +skip_optional_kwonly: + return_value = marshal_dump_impl(module, value, file, version, allow_code); exit: return return_value; } PyDoc_STRVAR(marshal_load__doc__, -"load($module, file, /)\n" +"load($module, file, /, *, allow_code=True)\n" "--\n" "\n" "Read one value from the open file and return it.\n" "\n" " file\n" " Must be readable binary file.\n" +" allow_code\n" +" Allow to load code objects.\n" "\n" "If no valid value is read (e.g. because the data has a different Python\n" "version\'s incompatible marshal format), raise EOFError, ValueError or\n" @@ -72,10 +118,66 @@ PyDoc_STRVAR(marshal_load__doc__, "dump(), load() will substitute None for the unmarshallable type."); #define MARSHAL_LOAD_METHODDEF \ - {"load", (PyCFunction)marshal_load, METH_O, marshal_load__doc__}, + {"load", _PyCFunction_CAST(marshal_load), METH_FASTCALL|METH_KEYWORDS, marshal_load__doc__}, + +static PyObject * +marshal_load_impl(PyObject *module, PyObject *file, int allow_code); + +static PyObject * +marshal_load(PyObject *module, PyObject *const *args, Py_ssize_t nargs, PyObject *kwnames) +{ + PyObject *return_value = NULL; + #if defined(Py_BUILD_CORE) && !defined(Py_BUILD_CORE_MODULE) + + #define NUM_KEYWORDS 1 + static struct { + PyGC_Head _this_is_not_used; + PyObject_VAR_HEAD + PyObject *ob_item[NUM_KEYWORDS]; + } _kwtuple = { + .ob_base = PyVarObject_HEAD_INIT(&PyTuple_Type, NUM_KEYWORDS) + .ob_item = { &_Py_ID(allow_code), }, + }; + #undef NUM_KEYWORDS + #define KWTUPLE (&_kwtuple.ob_base.ob_base) + + #else // !Py_BUILD_CORE + # define KWTUPLE NULL + #endif // !Py_BUILD_CORE + + static const char * const _keywords[] = {"", "allow_code", NULL}; + static _PyArg_Parser _parser = { + .keywords = _keywords, + .fname = "load", + .kwtuple = KWTUPLE, + }; + #undef KWTUPLE + PyObject *argsbuf[2]; + Py_ssize_t noptargs = nargs + (kwnames ? PyTuple_GET_SIZE(kwnames) : 0) - 1; + PyObject *file; + int allow_code = 1; + + args = _PyArg_UnpackKeywords(args, nargs, NULL, kwnames, &_parser, 1, 1, 0, argsbuf); + if (!args) { + goto exit; + } + file = args[0]; + if (!noptargs) { + goto skip_optional_kwonly; + } + allow_code = PyObject_IsTrue(args[1]); + if (allow_code < 0) { + goto exit; + } +skip_optional_kwonly: + return_value = marshal_load_impl(module, file, allow_code); + +exit: + return return_value; +} PyDoc_STRVAR(marshal_dumps__doc__, -"dumps($module, value, version=version, /)\n" +"dumps($module, value, version=version, /, *, allow_code=True)\n" "--\n" "\n" "Return the bytes object that would be written to a file by dump(value, file).\n" @@ -84,66 +186,150 @@ PyDoc_STRVAR(marshal_dumps__doc__, " Must be a supported type.\n" " version\n" " Indicates the data format that dumps should use.\n" +" allow_code\n" +" Allow to write code objects.\n" "\n" "Raise a ValueError exception if value has (or contains an object that has) an\n" "unsupported type."); #define MARSHAL_DUMPS_METHODDEF \ - {"dumps", _PyCFunction_CAST(marshal_dumps), METH_FASTCALL, marshal_dumps__doc__}, + {"dumps", _PyCFunction_CAST(marshal_dumps), METH_FASTCALL|METH_KEYWORDS, marshal_dumps__doc__}, static PyObject * -marshal_dumps_impl(PyObject *module, PyObject *value, int version); +marshal_dumps_impl(PyObject *module, PyObject *value, int version, + int allow_code); static PyObject * -marshal_dumps(PyObject *module, PyObject *const *args, Py_ssize_t nargs) +marshal_dumps(PyObject *module, PyObject *const *args, Py_ssize_t nargs, PyObject *kwnames) { PyObject *return_value = NULL; + #if defined(Py_BUILD_CORE) && !defined(Py_BUILD_CORE_MODULE) + + #define NUM_KEYWORDS 1 + static struct { + PyGC_Head _this_is_not_used; + PyObject_VAR_HEAD + PyObject *ob_item[NUM_KEYWORDS]; + } _kwtuple = { + .ob_base = PyVarObject_HEAD_INIT(&PyTuple_Type, NUM_KEYWORDS) + .ob_item = { &_Py_ID(allow_code), }, + }; + #undef NUM_KEYWORDS + #define KWTUPLE (&_kwtuple.ob_base.ob_base) + + #else // !Py_BUILD_CORE + # define KWTUPLE NULL + #endif // !Py_BUILD_CORE + + static const char * const _keywords[] = {"", "", "allow_code", NULL}; + static _PyArg_Parser _parser = { + .keywords = _keywords, + .fname = "dumps", + .kwtuple = KWTUPLE, + }; + #undef KWTUPLE + PyObject *argsbuf[3]; + Py_ssize_t noptargs = nargs + (kwnames ? PyTuple_GET_SIZE(kwnames) : 0) - 1; PyObject *value; int version = Py_MARSHAL_VERSION; + int allow_code = 1; - if (!_PyArg_CheckPositional("dumps", nargs, 1, 2)) { + args = _PyArg_UnpackKeywords(args, nargs, NULL, kwnames, &_parser, 1, 2, 0, argsbuf); + if (!args) { goto exit; } value = args[0]; if (nargs < 2) { - goto skip_optional; + goto skip_optional_posonly; } + noptargs--; version = PyLong_AsInt(args[1]); if (version == -1 && PyErr_Occurred()) { goto exit; } -skip_optional: - return_value = marshal_dumps_impl(module, value, version); +skip_optional_posonly: + if (!noptargs) { + goto skip_optional_kwonly; + } + allow_code = PyObject_IsTrue(args[2]); + if (allow_code < 0) { + goto exit; + } +skip_optional_kwonly: + return_value = marshal_dumps_impl(module, value, version, allow_code); exit: return return_value; } PyDoc_STRVAR(marshal_loads__doc__, -"loads($module, bytes, /)\n" +"loads($module, bytes, /, *, allow_code=True)\n" "--\n" "\n" "Convert the bytes-like object to a value.\n" "\n" +" allow_code\n" +" Allow to load code objects.\n" +"\n" "If no valid value is found, raise EOFError, ValueError or TypeError. Extra\n" "bytes in the input are ignored."); #define MARSHAL_LOADS_METHODDEF \ - {"loads", (PyCFunction)marshal_loads, METH_O, marshal_loads__doc__}, + {"loads", _PyCFunction_CAST(marshal_loads), METH_FASTCALL|METH_KEYWORDS, marshal_loads__doc__}, static PyObject * -marshal_loads_impl(PyObject *module, Py_buffer *bytes); +marshal_loads_impl(PyObject *module, Py_buffer *bytes, int allow_code); static PyObject * -marshal_loads(PyObject *module, PyObject *arg) +marshal_loads(PyObject *module, PyObject *const *args, Py_ssize_t nargs, PyObject *kwnames) { PyObject *return_value = NULL; + #if defined(Py_BUILD_CORE) && !defined(Py_BUILD_CORE_MODULE) + + #define NUM_KEYWORDS 1 + static struct { + PyGC_Head _this_is_not_used; + PyObject_VAR_HEAD + PyObject *ob_item[NUM_KEYWORDS]; + } _kwtuple = { + .ob_base = PyVarObject_HEAD_INIT(&PyTuple_Type, NUM_KEYWORDS) + .ob_item = { &_Py_ID(allow_code), }, + }; + #undef NUM_KEYWORDS + #define KWTUPLE (&_kwtuple.ob_base.ob_base) + + #else // !Py_BUILD_CORE + # define KWTUPLE NULL + #endif // !Py_BUILD_CORE + + static const char * const _keywords[] = {"", "allow_code", NULL}; + static _PyArg_Parser _parser = { + .keywords = _keywords, + .fname = "loads", + .kwtuple = KWTUPLE, + }; + #undef KWTUPLE + PyObject *argsbuf[2]; + Py_ssize_t noptargs = nargs + (kwnames ? PyTuple_GET_SIZE(kwnames) : 0) - 1; Py_buffer bytes = {NULL, NULL}; + int allow_code = 1; - if (PyObject_GetBuffer(arg, &bytes, PyBUF_SIMPLE) != 0) { + args = _PyArg_UnpackKeywords(args, nargs, NULL, kwnames, &_parser, 1, 1, 0, argsbuf); + if (!args) { + goto exit; + } + if (PyObject_GetBuffer(args[0], &bytes, PyBUF_SIMPLE) != 0) { + goto exit; + } + if (!noptargs) { + goto skip_optional_kwonly; + } + allow_code = PyObject_IsTrue(args[1]); + if (allow_code < 0) { goto exit; } - return_value = marshal_loads_impl(module, &bytes); +skip_optional_kwonly: + return_value = marshal_loads_impl(module, &bytes, allow_code); exit: /* Cleanup for bytes */ @@ -153,4 +339,4 @@ marshal_loads(PyObject *module, PyObject *arg) return return_value; } -/*[clinic end generated code: output=92d2d47aac9128ee input=a9049054013a1b77]*/ +/*[clinic end generated code: output=1575b9a3ae48ad3d input=a9049054013a1b77]*/ diff --git a/Python/compile.c b/Python/compile.c index 65ac05ad58d4dd..15e5cf38a37b97 100644 --- a/Python/compile.c +++ b/Python/compile.c @@ -160,7 +160,7 @@ _PyCompile_EnsureArrayLargeEnough(int idx, void **array, int *alloc, if (idx >= new_alloc) { new_alloc = idx + default_alloc; } - arr = PyObject_Calloc(new_alloc, item_size); + arr = PyMem_Calloc(new_alloc, item_size); if (arr == NULL) { PyErr_NoMemory(); return ERROR; @@ -181,7 +181,7 @@ _PyCompile_EnsureArrayLargeEnough(int idx, void **array, int *alloc, } assert(newsize > 0); - void *tmp = PyObject_Realloc(arr, newsize); + void *tmp = PyMem_Realloc(arr, newsize); if (tmp == NULL) { PyErr_NoMemory(); return ERROR; @@ -282,10 +282,10 @@ instr_sequence_insert_instruction(instr_sequence *seq, int pos, static void instr_sequence_fini(instr_sequence *seq) { - PyObject_Free(seq->s_labelmap); + PyMem_Free(seq->s_labelmap); seq->s_labelmap = NULL; - PyObject_Free(seq->s_instrs); + PyMem_Free(seq->s_instrs); seq->s_instrs = NULL; } @@ -690,7 +690,7 @@ compiler_unit_free(struct compiler_unit *u) Py_CLEAR(u->u_metadata.u_cellvars); Py_CLEAR(u->u_metadata.u_fasthidden); Py_CLEAR(u->u_private); - PyObject_Free(u); + PyMem_Free(u); } static int @@ -958,14 +958,15 @@ merge_consts_recursive(PyObject *const_cache, PyObject *o) return NULL; } - // t is borrowed reference - PyObject *t = PyDict_SetDefault(const_cache, key, key); - if (t != key) { - // o is registered in const_cache. Just use it. - Py_XINCREF(t); + PyObject *t; + int res = PyDict_SetDefaultRef(const_cache, key, key, &t); + if (res != 0) { + // o was not inserted into const_cache. t is either the existing value + // or NULL (on error). Py_DECREF(key); return t; } + Py_DECREF(t); // We registered o in const_cache. // When o is a tuple or frozenset, we want to merge its @@ -1262,8 +1263,7 @@ compiler_enter_scope(struct compiler *c, identifier name, struct compiler_unit *u; - u = (struct compiler_unit *)PyObject_Calloc(1, sizeof( - struct compiler_unit)); + u = (struct compiler_unit *)PyMem_Calloc(1, sizeof(struct compiler_unit)); if (!u) { PyErr_NoMemory(); return ERROR; @@ -2897,7 +2897,7 @@ compiler_jump_if(struct compiler *c, location loc, compiler_jump_if(c, loc, e->v.IfExp.test, next2, 0)); RETURN_IF_ERROR( compiler_jump_if(c, loc, e->v.IfExp.body, next, cond)); - ADDOP_JUMP(c, NO_LOCATION, JUMP, end); + ADDOP_JUMP(c, NO_LOCATION, JUMP_NO_INTERRUPT, end); USE_LABEL(c, next2); RETURN_IF_ERROR( @@ -2926,12 +2926,12 @@ compiler_jump_if(struct compiler *c, location loc, ADDOP(c, LOC(e), TO_BOOL); ADDOP_JUMP(c, LOC(e), cond ? POP_JUMP_IF_TRUE : POP_JUMP_IF_FALSE, next); NEW_JUMP_TARGET_LABEL(c, end); - ADDOP_JUMP(c, NO_LOCATION, JUMP, end); + ADDOP_JUMP(c, NO_LOCATION, JUMP_NO_INTERRUPT, end); USE_LABEL(c, cleanup); ADDOP(c, LOC(e), POP_TOP); if (!cond) { - ADDOP_JUMP(c, NO_LOCATION, JUMP, next); + ADDOP_JUMP(c, NO_LOCATION, JUMP_NO_INTERRUPT, next); } USE_LABEL(c, end); @@ -2963,7 +2963,7 @@ compiler_ifexp(struct compiler *c, expr_ty e) compiler_jump_if(c, LOC(e), e->v.IfExp.test, next, 0)); VISIT(c, expr, e->v.IfExp.body); - ADDOP_JUMP(c, NO_LOCATION, JUMP, end); + ADDOP_JUMP(c, NO_LOCATION, JUMP_NO_INTERRUPT, end); USE_LABEL(c, next); VISIT(c, expr, e->v.IfExp.orelse); @@ -3041,7 +3041,7 @@ compiler_if(struct compiler *c, stmt_ty s) VISIT_SEQ(c, stmt, s->v.If.body); if (asdl_seq_LEN(s->v.If.orelse)) { - ADDOP_JUMP(c, NO_LOCATION, JUMP, end); + ADDOP_JUMP(c, NO_LOCATION, JUMP_NO_INTERRUPT, end); USE_LABEL(c, next); VISIT_SEQ(c, stmt, s->v.If.orelse); @@ -3075,7 +3075,12 @@ compiler_for(struct compiler *c, stmt_ty s) ADDOP_JUMP(c, NO_LOCATION, JUMP, start); USE_LABEL(c, cleanup); + /* It is important for instrumentation that the `END_FOR` comes first. + * Iteration over a generator will jump to the first of these instructions, + * but a non-generator will jump to a later instruction. + */ ADDOP(c, NO_LOCATION, END_FOR); + ADDOP(c, NO_LOCATION, POP_TOP); compiler_pop_fblock(c, FOR_LOOP, start); @@ -3294,7 +3299,7 @@ compiler_try_finally(struct compiler *c, stmt_ty s) compiler_pop_fblock(c, FINALLY_TRY, body); VISIT_SEQ(c, stmt, s->v.Try.finalbody); - ADDOP_JUMP(c, NO_LOCATION, JUMP, exit); + ADDOP_JUMP(c, NO_LOCATION, JUMP_NO_INTERRUPT, exit); /* `finally` block */ USE_LABEL(c, end); @@ -3344,7 +3349,7 @@ compiler_try_star_finally(struct compiler *c, stmt_ty s) compiler_pop_fblock(c, FINALLY_TRY, body); VISIT_SEQ(c, stmt, s->v.TryStar.finalbody); - ADDOP_JUMP(c, NO_LOCATION, JUMP, exit); + ADDOP_JUMP(c, NO_LOCATION, JUMP_NO_INTERRUPT, exit); /* `finally` block */ USE_LABEL(c, end); @@ -3419,7 +3424,7 @@ compiler_try_except(struct compiler *c, stmt_ty s) if (s->v.Try.orelse && asdl_seq_LEN(s->v.Try.orelse)) { VISIT_SEQ(c, stmt, s->v.Try.orelse); } - ADDOP_JUMP(c, NO_LOCATION, JUMP, end); + ADDOP_JUMP(c, NO_LOCATION, JUMP_NO_INTERRUPT, end); n = asdl_seq_LEN(s->v.Try.handlers); USE_LABEL(c, except); @@ -3483,7 +3488,7 @@ compiler_try_except(struct compiler *c, stmt_ty s) compiler_nameop(c, NO_LOCATION, handler->v.ExceptHandler.name, Store)); RETURN_IF_ERROR( compiler_nameop(c, NO_LOCATION, handler->v.ExceptHandler.name, Del)); - ADDOP_JUMP(c, NO_LOCATION, JUMP, end); + ADDOP_JUMP(c, NO_LOCATION, JUMP_NO_INTERRUPT, end); /* except: */ USE_LABEL(c, cleanup_end); @@ -3511,7 +3516,7 @@ compiler_try_except(struct compiler *c, stmt_ty s) compiler_pop_fblock(c, HANDLER_CLEANUP, cleanup_body); ADDOP(c, NO_LOCATION, POP_BLOCK); ADDOP(c, NO_LOCATION, POP_EXCEPT); - ADDOP_JUMP(c, NO_LOCATION, JUMP, end); + ADDOP_JUMP(c, NO_LOCATION, JUMP_NO_INTERRUPT, end); } USE_LABEL(c, except); @@ -3599,7 +3604,7 @@ compiler_try_star_except(struct compiler *c, stmt_ty s) VISIT_SEQ(c, stmt, s->v.TryStar.body); compiler_pop_fblock(c, TRY_EXCEPT, body); ADDOP(c, NO_LOCATION, POP_BLOCK); - ADDOP_JUMP(c, NO_LOCATION, JUMP, orelse); + ADDOP_JUMP(c, NO_LOCATION, JUMP_NO_INTERRUPT, orelse); Py_ssize_t n = asdl_seq_LEN(s->v.TryStar.handlers); USE_LABEL(c, except); @@ -3681,7 +3686,7 @@ compiler_try_star_except(struct compiler *c, stmt_ty s) RETURN_IF_ERROR( compiler_nameop(c, NO_LOCATION, handler->v.ExceptHandler.name, Del)); } - ADDOP_JUMP(c, NO_LOCATION, JUMP, except); + ADDOP_JUMP(c, NO_LOCATION, JUMP_NO_INTERRUPT, except); /* except: */ USE_LABEL(c, cleanup_end); @@ -3698,11 +3703,11 @@ compiler_try_star_except(struct compiler *c, stmt_ty s) /* add exception raised to the res list */ ADDOP_I(c, NO_LOCATION, LIST_APPEND, 3); // exc ADDOP(c, NO_LOCATION, POP_TOP); // lasti - ADDOP_JUMP(c, NO_LOCATION, JUMP, except_with_error); + ADDOP_JUMP(c, NO_LOCATION, JUMP_NO_INTERRUPT, except_with_error); USE_LABEL(c, except); ADDOP(c, NO_LOCATION, NOP); // to hold a propagated location info - ADDOP_JUMP(c, NO_LOCATION, JUMP, except_with_error); + ADDOP_JUMP(c, NO_LOCATION, JUMP_NO_INTERRUPT, except_with_error); USE_LABEL(c, no_match); ADDOP(c, loc, POP_TOP); // match (None) @@ -3712,7 +3717,7 @@ compiler_try_star_except(struct compiler *c, stmt_ty s) if (i == n - 1) { /* Add exc to the list (if not None it's the unhandled part of the EG) */ ADDOP_I(c, NO_LOCATION, LIST_APPEND, 1); - ADDOP_JUMP(c, NO_LOCATION, JUMP, reraise_star); + ADDOP_JUMP(c, NO_LOCATION, JUMP_NO_INTERRUPT, reraise_star); } } /* artificial */ @@ -3728,7 +3733,7 @@ compiler_try_star_except(struct compiler *c, stmt_ty s) ADDOP(c, NO_LOCATION, POP_TOP); ADDOP(c, NO_LOCATION, POP_BLOCK); ADDOP(c, NO_LOCATION, POP_EXCEPT); - ADDOP_JUMP(c, NO_LOCATION, JUMP, end); + ADDOP_JUMP(c, NO_LOCATION, JUMP_NO_INTERRUPT, end); USE_LABEL(c, reraise); ADDOP(c, NO_LOCATION, POP_BLOCK); @@ -4626,7 +4631,7 @@ compiler_compare(struct compiler *c, expr_ty e) VISIT(c, expr, (expr_ty)asdl_seq_GET(e->v.Compare.comparators, n)); ADDOP_COMPARE(c, loc, asdl_seq_GET(e->v.Compare.ops, n)); NEW_JUMP_TARGET_LABEL(c, end); - ADDOP_JUMP(c, NO_LOCATION, JUMP, end); + ADDOP_JUMP(c, NO_LOCATION, JUMP_NO_INTERRUPT, end); USE_LABEL(c, cleanup); ADDOP_I(c, loc, SWAP, 2); @@ -5390,7 +5395,12 @@ compiler_sync_comprehension_generator(struct compiler *c, location loc, ADDOP_JUMP(c, elt_loc, JUMP, start); USE_LABEL(c, anchor); + /* It is important for instrumentation that the `END_FOR` comes first. + * Iteration over a generator will jump to the first of these instructions, + * but a non-generator will jump to a later instruction. + */ ADDOP(c, NO_LOCATION, END_FOR); + ADDOP(c, NO_LOCATION, POP_TOP); } return SUCCESS; @@ -5668,7 +5678,7 @@ pop_inlined_comprehension_state(struct compiler *c, location loc, } if (state.pushed_locals) { ADDOP(c, NO_LOCATION, POP_BLOCK); - ADDOP_JUMP(c, NO_LOCATION, JUMP, state.end); + ADDOP_JUMP(c, NO_LOCATION, JUMP_NO_INTERRUPT, state.end); // cleanup from an exception inside the comprehension USE_LABEL(c, state.cleanup); @@ -5916,7 +5926,7 @@ compiler_with_except_finish(struct compiler *c, jump_target_label cleanup) { ADDOP(c, NO_LOCATION, POP_TOP); ADDOP(c, NO_LOCATION, POP_TOP); NEW_JUMP_TARGET_LABEL(c, exit); - ADDOP_JUMP(c, NO_LOCATION, JUMP, exit); + ADDOP_JUMP(c, NO_LOCATION, JUMP_NO_INTERRUPT, exit); USE_LABEL(c, cleanup); POP_EXCEPT_AND_RERAISE(c, NO_LOCATION); @@ -6647,7 +6657,7 @@ ensure_fail_pop(struct compiler *c, pattern_context *pc, Py_ssize_t n) return SUCCESS; } Py_ssize_t needed = sizeof(jump_target_label) * size; - jump_target_label *resized = PyObject_Realloc(pc->fail_pop, needed); + jump_target_label *resized = PyMem_Realloc(pc->fail_pop, needed); if (resized == NULL) { PyErr_NoMemory(); return ERROR; @@ -6686,13 +6696,13 @@ emit_and_reset_fail_pop(struct compiler *c, location loc, USE_LABEL(c, pc->fail_pop[pc->fail_pop_size]); if (codegen_addop_noarg(INSTR_SEQUENCE(c), POP_TOP, loc) < 0) { pc->fail_pop_size = 0; - PyObject_Free(pc->fail_pop); + PyMem_Free(pc->fail_pop); pc->fail_pop = NULL; return ERROR; } } USE_LABEL(c, pc->fail_pop[0]); - PyObject_Free(pc->fail_pop); + PyMem_Free(pc->fail_pop); pc->fail_pop = NULL; return SUCCESS; } @@ -7196,7 +7206,7 @@ compiler_pattern_or(struct compiler *c, pattern_ty p, pattern_context *pc) Py_DECREF(pc->stores); *pc = old_pc; Py_INCREF(pc->stores); - // Need to NULL this for the PyObject_Free call in the error block. + // Need to NULL this for the PyMem_Free call in the error block. old_pc.fail_pop = NULL; // No match. Pop the remaining copy of the subject and fail: if (codegen_addop_noarg(INSTR_SEQUENCE(c), POP_TOP, LOC(p)) < 0 || @@ -7242,7 +7252,7 @@ compiler_pattern_or(struct compiler *c, pattern_ty p, pattern_context *pc) diff: compiler_error(c, LOC(p), "alternative patterns bind different names"); error: - PyObject_Free(old_pc.fail_pop); + PyMem_Free(old_pc.fail_pop); Py_DECREF(old_pc.stores); Py_XDECREF(control); return ERROR; @@ -7410,7 +7420,7 @@ compiler_match_inner(struct compiler *c, stmt_ty s, pattern_context *pc) ADDOP(c, LOC(m->pattern), POP_TOP); } VISIT_SEQ(c, stmt, m->body); - ADDOP_JUMP(c, NO_LOCATION, JUMP, end); + ADDOP_JUMP(c, NO_LOCATION, JUMP_NO_INTERRUPT, end); // If the pattern fails to match, we want the line number of the // cleanup to be associated with the failed pattern, not the last line // of the body @@ -7443,7 +7453,7 @@ compiler_match(struct compiler *c, stmt_ty s) pattern_context pc; pc.fail_pop = NULL; int result = compiler_match_inner(c, s, &pc); - PyObject_Free(pc.fail_pop); + PyMem_Free(pc.fail_pop); return result; } @@ -7518,22 +7528,26 @@ _PyCompile_ConstCacheMergeOne(PyObject *const_cache, PyObject **obj) return ERROR; } - // t is borrowed reference - PyObject *t = PyDict_SetDefault(const_cache, key, key); + PyObject *t; + int res = PyDict_SetDefaultRef(const_cache, key, key, &t); Py_DECREF(key); - if (t == NULL) { + if (res < 0) { return ERROR; } - if (t == key) { // obj is new constant. + if (res == 0) { // inserted: obj is new constant. + Py_DECREF(t); return SUCCESS; } if (PyTuple_CheckExact(t)) { - // t is still borrowed reference - t = PyTuple_GET_ITEM(t, 1); + PyObject *item = PyTuple_GET_ITEM(t, 1); + Py_SETREF(*obj, Py_NewRef(item)); + Py_DECREF(t); + } + else { + Py_SETREF(*obj, t); } - Py_SETREF(*obj, Py_NewRef(t)); return SUCCESS; } diff --git a/Python/condvar.h b/Python/condvar.h index d54db94f2c871d..dcabed6d55928c 100644 --- a/Python/condvar.h +++ b/Python/condvar.h @@ -260,13 +260,13 @@ PyMUTEX_UNLOCK(PyMUTEX_T *cs) return 0; } - Py_LOCAL_INLINE(int) PyCOND_INIT(PyCOND_T *cv) { InitializeConditionVariable(cv); return 0; } + Py_LOCAL_INLINE(int) PyCOND_FINI(PyCOND_T *cv) { @@ -279,27 +279,32 @@ PyCOND_WAIT(PyCOND_T *cv, PyMUTEX_T *cs) return SleepConditionVariableSRW(cv, cs, INFINITE, 0) ? 0 : -1; } -/* This implementation makes no distinction about timeouts. Signal - * 2 to indicate that we don't know. - */ +/* return 0 for success, 1 on timeout, -1 on error */ Py_LOCAL_INLINE(int) PyCOND_TIMEDWAIT(PyCOND_T *cv, PyMUTEX_T *cs, long long us) { - return SleepConditionVariableSRW(cv, cs, (DWORD)(us/1000), 0) ? 2 : -1; + BOOL success = SleepConditionVariableSRW(cv, cs, (DWORD)(us/1000), 0); + if (!success) { + if (GetLastError() == ERROR_TIMEOUT) { + return 1; + } + return -1; + } + return 0; } Py_LOCAL_INLINE(int) PyCOND_SIGNAL(PyCOND_T *cv) { - WakeConditionVariable(cv); - return 0; + WakeConditionVariable(cv); + return 0; } Py_LOCAL_INLINE(int) PyCOND_BROADCAST(PyCOND_T *cv) { - WakeAllConditionVariable(cv); - return 0; + WakeAllConditionVariable(cv); + return 0; } diff --git a/Python/context.c b/Python/context.c index c94c014219d0e4..e44fef705c36e0 100644 --- a/Python/context.c +++ b/Python/context.c @@ -64,12 +64,12 @@ static int contextvar_del(PyContextVar *var); -#if PyContext_MAXFREELIST > 0 +#ifdef WITH_FREELISTS static struct _Py_context_state * get_context_state(void) { - PyInterpreterState *interp = _PyInterpreterState_GET(); - return &interp->context; + _PyFreeListState *state = _PyFreeListState_GET(); + return &state->contexts; } #endif @@ -340,13 +340,9 @@ static inline PyContext * _context_alloc(void) { PyContext *ctx; -#if PyContext_MAXFREELIST > 0 +#ifdef WITH_FREELISTS struct _Py_context_state *state = get_context_state(); -#ifdef Py_DEBUG - // _context_alloc() must not be called after _PyContext_Fini() - assert(state->numfree != -1); -#endif - if (state->numfree) { + if (state->numfree > 0) { state->numfree--; ctx = state->freelist; state->freelist = (PyContext *)ctx->ctx_weakreflist; @@ -471,13 +467,9 @@ context_tp_dealloc(PyContext *self) } (void)context_tp_clear(self); -#if PyContext_MAXFREELIST > 0 +#ifdef WITH_FREELISTS struct _Py_context_state *state = get_context_state(); -#ifdef Py_DEBUG - // _context_alloc() must not be called after _PyContext_Fini() - assert(state->numfree != -1); -#endif - if (state->numfree < PyContext_MAXFREELIST) { + if (state->numfree >= 0 && state->numfree < PyContext_MAXFREELIST) { state->numfree++; self->ctx_weakreflist = (PyObject *)state->freelist; state->freelist = self; @@ -1275,27 +1267,19 @@ get_token_missing(void) void -_PyContext_ClearFreeList(PyInterpreterState *interp) +_PyContext_ClearFreeList(_PyFreeListState *freelist_state, int is_finalization) { -#if PyContext_MAXFREELIST > 0 - struct _Py_context_state *state = &interp->context; - for (; state->numfree; state->numfree--) { +#ifdef WITH_FREELISTS + struct _Py_context_state *state = &freelist_state->contexts; + for (; state->numfree > 0; state->numfree--) { PyContext *ctx = state->freelist; state->freelist = (PyContext *)ctx->ctx_weakreflist; ctx->ctx_weakreflist = NULL; PyObject_GC_Del(ctx); } -#endif -} - - -void -_PyContext_Fini(PyInterpreterState *interp) -{ - _PyContext_ClearFreeList(interp); -#if defined(Py_DEBUG) && PyContext_MAXFREELIST > 0 - struct _Py_context_state *state = &interp->context; - state->numfree = -1; + if (is_finalization) { + state->numfree = -1; + } #endif } diff --git a/Python/executor_cases.c.h b/Python/executor_cases.c.h index 14fb3a05a9f674..2d914b82dbf88f 100644 --- a/Python/executor_cases.c.h +++ b/Python/executor_cases.c.h @@ -3318,35 +3318,38 @@ case _GUARD_IS_TRUE_POP: { PyObject *flag; flag = stack_pointer[-1]; - if (Py_IsFalse(flag)) goto deoptimize; - assert(Py_IsTrue(flag)); stack_pointer += -1; + if (!Py_IsTrue(flag)) goto deoptimize; + assert(Py_IsTrue(flag)); break; } case _GUARD_IS_FALSE_POP: { PyObject *flag; flag = stack_pointer[-1]; - if (Py_IsTrue(flag)) goto deoptimize; - assert(Py_IsFalse(flag)); stack_pointer += -1; + if (!Py_IsFalse(flag)) goto deoptimize; + assert(Py_IsFalse(flag)); break; } case _GUARD_IS_NONE_POP: { PyObject *val; val = stack_pointer[-1]; - if (!Py_IsNone(val)) goto deoptimize; stack_pointer += -1; + if (!Py_IsNone(val)) { + Py_DECREF(val); + if (1) goto deoptimize; + } break; } case _GUARD_IS_NOT_NONE_POP: { PyObject *val; val = stack_pointer[-1]; + stack_pointer += -1; if (Py_IsNone(val)) goto deoptimize; Py_DECREF(val); - stack_pointer += -1; break; } @@ -3381,19 +3384,78 @@ break; } - case _INSERT: { - PyObject *top; - oparg = CURRENT_OPARG(); - top = stack_pointer[-1]; - // Inserts TOS at position specified by oparg; - memmove(&stack_pointer[-1 - oparg], &stack_pointer[-oparg], oparg * sizeof(stack_pointer[0])); - stack_pointer[-1 - oparg] = top; + case _CHECK_VALIDITY: { + TIER_TWO_ONLY + if (!current_executor->vm_data.valid) goto deoptimize; break; } - case _CHECK_VALIDITY: { + case _LOAD_CONST_INLINE: { + PyObject *value; + PyObject *ptr = (PyObject *)CURRENT_OPERAND(); TIER_TWO_ONLY - if (!current_executor->base.vm_data.valid) goto deoptimize; + value = Py_NewRef(ptr); + stack_pointer[0] = value; + stack_pointer += 1; + break; + } + + case _LOAD_CONST_INLINE_BORROW: { + PyObject *value; + PyObject *ptr = (PyObject *)CURRENT_OPERAND(); + TIER_TWO_ONLY + value = ptr; + stack_pointer[0] = value; + stack_pointer += 1; + break; + } + + case _LOAD_CONST_INLINE_WITH_NULL: { + PyObject *value; + PyObject *null; + PyObject *ptr = (PyObject *)CURRENT_OPERAND(); + TIER_TWO_ONLY + value = Py_NewRef(ptr); + null = NULL; + stack_pointer[0] = value; + stack_pointer[1] = null; + stack_pointer += 2; + break; + } + + case _LOAD_CONST_INLINE_BORROW_WITH_NULL: { + PyObject *value; + PyObject *null; + PyObject *ptr = (PyObject *)CURRENT_OPERAND(); + TIER_TWO_ONLY + value = ptr; + null = NULL; + stack_pointer[0] = value; + stack_pointer[1] = null; + stack_pointer += 2; + break; + } + + case _CHECK_GLOBALS: { + PyObject *dict = (PyObject *)CURRENT_OPERAND(); + TIER_TWO_ONLY + if (GLOBALS() != dict) goto deoptimize; + break; + } + + case _CHECK_BUILTINS: { + PyObject *dict = (PyObject *)CURRENT_OPERAND(); + TIER_TWO_ONLY + if (BUILTINS() != dict) goto deoptimize; + break; + } + + case _INTERNAL_INCREMENT_OPT_COUNTER: { + PyObject *opt; + opt = stack_pointer[-1]; + _PyCounterOptimizerObject *exe = (_PyCounterOptimizerObject *)opt; + exe->count++; + stack_pointer += -1; break; } diff --git a/Python/flowgraph.c b/Python/flowgraph.c index 0e6ffbc32e1526..1a648edf0880c0 100644 --- a/Python/flowgraph.c +++ b/Python/flowgraph.c @@ -145,6 +145,16 @@ basicblock_next_instr(basicblock *b) return b->b_iused++; } +static cfg_instr * +basicblock_last_instr(const basicblock *b) { + assert(b->b_iused >= 0); + if (b->b_iused > 0) { + assert(b->b_instr != NULL); + return &b->b_instr[b->b_iused - 1]; + } + return NULL; +} + /* Allocate a new block and return a pointer to it. Returns NULL on error. */ @@ -152,7 +162,7 @@ basicblock_next_instr(basicblock *b) static basicblock * cfg_builder_new_block(cfg_builder *g) { - basicblock *b = (basicblock *)PyObject_Calloc(1, sizeof(basicblock)); + basicblock *b = (basicblock *)PyMem_Calloc(1, sizeof(basicblock)); if (b == NULL) { PyErr_NoMemory(); return NULL; @@ -185,29 +195,35 @@ basicblock_addop(basicblock *b, int opcode, int oparg, location loc) return SUCCESS; } +static int +basicblock_add_jump(basicblock *b, int opcode, basicblock *target, location loc) +{ + cfg_instr *last = basicblock_last_instr(b); + if (last && is_jump(last)) { + return ERROR; + } + + RETURN_IF_ERROR( + basicblock_addop(b, opcode, target->b_label.id, loc)); + last = basicblock_last_instr(b); + assert(last && last->i_opcode == opcode); + last->i_target = target; + return SUCCESS; +} + static inline int -basicblock_append_instructions(basicblock *target, basicblock *source) +basicblock_append_instructions(basicblock *to, basicblock *from) { - for (int i = 0; i < source->b_iused; i++) { - int n = basicblock_next_instr(target); + for (int i = 0; i < from->b_iused; i++) { + int n = basicblock_next_instr(to); if (n < 0) { return ERROR; } - target->b_instr[n] = source->b_instr[i]; + to->b_instr[n] = from->b_instr[i]; } return SUCCESS; } -static cfg_instr * -basicblock_last_instr(const basicblock *b) { - assert(b->b_iused >= 0); - if (b->b_iused > 0) { - assert(b->b_instr != NULL); - return &b->b_instr[b->b_iused - 1]; - } - return NULL; -} - static inline int basicblock_nofallthrough(const basicblock *b) { cfg_instr *last = basicblock_last_instr(b); @@ -276,9 +292,9 @@ static void dump_basicblock(const basicblock *b) { const char *b_return = basicblock_returns(b) ? "return " : ""; - fprintf(stderr, "%d: [EH=%d CLD=%d WRM=%d NO_FT=%d %p] used: %d, depth: %d, %s\n", + fprintf(stderr, "%d: [EH=%d CLD=%d WRM=%d NO_FT=%d %p] used: %d, depth: %d, preds: %d %s\n", b->b_label.id, b->b_except_handler, b->b_cold, b->b_warm, BB_NO_FALLTHROUGH(b), b, b->b_iused, - b->b_startdepth, b_return); + b->b_startdepth, b->b_predecessors, b_return); if (b->b_instr) { int i; for (i = 0; i < b->b_iused; i++) { @@ -316,6 +332,16 @@ basicblock_exits_scope(const basicblock *b) { return last && IS_SCOPE_EXIT_OPCODE(last->i_opcode); } +static inline int +basicblock_has_eval_break(const basicblock *b) { + for (int i = 0; i < b->b_iused; i++) { + if (OPCODE_HAS_EVAL_BREAK(b->b_instr[i].i_opcode)) { + return true; + } + } + return false; +} + static bool cfg_builder_current_block_is_terminated(cfg_builder *g) { @@ -411,10 +437,10 @@ _PyCfgBuilder_Free(cfg_builder *g) basicblock *b = g->g_block_list; while (b != NULL) { if (b->b_instr) { - PyObject_Free((void *)b->b_instr); + PyMem_Free((void *)b->b_instr); } basicblock *next = b->b_list; - PyObject_Free((void *)b); + PyMem_Free((void *)b); b = next; } PyMem_Free(g); @@ -449,39 +475,57 @@ _PyCfgBuilder_Addop(cfg_builder *g, int opcode, int oparg, location loc) } +static basicblock * +next_nonempty_block(basicblock *b) +{ + while (b && b->b_iused == 0) { + b = b->b_next; + } + return b; +} + /***** debugging helpers *****/ #ifndef NDEBUG -static int remove_redundant_nops(basicblock *bb); +static int remove_redundant_nops(cfg_builder *g); static bool no_redundant_nops(cfg_builder *g) { - for (basicblock *b = g->g_entryblock; b != NULL; b = b->b_next) { - if (remove_redundant_nops(b) != 0) { - return false; - } + if (remove_redundant_nops(g) != 0) { + return false; } return true; } static bool -no_empty_basic_blocks(cfg_builder *g) { +no_redundant_jumps(cfg_builder *g) { for (basicblock *b = g->g_entryblock; b != NULL; b = b->b_next) { - if (b->b_iused == 0) { - return false; + cfg_instr *last = basicblock_last_instr(b); + if (last != NULL) { + if (IS_UNCONDITIONAL_JUMP_OPCODE(last->i_opcode)) { + basicblock *next = next_nonempty_block(b->b_next); + basicblock *jump_target = next_nonempty_block(last->i_target); + if (jump_target == next) { + assert(next); + if (last->i_loc.lineno == next->b_instr[0].i_loc.lineno) { + assert(0); + return false; + } + } + } } } return true; } static bool -no_redundant_jumps(cfg_builder *g) { - for (basicblock *b = g->g_entryblock; b != NULL; b = b->b_next) { - cfg_instr *last = basicblock_last_instr(b); - if (last != NULL) { - if (IS_UNCONDITIONAL_JUMP_OPCODE(last->i_opcode)) { - assert(last->i_target != b->b_next); - if (last->i_target == b->b_next) { +all_exits_have_lineno(basicblock *entryblock) { + for (basicblock *b = entryblock; b != NULL; b = b->b_next) { + for (int i = 0; i < b->b_iused; i++) { + cfg_instr *instr = &b->b_instr[i]; + if (instr->i_opcode == RETURN_VALUE) { + if (instr->i_loc.lineno < 0) { + assert(0); return false; } } @@ -489,7 +533,6 @@ no_redundant_jumps(cfg_builder *g) { } return true; } - #endif /***** CFG preprocessing (jump targets and exceptions) *****/ @@ -532,8 +575,8 @@ normalize_jumps_in_block(cfg_builder *g, basicblock *b) { if (backwards_jump == NULL) { return ERROR; } - basicblock_addop(backwards_jump, JUMP, target->b_label.id, last->i_loc); - backwards_jump->b_instr[0].i_target = target; + RETURN_IF_ERROR( + basicblock_add_jump(backwards_jump, JUMP, target, last->i_loc)); last->i_opcode = reversed_opcode; last->i_target = b->b_next; @@ -860,6 +903,7 @@ label_exception_targets(basicblock *entryblock) { } else if (instr->i_opcode == POP_BLOCK) { handler = pop_except_block(except_stack); + INSTR_SET_OP0(instr, NOP); } else if (is_jump(instr)) { instr->i_except = handler; @@ -926,7 +970,10 @@ label_exception_targets(basicblock *entryblock) { /***** CFG optimizations *****/ static int -mark_reachable(basicblock *entryblock) { +remove_unreachable(basicblock *entryblock) { + for (basicblock *b = entryblock; b != NULL; b = b->b_next) { + b->b_predecessors = 0; + } basicblock **stack = make_cfg_traversal_stack(entryblock); if (stack == NULL) { return ERROR; @@ -958,47 +1005,19 @@ mark_reachable(basicblock *entryblock) { } } PyMem_Free(stack); - return SUCCESS; -} -static void -eliminate_empty_basic_blocks(cfg_builder *g) { - /* Eliminate empty blocks */ - for (basicblock *b = g->g_entryblock; b != NULL; b = b->b_next) { - basicblock *next = b->b_next; - while (next && next->b_iused == 0) { - next = next->b_next; - } - b->b_next = next; - } - while(g->g_entryblock && g->g_entryblock->b_iused == 0) { - g->g_entryblock = g->g_entryblock->b_next; - } - int next_lbl = get_max_label(g->g_entryblock) + 1; - for (basicblock *b = g->g_entryblock; b != NULL; b = b->b_next) { - assert(b->b_iused > 0); - for (int i = 0; i < b->b_iused; i++) { - cfg_instr *instr = &b->b_instr[i]; - if (HAS_TARGET(instr->i_opcode)) { - basicblock *target = instr->i_target; - while (target->b_iused == 0) { - target = target->b_next; - } - if (instr->i_target != target) { - if (!IS_LABEL(target->b_label)) { - target->b_label.id = next_lbl++; - } - instr->i_target = target; - instr->i_oparg = target->b_label.id; - } - assert(instr->i_target && instr->i_target->b_iused > 0); - } - } + /* Delete unreachable instructions */ + for (basicblock *b = entryblock; b != NULL; b = b->b_next) { + if (b->b_predecessors == 0) { + b->b_iused = 0; + b->b_except_handler = 0; + } } + return SUCCESS; } static int -remove_redundant_nops(basicblock *bb) { +basicblock_remove_redundant_nops(basicblock *bb) { /* Remove NOPs when legal to do so. */ int dest = 0; int prev_lineno = -1; @@ -1025,10 +1044,7 @@ remove_redundant_nops(basicblock *bb) { } } else { - basicblock* next = bb->b_next; - while (next && next->b_iused == 0) { - next = next->b_next; - } + basicblock *next = next_nonempty_block(bb->b_next); /* or if last instruction in BB and next BB has same line number */ if (next) { location next_loc = NO_LOCATION; @@ -1060,6 +1076,17 @@ remove_redundant_nops(basicblock *bb) { return num_removed; } +static int +remove_redundant_nops(cfg_builder *g) { + int changes = 0; + for (basicblock *b = g->g_entryblock; b != NULL; b = b->b_next) { + int change = basicblock_remove_redundant_nops(b); + RETURN_IF_ERROR(change); + changes += change; + } + return changes; +} + static int remove_redundant_nops_and_pairs(basicblock *entryblock) { @@ -1070,7 +1097,7 @@ remove_redundant_nops_and_pairs(basicblock *entryblock) cfg_instr *prev_instr = NULL; cfg_instr *instr = NULL; for (basicblock *b = entryblock; b != NULL; b = b->b_next) { - remove_redundant_nops(b); + RETURN_IF_ERROR(basicblock_remove_redundant_nops(b)); if (IS_LABEL(b->b_label)) { /* this block is a jump target, forget instr */ instr = NULL; @@ -1110,50 +1137,54 @@ remove_redundant_jumps(cfg_builder *g) { * non-empty block reached through normal flow control is the target * of that jump. If it is, then the jump instruction is redundant and * can be deleted. + * + * Return the number of changes applied, or -1 on error. */ - assert(no_empty_basic_blocks(g)); - - bool remove_empty_blocks = false; + int changes = 0; for (basicblock *b = g->g_entryblock; b != NULL; b = b->b_next) { cfg_instr *last = basicblock_last_instr(b); - assert(last != NULL); + if (last == NULL) { + continue; + } assert(!IS_ASSEMBLER_OPCODE(last->i_opcode)); if (IS_UNCONDITIONAL_JUMP_OPCODE(last->i_opcode)) { - if (last->i_target == NULL) { + basicblock* jump_target = next_nonempty_block(last->i_target); + if (jump_target == NULL) { PyErr_SetString(PyExc_SystemError, "jump with NULL target"); return ERROR; } - if (last->i_target == b->b_next) { - assert(b->b_next->b_iused); - if (last->i_loc.lineno == NO_LOCATION.lineno) { - b->b_iused--; - if (b->b_iused == 0) { - remove_empty_blocks = true; - } - } - else { - INSTR_SET_OP0(last, NOP); - } + basicblock *next = next_nonempty_block(b->b_next); + if (jump_target == next) { + changes++; + INSTR_SET_OP0(last, NOP); } } } - if (remove_empty_blocks) { - eliminate_empty_basic_blocks(g); + + return changes; +} + +static inline bool +basicblock_has_no_lineno(basicblock *b) { + for (int i = 0; i < b->b_iused; i++) { + if (b->b_instr[i].i_loc.lineno >= 0) { + return false; + } } - assert(no_empty_basic_blocks(g)); - return SUCCESS; + return true; } /* Maximum size of basic block that should be copied in optimizer */ #define MAX_COPY_SIZE 4 -/* If this block ends with an unconditional jump to a small exit block, then +/* If this block ends with an unconditional jump to a small exit block or + * a block that has no line numbers (and no fallthrough), then * remove the jump and extend this block with the target. * Returns 1 if extended, 0 if no change, and -1 on error. */ static int -inline_small_exit_blocks(basicblock *bb) { +basicblock_inline_small_or_no_lineno_blocks(basicblock *bb) { cfg_instr *last = basicblock_last_instr(bb); if (last == NULL) { return 0; @@ -1162,29 +1193,67 @@ inline_small_exit_blocks(basicblock *bb) { return 0; } basicblock *target = last->i_target; - if (basicblock_exits_scope(target) && target->b_iused <= MAX_COPY_SIZE) { + bool small_exit_block = (basicblock_exits_scope(target) && + target->b_iused <= MAX_COPY_SIZE); + bool no_lineno_no_fallthrough = (basicblock_has_no_lineno(target) && + !BB_HAS_FALLTHROUGH(target)); + if (small_exit_block || no_lineno_no_fallthrough) { + assert(is_jump(last)); + int removed_jump_opcode = last->i_opcode; INSTR_SET_OP0(last, NOP); RETURN_IF_ERROR(basicblock_append_instructions(bb, target)); + if (no_lineno_no_fallthrough) { + last = basicblock_last_instr(bb); + if (IS_UNCONDITIONAL_JUMP_OPCODE(last->i_opcode) && + removed_jump_opcode == JUMP) + { + /* Make sure we don't lose eval breaker checks */ + last->i_opcode = JUMP; + } + } + target->b_predecessors--; return 1; } return 0; } +static int +inline_small_or_no_lineno_blocks(basicblock *entryblock) { + bool changes; + do { + changes = false; + for (basicblock *b = entryblock; b != NULL; b = b->b_next) { + int res = basicblock_inline_small_or_no_lineno_blocks(b); + RETURN_IF_ERROR(res); + if (res) { + changes = true; + } + } + } while(changes); /* every change removes a jump, ensuring convergence */ + return changes; +} + // Attempt to eliminate jumps to jumps by updating inst to jump to // target->i_target using the provided opcode. Return whether or not the // optimization was successful. static bool -jump_thread(cfg_instr *inst, cfg_instr *target, int opcode) +jump_thread(basicblock *bb, cfg_instr *inst, cfg_instr *target, int opcode) { assert(is_jump(inst)); assert(is_jump(target)); + assert(inst == basicblock_last_instr(bb)); // bpo-45773: If inst->i_target == target->i_target, then nothing actually // changes (and we fall into an infinite loop): - if ((inst->i_loc.lineno == target->i_loc.lineno || target->i_loc.lineno == -1) && - inst->i_target != target->i_target) - { - inst->i_target = target->i_target; - inst->i_opcode = opcode; + if (inst->i_target != target->i_target) { + /* Change inst to NOP and append a jump to target->i_target. The + * NOP will be removed later if it's not needed for the lineno. + */ + INSTR_SET_OP0(inst, NOP); + + RETURN_IF_ERROR( + basicblock_add_jump( + bb, opcode, target->i_target, target->i_loc)); + return true; } return false; @@ -1477,16 +1546,12 @@ apply_static_swaps(basicblock *block, int i) } static int -optimize_basic_block(PyObject *const_cache, basicblock *bb, PyObject *consts) +basicblock_optimize_load_const(PyObject *const_cache, basicblock *bb, PyObject *consts) { assert(PyDict_CheckExact(const_cache)); assert(PyList_CheckExact(consts)); - cfg_instr nop; - INSTR_SET_OP0(&nop, NOP); - cfg_instr *target = &nop; int opcode = 0; int oparg = 0; - int nextop = 0; for (int i = 0; i < bb->b_iused; i++) { cfg_instr *inst = &bb->b_instr[i]; bool is_copy_of_load_const = (opcode == LOAD_CONST && @@ -1495,118 +1560,148 @@ optimize_basic_block(PyObject *const_cache, basicblock *bb, PyObject *consts) if (! is_copy_of_load_const) { opcode = inst->i_opcode; oparg = inst->i_oparg; - if (HAS_TARGET(opcode)) { - assert(inst->i_target->b_iused > 0); - target = &inst->i_target->b_instr[0]; - assert(!IS_ASSEMBLER_OPCODE(target->i_opcode)); - } - else { - target = &nop; - } } - nextop = i+1 < bb->b_iused ? bb->b_instr[i+1].i_opcode : 0; assert(!IS_ASSEMBLER_OPCODE(opcode)); - switch (opcode) { - /* Remove LOAD_CONST const; conditional jump */ - case LOAD_CONST: + if (opcode != LOAD_CONST) { + continue; + } + int nextop = i+1 < bb->b_iused ? bb->b_instr[i+1].i_opcode : 0; + switch(nextop) { + case POP_JUMP_IF_FALSE: + case POP_JUMP_IF_TRUE: { - PyObject* cnt; - int is_true; - int jump_if_true; - switch(nextop) { - case POP_JUMP_IF_FALSE: - case POP_JUMP_IF_TRUE: - cnt = get_const_value(opcode, oparg, consts); - if (cnt == NULL) { - goto error; - } - is_true = PyObject_IsTrue(cnt); - Py_DECREF(cnt); - if (is_true == -1) { - goto error; - } - INSTR_SET_OP0(inst, NOP); - jump_if_true = nextop == POP_JUMP_IF_TRUE; - if (is_true == jump_if_true) { - bb->b_instr[i+1].i_opcode = JUMP; - } - else { - INSTR_SET_OP0(&bb->b_instr[i + 1], NOP); - } - break; - case IS_OP: - // Fold to POP_JUMP_IF_NONE: - // - LOAD_CONST(None) IS_OP(0) POP_JUMP_IF_TRUE - // - LOAD_CONST(None) IS_OP(1) POP_JUMP_IF_FALSE - // - LOAD_CONST(None) IS_OP(0) TO_BOOL POP_JUMP_IF_TRUE - // - LOAD_CONST(None) IS_OP(1) TO_BOOL POP_JUMP_IF_FALSE - // Fold to POP_JUMP_IF_NOT_NONE: - // - LOAD_CONST(None) IS_OP(0) POP_JUMP_IF_FALSE - // - LOAD_CONST(None) IS_OP(1) POP_JUMP_IF_TRUE - // - LOAD_CONST(None) IS_OP(0) TO_BOOL POP_JUMP_IF_FALSE - // - LOAD_CONST(None) IS_OP(1) TO_BOOL POP_JUMP_IF_TRUE - cnt = get_const_value(opcode, oparg, consts); - if (cnt == NULL) { - goto error; - } - if (!Py_IsNone(cnt)) { - Py_DECREF(cnt); - break; - } - if (bb->b_iused <= i + 2) { - break; - } - cfg_instr *is_instr = &bb->b_instr[i + 1]; - cfg_instr *jump_instr = &bb->b_instr[i + 2]; - // Get rid of TO_BOOL regardless: - if (jump_instr->i_opcode == TO_BOOL) { - INSTR_SET_OP0(jump_instr, NOP); - if (bb->b_iused <= i + 3) { - break; - } - jump_instr = &bb->b_instr[i + 3]; - } - bool invert = is_instr->i_oparg; - if (jump_instr->i_opcode == POP_JUMP_IF_FALSE) { - invert = !invert; - } - else if (jump_instr->i_opcode != POP_JUMP_IF_TRUE) { - break; - } - INSTR_SET_OP0(inst, NOP); - INSTR_SET_OP0(is_instr, NOP); - jump_instr->i_opcode = invert ? POP_JUMP_IF_NOT_NONE - : POP_JUMP_IF_NONE; - break; - case RETURN_VALUE: - INSTR_SET_OP0(inst, NOP); - INSTR_SET_OP1(&bb->b_instr[++i], RETURN_CONST, oparg); - break; - case TO_BOOL: - cnt = get_const_value(opcode, oparg, consts); - if (cnt == NULL) { - goto error; - } - is_true = PyObject_IsTrue(cnt); - Py_DECREF(cnt); - if (is_true == -1) { - goto error; - } - cnt = PyBool_FromLong(is_true); - int index = add_const(cnt, consts, const_cache); - if (index < 0) { - return ERROR; - } - INSTR_SET_OP0(inst, NOP); - INSTR_SET_OP1(&bb->b_instr[i + 1], LOAD_CONST, index); + /* Remove LOAD_CONST const; conditional jump */ + PyObject* cnt = get_const_value(opcode, oparg, consts); + if (cnt == NULL) { + return ERROR; + } + int is_true = PyObject_IsTrue(cnt); + Py_DECREF(cnt); + if (is_true == -1) { + return ERROR; + } + INSTR_SET_OP0(inst, NOP); + int jump_if_true = nextop == POP_JUMP_IF_TRUE; + if (is_true == jump_if_true) { + bb->b_instr[i+1].i_opcode = JUMP; + } + else { + INSTR_SET_OP0(&bb->b_instr[i + 1], NOP); + } + break; + } + case IS_OP: + { + // Fold to POP_JUMP_IF_NONE: + // - LOAD_CONST(None) IS_OP(0) POP_JUMP_IF_TRUE + // - LOAD_CONST(None) IS_OP(1) POP_JUMP_IF_FALSE + // - LOAD_CONST(None) IS_OP(0) TO_BOOL POP_JUMP_IF_TRUE + // - LOAD_CONST(None) IS_OP(1) TO_BOOL POP_JUMP_IF_FALSE + // Fold to POP_JUMP_IF_NOT_NONE: + // - LOAD_CONST(None) IS_OP(0) POP_JUMP_IF_FALSE + // - LOAD_CONST(None) IS_OP(1) POP_JUMP_IF_TRUE + // - LOAD_CONST(None) IS_OP(0) TO_BOOL POP_JUMP_IF_FALSE + // - LOAD_CONST(None) IS_OP(1) TO_BOOL POP_JUMP_IF_TRUE + PyObject *cnt = get_const_value(opcode, oparg, consts); + if (cnt == NULL) { + return ERROR; + } + if (!Py_IsNone(cnt)) { + Py_DECREF(cnt); + break; + } + if (bb->b_iused <= i + 2) { + break; + } + cfg_instr *is_instr = &bb->b_instr[i + 1]; + cfg_instr *jump_instr = &bb->b_instr[i + 2]; + // Get rid of TO_BOOL regardless: + if (jump_instr->i_opcode == TO_BOOL) { + INSTR_SET_OP0(jump_instr, NOP); + if (bb->b_iused <= i + 3) { break; + } + jump_instr = &bb->b_instr[i + 3]; + } + bool invert = is_instr->i_oparg; + if (jump_instr->i_opcode == POP_JUMP_IF_FALSE) { + invert = !invert; + } + else if (jump_instr->i_opcode != POP_JUMP_IF_TRUE) { + break; } + INSTR_SET_OP0(inst, NOP); + INSTR_SET_OP0(is_instr, NOP); + jump_instr->i_opcode = invert ? POP_JUMP_IF_NOT_NONE + : POP_JUMP_IF_NONE; break; } - /* Try to fold tuples of constants. - Skip over BUILD_TUPLE(1) UNPACK_SEQUENCE(1). - Replace BUILD_TUPLE(2) UNPACK_SEQUENCE(2) with SWAP(2). - Replace BUILD_TUPLE(3) UNPACK_SEQUENCE(3) with SWAP(3). */ + case RETURN_VALUE: + { + INSTR_SET_OP0(inst, NOP); + INSTR_SET_OP1(&bb->b_instr[++i], RETURN_CONST, oparg); + break; + } + case TO_BOOL: + { + PyObject *cnt = get_const_value(opcode, oparg, consts); + if (cnt == NULL) { + return ERROR; + } + int is_true = PyObject_IsTrue(cnt); + Py_DECREF(cnt); + if (is_true == -1) { + return ERROR; + } + cnt = PyBool_FromLong(is_true); + int index = add_const(cnt, consts, const_cache); + if (index < 0) { + return ERROR; + } + INSTR_SET_OP0(inst, NOP); + INSTR_SET_OP1(&bb->b_instr[i + 1], LOAD_CONST, index); + break; + } + } + } + return SUCCESS; +} + +static int +optimize_load_const(PyObject *const_cache, cfg_builder *g, PyObject *consts) { + for (basicblock *b = g->g_entryblock; b != NULL; b = b->b_next) { + RETURN_IF_ERROR(basicblock_optimize_load_const(const_cache, b, consts)); + } + return SUCCESS; +} + +static int +optimize_basic_block(PyObject *const_cache, basicblock *bb, PyObject *consts) +{ + assert(PyDict_CheckExact(const_cache)); + assert(PyList_CheckExact(consts)); + cfg_instr nop; + INSTR_SET_OP0(&nop, NOP); + for (int i = 0; i < bb->b_iused; i++) { + cfg_instr *inst = &bb->b_instr[i]; + cfg_instr *target; + int opcode = inst->i_opcode; + int oparg = inst->i_oparg; + if (HAS_TARGET(opcode)) { + assert(inst->i_target->b_iused > 0); + target = &inst->i_target->b_instr[0]; + assert(!IS_ASSEMBLER_OPCODE(target->i_opcode)); + } + else { + target = &nop; + } + int nextop = i+1 < bb->b_iused ? bb->b_instr[i+1].i_opcode : 0; + assert(!IS_ASSEMBLER_OPCODE(opcode)); + switch (opcode) { + /* Try to fold tuples of constants. + Skip over BUILD_TUPLE(1) UNPACK_SEQUENCE(1). + Replace BUILD_TUPLE(2) UNPACK_SEQUENCE(2) with SWAP(2). + Replace BUILD_TUPLE(3) UNPACK_SEQUENCE(3) with SWAP(3). */ case BUILD_TUPLE: if (nextop == UNPACK_SEQUENCE && oparg == bb->b_instr[i+1].i_oparg) { switch(oparg) { @@ -1631,25 +1726,30 @@ optimize_basic_block(PyObject *const_cache, basicblock *bb, PyObject *consts) case POP_JUMP_IF_NONE: switch (target->i_opcode) { case JUMP: - i -= jump_thread(inst, target, inst->i_opcode); + i -= jump_thread(bb, inst, target, inst->i_opcode); } break; case POP_JUMP_IF_FALSE: switch (target->i_opcode) { case JUMP: - i -= jump_thread(inst, target, POP_JUMP_IF_FALSE); + i -= jump_thread(bb, inst, target, POP_JUMP_IF_FALSE); } break; case POP_JUMP_IF_TRUE: switch (target->i_opcode) { case JUMP: - i -= jump_thread(inst, target, POP_JUMP_IF_TRUE); + i -= jump_thread(bb, inst, target, POP_JUMP_IF_TRUE); } break; case JUMP: + case JUMP_NO_INTERRUPT: switch (target->i_opcode) { case JUMP: - i -= jump_thread(inst, target, JUMP); + i -= jump_thread(bb, inst, target, JUMP); + continue; + case JUMP_NO_INTERRUPT: + i -= jump_thread(bb, inst, target, opcode); + continue; } break; case FOR_ITER: @@ -1660,7 +1760,7 @@ optimize_basic_block(PyObject *const_cache, basicblock *bb, PyObject *consts) * of FOR_ITER. */ /* - i -= jump_thread(inst, target, FOR_ITER); + i -= jump_thread(bb, inst, target, FOR_ITER); */ } break; @@ -1716,9 +1816,6 @@ optimize_basic_block(PyObject *const_cache, basicblock *bb, PyObject *consts) continue; } break; - default: - /* All OPCODE_HAS_CONST opcodes should be handled with LOAD_CONST */ - assert (!OPCODE_HAS_CONST(inst->i_opcode)); } } @@ -1736,6 +1833,7 @@ optimize_basic_block(PyObject *const_cache, basicblock *bb, PyObject *consts) return ERROR; } +static int resolve_line_numbers(cfg_builder *g, int firstlineno); /* Perform optimizations on a control flow graph. The consts object should still be in list form to allow new constants @@ -1745,37 +1843,31 @@ optimize_basic_block(PyObject *const_cache, basicblock *bb, PyObject *consts) NOPs. Later those NOPs are removed. */ static int -optimize_cfg(cfg_builder *g, PyObject *consts, PyObject *const_cache) +optimize_cfg(cfg_builder *g, PyObject *consts, PyObject *const_cache, int firstlineno) { assert(PyDict_CheckExact(const_cache)); RETURN_IF_ERROR(check_cfg(g)); - eliminate_empty_basic_blocks(g); - for (basicblock *b = g->g_entryblock; b != NULL; b = b->b_next) { - RETURN_IF_ERROR(inline_small_exit_blocks(b)); - } - assert(no_empty_basic_blocks(g)); + RETURN_IF_ERROR(inline_small_or_no_lineno_blocks(g->g_entryblock)); + RETURN_IF_ERROR(remove_unreachable(g->g_entryblock)); + RETURN_IF_ERROR(resolve_line_numbers(g, firstlineno)); + RETURN_IF_ERROR(optimize_load_const(const_cache, g, consts)); for (basicblock *b = g->g_entryblock; b != NULL; b = b->b_next) { RETURN_IF_ERROR(optimize_basic_block(const_cache, b, consts)); - assert(b->b_predecessors == 0); } RETURN_IF_ERROR(remove_redundant_nops_and_pairs(g->g_entryblock)); - for (basicblock *b = g->g_entryblock; b != NULL; b = b->b_next) { - RETURN_IF_ERROR(inline_small_exit_blocks(b)); - } - RETURN_IF_ERROR(mark_reachable(g->g_entryblock)); + RETURN_IF_ERROR(remove_unreachable(g->g_entryblock)); - /* Delete unreachable instructions */ - for (basicblock *b = g->g_entryblock; b != NULL; b = b->b_next) { - if (b->b_predecessors == 0) { - b->b_iused = 0; - } - } - for (basicblock *b = g->g_entryblock; b != NULL; b = b->b_next) { - remove_redundant_nops(b); - } - eliminate_empty_basic_blocks(g); - assert(no_redundant_nops(g)); - RETURN_IF_ERROR(remove_redundant_jumps(g)); + int removed_nops, removed_jumps; + do { + /* Convergence is guaranteed because the number of + * redundant jumps and nops only decreases. + */ + removed_nops = remove_redundant_nops(g); + RETURN_IF_ERROR(removed_nops); + removed_jumps = remove_redundant_jumps(g); + RETURN_IF_ERROR(removed_jumps); + } while(removed_nops + removed_jumps > 0); + assert(no_redundant_jumps(g)); return SUCCESS; } @@ -1795,7 +1887,7 @@ make_super_instruction(cfg_instr *inst1, cfg_instr *inst2, int super_op) INSTR_SET_OP0(inst2, NOP); } -static void +static int insert_superinstructions(cfg_builder *g) { for (basicblock *b = g->g_entryblock; b != NULL; b = b->b_next) { @@ -1822,11 +1914,9 @@ insert_superinstructions(cfg_builder *g) } } } - for (basicblock *b = g->g_entryblock; b != NULL; b = b->b_next) { - remove_redundant_nops(b); - } - eliminate_empty_basic_blocks(g); + int res = remove_redundant_nops(g); assert(no_redundant_nops(g)); + return res; } // helper functions for add_checks_for_loads_of_unknown_variables @@ -2192,7 +2282,13 @@ push_cold_blocks_to_end(cfg_builder *g) { if (!IS_LABEL(b->b_next->b_label)) { b->b_next->b_label.id = next_lbl++; } - basicblock_addop(explicit_jump, JUMP, b->b_next->b_label.id, NO_LOCATION); + cfg_instr *prev_instr = basicblock_last_instr(b); + // b cannot be empty because at the end of an exception handler + // there is always a POP_EXCEPT + RERAISE/RETURN + assert(prev_instr); + + basicblock_addop(explicit_jump, JUMP_NO_INTERRUPT, b->b_next->b_label.id, + prev_instr->i_loc); explicit_jump->b_cold = 1; explicit_jump->b_next = b->b_next; b->b_next = explicit_jump; @@ -2249,13 +2345,14 @@ push_cold_blocks_to_end(cfg_builder *g) { return SUCCESS; } -static void -convert_pseudo_ops(basicblock *entryblock) +static int +convert_pseudo_ops(cfg_builder *g) { + basicblock *entryblock = g->g_entryblock; for (basicblock *b = entryblock; b != NULL; b = b->b_next) { for (int i = 0; i < b->b_iused; i++) { cfg_instr *instr = &b->b_instr[i]; - if (is_block_push(instr) || instr->i_opcode == POP_BLOCK) { + if (is_block_push(instr)) { INSTR_SET_OP0(instr, NOP); } else if (instr->i_opcode == LOAD_CLOSURE) { @@ -2268,22 +2365,17 @@ convert_pseudo_ops(basicblock *entryblock) } } } - for (basicblock *b = entryblock; b != NULL; b = b->b_next) { - remove_redundant_nops(b); - } + return remove_redundant_nops(g); } static inline bool -is_exit_without_lineno(basicblock *b) { - if (!basicblock_exits_scope(b)) { - return false; +is_exit_or_eval_check_without_lineno(basicblock *b) { + if (basicblock_exits_scope(b) || basicblock_has_eval_break(b)) { + return basicblock_has_no_lineno(b); } - for (int i = 0; i < b->b_iused; i++) { - if (b->b_instr[i].i_loc.lineno >= 0) { - return false; - } + else { + return false; } - return true; } @@ -2299,8 +2391,6 @@ is_exit_without_lineno(basicblock *b) { static int duplicate_exits_without_lineno(cfg_builder *g) { - assert(no_empty_basic_blocks(g)); - int next_lbl = get_max_label(g->g_entryblock) + 1; /* Copy all exit blocks without line number that are targets of a jump. @@ -2308,10 +2398,12 @@ duplicate_exits_without_lineno(cfg_builder *g) basicblock *entryblock = g->g_entryblock; for (basicblock *b = entryblock; b != NULL; b = b->b_next) { cfg_instr *last = basicblock_last_instr(b); - assert(last != NULL); + if (last == NULL) { + continue; + } if (is_jump(last)) { - basicblock *target = last->i_target; - if (is_exit_without_lineno(target) && target->b_predecessors > 1) { + basicblock *target = next_nonempty_block(last->i_target); + if (is_exit_or_eval_check_without_lineno(target) && target->b_predecessors > 1) { basicblock *new_target = copy_basicblock(g, target); if (new_target == NULL) { return ERROR; @@ -2331,7 +2423,7 @@ duplicate_exits_without_lineno(cfg_builder *g) * fall through, and thus can only have a single predecessor */ for (basicblock *b = entryblock; b != NULL; b = b->b_next) { if (BB_HAS_FALLTHROUGH(b) && b->b_next && b->b_iused > 0) { - if (is_exit_without_lineno(b->b_next)) { + if (is_exit_or_eval_check_without_lineno(b->b_next)) { cfg_instr *last = basicblock_last_instr(b); assert(last != NULL); b->b_next->b_instr[0].i_loc = last->i_loc; @@ -2367,9 +2459,10 @@ propagate_line_numbers(basicblock *entryblock) { } } if (BB_HAS_FALLTHROUGH(b) && b->b_next->b_predecessors == 1) { - assert(b->b_next->b_iused); - if (b->b_next->b_instr[0].i_loc.lineno < 0) { - b->b_next->b_instr[0].i_loc = prev_location; + if (b->b_next->b_iused > 0) { + if (b->b_next->b_instr[0].i_loc.lineno < 0) { + b->b_next->b_instr[0].i_loc = prev_location; + } } } if (is_jump(last)) { @@ -2383,40 +2476,11 @@ propagate_line_numbers(basicblock *entryblock) { } } -/* Make sure that all returns have a line number, even if early passes - * have failed to propagate a correct line number. - * The resulting line number may not be correct according to PEP 626, - * but should be "good enough", and no worse than in older versions. */ -static void -guarantee_lineno_for_exits(basicblock *entryblock, int firstlineno) { - int lineno = firstlineno; - assert(lineno > 0); - for (basicblock *b = entryblock; b != NULL; b = b->b_next) { - cfg_instr *last = basicblock_last_instr(b); - if (last == NULL) { - continue; - } - if (last->i_loc.lineno < 0) { - if (last->i_opcode == RETURN_VALUE) { - for (int i = 0; i < b->b_iused; i++) { - assert(b->b_instr[i].i_loc.lineno < 0); - - b->b_instr[i].i_loc.lineno = lineno; - } - } - } - else { - lineno = last->i_loc.lineno; - } - } -} - static int resolve_line_numbers(cfg_builder *g, int firstlineno) { RETURN_IF_ERROR(duplicate_exits_without_lineno(g)); propagate_line_numbers(g->g_entryblock); - guarantee_lineno_for_exits(g->g_entryblock, firstlineno); return SUCCESS; } @@ -2432,14 +2496,15 @@ _PyCfg_OptimizeCodeUnit(cfg_builder *g, PyObject *consts, PyObject *const_cache, RETURN_IF_ERROR(label_exception_targets(g->g_entryblock)); /** Optimization **/ - RETURN_IF_ERROR(optimize_cfg(g, consts, const_cache)); + RETURN_IF_ERROR(optimize_cfg(g, consts, const_cache, firstlineno)); RETURN_IF_ERROR(remove_unused_consts(g->g_entryblock, consts)); RETURN_IF_ERROR( add_checks_for_loads_of_uninitialized_variables( g->g_entryblock, nlocals, nparams)); - insert_superinstructions(g); + RETURN_IF_ERROR(insert_superinstructions(g)); RETURN_IF_ERROR(push_cold_blocks_to_end(g)); + assert(all_exits_have_lineno(g->g_entryblock)); RETURN_IF_ERROR(resolve_line_numbers(g, firstlineno)); return SUCCESS; } @@ -2711,7 +2776,7 @@ _PyCfg_OptimizedCfgToInstructionSequence(cfg_builder *g, return ERROR; } - convert_pseudo_ops(g->g_entryblock); + RETURN_IF_ERROR(convert_pseudo_ops(g)); /* Order of basic blocks must have been determined by now */ diff --git a/Python/frame.c b/Python/frame.c index 2865b2eab603c2..ddf6ef6ba5465c 100644 --- a/Python/frame.c +++ b/Python/frame.c @@ -139,7 +139,6 @@ _PyFrame_ClearExceptCode(_PyInterpreterFrame *frame) for (int i = 0; i < frame->stacktop; i++) { Py_XDECREF(frame->localsplus[i]); } - Py_XDECREF(frame->frame_obj); Py_XDECREF(frame->f_locals); Py_DECREF(frame->f_funcobj); } diff --git a/Python/gc.c b/Python/gc.c new file mode 100644 index 00000000000000..46646760291526 --- /dev/null +++ b/Python/gc.c @@ -0,0 +1,1948 @@ +// This implements the reference cycle garbage collector. +// The Python module inteface to the collector is in gcmodule.c. +// See https://devguide.python.org/internals/garbage-collector/ + +#include "Python.h" +#include "pycore_ceval.h" // _Py_set_eval_breaker_bit() +#include "pycore_context.h" +#include "pycore_dict.h" // _PyDict_MaybeUntrack() +#include "pycore_initconfig.h" +#include "pycore_interp.h" // PyInterpreterState.gc +#include "pycore_object.h" +#include "pycore_object_alloc.h" // _PyObject_MallocWithType() +#include "pycore_pyerrors.h" +#include "pycore_pystate.h" // _PyThreadState_GET() +#include "pycore_weakref.h" // _PyWeakref_ClearRef() +#include "pydtrace.h" + +#ifndef Py_GIL_DISABLED + +typedef struct _gc_runtime_state GCState; + +#ifdef Py_DEBUG +# define GC_DEBUG +#endif + +#define GC_NEXT _PyGCHead_NEXT +#define GC_PREV _PyGCHead_PREV + +// update_refs() set this bit for all objects in current generation. +// subtract_refs() and move_unreachable() uses this to distinguish +// visited object is in GCing or not. +// +// move_unreachable() removes this flag from reachable objects. +// Only unreachable objects have this flag. +// +// No objects in interpreter have this flag after GC ends. +#define PREV_MASK_COLLECTING _PyGC_PREV_MASK_COLLECTING + +// Lowest bit of _gc_next is used for UNREACHABLE flag. +// +// This flag represents the object is in unreachable list in move_unreachable() +// +// Although this flag is used only in move_unreachable(), move_unreachable() +// doesn't clear this flag to skip unnecessary iteration. +// move_legacy_finalizers() removes this flag instead. +// Between them, unreachable list is not normal list and we can not use +// most gc_list_* functions for it. +#define NEXT_MASK_UNREACHABLE (1) + +#define AS_GC(op) _Py_AS_GC(op) +#define FROM_GC(gc) _Py_FROM_GC(gc) + +// Automatically choose the generation that needs collecting. +#define GENERATION_AUTO (-1) + +static inline int +gc_is_collecting(PyGC_Head *g) +{ + return (g->_gc_prev & PREV_MASK_COLLECTING) != 0; +} + +static inline void +gc_clear_collecting(PyGC_Head *g) +{ + g->_gc_prev &= ~PREV_MASK_COLLECTING; +} + +static inline Py_ssize_t +gc_get_refs(PyGC_Head *g) +{ + return (Py_ssize_t)(g->_gc_prev >> _PyGC_PREV_SHIFT); +} + +static inline void +gc_set_refs(PyGC_Head *g, Py_ssize_t refs) +{ + g->_gc_prev = (g->_gc_prev & ~_PyGC_PREV_MASK) + | ((uintptr_t)(refs) << _PyGC_PREV_SHIFT); +} + +static inline void +gc_reset_refs(PyGC_Head *g, Py_ssize_t refs) +{ + g->_gc_prev = (g->_gc_prev & _PyGC_PREV_MASK_FINALIZED) + | PREV_MASK_COLLECTING + | ((uintptr_t)(refs) << _PyGC_PREV_SHIFT); +} + +static inline void +gc_decref(PyGC_Head *g) +{ + _PyObject_ASSERT_WITH_MSG(FROM_GC(g), + gc_get_refs(g) > 0, + "refcount is too small"); + g->_gc_prev -= 1 << _PyGC_PREV_SHIFT; +} + + +#define GEN_HEAD(gcstate, n) (&(gcstate)->generations[n].head) + + +static GCState * +get_gc_state(void) +{ + PyInterpreterState *interp = _PyInterpreterState_GET(); + return &interp->gc; +} + + +void +_PyGC_InitState(GCState *gcstate) +{ +#define INIT_HEAD(GEN) \ + do { \ + GEN.head._gc_next = (uintptr_t)&GEN.head; \ + GEN.head._gc_prev = (uintptr_t)&GEN.head; \ + } while (0) + + for (int i = 0; i < NUM_GENERATIONS; i++) { + assert(gcstate->generations[i].count == 0); + INIT_HEAD(gcstate->generations[i]); + }; + gcstate->generation0 = GEN_HEAD(gcstate, 0); + INIT_HEAD(gcstate->permanent_generation); + +#undef INIT_HEAD +} + + +PyStatus +_PyGC_Init(PyInterpreterState *interp) +{ + GCState *gcstate = &interp->gc; + + gcstate->garbage = PyList_New(0); + if (gcstate->garbage == NULL) { + return _PyStatus_NO_MEMORY(); + } + + gcstate->callbacks = PyList_New(0); + if (gcstate->callbacks == NULL) { + return _PyStatus_NO_MEMORY(); + } + + return _PyStatus_OK(); +} + + +/* +_gc_prev values +--------------- + +Between collections, _gc_prev is used for doubly linked list. + +Lowest two bits of _gc_prev are used for flags. +PREV_MASK_COLLECTING is used only while collecting and cleared before GC ends +or _PyObject_GC_UNTRACK() is called. + +During a collection, _gc_prev is temporary used for gc_refs, and the gc list +is singly linked until _gc_prev is restored. + +gc_refs + At the start of a collection, update_refs() copies the true refcount + to gc_refs, for each object in the generation being collected. + subtract_refs() then adjusts gc_refs so that it equals the number of + times an object is referenced directly from outside the generation + being collected. + +PREV_MASK_COLLECTING + Objects in generation being collected are marked PREV_MASK_COLLECTING in + update_refs(). + + +_gc_next values +--------------- + +_gc_next takes these values: + +0 + The object is not tracked + +!= 0 + Pointer to the next object in the GC list. + Additionally, lowest bit is used temporary for + NEXT_MASK_UNREACHABLE flag described below. + +NEXT_MASK_UNREACHABLE + move_unreachable() then moves objects not reachable (whether directly or + indirectly) from outside the generation into an "unreachable" set and + set this flag. + + Objects that are found to be reachable have gc_refs set to 1. + When this flag is set for the reachable object, the object must be in + "unreachable" set. + The flag is unset and the object is moved back to "reachable" set. + + move_legacy_finalizers() will remove this flag from "unreachable" set. +*/ + +/*** list functions ***/ + +static inline void +gc_list_init(PyGC_Head *list) +{ + // List header must not have flags. + // We can assign pointer by simple cast. + list->_gc_prev = (uintptr_t)list; + list->_gc_next = (uintptr_t)list; +} + +static inline int +gc_list_is_empty(PyGC_Head *list) +{ + return (list->_gc_next == (uintptr_t)list); +} + +/* Append `node` to `list`. */ +static inline void +gc_list_append(PyGC_Head *node, PyGC_Head *list) +{ + PyGC_Head *last = (PyGC_Head *)list->_gc_prev; + + // last <-> node + _PyGCHead_SET_PREV(node, last); + _PyGCHead_SET_NEXT(last, node); + + // node <-> list + _PyGCHead_SET_NEXT(node, list); + list->_gc_prev = (uintptr_t)node; +} + +/* Remove `node` from the gc list it's currently in. */ +static inline void +gc_list_remove(PyGC_Head *node) +{ + PyGC_Head *prev = GC_PREV(node); + PyGC_Head *next = GC_NEXT(node); + + _PyGCHead_SET_NEXT(prev, next); + _PyGCHead_SET_PREV(next, prev); + + node->_gc_next = 0; /* object is not currently tracked */ +} + +/* Move `node` from the gc list it's currently in (which is not explicitly + * named here) to the end of `list`. This is semantically the same as + * gc_list_remove(node) followed by gc_list_append(node, list). + */ +static void +gc_list_move(PyGC_Head *node, PyGC_Head *list) +{ + /* Unlink from current list. */ + PyGC_Head *from_prev = GC_PREV(node); + PyGC_Head *from_next = GC_NEXT(node); + _PyGCHead_SET_NEXT(from_prev, from_next); + _PyGCHead_SET_PREV(from_next, from_prev); + + /* Relink at end of new list. */ + // list must not have flags. So we can skip macros. + PyGC_Head *to_prev = (PyGC_Head*)list->_gc_prev; + _PyGCHead_SET_PREV(node, to_prev); + _PyGCHead_SET_NEXT(to_prev, node); + list->_gc_prev = (uintptr_t)node; + _PyGCHead_SET_NEXT(node, list); +} + +/* append list `from` onto list `to`; `from` becomes an empty list */ +static void +gc_list_merge(PyGC_Head *from, PyGC_Head *to) +{ + assert(from != to); + if (!gc_list_is_empty(from)) { + PyGC_Head *to_tail = GC_PREV(to); + PyGC_Head *from_head = GC_NEXT(from); + PyGC_Head *from_tail = GC_PREV(from); + assert(from_head != from); + assert(from_tail != from); + + _PyGCHead_SET_NEXT(to_tail, from_head); + _PyGCHead_SET_PREV(from_head, to_tail); + + _PyGCHead_SET_NEXT(from_tail, to); + _PyGCHead_SET_PREV(to, from_tail); + } + gc_list_init(from); +} + +static Py_ssize_t +gc_list_size(PyGC_Head *list) +{ + PyGC_Head *gc; + Py_ssize_t n = 0; + for (gc = GC_NEXT(list); gc != list; gc = GC_NEXT(gc)) { + n++; + } + return n; +} + +/* Walk the list and mark all objects as non-collecting */ +static inline void +gc_list_clear_collecting(PyGC_Head *collectable) +{ + PyGC_Head *gc; + for (gc = GC_NEXT(collectable); gc != collectable; gc = GC_NEXT(gc)) { + gc_clear_collecting(gc); + } +} + +/* Append objects in a GC list to a Python list. + * Return 0 if all OK, < 0 if error (out of memory for list) + */ +static int +append_objects(PyObject *py_list, PyGC_Head *gc_list) +{ + PyGC_Head *gc; + for (gc = GC_NEXT(gc_list); gc != gc_list; gc = GC_NEXT(gc)) { + PyObject *op = FROM_GC(gc); + if (op != py_list) { + if (PyList_Append(py_list, op)) { + return -1; /* exception */ + } + } + } + return 0; +} + +// Constants for validate_list's flags argument. +enum flagstates {collecting_clear_unreachable_clear, + collecting_clear_unreachable_set, + collecting_set_unreachable_clear, + collecting_set_unreachable_set}; + +#ifdef GC_DEBUG +// validate_list checks list consistency. And it works as document +// describing when flags are expected to be set / unset. +// `head` must be a doubly-linked gc list, although it's fine (expected!) if +// the prev and next pointers are "polluted" with flags. +// What's checked: +// - The `head` pointers are not polluted. +// - The objects' PREV_MASK_COLLECTING and NEXT_MASK_UNREACHABLE flags are all +// `set or clear, as specified by the 'flags' argument. +// - The prev and next pointers are mutually consistent. +static void +validate_list(PyGC_Head *head, enum flagstates flags) +{ + assert((head->_gc_prev & PREV_MASK_COLLECTING) == 0); + assert((head->_gc_next & NEXT_MASK_UNREACHABLE) == 0); + uintptr_t prev_value = 0, next_value = 0; + switch (flags) { + case collecting_clear_unreachable_clear: + break; + case collecting_set_unreachable_clear: + prev_value = PREV_MASK_COLLECTING; + break; + case collecting_clear_unreachable_set: + next_value = NEXT_MASK_UNREACHABLE; + break; + case collecting_set_unreachable_set: + prev_value = PREV_MASK_COLLECTING; + next_value = NEXT_MASK_UNREACHABLE; + break; + default: + assert(! "bad internal flags argument"); + } + PyGC_Head *prev = head; + PyGC_Head *gc = GC_NEXT(head); + while (gc != head) { + PyGC_Head *trueprev = GC_PREV(gc); + PyGC_Head *truenext = (PyGC_Head *)(gc->_gc_next & ~NEXT_MASK_UNREACHABLE); + assert(truenext != NULL); + assert(trueprev == prev); + assert((gc->_gc_prev & PREV_MASK_COLLECTING) == prev_value); + assert((gc->_gc_next & NEXT_MASK_UNREACHABLE) == next_value); + prev = gc; + gc = truenext; + } + assert(prev == GC_PREV(head)); +} +#else +#define validate_list(x, y) do{}while(0) +#endif + +/*** end of list stuff ***/ + + +/* Set all gc_refs = ob_refcnt. After this, gc_refs is > 0 and + * PREV_MASK_COLLECTING bit is set for all objects in containers. + */ +static void +update_refs(PyGC_Head *containers) +{ + PyGC_Head *next; + PyGC_Head *gc = GC_NEXT(containers); + + while (gc != containers) { + next = GC_NEXT(gc); + /* Move any object that might have become immortal to the + * permanent generation as the reference count is not accurately + * reflecting the actual number of live references to this object + */ + if (_Py_IsImmortal(FROM_GC(gc))) { + gc_list_move(gc, &get_gc_state()->permanent_generation.head); + gc = next; + continue; + } + gc_reset_refs(gc, Py_REFCNT(FROM_GC(gc))); + /* Python's cyclic gc should never see an incoming refcount + * of 0: if something decref'ed to 0, it should have been + * deallocated immediately at that time. + * Possible cause (if the assert triggers): a tp_dealloc + * routine left a gc-aware object tracked during its teardown + * phase, and did something-- or allowed something to happen -- + * that called back into Python. gc can trigger then, and may + * see the still-tracked dying object. Before this assert + * was added, such mistakes went on to allow gc to try to + * delete the object again. In a debug build, that caused + * a mysterious segfault, when _Py_ForgetReference tried + * to remove the object from the doubly-linked list of all + * objects a second time. In a release build, an actual + * double deallocation occurred, which leads to corruption + * of the allocator's internal bookkeeping pointers. That's + * so serious that maybe this should be a release-build + * check instead of an assert? + */ + _PyObject_ASSERT(FROM_GC(gc), gc_get_refs(gc) != 0); + gc = next; + } +} + +/* A traversal callback for subtract_refs. */ +static int +visit_decref(PyObject *op, void *parent) +{ + OBJECT_STAT_INC(object_visits); + _PyObject_ASSERT(_PyObject_CAST(parent), !_PyObject_IsFreed(op)); + + if (_PyObject_IS_GC(op)) { + PyGC_Head *gc = AS_GC(op); + /* We're only interested in gc_refs for objects in the + * generation being collected, which can be recognized + * because only they have positive gc_refs. + */ + if (gc_is_collecting(gc)) { + gc_decref(gc); + } + } + return 0; +} + +/* Subtract internal references from gc_refs. After this, gc_refs is >= 0 + * for all objects in containers, and is GC_REACHABLE for all tracked gc + * objects not in containers. The ones with gc_refs > 0 are directly + * reachable from outside containers, and so can't be collected. + */ +static void +subtract_refs(PyGC_Head *containers) +{ + traverseproc traverse; + PyGC_Head *gc = GC_NEXT(containers); + for (; gc != containers; gc = GC_NEXT(gc)) { + PyObject *op = FROM_GC(gc); + traverse = Py_TYPE(op)->tp_traverse; + (void) traverse(op, + visit_decref, + op); + } +} + +/* A traversal callback for move_unreachable. */ +static int +visit_reachable(PyObject *op, void *arg) +{ + PyGC_Head *reachable = arg; + OBJECT_STAT_INC(object_visits); + if (!_PyObject_IS_GC(op)) { + return 0; + } + + PyGC_Head *gc = AS_GC(op); + const Py_ssize_t gc_refs = gc_get_refs(gc); + + // Ignore objects in other generation. + // This also skips objects "to the left" of the current position in + // move_unreachable's scan of the 'young' list - they've already been + // traversed, and no longer have the PREV_MASK_COLLECTING flag. + if (! gc_is_collecting(gc)) { + return 0; + } + // It would be a logic error elsewhere if the collecting flag were set on + // an untracked object. + assert(gc->_gc_next != 0); + + if (gc->_gc_next & NEXT_MASK_UNREACHABLE) { + /* This had gc_refs = 0 when move_unreachable got + * to it, but turns out it's reachable after all. + * Move it back to move_unreachable's 'young' list, + * and move_unreachable will eventually get to it + * again. + */ + // Manually unlink gc from unreachable list because the list functions + // don't work right in the presence of NEXT_MASK_UNREACHABLE flags. + PyGC_Head *prev = GC_PREV(gc); + PyGC_Head *next = (PyGC_Head*)(gc->_gc_next & ~NEXT_MASK_UNREACHABLE); + _PyObject_ASSERT(FROM_GC(prev), + prev->_gc_next & NEXT_MASK_UNREACHABLE); + _PyObject_ASSERT(FROM_GC(next), + next->_gc_next & NEXT_MASK_UNREACHABLE); + prev->_gc_next = gc->_gc_next; // copy NEXT_MASK_UNREACHABLE + _PyGCHead_SET_PREV(next, prev); + + gc_list_append(gc, reachable); + gc_set_refs(gc, 1); + } + else if (gc_refs == 0) { + /* This is in move_unreachable's 'young' list, but + * the traversal hasn't yet gotten to it. All + * we need to do is tell move_unreachable that it's + * reachable. + */ + gc_set_refs(gc, 1); + } + /* Else there's nothing to do. + * If gc_refs > 0, it must be in move_unreachable's 'young' + * list, and move_unreachable will eventually get to it. + */ + else { + _PyObject_ASSERT_WITH_MSG(op, gc_refs > 0, "refcount is too small"); + } + return 0; +} + +/* Move the unreachable objects from young to unreachable. After this, + * all objects in young don't have PREV_MASK_COLLECTING flag and + * unreachable have the flag. + * All objects in young after this are directly or indirectly reachable + * from outside the original young; and all objects in unreachable are + * not. + * + * This function restores _gc_prev pointer. young and unreachable are + * doubly linked list after this function. + * But _gc_next in unreachable list has NEXT_MASK_UNREACHABLE flag. + * So we can not gc_list_* functions for unreachable until we remove the flag. + */ +static void +move_unreachable(PyGC_Head *young, PyGC_Head *unreachable) +{ + // previous elem in the young list, used for restore gc_prev. + PyGC_Head *prev = young; + PyGC_Head *gc = GC_NEXT(young); + + /* Invariants: all objects "to the left" of us in young are reachable + * (directly or indirectly) from outside the young list as it was at entry. + * + * All other objects from the original young "to the left" of us are in + * unreachable now, and have NEXT_MASK_UNREACHABLE. All objects to the + * left of us in 'young' now have been scanned, and no objects here + * or to the right have been scanned yet. + */ + + while (gc != young) { + if (gc_get_refs(gc)) { + /* gc is definitely reachable from outside the + * original 'young'. Mark it as such, and traverse + * its pointers to find any other objects that may + * be directly reachable from it. Note that the + * call to tp_traverse may append objects to young, + * so we have to wait until it returns to determine + * the next object to visit. + */ + PyObject *op = FROM_GC(gc); + traverseproc traverse = Py_TYPE(op)->tp_traverse; + _PyObject_ASSERT_WITH_MSG(op, gc_get_refs(gc) > 0, + "refcount is too small"); + // NOTE: visit_reachable may change gc->_gc_next when + // young->_gc_prev == gc. Don't do gc = GC_NEXT(gc) before! + (void) traverse(op, + visit_reachable, + (void *)young); + // relink gc_prev to prev element. + _PyGCHead_SET_PREV(gc, prev); + // gc is not COLLECTING state after here. + gc_clear_collecting(gc); + prev = gc; + } + else { + /* This *may* be unreachable. To make progress, + * assume it is. gc isn't directly reachable from + * any object we've already traversed, but may be + * reachable from an object we haven't gotten to yet. + * visit_reachable will eventually move gc back into + * young if that's so, and we'll see it again. + */ + // Move gc to unreachable. + // No need to gc->next->prev = prev because it is single linked. + prev->_gc_next = gc->_gc_next; + + // We can't use gc_list_append() here because we use + // NEXT_MASK_UNREACHABLE here. + PyGC_Head *last = GC_PREV(unreachable); + // NOTE: Since all objects in unreachable set has + // NEXT_MASK_UNREACHABLE flag, we set it unconditionally. + // But this may pollute the unreachable list head's 'next' pointer + // too. That's semantically senseless but expedient here - the + // damage is repaired when this function ends. + last->_gc_next = (NEXT_MASK_UNREACHABLE | (uintptr_t)gc); + _PyGCHead_SET_PREV(gc, last); + gc->_gc_next = (NEXT_MASK_UNREACHABLE | (uintptr_t)unreachable); + unreachable->_gc_prev = (uintptr_t)gc; + } + gc = (PyGC_Head*)prev->_gc_next; + } + // young->_gc_prev must be last element remained in the list. + young->_gc_prev = (uintptr_t)prev; + // don't let the pollution of the list head's next pointer leak + unreachable->_gc_next &= ~NEXT_MASK_UNREACHABLE; +} + +static void +untrack_tuples(PyGC_Head *head) +{ + PyGC_Head *next, *gc = GC_NEXT(head); + while (gc != head) { + PyObject *op = FROM_GC(gc); + next = GC_NEXT(gc); + if (PyTuple_CheckExact(op)) { + _PyTuple_MaybeUntrack(op); + } + gc = next; + } +} + +/* Try to untrack all currently tracked dictionaries */ +static void +untrack_dicts(PyGC_Head *head) +{ + PyGC_Head *next, *gc = GC_NEXT(head); + while (gc != head) { + PyObject *op = FROM_GC(gc); + next = GC_NEXT(gc); + if (PyDict_CheckExact(op)) { + _PyDict_MaybeUntrack(op); + } + gc = next; + } +} + +/* Return true if object has a pre-PEP 442 finalization method. */ +static int +has_legacy_finalizer(PyObject *op) +{ + return Py_TYPE(op)->tp_del != NULL; +} + +/* Move the objects in unreachable with tp_del slots into `finalizers`. + * + * This function also removes NEXT_MASK_UNREACHABLE flag + * from _gc_next in unreachable. + */ +static void +move_legacy_finalizers(PyGC_Head *unreachable, PyGC_Head *finalizers) +{ + PyGC_Head *gc, *next; + assert((unreachable->_gc_next & NEXT_MASK_UNREACHABLE) == 0); + + /* March over unreachable. Move objects with finalizers into + * `finalizers`. + */ + for (gc = GC_NEXT(unreachable); gc != unreachable; gc = next) { + PyObject *op = FROM_GC(gc); + + _PyObject_ASSERT(op, gc->_gc_next & NEXT_MASK_UNREACHABLE); + gc->_gc_next &= ~NEXT_MASK_UNREACHABLE; + next = (PyGC_Head*)gc->_gc_next; + + if (has_legacy_finalizer(op)) { + gc_clear_collecting(gc); + gc_list_move(gc, finalizers); + } + } +} + +static inline void +clear_unreachable_mask(PyGC_Head *unreachable) +{ + /* Check that the list head does not have the unreachable bit set */ + assert(((uintptr_t)unreachable & NEXT_MASK_UNREACHABLE) == 0); + + PyGC_Head *gc, *next; + assert((unreachable->_gc_next & NEXT_MASK_UNREACHABLE) == 0); + for (gc = GC_NEXT(unreachable); gc != unreachable; gc = next) { + _PyObject_ASSERT((PyObject*)FROM_GC(gc), gc->_gc_next & NEXT_MASK_UNREACHABLE); + gc->_gc_next &= ~NEXT_MASK_UNREACHABLE; + next = (PyGC_Head*)gc->_gc_next; + } + validate_list(unreachable, collecting_set_unreachable_clear); +} + +/* A traversal callback for move_legacy_finalizer_reachable. */ +static int +visit_move(PyObject *op, void *arg) +{ + PyGC_Head *tolist = arg; + OBJECT_STAT_INC(object_visits); + if (_PyObject_IS_GC(op)) { + PyGC_Head *gc = AS_GC(op); + if (gc_is_collecting(gc)) { + gc_list_move(gc, tolist); + gc_clear_collecting(gc); + } + } + return 0; +} + +/* Move objects that are reachable from finalizers, from the unreachable set + * into finalizers set. + */ +static void +move_legacy_finalizer_reachable(PyGC_Head *finalizers) +{ + traverseproc traverse; + PyGC_Head *gc = GC_NEXT(finalizers); + for (; gc != finalizers; gc = GC_NEXT(gc)) { + /* Note that the finalizers list may grow during this. */ + traverse = Py_TYPE(FROM_GC(gc))->tp_traverse; + (void) traverse(FROM_GC(gc), + visit_move, + (void *)finalizers); + } +} + +/* Clear all weakrefs to unreachable objects, and if such a weakref has a + * callback, invoke it if necessary. Note that it's possible for such + * weakrefs to be outside the unreachable set -- indeed, those are precisely + * the weakrefs whose callbacks must be invoked. See gc_weakref.txt for + * overview & some details. Some weakrefs with callbacks may be reclaimed + * directly by this routine; the number reclaimed is the return value. Other + * weakrefs with callbacks may be moved into the `old` generation. Objects + * moved into `old` have gc_refs set to GC_REACHABLE; the objects remaining in + * unreachable are left at GC_TENTATIVELY_UNREACHABLE. When this returns, + * no object in `unreachable` is weakly referenced anymore. + */ +static int +handle_weakrefs(PyGC_Head *unreachable, PyGC_Head *old) +{ + PyGC_Head *gc; + PyObject *op; /* generally FROM_GC(gc) */ + PyWeakReference *wr; /* generally a cast of op */ + PyGC_Head wrcb_to_call; /* weakrefs with callbacks to call */ + PyGC_Head *next; + int num_freed = 0; + + gc_list_init(&wrcb_to_call); + + /* Clear all weakrefs to the objects in unreachable. If such a weakref + * also has a callback, move it into `wrcb_to_call` if the callback + * needs to be invoked. Note that we cannot invoke any callbacks until + * all weakrefs to unreachable objects are cleared, lest the callback + * resurrect an unreachable object via a still-active weakref. We + * make another pass over wrcb_to_call, invoking callbacks, after this + * pass completes. + */ + for (gc = GC_NEXT(unreachable); gc != unreachable; gc = next) { + PyWeakReference **wrlist; + + op = FROM_GC(gc); + next = GC_NEXT(gc); + + if (PyWeakref_Check(op)) { + /* A weakref inside the unreachable set must be cleared. If we + * allow its callback to execute inside delete_garbage(), it + * could expose objects that have tp_clear already called on + * them. Or, it could resurrect unreachable objects. One way + * this can happen is if some container objects do not implement + * tp_traverse. Then, wr_object can be outside the unreachable + * set but can be deallocated as a result of breaking the + * reference cycle. If we don't clear the weakref, the callback + * will run and potentially cause a crash. See bpo-38006 for + * one example. + */ + _PyWeakref_ClearRef((PyWeakReference *)op); + } + + if (! _PyType_SUPPORTS_WEAKREFS(Py_TYPE(op))) { + continue; + } + + /* It supports weakrefs. Does it have any? + * + * This is never triggered for static types so we can avoid the + * (slightly) more costly _PyObject_GET_WEAKREFS_LISTPTR(). + */ + wrlist = _PyObject_GET_WEAKREFS_LISTPTR_FROM_OFFSET(op); + + /* `op` may have some weakrefs. March over the list, clear + * all the weakrefs, and move the weakrefs with callbacks + * that must be called into wrcb_to_call. + */ + for (wr = *wrlist; wr != NULL; wr = *wrlist) { + PyGC_Head *wrasgc; /* AS_GC(wr) */ + + /* _PyWeakref_ClearRef clears the weakref but leaves + * the callback pointer intact. Obscure: it also + * changes *wrlist. + */ + _PyObject_ASSERT((PyObject *)wr, wr->wr_object == op); + _PyWeakref_ClearRef(wr); + _PyObject_ASSERT((PyObject *)wr, wr->wr_object == Py_None); + if (wr->wr_callback == NULL) { + /* no callback */ + continue; + } + + /* Headache time. `op` is going away, and is weakly referenced by + * `wr`, which has a callback. Should the callback be invoked? If wr + * is also trash, no: + * + * 1. There's no need to call it. The object and the weakref are + * both going away, so it's legitimate to pretend the weakref is + * going away first. The user has to ensure a weakref outlives its + * referent if they want a guarantee that the wr callback will get + * invoked. + * + * 2. It may be catastrophic to call it. If the callback is also in + * cyclic trash (CT), then although the CT is unreachable from + * outside the current generation, CT may be reachable from the + * callback. Then the callback could resurrect insane objects. + * + * Since the callback is never needed and may be unsafe in this case, + * wr is simply left in the unreachable set. Note that because we + * already called _PyWeakref_ClearRef(wr), its callback will never + * trigger. + * + * OTOH, if wr isn't part of CT, we should invoke the callback: the + * weakref outlived the trash. Note that since wr isn't CT in this + * case, its callback can't be CT either -- wr acted as an external + * root to this generation, and therefore its callback did too. So + * nothing in CT is reachable from the callback either, so it's hard + * to imagine how calling it later could create a problem for us. wr + * is moved to wrcb_to_call in this case. + */ + if (gc_is_collecting(AS_GC((PyObject *)wr))) { + /* it should already have been cleared above */ + assert(wr->wr_object == Py_None); + continue; + } + + /* Create a new reference so that wr can't go away + * before we can process it again. + */ + Py_INCREF(wr); + + /* Move wr to wrcb_to_call, for the next pass. */ + wrasgc = AS_GC((PyObject *)wr); + assert(wrasgc != next); /* wrasgc is reachable, but + next isn't, so they can't + be the same */ + gc_list_move(wrasgc, &wrcb_to_call); + } + } + + /* Invoke the callbacks we decided to honor. It's safe to invoke them + * because they can't reference unreachable objects. + */ + while (! gc_list_is_empty(&wrcb_to_call)) { + PyObject *temp; + PyObject *callback; + + gc = (PyGC_Head*)wrcb_to_call._gc_next; + op = FROM_GC(gc); + _PyObject_ASSERT(op, PyWeakref_Check(op)); + wr = (PyWeakReference *)op; + callback = wr->wr_callback; + _PyObject_ASSERT(op, callback != NULL); + + /* copy-paste of weakrefobject.c's handle_callback() */ + temp = PyObject_CallOneArg(callback, (PyObject *)wr); + if (temp == NULL) { + PyErr_WriteUnraisable(callback); + } + else { + Py_DECREF(temp); + } + + /* Give up the reference we created in the first pass. When + * op's refcount hits 0 (which it may or may not do right now), + * op's tp_dealloc will decref op->wr_callback too. Note + * that the refcount probably will hit 0 now, and because this + * weakref was reachable to begin with, gc didn't already + * add it to its count of freed objects. Example: a reachable + * weak value dict maps some key to this reachable weakref. + * The callback removes this key->weakref mapping from the + * dict, leaving no other references to the weakref (excepting + * ours). + */ + Py_DECREF(op); + if (wrcb_to_call._gc_next == (uintptr_t)gc) { + /* object is still alive -- move it */ + gc_list_move(gc, old); + } + else { + ++num_freed; + } + } + + return num_freed; +} + +static void +debug_cycle(const char *msg, PyObject *op) +{ + PySys_FormatStderr("gc: %s <%s %p>\n", + msg, Py_TYPE(op)->tp_name, op); +} + +/* Handle uncollectable garbage (cycles with tp_del slots, and stuff reachable + * only from such cycles). + * If _PyGC_DEBUG_SAVEALL, all objects in finalizers are appended to the module + * garbage list (a Python list), else only the objects in finalizers with + * __del__ methods are appended to garbage. All objects in finalizers are + * merged into the old list regardless. + */ +static void +handle_legacy_finalizers(PyThreadState *tstate, + GCState *gcstate, + PyGC_Head *finalizers, PyGC_Head *old) +{ + assert(!_PyErr_Occurred(tstate)); + assert(gcstate->garbage != NULL); + + PyGC_Head *gc = GC_NEXT(finalizers); + for (; gc != finalizers; gc = GC_NEXT(gc)) { + PyObject *op = FROM_GC(gc); + + if ((gcstate->debug & _PyGC_DEBUG_SAVEALL) || has_legacy_finalizer(op)) { + if (PyList_Append(gcstate->garbage, op) < 0) { + _PyErr_Clear(tstate); + break; + } + } + } + + gc_list_merge(finalizers, old); +} + +/* Run first-time finalizers (if any) on all the objects in collectable. + * Note that this may remove some (or even all) of the objects from the + * list, due to refcounts falling to 0. + */ +static void +finalize_garbage(PyThreadState *tstate, PyGC_Head *collectable) +{ + destructor finalize; + PyGC_Head seen; + + /* While we're going through the loop, `finalize(op)` may cause op, or + * other objects, to be reclaimed via refcounts falling to zero. So + * there's little we can rely on about the structure of the input + * `collectable` list across iterations. For safety, we always take the + * first object in that list and move it to a temporary `seen` list. + * If objects vanish from the `collectable` and `seen` lists we don't + * care. + */ + gc_list_init(&seen); + + while (!gc_list_is_empty(collectable)) { + PyGC_Head *gc = GC_NEXT(collectable); + PyObject *op = FROM_GC(gc); + gc_list_move(gc, &seen); + if (!_PyGC_FINALIZED(op) && + (finalize = Py_TYPE(op)->tp_finalize) != NULL) + { + _PyGC_SET_FINALIZED(op); + Py_INCREF(op); + finalize(op); + assert(!_PyErr_Occurred(tstate)); + Py_DECREF(op); + } + } + gc_list_merge(&seen, collectable); +} + +/* Break reference cycles by clearing the containers involved. This is + * tricky business as the lists can be changing and we don't know which + * objects may be freed. It is possible I screwed something up here. + */ +static void +delete_garbage(PyThreadState *tstate, GCState *gcstate, + PyGC_Head *collectable, PyGC_Head *old) +{ + assert(!_PyErr_Occurred(tstate)); + + while (!gc_list_is_empty(collectable)) { + PyGC_Head *gc = GC_NEXT(collectable); + PyObject *op = FROM_GC(gc); + + _PyObject_ASSERT_WITH_MSG(op, Py_REFCNT(op) > 0, + "refcount is too small"); + + if (gcstate->debug & _PyGC_DEBUG_SAVEALL) { + assert(gcstate->garbage != NULL); + if (PyList_Append(gcstate->garbage, op) < 0) { + _PyErr_Clear(tstate); + } + } + else { + inquiry clear; + if ((clear = Py_TYPE(op)->tp_clear) != NULL) { + Py_INCREF(op); + (void) clear(op); + if (_PyErr_Occurred(tstate)) { + PyErr_FormatUnraisable("Exception ignored in tp_clear of %s", + Py_TYPE(op)->tp_name); + } + Py_DECREF(op); + } + } + if (GC_NEXT(collectable) == gc) { + /* object is still alive, move it, it may die later */ + gc_clear_collecting(gc); + gc_list_move(gc, old); + } + } +} + + +// Show stats for objects in each generations +static void +show_stats_each_generations(GCState *gcstate) +{ + char buf[100]; + size_t pos = 0; + + for (int i = 0; i < NUM_GENERATIONS && pos < sizeof(buf); i++) { + pos += PyOS_snprintf(buf+pos, sizeof(buf)-pos, + " %zd", + gc_list_size(GEN_HEAD(gcstate, i))); + } + + PySys_FormatStderr( + "gc: objects in each generation:%s\n" + "gc: objects in permanent generation: %zd\n", + buf, gc_list_size(&gcstate->permanent_generation.head)); +} + +/* Deduce which objects among "base" are unreachable from outside the list + and move them to 'unreachable'. The process consist in the following steps: + +1. Copy all reference counts to a different field (gc_prev is used to hold + this copy to save memory). +2. Traverse all objects in "base" and visit all referred objects using + "tp_traverse" and for every visited object, subtract 1 to the reference + count (the one that we copied in the previous step). After this step, all + objects that can be reached directly from outside must have strictly positive + reference count, while all unreachable objects must have a count of exactly 0. +3. Identify all unreachable objects (the ones with 0 reference count) and move + them to the "unreachable" list. This step also needs to move back to "base" all + objects that were initially marked as unreachable but are referred transitively + by the reachable objects (the ones with strictly positive reference count). + +Contracts: + + * The "base" has to be a valid list with no mask set. + + * The "unreachable" list must be uninitialized (this function calls + gc_list_init over 'unreachable'). + +IMPORTANT: This function leaves 'unreachable' with the NEXT_MASK_UNREACHABLE +flag set but it does not clear it to skip unnecessary iteration. Before the +flag is cleared (for example, by using 'clear_unreachable_mask' function or +by a call to 'move_legacy_finalizers'), the 'unreachable' list is not a normal +list and we can not use most gc_list_* functions for it. */ +static inline void +deduce_unreachable(PyGC_Head *base, PyGC_Head *unreachable) { + validate_list(base, collecting_clear_unreachable_clear); + /* Using ob_refcnt and gc_refs, calculate which objects in the + * container set are reachable from outside the set (i.e., have a + * refcount greater than 0 when all the references within the + * set are taken into account). + */ + update_refs(base); // gc_prev is used for gc_refs + subtract_refs(base); + + /* Leave everything reachable from outside base in base, and move + * everything else (in base) to unreachable. + * + * NOTE: This used to move the reachable objects into a reachable + * set instead. But most things usually turn out to be reachable, + * so it's more efficient to move the unreachable things. It "sounds slick" + * to move the unreachable objects, until you think about it - the reason it + * pays isn't actually obvious. + * + * Suppose we create objects A, B, C in that order. They appear in the young + * generation in the same order. If B points to A, and C to B, and C is + * reachable from outside, then the adjusted refcounts will be 0, 0, and 1 + * respectively. + * + * When move_unreachable finds A, A is moved to the unreachable list. The + * same for B when it's first encountered. Then C is traversed, B is moved + * _back_ to the reachable list. B is eventually traversed, and then A is + * moved back to the reachable list. + * + * So instead of not moving at all, the reachable objects B and A are moved + * twice each. Why is this a win? A straightforward algorithm to move the + * reachable objects instead would move A, B, and C once each. + * + * The key is that this dance leaves the objects in order C, B, A - it's + * reversed from the original order. On all _subsequent_ scans, none of + * them will move. Since most objects aren't in cycles, this can save an + * unbounded number of moves across an unbounded number of later collections. + * It can cost more only the first time the chain is scanned. + * + * Drawback: move_unreachable is also used to find out what's still trash + * after finalizers may resurrect objects. In _that_ case most unreachable + * objects will remain unreachable, so it would be more efficient to move + * the reachable objects instead. But this is a one-time cost, probably not + * worth complicating the code to speed just a little. + */ + gc_list_init(unreachable); + move_unreachable(base, unreachable); // gc_prev is pointer again + validate_list(base, collecting_clear_unreachable_clear); + validate_list(unreachable, collecting_set_unreachable_set); +} + +/* Handle objects that may have resurrected after a call to 'finalize_garbage', moving + them to 'old_generation' and placing the rest on 'still_unreachable'. + + Contracts: + * After this function 'unreachable' must not be used anymore and 'still_unreachable' + will contain the objects that did not resurrect. + + * The "still_unreachable" list must be uninitialized (this function calls + gc_list_init over 'still_unreachable'). + +IMPORTANT: After a call to this function, the 'still_unreachable' set will have the +PREV_MARK_COLLECTING set, but the objects in this set are going to be removed so +we can skip the expense of clearing the flag to avoid extra iteration. */ +static inline void +handle_resurrected_objects(PyGC_Head *unreachable, PyGC_Head* still_unreachable, + PyGC_Head *old_generation) +{ + // Remove the PREV_MASK_COLLECTING from unreachable + // to prepare it for a new call to 'deduce_unreachable' + gc_list_clear_collecting(unreachable); + + // After the call to deduce_unreachable, the 'still_unreachable' set will + // have the PREV_MARK_COLLECTING set, but the objects are going to be + // removed so we can skip the expense of clearing the flag. + PyGC_Head* resurrected = unreachable; + deduce_unreachable(resurrected, still_unreachable); + clear_unreachable_mask(still_unreachable); + + // Move the resurrected objects to the old generation for future collection. + gc_list_merge(resurrected, old_generation); +} + + +/* Invoke progress callbacks to notify clients that garbage collection + * is starting or stopping + */ +static void +invoke_gc_callback(PyThreadState *tstate, const char *phase, + int generation, Py_ssize_t collected, + Py_ssize_t uncollectable) +{ + assert(!_PyErr_Occurred(tstate)); + + /* we may get called very early */ + GCState *gcstate = &tstate->interp->gc; + if (gcstate->callbacks == NULL) { + return; + } + + /* The local variable cannot be rebound, check it for sanity */ + assert(PyList_CheckExact(gcstate->callbacks)); + PyObject *info = NULL; + if (PyList_GET_SIZE(gcstate->callbacks) != 0) { + info = Py_BuildValue("{sisnsn}", + "generation", generation, + "collected", collected, + "uncollectable", uncollectable); + if (info == NULL) { + PyErr_FormatUnraisable("Exception ignored on invoking gc callbacks"); + return; + } + } + + PyObject *phase_obj = PyUnicode_FromString(phase); + if (phase_obj == NULL) { + Py_XDECREF(info); + PyErr_FormatUnraisable("Exception ignored on invoking gc callbacks"); + return; + } + + PyObject *stack[] = {phase_obj, info}; + for (Py_ssize_t i=0; icallbacks); i++) { + PyObject *r, *cb = PyList_GET_ITEM(gcstate->callbacks, i); + Py_INCREF(cb); /* make sure cb doesn't go away */ + r = PyObject_Vectorcall(cb, stack, 2, NULL); + if (r == NULL) { + PyErr_WriteUnraisable(cb); + } + else { + Py_DECREF(r); + } + Py_DECREF(cb); + } + Py_DECREF(phase_obj); + Py_XDECREF(info); + assert(!_PyErr_Occurred(tstate)); +} + + +/* Find the oldest generation (highest numbered) where the count + * exceeds the threshold. Objects in the that generation and + * generations younger than it will be collected. */ +static int +gc_select_generation(GCState *gcstate) +{ + for (int i = NUM_GENERATIONS-1; i >= 0; i--) { + if (gcstate->generations[i].count > gcstate->generations[i].threshold) { + /* Avoid quadratic performance degradation in number + of tracked objects (see also issue #4074): + + To limit the cost of garbage collection, there are two strategies; + - make each collection faster, e.g. by scanning fewer objects + - do less collections + This heuristic is about the latter strategy. + + In addition to the various configurable thresholds, we only trigger a + full collection if the ratio + + long_lived_pending / long_lived_total + + is above a given value (hardwired to 25%). + + The reason is that, while "non-full" collections (i.e., collections of + the young and middle generations) will always examine roughly the same + number of objects -- determined by the aforementioned thresholds --, + the cost of a full collection is proportional to the total number of + long-lived objects, which is virtually unbounded. + + Indeed, it has been remarked that doing a full collection every + of object creations entails a dramatic performance + degradation in workloads which consist in creating and storing lots of + long-lived objects (e.g. building a large list of GC-tracked objects would + show quadratic performance, instead of linear as expected: see issue #4074). + + Using the above ratio, instead, yields amortized linear performance in + the total number of objects (the effect of which can be summarized + thusly: "each full garbage collection is more and more costly as the + number of objects grows, but we do fewer and fewer of them"). + + This heuristic was suggested by Martin von Löwis on python-dev in + June 2008. His original analysis and proposal can be found at: + http://mail.python.org/pipermail/python-dev/2008-June/080579.html + */ + if (i == NUM_GENERATIONS - 1 + && gcstate->long_lived_pending < gcstate->long_lived_total / 4) + { + continue; + } + return i; + } + } + return -1; +} + + +/* This is the main function. Read this to understand how the + * collection process works. */ +static Py_ssize_t +gc_collect_main(PyThreadState *tstate, int generation, _PyGC_Reason reason) +{ + int i; + Py_ssize_t m = 0; /* # objects collected */ + Py_ssize_t n = 0; /* # unreachable objects that couldn't be collected */ + PyGC_Head *young; /* the generation we are examining */ + PyGC_Head *old; /* next older generation */ + PyGC_Head unreachable; /* non-problematic unreachable trash */ + PyGC_Head finalizers; /* objects with, & reachable from, __del__ */ + PyGC_Head *gc; + _PyTime_t t1 = 0; /* initialize to prevent a compiler warning */ + GCState *gcstate = &tstate->interp->gc; + + // gc_collect_main() must not be called before _PyGC_Init + // or after _PyGC_Fini() + assert(gcstate->garbage != NULL); + assert(!_PyErr_Occurred(tstate)); + + int expected = 0; + if (!_Py_atomic_compare_exchange_int(&gcstate->collecting, &expected, 1)) { + // Don't start a garbage collection if one is already in progress. + return 0; + } + + if (generation == GENERATION_AUTO) { + // Select the oldest generation that needs collecting. We will collect + // objects from that generation and all generations younger than it. + generation = gc_select_generation(gcstate); + if (generation < 0) { + // No generation needs to be collected. + _Py_atomic_store_int(&gcstate->collecting, 0); + return 0; + } + } + + assert(generation >= 0 && generation < NUM_GENERATIONS); + +#ifdef Py_STATS + if (_Py_stats) { + _Py_stats->object_stats.object_visits = 0; + } +#endif + GC_STAT_ADD(generation, collections, 1); + + if (reason != _Py_GC_REASON_SHUTDOWN) { + invoke_gc_callback(tstate, "start", generation, 0, 0); + } + + if (gcstate->debug & _PyGC_DEBUG_STATS) { + PySys_WriteStderr("gc: collecting generation %d...\n", generation); + show_stats_each_generations(gcstate); + t1 = _PyTime_GetPerfCounter(); + } + + if (PyDTrace_GC_START_ENABLED()) { + PyDTrace_GC_START(generation); + } + + /* update collection and allocation counters */ + if (generation+1 < NUM_GENERATIONS) { + gcstate->generations[generation+1].count += 1; + } + for (i = 0; i <= generation; i++) { + gcstate->generations[i].count = 0; + } + + /* merge younger generations with one we are currently collecting */ + for (i = 0; i < generation; i++) { + gc_list_merge(GEN_HEAD(gcstate, i), GEN_HEAD(gcstate, generation)); + } + + /* handy references */ + young = GEN_HEAD(gcstate, generation); + if (generation < NUM_GENERATIONS-1) { + old = GEN_HEAD(gcstate, generation+1); + } + else { + old = young; + } + validate_list(old, collecting_clear_unreachable_clear); + + deduce_unreachable(young, &unreachable); + + untrack_tuples(young); + /* Move reachable objects to next generation. */ + if (young != old) { + if (generation == NUM_GENERATIONS - 2) { + gcstate->long_lived_pending += gc_list_size(young); + } + gc_list_merge(young, old); + } + else { + /* We only un-track dicts in full collections, to avoid quadratic + dict build-up. See issue #14775. */ + untrack_dicts(young); + gcstate->long_lived_pending = 0; + gcstate->long_lived_total = gc_list_size(young); + } + + /* All objects in unreachable are trash, but objects reachable from + * legacy finalizers (e.g. tp_del) can't safely be deleted. + */ + gc_list_init(&finalizers); + // NEXT_MASK_UNREACHABLE is cleared here. + // After move_legacy_finalizers(), unreachable is normal list. + move_legacy_finalizers(&unreachable, &finalizers); + /* finalizers contains the unreachable objects with a legacy finalizer; + * unreachable objects reachable *from* those are also uncollectable, + * and we move those into the finalizers list too. + */ + move_legacy_finalizer_reachable(&finalizers); + + validate_list(&finalizers, collecting_clear_unreachable_clear); + validate_list(&unreachable, collecting_set_unreachable_clear); + + /* Print debugging information. */ + if (gcstate->debug & _PyGC_DEBUG_COLLECTABLE) { + for (gc = GC_NEXT(&unreachable); gc != &unreachable; gc = GC_NEXT(gc)) { + debug_cycle("collectable", FROM_GC(gc)); + } + } + + /* Clear weakrefs and invoke callbacks as necessary. */ + m += handle_weakrefs(&unreachable, old); + + validate_list(old, collecting_clear_unreachable_clear); + validate_list(&unreachable, collecting_set_unreachable_clear); + + /* Call tp_finalize on objects which have one. */ + finalize_garbage(tstate, &unreachable); + + /* Handle any objects that may have resurrected after the call + * to 'finalize_garbage' and continue the collection with the + * objects that are still unreachable */ + PyGC_Head final_unreachable; + handle_resurrected_objects(&unreachable, &final_unreachable, old); + + /* Call tp_clear on objects in the final_unreachable set. This will cause + * the reference cycles to be broken. It may also cause some objects + * in finalizers to be freed. + */ + m += gc_list_size(&final_unreachable); + delete_garbage(tstate, gcstate, &final_unreachable, old); + + /* Collect statistics on uncollectable objects found and print + * debugging information. */ + for (gc = GC_NEXT(&finalizers); gc != &finalizers; gc = GC_NEXT(gc)) { + n++; + if (gcstate->debug & _PyGC_DEBUG_UNCOLLECTABLE) + debug_cycle("uncollectable", FROM_GC(gc)); + } + if (gcstate->debug & _PyGC_DEBUG_STATS) { + double d = _PyTime_AsSecondsDouble(_PyTime_GetPerfCounter() - t1); + PySys_WriteStderr( + "gc: done, %zd unreachable, %zd uncollectable, %.4fs elapsed\n", + n+m, n, d); + } + + /* Append instances in the uncollectable set to a Python + * reachable list of garbage. The programmer has to deal with + * this if they insist on creating this type of structure. + */ + handle_legacy_finalizers(tstate, gcstate, &finalizers, old); + validate_list(old, collecting_clear_unreachable_clear); + + /* Clear free list only during the collection of the highest + * generation */ + if (generation == NUM_GENERATIONS-1) { + _PyGC_ClearAllFreeLists(tstate->interp); + } + + if (_PyErr_Occurred(tstate)) { + if (reason == _Py_GC_REASON_SHUTDOWN) { + _PyErr_Clear(tstate); + } + else { + PyErr_FormatUnraisable("Exception ignored in garbage collection"); + } + } + + /* Update stats */ + struct gc_generation_stats *stats = &gcstate->generation_stats[generation]; + stats->collections++; + stats->collected += m; + stats->uncollectable += n; + + GC_STAT_ADD(generation, objects_collected, m); +#ifdef Py_STATS + if (_Py_stats) { + GC_STAT_ADD(generation, object_visits, + _Py_stats->object_stats.object_visits); + _Py_stats->object_stats.object_visits = 0; + } +#endif + + if (PyDTrace_GC_DONE_ENABLED()) { + PyDTrace_GC_DONE(n + m); + } + + if (reason != _Py_GC_REASON_SHUTDOWN) { + invoke_gc_callback(tstate, "stop", generation, m, n); + } + + assert(!_PyErr_Occurred(tstate)); + _Py_atomic_store_int(&gcstate->collecting, 0); + return n + m; +} + +static int +referrersvisit(PyObject* obj, void *arg) +{ + PyObject *objs = arg; + Py_ssize_t i; + for (i = 0; i < PyTuple_GET_SIZE(objs); i++) { + if (PyTuple_GET_ITEM(objs, i) == obj) { + return 1; + } + } + return 0; +} + +static int +gc_referrers_for(PyObject *objs, PyGC_Head *list, PyObject *resultlist) +{ + PyGC_Head *gc; + PyObject *obj; + traverseproc traverse; + for (gc = GC_NEXT(list); gc != list; gc = GC_NEXT(gc)) { + obj = FROM_GC(gc); + traverse = Py_TYPE(obj)->tp_traverse; + if (obj == objs || obj == resultlist) { + continue; + } + if (traverse(obj, referrersvisit, objs)) { + if (PyList_Append(resultlist, obj) < 0) { + return 0; /* error */ + } + } + } + return 1; /* no error */ +} + +PyObject * +_PyGC_GetReferrers(PyInterpreterState *interp, PyObject *objs) +{ + PyObject *result = PyList_New(0); + if (!result) { + return NULL; + } + + GCState *gcstate = &interp->gc; + for (int i = 0; i < NUM_GENERATIONS; i++) { + if (!(gc_referrers_for(objs, GEN_HEAD(gcstate, i), result))) { + Py_DECREF(result); + return NULL; + } + } + return result; +} + +PyObject * +_PyGC_GetObjects(PyInterpreterState *interp, Py_ssize_t generation) +{ + assert(generation >= -1 && generation < NUM_GENERATIONS); + GCState *gcstate = &interp->gc; + + PyObject *result = PyList_New(0); + if (result == NULL) { + return NULL; + } + + if (generation == -1) { + /* If generation is -1, get all objects from all generations */ + for (int i = 0; i < NUM_GENERATIONS; i++) { + if (append_objects(result, GEN_HEAD(gcstate, i))) { + goto error; + } + } + } + else { + if (append_objects(result, GEN_HEAD(gcstate, generation))) { + goto error; + } + } + + return result; +error: + Py_DECREF(result); + return NULL; +} + +void +_PyGC_Freeze(PyInterpreterState *interp) +{ + GCState *gcstate = &interp->gc; + for (int i = 0; i < NUM_GENERATIONS; ++i) { + gc_list_merge(GEN_HEAD(gcstate, i), &gcstate->permanent_generation.head); + gcstate->generations[i].count = 0; + } +} + +void +_PyGC_Unfreeze(PyInterpreterState *interp) +{ + GCState *gcstate = &interp->gc; + gc_list_merge(&gcstate->permanent_generation.head, + GEN_HEAD(gcstate, NUM_GENERATIONS-1)); +} + +Py_ssize_t +_PyGC_GetFreezeCount(PyInterpreterState *interp) +{ + GCState *gcstate = &interp->gc; + return gc_list_size(&gcstate->permanent_generation.head); +} + +/* C API for controlling the state of the garbage collector */ +int +PyGC_Enable(void) +{ + GCState *gcstate = get_gc_state(); + int old_state = gcstate->enabled; + gcstate->enabled = 1; + return old_state; +} + +int +PyGC_Disable(void) +{ + GCState *gcstate = get_gc_state(); + int old_state = gcstate->enabled; + gcstate->enabled = 0; + return old_state; +} + +int +PyGC_IsEnabled(void) +{ + GCState *gcstate = get_gc_state(); + return gcstate->enabled; +} + +/* Public API to invoke gc.collect() from C */ +Py_ssize_t +PyGC_Collect(void) +{ + PyThreadState *tstate = _PyThreadState_GET(); + GCState *gcstate = &tstate->interp->gc; + + if (!gcstate->enabled) { + return 0; + } + + Py_ssize_t n; + PyObject *exc = _PyErr_GetRaisedException(tstate); + n = gc_collect_main(tstate, NUM_GENERATIONS - 1, _Py_GC_REASON_MANUAL); + _PyErr_SetRaisedException(tstate, exc); + + return n; +} + +Py_ssize_t +_PyGC_Collect(PyThreadState *tstate, int generation, _PyGC_Reason reason) +{ + return gc_collect_main(tstate, generation, reason); +} + +Py_ssize_t +_PyGC_CollectNoFail(PyThreadState *tstate) +{ + /* Ideally, this function is only called on interpreter shutdown, + and therefore not recursively. Unfortunately, when there are daemon + threads, a daemon thread can start a cyclic garbage collection + during interpreter shutdown (and then never finish it). + See http://bugs.python.org/issue8713#msg195178 for an example. + */ + return gc_collect_main(tstate, NUM_GENERATIONS - 1, _Py_GC_REASON_SHUTDOWN); +} + +void +_PyGC_DumpShutdownStats(PyInterpreterState *interp) +{ + GCState *gcstate = &interp->gc; + if (!(gcstate->debug & _PyGC_DEBUG_SAVEALL) + && gcstate->garbage != NULL && PyList_GET_SIZE(gcstate->garbage) > 0) { + const char *message; + if (gcstate->debug & _PyGC_DEBUG_UNCOLLECTABLE) { + message = "gc: %zd uncollectable objects at shutdown"; + } + else { + message = "gc: %zd uncollectable objects at shutdown; " \ + "use gc.set_debug(gc.DEBUG_UNCOLLECTABLE) to list them"; + } + /* PyErr_WarnFormat does too many things and we are at shutdown, + the warnings module's dependencies (e.g. linecache) may be gone + already. */ + if (PyErr_WarnExplicitFormat(PyExc_ResourceWarning, "gc", 0, + "gc", NULL, message, + PyList_GET_SIZE(gcstate->garbage))) + { + PyErr_WriteUnraisable(NULL); + } + if (gcstate->debug & _PyGC_DEBUG_UNCOLLECTABLE) { + PyObject *repr = NULL, *bytes = NULL; + repr = PyObject_Repr(gcstate->garbage); + if (!repr || !(bytes = PyUnicode_EncodeFSDefault(repr))) { + PyErr_WriteUnraisable(gcstate->garbage); + } + else { + PySys_WriteStderr( + " %s\n", + PyBytes_AS_STRING(bytes) + ); + } + Py_XDECREF(repr); + Py_XDECREF(bytes); + } + } +} + + +void +_PyGC_Fini(PyInterpreterState *interp) +{ + GCState *gcstate = &interp->gc; + Py_CLEAR(gcstate->garbage); + Py_CLEAR(gcstate->callbacks); + + /* We expect that none of this interpreters objects are shared + with other interpreters. + See https://github.com/python/cpython/issues/90228. */ +} + +/* for debugging */ +void +_PyGC_Dump(PyGC_Head *g) +{ + _PyObject_Dump(FROM_GC(g)); +} + + +#ifdef Py_DEBUG +static int +visit_validate(PyObject *op, void *parent_raw) +{ + PyObject *parent = _PyObject_CAST(parent_raw); + if (_PyObject_IsFreed(op)) { + _PyObject_ASSERT_FAILED_MSG(parent, + "PyObject_GC_Track() object is not valid"); + } + return 0; +} +#endif + + +/* extension modules might be compiled with GC support so these + functions must always be available */ + +void +PyObject_GC_Track(void *op_raw) +{ + PyObject *op = _PyObject_CAST(op_raw); + if (_PyObject_GC_IS_TRACKED(op)) { + _PyObject_ASSERT_FAILED_MSG(op, + "object already tracked " + "by the garbage collector"); + } + _PyObject_GC_TRACK(op); + +#ifdef Py_DEBUG + /* Check that the object is valid: validate objects traversed + by tp_traverse() */ + traverseproc traverse = Py_TYPE(op)->tp_traverse; + (void)traverse(op, visit_validate, op); +#endif +} + +void +PyObject_GC_UnTrack(void *op_raw) +{ + PyObject *op = _PyObject_CAST(op_raw); + /* Obscure: the Py_TRASHCAN mechanism requires that we be able to + * call PyObject_GC_UnTrack twice on an object. + */ + if (_PyObject_GC_IS_TRACKED(op)) { + _PyObject_GC_UNTRACK(op); + } +} + +int +PyObject_IS_GC(PyObject *obj) +{ + return _PyObject_IS_GC(obj); +} + +void +_Py_ScheduleGC(PyInterpreterState *interp) +{ + _Py_set_eval_breaker_bit(interp, _PY_GC_SCHEDULED_BIT, 1); +} + +void +_PyObject_GC_Link(PyObject *op) +{ + PyGC_Head *g = AS_GC(op); + assert(((uintptr_t)g & (sizeof(uintptr_t)-1)) == 0); // g must be correctly aligned + + PyThreadState *tstate = _PyThreadState_GET(); + GCState *gcstate = &tstate->interp->gc; + g->_gc_next = 0; + g->_gc_prev = 0; + gcstate->generations[0].count++; /* number of allocated GC objects */ + if (gcstate->generations[0].count > gcstate->generations[0].threshold && + gcstate->enabled && + gcstate->generations[0].threshold && + !_Py_atomic_load_int_relaxed(&gcstate->collecting) && + !_PyErr_Occurred(tstate)) + { + _Py_ScheduleGC(tstate->interp); + } +} + +void +_Py_RunGC(PyThreadState *tstate) +{ + gc_collect_main(tstate, GENERATION_AUTO, _Py_GC_REASON_HEAP); +} + +static PyObject * +gc_alloc(PyTypeObject *tp, size_t basicsize, size_t presize) +{ + PyThreadState *tstate = _PyThreadState_GET(); + if (basicsize > PY_SSIZE_T_MAX - presize) { + return _PyErr_NoMemory(tstate); + } + size_t size = presize + basicsize; + char *mem = _PyObject_MallocWithType(tp, size); + if (mem == NULL) { + return _PyErr_NoMemory(tstate); + } + ((PyObject **)mem)[0] = NULL; + ((PyObject **)mem)[1] = NULL; + PyObject *op = (PyObject *)(mem + presize); + _PyObject_GC_Link(op); + return op; +} + +PyObject * +_PyObject_GC_New(PyTypeObject *tp) +{ + size_t presize = _PyType_PreHeaderSize(tp); + PyObject *op = gc_alloc(tp, _PyObject_SIZE(tp), presize); + if (op == NULL) { + return NULL; + } + _PyObject_Init(op, tp); + return op; +} + +PyVarObject * +_PyObject_GC_NewVar(PyTypeObject *tp, Py_ssize_t nitems) +{ + PyVarObject *op; + + if (nitems < 0) { + PyErr_BadInternalCall(); + return NULL; + } + size_t presize = _PyType_PreHeaderSize(tp); + size_t size = _PyObject_VAR_SIZE(tp, nitems); + op = (PyVarObject *)gc_alloc(tp, size, presize); + if (op == NULL) { + return NULL; + } + _PyObject_InitVar(op, tp, nitems); + return op; +} + +PyObject * +PyUnstable_Object_GC_NewWithExtraData(PyTypeObject *tp, size_t extra_size) +{ + size_t presize = _PyType_PreHeaderSize(tp); + PyObject *op = gc_alloc(tp, _PyObject_SIZE(tp) + extra_size, presize); + if (op == NULL) { + return NULL; + } + memset(op, 0, _PyObject_SIZE(tp) + extra_size); + _PyObject_Init(op, tp); + return op; +} + +PyVarObject * +_PyObject_GC_Resize(PyVarObject *op, Py_ssize_t nitems) +{ + const size_t basicsize = _PyObject_VAR_SIZE(Py_TYPE(op), nitems); + const size_t presize = _PyType_PreHeaderSize(((PyObject *)op)->ob_type); + _PyObject_ASSERT((PyObject *)op, !_PyObject_GC_IS_TRACKED(op)); + if (basicsize > (size_t)PY_SSIZE_T_MAX - presize) { + return (PyVarObject *)PyErr_NoMemory(); + } + char *mem = (char *)op - presize; + mem = (char *)_PyObject_ReallocWithType(Py_TYPE(op), mem, presize + basicsize); + if (mem == NULL) { + return (PyVarObject *)PyErr_NoMemory(); + } + op = (PyVarObject *) (mem + presize); + Py_SET_SIZE(op, nitems); + return op; +} + +void +PyObject_GC_Del(void *op) +{ + size_t presize = _PyType_PreHeaderSize(((PyObject *)op)->ob_type); + PyGC_Head *g = AS_GC(op); + if (_PyObject_GC_IS_TRACKED(op)) { + gc_list_remove(g); +#ifdef Py_DEBUG + PyObject *exc = PyErr_GetRaisedException(); + if (PyErr_WarnExplicitFormat(PyExc_ResourceWarning, "gc", 0, + "gc", NULL, "Object of type %s is not untracked before destruction", + ((PyObject*)op)->ob_type->tp_name)) { + PyErr_WriteUnraisable(NULL); + } + PyErr_SetRaisedException(exc); +#endif + } + GCState *gcstate = get_gc_state(); + if (gcstate->generations[0].count > 0) { + gcstate->generations[0].count--; + } + PyObject_Free(((char *)op)-presize); +} + +int +PyObject_GC_IsTracked(PyObject* obj) +{ + if (_PyObject_IS_GC(obj) && _PyObject_GC_IS_TRACKED(obj)) { + return 1; + } + return 0; +} + +int +PyObject_GC_IsFinalized(PyObject *obj) +{ + if (_PyObject_IS_GC(obj) && _PyGC_FINALIZED(obj)) { + return 1; + } + return 0; +} + +void +PyUnstable_GC_VisitObjects(gcvisitobjects_t callback, void *arg) +{ + size_t i; + GCState *gcstate = get_gc_state(); + int origenstate = gcstate->enabled; + gcstate->enabled = 0; + for (i = 0; i < NUM_GENERATIONS; i++) { + PyGC_Head *gc_list, *gc; + gc_list = GEN_HEAD(gcstate, i); + for (gc = GC_NEXT(gc_list); gc != gc_list; gc = GC_NEXT(gc)) { + PyObject *op = FROM_GC(gc); + Py_INCREF(op); + int res = callback(op, arg); + Py_DECREF(op); + if (!res) { + goto done; + } + } + } +done: + gcstate->enabled = origenstate; +} + +#endif // Py_GIL_DISABLED diff --git a/Python/gc_free_threading.c b/Python/gc_free_threading.c new file mode 100644 index 00000000000000..93e1168002b6f7 --- /dev/null +++ b/Python/gc_free_threading.c @@ -0,0 +1,1730 @@ +// Cyclic garbage collector implementation for free-threaded build. +#include "Python.h" +#include "pycore_brc.h" // struct _brc_thread_state +#include "pycore_ceval.h" // _Py_set_eval_breaker_bit() +#include "pycore_context.h" +#include "pycore_dict.h" // _PyDict_MaybeUntrack() +#include "pycore_initconfig.h" +#include "pycore_interp.h" // PyInterpreterState.gc +#include "pycore_object.h" +#include "pycore_object_alloc.h" // _PyObject_MallocWithType() +#include "pycore_object_stack.h" +#include "pycore_pyerrors.h" +#include "pycore_pystate.h" // _PyThreadState_GET() +#include "pycore_tstate.h" // _PyThreadStateImpl +#include "pycore_weakref.h" // _PyWeakref_ClearRef() +#include "pydtrace.h" + +#ifdef Py_GIL_DISABLED + +typedef struct _gc_runtime_state GCState; + +#ifdef Py_DEBUG +# define GC_DEBUG +#endif + +// Automatically choose the generation that needs collecting. +#define GENERATION_AUTO (-1) + +// A linked list of objects using the `ob_tid` field as the next pointer. +// The linked list pointers are distinct from any real thread ids, because the +// thread ids returned by _Py_ThreadId() are also pointers to distinct objects. +// No thread will confuse its own id with a linked list pointer. +struct worklist { + uintptr_t head; +}; + +struct worklist_iter { + uintptr_t *ptr; // pointer to current object + uintptr_t *next; // next value of ptr +}; + +struct visitor_args { + size_t offset; // offset of PyObject from start of block +}; + +// Per-collection state +struct collection_state { + struct visitor_args base; + PyInterpreterState *interp; + GCState *gcstate; + Py_ssize_t collected; + Py_ssize_t uncollectable; + Py_ssize_t long_lived_total; + struct worklist unreachable; + struct worklist legacy_finalizers; + struct worklist wrcb_to_call; + struct worklist objs_to_decref; +}; + +// iterate over a worklist +#define WORKSTACK_FOR_EACH(stack, op) \ + for ((op) = (PyObject *)(stack)->head; (op) != NULL; (op) = (PyObject *)(op)->ob_tid) + +// iterate over a worklist with support for removing the current object +#define WORKSTACK_FOR_EACH_ITER(stack, iter, op) \ + for (worklist_iter_init((iter), &(stack)->head), (op) = (PyObject *)(*(iter)->ptr); \ + (op) != NULL; \ + worklist_iter_init((iter), (iter)->next), (op) = (PyObject *)(*(iter)->ptr)) + +static void +worklist_push(struct worklist *worklist, PyObject *op) +{ + assert(op->ob_tid == 0); + op->ob_tid = worklist->head; + worklist->head = (uintptr_t)op; +} + +static PyObject * +worklist_pop(struct worklist *worklist) +{ + PyObject *op = (PyObject *)worklist->head; + if (op != NULL) { + worklist->head = op->ob_tid; + op->ob_tid = 0; + } + return op; +} + +static void +worklist_iter_init(struct worklist_iter *iter, uintptr_t *next) +{ + iter->ptr = next; + PyObject *op = (PyObject *)*(iter->ptr); + if (op) { + iter->next = &op->ob_tid; + } +} + +static void +worklist_remove(struct worklist_iter *iter) +{ + PyObject *op = (PyObject *)*(iter->ptr); + *(iter->ptr) = op->ob_tid; + op->ob_tid = 0; + iter->next = iter->ptr; +} + +static inline int +gc_is_unreachable(PyObject *op) +{ + return (op->ob_gc_bits & _PyGC_BITS_UNREACHABLE) != 0; +} + +static void +gc_set_unreachable(PyObject *op) +{ + op->ob_gc_bits |= _PyGC_BITS_UNREACHABLE; +} + +static void +gc_clear_unreachable(PyObject *op) +{ + op->ob_gc_bits &= ~_PyGC_BITS_UNREACHABLE; +} + +// Initialize the `ob_tid` field to zero if the object is not already +// initialized as unreachable. +static void +gc_maybe_init_refs(PyObject *op) +{ + if (!gc_is_unreachable(op)) { + gc_set_unreachable(op); + op->ob_tid = 0; + } +} + +static inline Py_ssize_t +gc_get_refs(PyObject *op) +{ + return (Py_ssize_t)op->ob_tid; +} + +static inline void +gc_add_refs(PyObject *op, Py_ssize_t refs) +{ + assert(_PyObject_GC_IS_TRACKED(op)); + op->ob_tid += refs; +} + +static inline void +gc_decref(PyObject *op) +{ + op->ob_tid -= 1; +} + +static Py_ssize_t +merge_refcount(PyObject *op, Py_ssize_t extra) +{ + assert(_PyInterpreterState_GET()->stoptheworld.world_stopped); + + Py_ssize_t refcount = Py_REFCNT(op); + refcount += extra; + +#ifdef Py_REF_DEBUG + _Py_AddRefTotal(_PyInterpreterState_GET(), extra); +#endif + + // No atomics necessary; all other threads in this interpreter are paused. + op->ob_tid = 0; + op->ob_ref_local = 0; + op->ob_ref_shared = _Py_REF_SHARED(refcount, _Py_REF_MERGED); + return refcount; +} + +static void +gc_restore_tid(PyObject *op) +{ + mi_segment_t *segment = _mi_ptr_segment(op); + if (_Py_REF_IS_MERGED(op->ob_ref_shared)) { + op->ob_tid = 0; + } + else { + // NOTE: may change ob_tid if the object was re-initialized by + // a different thread or its segment was abandoned and reclaimed. + // The segment thread id might be zero, in which case we should + // ensure the refcounts are now merged. + op->ob_tid = segment->thread_id; + if (op->ob_tid == 0) { + merge_refcount(op, 0); + } + } +} + +static void +gc_restore_refs(PyObject *op) +{ + if (gc_is_unreachable(op)) { + gc_restore_tid(op); + gc_clear_unreachable(op); + } +} + +// Given a mimalloc memory block return the PyObject stored in it or NULL if +// the block is not allocated or the object is not tracked or is immortal. +static PyObject * +op_from_block(void *block, void *arg, bool include_frozen) +{ + struct visitor_args *a = arg; + if (block == NULL) { + return NULL; + } + PyObject *op = (PyObject *)((char*)block + a->offset); + assert(PyObject_IS_GC(op)); + if (!_PyObject_GC_IS_TRACKED(op)) { + return NULL; + } + if (!include_frozen && (op->ob_gc_bits & _PyGC_BITS_FROZEN) != 0) { + return NULL; + } + return op; +} + +static int +gc_visit_heaps_lock_held(PyInterpreterState *interp, mi_block_visit_fun *visitor, + struct visitor_args *arg) +{ + // Offset of PyObject header from start of memory block. + Py_ssize_t offset_base = 0; + if (_PyMem_DebugEnabled()) { + // The debug allocator adds two words at the beginning of each block. + offset_base += 2 * sizeof(size_t); + } + + // Objects with Py_TPFLAGS_PREHEADER have two extra fields + Py_ssize_t offset_pre = offset_base + 2 * sizeof(PyObject*); + + // visit each thread's heaps for GC objects + for (PyThreadState *p = interp->threads.head; p != NULL; p = p->next) { + struct _mimalloc_thread_state *m = &((_PyThreadStateImpl *)p)->mimalloc; + + arg->offset = offset_base; + if (!mi_heap_visit_blocks(&m->heaps[_Py_MIMALLOC_HEAP_GC], true, + visitor, arg)) { + return -1; + } + arg->offset = offset_pre; + if (!mi_heap_visit_blocks(&m->heaps[_Py_MIMALLOC_HEAP_GC_PRE], true, + visitor, arg)) { + return -1; + } + } + + // visit blocks in the per-interpreter abandoned pool (from dead threads) + mi_abandoned_pool_t *pool = &interp->mimalloc.abandoned_pool; + arg->offset = offset_base; + if (!_mi_abandoned_pool_visit_blocks(pool, _Py_MIMALLOC_HEAP_GC, true, + visitor, arg)) { + return -1; + } + arg->offset = offset_pre; + if (!_mi_abandoned_pool_visit_blocks(pool, _Py_MIMALLOC_HEAP_GC_PRE, true, + visitor, arg)) { + return -1; + } + return 0; +} + +// Visits all GC objects in the interpreter's heaps. +// NOTE: It is not safe to allocate or free any mimalloc managed memory while +// this function is running. +static int +gc_visit_heaps(PyInterpreterState *interp, mi_block_visit_fun *visitor, + struct visitor_args *arg) +{ + // Other threads in the interpreter must be paused so that we can safely + // traverse their heaps. + assert(interp->stoptheworld.world_stopped); + + int err; + HEAD_LOCK(&_PyRuntime); + err = gc_visit_heaps_lock_held(interp, visitor, arg); + HEAD_UNLOCK(&_PyRuntime); + return err; +} + +static void +merge_queued_objects(_PyThreadStateImpl *tstate, struct collection_state *state) +{ + struct _brc_thread_state *brc = &tstate->brc; + _PyObjectStack_Merge(&brc->local_objects_to_merge, &brc->objects_to_merge); + + PyObject *op; + while ((op = _PyObjectStack_Pop(&brc->local_objects_to_merge)) != NULL) { + // Subtract one when merging because the queue had a reference. + Py_ssize_t refcount = merge_refcount(op, -1); + + if (!_PyObject_GC_IS_TRACKED(op) && refcount == 0) { + // GC objects with zero refcount are handled subsequently by the + // GC as if they were cyclic trash, but we have to handle dead + // non-GC objects here. Add one to the refcount so that we can + // decref and deallocate the object once we start the world again. + op->ob_ref_shared += (1 << _Py_REF_SHARED_SHIFT); +#ifdef Py_REF_DEBUG + _Py_IncRefTotal(_PyInterpreterState_GET()); +#endif + worklist_push(&state->objs_to_decref, op); + } + } +} + +static void +merge_all_queued_objects(PyInterpreterState *interp, struct collection_state *state) +{ + HEAD_LOCK(&_PyRuntime); + for (PyThreadState *p = interp->threads.head; p != NULL; p = p->next) { + merge_queued_objects((_PyThreadStateImpl *)p, state); + } + HEAD_UNLOCK(&_PyRuntime); +} + +// Subtract an incoming reference from the computed "gc_refs" refcount. +static int +visit_decref(PyObject *op, void *arg) +{ + if (_PyObject_GC_IS_TRACKED(op) && !_Py_IsImmortal(op)) { + // If update_refs hasn't reached this object yet, mark it + // as (tentatively) unreachable and initialize ob_tid to zero. + gc_maybe_init_refs(op); + gc_decref(op); + } + return 0; +} + +// Compute the number of external references to objects in the heap +// by subtracting internal references from the refcount. The difference is +// computed in the ob_tid field (we restore it later). +static bool +update_refs(const mi_heap_t *heap, const mi_heap_area_t *area, + void *block, size_t block_size, void *args) +{ + PyObject *op = op_from_block(block, args, false); + if (op == NULL) { + return true; + } + + // Exclude immortal objects from garbage collection + if (_Py_IsImmortal(op)) { + op->ob_tid = 0; + _PyObject_GC_UNTRACK(op); + gc_clear_unreachable(op); + return true; + } + + // Untrack tuples and dicts as necessary in this pass. + if (PyTuple_CheckExact(op)) { + _PyTuple_MaybeUntrack(op); + if (!_PyObject_GC_IS_TRACKED(op)) { + gc_restore_refs(op); + return true; + } + } + else if (PyDict_CheckExact(op)) { + _PyDict_MaybeUntrack(op); + if (!_PyObject_GC_IS_TRACKED(op)) { + gc_restore_refs(op); + return true; + } + } + + Py_ssize_t refcount = Py_REFCNT(op); + _PyObject_ASSERT(op, refcount >= 0); + + // We repurpose ob_tid to compute "gc_refs", the number of external + // references to the object (i.e., from outside the GC heaps). This means + // that ob_tid is no longer a valid thread id until it is restored by + // scan_heap_visitor(). Until then, we cannot use the standard reference + // counting functions or allow other threads to run Python code. + gc_maybe_init_refs(op); + + // Add the actual refcount to ob_tid. + gc_add_refs(op, refcount); + + // Subtract internal references from ob_tid. Objects with ob_tid > 0 + // are directly reachable from outside containers, and so can't be + // collected. + Py_TYPE(op)->tp_traverse(op, visit_decref, NULL); + return true; +} + +static int +visit_clear_unreachable(PyObject *op, _PyObjectStack *stack) +{ + if (gc_is_unreachable(op)) { + _PyObject_ASSERT(op, _PyObject_GC_IS_TRACKED(op)); + gc_clear_unreachable(op); + return _PyObjectStack_Push(stack, op); + } + return 0; +} + +// Transitively clear the unreachable bit on all objects reachable from op. +static int +mark_reachable(PyObject *op) +{ + _PyObjectStack stack = { NULL }; + do { + traverseproc traverse = Py_TYPE(op)->tp_traverse; + if (traverse(op, (visitproc)&visit_clear_unreachable, &stack) < 0) { + _PyObjectStack_Clear(&stack); + return -1; + } + op = _PyObjectStack_Pop(&stack); + } while (op != NULL); + return 0; +} + +#ifdef GC_DEBUG +static bool +validate_gc_objects(const mi_heap_t *heap, const mi_heap_area_t *area, + void *block, size_t block_size, void *args) +{ + PyObject *op = op_from_block(block, args, false); + if (op == NULL) { + return true; + } + + _PyObject_ASSERT(op, gc_is_unreachable(op)); + _PyObject_ASSERT_WITH_MSG(op, gc_get_refs(op) >= 0, + "refcount is too small"); + return true; +} +#endif + +static bool +mark_heap_visitor(const mi_heap_t *heap, const mi_heap_area_t *area, + void *block, size_t block_size, void *args) +{ + PyObject *op = op_from_block(block, args, false); + if (op == NULL) { + return true; + } + + if (gc_is_unreachable(op) && gc_get_refs(op) != 0) { + // Object is reachable but currently marked as unreachable. + // Mark it as reachable and traverse its pointers to find + // any other object that may be directly reachable from it. + gc_clear_unreachable(op); + + // Transitively mark reachable objects by clearing the unreachable flag. + if (mark_reachable(op) < 0) { + return false; + } + } + + return true; +} + +/* Return true if object has a pre-PEP 442 finalization method. */ +static int +has_legacy_finalizer(PyObject *op) +{ + return Py_TYPE(op)->tp_del != NULL; +} + +static bool +scan_heap_visitor(const mi_heap_t *heap, const mi_heap_area_t *area, + void *block, size_t block_size, void *args) +{ + PyObject *op = op_from_block(block, args, false); + if (op == NULL) { + return true; + } + + struct collection_state *state = (struct collection_state *)args; + if (gc_is_unreachable(op)) { + // Merge and add one to the refcount to prevent deallocation while we + // are holding on to it in a worklist. + merge_refcount(op, 1); + + if (has_legacy_finalizer(op)) { + // would be unreachable, but has legacy finalizer + gc_clear_unreachable(op); + worklist_push(&state->legacy_finalizers, op); + } + else { + worklist_push(&state->unreachable, op); + } + } + else { + // object is reachable, restore `ob_tid`; we're done with these objects + gc_restore_tid(op); + state->long_lived_total++; + } + + return true; +} + +static int +move_legacy_finalizer_reachable(struct collection_state *state); + +static int +deduce_unreachable_heap(PyInterpreterState *interp, + struct collection_state *state) +{ + // Identify objects that are directly reachable from outside the GC heap + // by computing the difference between the refcount and the number of + // incoming references. + gc_visit_heaps(interp, &update_refs, &state->base); + +#ifdef GC_DEBUG + // Check that all objects are marked as unreachable and that the computed + // reference count difference (stored in `ob_tid`) is non-negative. + gc_visit_heaps(interp, &validate_gc_objects, &state->base); +#endif + + // Transitively mark reachable objects by clearing the + // _PyGC_BITS_UNREACHABLE flag. + if (gc_visit_heaps(interp, &mark_heap_visitor, &state->base) < 0) { + return -1; + } + + // Identify remaining unreachable objects and push them onto a stack. + // Restores ob_tid for reachable objects. + gc_visit_heaps(interp, &scan_heap_visitor, &state->base); + + if (state->legacy_finalizers.head) { + // There may be objects reachable from legacy finalizers that are in + // the unreachable set. We need to mark them as reachable. + if (move_legacy_finalizer_reachable(state) < 0) { + return -1; + } + } + + return 0; +} + +static int +move_legacy_finalizer_reachable(struct collection_state *state) +{ + // Clear the reachable bit on all objects transitively reachable + // from the objects with legacy finalizers. + PyObject *op; + WORKSTACK_FOR_EACH(&state->legacy_finalizers, op) { + if (mark_reachable(op) < 0) { + return -1; + } + } + + // Move the reachable objects from the unreachable worklist to the legacy + // finalizer worklist. + struct worklist_iter iter; + WORKSTACK_FOR_EACH_ITER(&state->unreachable, &iter, op) { + if (!gc_is_unreachable(op)) { + worklist_remove(&iter); + worklist_push(&state->legacy_finalizers, op); + } + } + + return 0; +} + +// Clear all weakrefs to unreachable objects. Weakrefs with callbacks are +// enqueued in `wrcb_to_call`, but not invoked yet. +static void +clear_weakrefs(struct collection_state *state) +{ + PyObject *op; + WORKSTACK_FOR_EACH(&state->unreachable, op) { + if (PyWeakref_Check(op)) { + // Clear weakrefs that are themselves unreachable to ensure their + // callbacks will not be executed later from a `tp_clear()` + // inside delete_garbage(). That would be unsafe: it could + // resurrect a dead object or access a an already cleared object. + // See bpo-38006 for one example. + _PyWeakref_ClearRef((PyWeakReference *)op); + } + + if (!_PyType_SUPPORTS_WEAKREFS(Py_TYPE(op))) { + continue; + } + + // NOTE: This is never triggered for static types so we can avoid the + // (slightly) more costly _PyObject_GET_WEAKREFS_LISTPTR(). + PyWeakReference **wrlist = _PyObject_GET_WEAKREFS_LISTPTR_FROM_OFFSET(op); + + // `op` may have some weakrefs. March over the list, clear + // all the weakrefs, and enqueue the weakrefs with callbacks + // that must be called into wrcb_to_call. + for (PyWeakReference *wr = *wrlist; wr != NULL; wr = *wrlist) { + // _PyWeakref_ClearRef clears the weakref but leaves + // the callback pointer intact. Obscure: it also + // changes *wrlist. + _PyObject_ASSERT((PyObject *)wr, wr->wr_object == op); + _PyWeakref_ClearRef(wr); + _PyObject_ASSERT((PyObject *)wr, wr->wr_object == Py_None); + + // We do not invoke callbacks for weakrefs that are themselves + // unreachable. This is partly for historical reasons: weakrefs + // predate safe object finalization, and a weakref that is itself + // unreachable may have a callback that resurrects other + // unreachable objects. + if (wr->wr_callback == NULL || gc_is_unreachable((PyObject *)wr)) { + continue; + } + + // Create a new reference so that wr can't go away before we can + // process it again. + merge_refcount((PyObject *)wr, 1); + + // Enqueue weakref to be called later. + worklist_push(&state->wrcb_to_call, (PyObject *)wr); + } + } +} + +static void +call_weakref_callbacks(struct collection_state *state) +{ + // Invoke the callbacks we decided to honor. + PyObject *op; + while ((op = worklist_pop(&state->wrcb_to_call)) != NULL) { + _PyObject_ASSERT(op, PyWeakref_Check(op)); + + PyWeakReference *wr = (PyWeakReference *)op; + PyObject *callback = wr->wr_callback; + _PyObject_ASSERT(op, callback != NULL); + + /* copy-paste of weakrefobject.c's handle_callback() */ + PyObject *temp = PyObject_CallOneArg(callback, (PyObject *)wr); + if (temp == NULL) { + PyErr_WriteUnraisable(callback); + } + else { + Py_DECREF(temp); + } + + gc_restore_tid(op); + Py_DECREF(op); // drop worklist reference + } +} + + +static GCState * +get_gc_state(void) +{ + PyInterpreterState *interp = _PyInterpreterState_GET(); + return &interp->gc; +} + + +void +_PyGC_InitState(GCState *gcstate) +{ + // TODO: move to pycore_runtime_init.h once the incremental GC lands. + gcstate->generations[0].threshold = 2000; +} + + +PyStatus +_PyGC_Init(PyInterpreterState *interp) +{ + GCState *gcstate = &interp->gc; + + gcstate->garbage = PyList_New(0); + if (gcstate->garbage == NULL) { + return _PyStatus_NO_MEMORY(); + } + + gcstate->callbacks = PyList_New(0); + if (gcstate->callbacks == NULL) { + return _PyStatus_NO_MEMORY(); + } + + return _PyStatus_OK(); +} + +static void +debug_cycle(const char *msg, PyObject *op) +{ + PySys_FormatStderr("gc: %s <%s %p>\n", + msg, Py_TYPE(op)->tp_name, op); +} + +/* Run first-time finalizers (if any) on all the objects in collectable. + * Note that this may remove some (or even all) of the objects from the + * list, due to refcounts falling to 0. + */ +static void +finalize_garbage(struct collection_state *state) +{ + // NOTE: the unreachable worklist holds a strong reference to the object + // to prevent it from being deallocated while we are holding on to it. + PyObject *op; + WORKSTACK_FOR_EACH(&state->unreachable, op) { + if (!_PyGC_FINALIZED(op)) { + destructor finalize = Py_TYPE(op)->tp_finalize; + if (finalize != NULL) { + _PyGC_SET_FINALIZED(op); + finalize(op); + assert(!_PyErr_Occurred(_PyThreadState_GET())); + } + } + } +} + +// Break reference cycles by clearing the containers involved. +static void +delete_garbage(struct collection_state *state) +{ + PyThreadState *tstate = _PyThreadState_GET(); + GCState *gcstate = state->gcstate; + + assert(!_PyErr_Occurred(tstate)); + + PyObject *op; + while ((op = worklist_pop(&state->objs_to_decref)) != NULL) { + Py_DECREF(op); + } + + while ((op = worklist_pop(&state->unreachable)) != NULL) { + _PyObject_ASSERT(op, gc_is_unreachable(op)); + + // Clear the unreachable flag. + gc_clear_unreachable(op); + + if (!_PyObject_GC_IS_TRACKED(op)) { + // Object might have been untracked by some other tp_clear() call. + Py_DECREF(op); // drop the reference from the worklist + continue; + } + + state->collected++; + + if (gcstate->debug & _PyGC_DEBUG_SAVEALL) { + assert(gcstate->garbage != NULL); + if (PyList_Append(gcstate->garbage, op) < 0) { + _PyErr_Clear(tstate); + } + } + else { + inquiry clear = Py_TYPE(op)->tp_clear; + if (clear != NULL) { + (void) clear(op); + if (_PyErr_Occurred(tstate)) { + PyErr_FormatUnraisable("Exception ignored in tp_clear of %s", + Py_TYPE(op)->tp_name); + } + } + } + + Py_DECREF(op); // drop the reference from the worklist + } +} + +static void +handle_legacy_finalizers(struct collection_state *state) +{ + GCState *gcstate = state->gcstate; + assert(gcstate->garbage != NULL); + + PyObject *op; + while ((op = worklist_pop(&state->legacy_finalizers)) != NULL) { + state->uncollectable++; + + if (gcstate->debug & _PyGC_DEBUG_UNCOLLECTABLE) { + debug_cycle("uncollectable", op); + } + + if ((gcstate->debug & _PyGC_DEBUG_SAVEALL) || has_legacy_finalizer(op)) { + if (PyList_Append(gcstate->garbage, op) < 0) { + PyErr_Clear(); + } + } + Py_DECREF(op); // drop worklist reference + } +} + +// Show stats for objects in each generations +static void +show_stats_each_generations(GCState *gcstate) +{ + // TODO +} + +// Traversal callback for handle_resurrected_objects. +static int +visit_decref_unreachable(PyObject *op, void *data) +{ + if (gc_is_unreachable(op) && _PyObject_GC_IS_TRACKED(op)) { + op->ob_ref_local -= 1; + } + return 0; +} + +// Handle objects that may have resurrected after a call to 'finalize_garbage'. +static int +handle_resurrected_objects(struct collection_state *state) +{ + // First, find externally reachable objects by computing the reference + // count difference in ob_ref_local. We can't use ob_tid here because + // that's already used to store the unreachable worklist. + PyObject *op; + struct worklist_iter iter; + WORKSTACK_FOR_EACH_ITER(&state->unreachable, &iter, op) { + assert(gc_is_unreachable(op)); + assert(_Py_REF_IS_MERGED(op->ob_ref_shared)); + + if (!_PyObject_GC_IS_TRACKED(op)) { + // Object was untracked by a finalizer. Schedule it for a Py_DECREF + // after we finish with the stop-the-world pause. + gc_clear_unreachable(op); + worklist_remove(&iter); + worklist_push(&state->objs_to_decref, op); + continue; + } + + Py_ssize_t refcount = (op->ob_ref_shared >> _Py_REF_SHARED_SHIFT); + if (refcount > INT32_MAX) { + // The refcount is too big to fit in `ob_ref_local`. Mark the + // object as immortal and bail out. + gc_clear_unreachable(op); + worklist_remove(&iter); + _Py_SetImmortal(op); + continue; + } + + op->ob_ref_local += (uint32_t)refcount; + + // Subtract one to account for the reference from the worklist. + op->ob_ref_local -= 1; + + traverseproc traverse = Py_TYPE(op)->tp_traverse; + (void) traverse(op, + (visitproc)visit_decref_unreachable, + NULL); + } + + // Find resurrected objects + bool any_resurrected = false; + WORKSTACK_FOR_EACH(&state->unreachable, op) { + int32_t gc_refs = (int32_t)op->ob_ref_local; + op->ob_ref_local = 0; // restore ob_ref_local + + _PyObject_ASSERT(op, gc_refs >= 0); + + if (gc_is_unreachable(op) && gc_refs > 0) { + // Clear the unreachable flag on any transitively reachable objects + // from this one. + any_resurrected = true; + gc_clear_unreachable(op); + if (mark_reachable(op) < 0) { + return -1; + } + } + } + + if (any_resurrected) { + // Remove resurrected objects from the unreachable list. + WORKSTACK_FOR_EACH_ITER(&state->unreachable, &iter, op) { + if (!gc_is_unreachable(op)) { + _PyObject_ASSERT(op, Py_REFCNT(op) > 1); + worklist_remove(&iter); + merge_refcount(op, -1); // remove worklist reference + } + } + } + +#ifdef GC_DEBUG + WORKSTACK_FOR_EACH(&state->unreachable, op) { + _PyObject_ASSERT(op, gc_is_unreachable(op)); + _PyObject_ASSERT(op, _PyObject_GC_IS_TRACKED(op)); + _PyObject_ASSERT(op, op->ob_ref_local == 0); + _PyObject_ASSERT(op, _Py_REF_IS_MERGED(op->ob_ref_shared)); + } +#endif + + return 0; +} + + +/* Invoke progress callbacks to notify clients that garbage collection + * is starting or stopping + */ +static void +invoke_gc_callback(PyThreadState *tstate, const char *phase, + int generation, Py_ssize_t collected, + Py_ssize_t uncollectable) +{ + assert(!_PyErr_Occurred(tstate)); + + /* we may get called very early */ + GCState *gcstate = &tstate->interp->gc; + if (gcstate->callbacks == NULL) { + return; + } + + /* The local variable cannot be rebound, check it for sanity */ + assert(PyList_CheckExact(gcstate->callbacks)); + PyObject *info = NULL; + if (PyList_GET_SIZE(gcstate->callbacks) != 0) { + info = Py_BuildValue("{sisnsn}", + "generation", generation, + "collected", collected, + "uncollectable", uncollectable); + if (info == NULL) { + PyErr_FormatUnraisable("Exception ignored on invoking gc callbacks"); + return; + } + } + + PyObject *phase_obj = PyUnicode_FromString(phase); + if (phase_obj == NULL) { + Py_XDECREF(info); + PyErr_FormatUnraisable("Exception ignored on invoking gc callbacks"); + return; + } + + PyObject *stack[] = {phase_obj, info}; + for (Py_ssize_t i=0; icallbacks); i++) { + PyObject *r, *cb = PyList_GET_ITEM(gcstate->callbacks, i); + Py_INCREF(cb); /* make sure cb doesn't go away */ + r = PyObject_Vectorcall(cb, stack, 2, NULL); + if (r == NULL) { + PyErr_WriteUnraisable(cb); + } + else { + Py_DECREF(r); + } + Py_DECREF(cb); + } + Py_DECREF(phase_obj); + Py_XDECREF(info); + assert(!_PyErr_Occurred(tstate)); +} + +static void +cleanup_worklist(struct worklist *worklist) +{ + PyObject *op; + while ((op = worklist_pop(worklist)) != NULL) { + gc_restore_tid(op); + gc_clear_unreachable(op); + Py_DECREF(op); + } +} + +static bool +gc_should_collect(GCState *gcstate) +{ + int count = _Py_atomic_load_int_relaxed(&gcstate->generations[0].count); + int threshold = gcstate->generations[0].threshold; + if (count <= threshold || threshold == 0 || !gcstate->enabled) { + return false; + } + // Avoid quadratic behavior by scaling threshold to the number of live + // objects. A few tests rely on immediate scheduling of the GC so we ignore + // the scaled threshold if generations[1].threshold is set to zero. + return (count > gcstate->long_lived_total / 4 || + gcstate->generations[1].threshold == 0); +} + +static void +gc_collect_internal(PyInterpreterState *interp, struct collection_state *state) +{ + _PyEval_StopTheWorld(interp); + // merge refcounts for all queued objects + merge_all_queued_objects(interp, state); + + // Find unreachable objects + int err = deduce_unreachable_heap(interp, state); + if (err < 0) { + _PyEval_StartTheWorld(interp); + goto error; + } + + // Print debugging information. + if (interp->gc.debug & _PyGC_DEBUG_COLLECTABLE) { + PyObject *op; + WORKSTACK_FOR_EACH(&state->unreachable, op) { + debug_cycle("collectable", op); + } + } + + // Clear weakrefs and enqueue callbacks (but do not call them). + clear_weakrefs(state); + _PyEval_StartTheWorld(interp); + + // Deallocate any object from the refcount merge step + cleanup_worklist(&state->objs_to_decref); + + // Call weakref callbacks and finalizers after unpausing other threads to + // avoid potential deadlocks. + call_weakref_callbacks(state); + finalize_garbage(state); + + // Handle any objects that may have resurrected after the finalization. + _PyEval_StopTheWorld(interp); + err = handle_resurrected_objects(state); + _PyEval_StartTheWorld(interp); + + if (err < 0) { + goto error; + } + + // Call tp_clear on objects in the unreachable set. This will cause + // the reference cycles to be broken. It may also cause some objects + // to be freed. + delete_garbage(state); + + // Append objects with legacy finalizers to the "gc.garbage" list. + handle_legacy_finalizers(state); + return; + +error: + cleanup_worklist(&state->unreachable); + cleanup_worklist(&state->legacy_finalizers); + cleanup_worklist(&state->wrcb_to_call); + cleanup_worklist(&state->objs_to_decref); + PyErr_NoMemory(); + PyErr_FormatUnraisable("Out of memory during garbage collection"); +} + +/* This is the main function. Read this to understand how the + * collection process works. */ +static Py_ssize_t +gc_collect_main(PyThreadState *tstate, int generation, _PyGC_Reason reason) +{ + int i; + Py_ssize_t m = 0; /* # objects collected */ + Py_ssize_t n = 0; /* # unreachable objects that couldn't be collected */ + _PyTime_t t1 = 0; /* initialize to prevent a compiler warning */ + GCState *gcstate = &tstate->interp->gc; + + // gc_collect_main() must not be called before _PyGC_Init + // or after _PyGC_Fini() + assert(gcstate->garbage != NULL); + assert(!_PyErr_Occurred(tstate)); + + int expected = 0; + if (!_Py_atomic_compare_exchange_int(&gcstate->collecting, &expected, 1)) { + // Don't start a garbage collection if one is already in progress. + return 0; + } + + if (reason == _Py_GC_REASON_HEAP && !gc_should_collect(gcstate)) { + // Don't collect if the threshold is not exceeded. + _Py_atomic_store_int(&gcstate->collecting, 0); + return 0; + } + + assert(generation >= 0 && generation < NUM_GENERATIONS); + +#ifdef Py_STATS + if (_Py_stats) { + _Py_stats->object_stats.object_visits = 0; + } +#endif + GC_STAT_ADD(generation, collections, 1); + + if (reason != _Py_GC_REASON_SHUTDOWN) { + invoke_gc_callback(tstate, "start", generation, 0, 0); + } + + if (gcstate->debug & _PyGC_DEBUG_STATS) { + PySys_WriteStderr("gc: collecting generation %d...\n", generation); + show_stats_each_generations(gcstate); + t1 = _PyTime_GetPerfCounter(); + } + + if (PyDTrace_GC_START_ENABLED()) { + PyDTrace_GC_START(generation); + } + + /* update collection and allocation counters */ + if (generation+1 < NUM_GENERATIONS) { + gcstate->generations[generation+1].count += 1; + } + for (i = 0; i <= generation; i++) { + gcstate->generations[i].count = 0; + } + + PyInterpreterState *interp = tstate->interp; + + struct collection_state state = { + .interp = interp, + .gcstate = gcstate, + }; + + gc_collect_internal(interp, &state); + + m = state.collected; + n = state.uncollectable; + gcstate->long_lived_total = state.long_lived_total; + + if (gcstate->debug & _PyGC_DEBUG_STATS) { + double d = _PyTime_AsSecondsDouble(_PyTime_GetPerfCounter() - t1); + PySys_WriteStderr( + "gc: done, %zd unreachable, %zd uncollectable, %.4fs elapsed\n", + n+m, n, d); + } + + // Clear free lists in all threads + _PyGC_ClearAllFreeLists(interp); + + if (_PyErr_Occurred(tstate)) { + if (reason == _Py_GC_REASON_SHUTDOWN) { + _PyErr_Clear(tstate); + } + else { + PyErr_FormatUnraisable("Exception ignored in garbage collection"); + } + } + + /* Update stats */ + struct gc_generation_stats *stats = &gcstate->generation_stats[generation]; + stats->collections++; + stats->collected += m; + stats->uncollectable += n; + + GC_STAT_ADD(generation, objects_collected, m); +#ifdef Py_STATS + if (_Py_stats) { + GC_STAT_ADD(generation, object_visits, + _Py_stats->object_stats.object_visits); + _Py_stats->object_stats.object_visits = 0; + } +#endif + + if (PyDTrace_GC_DONE_ENABLED()) { + PyDTrace_GC_DONE(n + m); + } + + if (reason != _Py_GC_REASON_SHUTDOWN) { + invoke_gc_callback(tstate, "stop", generation, m, n); + } + + assert(!_PyErr_Occurred(tstate)); + _Py_atomic_store_int(&gcstate->collecting, 0); + return n + m; +} + +struct get_referrers_args { + struct visitor_args base; + PyObject *objs; + struct worklist results; +}; + +static int +referrersvisit(PyObject* obj, void *arg) +{ + PyObject *objs = arg; + Py_ssize_t i; + for (i = 0; i < PyTuple_GET_SIZE(objs); i++) { + if (PyTuple_GET_ITEM(objs, i) == obj) { + return 1; + } + } + return 0; +} + +static bool +visit_get_referrers(const mi_heap_t *heap, const mi_heap_area_t *area, + void *block, size_t block_size, void *args) +{ + PyObject *op = op_from_block(block, args, true); + if (op == NULL) { + return true; + } + + struct get_referrers_args *arg = (struct get_referrers_args *)args; + if (Py_TYPE(op)->tp_traverse(op, referrersvisit, arg->objs)) { + op->ob_tid = 0; // we will restore the refcount later + worklist_push(&arg->results, op); + } + + return true; +} + +PyObject * +_PyGC_GetReferrers(PyInterpreterState *interp, PyObject *objs) +{ + PyObject *result = PyList_New(0); + if (!result) { + return NULL; + } + + _PyEval_StopTheWorld(interp); + + // Append all objects to a worklist. This abuses ob_tid. We will restore + // it later. NOTE: We can't append to the PyListObject during + // gc_visit_heaps() because PyList_Append() may reclaim an abandoned + // mimalloc segments while we are traversing them. + struct get_referrers_args args = { .objs = objs }; + gc_visit_heaps(interp, &visit_get_referrers, &args.base); + + bool error = false; + PyObject *op; + while ((op = worklist_pop(&args.results)) != NULL) { + gc_restore_tid(op); + if (op != objs && PyList_Append(result, op) < 0) { + error = true; + break; + } + } + + // In case of error, clear the remaining worklist + while ((op = worklist_pop(&args.results)) != NULL) { + gc_restore_tid(op); + } + + _PyEval_StartTheWorld(interp); + + if (error) { + Py_DECREF(result); + return NULL; + } + + return result; +} + +struct get_objects_args { + struct visitor_args base; + struct worklist objects; +}; + +static bool +visit_get_objects(const mi_heap_t *heap, const mi_heap_area_t *area, + void *block, size_t block_size, void *args) +{ + PyObject *op = op_from_block(block, args, true); + if (op == NULL) { + return true; + } + + struct get_objects_args *arg = (struct get_objects_args *)args; + op->ob_tid = 0; // we will restore the refcount later + worklist_push(&arg->objects, op); + + return true; +} + +PyObject * +_PyGC_GetObjects(PyInterpreterState *interp, Py_ssize_t generation) +{ + PyObject *result = PyList_New(0); + if (!result) { + return NULL; + } + + _PyEval_StopTheWorld(interp); + + // Append all objects to a worklist. This abuses ob_tid. We will restore + // it later. NOTE: We can't append to the list during gc_visit_heaps() + // because PyList_Append() may reclaim an abandoned mimalloc segment + // while we are traversing it. + struct get_objects_args args = { 0 }; + gc_visit_heaps(interp, &visit_get_objects, &args.base); + + bool error = false; + PyObject *op; + while ((op = worklist_pop(&args.objects)) != NULL) { + gc_restore_tid(op); + if (op != result && PyList_Append(result, op) < 0) { + error = true; + break; + } + } + + // In case of error, clear the remaining worklist + while ((op = worklist_pop(&args.objects)) != NULL) { + gc_restore_tid(op); + } + + _PyEval_StartTheWorld(interp); + + if (error) { + Py_DECREF(result); + return NULL; + } + + return result; +} + +static bool +visit_freeze(const mi_heap_t *heap, const mi_heap_area_t *area, + void *block, size_t block_size, void *args) +{ + PyObject *op = op_from_block(block, args, true); + if (op != NULL) { + op->ob_gc_bits |= _PyGC_BITS_FROZEN; + } + return true; +} + +void +_PyGC_Freeze(PyInterpreterState *interp) +{ + struct visitor_args args; + _PyEval_StopTheWorld(interp); + gc_visit_heaps(interp, &visit_freeze, &args); + _PyEval_StartTheWorld(interp); +} + +static bool +visit_unfreeze(const mi_heap_t *heap, const mi_heap_area_t *area, + void *block, size_t block_size, void *args) +{ + PyObject *op = op_from_block(block, args, true); + if (op != NULL) { + op->ob_gc_bits &= ~_PyGC_BITS_FROZEN; + } + return true; +} + +void +_PyGC_Unfreeze(PyInterpreterState *interp) +{ + struct visitor_args args; + _PyEval_StopTheWorld(interp); + gc_visit_heaps(interp, &visit_unfreeze, &args); + _PyEval_StartTheWorld(interp); +} + +struct count_frozen_args { + struct visitor_args base; + Py_ssize_t count; +}; + +static bool +visit_count_frozen(const mi_heap_t *heap, const mi_heap_area_t *area, + void *block, size_t block_size, void *args) +{ + PyObject *op = op_from_block(block, args, true); + if (op != NULL && (op->ob_gc_bits & _PyGC_BITS_FROZEN) != 0) { + struct count_frozen_args *arg = (struct count_frozen_args *)args; + arg->count++; + } + return true; +} + +Py_ssize_t +_PyGC_GetFreezeCount(PyInterpreterState *interp) +{ + struct count_frozen_args args = { .count = 0 }; + _PyEval_StopTheWorld(interp); + gc_visit_heaps(interp, &visit_count_frozen, &args.base); + _PyEval_StartTheWorld(interp); + return args.count; +} + +/* C API for controlling the state of the garbage collector */ +int +PyGC_Enable(void) +{ + GCState *gcstate = get_gc_state(); + int old_state = gcstate->enabled; + gcstate->enabled = 1; + return old_state; +} + +int +PyGC_Disable(void) +{ + GCState *gcstate = get_gc_state(); + int old_state = gcstate->enabled; + gcstate->enabled = 0; + return old_state; +} + +int +PyGC_IsEnabled(void) +{ + GCState *gcstate = get_gc_state(); + return gcstate->enabled; +} + +/* Public API to invoke gc.collect() from C */ +Py_ssize_t +PyGC_Collect(void) +{ + PyThreadState *tstate = _PyThreadState_GET(); + GCState *gcstate = &tstate->interp->gc; + + if (!gcstate->enabled) { + return 0; + } + + Py_ssize_t n; + PyObject *exc = _PyErr_GetRaisedException(tstate); + n = gc_collect_main(tstate, NUM_GENERATIONS - 1, _Py_GC_REASON_MANUAL); + _PyErr_SetRaisedException(tstate, exc); + + return n; +} + +Py_ssize_t +_PyGC_Collect(PyThreadState *tstate, int generation, _PyGC_Reason reason) +{ + return gc_collect_main(tstate, generation, reason); +} + +Py_ssize_t +_PyGC_CollectNoFail(PyThreadState *tstate) +{ + /* Ideally, this function is only called on interpreter shutdown, + and therefore not recursively. Unfortunately, when there are daemon + threads, a daemon thread can start a cyclic garbage collection + during interpreter shutdown (and then never finish it). + See http://bugs.python.org/issue8713#msg195178 for an example. + */ + return gc_collect_main(tstate, NUM_GENERATIONS - 1, _Py_GC_REASON_SHUTDOWN); +} + +void +_PyGC_DumpShutdownStats(PyInterpreterState *interp) +{ + GCState *gcstate = &interp->gc; + if (!(gcstate->debug & _PyGC_DEBUG_SAVEALL) + && gcstate->garbage != NULL && PyList_GET_SIZE(gcstate->garbage) > 0) { + const char *message; + if (gcstate->debug & _PyGC_DEBUG_UNCOLLECTABLE) { + message = "gc: %zd uncollectable objects at shutdown"; + } + else { + message = "gc: %zd uncollectable objects at shutdown; " \ + "use gc.set_debug(gc.DEBUG_UNCOLLECTABLE) to list them"; + } + /* PyErr_WarnFormat does too many things and we are at shutdown, + the warnings module's dependencies (e.g. linecache) may be gone + already. */ + if (PyErr_WarnExplicitFormat(PyExc_ResourceWarning, "gc", 0, + "gc", NULL, message, + PyList_GET_SIZE(gcstate->garbage))) + { + PyErr_WriteUnraisable(NULL); + } + if (gcstate->debug & _PyGC_DEBUG_UNCOLLECTABLE) { + PyObject *repr = NULL, *bytes = NULL; + repr = PyObject_Repr(gcstate->garbage); + if (!repr || !(bytes = PyUnicode_EncodeFSDefault(repr))) { + PyErr_WriteUnraisable(gcstate->garbage); + } + else { + PySys_WriteStderr( + " %s\n", + PyBytes_AS_STRING(bytes) + ); + } + Py_XDECREF(repr); + Py_XDECREF(bytes); + } + } +} + + +void +_PyGC_Fini(PyInterpreterState *interp) +{ + GCState *gcstate = &interp->gc; + Py_CLEAR(gcstate->garbage); + Py_CLEAR(gcstate->callbacks); + + /* We expect that none of this interpreters objects are shared + with other interpreters. + See https://github.com/python/cpython/issues/90228. */ +} + +/* for debugging */ + +#ifdef Py_DEBUG +static int +visit_validate(PyObject *op, void *parent_raw) +{ + PyObject *parent = _PyObject_CAST(parent_raw); + if (_PyObject_IsFreed(op)) { + _PyObject_ASSERT_FAILED_MSG(parent, + "PyObject_GC_Track() object is not valid"); + } + return 0; +} +#endif + + +/* extension modules might be compiled with GC support so these + functions must always be available */ + +void +PyObject_GC_Track(void *op_raw) +{ + PyObject *op = _PyObject_CAST(op_raw); + if (_PyObject_GC_IS_TRACKED(op)) { + _PyObject_ASSERT_FAILED_MSG(op, + "object already tracked " + "by the garbage collector"); + } + _PyObject_GC_TRACK(op); + +#ifdef Py_DEBUG + /* Check that the object is valid: validate objects traversed + by tp_traverse() */ + traverseproc traverse = Py_TYPE(op)->tp_traverse; + (void)traverse(op, visit_validate, op); +#endif +} + +void +PyObject_GC_UnTrack(void *op_raw) +{ + PyObject *op = _PyObject_CAST(op_raw); + /* Obscure: the Py_TRASHCAN mechanism requires that we be able to + * call PyObject_GC_UnTrack twice on an object. + */ + if (_PyObject_GC_IS_TRACKED(op)) { + _PyObject_GC_UNTRACK(op); + } +} + +int +PyObject_IS_GC(PyObject *obj) +{ + return _PyObject_IS_GC(obj); +} + +void +_Py_ScheduleGC(PyInterpreterState *interp) +{ + _Py_set_eval_breaker_bit(interp, _PY_GC_SCHEDULED_BIT, 1); +} + +void +_PyObject_GC_Link(PyObject *op) +{ + PyThreadState *tstate = _PyThreadState_GET(); + GCState *gcstate = &tstate->interp->gc; + gcstate->generations[0].count++; + + if (gc_should_collect(gcstate) && + !_Py_atomic_load_int_relaxed(&gcstate->collecting)) + { + _Py_ScheduleGC(tstate->interp); + } +} + +void +_Py_RunGC(PyThreadState *tstate) +{ + gc_collect_main(tstate, 0, _Py_GC_REASON_HEAP); +} + +static PyObject * +gc_alloc(PyTypeObject *tp, size_t basicsize, size_t presize) +{ + PyThreadState *tstate = _PyThreadState_GET(); + if (basicsize > PY_SSIZE_T_MAX - presize) { + return _PyErr_NoMemory(tstate); + } + size_t size = presize + basicsize; + char *mem = _PyObject_MallocWithType(tp, size); + if (mem == NULL) { + return _PyErr_NoMemory(tstate); + } + if (presize) { + ((PyObject **)mem)[0] = NULL; + ((PyObject **)mem)[1] = NULL; + } + PyObject *op = (PyObject *)(mem + presize); + _PyObject_GC_Link(op); + return op; +} + +PyObject * +_PyObject_GC_New(PyTypeObject *tp) +{ + size_t presize = _PyType_PreHeaderSize(tp); + PyObject *op = gc_alloc(tp, _PyObject_SIZE(tp), presize); + if (op == NULL) { + return NULL; + } + _PyObject_Init(op, tp); + return op; +} + +PyVarObject * +_PyObject_GC_NewVar(PyTypeObject *tp, Py_ssize_t nitems) +{ + PyVarObject *op; + + if (nitems < 0) { + PyErr_BadInternalCall(); + return NULL; + } + size_t presize = _PyType_PreHeaderSize(tp); + size_t size = _PyObject_VAR_SIZE(tp, nitems); + op = (PyVarObject *)gc_alloc(tp, size, presize); + if (op == NULL) { + return NULL; + } + _PyObject_InitVar(op, tp, nitems); + return op; +} + +PyObject * +PyUnstable_Object_GC_NewWithExtraData(PyTypeObject *tp, size_t extra_size) +{ + size_t presize = _PyType_PreHeaderSize(tp); + PyObject *op = gc_alloc(tp, _PyObject_SIZE(tp) + extra_size, presize); + if (op == NULL) { + return NULL; + } + memset(op, 0, _PyObject_SIZE(tp) + extra_size); + _PyObject_Init(op, tp); + return op; +} + +PyVarObject * +_PyObject_GC_Resize(PyVarObject *op, Py_ssize_t nitems) +{ + const size_t basicsize = _PyObject_VAR_SIZE(Py_TYPE(op), nitems); + const size_t presize = _PyType_PreHeaderSize(((PyObject *)op)->ob_type); + _PyObject_ASSERT((PyObject *)op, !_PyObject_GC_IS_TRACKED(op)); + if (basicsize > (size_t)PY_SSIZE_T_MAX - presize) { + return (PyVarObject *)PyErr_NoMemory(); + } + char *mem = (char *)op - presize; + mem = (char *)_PyObject_ReallocWithType(Py_TYPE(op), mem, presize + basicsize); + if (mem == NULL) { + return (PyVarObject *)PyErr_NoMemory(); + } + op = (PyVarObject *) (mem + presize); + Py_SET_SIZE(op, nitems); + return op; +} + +void +PyObject_GC_Del(void *op) +{ + size_t presize = _PyType_PreHeaderSize(((PyObject *)op)->ob_type); + if (_PyObject_GC_IS_TRACKED(op)) { +#ifdef Py_DEBUG + PyObject *exc = PyErr_GetRaisedException(); + if (PyErr_WarnExplicitFormat(PyExc_ResourceWarning, "gc", 0, + "gc", NULL, "Object of type %s is not untracked before destruction", + ((PyObject*)op)->ob_type->tp_name)) { + PyErr_WriteUnraisable(NULL); + } + PyErr_SetRaisedException(exc); +#endif + } + GCState *gcstate = get_gc_state(); + if (gcstate->generations[0].count > 0) { + gcstate->generations[0].count--; + } + PyObject_Free(((char *)op)-presize); +} + +int +PyObject_GC_IsTracked(PyObject* obj) +{ + if (_PyObject_IS_GC(obj) && _PyObject_GC_IS_TRACKED(obj)) { + return 1; + } + return 0; +} + +int +PyObject_GC_IsFinalized(PyObject *obj) +{ + if (_PyObject_IS_GC(obj) && _PyGC_FINALIZED(obj)) { + return 1; + } + return 0; +} + +struct custom_visitor_args { + struct visitor_args base; + gcvisitobjects_t callback; + void *arg; +}; + +static bool +custom_visitor_wrapper(const mi_heap_t *heap, const mi_heap_area_t *area, + void *block, size_t block_size, void *args) +{ + PyObject *op = op_from_block(block, args, false); + if (op == NULL) { + return true; + } + + struct custom_visitor_args *wrapper = (struct custom_visitor_args *)args; + if (!wrapper->callback(op, wrapper->arg)) { + return false; + } + + return true; +} + +void +PyUnstable_GC_VisitObjects(gcvisitobjects_t callback, void *arg) +{ + PyInterpreterState *interp = _PyInterpreterState_GET(); + struct custom_visitor_args wrapper = { + .callback = callback, + .arg = arg, + }; + + _PyEval_StopTheWorld(interp); + gc_visit_heaps(interp, &custom_visitor_wrapper, &wrapper.base); + _PyEval_StartTheWorld(interp); +} + +/* Clear all free lists + * All free lists are cleared during the collection of the highest generation. + * Allocated items in the free list may keep a pymalloc arena occupied. + * Clearing the free lists may give back memory to the OS earlier. + * Free-threading version: Since freelists are managed per thread, + * GC should clear all freelists by traversing all threads. + */ +void +_PyGC_ClearAllFreeLists(PyInterpreterState *interp) +{ + HEAD_LOCK(&_PyRuntime); + _PyThreadStateImpl *tstate = (_PyThreadStateImpl *)interp->threads.head; + while (tstate != NULL) { + _PyObject_ClearFreeLists(&tstate->freelist_state, 0); + tstate = (_PyThreadStateImpl *)tstate->base.next; + } + HEAD_UNLOCK(&_PyRuntime); +} + +#endif // Py_GIL_DISABLED diff --git a/Python/gc_gil.c b/Python/gc_gil.c new file mode 100644 index 00000000000000..5f1365f509deb0 --- /dev/null +++ b/Python/gc_gil.c @@ -0,0 +1,17 @@ +#include "Python.h" +#include "pycore_pystate.h" // _Py_ClearFreeLists() + +#ifndef Py_GIL_DISABLED + +/* Clear all free lists + * All free lists are cleared during the collection of the highest generation. + * Allocated items in the free list may keep a pymalloc arena occupied. + * Clearing the free lists may give back memory to the OS earlier. + */ +void +_PyGC_ClearAllFreeLists(PyInterpreterState *interp) +{ + _PyObject_ClearFreeLists(&interp->freelist_state, 0); +} + +#endif diff --git a/Python/generated_cases.c.h b/Python/generated_cases.c.h index ce31967b7912d7..16f1db30620d72 100644 --- a/Python/generated_cases.c.h +++ b/Python/generated_cases.c.h @@ -2342,17 +2342,9 @@ next_instr += 1; INSTRUCTION_STATS(END_FOR); PyObject *value; - // _POP_TOP value = stack_pointer[-1]; - { - Py_DECREF(value); - } - // _POP_TOP - value = stack_pointer[-2]; - { - Py_DECREF(value); - } - stack_pointer += -2; + Py_DECREF(value); + stack_pointer += -1; DISPATCH(); } @@ -2371,24 +2363,29 @@ } TARGET(ENTER_EXECUTOR) { - frame->instr_ptr = next_instr; + _Py_CODEUNIT *this_instr = frame->instr_ptr = next_instr; next_instr += 1; INSTRUCTION_STATS(ENTER_EXECUTOR); TIER_ONE_ONLY CHECK_EVAL_BREAKER(); PyCodeObject *code = _PyFrame_GetCode(frame); - _PyExecutorObject *executor = (_PyExecutorObject *)code->co_executors->executors[oparg&255]; - Py_INCREF(executor); - if (executor->execute == _PyUOpExecute) { - current_executor = (_PyUOpExecutorObject *)executor; + _PyExecutorObject *executor = code->co_executors->executors[oparg & 255]; + if (executor->vm_data.valid) { + Py_INCREF(executor); + current_executor = executor; GOTO_TIER_TWO(); } - next_instr = executor->execute(executor, frame, stack_pointer); - frame = tstate->current_frame; - if (next_instr == NULL) { - goto resume_with_error; + else { + /* ENTER_EXECUTOR will be the first code unit of the instruction */ + assert(oparg < 256); + code->co_executors->executors[oparg] = NULL; + opcode = this_instr->op.code = executor->vm_data.opcode; + this_instr->op.arg = executor->vm_data.oparg; + oparg = executor->vm_data.oparg; + Py_DECREF(executor); + next_instr = this_instr; + DISPATCH_GOTO(); } - stack_pointer = _PyFrame_GetStackPointer(frame); DISPATCH(); } @@ -2500,8 +2497,8 @@ next_instr[oparg].op.code == INSTRUMENTED_END_FOR); Py_DECREF(iter); STACK_SHRINK(1); - /* Jump forward oparg, then skip following END_FOR instruction */ - JUMPBY(oparg + 1); + /* Jump forward oparg, then skip following END_FOR and POP_TOP instruction */ + JUMPBY(oparg + 2); DISPATCH(); } // Common case: no jump, leave it to the code generator @@ -2562,8 +2559,8 @@ } Py_DECREF(iter); STACK_SHRINK(1); - /* Jump forward oparg, then skip following END_FOR instruction */ - JUMPBY(oparg + 1); + /* Jump forward oparg, then skip following END_FOR and POP_TOP instructions */ + JUMPBY(oparg + 2); DISPATCH(); } } @@ -2603,8 +2600,8 @@ if (r->len <= 0) { STACK_SHRINK(1); Py_DECREF(r); - // Jump over END_FOR instruction. - JUMPBY(oparg + 1); + // Jump over END_FOR and POP_TOP instructions. + JUMPBY(oparg + 2); DISPATCH(); } } @@ -2650,8 +2647,8 @@ } Py_DECREF(iter); STACK_SHRINK(1); - /* Jump forward oparg, then skip following END_FOR instruction */ - JUMPBY(oparg + 1); + /* Jump forward oparg, then skip following END_FOR and POP_TOP instructions */ + JUMPBY(oparg + 2); DISPATCH(); } } @@ -2947,9 +2944,8 @@ } PyErr_SetRaisedException(NULL); } - Py_DECREF(receiver); Py_DECREF(value); - stack_pointer += -2; + stack_pointer += -1; DISPATCH(); } @@ -3000,8 +2996,8 @@ next_instr[oparg].op.code == INSTRUMENTED_END_FOR); STACK_SHRINK(1); Py_DECREF(iter); - /* Skip END_FOR */ - target = next_instr + oparg + 1; + /* Skip END_FOR and POP_TOP */ + target = next_instr + oparg + 2; } INSTRUMENTED_JUMP(this_instr, target, PY_MONITORING_EVENT_BRANCH); DISPATCH(); @@ -3151,7 +3147,7 @@ if (err) goto error; if (frame->instr_ptr != this_instr) { /* Instrumentation has jumped */ - next_instr = this_instr; + next_instr = frame->instr_ptr; DISPATCH(); } } @@ -3286,12 +3282,18 @@ // Double-check that the opcode isn't instrumented or something: if (ucounter > threshold && this_instr->op.code == JUMP_BACKWARD) { OPT_STAT_INC(attempts); - int optimized = _PyOptimizer_BackEdge(frame, this_instr, next_instr, stack_pointer); + _Py_CODEUNIT *start = this_instr; + /* Back up over EXTENDED_ARGs so optimizer sees the whole instruction */ + while (oparg > 255) { + oparg >>= 8; + start--; + } + int optimized = _PyOptimizer_Optimize(frame, start, stack_pointer); if (optimized < 0) goto error; if (optimized) { // Rewind and enter the executor: - assert(this_instr->op.code == ENTER_EXECUTOR); - next_instr = this_instr; + assert(start->op.code == ENTER_EXECUTOR); + next_instr = start; this_instr[1].cache &= ((1 << OPTIMIZER_BITS_IN_COUNTER) - 1); } else { diff --git a/Python/getargs.c b/Python/getargs.c index 0c4ce282f48764..08e97ee3e627b5 100644 --- a/Python/getargs.c +++ b/Python/getargs.c @@ -8,6 +8,7 @@ #include "pycore_modsupport.h" // export _PyArg_NoKeywords() #include "pycore_pylifecycle.h" // _PyArg_Fini #include "pycore_tuple.h" // _PyTuple_ITEMS() +#include "pycore_pyerrors.h" // _Py_CalculateSuggestions() /* Export Stable ABIs (abi only) */ PyAPI_FUNC(int) _PyArg_Parse_SizeT(PyObject *, const char *, ...); @@ -1424,12 +1425,31 @@ error_unexpected_keyword_arg(PyObject *kwargs, PyObject *kwnames, PyObject *kwtu int match = PySequence_Contains(kwtuple, keyword); if (match <= 0) { if (!match) { - PyErr_Format(PyExc_TypeError, - "'%S' is an invalid keyword " - "argument for %.200s%s", - keyword, - (fname == NULL) ? "this function" : fname, - (fname == NULL) ? "" : "()"); + PyObject *kwlist = PySequence_List(kwtuple); + if (!kwlist) { + return; + } + PyObject *suggestion_keyword = _Py_CalculateSuggestions(kwlist, keyword); + Py_DECREF(kwlist); + + if (suggestion_keyword) { + PyErr_Format(PyExc_TypeError, + "%.200s%s got an unexpected keyword argument '%S'." + " Did you mean '%S'?", + (fname == NULL) ? "this function" : fname, + (fname == NULL) ? "" : "()", + keyword, + suggestion_keyword); + Py_DECREF(suggestion_keyword); + } + else { + PyErr_Format(PyExc_TypeError, + "%.200s%s got an unexpected keyword argument '%S'", + (fname == NULL) ? "this function" : fname, + (fname == NULL) ? "" : "()", + keyword); + } + } return; } @@ -1457,6 +1477,9 @@ PyArg_ValidateKeywordArguments(PyObject *kwargs) return 1; } +static PyObject * +new_kwtuple(const char * const *keywords, int total, int pos); + #define IS_END_OF_FORMAT(c) (c == '\0' || c == ';' || c == ':') static int @@ -1722,12 +1745,35 @@ vgetargskeywords(PyObject *args, PyObject *kwargs, const char *format, } } if (!match) { - PyErr_Format(PyExc_TypeError, - "'%U' is an invalid keyword " - "argument for %.200s%s", - key, - (fname == NULL) ? "this function" : fname, - (fname == NULL) ? "" : "()"); + PyObject *_pykwtuple = new_kwtuple(kwlist, len, pos); + if (!_pykwtuple) { + return cleanreturn(0, &freelist); + } + PyObject *pykwlist = PySequence_List(_pykwtuple); + Py_DECREF(_pykwtuple); + if (!pykwlist) { + return cleanreturn(0, &freelist); + } + PyObject *suggestion_keyword = _Py_CalculateSuggestions(pykwlist, key); + Py_DECREF(pykwlist); + + if (suggestion_keyword) { + PyErr_Format(PyExc_TypeError, + "%.200s%s got an unexpected keyword argument '%S'." + " Did you mean '%S'?", + (fname == NULL) ? "this function" : fname, + (fname == NULL) ? "" : "()", + key, + suggestion_keyword); + Py_DECREF(suggestion_keyword); + } + else { + PyErr_Format(PyExc_TypeError, + "%.200s%s got an unexpected keyword argument '%S'", + (fname == NULL) ? "this function" : fname, + (fname == NULL) ? "" : "()", + key); + } return cleanreturn(0, &freelist); } } diff --git a/Python/getcopyright.c b/Python/getcopyright.c index c1f1aad9b845b1..066c2ed66acddf 100644 --- a/Python/getcopyright.c +++ b/Python/getcopyright.c @@ -4,7 +4,7 @@ static const char cprt[] = "\ -Copyright (c) 2001-2023 Python Software Foundation.\n\ +Copyright (c) 2001-2024 Python Software Foundation.\n\ All Rights Reserved.\n\ \n\ Copyright (c) 2000 BeOpen.com.\n\ diff --git a/Python/import.c b/Python/import.c index 2dd95d8364a0be..2fd0c08a6bb5ae 100644 --- a/Python/import.c +++ b/Python/import.c @@ -3544,7 +3544,7 @@ _imp_get_frozen_object_impl(PyObject *module, PyObject *name, struct frozen_info info = {0}; Py_buffer buf = {0}; if (PyObject_CheckBuffer(dataobj)) { - if (PyObject_GetBuffer(dataobj, &buf, PyBUF_READ) != 0) { + if (PyObject_GetBuffer(dataobj, &buf, PyBUF_SIMPLE) != 0) { return NULL; } info.data = (const char *)buf.buf; diff --git a/Python/initconfig.c b/Python/initconfig.c index 06e317907b8ec9..a6d8c176156617 100644 --- a/Python/initconfig.c +++ b/Python/initconfig.c @@ -295,6 +295,7 @@ static const char usage_envvars[] = " running a local build).\n" "PYTHON_COLORS : If this variable is set to 1, the interpreter will" " colorize various kinds of output. Setting it to 0 deactivates this behavior.\n" +"PYTHON_HISTORY : the location of a .python_history file.\n" "These variables have equivalent command-line parameters (see --help for details):\n" "PYTHONDEBUG : enable parser debug mode (-d)\n" "PYTHONDONTWRITEBYTECODE : don't write .pyc files (-B)\n" diff --git a/Python/instrumentation.c b/Python/instrumentation.c index 35b0e7a8f35c56..533aece210202b 100644 --- a/Python/instrumentation.c +++ b/Python/instrumentation.c @@ -1576,13 +1576,11 @@ _Py_Instrument(PyCodeObject *code, PyInterpreterState *interp) } _Py_Executors_InvalidateDependency(interp, code); int code_len = (int)Py_SIZE(code); - /* code->_co_firsttraceable >= code_len indicates - * that no instrumentation can be inserted. - * Exit early to avoid creating instrumentation + /* Exit early to avoid creating instrumentation * data for potential statically allocated code * objects. * See https://github.com/python/cpython/issues/108390 */ - if (code->_co_firsttraceable >= code_len) { + if (code->co_flags & CO_NO_MONITORING_EVENTS) { return 0; } if (update_instrumentation_data(code, interp)) { diff --git a/Python/jit.c b/Python/jit.c new file mode 100644 index 00000000000000..22949c082da05a --- /dev/null +++ b/Python/jit.c @@ -0,0 +1,369 @@ +#ifdef _Py_JIT + +#include "Python.h" + +#include "pycore_abstract.h" +#include "pycore_call.h" +#include "pycore_ceval.h" +#include "pycore_dict.h" +#include "pycore_intrinsics.h" +#include "pycore_long.h" +#include "pycore_opcode_metadata.h" +#include "pycore_opcode_utils.h" +#include "pycore_optimizer.h" +#include "pycore_pyerrors.h" +#include "pycore_setobject.h" +#include "pycore_sliceobject.h" +#include "pycore_jit.h" + +#include "jit_stencils.h" + +// Memory management stuff: //////////////////////////////////////////////////// + +#ifndef MS_WINDOWS + #include +#endif + +static size_t +get_page_size(void) +{ +#ifdef MS_WINDOWS + SYSTEM_INFO si; + GetSystemInfo(&si); + return si.dwPageSize; +#else + return sysconf(_SC_PAGESIZE); +#endif +} + +static void +jit_error(const char *message) +{ +#ifdef MS_WINDOWS + int hint = GetLastError(); +#else + int hint = errno; +#endif + PyErr_Format(PyExc_RuntimeWarning, "JIT %s (%d)", message, hint); +} + +static char * +jit_alloc(size_t size) +{ + assert(size); + assert(size % get_page_size() == 0); +#ifdef MS_WINDOWS + int flags = MEM_COMMIT | MEM_RESERVE; + char *memory = VirtualAlloc(NULL, size, flags, PAGE_READWRITE); + int failed = memory == NULL; +#else + int flags = MAP_ANONYMOUS | MAP_PRIVATE; + char *memory = mmap(NULL, size, PROT_READ | PROT_WRITE, flags, -1, 0); + int failed = memory == MAP_FAILED; +#endif + if (failed) { + jit_error("unable to allocate memory"); + return NULL; + } + return memory; +} + +static int +jit_free(char *memory, size_t size) +{ + assert(size); + assert(size % get_page_size() == 0); +#ifdef MS_WINDOWS + int failed = !VirtualFree(memory, 0, MEM_RELEASE); +#else + int failed = munmap(memory, size); +#endif + if (failed) { + jit_error("unable to free memory"); + return -1; + } + return 0; +} + +static int +mark_executable(char *memory, size_t size) +{ + if (size == 0) { + return 0; + } + assert(size % get_page_size() == 0); + // Do NOT ever leave the memory writable! Also, don't forget to flush the + // i-cache (I cannot begin to tell you how horrible that is to debug): +#ifdef MS_WINDOWS + if (!FlushInstructionCache(GetCurrentProcess(), memory, size)) { + jit_error("unable to flush instruction cache"); + return -1; + } + int old; + int failed = !VirtualProtect(memory, size, PAGE_EXECUTE_READ, &old); +#else + __builtin___clear_cache((char *)memory, (char *)memory + size); + int failed = mprotect(memory, size, PROT_EXEC | PROT_READ); +#endif + if (failed) { + jit_error("unable to protect executable memory"); + return -1; + } + return 0; +} + +static int +mark_readable(char *memory, size_t size) +{ + if (size == 0) { + return 0; + } + assert(size % get_page_size() == 0); +#ifdef MS_WINDOWS + DWORD old; + int failed = !VirtualProtect(memory, size, PAGE_READONLY, &old); +#else + int failed = mprotect(memory, size, PROT_READ); +#endif + if (failed) { + jit_error("unable to protect readable memory"); + return -1; + } + return 0; +} + +// JIT compiler stuff: ///////////////////////////////////////////////////////// + +// Warning! AArch64 requires you to get your hands dirty. These are your gloves: + +// value[value_start : value_start + len] +static uint32_t +get_bits(uint64_t value, uint8_t value_start, uint8_t width) +{ + assert(width <= 32); + return (value >> value_start) & ((1ULL << width) - 1); +} + +// *loc[loc_start : loc_start + width] = value[value_start : value_start + width] +static void +set_bits(uint32_t *loc, uint8_t loc_start, uint64_t value, uint8_t value_start, + uint8_t width) +{ + assert(loc_start + width <= 32); + // Clear the bits we're about to patch: + *loc &= ~(((1ULL << width) - 1) << loc_start); + assert(get_bits(*loc, loc_start, width) == 0); + // Patch the bits: + *loc |= get_bits(value, value_start, width) << loc_start; + assert(get_bits(*loc, loc_start, width) == get_bits(value, value_start, width)); +} + +// See https://developer.arm.com/documentation/ddi0602/2023-09/Base-Instructions +// for instruction encodings: +#define IS_AARCH64_ADD_OR_SUB(I) (((I) & 0x11C00000) == 0x11000000) +#define IS_AARCH64_ADRP(I) (((I) & 0x9F000000) == 0x90000000) +#define IS_AARCH64_BRANCH(I) (((I) & 0x7C000000) == 0x14000000) +#define IS_AARCH64_LDR_OR_STR(I) (((I) & 0x3B000000) == 0x39000000) +#define IS_AARCH64_MOV(I) (((I) & 0x9F800000) == 0x92800000) + +// Fill all of stencil's holes in the memory pointed to by base, using the +// values in patches. +static void +patch(char *base, const Stencil *stencil, uint64_t *patches) +{ + for (uint64_t i = 0; i < stencil->holes_size; i++) { + const Hole *hole = &stencil->holes[i]; + void *location = base + hole->offset; + uint64_t value = patches[hole->value] + (uint64_t)hole->symbol + hole->addend; + uint32_t *loc32 = (uint32_t *)location; + uint64_t *loc64 = (uint64_t *)location; + // LLD is a great reference for performing relocations... just keep in + // mind that Tools/jit/build.py does filtering and preprocessing for us! + // Here's a good place to start for each platform: + // - aarch64-apple-darwin: + // - https://github.com/llvm/llvm-project/blob/main/lld/MachO/Arch/ARM64Common.cpp + // - https://github.com/llvm/llvm-project/blob/main/lld/MachO/Arch/ARM64Common.h + // - aarch64-unknown-linux-gnu: + // - https://github.com/llvm/llvm-project/blob/main/lld/ELF/Arch/AArch64.cpp + // - i686-pc-windows-msvc: + // - https://github.com/llvm/llvm-project/blob/main/lld/COFF/Chunks.cpp + // - x86_64-apple-darwin: + // - https://github.com/llvm/llvm-project/blob/main/lld/MachO/Arch/X86_64.cpp + // - x86_64-pc-windows-msvc: + // - https://github.com/llvm/llvm-project/blob/main/lld/COFF/Chunks.cpp + // - x86_64-unknown-linux-gnu: + // - https://github.com/llvm/llvm-project/blob/main/lld/ELF/Arch/X86_64.cpp + switch (hole->kind) { + case HoleKind_IMAGE_REL_I386_DIR32: + // 32-bit absolute address. + // Check that we're not out of range of 32 unsigned bits: + assert(value < (1ULL << 32)); + *loc32 = (uint32_t)value; + continue; + case HoleKind_ARM64_RELOC_UNSIGNED: + case HoleKind_IMAGE_REL_AMD64_ADDR64: + case HoleKind_R_AARCH64_ABS64: + case HoleKind_X86_64_RELOC_UNSIGNED: + case HoleKind_R_X86_64_64: + // 64-bit absolute address. + *loc64 = value; + continue; + case HoleKind_R_AARCH64_CALL26: + case HoleKind_R_AARCH64_JUMP26: + // 28-bit relative branch. + assert(IS_AARCH64_BRANCH(*loc32)); + value -= (uint64_t)location; + // Check that we're not out of range of 28 signed bits: + assert((int64_t)value >= -(1 << 27)); + assert((int64_t)value < (1 << 27)); + // Since instructions are 4-byte aligned, only use 26 bits: + assert(get_bits(value, 0, 2) == 0); + set_bits(loc32, 0, value, 2, 26); + continue; + case HoleKind_R_AARCH64_MOVW_UABS_G0_NC: + // 16-bit low part of an absolute address. + assert(IS_AARCH64_MOV(*loc32)); + // Check the implicit shift (this is "part 0 of 3"): + assert(get_bits(*loc32, 21, 2) == 0); + set_bits(loc32, 5, value, 0, 16); + continue; + case HoleKind_R_AARCH64_MOVW_UABS_G1_NC: + // 16-bit middle-low part of an absolute address. + assert(IS_AARCH64_MOV(*loc32)); + // Check the implicit shift (this is "part 1 of 3"): + assert(get_bits(*loc32, 21, 2) == 1); + set_bits(loc32, 5, value, 16, 16); + continue; + case HoleKind_R_AARCH64_MOVW_UABS_G2_NC: + // 16-bit middle-high part of an absolute address. + assert(IS_AARCH64_MOV(*loc32)); + // Check the implicit shift (this is "part 2 of 3"): + assert(get_bits(*loc32, 21, 2) == 2); + set_bits(loc32, 5, value, 32, 16); + continue; + case HoleKind_R_AARCH64_MOVW_UABS_G3: + // 16-bit high part of an absolute address. + assert(IS_AARCH64_MOV(*loc32)); + // Check the implicit shift (this is "part 3 of 3"): + assert(get_bits(*loc32, 21, 2) == 3); + set_bits(loc32, 5, value, 48, 16); + continue; + case HoleKind_ARM64_RELOC_GOT_LOAD_PAGE21: + // 21-bit count of pages between this page and an absolute address's + // page... I know, I know, it's weird. Pairs nicely with + // ARM64_RELOC_GOT_LOAD_PAGEOFF12 (below). + assert(IS_AARCH64_ADRP(*loc32)); + // Number of pages between this page and the value's page: + value = (value >> 12) - ((uint64_t)location >> 12); + // Check that we're not out of range of 21 signed bits: + assert((int64_t)value >= -(1 << 20)); + assert((int64_t)value < (1 << 20)); + // value[0:2] goes in loc[29:31]: + set_bits(loc32, 29, value, 0, 2); + // value[2:21] goes in loc[5:26]: + set_bits(loc32, 5, value, 2, 19); + continue; + case HoleKind_ARM64_RELOC_GOT_LOAD_PAGEOFF12: + // 12-bit low part of an absolute address. Pairs nicely with + // ARM64_RELOC_GOT_LOAD_PAGE21 (above). + assert(IS_AARCH64_LDR_OR_STR(*loc32) || IS_AARCH64_ADD_OR_SUB(*loc32)); + // There might be an implicit shift encoded in the instruction: + uint8_t shift = 0; + if (IS_AARCH64_LDR_OR_STR(*loc32)) { + shift = (uint8_t)get_bits(*loc32, 30, 2); + // If both of these are set, the shift is supposed to be 4. + // That's pretty weird, and it's never actually been observed... + assert(get_bits(*loc32, 23, 1) == 0 || get_bits(*loc32, 26, 1) == 0); + } + value = get_bits(value, 0, 12); + assert(get_bits(value, 0, shift) == 0); + set_bits(loc32, 10, value, shift, 12); + continue; + } + Py_UNREACHABLE(); + } +} + +static void +copy_and_patch(char *base, const Stencil *stencil, uint64_t *patches) +{ + memcpy(base, stencil->body, stencil->body_size); + patch(base, stencil, patches); +} + +static void +emit(const StencilGroup *group, uint64_t patches[]) +{ + copy_and_patch((char *)patches[HoleValue_CODE], &group->code, patches); + copy_and_patch((char *)patches[HoleValue_DATA], &group->data, patches); +} + +// Compiles executor in-place. Don't forget to call _PyJIT_Free later! +int +_PyJIT_Compile(_PyExecutorObject *executor, _PyUOpInstruction *trace, size_t length) +{ + // Loop once to find the total compiled size: + size_t code_size = 0; + size_t data_size = 0; + for (size_t i = 0; i < length; i++) { + _PyUOpInstruction *instruction = &trace[i]; + const StencilGroup *group = &stencil_groups[instruction->opcode]; + code_size += group->code.body_size; + data_size += group->data.body_size; + } + // Round up to the nearest page (code and data need separate pages): + size_t page_size = get_page_size(); + assert((page_size & (page_size - 1)) == 0); + code_size += page_size - (code_size & (page_size - 1)); + data_size += page_size - (data_size & (page_size - 1)); + char *memory = jit_alloc(code_size + data_size); + if (memory == NULL) { + return -1; + } + // Loop again to emit the code: + char *code = memory; + char *data = memory + code_size; + for (size_t i = 0; i < length; i++) { + _PyUOpInstruction *instruction = &trace[i]; + const StencilGroup *group = &stencil_groups[instruction->opcode]; + // Think of patches as a dictionary mapping HoleValue to uint64_t: + uint64_t patches[] = GET_PATCHES(); + patches[HoleValue_CODE] = (uint64_t)code; + patches[HoleValue_CONTINUE] = (uint64_t)code + group->code.body_size; + patches[HoleValue_DATA] = (uint64_t)data; + patches[HoleValue_EXECUTOR] = (uint64_t)executor; + patches[HoleValue_OPARG] = instruction->oparg; + patches[HoleValue_OPERAND] = instruction->operand; + patches[HoleValue_TARGET] = instruction->target; + patches[HoleValue_TOP] = (uint64_t)memory; + patches[HoleValue_ZERO] = 0; + emit(group, patches); + code += group->code.body_size; + data += group->data.body_size; + } + if (mark_executable(memory, code_size) || + mark_readable(memory + code_size, data_size)) + { + jit_free(memory, code_size + data_size); + return -1; + } + executor->jit_code = memory; + executor->jit_size = code_size + data_size; + return 0; +} + +void +_PyJIT_Free(_PyExecutorObject *executor) +{ + char *memory = (char *)executor->jit_code; + size_t size = executor->jit_size; + if (memory) { + executor->jit_code = NULL; + executor->jit_size = 0; + if (jit_free(memory, size)) { + PyErr_WriteUnraisable(NULL); + } + } +} + +#endif // _Py_JIT diff --git a/Python/marshal.c b/Python/marshal.c index 8940582c7f5328..daec7415b3fc7e 100644 --- a/Python/marshal.c +++ b/Python/marshal.c @@ -78,6 +78,7 @@ module marshal #define WFERR_UNMARSHALLABLE 1 #define WFERR_NESTEDTOODEEP 2 #define WFERR_NOMEMORY 3 +#define WFERR_CODE_NOT_ALLOWED 4 typedef struct { FILE *fp; @@ -89,6 +90,7 @@ typedef struct { char *buf; _Py_hashtable_t *hashtable; int version; + int allow_code; } WFILE; #define w_byte(c, p) do { \ @@ -225,6 +227,9 @@ w_short_pstring(const void *s, Py_ssize_t n, WFILE *p) w_byte((t) | flag, (p)); \ } while(0) +static PyObject * +_PyMarshal_WriteObjectToString(PyObject *x, int version, int allow_code); + static void w_PyLong(const PyLongObject *ob, char flag, WFILE *p) { @@ -520,7 +525,8 @@ w_complex_object(PyObject *v, char flag, WFILE *p) } Py_ssize_t i = 0; while (_PySet_NextEntry(v, &pos, &value, &hash)) { - PyObject *dump = PyMarshal_WriteObjectToString(value, p->version); + PyObject *dump = _PyMarshal_WriteObjectToString(value, + p->version, p->allow_code); if (dump == NULL) { p->error = WFERR_UNMARSHALLABLE; Py_DECREF(pairs); @@ -549,6 +555,10 @@ w_complex_object(PyObject *v, char flag, WFILE *p) Py_DECREF(pairs); } else if (PyCode_Check(v)) { + if (!p->allow_code) { + p->error = WFERR_CODE_NOT_ALLOWED; + return; + } PyCodeObject *co = (PyCodeObject *)v; PyObject *co_code = _PyCode_GetCode(co); if (co_code == NULL) { @@ -657,6 +667,7 @@ PyMarshal_WriteObjectToFile(PyObject *x, FILE *fp, int version) wf.end = wf.ptr + sizeof(buf); wf.error = WFERR_OK; wf.version = version; + wf.allow_code = 1; if (w_init_refs(&wf, version)) { return; /* caller must check PyErr_Occurred() */ } @@ -674,6 +685,7 @@ typedef struct { char *buf; Py_ssize_t buf_size; PyObject *refs; /* a list */ + int allow_code; } RFILE; static const char * @@ -1364,6 +1376,11 @@ r_object(RFILE *p) PyObject* linetable = NULL; PyObject *exceptiontable = NULL; + if (!p->allow_code) { + PyErr_SetString(PyExc_ValueError, + "unmarshalling code objects is disallowed"); + break; + } idx = r_ref_reserve(flag, p); if (idx < 0) break; @@ -1609,6 +1626,7 @@ PyMarshal_ReadObjectFromFile(FILE *fp) { RFILE rf; PyObject *result; + rf.allow_code = 1; rf.fp = fp; rf.readable = NULL; rf.depth = 0; @@ -1629,6 +1647,7 @@ PyMarshal_ReadObjectFromString(const char *str, Py_ssize_t len) { RFILE rf; PyObject *result; + rf.allow_code = 1; rf.fp = NULL; rf.readable = NULL; rf.ptr = str; @@ -1645,8 +1664,8 @@ PyMarshal_ReadObjectFromString(const char *str, Py_ssize_t len) return result; } -PyObject * -PyMarshal_WriteObjectToString(PyObject *x, int version) +static PyObject * +_PyMarshal_WriteObjectToString(PyObject *x, int version, int allow_code) { WFILE wf; @@ -1661,6 +1680,7 @@ PyMarshal_WriteObjectToString(PyObject *x, int version) wf.end = wf.ptr + PyBytes_GET_SIZE(wf.str); wf.error = WFERR_OK; wf.version = version; + wf.allow_code = allow_code; if (w_init_refs(&wf, version)) { Py_DECREF(wf.str); return NULL; @@ -1674,17 +1694,35 @@ PyMarshal_WriteObjectToString(PyObject *x, int version) } if (wf.error != WFERR_OK) { Py_XDECREF(wf.str); - if (wf.error == WFERR_NOMEMORY) + switch (wf.error) { + case WFERR_NOMEMORY: PyErr_NoMemory(); - else + break; + case WFERR_NESTEDTOODEEP: PyErr_SetString(PyExc_ValueError, - (wf.error==WFERR_UNMARSHALLABLE)?"unmarshallable object" - :"object too deeply nested to marshal"); + "object too deeply nested to marshal"); + break; + case WFERR_CODE_NOT_ALLOWED: + PyErr_SetString(PyExc_ValueError, + "marshalling code objects is disallowed"); + break; + default: + case WFERR_UNMARSHALLABLE: + PyErr_SetString(PyExc_ValueError, + "unmarshallable object"); + break; + } return NULL; } return wf.str; } +PyObject * +PyMarshal_WriteObjectToString(PyObject *x, int version) +{ + return _PyMarshal_WriteObjectToString(x, version, 1); +} + /* And an interface for Python programs... */ /*[clinic input] marshal.dump @@ -1696,6 +1734,9 @@ marshal.dump version: int(c_default="Py_MARSHAL_VERSION") = version Indicates the data format that dump should use. / + * + allow_code: bool = True + Allow to write code objects. Write the value on the open file. @@ -1706,14 +1747,14 @@ to the file. The object will not be properly read back by load(). static PyObject * marshal_dump_impl(PyObject *module, PyObject *value, PyObject *file, - int version) -/*[clinic end generated code: output=aaee62c7028a7cb2 input=6c7a3c23c6fef556]*/ + int version, int allow_code) +/*[clinic end generated code: output=429e5fd61c2196b9 input=041f7f6669b0aafb]*/ { /* XXX Quick hack -- need to do this differently */ PyObject *s; PyObject *res; - s = PyMarshal_WriteObjectToString(value, version); + s = _PyMarshal_WriteObjectToString(value, version, allow_code); if (s == NULL) return NULL; res = PyObject_CallMethodOneArg(file, &_Py_ID(write), s); @@ -1727,6 +1768,9 @@ marshal.load file: object Must be readable binary file. / + * + allow_code: bool = True + Allow to load code objects. Read one value from the open file and return it. @@ -1739,8 +1783,8 @@ dump(), load() will substitute None for the unmarshallable type. [clinic start generated code]*/ static PyObject * -marshal_load(PyObject *module, PyObject *file) -/*[clinic end generated code: output=f8e5c33233566344 input=c85c2b594cd8124a]*/ +marshal_load_impl(PyObject *module, PyObject *file, int allow_code) +/*[clinic end generated code: output=0c1aaf3546ae3ed3 input=2dca7b570653b82f]*/ { PyObject *data, *result; RFILE rf; @@ -1762,6 +1806,7 @@ marshal_load(PyObject *module, PyObject *file) result = NULL; } else { + rf.allow_code = allow_code; rf.depth = 0; rf.fp = NULL; rf.readable = file; @@ -1787,6 +1832,9 @@ marshal.dumps version: int(c_default="Py_MARSHAL_VERSION") = version Indicates the data format that dumps should use. / + * + allow_code: bool = True + Allow to write code objects. Return the bytes object that would be written to a file by dump(value, file). @@ -1795,10 +1843,11 @@ unsupported type. [clinic start generated code]*/ static PyObject * -marshal_dumps_impl(PyObject *module, PyObject *value, int version) -/*[clinic end generated code: output=9c200f98d7256cad input=a2139ea8608e9b27]*/ +marshal_dumps_impl(PyObject *module, PyObject *value, int version, + int allow_code) +/*[clinic end generated code: output=115f90da518d1d49 input=167eaecceb63f0a8]*/ { - return PyMarshal_WriteObjectToString(value, version); + return _PyMarshal_WriteObjectToString(value, version, allow_code); } /*[clinic input] @@ -1806,6 +1855,9 @@ marshal.loads bytes: Py_buffer / + * + allow_code: bool = True + Allow to load code objects. Convert the bytes-like object to a value. @@ -1814,13 +1866,14 @@ bytes in the input are ignored. [clinic start generated code]*/ static PyObject * -marshal_loads_impl(PyObject *module, Py_buffer *bytes) -/*[clinic end generated code: output=9fc65985c93d1bb1 input=6f426518459c8495]*/ +marshal_loads_impl(PyObject *module, Py_buffer *bytes, int allow_code) +/*[clinic end generated code: output=62c0c538d3edc31f input=14de68965b45aaa7]*/ { RFILE rf; char *s = bytes->buf; Py_ssize_t n = bytes->len; PyObject* result; + rf.allow_code = allow_code; rf.fp = NULL; rf.readable = NULL; rf.ptr = s; diff --git a/Python/object_stack.c b/Python/object_stack.c new file mode 100644 index 00000000000000..ced4460da00f44 --- /dev/null +++ b/Python/object_stack.c @@ -0,0 +1,108 @@ +// Stack of Python objects + +#include "Python.h" +#include "pycore_freelist.h" +#include "pycore_pystate.h" +#include "pycore_object_stack.h" + +extern _PyObjectStackChunk *_PyObjectStackChunk_New(void); +extern void _PyObjectStackChunk_Free(_PyObjectStackChunk *); + +static struct _Py_object_stack_state * +get_state(void) +{ + _PyFreeListState *state = _PyFreeListState_GET(); + return &state->object_stacks; +} + +_PyObjectStackChunk * +_PyObjectStackChunk_New(void) +{ + _PyObjectStackChunk *buf; + struct _Py_object_stack_state *state = get_state(); + if (state->numfree > 0) { + buf = state->free_list; + state->free_list = buf->prev; + state->numfree--; + } + else { + // NOTE: we use PyMem_RawMalloc() here because this is used by the GC + // during mimalloc heap traversal. In that context, it is not safe to + // allocate mimalloc memory, such as via PyMem_Malloc(). + buf = PyMem_RawMalloc(sizeof(_PyObjectStackChunk)); + if (buf == NULL) { + return NULL; + } + } + buf->prev = NULL; + buf->n = 0; + return buf; +} + +void +_PyObjectStackChunk_Free(_PyObjectStackChunk *buf) +{ + assert(buf->n == 0); + struct _Py_object_stack_state *state = get_state(); + if (state->numfree >= 0 && + state->numfree < _PyObjectStackChunk_MAXFREELIST) + { + buf->prev = state->free_list; + state->free_list = buf; + state->numfree++; + } + else { + PyMem_RawFree(buf); + } +} + +void +_PyObjectStack_Clear(_PyObjectStack *queue) +{ + while (queue->head != NULL) { + _PyObjectStackChunk *buf = queue->head; + buf->n = 0; + queue->head = buf->prev; + _PyObjectStackChunk_Free(buf); + } +} + +void +_PyObjectStack_Merge(_PyObjectStack *dst, _PyObjectStack *src) +{ + if (src->head == NULL) { + return; + } + + if (dst->head != NULL) { + // First, append dst to the bottom of src + _PyObjectStackChunk *last = src->head; + while (last->prev != NULL) { + last = last->prev; + } + last->prev = dst->head; + } + + // Now that src has all the chunks, set dst to src + dst->head = src->head; + src->head = NULL; +} + +void +_PyObjectStackChunk_ClearFreeList(_PyFreeListState *free_lists, int is_finalization) +{ + if (!is_finalization) { + // Ignore requests to clear the free list during GC. We use object + // stacks during GC, so emptying the free-list is counterproductive. + return; + } + + struct _Py_object_stack_state *state = &free_lists->object_stacks; + while (state->numfree > 0) { + _PyObjectStackChunk *buf = state->free_list; + state->free_list = buf->prev; + state->numfree--; + PyMem_RawFree(buf); + } + state->numfree = -1; +} diff --git a/Python/optimizer.c b/Python/optimizer.c index 0ff16191680a4b..d71ca0aef0e11a 100644 --- a/Python/optimizer.c +++ b/Python/optimizer.c @@ -2,12 +2,12 @@ #include "opcode.h" #include "pycore_interp.h" #include "pycore_bitutils.h" // _Py_popcount32() -#include "pycore_opcode_metadata.h" // _PyOpcode_OpName() +#include "pycore_opcode_metadata.h" // _PyOpcode_OpName[] #include "pycore_opcode_utils.h" // MAX_REAL_OPCODE #include "pycore_optimizer.h" // _Py_uop_analyze_and_optimize() #include "pycore_pystate.h" // _PyInterpreterState_GET() #include "pycore_uop_ids.h" -#include "pycore_uops.h" +#include "pycore_jit.h" #include "cpython/optimizer.h" #include #include @@ -17,6 +17,8 @@ #include "pycore_uop_metadata.h" // Uop tables #undef NEED_OPCODE_METADATA +#define UOP_MAX_TRACE_LENGTH 512 + #define MAX_EXECUTORS_SIZE 256 @@ -106,16 +108,14 @@ PyUnstable_Replace_Executor(PyCodeObject *code, _Py_CODEUNIT *instr, _PyExecutor } static int -error_optimize( +never_optimize( _PyOptimizerObject* self, - PyCodeObject *code, + _PyInterpreterFrame *frame, _Py_CODEUNIT *instr, _PyExecutorObject **exec, int Py_UNUSED(stack_entries)) { - assert(0); - PyErr_Format(PyExc_SystemError, "Should never call error_optimize"); - return -1; + return 0; } PyTypeObject _PyDefaultOptimizer_Type = { @@ -128,7 +128,7 @@ PyTypeObject _PyDefaultOptimizer_Type = { _PyOptimizerObject _PyOptimizer_Default = { PyObject_HEAD_INIT(&_PyDefaultOptimizer_Type) - .optimize = error_optimize, + .optimize = never_optimize, .resume_threshold = INT16_MAX, .backedge_threshold = INT16_MAX, }; @@ -162,24 +162,22 @@ PyUnstable_SetOptimizer(_PyOptimizerObject *optimizer) } int -_PyOptimizer_BackEdge(_PyInterpreterFrame *frame, _Py_CODEUNIT *src, _Py_CODEUNIT *dest, PyObject **stack_pointer) +_PyOptimizer_Optimize(_PyInterpreterFrame *frame, _Py_CODEUNIT *start, PyObject **stack_pointer) { - assert(src->op.code == JUMP_BACKWARD); PyCodeObject *code = (PyCodeObject *)frame->f_executable; assert(PyCode_Check(code)); PyInterpreterState *interp = _PyInterpreterState_GET(); - if (!has_space_for_executor(code, src)) { + if (!has_space_for_executor(code, start)) { return 0; } _PyOptimizerObject *opt = interp->optimizer; _PyExecutorObject *executor = NULL; - /* Start optimizing at the destination to guarantee forward progress */ - int err = opt->optimize(opt, code, dest, &executor, (int)(stack_pointer - _PyFrame_Stackbase(frame))); + int err = opt->optimize(opt, frame, start, &executor, (int)(stack_pointer - _PyFrame_Stackbase(frame))); if (err <= 0) { assert(executor == NULL); return err; } - int index = get_index_for_executor(code, src); + int index = get_index_for_executor(code, start); if (index < 0) { /* Out of memory. Don't raise and assume that the * error will show up elsewhere. @@ -190,7 +188,7 @@ _PyOptimizer_BackEdge(_PyInterpreterFrame *frame, _Py_CODEUNIT *src, _Py_CODEUNI Py_DECREF(executor); return 0; } - insert_executor(code, src, index, executor); + insert_executor(code, start, index, executor); Py_DECREF(executor); return 1; } @@ -212,27 +210,6 @@ PyUnstable_GetExecutor(PyCodeObject *code, int offset) return NULL; } -/** Test support **/ - - -typedef struct { - _PyOptimizerObject base; - int64_t count; -} _PyCounterOptimizerObject; - -typedef struct { - _PyExecutorObject executor; - _PyCounterOptimizerObject *optimizer; - _Py_CODEUNIT *next_instr; -} _PyCounterExecutorObject; - -static void -counter_dealloc(_PyCounterExecutorObject *self) { - _Py_ExecutorClear((_PyExecutorObject *)self); - Py_DECREF(self->optimizer); - PyObject_Free(self); -} - static PyObject * is_valid(PyObject *self, PyObject *Py_UNUSED(ignored)) { @@ -244,89 +221,14 @@ static PyMethodDef executor_methods[] = { { NULL, NULL }, }; -PyTypeObject _PyCounterExecutor_Type = { - PyVarObject_HEAD_INIT(&PyType_Type, 0) - .tp_name = "counting_executor", - .tp_basicsize = sizeof(_PyCounterExecutorObject), - .tp_itemsize = 0, - .tp_flags = Py_TPFLAGS_DEFAULT | Py_TPFLAGS_DISALLOW_INSTANTIATION, - .tp_dealloc = (destructor)counter_dealloc, - .tp_methods = executor_methods, -}; - -static _Py_CODEUNIT * -counter_execute(_PyExecutorObject *self, _PyInterpreterFrame *frame, PyObject **stack_pointer) -{ - ((_PyCounterExecutorObject *)self)->optimizer->count++; - _PyFrame_SetStackPointer(frame, stack_pointer); - Py_DECREF(self); - return ((_PyCounterExecutorObject *)self)->next_instr; -} - -static int -counter_optimize( - _PyOptimizerObject* self, - PyCodeObject *code, - _Py_CODEUNIT *instr, - _PyExecutorObject **exec_ptr, - int Py_UNUSED(curr_stackentries) -) -{ - _PyCounterExecutorObject *executor = (_PyCounterExecutorObject *)_PyObject_New(&_PyCounterExecutor_Type); - if (executor == NULL) { - return -1; - } - executor->executor.execute = counter_execute; - Py_INCREF(self); - executor->optimizer = (_PyCounterOptimizerObject *)self; - executor->next_instr = instr; - *exec_ptr = (_PyExecutorObject *)executor; - _PyBloomFilter empty; - _Py_BloomFilter_Init(&empty); - _Py_ExecutorInit((_PyExecutorObject *)executor, &empty); - return 1; -} - -static PyObject * -counter_get_counter(PyObject *self, PyObject *args) -{ - return PyLong_FromLongLong(((_PyCounterOptimizerObject *)self)->count); -} - -static PyMethodDef counter_optimizer_methods[] = { - { "get_count", counter_get_counter, METH_NOARGS, NULL }, - { NULL, NULL }, -}; - -PyTypeObject _PyCounterOptimizer_Type = { - PyVarObject_HEAD_INIT(&PyType_Type, 0) - .tp_name = "Counter optimizer", - .tp_basicsize = sizeof(_PyCounterOptimizerObject), - .tp_itemsize = 0, - .tp_flags = Py_TPFLAGS_DEFAULT | Py_TPFLAGS_DISALLOW_INSTANTIATION, - .tp_methods = counter_optimizer_methods, - .tp_dealloc = (destructor)PyObject_Del, -}; - -PyObject * -PyUnstable_Optimizer_NewCounter(void) -{ - _PyCounterOptimizerObject *opt = (_PyCounterOptimizerObject *)_PyObject_New(&_PyCounterOptimizer_Type); - if (opt == NULL) { - return NULL; - } - opt->base.optimize = counter_optimize; - opt->base.resume_threshold = INT16_MAX; - opt->base.backedge_threshold = 0; - opt->count = 0; - return (PyObject *)opt; -} - ///////////////////// Experimental UOp Optimizer ///////////////////// static void -uop_dealloc(_PyUOpExecutorObject *self) { - _Py_ExecutorClear((_PyExecutorObject *)self); +uop_dealloc(_PyExecutorObject *self) { + _Py_ExecutorClear(self); +#ifdef _Py_JIT + _PyJIT_Free(self); +#endif PyObject_Free(self); } @@ -337,13 +239,13 @@ _PyUOpName(int index) } static Py_ssize_t -uop_len(_PyUOpExecutorObject *self) +uop_len(_PyExecutorObject *self) { return Py_SIZE(self); } static PyObject * -uop_item(_PyUOpExecutorObject *self, Py_ssize_t index) +uop_item(_PyExecutorObject *self, Py_ssize_t index) { Py_ssize_t len = uop_len(self); if (index < 0 || index >= len) { @@ -381,7 +283,7 @@ PySequenceMethods uop_as_sequence = { PyTypeObject _PyUOpExecutor_Type = { PyVarObject_HEAD_INIT(&PyType_Type, 0) .tp_name = "uop_executor", - .tp_basicsize = sizeof(_PyUOpExecutorObject) - sizeof(_PyUOpInstruction), + .tp_basicsize = offsetof(_PyExecutorObject, trace), .tp_itemsize = sizeof(_PyUOpInstruction), .tp_flags = Py_TPFLAGS_DEFAULT | Py_TPFLAGS_DISALLOW_INSTANTIATION, .tp_dealloc = (destructor)uop_dealloc, @@ -415,38 +317,6 @@ BRANCH_TO_GUARD[4][2] = { #define CONFIDENCE_RANGE 1000 #define CONFIDENCE_CUTOFF 333 -/* Returns 1 on success, - * 0 if it failed to produce a worthwhile trace, - * and -1 on an error. - */ -static int -translate_bytecode_to_trace( - PyCodeObject *code, - _Py_CODEUNIT *instr, - _PyUOpInstruction *trace, - int buffer_size, - _PyBloomFilter *dependencies) -{ - PyCodeObject *initial_code = code; - _Py_BloomFilter_Add(dependencies, initial_code); - _Py_CODEUNIT *initial_instr = instr; - int trace_length = 0; - int max_length = buffer_size; - struct { - PyCodeObject *code; - _Py_CODEUNIT *instr; - } trace_stack[TRACE_STACK_SIZE]; - int trace_stack_depth = 0; - int confidence = CONFIDENCE_RANGE; // Adjusted by branch instructions - -#ifdef Py_DEBUG - char *python_lltrace = Py_GETENV("PYTHON_LLTRACE"); - int lltrace = 0; - if (python_lltrace != NULL && *python_lltrace >= '0') { - lltrace = *python_lltrace - '0'; // TODO: Parse an int and all that - } -#endif - #ifdef Py_DEBUG #define DPRINTF(level, ...) \ if (lltrace >= (level)) { printf(__VA_ARGS__); } @@ -491,7 +361,8 @@ translate_bytecode_to_trace( ADD_TO_TRACE(_EXIT_TRACE, 0, 0, 0); \ goto done; \ } \ - trace_stack[trace_stack_depth].code = code; \ + assert(func->func_code == (PyObject *)code); \ + trace_stack[trace_stack_depth].func = func; \ trace_stack[trace_stack_depth].instr = instr; \ trace_stack_depth++; #define TRACE_STACK_POP() \ @@ -499,9 +370,46 @@ translate_bytecode_to_trace( Py_FatalError("Trace stack underflow\n"); \ } \ trace_stack_depth--; \ - code = trace_stack[trace_stack_depth].code; \ + func = trace_stack[trace_stack_depth].func; \ + code = (PyCodeObject *)trace_stack[trace_stack_depth].func->func_code; \ instr = trace_stack[trace_stack_depth].instr; +/* Returns 1 on success, + * 0 if it failed to produce a worthwhile trace, + * and -1 on an error. + */ +static int +translate_bytecode_to_trace( + _PyInterpreterFrame *frame, + _Py_CODEUNIT *instr, + _PyUOpInstruction *trace, + int buffer_size, + _PyBloomFilter *dependencies) +{ + bool progress_needed = true; + PyCodeObject *code = (PyCodeObject *)frame->f_executable; + PyFunctionObject *func = (PyFunctionObject *)frame->f_funcobj; + assert(PyFunction_Check(func)); + PyCodeObject *initial_code = code; + _Py_BloomFilter_Add(dependencies, initial_code); + _Py_CODEUNIT *initial_instr = instr; + int trace_length = 0; + int max_length = buffer_size; + struct { + PyFunctionObject *func; + _Py_CODEUNIT *instr; + } trace_stack[TRACE_STACK_SIZE]; + int trace_stack_depth = 0; + int confidence = CONFIDENCE_RANGE; // Adjusted by branch instructions + +#ifdef Py_DEBUG + char *python_lltrace = Py_GETENV("PYTHON_LLTRACE"); + int lltrace = 0; + if (python_lltrace != NULL && *python_lltrace >= '0') { + lltrace = *python_lltrace - '0'; // TODO: Parse an int and all that + } +#endif + DPRINTF(4, "Optimizing %s (%s:%d) at byte offset %d\n", PyUnicode_AsUTF8(code->co_qualname), @@ -509,6 +417,7 @@ translate_bytecode_to_trace( code->co_firstlineno, 2 * INSTR_IP(initial_instr, code)); uint32_t target = 0; + top: // Jump here after _PUSH_FRAME or likely branches for (;;) { target = INSTR_IP(instr, code); @@ -518,11 +427,19 @@ translate_bytecode_to_trace( uint32_t opcode = instr->op.code; uint32_t oparg = instr->op.arg; - uint32_t extras = 0; + uint32_t extended = 0; + + if (opcode == ENTER_EXECUTOR) { + assert(oparg < 256); + _PyExecutorObject *executor = code->co_executors->executors[oparg]; + opcode = executor->vm_data.opcode; + DPRINTF(2, " * ENTER_EXECUTOR -> %s\n", _PyOpcode_OpName[opcode]); + oparg = executor->vm_data.oparg; + } if (opcode == EXTENDED_ARG) { instr++; - extras += 1; + extended = 1; opcode = instr->op.code; oparg = (oparg << 8) | instr->op.arg; if (opcode == EXTENDED_ARG) { @@ -530,13 +447,23 @@ translate_bytecode_to_trace( goto done; } } - - if (opcode == ENTER_EXECUTOR) { - _PyExecutorObject *executor = - (_PyExecutorObject *)code->co_executors->executors[oparg&255]; - opcode = executor->vm_data.opcode; - DPRINTF(2, " * ENTER_EXECUTOR -> %s\n", _PyOpcode_OpName[opcode]); - oparg = (oparg & 0xffffff00) | executor->vm_data.oparg; + assert(opcode != ENTER_EXECUTOR && opcode != EXTENDED_ARG); + + /* Special case the first instruction, + * so that we can guarantee forward progress */ + if (progress_needed) { + progress_needed = false; + if (opcode == JUMP_BACKWARD || opcode == JUMP_BACKWARD_NO_INTERRUPT) { + instr += 1 + _PyOpcode_Caches[opcode] - (int32_t)oparg; + initial_instr = instr; + continue; + } + else { + if (OPCODE_HAS_DEOPT(opcode)) { + opcode = _PyOpcode_Deopt[opcode]; + } + assert(!OPCODE_HAS_DEOPT(opcode)); + } } switch (opcode) { @@ -561,24 +488,28 @@ translate_bytecode_to_trace( goto done; } uint32_t uopcode = BRANCH_TO_GUARD[opcode - POP_JUMP_IF_FALSE][jump_likely]; - _Py_CODEUNIT *next_instr = instr + 1 + _PyOpcode_Caches[_PyOpcode_Deopt[opcode]]; DPRINTF(2, "%s(%d): counter=%x, bitcount=%d, likely=%d, confidence=%d, uopcode=%s\n", - _PyUOpName(opcode), oparg, + _PyOpcode_OpName[opcode], oparg, counter, bitcount, jump_likely, confidence, _PyUOpName(uopcode)); - ADD_TO_TRACE(uopcode, max_length, 0, target); + _Py_CODEUNIT *next_instr = instr + 1 + _PyOpcode_Caches[_PyOpcode_Deopt[opcode]]; + _Py_CODEUNIT *target_instr = next_instr + oparg; if (jump_likely) { - _Py_CODEUNIT *target_instr = next_instr + oparg; DPRINTF(2, "Jump likely (%x = %d bits), continue at byte offset %d\n", instr[1].cache, bitcount, 2 * INSTR_IP(target_instr, code)); instr = target_instr; + ADD_TO_TRACE(uopcode, max_length, 0, INSTR_IP(next_instr, code)); goto top; } + ADD_TO_TRACE(uopcode, max_length, 0, INSTR_IP(target_instr, code)); break; } case JUMP_BACKWARD: + case JUMP_BACKWARD_NO_INTERRUPT: { - if (instr + 2 - oparg == initial_instr && code == initial_code) { + _Py_CODEUNIT *target = instr + 1 + _PyOpcode_Caches[opcode] - (int)oparg; + if (target == initial_instr) { + /* We have looped round to the start */ RESERVE(1); ADD_TO_TRACE(_JUMP_TO_TOP, 0, 0, 0); } @@ -623,15 +554,7 @@ translate_bytecode_to_trace( int offset = expansion->uops[i].offset + 1; switch (expansion->uops[i].size) { case OPARG_FULL: - if (extras && OPCODE_HAS_JUMP(opcode)) { - if (opcode == JUMP_BACKWARD_NO_INTERRUPT) { - oparg -= extras; - } - else { - assert(opcode != JUMP_BACKWARD); - oparg += extras; - } - } + assert(opcode != JUMP_BACKWARD_NO_INTERRUPT && opcode != JUMP_BACKWARD); break; case OPARG_CACHE_1: operand = read_u16(&instr[offset].cache); @@ -656,9 +579,10 @@ translate_bytecode_to_trace( uop = _PyUOp_Replacements[uop]; assert(uop != 0); if (uop == _FOR_ITER_TIER_TWO) { - target += 1 + INLINE_CACHE_ENTRIES_FOR_ITER + oparg + 1; - assert(_PyCode_CODE(code)[target-1].op.code == END_FOR || - _PyCode_CODE(code)[target-1].op.code == INSTRUMENTED_END_FOR); + target += 1 + INLINE_CACHE_ENTRIES_FOR_ITER + oparg + 2 + extended; + assert(_PyCode_CODE(code)[target-2].op.code == END_FOR || + _PyCode_CODE(code)[target-2].op.code == INSTRUMENTED_END_FOR); + assert(_PyCode_CODE(code)[target-1].op.code == POP_TOP); } break; default: @@ -672,6 +596,9 @@ translate_bytecode_to_trace( ADD_TO_TRACE(uop, oparg, operand, target); if (uop == _POP_FRAME) { TRACE_STACK_POP(); + /* Set the operand to the function object returned to, + * to assist optimization passes */ + trace[trace_length-1].operand = (uintptr_t)func; DPRINTF(2, "Returning to %s (%s:%d) at byte offset %d\n", PyUnicode_AsUTF8(code->co_qualname), @@ -687,10 +614,10 @@ translate_bytecode_to_trace( // Add one to account for the actual opcode/oparg pair: + 1; uint32_t func_version = read_u32(&instr[func_version_offset].cache); - PyFunctionObject *func = _PyFunction_LookupByVersion(func_version); + PyFunctionObject *new_func = _PyFunction_LookupByVersion(func_version); DPRINTF(3, "Function object: %p\n", func); - if (func != NULL) { - PyCodeObject *new_code = (PyCodeObject *)PyFunction_GET_CODE(func); + if (new_func != NULL) { + PyCodeObject *new_code = (PyCodeObject *)PyFunction_GET_CODE(new_func); if (new_code == code) { // Recursive call, bail (we could be here forever). DPRINTF(2, "Bailing on recursive call to %s (%s:%d)\n", @@ -713,7 +640,11 @@ translate_bytecode_to_trace( instr += _PyOpcode_Caches[_PyOpcode_Deopt[opcode]] + 1; TRACE_STACK_PUSH(); _Py_BloomFilter_Add(dependencies, new_code); + /* Set the operand to the callee's code object, + * to assist optimization passes */ + trace[trace_length-1].operand = (uintptr_t)new_func; code = new_code; + func = new_func; instr = _PyCode_CODE(code); DPRINTF(2, "Continuing in %s (%s:%d) at byte offset %d\n", @@ -729,7 +660,7 @@ translate_bytecode_to_trace( } break; } - DPRINTF(2, "Unsupported opcode %s\n", _PyUOpName(opcode)); + DPRINTF(2, "Unsupported opcode %s\n", _PyOpcode_OpName[opcode]); OPT_UNSUPPORTED_OPCODE(opcode); goto done; // Break out of loop } // End default @@ -747,19 +678,7 @@ translate_bytecode_to_trace( } assert(code == initial_code); // Skip short traces like _SET_IP, LOAD_FAST, _SET_IP, _EXIT_TRACE - if (trace_length > 4) { - ADD_TO_TRACE(_EXIT_TRACE, 0, 0, target); - DPRINTF(1, - "Created a trace for %s (%s:%d) at byte offset %d -- length %d\n", - PyUnicode_AsUTF8(code->co_qualname), - PyUnicode_AsUTF8(code->co_filename), - code->co_firstlineno, - 2 * INSTR_IP(initial_instr, code), - trace_length); - OPT_HIST(trace_length + buffer_size - max_length, trace_length_hist); - return 1; - } - else { + if (progress_needed || trace_length < 5) { OPT_STAT_INC(trace_too_short); DPRINTF(4, "No trace for %s (%s:%d) at byte offset %d\n", @@ -767,15 +686,25 @@ translate_bytecode_to_trace( PyUnicode_AsUTF8(code->co_filename), code->co_firstlineno, 2 * INSTR_IP(initial_instr, code)); + return 0; } - return 0; + ADD_TO_TRACE(_EXIT_TRACE, 0, 0, target); + DPRINTF(1, + "Created a trace for %s (%s:%d) at byte offset %d -- length %d\n", + PyUnicode_AsUTF8(code->co_qualname), + PyUnicode_AsUTF8(code->co_filename), + code->co_firstlineno, + 2 * INSTR_IP(initial_instr, code), + trace_length); + OPT_HIST(trace_length + buffer_size - max_length, trace_length_hist); + return 1; +} #undef RESERVE #undef RESERVE_RAW #undef INSTR_IP #undef ADD_TO_TRACE #undef DPRINTF -} #define UNSET_BIT(array, bit) (array[(bit)>>5] &= ~(1<<((bit)&31))) #define SET_BIT(array, bit) (array[(bit)>>5] |= (1<<((bit)&31))) @@ -791,7 +720,7 @@ compute_used(_PyUOpInstruction *buffer, uint32_t *used) { int count = 0; SET_BIT(used, 0); - for (int i = 0; i < _Py_UOP_MAX_TRACE_LENGTH; i++) { + for (int i = 0; i < UOP_MAX_TRACE_LENGTH; i++) { if (!BIT_IS_SET(used, i)) { continue; } @@ -823,15 +752,15 @@ compute_used(_PyUOpInstruction *buffer, uint32_t *used) static _PyExecutorObject * make_executor_from_uops(_PyUOpInstruction *buffer, _PyBloomFilter *dependencies) { - uint32_t used[(_Py_UOP_MAX_TRACE_LENGTH + 31)/32] = { 0 }; + uint32_t used[(UOP_MAX_TRACE_LENGTH + 31)/32] = { 0 }; int length = compute_used(buffer, used); - _PyUOpExecutorObject *executor = PyObject_NewVar(_PyUOpExecutorObject, &_PyUOpExecutor_Type, length); + _PyExecutorObject *executor = PyObject_NewVar(_PyExecutorObject, &_PyUOpExecutor_Type, length); if (executor == NULL) { return NULL; } int dest = length - 1; /* Scan backwards, so that we see the destinations of jumps before the jumps themselves. */ - for (int i = _Py_UOP_MAX_TRACE_LENGTH-1; i >= 0; i--) { + for (int i = UOP_MAX_TRACE_LENGTH-1; i >= 0; i--) { if (!BIT_IS_SET(used, i)) { continue; } @@ -850,8 +779,7 @@ make_executor_from_uops(_PyUOpInstruction *buffer, _PyBloomFilter *dependencies) dest--; } assert(dest == -1); - executor->base.execute = _PyUOpExecute; - _Py_ExecutorInit((_PyExecutorObject *)executor, dependencies); + _Py_ExecutorInit(executor, dependencies); #ifdef Py_DEBUG char *python_lltrace = Py_GETENV("PYTHON_LLTRACE"); int lltrace = 0; @@ -870,21 +798,29 @@ make_executor_from_uops(_PyUOpInstruction *buffer, _PyBloomFilter *dependencies) } } #endif - return (_PyExecutorObject *)executor; +#ifdef _Py_JIT + executor->jit_code = NULL; + executor->jit_size = 0; + if (_PyJIT_Compile(executor, executor->trace, Py_SIZE(executor))) { + Py_DECREF(executor); + return NULL; + } +#endif + return executor; } static int uop_optimize( _PyOptimizerObject *self, - PyCodeObject *code, + _PyInterpreterFrame *frame, _Py_CODEUNIT *instr, _PyExecutorObject **exec_ptr, int curr_stackentries) { _PyBloomFilter dependencies; _Py_BloomFilter_Init(&dependencies); - _PyUOpInstruction buffer[_Py_UOP_MAX_TRACE_LENGTH]; - int err = translate_bytecode_to_trace(code, instr, buffer, _Py_UOP_MAX_TRACE_LENGTH, &dependencies); + _PyUOpInstruction buffer[UOP_MAX_TRACE_LENGTH]; + int err = translate_bytecode_to_trace(frame, instr, buffer, UOP_MAX_TRACE_LENGTH, &dependencies); if (err <= 0) { // Error or nothing translated return err; @@ -892,9 +828,10 @@ uop_optimize( OPT_STAT_INC(traces_created); char *uop_optimize = Py_GETENV("PYTHONUOPSOPTIMIZE"); if (uop_optimize == NULL || *uop_optimize > '0') { - err = _Py_uop_analyze_and_optimize(code, buffer, _Py_UOP_MAX_TRACE_LENGTH, curr_stackentries); - if (err < 0) { - return -1; + err = _Py_uop_analyze_and_optimize(frame, buffer, + UOP_MAX_TRACE_LENGTH, curr_stackentries, &dependencies); + if (err <= 0) { + return err; } } _PyExecutorObject *executor = make_executor_from_uops(buffer, &dependencies); @@ -906,15 +843,6 @@ uop_optimize( return 1; } -/* Dummy execute() function for UOp Executor. - * The actual implementation is inlined in ceval.c, - * in _PyEval_EvalFrameDefault(). */ -_Py_CODEUNIT * -_PyUOpExecute(_PyExecutorObject *executor, _PyInterpreterFrame *frame, PyObject **stack_pointer) -{ - Py_FatalError("Tier 2 is now inlined into Tier 1"); -} - static void uop_opt_dealloc(PyObject *self) { PyObject_Free(self); @@ -944,6 +872,95 @@ PyUnstable_Optimizer_NewUOpOptimizer(void) return (PyObject *)opt; } +static void +counter_dealloc(_PyExecutorObject *self) { + PyObject *opt = (PyObject *)self->trace[0].operand; + Py_DECREF(opt); + uop_dealloc(self); +} + +PyTypeObject _PyCounterExecutor_Type = { + PyVarObject_HEAD_INIT(&PyType_Type, 0) + .tp_name = "counting_executor", + .tp_basicsize = offsetof(_PyExecutorObject, trace), + .tp_itemsize = sizeof(_PyUOpInstruction), + .tp_flags = Py_TPFLAGS_DEFAULT | Py_TPFLAGS_DISALLOW_INSTANTIATION, + .tp_dealloc = (destructor)counter_dealloc, + .tp_methods = executor_methods, +}; + +static int +counter_optimize( + _PyOptimizerObject* self, + _PyInterpreterFrame *frame, + _Py_CODEUNIT *instr, + _PyExecutorObject **exec_ptr, + int Py_UNUSED(curr_stackentries) +) +{ + PyCodeObject *code = (PyCodeObject *)frame->f_executable; + int oparg = instr->op.arg; + while (instr->op.code == EXTENDED_ARG) { + instr++; + oparg = (oparg << 8) | instr->op.arg; + } + if (instr->op.code != JUMP_BACKWARD) { + /* Counter optimizer can only handle backward edges */ + return 0; + } + _Py_CODEUNIT *target = instr + 1 + _PyOpcode_Caches[JUMP_BACKWARD] - oparg; + _PyUOpInstruction buffer[3] = { + { .opcode = _LOAD_CONST_INLINE_BORROW, .operand = (uintptr_t)self }, + { .opcode = _INTERNAL_INCREMENT_OPT_COUNTER }, + { .opcode = _EXIT_TRACE, .target = (uint32_t)(target - _PyCode_CODE(code)) } + }; + _PyBloomFilter empty; + _Py_BloomFilter_Init(&empty); + _PyExecutorObject *executor = make_executor_from_uops(buffer, &empty); + if (executor == NULL) { + return -1; + } + Py_INCREF(self); + Py_SET_TYPE(executor, &_PyCounterExecutor_Type); + *exec_ptr = executor; + return 1; +} + +static PyObject * +counter_get_counter(PyObject *self, PyObject *args) +{ + return PyLong_FromLongLong(((_PyCounterOptimizerObject *)self)->count); +} + +static PyMethodDef counter_optimizer_methods[] = { + { "get_count", counter_get_counter, METH_NOARGS, NULL }, + { NULL, NULL }, +}; + +PyTypeObject _PyCounterOptimizer_Type = { + PyVarObject_HEAD_INIT(&PyType_Type, 0) + .tp_name = "Counter optimizer", + .tp_basicsize = sizeof(_PyCounterOptimizerObject), + .tp_itemsize = 0, + .tp_flags = Py_TPFLAGS_DEFAULT | Py_TPFLAGS_DISALLOW_INSTANTIATION, + .tp_methods = counter_optimizer_methods, + .tp_dealloc = (destructor)PyObject_Del, +}; + +PyObject * +PyUnstable_Optimizer_NewCounter(void) +{ + _PyCounterOptimizerObject *opt = (_PyCounterOptimizerObject *)_PyObject_New(&_PyCounterOptimizer_Type); + if (opt == NULL) { + return NULL; + } + opt->base.optimize = counter_optimize; + opt->base.resume_threshold = INT16_MAX; + opt->base.backedge_threshold = 0; + opt->count = 0; + return (PyObject *)opt; +} + /***************************************** * Executor management diff --git a/Python/optimizer_analysis.c b/Python/optimizer_analysis.c index 8b471d70a10d7d..2cfbf4b349d0f5 100644 --- a/Python/optimizer_analysis.c +++ b/Python/optimizer_analysis.c @@ -1,10 +1,12 @@ #include "Python.h" #include "opcode.h" +#include "pycore_dict.h" #include "pycore_interp.h" #include "pycore_opcode_metadata.h" #include "pycore_opcode_utils.h" #include "pycore_pystate.h" // _PyInterpreterState_GET() -#include "pycore_uops.h" +#include "pycore_uop_metadata.h" +#include "pycore_dict.h" #include "pycore_long.h" #include "cpython/optimizer.h" #include @@ -12,6 +14,249 @@ #include #include "pycore_optimizer.h" +static int +get_mutations(PyObject* dict) { + assert(PyDict_CheckExact(dict)); + PyDictObject *d = (PyDictObject *)dict; + return (d->ma_version_tag >> DICT_MAX_WATCHERS) & ((1 << DICT_WATCHED_MUTATION_BITS)-1); +} + +static void +increment_mutations(PyObject* dict) { + assert(PyDict_CheckExact(dict)); + PyDictObject *d = (PyDictObject *)dict; + d->ma_version_tag += (1 << DICT_MAX_WATCHERS); +} + +static int +globals_watcher_callback(PyDict_WatchEvent event, PyObject* dict, + PyObject* key, PyObject* new_value) +{ + if (event == PyDict_EVENT_CLONED) { + return 0; + } + uint64_t watched_mutations = get_mutations(dict); + if (watched_mutations < _Py_MAX_ALLOWED_GLOBALS_MODIFICATIONS) { + _Py_Executors_InvalidateDependency(_PyInterpreterState_GET(), dict); + increment_mutations(dict); + } + else { + PyDict_Unwatch(1, dict); + } + return 0; +} + + +static void +global_to_const(_PyUOpInstruction *inst, PyObject *obj) +{ + assert(inst->opcode == _LOAD_GLOBAL_MODULE || inst->opcode == _LOAD_GLOBAL_BUILTINS); + assert(PyDict_CheckExact(obj)); + PyDictObject *dict = (PyDictObject *)obj; + assert(dict->ma_keys->dk_kind == DICT_KEYS_UNICODE); + PyDictUnicodeEntry *entries = DK_UNICODE_ENTRIES(dict->ma_keys); + assert(inst->operand <= UINT16_MAX); + PyObject *res = entries[inst->operand].me_value; + if (res == NULL) { + return; + } + if (_Py_IsImmortal(res)) { + inst->opcode = (inst->oparg & 1) ? _LOAD_CONST_INLINE_BORROW_WITH_NULL : _LOAD_CONST_INLINE_BORROW; + } + else { + inst->opcode = (inst->oparg & 1) ? _LOAD_CONST_INLINE_WITH_NULL : _LOAD_CONST_INLINE; + } + inst->operand = (uint64_t)res; +} + +static int +incorrect_keys(_PyUOpInstruction *inst, PyObject *obj) +{ + if (!PyDict_CheckExact(obj)) { + return 1; + } + PyDictObject *dict = (PyDictObject *)obj; + if (dict->ma_keys->dk_version != inst->operand) { + return 1; + } + return 0; +} + +/* The first two dict watcher IDs are reserved for CPython, + * so we don't need to check that they haven't been used */ +#define BUILTINS_WATCHER_ID 0 +#define GLOBALS_WATCHER_ID 1 + +/* Returns 1 if successfully optimized + * 0 if the trace is not suitable for optimization (yet) + * -1 if there was an error. */ +static int +remove_globals(_PyInterpreterFrame *frame, _PyUOpInstruction *buffer, + int buffer_size, _PyBloomFilter *dependencies) +{ + PyInterpreterState *interp = _PyInterpreterState_GET(); + PyObject *builtins = frame->f_builtins; + if (builtins != interp->builtins) { + return 1; + } + PyObject *globals = frame->f_globals; + assert(PyFunction_Check(((PyFunctionObject *)frame->f_funcobj))); + assert(((PyFunctionObject *)frame->f_funcobj)->func_builtins == builtins); + assert(((PyFunctionObject *)frame->f_funcobj)->func_globals == globals); + /* In order to treat globals as constants, we need to + * know that the globals dict is the one we expected, and + * that it hasn't changed + * In order to treat builtins as constants, we need to + * know that the builtins dict is the one we expected, and + * that it hasn't changed and that the global dictionary's + * keys have not changed */ + + /* These values represent stacks of booleans (one bool per bit). + * Pushing a frame shifts left, popping a frame shifts right. */ + uint32_t builtins_checked = 0; + uint32_t builtins_watched = 0; + uint32_t globals_checked = 0; + uint32_t globals_watched = 0; + if (interp->dict_state.watchers[1] == NULL) { + interp->dict_state.watchers[1] = globals_watcher_callback; + } + for (int pc = 0; pc < buffer_size; pc++) { + _PyUOpInstruction *inst = &buffer[pc]; + int opcode = inst->opcode; + switch(opcode) { + case _GUARD_BUILTINS_VERSION: + if (incorrect_keys(inst, builtins)) { + return 0; + } + if (interp->rare_events.builtin_dict >= _Py_MAX_ALLOWED_BUILTINS_MODIFICATIONS) { + continue; + } + if ((builtins_watched & 1) == 0) { + PyDict_Watch(BUILTINS_WATCHER_ID, builtins); + builtins_watched |= 1; + } + if (builtins_checked & 1) { + buffer[pc].opcode = NOP; + } + else { + buffer[pc].opcode = _CHECK_BUILTINS; + buffer[pc].operand = (uintptr_t)builtins; + builtins_checked |= 1; + } + break; + case _GUARD_GLOBALS_VERSION: + if (incorrect_keys(inst, globals)) { + return 0; + } + uint64_t watched_mutations = get_mutations(globals); + if (watched_mutations >= _Py_MAX_ALLOWED_GLOBALS_MODIFICATIONS) { + continue; + } + if ((globals_watched & 1) == 0) { + PyDict_Watch(GLOBALS_WATCHER_ID, globals); + _Py_BloomFilter_Add(dependencies, globals); + globals_watched |= 1; + } + if (globals_checked & 1) { + buffer[pc].opcode = NOP; + } + else { + buffer[pc].opcode = _CHECK_GLOBALS; + buffer[pc].operand = (uintptr_t)globals; + globals_checked |= 1; + } + break; + case _LOAD_GLOBAL_BUILTINS: + if (globals_checked & builtins_checked & globals_watched & builtins_watched & 1) { + global_to_const(inst, builtins); + } + break; + case _LOAD_GLOBAL_MODULE: + if (globals_checked & globals_watched & 1) { + global_to_const(inst, globals); + } + break; + case _PUSH_FRAME: + { + globals_checked <<= 1; + globals_watched <<= 1; + builtins_checked <<= 1; + builtins_watched <<= 1; + PyFunctionObject *func = (PyFunctionObject *)buffer[pc].operand; + if (func == NULL) { + return 1; + } + assert(PyFunction_Check(func)); + globals = func->func_globals; + builtins = func->func_builtins; + if (builtins != interp->builtins) { + return 1; + } + break; + } + case _POP_FRAME: + { + globals_checked >>= 1; + globals_watched >>= 1; + builtins_checked >>= 1; + builtins_watched >>= 1; + PyFunctionObject *func = (PyFunctionObject *)buffer[pc].operand; + assert(PyFunction_Check(func)); + globals = func->func_globals; + builtins = func->func_builtins; + break; + } + case _JUMP_TO_TOP: + case _EXIT_TRACE: + return 1; + } + } + return 0; +} + +static void +peephole_opt(_PyInterpreterFrame *frame, _PyUOpInstruction *buffer, int buffer_size) +{ + PyCodeObject *co = (PyCodeObject *)frame->f_executable; + for (int pc = 0; pc < buffer_size; pc++) { + int opcode = buffer[pc].opcode; + switch(opcode) { + case _LOAD_CONST: { + assert(co != NULL); + PyObject *val = PyTuple_GET_ITEM(co->co_consts, buffer[pc].oparg); + buffer[pc].opcode = _Py_IsImmortal(val) ? _LOAD_CONST_INLINE_BORROW : _LOAD_CONST_INLINE; + buffer[pc].operand = (uintptr_t)val; + break; + } + case _CHECK_PEP_523: + { + /* Setting the eval frame function invalidates + * all executors, so no need to check dynamically */ + if (_PyInterpreterState_GET()->eval_frame == NULL) { + buffer[pc].opcode = _NOP; + } + break; + } + case _PUSH_FRAME: + case _POP_FRAME: + { + PyFunctionObject *func = (PyFunctionObject *)buffer[pc].operand; + if (func == NULL) { + co = NULL; + } + else { + assert(PyFunction_Check(func)); + co = (PyCodeObject *)func->func_code; + } + break; + } + case _JUMP_TO_TOP: + case _EXIT_TRACE: + return; + } + } +} + static void remove_unneeded_uops(_PyUOpInstruction *buffer, int buffer_size) { @@ -35,13 +280,13 @@ remove_unneeded_uops(_PyUOpInstruction *buffer, int buffer_size) break; } else { - if (OPCODE_HAS_ESCAPES(opcode)) { + if (_PyUop_Flags[opcode] & HAS_ESCAPES_FLAG) { maybe_invalid = true; if (last_set_ip >= 0) { buffer[last_set_ip].opcode = _SET_IP; } } - if (OPCODE_HAS_ERROR(opcode) || opcode == _PUSH_FRAME) { + if ((_PyUop_Flags[opcode] & HAS_ERROR_FLAG) || opcode == _PUSH_FRAME) { if (last_set_ip >= 0) { buffer[last_set_ip].opcode = _SET_IP; } @@ -50,15 +295,20 @@ remove_unneeded_uops(_PyUOpInstruction *buffer, int buffer_size) } } - int _Py_uop_analyze_and_optimize( - PyCodeObject *co, + _PyInterpreterFrame *frame, _PyUOpInstruction *buffer, int buffer_size, - int curr_stacklen + int curr_stacklen, + _PyBloomFilter *dependencies ) { + int err = remove_globals(frame, buffer, buffer_size, dependencies); + if (err <= 0) { + return err; + } + peephole_opt(frame, buffer, buffer_size); remove_unneeded_uops(buffer, buffer_size); - return 0; + return 1; } diff --git a/Python/parking_lot.c b/Python/parking_lot.c index d44c1b4b93b4d2..8ba50fc1353ebd 100644 --- a/Python/parking_lot.c +++ b/Python/parking_lot.c @@ -158,11 +158,15 @@ _PySemaphore_PlatformWait(_PySemaphore *sema, _PyTime_t timeout) if (sema->counter == 0) { if (timeout >= 0) { struct timespec ts; - +#if defined(HAVE_PTHREAD_COND_TIMEDWAIT_RELATIVE_NP) + _PyTime_AsTimespec_clamp(timeout, &ts); + err = pthread_cond_timedwait_relative_np(&sema->cond, &sema->mutex, &ts); +#else _PyTime_t deadline = _PyTime_Add(_PyTime_GetSystemClock(), timeout); _PyTime_AsTimespec_clamp(deadline, &ts); err = pthread_cond_timedwait(&sema->cond, &sema->mutex, &ts); +#endif // HAVE_PTHREAD_COND_TIMEDWAIT_RELATIVE_NP } else { err = pthread_cond_wait(&sema->cond, &sema->mutex); @@ -240,6 +244,7 @@ dequeue(Bucket *bucket, const void *address) if (wait->addr == (uintptr_t)address) { llist_remove(node); --bucket->num_waiters; + wait->is_unparking = true; return wait; } } @@ -258,6 +263,7 @@ dequeue_all(Bucket *bucket, const void *address, struct llist_node *dst) llist_remove(node); llist_insert_tail(dst, node); --bucket->num_waiters; + wait->is_unparking = true; } } } @@ -333,8 +339,6 @@ _PyParkingLot_Unpark(const void *addr, _Py_unpark_fn_t *fn, void *arg) _PyRawMutex_Lock(&bucket->mutex); struct wait_entry *waiter = dequeue(bucket, addr); if (waiter) { - waiter->is_unparking = true; - int has_more_waiters = (bucket->num_waiters > 0); fn(arg, waiter->park_arg, has_more_waiters); } diff --git a/Python/pylifecycle.c b/Python/pylifecycle.c index 0ec29846b0850b..61c9d4f9ea9575 100644 --- a/Python/pylifecycle.c +++ b/Python/pylifecycle.c @@ -32,6 +32,8 @@ #include "pycore_typevarobject.h" // _Py_clear_generic_types() #include "pycore_unicodeobject.h" // _PyUnicode_InitTypes() #include "pycore_weakref.h" // _PyWeakref_GET_REF() +#include "cpython/optimizer.h" // _Py_MAX_ALLOWED_BUILTINS_MODIFICATIONS +#include "pycore_obmalloc.h" // _PyMem_init_obmalloc() #include "opcode.h" @@ -605,6 +607,16 @@ init_interp_create_gil(PyThreadState *tstate, int gil) _PyEval_InitGIL(tstate, own_gil); } +static int +builtins_dict_watcher(PyDict_WatchEvent event, PyObject *dict, PyObject *key, PyObject *new_value) +{ + PyInterpreterState *interp = _PyInterpreterState_GET(); + if (event != PyDict_EVENT_CLONED && interp->rare_events.builtin_dict < _Py_MAX_ALLOWED_BUILTINS_MODIFICATIONS) { + _Py_Executors_InvalidateAll(interp); + } + RARE_EVENT_INTERP_INC(interp, builtin_dict); + return 0; +} static PyStatus pycore_create_interpreter(_PyRuntimeState *runtime, @@ -639,6 +651,13 @@ pycore_create_interpreter(_PyRuntimeState *runtime, return status; } + // initialize the interp->obmalloc state. This must be done after + // the settings are loaded (so that feature_flags are set) but before + // any calls are made to obmalloc functions. + if (_PyMem_init_obmalloc(interp) < 0) { + return _PyStatus_NO_MEMORY(); + } + PyThreadState *tstate = _PyThreadState_New(interp, _PyThreadState_WHENCE_INTERP); if (tstate == NULL) { @@ -1226,12 +1245,19 @@ init_interp_main(PyThreadState *tstate) // Turn on experimental tier 2 (uops-based) optimizer if (is_main_interp) { +#ifndef _Py_JIT + // No JIT, maybe use the tier two interpreter: char *envvar = Py_GETENV("PYTHON_UOPS"); int enabled = envvar != NULL && *envvar > '0'; if (_Py_get_xoption(&config->xoptions, L"uops") != NULL) { enabled = 1; } if (enabled) { +#else + // Always enable tier two for JIT builds (ignoring the environment + // variable and command-line option above): + if (true) { +#endif PyObject *opt = PyUnstable_Optimizer_NewUOpOptimizer(); if (opt == NULL) { return _PyStatus_ERR("can't initialize optimizer"); @@ -1266,6 +1292,12 @@ init_interp_main(PyThreadState *tstate) } } + + interp->dict_state.watchers[0] = &builtins_dict_watcher; + if (PyDict_Watch(0, interp->builtins) != 0) { + return _PyStatus_ERR("failed to set builtin dict watcher"); + } + assert(!_PyErr_Occurred(tstate)); return _PyStatus_OK(); @@ -1592,6 +1624,15 @@ static void finalize_modules(PyThreadState *tstate) { PyInterpreterState *interp = tstate->interp; + + // Invalidate all executors and turn off tier 2 optimizer + _Py_Executors_InvalidateAll(interp); + Py_XDECREF(interp->optimizer); + interp->optimizer = &_PyOptimizer_Default; + + // Stop watching __builtin__ modifications + PyDict_Unwatch(0, interp->builtins); + PyObject *modules = _PyImport_GetModules(interp); if (modules == NULL) { // Already done @@ -1735,8 +1776,6 @@ finalize_interp_types(PyInterpreterState *interp) _PySys_FiniTypes(interp); _PyXI_FiniTypes(interp); _PyExc_Fini(interp); - _PyAsyncGen_Fini(interp); - _PyContext_Fini(interp); _PyFloat_FiniType(interp); _PyLong_FiniTypes(interp); _PyThread_FiniType(interp); @@ -1751,14 +1790,15 @@ finalize_interp_types(PyInterpreterState *interp) // a dict internally. _PyUnicode_ClearInterned(interp); - _PyDict_Fini(interp); - _PyList_Fini(interp); - _PyTuple_Fini(interp); + _PyUnicode_Fini(interp); - _PySlice_Fini(interp); +#ifndef Py_GIL_DISABLED + // With Py_GIL_DISABLED: + // the freelists for the current thread state have already been cleared. + _PyFreeListState *state = _PyFreeListState_GET(); + _PyObject_ClearFreeLists(state, 1); +#endif - _PyUnicode_Fini(interp); - _PyFloat_Fini(interp); #ifdef Py_DEBUG _PyStaticObjects_CheckRefcnt(interp); #endif @@ -1794,6 +1834,10 @@ finalize_interp_clear(PyThreadState *tstate) } finalize_interp_types(tstate->interp); + + /* finalize_interp_types may allocate Python objects so we may need to + abandon mimalloc segments again */ + _PyThreadState_ClearMimallocHeaps(tstate); } @@ -2121,6 +2165,14 @@ new_interpreter(PyThreadState **tstate_p, const PyInterpreterConfig *config) goto error; } + // initialize the interp->obmalloc state. This must be done after + // the settings are loaded (so that feature_flags are set) but before + // any calls are made to obmalloc functions. + if (_PyMem_init_obmalloc(interp) < 0) { + status = _PyStatus_NO_MEMORY(); + goto error; + } + tstate = _PyThreadState_New(interp, _PyThreadState_WHENCE_INTERP); if (tstate == NULL) { status = _PyStatus_NO_MEMORY(); diff --git a/Python/pystate.c b/Python/pystate.c index 632a119ea6d4f8..937c43033b068d 100644 --- a/Python/pystate.c +++ b/Python/pystate.c @@ -10,6 +10,7 @@ #include "pycore_frame.h" #include "pycore_initconfig.h" // _PyStatus_OK() #include "pycore_object.h" // _PyType_InitCache() +#include "pycore_object_stack.h" // _PyObjectStackChunk_ClearFreeList() #include "pycore_parking_lot.h" // _PyParkingLot_AfterFork() #include "pycore_pyerrors.h" // _PyErr_Clear() #include "pycore_pylifecycle.h" // _PyAST_Fini() @@ -17,6 +18,7 @@ #include "pycore_pystate.h" #include "pycore_runtime_init.h" // _PyRuntimeState_INIT #include "pycore_sysmodule.h" // _PySys_Audit() +#include "pycore_obmalloc.h" // _PyMem_obmalloc_state_on_heap() /* -------------------------------------------------------------------------- CAUTION @@ -65,7 +67,7 @@ _Py_thread_local PyThreadState *_Py_tss_tstate = NULL; #endif static inline PyThreadState * -current_fast_get(_PyRuntimeState *Py_UNUSED(runtime)) +current_fast_get(void) { #ifdef HAVE_THREAD_LOCAL return _Py_tss_tstate; @@ -99,14 +101,14 @@ current_fast_clear(_PyRuntimeState *Py_UNUSED(runtime)) } #define tstate_verify_not_active(tstate) \ - if (tstate == current_fast_get((tstate)->interp->runtime)) { \ + if (tstate == current_fast_get()) { \ _Py_FatalErrorFormat(__func__, "tstate %p is still current", tstate); \ } PyThreadState * _PyThreadState_GetCurrent(void) { - return current_fast_get(&_PyRuntime); + return current_fast_get(); } @@ -236,6 +238,8 @@ tstate_is_bound(PyThreadState *tstate) static void bind_gilstate_tstate(PyThreadState *); static void unbind_gilstate_tstate(PyThreadState *); +static void tstate_mimalloc_bind(PyThreadState *); + static void bind_tstate(PyThreadState *tstate) { @@ -256,6 +260,9 @@ bind_tstate(PyThreadState *tstate) tstate->native_thread_id = PyThread_get_thread_native_id(); #endif + // mimalloc state needs to be initialized from the active thread. + tstate_mimalloc_bind(tstate); + tstate->_status.bound = 1; } @@ -353,10 +360,9 @@ holds_gil(PyThreadState *tstate) // XXX Fall back to tstate->interp->runtime->ceval.gil.last_holder // (and tstate->interp->runtime->ceval.gil.locked). assert(tstate != NULL); - _PyRuntimeState *runtime = tstate->interp->runtime; /* Must be the tstate for this thread */ - assert(tstate == gilstate_tss_get(runtime)); - return tstate == current_fast_get(runtime); + assert(tstate == gilstate_tss_get(tstate->interp->runtime)); + return tstate == current_fast_get(); } @@ -511,6 +517,8 @@ _PyRuntimeState_ReInitThreads(_PyRuntimeState *runtime) return _PyStatus_NO_MEMORY(); } + _PyThread_AfterFork(&runtime->threads); + return _PyStatus_OK(); } #endif @@ -547,6 +555,11 @@ free_interpreter(PyInterpreterState *interp) // The main interpreter is statically allocated so // should not be freed. if (interp != &_PyRuntime._main_interpreter) { + if (_PyMem_obmalloc_state_on_heap(interp)) { + // interpreter has its own obmalloc state, free it + PyMem_RawFree(interp->obmalloc); + interp->obmalloc = NULL; + } PyMem_RawFree(interp); } } @@ -589,14 +602,6 @@ init_interpreter(PyInterpreterState *interp, assert(next != NULL || (interp == runtime->interpreters.main)); interp->next = next; - /* Initialize obmalloc, but only for subinterpreters, - since the main interpreter is initialized statically. */ - if (interp != &runtime->_main_interpreter) { - poolp temp[OBMALLOC_USED_POOLS_SIZE] = \ - _obmalloc_pools_INIT(interp->obmalloc.pools); - memcpy(&interp->obmalloc.pools.used, temp, sizeof(temp)); - } - PyStatus status = _PyObject_InitState(interp); if (_PyStatus_EXCEPTION(status)) { return status; @@ -606,6 +611,9 @@ init_interpreter(PyInterpreterState *interp, _PyGC_InitState(&interp->gc); PyConfig_InitPythonConfig(&interp->config); _PyType_InitCache(interp); +#ifdef Py_GIL_DISABLED + _Py_brc_init_state(interp); +#endif for (int i = 0; i < _PY_MONITORING_UNGROUPED_EVENTS; i++) { interp->monitors.tools[i] = 0; } @@ -719,7 +727,7 @@ PyInterpreterState * PyInterpreterState_New(void) { // tstate can be NULL - PyThreadState *tstate = current_fast_get(&_PyRuntime); + PyThreadState *tstate = current_fast_get(); PyInterpreterState *interp; PyStatus status = _PyInterpreterState_New(tstate, &interp); @@ -878,7 +886,7 @@ PyInterpreterState_Clear(PyInterpreterState *interp) // Use the current Python thread state to call audit hooks and to collect // garbage. It can be different than the current Python thread state // of 'interp'. - PyThreadState *current_tstate = current_fast_get(interp->runtime); + PyThreadState *current_tstate = current_fast_get(); _PyImport_ClearCore(interp); interpreter_clear(interp, current_tstate); } @@ -904,7 +912,7 @@ PyInterpreterState_Delete(PyInterpreterState *interp) // XXX Clearing the "current" thread state should happen before // we start finalizing the interpreter (or the current thread state). - PyThreadState *tcur = current_fast_get(runtime); + PyThreadState *tcur = current_fast_get(); if (tcur != NULL && interp == tcur->interp) { /* Unset current thread. After this, many C API calls become crashy. */ _PyThreadState_Detach(tcur); @@ -1006,7 +1014,7 @@ _PyInterpreterState_SetRunningMain(PyInterpreterState *interp) if (_PyInterpreterState_FailIfRunningMain(interp) < 0) { return -1; } - PyThreadState *tstate = current_fast_get(&_PyRuntime); + PyThreadState *tstate = current_fast_get(); _Py_EnsureTstateNotNULL(tstate); if (tstate->interp != interp) { PyErr_SetString(PyExc_RuntimeError, @@ -1021,7 +1029,7 @@ void _PyInterpreterState_SetNotRunningMain(PyInterpreterState *interp) { PyThreadState *tstate = interp->threads.main; - assert(tstate == current_fast_get(&_PyRuntime)); + assert(tstate == current_fast_get()); if (tstate->on_delete != NULL) { // The threading module was imported for the first time in this @@ -1174,7 +1182,7 @@ PyInterpreterState_GetDict(PyInterpreterState *interp) PyInterpreterState* PyInterpreterState_Get(void) { - PyThreadState *tstate = current_fast_get(&_PyRuntime); + PyThreadState *tstate = current_fast_get(); _Py_EnsureTstateNotNULL(tstate); PyInterpreterState *interp = tstate->interp; if (interp == NULL) { @@ -1331,6 +1339,16 @@ init_threadstate(_PyThreadStateImpl *_tstate, tstate->datastack_limit = NULL; tstate->what_event = -1; +#ifdef Py_GIL_DISABLED + // Initialize biased reference counting inter-thread queue + _Py_brc_init_thread(tstate); +#endif + + if (interp->stoptheworld.requested || _PyRuntime.stoptheworld.requested) { + // Start in the suspended state if there is an ongoing stop-the-world. + tstate->state = _Py_THREAD_SUSPENDED; + } + tstate->_status.initialized = 1; } @@ -1454,7 +1472,7 @@ void PyThreadState_Clear(PyThreadState *tstate) { assert(tstate->_status.initialized && !tstate->_status.cleared); - assert(current_fast_get(&_PyRuntime)->interp == tstate->interp); + assert(current_fast_get()->interp == tstate->interp); // XXX assert(!tstate->_status.bound || tstate->_status.unbound); tstate->_status.finalizing = 1; // just in case @@ -1532,6 +1550,16 @@ PyThreadState_Clear(PyThreadState *tstate) // don't call _PyInterpreterState_SetNotRunningMain() yet. tstate->on_delete(tstate->on_delete_data); } +#ifdef Py_GIL_DISABLED + // Each thread should clear own freelists in free-threading builds. + _PyFreeListState *freelist_state = _PyFreeListState_GET(); + _PyObject_ClearFreeLists(freelist_state, 1); + + // Remove ourself from the biased reference counting table of threads. + _Py_brc_remove_thread(tstate); +#endif + + _PyThreadState_ClearMimallocHeaps(tstate); tstate->_status.cleared = 1; @@ -1539,6 +1567,9 @@ PyThreadState_Clear(PyThreadState *tstate) // XXX Do it as early in the function as possible. } +static void +decrement_stoptheworld_countdown(struct _stoptheworld_state *stw); + /* Common code for PyThreadState_Delete() and PyThreadState_DeleteCurrent() */ static void tstate_delete_common(PyThreadState *tstate) @@ -1562,6 +1593,16 @@ tstate_delete_common(PyThreadState *tstate) if (tstate->next) { tstate->next->prev = tstate->prev; } + if (tstate->state != _Py_THREAD_SUSPENDED) { + // Any ongoing stop-the-world request should not wait for us because + // our thread is getting deleted. + if (interp->stoptheworld.requested) { + decrement_stoptheworld_countdown(&interp->stoptheworld); + } + if (runtime->stoptheworld.requested) { + decrement_stoptheworld_countdown(&runtime->stoptheworld); + } + } HEAD_UNLOCK(runtime); // XXX Unbind in PyThreadState_Clear(), or earlier @@ -1615,7 +1656,7 @@ _PyThreadState_DeleteCurrent(PyThreadState *tstate) void PyThreadState_DeleteCurrent(void) { - PyThreadState *tstate = current_fast_get(&_PyRuntime); + PyThreadState *tstate = current_fast_get(); _PyThreadState_DeleteCurrent(tstate); } @@ -1691,7 +1732,7 @@ _PyThreadState_GetDict(PyThreadState *tstate) PyObject * PyThreadState_GetDict(void) { - PyThreadState *tstate = current_fast_get(&_PyRuntime); + PyThreadState *tstate = current_fast_get(); if (tstate == NULL) { return NULL; } @@ -1767,13 +1808,9 @@ tstate_try_attach(PyThreadState *tstate) { #ifdef Py_GIL_DISABLED int expected = _Py_THREAD_DETACHED; - if (_Py_atomic_compare_exchange_int( - &tstate->state, - &expected, - _Py_THREAD_ATTACHED)) { - return 1; - } - return 0; + return _Py_atomic_compare_exchange_int(&tstate->state, + &expected, + _Py_THREAD_ATTACHED); #else assert(tstate->state == _Py_THREAD_DETACHED); tstate->state = _Py_THREAD_ATTACHED; @@ -1792,6 +1829,20 @@ tstate_set_detached(PyThreadState *tstate) #endif } +static void +tstate_wait_attach(PyThreadState *tstate) +{ + do { + int expected = _Py_THREAD_SUSPENDED; + + // Wait until we're switched out of SUSPENDED to DETACHED. + _PyParkingLot_Park(&tstate->state, &expected, sizeof(tstate->state), + /*timeout=*/-1, NULL, /*detach=*/0); + + // Once we're back in DETACHED we can re-attach + } while (!tstate_try_attach(tstate)); +} + void _PyThreadState_Attach(PyThreadState *tstate) { @@ -1802,7 +1853,7 @@ _PyThreadState_Attach(PyThreadState *tstate) #endif _Py_EnsureTstateNotNULL(tstate); - if (current_fast_get(&_PyRuntime) != NULL) { + if (current_fast_get() != NULL) { Py_FatalError("non-NULL old thread state"); } @@ -1813,10 +1864,7 @@ _PyThreadState_Attach(PyThreadState *tstate) tstate_activate(tstate); if (!tstate_try_attach(tstate)) { - // TODO: Once stop-the-world GC is implemented for --disable-gil builds - // this will need to wait until the GC completes. For now, this case - // should never happen. - Py_FatalError("thread attach failed"); + tstate_wait_attach(tstate); } // Resume previous critical section. This acquires the lock(s) from the @@ -1830,21 +1878,237 @@ _PyThreadState_Attach(PyThreadState *tstate) #endif } -void -_PyThreadState_Detach(PyThreadState *tstate) +static void +detach_thread(PyThreadState *tstate, int detached_state) { // XXX assert(tstate_is_alive(tstate) && tstate_is_bound(tstate)); assert(tstate->state == _Py_THREAD_ATTACHED); - assert(tstate == current_fast_get(&_PyRuntime)); + assert(tstate == current_fast_get()); if (tstate->critical_section != 0) { _PyCriticalSection_SuspendAll(tstate); } - tstate_set_detached(tstate); tstate_deactivate(tstate); + tstate_set_detached(tstate); current_fast_clear(&_PyRuntime); _PyEval_ReleaseLock(tstate->interp, tstate); } +void +_PyThreadState_Detach(PyThreadState *tstate) +{ + detach_thread(tstate, _Py_THREAD_DETACHED); +} + +void +_PyThreadState_Suspend(PyThreadState *tstate) +{ + _PyRuntimeState *runtime = &_PyRuntime; + + assert(tstate->state == _Py_THREAD_ATTACHED); + + struct _stoptheworld_state *stw = NULL; + HEAD_LOCK(runtime); + if (runtime->stoptheworld.requested) { + stw = &runtime->stoptheworld; + } + else if (tstate->interp->stoptheworld.requested) { + stw = &tstate->interp->stoptheworld; + } + HEAD_UNLOCK(runtime); + + if (stw == NULL) { + // Switch directly to "detached" if there is no active stop-the-world + // request. + detach_thread(tstate, _Py_THREAD_DETACHED); + return; + } + + // Switch to "suspended" state. + detach_thread(tstate, _Py_THREAD_SUSPENDED); + + // Decrease the count of remaining threads needing to park. + HEAD_LOCK(runtime); + decrement_stoptheworld_countdown(stw); + HEAD_UNLOCK(runtime); +} + +// Decrease stop-the-world counter of remaining number of threads that need to +// pause. If we are the final thread to pause, notify the requesting thread. +static void +decrement_stoptheworld_countdown(struct _stoptheworld_state *stw) +{ + assert(stw->thread_countdown > 0); + if (--stw->thread_countdown == 0) { + _PyEvent_Notify(&stw->stop_event); + } +} + +#ifdef Py_GIL_DISABLED +// Interpreter for _Py_FOR_EACH_THREAD(). For global stop-the-world events, +// we start with the first interpreter and then iterate over all interpreters. +// For per-interpreter stop-the-world events, we only operate on the one +// interpreter. +static PyInterpreterState * +interp_for_stop_the_world(struct _stoptheworld_state *stw) +{ + return (stw->is_global + ? PyInterpreterState_Head() + : _Py_CONTAINER_OF(stw, PyInterpreterState, stoptheworld)); +} + +// Loops over threads for a stop-the-world event. +// For global: all threads in all interpreters +// For per-interpreter: all threads in the interpreter +#define _Py_FOR_EACH_THREAD(stw, i, t) \ + for (i = interp_for_stop_the_world((stw)); \ + i != NULL; i = ((stw->is_global) ? i->next : NULL)) \ + for (t = i->threads.head; t; t = t->next) + + +// Try to transition threads atomically from the "detached" state to the +// "gc stopped" state. Returns true if all threads are in the "gc stopped" +static bool +park_detached_threads(struct _stoptheworld_state *stw) +{ + int num_parked = 0; + PyInterpreterState *i; + PyThreadState *t; + _Py_FOR_EACH_THREAD(stw, i, t) { + int state = _Py_atomic_load_int_relaxed(&t->state); + if (state == _Py_THREAD_DETACHED) { + // Atomically transition to "suspended" if in "detached" state. + if (_Py_atomic_compare_exchange_int(&t->state, + &state, _Py_THREAD_SUSPENDED)) { + num_parked++; + } + } + else if (state == _Py_THREAD_ATTACHED && t != stw->requester) { + // TODO: set this per-thread, rather than per-interpreter. + _Py_set_eval_breaker_bit(t->interp, _PY_EVAL_PLEASE_STOP_BIT, 1); + } + } + stw->thread_countdown -= num_parked; + assert(stw->thread_countdown >= 0); + return num_parked > 0 && stw->thread_countdown == 0; +} + +static void +stop_the_world(struct _stoptheworld_state *stw) +{ + _PyRuntimeState *runtime = &_PyRuntime; + + PyMutex_Lock(&stw->mutex); + if (stw->is_global) { + _PyRWMutex_Lock(&runtime->stoptheworld_mutex); + } + else { + _PyRWMutex_RLock(&runtime->stoptheworld_mutex); + } + + HEAD_LOCK(runtime); + stw->requested = 1; + stw->thread_countdown = 0; + stw->stop_event = (PyEvent){0}; // zero-initialize (unset) + stw->requester = _PyThreadState_GET(); // may be NULL + + PyInterpreterState *i; + PyThreadState *t; + _Py_FOR_EACH_THREAD(stw, i, t) { + if (t != stw->requester) { + // Count all the other threads (we don't wait on ourself). + stw->thread_countdown++; + } + } + + if (stw->thread_countdown == 0) { + HEAD_UNLOCK(runtime); + stw->world_stopped = 1; + return; + } + + for (;;) { + // Switch threads that are detached to the GC stopped state + bool stopped_all_threads = park_detached_threads(stw); + HEAD_UNLOCK(runtime); + + if (stopped_all_threads) { + break; + } + + _PyTime_t wait_ns = 1000*1000; // 1ms (arbitrary, may need tuning) + if (PyEvent_WaitTimed(&stw->stop_event, wait_ns)) { + assert(stw->thread_countdown == 0); + break; + } + + HEAD_LOCK(runtime); + } + stw->world_stopped = 1; +} + +static void +start_the_world(struct _stoptheworld_state *stw) +{ + _PyRuntimeState *runtime = &_PyRuntime; + assert(PyMutex_IsLocked(&stw->mutex)); + + HEAD_LOCK(runtime); + stw->requested = 0; + stw->world_stopped = 0; + // Switch threads back to the detached state. + PyInterpreterState *i; + PyThreadState *t; + _Py_FOR_EACH_THREAD(stw, i, t) { + if (t != stw->requester) { + assert(t->state == _Py_THREAD_SUSPENDED); + _Py_atomic_store_int(&t->state, _Py_THREAD_DETACHED); + _PyParkingLot_UnparkAll(&t->state); + } + } + stw->requester = NULL; + HEAD_UNLOCK(runtime); + if (stw->is_global) { + _PyRWMutex_Unlock(&runtime->stoptheworld_mutex); + } + else { + _PyRWMutex_RUnlock(&runtime->stoptheworld_mutex); + } + PyMutex_Unlock(&stw->mutex); +} +#endif // Py_GIL_DISABLED + +void +_PyEval_StopTheWorldAll(_PyRuntimeState *runtime) +{ +#ifdef Py_GIL_DISABLED + stop_the_world(&runtime->stoptheworld); +#endif +} + +void +_PyEval_StartTheWorldAll(_PyRuntimeState *runtime) +{ +#ifdef Py_GIL_DISABLED + start_the_world(&runtime->stoptheworld); +#endif +} + +void +_PyEval_StopTheWorld(PyInterpreterState *interp) +{ +#ifdef Py_GIL_DISABLED + stop_the_world(&interp->stoptheworld); +#endif +} + +void +_PyEval_StartTheWorld(PyInterpreterState *interp) +{ +#ifdef Py_GIL_DISABLED + start_the_world(&interp->stoptheworld); +#endif +} + //---------- // other API //---------- @@ -1904,14 +2168,14 @@ PyThreadState_SetAsyncExc(unsigned long id, PyObject *exc) PyThreadState * PyThreadState_GetUnchecked(void) { - return current_fast_get(&_PyRuntime); + return current_fast_get(); } PyThreadState * PyThreadState_Get(void) { - PyThreadState *tstate = current_fast_get(&_PyRuntime); + PyThreadState *tstate = current_fast_get(); _Py_EnsureTstateNotNULL(tstate); return tstate; } @@ -1919,7 +2183,7 @@ PyThreadState_Get(void) PyThreadState * _PyThreadState_Swap(_PyRuntimeState *runtime, PyThreadState *newts) { - PyThreadState *oldts = current_fast_get(runtime); + PyThreadState *oldts = current_fast_get(); if (oldts != NULL) { _PyThreadState_Detach(oldts); } @@ -2014,7 +2278,7 @@ PyObject * _PyThread_CurrentFrames(void) { _PyRuntimeState *runtime = &_PyRuntime; - PyThreadState *tstate = current_fast_get(runtime); + PyThreadState *tstate = current_fast_get(); if (_PySys_Audit(tstate, "sys._current_frames", NULL) < 0) { return NULL; } @@ -2075,7 +2339,7 @@ PyObject * _PyThread_CurrentExceptions(void) { _PyRuntimeState *runtime = &_PyRuntime; - PyThreadState *tstate = current_fast_get(runtime); + PyThreadState *tstate = current_fast_get(); _Py_EnsureTstateNotNULL(tstate); @@ -2217,12 +2481,22 @@ PyGILState_Check(void) return 1; } - PyThreadState *tstate = current_fast_get(runtime); + PyThreadState *tstate = current_fast_get(); if (tstate == NULL) { return 0; } - return (tstate == gilstate_tss_get(runtime)); +#ifdef MS_WINDOWS + int err = GetLastError(); +#endif + + PyThreadState *tcur = gilstate_tss_get(runtime); + +#ifdef MS_WINDOWS + SetLastError(err); +#endif + + return (tstate == tcur); } PyGILState_STATE @@ -2315,7 +2589,7 @@ PyGILState_Release(PyGILState_STATE oldstate) * races; see bugs 225673 and 1061968 (that nasty bug has a * habit of coming back). */ - assert(current_fast_get(runtime) == tstate); + assert(current_fast_get() == tstate); _PyThreadState_DeleteCurrent(tstate); } /* Release the lock if necessary */ @@ -2344,11 +2618,16 @@ _PyInterpreterState_SetEvalFrameFunc(PyInterpreterState *interp, _PyFrameEvalFunction eval_frame) { if (eval_frame == _PyEval_EvalFrameDefault) { - interp->eval_frame = NULL; + eval_frame = NULL; } - else { - interp->eval_frame = eval_frame; + if (eval_frame == interp->eval_frame) { + return; } + if (eval_frame != NULL) { + _Py_Executors_InvalidateAll(interp); + } + RARE_EVENT_INC(set_eval_frame_func); + interp->eval_frame = eval_frame; } @@ -2376,9 +2655,8 @@ _PyInterpreterState_GetConfigCopy(PyConfig *config) const PyConfig* _Py_GetConfig(void) { - _PyRuntimeState *runtime = &_PyRuntime; assert(PyGILState_Check()); - PyThreadState *tstate = current_fast_get(runtime); + PyThreadState *tstate = current_fast_get(); _Py_EnsureTstateNotNULL(tstate); return _PyInterpreterState_GetConfig(tstate->interp); } @@ -2509,3 +2787,56 @@ _PyThreadState_MustExit(PyThreadState *tstate) } return 1; } + +/********************/ +/* mimalloc support */ +/********************/ + +static void +tstate_mimalloc_bind(PyThreadState *tstate) +{ +#ifdef Py_GIL_DISABLED + struct _mimalloc_thread_state *mts = &((_PyThreadStateImpl*)tstate)->mimalloc; + + // Initialize the mimalloc thread state. This must be called from the + // same thread that will use the thread state. The "mem" heap doubles as + // the "backing" heap. + mi_tld_t *tld = &mts->tld; + _mi_tld_init(tld, &mts->heaps[_Py_MIMALLOC_HEAP_MEM]); + + // Exiting threads push any remaining in-use segments to the abandoned + // pool to be re-claimed later by other threads. We use per-interpreter + // pools to keep Python objects from different interpreters separate. + tld->segments.abandoned = &tstate->interp->mimalloc.abandoned_pool; + + // Initialize each heap + for (uint8_t i = 0; i < _Py_MIMALLOC_HEAP_COUNT; i++) { + _mi_heap_init_ex(&mts->heaps[i], tld, _mi_arena_id_none(), false, i); + } + + // By default, object allocations use _Py_MIMALLOC_HEAP_OBJECT. + // _PyObject_GC_New() and similar functions temporarily override this to + // use one of the GC heaps. + mts->current_object_heap = &mts->heaps[_Py_MIMALLOC_HEAP_OBJECT]; +#endif +} + +void +_PyThreadState_ClearMimallocHeaps(PyThreadState *tstate) +{ +#ifdef Py_GIL_DISABLED + if (!tstate->_status.bound) { + // The mimalloc heaps are only initialized when the thread is bound. + return; + } + + _PyThreadStateImpl *tstate_impl = (_PyThreadStateImpl *)tstate; + for (Py_ssize_t i = 0; i < _Py_MIMALLOC_HEAP_COUNT; i++) { + // Abandon all segments in use by this thread. This pushes them to + // a shared pool to later be reclaimed by other threads. It's important + // to do this before the thread state is destroyed so that objects + // remain visible to the GC. + _mi_heap_collect_abandon(&tstate_impl->mimalloc.heaps[i]); + } +#endif +} diff --git a/Python/specialize.c b/Python/specialize.c index 369b962a545f4e..e38e3556a6d642 100644 --- a/Python/specialize.c +++ b/Python/specialize.c @@ -267,6 +267,16 @@ print_optimization_stats(FILE *out, OptimizationStats *stats) } } +static void +print_rare_event_stats(FILE *out, RareEventStats *stats) +{ + fprintf(out, "Rare event (set_class): %" PRIu64 "\n", stats->set_class); + fprintf(out, "Rare event (set_bases): %" PRIu64 "\n", stats->set_bases); + fprintf(out, "Rare event (set_eval_frame_func): %" PRIu64 "\n", stats->set_eval_frame_func); + fprintf(out, "Rare event (builtin_dict): %" PRIu64 "\n", stats->builtin_dict); + fprintf(out, "Rare event (func_modification): %" PRIu64 "\n", stats->func_modification); +} + static void print_stats(FILE *out, PyStats *stats) { @@ -275,6 +285,7 @@ print_stats(FILE *out, PyStats *stats) print_object_stats(out, &stats->object_stats); print_gc_stats(out, stats->gc_stats); print_optimization_stats(out, &stats->optimization_stats); + print_rare_event_stats(out, &stats->rare_event_stats); } void @@ -529,6 +540,7 @@ _PyCode_Quicken(PyCodeObject *code) #define SPEC_FAIL_CALL_METHOD_WRAPPER 28 #define SPEC_FAIL_CALL_OPERATOR_WRAPPER 29 #define SPEC_FAIL_CALL_INIT_NOT_SIMPLE 30 +#define SPEC_FAIL_CALL_METACLASS 31 /* COMPARE_OP */ #define SPEC_FAIL_COMPARE_OP_DIFFERENT_TYPES 12 @@ -586,6 +598,7 @@ _PyCode_Quicken(PyCodeObject *code) static int function_kind(PyCodeObject *code); static bool function_check_args(PyObject *o, int expected_argcount, int opcode); static uint32_t function_get_version(PyObject *o, int opcode); +static uint32_t type_get_version(PyTypeObject *t, int opcode); static int specialize_module_load_attr( @@ -874,6 +887,9 @@ _Py_Specialize_LoadAttr(PyObject *owner, _Py_CODEUNIT *instr, PyObject *name) PyObject *descr = NULL; DescriptorClassification kind = analyze_descriptor(type, name, &descr, 0); assert(descr != NULL || kind == ABSENT || kind == GETSET_OVERRIDDEN); + if (type_get_version(type, LOAD_ATTR) == 0) { + goto fail; + } switch(kind) { case OVERRIDING: SPECIALIZATION_FAIL(LOAD_ATTR, SPEC_FAIL_ATTR_OVERRIDING_DESCRIPTOR); @@ -1057,6 +1073,9 @@ _Py_Specialize_StoreAttr(PyObject *owner, _Py_CODEUNIT *instr, PyObject *name) } PyObject *descr; DescriptorClassification kind = analyze_descriptor(type, name, &descr, 1); + if (type_get_version(type, STORE_ATTR) == 0) { + goto fail; + } switch(kind) { case OVERRIDING: SPECIALIZATION_FAIL(STORE_ATTR, SPEC_FAIL_ATTR_OVERRIDING_DESCRIPTOR); @@ -1183,6 +1202,9 @@ specialize_class_load_attr(PyObject *owner, _Py_CODEUNIT *instr, PyObject *descr = NULL; DescriptorClassification kind = 0; kind = analyze_descriptor((PyTypeObject *)owner, name, &descr, 0); + if (type_get_version((PyTypeObject *)owner, LOAD_ATTR) == 0) { + return -1; + } switch (kind) { case METHOD: case NON_DESCRIPTOR: @@ -1455,6 +1477,18 @@ function_get_version(PyObject *o, int opcode) return version; } +/* Returning 0 indicates a failure. */ +static uint32_t +type_get_version(PyTypeObject *t, int opcode) +{ + uint32_t version = t->tp_version_tag; + if (version == 0) { + SPECIALIZATION_FAIL(opcode, SPEC_FAIL_OUT_OF_VERSIONS); + return 0; + } + return version; +} + void _Py_Specialize_BinarySubscr( PyObject *container, PyObject *sub, _Py_CODEUNIT *instr) @@ -1724,8 +1758,15 @@ specialize_class_call(PyObject *callable, _Py_CODEUNIT *instr, int nargs) SPEC_FAIL_CALL_STR : SPEC_FAIL_CALL_CLASS_NO_VECTORCALL); return -1; } + if (Py_TYPE(tp) != &PyType_Type) { + SPECIALIZATION_FAIL(CALL, SPEC_FAIL_CALL_METACLASS); + return -1; + } if (tp->tp_new == PyBaseObject_Type.tp_new) { PyFunctionObject *init = get_init_for_simple_managed_python_class(tp); + if (type_get_version(tp, CALL) == 0) { + return -1; + } if (init != NULL) { if (((PyCodeObject *)init->func_code)->co_argcount != nargs+1) { SPECIALIZATION_FAIL(CALL, SPEC_FAIL_WRONG_NUMBER_ARGUMENTS); @@ -2466,7 +2507,10 @@ _Py_Specialize_ToBool(PyObject *value, _Py_CODEUNIT *instr) SPECIALIZATION_FAIL(TO_BOOL, SPEC_FAIL_OUT_OF_VERSIONS); goto failure; } - uint32_t version = Py_TYPE(value)->tp_version_tag; + uint32_t version = type_get_version(Py_TYPE(value), TO_BOOL); + if (version == 0) { + goto failure; + } instr->op.code = TO_BOOL_ALWAYS_TRUE; write_u32(cache->version, version); assert(version); @@ -2534,7 +2578,7 @@ const struct _PyCode_DEF(8) _Py_InitCleanup = { .co_consts = (PyObject *)&_Py_SINGLETON(tuple_empty), .co_names = (PyObject *)&_Py_SINGLETON(tuple_empty), .co_exceptiontable = (PyObject *)&_Py_SINGLETON(bytes_empty), - .co_flags = CO_OPTIMIZED, + .co_flags = CO_OPTIMIZED | CO_NO_MONITORING_EVENTS, .co_localsplusnames = (PyObject *)&_Py_SINGLETON(tuple_empty), .co_localspluskinds = (PyObject *)&_Py_SINGLETON(bytes_empty), .co_filename = &_Py_ID(__init__), diff --git a/Python/stdlib_module_names.h b/Python/stdlib_module_names.h index 701bfc35cc8182..2445a5c838a7d7 100644 --- a/Python/stdlib_module_names.h +++ b/Python/stdlib_module_names.h @@ -76,6 +76,7 @@ static const char* _Py_stdlib_module_names[] = { "_string", "_strptime", "_struct", +"_suggestions", "_symtable", "_sysconfig", "_thread", diff --git a/Python/structmember.c b/Python/structmember.c index 7a5a6a49d23116..c9f03a464078d0 100644 --- a/Python/structmember.c +++ b/Python/structmember.c @@ -197,45 +197,74 @@ PyMember_SetOne(char *addr, PyMemberDef *l, PyObject *v) WARN("Truncation of value to int"); break; } - case Py_T_UINT:{ - unsigned long ulong_val = PyLong_AsUnsignedLong(v); - if ((ulong_val == (unsigned long)-1) && PyErr_Occurred()) { - /* XXX: For compatibility, accept negative int values - as well. */ - PyErr_Clear(); - ulong_val = PyLong_AsLong(v); - if ((ulong_val == (unsigned long)-1) && - PyErr_Occurred()) + case Py_T_UINT: { + /* XXX: For compatibility, accept negative int values + as well. */ + int overflow; + long long_val = PyLong_AsLongAndOverflow(v, &overflow); + if (long_val == -1 && PyErr_Occurred()) { + return -1; + } + if (overflow < 0) { + PyErr_SetString(PyExc_OverflowError, + "Python int too large to convert to C long"); + return -1; + } + else if (!overflow) { + *(unsigned int *)addr = (unsigned int)(unsigned long)long_val; + if (long_val < 0) { + WARN("Writing negative value into unsigned field"); + } + else if ((unsigned long)long_val > UINT_MAX) { + WARN("Truncation of value to unsigned short"); + } + } + else { + unsigned long ulong_val = PyLong_AsUnsignedLong(v); + if (ulong_val == (unsigned long)-1 && PyErr_Occurred()) { return -1; - *(unsigned int *)addr = (unsigned int)ulong_val; - WARN("Writing negative value into unsigned field"); - } else - *(unsigned int *)addr = (unsigned int)ulong_val; - if (ulong_val > UINT_MAX) - WARN("Truncation of value to unsigned int"); - break; + } + *(unsigned int*)addr = (unsigned int)ulong_val; + if (ulong_val > UINT_MAX) { + WARN("Truncation of value to unsigned int"); + } } + break; + } case Py_T_LONG:{ *(long*)addr = PyLong_AsLong(v); if ((*(long*)addr == -1) && PyErr_Occurred()) return -1; break; } - case Py_T_ULONG:{ - *(unsigned long*)addr = PyLong_AsUnsignedLong(v); - if ((*(unsigned long*)addr == (unsigned long)-1) - && PyErr_Occurred()) { - /* XXX: For compatibility, accept negative int values - as well. */ - PyErr_Clear(); - *(unsigned long*)addr = PyLong_AsLong(v); - if ((*(unsigned long*)addr == (unsigned long)-1) - && PyErr_Occurred()) + case Py_T_ULONG: { + /* XXX: For compatibility, accept negative int values + as well. */ + int overflow; + long long_val = PyLong_AsLongAndOverflow(v, &overflow); + if (long_val == -1 && PyErr_Occurred()) { + return -1; + } + if (overflow < 0) { + PyErr_SetString(PyExc_OverflowError, + "Python int too large to convert to C long"); + return -1; + } + else if (!overflow) { + *(unsigned long *)addr = (unsigned long)long_val; + if (long_val < 0) { + WARN("Writing negative value into unsigned field"); + } + } + else { + unsigned long ulong_val = PyLong_AsUnsignedLong(v); + if (ulong_val == (unsigned long)-1 && PyErr_Occurred()) { return -1; - WARN("Writing negative value into unsigned field"); + } + *(unsigned long*)addr = ulong_val; } break; - } + } case Py_T_PYSSIZET:{ *(Py_ssize_t*)addr = PyLong_AsSsize_t(v); if ((*(Py_ssize_t*)addr == (Py_ssize_t)-1) diff --git a/Python/suggestions.c b/Python/suggestions.c index 1ad359b18923f3..a09b3ce6d9dab2 100644 --- a/Python/suggestions.c +++ b/Python/suggestions.c @@ -4,8 +4,6 @@ #include "pycore_pyerrors.h" // export _Py_UTF8_Edit_Cost() #include "pycore_runtime.h" // _Py_ID() -#include "stdlib_module_names.h" // _Py_stdlib_module_names - #define MAX_CANDIDATE_ITEMS 750 #define MAX_STRING_SIZE 40 @@ -178,225 +176,6 @@ _Py_CalculateSuggestions(PyObject *dir, return Py_XNewRef(suggestion); } -static PyObject * -get_suggestions_for_attribute_error(PyAttributeErrorObject *exc) -{ - PyObject *name = exc->name; // borrowed reference - PyObject *obj = exc->obj; // borrowed reference - - // Abort if we don't have an attribute name or we have an invalid one - if (name == NULL || obj == NULL || !PyUnicode_CheckExact(name)) { - return NULL; - } - - PyObject *dir = PyObject_Dir(obj); - if (dir == NULL) { - return NULL; - } - - PyObject *suggestions = _Py_CalculateSuggestions(dir, name); - Py_DECREF(dir); - return suggestions; -} - -static PyObject * -offer_suggestions_for_attribute_error(PyAttributeErrorObject *exc) -{ - PyObject* suggestion = get_suggestions_for_attribute_error(exc); - if (suggestion == NULL) { - return NULL; - } - // Add a trailer ". Did you mean: (...)?" - PyObject* result = PyUnicode_FromFormat(". Did you mean: %R?", suggestion); - Py_DECREF(suggestion); - return result; -} - -static PyObject * -get_suggestions_for_name_error(PyObject* name, PyFrameObject* frame) -{ - PyCodeObject *code = PyFrame_GetCode(frame); - assert(code != NULL && code->co_localsplusnames != NULL); - - PyObject *varnames = _PyCode_GetVarnames(code); - Py_DECREF(code); - if (varnames == NULL) { - return NULL; - } - PyObject *dir = PySequence_List(varnames); - Py_DECREF(varnames); - if (dir == NULL) { - return NULL; - } - - // Are we inside a method and the instance has an attribute called 'name'? - int res = PySequence_Contains(dir, &_Py_ID(self)); - if (res < 0) { - goto error; - } - if (res > 0) { - PyObject* locals = PyFrame_GetLocals(frame); - if (!locals) { - goto error; - } - PyObject* self = PyDict_GetItemWithError(locals, &_Py_ID(self)); /* borrowed */ - if (!self) { - Py_DECREF(locals); - goto error; - } - - res = PyObject_HasAttrWithError(self, name); - Py_DECREF(locals); - if (res < 0) { - goto error; - } - if (res) { - Py_DECREF(dir); - return PyUnicode_FromFormat("self.%U", name); - } - } - - PyObject *suggestions = _Py_CalculateSuggestions(dir, name); - Py_DECREF(dir); - if (suggestions != NULL || PyErr_Occurred()) { - return suggestions; - } - - dir = PySequence_List(frame->f_frame->f_globals); - if (dir == NULL) { - return NULL; - } - suggestions = _Py_CalculateSuggestions(dir, name); - Py_DECREF(dir); - if (suggestions != NULL || PyErr_Occurred()) { - return suggestions; - } - - dir = PySequence_List(frame->f_frame->f_builtins); - if (dir == NULL) { - return NULL; - } - suggestions = _Py_CalculateSuggestions(dir, name); - Py_DECREF(dir); - - return suggestions; - -error: - Py_DECREF(dir); - return NULL; -} - -static bool -is_name_stdlib_module(PyObject* name) -{ - const char* the_name = PyUnicode_AsUTF8(name); - Py_ssize_t len = Py_ARRAY_LENGTH(_Py_stdlib_module_names); - for (Py_ssize_t i = 0; i < len; i++) { - if (strcmp(the_name, _Py_stdlib_module_names[i]) == 0) { - return 1; - } - } - return 0; -} - -static PyObject * -offer_suggestions_for_name_error(PyNameErrorObject *exc) -{ - PyObject *name = exc->name; // borrowed reference - PyTracebackObject *traceback = (PyTracebackObject *) exc->traceback; // borrowed reference - // Abort if we don't have a variable name or we have an invalid one - // or if we don't have a traceback to work with - if (name == NULL || !PyUnicode_CheckExact(name) || - traceback == NULL || !Py_IS_TYPE(traceback, &PyTraceBack_Type) - ) { - return NULL; - } - - // Move to the traceback of the exception - while (1) { - PyTracebackObject *next = traceback->tb_next; - if (next == NULL || !Py_IS_TYPE(next, &PyTraceBack_Type)) { - break; - } - else { - traceback = next; - } - } - - PyFrameObject *frame = traceback->tb_frame; - assert(frame != NULL); - - PyObject* suggestion = get_suggestions_for_name_error(name, frame); - if (suggestion == NULL && PyErr_Occurred()) { - return NULL; - } - - // Add a trailer ". Did you mean: (...)?" - PyObject* result = NULL; - if (!is_name_stdlib_module(name)) { - if (suggestion == NULL) { - return NULL; - } - result = PyUnicode_FromFormat(". Did you mean: %R?", suggestion); - } else if (suggestion == NULL) { - result = PyUnicode_FromFormat(". Did you forget to import %R?", name); - } else { - result = PyUnicode_FromFormat(". Did you mean: %R? Or did you forget to import %R?", suggestion, name); - } - Py_XDECREF(suggestion); - return result; -} - -static PyObject * -offer_suggestions_for_import_error(PyImportErrorObject *exc) -{ - PyObject *mod_name = exc->name; // borrowed reference - PyObject *name = exc->name_from; // borrowed reference - if (name == NULL || mod_name == NULL || name == Py_None || - !PyUnicode_CheckExact(name) || !PyUnicode_CheckExact(mod_name)) { - return NULL; - } - - PyObject* mod = PyImport_GetModule(mod_name); - if (mod == NULL) { - return NULL; - } - - PyObject *dir = PyObject_Dir(mod); - Py_DECREF(mod); - if (dir == NULL) { - return NULL; - } - - PyObject *suggestion = _Py_CalculateSuggestions(dir, name); - Py_DECREF(dir); - if (!suggestion) { - return NULL; - } - - PyObject* result = PyUnicode_FromFormat(". Did you mean: %R?", suggestion); - Py_DECREF(suggestion); - return result; -} - -// Offer suggestions for a given exception. Returns a python string object containing the -// suggestions. This function returns NULL if no suggestion was found or if an exception happened, -// users must call PyErr_Occurred() to disambiguate. -PyObject * -_Py_Offer_Suggestions(PyObject *exception) -{ - PyObject *result = NULL; - assert(!PyErr_Occurred()); - if (Py_IS_TYPE(exception, (PyTypeObject*)PyExc_AttributeError)) { - result = offer_suggestions_for_attribute_error((PyAttributeErrorObject *) exception); - } else if (Py_IS_TYPE(exception, (PyTypeObject*)PyExc_NameError)) { - result = offer_suggestions_for_name_error((PyNameErrorObject *) exception); - } else if (Py_IS_TYPE(exception, (PyTypeObject*)PyExc_ImportError)) { - result = offer_suggestions_for_import_error((PyImportErrorObject *) exception); - } - return result; -} - Py_ssize_t _Py_UTF8_Edit_Cost(PyObject *a, PyObject *b, Py_ssize_t max_cost) { diff --git a/Python/symtable.c b/Python/symtable.c index 52d5932896b263..d69516351efba2 100644 --- a/Python/symtable.c +++ b/Python/symtable.c @@ -386,11 +386,6 @@ symtable_new(void) return NULL; } -/* Using a scaling factor means this should automatically adjust when - the recursion limit is adjusted for small or large C stack allocations. -*/ -#define COMPILER_STACK_FRAME_SCALE 2 - struct symtable * _PySymtable_Build(mod_ty mod, PyObject *filename, PyFutureFeatures *future) { @@ -417,9 +412,9 @@ _PySymtable_Build(mod_ty mod, PyObject *filename, PyFutureFeatures *future) } /* Be careful here to prevent overflow. */ int recursion_depth = Py_C_RECURSION_LIMIT - tstate->c_recursion_remaining; - starting_recursion_depth = recursion_depth * COMPILER_STACK_FRAME_SCALE; + starting_recursion_depth = recursion_depth; st->recursion_depth = starting_recursion_depth; - st->recursion_limit = Py_C_RECURSION_LIMIT * COMPILER_STACK_FRAME_SCALE; + st->recursion_limit = Py_C_RECURSION_LIMIT; /* Make the initial symbol information gathering pass */ if (!symtable_enter_block(st, &_Py_ID(top), ModuleBlock, (void *)mod, 0, 0, 0, 0)) { @@ -763,6 +758,8 @@ inline_comprehension(PySTEntryObject *ste, PySTEntryObject *comp, { PyObject *k, *v; Py_ssize_t pos = 0; + int remove_dunder_class = 0; + while (PyDict_Next(comp->ste_symbols, &pos, &k, &v)) { // skip comprehension parameter long comp_flags = PyLong_AS_LONG(v); @@ -784,6 +781,19 @@ inline_comprehension(PySTEntryObject *ste, PySTEntryObject *comp, if (!existing) { // name does not exist in scope, copy from comprehension assert(scope != FREE || PySet_Contains(comp_free, k) == 1); + if (scope == FREE && ste->ste_type == ClassBlock && + _PyUnicode_EqualToASCIIString(k, "__class__")) { + // if __class__ is unbound in the enclosing class scope and free + // in the comprehension scope, it needs special handling; just + // letting it be marked as free in class scope will break due to + // drop_class_free + scope = GLOBAL_IMPLICIT; + only_flags &= ~DEF_FREE; + if (PySet_Discard(comp_free, k) < 0) { + return 0; + } + remove_dunder_class = 1; + } PyObject *v_flags = PyLong_FromLong(only_flags); if (v_flags == NULL) { return 0; @@ -808,6 +818,10 @@ inline_comprehension(PySTEntryObject *ste, PySTEntryObject *comp, } } } + comp->ste_free = PySet_Size(comp_free) > 0; + if (remove_dunder_class && PyDict_DelItemString(comp->ste_symbols, "__class__") < 0) { + return 0; + } return 1; } @@ -977,6 +991,12 @@ update_symbols(PyObject *symbols, PyObject *scopes, } Py_DECREF(name); } + + /* Check if loop ended because of exception in PyIter_Next */ + if (PyErr_Occurred()) { + goto error; + } + Py_DECREF(itr); Py_DECREF(v_free); return 1; diff --git a/Python/sysmodule.c b/Python/sysmodule.c index 57dc4a1226ce75..437d7f8dfc4958 100644 --- a/Python/sysmodule.c +++ b/Python/sysmodule.c @@ -1420,7 +1420,7 @@ sys_set_asyncgen_hooks(PyObject *self, PyObject *args, PyObject *kw) } PyDoc_STRVAR(set_asyncgen_hooks_doc, -"set_asyncgen_hooks(* [, firstiter] [, finalizer])\n\ +"set_asyncgen_hooks([firstiter] [, finalizer])\n\ \n\ Set a finalizer for async generators objects." ); @@ -1715,6 +1715,13 @@ static PyObject * sys__enablelegacywindowsfsencoding_impl(PyObject *module) /*[clinic end generated code: output=f5c3855b45e24fe9 input=2bfa931a20704492]*/ { + if (PyErr_WarnEx(PyExc_DeprecationWarning, + "sys._enablelegacywindowsfsencoding() is deprecated and will be " + "removed in Python 3.16. Use PYTHONLEGACYWINDOWSFSENCODING " + "instead.", 1)) + { + return NULL; + } if (_PyUnicode_EnableLegacyWindowsFSEncoding() < 0) { return NULL; } @@ -1871,7 +1878,15 @@ _PySys_GetSizeOf(PyObject *o) return (size_t)-1; } - return (size_t)size + _PyType_PreHeaderSize(Py_TYPE(o)); + size_t presize = 0; + if (!Py_IS_TYPE(o, &PyType_Type) || + PyType_HasFeature((PyTypeObject *)o, Py_TPFLAGS_HEAPTYPE)) + { + /* Add the size of the pre-header if "o" is not a static type */ + presize = _PyType_PreHeaderSize(Py_TYPE(o)); + } + + return (size_t)size + presize; } static PyObject * diff --git a/Python/thread_nt.h b/Python/thread_nt.h index 14b9cddc24c0ec..ad467e0e7840e7 100644 --- a/Python/thread_nt.h +++ b/Python/thread_nt.h @@ -242,10 +242,6 @@ PyThread_detach_thread(PyThread_handle_t handle) { return (CloseHandle(hThread) == 0); } -void -PyThread_update_thread_after_fork(PyThread_ident_t* ident, PyThread_handle_t* handle) { -} - /* * Return the thread Id instead of a handle. The Id is said to uniquely identify the * thread in the system @@ -444,16 +440,7 @@ PyThread_set_key_value(int key, void *value) void * PyThread_get_key_value(int key) { - /* because TLS is used in the Py_END_ALLOW_THREAD macro, - * it is necessary to preserve the windows error state, because - * it is assumed to be preserved across the call to the macro. - * Ideally, the macro should be fixed, but it is simpler to - * do it here. - */ - DWORD error = GetLastError(); - void *result = TlsGetValue(key); - SetLastError(error); - return result; + return TlsGetValue(key); } void @@ -525,14 +512,5 @@ void * PyThread_tss_get(Py_tss_t *key) { assert(key != NULL); - /* because TSS is used in the Py_END_ALLOW_THREAD macro, - * it is necessary to preserve the windows error state, because - * it is assumed to be preserved across the call to the macro. - * Ideally, the macro should be fixed, but it is simpler to - * do it here. - */ - DWORD error = GetLastError(); - void *result = TlsGetValue(key->_key); - SetLastError(error); - return result; + return TlsGetValue(key->_key); } diff --git a/Python/thread_pthread.h b/Python/thread_pthread.h index fb3b79fc160502..556e3de0b071f8 100644 --- a/Python/thread_pthread.h +++ b/Python/thread_pthread.h @@ -339,16 +339,6 @@ PyThread_detach_thread(PyThread_handle_t th) { return pthread_detach((pthread_t) th); } -void -PyThread_update_thread_after_fork(PyThread_ident_t* ident, PyThread_handle_t* handle) { - // The thread id might have been updated in the forked child - pthread_t th = pthread_self(); - *ident = (PyThread_ident_t) th; - *handle = (PyThread_handle_t) th; - assert(th == (pthread_t) *ident); - assert(th == (pthread_t) *handle); -} - /* XXX This implementation is considered (to quote Tim Peters) "inherently hosed" because: - It does not guarantee the promise that a non-zero integer is returned. diff --git a/Python/traceback.c b/Python/traceback.c index abd429ac6c1f71..7a188e56c939c0 100644 --- a/Python/traceback.c +++ b/Python/traceback.c @@ -965,7 +965,11 @@ dump_traceback(int fd, PyThreadState *tstate, int write_header) unsigned int depth = 0; while (1) { if (MAX_FRAME_DEPTH <= depth) { - PUTS(fd, " ...\n"); + if (MAX_FRAME_DEPTH < depth) { + PUTS(fd, "plus "); + _Py_DumpDecimal(fd, depth); + PUTS(fd, " frames\n"); + } break; } dump_frame(fd, frame); diff --git a/README.rst b/README.rst index 4c8602c97ac8ef..fbfae16a7dbb0b 100644 --- a/README.rst +++ b/README.rst @@ -1,4 +1,4 @@ -This is Python version 3.13.0 alpha 2 +This is Python version 3.13.0 alpha 3 ===================================== .. image:: https://github.com/python/cpython/workflows/Tests/badge.svg @@ -14,7 +14,7 @@ This is Python version 3.13.0 alpha 2 :target: https://discuss.python.org/ -Copyright © 2001-2023 Python Software Foundation. All rights reserved. +Copyright © 2001-2024 Python Software Foundation. All rights reserved. See the end of this file for further copyright and license information. @@ -206,8 +206,8 @@ directories installed using ``make altinstall`` contain the major and minor version and can thus live side-by-side. ``make install`` also creates ``${prefix}/bin/python3`` which refers to ``${prefix}/bin/python3.X``. If you intend to install multiple versions using the same prefix you must decide which -version (if any) is your "primary" version. Install that version using ``make -install``. Install all other versions using ``make altinstall``. +version (if any) is your "primary" version. Install that version using +``make install``. Install all other versions using ``make altinstall``. For example, if you want to install Python 2.7, 3.6, and 3.13 with 3.13 being the primary version, you would execute ``make install`` in your 3.13 build directory @@ -224,7 +224,7 @@ Copyright and License Information --------------------------------- -Copyright © 2001-2023 Python Software Foundation. All rights reserved. +Copyright © 2001-2024 Python Software Foundation. All rights reserved. Copyright © 2000 BeOpen.com. All rights reserved. diff --git a/Tools/build/freeze_modules.py b/Tools/build/freeze_modules.py index 6a54f45bac3a86..a541b4b33c519b 100644 --- a/Tools/build/freeze_modules.py +++ b/Tools/build/freeze_modules.py @@ -658,7 +658,7 @@ def regen_pcbuild(modules): filterlines = [] corelines = [] deepfreezemappingsfile = f'$(IntDir)\\{DEEPFREEZE_MAPPING_FNAME}' - deepfreezerules = [f' '] + deepfreezerules = [f' '] deepfreezemappings = [] for src in _iter_sources(modules): pyfile = relpath_for_windows_display(src.pyfile, ROOT_DIR) @@ -667,15 +667,15 @@ def regen_pcbuild(modules): projlines.append(f' ') projlines.append(f' {src.frozenid}') projlines.append(f' $(IntDir){intfile}') - projlines.append(f' $(PySourcePath){header}') + projlines.append(f' $(GeneratedFrozenModulesDir){header}') projlines.append(f' ') filterlines.append(f' ') filterlines.append(' Python Files') filterlines.append(' ') - deepfreezemappings.append(f' \n') + deepfreezemappings.append(f' \n') - corelines.append(f' ') + corelines.append(f' ') print(f'# Updating {os.path.relpath(PCBUILD_PROJECT)}') with updating_file_with_tmpfile(PCBUILD_PROJECT) as (infile, outfile): diff --git a/Tools/build/generate_global_objects.py b/Tools/build/generate_global_objects.py index ded19ee489e79b..33d1b323fc1753 100644 --- a/Tools/build/generate_global_objects.py +++ b/Tools/build/generate_global_objects.py @@ -123,6 +123,14 @@ '__rdivmod__', '__buffer__', '__release_buffer__', + + #Workarounds for GH-108918 + 'alias', + 'args', + 'exc_type', + 'exc_value', + 'self', + 'traceback', ] NON_GENERATED_IMMORTAL_OBJECTS = [ diff --git a/Tools/build/generate_sbom.py b/Tools/build/generate_sbom.py index 93d0d8a3762df3..442487f2d2546b 100644 --- a/Tools/build/generate_sbom.py +++ b/Tools/build/generate_sbom.py @@ -8,6 +8,7 @@ import subprocess import sys import typing +import zipfile from urllib.request import urlopen CPYTHON_ROOT_DIR = pathlib.Path(__file__).parent.parent.parent @@ -16,10 +17,16 @@ # the license expression is a valid SPDX license expression: # See: https://spdx.org/licenses ALLOWED_LICENSE_EXPRESSIONS = { - "MIT", - "CC0-1.0", "Apache-2.0", + "Apache-2.0 OR BSD-2-Clause", "BSD-2-Clause", + "BSD-3-Clause", + "CC0-1.0", + "ISC", + "LGPL-2.1-only", + "MIT", + "MPL-2.0", + "Python-2.0.1", } # Properties which are required for our purposes. @@ -31,14 +38,13 @@ "checksums", "licenseConcluded", "externalRefs", - "originator", "primaryPackagePurpose", ]) class PackageFiles(typing.NamedTuple): """Structure for describing the files of a package""" - include: list[str] + include: list[str] | None exclude: list[str] | None = None @@ -82,6 +88,14 @@ def spdx_id(value: str) -> str: return re.sub(r"[^a-zA-Z0-9.\-]+", "-", value) +def error_if(value: bool, error_message: str) -> None: + """Prints an error if a comparison fails along with a link to the devguide""" + if value: + print(error_message) + print("See 'https://devguide.python.org/developer-workflow/sbom' for more information.") + sys.exit(1) + + def filter_gitignored_paths(paths: list[str]) -> list[str]: """ Filter out paths excluded by the gitignore file. @@ -98,6 +112,7 @@ def filter_gitignored_paths(paths: list[str]) -> list[str]: # Non-matching files show up as '::' git_check_ignore_proc = subprocess.run( ["git", "check-ignore", "--verbose", "--non-matching", *paths], + cwd=CPYTHON_ROOT_DIR, check=False, stdout=subprocess.PIPE, ) @@ -109,62 +124,209 @@ def filter_gitignored_paths(paths: list[str]) -> list[str]: return sorted([line.split()[-1] for line in git_check_ignore_lines if line.startswith("::")]) +def fetch_package_metadata_from_pypi(project: str, version: str, filename: str | None = None) -> tuple[str, str] | None: + """ + Fetches the SHA256 checksum and download location from PyPI. + If we're given a filename then we match with that, otherwise we use wheels. + """ + # Get pip's download location from PyPI. Check that the checksum is correct too. + try: + raw_text = urlopen(f"https://pypi.org/pypi/{project}/{version}/json").read() + release_metadata = json.loads(raw_text) + url: dict[str, typing.Any] + + # Look for a matching artifact filename and then check + # its remote checksum to the local one. + for url in release_metadata["urls"]: + # pip can only use Python-only dependencies, so there's + # no risk of picking the 'incorrect' wheel here. + if ( + (filename is None and url["packagetype"] == "bdist_wheel") + or (filename is not None and url["filename"] == filename) + ): + break + else: + raise ValueError(f"No matching filename on PyPI for '{filename}'") + + # Successfully found the download URL for the matching artifact. + download_url = url["url"] + checksum_sha256 = url["digests"]["sha256"] + return download_url, checksum_sha256 + + except (OSError, ValueError) as e: + # Fail if we're running in CI where we should have an internet connection. + error_if( + "CI" in os.environ, + f"Couldn't fetch metadata for project '{project}' from PyPI: {e}" + ) + return None + + +def find_ensurepip_pip_wheel() -> pathlib.Path | None: + """Try to find the pip wheel bundled in ensurepip. If missing return None""" + + ensurepip_bundled_dir = CPYTHON_ROOT_DIR / "Lib/ensurepip/_bundled" + + pip_wheels = [] + try: + for wheel_filename in os.listdir(ensurepip_bundled_dir): + if wheel_filename.startswith("pip-"): + pip_wheels.append(wheel_filename) + else: + print(f"Unexpected wheel in ensurepip: '{wheel_filename}'") + sys.exit(1) + + # Ignore this error, likely caused by downstream distributors + # deleting the 'ensurepip/_bundled' directory. + except FileNotFoundError: + pass + + if len(pip_wheels) == 0: + return None + elif len(pip_wheels) > 1: + print("Multiple pip wheels detected in 'Lib/ensurepip/_bundled'") + sys.exit(1) + # Otherwise return the one pip wheel. + return ensurepip_bundled_dir / pip_wheels[0] + + +def maybe_remove_pip_and_deps_from_sbom(sbom_data: dict[str, typing.Any]) -> None: + """ + Removes pip and its dependencies from the SBOM data + if the pip wheel is removed from ensurepip. This is done + by redistributors of Python and pip. + """ + + # If there's a wheel we don't remove anything. + if find_ensurepip_pip_wheel() is not None: + return + + # Otherwise we traverse the relationships + # to find dependent packages to remove. + sbom_pip_spdx_id = spdx_id("SPDXRef-PACKAGE-pip") + sbom_spdx_ids_to_remove = {sbom_pip_spdx_id} + + # Find all package SPDXIDs that pip depends on. + for sbom_relationship in sbom_data["relationships"]: + if ( + sbom_relationship["relationshipType"] == "DEPENDS_ON" + and sbom_relationship["spdxElementId"] == sbom_pip_spdx_id + ): + sbom_spdx_ids_to_remove.add(sbom_relationship["relatedSpdxElement"]) + + # Remove all the packages and relationships. + sbom_data["packages"] = [ + sbom_package for sbom_package in sbom_data["packages"] + if sbom_package["SPDXID"] not in sbom_spdx_ids_to_remove + ] + sbom_data["relationships"] = [ + sbom_relationship for sbom_relationship in sbom_data["relationships"] + if sbom_relationship["relatedSpdxElement"] not in sbom_spdx_ids_to_remove + ] + + def discover_pip_sbom_package(sbom_data: dict[str, typing.Any]) -> None: """pip is a part of a packaging ecosystem (Python, surprise!) so it's actually automatable to discover the metadata we need like the version and checksums - so let's do that on behalf of our friends at the PyPA. + so let's do that on behalf of our friends at the PyPA. This function also + discovers vendored packages within pip and fetches their metadata. """ global PACKAGE_TO_FILES - ensurepip_bundled_dir = CPYTHON_ROOT_DIR / "Lib/ensurepip/_bundled" - pip_wheels = [] - - # Find the hopefully one pip wheel in the bundled directory. - for wheel_filename in os.listdir(ensurepip_bundled_dir): - if wheel_filename.startswith("pip-"): - pip_wheels.append(wheel_filename) - if len(pip_wheels) != 1: - print("Zero or multiple pip wheels detected in 'Lib/ensurepip/_bundled'") - sys.exit(1) - pip_wheel_filename = pip_wheels[0] + pip_wheel_filepath = find_ensurepip_pip_wheel() + if pip_wheel_filepath is None: + return # There's no pip wheel, nothing to discover. # Add the wheel filename to the list of files so the SBOM file # and relationship generator can work its magic on the wheel too. PACKAGE_TO_FILES["pip"] = PackageFiles( - include=[f"Lib/ensurepip/_bundled/{pip_wheel_filename}"] + include=[str(pip_wheel_filepath.relative_to(CPYTHON_ROOT_DIR))] ) # Wheel filename format puts the version right after the project name. - pip_version = pip_wheel_filename.split("-")[1] + pip_version = pip_wheel_filepath.name.split("-")[1] pip_checksum_sha256 = hashlib.sha256( - (ensurepip_bundled_dir / pip_wheel_filename).read_bytes() + pip_wheel_filepath.read_bytes() ).hexdigest() - # Get pip's download location from PyPI. Check that the checksum is correct too. - try: - raw_text = urlopen(f"https://pypi.org/pypi/pip/{pip_version}/json").read() - pip_release_metadata = json.loads(raw_text) - url: dict[str, typing.Any] + pip_metadata = fetch_package_metadata_from_pypi( + project="pip", + version=pip_version, + filename=pip_wheel_filepath.name, + ) + # We couldn't fetch any metadata from PyPI, + # so we give up on verifying if we're not in CI. + if pip_metadata is None: + return + + pip_download_url, pip_actual_sha256 = pip_metadata + if pip_actual_sha256 != pip_checksum_sha256: + raise ValueError("Unexpected") + + # Parse 'pip/_vendor/vendor.txt' from the wheel for sub-dependencies. + with zipfile.ZipFile(pip_wheel_filepath) as whl: + vendor_txt_data = whl.read("pip/_vendor/vendor.txt").decode() + + # With this version regex we're assuming that pip isn't using pre-releases. + # If any version doesn't match we get a failure below, so we're safe doing this. + version_pin_re = re.compile(r"^([a-zA-Z0-9_.-]+)==([0-9.]*[0-9])$") + sbom_pip_dependency_spdx_ids = set() + for line in vendor_txt_data.splitlines(): + line = line.partition("#")[0].strip() # Strip comments and whitespace. + if not line: # Skip empty lines. + continue + + # Non-empty lines we must be able to match. + match = version_pin_re.match(line) + error_if(match is None, f"Couldn't parse line from pip vendor.txt: '{line}'") + assert match is not None # Make mypy happy. + + # Parse out and normalize the project name. + project_name, project_version = match.groups() + project_name = project_name.lower() + + # At this point if pip's metadata fetch succeeded we should + # expect this request to also succeed. + project_metadata = ( + fetch_package_metadata_from_pypi(project_name, project_version) + ) + assert project_metadata is not None + project_download_url, project_checksum_sha256 = project_metadata + + # Update our SBOM data with what we received from PyPI. + # Don't overwrite any existing values. + sbom_project_spdx_id = spdx_id(f"SPDXRef-PACKAGE-{project_name}") + sbom_pip_dependency_spdx_ids.add(sbom_project_spdx_id) + for package in sbom_data["packages"]: + if package["SPDXID"] != sbom_project_spdx_id: + continue - # Look for a matching artifact filename and then check - # its remote checksum to the local one. - for url in pip_release_metadata["urls"]: - if url["filename"] == pip_wheel_filename: + # Only thing missing from this blob is the `licenseConcluded`, + # that needs to be triaged by human maintainers if the list changes. + package.update({ + "SPDXID": sbom_project_spdx_id, + "name": project_name, + "versionInfo": project_version, + "downloadLocation": project_download_url, + "checksums": [ + {"algorithm": "SHA256", "checksumValue": project_checksum_sha256} + ], + "externalRefs": [ + { + "referenceCategory": "PACKAGE_MANAGER", + "referenceLocator": f"pkg:pypi/{project_name}@{project_version}", + "referenceType": "purl", + }, + ], + "primaryPackagePurpose": "SOURCE" + }) break - else: - raise ValueError(f"No matching filename on PyPI for '{pip_wheel_filename}'") - if url["digests"]["sha256"] != pip_checksum_sha256: - raise ValueError(f"Local pip checksum doesn't match artifact on PyPI") - # Successfully found the download URL for the matching artifact. - pip_download_url = url["url"] - - except (OSError, ValueError) as e: - print(f"Couldn't fetch pip's metadata from PyPI: {e}") - sys.exit(1) + PACKAGE_TO_FILES[project_name] = PackageFiles(include=None) # Remove pip from the existing SBOM packages if it's there # and then overwrite its entry with our own generated one. + sbom_pip_spdx_id = spdx_id("SPDXRef-PACKAGE-pip") sbom_data["packages"] = [ sbom_package for sbom_package in sbom_data["packages"] @@ -172,11 +334,11 @@ def discover_pip_sbom_package(sbom_data: dict[str, typing.Any]) -> None: ] sbom_data["packages"].append( { - "SPDXID": spdx_id("SPDXRef-PACKAGE-pip"), + "SPDXID": sbom_pip_spdx_id, "name": "pip", "versionInfo": pip_version, "originator": "Organization: Python Packaging Authority", - "licenseConcluded": "MIT", + "licenseConcluded": "NOASSERTION", "downloadLocation": pip_download_url, "checksums": [ {"algorithm": "SHA256", "checksumValue": pip_checksum_sha256} @@ -196,47 +358,88 @@ def discover_pip_sbom_package(sbom_data: dict[str, typing.Any]) -> None: "primaryPackagePurpose": "SOURCE", } ) + for sbom_dep_spdx_id in sorted(sbom_pip_dependency_spdx_ids): + sbom_data["relationships"].append({ + "spdxElementId": sbom_pip_spdx_id, + "relatedSpdxElement": sbom_dep_spdx_id, + "relationshipType": "DEPENDS_ON" + }) def main() -> None: sbom_path = CPYTHON_ROOT_DIR / "Misc/sbom.spdx.json" sbom_data = json.loads(sbom_path.read_bytes()) + # Check if pip should be removed if the wheel is missing. + # We can't reset the SBOM relationship data until checking this. + maybe_remove_pip_and_deps_from_sbom(sbom_data) + + # We regenerate all of this information. Package information + # should be preserved though since that is edited by humans. + sbom_data["files"] = [] + sbom_data["relationships"] = [] + # Insert pip's SBOM metadata from the wheel. discover_pip_sbom_package(sbom_data) # Ensure all packages in this tool are represented also in the SBOM file. - assert {package["name"] for package in sbom_data["packages"]} == set(PACKAGE_TO_FILES) + actual_names = {package["name"] for package in sbom_data["packages"]} + expected_names = set(PACKAGE_TO_FILES) + error_if( + actual_names != expected_names, + f"Packages defined in SBOM tool don't match those defined in SBOM file: {actual_names}, {expected_names}", + ) # Make a bunch of assertions about the SBOM data to ensure it's consistent. for package in sbom_data["packages"]: - # Properties and ID must be properly formed. - assert set(package.keys()) == REQUIRED_PROPERTIES_PACKAGE - assert package["SPDXID"] == spdx_id(f"SPDXRef-PACKAGE-{package['name']}") + error_if( + "name" not in package, + "Package is missing the 'name' field" + ) + missing_required_keys = REQUIRED_PROPERTIES_PACKAGE - set(package.keys()) + error_if( + bool(missing_required_keys), + f"Package '{package['name']}' is missing required fields: {missing_required_keys}", + ) + error_if( + package["SPDXID"] != spdx_id(f"SPDXRef-PACKAGE-{package['name']}"), + f"Package '{package['name']}' has a malformed SPDXID", + ) # Version must be in the download and external references. version = package["versionInfo"] - assert version in package["downloadLocation"] - assert all(version in ref["referenceLocator"] for ref in package["externalRefs"]) + error_if( + version not in package["downloadLocation"], + f"Version '{version}' for package '{package['name']} not in 'downloadLocation' field", + ) + error_if( + any(version not in ref["referenceLocator"] for ref in package["externalRefs"]), + ( + f"Version '{version}' for package '{package['name']} not in " + f"all 'externalRefs[].referenceLocator' fields" + ), + ) # License must be on the approved list for SPDX. - assert package["licenseConcluded"] in ALLOWED_LICENSE_EXPRESSIONS, package["licenseConcluded"] - - # Regenerate file information from current data. - sbom_files = [] - sbom_relationships = [] + license_concluded = package["licenseConcluded"] + error_if( + license_concluded != "NOASSERTION", + f"License identifier must be 'NOASSERTION'" + ) # We call 'sorted()' here a lot to avoid filesystem scan order issues. for name, files in sorted(PACKAGE_TO_FILES.items()): package_spdx_id = spdx_id(f"SPDXRef-PACKAGE-{name}") exclude = files.exclude or () - for include in sorted(files.include): - + for include in sorted(files.include or ()): # Find all the paths and then filter them through .gitignore. paths = glob.glob(include, root_dir=CPYTHON_ROOT_DIR, recursive=True) paths = filter_gitignored_paths(paths) - assert paths, include # Make sure that every value returns something! + error_if( + len(paths) == 0, + f"No valid paths found at path '{include}' for package '{name}", + ) for path in paths: # Skip directories and excluded files @@ -249,7 +452,7 @@ def main() -> None: checksum_sha256 = hashlib.sha256(data).hexdigest() file_spdx_id = spdx_id(f"SPDXRef-FILE-{path}") - sbom_files.append({ + sbom_data["files"].append({ "SPDXID": file_spdx_id, "fileName": path, "checksums": [ @@ -259,15 +462,13 @@ def main() -> None: }) # Tie each file back to its respective package. - sbom_relationships.append({ + sbom_data["relationships"].append({ "spdxElementId": package_spdx_id, "relatedSpdxElement": file_spdx_id, "relationshipType": "CONTAINS", }) # Update the SBOM on disk - sbom_data["files"] = sbom_files - sbom_data["relationships"] = sbom_relationships sbom_path.write_text(json.dumps(sbom_data, indent=2, sort_keys=True)) diff --git a/Tools/build/stable_abi.py b/Tools/build/stable_abi.py index 85c437d521a15a..83146622c74f94 100644 --- a/Tools/build/stable_abi.py +++ b/Tools/build/stable_abi.py @@ -278,6 +278,13 @@ def gen_ctypes_test(manifest, args, outfile): from _testcapi import get_feature_macros feature_macros = get_feature_macros() + + # Stable ABI is incompatible with Py_TRACE_REFS builds due to PyObject + # layout differences. + # See https://github.com/python/cpython/issues/88299#issuecomment-1113366226 + if feature_macros['Py_TRACE_REFS']: + raise unittest.SkipTest("incompatible with Py_TRACE_REFS.") + ctypes_test = import_module('ctypes') class TestStableABIAvailability(unittest.TestCase): @@ -308,16 +315,11 @@ def test_windows_feature_macros(self): {'function', 'data'}, include_abi_only=True, ) - optional_items = {} + feature_macros = list(manifest.select({'feature_macro'})) + optional_items = {m.name: [] for m in feature_macros} for item in items: - if item.name in ( - # Some symbols aren't exported on all platforms. - # This is a bug: https://bugs.python.org/issue44133 - 'PyModule_Create2', 'PyModule_FromDefAndSpec2', - ): - continue if item.ifdef: - optional_items.setdefault(item.ifdef, []).append(item.name) + optional_items[item.ifdef].append(item.name) else: write(f' "{item.name}",') write(")") @@ -328,7 +330,6 @@ def test_windows_feature_macros(self): write(f" {name!r},") write(" )") write("") - feature_macros = list(manifest.select({'feature_macro'})) feature_names = sorted(m.name for m in feature_macros) write(f"EXPECTED_FEATURE_MACROS = set({pprint.pformat(feature_names)})") diff --git a/Tools/build/verify_ensurepip_wheels.py b/Tools/build/verify_ensurepip_wheels.py index 29897425da6c03..a37da2f70757e5 100755 --- a/Tools/build/verify_ensurepip_wheels.py +++ b/Tools/build/verify_ensurepip_wheels.py @@ -14,7 +14,6 @@ from pathlib import Path from urllib.request import urlopen -PACKAGE_NAMES = ("pip",) ENSURE_PIP_ROOT = Path(__file__).parent.parent.parent / "Lib/ensurepip" WHEEL_DIR = ENSURE_PIP_ROOT / "_bundled" ENSURE_PIP_INIT_PY_TEXT = (ENSURE_PIP_ROOT / "__init__.py").read_text(encoding="utf-8") @@ -97,8 +96,5 @@ def verify_wheel(package_name: str) -> bool: if __name__ == "__main__": - exit_status = 0 - for package_name in PACKAGE_NAMES: - if not verify_wheel(package_name): - exit_status = 1 + exit_status = int(not verify_wheel("pip")) raise SystemExit(exit_status) diff --git a/Tools/c-analyzer/cpython/_parser.py b/Tools/c-analyzer/cpython/_parser.py index 239ed4e0266a75..444063d2148934 100644 --- a/Tools/c-analyzer/cpython/_parser.py +++ b/Tools/c-analyzer/cpython/_parser.py @@ -70,7 +70,6 @@ def clean_lines(text): Python/thread_pthread_stubs.h # only huge constants (safe but parsing is slow) -Modules/_ssl_data.h Modules/_ssl_data_31.h Modules/_ssl_data_300.h Modules/_ssl_data_111.h @@ -333,7 +332,7 @@ def clean_lines(text): _abs('Python/stdlib_module_names.h'): (5_000, 500), # These large files are currently ignored (see above). - _abs('Modules/_ssl_data.h'): (80_000, 10_000), + _abs('Modules/_ssl_data_31.h'): (80_000, 10_000), _abs('Modules/_ssl_data_300.h'): (80_000, 10_000), _abs('Modules/_ssl_data_111.h'): (80_000, 10_000), _abs('Modules/cjkcodecs/mappings_*.h'): (160_000, 2_000), diff --git a/Tools/c-analyzer/cpython/globals-to-fix.tsv b/Tools/c-analyzer/cpython/globals-to-fix.tsv index e3a1b5d532bda2..1d9576d083d8dc 100644 --- a/Tools/c-analyzer/cpython/globals-to-fix.tsv +++ b/Tools/c-analyzer/cpython/globals-to-fix.tsv @@ -197,6 +197,7 @@ Objects/exceptions.c - _PyExc_AttributeError - Objects/exceptions.c - _PyExc_SyntaxError - Objects/exceptions.c - _PyExc_IndentationError - Objects/exceptions.c - _PyExc_TabError - +Objects/exceptions.c - _PyExc_IncompleteInputError - Objects/exceptions.c - _PyExc_LookupError - Objects/exceptions.c - _PyExc_IndexError - Objects/exceptions.c - _PyExc_KeyError - @@ -261,6 +262,7 @@ Objects/exceptions.c - PyExc_AttributeError - Objects/exceptions.c - PyExc_SyntaxError - Objects/exceptions.c - PyExc_IndentationError - Objects/exceptions.c - PyExc_TabError - +Objects/exceptions.c - PyExc_IncompleteInputError - Objects/exceptions.c - PyExc_LookupError - Objects/exceptions.c - PyExc_IndexError - Objects/exceptions.c - PyExc_KeyError - @@ -332,6 +334,7 @@ Modules/_testclinic.c - DeprStarNew - Modules/_testclinic.c - DeprKwdInit - Modules/_testclinic.c - DeprKwdInitNoInline - Modules/_testclinic.c - DeprKwdNew - +Modules/_testclinic.c - TestClass - ################################## diff --git a/Tools/c-analyzer/cpython/ignored.tsv b/Tools/c-analyzer/cpython/ignored.tsv index 2f9e80d6ab6737..c75aff8c1723c1 100644 --- a/Tools/c-analyzer/cpython/ignored.tsv +++ b/Tools/c-analyzer/cpython/ignored.tsv @@ -325,7 +325,8 @@ Objects/obmalloc.c - _PyMem_Debug - Objects/obmalloc.c - _PyMem_Raw - Objects/obmalloc.c - _PyObject - Objects/obmalloc.c - last_final_leaks - -Objects/obmalloc.c - usedpools - +Objects/obmalloc.c - obmalloc_state_main - +Objects/obmalloc.c - obmalloc_state_initialized - Objects/typeobject.c - name_op - Objects/typeobject.c - slotdefs - Objects/unicodeobject.c - stripfuncnames - diff --git a/Tools/cases_generator/README.md b/Tools/cases_generator/README.md index ed802e44f31ad5..7fec8a882336cd 100644 --- a/Tools/cases_generator/README.md +++ b/Tools/cases_generator/README.md @@ -5,16 +5,30 @@ Documentation for the instruction definitions in `Python/bytecodes.c` What's currently here: +- `analyzer.py`: code for converting `AST` generated by `Parser` + to more high-level structure for easier interaction - `lexer.py`: lexer for C, originally written by Mark Shannon - `plexer.py`: OO interface on top of lexer.py; main class: `PLexer` -- `parsing.py`: Parser for instruction definition DSL; main class `Parser` -- `generate_cases.py`: driver script to read `Python/bytecodes.c` and +- `parsing.py`: Parser for instruction definition DSL; main class: `Parser` +- `parser.py` helper for interactions with `parsing.py` +- `tierN_generator.py`: a couple of driver scripts to read `Python/bytecodes.c` and write `Python/generated_cases.c.h` (and several other files) -- `analysis.py`: `Analyzer` class used to read the input files -- `flags.py`: abstractions related to metadata flags for instructions -- `formatting.py`: `Formatter` class used to write the output files -- `instructions.py`: classes to analyze and write instructions -- `stacking.py`: code to handle generalized stack effects +- `stack.py`: code to handle generalized stack effects +- `cwriter.py`: code which understands tokens and how to format C code; + main class: `CWriter` +- `generators_common.py`: helpers for generators +- `opcode_id_generator.py`: generate a list of opcodes and write them to + `Include/opcode_ids.h` +- `opcode_metadata_generator.py`: reads the instruction definitions and + write the metadata to `Include/internal/pycore_opcode_metadata.h` +- `py_metadata_generator.py`: reads the instruction definitions and + write the metadata to `Lib/_opcode_metadata.py` +- `target_generator.py`: generate targets for computed goto dispatch and + write them to `Python/opcode_targets.h` +- `uop_id_generator.py`: generate a list of uop IDs and write them to + `Include/internal/pycore_uop_ids.h` +- `uop_metadata_generator.py`: reads the instruction definitions and + write the metadata to `Include/internal/pycore_uop_metadata.h` Note that there is some dummy C code at the top and bottom of `Python/bytecodes.c` diff --git a/Tools/cases_generator/analyzer.py b/Tools/cases_generator/analyzer.py index 82ef8888bfcee5..b80fa66e2a159a 100644 --- a/Tools/cases_generator/analyzer.py +++ b/Tools/cases_generator/analyzer.py @@ -1,4 +1,4 @@ -from dataclasses import dataclass +from dataclasses import dataclass, field import lexer import parser from typing import Optional @@ -22,6 +22,10 @@ class Properties: uses_locals: bool has_free: bool + pure: bool + passthrough: bool + guard: bool + def dump(self, indent: str) -> None: print(indent, end="") text = ", ".join([f"{key}: {value}" for (key, value) in self.__dict__.items()]) @@ -45,6 +49,9 @@ def from_list(properties: list["Properties"]) -> "Properties": uses_co_names=any(p.uses_co_names for p in properties), uses_locals=any(p.uses_locals for p in properties), has_free=any(p.has_free for p in properties), + pure=all(p.pure for p in properties), + passthrough=all(p.passthrough for p in properties), + guard=all(p.guard for p in properties), ) @@ -64,6 +71,9 @@ def from_list(properties: list["Properties"]) -> "Properties": uses_co_names=False, uses_locals=False, has_free=False, + pure=False, + passthrough=False, + guard=False, ) @@ -88,6 +98,9 @@ class StackItem: condition: str | None size: str peek: bool = False + type_prop: None | tuple[str, None | str] = field( + default_factory=lambda: None, init=True, compare=False, hash=False + ) def __str__(self) -> str: cond = f" if ({self.condition})" if self.condition else "" @@ -259,7 +272,9 @@ def override_error( def convert_stack_item(item: parser.StackEffect) -> StackItem: - return StackItem(item.name, item.type, item.cond, (item.size or "1")) + return StackItem( + item.name, item.type, item.cond, (item.size or "1"), type_prop=item.type_prop + ) def analyze_stack(op: parser.InstDef) -> StackEffect: @@ -377,7 +392,6 @@ def makes_escaping_api_call(instr: parser.InstDef) -> bool: return False - EXITS = { "DISPATCH", "GO_TO_INSTRUCTION", @@ -417,29 +431,49 @@ def always_exits(op: parser.InstDef) -> bool: return False +def stack_effect_only_peeks(instr: parser.InstDef) -> bool: + stack_inputs = [s for s in instr.inputs if not isinstance(s, parser.CacheEffect)] + if len(stack_inputs) != len(instr.outputs): + return False + if len(stack_inputs) == 0: + return False + if any(s.cond for s in stack_inputs) or any(s.cond for s in instr.outputs): + return False + return all( + (s.name == other.name and s.type == other.type and s.size == other.size) + for s, other in zip(stack_inputs, instr.outputs) + ) + + def compute_properties(op: parser.InstDef) -> Properties: has_free = ( variable_used(op, "PyCell_New") or variable_used(op, "PyCell_GET") or variable_used(op, "PyCell_SET") ) + infallible = is_infallible(op) + deopts = variable_used(op, "DEOPT_IF") + passthrough = stack_effect_only_peeks(op) and infallible return Properties( escapes=makes_escaping_api_call(op), - infallible=is_infallible(op), - deopts=variable_used(op, "DEOPT_IF"), + infallible=infallible, + deopts=deopts, oparg=variable_used(op, "oparg"), jumps=variable_used(op, "JUMPBY"), eval_breaker=variable_used(op, "CHECK_EVAL_BREAKER"), ends_with_eval_breaker=eval_breaker_at_end(op), needs_this=variable_used(op, "this_instr"), always_exits=always_exits(op), - stores_sp=variable_used(op, "STORE_SP"), + stores_sp=variable_used(op, "SYNC_SP"), tier_one_only=variable_used(op, "TIER_ONE_ONLY"), uses_co_consts=variable_used(op, "FRAME_CO_CONSTS"), uses_co_names=variable_used(op, "FRAME_CO_NAMES"), uses_locals=(variable_used(op, "GETLOCAL") or variable_used(op, "SETLOCAL")) and not has_free, has_free=has_free, + pure="pure" in op.annotations, + passthrough=passthrough, + guard=passthrough and deopts, ) @@ -686,9 +720,7 @@ def analyze_forest(forest: list[parser.AstNode]) -> Analysis: inst = instructions["BINARY_OP_INPLACE_ADD_UNICODE"] inst.family = families["BINARY_OP"] families["BINARY_OP"].members.append(inst) - opmap, first_arg, min_instrumented = assign_opcodes( - instructions, families, pseudos - ) + opmap, first_arg, min_instrumented = assign_opcodes(instructions, families, pseudos) return Analysis( instructions, uops, families, pseudos, opmap, first_arg, min_instrumented ) diff --git a/Tools/cases_generator/generators_common.py b/Tools/cases_generator/generators_common.py index 5a42a05c5c2ef2..2fc2ab115321cf 100644 --- a/Tools/cases_generator/generators_common.py +++ b/Tools/cases_generator/generators_common.py @@ -26,7 +26,9 @@ def root_relative_path(filename: str) -> str: return filename -def write_header(generator: str, sources: list[str], outfile: TextIO, comment: str = "//") -> None: +def write_header( + generator: str, sources: list[str], outfile: TextIO, comment: str = "//" +) -> None: outfile.write( f"""{comment} This file is generated by {root_relative_path(generator)} {comment} from: @@ -122,7 +124,7 @@ def replace_decrefs( out.emit(f"Py_DECREF({var.name});\n") -def replace_store_sp( +def replace_sync_sp( out: CWriter, tkn: Token, tkn_iter: Iterator[Token], @@ -133,9 +135,7 @@ def replace_store_sp( next(tkn_iter) next(tkn_iter) next(tkn_iter) - out.emit_at("", tkn) stack.flush(out) - out.emit("_PyFrame_SetStackPointer(frame, stack_pointer);\n") def replace_check_eval_breaker( @@ -158,7 +158,7 @@ def replace_check_eval_breaker( "ERROR_IF": replace_error, "DECREF_INPUTS": replace_decrefs, "CHECK_EVAL_BREAKER": replace_check_eval_breaker, - "STORE_SP": replace_store_sp, + "SYNC_SP": replace_sync_sp, } ReplacementFunctionType = Callable[ @@ -209,6 +209,10 @@ def cflags(p: Properties) -> str: flags.append("HAS_ERROR_FLAG") if p.escapes: flags.append("HAS_ESCAPES_FLAG") + if p.pure: + flags.append("HAS_PURE_FLAG") + if p.passthrough: + flags.append("HAS_PASSTHROUGH_FLAG") if flags: return " | ".join(flags) else: diff --git a/Tools/cases_generator/interpreter_definition.md b/Tools/cases_generator/interpreter_definition.md index 5c4238756748a7..e87aff43762b11 100644 --- a/Tools/cases_generator/interpreter_definition.md +++ b/Tools/cases_generator/interpreter_definition.md @@ -6,7 +6,7 @@ The CPython interpreter is defined in C, meaning that the semantics of the bytecode instructions, the dispatching mechanism, error handling, and tracing and instrumentation are all intermixed. -This document proposes defining a custom C-like DSL for defining the +This document proposes defining a custom C-like DSL for defining the instruction semantics and tools for generating the code deriving from the instruction definitions. @@ -15,6 +15,7 @@ These tools would be used to: * Generate the tier 2 interpreter * Generate documentation for instructions * Generate metadata about instructions, such as stack use (done). +* Generate the tier 2 optimizer's abstract interpreter. Having a single definition file ensures that there is a single source of truth for bytecode semantics. @@ -45,7 +46,7 @@ passes from the semantic definition, reducing errors. As we improve the performance of CPython, we need to optimize larger regions of code, use more complex optimizations and, ultimately, translate to machine -code. +code. All of these steps introduce the possibility of more bugs, and require more code to be written. One way to mitigate this is through the use of code generators. @@ -61,7 +62,7 @@ blocks as the instructions for the tier 1 (PEP 659) interpreter. Rewriting all the instructions is tedious and error-prone, and changing the instructions is a maintenance headache as both versions need to be kept in sync. -By using a code generator and using a common source for the instructions, or +By using a code generator and using a common source for the instructions, or parts of instructions, we can reduce the potential for errors considerably. @@ -74,7 +75,7 @@ We update it as the need arises. Each op definition has a kind, a name, a stack and instruction stream effect, and a piece of C code describing its semantics:: - + ``` file: (definition | family | pseudo)+ @@ -85,7 +86,7 @@ and a piece of C code describing its semantics:: "op" "(" NAME "," stack_effect ")" "{" C-code "}" | "macro" "(" NAME ")" "=" uop ("+" uop)* ";" - + stack_effect: "(" [inputs] "--" [outputs] ")" @@ -108,7 +109,10 @@ and a piece of C code describing its semantics:: NAME [":" type] [ "if" "(" C-expression ")" ] type: - NAME ["*"] + NAME ["*"] | type_prop + + type_prop: + "&" "(" NAME ["+" NAME] ")" stream: NAME "/" size @@ -138,7 +142,27 @@ The following definitions may occur: The optional `type` in an `object` is the C type. It defaults to `PyObject *`. The objects before the "--" are the objects on top of the stack at the start of the instruction. Those after the "--" are the objects on top of the stack at the -end of the instruction. +end of the instruction. When prefixed by a `&`, the `type` production rule follows the +`type_prop` production rule. This indicates the type of the value is of that specific type +after the operation. In this case, the type may also contain 64-bit refinement information +that is fetched from a previously defined operand in the instruction header, such as +a type version tag. This follows the format `type + refinement`. The list of possible types +and their refinements are below. They obey the following predicates: + + +* `PYLONG_TYPE`: `Py_TYPE(val) == &PyLong_Type` +* `PYFLOAT_TYPE`: `Py_TYPE(val) == &PyFloat_Type` +* `PYUNICODE_TYPE`: `Py_TYPE(val) == &PYUNICODE_TYPE` +* `NULL_TYPE`: `val == NULL` +* `GUARD_TYPE_VERSION_TYPE`: `type->tp_version_tag == auxillary` +* `GUARD_DORV_VALUES_TYPE`: `_PyDictOrValues_IsValues(obj)` +* `GUARD_DORV_VALUES_INST_ATTR_FROM_DICT_TYPE`: + `_PyDictOrValues_IsValues(obj) || _PyObject_MakeInstanceAttributesFromDict(obj, dorv)` +* `GUARD_KEYS_VERSION_TYPE`: `owner_heap_type->ht_cached_keys->dk_version == auxillary` +* `PYMETHOD_TYPE`: `Py_TYPE(val) == &PyMethod_Type` +* `PYFUNCTION_TYPE_VERSION_TYPE`: + `PyFunction_Check(callable) && func->func_version == auxillary && code->co_argcount == oparg + (self_or_null != NULL)` + An `inst` without `stack_effect` is a transitional form to allow the original C code definitions to be copied. It lacks information to generate anything other than the @@ -158,6 +182,15 @@ By convention cache effects (`stream`) must precede the input effects. The name `oparg` is pre-defined as a 32 bit value fetched from the instruction stream. +### Special instruction annotations + +Instruction headers may be prefixed by one or more annotations. The non-exhaustive +list of annotations and their meanings are as follows: + +* `override`. For external use by other interpreter definitions to override the current + instruction definition. +* `pure`. This instruction has no side effects. + ### Special functions/macros The C code may include special functions that are understood by the tools as @@ -168,6 +201,7 @@ Those functions include: * `DEOPT_IF(cond, instruction)`. Deoptimize if `cond` is met. * `ERROR_IF(cond, label)`. Jump to error handler at `label` if `cond` is true. * `DECREF_INPUTS()`. Generate `Py_DECREF()` calls for the input stack effects. +* `SYNC_SP()`. Synchronizes the physical stack pointer with the stack effects. Note that the use of `DECREF_INPUTS()` is optional -- manual calls to `Py_DECREF()` or other approaches are also acceptable @@ -412,7 +446,7 @@ rather than popping and pushing, such that `LOAD_ATTR_SLOT` would look something stack_pointer += 1; } s1 = res; - } + } next_instr += (1 + 1 + 2 + 1 + 4); stack_pointer[-1] = s1; DISPATCH(); diff --git a/Tools/cases_generator/lexer.py b/Tools/cases_generator/lexer.py index c3c2954a42083f..4f8d01c5492f51 100644 --- a/Tools/cases_generator/lexer.py +++ b/Tools/cases_generator/lexer.py @@ -216,7 +216,13 @@ def choice(*opts: str) -> str: keywords = {name.lower(): name for name in kwds} ANNOTATION = "ANNOTATION" -annotations = {"specializing", "guard", "override", "register", "replaced"} +annotations = { + "specializing", + "override", + "register", + "replaced", + "pure", +} __all__ = [] __all__.extend(kwds) @@ -324,7 +330,9 @@ def tokenize(src: str, line: int = 1, filename: str = "") -> Iterator[Token]: else: begin = line, start - linestart if kind != "\n": - yield Token(filename, kind, text, begin, (line, start - linestart + len(text))) + yield Token( + filename, kind, text, begin, (line, start - linestart + len(text)) + ) def to_text(tkns: list[Token], dedent: int = 0) -> str: diff --git a/Tools/cases_generator/opcode_id_generator.py b/Tools/cases_generator/opcode_id_generator.py index dbea3d0b622c87..5a3009a5c04c27 100644 --- a/Tools/cases_generator/opcode_id_generator.py +++ b/Tools/cases_generator/opcode_id_generator.py @@ -1,6 +1,6 @@ """Generate the list of opcode IDs. Reads the instruction definitions from bytecodes.c. -Writes the IDs to opcode._ids.h by default. +Writes the IDs to opcode_ids.h by default. """ import argparse diff --git a/Tools/cases_generator/opcode_metadata_generator.py b/Tools/cases_generator/opcode_metadata_generator.py index 9b7df9a54c7b3b..3e9fa3e26daa53 100644 --- a/Tools/cases_generator/opcode_metadata_generator.py +++ b/Tools/cases_generator/opcode_metadata_generator.py @@ -1,6 +1,6 @@ -"""Generate uop metedata. +"""Generate opcode metadata. Reads the instruction definitions from bytecodes.c. -Writes the metadata to pycore_uop_metadata.h by default. +Writes the metadata to pycore_opcode_metadata.h by default. """ import argparse @@ -50,6 +50,8 @@ "DEOPT", "ERROR", "ESCAPES", + "PURE", + "PASSTHROUGH", ] diff --git a/Tools/cases_generator/parsing.py b/Tools/cases_generator/parsing.py index 60c185dcef58e9..307919cb37ce1e 100644 --- a/Tools/cases_generator/parsing.py +++ b/Tools/cases_generator/parsing.py @@ -75,6 +75,11 @@ class StackEffect(Node): size: str = "" # Optional `[size]` # Note: size cannot be combined with type or cond + # Optional `(type, refinement)` + type_prop: None | tuple[str, None | str] = field( + default_factory=lambda: None, init=True, compare=False, hash=False + ) + def __repr__(self) -> str: items = [self.name, self.type, self.cond, self.size] while items and items[-1] == "": @@ -138,11 +143,13 @@ class Family(Node): @dataclass class Pseudo(Node): name: str - flags: list[str] # instr flags to set on the pseudo instruction - targets: list[str] # opcodes this can be replaced by + flags: list[str] # instr flags to set on the pseudo instruction + targets: list[str] # opcodes this can be replaced by + AstNode = InstDef | Macro | Pseudo | Family + class Parser(PLexer): @contextual def definition(self) -> AstNode | None: @@ -253,14 +260,25 @@ def cache_effect(self) -> CacheEffect | None: @contextual def stack_effect(self) -> StackEffect | None: - # IDENTIFIER [':' IDENTIFIER [TIMES]] ['if' '(' expression ')'] + # IDENTIFIER [':' [IDENTIFIER [TIMES]] ['&' '(' IDENTIFIER ['+' IDENTIFIER] ')']] ['if' '(' expression ')'] # | IDENTIFIER '[' expression ']' if tkn := self.expect(lx.IDENTIFIER): type_text = "" + type_prop = None if self.expect(lx.COLON): - type_text = self.require(lx.IDENTIFIER).text.strip() - if self.expect(lx.TIMES): - type_text += " *" + if i := self.expect(lx.IDENTIFIER): + type_text = i.text.strip() + if self.expect(lx.TIMES): + type_text += " *" + if self.expect(lx.AND): + consumed_bracket = self.expect(lx.LPAREN) is not None + type_prop_text = self.require(lx.IDENTIFIER).text.strip() + refinement = None + if self.expect(lx.PLUS): + refinement = self.require(lx.IDENTIFIER).text.strip() + type_prop = (type_prop_text, refinement) + if consumed_bracket: + self.require(lx.RPAREN) cond_text = "" if self.expect(lx.IF): self.require(lx.LPAREN) @@ -277,7 +295,7 @@ def stack_effect(self) -> StackEffect | None: self.require(lx.RBRACKET) type_text = "PyObject **" size_text = size.text.strip() - return StackEffect(tkn.text, type_text, cond_text, size_text) + return StackEffect(tkn.text, type_text, cond_text, size_text, type_prop) return None @contextual @@ -364,7 +382,9 @@ def family_def(self) -> Family | None: if self.expect(lx.COMMA): if not (size := self.expect(lx.IDENTIFIER)): if not (size := self.expect(lx.NUMBER)): - raise self.make_syntax_error("Expected identifier or number") + raise self.make_syntax_error( + "Expected identifier or number" + ) if self.expect(lx.RPAREN): if self.expect(lx.EQUALS): if not self.expect(lx.LBRACE): diff --git a/Tools/cases_generator/py_metadata_generator.py b/Tools/cases_generator/py_metadata_generator.py index 43811fdacc8a9e..0dbcd599f9d4d9 100644 --- a/Tools/cases_generator/py_metadata_generator.py +++ b/Tools/cases_generator/py_metadata_generator.py @@ -1,6 +1,6 @@ -"""Generate uop metedata. +"""Generate opcode metadata for Python. Reads the instruction definitions from bytecodes.c. -Writes the metadata to pycore_uop_metadata.h by default. +Writes the metadata to _opcode_metadata.py by default. """ import argparse diff --git a/Tools/cases_generator/stack.py b/Tools/cases_generator/stack.py index d351037a663ca2..f62ece43c1be7f 100644 --- a/Tools/cases_generator/stack.py +++ b/Tools/cases_generator/stack.py @@ -3,6 +3,8 @@ from dataclasses import dataclass from cwriter import CWriter +UNUSED = {"unused"} + def maybe_parenthesize(sym: str) -> str: """Add parentheses around a string if it contains an operator @@ -29,6 +31,7 @@ def var_size(var: StackItem) -> str: else: return var.size + @dataclass class StackOffset: "The stack offset of the virtual base of the stack from the physical stack pointer" @@ -47,10 +50,7 @@ def push(self, item: StackItem) -> None: self.pushed.append(var_size(item)) def __sub__(self, other: "StackOffset") -> "StackOffset": - return StackOffset( - self.popped + other.pushed, - self.pushed + other.popped - ) + return StackOffset(self.popped + other.pushed, self.pushed + other.popped) def __neg__(self) -> "StackOffset": return StackOffset(self.pushed, self.popped) @@ -134,18 +134,18 @@ def pop(self, var: StackItem) -> str: ) if popped.name == var.name: return "" - elif popped.name == "unused": + elif popped.name in UNUSED: self.defined.add(var.name) return ( f"{var.name} = {indirect}stack_pointer[{self.top_offset.to_c()}];\n" ) - elif var.name == "unused": + elif var.name in UNUSED: return "" else: self.defined.add(var.name) return f"{var.name} = {popped.name};\n" self.base_offset.pop(var) - if var.name == "unused": + if var.name in UNUSED: return "" else: self.defined.add(var.name) @@ -159,7 +159,7 @@ def pop(self, var: StackItem) -> str: def push(self, var: StackItem) -> str: self.variables.append(var) - if var.is_array() and var.name not in self.defined and var.name != "unused": + if var.is_array() and var.name not in self.defined and var.name not in UNUSED: c_offset = self.top_offset.to_c() self.top_offset.push(var) self.defined.add(var.name) @@ -169,10 +169,11 @@ def push(self, var: StackItem) -> str: return "" def flush(self, out: CWriter) -> None: + out.start_line() for var in self.variables: if not var.peek: cast = "(PyObject *)" if var.type else "" - if var.name != "unused" and not var.is_array(): + if var.name not in UNUSED and not var.is_array(): if var.condition: out.emit(f"if ({var.condition}) ") out.emit( @@ -189,6 +190,7 @@ def flush(self, out: CWriter) -> None: self.base_offset.clear() self.top_offset.clear() self.peek_offset.clear() + out.start_line() def as_comment(self) -> str: return f"/* Variables: {[v.name for v in self.variables]}. Base offset: {self.base_offset.to_c()}. Top offset: {self.top_offset.to_c()} */" diff --git a/Tools/cases_generator/uop_metadata_generator.py b/Tools/cases_generator/uop_metadata_generator.py index d4f3a096d2acc1..9083ecc48bdf5b 100644 --- a/Tools/cases_generator/uop_metadata_generator.py +++ b/Tools/cases_generator/uop_metadata_generator.py @@ -1,4 +1,4 @@ -"""Generate uop metedata. +"""Generate uop metadata. Reads the instruction definitions from bytecodes.c. Writes the metadata to pycore_uop_metadata.h by default. """ diff --git a/Tools/clinic/.ruff.toml b/Tools/clinic/.ruff.toml index cbb3a9a8f3a8c2..c019572d0cb186 100644 --- a/Tools/clinic/.ruff.toml +++ b/Tools/clinic/.ruff.toml @@ -1,5 +1,7 @@ target-version = "py310" fix = true + +[lint] select = [ "F", # Enable all pyflakes rules "UP", # Enable all pyupgrade rules by default diff --git a/Tools/clinic/clinic.py b/Tools/clinic/clinic.py index e8ba805bd53b45..db57d17899af93 100755 --- a/Tools/clinic/clinic.py +++ b/Tools/clinic/clinic.py @@ -13,11 +13,9 @@ import collections import contextlib import copy -import cpp import dataclasses as dc import enum import functools -import hashlib import inspect import io import itertools @@ -25,7 +23,6 @@ import pprint import re import shlex -import string import sys import textwrap @@ -50,6 +47,13 @@ overload, ) + +# Local imports. +import libclinic +import libclinic.cpp +from libclinic import ClinicError + + # TODO: # # soon: @@ -61,23 +65,6 @@ # and keyword-only # -NO_VARARG = "PY_SSIZE_T_MAX" -CLINIC_PREFIX = "__clinic_" -CLINIC_PREFIXED_ARGS = { - "_keywords", - "_parser", - "args", - "argsbuf", - "fastargs", - "kwargs", - "kwnames", - "nargs", - "noptargs", - "return_value", -} - -# '#include "header.h" // reason': column of '//' comment -INCLUDE_COMMENT_COLUMN = 35 # match '#define Py_LIMITED_API' LIMITED_CAPI_REGEX = re.compile(r'#define +Py_LIMITED_API') @@ -103,65 +90,8 @@ def __repr__(self) -> str: NULL = Null() -sig_end_marker = '--' - -Appender = Callable[[str], None] -Outputter = Callable[[], str] TemplateDict = dict[str, str] -class _TextAccumulator(NamedTuple): - text: list[str] - append: Appender - output: Outputter - -def _text_accumulator() -> _TextAccumulator: - text: list[str] = [] - def output() -> str: - s = ''.join(text) - text.clear() - return s - return _TextAccumulator(text, text.append, output) - - -class TextAccumulator(NamedTuple): - append: Appender - output: Outputter - -def text_accumulator() -> TextAccumulator: - """ - Creates a simple text accumulator / joiner. - - Returns a pair of callables: - append, output - "append" appends a string to the accumulator. - "output" returns the contents of the accumulator - joined together (''.join(accumulator)) and - empties the accumulator. - """ - text, append, output = _text_accumulator() - return TextAccumulator(append, output) - - -@dc.dataclass -class ClinicError(Exception): - message: str - _: dc.KW_ONLY - lineno: int | None = None - filename: str | None = None - - def __post_init__(self) -> None: - super().__init__(self.message) - - def report(self, *, warn_only: bool = False) -> str: - msg = "Warning" if warn_only else "Error" - if self.filename is not None: - msg += f" in file {self.filename!r}" - if self.lineno is not None: - msg += f" on line {self.lineno}" - msg += ":\n" - msg += f"{self.message}\n" - return msg - @overload def warn_or_fail( @@ -213,33 +143,6 @@ def fail( warn_or_fail(*args, filename=filename, line_number=line_number, fail=True) -def quoted_for_c_string(s: str) -> str: - for old, new in ( - ('\\', '\\\\'), # must be first! - ('"', '\\"'), - ("'", "\\'"), - ): - s = s.replace(old, new) - return s - -def c_repr(s: str) -> str: - return '"' + s + '"' - - -def wrapped_c_string_literal( - text: str, - *, - width: int = 72, - suffix: str = '', - initial_indent: int = 0, - subsequent_indent: int = 4 -) -> str: - wrapped = textwrap.wrap(text, width=width, replace_whitespace=False, - drop_whitespace=False, break_on_hyphens=False) - separator = '"' + suffix + '\n' + subsequent_indent * ' ' + '"' - return initial_indent * ' ' + '"' + separator.join(wrapped) + '"' - - is_legal_c_identifier = re.compile('^[A-Za-z_][A-Za-z0-9_]*$').match def is_legal_py_identifier(s: str) -> bool: @@ -264,20 +167,6 @@ def ensure_legal_c_identifier(s: str) -> str: return s + "_value" return s -def rstrip_lines(s: str) -> str: - text, add, output = _text_accumulator() - for line in s.split('\n'): - add(line.rstrip()) - add('\n') - text.pop() - return output() - -def format_escape(s: str) -> str: - # double up curly-braces, this string will be used - # as part of a format_map() template later - s = s.replace('{', '{{') - s = s.replace('}', '}}') - return s def linear_format(s: str, **kwargs: str) -> str: """ @@ -293,19 +182,16 @@ def linear_format(s: str, **kwargs: str) -> str: by the indent of the source line. * A newline will be added to the end. """ - - add, output = text_accumulator() + lines = [] for line in s.split('\n'): indent, curly, trailing = line.partition('{') if not curly: - add(line) - add('\n') + lines.extend([line, "\n"]) continue name, curly, trailing = trailing.partition('}') if not curly or name not in kwargs: - add(line) - add('\n') + lines.extend([line, "\n"]) continue if trailing: @@ -319,58 +205,11 @@ def linear_format(s: str, **kwargs: str) -> str: if not value: continue - value = textwrap.indent(rstrip_lines(value), indent) - add(value) - add('\n') - - return output()[:-1] - -def indent_all_lines(s: str, prefix: str) -> str: - """ - Returns 's', with 'prefix' prepended to all lines. - - If the last line is empty, prefix is not prepended - to it. (If s is blank, returns s unchanged.) - - (textwrap.indent only adds to non-blank lines.) - """ - split = s.split('\n') - last = split.pop() - final = [] - for line in split: - final.append(prefix) - final.append(line) - final.append('\n') - if last: - final.append(prefix) - final.append(last) - return ''.join(final) - -def suffix_all_lines(s: str, suffix: str) -> str: - """ - Returns 's', with 'suffix' appended to all lines. + stripped = [line.rstrip() for line in value.split("\n")] + value = textwrap.indent("\n".join(stripped), indent) + lines.extend([value, "\n"]) - If the last line is empty, suffix is not appended - to it. (If s is blank, returns s unchanged.) - """ - split = s.split('\n') - last = split.pop() - final = [] - for line in split: - final.append(line) - final.append(suffix) - final.append('\n') - if last: - final.append(last) - final.append(suffix) - return ''.join(final) - - -def pprint_words(items: list[str]) -> str: - if len(items) <= 2: - return " and ".join(items) - else: - return ", ".join(items[:-1]) + " and " + items[-1] + return "".join(lines[:-1]) class CRenderData: @@ -430,24 +269,6 @@ def __init__(self) -> None: self.unlock: list[str] = [] -class FormatCounterFormatter(string.Formatter): - """ - This counts how many instances of each formatter - "replacement string" appear in the format string. - - e.g. after evaluating "string {a}, {b}, {c}, {a}" - the counts dict would now look like - {'a': 2, 'b': 1, 'c': 1} - """ - def __init__(self) -> None: - self.counts = collections.Counter[str]() - - def get_value( - self, key: str, args: object, kwargs: object # type: ignore[override] - ) -> Literal['']: - self.counts[key] += 1 - return '' - class Language(metaclass=abc.ABCMeta): start_line = "" @@ -501,7 +322,7 @@ def assert_only_one( fields = ['dsl_name'] fields.extend(additional_fields) line: str = getattr(self, attr) - fcf = FormatCounterFormatter() + fcf = libclinic.FormatCounterFormatter() fcf.format(line) def local_fail(should_be_there_but_isnt: bool) -> None: if should_be_there_but_isnt: @@ -608,34 +429,6 @@ def permute_optional_groups( return tuple(accumulator) -def strip_leading_and_trailing_blank_lines(s: str) -> str: - lines = s.rstrip().split('\n') - while lines: - line = lines[0] - if line.strip(): - break - del lines[0] - return '\n'.join(lines) - -@functools.lru_cache() -def normalize_snippet( - s: str, - *, - indent: int = 0 -) -> str: - """ - Reformats s: - * removes leading and trailing blank lines - * ensures that it does not end with a newline - * dedents so the first nonwhite character on any line is at column "indent" - """ - s = strip_leading_and_trailing_blank_lines(s) - s = textwrap.dedent(s) - if indent: - s = textwrap.indent(s, ' ' * indent) - return s - - def declare_parser( f: Function, *, @@ -706,62 +499,7 @@ def declare_parser( }}; #undef KWTUPLE """ % (format_ or fname) - return normalize_snippet(declarations) - - -def wrap_declarations( - text: str, - length: int = 78 -) -> str: - """ - A simple-minded text wrapper for C function declarations. - - It views a declaration line as looking like this: - xxxxxxxx(xxxxxxxxx,xxxxxxxxx) - If called with length=30, it would wrap that line into - xxxxxxxx(xxxxxxxxx, - xxxxxxxxx) - (If the declaration has zero or one parameters, this - function won't wrap it.) - - If this doesn't work properly, it's probably better to - start from scratch with a more sophisticated algorithm, - rather than try and improve/debug this dumb little function. - """ - lines = [] - for line in text.split('\n'): - prefix, _, after_l_paren = line.partition('(') - if not after_l_paren: - lines.append(line) - continue - in_paren, _, after_r_paren = after_l_paren.partition(')') - if not _: - lines.append(line) - continue - if ',' not in in_paren: - lines.append(line) - continue - parameters = [x.strip() + ", " for x in in_paren.split(',')] - prefix += "(" - if len(prefix) < length: - spaces = " " * len(prefix) - else: - spaces = " " * 4 - - while parameters: - line = prefix - first = True - while parameters: - if (not first and - (len(line) + len(parameters[0]) > length)): - break - line += parameters.pop(0) - first = False - if not parameters: - line = line.rstrip(", ") + ")" + after_r_paren - lines.append(line.rstrip()) - prefix = spaces - return "\n".join(lines) + return libclinic.normalize_snippet(declarations) class CLanguage(Language): @@ -773,67 +511,69 @@ class CLanguage(Language): stop_line = "[{dsl_name} start generated code]*/" checksum_line = "/*[{dsl_name} end generated code: {arguments}]*/" - PARSER_PROTOTYPE_KEYWORD: Final[str] = normalize_snippet(""" + NO_VARARG: Final[str] = "PY_SSIZE_T_MAX" + + PARSER_PROTOTYPE_KEYWORD: Final[str] = libclinic.normalize_snippet(""" static PyObject * {c_basename}({self_type}{self_name}, PyObject *args, PyObject *kwargs) """) - PARSER_PROTOTYPE_KEYWORD___INIT__: Final[str] = normalize_snippet(""" + PARSER_PROTOTYPE_KEYWORD___INIT__: Final[str] = libclinic.normalize_snippet(""" static int {c_basename}({self_type}{self_name}, PyObject *args, PyObject *kwargs) """) - PARSER_PROTOTYPE_VARARGS: Final[str] = normalize_snippet(""" + PARSER_PROTOTYPE_VARARGS: Final[str] = libclinic.normalize_snippet(""" static PyObject * {c_basename}({self_type}{self_name}, PyObject *args) """) - PARSER_PROTOTYPE_FASTCALL: Final[str] = normalize_snippet(""" + PARSER_PROTOTYPE_FASTCALL: Final[str] = libclinic.normalize_snippet(""" static PyObject * {c_basename}({self_type}{self_name}, PyObject *const *args, Py_ssize_t nargs) """) - PARSER_PROTOTYPE_FASTCALL_KEYWORDS: Final[str] = normalize_snippet(""" + PARSER_PROTOTYPE_FASTCALL_KEYWORDS: Final[str] = libclinic.normalize_snippet(""" static PyObject * {c_basename}({self_type}{self_name}, PyObject *const *args, Py_ssize_t nargs, PyObject *kwnames) """) - PARSER_PROTOTYPE_DEF_CLASS: Final[str] = normalize_snippet(""" + PARSER_PROTOTYPE_DEF_CLASS: Final[str] = libclinic.normalize_snippet(""" static PyObject * {c_basename}({self_type}{self_name}, PyTypeObject *{defining_class_name}, PyObject *const *args, Py_ssize_t nargs, PyObject *kwnames) """) - PARSER_PROTOTYPE_NOARGS: Final[str] = normalize_snippet(""" + PARSER_PROTOTYPE_NOARGS: Final[str] = libclinic.normalize_snippet(""" static PyObject * {c_basename}({self_type}{self_name}, PyObject *Py_UNUSED(ignored)) """) - PARSER_PROTOTYPE_GETTER: Final[str] = normalize_snippet(""" + PARSER_PROTOTYPE_GETTER: Final[str] = libclinic.normalize_snippet(""" static PyObject * {c_basename}({self_type}{self_name}, void *Py_UNUSED(context)) """) - PARSER_PROTOTYPE_SETTER: Final[str] = normalize_snippet(""" + PARSER_PROTOTYPE_SETTER: Final[str] = libclinic.normalize_snippet(""" static int {c_basename}({self_type}{self_name}, PyObject *value, void *Py_UNUSED(context)) """) - METH_O_PROTOTYPE: Final[str] = normalize_snippet(""" + METH_O_PROTOTYPE: Final[str] = libclinic.normalize_snippet(""" static PyObject * {c_basename}({impl_parameters}) """) - DOCSTRING_PROTOTYPE_VAR: Final[str] = normalize_snippet(""" + DOCSTRING_PROTOTYPE_VAR: Final[str] = libclinic.normalize_snippet(""" PyDoc_VAR({c_basename}__doc__); """) - DOCSTRING_PROTOTYPE_STRVAR: Final[str] = normalize_snippet(""" + DOCSTRING_PROTOTYPE_STRVAR: Final[str] = libclinic.normalize_snippet(""" PyDoc_STRVAR({c_basename}__doc__, {docstring}); """) - GETSET_DOCSTRING_PROTOTYPE_STRVAR: Final[str] = normalize_snippet(""" + GETSET_DOCSTRING_PROTOTYPE_STRVAR: Final[str] = libclinic.normalize_snippet(""" PyDoc_STRVAR({getset_basename}__doc__, {docstring}); #define {getset_basename}_HAS_DOCSTR """) - IMPL_DEFINITION_PROTOTYPE: Final[str] = normalize_snippet(""" + IMPL_DEFINITION_PROTOTYPE: Final[str] = libclinic.normalize_snippet(""" static {impl_return_type} {c_basename}_impl({impl_parameters}) """) - METHODDEF_PROTOTYPE_DEFINE: Final[str] = normalize_snippet(r""" + METHODDEF_PROTOTYPE_DEFINE: Final[str] = libclinic.normalize_snippet(r""" #define {methoddef_name} \ {{"{name}", {methoddef_cast}{c_basename}{methoddef_cast_end}, {methoddef_flags}, {c_basename}__doc__}}, """) - GETTERDEF_PROTOTYPE_DEFINE: Final[str] = normalize_snippet(r""" + GETTERDEF_PROTOTYPE_DEFINE: Final[str] = libclinic.normalize_snippet(r""" #if defined({getset_basename}_HAS_DOCSTR) # define {getset_basename}_DOCSTR {getset_basename}__doc__ #else @@ -846,7 +586,7 @@ class CLanguage(Language): # define {getset_name}_GETSETDEF {{"{name}", (getter){getset_basename}_get, NULL, {getset_basename}_DOCSTR}}, #endif """) - SETTERDEF_PROTOTYPE_DEFINE: Final[str] = normalize_snippet(r""" + SETTERDEF_PROTOTYPE_DEFINE: Final[str] = libclinic.normalize_snippet(r""" #if defined({getset_name}_HAS_DOCSTR) # define {getset_basename}_DOCSTR {getset_basename}__doc__ #else @@ -859,7 +599,7 @@ class CLanguage(Language): # define {getset_name}_GETSETDEF {{"{name}", NULL, (setter){getset_basename}_set, NULL}}, #endif """) - METHODDEF_PROTOTYPE_IFNDEF: Final[str] = normalize_snippet(""" + METHODDEF_PROTOTYPE_IFNDEF: Final[str] = libclinic.normalize_snippet(""" #ifndef {methoddef_name} #define {methoddef_name} #endif /* !defined({methoddef_name}) */ @@ -888,8 +628,7 @@ class CLanguage(Language): def __init__(self, filename: str) -> None: super().__init__(filename) - self.cpp = cpp.Monitor(filename) - self.cpp.fail = fail # type: ignore[method-assign] + self.cpp = libclinic.cpp.Monitor(filename) def parse_line(self, line: str) -> None: self.cpp.writeline(line) @@ -926,9 +665,9 @@ def compiler_deprecated_warning( code = self.COMPILER_DEPRECATION_WARNING_PROTOTYPE.format( major=minversion[0], minor=minversion[1], - message=c_repr(message), + message=libclinic.c_repr(message), ) - return normalize_snippet(code) + return libclinic.normalize_snippet(code) def deprecate_positional_use( self, @@ -957,7 +696,7 @@ def deprecate_positional_use( params.values(), key=attrgetter("deprecated_positional") ): names = [repr(p.name) for p in group] - pstr = pprint_words(names) + pstr = libclinic.pprint_words(names) if len(names) == 1: message += ( f" Parameter {pstr} will become a keyword-only parameter " @@ -976,10 +715,10 @@ def deprecate_positional_use( code = self.DEPRECATION_WARNING_PROTOTYPE.format( condition=condition, errcheck="", - message=wrapped_c_string_literal(message, width=64, - subsequent_indent=20), + message=libclinic.wrapped_c_string_literal(message, width=64, + subsequent_indent=20), ) - return normalize_snippet(code, indent=4) + return libclinic.normalize_snippet(code, indent=4) def deprecate_keyword_use( self, @@ -1026,7 +765,7 @@ def deprecate_keyword_use( else: condition = f"kwargs && PyDict_GET_SIZE(kwargs) && {condition}" names = [repr(p.name) for p in params.values()] - pstr = pprint_words(names) + pstr = libclinic.pprint_words(names) pl = 's' if len(params) != 1 else '' message = ( f"Passing keyword argument{pl} {pstr} to " @@ -1037,7 +776,7 @@ def deprecate_keyword_use( params.values(), key=attrgetter("deprecated_keyword") ): names = [repr(p.name) for p in group] - pstr = pprint_words(names) + pstr = libclinic.pprint_words(names) pl = 's' if len(names) != 1 else '' message += ( f" Parameter{pl} {pstr} will become positional-only " @@ -1059,30 +798,10 @@ def deprecate_keyword_use( code = self.DEPRECATION_WARNING_PROTOTYPE.format( condition=condition, errcheck=errcheck, - message=wrapped_c_string_literal(message, width=64, - subsequent_indent=20), + message=libclinic.wrapped_c_string_literal(message, width=64, + subsequent_indent=20), ) - return normalize_snippet(code, indent=4) - - def docstring_for_c_string( - self, - f: Function - ) -> str: - text, add, output = _text_accumulator() - # turn docstring into a properly quoted C string - for line in f.docstring.split('\n'): - add('"') - add(quoted_for_c_string(line)) - add('\\n"\n') - - if text[-2] == sig_end_marker: - # If we only have a signature, add the blank line that the - # __text_signature__ getter expects to be there. - add('"\\n"') - else: - text.pop() - add('"') - return ''.join(text) + return libclinic.normalize_snippet(code, indent=4) def output_templates( self, @@ -1099,12 +818,6 @@ def output_templates( del parameters[0] converters = [p.converter for p in parameters] - # Copy includes from parameters to Clinic - for converter in converters: - include = converter.include - if include: - clinic.add_include(include.filename, include.reason, - condition=include.condition) if f.critical_section: clinic.add_include('pycore_critical_section.h', 'Py_BEGIN_CRITICAL_SECTION()') has_option_groups = parameters and (parameters[0].group or parameters[-1].group) @@ -1112,7 +825,7 @@ def output_templates( and not f.critical_section) new_or_init = f.kind.new_or_init - vararg: int | str = NO_VARARG + vararg: int | str = self.NO_VARARG pos_only = min_pos = max_pos = min_kw_only = pseudo_args = 0 for i, p in enumerate(parameters, 1): if p.is_keyword_only(): @@ -1120,12 +833,12 @@ def output_templates( if not p.is_optional(): min_kw_only = i - max_pos elif p.is_vararg(): - if vararg != NO_VARARG: + if vararg != self.NO_VARARG: fail("Too many var args") pseudo_args += 1 vararg = i - 1 else: - if vararg == NO_VARARG: + if vararg == self.NO_VARARG: max_pos = i if p.is_positional_only(): pos_only = i @@ -1183,18 +896,18 @@ def parser_body( declarations: str = '' ) -> str: nonlocal parser_body_fields - add, output = text_accumulator() - add(prototype) + lines = [] + lines.append(prototype) parser_body_fields = fields - preamble = normalize_snippet(""" + preamble = libclinic.normalize_snippet(""" {{ {return_value_declaration} {parser_declarations} {declarations} {initializers} """) + "\n" - finale = normalize_snippet(""" + finale = libclinic.normalize_snippet(""" {modifications} {lock} {return_value} = {c_basename}_impl({impl_arguments}); @@ -1208,9 +921,8 @@ def parser_body( }} """) for field in preamble, *fields, finale: - add('\n') - add(field) - return linear_format(output(), parser_declarations=declarations) + lines.append(field) + return linear_format("\n".join(lines), parser_declarations=declarations) fastcall = not new_or_init limited_capi = clinic.limited_capi @@ -1247,8 +959,8 @@ def parser_body( parser_prototype = self.PARSER_PROTOTYPE_DEF_CLASS return_error = ('return NULL;' if simple_return else 'goto exit;') - parser_code = [normalize_snippet(""" - if (nargs) {{ + parser_code = [libclinic.normalize_snippet(""" + if (nargs || (kwnames && PyTuple_GET_SIZE(kwnames))) {{ PyErr_SetString(PyExc_TypeError, "{name}() takes no arguments"); %s }} @@ -1287,7 +999,7 @@ def parser_body( argname = 'arg' if parameters[0].name == argname: argname += '_' - parser_prototype = normalize_snippet(""" + parser_prototype = libclinic.normalize_snippet(""" static PyObject * {c_basename}({self_type}{self_name}, PyObject *%s) """ % argname) @@ -1301,7 +1013,7 @@ def parser_body( }} """ % argname parser_definition = parser_body(parser_prototype, - normalize_snippet(parsearg, indent=4)) + libclinic.normalize_snippet(parsearg, indent=4)) elif has_option_groups: # positional parameters with option groups @@ -1335,15 +1047,16 @@ def parser_body( argname_fmt = 'PyTuple_GET_ITEM(args, %d)' left_args = f"{nargs} - {max_pos}" - max_args = NO_VARARG if (vararg != NO_VARARG) else max_pos + max_args = self.NO_VARARG if (vararg != self.NO_VARARG) else max_pos if limited_capi: parser_code = [] if nargs != 'nargs': - parser_code.append(normalize_snippet(f'Py_ssize_t nargs = {nargs};', indent=4)) + nargs_def = f'Py_ssize_t nargs = {nargs};' + parser_code.append(libclinic.normalize_snippet(nargs_def, indent=4)) nargs = 'nargs' if min_pos == max_args: pl = '' if min_pos == 1 else 's' - parser_code.append(normalize_snippet(f""" + parser_code.append(libclinic.normalize_snippet(f""" if ({nargs} != {min_pos}) {{{{ PyErr_Format(PyExc_TypeError, "{{name}} expected {min_pos} argument{pl}, got %zd", {nargs}); goto exit; @@ -1353,16 +1066,16 @@ def parser_body( else: if min_pos: pl = '' if min_pos == 1 else 's' - parser_code.append(normalize_snippet(f""" + parser_code.append(libclinic.normalize_snippet(f""" if ({nargs} < {min_pos}) {{{{ PyErr_Format(PyExc_TypeError, "{{name}} expected at least {min_pos} argument{pl}, got %zd", {nargs}); goto exit; }}}} """, indent=4)) - if max_args != NO_VARARG: + if max_args != self.NO_VARARG: pl = '' if max_args == 1 else 's' - parser_code.append(normalize_snippet(f""" + parser_code.append(libclinic.normalize_snippet(f""" if ({nargs} > {max_args}) {{{{ PyErr_Format(PyExc_TypeError, "{{name}} expected at most {max_args} argument{pl}, got %zd", {nargs}); goto exit; @@ -1372,7 +1085,7 @@ def parser_body( else: clinic.add_include('pycore_modsupport.h', '_PyArg_CheckPositional()') - parser_code = [normalize_snippet(f""" + parser_code = [libclinic.normalize_snippet(f""" if (!_PyArg_CheckPositional("{{name}}", {nargs}, {min_pos}, {max_args})) {{{{ goto exit; }}}} @@ -1382,7 +1095,7 @@ def parser_body( for i, p in enumerate(parameters): if p.is_vararg(): if fastcall: - parser_code.append(normalize_snippet(""" + parser_code.append(libclinic.normalize_snippet(""" %s = PyTuple_New(%s); if (!%s) {{ goto exit; @@ -1399,7 +1112,7 @@ def parser_body( max_pos ), indent=4)) else: - parser_code.append(normalize_snippet(""" + parser_code.append(libclinic.normalize_snippet(""" %s = PyTuple_GetSlice(%d, -1); """ % ( p.converter.parser_name, @@ -1415,12 +1128,12 @@ def parser_body( break if has_optional or p.is_optional(): has_optional = True - parser_code.append(normalize_snippet(""" + parser_code.append(libclinic.normalize_snippet(""" if (%s < %d) {{ goto skip_optional; }} """, indent=4) % (nargs, i + 1)) - parser_code.append(normalize_snippet(parsearg, indent=4)) + parser_code.append(libclinic.normalize_snippet(parsearg, indent=4)) if parser_code is not None: if has_optional: @@ -1431,7 +1144,7 @@ def parser_body( if fastcall: clinic.add_include('pycore_modsupport.h', '_PyArg_ParseStack()') - parser_code = [normalize_snippet(""" + parser_code = [libclinic.normalize_snippet(""" if (!_PyArg_ParseStack(args, nargs, "{format_units}:{name}", {parse_arguments})) {{ goto exit; @@ -1440,7 +1153,7 @@ def parser_body( else: flags = "METH_VARARGS" parser_prototype = self.PARSER_PROTOTYPE_VARARGS - parser_code = [normalize_snippet(""" + parser_code = [libclinic.normalize_snippet(""" if (!PyArg_ParseTuple(args, "{format_units}:{name}", {parse_arguments})) {{ goto exit; @@ -1457,13 +1170,16 @@ def parser_body( if p.deprecated_keyword: deprecated_keywords[i] = p - has_optional_kw = (max(pos_only, min_pos) + min_kw_only < len(converters) - int(vararg != NO_VARARG)) + has_optional_kw = ( + max(pos_only, min_pos) + min_kw_only + < len(converters) - int(vararg != self.NO_VARARG) + ) if limited_capi: parser_code = None fastcall = False else: - if vararg == NO_VARARG: + if vararg == self.NO_VARARG: clinic.add_include('pycore_modsupport.h', '_PyArg_UnpackKeywords()') args_declaration = "_PyArg_UnpackKeywords", "%s, %s, %s" % ( @@ -1492,7 +1208,7 @@ def parser_body( declarations += "\nPyObject *argsbuf[%s];" % len(converters) if has_optional_kw: declarations += "\nPy_ssize_t noptargs = %s + (kwnames ? PyTuple_GET_SIZE(kwnames) : 0) - %d;" % (nargs, min_pos + min_kw_only) - parser_code = [normalize_snippet(""" + parser_code = [libclinic.normalize_snippet(""" args = %s(args, nargs, NULL, kwnames, &_parser, %s, argsbuf); if (!args) {{ goto exit; @@ -1510,7 +1226,7 @@ def parser_body( declarations += "\nPy_ssize_t nargs = PyTuple_GET_SIZE(args);" if has_optional_kw: declarations += "\nPy_ssize_t noptargs = %s + (kwargs ? PyDict_GET_SIZE(kwargs) : 0) - %d;" % (nargs, min_pos + min_kw_only) - parser_code = [normalize_snippet(""" + parser_code = [libclinic.normalize_snippet(""" fastargs = %s(_PyTuple_CAST(args)->ob_item, nargs, kwargs, NULL, &_parser, %s, argsbuf); if (!fastargs) {{ goto exit; @@ -1543,19 +1259,19 @@ def parser_body( parser_code.append("%s:" % add_label) add_label = None if not p.is_optional(): - parser_code.append(normalize_snippet(parsearg, indent=4)) + parser_code.append(libclinic.normalize_snippet(parsearg, indent=4)) elif i < pos_only: add_label = 'skip_optional_posonly' - parser_code.append(normalize_snippet(""" + parser_code.append(libclinic.normalize_snippet(""" if (nargs < %d) {{ goto %s; }} """ % (i + 1, add_label), indent=4)) if has_optional_kw: - parser_code.append(normalize_snippet(""" + parser_code.append(libclinic.normalize_snippet(""" noptargs--; """, indent=4)) - parser_code.append(normalize_snippet(parsearg, indent=4)) + parser_code.append(libclinic.normalize_snippet(parsearg, indent=4)) else: if i < max_pos: label = 'skip_optional_pos' @@ -1563,24 +1279,24 @@ def parser_body( else: label = 'skip_optional_kwonly' first_opt = max_pos + min_kw_only - if vararg != NO_VARARG: + if vararg != self.NO_VARARG: first_opt += 1 if i == first_opt: add_label = label - parser_code.append(normalize_snippet(""" + parser_code.append(libclinic.normalize_snippet(""" if (!noptargs) {{ goto %s; }} """ % add_label, indent=4)) if i + 1 == len(parameters): - parser_code.append(normalize_snippet(parsearg, indent=4)) + parser_code.append(libclinic.normalize_snippet(parsearg, indent=4)) else: add_label = label - parser_code.append(normalize_snippet(""" + parser_code.append(libclinic.normalize_snippet(""" if (%s) {{ """ % (argname_fmt % i), indent=4)) - parser_code.append(normalize_snippet(parsearg, indent=8)) - parser_code.append(normalize_snippet(""" + parser_code.append(libclinic.normalize_snippet(parsearg, indent=8)) + parser_code.append(libclinic.normalize_snippet(""" if (!--noptargs) {{ goto %s; }} @@ -1599,7 +1315,7 @@ def parser_body( assert not fastcall flags = "METH_VARARGS|METH_KEYWORDS" parser_prototype = self.PARSER_PROTOTYPE_KEYWORD - parser_code = [normalize_snippet(""" + parser_code = [libclinic.normalize_snippet(""" if (!PyArg_ParseTupleAndKeywords(args, kwargs, "{format_units}:{name}", _keywords, {parse_arguments})) goto exit; @@ -1611,7 +1327,7 @@ def parser_body( elif fastcall: clinic.add_include('pycore_modsupport.h', '_PyArg_ParseStackAndKeywords()') - parser_code = [normalize_snippet(""" + parser_code = [libclinic.normalize_snippet(""" if (!_PyArg_ParseStackAndKeywords(args, nargs, kwnames, &_parser{parse_arguments_comma} {parse_arguments})) {{ goto exit; @@ -1620,7 +1336,7 @@ def parser_body( else: clinic.add_include('pycore_modsupport.h', '_PyArg_ParseTupleAndKeywordsFast()') - parser_code = [normalize_snippet(""" + parser_code = [libclinic.normalize_snippet(""" if (!_PyArg_ParseTupleAndKeywordsFast(args, kwargs, &_parser, {parse_arguments})) {{ goto exit; @@ -1645,6 +1361,13 @@ def parser_body( declarations=declarations) + # Copy includes from parameters to Clinic after parse_arg() has been + # called above. + for converter in converters: + for include in converter.includes: + clinic.add_include(include.filename, include.reason, + condition=include.condition) + if new_or_init: methoddef_define = '' @@ -1667,7 +1390,7 @@ def parser_body( declarations = '{base_type_ptr}' clinic.add_include('pycore_modsupport.h', '_PyArg_NoKeywords()') - fields.insert(0, normalize_snippet(""" + fields.insert(0, libclinic.normalize_snippet(""" if ({self_type_check}!_PyArg_NoKeywords("{name}", kwargs)) {{ goto exit; }} @@ -1675,7 +1398,7 @@ def parser_body( if not parses_positional: clinic.add_include('pycore_modsupport.h', '_PyArg_NoPositional()') - fields.insert(0, normalize_snippet(""" + fields.insert(0, libclinic.normalize_snippet(""" if ({self_type_check}!_PyArg_NoPositional("{name}", args)) {{ goto exit; }} @@ -1785,7 +1508,7 @@ def render_option_group_parsing( # Clinic prefers groups on the left. So in the above example, # five arguments would map to B+C, not C+D. - add, output = text_accumulator() + out = [] parameters = list(f.parameters.values()) if isinstance(parameters[0].converter, self_converter): del parameters[0] @@ -1817,14 +1540,14 @@ def render_option_group_parsing( nargs = 'PyTuple_Size(args)' else: nargs = 'PyTuple_GET_SIZE(args)' - add(f"switch ({nargs}) {{\n") + out.append(f"switch ({nargs}) {{\n") for subset in permute_optional_groups(left, required, right): count = len(subset) count_min = min(count_min, count) count_max = max(count_max, count) if count == 0: - add(""" case 0: + out.append(""" case 0: break; """) continue @@ -1856,14 +1579,15 @@ def render_option_group_parsing( """ s = linear_format(s, group_booleans=lines) s = s.format_map(d) - add(s) + out.append(s) - add(" default:\n") + out.append(" default:\n") s = ' PyErr_SetString(PyExc_TypeError, "{} requires {} to {} arguments");\n' - add(s.format(f.full_name, count_min, count_max)) - add(' goto exit;\n') - add("}") - template_dict['option_group_parsing'] = format_escape(output()) + out.append(s.format(f.full_name, count_min, count_max)) + out.append(' goto exit;\n') + out.append("}") + + template_dict['option_group_parsing'] = libclinic.format_escape("".join(out)) def render_function( self, @@ -1873,7 +1597,6 @@ def render_function( if f is None or clinic is None: return "" - add, output = text_accumulator() data = CRenderData() assert f.parameters, "We should always have a 'self' at this point!" @@ -1961,8 +1684,7 @@ def render_function( template_dict['methoddef_name'] = f.c_basename.upper() + "_METHODDEF" template_dict['c_basename'] = f.c_basename - template_dict['docstring'] = self.docstring_for_c_string(f) - + template_dict['docstring'] = libclinic.docstring_for_c_string(f.docstring) template_dict['self_name'] = template_dict['self_type'] = template_dict['self_type_check'] = '' template_dict['target_critical_section'] = ', '.join(f.target_critical_section) for converter in converters: @@ -1975,7 +1697,7 @@ def render_function( else: template_dict['impl_return_type'] = f.return_converter.type - template_dict['declarations'] = format_escape("\n".join(data.declarations)) + template_dict['declarations'] = libclinic.format_escape("\n".join(data.declarations)) template_dict['initializers'] = "\n\n".join(data.initializers) template_dict['modifications'] = '\n\n'.join(data.modifications) template_dict['keywords_c'] = ' '.join('"' + k + '",' @@ -1991,9 +1713,11 @@ def render_function( template_dict['parse_arguments_comma'] = ''; template_dict['impl_parameters'] = ", ".join(data.impl_parameters) template_dict['impl_arguments'] = ", ".join(data.impl_arguments) - template_dict['return_conversion'] = format_escape("".join(data.return_conversion).rstrip()) - template_dict['post_parsing'] = format_escape("".join(data.post_parsing).rstrip()) - template_dict['cleanup'] = format_escape("".join(data.cleanup)) + + template_dict['return_conversion'] = libclinic.format_escape("".join(data.return_conversion).rstrip()) + template_dict['post_parsing'] = libclinic.format_escape("".join(data.post_parsing).rstrip()) + template_dict['cleanup'] = libclinic.format_escape("".join(data.cleanup)) + template_dict['return_value'] = data.return_value template_dict['lock'] = "\n".join(data.lock) template_dict['unlock'] = "\n".join(data.unlock) @@ -2037,33 +1761,18 @@ def render_function( # mild hack: # reflow long impl declarations if name in {"impl_prototype", "impl_definition"}: - s = wrap_declarations(s) + s = libclinic.wrap_declarations(s) if clinic.line_prefix: - s = indent_all_lines(s, clinic.line_prefix) + s = libclinic.indent_all_lines(s, clinic.line_prefix) if clinic.line_suffix: - s = suffix_all_lines(s, clinic.line_suffix) + s = libclinic.suffix_all_lines(s, clinic.line_suffix) destination.append(s) return clinic.get_destination('block').dump() -def create_regex( - before: str, - after: str, - word: bool = True, - whole_line: bool = True -) -> re.Pattern[str]: - """Create an re object for matching marker lines.""" - group_re = r"\w+" if word else ".+" - pattern = r'{}({}){}' - if whole_line: - pattern = '^' + pattern + '$' - pattern = pattern.format(re.escape(before), group_re, re.escape(after)) - return re.compile(pattern) - - @dc.dataclass(slots=True, repr=False) class Block: r""" @@ -2162,8 +1871,9 @@ def __init__( self.language = language before, _, after = language.start_line.partition('{dsl_name}') assert _ == '{dsl_name}' - self.find_start_re = create_regex(before, after, whole_line=False) - self.start_re = create_regex(before, after) + self.find_start_re = libclinic.create_regex(before, after, + whole_line=False) + self.start_re = libclinic.create_regex(before, after) self.verify = verify self.last_checksum_re: re.Pattern[str] | None = None self.last_dsl_name: str | None = None @@ -2202,7 +1912,7 @@ def _line(self, lookahead: bool = False) -> str: return line def parse_verbatim_block(self) -> Block: - add, output = text_accumulator() + lines = [] self.block_start_line_number = self.line_number while self.input: @@ -2211,12 +1921,12 @@ def parse_verbatim_block(self) -> Block: if dsl_name: self.dsl_name = dsl_name break - add(line) + lines.append(line) - return Block(output()) + return Block("".join(lines)) def parse_clinic_block(self, dsl_name: str) -> Block: - input_add, input_output = text_accumulator() + in_lines = [] self.block_start_line_number = self.line_number + 1 stop_line = self.language.stop_line.format(dsl_name=dsl_name) body_prefix = self.language.body_prefix.format(dsl_name=dsl_name) @@ -2244,7 +1954,7 @@ def is_stop_line(line: str) -> bool: line = line.lstrip() assert line.startswith(body_prefix) line = line.removeprefix(body_prefix) - input_add(line) + in_lines.append(line) # consume output and checksum line, if present. if self.last_dsl_name == dsl_name: @@ -2252,13 +1962,13 @@ def is_stop_line(line: str) -> bool: else: before, _, after = self.language.checksum_line.format(dsl_name=dsl_name, arguments='{arguments}').partition('{arguments}') assert _ == '{arguments}' - checksum_re = create_regex(before, after, word=False) + checksum_re = libclinic.create_regex(before, after, word=False) self.last_dsl_name = dsl_name self.last_checksum_re = checksum_re assert checksum_re is not None # scan forward for checksum line - output_add, output_output = text_accumulator() + out_lines = [] arguments = None while self.input: line = self._line(lookahead=True) @@ -2266,12 +1976,12 @@ def is_stop_line(line: str) -> bool: arguments = match.group(1) if match else None if arguments: break - output_add(line) + out_lines.append(line) if self.is_start_line(line): break output: str | None - output = output_output() + output = "".join(out_lines) if arguments: d = {} for field in shlex.split(arguments): @@ -2286,7 +1996,7 @@ def is_stop_line(line: str) -> bool: else: checksum = d['checksum'] - computed = compute_checksum(output, len(checksum)) + computed = libclinic.compute_checksum(output, len(checksum)) if checksum != computed: fail("Checksum mismatch! " f"Expected {checksum!r}, computed {computed!r}. " @@ -2299,7 +2009,7 @@ def is_stop_line(line: str) -> bool: self.input.extend(reversed(output_lines)) output = None - return Block(input_output(), dsl_name, output=output) + return Block("".join(in_lines), dsl_name, output=output) @dc.dataclass(slots=True, frozen=True) @@ -2327,6 +2037,9 @@ class BlockPrinter: language: Language f: io.StringIO = dc.field(default_factory=io.StringIO) + # '#include "header.h" // reason': column of '//' comment + INCLUDE_COMMENT_COLUMN: Final[int] = 35 + def print_block( self, block: Block, @@ -2382,7 +2095,7 @@ def print_block( line = f'#include "{include.filename}"' if include.reason: comment = f'// {include.reason}\n' - line = line.ljust(INCLUDE_COMMENT_COLUMN - 1) + comment + line = line.ljust(self.INCLUDE_COMMENT_COLUMN - 1) + comment output += line if current_condition: @@ -2396,8 +2109,8 @@ def print_block( write(output) arguments = "output={output} input={input}".format( - output=compute_checksum(output, 16), - input=compute_checksum(input, 16) + output=libclinic.compute_checksum(output, 16), + input=libclinic.compute_checksum(input, 16) ) write(self.language.checksum_line.format(dsl_name=dsl_name, arguments=arguments)) write("\n") @@ -2419,26 +2132,26 @@ class BufferSeries: def __init__(self) -> None: self._start = 0 - self._array: list[_TextAccumulator] = [] - self._constructor = _text_accumulator + self._array: list[list[str]] = [] - def __getitem__(self, i: int) -> _TextAccumulator: + def __getitem__(self, i: int) -> list[str]: i -= self._start if i < 0: self._start += i - prefix = [self._constructor() for x in range(-i)] + prefix: list[list[str]] = [[] for x in range(-i)] self._array = prefix + self._array i = 0 while i >= len(self._array): - self._array.append(self._constructor()) + self._array.append([]) return self._array[i] def clear(self) -> None: for ta in self._array: - ta.text.clear() + ta.clear() def dump(self) -> str: - texts = [ta.output() for ta in self._array] + texts = ["".join(ta) for ta in self._array] + self.clear() return "".join(texts) @@ -2499,27 +2212,6 @@ def dump(self) -> str: extensions['py'] = PythonLanguage -def write_file(filename: str, new_contents: str) -> None: - try: - with open(filename, encoding="utf-8") as fp: - old_contents = fp.read() - - if old_contents == new_contents: - # no change: avoid modifying the file modification time - return - except FileNotFoundError: - pass - # Atomic write using a temporary file and os.replace() - filename_new = f"{filename}.new" - with open(filename_new, "w", encoding="utf-8") as fp: - fp.write(new_contents) - try: - os.replace(filename_new, filename) - except: - os.unlink(filename_new) - raise - - ClassDict = dict[str, "Class"] DestinationDict = dict[str, Destination] ModuleDict = dict[str, "Module"] @@ -2624,7 +2316,7 @@ def __init__( 'impl_definition': d('block'), } - DestBufferType = dict[str, _TextAccumulator] + DestBufferType = dict[str, list[str]] DestBufferList = list[DestBufferType] self.destination_buffers_stack: DestBufferList = [] @@ -2696,7 +2388,7 @@ def get_destination_buffer( self, name: str, item: int = 0 - ) -> _TextAccumulator: + ) -> list[str]: d = self.get_destination(name) return d.buffers[item] @@ -2759,7 +2451,8 @@ def parse(self, input: str) -> str: core_includes=True, limited_capi=self.limited_capi, header_includes=self.includes) - write_file(destination.filename, printer_2.f.getvalue()) + libclinic.write_file(destination.filename, + printer_2.f.getvalue()) continue return printer.f.getvalue() @@ -2832,18 +2525,7 @@ def parse_file( limited_capi=limited_capi) cooked = clinic.parse(raw) - write_file(output, cooked) - - -def compute_checksum( - input: str | None, - length: int | None = None -) -> str: - input = input or '' - s = hashlib.sha1(input.encode('utf-8')).hexdigest() - if length: - s = s[:length] - return s + libclinic.write_file(output, cooked) class PythonParser: @@ -3150,11 +2832,9 @@ def get_displayname(self, i: int) -> str: return f'argument {i}' def render_docstring(self) -> str: - add, out = text_accumulator() - add(f" {self.name}\n") - for line in self.docstring.split("\n"): - add(f" {line}\n") - return out().rstrip() + lines = [f" {self.name}"] + lines.extend(f" {line}" for line in self.docstring.split("\n")) + return "\n".join(lines).rstrip() CConverterClassT = TypeVar("CConverterClassT", bound=type["CConverter"]) @@ -3309,7 +2989,6 @@ class CConverter(metaclass=CConverterAutoRegister): # Only set by self_converter. signature_name: str | None = None - include: Include | None = None broken_limited_capi: bool = False # keep in sync with self_converter.__init__! @@ -3329,6 +3008,7 @@ def __init__(self, self.name = ensure_legal_c_identifier(name) self.py_name = py_name self.unused = unused + self.includes: list[Include] = [] if default is not unspecified: if (self.default_type @@ -3472,7 +3152,7 @@ def parse_argument(self, args: list[str]) -> None: args.append(self.converter) if self.encoding: - args.append(c_repr(self.encoding)) + args.append(libclinic.c_repr(self.encoding)) elif self.subclass_of: args.append(self.subclass_of) @@ -3584,8 +3264,7 @@ def bad_argument(self, displayname: str, expected: str, *, limited_capi: bool, e else: if expected_literal: expected = f'"{expected}"' - if clinic is not None: - clinic.add_include('pycore_modsupport.h', '_PyArg_BadArgument()') + self.add_include('pycore_modsupport.h', '_PyArg_BadArgument()') return f'_PyArg_BadArgument("{{{{name}}}}", "{displayname}", {expected}, {{argname}});' def format_code(self, fmt: str, *, @@ -3650,16 +3329,15 @@ def set_template_dict(self, template_dict: TemplateDict) -> None: @property def parser_name(self) -> str: - if self.name in CLINIC_PREFIXED_ARGS: # bpo-39741 - return CLINIC_PREFIX + self.name + if self.name in libclinic.CLINIC_PREFIXED_ARGS: # bpo-39741 + return libclinic.CLINIC_PREFIX + self.name else: return self.name def add_include(self, name: str, reason: str, *, condition: str | None = None) -> None: - if self.include is not None: - raise ValueError("a converter only supports a single include") - self.include = Include(name, reason, condition) + include = Include(name, reason, condition) + self.includes.append(include) type_checks = { '&PyLong_Type': ('PyLong_Check', 'int'), @@ -4690,7 +4368,9 @@ def parse_arg(self, argname: str, displayname: str, *, limited_capi: bool) -> st if (ptr == NULL) {{{{ goto exit; }}}} - PyBuffer_FillInfo(&{paramname}, {argname}, (void *)ptr, len, 1, 0); + if (PyBuffer_FillInfo(&{paramname}, {argname}, (void *)ptr, len, 1, PyBUF_SIMPLE) < 0) {{{{ + goto exit; + }}}} }}}} else {{{{ /* any bytes-like object */ if (PyObject_GetBuffer({argname}, &{paramname}, PyBUF_SIMPLE) != 0) {{{{ @@ -5933,7 +5613,7 @@ def bad_node(self, node: ast.AST) -> None: if isinstance(value, (bool, NoneType)): c_default = "Py_" + py_default elif isinstance(value, str): - c_default = c_repr(value) + c_default = libclinic.c_repr(value) else: c_default = py_default @@ -6220,15 +5900,15 @@ def state_function_docstring(self, line: str) -> None: def format_docstring_signature( self, f: Function, parameters: list[Parameter] ) -> str: - text, add, output = _text_accumulator() - add(f.displayname) + lines = [] + lines.append(f.displayname) if self.forced_text_signature: - add(self.forced_text_signature) + lines.append(self.forced_text_signature) elif f.kind in {GETTER, SETTER}: # @getter and @setter do not need signatures like a method or a function. return '' else: - add('(') + lines.append('(') # populate "right_bracket_count" field for every parameter assert parameters, "We should always have a self parameter. " + repr(f) @@ -6282,7 +5962,7 @@ def fix_right_bracket_count(desired: int) -> str: first_parameter = True last_p = parameters[-1] - line_length = len(''.join(text)) + line_length = len(''.join(lines)) indent = " " * line_length def add_parameter(text: str) -> None: nonlocal line_length @@ -6293,12 +5973,11 @@ def add_parameter(text: str) -> None: else: s = ' ' + text if line_length + len(s) >= 72: - add('\n') - add(indent) + lines.extend(["\n", indent]) line_length = len(indent) s = text line_length += len(s) - add(s) + lines.append(s) for p in parameters: if not p.converter.show_in_signature: @@ -6321,8 +6000,7 @@ def add_parameter(text: str) -> None: added_star = True add_parameter('*,') - p_add, p_output = text_accumulator() - p_add(fix_right_bracket_count(p.right_bracket_count)) + p_lines = [fix_right_bracket_count(p.right_bracket_count)] if isinstance(p.converter, self_converter): # annotate first parameter as being a "self". @@ -6340,30 +6018,31 @@ def add_parameter(text: str) -> None: # have a docstring.) if this is an __init__ # (or __new__), then this signature is for # calling the class to construct a new instance. - p_add('$') + p_lines.append('$') if p.is_vararg(): - p_add("*") + p_lines.append("*") name = p.converter.signature_name or p.name - p_add(name) + p_lines.append(name) if not p.is_vararg() and p.converter.is_optional(): - p_add('=') + p_lines.append('=') value = p.converter.py_default if not value: value = repr(p.converter.default) - p_add(value) + p_lines.append(value) if (p != last_p) or need_a_trailing_slash: - p_add(',') + p_lines.append(',') - add_parameter(p_output()) + p_output = "".join(p_lines) + add_parameter(p_output) - add(fix_right_bracket_count(0)) + lines.append(fix_right_bracket_count(0)) if need_a_trailing_slash: add_parameter('/') - add(')') + lines.append(')') # PEP 8 says: # @@ -6375,13 +6054,13 @@ def add_parameter(text: str) -> None: # therefore this is commented out: # # if f.return_converter.py_default: - # add(' -> ') - # add(f.return_converter.py_default) + # lines.append(' -> ') + # lines.append(f.return_converter.py_default) if not f.docstring_only: - add("\n" + sig_end_marker + "\n") + lines.append("\n" + libclinic.SIG_END_MARKER + "\n") - signature_line = output() + signature_line = "".join(lines) # now fix up the places where the brackets look wrong return signature_line.replace(', ]', ',] ') @@ -6389,12 +6068,7 @@ def add_parameter(text: str) -> None: @staticmethod def format_docstring_parameters(params: list[Parameter]) -> str: """Create substitution text for {parameters}""" - add, output = text_accumulator() - for p in params: - if p.docstring: - add(p.render_docstring()) - add('\n') - return output() + return "".join(p.render_docstring() + "\n" for p in params if p.docstring) def format_docstring(self) -> str: assert self.function is not None diff --git a/Tools/clinic/libclinic/__init__.py b/Tools/clinic/libclinic/__init__.py new file mode 100644 index 00000000000000..1b300b55acc21e --- /dev/null +++ b/Tools/clinic/libclinic/__init__.py @@ -0,0 +1,64 @@ +from typing import Final + +from .errors import ( + ClinicError, +) +from .formatting import ( + SIG_END_MARKER, + c_repr, + docstring_for_c_string, + format_escape, + indent_all_lines, + normalize_snippet, + pprint_words, + suffix_all_lines, + wrap_declarations, + wrapped_c_string_literal, +) +from .utils import ( + FormatCounterFormatter, + compute_checksum, + create_regex, + write_file, +) + + +__all__ = [ + # Error handling + "ClinicError", + + # Formatting helpers + "SIG_END_MARKER", + "c_repr", + "docstring_for_c_string", + "format_escape", + "indent_all_lines", + "normalize_snippet", + "pprint_words", + "suffix_all_lines", + "wrap_declarations", + "wrapped_c_string_literal", + + # Utility functions + "FormatCounterFormatter", + "compute_checksum", + "create_regex", + "write_file", +] + + +CLINIC_PREFIX: Final = "__clinic_" +CLINIC_PREFIXED_ARGS: Final = frozenset( + { + "_keywords", + "_parser", + "args", + "argsbuf", + "fastargs", + "kwargs", + "kwnames", + "nargs", + "noptargs", + "return_value", + } +) diff --git a/Tools/clinic/cpp.py b/Tools/clinic/libclinic/cpp.py similarity index 90% rename from Tools/clinic/cpp.py rename to Tools/clinic/libclinic/cpp.py index 16eee6fc399491..e115d65a88e1b6 100644 --- a/Tools/clinic/cpp.py +++ b/Tools/clinic/libclinic/cpp.py @@ -3,6 +3,11 @@ import sys from typing import NoReturn +from .errors import ParseError + + +__all__ = ["Monitor"] + TokenAndCondition = tuple[str, str] TokenStack = list[TokenAndCondition] @@ -32,7 +37,7 @@ class Monitor: Anyway this implementation seems to work well enough for the CPython sources. """ - filename: str | None = None + filename: str _: dc.KW_ONLY verbose: bool = False @@ -59,14 +64,8 @@ def condition(self) -> str: """ return " && ".join(condition for token, condition in self.stack) - def fail(self, *a: object) -> NoReturn: - if self.filename: - filename = " " + self.filename - else: - filename = '' - print("Error at" + filename, "line", self.line_number, ":") - print(" ", ' '.join(str(x) for x in a)) - sys.exit(-1) + def fail(self, msg: str) -> NoReturn: + raise ParseError(msg, filename=self.filename, lineno=self.line_number) def writeline(self, line: str) -> None: self.line_number += 1 @@ -74,7 +73,7 @@ def writeline(self, line: str) -> None: def pop_stack() -> TokenAndCondition: if not self.stack: - self.fail("#" + token + " without matching #if / #ifdef / #ifndef!") + self.fail(f"#{token} without matching #if / #ifdef / #ifndef!") return self.stack.pop() if self.continuation: @@ -145,7 +144,7 @@ def pop_stack() -> TokenAndCondition: if token in {'if', 'ifdef', 'ifndef', 'elif'}: if not condition: - self.fail("Invalid format for #" + token + " line: no argument!") + self.fail(f"Invalid format for #{token} line: no argument!") if token in {'if', 'elif'}: if not is_a_simple_defined(condition): condition = "(" + condition + ")" @@ -155,7 +154,8 @@ def pop_stack() -> TokenAndCondition: else: fields = condition.split() if len(fields) != 1: - self.fail("Invalid format for #" + token + " line: should be exactly one argument!") + self.fail(f"Invalid format for #{token} line: " + "should be exactly one argument!") symbol = fields[0] condition = 'defined(' + symbol + ')' if token == 'ifndef': diff --git a/Tools/clinic/libclinic/errors.py b/Tools/clinic/libclinic/errors.py new file mode 100644 index 00000000000000..afb21b02386fe7 --- /dev/null +++ b/Tools/clinic/libclinic/errors.py @@ -0,0 +1,26 @@ +import dataclasses as dc + + +@dc.dataclass +class ClinicError(Exception): + message: str + _: dc.KW_ONLY + lineno: int | None = None + filename: str | None = None + + def __post_init__(self) -> None: + super().__init__(self.message) + + def report(self, *, warn_only: bool = False) -> str: + msg = "Warning" if warn_only else "Error" + if self.filename is not None: + msg += f" in file {self.filename!r}" + if self.lineno is not None: + msg += f" on line {self.lineno}" + msg += ":\n" + msg += f"{self.message}\n" + return msg + + +class ParseError(ClinicError): + pass diff --git a/Tools/clinic/libclinic/formatting.py b/Tools/clinic/libclinic/formatting.py new file mode 100644 index 00000000000000..8b3ad7ba566bc8 --- /dev/null +++ b/Tools/clinic/libclinic/formatting.py @@ -0,0 +1,173 @@ +"""A collection of string formatting helpers.""" + +import functools +import textwrap +from typing import Final + + +SIG_END_MARKER: Final = "--" + + +def docstring_for_c_string(docstring: str) -> str: + lines = [] + # Turn docstring into a properly quoted C string. + for line in docstring.split("\n"): + lines.append('"') + lines.append(_quoted_for_c_string(line)) + lines.append('\\n"\n') + + if lines[-2] == SIG_END_MARKER: + # If we only have a signature, add the blank line that the + # __text_signature__ getter expects to be there. + lines.append('"\\n"') + else: + lines.pop() + lines.append('"') + return "".join(lines) + + +def _quoted_for_c_string(text: str) -> str: + """Helper for docstring_for_c_string().""" + for old, new in ( + ("\\", "\\\\"), # must be first! + ('"', '\\"'), + ("'", "\\'"), + ): + text = text.replace(old, new) + return text + + +def c_repr(text: str) -> str: + return '"' + text + '"' + + +def wrapped_c_string_literal( + text: str, + *, + width: int = 72, + suffix: str = "", + initial_indent: int = 0, + subsequent_indent: int = 4 +) -> str: + wrapped = textwrap.wrap( + text, + width=width, + replace_whitespace=False, + drop_whitespace=False, + break_on_hyphens=False, + ) + separator = c_repr(suffix + "\n" + subsequent_indent * " ") + return initial_indent * " " + c_repr(separator.join(wrapped)) + + +def _add_prefix_and_suffix(text: str, *, prefix: str = "", suffix: str = "") -> str: + """Return 'text' with 'prefix' prepended and 'suffix' appended to all lines. + + If the last line is empty, it remains unchanged. + If text is blank, return text unchanged. + + (textwrap.indent only adds to non-blank lines.) + """ + *split, last = text.split("\n") + lines = [prefix + line + suffix + "\n" for line in split] + if last: + lines.append(prefix + last + suffix) + return "".join(lines) + + +def indent_all_lines(text: str, prefix: str) -> str: + return _add_prefix_and_suffix(text, prefix=prefix) + + +def suffix_all_lines(text: str, suffix: str) -> str: + return _add_prefix_and_suffix(text, suffix=suffix) + + +def pprint_words(items: list[str]) -> str: + if len(items) <= 2: + return " and ".join(items) + return ", ".join(items[:-1]) + " and " + items[-1] + + +def _strip_leading_and_trailing_blank_lines(text: str) -> str: + lines = text.rstrip().split("\n") + while lines: + line = lines[0] + if line.strip(): + break + del lines[0] + return "\n".join(lines) + + +@functools.lru_cache() +def normalize_snippet(text: str, *, indent: int = 0) -> str: + """ + Reformats 'text': + * removes leading and trailing blank lines + * ensures that it does not end with a newline + * dedents so the first nonwhite character on any line is at column "indent" + """ + text = _strip_leading_and_trailing_blank_lines(text) + text = textwrap.dedent(text) + if indent: + text = textwrap.indent(text, " " * indent) + return text + + +def format_escape(text: str) -> str: + # double up curly-braces, this string will be used + # as part of a format_map() template later + text = text.replace("{", "{{") + text = text.replace("}", "}}") + return text + + +def wrap_declarations(text: str, length: int = 78) -> str: + """ + A simple-minded text wrapper for C function declarations. + + It views a declaration line as looking like this: + xxxxxxxx(xxxxxxxxx,xxxxxxxxx) + If called with length=30, it would wrap that line into + xxxxxxxx(xxxxxxxxx, + xxxxxxxxx) + (If the declaration has zero or one parameters, this + function won't wrap it.) + + If this doesn't work properly, it's probably better to + start from scratch with a more sophisticated algorithm, + rather than try and improve/debug this dumb little function. + """ + lines = [] + for line in text.split("\n"): + prefix, _, after_l_paren = line.partition("(") + if not after_l_paren: + lines.append(line) + continue + in_paren, _, after_r_paren = after_l_paren.partition(")") + if not _: + lines.append(line) + continue + if "," not in in_paren: + lines.append(line) + continue + parameters = [x.strip() + ", " for x in in_paren.split(",")] + prefix += "(" + if len(prefix) < length: + spaces = " " * len(prefix) + else: + spaces = " " * 4 + + while parameters: + line = prefix + first = True + while parameters: + if not first and (len(line) + len(parameters[0]) > length): + break + line += parameters.pop(0) + first = False + if not parameters: + line = line.rstrip(", ") + ")" + after_r_paren + lines.append(line.rstrip()) + prefix = spaces + return "\n".join(lines) diff --git a/Tools/clinic/libclinic/utils.py b/Tools/clinic/libclinic/utils.py new file mode 100644 index 00000000000000..d2d09387a73d1e --- /dev/null +++ b/Tools/clinic/libclinic/utils.py @@ -0,0 +1,68 @@ +import collections +import hashlib +import os +import re +import string +from typing import Literal + + +def write_file(filename: str, new_contents: str) -> None: + """Write new content to file, iff the content changed.""" + try: + with open(filename, encoding="utf-8") as fp: + old_contents = fp.read() + + if old_contents == new_contents: + # no change: avoid modifying the file modification time + return + except FileNotFoundError: + pass + # Atomic write using a temporary file and os.replace() + filename_new = f"{filename}.new" + with open(filename_new, "w", encoding="utf-8") as fp: + fp.write(new_contents) + try: + os.replace(filename_new, filename) + except: + os.unlink(filename_new) + raise + + +def compute_checksum(input_: str, length: int | None = None) -> str: + checksum = hashlib.sha1(input_.encode("utf-8")).hexdigest() + if length: + checksum = checksum[:length] + return checksum + + +def create_regex( + before: str, after: str, word: bool = True, whole_line: bool = True +) -> re.Pattern[str]: + """Create a regex object for matching marker lines.""" + group_re = r"\w+" if word else ".+" + before = re.escape(before) + after = re.escape(after) + pattern = rf"{before}({group_re}){after}" + if whole_line: + pattern = rf"^{pattern}$" + return re.compile(pattern) + + +class FormatCounterFormatter(string.Formatter): + """ + This counts how many instances of each formatter + "replacement string" appear in the format string. + + e.g. after evaluating "string {a}, {b}, {c}, {a}" + the counts dict would now look like + {'a': 2, 'b': 1, 'c': 1} + """ + + def __init__(self) -> None: + self.counts = collections.Counter[str]() + + def get_value( + self, key: str, args: object, kwargs: object # type: ignore[override] + ) -> Literal[""]: + self.counts[key] += 1 + return "" diff --git a/Tools/gdb/libpython.py b/Tools/gdb/libpython.py index 5ef55524c11be2..483f28b46dfec7 100755 --- a/Tools/gdb/libpython.py +++ b/Tools/gdb/libpython.py @@ -70,6 +70,14 @@ def _type_unsigned_int_ptr(): def _sizeof_void_p(): return gdb.lookup_type('void').pointer().sizeof +def _managed_dict_offset(): + # See pycore_object.h + pyobj = gdb.lookup_type("PyObject") + if any(field.name == "ob_ref_local" for field in pyobj.fields()): + return -1 * _sizeof_void_p() + else: + return -3 * _sizeof_void_p() + Py_TPFLAGS_MANAGED_DICT = (1 << 4) Py_TPFLAGS_HEAPTYPE = (1 << 9) @@ -457,7 +465,7 @@ def get_attr_dict(self): if dictoffset < 0: if int_from_int(typeobj.field('tp_flags')) & Py_TPFLAGS_MANAGED_DICT: assert dictoffset == -1 - dictoffset = -3 * _sizeof_void_p() + dictoffset = _managed_dict_offset() else: type_PyVarObject_ptr = gdb.lookup_type('PyVarObject').pointer() tsize = int_from_int(self._gdbval.cast(type_PyVarObject_ptr)['ob_size']) @@ -485,9 +493,8 @@ def get_keys_values(self): has_values = int_from_int(typeobj.field('tp_flags')) & Py_TPFLAGS_MANAGED_DICT if not has_values: return None - charptrptr_t = _type_char_ptr().pointer() - ptr = self._gdbval.cast(charptrptr_t) - 3 - char_ptr = ptr.dereference() + ptr = self._gdbval.cast(_type_char_ptr()) + _managed_dict_offset() + char_ptr = ptr.cast(_type_char_ptr().pointer()).dereference() if (int(char_ptr) & 1) == 0: return None char_ptr += 1 diff --git a/Tools/importbench/importbench.py b/Tools/importbench/importbench.py index 0c4b3bc73517c5..eb101fe616c587 100644 --- a/Tools/importbench/importbench.py +++ b/Tools/importbench/importbench.py @@ -165,8 +165,8 @@ def using_bytecode_benchmark(seconds, repeat): def main(import_, options): if options.source_file: - with options.source_file: - prev_results = json.load(options.source_file) + with open(options.source_file, 'r', encoding='utf-8') as source_file: + prev_results = json.load(source_file) else: prev_results = {} __builtins__.__import__ = import_ @@ -218,8 +218,8 @@ def main(import_, options): new_result/old_result) print(benchmark_name, ':', result) if options.dest_file: - with options.dest_file: - json.dump(new_results, options.dest_file, indent=2) + with open(options.dest_file, 'w', encoding='utf-8') as dest_file: + json.dump(new_results, dest_file, indent=2) if __name__ == '__main__': @@ -229,11 +229,9 @@ def main(import_, options): parser.add_argument('-b', '--builtin', dest='builtin', action='store_true', default=False, help="use the built-in __import__") parser.add_argument('-r', '--read', dest='source_file', - type=argparse.FileType('r'), help='file to read benchmark data from to compare ' 'against') parser.add_argument('-w', '--write', dest='dest_file', - type=argparse.FileType('w'), help='file to write benchmark data to') parser.add_argument('--benchmark', dest='benchmark', help='specific benchmark to run') diff --git a/Tools/jit/README.md b/Tools/jit/README.md new file mode 100644 index 00000000000000..04a6c0780bf972 --- /dev/null +++ b/Tools/jit/README.md @@ -0,0 +1,46 @@ +The JIT Compiler +================ + +This version of CPython can be built with an experimental just-in-time compiler. While most everything you already know about building and using CPython is unchanged, you will probably need to install a compatible version of LLVM first. + +## Installing LLVM + +The JIT compiler does not require end users to install any third-party dependencies, but part of it must be *built* using LLVM[^why-llvm]. You are *not* required to build the rest of CPython using LLVM, or even the same version of LLVM (in fact, this is uncommon). + +LLVM version 16 is required. Both `clang` and `llvm-readobj` need to be installed and discoverable (version suffixes, like `clang-16`, are okay). It's highly recommended that you also have `llvm-objdump` available, since this allows the build script to dump human-readable assembly for the generated code. + +It's easy to install all of the required tools: + +### Linux + +Install LLVM 16 on Ubuntu/Debian: + +```sh +wget https://apt.llvm.org/llvm.sh +chmod +x llvm.sh +sudo ./llvm.sh 16 +``` + +### macOS + +Install LLVM 16 with [Homebrew](https://brew.sh): + +```sh +brew install llvm@16 +``` + +Homebrew won't add any of the tools to your `$PATH`. That's okay; the build script knows how to find them. + +### Windows + +Install LLVM 16 [by searching for it on LLVM's GitHub releases page](https://github.com/llvm/llvm-project/releases?q=16), clicking on "Assets", downloading the appropriate Windows installer for your platform (likely the file ending with `-win64.exe`), and running it. **When installing, be sure to select the option labeled "Add LLVM to the system PATH".** + +## Building + +For `PCbuild`-based builds, pass the new `--experimental-jit` option to `build.bat`. + +For all other builds, pass the new `--enable-experimental-jit` option to `configure`. + +Otherwise, just configure and build as you normally would. Cross-compiling "just works", since the JIT is built for the host platform. + +[^why-llvm]: Clang is specifically needed because it's the only C compiler with support for guaranteed tail calls (`musttail`), which are required by CPython's continuation-passing-style approach to JIT compilation. Since LLVM also includes other functionalities we need (namely, object file parsing and disassembly), it's convenient to only support one toolchain at this time. diff --git a/Tools/jit/_llvm.py b/Tools/jit/_llvm.py new file mode 100644 index 00000000000000..603bbef59ba2e6 --- /dev/null +++ b/Tools/jit/_llvm.py @@ -0,0 +1,99 @@ +"""Utilities for invoking LLVM tools.""" +import asyncio +import functools +import os +import re +import shlex +import subprocess +import typing + +_LLVM_VERSION = 16 +_LLVM_VERSION_PATTERN = re.compile(rf"version\s+{_LLVM_VERSION}\.\d+\.\d+\s+") + +_P = typing.ParamSpec("_P") +_R = typing.TypeVar("_R") +_C = typing.Callable[_P, typing.Awaitable[_R]] + + +def _async_cache(f: _C[_P, _R]) -> _C[_P, _R]: + cache = {} + lock = asyncio.Lock() + + @functools.wraps(f) + async def wrapper( + *args: _P.args, **kwargs: _P.kwargs # pylint: disable = no-member + ) -> _R: + async with lock: + if args not in cache: + cache[args] = await f(*args, **kwargs) + return cache[args] + + return wrapper + + +_CORES = asyncio.BoundedSemaphore(os.cpu_count() or 1) + + +async def _run(tool: str, args: typing.Iterable[str], echo: bool = False) -> str | None: + command = [tool, *args] + async with _CORES: + if echo: + print(shlex.join(command)) + try: + process = await asyncio.create_subprocess_exec( + *command, stdout=subprocess.PIPE + ) + except FileNotFoundError: + return None + out, _ = await process.communicate() + if process.returncode: + raise RuntimeError(f"{tool} exited with return code {process.returncode}") + return out.decode() + + +@_async_cache +async def _check_tool_version(name: str, *, echo: bool = False) -> bool: + output = await _run(name, ["--version"], echo=echo) + return bool(output and _LLVM_VERSION_PATTERN.search(output)) + + +@_async_cache +async def _get_brew_llvm_prefix(*, echo: bool = False) -> str | None: + output = await _run("brew", ["--prefix", f"llvm@{_LLVM_VERSION}"], echo=echo) + return output and output.removesuffix("\n") + + +@_async_cache +async def _find_tool(tool: str, *, echo: bool = False) -> str | None: + # Unversioned executables: + path = tool + if await _check_tool_version(path, echo=echo): + return path + # Versioned executables: + path = f"{tool}-{_LLVM_VERSION}" + if await _check_tool_version(path, echo=echo): + return path + # Homebrew-installed executables: + prefix = await _get_brew_llvm_prefix(echo=echo) + if prefix is not None: + path = os.path.join(prefix, "bin", tool) + if await _check_tool_version(path, echo=echo): + return path + # Nothing found: + return None + + +async def maybe_run( + tool: str, args: typing.Iterable[str], echo: bool = False +) -> str | None: + """Run an LLVM tool if it can be found. Otherwise, return None.""" + path = await _find_tool(tool, echo=echo) + return path and await _run(path, args, echo=echo) + + +async def run(tool: str, args: typing.Iterable[str], echo: bool = False) -> str: + """Run an LLVM tool if it can be found. Otherwise, raise RuntimeError.""" + output = await maybe_run(tool, args, echo=echo) + if output is None: + raise RuntimeError(f"Can't find {tool}-{_LLVM_VERSION}!") + return output diff --git a/Tools/jit/_schema.py b/Tools/jit/_schema.py new file mode 100644 index 00000000000000..8eeb78e6cd69ee --- /dev/null +++ b/Tools/jit/_schema.py @@ -0,0 +1,99 @@ +"""Schema for the JSON produced by llvm-readobj --elf-output-style=JSON.""" +import typing + +HoleKind: typing.TypeAlias = typing.Literal[ + "ARM64_RELOC_GOT_LOAD_PAGE21", + "ARM64_RELOC_GOT_LOAD_PAGEOFF12", + "ARM64_RELOC_UNSIGNED", + "IMAGE_REL_AMD64_ADDR64", + "IMAGE_REL_I386_DIR32", + "R_AARCH64_ABS64", + "R_AARCH64_CALL26", + "R_AARCH64_JUMP26", + "R_AARCH64_MOVW_UABS_G0_NC", + "R_AARCH64_MOVW_UABS_G1_NC", + "R_AARCH64_MOVW_UABS_G2_NC", + "R_AARCH64_MOVW_UABS_G3", + "R_X86_64_64", + "X86_64_RELOC_UNSIGNED", +] + + +class COFFRelocation(typing.TypedDict): + """A COFF object file relocation record.""" + + Type: dict[typing.Literal["Value"], HoleKind] + Symbol: str + Offset: int + + +class ELFRelocation(typing.TypedDict): + """An ELF object file relocation record.""" + + Addend: int + Offset: int + Symbol: dict[typing.Literal["Value"], str] + Type: dict[typing.Literal["Value"], HoleKind] + + +class MachORelocation(typing.TypedDict): + """A Mach-O object file relocation record.""" + + Offset: int + Section: typing.NotRequired[dict[typing.Literal["Value"], str]] + Symbol: typing.NotRequired[dict[typing.Literal["Value"], str]] + Type: dict[typing.Literal["Value"], HoleKind] + + +class _COFFSymbol(typing.TypedDict): + Name: str + Value: int + + +class _ELFSymbol(typing.TypedDict): + Name: dict[typing.Literal["Value"], str] + Value: int + + +class _MachOSymbol(typing.TypedDict): + Name: dict[typing.Literal["Value"], str] + Value: int + + +class COFFSection(typing.TypedDict): + """A COFF object file section.""" + + Characteristics: dict[ + typing.Literal["Flags"], list[dict[typing.Literal["Name"], str]] + ] + Number: int + RawDataSize: int + Relocations: list[dict[typing.Literal["Relocation"], COFFRelocation]] + SectionData: typing.NotRequired[dict[typing.Literal["Bytes"], list[int]]] + Symbols: list[dict[typing.Literal["Symbol"], _COFFSymbol]] + + +class ELFSection(typing.TypedDict): + """An ELF object file section.""" + + Flags: dict[typing.Literal["Flags"], list[dict[typing.Literal["Name"], str]]] + Index: int + Info: int + Relocations: list[dict[typing.Literal["Relocation"], ELFRelocation]] + SectionData: dict[typing.Literal["Bytes"], list[int]] + Symbols: list[dict[typing.Literal["Symbol"], _ELFSymbol]] + Type: dict[typing.Literal["Value"], str] + + +class MachOSection(typing.TypedDict): + """A Mach-O object file section.""" + + Address: int + Attributes: dict[typing.Literal["Flags"], list[dict[typing.Literal["Name"], str]]] + Index: int + Name: dict[typing.Literal["Value"], str] + Relocations: typing.NotRequired[ + list[dict[typing.Literal["Relocation"], MachORelocation]] + ] + SectionData: typing.NotRequired[dict[typing.Literal["Bytes"], list[int]]] + Symbols: typing.NotRequired[list[dict[typing.Literal["Symbol"], _MachOSymbol]]] diff --git a/Tools/jit/_stencils.py b/Tools/jit/_stencils.py new file mode 100644 index 00000000000000..71c678e04fbfd5 --- /dev/null +++ b/Tools/jit/_stencils.py @@ -0,0 +1,220 @@ +"""Core data structures for compiled code templates.""" +import dataclasses +import enum +import sys + +import _schema + + +@enum.unique +class HoleValue(enum.Enum): + """ + Different "base" values that can be patched into holes (usually combined with the + address of a symbol and/or an addend). + """ + + # The base address of the machine code for the current uop (exposed as _JIT_ENTRY): + CODE = enum.auto() + # The base address of the machine code for the next uop (exposed as _JIT_CONTINUE): + CONTINUE = enum.auto() + # The base address of the read-only data for this uop: + DATA = enum.auto() + # The address of the current executor (exposed as _JIT_EXECUTOR): + EXECUTOR = enum.auto() + # The base address of the "global" offset table located in the read-only data. + # Shouldn't be present in the final stencils, since these are all replaced with + # equivalent DATA values: + GOT = enum.auto() + # The current uop's oparg (exposed as _JIT_OPARG): + OPARG = enum.auto() + # The current uop's operand (exposed as _JIT_OPERAND): + OPERAND = enum.auto() + # The current uop's target (exposed as _JIT_TARGET): + TARGET = enum.auto() + # The base address of the machine code for the first uop (exposed as _JIT_TOP): + TOP = enum.auto() + # A hardcoded value of zero (used for symbol lookups): + ZERO = enum.auto() + + +@dataclasses.dataclass +class Hole: + """ + A "hole" in the stencil to be patched with a computed runtime value. + + Analogous to relocation records in an object file. + """ + + offset: int + kind: _schema.HoleKind + # Patch with this base value: + value: HoleValue + # ...plus the address of this symbol: + symbol: str | None + # ...plus this addend: + addend: int + # Convenience method: + replace = dataclasses.replace + + def as_c(self) -> str: + """Dump this hole as an initialization of a C Hole struct.""" + parts = [ + f"{self.offset:#x}", + f"HoleKind_{self.kind}", + f"HoleValue_{self.value.name}", + f"&{self.symbol}" if self.symbol else "NULL", + _format_addend(self.addend), + ] + return f"{{{', '.join(parts)}}}" + + +@dataclasses.dataclass +class Stencil: + """ + A contiguous block of machine code or data to be copied-and-patched. + + Analogous to a section or segment in an object file. + """ + + body: bytearray = dataclasses.field(default_factory=bytearray, init=False) + holes: list[Hole] = dataclasses.field(default_factory=list, init=False) + disassembly: list[str] = dataclasses.field(default_factory=list, init=False) + + def pad(self, alignment: int) -> None: + """Pad the stencil to the given alignment.""" + offset = len(self.body) + padding = -offset % alignment + self.disassembly.append(f"{offset:x}: {' '.join(['00'] * padding)}") + self.body.extend([0] * padding) + + def emit_aarch64_trampoline(self, hole: Hole) -> None: + """Even with the large code model, AArch64 Linux insists on 28-bit jumps.""" + base = len(self.body) + where = slice(hole.offset, hole.offset + 4) + instruction = int.from_bytes(self.body[where], sys.byteorder) + instruction &= 0xFC000000 + instruction |= ((base - hole.offset) >> 2) & 0x03FFFFFF + self.body[where] = instruction.to_bytes(4, sys.byteorder) + self.disassembly += [ + f"{base + 4 * 0: x}: d2800008 mov x8, #0x0", + f"{base + 4 * 0:016x}: R_AARCH64_MOVW_UABS_G0_NC {hole.symbol}", + f"{base + 4 * 1:x}: f2a00008 movk x8, #0x0, lsl #16", + f"{base + 4 * 1:016x}: R_AARCH64_MOVW_UABS_G1_NC {hole.symbol}", + f"{base + 4 * 2:x}: f2c00008 movk x8, #0x0, lsl #32", + f"{base + 4 * 2:016x}: R_AARCH64_MOVW_UABS_G2_NC {hole.symbol}", + f"{base + 4 * 3:x}: f2e00008 movk x8, #0x0, lsl #48", + f"{base + 4 * 3:016x}: R_AARCH64_MOVW_UABS_G3 {hole.symbol}", + f"{base + 4 * 4:x}: d61f0100 br x8", + ] + for code in [ + 0xD2800008.to_bytes(4, sys.byteorder), + 0xF2A00008.to_bytes(4, sys.byteorder), + 0xF2C00008.to_bytes(4, sys.byteorder), + 0xF2E00008.to_bytes(4, sys.byteorder), + 0xD61F0100.to_bytes(4, sys.byteorder), + ]: + self.body.extend(code) + for i, kind in enumerate( + [ + "R_AARCH64_MOVW_UABS_G0_NC", + "R_AARCH64_MOVW_UABS_G1_NC", + "R_AARCH64_MOVW_UABS_G2_NC", + "R_AARCH64_MOVW_UABS_G3", + ] + ): + self.holes.append(hole.replace(offset=base + 4 * i, kind=kind)) + + +@dataclasses.dataclass +class StencilGroup: + """ + Code and data corresponding to a given micro-opcode. + + Analogous to an entire object file. + """ + + code: Stencil = dataclasses.field(default_factory=Stencil, init=False) + data: Stencil = dataclasses.field(default_factory=Stencil, init=False) + symbols: dict[int | str, tuple[HoleValue, int]] = dataclasses.field( + default_factory=dict, init=False + ) + _got: dict[str, int] = dataclasses.field(default_factory=dict, init=False) + + def process_relocations(self, *, alignment: int = 1) -> None: + """Fix up all GOT and internal relocations for this stencil group.""" + self.code.pad(alignment) + self.data.pad(8) + for stencil in [self.code, self.data]: + holes = [] + for hole in stencil.holes: + if hole.value is HoleValue.GOT: + assert hole.symbol is not None + hole.value = HoleValue.DATA + hole.addend += self._global_offset_table_lookup(hole.symbol) + hole.symbol = None + elif hole.symbol in self.symbols: + hole.value, addend = self.symbols[hole.symbol] + hole.addend += addend + hole.symbol = None + elif ( + hole.kind in {"R_AARCH64_CALL26", "R_AARCH64_JUMP26"} + and hole.value is HoleValue.ZERO + ): + self.code.emit_aarch64_trampoline(hole) + continue + holes.append(hole) + stencil.holes[:] = holes + self.code.pad(alignment) + self._emit_global_offset_table() + self.code.holes.sort(key=lambda hole: hole.offset) + self.data.holes.sort(key=lambda hole: hole.offset) + + def _global_offset_table_lookup(self, symbol: str) -> int: + return len(self.data.body) + self._got.setdefault(symbol, 8 * len(self._got)) + + def _emit_global_offset_table(self) -> None: + got = len(self.data.body) + for s, offset in self._got.items(): + if s in self.symbols: + value, addend = self.symbols[s] + symbol = None + else: + value, symbol = symbol_to_value(s) + addend = 0 + self.data.holes.append( + Hole(got + offset, "R_X86_64_64", value, symbol, addend) + ) + value_part = value.name if value is not HoleValue.ZERO else "" + if value_part and not symbol and not addend: + addend_part = "" + else: + addend_part = f"&{symbol}" if symbol else "" + addend_part += _format_addend(addend, signed=symbol is not None) + if value_part: + value_part += "+" + self.data.disassembly.append( + f"{len(self.data.body):x}: {value_part}{addend_part}" + ) + self.data.body.extend([0] * 8) + + +def symbol_to_value(symbol: str) -> tuple[HoleValue, str | None]: + """ + Convert a symbol name to a HoleValue and a symbol name. + + Some symbols (starting with "_JIT_") are special and are converted to their + own HoleValues. + """ + if symbol.startswith("_JIT_"): + try: + return HoleValue[symbol.removeprefix("_JIT_")], None + except KeyError: + pass + return HoleValue.ZERO, symbol + + +def _format_addend(addend: int, signed: bool = False) -> str: + addend %= 1 << 64 + if addend & (1 << 63): + addend -= 1 << 64 + return f"{addend:{'+#x' if signed else '#x'}}" diff --git a/Tools/jit/_targets.py b/Tools/jit/_targets.py new file mode 100644 index 00000000000000..51b091eb246413 --- /dev/null +++ b/Tools/jit/_targets.py @@ -0,0 +1,394 @@ +"""Target-specific code generation, parsing, and processing.""" +import asyncio +import dataclasses +import hashlib +import json +import os +import pathlib +import re +import sys +import tempfile +import typing + +import _llvm +import _schema +import _stencils +import _writer + +if sys.version_info < (3, 11): + raise RuntimeError("Building the JIT compiler requires Python 3.11 or newer!") + +TOOLS_JIT_BUILD = pathlib.Path(__file__).resolve() +TOOLS_JIT = TOOLS_JIT_BUILD.parent +TOOLS = TOOLS_JIT.parent +CPYTHON = TOOLS.parent +PYTHON_EXECUTOR_CASES_C_H = CPYTHON / "Python" / "executor_cases.c.h" +TOOLS_JIT_TEMPLATE_C = TOOLS_JIT / "template.c" + + +_S = typing.TypeVar("_S", _schema.COFFSection, _schema.ELFSection, _schema.MachOSection) +_R = typing.TypeVar( + "_R", _schema.COFFRelocation, _schema.ELFRelocation, _schema.MachORelocation +) + + +@dataclasses.dataclass +class _Target(typing.Generic[_S, _R]): + triple: str + _: dataclasses.KW_ONLY + alignment: int = 1 + prefix: str = "" + debug: bool = False + force: bool = False + verbose: bool = False + + def _compute_digest(self, out: pathlib.Path) -> str: + hasher = hashlib.sha256() + hasher.update(self.triple.encode()) + hasher.update(self.alignment.to_bytes()) + hasher.update(self.prefix.encode()) + # These dependencies are also reflected in _JITSources in regen.targets: + hasher.update(PYTHON_EXECUTOR_CASES_C_H.read_bytes()) + hasher.update((out / "pyconfig.h").read_bytes()) + for dirpath, _, filenames in sorted(os.walk(TOOLS_JIT)): + for filename in filenames: + hasher.update(pathlib.Path(dirpath, filename).read_bytes()) + return hasher.hexdigest() + + async def _parse(self, path: pathlib.Path) -> _stencils.StencilGroup: + group = _stencils.StencilGroup() + args = ["--disassemble", "--reloc", f"{path}"] + output = await _llvm.maybe_run("llvm-objdump", args, echo=self.verbose) + if output is not None: + group.code.disassembly.extend( + line.expandtabs().strip() + for line in output.splitlines() + if not line.isspace() + ) + args = [ + "--elf-output-style=JSON", + "--expand-relocs", + # "--pretty-print", + "--section-data", + "--section-relocations", + "--section-symbols", + "--sections", + f"{path}", + ] + output = await _llvm.run("llvm-readobj", args, echo=self.verbose) + # --elf-output-style=JSON is only *slightly* broken on Mach-O... + output = output.replace("PrivateExtern\n", "\n") + output = output.replace("Extern\n", "\n") + # ...and also COFF: + output = output[output.index("[", 1, None) :] + output = output[: output.rindex("]", None, -1) + 1] + sections: list[dict[typing.Literal["Section"], _S]] = json.loads(output) + for wrapped_section in sections: + self._handle_section(wrapped_section["Section"], group) + assert group.symbols["_JIT_ENTRY"] == (_stencils.HoleValue.CODE, 0) + if group.data.body: + line = f"0: {str(bytes(group.data.body)).removeprefix('b')}" + group.data.disassembly.append(line) + group.process_relocations() + return group + + def _handle_section(self, section: _S, group: _stencils.StencilGroup) -> None: + raise NotImplementedError(type(self)) + + def _handle_relocation( + self, base: int, relocation: _R, raw: bytes + ) -> _stencils.Hole: + raise NotImplementedError(type(self)) + + async def _compile( + self, opname: str, c: pathlib.Path, tempdir: pathlib.Path + ) -> _stencils.StencilGroup: + o = tempdir / f"{opname}.o" + args = [ + f"--target={self.triple}", + "-DPy_BUILD_CORE", + "-D_DEBUG" if self.debug else "-DNDEBUG", + f"-D_JIT_OPCODE={opname}", + "-D_PyJIT_ACTIVE", + "-D_Py_JIT", + "-I.", + f"-I{CPYTHON / 'Include'}", + f"-I{CPYTHON / 'Include' / 'internal'}", + f"-I{CPYTHON / 'Include' / 'internal' / 'mimalloc'}", + f"-I{CPYTHON / 'Python'}", + "-O3", + "-c", + "-fno-asynchronous-unwind-tables", + # SET_FUNCTION_ATTRIBUTE on 32-bit Windows debug builds: + "-fno-jump-tables", + # Position-independent code adds indirection to every load and jump: + "-fno-pic", + # Don't make calls to weird stack-smashing canaries: + "-fno-stack-protector", + # We have three options for code model: + # - "small": the default, assumes that code and data reside in the + # lowest 2GB of memory (128MB on aarch64) + # - "medium": assumes that code resides in the lowest 2GB of memory, + # and makes no assumptions about data (not available on aarch64) + # - "large": makes no assumptions about either code or data + "-mcmodel=large", + "-o", + f"{o}", + "-std=c11", + f"{c}", + ] + await _llvm.run("clang", args, echo=self.verbose) + return await self._parse(o) + + async def _build_stencils(self) -> dict[str, _stencils.StencilGroup]: + generated_cases = PYTHON_EXECUTOR_CASES_C_H.read_text() + opnames = sorted(re.findall(r"\n {8}case (\w+): \{\n", generated_cases)) + tasks = [] + with tempfile.TemporaryDirectory() as tempdir: + work = pathlib.Path(tempdir).resolve() + async with asyncio.TaskGroup() as group: + for opname in opnames: + coro = self._compile(opname, TOOLS_JIT_TEMPLATE_C, work) + tasks.append(group.create_task(coro, name=opname)) + return {task.get_name(): task.result() for task in tasks} + + def build(self, out: pathlib.Path, *, comment: str = "") -> None: + """Build jit_stencils.h in the given directory.""" + digest = f"// {self._compute_digest(out)}\n" + jit_stencils = out / "jit_stencils.h" + if ( + not self.force + and jit_stencils.exists() + and jit_stencils.read_text().startswith(digest) + ): + return + stencil_groups = asyncio.run(self._build_stencils()) + with jit_stencils.open("w") as file: + file.write(digest) + if comment: + file.write(f"// {comment}\n") + file.write("") + for line in _writer.dump(stencil_groups): + file.write(f"{line}\n") + + +class _COFF( + _Target[_schema.COFFSection, _schema.COFFRelocation] +): # pylint: disable = too-few-public-methods + def _handle_section( + self, section: _schema.COFFSection, group: _stencils.StencilGroup + ) -> None: + flags = {flag["Name"] for flag in section["Characteristics"]["Flags"]} + if "SectionData" in section: + section_data_bytes = section["SectionData"]["Bytes"] + else: + # Zeroed BSS data, seen with printf debugging calls: + section_data_bytes = [0] * section["RawDataSize"] + if "IMAGE_SCN_MEM_EXECUTE" in flags: + value = _stencils.HoleValue.CODE + stencil = group.code + elif "IMAGE_SCN_MEM_READ" in flags: + value = _stencils.HoleValue.DATA + stencil = group.data + else: + return + base = len(stencil.body) + group.symbols[section["Number"]] = value, base + stencil.body.extend(section_data_bytes) + for wrapped_symbol in section["Symbols"]: + symbol = wrapped_symbol["Symbol"] + offset = base + symbol["Value"] + name = symbol["Name"] + name = name.removeprefix(self.prefix) + group.symbols[name] = value, offset + for wrapped_relocation in section["Relocations"]: + relocation = wrapped_relocation["Relocation"] + hole = self._handle_relocation(base, relocation, stencil.body) + stencil.holes.append(hole) + + def _handle_relocation( + self, base: int, relocation: _schema.COFFRelocation, raw: bytes + ) -> _stencils.Hole: + match relocation: + case { + "Offset": offset, + "Symbol": s, + "Type": {"Value": "IMAGE_REL_AMD64_ADDR64" as kind}, + }: + offset += base + s = s.removeprefix(self.prefix) + value, symbol = _stencils.symbol_to_value(s) + addend = int.from_bytes(raw[offset : offset + 8], "little") + case { + "Offset": offset, + "Symbol": s, + "Type": {"Value": "IMAGE_REL_I386_DIR32" as kind}, + }: + offset += base + s = s.removeprefix(self.prefix) + value, symbol = _stencils.symbol_to_value(s) + addend = int.from_bytes(raw[offset : offset + 4], "little") + case _: + raise NotImplementedError(relocation) + return _stencils.Hole(offset, kind, value, symbol, addend) + + +class _ELF( + _Target[_schema.ELFSection, _schema.ELFRelocation] +): # pylint: disable = too-few-public-methods + def _handle_section( + self, section: _schema.ELFSection, group: _stencils.StencilGroup + ) -> None: + section_type = section["Type"]["Value"] + flags = {flag["Name"] for flag in section["Flags"]["Flags"]} + if section_type == "SHT_RELA": + assert "SHF_INFO_LINK" in flags, flags + assert not section["Symbols"] + value, base = group.symbols[section["Info"]] + if value is _stencils.HoleValue.CODE: + stencil = group.code + else: + assert value is _stencils.HoleValue.DATA + stencil = group.data + for wrapped_relocation in section["Relocations"]: + relocation = wrapped_relocation["Relocation"] + hole = self._handle_relocation(base, relocation, stencil.body) + stencil.holes.append(hole) + elif section_type == "SHT_PROGBITS": + if "SHF_ALLOC" not in flags: + return + if "SHF_EXECINSTR" in flags: + value = _stencils.HoleValue.CODE + stencil = group.code + else: + value = _stencils.HoleValue.DATA + stencil = group.data + group.symbols[section["Index"]] = value, len(stencil.body) + for wrapped_symbol in section["Symbols"]: + symbol = wrapped_symbol["Symbol"] + offset = len(stencil.body) + symbol["Value"] + name = symbol["Name"]["Value"] + name = name.removeprefix(self.prefix) + group.symbols[name] = value, offset + stencil.body.extend(section["SectionData"]["Bytes"]) + assert not section["Relocations"] + else: + assert section_type in { + "SHT_GROUP", + "SHT_LLVM_ADDRSIG", + "SHT_NULL", + "SHT_STRTAB", + "SHT_SYMTAB", + }, section_type + + def _handle_relocation( + self, base: int, relocation: _schema.ELFRelocation, raw: bytes + ) -> _stencils.Hole: + match relocation: + case { + "Addend": addend, + "Offset": offset, + "Symbol": {"Value": s}, + "Type": {"Value": kind}, + }: + offset += base + s = s.removeprefix(self.prefix) + value, symbol = _stencils.symbol_to_value(s) + case _: + raise NotImplementedError(relocation) + return _stencils.Hole(offset, kind, value, symbol, addend) + + +class _MachO( + _Target[_schema.MachOSection, _schema.MachORelocation] +): # pylint: disable = too-few-public-methods + def _handle_section( + self, section: _schema.MachOSection, group: _stencils.StencilGroup + ) -> None: + assert section["Address"] >= len(group.code.body) + assert "SectionData" in section + flags = {flag["Name"] for flag in section["Attributes"]["Flags"]} + name = section["Name"]["Value"] + name = name.removeprefix(self.prefix) + if "SomeInstructions" in flags: + value = _stencils.HoleValue.CODE + stencil = group.code + start_address = 0 + group.symbols[name] = value, section["Address"] - start_address + else: + value = _stencils.HoleValue.DATA + stencil = group.data + start_address = len(group.code.body) + group.symbols[name] = value, len(group.code.body) + base = section["Address"] - start_address + group.symbols[section["Index"]] = value, base + stencil.body.extend( + [0] * (section["Address"] - len(group.code.body) - len(group.data.body)) + ) + stencil.body.extend(section["SectionData"]["Bytes"]) + assert "Symbols" in section + for wrapped_symbol in section["Symbols"]: + symbol = wrapped_symbol["Symbol"] + offset = symbol["Value"] - start_address + name = symbol["Name"]["Value"] + name = name.removeprefix(self.prefix) + group.symbols[name] = value, offset + assert "Relocations" in section + for wrapped_relocation in section["Relocations"]: + relocation = wrapped_relocation["Relocation"] + hole = self._handle_relocation(base, relocation, stencil.body) + stencil.holes.append(hole) + + def _handle_relocation( + self, base: int, relocation: _schema.MachORelocation, raw: bytes + ) -> _stencils.Hole: + symbol: str | None + match relocation: + case { + "Offset": offset, + "Symbol": {"Value": s}, + "Type": { + "Value": "ARM64_RELOC_GOT_LOAD_PAGE21" + | "ARM64_RELOC_GOT_LOAD_PAGEOFF12" as kind + }, + }: + offset += base + s = s.removeprefix(self.prefix) + value, symbol = _stencils.HoleValue.GOT, s + addend = 0 + case { + "Offset": offset, + "Section": {"Value": s}, + "Type": {"Value": kind}, + } | { + "Offset": offset, + "Symbol": {"Value": s}, + "Type": {"Value": kind}, + }: + offset += base + s = s.removeprefix(self.prefix) + value, symbol = _stencils.symbol_to_value(s) + addend = 0 + case _: + raise NotImplementedError(relocation) + # Turn Clang's weird __bzero calls into normal bzero calls: + if symbol == "__bzero": + symbol = "bzero" + return _stencils.Hole(offset, kind, value, symbol, addend) + + +def get_target(host: str) -> _COFF | _ELF | _MachO: + """Build a _Target for the given host "triple" and options.""" + if re.fullmatch(r"aarch64-apple-darwin.*", host): + return _MachO(host, alignment=8, prefix="_") + if re.fullmatch(r"aarch64-.*-linux-gnu", host): + return _ELF(host, alignment=8) + if re.fullmatch(r"i686-pc-windows-msvc", host): + return _COFF(host, prefix="_") + if re.fullmatch(r"x86_64-apple-darwin.*", host): + return _MachO(host, prefix="_") + if re.fullmatch(r"x86_64-pc-windows-msvc", host): + return _COFF(host) + if re.fullmatch(r"x86_64-.*-linux-gnu", host): + return _ELF(host) + raise ValueError(host) diff --git a/Tools/jit/_writer.py b/Tools/jit/_writer.py new file mode 100644 index 00000000000000..8a2a42e75cfb9b --- /dev/null +++ b/Tools/jit/_writer.py @@ -0,0 +1,95 @@ +"""Utilities for writing StencilGroups out to a C header file.""" +import typing + +import _schema +import _stencils + + +def _dump_header() -> typing.Iterator[str]: + yield "typedef enum {" + for kind in typing.get_args(_schema.HoleKind): + yield f" HoleKind_{kind}," + yield "} HoleKind;" + yield "" + yield "typedef enum {" + for value in _stencils.HoleValue: + yield f" HoleValue_{value.name}," + yield "} HoleValue;" + yield "" + yield "typedef struct {" + yield " const uint64_t offset;" + yield " const HoleKind kind;" + yield " const HoleValue value;" + yield " const void *symbol;" + yield " const uint64_t addend;" + yield "} Hole;" + yield "" + yield "typedef struct {" + yield " const size_t body_size;" + yield " const unsigned char * const body;" + yield " const size_t holes_size;" + yield " const Hole * const holes;" + yield "} Stencil;" + yield "" + yield "typedef struct {" + yield " const Stencil code;" + yield " const Stencil data;" + yield "} StencilGroup;" + yield "" + + +def _dump_footer(opnames: typing.Iterable[str]) -> typing.Iterator[str]: + yield "#define INIT_STENCIL(STENCIL) { \\" + yield " .body_size = Py_ARRAY_LENGTH(STENCIL##_body) - 1, \\" + yield " .body = STENCIL##_body, \\" + yield " .holes_size = Py_ARRAY_LENGTH(STENCIL##_holes) - 1, \\" + yield " .holes = STENCIL##_holes, \\" + yield "}" + yield "" + yield "#define INIT_STENCIL_GROUP(OP) { \\" + yield " .code = INIT_STENCIL(OP##_code), \\" + yield " .data = INIT_STENCIL(OP##_data), \\" + yield "}" + yield "" + yield "static const StencilGroup stencil_groups[512] = {" + for opname in opnames: + yield f" [{opname}] = INIT_STENCIL_GROUP({opname})," + yield "};" + yield "" + yield "#define GET_PATCHES() { \\" + for value in _stencils.HoleValue: + yield f" [HoleValue_{value.name}] = (uint64_t)0xBADBADBADBADBADB, \\" + yield "}" + + +def _dump_stencil(opname: str, group: _stencils.StencilGroup) -> typing.Iterator[str]: + yield f"// {opname}" + for part, stencil in [("code", group.code), ("data", group.data)]: + for line in stencil.disassembly: + yield f"// {line}" + if stencil.body: + size = len(stencil.body) + 1 + yield f"static const unsigned char {opname}_{part}_body[{size}] = {{" + for i in range(0, len(stencil.body), 8): + row = " ".join(f"{byte:#04x}," for byte in stencil.body[i : i + 8]) + yield f" {row}" + yield "};" + else: + yield f"static const unsigned char {opname}_{part}_body[1];" + if stencil.holes: + size = len(stencil.holes) + 1 + yield f"static const Hole {opname}_{part}_holes[{size}] = {{" + for hole in stencil.holes: + yield f" {hole.as_c()}," + yield "};" + else: + yield f"static const Hole {opname}_{part}_holes[1];" + yield "" + + +def dump(groups: dict[str, _stencils.StencilGroup]) -> typing.Iterator[str]: + """Yield a JIT compiler line-by-line as a C header file.""" + yield from _dump_header() + for opname, group in groups.items(): + yield from _dump_stencil(opname, group) + yield from _dump_footer(groups) diff --git a/Tools/jit/build.py b/Tools/jit/build.py new file mode 100644 index 00000000000000..4d4ace14ebf26c --- /dev/null +++ b/Tools/jit/build.py @@ -0,0 +1,28 @@ +"""Build an experimental just-in-time compiler for CPython.""" +import argparse +import pathlib +import shlex +import sys + +import _targets + +if __name__ == "__main__": + comment = f"$ {shlex.join([sys.executable] + sys.argv)}" + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument( + "target", type=_targets.get_target, help="a PEP 11 target triple to compile for" + ) + parser.add_argument( + "-d", "--debug", action="store_true", help="compile for a debug build of Python" + ) + parser.add_argument( + "-f", "--force", action="store_true", help="force the entire JIT to be rebuilt" + ) + parser.add_argument( + "-v", "--verbose", action="store_true", help="echo commands as they are run" + ) + args = parser.parse_args() + args.target.debug = args.debug + args.target.force = args.force + args.target.verbose = args.verbose + args.target.build(pathlib.Path.cwd(), comment=comment) diff --git a/Tools/jit/mypy.ini b/Tools/jit/mypy.ini new file mode 100644 index 00000000000000..768d0028516abd --- /dev/null +++ b/Tools/jit/mypy.ini @@ -0,0 +1,5 @@ +[mypy] +files = Tools/jit +pretty = True +python_version = 3.11 +strict = True diff --git a/Tools/jit/template.c b/Tools/jit/template.c new file mode 100644 index 00000000000000..12303a550d8879 --- /dev/null +++ b/Tools/jit/template.c @@ -0,0 +1,98 @@ +#include "Python.h" + +#include "pycore_call.h" +#include "pycore_ceval.h" +#include "pycore_dict.h" +#include "pycore_emscripten_signal.h" +#include "pycore_intrinsics.h" +#include "pycore_jit.h" +#include "pycore_long.h" +#include "pycore_opcode_metadata.h" +#include "pycore_opcode_utils.h" +#include "pycore_range.h" +#include "pycore_setobject.h" +#include "pycore_sliceobject.h" + +#include "ceval_macros.h" + +#undef CURRENT_OPARG +#define CURRENT_OPARG() (_oparg) + +#undef CURRENT_OPERAND +#define CURRENT_OPERAND() (_operand) + +#undef DEOPT_IF +#define DEOPT_IF(COND, INSTNAME) \ + do { \ + if ((COND)) { \ + goto deoptimize; \ + } \ + } while (0) + +#undef ENABLE_SPECIALIZATION +#define ENABLE_SPECIALIZATION (0) + +#undef GOTO_ERROR +#define GOTO_ERROR(LABEL) \ + do { \ + goto LABEL ## _tier_two; \ + } while (0) + +#undef LOAD_IP +#define LOAD_IP(UNUSED) \ + do { \ + } while (0) + +#define PATCH_VALUE(TYPE, NAME, ALIAS) \ + extern void ALIAS; \ + TYPE NAME = (TYPE)(uint64_t)&ALIAS; + +#define PATCH_JUMP(ALIAS) \ + extern void ALIAS; \ + __attribute__((musttail)) \ + return ((jit_func)&ALIAS)(frame, stack_pointer, tstate); + +_Py_CODEUNIT * +_JIT_ENTRY(_PyInterpreterFrame *frame, PyObject **stack_pointer, PyThreadState *tstate) +{ + // Locals that the instruction implementations expect to exist: + PATCH_VALUE(_PyExecutorObject *, current_executor, _JIT_EXECUTOR) + int oparg; + int opcode = _JIT_OPCODE; + _PyUOpInstruction *next_uop; + // Other stuff we need handy: + PATCH_VALUE(uint16_t, _oparg, _JIT_OPARG) + PATCH_VALUE(uint64_t, _operand, _JIT_OPERAND) + PATCH_VALUE(uint32_t, _target, _JIT_TARGET) + // The actual instruction definitions (only one will be used): + if (opcode == _JUMP_TO_TOP) { + CHECK_EVAL_BREAKER(); + PATCH_JUMP(_JIT_TOP); + } + switch (opcode) { +#include "executor_cases.c.h" + default: + Py_UNREACHABLE(); + } + PATCH_JUMP(_JIT_CONTINUE); + // Labels that the instruction implementations expect to exist: +unbound_local_error_tier_two: + _PyEval_FormatExcCheckArg( + tstate, PyExc_UnboundLocalError, UNBOUNDLOCAL_ERROR_MSG, + PyTuple_GetItem(_PyFrame_GetCode(frame)->co_localsplusnames, oparg)); + goto error_tier_two; +pop_4_error_tier_two: + STACK_SHRINK(1); +pop_3_error_tier_two: + STACK_SHRINK(1); +pop_2_error_tier_two: + STACK_SHRINK(1); +pop_1_error_tier_two: + STACK_SHRINK(1); +error_tier_two: + _PyFrame_SetStackPointer(frame, stack_pointer); + return NULL; +deoptimize: + _PyFrame_SetStackPointer(frame, stack_pointer); + return _PyCode_CODE(_PyFrame_GetCode(frame)) + _target; +} diff --git a/Tools/msi/build.bat b/Tools/msi/build.bat index b9aab887c4939b..2fe8a475e7e3a3 100644 --- a/Tools/msi/build.bat +++ b/Tools/msi/build.bat @@ -22,6 +22,9 @@ if "%~1" EQU "--no-test-marker" (set BUILDTEST=) && shift && goto CheckOpts if "%~1" EQU "--test-marker" (set BUILDTEST=--test-marker) && shift && goto CheckOpts if "%~1" EQU "--pack" (set BUILDPACK=1) && shift && goto CheckOpts if "%~1" EQU "-r" (set REBUILD=-r) && shift && goto CheckOpts +rem %IncludeFreethreaded% is recognised by the MSI build, but not the regular build. +rem We use it to build twice and then build the installer with its extra option +if /I "%~1" EQU "--disable-gil" (set IncludeFreethreaded=true) && shift && goto CheckOpts if not defined BUILDX86 if not defined BUILDX64 if not defined BUILDARM64 (set BUILDX86=1) && (set BUILDX64=1) @@ -44,6 +47,20 @@ if errorlevel 1 exit /B %ERRORLEVEL% if defined BUILDARM64 call "%PCBUILD%build.bat" -p ARM64 -e %REBUILD% %BUILDTEST% if errorlevel 1 exit /B %ERRORLEVEL% +if /I "%IncludeFreethreaded%"=="true" ( + rem Cannot "exit /B" inside an if block because %ERRORLEVEL% will be wrong. + rem We just skip everything after the first "errorlevel 1" and then exit after + if defined BUILDX86 call "%PCBUILD%build.bat" -p Win32 -d -e %REBUILD% %BUILDTEST% --disable-gil + if not errorlevel 1 if defined BUILDX86 call "%PCBUILD%build.bat" -p Win32 -e %REBUILD% %BUILDTEST% --disable-gil + + if not errorlevel 1 if defined BUILDX64 call "%PCBUILD%build.bat" -p x64 -d -e %REBUILD% %BUILDTEST% --disable-gil + if not errorlevel 1 if defined BUILDX64 call "%PCBUILD%build.bat" -p x64 -e %REBUILD% %BUILDTEST% --disable-gil + + if not errorlevel 1 if defined BUILDARM64 call "%PCBUILD%build.bat" -p ARM64 -d -e %REBUILD% %BUILDTEST% --disable-gil + if not errorlevel 1 if defined BUILDARM64 call "%PCBUILD%build.bat" -p ARM64 -e %REBUILD% %BUILDTEST% --disable-gil +) +if errorlevel 1 exit /B %ERRORLEVEL% + if defined BUILDDOC call "%PCBUILD%..\Doc\make.bat" html if errorlevel 1 exit /B %ERRORLEVEL% diff --git a/Tools/msi/bundle/Default.thm b/Tools/msi/bundle/Default.thm index d1b0f5bd9c1d4a..471d37acc33b3d 100644 --- a/Tools/msi/bundle/Default.thm +++ b/Tools/msi/bundle/Default.thm @@ -83,6 +83,7 @@ #(loc.PrecompileLabel) #(loc.Include_symbolsLabel) #(loc.Include_debugLabel) + #(loc.Include_freethreadedLabel) #(loc.CustomLocationLabel) diff --git a/Tools/msi/bundle/Default.wxl b/Tools/msi/bundle/Default.wxl index 6f8befba3a2523..1540f050159a54 100644 --- a/Tools/msi/bundle/Default.wxl +++ b/Tools/msi/bundle/Default.wxl @@ -91,6 +91,7 @@ Select Customize to review current options. &Precompile standard library Download debugging &symbols Download debu&g binaries (requires VS 2017 or later) + Download &free-threaded binaries (experimental) [ActionLikeInstallation] Progress [ActionLikeInstalling]: diff --git a/Tools/msi/bundle/bundle.targets b/Tools/msi/bundle/bundle.targets index 9c7410fe514d19..cb3effb4434843 100644 --- a/Tools/msi/bundle/bundle.targets +++ b/Tools/msi/bundle/bundle.targets @@ -72,6 +72,7 @@ + diff --git a/Tools/msi/bundle/bundle.wxs b/Tools/msi/bundle/bundle.wxs index 8b12baae31105e..9b4f072152d5c0 100644 --- a/Tools/msi/bundle/bundle.wxs +++ b/Tools/msi/bundle/bundle.wxs @@ -82,6 +82,13 @@ + + + + + + + @@ -104,6 +111,9 @@ + + + diff --git a/Tools/msi/bundle/packagegroups/freethreaded.wxs b/Tools/msi/bundle/packagegroups/freethreaded.wxs new file mode 100644 index 00000000000000..121ca34ab66157 --- /dev/null +++ b/Tools/msi/bundle/packagegroups/freethreaded.wxs @@ -0,0 +1,62 @@ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + \ No newline at end of file diff --git a/Tools/msi/freethreaded/freethreaded.wixproj b/Tools/msi/freethreaded/freethreaded.wixproj new file mode 100644 index 00000000000000..0b4bd055d77977 --- /dev/null +++ b/Tools/msi/freethreaded/freethreaded.wixproj @@ -0,0 +1,20 @@ + + + + {1B4502D5-B627-4F50-ABEA-4CC5A8E88265} + 2.0 + freethreaded + Package + + + + + + + + + + + + + \ No newline at end of file diff --git a/Tools/msi/freethreaded/freethreaded.wxs b/Tools/msi/freethreaded/freethreaded.wxs new file mode 100644 index 00000000000000..063aa28bf09fce --- /dev/null +++ b/Tools/msi/freethreaded/freethreaded.wxs @@ -0,0 +1,18 @@ + + + + + + + + + + + + + + + + + + diff --git a/Tools/msi/freethreaded/freethreaded_d.wixproj b/Tools/msi/freethreaded/freethreaded_d.wixproj new file mode 100644 index 00000000000000..e1563d4f907126 --- /dev/null +++ b/Tools/msi/freethreaded/freethreaded_d.wixproj @@ -0,0 +1,20 @@ + + + + {D3677DCF-098A-4398-9FA5-8E74AC37E0DF} + 2.0 + freethreaded_d + Package + + + + + + + + + + + + + \ No newline at end of file diff --git a/Tools/msi/freethreaded/freethreaded_d.wxs b/Tools/msi/freethreaded/freethreaded_d.wxs new file mode 100644 index 00000000000000..cddf22a6c803d3 --- /dev/null +++ b/Tools/msi/freethreaded/freethreaded_d.wxs @@ -0,0 +1,16 @@ + + + + + + + + + + + + + + + + diff --git a/Tools/msi/freethreaded/freethreaded_en-US.wxl_template b/Tools/msi/freethreaded/freethreaded_en-US.wxl_template new file mode 100644 index 00000000000000..b9747eb256d24b --- /dev/null +++ b/Tools/msi/freethreaded/freethreaded_en-US.wxl_template @@ -0,0 +1,8 @@ + + + Freethreaded Interpreter + freethreaded + Python {{ShortVersion}} ({{Bitness}}, freethreaded) + Launches the !(loc.ProductName) freethreaded interpreter. + https://www.python.org/ + diff --git a/Tools/msi/freethreaded/freethreaded_files.wxs b/Tools/msi/freethreaded/freethreaded_files.wxs new file mode 100644 index 00000000000000..adaf63c69d5ade --- /dev/null +++ b/Tools/msi/freethreaded/freethreaded_files.wxs @@ -0,0 +1,175 @@ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + diff --git a/Tools/msi/freethreaded/freethreaded_pdb.wixproj b/Tools/msi/freethreaded/freethreaded_pdb.wixproj new file mode 100644 index 00000000000000..789a4f55ae5191 --- /dev/null +++ b/Tools/msi/freethreaded/freethreaded_pdb.wixproj @@ -0,0 +1,20 @@ + + + + {E98E7539-64E7-4DCE-AACD-01E3ADE40EFD} + 2.0 + freethreaded_pdb + Package + + + + + + + + + + + + + \ No newline at end of file diff --git a/Tools/msi/freethreaded/freethreaded_pdb.wxs b/Tools/msi/freethreaded/freethreaded_pdb.wxs new file mode 100644 index 00000000000000..302ac416fe9275 --- /dev/null +++ b/Tools/msi/freethreaded/freethreaded_pdb.wxs @@ -0,0 +1,15 @@ + + + + + + + + + + + + + + + diff --git a/Tools/msi/lib/lib_files.wxs b/Tools/msi/lib/lib_files.wxs index a82cad596d47a6..b8e16b5fe238a0 100644 --- a/Tools/msi/lib/lib_files.wxs +++ b/Tools/msi/lib/lib_files.wxs @@ -26,10 +26,10 @@ - + - + @@ -63,10 +63,10 @@ - + - + @@ -100,10 +100,10 @@ - + - + diff --git a/Tools/msi/msi.props b/Tools/msi/msi.props index cfb3ca9e76e24c..372c4823bce07f 100644 --- a/Tools/msi/msi.props +++ b/Tools/msi/msi.props @@ -24,14 +24,14 @@ This URI is used to generate the various GUIDs used by the installer. Installers built with the same URI will upgrade each other or block when attempting to downgrade. - + By default, this is the local computer name, which will produce installers that do not interfere with other installers. Products that intend to bundle Python should rebuild these modules with their own URI to avoid conflicting with the official releases. - + The official releases use "https://www.python.org/$(ArchName)" - + This is not the same as the DownloadUrl property used in the bundle projects. --> @@ -39,7 +39,7 @@ $(ReleaseUri)/ - + @@ -63,13 +63,17 @@ $(MajorVersionNumber).$(MinorVersionNumber).$(Field3Value).0 - + + + false + + $([System.Math]::Floor($([System.DateTime]::Now.Subtract($([System.DateTime]::new(2001, 1, 1))).TotalDays))) $(MajorVersionNumber).$(MinorVersionNumber).$(MicroVersionNumber)dev$(RevisionNumber) $(MajorVersionNumber).$(MinorVersionNumber).$(RevisionNumber).0 - + 32-bit 64-bit @@ -91,9 +95,12 @@ PyDebugExt=$(PyDebugExt); PyArchExt=$(PyArchExt); PyTestExt=$(PyTestExt); + PydTag=$(PydTag); + FreethreadedPydTag=$(FreethreadedPydTag); OptionalFeatureName=$(OutputName); ssltag=$(OpenSSLDLLSuffix); Suffix32=$(PyArchExt); + IncludeFreethreaded=$(IncludeFreethreaded); $(DefineConstants);CRTRedist=$(CRTRedist); @@ -139,7 +146,7 @@ - + <_Uuid Include="CoreUpgradeCode"> upgradecode @@ -162,6 +169,12 @@ <_Uuid Include="PythonRegComponentGuid"> registry/$(OutputName) + <_Uuid Include="FreethreadedPythonExeComponentGuid" Condition="$(IncludeFreethreaded)"> + freethreaded/python.exe + + <_Uuid Include="FreethreadedPythonwExeComponentGuid" Condition="$(IncludeFreethreaded)"> + freethreaded/pythonw.exe + @(_Uuid->'("%(Identity)", "$(MajorVersionNumber).$(MinorVersionNumber)/%(Uri)")',',') <_GenerateCommand>import uuid; print('\n'.join('{}={}'.format(i, uuid.uuid5(uuid.UUID('c8d9733e-a70c-43ff-ab0c-e26456f11083'), '$(ReleaseUri.Replace(`{arch}`, `$(ArchName)`))' + j)) for i,j in [$(_Uuids.Replace(`"`,`'`))])) - + - + - + $(DefineConstants);@(_UuidValue,';'); diff --git a/Tools/peg_generator/pegen/build.py b/Tools/peg_generator/pegen/build.py index 7df39a3b0ae48e..67a7c0c4788e9d 100644 --- a/Tools/peg_generator/pegen/build.py +++ b/Tools/peg_generator/pegen/build.py @@ -139,6 +139,7 @@ def compile_c_extension( ] include_dirs = [ str(MOD_DIR.parent.parent.parent / "Include" / "internal"), + str(MOD_DIR.parent.parent.parent / "Include" / "internal" / "mimalloc"), str(MOD_DIR.parent.parent.parent / "Parser"), str(MOD_DIR.parent.parent.parent / "Parser" / "lexer"), str(MOD_DIR.parent.parent.parent / "Parser" / "tokenizer"), @@ -219,6 +220,9 @@ def compile_c_extension( ) else: objects = compiler.object_filenames(extension.sources, output_dir=cmd.build_temp) + # The cmd.get_libraries() call needs a valid compiler attribute or we will + # get an incorrect library name on the free-threaded Windows build. + cmd.compiler = compiler # Now link the object files together into a "shared object" compiler.link_shared_object( objects, diff --git a/Tools/peg_generator/pegen/keywordgen.py b/Tools/peg_generator/pegen/keywordgen.py index 82d717b72976e5..52611eae044e58 100644 --- a/Tools/peg_generator/pegen/keywordgen.py +++ b/Tools/peg_generator/pegen/keywordgen.py @@ -41,25 +41,24 @@ def main() -> None: description="Generate the Lib/keywords.py file from the grammar." ) parser.add_argument( - "grammar", type=str, help="The file with the grammar definition in PEG format" + "grammar", help="The file with the grammar definition in PEG format" ) parser.add_argument( - "tokens_file", type=argparse.FileType("r"), help="The file with the token definitions" + "tokens_file", help="The file with the token definitions" ) parser.add_argument( "keyword_file", - type=argparse.FileType("w"), help="The path to write the keyword definitions", ) args = parser.parse_args() grammar, _, _ = build_parser(args.grammar) - with args.tokens_file as tok_file: + with open(args.tokens_file) as tok_file: all_tokens, exact_tok, non_exact_tok = generate_token_definitions(tok_file) gen = CParserGenerator(grammar, all_tokens, exact_tok, non_exact_tok, file=None) gen.collect_rules() - with args.keyword_file as thefile: + with open(args.keyword_file, 'w') as thefile: all_keywords = sorted(list(gen.keywords.keys())) all_soft_keywords = sorted(gen.soft_keywords) diff --git a/Tools/requirements-dev.txt b/Tools/requirements-dev.txt index b89f86a35d6115..c0a63b40ff4155 100644 --- a/Tools/requirements-dev.txt +++ b/Tools/requirements-dev.txt @@ -3,5 +3,5 @@ mypy==1.8.0 # needed for peg_generator: -types-psutil==5.9.5.17 -types-setuptools==69.0.0.0 +types-psutil==5.9.5.20240106 +types-setuptools==69.0.0.20240125 diff --git a/Tools/requirements-hypothesis.txt b/Tools/requirements-hypothesis.txt index 1bca5d2367f4b2..064731a236ee86 100644 --- a/Tools/requirements-hypothesis.txt +++ b/Tools/requirements-hypothesis.txt @@ -1,4 +1,4 @@ # Requirements file for hypothesis that # we use to run our property-based tests in CI. -hypothesis==6.91.0 +hypothesis==6.97.4 diff --git a/Tools/scripts/summarize_stats.py b/Tools/scripts/summarize_stats.py index 80a1280c025aca..9b7e7b999ea7c7 100644 --- a/Tools/scripts/summarize_stats.py +++ b/Tools/scripts/summarize_stats.py @@ -412,6 +412,14 @@ def get_histogram(self, prefix: str) -> list[tuple[int, int]]: rows.sort() return rows + def get_rare_events(self) -> list[tuple[str, int]]: + prefix = "Rare event " + return [ + (key[len(prefix) + 1:-1], val) + for key, val in self._data.items() + if key.startswith(prefix) + ] + class Count(int): def markdown(self) -> str: @@ -460,8 +468,11 @@ class JoinMode(enum.Enum): # second column of each input table as a new column CHANGE = 1 # Join using the first column as a key, indicating the change in the second - # column of each input table as a ne column, and omit all other columns + # column of each input table as a new column, and omit all other columns CHANGE_ONE_COLUMN = 2 + # Join using the first column as a key, and indicate the change as a new + # column, but don't sort by the amount of change. + CHANGE_NO_SORT = 3 class Table: @@ -484,7 +495,7 @@ def join_row(self, key: str, row_a: tuple, row_b: tuple) -> tuple: match self.join_mode: case JoinMode.SIMPLE: return (key, *row_a, *row_b) - case JoinMode.CHANGE: + case JoinMode.CHANGE | JoinMode.CHANGE_NO_SORT: return (key, *row_a, *row_b, DiffRatio(row_a[0], row_b[0])) case JoinMode.CHANGE_ONE_COLUMN: return (key, row_a[0], row_b[0], DiffRatio(row_a[0], row_b[0])) @@ -497,7 +508,7 @@ def join_columns(self, columns: Columns) -> Columns: *("Base " + x for x in columns[1:]), *("Head " + x for x in columns[1:]), ) - case JoinMode.CHANGE: + case JoinMode.CHANGE | JoinMode.CHANGE_NO_SORT: return ( columns[0], *("Base " + x for x in columns[1:]), @@ -1027,7 +1038,7 @@ def iter_optimization_tables(base_stats: Stats, head_stats: Stats | None = None) Table( ("Range", "Count:", "Ratio:"), calc_histogram_table(name, den), - JoinMode.CHANGE, + JoinMode.CHANGE_NO_SORT, ) ], ) @@ -1061,6 +1072,17 @@ def iter_optimization_tables(base_stats: Stats, head_stats: Stats | None = None) ) +def rare_event_section() -> Section: + def calc_rare_event_table(stats: Stats) -> Table: + return [(x, Count(y)) for x, y in stats.get_rare_events()] + + return Section( + "Rare events", + "Counts of rare/unlikely events", + [Table(("Event", "Count:"), calc_rare_event_table, JoinMode.CHANGE)], + ) + + def meta_stats_section() -> Section: def calc_rows(stats: Stats) -> Rows: return [("Number of data files", Count(stats.get("__nfiles__")))] @@ -1082,6 +1104,7 @@ def calc_rows(stats: Stats) -> Rows: object_stats_section(), gc_stats_section(), optimization_section(), + rare_event_section(), meta_stats_section(), ] @@ -1154,12 +1177,13 @@ def to_markdown(x): print("Stats gathered on:", date.today(), file=out) -def output_stats(inputs: list[Path], json_output=TextIO | None): +def output_stats(inputs: list[Path], json_output=str | None): match len(inputs): case 1: data = load_raw_data(Path(inputs[0])) if json_output is not None: - save_raw_data(data, json_output) # type: ignore + with open(json_output, "w", encoding="utf-8") as f: + save_raw_data(data, f) # type: ignore stats = Stats(data) output_markdown(sys.stdout, LAYOUT, stats) case 2: @@ -1195,7 +1219,6 @@ def main(): parser.add_argument( "--json-output", nargs="?", - type=argparse.FileType("w"), help="Output complete raw results to the given JSON file.", ) diff --git a/Tools/ssl/make_ssl_data.py b/Tools/ssl/make_ssl_data.py index ab1134ed8c4f77..98608716576792 100755 --- a/Tools/ssl/make_ssl_data.py +++ b/Tools/ssl/make_ssl_data.py @@ -23,7 +23,7 @@ ) parser.add_argument("srcdir", help="OpenSSL source directory") parser.add_argument( - "output", nargs="?", type=argparse.FileType("w"), default=sys.stdout + "output", nargs="?", default=None ) @@ -126,8 +126,13 @@ def main(): lines.append("") lines.extend(gen_error_codes(args)) - for line in lines: - args.output.write(line + "\n") + if args.output is None: + for line in lines: + print(line) + else: + with open(args.output, 'w') as output: + for line in lines: + print(line, file=output) if __name__ == "__main__": diff --git a/Tools/ssl/multissltests.py b/Tools/ssl/multissltests.py index 120e3883adc795..baa16102068aa0 100755 --- a/Tools/ssl/multissltests.py +++ b/Tools/ssl/multissltests.py @@ -47,8 +47,9 @@ OPENSSL_RECENT_VERSIONS = [ "1.1.1w", - "3.0.11", - "3.1.3", + "3.0.13", + "3.1.5", + "3.2.1", ] LIBRESSL_OLD_VERSIONS = [ diff --git a/Tools/wasm/README.md b/Tools/wasm/README.md index beb857f69e40da..23b38c8e93638a 100644 --- a/Tools/wasm/README.md +++ b/Tools/wasm/README.md @@ -83,7 +83,7 @@ embuilder --pic build zlib bzip2 MINIMAL_PIC ``` -#### Compile a build Python interpreter +### Compile and build Python interpreter From within the container, run the following command: diff --git a/Tools/wasm/wasi.py b/Tools/wasm/wasi.py index 34c0e9375e24c8..46ecae74a9ecea 100644 --- a/Tools/wasm/wasi.py +++ b/Tools/wasm/wasi.py @@ -17,11 +17,15 @@ CHECKOUT = pathlib.Path(__file__).parent.parent.parent + CROSS_BUILD_DIR = CHECKOUT / "cross-build" BUILD_DIR = CROSS_BUILD_DIR / "build" HOST_TRIPLE = "wasm32-wasi" HOST_DIR = CROSS_BUILD_DIR / HOST_TRIPLE +LOCAL_SETUP = CHECKOUT / "Modules" / "Setup.local" +LOCAL_SETUP_MARKER = "# Generated by Tools/wasm/wasi.py\n".encode("utf-8") + def updated_env(updates={}): """Create a new dict representing the environment to use. @@ -64,7 +68,8 @@ def wrapper(context): terminal_width = 80 print("⎯" * terminal_width) print("📁", working_dir) - if clean_ok and context.clean and working_dir.exists(): + if (clean_ok and getattr(context, "clean", False) and + working_dir.exists()): print(f"🚮 Deleting directory (--clean)...") shutil.rmtree(working_dir) @@ -119,12 +124,11 @@ def build_python_path(): @subdir(BUILD_DIR, clean_ok=True) def configure_build_python(context, working_dir): """Configure the build/host Python.""" - local_setup = CHECKOUT / "Modules" / "Setup.local" - if local_setup.exists(): - print(f"👍 {local_setup} exists ...") + if LOCAL_SETUP.exists(): + print(f"👍 {LOCAL_SETUP} exists ...") else: - print(f"📝 Touching {local_setup} ...") - local_setup.touch() + print(f"📝 Touching {LOCAL_SETUP} ...") + LOCAL_SETUP.write_bytes(LOCAL_SETUP_MARKER) configure = [os.path.relpath(CHECKOUT / 'configure', working_dir)] if context.args: @@ -233,9 +237,10 @@ def configure_wasi_python(context, working_dir): env=updated_env(env_additions | wasi_sdk_env(context)), quiet=context.quiet) + python_wasm = working_dir / "python.wasm" exec_script = working_dir / "python.sh" with exec_script.open("w", encoding="utf-8") as file: - file.write(f'#!/bin/sh\nexec {host_runner} "$@"\n') + file.write(f'#!/bin/sh\nexec {host_runner} {python_wasm} "$@"\n') exec_script.chmod(0o755) print(f"🏃‍♀️ Created {exec_script} ... ") sys.stdout.flush() @@ -259,6 +264,17 @@ def build_all(context): for step in steps: step(context) +def clean_contents(context): + """Delete all files created by this script.""" + if CROSS_BUILD_DIR.exists(): + print(f"🧹 Deleting {CROSS_BUILD_DIR} ...") + shutil.rmtree(CROSS_BUILD_DIR) + + if LOCAL_SETUP.exists(): + with LOCAL_SETUP.open("rb") as file: + if file.read(len(LOCAL_SETUP_MARKER)) == LOCAL_SETUP_MARKER: + print(f"🧹 Deleting generated {LOCAL_SETUP} ...") + def main(): default_host_runner = (f"{shutil.which('wasmtime')} run " @@ -272,9 +288,7 @@ def main(): # Map the checkout to / to load the stdlib from /Lib. "--dir {HOST_DIR}::{GUEST_DIR} " # Set PYTHONPATH to the sysconfig data. - "--env {ENV_VAR_NAME}={ENV_VAR_VALUE} " - # Path to the WASM binary. - "{PYTHON_WASM}") + "--env {ENV_VAR_NAME}={ENV_VAR_VALUE}") parser = argparse.ArgumentParser() subcommands = parser.add_subparsers(dest="subcommand") @@ -291,11 +305,13 @@ def main(): "Python)") make_host = subcommands.add_parser("make-host", help="Run `make` for the host/WASI") + clean = subcommands.add_parser("clean", help="Delete files and directories " + "created by this script") for subcommand in build, configure_build, make_build, configure_host, make_host: subcommand.add_argument("--quiet", action="store_true", default=False, dest="quiet", help="Redirect output from subprocesses to a log file") - for subcommand in build, configure_build, configure_host: + for subcommand in configure_build, configure_host: subcommand.add_argument("--clean", action="store_true", default=False, dest="clean", help="Delete any relevant directories before building") @@ -310,8 +326,8 @@ def main(): "$WASI_SDK_PATH or /opt/wasi-sdk") subcommand.add_argument("--host-runner", action="store", default=default_host_runner, dest="host_runner", - help="Command template for running the WebAssembly " - "code (default meant for wasmtime 14 or newer: " + help="Command template for running the WASI host " + "(default designed for wasmtime 14 or newer: " f"`{default_host_runner}`)") context = parser.parse_args() @@ -320,7 +336,8 @@ def main(): "make-build-python": make_build_python, "configure-host": configure_wasi_python, "make-host": make_wasi_python, - "build": build_all} + "build": build_all, + "clean": clean_contents} dispatch[context.subcommand](context) diff --git a/configure b/configure index 7e50abc29d0c1a..705a778cafced3 100755 --- a/configure +++ b/configure @@ -920,6 +920,8 @@ LLVM_AR PROFILE_TASK DEF_MAKE_RULE DEF_MAKE_ALL_RULE +JIT_STENCILS_H +REGEN_JIT_COMMAND ABIFLAGS LN MKDIR_P @@ -1074,6 +1076,7 @@ with_pydebug with_trace_refs enable_pystats with_assertions +enable_experimental_jit enable_optimizations with_lto enable_bolt @@ -1082,6 +1085,7 @@ with_dsymutil with_address_sanitizer with_memory_sanitizer with_undefined_behavior_sanitizer +with_thread_sanitizer with_hash_algorithm with_tzpath with_libs @@ -1800,6 +1804,9 @@ Optional Features: --disable-gil enable experimental support for running without the GIL (default is no) --enable-pystats enable internal statistics gathering (default is no) + --enable-experimental-jit + build the experimental just-in-time compiler + (default is no) --enable-optimizations enable expensive, stable optimizations (PGO, etc.) (default is no) --enable-bolt enable usage of the llvm-bolt post-link optimizer @@ -1860,6 +1867,8 @@ Optional Packages: --with-undefined-behavior-sanitizer enable UndefinedBehaviorSanitizer undefined behaviour detector, 'ubsan' (default is no) + --with-thread-sanitizer enable ThreadSanitizer data race detector, 'tsan' + (default is no) --with-hash-algorithm=[fnv|siphash13|siphash24] select hash algorithm for use in Python/pyhash.c (default is SipHash13) @@ -6796,6 +6805,8 @@ case $host/$ac_cv_cc_name in #( aarch64-*-linux-gnu/clang) : PY_SUPPORT_TIER=2 ;; #( powerpc64le-*-linux-gnu/gcc) : + PY_SUPPORT_TIER=2 ;; #( + wasm32-unknown-wasi/clang) : PY_SUPPORT_TIER=2 ;; #( x86_64-*-linux-gnu/clang) : PY_SUPPORT_TIER=2 ;; #( @@ -6808,10 +6819,6 @@ case $host/$ac_cv_cc_name in #( PY_SUPPORT_TIER=3 ;; #( s390x-*-linux-gnu/gcc) : PY_SUPPORT_TIER=3 ;; #( - wasm32-unknown-emscripten/clang) : - PY_SUPPORT_TIER=3 ;; #( - wasm32-unknown-wasi/clang) : - PY_SUPPORT_TIER=3 ;; #( x86_64-*-freebsd*/clang) : PY_SUPPORT_TIER=3 ;; #( *) : @@ -7994,6 +8001,34 @@ else printf "%s\n" "no" >&6; } fi +# Check for --enable-experimental-jit: +{ printf "%s\n" "$as_me:${as_lineno-$LINENO}: checking for --enable-experimental-jit" >&5 +printf %s "checking for --enable-experimental-jit... " >&6; } +# Check whether --enable-experimental-jit was given. +if test ${enable_experimental_jit+y} +then : + enableval=$enable_experimental_jit; +else $as_nop + enable_experimental_jit=no +fi + +if test "x$enable_experimental_jit" = xno +then : + +else $as_nop + as_fn_append CFLAGS_NODIST " -D_Py_JIT" + REGEN_JIT_COMMAND="\$(PYTHON_FOR_REGEN) \$(srcdir)/Tools/jit/build.py $host" + JIT_STENCILS_H="jit_stencils.h" + if test "x$Py_DEBUG" = xtrue +then : + as_fn_append REGEN_JIT_COMMAND " --debug" +fi +fi + + +{ printf "%s\n" "$as_me:${as_lineno-$LINENO}: result: $enable_experimental_jit" >&5 +printf "%s\n" "$enable_experimental_jit" >&6; } + # Enable optimization flags @@ -12506,6 +12541,28 @@ with_ubsan="no" fi +{ printf "%s\n" "$as_me:${as_lineno-$LINENO}: checking for --with-thread-sanitizer" >&5 +printf %s "checking for --with-thread-sanitizer... " >&6; } + +# Check whether --with-thread_sanitizer was given. +if test ${with_thread_sanitizer+y} +then : + withval=$with_thread_sanitizer; +{ printf "%s\n" "$as_me:${as_lineno-$LINENO}: result: $withval" >&5 +printf "%s\n" "$withval" >&6; } +BASECFLAGS="-fsanitize=thread $BASECFLAGS" +LDFLAGS="-fsanitize=thread $LDFLAGS" +with_tsan="yes" + +else $as_nop + +{ printf "%s\n" "$as_me:${as_lineno-$LINENO}: result: no" >&5 +printf "%s\n" "no" >&6; } +with_tsan="no" + +fi + + # Set info about shared libraries. @@ -16513,7 +16570,7 @@ ipv6type=unknown ipv6lib=none ipv6trylibc=no -if test "$ipv6" = "yes"; then +if test "$ipv6" = yes -a "$cross_compiling" = no; then { printf "%s\n" "$as_me:${as_lineno-$LINENO}: checking ipv6 stack type" >&5 printf %s "checking ipv6 stack type... " >&6; } for i in inria kame linux-glibc linux-inet6 solaris toshiba v6d zeta; @@ -17419,6 +17476,12 @@ if test "x$ac_cv_func_getgid" = xyes then : printf "%s\n" "#define HAVE_GETGID 1" >>confdefs.h +fi +ac_fn_c_check_func "$LINENO" "getgrent" "ac_cv_func_getgrent" +if test "x$ac_cv_func_getgrent" = xyes +then : + printf "%s\n" "#define HAVE_GETGRENT 1" >>confdefs.h + fi ac_fn_c_check_func "$LINENO" "getgrgid" "ac_cv_func_getgrgid" if test "x$ac_cv_func_getgrgid" = xyes @@ -17581,6 +17644,12 @@ if test "x$ac_cv_func_getwd" = xyes then : printf "%s\n" "#define HAVE_GETWD 1" >>confdefs.h +fi +ac_fn_c_check_func "$LINENO" "grantpt" "ac_cv_func_grantpt" +if test "x$ac_cv_func_grantpt" = xyes +then : + printf "%s\n" "#define HAVE_GRANTPT 1" >>confdefs.h + fi ac_fn_c_check_func "$LINENO" "if_nameindex" "ac_cv_func_if_nameindex" if test "x$ac_cv_func_if_nameindex" = xyes @@ -17767,6 +17836,12 @@ if test "x$ac_cv_func_posix_fallocate" = xyes then : printf "%s\n" "#define HAVE_POSIX_FALLOCATE 1" >>confdefs.h +fi +ac_fn_c_check_func "$LINENO" "posix_openpt" "ac_cv_func_posix_openpt" +if test "x$ac_cv_func_posix_openpt" = xyes +then : + printf "%s\n" "#define HAVE_POSIX_OPENPT 1" >>confdefs.h + fi ac_fn_c_check_func "$LINENO" "posix_spawn" "ac_cv_func_posix_spawn" if test "x$ac_cv_func_posix_spawn" = xyes @@ -17803,6 +17878,12 @@ if test "x$ac_cv_func_preadv2" = xyes then : printf "%s\n" "#define HAVE_PREADV2 1" >>confdefs.h +fi +ac_fn_c_check_func "$LINENO" "pthread_cond_timedwait_relative_np" "ac_cv_func_pthread_cond_timedwait_relative_np" +if test "x$ac_cv_func_pthread_cond_timedwait_relative_np" = xyes +then : + printf "%s\n" "#define HAVE_PTHREAD_COND_TIMEDWAIT_RELATIVE_NP 1" >>confdefs.h + fi ac_fn_c_check_func "$LINENO" "pthread_condattr_setclock" "ac_cv_func_pthread_condattr_setclock" if test "x$ac_cv_func_pthread_condattr_setclock" = xyes @@ -17821,6 +17902,18 @@ if test "x$ac_cv_func_pthread_kill" = xyes then : printf "%s\n" "#define HAVE_PTHREAD_KILL 1" >>confdefs.h +fi +ac_fn_c_check_func "$LINENO" "ptsname" "ac_cv_func_ptsname" +if test "x$ac_cv_func_ptsname" = xyes +then : + printf "%s\n" "#define HAVE_PTSNAME 1" >>confdefs.h + +fi +ac_fn_c_check_func "$LINENO" "ptsname_r" "ac_cv_func_ptsname_r" +if test "x$ac_cv_func_ptsname_r" = xyes +then : + printf "%s\n" "#define HAVE_PTSNAME_R 1" >>confdefs.h + fi ac_fn_c_check_func "$LINENO" "pwrite" "ac_cv_func_pwrite" if test "x$ac_cv_func_pwrite" = xyes @@ -18229,6 +18322,12 @@ if test "x$ac_cv_func_unlinkat" = xyes then : printf "%s\n" "#define HAVE_UNLINKAT 1" >>confdefs.h +fi +ac_fn_c_check_func "$LINENO" "unlockpt" "ac_cv_func_unlockpt" +if test "x$ac_cv_func_unlockpt" = xyes +then : + printf "%s\n" "#define HAVE_UNLOCKPT 1" >>confdefs.h + fi ac_fn_c_check_func "$LINENO" "utimensat" "ac_cv_func_utimensat" if test "x$ac_cv_func_utimensat" = xyes @@ -21805,6 +21904,7 @@ cat confdefs.h - <<_ACEOF >conftest.$ac_ext #if defined(MAJOR_IN_MKDEV) #include #elif defined(MAJOR_IN_SYSMACROS) +#include #include #else #include @@ -27478,7 +27578,12 @@ then : else $as_nop - rpath_arg="-Wl,-rpath=" + if test "$ac_sys_system" = "Darwin" + then + rpath_arg="-Wl,-rpath," + else + rpath_arg="-Wl,-rpath=" + fi fi @@ -28870,7 +28975,8 @@ then : if true then : - if test "$ac_cv_func_getgrgid" = yes -o "$ac_cv_func_getgrgid_r" = yes + if test "$ac_cv_func_getgrent" = "yes" && + { test "$ac_cv_func_getgrgid" = "yes" || test "$ac_cv_func_getgrgid_r" = "yes"; } then : py_cv_module_grp=yes else $as_nop diff --git a/configure.ac b/configure.ac index e064848af9ed1b..dee7ed552b370f 100644 --- a/configure.ac +++ b/configure.ac @@ -973,14 +973,13 @@ AS_CASE([$host/$ac_cv_cc_name], [aarch64-*-linux-gnu/gcc], [PY_SUPPORT_TIER=2], dnl Linux ARM64, glibc, gcc+clang [aarch64-*-linux-gnu/clang], [PY_SUPPORT_TIER=2], [powerpc64le-*-linux-gnu/gcc], [PY_SUPPORT_TIER=2], dnl Linux on PPC64 little endian, glibc, gcc + [wasm32-unknown-wasi/clang], [PY_SUPPORT_TIER=2], dnl WebAssembly System Interface, clang [x86_64-*-linux-gnu/clang], [PY_SUPPORT_TIER=2], dnl Linux on AMD64, any vendor, glibc, clang [aarch64-pc-windows-msvc/msvc], [PY_SUPPORT_TIER=3], dnl Windows ARM64, MSVC [armv7l-*-linux-gnueabihf/gcc], [PY_SUPPORT_TIER=3], dnl ARMv7 LE with hardware floats, any vendor, glibc, gcc [powerpc64le-*-linux-gnu/clang], [PY_SUPPORT_TIER=3], dnl Linux on PPC64 little endian, glibc, clang [s390x-*-linux-gnu/gcc], [PY_SUPPORT_TIER=3], dnl Linux on 64bit s390x (big endian), glibc, gcc - [wasm32-unknown-emscripten/clang], [PY_SUPPORT_TIER=3], dnl WebAssembly Emscripten - [wasm32-unknown-wasi/clang], [PY_SUPPORT_TIER=3], dnl WebAssembly System Interface [x86_64-*-freebsd*/clang], [PY_SUPPORT_TIER=3], dnl FreeBSD on AMD64 [PY_SUPPORT_TIER=0] ) @@ -1579,6 +1578,28 @@ else AC_MSG_RESULT([no]) fi +# Check for --enable-experimental-jit: +AC_MSG_CHECKING([for --enable-experimental-jit]) +AC_ARG_ENABLE([experimental-jit], + [AS_HELP_STRING([--enable-experimental-jit], + [build the experimental just-in-time compiler (default is no)])], + [], + [enable_experimental_jit=no]) +AS_VAR_IF([enable_experimental_jit], + [no], + [], + [AS_VAR_APPEND([CFLAGS_NODIST], [" -D_Py_JIT"]) + AS_VAR_SET([REGEN_JIT_COMMAND], + ["\$(PYTHON_FOR_REGEN) \$(srcdir)/Tools/jit/build.py $host"]) + AS_VAR_SET([JIT_STENCILS_H], ["jit_stencils.h"]) + AS_VAR_IF([Py_DEBUG], + [true], + [AS_VAR_APPEND([REGEN_JIT_COMMAND], [" --debug"])], + [])]) +AC_SUBST([REGEN_JIT_COMMAND]) +AC_SUBST([JIT_STENCILS_H]) +AC_MSG_RESULT([$enable_experimental_jit]) + # Enable optimization flags AC_SUBST([DEF_MAKE_ALL_RULE]) AC_SUBST([DEF_MAKE_RULE]) @@ -3067,6 +3088,24 @@ AC_MSG_RESULT([no]) with_ubsan="no" ]) +AC_MSG_CHECKING([for --with-thread-sanitizer]) +AC_ARG_WITH( + [thread_sanitizer], + [AS_HELP_STRING( + [--with-thread-sanitizer], + [enable ThreadSanitizer data race detector, 'tsan' (default is no)] + )], +[ +AC_MSG_RESULT([$withval]) +BASECFLAGS="-fsanitize=thread $BASECFLAGS" +LDFLAGS="-fsanitize=thread $LDFLAGS" +with_tsan="yes" +], +[ +AC_MSG_RESULT([no]) +with_tsan="no" +]) + # Set info about shared libraries. AC_SUBST([SHLIB_SUFFIX]) AC_SUBST([LDSHARED]) @@ -4349,7 +4388,7 @@ ipv6type=unknown ipv6lib=none ipv6trylibc=no -if test "$ipv6" = "yes"; then +if test "$ipv6" = yes -a "$cross_compiling" = no; then AC_MSG_CHECKING([ipv6 stack type]) for i in inria kame linux-glibc linux-inet6 solaris toshiba v6d zeta; do @@ -4749,17 +4788,17 @@ AC_CHECK_FUNCS([ \ copy_file_range ctermid dup dup3 execv explicit_bzero explicit_memset \ faccessat fchmod fchmodat fchown fchownat fdopendir fdwalk fexecve \ fork fork1 fpathconf fstatat ftime ftruncate futimens futimes futimesat \ - gai_strerror getegid getentropy geteuid getgid getgrgid getgrgid_r \ + gai_strerror getegid getentropy geteuid getgid getgrent getgrgid getgrgid_r \ getgrnam_r getgrouplist getgroups gethostname getitimer getloadavg getlogin \ getpeername getpgid getpid getppid getpriority _getpty \ getpwent getpwnam_r getpwuid getpwuid_r getresgid getresuid getrusage getsid getspent \ - getspnam getuid getwd if_nameindex initgroups kill killpg lchown linkat \ + getspnam getuid getwd grantpt if_nameindex initgroups kill killpg lchown linkat \ lockf lstat lutimes madvise mbrtowc memrchr mkdirat mkfifo mkfifoat \ mknod mknodat mktime mmap mremap nice openat opendir pathconf pause pipe \ - pipe2 plock poll posix_fadvise posix_fallocate posix_spawn posix_spawnp \ + pipe2 plock poll posix_fadvise posix_fallocate posix_openpt posix_spawn posix_spawnp \ posix_spawn_file_actions_addclosefrom_np \ - pread preadv preadv2 pthread_condattr_setclock pthread_init pthread_kill \ - pwrite pwritev pwritev2 readlink readlinkat readv realpath renameat \ + pread preadv preadv2 pthread_cond_timedwait_relative_np pthread_condattr_setclock pthread_init \ + pthread_kill ptsname ptsname_r pwrite pwritev pwritev2 readlink readlinkat readv realpath renameat \ rtpSpawn sched_get_priority_max sched_rr_get_interval sched_setaffinity \ sched_setparam sched_setscheduler sem_clockwait sem_getvalue sem_open \ sem_timedwait sem_unlink sendfile setegid seteuid setgid sethostname \ @@ -4768,7 +4807,7 @@ AC_CHECK_FUNCS([ \ sigfillset siginterrupt sigpending sigrelse sigtimedwait sigwait \ sigwaitinfo snprintf splice strftime strlcpy strsignal symlinkat sync \ sysconf system tcgetpgrp tcsetpgrp tempnam timegm times tmpfile \ - tmpnam tmpnam_r truncate ttyname umask uname unlinkat utimensat utimes vfork \ + tmpnam tmpnam_r truncate ttyname umask uname unlinkat unlockpt utimensat utimes vfork \ wait wait3 wait4 waitid waitpid wcscoll wcsftime wcsxfrm wmemcmp writev \ ]) @@ -5102,6 +5141,7 @@ AC_LINK_IFELSE([AC_LANG_PROGRAM([[ #if defined(MAJOR_IN_MKDEV) #include #elif defined(MAJOR_IN_SYSMACROS) +#include #include #else #include @@ -6808,7 +6848,12 @@ AX_CHECK_OPENSSL([have_openssl=yes],[have_openssl=no]) AS_VAR_IF([GNULD], [yes], [ rpath_arg="-Wl,--enable-new-dtags,-rpath=" ], [ - rpath_arg="-Wl,-rpath=" + if test "$ac_sys_system" = "Darwin" + then + rpath_arg="-Wl,-rpath," + else + rpath_arg="-Wl,-rpath=" + fi ]) AC_MSG_CHECKING([for --with-openssl-rpath]) @@ -7269,7 +7314,9 @@ PY_STDLIB_MOD([_socket], -a "$ac_cv_header_netinet_in_h" = "yes"])) dnl platform specific extensions -PY_STDLIB_MOD([grp], [], [test "$ac_cv_func_getgrgid" = yes -o "$ac_cv_func_getgrgid_r" = yes]) +PY_STDLIB_MOD([grp], [], + [test "$ac_cv_func_getgrent" = "yes" && + { test "$ac_cv_func_getgrgid" = "yes" || test "$ac_cv_func_getgrgid_r" = "yes"; }]) PY_STDLIB_MOD([pwd], [], [test "$ac_cv_func_getpwuid" = yes -o "$ac_cv_func_getpwuid_r" = yes]) PY_STDLIB_MOD([resource], [], [test "$ac_cv_header_sys_resource_h" = yes]) PY_STDLIB_MOD([_scproxy], diff --git a/pyconfig.h.in b/pyconfig.h.in index d8a9f68951afbd..2b4bb1a2b52866 100644 --- a/pyconfig.h.in +++ b/pyconfig.h.in @@ -474,6 +474,9 @@ /* Define to 1 if you have the `getgid' function. */ #undef HAVE_GETGID +/* Define to 1 if you have the `getgrent' function. */ +#undef HAVE_GETGRENT + /* Define to 1 if you have the `getgrgid' function. */ #undef HAVE_GETGRGID @@ -601,6 +604,9 @@ bcopy. */ #undef HAVE_GLIBC_MEMMOVE_BUG +/* Define to 1 if you have the `grantpt' function. */ +#undef HAVE_GRANTPT + /* Define to 1 if you have the header file. */ #undef HAVE_GRP_H @@ -899,6 +905,9 @@ /* Define to 1 if you have the `posix_fallocate' function. */ #undef HAVE_POSIX_FALLOCATE +/* Define to 1 if you have the `posix_openpt' function. */ +#undef HAVE_POSIX_OPENPT + /* Define to 1 if you have the `posix_spawn' function. */ #undef HAVE_POSIX_SPAWN @@ -930,6 +939,10 @@ /* Define to 1 if you have the `pthread_condattr_setclock' function. */ #undef HAVE_PTHREAD_CONDATTR_SETCLOCK +/* Define to 1 if you have the `pthread_cond_timedwait_relative_np' function. + */ +#undef HAVE_PTHREAD_COND_TIMEDWAIT_RELATIVE_NP + /* Defined for Solaris 2.6 bug in pthread header. */ #undef HAVE_PTHREAD_DESTRUCTOR @@ -951,6 +964,12 @@ /* Define if platform requires stubbed pthreads support */ #undef HAVE_PTHREAD_STUBS +/* Define to 1 if you have the `ptsname' function. */ +#undef HAVE_PTSNAME + +/* Define to 1 if you have the `ptsname_r' function. */ +#undef HAVE_PTSNAME_R + /* Define to 1 if you have the header file. */ #undef HAVE_PTY_H @@ -1459,6 +1478,9 @@ /* Define to 1 if you have the `unlinkat' function. */ #undef HAVE_UNLINKAT +/* Define to 1 if you have the `unlockpt' function. */ +#undef HAVE_UNLOCKPT + /* Define to 1 if you have the `unshare' function. */ #undef HAVE_UNSHARE