Merge remote-tracking branch 'upstream/master' into fix_4610

.github/workflows/mypy_primer.yml

@@ -0,0 +1,57 @@
+name: Run mypy_primer
+
+on:
+  # Only run on PR, since we diff against master
+  pull_request:
+    paths-ignore:
+    - 'docs/**'
+    - '**/*.rst'
+    - '**/*.md'
+    - 'mypyc/**'
+
+jobs:
+  mypy_primer:
+    name: Run
+    runs-on: ubuntu-latest
+    strategy:
+      matrix:
+        shard-index: [0, 1, 2]
+      fail-fast: false
+    steps:
+      - uses: actions/checkout@v2
+        with:
+          path: mypy_to_test
+          fetch-depth: 0
+      - uses: actions/setup-python@v2
+        with:
+          python-version: 3.8
+      - name: Install dependencies
+        run: |
+          python -m pip install -U pip
+          pip install git+https://github.com/hauntsaninja/mypy_primer.git
+      - name: Run mypy_primer
+        shell: bash
+        run: |
+          cd mypy_to_test
+          echo "new commit"
+          git rev-list --format=%s --max-count=1 $GITHUB_SHA
+          git checkout -b upstream_master origin/master
+          echo "base commit"
+          git rev-list --format=%s --max-count=1 upstream_master
+          echo ''
+          cd ..
+          # fail action if exit code isn't zero or one
+          (
+            mypy_primer \
+            --repo mypy_to_test \
+            --new $GITHUB_SHA --old upstream_master \
+            --num-shards 3 --shard-index ${{ matrix.shard-index }} \
+            --debug \
+            --output concise \
+            | tee diff.txt
+          ) || [ $? -eq 1 ]
+      - name: Upload mypy_primer diff
+        uses: actions/upload-artifact@v2
+        with:
+          name: mypy_primer_diff_${{ matrix.shard-index }}
+          path: diff.txt

.github/workflows/mypy_primer_comment.yml

@@ -0,0 +1,104 @@
+name: Comment with mypy_primer diff
+
+on:
+  # pull_request_target gives us access to a write token which we need to post a comment
+  # The presence of a write token means that we can't run any untrusted code (i.e. malicious PRs),
+  # which is why this its own workflow. Github Actions doesn't make it easy for workflows to talk to
+  # each other, so the approach here is to poll for workflow runs, find the mypy_primer run for our
+  # commit, wait till it's completed, and download and post the diff.
+  pull_request_target:
+    paths-ignore:
+    - 'docs/**'
+    - '**/*.rst'
+    - '**/*.md'
+    - 'mypyc/**'
+
+jobs:
+  mypy_primer:
+    name: Comment
+    runs-on: ubuntu-latest
+    steps:
+      - name: Install dependencies
+        run: npm install adm-zip
+      - name: Post comment
+        uses: actions/github-script@v3
+        with:
+          github-token: ${{secrets.GITHUB_TOKEN}}
+          script: |
+            const AdmZip = require(`${process.env.GITHUB_WORKSPACE}/node_modules/adm-zip`)
+
+            // Because of pull_request_target, context.sha is the PR base branch
+            // So we need to ask Github for the SHA of the PR's head commit
+            const pull_request = await github.pulls.get({
+                owner: context.repo.owner,
+                repo: context.repo.repo,
+                pull_number: context.issue.number,
+            })
+            const pr_commit_sha = pull_request.data.head.sha
+            console.log("Looking for mypy_primer run for commit:", pr_commit_sha)
+
+            // Find the mypy_primer run for our commit and wait till it's completed
+            // We wait up to an hour before timing out
+            async function check_mypy_primer() {
+              // We're only looking at the first page, so in theory if we open enough PRs around
+              // the same time, this will fail to find the run.
+              const response = await github.actions.listWorkflowRuns({
+                owner: context.repo.owner,
+                repo: context.repo.repo,
+                workflow_id: "mypy_primer.yml",
+              })
+              if (response) {
+                return response.data.workflow_runs.find(run => run.head_sha == pr_commit_sha)
+              }
+              return undefined
+            }
+
+            const end_time = Number(new Date()) + 60 * 60 * 1000
+            let primer_run = await check_mypy_primer()
+            while (!primer_run || primer_run.status != "completed") {
+              if (Number(new Date()) > end_time) {
+                throw Error("Timed out waiting for mypy_primer")
+              }
+              console.log("Waiting for mypy_primer to complete...")
+              await new Promise(r => setTimeout(r, 10000))
+              primer_run = await check_mypy_primer()
+            }
+            console.log("Found mypy_primer run!")
+            console.log(primer_run)
+
+            // Download artifact(s) from the run
+            const artifacts = await github.actions.listWorkflowRunArtifacts({
+                owner: context.repo.owner,
+                repo: context.repo.repo,
+                run_id: primer_run.id,
+            })
+            const filtered_artifacts = artifacts.data.artifacts.filter(
+              a => a.name.startsWith("mypy_primer_diff")
+            )
+            console.log("Artifacts from mypy_primer:")
+            console.log(filtered_artifacts)
+
+            async function get_artifact_data(artifact) {
+              const zip = await github.actions.downloadArtifact({
+                  owner: context.repo.owner,
+                  repo: context.repo.repo,
+                  artifact_id: artifact.id,
+                  archive_format: "zip",
+              })
+              const adm = new AdmZip(Buffer.from(zip.data))
+              return adm.readAsText(adm.getEntry("diff.txt"))
+            }
+
+            const all_data = await Promise.all(filtered_artifacts.map(get_artifact_data))
+            const data = all_data.join("\n")
+
+            console.log("Diff from mypy_primer:")
+            console.log(data)
+            if (data.trim()) {
+              await github.issues.createComment({
+                issue_number: context.issue.number,
+                owner: context.repo.owner,
+                repo: context.repo.repo,
+                body: 'Diff from [mypy_primer](https://github.com/hauntsaninja/mypy_primer), showing the effect of this PR on open source code:\n```diff\n' + data + '```'
+              })
+            }

.github/workflows/test.yml

@@ -42,8 +42,8 @@ jobs:
         python-version: ${{ matrix.python }}
         architecture: ${{ matrix.arch }}
     - name: install tox
-      run: pip install --upgrade 'setuptools!=50' 'virtualenv<20' tox==3.9.0
+      run: pip install --upgrade 'setuptools!=50' 'virtualenv<20' tox==3.20.1
     - name: setup tox environment
       run: tox -e ${{ matrix.toxenv }} --notest
     - name: test
-      run: tox -e ${{ matrix.toxenv }}
+      run: tox -e ${{ matrix.toxenv }} --skip-pkg-install

.github/workflows/test_stubgenc.yml

@@ -0,0 +1,32 @@
+name: Test stubgenc on pybind11-mypy-demo
+
+on:
+  push:
+    branches: [master]
+    tags: ['*']
+  pull_request:
+    paths:
+    - 'misc/test-stubgenc.sh'
+    - 'mypy/stubgenc.py'
+    - 'mypy/stubdoc.py'
+    - 'test-data/stubgen/**'
+
+jobs:
+  stubgenc:
+    # Check stub file generation for a small pybind11 project
+    # (full text match is required to pass)
+    runs-on: ubuntu-latest
+    steps:
+
+    - uses: actions/checkout@v2
+
+    - name: initialize submodules
+      run: git submodule update --init
+
+    - name: Setup 🐍 3.8
+      uses: actions/setup-python@v2
+      with:
+        python-version: 3.8
+
+    - name: Test stubgenc
+      run: misc/test-stubgenc.sh

.gitignore

@@ -2,7 +2,6 @@ build/
 __pycache__
 *.py[cod]
 *~
-@*
 /build
 /env*/
 docs/build/

.gitmodules

@@ -1,3 +0,0 @@
-[submodule "typeshed"]
-	path = mypy/typeshed
-	url = https://github.com/python/typeshed

.travis.yml

@@ -42,7 +42,7 @@ jobs:
   - name: "run test suite with python 3.8"
     python: 3.8
   - name: "run test suite with python 3.9"
-    python: 3.9-dev
+    python: 3.9
   - name: "run mypyc runtime tests with python 3.6 debug build"
     language: generic
     env:
@@ -82,20 +82,21 @@ jobs:
   #   - EXTRA_ARGS=
 
 install:
-- pip install -U pip setuptools
+# pip 21.0 no longer works on Python 3.5
+- pip install -U pip==20.3.4 setuptools
 - pip install -U 'virtualenv<20'
-- pip install -U tox==3.9.0
+- pip install -U tox==3.20.1
 - python2 -m pip install --user -U typing
 - tox --notest
 
 # This is a big hack and only works because the layout of our directories
 # means that tox picks up the mypy from the source directories instead of
 # the version it installed into a venv. This is also *why* we need to do this,
 # since if we arranged for tox to build with mypyc, pytest wouldn't use it.
-- if [[ $TEST_MYPYC == 1 ]]; then pip install -r mypy-requirements.txt; CC=clang MYPYC_OPT_LEVEL=0 python3 setup.py --use-mypyc build_ext --inplace; fi
+- if [[ $TEST_MYPYC == 1 ]]; then pip install -r test-requirements.txt; CC=clang MYPYC_OPT_LEVEL=0 python3 setup.py --use-mypyc build_ext --inplace; fi
 
 script:
-- tox -- $EXTRA_ARGS
+- tox --skip-pkg-install -- $EXTRA_ARGS
 
 # Getting our hands on a debug build or the right OS X build is
 # annoying, unfortunately.

LICENSE

@@ -28,8 +28,9 @@ DEALINGS IN THE SOFTWARE.
 
 Portions of mypy and mypyc are licensed under different licenses.  The
 files under stdlib-samples as well as the files
-mypyc/lib-rt/pythonsupport.h and mypyc/lib-rt/getargs.c are licensed
-under the PSF 2 License, reproduced below.
+mypyc/lib-rt/pythonsupport.h, mypyc/lib-rt/getargs.c and
+mypyc/lib-rt/getargsfast.c are licensed under the PSF 2 License, reproduced
+below.
 
 = = = = =
 

MANIFEST.in

@@ -1,23 +1,45 @@
-recursive-include scripts *
-recursive-include test-data *
-recursive-include extensions *
-recursive-include docs *
-recursive-include misc proper_plugin.py
-recursive-include mypy/typeshed *.py *.pyi
-recursive-include mypy/xml *.xsd *.xslt *.css
-recursive-include mypyc/lib-rt *.c *.h *.tmpl *.py *.cc
-recursive-include mypyc/ir *.py
-recursive-include mypyc/codegen *.py
-recursive-include mypyc/irbuild *.py
-recursive-include mypyc/primitives *.py
-recursive-include mypyc/transform *.py
-recursive-include mypyc/external *.cc *.h Makefile *.pump LICENSE README
-recursive-include mypyc/test *.py
-recursive-include mypyc/test-data *.test
-recursive-include mypyc/test-data/fixtures *.py *.pyi
-recursive-include mypyc/doc *.rst *.py *.md Makefile *.bat
+# some of the prunes here are so that check-manifest doesn't complain about their exclusion
+# as such, be judicious in your use of prune
+
+# stubs
+prune mypy/typeshed
+include mypy/typeshed/stdlib/VERSIONS
+recursive-include mypy/typeshed *.pyi
+
+# mypy and mypyc
+include mypy/py.typed
+recursive-include mypy *.py
+recursive-include mypyc *.py
+
+# random
 include mypy_bootstrap.ini
+graft mypy/xml
+graft scripts
+
+# docs
+graft docs
+prune docs/build
+prune docs/source/_build
+
+# assorted mypyc requirements
+graft mypyc/external
+graft mypyc/lib-rt
+graft mypyc/test-data
+graft mypyc/doc
+
+# files necessary for testing sdist
+include mypy-requirements.txt
+include test-requirements.txt
 include mypy_self_check.ini
-include LICENSE
+prune misc
+include misc/proper_plugin.py
+graft test-data
+include conftest.py
 include runtests.py
 include pytest.ini
+
+include LICENSE mypyc/README.md
+exclude .gitmodules CONTRIBUTING.md CREDITS ROADMAP.md tox.ini
+
+global-exclude *.py[cod]
+global-exclude .DS_Store

README.md

@@ -184,17 +184,7 @@ Quick start for contributing to mypy
 
 If you want to contribute, first clone the mypy git repository:
 
-    $ git clone --recurse-submodules https://github.com/python/mypy.git
-
-If you've already cloned the repo without `--recurse-submodules`,
-you need to pull in the typeshed repo as follows:
-
-    $ git submodule init
-    $ git submodule update
-
-Either way you should now have a subdirectory `typeshed` inside your mypy repo,
-your folders tree should be like `mypy/mypy/typeshed`, containing a
-clone of the typeshed repo (`https://github.com/python/typeshed`).
+    $ git clone https://github.com/python/mypy.git
 
 From the mypy directory, use pip to install mypy:
 
@@ -209,22 +199,8 @@ the above as root. For example, in Ubuntu:
 Now you can use the `mypy` program just as above.  In case of trouble
 see "Troubleshooting" above.
 
-> NOTE: Installing with sudo can be a security risk, please try with flag `--user` first.
-    $ python3 -m pip install --user -U . 
-
-Working with the git version of mypy
-------------------------------------
-
-mypy contains a submodule, "typeshed". See https://github.com/python/typeshed.
-This submodule contains types for the Python standard library.
-
-Due to the way git submodules work, you'll have to do
-```
-  git submodule update mypy/typeshed
-```
-whenever you change branches, merge, rebase, or pull.
-
-(It's possible to automate this: Search Google for "git hook update submodule")
+> NOTE: Installing with sudo can be a security risk. Please try with the `--user` flag first.
+    $ python3 -m pip install --user -U .
 
 
 Tests
@@ -244,7 +220,8 @@ Development status
 ------------------
 
 Mypy is beta software, but it has already been used in production
-for several years at Dropbox, and it has an extensive test suite.
+for several years at Dropbox and in many other organizations, and
+it has an extensive test suite.
 
 See [the roadmap](ROADMAP.md) if you are interested in plans for the
 future.