Merge branch 'release/1.2-dev' into refactor/5459

Lightning-AI · Feb 9, 2021 · e59c0c3 · e59c0c3
2 parents f0eb77e + 774e9bd
commit e59c0c3
Show file tree

Hide file tree

Showing 235 changed files with 4,110 additions and 2,954 deletions.
diff --git a/.drone.jsonnet b/.drone.jsonnet
diff --git a/.drone.yml b/.drone.yml
@@ -37,11 +37,11 @@ steps:
     - pip install -r ./requirements/devel.txt --upgrade-strategy only-if-needed --no-cache-dir
     - pip install git+https://${AUTH_TOKEN}@github.com/PyTorchLightning/[email protected] --no-cache-dir
     # when Image has defined CUDa version we can switch to this package spec "nvidia-dali-cuda${CUDA_VERSION%%.*}0"
-    - pip install --extra-index-url https://developer.download.nvidia.com/compute/redist nvidia-dali-cuda100 --upgrade-strategy only-if-needed
+    #- pip install --extra-index-url https://developer.download.nvidia.com/compute/redist nvidia-dali-cuda100 --upgrade-strategy only-if-needed
     - pip list
-    # todo: remove unzip install after new nigtly docker is created
-    - apt-get update -qq
-    - apt-get install -y --no-install-recommends unzip
+    # todo: remove unzip install after new nightly docker is created
+    #- apt-get update -qq
+    #- apt-get install -y --no-install-recommends unzip
     # get legacy checkpoints
     - wget https://pl-public-data.s3.amazonaws.com/legacy/checkpoints.zip -P legacy/
     - unzip -o legacy/checkpoints.zip -d legacy/

diff --git a/.github/CODEOWNERS b/.github/CODEOWNERS
@@ -5,48 +5,49 @@
 # the repo. Unless a later match takes precedence,
 # @global-owner1 and @global-owner2 will be requested for
 # review when someone opens a pull request.
-* @williamfalcon @borda @tchaton @SeanNaren @awaelchli @justusschock
+* @williamfalcon @borda @tchaton @SeanNaren @carmocca @awaelchli @justusschock
 
-# Metrics
-/pytorch_lightning/metrics/   @teddykoker @ananyahjha93 @justusschock
-/tests/metrics/               @teddykoker @ananyahjha93 @justusschock
-/docs/source/metrics.rst      @teddykoker @ananyahjha93 @justusschock
+/.github/CODEOWNERS @williamfalcon
+/README.md          @williamfalcon @edenlightning @borda
+/setup.py           @williamfalcon @borda
 
-# API
-/pytorch_lightning/callbacks/base.py           @williamfalcon
-/pytorch_lightning/core/datamodule.py          @williamfalcon
-/pytorch_lightning/trainer/trainer.py          @williamfalcon @tchaton
-/pytorch_lightning/core/hooks.py               @williamfalcon
-/pytorch_lightning/core/lightning.py           @williamfalcon @tchaton
-/pytorch_lightning/core/optimizer.py           @tchaton
-/pytorch_lightning/trainer/training_loop.py    @tchaton @SeanNaren
-/pytorch_lightning/trainer/evaluation_loop.py  @tchaton @SeanNaren
-
-# Connectors
-/pytorch_lightning/trainer/connectors/         @tchaton @SeanNaren
-
-# accelerators
-/pytorch_lightning/accelerators/               @williamfalcon @tchaton @SeanNaren @awaelchli @justusschock
-
-# owners
-/.github/CODEOWNERS   @williamfalcon
-# main
-/README.md            @williamfalcon @edenlightning
-# installation
-/setup.py             @borda @williamfalcon
-
-# CI/CD
-/.github/workflows/   @borda @tchaton
-/.github/*.py         @borda @tchaton
-/dockers/             @borda @tchaton
-# configs in root
-/*.yml                @borda @tchaton
+# CI/CD and configs
+/.github/ @borda @tchaton
+/dockers/ @borda @tchaton
+*.yml     @borda @tchaton
 
 # Docs
-/docs/                          @edenlightning @tchaton @borda @awaelchli
-/.github/*.md                   @edenlightning @williamfalcon @borda
-/.github/ISSUE_TEMPLATE/*.md    @edenlightning @borda @tchaton
-/docs/source/conf.py            @borda @awaelchli
+/docs/                   @edenlightning @tchaton @borda @awaelchli
+/.github/*.md            @edenlightning @williamfalcon @borda
+/.github/ISSUE_TEMPLATE/ @edenlightning @borda @tchaton
+/docs/source/conf.py     @borda @awaelchli
+
+# Packages
+/pytorch_lightning/accelerators         @williamfalcon @tchaton @SeanNaren @awaelchli @justusschock
+/pytorch_lightning/callbacks            @williamfalcon @tchaton @carmocca @borda
+/pytorch_lightning/cluster_environments @borda @tchaton @SeanNaren @carmocca
+/pytorch_lightning/core                 @tchaton @SeanNaren @borda @carmocca @justusschock
+/pytorch_lightning/distributed          @williamfalcon @tchaton @awaelchli
+/pytorch_lightning/loggers              @tchaton @awaelchli @borda
+/pytorch_lightning/overrides            @tchaton @SeanNaren @borda
+/pytorch_lightning/plugins              @tchaton @SeanNaren @awaelchli @justusschock
+/pytorch_lightning/profiler             @williamfalcon @tchaton @borda
+/pytorch_lightning/trainer              @williamfalcon @borda @tchaton @SeanNaren @carmocca @awaelchli @justusschock
+/pytorch_lightning/trainer/connectors   @tchaton @SeanNaren @carmocca @borda
+/pytorch_lightning/tuner                @SkafteNicki @borda @awaelchli
+/pytorch_lightning/utilities            @borda @tchaton @SeanNaren @carmocca
+
+# Metrics
+/pytorch_lightning/metrics/             @teddykoker @ananyahjha93 @justusschock
+/tests/metrics/                         @teddykoker @ananyahjha93 @justusschock
+/docs/source/metrics.rst                @teddykoker @ananyahjha93 @justusschock
+
+# API
+/pytorch_lightning/callbacks/base.py  @williamfalcon
+/pytorch_lightning/core/datamodule.py @williamFalcon
+/pytorch_lightning/trainer/trainer.py @williamfalcon @tchaton
+/pytorch_lightning/core/hooks.py      @williamfalcon
+/pytorch_lightning/core/lightning.py  @williamfalcon @tchaton
 
 # Testing
-/tests/base/boring_model.py @williamfalcon
+/tests/base/boring_model.py @williamfalcon @tchaton @borda
diff --git a/.github/CONTRIBUTING.md b/.github/CONTRIBUTING.md
@@ -104,6 +104,12 @@ Most of the tests in PyTorch Lightning train a trial MNIST model under various t
 
 ## Guidelines
 
+### Developments scripts
+To build the documentation locally, simply execute the following commands from project root (only for Unix):
+- `make clean` cleans repo from temp/generated files
+- `make docs` builds documentation under _docs/build/html_
+- `make test` runs all project's tests
+
 ### Original code
 
 All added or edited code shall be the own original work of the particular contributor.
@@ -223,60 +229,44 @@ We welcome any useful contribution! For your convenience here's a recommended wo
 
 ### Question & Answer
 
-1. **How can I help/contribute?**
-
-   All help is extremely welcome - reporting bugs, fixing documentation, adding test cases, solving issues and preparing bug fixes. To solve some issues you can start with label [good first issue](https://github.com/PyTorchLightning/pytorch-lightning/issues?q=is%3Aopen+is%3Aissue+label%3A%22good+first+issue%22) or chose something close to your domain with label [help wanted](https://github.com/PyTorchLightning/pytorch-lightning/issues?q=is%3Aopen+is%3Aissue+label%3A%22help+wanted%22). Before you start to implement anything check that the issue description that it is clear and self-assign the task to you (if it is not possible, just comment that you take it and we assign it to you...).
-
-2. **Is there a recommendation for branch names?**
-
-   We do not rely on the name convention so far you are working with your own fork. Anyway it would be nice to follow this convention `<type>/<issue-id>_<short-name>` where the types are: `bugfix`, `feature`, `docs`, `tests`, ...
-
-3. **How to rebase my PR?**
+#### How can I help/contribute?
 
-   We recommend creating a PR in separate branch other than `master`, especially if you plan submitting several changes and do not want to wait until the first one is resolved (we can work on them in parallel).
+All types of contributions are welcome - reporting bugs, fixing documentation, adding test cases, solving issues, and preparing bug fixes. 
+To get started with code contributions, look for issues marked with the label [good first issue](https://github.com/PyTorchLightning/pytorch-lightning/issues?q=is%3Aopen+is%3Aissue+label%3A%22good+first+issue%22) or chose something close to your domain with the label [help wanted](https://github.com/PyTorchLightning/pytorch-lightning/issues?q=is%3Aopen+is%3Aissue+label%3A%22help+wanted%22). Before coding, make sure that the issue description is clear and comment on the issue so that we can assign it to you (or simply self-assign if you can).
 
-   First, make sure you have set [upstream](https://help.github.com/en/github/collaborating-with-issues-and-pull-requests/configuring-a-remote-for-a-fork) by running:
+#### Is there a recommendation for branch names?
 
-   ```bash
-   git remote add upstream https://github.com/PyTorchLightning/pytorch-lightning.git
-   ```
+We recommend you follow this convention `<type>/<issue-id>_<short-name>` where the types are: `bugfix`, `feature`, `docs`, or `tests` (but if you are using your own fork that's optional).
 
-   You'll know its set up right if you run `git remote -v` and see something similar to this:
+#### How to rebase my PR?
 
-   ```bash
-   origin  https://github.com/{YOUR_USERNAME}/pytorch-lightning.git (fetch)
-   origin  https://github.com/{YOUR_USERNAME}/pytorch-lightning.git (push)
-   upstream        https://github.com/PyTorchLightning/pytorch-lightning.git (fetch)
-   upstream        https://github.com/PyTorchLightning/pytorch-lightning.git (push)
-   ```
+We recommend creating a PR in a separate branch other than `master`, especially if you plan to submit several changes and do not want to wait until the first one is resolved (we can work on them in parallel).
 
-   Now you can update your master with upstream's master by running:
+First, make sure you have set [upstream](https://help.github.com/en/github/collaborating-with-issues-and-pull-requests/configuring-a-remote-for-a-fork) by running:
 
-   ```bash
-   git fetch --all --prune
-   git checkout master
-   git merge upstream/master
-   ```
+```bash
+git remote add upstream https://github.com/PyTorchLightning/pytorch-lightning.git
+```
 
-   Finally, checkout your feature branch and rebase it with master before pushing up your feature branch:
+You'll know its set up right if you run `git remote -v` and see something similar to this:
 
-   ```bash
-   git checkout my-PR-branch
-   git rebase master
-   # follow git instructions to resolve conflicts
-   git push -f
-   ```
+```bash
+origin  https://github.com/{YOUR_USERNAME}/pytorch-lightning.git (fetch)
+origin  https://github.com/{YOUR_USERNAME}/pytorch-lightning.git (push)
+upstream        https://github.com/PyTorchLightning/pytorch-lightning.git (fetch)
+upstream        https://github.com/PyTorchLightning/pytorch-lightning.git (push)
+```
 
-   Eventually, you can perform the rebasing directly from upstream after setting it up:
+Checkout your feature branch and rebase it with upstream's master before pushing up your feature branch:
 
-   ```bash
-   git fetch --all --prune
-   git rebase upstream/master
-   # follow git instructions to resolve conflicts
-   git push -f
-   ```
+```bash
+git fetch --all --prune
+git rebase upstream/master
+# follow git instructions to resolve conflicts
+git push -f
+```
 
-4. **How to add new tests**
+#### How to add new tests?**
 
 We are using [pytest](https://docs.pytest.org/en/stable/) in Pytorch Lightning.
 
@@ -293,19 +283,16 @@ Here is the process to create a new test
 
 ```python
 # TEST SHOULD BE IN YOUR FILE: tests/..../...py
+# TEST CODE TEMPLATE
 
-# RUN OUR TEST WITH: pytest tests/..../...py::test_explain_what_is_being_tested --verbose --capture=no
-
-# TEST CODE TEMPLATE
-
-# pytest decorator
+# [OPTIONAL] pytest decorator
 # @pytest.mark.skipif(not torch.cuda.is_available(), reason="test requires GPU machine")
 def test_explain_what_is_being_tested(tmpdir):
     """
     Test description about text reason to be
     """
 
-    # os.environ["PL_DEV_DEBUG"] = '1' optional. When activated, you can use internal trainer.dev_debugger
+    # os.environ["PL_DEV_DEBUG"] = '1' # [OPTIONAL] When activated, you can use internal trainer.dev_debugger
 
     class ExtendedModel(BoringModel):
         ...
@@ -320,12 +307,60 @@ def test_explain_what_is_being_tested(tmpdir):
         ...
     )
     trainer.fit(model)
-    result = trainer.test()
+    trainer.test()  # [OPTIONAL]
 
     # assert the behaviour is correct.
     assert ...
-    assert ...
 ```
+run our/your test with
+```bash
+python -m pytest tests/..../...py::test_explain_what_is_being_tested --verbose --capture=no
+```
+
+#### How to contribute bugfixes/features?
+
+Currently we have separate streams/branches for bugfixes/features and release from the default branch (`master`).
+Bugfixes should land in this `master` branch and features should land in `release/X.y-dev`.
+This means that when starting your contribution and creating a branch according to question 2) you should start this new branch from master or future release dev branch.
+Later in PR creation also pay attention to properly set the target branch, usually the starting (base) and target branch are the same.
+
+_Note, that this flow may change after the 1.2 release as we will adjust releasing strategy._
+
+#### How to fix PR with mixed base and target branches?
+
+Sometimes you start your PR as a bug-fix but it turns out to be more of a feature (or the other way around).
+Do not panic, the solution is very straightforward and quite simple.
+All you need to do are these two steps in arbitrary order:
+   - Ask someone from Core to change the base/target branch to the correct one
+   - Rebase or cherry-pick your commits onto the correct base branch...
+
+Let's show how to deal with the git...
+the sample case is moving a PR from `master` to `release/1.2-dev` assuming my branch name is `my-branch`
+and the last true master commit is `ccc111` and your first commit is `mmm222`.
+   * **Cherry-picking** way
+     ```bash
+     git checkout my-branch
+     # create a local backup of your branch
+     git checkout -b my-branch-backup
+     # reset your branch to the correct base
+     git reset release/1.2-dev --hard
+     # ACTION: this step is much easier to do with IDE
+     #  so open one and cherry-pick your last commits from `my-branch-backup`
+     #  resolve all eventual conflict as the new base may contain different code
+     # when all done, push back to the open PR
+     git push -f 
+     ```
+   * **Rebasing way**, see more about [rebase onto usage](https://womanonrails.com/git-rebase-onto)
+     ```bash
+     git checkout my-branch
+     # rebase your commits on the correct branch
+     git rebase --onto release/1.2-dev ccc111
+     # if there is no collision you shall see just success
+     #  eventually you would need to resolve collision and in such case follow the instruction in terminal
+     # when all done, push back to the open PR
+     git push -f
+     ```
+
 
 ### Bonus Workflow Tip
 

diff --git a/.github/mergify.yml b/.github/mergify.yml
@@ -40,6 +40,7 @@ pull_request_rules:
     conditions:
       - -conflict
       - -draft # filter-out GH draft PRs
+      - base=master  # apply only on master
       - -title~=(?i)wip # skip all PR that title contains “WIP” (ignoring case)
       - "#approved-reviews-by>=3" # number of review approvals
     actions:

diff --git a/.github/workflows/ci_test-conda.yml b/.github/workflows/ci_test-conda.yml
@@ -40,9 +40,6 @@ jobs:
     - name: Pull checkpoints from S3
       # todo: consider adding coma caching, but ATM all models have less then 100KB
       run: |
-        # todo: remove unzip install after new nigtly docker is created
-        apt-get update -qq
-        apt-get install -y --no-install-recommends unzip
         # enter legacy and update checkpoints from S3
         cd legacy
         curl https://pl-public-data.s3.amazonaws.com/legacy/checkpoints.zip --output checkpoints.zip

diff --git a/.github/workflows/ci_test-tpu.yml b/.github/workflows/ci_test-tpu.yml
@@ -113,6 +113,7 @@ jobs:
         cat xx00 && \
         echo "Done with log retrieval attempt." && \
         gcloud container images delete "$IMAGE:$IMAGETAG" --force-delete-tags && \
+        echo "Status code: $status_code"
         exit $status_code
       shell: bash
 

diff --git a/.github/workflows/code-formatting.yml b/.github/workflows/code-formatting.yml
@@ -69,7 +69,7 @@ jobs:
           python-version: 3.8
       - name: Install mypy
         run: |
-          pip install "mypy==0.790"
+          pip install mypy==0.790
           pip list
       - name: mypy check
         run: |