setup basic package functionality with CLI

.github/workflows/check-docs.yml

@@ -20,19 +20,6 @@ on:
         type: number
 
 jobs:
-  check:
-    runs-on: ubuntu-20.04
-
-    steps:
-    - uses: actions/checkout@v3
-
-    - uses: ammaraskar/sphinx-action@master
-      with:
-        # git is required to clone the docs theme
-        # before custom requirement are resolved https://github.com/ammaraskar/sphinx-action/issues/16
-        pre-build-command: "apt-get update -y && apt-get install -y git && pip install -r requirements/docs.txt"
-        docs-folder: "docs/"
-        repo-token: "${{ secrets.GITHUB_TOKEN }}"
 
   doctest:
     runs-on: ubuntu-20.04
@@ -47,7 +34,7 @@ jobs:
       - name: Install dependencies
         run: |
           pip install -e . -U -f https://download.pytorch.org/whl/cpu/torch_stable.html
-          pip install -r docs/requirements.txt
+          pip install -r requirements/docs.txt
           python --version
           pip --version
           pip list
@@ -73,7 +60,7 @@ jobs:
 
       - name: Install dependencies
         run: |
-          pip install -e . -U -r docs/requirements.txt -f https://download.pytorch.org/whl/torch_stable.html
+          pip install -e . -U -r requirements/docs.txt -f https://download.pytorch.org/whl/torch_stable.html
           sudo apt-get update --fix-missing
           sudo apt-get install -y texlive-latex-extra dvipng texlive-pictures
           python --version

.github/workflows/ci_testing.yml

@@ -34,10 +34,8 @@ jobs:
     - name: Set oldest dependencies
       if: matrix.requires == 'oldest'
       run: |
-        for fpath in ('requirements.txt', 'tests/requirements.txt'):
-            req = open(fpath).read().replace('>=', '==')
-            open(fpath, 'w').write(req)
-      shell: python
+        pip install -e .
+        python -m pl_devtools requirements set-oldest
 
     - name: Get pip cache
       id: pip-cache
@@ -48,18 +46,17 @@ jobs:
       uses: actions/cache@v3
       with:
         path: ${{ steps.pip-cache.outputs.dir }}
-        key: ${{ runner.os }}-${{ matrix.python-version }}-${{ matrix.requires }}-pip-${{ hashFiles('requirements.txt') }}
+        key: ${{ runner.os }}-${{ matrix.python-version }}-${{ matrix.requires }}-pip-${{ hashFiles('requirements/*.txt') }}
         restore-keys: |
           ${{ runner.os }}-${{ matrix.python-version }}-${{ matrix.requires }}-pip-
 
     - name: Install dependencies
       run: |
         pip install -e . -U --find-links https://download.pytorch.org/whl/cpu/torch_stable.html
-        pip install -q -r tests/requirements.txt
+        pip install -q -r requirements/test.txt
         python --version
         pip --version
         pip list
-      shell: bash
 
     - name: Tests
       run: |

.github/workflows/docs-deploy.yml

@@ -23,14 +23,14 @@ jobs:
         uses: actions/cache@v2
         with:
           path: ~/.cache/pip
-          key: ${{ runner.os }}-pip-${{ hashFiles('requirements.txt') }}
+          key: ${{ runner.os }}-pip-${{ hashFiles('requirements/*.txt') }}
           restore-keys: |
             ${{ runner.os }}-pip-
 
       - name: Install dependencies
         run: |
-          pip install  --requirement requirements.txt -U -f https://download.pytorch.org/whl/cpu/torch_stable.html  --quiet
-          pip install  --requirement docs/requirements.txt
+          pip install  -e . -U -f https://download.pytorch.org/whl/cpu/torch_stable.html  --quiet
+          pip install  --requirement requirements/docs.txt
           # install Texlive, see https://linuxconfig.org/how-to-install-latex-on-ubuntu-20-04-focal-fossa-linux
           sudo apt-get update
           sudo apt-get install -y texlive-latex-extra dvipng texlive-pictures

.readthedocs.yml

@@ -15,6 +15,8 @@ formats: all
 
 # Optionally set the version of Python and requirements required to build your docs
 python:
-  version: 3.7
+  version: 3.8
   install:
-    - requirements: docs/requirements.txt
+    - requirements: requirements/docs.txt
+    - method: pip
+      path: .

MANIFEST.in

@@ -27,7 +27,7 @@ recursive-exclude docs *
 exclude docs
 
 # Include the Requirements
-include requirements.txt
+recursive-include requirements *.txt
 
 # Exclude Makefile
 exclude Makefile

Makefile

@@ -6,8 +6,8 @@ export SLURM_LOCALID=0
 export SPHINX_MOCK_REQUIREMENTS=0
 
 test:
-	pip install -q -r requirements.txt
-	pip install -q -r tests/requirements.txt
+	pip install -q -r requirements/base.txt
+	pip install -q -r requirements/test.txt
 
 	# use this to run tests
 	rm -rf _ckpt_*
@@ -19,7 +19,7 @@ test:
 	# python -m coverage run --source src/pl_devtools -m pytest --flake8 --durations=0 -v -k
 
 docs: clean
-	pip install -e . -r docs/requirements.txt
+	pip install -e . -q -r requirements/docs.txt
 	cd docs && $(MAKE) html
 
 clean:

docs/source/conf.py

@@ -10,42 +10,31 @@
 # If extensions (or modules to document with autodoc) are in another directory,
 # add these directories to sys.path here. If the directory is relative to the
 # documentation root, use os.path.abspath to make it absolute, like shown here.
-
 import glob
 import inspect
 import os
 import re
-import shutil
 import sys
-from importlib.util import module_from_spec, spec_from_file_location
 
 import pt_lightning_sphinx_theme
 
+import pl_devtools
+
 _PATH_HERE = os.path.abspath(os.path.dirname(__file__))
 _PATH_ROOT = os.path.realpath(os.path.join(_PATH_HERE, "..", ".."))
-sys.path.insert(0, os.path.abspath(_PATH_ROOT))
-
 SPHINX_MOCK_REQUIREMENTS = int(os.environ.get("SPHINX_MOCK_REQUIREMENTS", True))
 
-# alternative https://stackoverflow.com/a/67692/4521646
-spec = spec_from_file_location(
-    "pl_devtools/__about__.py",
-    os.path.join(_PATH_ROOT, "src", "pl_devtools", "__about__.py"),
-)
-about = module_from_spec(spec)
-spec.loader.exec_module(about)
-
 # -- Project information -----------------------------------------------------
 
 # this name shall match the project name in Github as it is used for linking to code
 project = "Lightning-Dev-Tools"
-copyright = about.__copyright__
-author = about.__author__
+copyright = pl_devtools.__copyright__
+author = pl_devtools.__author__
 
 # The short X.Y version
-version = about.__version__
+version = pl_devtools.__version__
 # The full version, including alpha/beta/rc tags
-release = about.__version__
+release = pl_devtools.__version__
 
 # Options for the linkcode extension
 # ----------------------------------
@@ -173,8 +162,8 @@ def _convert_markdown(path_in: str, path_out: str) -> None:
 # documentation.
 
 html_theme_options = {
-    "pytorch_project": about.__homepage__,
-    "canonical_url": about.__homepage__,
+    "pytorch_project": pl_devtools.__homepage__,
+    "canonical_url": pl_devtools.__homepage__,
     "collapse_navigation": False,
     "display_version": True,
     "logo_only": False,
@@ -240,7 +229,7 @@ def _convert_markdown(path_in: str, path_out: str) -> None:
         project + " Documentation",
         author,
         project,
-        about.__docs__,
+        pl_devtools.__docs__,
         "Miscellaneous",
     ),
 ]
@@ -285,15 +274,6 @@ def setup(app):
     app.add_js_file("copybutton.js")
 
 
-# copy all notebooks to local folder
-path_nbs = os.path.join(_PATH_HERE, "notebooks")
-if not os.path.isdir(path_nbs):
-    os.mkdir(path_nbs)
-for path_ipynb in glob.glob(os.path.join(_PATH_ROOT, "notebooks", "*.ipynb")):
-    path_ipynb2 = os.path.join(path_nbs, os.path.basename(path_ipynb))
-    shutil.copy(path_ipynb, path_ipynb2)
-
-
 # Ignoring Third-party packages
 # https://stackoverflow.com/questions/15889621/sphinx-how-to-exclude-imports-in-automodule
 def _package_list_from_file(file):
@@ -315,7 +295,7 @@ def _package_list_from_file(file):
 MOCK_PACKAGES = []
 if SPHINX_MOCK_REQUIREMENTS:
     # mock also base packages when we are on RTD since we don't install them there
-    MOCK_PACKAGES += _package_list_from_file(os.path.join(_PATH_ROOT, "requirements.txt"))
+    MOCK_PACKAGES += _package_list_from_file(os.path.join(_PATH_ROOT, "requirements", "base.txt"))
 MOCK_PACKAGES = [PACKAGE_MAPPING.get(pkg, pkg) for pkg in MOCK_PACKAGES]
 
 autodoc_mock_imports = MOCK_PACKAGES
@@ -392,13 +372,6 @@ def find_source():
 # only run doctests marked with a ".. doctest::" directive
 doctest_test_doctest_blocks = ""
 doctest_global_setup = """
-
-import importlib
 import os
-import torch
-
-import pytorch_lightning as pl
-from pytorch_lightning import Trainer, LightningModule
-
 """
 coverage_skip_undoc_in_source = True

requirements/base.txt

@@ -0,0 +1 @@
+fire

setup.py

@@ -3,6 +3,7 @@
 import os
 from importlib.util import module_from_spec, spec_from_file_location
 
+from pkg_resources import parse_requirements
 from setuptools import find_packages, setup
 
 _PATH_ROOT = os.path.realpath(os.path.dirname(__file__))
@@ -18,9 +19,10 @@ def _load_py_module(fname, pkg="pl_devtools"):
 
 
 about = _load_py_module("__about__.py")
-setup_tools = _load_py_module("setup_tools.py")
-long_description = setup_tools._load_readme_description(_PATH_ROOT, homepage=about.__homepage__, ver=about.__version__)
-
+with open(os.path.join(_PATH_REQUIRE, "base.txt")) as fp:
+    requirements = list(map(str, parse_requirements(fp.readline())))
+with open(os.path.join(_PATH_ROOT, "README.md")) as fp:
+    readme = fp.read()
 
 setup(
     name="lightning-devtools",
@@ -33,14 +35,14 @@ def _load_py_module(fname, pkg="pl_devtools"):
     license=about.__license__,
     packages=find_packages(where="src"),
     package_dir={"": "src"},
-    long_description=long_description,
+    long_description=readme,
     long_description_content_type="text/markdown",
     include_package_data=True,
     zip_safe=False,
-    keywords=["deep learning", "pytorch", "AI"],
+    keywords=["DevOps", "CI/CD"],
     python_requires=">=3.7",
     setup_requires=[],
-    install_requires=setup_tools._load_requirements(path_dir=_PATH_ROOT, file_name="requirements.txt"),
+    install_requires=requirements,
     project_urls={
         "Bug Tracker": "https://github.com/Lightning-AI/dev-toolbox/issues",
         "Documentation": "https://dev-toolbox.rtfd.io/en/latest/",
@@ -54,9 +56,6 @@ def _load_py_module(fname, pkg="pl_devtools"):
         "Development Status :: 3 - Alpha",
         # Indicate who your project is intended for
         "Intended Audience :: Developers",
-        "Topic :: Scientific/Engineering :: Artificial Intelligence",
-        "Topic :: Scientific/Engineering :: Image Recognition",
-        "Topic :: Scientific/Engineering :: Information Analysis",
         # Pick your license as you wish
         # 'License :: OSI Approved :: BSD License',
         "Operating System :: OS Independent",

src/pl_devtools/__main__.py

@@ -0,0 +1,18 @@
+import fire
+
+from pl_devtools.dependencies import replace_oldest_ver, requirements_prune_pkgs
+
+
+def main() -> None:
+    fire.Fire(
+        {
+            "requirements": {
+                "prune-pkgs": requirements_prune_pkgs,
+                "set-oldest": replace_oldest_ver,
+            }
+        }
+    )
+
+
+if __name__ == "__main__":
+    main()

src/pl_devtools/dependencies.py

@@ -0,0 +1,44 @@
+import glob
+import os.path
+from pprint import pprint
+from typing import Sequence
+
+REQUIREMENT_ROOT = "requirements.txt"
+REQUIREMENT_FILES_ALL: list = glob.glob(os.path.join("requirements", "*.txt"))
+REQUIREMENT_FILES_ALL += glob.glob(os.path.join("requirements", "**", "*.txt"), recursive=True)
+if os.path.isfile(REQUIREMENT_ROOT):
+    REQUIREMENT_FILES_ALL += [REQUIREMENT_ROOT]
+
+
+def requirements_prune_pkgs(packages: Sequence[str], req_files: Sequence[str] = REQUIREMENT_FILES_ALL) -> None:
+    """Remove some packages from given requirement files."""
+    if isinstance(req_files, str):
+        req_files = [req_files]
+    for req in req_files:
+        _prune_packages(req, packages)
+
+
+def _prune_packages(req_file: str, packages: Sequence[str]) -> None:
+    """Remove some packages from given requirement files."""
+    with open(req_file) as fp:
+        lines = fp.readlines()
+
+    if isinstance(packages, str):
+        packages = [packages]
+    for pkg in packages:
+        lines = [ln for ln in lines if not ln.startswith(pkg)]
+    pprint(lines)
+
+    with open(req_file, "w") as fp:
+        fp.writelines(lines)
+
+
+def _replace_min(fname: str) -> None:
+    req = open(fname).read().replace(">=", "==")
+    open(fname, "w").write(req)
+
+
+def replace_oldest_ver(req_files: Sequence[str] = REQUIREMENT_FILES_ALL) -> None:
+    """Replace the min package version by fixed one."""
+    for fname in req_files:
+        _replace_min(fname)