diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml index 6d814bc..8172336 100644 --- a/.github/workflows/release.yml +++ b/.github/workflows/release.yml @@ -19,6 +19,10 @@ on: description: 'Skip release' required: false default: 'false' + update_notebook_visualisations: + description: 'Update notebook visualisations' + required: false + default: 'false' jobs: version_number: @@ -524,4 +528,131 @@ jobs: VERSION="${{ needs.version_number.outputs.version_number }}" git tag -m "Release $VERSION" -a "$VERSION" git push - git push --tags \ No newline at end of file + git push --tags + + + run_notebooks_and_release_workspaces: + runs-on: ubuntu-latest + env: + TWINE_REPOSITORY: pypi + TWINE_USERNAME: __token__ + TWINE_PASSWORD: ${{ secrets.PYPI }} + needs: + - run_scripts + - release_test_pypi + - version_number + - find_scripts + strategy: + fail-fast: false + matrix: + python-version: [ 3.11 ] + project: + - name: autofit + repository: rhayes777/PyAutoFit + workspace_repository: Jammy2211/autofit_workspace + pat: PAT_RICH + - name: autogalaxy + repository: Jammy2211/PyAutoGalaxy + workspace_repository: Jammy2211/autogalaxy_workspace + pat: PAT_JAMES + - name: autolens + repository: Jammy2211/PyAutoLens + workspace_repository: Jammy2211/autolens_workspace + pat: PAT_JAMES + - name: autocti + repository: Jammy2211/PyAutoCTI + workspace_repository: Jammy2211/autocti_workspace + pat: PAT_JAMES + steps: + - name: Configure + id: configure + if: "${{ github.event.inputs.update_notebook_visualisations == 'true' }}" + run: | + echo "::set-output name=repository::${{ matrix.project.repository }}" + echo "::set-output name=workspace_repository::${{ matrix.project.workspace_repository }}" + echo "::set-output name=project::${{ matrix.project.name }}" + echo "::set-output name=pat::${{ matrix.project.pat }}" + - uses: actions/checkout@v2 + if: "${{ github.event.inputs.update_notebook_visualisations == 'true' }}" + with: + path: PyAutoBuild + - name: Checkout project + if: "${{ github.event.inputs.update_notebook_visualisations == 'true' }}" + uses: actions/checkout@v2 + with: + repository: "${{ steps.configure.outputs.repository }}" + path: project + - name: Checkout workspace + if: "${{ github.event.inputs.update_notebook_visualisations == 'true' }}" + run: | + PAT="${{ secrets[steps.configure.outputs.pat] }}" + git clone -b main "https://$PAT@github.com/${{ steps.configure.outputs.workspace_repository }}.git" workspace + - name: Set up Python ${{ matrix.python-version }} + if: "${{ github.event.inputs.update_notebook_visualisations == 'true' }}" + uses: actions/setup-python@v2 + with: + python-version: ${{ matrix.python-version }} + - name: Install optional requirements + if: "${{ github.event.inputs.update_notebook_visualisations == 'true' }}" + run: | + if [ -f "project/optional_requirements.txt" ] + then + echo "Installing optional requirements" + python3 -m pip install -r "project/requirements.txt" + python3 -m pip install -r "project/optional_requirements.txt" + pip install matplotlib==3.6.0 + fi + - name: Install project + if: "${{ github.event.inputs.update_notebook_visualisations == 'true' }}" + run: | + python3 -m pip install --index-url https://test.pypi.org/simple/ --extra-index-url https://pypi.org/simple "${{ steps.configure.outputs.project }}==${{ needs.version_number.outputs.version_number }}" + - name: Install Jupyter dependency + if: "${{ github.event.inputs.update_notebook_visualisations == 'true' }}" + run: | + python3 -m pip install jupyter ipynb-py-convert + - name: Generate Jupyter notebooks + if: "${{ github.event.inputs.update_notebook_visualisations == 'true' }}" + run: | + export PYTHONPATH=$PYTHONPATH:$(pwd)/PyAutoBuild + AUTOBUILD_PATH="$(pwd)/PyAutoBuild/autobuild" + pushd "workspace" + python3 "$AUTOBUILD_PATH/generate.py" ${{ matrix.project.name }} + - name: Run Jupyter notebooks + if: "${{ github.event.inputs.update_notebook_visualisations == 'true' }}" + run: | + if [[ ${{ matrix.project.name }} == *_test ]] + then + export PYAUTOFIT_TEST_MODE=0 + else + export PYAUTOFIT_TEST_MODE=1 + fi + export PYTHONPATH=$PYTHONPATH:$(pwd)/PyAutoBuild + AUTOBUILD_PATH="$(pwd)/PyAutoBuild/autobuild" + pushd workspace + python3 "$AUTOBUILD_PATH/run.py" ${{ matrix.project.name }} "notebooks/${{ matrix.project.directory }}" --visualise + - name: Configure Git + if: "${{ github.event.inputs.update_notebook_visualisations == 'true' }}" + run: | + git config --global user.email "richard@rghsoftware.co.uk" + git config --global user.name "GitHub Actions bot" + - name: Switch to release branch + if: "${{ github.event.inputs.update_notebook_visualisations == 'true' }}" + run: | + cd workspace + git checkout release + - name: Merge main into release + if: "${{ github.event.inputs.update_notebook_visualisations == 'true' }}" + run: | + cd workspace + git merge main + - name: Commit visualizations + if: "${{ github.event.inputs.update_notebook_visualisations == 'true' }}" + run: | + cd workspace + git add . + git commit -m "Updated notebooks with visualizations" || true + - name: Push to release branch + if: "${{ github.event.inputs.update_notebook_visualisations == 'true' }}" + run: | + cd workspace + git push origin release \ No newline at end of file diff --git a/autobuild/build_util.py b/autobuild/build_util.py index de5bdf8..24b21a9 100644 --- a/autobuild/build_util.py +++ b/autobuild/build_util.py @@ -68,13 +68,19 @@ def execute_notebook(f): sys.exit(1) -def execute_notebooks_in_folder(directory, no_run_list): +def execute_notebooks_in_folder( + directory, + no_run_list, + visualise_list=None, +): no_run_list.extend(["__init__", "README"]) files = list(Path.cwd().rglob(f"{directory}/**/*.ipynb")) print(f"Found {len(files)} notebooks") for file in sorted(files): + if visualise_list is not None and file.stem not in visualise_list: + continue if file.stem not in no_run_list: execute_notebook(file) diff --git a/autobuild/config/visualise_notebooks.yaml b/autobuild/config/visualise_notebooks.yaml new file mode 100644 index 0000000..d0912b2 --- /dev/null +++ b/autobuild/config/visualise_notebooks.yaml @@ -0,0 +1,6 @@ +autofit: [] +autogalaxy: [] +autolens: [] +autocti: [] +autofit_test: [] +autolens_test: [] \ No newline at end of file diff --git a/autobuild/run.py b/autobuild/run.py index 2f3f006..2087e0e 100644 --- a/autobuild/run.py +++ b/autobuild/run.py @@ -1,19 +1,41 @@ -import sys import yaml from pathlib import Path import build_util -project = sys.argv[1] -directory = sys.argv[2] +from argparse import ArgumentParser + + +parser = ArgumentParser() + +parser.add_argument("project", type=str, help="The project to build") +parser.add_argument("directory", type=str, help="The directory to build") +parser.add_argument( + "--visualise", + action="store_true", + help="Only run notebooks for which we want to create visualisations", +) + +args = parser.parse_args() + +project = args.project +directory = args.directory +visualise = args.visualise CONFIG_PATH = Path(__file__).parent / "config" with open(CONFIG_PATH / "no_run.yaml") as f: no_run_dict = yaml.safe_load(f) +if visualise: + with open(CONFIG_PATH / "visualise_notebooks.yaml") as f: + visualise_list = yaml.safe_load(f)[project] +else: + visualise_dict = None + if __name__ == "__main__": build_util.execute_notebooks_in_folder( no_run_list=no_run_dict[project], + visualise_list=visualise_list, directory=directory, )