diff --git a/.flake8 b/.flake8 index 460421cf5f..a609f12d9c 100644 --- a/.flake8 +++ b/.flake8 @@ -1,4 +1,4 @@ [flake8] max-line-length = 88 -select = C,E,F,W,B,B950 +select = C, E, F, W, B, B950 extend-ignore = E203, E501, W503 diff --git a/.github/ISSUE_TEMPLATE/bug_report.yml b/.github/ISSUE_TEMPLATE/bug_report.yml index 3dc95c4f7f..064028d4ca 100644 --- a/.github/ISSUE_TEMPLATE/bug_report.yml +++ b/.github/ISSUE_TEMPLATE/bug_report.yml @@ -1,6 +1,5 @@ name: Bug report description: Create a report to help us improve -title: "[BUG] - " labels: ["type:Bug"] body: @@ -10,7 +9,7 @@ body: Hi! Thanks for using the Jupyter Docker Stacks and taking some time to contribute to this project. We'd appreciate it if you could check out the [Troubleshooting common problems](https://jupyter-docker-stacks.readthedocs.io/en/latest/using/troubleshooting.html) section in the documentation, - as well as [existing issues](https://github.com/jupyter/docker-stacks/issues) prior to submitting an issue to avoid duplication. + as well as [existing issues](https://github.com/jupyter/docker-stacks/issues?q=is%3Aissue) prior to submitting an issue to avoid duplication. Please answer the following sections to help us troubleshoot the problem. @@ -23,9 +22,11 @@ body: - all-spark-notebook - base-notebook - datascience-notebook + - docker-stacks-foundation - julia-notebook - minimal-notebook - pyspark-notebook + - pytorch-notebook - r-notebook - scipy-notebook - tensorflow-notebook @@ -34,10 +35,19 @@ body: - type: input attributes: - label: Host OS system and architecture running docker image + label: Host OS system placeholder: | Example: - Ubuntu 22.04 / aarch64 + Ubuntu 22.04 + validations: + required: true + + - type: dropdown + attributes: + label: Host architecture + options: + - x86_64 + - aarch64 validations: required: true @@ -48,7 +58,7 @@ body: What complete docker command do you run to launch the container (omitting sensitive values)? placeholder: | Example: - `docker run -it --rm -p 8888:8888 jupyter/all-spark-notebook` + `docker run -it --rm -p 8888:8888 quay.io/jupyter/base-notebook` validations: required: true @@ -96,7 +106,7 @@ body: description: | A clear and concise description of what the bug is. placeholder: | - Example: No output is visible in the notebook and the notebook server log contains messages about ... + Example: No output is visible in the notebook and the Server log contains messages about ... validations: required: true @@ -115,5 +125,5 @@ body: label: Latest Docker version description: You should try to use the latest Docker version options: - - label: I've updated my Docker version to the latest available, and the issue still persists + - label: I've updated my Docker version to the latest available, and the issue persists required: true diff --git a/.github/ISSUE_TEMPLATE/feature_request.yml b/.github/ISSUE_TEMPLATE/feature_request.yml index 89242a2bb3..f0da40624e 100644 --- a/.github/ISSUE_TEMPLATE/feature_request.yml +++ b/.github/ISSUE_TEMPLATE/feature_request.yml @@ -1,6 +1,5 @@ name: Feature request description: Suggest a new feature for this project -title: "[ENH] - <title>" labels: ["type:Enhancement"] body: @@ -21,9 +20,11 @@ body: - all-spark-notebook - base-notebook - datascience-notebook + - docker-stacks-foundation - julia-notebook - minimal-notebook - pyspark-notebook + - pytorch-notebook - r-notebook - scipy-notebook - tensorflow-notebook @@ -33,7 +34,7 @@ body: - type: textarea attributes: - label: What changes are you proposing? + label: What change(s) are you proposing? description: | Be concise and feel free to add supporting links or references. placeholder: | @@ -52,7 +53,7 @@ body: Example: - Altair is a declarative statistical visualization library for Python, based on Vega and Vega-Lite, and the source is available on GitHub. - With Altair, you can spend more time understanding your data and its meaning. - - Altair's API is simple, friendly and consistent and built on top of the powerful Vega-Lite visualization grammar. + - Altair's API is simple, friendly, and consistent and built on top of the powerful Vega-Lite visualization grammar. - This elegant simplicity produces beautiful and effective visualizations with a minimal amount of code. validations: required: true diff --git a/.github/actions/create-dev-env/action.yml b/.github/actions/create-dev-env/action.yml new file mode 100644 index 0000000000..7323494f6c --- /dev/null +++ b/.github/actions/create-dev-env/action.yml @@ -0,0 +1,20 @@ +name: Build environment +description: Create a build environment + +runs: + using: composite + steps: + # actions/setup-python doesn't support Linux aarch64 runners + # See: https://github.com/actions/setup-python/issues/108 + # python3 is manually preinstalled in the aarch64 VM self-hosted runner + - name: Set Up Python ๐Ÿ + uses: actions/setup-python@v5 + with: + python-version: 3.x + if: runner.arch == 'X64' + + - name: Install Dev Dependencies ๐Ÿ“ฆ + run: | + pip install --upgrade pip + pip install --upgrade -r requirements-dev.txt + shell: bash diff --git a/.github/actions/load-image/action.yml b/.github/actions/load-image/action.yml new file mode 100644 index 0000000000..cf53e4a6f6 --- /dev/null +++ b/.github/actions/load-image/action.yml @@ -0,0 +1,27 @@ +name: Load Docker image +description: Download the image tar and load it to Docker + +inputs: + image: + description: Image name + required: true + platform: + description: Image platform + required: true + variant: + description: Variant tag prefix + required: true + +runs: + using: composite + steps: + - name: Download built image ๐Ÿ“ฅ + uses: actions/download-artifact@v4 + with: + name: ${{ inputs.image }}-${{ inputs.platform }}-${{ inputs.variant }} + path: /tmp/jupyter/images/ + - name: Load downloaded image to docker ๐Ÿ“ฅ + run: | + zstd --uncompress --stdout --rm /tmp/jupyter/images/${{ inputs.image }}-${{ inputs.platform }}-${{ inputs.variant }}.tar.zst | docker load + docker image ls --all + shell: bash diff --git a/.github/dependabot.yml b/.github/dependabot.yml index 136832b017..8f17357fbb 100644 --- a/.github/dependabot.yml +++ b/.github/dependabot.yml @@ -1,7 +1,7 @@ # To get started with Dependabot version updates, you'll need to specify which # package ecosystems to update and where the package manifests are located. # Please see the documentation for all configuration options: -# https://docs.github.com/en/code-security/supply-chain-security/keeping-your-dependencies-updated-automatically/configuration-options-for-dependency-updates +# https://docs.github.com/en/code-security/dependabot/dependabot-version-updates/configuration-options-for-the-dependabot.yml-file version: 2 updates: @@ -9,3 +9,11 @@ updates: directory: / schedule: interval: weekly + - package-ecosystem: github-actions + directory: .github/actions/create-dev-env/ + schedule: + interval: weekly + - package-ecosystem: github-actions + directory: .github/actions/load-image/ + schedule: + interval: weekly diff --git a/.github/workflows/docker-build-test-upload.yml b/.github/workflows/docker-build-test-upload.yml index 539e4701ae..1dacad4179 100644 --- a/.github/workflows/docker-build-test-upload.yml +++ b/.github/workflows/docker-build-test-upload.yml @@ -1,24 +1,35 @@ -name: Download parent image, build a new one and test it; then upload the image, tags and manifests to GitHub artifacts +name: Download a parent image, build a new one, and test it; then upload the image, tags, and manifests to GitHub artifacts env: + REGISTRY: quay.io OWNER: ${{ github.repository_owner }} on: workflow_call: inputs: - parentImage: + parent-image: description: Parent image name required: true type: string + parent-variant: + description: Parent variant tag prefix + required: false + type: string + default: default image: description: Image name required: true type: string - # platform: - # description: Image platform - # required: true - # type: string - runsOn: + variant: + description: Variant tag prefix + required: false + type: string + default: default + platform: + description: Image platform + required: true + type: string + runs-on: description: GitHub Actions Runner image required: true type: string @@ -34,109 +45,60 @@ on: jobs: build-test-upload: - runs-on: ${{ inputs.runsOn }} - # This will create a registry that we use for docker images - # as an intermediary between build and test. - # services: - # registry: - # image: registry:2 - # ports: - # - 5000:5000 + runs-on: ${{ inputs.runs-on }} steps: - - name: Checkout Repo โšก๏ธ - uses: actions/checkout@v3 - - - name: Set Up Python ๐Ÿ - uses: actions/setup-python@v4 - with: - python-version: 3.x - - - uses: actions/cache@v3 - name: Cache Python + # Image with CUDA needs extra disk space + - name: Free disk space ๐Ÿงน + if: contains(inputs.variant, 'cuda') && inputs.platform == 'x86_64' + uses: jlumbroso/free-disk-space@54081f138730dfa15788a46383842cd2f914a1be with: - path: ${{ env.pythonLocation }} - key: ${{ env.pythonLocation }}-${{ hashFiles('setup.py') }}-${{ hashFiles('requirements.txt') }} + tool-cache: false + android: true + dotnet: true + haskell: true + large-packages: false + docker-images: false + swap-storage: false - - name: Install Dev Dependencies ๐Ÿ“ฆ + - name: Checkout Repo โšก๏ธ + uses: actions/checkout@v4 + - name: Create dev environment ๐Ÿ“ฆ + uses: ./.github/actions/create-dev-env + + # Self-hosted runners share a state (whole VM) between runs + # Also, they might have running or stopped containers, + # which are not cleaned up by `docker system prune` + - name: Reset docker state and cleanup artifacts ๐Ÿ—‘๏ธ + if: inputs.platform != 'x86_64' run: | pip install --upgrade pip pip install --upgrade -r requirements-dev.txt shell: bash - - name: Set up Docker Buildx - uses: docker/setup-buildx-action@v2 - with: - driver-opts: | - network=host - - - name: Download artifact ๐Ÿ“ฅ - if: ${{ inputs.parentImage != '' }} - uses: actions/download-artifact@v3 - with: - name: ${{ inputs.parentImage }} - path: /tmp/ - - - name: Load parent built image to Docker ๐Ÿณ - if: ${{ inputs.parentImage != '' }} - run: | - docker load --input /tmp/${{ inputs.parentImage }}.tar - - - name: Login to Docker Hub ๐Ÿ” - uses: docker/login-action@f4ef78c080cd8ba55a85445d5b36e214a81df20a # v2.1.0 - with: - username: ${{ secrets.DOCKERHUB_USERNAME }} - password: ${{ secrets.DOCKERHUB_TOKEN }} - logout: true - - - name: Login to Containers ๐Ÿซ™ - uses: docker/login-action@f4ef78c080cd8ba55a85445d5b36e214a81df20a # v2.1.0 - with: - registry: containers.renci.org - username: ${{ secrets.CONTAINERHUB_USERNAME }} - password: ${{ secrets.CONTAINERHUB_TOKEN }} - logout: true - - - name: Docker meta - id: meta - uses: docker/metadata-action@v4 - with: - # list of Docker images to use as base name for tags - images: | - ${{ env.OWNER }}/${{ inputs.image }} - containers.renci.org/${{ env.OWNER }}/jupyter/docker-stacks/${{ inputs.image }} - # generate Docker tags based on the following events/attributes - tags: | - type=ref,event=branch - type=sha - - - name: Build Containers ๐Ÿ› ๏ธ - uses: docker/build-push-action@v4 + - name: Load parent built image to Docker ๐Ÿ“ฅ + if: inputs.parent-image != '' + uses: ./.github/actions/load-image with: - context: ${{ inputs.image }}/ - push: false - build-args: | - NB_GID=0 - ROOT_CONTAINER=nvidia/cuda:12.1.1-cudnn8-runtime-ubuntu22.04 - # Push to renci-registry and dockerhub here. - tags: | - ${{ steps.meta.outputs.tags }} - ${{ env.OWNER }}/${{ inputs.image }}:latest - ${{ env.OWNER }}/${{ inputs.image }}:cuda - outputs: type=docker, dest=/tmp/${{ inputs.image }}.tar - cache-from: type=registry,ref=${{ env.OWNER }}/${{ inputs.image }}:buildcache-${{ inputs.image }} - cache-to: type=registry,ref=${{ env.OWNER }}/${{ inputs.image }}:buildcache-${{ inputs.image }},mode=max + image: ${{ inputs.parent-image }} + platform: ${{ inputs.platform }} + variant: ${{ inputs.parent-variant }} - - name: Load image to docker - run: | - docker load --input /tmp/${{ inputs.image }}.tar + - name: Pull base ubuntu image ๐Ÿ“ฅ + if: inputs.parent-image == '' + run: docker pull ubuntu:24.04 + shell: bash - # - name: Run Test Docker Image - # run: | - # docker run --force-rm --rm ${{ env.OWNER }}/${{ inputs.image }}:latest + - name: Build image ๐Ÿ›  + run: docker build --rm --force-rm --tag ${{ env.REGISTRY }}/${{ env.OWNER }}/${{ inputs.image }} images/${{ inputs.image }}/${{ inputs.variant != 'default' && inputs.variant || '.' }}/ --build-arg REGISTRY=${{ env.REGISTRY }} --build-arg OWNER=${{ env.OWNER }} + env: + DOCKER_BUILDKIT: 1 + # Full logs for CI build + BUILDKIT_PROGRESS: plain + shell: bash - name: Run tests โœ… - run: python3 -m tests.run_tests --short-image-name ${{ inputs.image }} --owner ${{ env.OWNER }} + run: python3 -m tests.run_tests --short-image-name ${{ inputs.image }} --registry ${{ env.REGISTRY }} --owner ${{ env.OWNER }} shell: bash # We do not want to push "latest" tag unless @@ -146,31 +108,39 @@ jobs: - name: Remove latest tag if Develop Branch if: ${{ github.ref != 'refs/heads/main' }} run: | - docker rmi ${{ env.OWNER }}/${{ inputs.image }}:latest - - - name: Upload artifact ๐Ÿ“ค - uses: actions/upload-artifact@v3 + python3 -m tagging.write_tags_file --short-image-name ${{ inputs.image }} --tags-dir /tmp/jupyter/tags/ --registry ${{ env.REGISTRY }} --owner ${{ env.OWNER }} --variant ${{ inputs.variant }} + shell: bash + - name: Upload tags file ๐Ÿ’พ + uses: actions/upload-artifact@v4 with: - name: ${{ inputs.image }} - path: /tmp/${{ inputs.image }}.tar + name: ${{ inputs.image }}-${{ inputs.platform }}-${{ inputs.variant }}-tags + path: /tmp/jupyter/tags/${{ inputs.platform }}-${{ inputs.variant }}-${{ inputs.image }}.txt + retention-days: 3 - - name: Push to Docker ๐Ÿ› ๏ธ - run: | - docker push --all-tags ${{ env.OWNER }}/${{ inputs.image }} - - # Because Containers often fails, we keep going and display a message. - - name: Push to Containers ๐Ÿซ™ - id: containers - run: | - docker push --all-tags containers.renci.org/${{ env.OWNER }}/jupyter/docker-stacks/${{ inputs.image }} - continue-on-error: true + - name: Write manifest and build history file ๐Ÿท + run: python3 -m tagging.write_manifest --short-image-name ${{ inputs.image }} --hist-lines-dir /tmp/jupyter/hist_lines/ --manifests-dir /tmp/jupyter/manifests/ --registry ${{ env.REGISTRY }} --owner ${{ env.OWNER }} --variant ${{ inputs.variant }} + shell: bash + - name: Upload manifest file ๐Ÿ’พ + uses: actions/upload-artifact@v4 + with: + name: ${{ inputs.image }}-${{ inputs.platform }}-${{ inputs.variant }}-manifest + path: /tmp/jupyter/manifests/${{ inputs.platform }}-${{ inputs.variant }}-${{ inputs.image }}-*.md + retention-days: 3 + - name: Upload build history line ๐Ÿ’พ + uses: actions/upload-artifact@v4 + with: + name: ${{ inputs.image }}-${{ inputs.platform }}-${{ inputs.variant }}-history_line + path: /tmp/jupyter/hist_lines/${{ inputs.platform }}-${{ inputs.variant }}-${{ inputs.image }}-*.txt + retention-days: 3 - # Sometimes containers.recnci.org fails so we try again. - - name: If Containers Failed - if: ${{ steps.containers.conclusion == 'failure' }} + - name: Save image as a tar for later use ๐Ÿ’พ run: | - echo "Last push failed retrying " - docker push --all-tags containers.renci.org/${{ env.OWNER }}/jupyter/docker-stacks/${{ inputs.image }} - echo "Completed!" - continue-on-error: true - + mkdir -p /tmp/jupyter/images/ + docker save ${{ env.REGISTRY }}/${{ env.OWNER }}/${{ inputs.image }} | zstd > /tmp/jupyter/images/${{ inputs.image }}-${{ inputs.platform }}-${{ inputs.variant }}.tar.zst + shell: bash + - name: Upload image as artifact ๐Ÿ’พ + uses: actions/upload-artifact@v4 + with: + name: ${{ inputs.image }}-${{ inputs.platform }}-${{ inputs.variant }} + path: /tmp/jupyter/images/${{ inputs.image }}-${{ inputs.platform }}-${{ inputs.variant }}.tar.zst + retention-days: 3 diff --git a/.github/workflows/docker-merge-tags.yml b/.github/workflows/docker-merge-tags.yml new file mode 100644 index 0000000000..a476879408 --- /dev/null +++ b/.github/workflows/docker-merge-tags.yml @@ -0,0 +1,66 @@ +name: Download all tags from GitHub artifacts and create multi-platform manifests + +env: + OWNER: ${{ github.repository_owner }} + PUSH_TO_REGISTRY: ${{ (github.repository_owner == 'jupyter' || github.repository_owner == 'mathbunnyru') && (github.ref == 'refs/heads/main' || github.event_name == 'schedule') }} + +on: + workflow_call: + inputs: + variant: + description: Variant tag prefix + required: true + type: string + image: + description: Image name + required: true + type: string + secrets: + REGISTRY_USERNAME: + required: true + REGISTRY_TOKEN: + required: true + +jobs: + merge-tags: + runs-on: ubuntu-latest + + steps: + - name: Checkout Repo โšก๏ธ + uses: actions/checkout@v4 + - name: Create dev environment ๐Ÿ“ฆ + uses: ./.github/actions/create-dev-env + + - name: Download x86_64 tags file ๐Ÿ“ฅ + uses: actions/download-artifact@v4 + with: + name: ${{ inputs.image }}-x86_64-${{ inputs.variant }}-tags + path: /tmp/jupyter/tags/ + - name: Download aarch64 tags file ๐Ÿ“ฅ + uses: actions/download-artifact@v4 + with: + name: ${{ inputs.image }}-aarch64-${{ inputs.variant }}-tags + path: /tmp/jupyter/tags/ + if: github.repository_owner == 'jupyter' && !contains(inputs.variant, 'cuda') + + # Docker might be stuck when pulling images + # https://github.com/docker/for-mac/issues/2083 + # https://stackoverflow.com/questions/38087027/docker-compose-stuck-downloading-or-pulling-fs-layer + - name: Reset docker state ๐Ÿ—‘๏ธ + run: | + docker system prune --all --force + sudo systemctl restart docker + shell: bash + + - name: Login to Registry ๐Ÿ” + if: env.PUSH_TO_REGISTRY == 'true' + uses: docker/login-action@9780b0c442fbb1117ed29e0efdff1e18412f7567 # v3.3.0 + with: + registry: quay.io + username: ${{ secrets.REGISTRY_USERNAME }} + password: ${{ secrets.REGISTRY_TOKEN }} + + - name: Merge tags for the images ๐Ÿ”€ + if: env.PUSH_TO_REGISTRY == 'true' + run: python3 -m tagging.merge_tags --short-image-name ${{ inputs.image }} --tags-dir /tmp/jupyter/tags/ --variant ${{ inputs.variant }} + shell: bash diff --git a/.github/workflows/docker-tag-push.yml b/.github/workflows/docker-tag-push.yml new file mode 100644 index 0000000000..887e2d2e01 --- /dev/null +++ b/.github/workflows/docker-tag-push.yml @@ -0,0 +1,80 @@ +name: Download a Docker image and its tags from GitHub artifacts, apply them, and push the image to the Registry + +env: + REGISTRY: quay.io + OWNER: ${{ github.repository_owner }} + PUSH_TO_REGISTRY: ${{ (github.repository_owner == 'jupyter' || github.repository_owner == 'mathbunnyru') && (github.ref == 'refs/heads/main' || github.event_name == 'schedule') }} + +on: + workflow_call: + inputs: + image: + description: Image name + required: true + type: string + platform: + description: Image platform + required: true + type: string + variant: + description: Variant tag prefix + required: true + type: string + secrets: + REGISTRY_USERNAME: + required: true + REGISTRY_TOKEN: + required: true + +jobs: + tag-push: + runs-on: ubuntu-latest + + steps: + # Image with CUDA needs extra disk space + - name: Free disk space ๐Ÿงน + if: contains(inputs.variant, 'cuda') && inputs.platform == 'x86_64' + uses: jlumbroso/free-disk-space@54081f138730dfa15788a46383842cd2f914a1be + with: + tool-cache: false + android: true + dotnet: true + haskell: true + large-packages: false + docker-images: false + swap-storage: false + + - name: Checkout Repo โšก๏ธ + uses: actions/checkout@v4 + - name: Create dev environment ๐Ÿ“ฆ + uses: ./.github/actions/create-dev-env + - name: Load image to Docker ๐Ÿ“ฅ + uses: ./.github/actions/load-image + with: + image: ${{ inputs.image }} + platform: ${{ inputs.platform }} + variant: ${{ inputs.variant }} + + - name: Login to Registry ๐Ÿ” + if: env.PUSH_TO_REGISTRY == 'true' + uses: docker/login-action@9780b0c442fbb1117ed29e0efdff1e18412f7567 # v3.3.0 + with: + registry: quay.io + username: ${{ secrets.REGISTRY_USERNAME }} + password: ${{ secrets.REGISTRY_TOKEN }} + + - name: Download tags file ๐Ÿ“ฅ + uses: actions/download-artifact@v4 + with: + name: ${{ inputs.image }}-${{ inputs.platform }}-${{ inputs.variant }}-tags + path: /tmp/jupyter/tags/ + - name: Apply tags to the loaded image ๐Ÿท + run: python3 -m tagging.apply_tags --short-image-name ${{ inputs.image }} --tags-dir /tmp/jupyter/tags/ --platform ${{ inputs.platform }} --variant ${{ inputs.variant }} --registry ${{ env.REGISTRY }} --owner ${{ env.OWNER }} + # This step is needed to prevent pushing non-multi-arch "latest" tag + - name: Remove the "latest" tag from the image ๐Ÿ—‘๏ธ + run: docker image rmi ${{ env.REGISTRY }}/${{ env.OWNER }}/${{ inputs.image }}:latest + + - name: Push Images to Registry ๐Ÿ“ค + if: env.PUSH_TO_REGISTRY == 'true' + run: docker push --all-tags ${{ env.REGISTRY }}/${{ env.OWNER }}/${{ inputs.image }} + shell: bash diff --git a/.github/workflows/docker-wiki-update.yml b/.github/workflows/docker-wiki-update.yml new file mode 100644 index 0000000000..b0abefc513 --- /dev/null +++ b/.github/workflows/docker-wiki-update.yml @@ -0,0 +1,48 @@ +name: Download build manifests from GitHub artifacts and push them to GitHub wiki +# We're doing everything in one workflow on purpose +# This way we make sure we don't access wiki pages from several jobs simultaneously + +env: + PUSH_TO_REGISTRY: ${{ (github.repository_owner == 'jupyter' || github.repository_owner == 'mathbunnyru') && (github.ref == 'refs/heads/main' || github.event_name == 'schedule') }} + +on: + workflow_call: + +jobs: + wiki-update: + runs-on: ubuntu-latest + + steps: + - name: Checkout Repo โšก๏ธ + uses: actions/checkout@v4 + - name: Create dev environment ๐Ÿ“ฆ + uses: ./.github/actions/create-dev-env + + - name: Download all history lines ๐Ÿ“ฅ + uses: actions/download-artifact@v4 + with: + pattern: "*-history_line" + path: /tmp/jupyter/hist_lines/ + + - name: Download all manifests ๐Ÿ“ฅ + uses: actions/download-artifact@v4 + with: + pattern: "*-manifest" + path: /tmp/jupyter/manifests/ + + - name: Checkout Wiki Repo ๐Ÿ“ƒ + uses: actions/checkout@v4 + with: + repository: ${{ github.repository }}.wiki + path: wiki/ + + - name: Update wiki ๐Ÿท + run: python3 -m tagging.update_wiki --wiki-dir wiki/ --hist-lines-dir /tmp/jupyter/hist_lines/ --manifests-dir /tmp/jupyter/manifests/ + shell: bash + + - name: Push Wiki to GitHub ๐Ÿ“ค + if: env.PUSH_TO_REGISTRY == 'true' + uses: stefanzweifel/git-auto-commit-action@8621497c8c39c72f3e2a999a26b4ca1b5058a842 # v5.0.1 + with: + commit_message: "Automated wiki publish for ${{ github.sha }}" + repository: wiki/ diff --git a/.github/workflows/docker.yml b/.github/workflows/docker.yml index 448417b377..a3163abe6b 100644 --- a/.github/workflows/docker.yml +++ b/.github/workflows/docker.yml @@ -1,152 +1,508 @@ -name: Build, test and push Docker Images +name: Build, test, and push Docker Images + +# [FAST_BUILD] in the PR title makes this workflow only build +# the `jupyter/docker-stacks-foundation` and `jupyter/base-notebook` images +# This allows to run CI faster if a full build is not required +# This only works for a `pull_request` event and does not affect `push` to the `main` branch on: - # schedule: - # # Weekly, at 03:00 on Monday UTC time - # - cron: "0 3 * * 1" + schedule: + # Weekly, at 03:00 on Monday UTC + - cron: "0 3 * * 1" + pull_request: + paths: + - ".github/workflows/docker.yml" + # We use local reusable workflows to make architecture clean and simple + # https://docs.github.com/en/actions/using-workflows/reusing-workflows + - ".github/workflows/docker-build-test-upload.yml" + - ".github/workflows/docker-merge-tags.yml" + - ".github/workflows/docker-tag-push.yml" + - ".github/workflows/docker-wiki-update.yml" + + # We use local composite actions to combine multiple workflow steps within one action + # https://docs.github.com/en/actions/creating-actions/about-custom-actions#composite-actions + - ".github/actions/create-dev-env/action.yml" + - ".github/actions/load-image/action.yml" + + - "images/**" + - "!images/*/README.md" + - "tagging/**" + - "!tagging/README.md" + - "tests/**" + - "!tests/README.md" + - "requirements-dev.txt" push: branches: - main - - develop - # paths-ignore: - # - ".github/workflows/*" - # - ".github/*" + paths: + - ".github/workflows/docker.yml" + - ".github/workflows/docker-build-test-upload.yml" + - ".github/workflows/docker-merge-tags.yml" + - ".github/workflows/docker-tag-push.yml" + - ".github/workflows/docker-wiki-update.yml" + + - ".github/actions/create-dev-env/action.yml" + - ".github/actions/load-image/action.yml" + + - "images/**" + - "!images/*/README.md" + - "tagging/**" + - "!tagging/README.md" + - "tests/**" + - "!tests/README.md" + - "requirements-dev.txt" workflow_dispatch: # https://docs.github.com/en/actions/using-jobs/using-concurrency concurrency: - # only cancel in-progress jobs or runs for the current workflow - matches against branch & tags + # Only cancel in-progress jobs or runs for the current workflow - matches against branch & tags group: ${{ github.workflow }}-${{ github.ref }} cancel-in-progress: true jobs: + aarch64-foundation: + uses: ./.github/workflows/docker-build-test-upload.yml + with: + parent-image: "" + image: docker-stacks-foundation + platform: aarch64 + runs-on: ARM64_FAST + if: github.repository_owner == 'jupyter' foundation: uses: ./.github/workflows/docker-build-test-upload.yml with: - parentImage: "" + parent-image: "" image: docker-stacks-foundation - runsOn: ubuntu-latest - secrets: - DOCKERHUB_USERNAME: ${{ secrets.DOCKERHUB_USERNAME }} - DOCKERHUB_TOKEN: ${{ secrets.DOCKERHUB_TOKEN }} - CONTAINERHUB_USERNAME: ${{ secrets.CONTAINERHUB_USERNAME }} - CONTAINERHUB_TOKEN: ${{ secrets.CONTAINERHUB_TOKEN }} + platform: x86_64 + runs-on: ubuntu-latest - base: + aarch64-base: uses: ./.github/workflows/docker-build-test-upload.yml with: - parentImage: docker-stacks-foundation + parent-image: docker-stacks-foundation image: base-notebook - runsOn: ubuntu-latest - secrets: - DOCKERHUB_USERNAME: ${{ secrets.DOCKERHUB_USERNAME }} - DOCKERHUB_TOKEN: ${{ secrets.DOCKERHUB_TOKEN }} - CONTAINERHUB_USERNAME: ${{ secrets.CONTAINERHUB_USERNAME }} - CONTAINERHUB_TOKEN: ${{ secrets.CONTAINERHUB_TOKEN }} - needs: [foundation] + platform: aarch64 + runs-on: ARM64_FAST + needs: [aarch64-foundation] + if: github.repository_owner == 'jupyter' + + x86_64-base: + uses: ./.github/workflows/docker-build-test-upload.yml + with: + parent-image: docker-stacks-foundation + image: base-notebook + platform: x86_64 + runs-on: ubuntu-latest + needs: [x86_64-foundation] + + aarch64-minimal: + uses: ./.github/workflows/docker-build-test-upload.yml + with: + parent-image: base-notebook + image: minimal-notebook + platform: aarch64 + runs-on: ARM64_FAST + needs: [aarch64-base] + if: github.repository_owner == 'jupyter' && !contains(github.event.pull_request.title, '[FAST_BUILD]') minimal: uses: ./.github/workflows/docker-build-test-upload.yml with: - parentImage: base-notebook + parent-image: base-notebook image: minimal-notebook - runsOn: ubuntu-latest - secrets: - DOCKERHUB_USERNAME: ${{ secrets.DOCKERHUB_USERNAME }} - DOCKERHUB_TOKEN: ${{ secrets.DOCKERHUB_TOKEN }} - CONTAINERHUB_USERNAME: ${{ secrets.CONTAINERHUB_USERNAME }} - CONTAINERHUB_TOKEN: ${{ secrets.CONTAINERHUB_TOKEN }} - needs: [base] + platform: x86_64 + runs-on: ubuntu-latest + needs: [x86_64-base] + if: ${{ !contains(github.event.pull_request.title, '[FAST_BUILD]') }} + + aarch64-scipy: + uses: ./.github/workflows/docker-build-test-upload.yml + with: + parent-image: minimal-notebook + image: scipy-notebook + platform: aarch64 + runs-on: ARM64_FAST + needs: [aarch64-minimal] + if: github.repository_owner == 'jupyter' && !contains(github.event.pull_request.title, '[FAST_BUILD]') scipy: uses: ./.github/workflows/docker-build-test-upload.yml with: - parentImage: minimal-notebook + parent-image: minimal-notebook image: scipy-notebook - runsOn: ubuntu-latest - secrets: - DOCKERHUB_USERNAME: ${{ secrets.DOCKERHUB_USERNAME }} - DOCKERHUB_TOKEN: ${{ secrets.DOCKERHUB_TOKEN }} - CONTAINERHUB_USERNAME: ${{ secrets.CONTAINERHUB_USERNAME }} - CONTAINERHUB_TOKEN: ${{ secrets.CONTAINERHUB_TOKEN }} - needs: [minimal] + platform: x86_64 + runs-on: ubuntu-latest + needs: [x86_64-minimal] + if: ${{ !contains(github.event.pull_request.title, '[FAST_BUILD]') }} + + aarch64-r: + uses: ./.github/workflows/docker-build-test-upload.yml + with: + parent-image: minimal-notebook + image: r-notebook + platform: aarch64 + runs-on: ARM64_SLOW + needs: [aarch64-minimal] + if: github.repository_owner == 'jupyter' && !contains(github.event.pull_request.title, '[FAST_BUILD]') r: uses: ./.github/workflows/docker-build-test-upload.yml with: - parentImage: minimal-notebook + parent-image: minimal-notebook image: r-notebook - runsOn: ubuntu-latest - secrets: - DOCKERHUB_USERNAME: ${{ secrets.DOCKERHUB_USERNAME }} - DOCKERHUB_TOKEN: ${{ secrets.DOCKERHUB_TOKEN }} - CONTAINERHUB_USERNAME: ${{ secrets.CONTAINERHUB_USERNAME }} - CONTAINERHUB_TOKEN: ${{ secrets.CONTAINERHUB_TOKEN }} - needs: [minimal] + platform: x86_64 + runs-on: ubuntu-latest + needs: [x86_64-minimal] + if: ${{ !contains(github.event.pull_request.title, '[FAST_BUILD]') }} - tensorflow: + aarch64-julia: uses: ./.github/workflows/docker-build-test-upload.yml with: - parentImage: scipy-notebook + parent-image: minimal-notebook + image: julia-notebook + platform: aarch64 + runs-on: ARM64_SLOW + needs: [aarch64-minimal] + if: github.repository_owner == 'jupyter' && !contains(github.event.pull_request.title, '[FAST_BUILD]') + + x86_64-julia: + uses: ./.github/workflows/docker-build-test-upload.yml + with: + parent-image: minimal-notebook + image: julia-notebook + platform: x86_64 + runs-on: ubuntu-latest + needs: [x86_64-minimal] + if: ${{ !contains(github.event.pull_request.title, '[FAST_BUILD]') }} + + aarch64-tensorflow: + uses: ./.github/workflows/docker-build-test-upload.yml + with: + parent-image: scipy-notebook image: tensorflow-notebook - runsOn: ubuntu-latest - secrets: - DOCKERHUB_USERNAME: ${{ secrets.DOCKERHUB_USERNAME }} - DOCKERHUB_TOKEN: ${{ secrets.DOCKERHUB_TOKEN }} - CONTAINERHUB_USERNAME: ${{ secrets.CONTAINERHUB_USERNAME }} - CONTAINERHUB_TOKEN: ${{ secrets.CONTAINERHUB_TOKEN }} - needs: [scipy] + platform: aarch64 + runs-on: ARM64_SLOW + needs: [aarch64-scipy] + if: github.repository_owner == 'jupyter' && !contains(github.event.pull_request.title, '[FAST_BUILD]') + + x86_64-tensorflow: + uses: ./.github/workflows/docker-build-test-upload.yml + with: + parent-image: scipy-notebook + image: tensorflow-notebook + platform: x86_64 + runs-on: ubuntu-latest + needs: [x86_64-scipy] + if: ${{ !contains(github.event.pull_request.title, '[FAST_BUILD]') }} + + x86_64-tensorflow-cuda: + uses: ./.github/workflows/docker-build-test-upload.yml + with: + parent-image: scipy-notebook + image: tensorflow-notebook + variant: cuda + platform: x86_64 + runs-on: ubuntu-latest + needs: [x86_64-scipy] + if: ${{ !contains(github.event.pull_request.title, '[FAST_BUILD]') }} + + aarch64-pytorch: + uses: ./.github/workflows/docker-build-test-upload.yml + with: + parent-image: scipy-notebook + image: pytorch-notebook + platform: aarch64 + runs-on: ARM64_SLOW + needs: [aarch64-scipy] + if: github.repository_owner == 'jupyter' && !contains(github.event.pull_request.title, '[FAST_BUILD]') + + x86_64-pytorch: + uses: ./.github/workflows/docker-build-test-upload.yml + with: + parent-image: scipy-notebook + image: pytorch-notebook + platform: x86_64 + runs-on: ubuntu-latest + needs: [x86_64-scipy] + if: ${{ !contains(github.event.pull_request.title, '[FAST_BUILD]') }} + + x86_64-pytorch-cuda11: + uses: ./.github/workflows/docker-build-test-upload.yml + with: + parent-image: scipy-notebook + image: pytorch-notebook + variant: cuda11 + platform: x86_64 + runs-on: ubuntu-latest + needs: [x86_64-scipy] + if: ${{ !contains(github.event.pull_request.title, '[FAST_BUILD]') }} + + x86_64-pytorch-cuda12: + uses: ./.github/workflows/docker-build-test-upload.yml + with: + parent-image: scipy-notebook + image: pytorch-notebook + variant: cuda12 + platform: x86_64 + runs-on: ubuntu-latest + needs: [x86_64-scipy] + if: ${{ !contains(github.event.pull_request.title, '[FAST_BUILD]') }} datascience: uses: ./.github/workflows/docker-build-test-upload.yml with: - parentImage: scipy-notebook + parent-image: scipy-notebook image: datascience-notebook - runsOn: ubuntu-latest - secrets: - DOCKERHUB_USERNAME: ${{ secrets.DOCKERHUB_USERNAME }} - DOCKERHUB_TOKEN: ${{ secrets.DOCKERHUB_TOKEN }} - CONTAINERHUB_USERNAME: ${{ secrets.CONTAINERHUB_USERNAME }} - CONTAINERHUB_TOKEN: ${{ secrets.CONTAINERHUB_TOKEN }} - needs: [scipy] + platform: aarch64 + runs-on: ARM64_SLOW + needs: [aarch64-scipy] + if: github.repository_owner == 'jupyter' && !contains(github.event.pull_request.title, '[FAST_BUILD]') - julia: + x86_64-datascience: uses: ./.github/workflows/docker-build-test-upload.yml with: - parentImage: minimal-notebook - image: julia-notebook - runsOn: ubuntu-latest - secrets: - DOCKERHUB_USERNAME: ${{ secrets.DOCKERHUB_USERNAME }} - DOCKERHUB_TOKEN: ${{ secrets.DOCKERHUB_TOKEN }} - CONTAINERHUB_USERNAME: ${{ secrets.CONTAINERHUB_USERNAME }} - CONTAINERHUB_TOKEN: ${{ secrets.CONTAINERHUB_TOKEN }} - needs: [minimal] + parent-image: scipy-notebook + image: datascience-notebook + platform: x86_64 + runs-on: ubuntu-latest + needs: [x86_64-scipy] + if: ${{ !contains(github.event.pull_request.title, '[FAST_BUILD]') }} + + aarch64-pyspark: + uses: ./.github/workflows/docker-build-test-upload.yml + with: + parent-image: scipy-notebook + image: pyspark-notebook + platform: aarch64 + runs-on: ARM64_FAST + needs: [aarch64-scipy] + if: github.repository_owner == 'jupyter' && !contains(github.event.pull_request.title, '[FAST_BUILD]') pyspark: uses: ./.github/workflows/docker-build-test-upload.yml with: - parentImage: scipy-notebook + parent-image: scipy-notebook image: pyspark-notebook - runsOn: ubuntu-latest - secrets: - DOCKERHUB_USERNAME: ${{ secrets.DOCKERHUB_USERNAME }} - DOCKERHUB_TOKEN: ${{ secrets.DOCKERHUB_TOKEN }} - CONTAINERHUB_USERNAME: ${{ secrets.CONTAINERHUB_USERNAME }} - CONTAINERHUB_TOKEN: ${{ secrets.CONTAINERHUB_TOKEN }} - needs: [scipy] + platform: x86_64 + runs-on: ubuntu-latest + needs: [x86_64-scipy] + if: ${{ !contains(github.event.pull_request.title, '[FAST_BUILD]') }} - all-spark: + aarch64-all-spark: uses: ./.github/workflows/docker-build-test-upload.yml with: - parentImage: pyspark-notebook + parent-image: pyspark-notebook image: all-spark-notebook - runsOn: ubuntu-latest + platform: aarch64 + runs-on: ARM64_FAST + needs: [aarch64-pyspark] + if: github.repository_owner == 'jupyter' && !contains(github.event.pull_request.title, '[FAST_BUILD]') + + x86_64-all-spark: + uses: ./.github/workflows/docker-build-test-upload.yml + with: + parent-image: pyspark-notebook + image: all-spark-notebook + platform: x86_64 + runs-on: ubuntu-latest + needs: [x86_64-pyspark] + if: ${{ !contains(github.event.pull_request.title, '[FAST_BUILD]') }} + + aarch64-images-tag-push: + uses: ./.github/workflows/docker-tag-push.yml + with: + platform: aarch64 + image: ${{ matrix.image-variant.image }} + variant: ${{ matrix.image-variant.variant }} + secrets: + REGISTRY_USERNAME: ${{ secrets.QUAY_USERNAME }} + REGISTRY_TOKEN: ${{ secrets.QUAY_ROBOT_TOKEN }} + strategy: + matrix: + image-variant: + [ + { image: docker-stacks-foundation, variant: default }, + { image: base-notebook, variant: default }, + { image: minimal-notebook, variant: default }, + { image: scipy-notebook, variant: default }, + { image: r-notebook, variant: default }, + { image: julia-notebook, variant: default }, + { image: tensorflow-notebook, variant: default }, + { image: pytorch-notebook, variant: default }, + { image: datascience-notebook, variant: default }, + { image: pyspark-notebook, variant: default }, + { image: all-spark-notebook, variant: default }, + ] + needs: + [ + aarch64-foundation, + aarch64-base, + aarch64-minimal, + aarch64-scipy, + aarch64-r, + aarch64-julia, + aarch64-tensorflow, + aarch64-pytorch, + aarch64-datascience, + aarch64-pyspark, + aarch64-all-spark, + ] + if: github.repository_owner == 'jupyter' && !contains(github.event.pull_request.title, '[FAST_BUILD]') + + aarch64-images-tag-push-fast: + uses: ./.github/workflows/docker-tag-push.yml + with: + platform: aarch64 + image: ${{ matrix.image-variant.image }} + variant: ${{ matrix.image-variant.variant }} + secrets: + REGISTRY_USERNAME: ${{ secrets.QUAY_USERNAME }} + REGISTRY_TOKEN: ${{ secrets.QUAY_ROBOT_TOKEN }} + strategy: + matrix: + image-variant: + [ + { image: docker-stacks-foundation, variant: default }, + { image: base-notebook, variant: default }, + ] + needs: [aarch64-foundation, aarch64-base] + if: github.repository_owner == 'jupyter' && contains(github.event.pull_request.title, '[FAST_BUILD]') + + x86_64-images-tag-push: + uses: ./.github/workflows/docker-tag-push.yml + with: + platform: x86_64 + image: ${{ matrix.image-variant.image }} + variant: ${{ matrix.image-variant.variant }} + secrets: + REGISTRY_USERNAME: ${{ secrets.QUAY_USERNAME }} + REGISTRY_TOKEN: ${{ secrets.QUAY_ROBOT_TOKEN }} + strategy: + matrix: + image-variant: + [ + { image: docker-stacks-foundation, variant: default }, + { image: base-notebook, variant: default }, + { image: minimal-notebook, variant: default }, + { image: scipy-notebook, variant: default }, + { image: r-notebook, variant: default }, + { image: julia-notebook, variant: default }, + { image: tensorflow-notebook, variant: default }, + { image: tensorflow-notebook, variant: cuda }, + { image: pytorch-notebook, variant: default }, + { image: pytorch-notebook, variant: cuda11 }, + { image: pytorch-notebook, variant: cuda12 }, + { image: datascience-notebook, variant: default }, + { image: pyspark-notebook, variant: default }, + { image: all-spark-notebook, variant: default }, + ] + needs: + [ + x86_64-foundation, + x86_64-base, + x86_64-minimal, + x86_64-scipy, + x86_64-r, + x86_64-julia, + x86_64-tensorflow, + x86_64-pytorch, + x86_64-datascience, + x86_64-pyspark, + x86_64-all-spark, + ] + if: ${{ !contains(github.event.pull_request.title, '[FAST_BUILD]') }} + + x86_64-images-tag-push-fast: + uses: ./.github/workflows/docker-tag-push.yml + with: + platform: x86_64 + image: ${{ matrix.image-variant.image }} + variant: ${{ matrix.image-variant.variant }} + secrets: + REGISTRY_USERNAME: ${{ secrets.QUAY_USERNAME }} + REGISTRY_TOKEN: ${{ secrets.QUAY_ROBOT_TOKEN }} + strategy: + matrix: + image-variant: + [ + { image: docker-stacks-foundation, variant: default }, + { image: base-notebook, variant: default }, + ] + needs: [x86_64-foundation, x86_64-base] + if: contains(github.event.pull_request.title, '[FAST_BUILD]') + + merge-tags: + uses: ./.github/workflows/docker-merge-tags.yml + with: + image: ${{ matrix.image-variant.image }} + variant: ${{ matrix.image-variant.variant }} secrets: - DOCKERHUB_USERNAME: ${{ secrets.DOCKERHUB_USERNAME }} - DOCKERHUB_TOKEN: ${{ secrets.DOCKERHUB_TOKEN }} - CONTAINERHUB_USERNAME: ${{ secrets.CONTAINERHUB_USERNAME }} - CONTAINERHUB_TOKEN: ${{ secrets.CONTAINERHUB_TOKEN }} - needs: [pyspark] - \ No newline at end of file + REGISTRY_USERNAME: ${{ secrets.QUAY_USERNAME }} + REGISTRY_TOKEN: ${{ secrets.QUAY_ROBOT_TOKEN }} + strategy: + matrix: + image-variant: + [ + { image: docker-stacks-foundation, variant: default }, + { image: base-notebook, variant: default }, + { image: minimal-notebook, variant: default }, + { image: scipy-notebook, variant: default }, + { image: r-notebook, variant: default }, + { image: julia-notebook, variant: default }, + { image: tensorflow-notebook, variant: default }, + { image: tensorflow-notebook, variant: cuda }, + { image: pytorch-notebook, variant: default }, + { image: pytorch-notebook, variant: cuda11 }, + { image: pytorch-notebook, variant: cuda12 }, + { image: datascience-notebook, variant: default }, + { image: pyspark-notebook, variant: default }, + { image: all-spark-notebook, variant: default }, + ] + needs: [aarch64-images-tag-push, x86_64-images-tag-push] + if: | + always() && + needs.x86_64-images-tag-push.result == 'success' && + (needs.aarch64-images-tag-push.result == 'success' || needs.aarch64-images-tag-push.result == 'skipped') && + !contains(github.event.pull_request.title, '[FAST_BUILD]') + + merge-tags-fast: + uses: ./.github/workflows/docker-merge-tags.yml + with: + image: ${{ matrix.image-variant.image }} + variant: ${{ matrix.image-variant.variant }} + secrets: + REGISTRY_USERNAME: ${{ secrets.QUAY_USERNAME }} + REGISTRY_TOKEN: ${{ secrets.QUAY_ROBOT_TOKEN }} + strategy: + matrix: + image-variant: + [ + { image: docker-stacks-foundation, variant: default }, + { image: base-notebook, variant: default }, + ] + needs: [aarch64-images-tag-push-fast, x86_64-images-tag-push-fast] + if: | + always() && + needs.x86_64-images-tag-push-fast.result == 'success' && + (needs.aarch64-images-tag-push-fast.result == 'success' || needs.aarch64-images-tag-push-fast.result == 'skipped') && + contains(github.event.pull_request.title, '[FAST_BUILD]') + + wiki-update: + uses: ./.github/workflows/docker-wiki-update.yml + needs: [aarch64-images-tag-push, x86_64-images-tag-push] + if: github.repository_owner == 'jupyter' && !contains(github.event.pull_request.title, '[FAST_BUILD]') + permissions: + contents: write + + wiki-update-fast: + uses: ./.github/workflows/docker-wiki-update.yml + needs: [aarch64-images-tag-push-fast, x86_64-images-tag-push-fast] + if: github.repository_owner == 'jupyter' && contains(github.event.pull_request.title, '[FAST_BUILD]') + permissions: + contents: write + + contributed-recipes: + uses: ./.github/workflows/contributed-recipes.yml + needs: [merge-tags] + if: github.repository_owner == 'jupyter' && (github.ref == 'refs/heads/main' || github.event_name == 'schedule') diff --git a/.github/workflows/helx-build-test-upload.yml b/.github/workflows/helx-build-test-upload.yml new file mode 100644 index 0000000000..d56693b904 --- /dev/null +++ b/.github/workflows/helx-build-test-upload.yml @@ -0,0 +1,165 @@ +name: Download parent image, build a new one and test it; then upload the image, tags and manifests to GitHub artifacts + +env: + OWNER: ${{ github.repository_owner }} + +on: + workflow_call: + inputs: + parentImage: + description: Parent image name + required: true + type: string + image: + description: Image name + required: true + type: string + # platform: + # description: Image platform + # required: true + # type: string + runsOn: + description: GitHub Actions Runner image + required: true + type: string + secrets: + DOCKERHUB_USERNAME: + required: true + DOCKERHUB_TOKEN: + required: true + CONTAINERHUB_USERNAME: + required: true + CONTAINERHUB_TOKEN: + required: true + +jobs: + build-test-upload: + runs-on: ${{ inputs.runsOn }} + + steps: + - name: Checkout Repo โšก๏ธ + uses: actions/checkout@v4 + + - name: Set Up Python ๐Ÿ + uses: actions/setup-python@v5 + with: + python-version: 3.x + + - uses: actions/cache@v3 + name: Cache Python + with: + path: ${{ env.pythonLocation }} + key: ${{ env.pythonLocation }}-${{ hashFiles('setup.py') }}-${{ hashFiles('requirements.txt') }} + + - name: Install Dev Dependencies ๐Ÿ“ฆ + run: | + pip install --upgrade pip + pip install --upgrade -r requirements-dev.txt + shell: bash + + - name: Set up Docker Buildx + uses: docker/setup-buildx-action@v3 + with: + driver-opts: | + network=host + - + name: Download artifact ๐Ÿ“ฅ + if: ${{ inputs.parentImage != '' }} + uses: actions/download-artifact@v4 + with: + name: ${{ inputs.parentImage }} + path: /tmp/jupyter/images/ + + - name: Load parent built image to Docker ๐Ÿณ + if: ${{ inputs.parentImage != '' }} + run: | + docker load --input /tmp/jupyter/images/${{ inputs.parentImage }}.tar + + - name: Login to Docker Hub ๐Ÿ” + uses: docker/login-action@v3 + with: + username: ${{ secrets.DOCKERHUB_USERNAME }} + password: ${{ secrets.DOCKERHUB_TOKEN }} + logout: true + + - name: Login to Containers ๐Ÿซ™ + uses: docker/login-action@v3 + with: + registry: containers.renci.org + username: ${{ secrets.CONTAINERHUB_USERNAME }} + password: ${{ secrets.CONTAINERHUB_TOKEN }} + logout: true + + - name: Docker meta + id: meta + uses: docker/metadata-action@v5 + with: + # list of Docker images to use as base name for tags + images: | + ${{ env.OWNER }}/${{ inputs.image }} + # containers.renci.org/${{ env.OWNER }}/jupyter/docker-stacks/${{ inputs.image }} + # generate Docker tags based on the following events/attributes + # using the type=raw,value="" syntax allows for custom tags. + tags: | + type=ref,event=branch + type=sha + type=raw,value=cuda + type=raw,value=latest + + - name: Build Containers ๐Ÿ› ๏ธ + uses: docker/build-push-action@v6 + with: + context: /images/${{ inputs.image }}/ + push: false + build-args: | + NB_GID=0 + ROOT_CONTAINER=nvidia/cuda:12.2.2-cudnn8-runtime-ubuntu22.04 + # Push to renci-registry and dockerhub here. + tags: | + ${{ steps.meta.outputs.tags }} + outputs: type=docker, dest=/tmp/jupyter/images/${{ inputs.image }}.tar + cache-from: type=registry,ref=${{ env.OWNER }}/${{ inputs.image }}:buildcache-${{ inputs.image }} + cache-to: type=registry,ref=${{ env.OWNER }}/${{ inputs.image }}:buildcache-${{ inputs.image }},mode=max + + - name: Load image to docker + run: | + docker load --input /tmp/jupyter/images/${{ inputs.image }}.tar + + - name: Run tests โœ… + run: python3 -m tests.run_tests --short-image-name ${{ inputs.image }} --owner ${{ env.OWNER }} + shell: bash + + # We do not want to push "latest" tag unless + # the images being built are in the main branch. + # the 'Run tests' step however expects "latest" tag; + # hence - we just delete the tag after the image is tested. + - name: Remove latest tag if Develop Branch + if: ${{ github.ref != 'refs/heads/main' }} + run: | + docker rmi ${{ env.OWNER }}/${{ inputs.image }}:latest + + - name: Upload artifact ๐Ÿ“ค + uses: actions/upload-artifact@v4 + with: + name: ${{ inputs.image }} + path: /tmp/jupyter/images/${{ inputs.image }}.tar + + - name: Push to Docker ๐Ÿ› ๏ธ + run: | + docker push --all-tags ${{ env.OWNER }}/${{ inputs.image }} + + # Because Containers often fails, we keep going and display a message. + - name: Push to Containers ๐Ÿซ™ + id: containers + run: | + docker push --all-tags containers.renci.org/${{ env.OWNER }}/jupyter/docker-stacks/${{ inputs.image }} + continue-on-error: true + + # Sometimes containers.recnci.org fails so we try again. + - name: If Containers Failed + if: ${{ steps.containers.conclusion == 'failure' }} + run: | + echo "Last push failed retrying " + docker push --all-tags containers.renci.org/${{ env.OWNER }}/jupyter/docker-stacks/${{ inputs.image }} + echo "Completed!" + continue-on-error: true \ No newline at end of file diff --git a/.github/workflows/helx-docker.yml b/.github/workflows/helx-docker.yml new file mode 100644 index 0000000000..c83eba70ad --- /dev/null +++ b/.github/workflows/helx-docker.yml @@ -0,0 +1,148 @@ +name: Build, test and push Docker Images + +on: + push: + branches: + - main + - develop + # paths-ignore: + # - ".github/workflows/*" + # - ".github/*" + workflow_dispatch: + +# https://docs.github.com/en/actions/using-jobs/using-concurrency +concurrency: + # only cancel in-progress jobs or runs for the current workflow - matches against branch & tags + group: ${{ github.workflow }}-${{ github.ref }} + cancel-in-progress: true + +jobs: + + foundation: + uses: ./.github/workflows/docker-build-test-upload.yml + with: + parentImage: "" + image: docker-stacks-foundation + runsOn: ubuntu-latest + secrets: + DOCKERHUB_USERNAME: ${{ secrets.DOCKERHUB_USERNAME }} + DOCKERHUB_TOKEN: ${{ secrets.DOCKERHUB_TOKEN }} + CONTAINERHUB_USERNAME: ${{ secrets.CONTAINERHUB_USERNAME }} + CONTAINERHUB_TOKEN: ${{ secrets.CONTAINERHUB_TOKEN }} + + base: + uses: ./.github/workflows/docker-build-test-upload.yml + with: + parentImage: docker-stacks-foundation + image: base-notebook + runsOn: ubuntu-latest + secrets: + DOCKERHUB_USERNAME: ${{ secrets.DOCKERHUB_USERNAME }} + DOCKERHUB_TOKEN: ${{ secrets.DOCKERHUB_TOKEN }} + CONTAINERHUB_USERNAME: ${{ secrets.CONTAINERHUB_USERNAME }} + CONTAINERHUB_TOKEN: ${{ secrets.CONTAINERHUB_TOKEN }} + needs: [foundation] + + minimal: + uses: ./.github/workflows/docker-build-test-upload.yml + with: + parentImage: base-notebook + image: minimal-notebook + runsOn: ubuntu-latest + secrets: + DOCKERHUB_USERNAME: ${{ secrets.DOCKERHUB_USERNAME }} + DOCKERHUB_TOKEN: ${{ secrets.DOCKERHUB_TOKEN }} + CONTAINERHUB_USERNAME: ${{ secrets.CONTAINERHUB_USERNAME }} + CONTAINERHUB_TOKEN: ${{ secrets.CONTAINERHUB_TOKEN }} + needs: [base] + + scipy: + uses: ./.github/workflows/docker-build-test-upload.yml + with: + parentImage: minimal-notebook + image: scipy-notebook + runsOn: ubuntu-latest + secrets: + DOCKERHUB_USERNAME: ${{ secrets.DOCKERHUB_USERNAME }} + DOCKERHUB_TOKEN: ${{ secrets.DOCKERHUB_TOKEN }} + CONTAINERHUB_USERNAME: ${{ secrets.CONTAINERHUB_USERNAME }} + CONTAINERHUB_TOKEN: ${{ secrets.CONTAINERHUB_TOKEN }} + needs: [minimal] + + r: + uses: ./.github/workflows/docker-build-test-upload.yml + with: + parentImage: minimal-notebook + image: r-notebook + runsOn: ubuntu-latest + secrets: + DOCKERHUB_USERNAME: ${{ secrets.DOCKERHUB_USERNAME }} + DOCKERHUB_TOKEN: ${{ secrets.DOCKERHUB_TOKEN }} + CONTAINERHUB_USERNAME: ${{ secrets.CONTAINERHUB_USERNAME }} + CONTAINERHUB_TOKEN: ${{ secrets.CONTAINERHUB_TOKEN }} + needs: [minimal] + + tensorflow: + uses: ./.github/workflows/docker-build-test-upload.yml + with: + parentImage: scipy-notebook + image: tensorflow-notebook + runsOn: ubuntu-latest + secrets: + DOCKERHUB_USERNAME: ${{ secrets.DOCKERHUB_USERNAME }} + DOCKERHUB_TOKEN: ${{ secrets.DOCKERHUB_TOKEN }} + CONTAINERHUB_USERNAME: ${{ secrets.CONTAINERHUB_USERNAME }} + CONTAINERHUB_TOKEN: ${{ secrets.CONTAINERHUB_TOKEN }} + needs: [scipy] + + datascience: + uses: ./.github/workflows/docker-build-test-upload.yml + with: + parentImage: scipy-notebook + image: datascience-notebook + runsOn: ubuntu-latest + secrets: + DOCKERHUB_USERNAME: ${{ secrets.DOCKERHUB_USERNAME }} + DOCKERHUB_TOKEN: ${{ secrets.DOCKERHUB_TOKEN }} + CONTAINERHUB_USERNAME: ${{ secrets.CONTAINERHUB_USERNAME }} + CONTAINERHUB_TOKEN: ${{ secrets.CONTAINERHUB_TOKEN }} + needs: [scipy] + + julia: + uses: ./.github/workflows/docker-build-test-upload.yml + with: + parentImage: minimal-notebook + image: julia-notebook + runsOn: ubuntu-latest + secrets: + DOCKERHUB_USERNAME: ${{ secrets.DOCKERHUB_USERNAME }} + DOCKERHUB_TOKEN: ${{ secrets.DOCKERHUB_TOKEN }} + CONTAINERHUB_USERNAME: ${{ secrets.CONTAINERHUB_USERNAME }} + CONTAINERHUB_TOKEN: ${{ secrets.CONTAINERHUB_TOKEN }} + needs: [minimal] + + pyspark: + uses: ./.github/workflows/docker-build-test-upload.yml + with: + parentImage: scipy-notebook + image: pyspark-notebook + runsOn: ubuntu-latest + secrets: + DOCKERHUB_USERNAME: ${{ secrets.DOCKERHUB_USERNAME }} + DOCKERHUB_TOKEN: ${{ secrets.DOCKERHUB_TOKEN }} + CONTAINERHUB_USERNAME: ${{ secrets.CONTAINERHUB_USERNAME }} + CONTAINERHUB_TOKEN: ${{ secrets.CONTAINERHUB_TOKEN }} + needs: [scipy] + + all-spark: + uses: ./.github/workflows/docker-build-test-upload.yml + with: + parentImage: pyspark-notebook + image: all-spark-notebook + runsOn: ubuntu-latest + secrets: + DOCKERHUB_USERNAME: ${{ secrets.DOCKERHUB_USERNAME }} + DOCKERHUB_TOKEN: ${{ secrets.DOCKERHUB_TOKEN }} + CONTAINERHUB_USERNAME: ${{ secrets.CONTAINERHUB_USERNAME }} + CONTAINERHUB_TOKEN: ${{ secrets.CONTAINERHUB_TOKEN }} + needs: [pyspark] \ No newline at end of file diff --git a/.github/workflows/hub-overview.yml b/.github/workflows/hub-overview.yml index 929367d53e..ce0f1f5c45 100644 --- a/.github/workflows/hub-overview.yml +++ b/.github/workflows/hub-overview.yml @@ -65,3 +65,4 @@ jobs: # description: "Python and Spark Jupyter Notebook Stack from https://github.com/jupyter/docker-stacks" # - image: all-spark-notebook # description: "Python, Scala, R and Spark Jupyter Notebook Stack from https://github.com/jupyter/docker-stacks" + diff --git a/.github/workflows/pre-commit.yml b/.github/workflows/pre-commit.yml deleted file mode 100644 index 20bcf94f8f..0000000000 --- a/.github/workflows/pre-commit.yml +++ /dev/null @@ -1,33 +0,0 @@ -name: Run pre-commit hooks - -on: - pull_request: - push: - branches: - - main - workflow_dispatch: - -permissions: - contents: read - -jobs: - run-hooks: - name: Run pre-commit hooks - runs-on: ubuntu-latest - - steps: - - name: Checkout Repo โšก๏ธ - uses: actions/checkout@v3 - - - name: Set Up Python ๐Ÿ - uses: actions/setup-python@v4 - with: - python-version: 3.x - - - name: Install pre-commit ๐Ÿ“ฆ - run: | - pip install --upgrade pip - pip install --upgrade pre-commit - - - name: Run pre-commit hooks โœ… - run: pre-commit run --all-files --hook-stage manual diff --git a/.hadolint.yaml b/.hadolint.yaml index 3d6d83e9da..11ee226343 100644 --- a/.hadolint.yaml +++ b/.hadolint.yaml @@ -2,3 +2,4 @@ ignored: - DL3006 - DL3008 + - DL3013 diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index 34df15e0fc..25658fa145 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -14,51 +14,62 @@ repos: # Autoupdate: Python code - repo: https://github.com/asottile/pyupgrade - rev: v3.6.0 + rev: v3.17.0 hooks: - id: pyupgrade args: [--py39-plus] # Automatically sort python imports - - repo: https://github.com/pycqa/isort - rev: 5.12.0 + - repo: https://github.com/PyCQA/isort + rev: 5.13.2 hooks: - id: isort args: [--profile, black] # Autoformat: Python code - repo: https://github.com/psf/black - rev: 23.3.0 + rev: 24.8.0 hooks: - id: black args: [--target-version=py39] # Check python code static typing - repo: https://github.com/pre-commit/mirrors-mypy - rev: v1.3.0 + rev: v1.11.2 hooks: - id: mypy args: [--config, ./mypy.ini] additional_dependencies: - ["numpy", "pytest", "requests", "types-requests", "types-tabulate"] - # Unfortunately, `pre-commit` only runs on changed files + [ + "beautifulsoup4", + "numpy", + "pytest", + "requests", + "urllib3", + "types-beautifulsoup4", + "types-PyYAML", + "types-requests", + "types-tabulate", + "types-urllib3", + ] + # Unfortunately, `pre-commit` only runs on modified files # This doesn't work well with `mypy --follow-imports error` # See: https://github.com/pre-commit/mirrors-mypy/issues/34#issuecomment-1062160321 # # To work around this we run `mypy` only in manual mode - # So it won't run as part of `git commit` command - # But it will still be run as part of `pre-commit` workflow and give expected results + # So it won't run as part of `git commit` command, + # but it will still be run as part of `pre-commit` workflow and give expected results stages: [manual] # Autoformat: YAML, JSON, Markdown, etc. - repo: https://github.com/pre-commit/mirrors-prettier - rev: v3.0.0-alpha.9-for-vscode + rev: v4.0.0-alpha.8 hooks: - id: prettier # `pre-commit sample-config` default hooks - repo: https://github.com/pre-commit/pre-commit-hooks - rev: v4.4.0 + rev: v4.6.0 hooks: - id: check-added-large-files - id: end-of-file-fixer @@ -66,22 +77,33 @@ repos: - id: trailing-whitespace # Lint: Dockerfile - - repo: https://github.com/hadolint/hadolint.git - rev: v2.12.1-beta + - repo: https://github.com/hadolint/hadolint + rev: v2.13.0-beta hooks: - id: hadolint-docker entry: hadolint/hadolint:v2.12.1-beta hadolint + # Lint: Dockerfile + # We're linting .dockerfile files as well + - repo: https://github.com/hadolint/hadolint + rev: v2.13.0-beta + hooks: + - id: hadolint-docker + name: Lint *.dockerfile Dockerfiles + entry: hadolint/hadolint:v2.12.1-beta hadolint + types: [file] + files: \.dockerfile$ + # Lint: YAML - - repo: https://github.com/adrienverge/yamllint.git - rev: v1.32.0 + - repo: https://github.com/adrienverge/yamllint + rev: v1.35.1 hooks: - id: yamllint args: ["-d {extends: relaxed, rules: {line-length: disable}}", "-s"] files: \.(yaml|yml)$ # Lint: Bash scripts - - repo: https://github.com/openstack-dev/bashate.git + - repo: https://github.com/openstack/bashate rev: 2.1.1 hooks: - id: bashate @@ -89,34 +111,34 @@ repos: # Lint: Shell scripts - repo: https://github.com/shellcheck-py/shellcheck-py - rev: v0.9.0.5 + rev: v0.10.0.1 hooks: - id: shellcheck args: ["-x"] # Lint: Python - repo: https://github.com/PyCQA/flake8 - rev: 6.0.0 + rev: 7.1.1 hooks: - id: flake8 # Lint: Markdown - repo: https://github.com/igorshubovych/markdownlint-cli - rev: v0.34.0 + rev: v0.41.0 hooks: - id: markdownlint args: ["--fix"] # Strip output from Jupyter notebooks - repo: https://github.com/kynan/nbstripout - rev: 0.6.1 + rev: 0.7.1 hooks: - id: nbstripout # nbQA provides tools from the Python ecosystem like # pyupgrade, isort, black, and flake8, adjusted for notebooks. - repo: https://github.com/nbQA-dev/nbQA - rev: 1.7.0 + rev: 1.8.7 hooks: - id: nbqa-pyupgrade args: [--py39-plus] @@ -126,13 +148,13 @@ repos: - id: nbqa-flake8 # Run black on python code blocks in documentation files. - - repo: https://github.com/asottile/blacken-docs - rev: 1.13.0 + - repo: https://github.com/adamchainz/blacken-docs + rev: 1.18.0 hooks: - id: blacken-docs # --skip-errors is added to allow us to have python syntax highlighting even if - # the python code blocks includes jupyter specific additions such as % or ! - # See https://github.com/asottile/blacken-docs/issues/127 for an upstream + # the python code blocks include jupyter-specific additions such as % or ! + # See https://github.com/adamchainz/blacken-docs/issues/127 for an upstream # feature request about this. args: [--target-version=py39, --skip-errors] diff --git a/.readthedocs.yaml b/.readthedocs.yaml index df3d6c324b..a1a47a1313 100644 --- a/.readthedocs.yaml +++ b/.readthedocs.yaml @@ -1,30 +1,38 @@ ---- -# .readthedocs.yaml -# Read the Docs configuration file +# Read the Docs configuration file for Sphinx projects # See https://docs.readthedocs.io/en/stable/config-file/v2.html for details # Required version: 2 -# Set the version of Python and other tools you might need +# Set the OS, Python version and other tools you might need build: os: ubuntu-22.04 tools: - python: "3.11" + python: "3.12" # You can also specify other tool versions: - # nodejs: "19" - # rust: "1.64" - # golang: "1.19" + # nodejs: "20" + # rust: "1.70" + # golang: "1.20" + jobs: + post_checkout: + - git fetch --unshallow || true -# Build documentation in the docs/ directory with Sphinx +# Build documentation in the "docs/" directory with Sphinx sphinx: configuration: docs/conf.py + # You can configure Sphinx to use a different builder, for instance use the dirhtml builder for simpler URLs + # builder: "dirhtml" + # Fail on all warnings to avoid broken references + # fail_on_warning: true -# If using Sphinx, optionally build your docs in additional formats such as PDF +# Optionally build your docs in additional formats such as PDF and ePub # formats: -# - pdf +# - pdf +# - epub -# Optionally declare the Python requirements required to build your docs +# Optional but recommended, declare the Python requirements required +# to build your documentation +# See https://docs.readthedocs.io/en/stable/guides/reproducible-builds.html python: install: - requirements: docs/requirements.txt diff --git a/CHANGELOG.md b/CHANGELOG.md new file mode 100644 index 0000000000..bbf2f7aeb5 --- /dev/null +++ b/CHANGELOG.md @@ -0,0 +1,9 @@ +# Changelog + +## 2024-10-09 + +### Changed + +_This change might only breaking if you build your custom set of images._ + +- **Breaking:** Rename: `ROOT_CONTAINER`->`ROOT_IMAGE`, `BASE_CONTAINER`->`BASE_IMAGE` ([#2154](https://github.com/jupyter/docker-stacks/pull/2154), [#2155](https://github.com/jupyter/docker-stacks/pull/2155)) diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index 0685e10505..c14c69809d 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -5,6 +5,6 @@ for information about how to contribute [features](https://jupyter-docker-stacks.readthedocs.io/en/latest/contributing/features.html), [recipes](https://jupyter-docker-stacks.readthedocs.io/en/latest/contributing/recipes.html), [tests](https://jupyter-docker-stacks.readthedocs.io/en/latest/contributing/tests.html), -[community-maintained stacks](https://jupyter-docker-stacks.readthedocs.io/en/latest/contributing/stacks.html). +and [community-maintained stacks](https://jupyter-docker-stacks.readthedocs.io/en/latest/contributing/stacks.html). <!-- markdownlint-disable-file MD041 --> diff --git a/Makefile b/Makefile index c1a33edb0d..9222d6777d 100644 --- a/Makefile +++ b/Makefile @@ -2,12 +2,14 @@ # Distributed under the terms of the Modified BSD License. .PHONY: docs help test -# Use bash for inline if-statements in arch_patch target SHELL:=bash +REGISTRY?=quay.io OWNER?=jupyter -# Need to list the images in build dependency order -# All of the images +# Enable BuildKit for Docker build +export DOCKER_BUILDKIT:=1 + +# All the images listed in the build dependency order ALL_IMAGES:= \ docker-stacks-foundation \ base-notebook \ @@ -16,13 +18,11 @@ ALL_IMAGES:= \ julia-notebook \ scipy-notebook \ tensorflow-notebook \ + pytorch-notebook \ datascience-notebook \ pyspark-notebook \ all-spark-notebook -# Enable BuildKit for Docker build -export DOCKER_BUILDKIT:=1 - # https://marmelab.com/blog/2016/02/29/auto-documented-makefile.html @@ -36,84 +36,78 @@ help: build/%: DOCKER_BUILD_ARGS?= +build/%: ROOT_IMAGE?=ubuntu:24.04 build/%: ## build the latest image for a stack using the system's architecture - docker build $(DOCKER_BUILD_ARGS) --rm --force-rm --tag $(OWNER)/$(notdir $@):latest ./$(notdir $@) --build-arg OWNER=$(OWNER) + docker build $(DOCKER_BUILD_ARGS) --rm --force-rm --tag "$(REGISTRY)/$(OWNER)/$(notdir $@):latest" "./images/$(notdir $@)" --build-arg REGISTRY="$(REGISTRY)" --build-arg OWNER="$(OWNER)" --build-arg ROOT_IMAGE="$(ROOT_IMAGE)" @echo -n "Built image size: " - @docker images $(OWNER)/$(notdir $@):latest --format "{{.Size}}" + @docker images "$(REGISTRY)/$(OWNER)/$(notdir $@):latest" --format "{{.Size}}" build-all: $(foreach I, $(ALL_IMAGES), build/$(I)) ## build all stacks -check-outdated/%: ## check the outdated mamba/conda packages in a stack and produce a report (experimental) - @TEST_IMAGE="$(OWNER)/$(notdir $@)" pytest tests/base-notebook/test_outdated.py + +check-outdated/%: ## check the outdated mamba/conda packages in a stack and produce a report + @TEST_IMAGE="$(REGISTRY)/$(OWNER)/$(notdir $@)" pytest tests/docker-stacks-foundation/test_outdated.py check-outdated-all: $(foreach I, $(ALL_IMAGES), check-outdated/$(I)) ## check all the stacks for outdated packages -cont-clean-all: cont-stop-all cont-rm-all ## clean all containers (stop + rm) cont-stop-all: ## stop all containers @echo "Stopping all containers ..." -docker stop --time 0 $(shell docker ps --all --quiet) 2> /dev/null cont-rm-all: ## remove all containers @echo "Removing all containers ..." -docker rm --force $(shell docker ps --all --quiet) 2> /dev/null +cont-clean-all: cont-stop-all cont-rm-all ## clean all containers (stop + rm) docs: ## build HTML documentation sphinx-build -W --keep-going --color docs/ docs/_build/ - linkcheck-docs: ## check broken links sphinx-build -W --keep-going --color -b linkcheck docs/ docs/_build/ +hook/%: VARIANT?=default hook/%: ## run post-build hooks for an image - python3 -m tagging.tag_image --short-image-name "$(notdir $@)" --owner "$(OWNER)" && \ - python3 -m tagging.write_manifest --short-image-name "$(notdir $@)" --hist-line-dir /tmp/jupyter/hist_lines/ --manifest-dir /tmp/jupyter/manifests/ --owner "$(OWNER)" + python3 -m tagging.write_tags_file --short-image-name "$(notdir $@)" --tags-dir /tmp/jupyter/tags/ --registry "$(REGISTRY)" --owner "$(OWNER)" --variant "$(VARIANT)" && \ + python3 -m tagging.write_manifest --short-image-name "$(notdir $@)" --hist-lines-dir /tmp/jupyter/hist_lines/ --manifests-dir /tmp/jupyter/manifests/ --registry "$(REGISTRY)" --owner "$(OWNER)" --variant "$(VARIANT)" && \ + python3 -m tagging.apply_tags --short-image-name "$(notdir $@)" --tags-dir /tmp/jupyter/tags/ --platform "$(shell uname -m)" --variant "$(VARIANT)" --registry "$(REGISTRY)" --owner "$(OWNER)" hook-all: $(foreach I, $(ALL_IMAGES), hook/$(I)) ## run post-build hooks for all images -img-clean: img-rm-dang img-rm ## clean dangling and jupyter images img-list: ## list jupyter images @echo "Listing $(OWNER) images ..." docker images "$(OWNER)/*" -img-rm: ## remove jupyter images - @echo "Removing $(OWNER) images ..." - -docker rmi --force $(shell docker images --quiet "$(OWNER)/*") 2> /dev/null + docker images "*/$(OWNER)/*" img-rm-dang: ## remove dangling images (tagged None) @echo "Removing dangling images ..." -docker rmi --force $(shell docker images -f "dangling=true" --quiet) 2> /dev/null - - - -pre-commit-all: ## run pre-commit hook on all files - @pre-commit run --all-files --hook-stage manual -pre-commit-install: ## set up the git hook scripts - @pre-commit --version - @pre-commit install +img-rm-jupyter: ## remove jupyter images + @echo "Removing $(OWNER) images ..." + -docker rmi --force $(shell docker images --quiet "$(OWNER)/*") 2> /dev/null + -docker rmi --force $(shell docker images --quiet "*/$(OWNER)/*") 2> /dev/null +img-rm: img-rm-dang img-rm-jupyter ## remove dangling and jupyter images pull/%: ## pull a jupyter image - docker pull $(OWNER)/$(notdir $@) + docker pull "$(REGISTRY)/$(OWNER)/$(notdir $@)" pull-all: $(foreach I, $(ALL_IMAGES), pull/$(I)) ## pull all images - - push/%: ## push all tags for a jupyter image - docker push --all-tags $(OWNER)/$(notdir $@) + docker push --all-tags "$(REGISTRY)/$(OWNER)/$(notdir $@)" push-all: $(foreach I, $(ALL_IMAGES), push/$(I)) ## push all tagged images run-shell/%: ## run a bash in interactive mode in a stack - docker run -it --rm $(OWNER)/$(notdir $@) $(SHELL) - -run-sudo-shell/%: ## run a bash in interactive mode as root in a stack - docker run -it --rm --user root $(OWNER)/$(notdir $@) $(SHELL) + docker run -it --rm "$(REGISTRY)/$(OWNER)/$(notdir $@)" $(SHELL) +run-sudo-shell/%: ## run bash in interactive mode as root in a stack + docker run -it --rm --user root "$(REGISTRY)/$(OWNER)/$(notdir $@)" $(SHELL) test/%: ## run tests against a stack - python3 -m tests.run_tests --short-image-name "$(notdir $@)" --owner "$(OWNER)" + python3 -m tests.run_tests --short-image-name "$(notdir $@)" --registry "$(REGISTRY)" --owner "$(OWNER)" test-all: $(foreach I, $(ALL_IMAGES), test/$(I)) ## test all stacks diff --git a/README.md b/README.md index 19bacd8128..6eb4981089 100644 --- a/README.md +++ b/README.md @@ -3,50 +3,53 @@ [![GitHub actions badge](https://github.com/helxplatform/jupyter-docker-stacks/actions/workflows/docker.yml/badge.svg)](https://github.com/helxplatform/jupyter-docker-stacks/actions/workflows/docker.yml "Docker images build status") -Jupyter Docker Stacks are a set of ready-to-run [Docker images](https://hub.docker.com/u/jupyter) containing Jupyter applications and interactive computing tools. +Jupyter Docker Stacks are a set of ready-to-run [Docker images](https://quay.io/organization/jupyter) containing Jupyter applications and interactive computing tools. You can use a stack image to do any of the following (and more): - Start a personal Jupyter Server with the JupyterLab frontend (default) - Run JupyterLab for a team using JupyterHub -- Start a personal Jupyter Notebook server in a local Docker container +- Start a personal Jupyter Server with the Jupyter Notebook frontend in a local Docker container - Write your own project Dockerfile ## Quick Start -You can try a [relatively recent build of the jupyter/base-notebook image on mybinder.org](https://mybinder.org/v2/gh/jupyter/docker-stacks/main?urlpath=lab/tree/README.ipynb) -by simply clicking the preceding link. -Otherwise, the examples below may help you get started if you [have Docker installed](https://docs.docker.com/get-docker/), -know [which Docker image](https://jupyter-docker-stacks.readthedocs.io/en/latest/using/selecting.html) you want to use -and want to launch a single Jupyter Server in a container. +You can [try a relatively recent build of the quay.io/jupyter/base-notebook image on mybinder.org](https://mybinder.org/v2/gh/jupyter/docker-stacks/main?urlpath=lab/tree/README.ipynb). +Otherwise, the examples below may help you get started if you [have Docker installed](https://docs.docker.com/get-started/get-docker/), +know [which Docker image](https://jupyter-docker-stacks.readthedocs.io/en/latest/using/selecting.html) you want to use, and want to launch a single Jupyter Application in a container. The [User Guide on ReadTheDocs](https://jupyter-docker-stacks.readthedocs.io/en/latest/) describes additional uses and features in detail. -**Example 1:** +```{note} +Since `2023-10-20` our images are only pushed to `Quay.io` registry. +Older images are available on Docker Hub, but they will no longer be updated. +``` + +### Example 1 -This command pulls the `jupyter/scipy-notebook` image tagged `2023-06-01` from Docker Hub if it is not already present on the local host. -It then starts a container running a Jupyter Server and exposes the container's internal port `8888` to port `10000` of the host machine: +This command pulls the `jupyter/scipy-notebook` image tagged `2024-10-07` from Quay.io if it is not already present on the local host. +It then starts a container running a Jupyter Server with the JupyterLab frontend and exposes the container's internal port `8888` to port `10000` of the host machine: ```bash -docker run -p 10000:8888 jupyter/scipy-notebook:2023-06-01 +docker run -p 10000:8888 quay.io/jupyter/scipy-notebook:2024-10-07 ``` -You can modify the port on which the container's port is exposed by [changing the value of the `-p` option](https://docs.docker.com/engine/reference/run/#expose-incoming-ports) to `-p 8888:8888`. +You can modify the port on which the container's port is exposed by [changing the value of the `-p` option](https://docs.docker.com/engine/containers/run/#exposed-ports) to `-p 8888:8888`. Visiting `http://<hostname>:10000/?token=<token>` in a browser loads JupyterLab, where: -- `hostname` is the name of the computer running Docker -- `token` is the secret token printed in the console. +- The `hostname` is the name of the computer running Docker +- The `token` is the secret token printed in the console. -The container remains intact for restart after the Jupyter Server exits. +The container remains intact for restart after the Server exits. -**Example 2:** +### Example 2 -This command pulls the `jupyter/datascience-notebook` image tagged `2023-06-01` from Docker Hub if it is not already present on the local host. -It then starts an _ephemeral_ container running a Jupyter Server and exposes the server on host port 10000. +This command pulls the `jupyter/datascience-notebook` image tagged `2024-10-07` from Quay.io if it is not already present on the local host. +It then starts an _ephemeral_ container running a Jupyter Server with the JupyterLab frontend and exposes the server on host port 10000. ```bash -docker run -it --rm -p 10000:8888 -v "${PWD}":/home/jovyan/work jupyter/datascience-notebook:2023-06-01 +docker run -it --rm -p 10000:8888 -v "${PWD}":/home/jovyan/work quay.io/jupyter/datascience-notebook:2024-10-07 ``` The use of the `-v` flag in the command mounts the current working directory on the host (`${PWD}` in the example command) as `/home/jovyan/work` in the container. @@ -54,72 +57,53 @@ The server logs appear in the terminal. Visiting `http://<hostname>:10000/?token=<token>` in a browser loads JupyterLab. -Due to the usage of [the flag `--rm`](https://docs.docker.com/engine/reference/run/#clean-up---rm) Docker automatically cleans up the container and removes the file -system when the container exits, but any changes made to the `~/work` directory and its files in the container will remain intact on the host. -[The `-it` flag](https://docs.docker.com/engine/reference/commandline/run/#name) allocates pseudo-TTY. - -## Contributing - -Please see the [Contributor Guide on ReadTheDocs](https://jupyter-docker-stacks.readthedocs.io/en/latest/) -for information about how to contribute recipes, features, tests, and community maintained stacks. - -## Maintainer Help Wanted +Due to the usage of [the `--rm` flag](https://docs.docker.com/reference/cli/docker/container/run/#rm) +Docker automatically cleans up the container and removes the file system when the container exits, +but any changes made to the `~/work` directory and its files in the container will remain intact on the host. +[The `-i` flag](https://docs.docker.com/reference/cli/docker/container/run/#interactive) keeps the container's `STDIN` open, and lets you send input to the container through standard input. +[The `-t` flag](https://docs.docker.com/reference/cli/docker/container/run/#tty) attaches a pseudo-TTY to the container. -We value all positive contributions to the Docker stacks project, -from [bug reports](https://jupyter-docker-stacks.readthedocs.io/en/latest/contributing/issues.html) -to [pull requests](https://jupyter-docker-stacks.readthedocs.io/en/latest/contributing/features.html#submitting-a-pull-request) -to help with answering questions. -We'd also like to invite members of the community to help with two maintainer activities: +```{note} +By default, [jupyter's root_dir](https://jupyter-server.readthedocs.io/en/latest/other/full-config.html) is `/home/jovyan`. +So, new notebooks will be saved there, unless you change the directory in the file browser. -- **Issue triaging**: Reading and providing a first response to issues, labeling issues appropriately, - redirecting cross-project questions to Jupyter Discourse -- **Pull request reviews**: Reading proposed documentation and code changes, working with the submitter - to improve the contribution, deciding if the contribution should take another form (e.g., a recipe - instead of a permanent change to the images) - -Anyone in the community can jump in and help with these activities anytime. -We will happily grant additional permissions (e.g., the ability to merge PRs) to anyone who shows an ongoing interest in working on the project. +To change the default directory, you must specify `ServerApp.root_dir` by adding this line to the previous command: `start-notebook.py --ServerApp.root_dir=/home/jovyan/work`. +``` -## Jupyter Notebook Deprecation Notice +## Choosing Jupyter frontend -Following [Jupyter Notebook notice](https://github.com/jupyter/notebook#notice), JupyterLab is now the default for all the Jupyter Docker stack images. +JupyterLab is the default for all the Jupyter Docker Stacks images. It is still possible to switch back to Jupyter Notebook (or to launch a different startup command). You can achieve this by passing the environment variable `DOCKER_STACKS_JUPYTER_CMD=notebook` (or any other valid `jupyter` subcommand) at container startup; more information is available in the [documentation](https://jupyter-docker-stacks.readthedocs.io/en/latest/using/common.html#alternative-commands). -According to the Jupyter Notebook project status and its compatibility with JupyterLab, -these Docker images may remove the classic Jupyter Notebook interface altogether in favor of another _classic-like_ UI built atop JupyterLab. - -This change is tracked in the issue [#1217](https://github.com/jupyter/docker-stacks/issues/1217); please check its content for more information. - -## Alternatives - -- [jupyter/repo2docker](https://github.com/jupyterhub/repo2docker) - Turn git repositories into - Jupyter-enabled Docker Images -- [openshift/source-to-image](https://github.com/openshift/source-to-image) - A tool for - building artifacts from source and injecting them into docker images -- [jupyter-on-openshift/jupyter-notebooks](https://github.com/jupyter-on-openshift/jupyter-notebooks) - - OpenShift compatible S2I builder for basic notebook images - ## Resources - [Documentation on ReadTheDocs](https://jupyter-docker-stacks.readthedocs.io/en/latest/) - [Issue Tracker on GitHub](https://github.com/jupyter/docker-stacks/issues) - [Jupyter Discourse Forum](https://discourse.jupyter.org/) - [Jupyter Website](https://jupyter.org) -- [Images on DockerHub](https://hub.docker.com/u/jupyter) +- [Images on Quay.io](https://quay.io/organization/jupyter) + +## Acknowledgments + +- Starting from `2022-07-05`, `aarch64` self-hosted runners were sponsored by [`@mathbunnyru`](https://github.com/mathbunnyru/). + Please, consider [sponsoring his work](https://github.com/sponsors/mathbunnyru) on GitHub +- Starting from `2023-10-31`, `aarch64` self-hosted runners are sponsored by an amazing [`2i2c non-profit organization`](https://2i2c.org) ## CPU Architectures - We publish containers for both `x86_64` and `aarch64` platforms -- Single-platform images have either `aarch64-` or `x86_64-` tag prefixes, for example, `jupyter/base-notebook:aarch64-python-3.10.5` +- Single-platform images have either `aarch64-` or `x86_64-` tag prefixes, for example, `quay.io/jupyter/base-notebook:aarch64-python-3.11.6` - Starting from `2022-09-21`, we create multi-platform images (except `tensorflow-notebook`) -- Starting from `2023-06-01`, we create multi-platform `tensorflow-notebook` image as well +- Starting from `2023-06-01`, we create a multi-platform `tensorflow-notebook` image as well +- Starting from `2024-02-24`, we create CUDA enabled variants of `pytorch-notebook` image for `x86_64` platform +- Starting from `2024-03-26`, we create CUDA enabled variant of `tensorflow-notebook` image for `x86_64` platform ## Using old images This project only builds one set of images at a time. -If you want to use older `Ubuntu` and/or `python` version, you can use following images: +If you want to use the older `Ubuntu` and/or `Python` version, you can use the following images: | Build Date | Ubuntu | Python | Tag | | ------------ | ------ | ------ | -------------- | @@ -131,4 +115,22 @@ If you want to use older `Ubuntu` and/or `python` version, you can use following | 2022-10-09 | 22.04 | 3.8 | `7285848c0a11` | | 2022-10-09 | 22.04 | 3.9 | `ed2908bbb62e` | | 2023-05-30 | 22.04 | 3.10 | `4d70cf8da953` | -| weekly build | 22.04 | 3.11 | `latest` | +| 2024-08-26 | 22.04 | 3.11 | `00987883e58d` | +| weekly build | 24.04 | 3.11 | `latest` | + +## Contributing + +Please see the [Contributor Guide on ReadTheDocs](https://jupyter-docker-stacks.readthedocs.io/en/latest/) +for information about how to contribute recipes, features, tests, and community-maintained stacks. + +## Alternatives + +- [rocker/binder](https://rocker-project.org/images/versioned/binder.html) - + From the R focused [rocker-project](https://rocker-project.org), + lets you run both RStudio and Jupyter either standalone or in a JupyterHub +- [jupyter/repo2docker](https://github.com/jupyterhub/repo2docker) - + Turn git repositories into Jupyter-enabled Docker Images +- [openshift/source-to-image](https://github.com/openshift/source-to-image) - + A tool for building artifacts from source code and injecting them into docker images +- [jupyter-on-openshift/jupyter-notebooks](https://github.com/jupyter-on-openshift/jupyter-notebooks) - + OpenShift compatible S2I builder for basic notebook images diff --git a/aarch64-runner/setup.sh b/aarch64-runner/setup.sh index 96fb247abb..fb6e05b176 100755 --- a/aarch64-runner/setup.sh +++ b/aarch64-runner/setup.sh @@ -3,7 +3,7 @@ set -ex GITHUB_RUNNER_USER="runner-user" -if [ "$EUID" -ne 0 ]; then +if [ "${EUID}" -ne 0 ]; then echo "Please run as root" exit 1 fi @@ -14,14 +14,26 @@ apt-get upgrade --yes echo "Setting up runner-user, who will run GitHub Actions runner" adduser --disabled-password --gecos "" ${GITHUB_RUNNER_USER} mkdir /home/${GITHUB_RUNNER_USER}/.ssh/ +set +e cp "/home/${SUDO_USER}/.ssh/authorized_keys" "/home/${GITHUB_RUNNER_USER}/.ssh/authorized_keys" -chown ${GITHUB_RUNNER_USER}:${GITHUB_RUNNER_USER} /home/${GITHUB_RUNNER_USER}/.ssh/authorized_keys +set -e +chown --recursive ${GITHUB_RUNNER_USER}:${GITHUB_RUNNER_USER} /home/${GITHUB_RUNNER_USER}/.ssh echo "Setting up python3" apt-get install --yes --no-install-recommends python3 curl -sS https://bootstrap.pypa.io/get-pip.py | python3 echo "Setting up docker" -apt-get install --yes --no-install-recommends docker.io +apt-get remove --yes docker.io docker-doc docker-compose podman-docker containerd runc +apt-get update --yes +apt-get install --yes ca-certificates curl gnupg +install -m 0755 -d /etc/apt/keyrings +curl -fsSL https://download.docker.com/linux/ubuntu/gpg | gpg --dearmor -o /etc/apt/keyrings/docker.gpg +chmod a+r /etc/apt/keyrings/docker.gpg +echo "deb [arch=$(dpkg --print-architecture) signed-by=/etc/apt/keyrings/docker.gpg] https://download.docker.com/linux/ubuntu $(lsb_release -cs) stable" | \ + tee /etc/apt/sources.list.d/docker.list > /dev/null +apt-get update --yes +apt-get install --yes docker-ce docker-ce-cli containerd.io docker-buildx-plugin docker-compose-plugin + usermod -aG docker ${GITHUB_RUNNER_USER} chmod 666 /var/run/docker.sock diff --git a/base-notebook/docker_healthcheck.py b/base-notebook/docker_healthcheck.py deleted file mode 100755 index 7c35a6b115..0000000000 --- a/base-notebook/docker_healthcheck.py +++ /dev/null @@ -1,21 +0,0 @@ -#!/usr/bin/env python3 -# Copyright (c) Jupyter Development Team. -# Distributed under the terms of the Modified BSD License. -import json -import os -from pathlib import Path - -import requests - -# A number of operations below deliberately don't check for possible errors -# As this is a healthcheck, it should succeed or raise an exception on error - -runtime_dir = Path("/home/") / os.environ["NB_USER"] / ".local/share/jupyter/runtime/" -json_file = next(runtime_dir.glob("*server-*.json")) - -url = json.loads(json_file.read_bytes())["url"] -url = url + "api" - -r = requests.get(url, verify=False) # request without SSL verification -r.raise_for_status() -print(r.content) diff --git a/base-notebook/start-notebook.sh b/base-notebook/start-notebook.sh deleted file mode 100755 index 4f673d22a5..0000000000 --- a/base-notebook/start-notebook.sh +++ /dev/null @@ -1,22 +0,0 @@ -#!/bin/bash -# Copyright (c) Jupyter Development Team. -# Distributed under the terms of the Modified BSD License. - -set -e - -# The Jupyter command to launch -# JupyterLab by default -DOCKER_STACKS_JUPYTER_CMD="${DOCKER_STACKS_JUPYTER_CMD:=lab}" - -if [[ -n "${JUPYTERHUB_API_TOKEN}" ]]; then - echo "WARNING: using start-singleuser.sh instead of start-notebook.sh to start a server associated with JupyterHub." - exec /usr/local/bin/start-singleuser.sh "$@" -fi - -wrapper="" -if [[ "${RESTARTABLE}" == "yes" ]]; then - wrapper="run-one-constantly" -fi - -# shellcheck disable=SC1091,SC2086 -exec /usr/local/bin/start.sh ${wrapper} jupyter ${DOCKER_STACKS_JUPYTER_CMD} ${NOTEBOOK_ARGS} "$@" diff --git a/base-notebook/start-singleuser.sh b/base-notebook/start-singleuser.sh deleted file mode 100755 index a2166e2c6d..0000000000 --- a/base-notebook/start-singleuser.sh +++ /dev/null @@ -1,13 +0,0 @@ -#!/bin/bash -# Copyright (c) Jupyter Development Team. -# Distributed under the terms of the Modified BSD License. - -set -e - -# set default ip to 0.0.0.0 -if [[ "${NOTEBOOK_ARGS} $*" != *"--ip="* ]]; then - NOTEBOOK_ARGS="--ip=0.0.0.0 ${NOTEBOOK_ARGS}" -fi - -# shellcheck disable=SC1091,SC2086 -. /usr/local/bin/start.sh jupyterhub-singleuser ${NOTEBOOK_ARGS} "$@" diff --git a/binder/Dockerfile b/binder/Dockerfile index 4986e2f400..28ccdd9e62 100644 --- a/binder/Dockerfile +++ b/binder/Dockerfile @@ -1,10 +1,11 @@ # Copyright (c) Jupyter Development Team. # Distributed under the terms of the Modified BSD License. -# https://hub.docker.com/r/jupyter/base-notebook/tags +# https://quay.io/repository/jupyter/base-notebook?tab=tags +ARG REGISTRY=quay.io ARG OWNER=jupyter -ARG BASE_CONTAINER=$OWNER/base-notebook:2023-06-01 -FROM $BASE_CONTAINER +ARG BASE_IMAGE=$REGISTRY/$OWNER/base-notebook:2024-10-07 +FROM $BASE_IMAGE LABEL maintainer="Jupyter Project <jupyter@googlegroups.com>" @@ -12,6 +13,8 @@ LABEL maintainer="Jupyter Project <jupyter@googlegroups.com>" # Fix: https://github.com/koalaman/shellcheck/wiki/SC3014 SHELL ["/bin/bash", "-o", "pipefail", "-c"] -ENV TAG="2023-06-01" +ENV TAG="2024-10-07" COPY --chown=${NB_UID}:${NB_GID} binder/README.ipynb "${HOME}"/README.ipynb + +RUN jupyter labextension disable "@jupyterlab/apputils-extension:announcements" diff --git a/binder/README.ipynb b/binder/README.ipynb index 4116a188ff..66630c9e80 100644 --- a/binder/README.ipynb +++ b/binder/README.ipynb @@ -28,7 +28,7 @@ "cell_type": "markdown", "metadata": {}, "source": [ - "The notebook server is running as the following user." + "The Server is running as the following user." ] }, { @@ -128,7 +128,7 @@ "name": "python", "nbconvert_exporter": "python", "pygments_lexer": "ipython3", - "version": "3.9.10" + "version": "3.11.4" } }, "nbformat": 4, diff --git a/docs/_static/contributing/stacks/docker-org-create-token.png b/docs/_static/contributing/stacks/docker-org-create-token.png new file mode 100644 index 0000000000..21105bc297 Binary files /dev/null and b/docs/_static/contributing/stacks/docker-org-create-token.png differ diff --git a/docs/_static/contributing/stacks/docker-org-security.png b/docs/_static/contributing/stacks/docker-org-security.png new file mode 100644 index 0000000000..831b3d54f2 Binary files /dev/null and b/docs/_static/contributing/stacks/docker-org-security.png differ diff --git a/docs/_static/contributing/stacks/docker-repo-name.png b/docs/_static/contributing/stacks/docker-repo-name.png new file mode 100644 index 0000000000..baf24d5dbb Binary files /dev/null and b/docs/_static/contributing/stacks/docker-repo-name.png differ diff --git a/docs/_static/contributing/stacks/docker-user-dropdown.png b/docs/_static/contributing/stacks/docker-user-dropdown.png new file mode 100644 index 0000000000..1cc107e913 Binary files /dev/null and b/docs/_static/contributing/stacks/docker-user-dropdown.png differ diff --git a/docs/_static/contributing/stacks/github-actions-tab.png b/docs/_static/contributing/stacks/github-actions-tab.png new file mode 100644 index 0000000000..1c233629bd Binary files /dev/null and b/docs/_static/contributing/stacks/github-actions-tab.png differ diff --git a/docs/_static/contributing/stacks/github-actions-workflow.png b/docs/_static/contributing/stacks/github-actions-workflow.png new file mode 100644 index 0000000000..2dc4a9cf89 Binary files /dev/null and b/docs/_static/contributing/stacks/github-actions-workflow.png differ diff --git a/docs/_static/contributing/stacks/github-create-secrets.png b/docs/_static/contributing/stacks/github-create-secrets.png new file mode 100644 index 0000000000..1336d0dacb Binary files /dev/null and b/docs/_static/contributing/stacks/github-create-secrets.png differ diff --git a/docs/_static/contributing/stacks/github-secret-token.png b/docs/_static/contributing/stacks/github-secret-token.png new file mode 100644 index 0000000000..5901c26473 Binary files /dev/null and b/docs/_static/contributing/stacks/github-secret-token.png differ diff --git a/docs/_static/docker-github-settings.png b/docs/_static/docker-github-settings.png deleted file mode 100644 index 8a07fe0c1b..0000000000 Binary files a/docs/_static/docker-github-settings.png and /dev/null differ diff --git a/docs/_static/docker-org-create-token.png b/docs/_static/docker-org-create-token.png deleted file mode 100644 index ae0163310b..0000000000 Binary files a/docs/_static/docker-org-create-token.png and /dev/null differ diff --git a/docs/_static/docker-org-security.png b/docs/_static/docker-org-security.png deleted file mode 100644 index 2dfa2a7851..0000000000 Binary files a/docs/_static/docker-org-security.png and /dev/null differ diff --git a/docs/_static/docker-org-select.png b/docs/_static/docker-org-select.png deleted file mode 100644 index 3d2a2b8ca7..0000000000 Binary files a/docs/_static/docker-org-select.png and /dev/null differ diff --git a/docs/_static/docker-repo-name.png b/docs/_static/docker-repo-name.png deleted file mode 100644 index b4088f9d54..0000000000 Binary files a/docs/_static/docker-repo-name.png and /dev/null differ diff --git a/docs/_static/github-actions-tab.png b/docs/_static/github-actions-tab.png deleted file mode 100644 index 932e3c0468..0000000000 Binary files a/docs/_static/github-actions-tab.png and /dev/null differ diff --git a/docs/_static/github-actions-workflow.png b/docs/_static/github-actions-workflow.png deleted file mode 100644 index 1af7c512ea..0000000000 Binary files a/docs/_static/github-actions-workflow.png and /dev/null differ diff --git a/docs/_static/github-create-secrets.png b/docs/_static/github-create-secrets.png deleted file mode 100644 index 3d1e35a4e1..0000000000 Binary files a/docs/_static/github-create-secrets.png and /dev/null differ diff --git a/docs/_static/github-secret-token.png b/docs/_static/github-secret-token.png deleted file mode 100644 index 68cb72c40e..0000000000 Binary files a/docs/_static/github-secret-token.png and /dev/null differ diff --git a/docs/_static/github-secrets-completed.png b/docs/_static/github-secrets-completed.png deleted file mode 100644 index c26cfd4280..0000000000 Binary files a/docs/_static/github-secrets-completed.png and /dev/null differ diff --git a/docs/_static/using/troubleshooting/vscode-jupyter-settings.png b/docs/_static/using/troubleshooting/vscode-jupyter-settings.png index 177c385522..6c60389ae5 100644 Binary files a/docs/_static/using/troubleshooting/vscode-jupyter-settings.png and b/docs/_static/using/troubleshooting/vscode-jupyter-settings.png differ diff --git a/docs/conf.py b/docs/conf.py index dad3b3cbeb..7957c246c2 100644 --- a/docs/conf.py +++ b/docs/conf.py @@ -7,7 +7,7 @@ # https://www.sphinx-doc.org/en/master/usage/configuration.html#project-information project = "docker-stacks" -copyright = "2023, Project Jupyter" +copyright = "2024, Project Jupyter" author = "Project Jupyter" version = "latest" @@ -29,7 +29,7 @@ html_theme = "alabaster" html_static_path = ["_static"] -# File above was generated using sphinx 6.2.1 with this command: +# The file above was generated using sphinx 7.2.6 with this command: # sphinx-quickstart --project "docker-stacks" --author "Project Jupyter" -v "latest" -r "latest" -l en --no-sep --no-makefile --no-batchfile # These are custom options for this project @@ -37,16 +37,17 @@ html_title = "Docker Stacks documentation" html_logo = "_static/jupyter-logo.svg" html_theme_options = { + "logo": { + "text": html_title, + }, + "navigation_with_keys": False, "path_to_docs": "docs", - "repository_url": "https://github.com/jupyter/docker-stacks", "repository_branch": "main", + "repository_url": "https://github.com/jupyter/docker-stacks", + "use_download_button": True, "use_edit_page_button": True, "use_issues_button": True, "use_repository_button": True, - "use_download_button": True, - "logo": { - "text": html_title, - }, } html_last_updated_fmt = "%Y-%m-%d" @@ -57,7 +58,7 @@ } pygments_style = "sphinx" -# MyST configuration reference: https://myst-parser.readthedocs.io/en/latest/sphinx/reference.html +# MyST configuration reference: https://myst-parser.readthedocs.io/en/latest/configuration.html myst_heading_anchors = 3 linkcheck_ignore = [ @@ -65,6 +66,7 @@ r"https://github\.com/jupyter/docker-stacks/settings/actions/runners/new\?arch=arm64\&os=linux", # only works for users with permissions to change runners r"http://127\.0\.0\.1:.*", # various examples r"https://mybinder\.org/v2/gh/.*", # lots of 500 errors + r"https://packages\.ubuntu\.com/search\?keywords=openjdk", # frequent read timeouts ] linkcheck_allowed_redirects = { diff --git a/docs/contributing/features.md b/docs/contributing/features.md index 08c1331f27..7da372de13 100644 --- a/docs/contributing/features.md +++ b/docs/contributing/features.md @@ -1,36 +1,43 @@ # New Features -Thank you for contributing to the Jupyter Docker Stacks! We review pull requests for new features -(e.g., new packages, new scripts, new flags) to balance the value of the images to the Jupyter -community with the cost of maintaining the images over time. +Thank you for contributing to the Jupyter Docker Stacks! +We review pull requests for new features (e.g., new packages, new scripts, new flags) +to balance the value of the images to the Jupyter community with the cost of maintaining the images over time. ## Suggesting a New Feature Please follow the process below to suggest a new feature for inclusion in one of the core stacks: -1. Open a [GitHub feature request issue](https://github.com/jupyter/docker-stacks/issues/new?assignees=&labels=type%3AEnhancement&template=feature_request.md&title=) +1. Open a [GitHub feature request issue](https://github.com/jupyter/docker-stacks/issues/new?assignees=&labels=type%3AEnhancement&projects=&template=feature_request.yml) describing the feature you'd like to contribute. -2. Discuss with the maintainers whether the addition makes sense in - [one of the core stacks](../using/selecting.md#core-stacks), as a - [recipe in the documentation](recipes.md), as a [community stack](stacks.md), or as something - else entirely. +2. Discuss with the maintainers whether the addition makes sense + in [one of the core stacks](../using/selecting.md#core-stacks), + as a [way to build a custom set of images](../using/custom-images.md), + as a [recipe in the documentation](recipes.md), + as a [community stack](stacks.md), + or as something else entirely. ## Selection Criteria Roughly speaking, we evaluate new features based on the following criteria: -- **Usefulness to Jupyter users**: Is the feature generally applicable across domains? Does it work - with Jupyter Notebook, JupyterLab, JupyterHub, etc.? -- **Fit with the image purpose**: Does the feature match the theme of the stack in which it will be - added? Would it fit better in a new community stack? -- **Complexity of build/runtime configuration**: How many lines of code does the feature require - in one of the Dockerfiles or startup scripts? Does it require new scripts entirely? Do users need - to adjust how they use the images? -- **Impact on image metrics**: How many bytes does the feature and its dependencies add to the - image(s)? How many minutes do they add to the build time? -- **Ability to support the addition**: Can existing maintainers answer user questions and address - future build issues? Are the contributors interested in helping with long-term maintenance? Can we - write tests to ensure the feature continues to work over time? +- **Usefulness to Jupyter users**: + Is the feature generally applicable across domains? + Does it work with JupyterLab, Jupyter Notebook, JupyterHub, etc.? +- **Fit with the image purpose**: + Does the feature match the theme of the stack to which it will be added? + Would it fit better in a new community stack? +- **The complexity of build/runtime configuration**: + How many lines of code does the feature require in one of the Dockerfiles or startup scripts? + Does it require new scripts entirely? + Do users need to adjust how they use the images? +- **Impact on image metrics**: + How many bytes does the feature and its dependencies add to the image(s)? + How many minutes do they add to the build time? +- **Ability to support the addition**: + Can existing maintainers answer user questions and address future build issues? + Are the contributors interested in helping with long-term maintenance? + Can we write tests to ensure the feature continues to work over the years? ## Submitting a Pull Request @@ -43,7 +50,7 @@ If there's agreement that the feature belongs in one or more of the core stacks: If you use `make`, call: ```bash - make build/somestack-notebook + make build/<somestack> ``` 3. [Submit a pull request](https://github.com/PointCloudLibrary/pcl/wiki/A-step-by-step-guide-on-preparing-and-submitting-a-pull-request)(PR) with your changes. diff --git a/docs/contributing/issues.md b/docs/contributing/issues.md index 0f133c09f3..680d584b7d 100644 --- a/docs/contributing/issues.md +++ b/docs/contributing/issues.md @@ -9,7 +9,7 @@ Please review the following guidelines when reporting your problem. - If you think your problem is unique to the Jupyter Docker Stacks images, please search the [jupyter/docker-stacks issue tracker](https://github.com/jupyter/docker-stacks/issues) to see if someone else has already reported the same problem. - If not, please open a [GitHub bug report issue](https://github.com/jupyter/docker-stacks/issues/new?assignees=&labels=type%3ABug&template=bug_report.md&title=) + If not, please open a [GitHub bug report issue](https://github.com/jupyter/docker-stacks/issues/new?assignees=&labels=type%3ABug&projects=&template=bug_report.yml) and provide all the information requested in the issue template. Additionally, check the [Troubleshooting Common Problems](../using/troubleshooting.md) page in the documentation before submitting an issue. - If the issue you're seeing is with one of the open-source libraries included in the Docker images and is reproducible outside the images, diff --git a/docs/contributing/lint.md b/docs/contributing/lint.md index 4fb83bf7e3..350f07dd27 100644 --- a/docs/contributing/lint.md +++ b/docs/contributing/lint.md @@ -2,7 +2,7 @@ To enforce some rules, **linters** are used in this project. Linters can be run either during the **development phase** (by the developer) or the **integration phase** (by GitHub Actions). -To integrate and enforce this process in the project lifecycle, we are using **git hooks** through [pre-commit][pre-commit]. +To integrate and enforce this process in the project lifecycle, we are using **git hooks** through [pre-commit](https://pre-commit.com/). ## Using pre-commit hooks @@ -13,7 +13,7 @@ To achieve this, use the generic task to install all Python development dependen ```sh # Install all development dependencies for the project -pip install requirements-dev.txt +pip install --upgrade -r requirements-dev.txt # It can also be installed directly pip install pre-commit ``` @@ -21,7 +21,7 @@ pip install pre-commit Then the git hooks scripts configured for the project in `.pre-commit-config.yaml` need to be installed in the local git repository. ```sh -make pre-commit-install +pre-commit install ``` ### Run @@ -34,22 +34,29 @@ Hadolint pre-commit uses Docker to run, so `docker` should be running while runn ``` ```sh -make pre-commit-all +pre-commit run --all-files --hook-stage manual +``` + +```{note} +We're running `pre-commit` with `--hook-stage manual`, because `pre-commit` is run on modified files only, which doesn't work well with `mypy --follow-imports error`. +More information can be found in [`.pre-commit-config.yaml` file](https://github.com/jupyter/docker-stacks/blob/main/.pre-commit-config.yaml) ``` ## Image Lint -To comply with [Docker best practices][dbp], we are using the [Hadolint][hadolint] tool to analyse each `Dockerfile`. +To comply with [Docker best practices](https://docs.docker.com/build/building/best-practices/), +we are using the [Hadolint](https://github.com/hadolint/hadolint) tool to analyze each `Dockerfile`. ### Ignoring Rules -Sometimes it is necessary to ignore [some rules][rules]. +Sometimes it is necessary to ignore [some rules](https://github.com/hadolint/hadolint#rules). The following rules are ignored by default for all images in the `.hadolint.yaml` file. - [`DL3006`][dl3006]: We use a specific policy to manage image tags. - - `base-notebook` `FROM` clause is fixed but based on an argument (`ARG`). + - The `docker-stacks-foundation` `FROM` clause is fixed but based on an argument (`ARG`). - Building downstream images from (`FROM`) the latest is done on purpose. - [`DL3008`][dl3008]: System packages are always updated (`apt-get`) to the latest version. +- [`DL3013`][dl3013]: We always install the latest packages using `pip` The preferred way to do it for other rules is to flag ignored ones in the `Dockerfile`. @@ -64,9 +71,6 @@ FROM ubuntu RUN cd /tmp && echo "hello!" ``` -[hadolint]: https://github.com/hadolint/hadolint -[dbp]: https://docs.docker.com/develop/develop-images/dockerfile_best-practices -[rules]: https://github.com/hadolint/hadolint#rules [dl3006]: https://github.com/hadolint/hadolint/wiki/DL3006 [dl3008]: https://github.com/hadolint/hadolint/wiki/DL3008 -[pre-commit]: https://pre-commit.com/ +[dl3013]: https://github.com/hadolint/hadolint/wiki/DL3013 diff --git a/docs/contributing/packages.md b/docs/contributing/packages.md index 45537bd354..d6b7706047 100644 --- a/docs/contributing/packages.md +++ b/docs/contributing/packages.md @@ -6,13 +6,13 @@ All this means that packages might have old versions. Images are rebuilt weekly, so usually, packages receive updates quite frequently. ```{note} -We pin major.minor version of python, so this will stay the same even after invoking the `mamba update` command. +We pin major.minor version of Python, so this will stay the same even after invoking the `mamba update` command. ``` ## Outdated packages -To help to identify packages that can be updated, you can use the following helper tool. -It will list all the updateable packages installed in the `Dockerfile` -- +To help identify packages that can be updated, you can use the following helper tool. +It will list all the outdated packages installed in the `Dockerfile` -- dependencies are filtered to focus only on requested packages. ```bash diff --git a/docs/contributing/recipes.md b/docs/contributing/recipes.md index 313d755cc8..45afb60c50 100644 --- a/docs/contributing/recipes.md +++ b/docs/contributing/recipes.md @@ -1,10 +1,12 @@ # New Recipes -We welcome contributions of [recipes](../using/recipes.md), short examples of using, configuring, or extending the Docker Stacks for inclusion in the documentation site. +We welcome contributions of [recipes](../using/recipes.md), and short examples of using, configuring, or extending the Docker Stacks for inclusion in the documentation site. Follow the process below to add a new recipe: 1. Open the `docs/using/recipes.md` source file. -2. Add a second-level Markdown heading naming your recipe at the bottom of the file (e.g., `## Add the RISE extension`) -3. Write the body of your recipe under the heading, including whatever command line, Dockerfile, links, etc. you need. -4. [Submit a pull request](https://github.com/PointCloudLibrary/pcl/wiki/A-step-by-step-guide-on-preparing-and-submitting-a-pull-request) (PR) with your changes. +2. Add a second-level Markdown heading naming your recipe at the bottom of the file (e.g., `## Slideshows with JupyterLab and RISE`) +3. Write the body of your recipe under the heading, including whatever command line, links, etc. you need. +4. If you have a Dockerfile, please put it in a `recipe_code` subdirectory. + This file will be built automatically by [contributed-recipes workflow](https://github.com/jupyter/docker-stacks/blob/main/.github/workflows/contributed-recipes.yml). +5. [Submit a pull request](https://github.com/PointCloudLibrary/pcl/wiki/A-step-by-step-guide-on-preparing-and-submitting-a-pull-request) (PR) with your changes. Maintainers will respond and work with you to address any formatting or content issues. diff --git a/docs/contributing/stacks.md b/docs/contributing/stacks.md index d93a6950f1..37b474e6f8 100644 --- a/docs/contributing/stacks.md +++ b/docs/contributing/stacks.md @@ -8,15 +8,15 @@ Following these steps will: 1. Set up a project on GitHub containing a Dockerfile based on any image we provide. 2. Configure GitHub Actions to build and test your image when users submit pull requests to your repository. -3. Configure Docker Hub to build and host your images for others to use. +3. Configure Quay.io to host your images for others to use. 4. Update the [list of community stacks](../using/selecting.md#community-stacks) in this documentation to include your image. This approach mirrors how we build and share the core stack images. -Feel free to follow it or pave your own path using alternative services and build tools. +Feel free to follow it or pave your path using alternative services and build tools. ## Creating a Project -First, install [cookiecutter](https://github.com/cookiecutter/cookiecutter) using _pip_ or _conda_: +First, install [cookiecutter](https://github.com/cookiecutter/cookiecutter) using _pip_ or _mamba_: ```bash pip install cookiecutter # or mamba install cookiecutter @@ -35,17 +35,17 @@ This will serve as both the git repository name and the part of the Docker image stack_name [my-jupyter-stack]: ``` -Enter the user or organization name under which this stack will reside on Docker Hub. -You must have access to manage this Docker Hub organization to push images here and set up automated builds. +Enter the user or organization name under which this stack will reside on Quay.io. +You must have access to manage this Quay.io organization to push images here. ```text stack_org [my-project]: ``` -Select an image from the jupyter/docker-stacks project that will serve as the base for your new image. +Select an image from the `jupyter/docker-stacks` project that will serve as the base for your new image. ```text -stack_base_image [jupyter/base-notebook]: +stack_base_image [quay.io/jupyter/base-notebook]: ``` Enter a longer description of the stack for your README. @@ -54,6 +54,7 @@ Enter a longer description of the stack for your README. stack_description [my-jupyter-stack is a community-maintained Jupyter Docker Stack image]: ``` +Create a GitHub repository to store your project. Initialize your project as a Git repository and push it to GitHub. ```bash @@ -66,90 +67,88 @@ git remote add origin <url from github> git push -u origin main ``` -## Configuring GitHub actions +## Exploring GitHub Actions -The cookiecutter template comes with a `.github/workflows/docker.yml` file, which allows you to use GitHub actions to build your Docker image whenever you or someone else submits a pull request. +1. By default, the newly `.github/workflows/docker.yaml` will trigger the CI pipeline whenever you push to your `main` branch + and when any Pull Requests are made to your repository. + For more details on this configuration, visit the [GitHub actions documentation on triggers](https://docs.github.com/en/actions/writing-workflows/choosing-when-your-workflow-runs/events-that-trigger-workflows). -1. By default, the `.github/workflows/docker.yaml` file has the following triggers configuration: +2. Go to your repository and click on the **Actions** tab. + From there, you can click on the workflows on the left-hand side of the screen. - ```yaml - on: - pull_request: - paths-ignore: - - "*.md" - push: - branches: - - main - paths-ignore: - - "*.md" - ``` + ![GitHub page for jupyter/docker-stacks with the Actions tab active and a rectangle around the "Build Docker Images" workflow in the UI](../_static/contributing/stacks/github-actions-tab.png) - This will trigger the CI pipeline whenever you push to your `main` branch and when any Pull Requests are made to your repository. - For more details on this configuration, visit the [GitHub actions documentation on triggers](https://docs.github.com/en/actions/using-workflows/events-that-trigger-workflows). + ```{note} + The first run is expected to fail because we haven't yet added Docker credentials to push the image + ``` -2. Commit your changes and push them to GitHub. -3. Head back to your repository and click on the **Actions** tab. - ![GitHub page for jupyter/docker-stacks with the Actions tab active and a rectangle around the "Build Docker Images" workflow in the UI](../_static/github-actions-tab.png) - From there, you can click on the workflows on the left-hand side of the screen. -4. In the next screen, you will see information about the workflow run and duration. +3. In the next screen, you will see information about the workflow run and duration. If you click the button with the workflow name again, you will see the logs for the workflow steps. - ![GitHub Actions page showing the "Build Docker Images" workflow](../_static/github-actions-workflow.png) + + ![GitHub Actions page showing the "Build Docker Images" workflow](../_static/contributing/stacks/github-actions-workflow.png) ## Configuring Docker Hub +```{note} +Jupyter Docker Stacks are hosted on Quay.io, but in this example, we show you how to host your image on Docker Hub. +``` + Now, configure Docker Hub to build your stack image and push it to the Docker Hub repository whenever you merge a GitHub pull request to the main branch of your project. 1. Visit [https://hub.docker.com/](https://hub.docker.com/) and log in. -2. Select the account or organization matching the one you entered when prompted with `stack_org` by the cookiecutter. - ![DockerHub page zoomed into the user's settings and accounts menu.](../_static/docker-org-select.png) -3. Scroll to the bottom of the page and click **Create repository**. -4. Enter the name of the image matching the one you entered when prompted with `stack_name` by the cookiecutter. - ![DockerHub - Create Repository page with the name field set to "My specialized jupyter stack"](../_static/docker-repo-name.png) -5. Enter a description for your image. -6. Click **GitHub** under the **Build Settings** and follow the prompts to connect your account if it is not already connected. -7. Select the GitHub organization and repository containing your image definition from the dropdowns. - ![Dockerhub - Create Repository page focusing on the "Select Repository" dropdown menu](../_static/docker-github-settings.png) -8. Click the **Create and Build** button. -9. Click on your avatar in the top-right corner and select Account settings. - ![DockerHub page zoomed into the user's settings and accounts menu](../_static/docker-org-select.png) -10. Click on **Security** and then click on the **New Access Token** button. - ![DockerHub - Account page with the "Security" tab active and a rectangle highlighting the "New Access Token" button in the UI](../_static/docker-org-security.png) -11. Enter a meaningful name for your token and click on **Create** - ![DockerHub - New Access Token page with the name field set to "my-jupyter-docker-token"](../_static/docker-org-create-token.png) -12. Copy the personal access token displayed on the next screen. - - ```{note} - you will not be able to see it again after you close the pop-up window**. - ``` - -13. Head back to your GitHub repository and click on the **Settings tab**. - ![GitHub page with the "Setting" tab active and a rectangle highlighting the "New repository secret" button in the UI](../_static/github-create-secrets.png) -14. Click on the **Secrets** section and then on the **New repository secret** button in the top right corner (see image above). -15. Create a **DOCKERHUB_TOKEN** secret and paste the Personal Access Token from DockerHub in the **value** field. - ![GitHub - Actions/New secret page with the Name field set to "DOCKERHUB_TOKEN"](../_static/github-secret-token.png) -16. Repeat the above step but creating a **DOCKERHUB_USERNAME** and replacing the _value_ field with your DockerHub username. - Once you have completed these steps, your repository secrets section should look something like this: - ![GitHub - Repository secrets page showing the existing "DOCKERHUB_TOKEN" and "DOCKERHUB_USERNAME" secrets](../_static/github-secrets-completed.png) +2. Create a new repository - make sure to use the correct namespace (account or organization). + Enter the name of the image matching the one you entered when prompted with `stack_name` by the cookiecutter. + + ![Docker Hub - 'Create repository' page with the name field set to "My specialized jupyter stack"](../_static/contributing/stacks/docker-repo-name.png) + +3. Enter a description for your image. +4. Click on your avatar in the top-right corner and select Account Settings. + + ![The Docker Hub page zoomed into the user's settings and accounts menu](../_static/contributing/stacks/docker-user-dropdown.png) + +5. Click on **Security** and then click on the **New Access Token** button. + + ![Docker Hub - Account page with the "Security" tab active and a rectangle highlighting the "New Access Token" button in the UI](../_static/contributing/stacks/docker-org-security.png) + +6. Enter a meaningful name for your token and click on **Generate** + + ![Docker Hub - New Access Token page with the name field set to "test-stack token"](../_static/contributing/stacks/docker-org-create-token.png) + +7. Copy the personal access token displayed on the next screen. + + ```{note} + **You will not be able to see it again after you close the pop-up window**. + ``` + +8. Head back to your GitHub repository and click on the **Settings tab**. +9. Click on the **Secrets and variables->Actions** section and then on the **New repository secret** button in the top right corner. + + ![GitHub page with the "Setting" tab active and a rectangle highlighting the "New repository secret" button in the UI](../_static/contributing/stacks/github-create-secrets.png) + +10. Create a **DOCKERHUB_TOKEN** secret and paste the Personal Access Token from Docker Hub in the **value** field. + + ![GitHub - Actions/New secret page with the Name field set to "DOCKERHUB_TOKEN"](../_static/contributing/stacks/github-secret-token.png) + +11. Now you're ready to go and you can restart a failed workflow. ## Defining Your Image -Make edits to the Dockerfile in your project to add third-party libraries and configure Jupyter -applications. -Refer to the Dockerfiles for the core stacks (e.g., [jupyter/datascience-notebook](https://github.com/jupyter/docker-stacks/blob/main/datascience-notebook/Dockerfile)) +Make edits to the Dockerfile in your project to add third-party libraries and configure Jupyter applications. +Refer to the Dockerfiles for the core stacks (e.g., [jupyter/datascience-notebook](https://github.com/jupyter/docker-stacks/blob/main/images/datascience-notebook/Dockerfile)) to get a feel for what's possible and the best practices. [Submit pull requests](https://github.com/PointCloudLibrary/pcl/wiki/A-step-by-step-guide-on-preparing-and-submitting-a-pull-request) to your project repository on GitHub. -Ensure your image builds correctly on GitHub actions before merging to the main branch. -Refer to Docker Hub to build the main branch that you can `docker pull`. +Ensure your image builds correctly on GitHub Actions before merging to the main branch. +After merging to the main branch, your image will be built and pushed to the Docker Hub automatically. ## Sharing Your Image Finally, if you'd like to add a link to your project to this documentation site, please do the following: -1. Clone the [jupyter/docker-stacks](https://github.com/jupyter/docker-stacks) GitHub repository. -2. Open the `docs/using/selecting.md` source file and locate the **Community Stacks** section. -3. Add a table entry with a link to your project, a binder link and a short description of what your Docker image contains. +1. Fork the [jupyter/docker-stacks](https://github.com/jupyter/docker-stacks) GitHub repository. +2. Open the `docs/using/selecting.md` source file and locate the **Community Stacks** section in your fork. +3. Add a table entry with a link to your project, a binder link, and a short description of what your Docker image contains. 4. [Submit a pull request](https://github.com/PointCloudLibrary/pcl/wiki/A-step-by-step-guide-on-preparing-and-submitting-a-pull-request)(PR) with your changes. Maintainers will respond and work with you to address any formatting or content issues. diff --git a/docs/contributing/tests.md b/docs/contributing/tests.md index b27a3046b1..3d7888bf57 100644 --- a/docs/contributing/tests.md +++ b/docs/contributing/tests.md @@ -1,6 +1,6 @@ # Image Tests -We greatly appreciate pull requests that extend the automated tests that vet the basic functionality of the Docker images. +We greatly appreciate Pull Requests that extend the automated tests that vet the basic functionality of the Docker images. ## How the Tests Work @@ -14,7 +14,7 @@ More info on `pytest` can be found [here](https://docs.pytest.org/en/latest/cont The actual image-specific test files are located in folders like `tests/<somestack>/` (e.g., `tests/docker-stacks-foundation/`, `tests/minimal-notebook/`, etc.). ```{note} -If your test is located in `tests/<somestack>/`, it will be run against `jupyter/<somestack>` image and against all the [images inherited from this image](https://jupyter-docker-stacks.readthedocs.io/en/latest/using/selecting.html#image-relationships. +If your test is located in `tests/<somestack>/`, it will be run against the `jupyter/<somestack>` image and against all the [images inherited from this image](https://jupyter-docker-stacks.readthedocs.io/en/latest/using/selecting.html#image-relationships. ``` Many tests make use of global [pytest fixtures](https://docs.pytest.org/en/latest/reference/fixtures.html) @@ -22,16 +22,16 @@ defined in the [conftest.py](https://github.com/jupyter/docker-stacks/blob/main/ ## Unit tests -You can add a unit test if you want to run a python script in one of our images. -You should create a `tests/<somestack>/units/` directory, if it doesn't already exist and put your file there. +You can add a unit test if you want to run a Python script in one of our images. +You should create a `tests/<somestack>/units/` directory, if it doesn't already exist, and put your file there. Files in this folder will be executed in the container when tests are run. -You can see an [example for the TensorFlow package here](https://github.com/jupyter/docker-stacks/blob/HEAD/tests/tensorflow-notebook/units/unit_tensorflow.py). +You can see an [TensorFlow package example here](https://github.com/jupyter/docker-stacks/blob/HEAD/tests/tensorflow-notebook/units/unit_tensorflow.py). ## Contributing New Tests Please follow the process below to add new tests: -1. Add your test code to one of the modules in `tests/<somestack>/` directory or create a new module. +1. Add your test code to one of the modules in the `tests/<somestack>/` directory or create a new module. 2. Build one or more images you intend to test and run the tests locally. If you use `make`, call: diff --git a/docs/images/inherit.svg b/docs/images/inherit.svg index 0f5467a403..e98db38019 100644 --- a/docs/images/inherit.svg +++ b/docs/images/inherit.svg @@ -1,6 +1,6 @@ <?xml version='1.0' encoding='UTF-8'?> <!DOCTYPE svg PUBLIC "-//W3C//DTD SVG 1.0//EN" "http://www.w3.org/TR/2001/REC-SVG-20010904/DTD/svg10.dtd"> -<svg viewBox="0 0 640 600" xmlns="http://www.w3.org/2000/svg" xmlns:inkspace="http://www.inkscape.org/namespaces/inkscape" xmlns:xlink="http://www.w3.org/1999/xlink"> +<svg viewBox="0 0 832 600" xmlns="http://www.w3.org/2000/svg" xmlns:inkspace="http://www.inkscape.org/namespaces/inkscape" xmlns:xlink="http://www.w3.org/1999/xlink"> <defs id="defs_block"> <filter height="1.504" id="filter_blur" inkspace:collect="always" width="1.1575" x="-0.07875" y="-0.252"> <feGaussianBlur id="feGaussianBlur3780" inkspace:collect="always" stdDeviation="4.2" /> @@ -18,7 +18,8 @@ <rect fill="rgb(0,0,0)" height="40" stroke="rgb(0,0,0)" style="filter:url(#filter_blur);opacity:0.7;fill-opacity:1" width="128" x="67" y="446" /> <rect fill="rgb(0,0,0)" height="40" stroke="rgb(0,0,0)" style="filter:url(#filter_blur);opacity:0.7;fill-opacity:1" width="128" x="259" y="446" /> <rect fill="rgb(0,0,0)" height="40" stroke="rgb(0,0,0)" style="filter:url(#filter_blur);opacity:0.7;fill-opacity:1" width="128" x="451" y="446" /> - <rect fill="rgb(0,0,0)" height="40" stroke="rgb(0,0,0)" style="filter:url(#filter_blur);opacity:0.7;fill-opacity:1" width="128" x="451" y="526" /> + <rect fill="rgb(0,0,0)" height="40" stroke="rgb(0,0,0)" style="filter:url(#filter_blur);opacity:0.7;fill-opacity:1" width="128" x="643" y="446" /> + <rect fill="rgb(0,0,0)" height="40" stroke="rgb(0,0,0)" style="filter:url(#filter_blur);opacity:0.7;fill-opacity:1" width="128" x="643" y="526" /> <rect fill="rgb(255,255,255)" height="40" stroke="rgb(0,0,0)" width="128" x="64" y="40" /> <text fill="rgb(0,0,0)" font-family="sans-serif" font-size="9" font-style="normal" font-weight="normal" text-anchor="middle" textLength="30" x="128" y="59">ubuntu</text> <text fill="rgb(0,0,0)" font-family="sans-serif" font-size="9" font-style="normal" font-weight="normal" text-anchor="middle" textLength="119" x="128" y="70">(LTS with point release)</text> @@ -35,13 +36,17 @@ <rect fill="rgb(255,255,255)" height="40" stroke="rgb(0,0,0)" width="128" x="448" y="360" /> <text fill="rgb(0,0,0)" font-family="sans-serif" font-size="9" font-style="normal" font-weight="normal" text-anchor="middle" textLength="70" x="512" y="385">julia-notebook</text> <rect fill="rgb(255,255,255)" height="40" stroke="rgb(0,0,0)" width="128" x="64" y="440" /> - <text fill="rgb(0,0,0)" font-family="sans-serif" font-size="9" font-style="normal" font-weight="normal" text-anchor="middle" textLength="95" x="128" y="465">tensorflow-notebook</text> + <text fill="rgb(0,0,0)" font-family="sans-serif" font-size="9" font-style="normal" font-weight="normal" text-anchor="middle" textLength="95" x="128" y="459">tensorflow-notebook</text> + <text fill="rgb(0,0,0)" font-family="sans-serif" font-size="9" font-style="normal" font-weight="normal" text-anchor="middle" textLength="65" x="128" y="470">+cuda variant</text> <rect fill="rgb(255,255,255)" height="40" stroke="rgb(0,0,0)" width="128" x="256" y="440" /> - <text fill="rgb(0,0,0)" font-family="sans-serif" font-size="9" font-style="normal" font-weight="normal" text-anchor="middle" textLength="100" x="320" y="465">datascience-notebook</text> + <text fill="rgb(0,0,0)" font-family="sans-serif" font-size="9" font-style="normal" font-weight="normal" text-anchor="middle" textLength="80" x="320" y="459">pytorch-notebook</text> + <text fill="rgb(0,0,0)" font-family="sans-serif" font-size="9" font-style="normal" font-weight="normal" text-anchor="middle" textLength="114" x="320" y="470">+cuda11/cuda12 variants</text> <rect fill="rgb(255,255,255)" height="40" stroke="rgb(0,0,0)" width="128" x="448" y="440" /> - <text fill="rgb(0,0,0)" font-family="sans-serif" font-size="9" font-style="normal" font-weight="normal" text-anchor="middle" textLength="80" x="512" y="465">pyspark-notebook</text> - <rect fill="rgb(255,255,255)" height="40" stroke="rgb(0,0,0)" width="128" x="448" y="520" /> - <text fill="rgb(0,0,0)" font-family="sans-serif" font-size="9" font-style="normal" font-weight="normal" text-anchor="middle" textLength="90" x="512" y="545">all-spark-notebook</text> + <text fill="rgb(0,0,0)" font-family="sans-serif" font-size="9" font-style="normal" font-weight="normal" text-anchor="middle" textLength="100" x="512" y="465">datascience-notebook</text> + <rect fill="rgb(255,255,255)" height="40" stroke="rgb(0,0,0)" width="128" x="640" y="440" /> + <text fill="rgb(0,0,0)" font-family="sans-serif" font-size="9" font-style="normal" font-weight="normal" text-anchor="middle" textLength="80" x="704" y="465">pyspark-notebook</text> + <rect fill="rgb(255,255,255)" height="40" stroke="rgb(0,0,0)" width="128" x="640" y="520" /> + <text fill="rgb(0,0,0)" font-family="sans-serif" font-size="9" font-style="normal" font-weight="normal" text-anchor="middle" textLength="90" x="704" y="545">all-spark-notebook</text> <path d="M 128 80 L 128 112" fill="none" stroke="rgb(0,0,0)" /> <polygon fill="rgb(0,0,0)" points="128,119 124,112 132,112 128,119" stroke="rgb(0,0,0)" /> <path d="M 128 160 L 128 192" fill="none" stroke="rgb(0,0,0)" /> @@ -59,15 +64,19 @@ <path d="M 512 340 L 512 352" fill="none" stroke="rgb(0,0,0)" /> <polygon fill="rgb(0,0,0)" points="512,359 508,352 516,352 512,359" stroke="rgb(0,0,0)" /> <path d="M 128 400 L 128 420" fill="none" stroke="rgb(0,0,0)" /> - <path d="M 128 420 L 512 420" fill="none" stroke="rgb(0,0,0)" /> - <path d="M 512 420 L 512 432" fill="none" stroke="rgb(0,0,0)" /> - <polygon fill="rgb(0,0,0)" points="512,439 508,432 516,432 512,439" stroke="rgb(0,0,0)" /> - <path d="M 128 400 L 128 432" fill="none" stroke="rgb(0,0,0)" /> - <polygon fill="rgb(0,0,0)" points="128,439 124,432 132,432 128,439" stroke="rgb(0,0,0)" /> - <path d="M 128 400 L 128 420" fill="none" stroke="rgb(0,0,0)" /> <path d="M 128 420 L 320 420" fill="none" stroke="rgb(0,0,0)" /> <path d="M 320 420 L 320 432" fill="none" stroke="rgb(0,0,0)" /> <polygon fill="rgb(0,0,0)" points="320,439 316,432 324,432 320,439" stroke="rgb(0,0,0)" /> - <path d="M 512 480 L 512 512" fill="none" stroke="rgb(0,0,0)" /> - <polygon fill="rgb(0,0,0)" points="512,519 508,512 516,512 512,519" stroke="rgb(0,0,0)" /> + <path d="M 128 400 L 128 420" fill="none" stroke="rgb(0,0,0)" /> + <path d="M 128 420 L 704 420" fill="none" stroke="rgb(0,0,0)" /> + <path d="M 704 420 L 704 432" fill="none" stroke="rgb(0,0,0)" /> + <polygon fill="rgb(0,0,0)" points="704,439 700,432 708,432 704,439" stroke="rgb(0,0,0)" /> + <path d="M 128 400 L 128 432" fill="none" stroke="rgb(0,0,0)" /> + <polygon fill="rgb(0,0,0)" points="128,439 124,432 132,432 128,439" stroke="rgb(0,0,0)" /> + <path d="M 128 400 L 128 420" fill="none" stroke="rgb(0,0,0)" /> + <path d="M 128 420 L 512 420" fill="none" stroke="rgb(0,0,0)" /> + <path d="M 512 420 L 512 432" fill="none" stroke="rgb(0,0,0)" /> + <polygon fill="rgb(0,0,0)" points="512,439 508,432 516,432 512,439" stroke="rgb(0,0,0)" /> + <path d="M 704 480 L 704 512" fill="none" stroke="rgb(0,0,0)" /> + <polygon fill="rgb(0,0,0)" points="704,519 700,512 708,512 704,519" stroke="rgb(0,0,0)" /> </svg> diff --git a/docs/index.rst b/docs/index.rst index 875ea2c57f..23aa0e90de 100644 --- a/docs/index.rst +++ b/docs/index.rst @@ -13,6 +13,7 @@ Table of Contents using/common using/specifics using/recipes + using/custom-images using/troubleshooting using/faq @@ -32,6 +33,7 @@ Table of Contents :maxdepth: 2 :caption: Maintainer Guide + maintaining/new-images-and-packages-policy maintaining/tasks maintaining/aarch64-runner diff --git a/docs/maintaining/aarch64-runner.md b/docs/maintaining/aarch64-runner.md index 45f9c9b3f8..1f27d084a7 100644 --- a/docs/maintaining/aarch64-runner.md +++ b/docs/maintaining/aarch64-runner.md @@ -6,7 +6,7 @@ Each runner is recommended to have at least _2 cores_ and _30 GB_ of disk space. Add a new runner: -- To use [Oracle OCI](https://www.oracle.com/cloud/), create a compute instance `VM.Standard.A1.Flex` . +- To use [Oracle OCI](https://www.oracle.com/cloud-0/), create a compute instance `VM.Standard.A1.Flex`. - To use [Google Cloud](https://cloud.google.com), use [this instruction](https://cloud.google.com/compute/docs/instances/create-arm-vm-instance#armpublicimage). Configure your runner: @@ -19,7 +19,7 @@ Configure your runner: This will perform the initial runner setup and create a user `runner-user` without `sudo` capabilities. -2. Setup new GitHub Runner under `runner-user` using [GitHub Instructions](https://github.com/jupyter/docker-stacks/settings/actions/runners/new?arch=arm64&os=linux). +2. Set up a new GitHub Runner under `runner-user` using [GitHub Instructions](https://github.com/jupyter/docker-stacks/settings/actions/runners/new?arch=arm64&os=linux). **Do not `./run.sh` yet**. 3. Run under `root`: diff --git a/docs/maintaining/new-images-and-packages-policy.md b/docs/maintaining/new-images-and-packages-policy.md new file mode 100644 index 0000000000..ec297451a1 --- /dev/null +++ b/docs/maintaining/new-images-and-packages-policy.md @@ -0,0 +1,35 @@ +# Policy on adding new images and packages + +There are many things we consider while adding new images and packages. + +Here is a non-exhaustive list of things we do care about: + +1. **Software health**, details, and maintenance status + - reasonable versioning is adopted, and the version is considered to be stable + - has been around for several years + - the package maintains documentation + - a changelog is actively maintained + - a release procedure with helpful automation is established + - multiple people are involved in the maintenance of the project + - provides a `conda-forge` package besides a `pypi` package, where both are kept up to date + - supports both `x86_64` and `aarch64` architectures +2. **Installation consequences** + - GitHub Actions build time + - Image sizes + - All requirements should be installed as well +3. Jupyter Docker Stacks _**image fit**_ + - new package or stack is changing (or inherits from) the most suitable stack +4. **Software impact** for users of docker-stacks images + - How this image can help existing users, or maybe reduce the need to build new images +5. Why it shouldn't just be a documented **recipe** +6. Impact on **security** + - Does the package open additional ports, or add new web endpoints, that could be exploited? + +With all this in mind, we have a voting group, that consists of +[@mathbunnyru](https://github.com/mathbunnyru), +[@consideRatio](https://github.com/consideRatio), +[@yuvipanda](https://github.com/yuvipanda), and +[@manics](https://github.com/manics). + +This voting group is responsible for accepting or declining new packages and stacks. +The change is accepted, if there are **at least 2 positive votes**. diff --git a/docs/maintaining/tasks.md b/docs/maintaining/tasks.md index ce37a24c1b..274bded73a 100644 --- a/docs/maintaining/tasks.md +++ b/docs/maintaining/tasks.md @@ -2,7 +2,7 @@ ## Merging Pull Requests -To build new images and publish them to the Docker Hub registry, do the following: +To build new images and publish them to the Registry, do the following: 1. Make sure GitHub Actions status checks pass for the PR. 2. Merge the PR. @@ -10,59 +10,62 @@ To build new images and publish them to the Docker Hub registry, do the followin ```{note} GitHub Actions are pretty reliable, so please investigate if some error occurs. - Building Docker images in PRs is the same after merging to the main branch, except there is an additional `push` step. + Building Docker images in PRs is the same as building them in the default branch, + except single-platform images are pushed to Registry and then tags are merged for `x86_64` and `aarch64`. ``` -4. Avoid merging another PR to the main branch until all pending builds are complete. - This way, you will know which commit might have broken the build and also have the correct tags for moving tags (like the `python` version). +4. Avoid merging another PR to the main branch until all pending builds in the main branch are complete. + This way, you will know which commit might have broken the build + and also have the correct tags for moving tags (like the `Python` version). ## Updating Python version -When a new `Python` version is released, we wait for two things: +When a new `Python` version is released, we wait for: - all the dependencies to be available (as wheels or in `conda-forge`). -- the first `python` patch release for this version. +- the first `Python` patch release for this version. This allows us to avoid many bugs, which can happen in a major release. ## Updating the Ubuntu Base Image -`docker-stacks-foundation` is based on the LTS Ubuntu docker image. -We wait for the first point release of the new LTS Ubuntu before updating the version. +`jupyter/docker-stacks-foundation` is based on the LTS Ubuntu docker image. +We are waiting for the first point release of the new LTS Ubuntu before updating the version. Other images are directly or indirectly inherited from `docker-stacks-foundation`. We rebuild our images automatically each week, which means they frequently receive updates. -When there's a security fix in the Ubuntu base image, it's a good idea to manually trigger images rebuild [from the GitHub actions workflow UI](https://github.com/jupyter/docker-stacks/actions/workflows/docker.yml). +When there's a security fix in the Ubuntu base image, it's a good idea to manually trigger the rebuild of images +[from the GitHub actions workflow UI](https://github.com/jupyter/docker-stacks/actions/workflows/docker.yml). Pushing the `Run Workflow` button will trigger this process. -## Adding a New Core Image to Docker Hub +## Adding a New Core Image to the Registry ```{note} -In general, we do not add new core images and ask contributors to either create a [recipe](../using/recipes.md) or [community stack](../contributing/stacks.md). +In general, we do not add new core images and ask contributors to either +create a [recipe](../using/recipes.md) or [community stack](../contributing/stacks.md). +We have a [policy](./new-images-and-packages-policy.md), which we consider when adding new images or new packages to existing images. ``` -When there's a new stack definition, do the following before merging the PR with the new stack: +You can see an example of adding a new image [here](https://github.com/jupyter/docker-stacks/pull/1936/files). -1. Ensure the PR includes an update to the stack overview diagram - [in the documentation](https://github.com/jupyter/docker-stacks/blob/main/docs/using/selecting.md#image-relationships). - The image links to the [blockdiag source](http://interactive.blockdiag.com/) used to create it. -2. Ensure the PR updates the [Makefile](https://github.com/jupyter/docker-stacks/blob/main/Makefile), which is used to build the stacks in order on GitHub Actions. -3. Ensure necessary tags/manifests are added for the new image in the [tagging](https://github.com/jupyter/docker-stacks/tree/main/tagging) folder. -4. Create a new repository in the `jupyter` org on Docker Hub named after the stack folder in the - git repo. -5. Grant the `stacks` team permission to write to the repo. +When there's a new stack definition, check before merging the PR: -## Adding a New Maintainer Account +1. PR includes an update to the stack overview diagram + [in the documentation](../using/selecting.md#image-relationships). + The image links to the [blockdiag source](http://interactive.blockdiag.com/) used to create it. +2. PR updates the [Makefile](https://github.com/jupyter/docker-stacks/blob/main/Makefile), + which is used to build the stacks in order on GitHub Actions. +3. Necessary Tagger(s)/Manifest(s) are added for the new image + in the [tagging](https://github.com/jupyter/docker-stacks/tree/main/tagging) folder. +4. A new repository is created in the `jupyter` organization in the Registry, + and it's named after the stack folder in the git repo. +5. Robot `Write` permission is added in the `Repository Settings`. -1. Visit <https://hub.docker.com/app/jupyter/team/stacks/users> -2. Add the maintainer's Docker Hub username. -3. Visit <https://github.com/orgs/jupyter/teams/docker-image-maintainers/members> -4. Add the maintainer's GitHub username. +## Adding a New Registry Owner Account -## Pushing a Build Manually +1. Visit <https://quay.io/organization/jupyter/teams/owners> +2. Add the maintainer's username. -If an automated build in GitHub Actions has got you down, do the following to push a build manually: +## Restarting a failed build -1. Clone this repository. -2. Check out the git SHA you want to build and publish. -3. `docker login` with your Docker Hub credentials. -4. Run `make push-all`. +If an automated build in GitHub Actions has got you down, you can restart failed steps on GitHub. +You can also download the artifacts and investigate them for any issues. diff --git a/docs/using/common.md b/docs/using/common.md index 1161439e5b..ee8e587875 100644 --- a/docs/using/common.md +++ b/docs/using/common.md @@ -1,34 +1,34 @@ # Common Features -Except for `jupyter/docker-stacks-foundation`, a container launched from any Jupyter Docker Stacks image runs a Jupyter Server with a JupyterLab frontend. -The container does so by executing a `start-notebook.sh` script. +Except for `jupyter/docker-stacks-foundation`, a container launched from any Jupyter Docker Stacks image runs a Jupyter Server with the JupyterLab frontend. +The container does so by executing a `start-notebook.py` script. This script configures the internal container environment and then runs `jupyter lab`, passing any command-line arguments received. This page describes the options supported by the startup script and how to bypass it to run alternative commands. ## Jupyter Server Options -You can pass [Jupyter server options](https://jupyter-server.readthedocs.io/en/latest/operators/public-server.html) to the `start-notebook.sh` script when launching the container. +You can pass [Jupyter Server options](https://jupyter-server.readthedocs.io/en/latest/operators/public-server.html) to the `start-notebook.py` script when launching the container. -1. For example, to secure the Notebook server with a [custom password](https://jupyter-server.readthedocs.io/en/latest/operators/public-server.html#preparing-a-hashed-password) - hashed using `jupyter_server.auth.security.passwd()` instead of the default token, +1. For example, to secure the Jupyter Server with a [custom password](https://jupyter-server.readthedocs.io/en/latest/operators/public-server.html#preparing-a-hashed-password) + hashed using `jupyter_server.auth.passwd()` instead of the default token, you can run the following (this hash was generated for the `my-password` password): ```bash - docker run -it --rm -p 8888:8888 jupyter/base-notebook \ - start-notebook.sh --NotebookApp.password='argon2:$argon2id$v=19$m=10240,t=10,p=8$JdAN3fe9J45NvK/EPuGCvA$O/tbxglbwRpOFuBNTYrymAEH6370Q2z+eS1eF4GM6Do' + docker run -it --rm -p 8888:8888 quay.io/jupyter/base-notebook \ + start-notebook.py --PasswordIdentityProvider.hashed_password='argon2:$argon2id$v=19$m=10240,t=10,p=8$JdAN3fe9J45NvK/EPuGCvA$O/tbxglbwRpOFuBNTYrymAEH6370Q2z+eS1eF4GM6Do' ``` -2. To set the [base URL](https://jupyter-server.readthedocs.io/en/latest/operators/public-server.html#running-the-notebook-with-a-customized-url-prefix) of the notebook server, you can run the following: +2. To set the [base URL](https://jupyter-server.readthedocs.io/en/latest/operators/public-server.html#running-the-notebook-with-a-customized-url-prefix) of the Jupyter Server, you can run the following: ```bash - docker run -it --rm -p 8888:8888 jupyter/base-notebook \ - start-notebook.sh --NotebookApp.base_url=/customized/url/prefix/ + docker run -it --rm -p 8888:8888 quay.io/jupyter/base-notebook \ + start-notebook.py --ServerApp.base_url=/customized/url/prefix/ ``` ## Docker Options -You may instruct the `start-notebook.sh` script to customize the container environment before launching the notebook server. +You may instruct the `start-notebook.py` script to customize the container environment before launching the Server. You do so by passing arguments to the `docker run` command. ### User-related configurations @@ -37,7 +37,7 @@ You do so by passing arguments to the `docker run` command. The default value is `jovyan`. Setting `NB_USER` refits the `jovyan` default user and ensures that the desired user has the correct file permissions for the new home directory created at `/home/<username>`. - For this option to take effect, you **must** run the container with `--user root`, set the working directory `-w "/home/${NB_USER}"` + For this option to take effect, you **must** run the container with `--user root`, set the working directory `-w "/home/<username>"` and set the environment variable `-e CHOWN_HOME=yes`. _Example usage:_ @@ -48,8 +48,13 @@ You do so by passing arguments to the `docker run` command. --user root \ -e NB_USER="my-username" \ -e CHOWN_HOME=yes \ - -w "/home/${NB_USER}" \ - jupyter/base-notebook + -w "/home/my-username" \ + quay.io/jupyter/base-notebook + ``` + + ```{note} + If you set `NB_USER` to `root`, the `root` home dir will be set to `/home/root`. + See discussion [here](https://github.com/jupyter/docker-stacks/issues/2042). ``` - `-e NB_UID=<numeric uid>` - Instructs the startup script to switch the numeric user ID of `${NB_USER}` to the given value. @@ -57,8 +62,8 @@ You do so by passing arguments to the `docker run` command. This feature is useful when mounting host volumes with specific owner permissions. You **must** run the container with `--user root` for this option to take effect. (The startup script will `su ${NB_USER}` after adjusting the user ID.) - Instead, you might consider using the modern Docker-native options [`--user`](https://docs.docker.com/engine/reference/run/#user) and - [`--group-add`](https://docs.docker.com/engine/reference/run/#additional-groups) - see the last bullet in this section for more details. + Instead, you might consider using the modern Docker-native options [`--user`](https://docs.docker.com/engine/containers/run/#user) and + [`--group-add`](https://docs.docker.com/engine/containers/run/#additional-groups) - see the last bullet in this section for more details. See bullet points regarding `--user` and `--group-add`. - `-e NB_GID=<numeric gid>` - Instructs the startup script to change the primary group of `${NB_USER}` to `${NB_GID}` @@ -86,7 +91,7 @@ You do so by passing arguments to the `docker run` command. ```{note} `NB_UMASK` when set only applies to the Jupyter process itself - - you cannot use it to set a `umask` for additional files created during run-hooks. + you cannot use it to set a `umask` for additional files created during `run-hooks.sh`. For example, via `pip` or `conda`. If you need to set a `umask` for these, you **must** set the `umask` value for each command. ``` @@ -104,7 +109,7 @@ You do so by passing arguments to the `docker run` command. You do **not** need this option to allow the user to `conda` or `pip` install additional packages. This option is helpful for cases when you wish to give `${NB_USER}` the ability to install OS packages with `apt` or modify other root-owned files in the container. You **must** run the container with `--user root` for this option to take effect. - (The `start-notebook.sh` script will `su ${NB_USER}` after adding `${NB_USER}` to sudoers.) + (The `start-notebook.py` script will `su ${NB_USER}` after adding `${NB_USER}` to sudoers.) **You should only enable `sudo` if you trust the user or if the container runs on an isolated host.** ### Additional runtime configurations @@ -133,9 +138,9 @@ or executables (`chmod +x`) to be run to the paths below: - `/usr/local/bin/start-notebook.d/` - handled **before** any of the standard options noted above are applied - `/usr/local/bin/before-notebook.d/` - handled **after** all the standard options noted above are applied - and ran right before the notebook server launches + and ran right before the Server launches -See the `run-hooks` function in the [`jupyter/base-notebook start.sh`](https://github.com/jupyter/docker-stacks/blob/main/docker-stacks-foundation/start.sh) +See the `run-hooks.sh` script [here](https://github.com/jupyter/docker-stacks/blob/main/images/docker-stacks-foundation/run-hooks.sh) and how it's used in the [`start.sh`](https://github.com/jupyter/docker-stacks/blob/main/images/docker-stacks-foundation/start.sh) script for execution details. ## SSL Certificates @@ -146,10 +151,10 @@ For example, to mount a host folder containing a `notebook.key` and `notebook.cr ```bash docker run -it --rm -p 8888:8888 \ -v /some/host/folder:/etc/ssl/notebook \ - jupyter/base-notebook \ - start-notebook.sh \ - --NotebookApp.keyfile=/etc/ssl/notebook/notebook.key \ - --NotebookApp.certfile=/etc/ssl/notebook/notebook.crt + quay.io/jupyter/base-notebook \ + start-notebook.py \ + --ServerApp.keyfile=/etc/ssl/notebook/notebook.key \ + --ServerApp.certfile=/etc/ssl/notebook/notebook.crt ``` Alternatively, you may mount a single PEM file containing both the key and certificate. @@ -158,12 +163,12 @@ For example: ```bash docker run -it --rm -p 8888:8888 \ -v /some/host/folder/notebook.pem:/etc/ssl/notebook.pem \ - jupyter/base-notebook \ - start-notebook.sh \ - --NotebookApp.certfile=/etc/ssl/notebook.pem + quay.io/jupyter/base-notebook \ + start-notebook.py \ + --ServerApp.certfile=/etc/ssl/notebook.pem ``` -In either case, Jupyter Notebook expects the key and certificate to be a **base64 encoded text file**. +In either case, Jupyter Server expects the key and certificate to be a **base64 encoded text file**. The certificate file or PEM may contain one or more certificates (e.g., server, intermediate, and root). For additional information about using SSL, see the following: @@ -171,10 +176,10 @@ For additional information about using SSL, see the following: - The [docker-stacks/examples](https://github.com/jupyter/docker-stacks/tree/main/examples) for information about how to use [Let's Encrypt](https://letsencrypt.org/) certificates when you run these stacks on a publicly visible domain. -- The [`jupyter_server_config.py`](https://github.com/jupyter/docker-stacks/blob/main/base-notebook/jupyter_server_config.py) +- The [`jupyter_server_config.py`](https://github.com/jupyter/docker-stacks/blob/main/images/base-notebook/jupyter_server_config.py) file for how this Docker image generates a self-signed certificate. - The [Jupyter Server documentation](https://jupyter-server.readthedocs.io/en/latest/operators/public-server.html#securing-a-jupyter-server) - for best practices about securing a public notebook server in general. + for best practices about securing a public Server in general. ## Alternative Commands @@ -184,63 +189,61 @@ JupyterLab, built on top of Jupyter Server, is now the default for all the image However, switching back to the classic notebook or using a different startup command is still possible. You can achieve this by setting the environment variable `DOCKER_STACKS_JUPYTER_CMD` at container startup. The table below shows some options. - -| `DOCKER_STACKS_JUPYTER_CMD` | Backend | Frontend | -| --------------------------- | ---------------- | ---------------- | -| `lab` (default) | Jupyter Server | JupyterLab | -| `notebook` | Jupyter Notebook | Jupyter Notebook | -| `nbclassic` | Jupyter Server | Jupyter Notebook | -| `server` | Jupyter Server | None | -| `retro`\* | Jupyter Server | RetroLab | - -Notes: - -- \*Not installed at this time, but it could be the case in the future or in a community stack. -- Any other valid `jupyter` command that starts the Jupyter server can be used. +Since `Jupyter Notebook v7` `jupyter-server` is used as a backend. + +| `DOCKER_STACKS_JUPYTER_CMD` | Frontend | +| --------------------------- | ---------------- | +| `lab` (default) | JupyterLab | +| `notebook` | Jupyter Notebook | +| `nbclassic` | NbClassic | +| `server` | None | +| `retro`\* | RetroLab | + +```{note} +- Changing frontend for **JupyterHub singleuser image** is described in [JupyterHub docs](https://jupyterhub.readthedocs.io/en/latest/howto/configuration/config-user-env.html#switching-back-to-the-classic-notebook). +- \* `retro` is not installed at this time, but it could be the case in the future or in a community stack. +- Any other valid `jupyter` subcommand that starts the Jupyter Application can be used. +``` Example: ```bash -# Run Jupyter Notebook on Jupyter Server +# Run Jupyter Server with the Jupyter Notebook frontend docker run -it --rm \ -p 8888:8888 \ -e DOCKER_STACKS_JUPYTER_CMD=notebook \ - jupyter/base-notebook + quay.io/jupyter/base-notebook # Executing the command: jupyter notebook ... -# Run Jupyter Notebook classic +# Use Jupyter NBClassic frontend docker run -it --rm \ -p 8888:8888 \ -e DOCKER_STACKS_JUPYTER_CMD=nbclassic \ - jupyter/base-notebook + quay.io/jupyter/base-notebook # Executing the command: jupyter nbclassic ... ``` ### `start.sh` -The `start-notebook.sh` script inherits most of its option handling capability from a more generic `start.sh` script. -The `start.sh` script supports all the features described above but allows you to specify an arbitrary command to execute. +Most of the configuration options in the `start-notebook.py` script are handled by an internal `start.sh` script that automatically runs before the command provided to the container +(it's set as the container entrypoint). +This allows you to specify an arbitrary command that takes advantage of all these features. For example, to run the text-based `ipython` console in a container, do the following: ```bash -docker run -it --rm jupyter/base-notebook start.sh ipython +docker run -it --rm quay.io/jupyter/base-notebook ipython ``` This script is handy when you derive a new Dockerfile from this image and install additional Jupyter applications with subcommands like `jupyter console`, `jupyter kernelgateway`, etc. -### Others - -You can bypass the provided scripts and specify an arbitrary start command. -If you do, keep in mind that features supported by the `start.sh` script and its kin will not function (e.g., `GRANT_SUDO`). - ## Conda Environments -The default Python 3.x [Conda environment](https://conda.io/projects/conda/en/latest/user-guide/concepts/environments.html) resides in `/opt/conda`. +The default Python 3.x [Conda environment](https://docs.conda.io/projects/conda/en/latest/user-guide/concepts/environments.html) resides in `/opt/conda`. The `/opt/conda/bin` directory is part of the default `jovyan` user's `${PATH}`. That directory is also searched for binaries when run using `sudo` (`sudo my_binary` will search for `my_binary` in `/opt/conda/bin/` The `jovyan` user has full read/write access to the `/opt/conda` directory. -You can use either `mamba`, `pip` or `conda` (`mamba` is recommended) to install new packages without any additional permissions. +You can use either `mamba`, `pip`, or `conda` (`mamba` is recommended) to install new packages without any additional permissions. ```bash # install a package into the default (python 3.x) environment and cleanup it after @@ -260,7 +263,7 @@ conda install --yes some-package && \ fix-permissions "/home/${NB_USER}" ``` -### Using alternative channels +### Using Alternative Channels Conda is configured by default to use only the [`conda-forge`](https://anaconda.org/conda-forge) channel. However, you can use alternative channels, either one-shot by overwriting the default channel in the installation command or by configuring `mamba` to use different channels. diff --git a/docs/using/custom-images.md b/docs/using/custom-images.md new file mode 100644 index 0000000000..0c61e36bdc --- /dev/null +++ b/docs/using/custom-images.md @@ -0,0 +1,88 @@ +# Building a custom set of images + +This section describes how to build a custom set of images. +It may be helpful if you need to change the Ubuntu or Python version, or to make a significant change to the build process itself. + +This project only builds one set of images at a time. +If you want to use older images, take a look [here](https://jupyter-docker-stacks.readthedocs.io/en/latest/#using-old-images). + +## Automating your build using template cookiecutter project + +If you wish to build your own image on top of one of our images and automate your build process, +please, [take a look at cookiecutter template](../contributing/stacks.md). + +## Custom arguments + +Our repository provides several customization points: + +- `ROOT_IMAGE` (docker argument) - the parent image for `docker-stacks-foundation` image +- `PYTHON_VERSION` (docker argument) - the Python version to install in `docker-stacks-foundation` image +- `REGISTRY`, `OWNER`, `BASE_IMAGE` (docker arguments) - they allow to specify parent image for all the other images +- `REGISTRY`, `OWNER` (part of `env` in some GitHub workflows) - these allow to properly tag and refer to images during following steps: + [`build-test-upload`](https://github.com/jupyter/docker-stacks/blob/main/.github/workflows/docker-build-test-upload.yml), + [`tag-push`](https://github.com/jupyter/docker-stacks/blob/main/.github/workflows/docker-tag-push.yml) and + [`merge-tags`](https://github.com/jupyter/docker-stacks/blob/main/.github/workflows/docker-merge-tags.yml) + +These customization points can't be changed during runtime. +Read more about [Docker build arguments](https://docs.docker.com/build/building/variables/#arg-usage-example) and [GitHub environment variables for a single workflow](https://docs.github.com/en/actions/writing-workflows/choosing-what-your-workflow-does/store-information-in-variables#defining-environment-variables-for-a-single-workflow). + +## Building stack images with custom arguments + +A selection of prebuilt images are available from [Quay.io](https://quay.io/organization/jupyter), +however, it's impossible to cater to everybody's needs. +For extensive customization with an automated build pipeline, +you may wish to create a [community-maintained stack](../contributing/stacks), +however, for minor customizations, this may be overkill. +For example, you may wish to use the same Jupyter stacks but built on a different base image, +or built with a different Python version. + +To achieve this you can use [Docker Bake](https://docs.docker.com/build/bake/) +to build the stacks locally with custom arguments. + +```{note} +Custom arguments may result in build errors due to incompatibility. +If so your use-case may require a fully customized stack. +``` + +As a basic example, if you want to build a custom image based on the `minimal-notebook` image using `Python 3.12`, +then with a Dockerfile like: + +```{code-block} Dockerfile +:caption: Dockerfile + +ARG BASE_IMAGE=minimal-notebook +FROM $BASE_IMAGE +... +``` + +Include the below file in your project: + +```{literalinclude} recipe_code/docker-bake.python312.hcl +:force: +:language: hcl +:caption: docker-bake.hcl +``` + +To build this stack, in the same directory run: + +```bash +docker buildx bake +``` + +Docker Bake then determines the correct build order from the `contexts` parameters +and builds the stack as requested. + +This image can then be run the same way as any other image provided by this project, for example: + +```bash +docker run -it --rm -p 8888:8888 custom-jupyter +``` + +or referenced in a Docker Compose file. + +## Forking our repository + +If for some reason, you need to change more things in our images, feel free to fork it and change it any way you want. +If your customization is easy to backport to the main repo and might be helpful for other users, feel free to create a PR. + +It is almost always a great idea to keep your diff as small as possible and to merge/rebase the latest version of our repo in your project. diff --git a/docs/using/faq.md b/docs/using/faq.md index 6c4856e8d3..7f45d73e79 100644 --- a/docs/using/faq.md +++ b/docs/using/faq.md @@ -1,10 +1,32 @@ # Frequently Asked Questions (FAQ) +## How to persist user data + +There are 2 types of data, which you might want to persist. + +1. If you want to persist your environment (i.e. packages installed by `mamba`, `conda`, `pip`, `apt-get`, and so on), + then you should create an inherited image and install packages only once while building your Dockerfile. + An example of using `mamba` and `pip` in a child image is available + [here](./recipes.md#using-mamba-install-recommended-or-pip-install-in-a-child-docker-image). + + ```{note} + If you install a package inside a running container (for example you run `pip install <package>` in a terminal), + it won't be preserved when you next run your image. + To make it work, install this package in your inherited image and rerun `docker build` command. + ``` + +2. If you want to persist user data (files created by you, like `Python` scripts, notebooks, text files, and so on), + then you should use a + [Docker bind mount](https://docs.docker.com/engine/storage/bind-mounts/) or + [Docker Volume](https://docs.docker.com/engine/storage/volumes/). + You can find [an example of using a bind mount here](./running.md#example-2). + There is also [a mount troubleshooting section](./troubleshooting.md#permission-denied-when-mounting-volumes) if you experience any issues. + ## Why we do not add your favorite package We have lots of users with different packages they want to use. Adding them all is impossible, so we have several images to choose from. -[Choose the image](selecting.md), that is closest to your needs and feel free to [add your package on top of our images](recipes.md#using-mamba-install-or-pip-install-in-a-child-docker-image). +[Choose the image](selecting.md), that is closest to your needs, and feel free to [add your package on top of our images](recipes.md#using-mamba-install-recommended-or-pip-install-in-a-child-docker-image). ## Who is `jovyan` diff --git a/docs/using/recipe_code/custom_environment.dockerfile b/docs/using/recipe_code/custom_environment.dockerfile new file mode 100644 index 0000000000..547245c03f --- /dev/null +++ b/docs/using/recipe_code/custom_environment.dockerfile @@ -0,0 +1,48 @@ +FROM quay.io/jupyter/minimal-notebook + +# Name your environment and choose the Python version +ARG env_name=python310 +ARG py_ver=3.10 + +# You can add additional libraries here +RUN mamba create --yes -p "${CONDA_DIR}/envs/${env_name}" \ + python=${py_ver} \ + 'ipykernel' \ + 'jupyterlab' && \ + mamba clean --all -f -y + +# Alternatively, you can comment out the lines above and uncomment those below +# if you'd prefer to use a YAML file present in the docker build context + +# COPY --chown=${NB_UID}:${NB_GID} environment.yml /tmp/ +# RUN mamba env create -p "${CONDA_DIR}/envs/${env_name}" -f /tmp/environment.yml && \ +# mamba clean --all -f -y + +# Create Python kernel and link it to jupyter +RUN "${CONDA_DIR}/envs/${env_name}/bin/python" -m ipykernel install --user --name="${env_name}" && \ + fix-permissions "${CONDA_DIR}" && \ + fix-permissions "/home/${NB_USER}" + +# Any additional `pip` installs can be added by using the following line +# Using `mamba` is highly recommended though +RUN "${CONDA_DIR}/envs/${env_name}/bin/pip" install --no-cache-dir \ + 'flake8' + +# This changes the custom Python kernel so that the custom environment will +# be activated for the respective Jupyter Notebook and Jupyter Console +# hadolint ignore=DL3059 +RUN /opt/setup-scripts/activate_notebook_custom_env.py "${env_name}" + +# Comment the line above and uncomment the section below instead to activate the custom environment by default +# Note: uncommenting this section makes "${env_name}" default both for Jupyter Notebook and Terminals +# More information here: https://github.com/jupyter/docker-stacks/pull/2047 +# USER root +# RUN \ +# # This changes a startup hook, which will activate the custom environment for the process +# echo conda activate "${env_name}" >> /usr/local/bin/before-notebook.d/10activate-conda-env.sh && \ +# # This makes the custom environment default in Jupyter Terminals for all users which might be created later +# echo conda activate "${env_name}" >> /etc/skel/.bashrc && \ +# # This makes the custom environment default in Jupyter Terminals for already existing NB_USER +# echo conda activate "${env_name}" >> "/home/${NB_USER}/.bashrc" + +USER ${NB_UID} diff --git a/docs/using/recipe_code/dask_jupyterlab.dockerfile b/docs/using/recipe_code/dask_jupyterlab.dockerfile new file mode 100644 index 0000000000..f1cd0e68e1 --- /dev/null +++ b/docs/using/recipe_code/dask_jupyterlab.dockerfile @@ -0,0 +1,10 @@ +FROM quay.io/jupyter/base-notebook + +# Install the Dask dashboard +RUN mamba install --yes 'dask-labextension' && \ + mamba clean --all -f -y && \ + fix-permissions "${CONDA_DIR}" && \ + fix-permissions "/home/${NB_USER}" + +# Dask Scheduler port +EXPOSE 8787 diff --git a/docs/using/recipe_code/docker-bake.python312.hcl b/docs/using/recipe_code/docker-bake.python312.hcl new file mode 100644 index 0000000000..aadbd18771 --- /dev/null +++ b/docs/using/recipe_code/docker-bake.python312.hcl @@ -0,0 +1,44 @@ +group "default" { + targets = ["custom-notebook"] +} + +target "foundation" { + context = "https://github.com/jupyter/docker-stacks.git#main:images/docker-stacks-foundation" + args = { + PYTHON_VERSION = "3.12" + } + tags = ["docker-stacks-foundation"] +} + +target "base-notebook" { + context = "https://github.com/jupyter/docker-stacks.git#main:images/base-notebook" + contexts = { + docker-stacks-foundation = "target:foundation" + } + args = { + BASE_IMAGE = "docker-stacks-foundation" + } + tags = ["base-notebook"] +} + +target "minimal-notebook" { + context = "https://github.com/jupyter/docker-stacks.git#main:images/minimal-notebook" + contexts = { + base-notebook = "target:base-notebook" + } + args = { + BASE_IMAGE = "base-notebook" + } + tags = ["minimal-notebook"] +} + +target "custom-notebook" { + context = "." + contexts = { + minimal-notebook = "target:minimal-notebook" + } + args = { + BASE_IMAGE = "minimal-notebook" + } + tags = ["custom-jupyter"] +} diff --git a/docs/using/recipe_code/generate_matrix.py b/docs/using/recipe_code/generate_matrix.py new file mode 100755 index 0000000000..f44736d245 --- /dev/null +++ b/docs/using/recipe_code/generate_matrix.py @@ -0,0 +1,26 @@ +#!/usr/bin/env python3 +# Copyright (c) Jupyter Development Team. +# Distributed under the terms of the Modified BSD License. +import json +import os +from pathlib import Path +from typing import Any + +THIS_DIR = Path(__file__).parent.resolve() +REPOSITORY_OWNER = os.environ["REPOSITORY_OWNER"] + + +def generate_matrix() -> dict[str, Any]: + dockerfiles = sorted(file.name for file in THIS_DIR.glob("*.dockerfile")) + runs_on = ["ubuntu-latest"] + if REPOSITORY_OWNER == "jupyter": + runs_on.append("ARM64") + return { + "dockerfile": dockerfiles, + "runs-on": runs_on, + "exclude": [{"dockerfile": "oracledb.dockerfile", "runs-on": "ARM64"}], + } + + +if __name__ == "__main__": + print("matrix=" + json.dumps(generate_matrix())) diff --git a/docs/using/recipe_code/ijavascript.dockerfile b/docs/using/recipe_code/ijavascript.dockerfile new file mode 100644 index 0000000000..362fffdd80 --- /dev/null +++ b/docs/using/recipe_code/ijavascript.dockerfile @@ -0,0 +1,23 @@ +FROM quay.io/jupyter/base-notebook + +USER root + +RUN apt-get update --yes && \ + apt-get install --yes --no-install-recommends \ + make \ + g++ && \ + apt-get clean && rm -rf /var/lib/apt/lists/* + +USER ${NB_UID} + +# NodeJS <= 20 is required +# https://github.com/n-riesco/ijavascript/issues/184 +RUN mamba install --yes nodejs=20.* && \ + mamba clean --all -f -y && \ + fix-permissions "${CONDA_DIR}" && \ + fix-permissions "/home/${NB_USER}" + +# hadolint ignore=DL3016 +RUN npm install -g ijavascript +# hadolint ignore=DL3059 +RUN ijsinstall diff --git a/docs/using/recipe_code/jupyterhub_version.dockerfile b/docs/using/recipe_code/jupyterhub_version.dockerfile new file mode 100644 index 0000000000..7fd530186a --- /dev/null +++ b/docs/using/recipe_code/jupyterhub_version.dockerfile @@ -0,0 +1,6 @@ +FROM quay.io/jupyter/base-notebook + +RUN mamba install --yes 'jupyterhub==4.0.1' && \ + mamba clean --all -f -y && \ + fix-permissions "${CONDA_DIR}" && \ + fix-permissions "/home/${NB_USER}" diff --git a/docs/using/recipe_code/mamba_install.dockerfile b/docs/using/recipe_code/mamba_install.dockerfile new file mode 100644 index 0000000000..2c4d2c4e19 --- /dev/null +++ b/docs/using/recipe_code/mamba_install.dockerfile @@ -0,0 +1,13 @@ +FROM quay.io/jupyter/base-notebook + +RUN mamba install --yes 'flake8' && \ + mamba clean --all -f -y && \ + fix-permissions "${CONDA_DIR}" && \ + fix-permissions "/home/${NB_USER}" + +# Install from the requirements.txt file +COPY --chown=${NB_UID}:${NB_GID} requirements.txt /tmp/ +RUN mamba install --yes --file /tmp/requirements.txt && \ + mamba clean --all -f -y && \ + fix-permissions "${CONDA_DIR}" && \ + fix-permissions "/home/${NB_USER}" diff --git a/docs/using/recipe_code/manpage_install.dockerfile b/docs/using/recipe_code/manpage_install.dockerfile new file mode 100644 index 0000000000..ed8d91d12f --- /dev/null +++ b/docs/using/recipe_code/manpage_install.dockerfile @@ -0,0 +1,16 @@ +FROM quay.io/jupyter/base-notebook + +# Fix: https://github.com/hadolint/hadolint/wiki/DL4006 +# Fix: https://github.com/koalaman/shellcheck/wiki/SC3014 +SHELL ["/bin/bash", "-o", "pipefail", "-c"] + +USER root + +# `/etc/dpkg/dpkg.cfg.d/excludes` contains several `path-exclude`s, including man pages +# Remove it, then install man, install docs +RUN rm /etc/dpkg/dpkg.cfg.d/excludes && \ + apt-get update --yes && \ + dpkg -l | grep ^ii | cut -d' ' -f3 | xargs apt-get install --yes --no-install-recommends --reinstall man && \ + apt-get clean && rm -rf /var/lib/apt/lists/* + +USER ${NB_UID} diff --git a/docs/using/recipe_code/microsoft_odbc.dockerfile b/docs/using/recipe_code/microsoft_odbc.dockerfile new file mode 100644 index 0000000000..8642fdf4f6 --- /dev/null +++ b/docs/using/recipe_code/microsoft_odbc.dockerfile @@ -0,0 +1,30 @@ +FROM quay.io/jupyter/base-notebook:ubuntu-22.04 + +# Fix: https://github.com/hadolint/hadolint/wiki/DL4006 +# Fix: https://github.com/koalaman/shellcheck/wiki/SC3014 +SHELL ["/bin/bash", "-o", "pipefail", "-c"] + +USER root + +ENV MSSQL_DRIVER="ODBC Driver 18 for SQL Server" +ENV PATH="/opt/mssql-tools18/bin:${PATH}" + +RUN apt-get update --yes && \ + apt-get install --yes --no-install-recommends curl gnupg2 lsb-release && \ + curl https://packages.microsoft.com/keys/microsoft.asc | apt-key add - && \ + curl "https://packages.microsoft.com/config/ubuntu/$(lsb_release -rs)/prod.list" > /etc/apt/sources.list.d/mssql-release.list && \ + apt-get update --yes && \ + ACCEPT_EULA=Y apt-get install --yes --no-install-recommends msodbcsql18 && \ + # optional: for bcp and sqlcmd + ACCEPT_EULA=Y apt-get install --yes --no-install-recommends mssql-tools18 && \ + # optional: for unixODBC development headers + apt-get install --yes --no-install-recommends unixodbc-dev && \ + apt-get clean && rm -rf /var/lib/apt/lists/* + +# Switch back to jovyan to avoid accidental container runs as root +USER ${NB_UID} + +RUN mamba install --yes 'pyodbc' && \ + mamba clean --all -f -y && \ + fix-permissions "${CONDA_DIR}" && \ + fix-permissions "/home/${NB_USER}" diff --git a/docs/using/recipe_code/oracledb.dockerfile b/docs/using/recipe_code/oracledb.dockerfile new file mode 100644 index 0000000000..44ff2cd44c --- /dev/null +++ b/docs/using/recipe_code/oracledb.dockerfile @@ -0,0 +1,60 @@ +FROM quay.io/jupyter/base-notebook:ubuntu-22.04 + +USER root + +# Install Java & Oracle SQL Instant Client +RUN apt-get update --yes && \ + apt-get install --yes --no-install-recommends software-properties-common && \ + add-apt-repository universe && \ + apt-get update --yes && \ + apt-get install --yes --no-install-recommends alien default-jre default-jdk openjdk-11-jdk libaio1 && \ + apt-get clean && rm -rf /var/lib/apt/lists/* + +# Oracle +ARG INSTANTCLIENT_MAJOR_VERSION=23 +ARG INSTANTCLIENT_BIN_SUFFIX=${INSTANTCLIENT_MAJOR_VERSION}.4.0.24.05-1.el9.x86_64.rpm +ARG INSTANTCLIENT_URL=https://download.oracle.com/otn_software/linux/instantclient/2340000 + +# Then install Oracle SQL Instant client, SQL+Plus, tools, and JDBC. +# Note: You may need to change the URL to a newer version. +# See: https://www.oracle.com/es/database/technologies/instant-client/linux-x86-64-downloads.html +RUN mkdir "/opt/oracle" +WORKDIR "/opt/oracle" +# alien doesn't work well with sqlplus, so skipping it for now +RUN wget --progress=dot:giga "${INSTANTCLIENT_URL}/oracle-instantclient-basiclite-${INSTANTCLIENT_BIN_SUFFIX}" && \ + alien --install --scripts "oracle-instantclient-basiclite-${INSTANTCLIENT_BIN_SUFFIX}" && \ + wget --progress=dot:giga "${INSTANTCLIENT_URL}/oracle-instantclient-sqlplus-${INSTANTCLIENT_BIN_SUFFIX}" && \ + # alien --install --scripts "oracle-instantclient-sqlplus-${INSTANTCLIENT_BIN_SUFFIX}" && \ + wget --progress=dot:giga "${INSTANTCLIENT_URL}/oracle-instantclient-tools-${INSTANTCLIENT_BIN_SUFFIX}" && \ + alien --install --scripts "oracle-instantclient-tools-${INSTANTCLIENT_BIN_SUFFIX}" && \ + wget --progress=dot:giga "${INSTANTCLIENT_URL}/oracle-instantclient-jdbc-${INSTANTCLIENT_BIN_SUFFIX}" && \ + alien --install --scripts "oracle-instantclient-jdbc-${INSTANTCLIENT_BIN_SUFFIX}" && \ + chown -R "${NB_UID}":"${NB_GID}" "${HOME}/.rpmdb" && \ + rm -f ./*.rpm + +# And configure variables +RUN echo "ORACLE_HOME=/usr/lib/oracle/${INSTANTCLIENT_MAJOR_VERSION}/client64" >> "${HOME}/.bashrc" && \ + echo "PATH=\"${ORACLE_HOME}/bin:${PATH}\"" >> "${HOME}/.bashrc" && \ + echo "LD_LIBRARY_PATH=\"${ORACLE_HOME}/lib:${LD_LIBRARY_PATH}\"" >> "${HOME}/.bashrc" && \ + echo "export ORACLE_HOME" >> "${HOME}/.bashrc" && \ + echo "export PATH" >> "${HOME}/.bashrc" && \ + echo "export LD_LIBRARY_PATH" >> "${HOME}/.bashrc" + +# Add credentials for /redacted/ using Oracle DB. +WORKDIR /usr/lib/oracle/${INSTANTCLIENT_MAJOR_VERSION}/client64/lib/network/admin/ +# Add a wildcard `[]` on the last letter of the filename to avoid throwing an error if the file does not exist. +# See: https://stackoverflow.com/questions/31528384/conditional-copy-add-in-dockerfile +COPY cwallet.ss[o] ./ +COPY sqlnet.or[a] ./ +COPY tnsnames.or[a] ./ + +# Switch back to jovyan to avoid accidental container runs as root +USER "${NB_UID}" + +WORKDIR "${HOME}" + +# Install `oracledb` Python library to use Oracle SQL Instant Client +RUN mamba install --yes 'oracledb' && \ + mamba clean --all -f -y && \ + fix-permissions "${CONDA_DIR}" && \ + fix-permissions "/home/${NB_USER}" diff --git a/docs/using/recipe_code/pip_install.dockerfile b/docs/using/recipe_code/pip_install.dockerfile new file mode 100644 index 0000000000..fc6508bc73 --- /dev/null +++ b/docs/using/recipe_code/pip_install.dockerfile @@ -0,0 +1,12 @@ +FROM quay.io/jupyter/base-notebook + +# Install in the default python3 environment +RUN pip install --no-cache-dir 'flake8' && \ + fix-permissions "${CONDA_DIR}" && \ + fix-permissions "/home/${NB_USER}" + +# Install from the requirements.txt file +COPY --chown=${NB_UID}:${NB_GID} requirements.txt /tmp/ +RUN pip install --no-cache-dir --requirement /tmp/requirements.txt && \ + fix-permissions "${CONDA_DIR}" && \ + fix-permissions "/home/${NB_USER}" diff --git a/docs/using/recipe_code/requirements.txt b/docs/using/recipe_code/requirements.txt new file mode 100644 index 0000000000..3379e2a569 --- /dev/null +++ b/docs/using/recipe_code/requirements.txt @@ -0,0 +1 @@ +autoflake diff --git a/docs/using/recipe_code/rise_jupyterlab.dockerfile b/docs/using/recipe_code/rise_jupyterlab.dockerfile new file mode 100644 index 0000000000..7d796ca991 --- /dev/null +++ b/docs/using/recipe_code/rise_jupyterlab.dockerfile @@ -0,0 +1,6 @@ +FROM quay.io/jupyter/base-notebook + +RUN mamba install --yes 'jupyterlab_rise' && \ + mamba clean --all -f -y && \ + fix-permissions "${CONDA_DIR}" && \ + fix-permissions "/home/${NB_USER}" diff --git a/docs/using/recipe_code/spellcheck_notebook_v6.dockerfile b/docs/using/recipe_code/spellcheck_notebook_v6.dockerfile new file mode 100644 index 0000000000..1e12b562bc --- /dev/null +++ b/docs/using/recipe_code/spellcheck_notebook_v6.dockerfile @@ -0,0 +1,9 @@ +# Using Docker Hub here, because this image is old and not pushed to Quay.io +FROM docker.io/jupyter/base-notebook:notebook-6.5.4 + +RUN pip install --no-cache-dir 'jupyter_contrib_nbextensions' && \ + jupyter contrib nbextension install --user && \ + # can modify or enable additional extensions here + jupyter nbclassic-extension enable spellchecker/main --user && \ + fix-permissions "${CONDA_DIR}" && \ + fix-permissions "/home/${NB_USER}" diff --git a/docs/using/recipe_code/xgboost.dockerfile b/docs/using/recipe_code/xgboost.dockerfile new file mode 100644 index 0000000000..14afc79e35 --- /dev/null +++ b/docs/using/recipe_code/xgboost.dockerfile @@ -0,0 +1,6 @@ +FROM quay.io/jupyter/base-notebook + +RUN mamba install --yes 'py-xgboost' && \ + mamba clean --all -f -y && \ + fix-permissions "${CONDA_DIR}" && \ + fix-permissions "/home/${NB_USER}" diff --git a/docs/using/recipes.md b/docs/using/recipes.md index 71af470f71..76cb1dea35 100644 --- a/docs/using/recipes.md +++ b/docs/using/recipes.md @@ -4,6 +4,9 @@ Users sometimes share interesting ways of using the Jupyter Docker Stacks. We encourage users to [contribute these recipes](../contributing/recipes.md) to the documentation in case they prove helpful to other community members by submitting a pull request to `docs/using/recipes.md`. The sections below capture this knowledge. +All the recipes here assume you would like to use an image built by this project and install some things on top of it. +If you would like to build a custom set of images, [take a look at the docs](custom-images.md). + ## Using `sudo` within a container Password authentication is disabled for the `NB_USER` (e.g., `jovyan`). @@ -17,95 +20,50 @@ For example: docker run -it --rm \ --user root \ -e GRANT_SUDO=yes \ - jupyter/minimal-notebook + quay.io/jupyter/base-notebook ``` **You should only enable `sudo` if you trust the user and/or if the container is running on an isolated host.** See [Docker security documentation](https://docs.docker.com/engine/security/userns-remap/) for more information about running containers as `root`. -## Using `mamba install` or `pip install` in a Child Docker image +## Using `mamba install` (recommended) or `pip install` in a Child Docker image Create a new Dockerfile like the one shown below. +To use a requirements.txt file, first, create your `requirements.txt` file with the listing of packages desired. -```dockerfile -# Start from a core stack version -FROM jupyter/datascience-notebook:2023-06-01 -# Install in the default python3 environment -RUN pip install --no-cache-dir 'flake8==3.9.2' && \ - fix-permissions "${CONDA_DIR}" && \ - fix-permissions "/home/${NB_USER}" +```{literalinclude} recipe_code/mamba_install.dockerfile +:language: docker ``` -Then build a new image. +`pip` usage is similar: -```bash -docker build --rm -t jupyter/my-datascience-notebook . +```{literalinclude} recipe_code/pip_install.dockerfile +:language: docker ``` -To use a requirements.txt file, first, create your `requirements.txt` file with the listing of -packages desired. -Next, create a new Dockerfile like the one shown below. +Then build a new image. -```dockerfile -# Start from a core stack version -FROM jupyter/datascience-notebook:2023-06-01 -# Install from the requirements.txt file -COPY --chown=${NB_UID}:${NB_GID} requirements.txt /tmp/ -RUN pip install --no-cache-dir --requirement /tmp/requirements.txt && \ - fix-permissions "${CONDA_DIR}" && \ - fix-permissions "/home/${NB_USER}" +```bash +docker build --rm --tag my-custom-image . ``` -For conda, the Dockerfile is similar: +You can then run the image as follows: -```dockerfile -# Start from a core stack version -FROM jupyter/datascience-notebook:2023-06-01 -# Install from the requirements.txt file -COPY --chown=${NB_UID}:${NB_GID} requirements.txt /tmp/ -RUN mamba install --yes --file /tmp/requirements.txt && \ - mamba clean --all -f -y && \ - fix-permissions "${CONDA_DIR}" && \ - fix-permissions "/home/${NB_USER}" +```bash +docker run -it --rm \ + -p 8888:8888 \ + my-custom-image ``` -Ref: [docker-stacks/commit/79169618d571506304934a7b29039085e77db78c](https://github.com/jupyter/docker-stacks/commit/79169618d571506304934a7b29039085e77db78c#r15960081) - ## Add a custom conda environment and Jupyter kernel -The default version of Python that ships with the image may not be the version you want. -The instructions below permit adding a conda environment with a different Python version and making it accessible to Jupyter. - -```dockerfile -# Choose your desired base image -FROM jupyter/minimal-notebook:latest - -# name your environment and choose the python version -ARG conda_env=python37 -ARG py_ver=3.7 - -# you can add additional libraries you want mamba to install by listing them below the first line and ending with "&& \" -RUN mamba create --yes -p "${CONDA_DIR}/envs/${conda_env}" python=${py_ver} ipython ipykernel && \ - mamba clean --all -f -y - -# alternatively, you can comment out the lines above and uncomment those below -# if you'd prefer to use a YAML file present in the docker build context - -# COPY --chown=${NB_UID}:${NB_GID} environment.yml "/home/${NB_USER}/tmp/" -# RUN cd "/home/${NB_USER}/tmp/" && \ -# mamba env create -p "${CONDA_DIR}/envs/${conda_env}" -f environment.yml && \ -# mamba clean --all -f -y - -# create Python kernel and link it to jupyter -RUN "${CONDA_DIR}/envs/${conda_env}/bin/python" -m ipykernel install --user --name="${conda_env}" && \ - fix-permissions "${CONDA_DIR}" && \ - fix-permissions "/home/${NB_USER}" - -# any additional pip installs can be added by uncommenting the following line -# RUN "${CONDA_DIR}/envs/${conda_env}/bin/pip" install --no-cache-dir +The default version of `Python` that ships with the image may not be the version you want. +The instructions below permit adding a conda environment with a different `Python` version and making it accessible to Jupyter. +You may also use older images like `jupyter/base-notebook:python-3.10`. +A list of all tags can be found [here](https://github.com/jupyter/docker-stacks/wiki) -# if you want this environment to be the default one, uncomment the following line: -# RUN echo "conda activate ${conda_env}" >> "${HOME}/.bashrc" +```{literalinclude} recipe_code/custom_environment.dockerfile +:language: docker ``` ## Dask JupyterLab Extension @@ -113,26 +71,14 @@ RUN "${CONDA_DIR}/envs/${conda_env}/bin/python" -m ipykernel install --user --na [Dask JupyterLab Extension](https://github.com/dask/dask-labextension) provides a JupyterLab extension to manage Dask clusters, as well as embed Dask's dashboard plots directly into JupyterLab panes. Create the Dockerfile as: -```dockerfile -# Start from a core stack version -FROM jupyter/scipy-notebook:latest - -# Install the Dask dashboard -RUN pip install --no-cache-dir dask-labextension && \ - fix-permissions "${CONDA_DIR}" && \ - fix-permissions "/home/${NB_USER}" - -# Dask Scheduler & Bokeh ports -EXPOSE 8787 -EXPOSE 8786 - -ENTRYPOINT ["jupyter", "lab", "--ip=0.0.0.0", "--allow-root"] +```{literalinclude} recipe_code/dask_jupyterlab.dockerfile +:language: docker ``` And build the image as: ```bash -docker build --tag jupyter/scipy-dasklabextension:latest . +docker build --rm --tag my-custom-image . ``` Once built, run using the command: @@ -140,55 +86,47 @@ Once built, run using the command: ```bash docker run -it --rm \ -p 8888:8888 \ - -p 8787:8787 jupyter/scipy-dasklabextension:latest + -p 8787:8787 \ + my-custom-image ``` -Ref: <https://github.com/jupyter/docker-stacks/issues/999> +## Let's Encrypt a Server -## Let's Encrypt a Notebook server +```{warning} +This recipe is not tested and might be broken. +``` -See the README for a basic automation here +See the README for basic automation here <https://github.com/jupyter/docker-stacks/tree/main/examples/make-deploy> which includes steps for requesting and renewing a Let's Encrypt certificate. Ref: <https://github.com/jupyter/docker-stacks/issues/78> -## Slideshows with Jupyter and RISE +## Slideshows with JupyterLab and RISE -[RISE](https://github.com/damianavila/RISE) allows via an extension to create live slideshows of your -notebooks, with no conversion, adding javascript Reveal.js: +[RISE](https://github.com/jupyterlab-contrib/rise): "Live" Reveal.js JupyterLab Slideshow Extension. -```bash -# Add Live slideshows with RISE -RUN mamba install --yes -c damianavila82 rise && \ - mamba clean --all -f -y && \ - fix-permissions "${CONDA_DIR}" && \ - fix-permissions "/home/${NB_USER}" +```{note} +We're providing the recipe to install the JupyterLab extension. +You can find the original Jupyter Notebook extension [here](https://github.com/damianavila/RISE) ``` -Credit: [Paolo D.](https://github.com/pdonorio) based on -[docker-stacks/issues/43](https://github.com/jupyter/docker-stacks/issues/43) +```{literalinclude} recipe_code/rise_jupyterlab.dockerfile +:language: docker +``` ## xgboost -You need to install conda-forge's gcc for Python xgboost to work correctly. -Otherwise, you'll get an exception about libgomp.so.1 missing GOMP_4.0. - -```bash -mamba install --yes gcc && \ - mamba clean --all -f -y && \ - fix-permissions "${CONDA_DIR}" && \ - fix-permissions "/home/${NB_USER}" - -pip install --no-cache-dir xgboost && \ - fix-permissions "${CONDA_DIR}" && \ - fix-permissions "/home/${NB_USER}" - -# run "import xgboost" in python +```{literalinclude} recipe_code/xgboost.dockerfile +:language: docker ``` ## Running behind an nginx proxy +```{warning} +This recipe is not tested and might be broken. +``` + Sometimes it is helpful to run the Jupyter instance behind an nginx proxy, for example: - you would prefer to access the notebook at a server URL with a path @@ -197,8 +135,8 @@ Sometimes it is helpful to run the Jupyter instance behind an nginx proxy, for e and want nginx to help improve server performance in managing the connections Here is a [quick example of NGINX configuration](https://gist.github.com/cboettig/8643341bd3c93b62b5c2) to get started. -You'll need a server, a `.crt` and `.key` file for your server, and `docker` & `docker-compose` installed. -Then download the files at that gist and run `docker-compose up -d` to test it out. +You'll need a server, a `.crt`, and a `.key` file for your server, and `docker` & `docker-compose` installed. +Then download the files at that gist and run `docker-compose up` to test it out. Customize the `nginx.conf` file to set the desired paths and add other services. ## Host volume mounts and notebook errors @@ -213,101 +151,63 @@ Ref: <https://github.com/jupyter/docker-stacks/issues/199> ## Manpage installation -Most containers, including our Ubuntu base image, ship without manpages installed to save space. +Most images, including our Ubuntu base image, ship without manpages installed to save space. You can use the following Dockerfile to inherit from one of our images to enable manpages: -```dockerfile -# Choose your desired base image -ARG BASE_CONTAINER=jupyter/datascience-notebook:latest -FROM $BASE_CONTAINER - -USER root - -# `/etc/dpkg/dpkg.cfg.d/excludes` contains several `path-exclude`s, including man pages -# Remove it, then install man, install docs -RUN rm /etc/dpkg/dpkg.cfg.d/excludes && \ - apt-get update --yes && \ - dpkg -l | grep ^ii | cut -d' ' -f3 | xargs apt-get install --yes --no-install-recommends --reinstall man && \ - apt-get clean && rm -rf /var/lib/apt/lists/* - -USER ${NB_UID} +```{literalinclude} recipe_code/manpage_install.dockerfile +:language: docker ``` -Adding the documentation on top of the existing single-user image wastes a lot of space +Adding the documentation on top of the existing image wastes a lot of space and requires reinstalling every system package, which can take additional time and bandwidth. -The `datascience-notebook` image has been shown to grow by almost 3GB when adding manpages in this way. -Enabling manpages in the base Ubuntu layer prevents this container bloat. -To achieve this, use the previous `Dockerfile`'s commands with the original `ubuntu` image as your base container: - -```dockerfile -ARG BASE_CONTAINER=ubuntu:22.04 -``` - -For Ubuntu 18.04 (bionic) and earlier, you may also require to a workaround for a mandb bug, which was fixed in mandb >= 2.8.6.1: +Enabling manpages in the base Ubuntu layer prevents this image bloat. +To achieve this, use the previous `Dockerfile`'s commands with the original `ubuntu` image as your base image: ```dockerfile -# https://git.savannah.gnu.org/cgit/man-db.git/commit/?id=8197d7824f814c5d4b992b4c8730b5b0f7ec589a -# https://launchpadlibrarian.net/435841763/man-db_2.8.5-2_2.8.6-1.diff.gz - -RUN echo "MANPATH_MAP ${CONDA_DIR}/bin ${CONDA_DIR}/man" >> /etc/manpath.config && \ - echo "MANPATH_MAP ${CONDA_DIR}/bin ${CONDA_DIR}/share/man" >> /etc/manpath.config && \ - mandb +FROM ubuntu:24.04 ``` -Be sure to check the current base image in `base-notebook` before building. +Be sure to check the current base image in `jupyter/docker-stacks-foundation` before building. ## JupyterHub We also have contributed recipes for using JupyterHub. -### Use JupyterHub's dockerspawner - -In most cases for use with DockerSpawner, given an image that already has a notebook stack set up, -you would only need to add: +### Use JupyterHub's DockerSpawner -1. install the jupyterhub-singleuser script (for the correct Python version) -2. change the command to launch the single-user server - -Swapping out the `FROM` line in the `jupyterhub/singleuser` Dockerfile should be enough for most -cases. - -Credit: [Justin Tyberg](https://github.com/jtyberg), [quanghoc](https://github.com/quanghoc), and -[Min RK](https://github.com/minrk) based on -[docker-stacks/issues/124](https://github.com/jupyter/docker-stacks/issues/124) and -[docker-stacks/pull/185](https://github.com/jupyter/docker-stacks/pull/185) +You can find an example of using DockerSpawner [here](https://github.com/jupyterhub/jupyterhub-deploy-docker/tree/main/basic-example). ### Containers with a specific version of JupyterHub -To use a specific version of JupyterHub, the version of `jupyterhub` in your image should match the -version in the Hub itself. +The version of `jupyterhub` in your image should match the +version in JupyterHub itself. +To use a specific version of JupyterHub, do the following: -```dockerfile -FROM jupyter/base-notebook:2023-06-01 -RUN pip install --no-cache-dir jupyterhub==1.4.1 && \ - fix-permissions "${CONDA_DIR}" && \ - fix-permissions "/home/${NB_USER}" +```{literalinclude} recipe_code/jupyterhub_version.dockerfile +:language: docker ``` -Credit: [MinRK](https://github.com/jupyter/docker-stacks/issues/423#issuecomment-322767742) - -Ref: <https://github.com/jupyter/docker-stacks/issues/177> - ## Spark -A few suggestions have been made regarding using Docker Stacks with spark. +A few suggestions have been made regarding using Docker Stacks with Spark. ### Using PySpark with AWS S3 +```{warning} +This recipe is not tested and might be broken. +``` + Using Spark session for Hadoop 2.7.3 ```python import os -# !ls /usr/local/spark/jars/hadoop* # to figure out what version of Hadoop -os.environ[ - "PYSPARK_SUBMIT_ARGS" -] = '--packages "org.apache.hadoop:hadoop-aws:2.7.3" pyspark-shell' +# To figure out what version of Hadoop, run: +# ls /usr/local/spark/jars/hadoop* +os.environ["PYSPARK_SUBMIT_ARGS"] = ( + '--packages "org.apache.hadoop:hadoop-aws:2.7.3" pyspark-shell' +) import pyspark @@ -329,9 +229,9 @@ Using Spark context for Hadoop 2.6.0 ```python import os -os.environ[ - "PYSPARK_SUBMIT_ARGS" -] = "--packages com.amazonaws:aws-java-sdk:1.10.34,org.apache.hadoop:hadoop-aws:2.6.0 pyspark-shell" +os.environ["PYSPARK_SUBMIT_ARGS"] = ( + "--packages com.amazonaws:aws-java-sdk:1.10.34,org.apache.hadoop:hadoop-aws:2.6.0 pyspark-shell" +) import pyspark @@ -355,12 +255,16 @@ Ref: <https://github.com/jupyter/docker-stacks/issues/127> ### Using Local Spark JARs +```{warning} +This recipe is not tested and might be broken. +``` + ```python import os -os.environ[ - "PYSPARK_SUBMIT_ARGS" -] = "--jars /home/jovyan/spark-streaming-kafka-assembly_2.10-1.6.1.jar pyspark-shell" +os.environ["PYSPARK_SUBMIT_ARGS"] = ( + "--jars /home/jovyan/spark-streaming-kafka-assembly_2.10-1.6.1.jar pyspark-shell" +) import pyspark from pyspark.streaming.kafka import KafkaUtils from pyspark.streaming import StreamingContext @@ -379,6 +283,10 @@ Ref: <https://github.com/jupyter/docker-stacks/issues/154> ### Using spark-packages.org +```{warning} +This recipe is not tested and might be broken. +``` + If you'd like to use packages from [spark-packages.org](https://spark-packages.org/), see [https://gist.github.com/parente/c95fdaba5a9a066efaab](https://gist.github.com/parente/c95fdaba5a9a066efaab) for an example of how to specify the package identifier in the environment before creating a @@ -388,14 +296,18 @@ Ref: <https://github.com/jupyter/docker-stacks/issues/43> ### Use jupyter/all-spark-notebooks with an existing Spark/YARN cluster +```{warning} +This recipe is not tested and might be broken. +``` + ```dockerfile -FROM jupyter/all-spark-notebook +FROM quay.io/jupyter/all-spark-notebook # Set env vars for pydoop -ENV HADOOP_HOME /usr/local/hadoop-2.7.3 -ENV JAVA_HOME /usr/lib/jvm/java-8-openjdk-amd64 -ENV HADOOP_CONF_HOME /usr/local/hadoop-2.7.3/etc/hadoop -ENV HADOOP_CONF_DIR /usr/local/hadoop-2.7.3/etc/hadoop +ENV HADOOP_HOME=/usr/local/hadoop-2.7.3 +ENV JAVA_HOME=/usr/lib/jvm/java-8-openjdk-amd64 +ENV HADOOP_CONF_HOME=/usr/local/hadoop-2.7.3/etc/hadoop +ENV HADOOP_CONF_DIR=/usr/local/hadoop-2.7.3/etc/hadoop USER root # Add proper open-jdk-8 not the jre only, needed for pydoop @@ -422,7 +334,7 @@ RUN echo 'deb https://cdn-fastly.deb.debian.org/debian jessie-backports main' > COPY example-hadoop-conf/ /usr/local/hadoop-2.7.3/etc/hadoop/ # Spark-Submit doesn't work unless I set the following -RUN echo "spark.driver.extraJavaOptions -Dhdp.version=2.5.3.0-37" >> /usr/local/spark/conf/spark-defaults.conf && \ +RUN echo "spark.driver.extraJavaOptions -Dhdp.version=2.5.3.0-37" >> /usr/local/spark/conf/spark-defaults.conf && \ echo "spark.yarn.am.extraJavaOptions -Dhdp.version=2.5.3.0-37" >> /usr/local/spark/conf/spark-defaults.conf && \ echo "spark.master=yarn" >> /usr/local/spark/conf/spark-defaults.conf && \ echo "spark.hadoop.yarn.timeline-service.enabled=false" >> /usr/local/spark/conf/spark-defaults.conf && \ @@ -438,9 +350,9 @@ USER ${NB_UID} # - Dashboards # - PyDoop # - PyHive -RUN pip install --no-cache-dir jupyter_dashboards faker && \ +RUN pip install --no-cache-dir 'jupyter_dashboards' 'faker' && \ jupyter dashboards quick-setup --sys-prefix && \ - pip2 install --no-cache-dir pyhive pydoop thrift sasl thrift_sasl faker && \ + pip2 install --no-cache-dir 'pyhive' 'pydoop' 'thrift' 'sasl' 'thrift_sasl' 'faker' && \ fix-permissions "${CONDA_DIR}" && \ fix-permissions "/home/${NB_USER}" @@ -461,63 +373,57 @@ USER ${NB_UID} Credit: [britishbadger](https://github.com/britishbadger) from [docker-stacks/issues/369](https://github.com/jupyter/docker-stacks/issues/369) -## Run Jupyter Notebook/Lab inside an already secured environment (i.e., with no token) - -(Adapted from [issue 728](https://github.com/jupyter/docker-stacks/issues/728)) +## Run Server inside an already secured environment (i.e., with no token) The default security is very good. There are use cases, encouraged by containers, where the jupyter container and the system it runs within lie inside the security boundary. It is convenient to launch the server without a password or token in these use cases. -In this case, you should use the `start.sh` script to launch the server with no token: +In this case, you should use the `start-notebook.py` script to launch the server with no token: For JupyterLab: ```bash docker run -it --rm \ - jupyter/base-notebook:2023-06-01 \ - start.sh jupyter lab --LabApp.token='' + quay.io/jupyter/base-notebook \ + start-notebook.py --IdentityProvider.token='' ``` -For jupyter classic: +For Jupyter Notebook: ```bash docker run -it --rm \ - jupyter/base-notebook:2023-06-01 \ - start.sh jupyter notebook --NotebookApp.token='' + -e DOCKER_STACKS_JUPYTER_CMD=notebook \ + quay.io/jupyter/base-notebook \ + start-notebook.py --IdentityProvider.token='' ``` -## Enable nbextension spellchecker for markdown (or any other nbextension) - -NB: this works for classic notebooks only +## Enable nbclassic-extension spellchecker for markdown (or any other nbclassic-extension) -```dockerfile -# Update with your base image of choice -FROM jupyter/minimal-notebook:latest - -USER ${NB_UID} - -RUN pip install --no-cache-dir jupyter_contrib_nbextensions && \ - jupyter contrib nbextension install --user && \ - # can modify or enable additional extensions here - jupyter nbextension enable spellchecker/main --user && \ - fix-permissions "${CONDA_DIR}" && \ - fix-permissions "/home/${NB_USER}" +```{note} +This recipe only works for NBCassic with Jupyter Notebook < 7. +It is recommended to use [jupyterlab-spellchecker](https://github.com/jupyterlab-contrib/spellchecker) in modern environments. ``` -Ref: <https://github.com/jupyter/docker-stacks/issues/675> +```{literalinclude} recipe_code/spellcheck_notebook_v6.dockerfile +:language: docker +``` ## Enable Delta Lake in Spark notebooks +```{warning} +This recipe is not tested and might be broken. +``` + Please note that the [Delta Lake](https://delta.io/) packages are only available for Spark version > `3.0`. By adding the properties to `spark-defaults.conf`, the user no longer needs to enable Delta support in each notebook. ```dockerfile -FROM jupyter/pyspark-notebook:latest +FROM quay.io/jupyter/pyspark-notebook -ARG DELTA_CORE_VERSION="1.2.1" -RUN pip install --no-cache-dir delta-spark==${DELTA_CORE_VERSION} && \ - fix-permissions "${HOME}" && \ - fix-permissions "${CONDA_DIR}" +RUN mamba install --yes 'delta-spark' && \ + mamba clean --all -f -y && \ + fix-permissions "${CONDA_DIR}" && \ + fix-permissions "/home/${NB_USER}" USER root @@ -536,10 +442,14 @@ RUN echo "from pyspark.sql import SparkSession" > /tmp/init-delta.py && \ ## Add Custom Fonts in Scipy notebook +```{warning} +This recipe is not tested and might be broken. +``` + The example below is a Dockerfile to load Source Han Sans with normal weight, usually used for the web. ```dockerfile -FROM jupyter/scipy-notebook:latest +FROM quay.io/jupyter/scipy-notebook RUN PYV=$(ls "${CONDA_DIR}/lib" | grep ^python) && \ MPL_DATA="${CONDA_DIR}/lib/${PYV}/site-packages/matplotlib/mpl-data" && \ @@ -553,6 +463,10 @@ RUN PYV=$(ls "${CONDA_DIR}/lib" | grep ^python) && \ ## Enable clipboard in pandas on Linux systems +```{warning} +This recipe is not tested and might be broken. +``` + ```{admonition} Additional notes This solution works on Linux host systems. It is not required on Windows and won't work on macOS. @@ -566,51 +480,43 @@ and add these options when running `docker`: `-e DISPLAY -v /tmp/.X11-unix:/tmp/ docker run -it --rm \ -e DISPLAY \ -v /tmp/.X11-unix:/tmp/.X11-unix \ - jupyter/minimal-notebook + quay.io/jupyter/minimal-notebook ``` -## Add ijavascript kernel to container - -The example below is a Dockerfile to install the [ijavascript kernel](https://github.com/n-riesco/ijavascript). +## Install ijavascript kernel in your image -```dockerfile -# use one of the Jupyter Docker Stacks images -FROM jupyter/scipy-notebook:2023-06-01 +The example below is a Dockerfile to install the [IJavascript kernel](https://github.com/n-riesco/ijavascript). -# install ijavascript -RUN npm install -g ijavascript -RUN ijsinstall +```{literalinclude} recipe_code/ijavascript.dockerfile +:language: docker ``` ## Add Microsoft SQL Server ODBC driver The following recipe demonstrates how to add functionality to read from and write to an instance of Microsoft SQL server in your notebook. -```dockerfile -ARG BASE_IMAGE=jupyter/tensorflow-notebook - -FROM $BASE_IMAGE - -USER root +```{literalinclude} recipe_code/microsoft_odbc.dockerfile +:language: docker +``` -ENV MSSQL_DRIVER "ODBC Driver 18 for SQL Server" -ENV PATH="/opt/mssql-tools18/bin:${PATH}" +You can now use `pyodbc` and `sqlalchemy` to interact with the database. -RUN apt-get update --yes && \ - apt-get install --yes --no-install-recommends gnupg2 && \ - wget --progress=dot:giga https://packages.microsoft.com/keys/microsoft.asc | gpg --dearmor > /usr/share/keyrings/microsoft.gpg && \ - apt-get purge --yes gnupg2 && \ - echo "deb [arch=amd64,armhf,arm64 signed-by=/usr/share/keyrings/microsoft.gpg] https://packages.microsoft.com/ubuntu/22.04/prod jammy main" > /etc/apt/sources.list.d/microsoft.list && \ - apt-get update --yes && \ - ACCEPT_EULA=Y apt-get install --yes --no-install-recommends msodbcsql18 && \ - apt-get clean && rm -rf /var/lib/apt/lists/* +Pre-built images are hosted in the [Realiserad/jupyter-docker-mssql](https://github.com/Realiserad/jupyter-docker-mssql) repository. -# Switch back to jovyan to avoid accidental container runs as root -USER ${NB_UID} +## Add Oracle SQL Instant client, SQL\*Plus, and other tools (Version 21.x) -RUN pip install --no-cache-dir pyodbc +```{note} +This recipe only works for x86_64 architecture. ``` -You can now use `pyodbc` and `sqlalchemy` to interact with the database. +The following recipe demonstrates how to add functionality to connect to an Oracle Database using [Oracle Instant Client](https://www.oracle.com/database/technologies/instant-client.html) +in your notebook. +This recipe installs version `21.11.0.0.0`. + +Nonetheless, go to the [Oracle Instant Client Download page](https://www.oracle.com/es/database/technologies/instant-client/linux-x86-64-downloads.html) for the complete list of versions available. +You may need to perform different steps for older versions; +they may be explained in the "Installation instructions" section of the Downloads page. -Pre-built images are hosted in the [realiserad/jupyter-docker-mssql](https://github.com/Realiserad/jupyter-docker-mssql) repository. +```{literalinclude} recipe_code/oracledb.dockerfile +:language: docker +``` diff --git a/docs/using/running.md b/docs/using/running.md index 1f330b540e..4c2290ab92 100644 --- a/docs/using/running.md +++ b/docs/using/running.md @@ -9,18 +9,18 @@ This section provides details about the second. ## Using the Docker CLI -You can launch a local Docker container from the Jupyter Docker Stacks using the [Docker command-line interface](https://docs.docker.com/engine/reference/commandline/cli/). -There are numerous ways to configure containers using the CLI. +You can launch a local Docker container from the Jupyter Docker Stacks using the [Docker command-line interface](https://docs.docker.com/reference/cli/docker/). +There are numerous ways to configure containers using CLI. The following are some common patterns. -**Example 1:** +### Example 1 -This command pulls the `jupyter/scipy-notebook` image tagged `2023-06-01` from Docker Hub if it is not already present on the local host. -It then starts a container running a Jupyter Notebook server and exposes the server on host port 8888. -The server logs appear in the terminal and include a URL to the notebook server. +This command pulls the `jupyter/scipy-notebook` image tagged `2024-10-07` from Quay.io if it is not already present on the local host. +It then starts a container running Jupyter Server with the JupyterLab frontend and exposes the server on host port 8888. +The server logs appear in the terminal and include a URL to the server. ```bash -docker run -it -p 8888:8888 jupyter/scipy-notebook:2023-06-01 +docker run -it -p 8888:8888 quay.io/jupyter/scipy-notebook:2024-10-07 # Entered start.sh with args: jupyter lab @@ -29,56 +29,63 @@ docker run -it -p 8888:8888 jupyter/scipy-notebook:2023-06-01 # To access the server, open this file in a browser: # file:///home/jovyan/.local/share/jupyter/runtime/jpserver-7-open.html # Or copy and paste one of these URLs: -# http://042fc8ac2b0c:8888/lab?token=f31f2625f13d131f578fced0fc76b81d10f6c629e92c7099 -# or http://127.0.0.1:8888/lab?token=f31f2625f13d131f578fced0fc76b81d10f6c629e92c7099 +# http://eca4aa01751c:8888/lab?token=d4ac9278f5f5388e88097a3a8ebbe9401be206cfa0b83099 +# http://127.0.0.1:8888/lab?token=d4ac9278f5f5388e88097a3a8ebbe9401be206cfa0b83099 ``` -Pressing `Ctrl-C` twice shuts down the notebook server but leaves the container intact on disk for later restart or permanent deletion using commands like the following: +Pressing `Ctrl-C` twice shuts down the Server but leaves the container intact on disk for later restart or permanent deletion using commands like the following: ```bash # list containers docker ps --all -# CONTAINER ID IMAGE COMMAND CREATED STATUS PORTS NAMES -# 221331c047c4 jupyter/scipy-notebook:2023-06-01 "tini -g -- start-noโ€ฆ" 11 seconds ago Exited (0) 8 seconds ago cranky_benz +# CONTAINER ID IMAGE COMMAND CREATED STATUS PORTS NAMES +# eca4aa01751c quay.io/jupyter/scipy-notebook:2024-10-07 "tini -g -- start-noโ€ฆ" About a minute ago Exited (0) 5 seconds ago silly_panini # start the stopped container -docker start --attach 221331c047c4 +docker start --attach -i eca4aa01751c # Entered start.sh with args: jupyter lab # ... # remove the stopped container -docker rm 221331c047c4 -# 221331c047c4 +docker rm eca4aa01751c +# eca4aa01751c ``` -**Example 2:** +### Example 2 -This command pulls the `jupyter/r-notebook` image tagged `2023-06-01` from Docker Hub if it is not already present on the local host. -It then starts a container running a Jupyter Notebook server and exposes the server on host port 10000. -The server logs appear in the terminal and include a URL to the notebook server, but with the internal container port (8888) instead of the correct host port (10000). +This command pulls the `jupyter/r-notebook` image tagged `2024-10-07` from Quay.io if it is not already present on the local host. +It then starts a container running Server and exposes the server on host port 10000. +The server logs appear in the terminal and include a URL to the Server but with the internal container port (8888) instead of the correct host port (10000). ```bash -docker run -it --rm -p 10000:8888 -v "${PWD}":/home/jovyan/work jupyter/r-notebook:2023-06-01 +docker run -it --rm -p 10000:8888 -v "${PWD}":/home/jovyan/work quay.io/jupyter/r-notebook:2024-10-07 ``` -Pressing `Ctrl-C` twice shuts down the notebook server and immediately destroys the Docker container. +Pressing `Ctrl-C` twice shuts down the Server and immediately destroys the Docker container. New files and changes in `~/work` in the container will be preserved. Any other changes made in the container will be lost. -**Example 3:** +```{note} +By default, [jupyter's root_dir](https://jupyter-server.readthedocs.io/en/latest/other/full-config.html) is `/home/jovyan`. +So, new notebooks will be saved there, unless you change the directory in the file browser. + +To change the default directory, you will need to specify `ServerApp.root_dir` by adding this line to the previous command: `start-notebook.py --ServerApp.root_dir=/home/jovyan/work`. +``` + +### Example 3 -This command pulls the `jupyter/all-spark-notebook` image currently tagged `latest` from Docker Hub if an image tagged `latest` is not already present on the local host. +This command pulls the `jupyter/all-spark-notebook` image currently tagged `latest` from Quay.io if an image tagged `latest` is not already present on the local host. It then starts a container named `notebook` running a JupyterLab server and exposes the server on a randomly selected port. ```bash -docker run --detach -P --name notebook jupyter/all-spark-notebook +docker run --detach -P --name notebook quay.io/jupyter/all-spark-notebook ``` where: - `--detach`: will run the container in detached mode -You can also use the following docker commands to see the port and notebook server token: +You can also use the following docker commands to see the port and Jupyter Server token: ```bash # get the random host port assigned to the container port 8888 @@ -109,9 +116,10 @@ docker rm notebook ## Using the Podman CLI -An alternative to using the Docker CLI is to use the Podman CLI. Podman is mostly compatible with Docker. +An alternative to using the Docker CLI is to use the Podman CLI. +Podman is mostly compatible with Docker. -**Example 4:** +### Podman example If we use Podman instead of Docker in the situation given in _Example 2_, it will look like this: @@ -130,16 +138,16 @@ subuidSize=$(( $(podman info --format "{{ range .Host.IDMappings.UIDMap }}+{{.Si subgidSize=$(( $(podman info --format "{{ range .Host.IDMappings.GIDMap }}+{{.Size }}{{end }}" ) - 1 )) ``` -This command pulls the `docker.io/jupyter/r-notebook` image tagged `2023-06-01` from Docker Hub if it is not already present on the local host. -It then starts a container running a Jupyter Server and exposes the server on host port 10000. -The server logs appear in the terminal and include a URL to the notebook server, but with the internal container port (8888) instead of the correct host port (10000). +This command pulls the `quay.io/jupyter/r-notebook` image tagged `2024-10-07` from Quay.io if it is not already present on the local host. +It then starts a container running a Jupyter Server with the JupyterLab frontend and exposes the server on host port 10000. +The server logs appear in the terminal and include a URL to the server but with the internal container port (8888) instead of the correct host port (10000). ```bash podman run -it --rm -p 10000:8888 \ -v "${PWD}":/home/jovyan/work --user $uid:$gid \ --uidmap $uid:0:1 --uidmap 0:1:$uid --uidmap $(($uid+1)):$(($uid+1)):$(($subuidSize-$uid)) \ --gidmap $gid:0:1 --gidmap 0:1:$gid --gidmap $(($gid+1)):$(($gid+1)):$(($subgidSize-$gid)) \ - docker.io/jupyter/r-notebook:2023-06-01 + quay.io/jupyter/r-notebook:2024-10-07 ``` ```{warning} @@ -156,17 +164,17 @@ The `podman run` option `--userns=auto` will, for instance, not be possible to u The example could be improved by investigating more in detail which UIDs and GIDs need to be available in the container and then only map them. ``` -Pressing `Ctrl-C` twice shuts down the notebook server and immediately destroys the Docker container. +Pressing `Ctrl-C` twice shuts down the Server and immediately destroys the Docker container. New files and changes in `~/work` in the container will be preserved. Any other changes made in the container will be lost. ## Using Binder -[Binder](https://mybinder.org/) is a service that allows you to create and share custom computing environments for projects in version control. +A [Binder](https://mybinder.org/) is a service that allows you to create and share custom computing environments for projects in version control. You can use any of the Jupyter Docker Stacks images as a basis for a Binder-compatible Dockerfile. See the [docker-stacks example](https://mybinder.readthedocs.io/en/latest/examples/sample_repos.html#using-a-docker-image-from-the-jupyter-docker-stacks-repository) and -[Using a Dockerfile](https://mybinder.readthedocs.io/en/latest/tutorials/dockerfile.html) sections in the +[Using a Dockerfile](https://mybinder.readthedocs.io/en/latest/tutorials/dockerfile.html) section in the [Binder documentation](https://mybinder.readthedocs.io/en/latest/index.html) for instructions. ## Using JupyterHub @@ -180,5 +188,5 @@ instructions for the [dockerspawner](https://github.com/jupyterhub/dockerspawner ## Using Other Tools and Services You can use the Jupyter Docker Stacks with any Docker-compatible technology -(e.g., [Docker Compose](https://docs.docker.com/compose/), [docker-py](https://github.com/docker/docker-py), your favorite cloud container service). +(e.g., [Docker Compose](https://docs.docker.com/compose/), [docker-py](https://github.com/docker/docker-py), or your favorite cloud container service). See the documentation of the tool, library, or service for details about how to reference, configure, and launch containers from these images. diff --git a/docs/using/selecting.md b/docs/using/selecting.md index f93aad7072..3ea4e85e6f 100644 --- a/docs/using/selecting.md +++ b/docs/using/selecting.md @@ -18,14 +18,14 @@ The following sections describe these images, including their contents, relation ### jupyter/docker-stacks-foundation -[Source on GitHub](https://github.com/jupyter/docker-stacks/tree/main/docker-stacks-foundation) | -[Dockerfile commit history](https://github.com/jupyter/docker-stacks/commits/main/docker-stacks-foundation/Dockerfile) | -[Docker Hub image tags](https://hub.docker.com/r/jupyter/docker-stacks-foundation/tags/) +[Source on GitHub](https://github.com/jupyter/docker-stacks/tree/main/images/docker-stacks-foundation) | +[Dockerfile commit history](https://github.com/jupyter/docker-stacks/commits/main/images/docker-stacks-foundation/Dockerfile) | +[Quay.io image tags](https://quay.io/repository/jupyter/docker-stacks-foundation?tab=tags) `jupyter/docker-stacks-foundation` is a small image supporting a majority of [options common across all core stacks](common.md). It is the basis for all other stacks on which Jupyter-related applications can be built (e.g., kernel-based containers, [nbclient](https://github.com/jupyter/nbclient) applications, etc.). -As such, it does not contain application-level software like Jupyter Notebook server, Jupyter Lab or Jupyter Hub. +As such, it does not contain application-level software like JupyterLab, Jupyter Notebook, or JupyterHub. It contains: @@ -34,34 +34,43 @@ It contains: - [mamba](https://github.com/mamba-org/mamba): "reimplementation of the conda package manager in C++". We use this package manager by default when installing packages. - Unprivileged user `jovyan` (`uid=1000`, configurable, [see options in the common features section](./common.md) of this documentation) in group `users` (`gid=100`) with ownership over the `/home/jovyan` and `/opt/conda` paths -- `tini` as the container entry point -- A `start.sh` script as the default command - useful for running alternative commands in the container as applications are added (e.g. `ipython`, `jupyter kernelgateway`, `jupyter lab`) +- `tini` and a `start.sh` script as the container entry point - useful for running alternative commands in the container as applications are added (e.g. `ipython`, `jupyter kernelgateway`, `jupyter lab`) +- A `run-hooks.sh` script, which can source/run files in a given directory - Options for a passwordless sudo +- Common system libraries like `bzip2`, `ca-certificates`, `locales` +- `wget` to download external files - No preinstalled scientific computing packages ### jupyter/base-notebook -[Source on GitHub](https://github.com/jupyter/docker-stacks/tree/main/base-notebook) | -[Dockerfile commit history](https://github.com/jupyter/docker-stacks/commits/main/base-notebook/Dockerfile) | -[Docker Hub image tags](https://hub.docker.com/r/jupyter/base-notebook/tags/) +[Source on GitHub](https://github.com/jupyter/docker-stacks/tree/main/images/base-notebook) | +[Dockerfile commit history](https://github.com/jupyter/docker-stacks/commits/main/images/base-notebook/Dockerfile) | +[Quay.io image tags](https://quay.io/repository/jupyter/base-notebook?tab=tags) -`jupyter/base-notebook` adds base Jupyter server applications like Notebook, Jupyter Lab and Jupyter Hub +`jupyter/base-notebook` adds base Jupyter Applications like JupyterLab, Jupyter Notebook, JupyterHub, and NBClassic and serves as the basis for all other stacks besides `jupyter/docker-stacks-foundation`. It contains: - Everything in `jupyter/docker-stacks-foundation` -- Minimally functional Jupyter Notebook server (e.g., no LaTeX support for saving notebooks as PDFs) -- `notebook`, `jupyterhub` and `jupyterlab` packages -- A `start-notebook.sh` script as the default command -- A `start-singleuser.sh` script useful for launching containers in JupyterHub +- Minimally functional Server (e.g., no LaTeX support for saving notebooks as PDFs) +- `notebook`, `jupyterhub`, and `jupyterlab` packages +- A `start-notebook.py` script as the default command +- A `start-singleuser.py` script useful for launching containers in JupyterHub - Options for a self-signed HTTPS certificate +```{warning} +`jupyter/base-notebook` also contains `start-notebook.sh` and `start-singleuser.sh` files to maintain backward compatibility. +External config that explicitly refers to those files should instead +update to refer to `start-notebook.py` and `start-singleuser.py`. +The shim `.sh` files will be removed at some future date. +``` + ### jupyter/minimal-notebook -[Source on GitHub](https://github.com/jupyter/docker-stacks/tree/main/minimal-notebook) | -[Dockerfile commit history](https://github.com/jupyter/docker-stacks/commits/main/minimal-notebook/Dockerfile) | -[Docker Hub image tags](https://hub.docker.com/r/jupyter/minimal-notebook/tags/) +[Source on GitHub](https://github.com/jupyter/docker-stacks/tree/main/images/minimal-notebook) | +[Dockerfile commit history](https://github.com/jupyter/docker-stacks/commits/main/images/minimal-notebook/Dockerfile) | +[Quay.io image tags](https://quay.io/repository/jupyter/minimal-notebook?tab=tags) `jupyter/minimal-notebook` adds command-line tools useful when working in Jupyter applications. @@ -69,18 +78,19 @@ It contains: - Everything in `jupyter/base-notebook` - Common useful utilities like + [curl](https://curl.se), [git](https://git-scm.com/), [nano](https://www.nano-editor.org/) (actually `nano-tiny`), [tzdata](https://www.iana.org/time-zones), - [unzip](https://code.launchpad.net/ubuntu/+source/unzip) + [unzip](https://code.launchpad.net/ubuntu/+source/unzip), and [vi](https://www.vim.org) (actually `vim-tiny`), - [TeX Live](https://www.tug.org/texlive/) for notebook document conversion ### jupyter/r-notebook -[Source on GitHub](https://github.com/jupyter/docker-stacks/tree/main/r-notebook) | -[Dockerfile commit history](https://github.com/jupyter/docker-stacks/commits/main/r-notebook/Dockerfile) | -[Docker Hub image tags](https://hub.docker.com/r/jupyter/r-notebook/tags/) +[Source on GitHub](https://github.com/jupyter/docker-stacks/tree/main/images/r-notebook) | +[Dockerfile commit history](https://github.com/jupyter/docker-stacks/commits/main/images/r-notebook/Dockerfile) | +[Quay.io image tags](https://quay.io/repository/jupyter/r-notebook?tab=tags) `jupyter/r-notebook` includes popular packages from the R ecosystem listed below: @@ -109,21 +119,23 @@ It contains: ### jupyter/julia-notebook -[Source on GitHub](https://github.com/jupyter/docker-stacks/tree/main/julia-notebook) | -[Dockerfile commit history](https://github.com/jupyter/docker-stacks/commits/main/julia-notebook/Dockerfile) | -[Docker Hub image tags](https://hub.docker.com/r/jupyter/julia-notebook/tags/) +[Source on GitHub](https://github.com/jupyter/docker-stacks/tree/main/images/julia-notebook) | +[Dockerfile commit history](https://github.com/jupyter/docker-stacks/commits/main/images/julia-notebook/Dockerfile) | +[Quay.io image tags](https://quay.io/repository/jupyter/julia-notebook?tab=tags) `jupyter/julia-notebook` includes popular packages from the Julia ecosystem listed below: - Everything in `jupyter/minimal-notebook` and its ancestor images -- The [Julia Programming Language](https://julialang.org/) +- The [Julia](https://julialang.org/) compiler and base environment - [IJulia](https://github.com/JuliaLang/IJulia.jl) to support Julia code in Jupyter notebook +- [Pluto.jl](https://plutojl.org/) reactive Julia notebook interface, made accessible with [jupyter-pluto-proxy](https://github.com/yuvipanda/jupyter-pluto-proxy) +- [HDF5](https://github.com/JuliaIO/HDF5.jl) package ### jupyter/scipy-notebook -[Source on GitHub](https://github.com/jupyter/docker-stacks/tree/main/scipy-notebook) | -[Dockerfile commit history](https://github.com/jupyter/docker-stacks/commits/main/scipy-notebook/Dockerfile) | -[Docker Hub image tags](https://hub.docker.com/r/jupyter/scipy-notebook/tags/) +[Source on GitHub](https://github.com/jupyter/docker-stacks/tree/main/images/scipy-notebook) | +[Dockerfile commit history](https://github.com/jupyter/docker-stacks/commits/main/images/scipy-notebook/Dockerfile) | +[Quay.io image tags](https://quay.io/repository/jupyter/scipy-notebook?tab=tags) `jupyter/scipy-notebook` includes popular packages from the scientific Python ecosystem. @@ -165,87 +177,108 @@ It contains: ### jupyter/tensorflow-notebook -[Source on GitHub](https://github.com/jupyter/docker-stacks/tree/main/tensorflow-notebook) | -[Dockerfile commit history](https://github.com/jupyter/docker-stacks/commits/main/tensorflow-notebook/Dockerfile) | -[Docker Hub image tags](https://hub.docker.com/r/jupyter/tensorflow-notebook/tags/) +[Source on GitHub](https://github.com/jupyter/docker-stacks/tree/main/images/tensorflow-notebook) | +[Dockerfile commit history](https://github.com/jupyter/docker-stacks/commits/main/images/tensorflow-notebook/Dockerfile) | +[Quay.io image tags](https://quay.io/repository/jupyter/tensorflow-notebook?tab=tags) `jupyter/tensorflow-notebook` includes popular Python deep learning libraries. - Everything in `jupyter/scipy-notebook` and its ancestor images -- [tensorflow](https://www.tensorflow.org/) machine learning library +- [TensorFlow](https://www.tensorflow.org/) machine learning library +- [Jupyter Server Proxy](https://jupyter-server-proxy.readthedocs.io/en/latest/) to support [TensorBoard](https://www.tensorflow.org/tensorboard) + +### jupyter/pytorch-notebook + +[Source on GitHub](https://github.com/jupyter/docker-stacks/tree/main/images/pytorch-notebook) | +[Dockerfile commit history](https://github.com/jupyter/docker-stacks/commits/main/images/pytorch-notebook/Dockerfile) | +[Quay.io image tags](https://quay.io/repository/jupyter/pytorch-notebook?tab=tags) + +`jupyter/pytorch-notebook` includes popular Python deep learning libraries. + +- Everything in `jupyter/scipy-notebook` and its ancestor images +- [pytorch](https://pytorch.org/) machine learning library ### jupyter/datascience-notebook -[Source on GitHub](https://github.com/jupyter/docker-stacks/tree/main/datascience-notebook) | -[Dockerfile commit history](https://github.com/jupyter/docker-stacks/commits/main/datascience-notebook/Dockerfile) | -[Docker Hub image tags](https://hub.docker.com/r/jupyter/datascience-notebook/tags/) +[Source on GitHub](https://github.com/jupyter/docker-stacks/tree/main/images/datascience-notebook) | +[Dockerfile commit history](https://github.com/jupyter/docker-stacks/commits/main/images/datascience-notebook/Dockerfile) | +[Quay.io image tags](https://quay.io/repository/jupyter/datascience-notebook?tab=tags) -`jupyter/datascience-notebook` includes libraries for data analysis from the Julia, Python, and R -communities. +`jupyter/datascience-notebook` includes libraries for data analysis from the Python, R, and Julia communities. -- Everything in the `jupyter/scipy-notebook` and `jupyter/r-notebook` images and their ancestor +- Everything in the `jupyter/scipy-notebook`, `jupyter/r-notebook`, and `jupyter/julia-notebook` images and their ancestor images - [rpy2](https://rpy2.github.io/doc/latest/html/index.html) package -- The [Julia](https://julialang.org/) compiler and base environment -- [IJulia](https://github.com/JuliaLang/IJulia.jl) to support Julia code in Jupyter notebooks -- [HDF5](https://github.com/JuliaIO/HDF5.jl) package ### jupyter/pyspark-notebook -[Source on GitHub](https://github.com/jupyter/docker-stacks/tree/main/pyspark-notebook) | -[Dockerfile commit history](https://github.com/jupyter/docker-stacks/commits/main/pyspark-notebook/Dockerfile) | -[Docker Hub image tags](https://hub.docker.com/r/jupyter/pyspark-notebook/tags/) +[Source on GitHub](https://github.com/jupyter/docker-stacks/tree/main/images/pyspark-notebook) | +[Dockerfile commit history](https://github.com/jupyter/docker-stacks/commits/main/images/pyspark-notebook/Dockerfile) | +[Quay.io image tags](https://quay.io/repository/jupyter/pyspark-notebook?tab=tags) `jupyter/pyspark-notebook` includes Python support for Apache Spark. - Everything in `jupyter/scipy-notebook` and its ancestor images - [Apache Spark](https://spark.apache.org/) with Hadoop binaries -- [pyarrow](https://arrow.apache.org/docs/python/) library +- [grpcio-status](https://github.com/grpc/grpc/tree/master/src/python/grpcio_status) +- [grpcio](https://grpc.io/docs/languages/python/quickstart/) +- [pyarrow](https://arrow.apache.org/docs/python/) ### jupyter/all-spark-notebook -[Source on GitHub](https://github.com/jupyter/docker-stacks/tree/main/all-spark-notebook) | -[Dockerfile commit history](https://github.com/jupyter/docker-stacks/commits/main/all-spark-notebook/Dockerfile) | -[Docker Hub image tags](https://hub.docker.com/r/jupyter/all-spark-notebook/tags/) +[Source on GitHub](https://github.com/jupyter/docker-stacks/tree/main/images/all-spark-notebook) | +[Dockerfile commit history](https://github.com/jupyter/docker-stacks/commits/main/images/all-spark-notebook/Dockerfile) | +[Quay.io image tags](https://quay.io/repository/jupyter/all-spark-notebook?tab=tags) `jupyter/all-spark-notebook` includes Python and R support for Apache Spark. - Everything in `jupyter/pyspark-notebook` and its ancestor images - [IRKernel](https://irkernel.github.io/) to support R code in Jupyter notebooks - [rcurl](https://cran.r-project.org/web/packages/RCurl/index.html), - [sparklyr](https://spark.rstudio.com), + [sparklyr](https://spark.posit.co), [ggplot2](https://ggplot2.tidyverse.org) packages +### CUDA enabled variants + +We provide CUDA accelerated versions of the `pytorch-notebook` and `tensorflow-notebook` images. +Prepend a CUDA prefix (versioned prefix like `cuda12-` for `pytorch-notebook` or just `cuda-` for `tensorflow-notebook`) to the image tag +to allow PyTorch or TensorFlow operations to use compatible NVIDIA GPUs for accelerated computation. +We only build `pytorch-notebook` for last two major versions of CUDA. +The `tensorflow-notebook` image only supports the latest CUDA version listed in the [officially tested build configurations](https://www.tensorflow.org/install/source#gpu). + +For example, you could use the image `quay.io/jupyter/pytorch-notebook:cuda12-python-3.11.8` or `quay.io/jupyter/tensorflow-notebook:cuda-latest`. + ### Image Relationships The following diagram depicts the build dependency tree of the core images. (i.e., the `FROM` statements in their Dockerfiles). Any given image inherits the complete content of all ancestor images pointing to it. [![Image inheritance -diagram](../images/inherit.svg)](http://interactive.blockdiag.com/?compression=deflate&src=eJyFjkFuwkAMRfecwsqKLuYACMEJuqPLSshJHDAZ7GjGIwSIuzPTRaWJWmX7_vP_br12Y894gucKoKcBk7fjoGKRHwQ72Gwz18AkhsYqGU0aLCDbdpWjJrVJLH3L-vPrADe2c85ZDAJ5wkgfDbg99HmFgouG3RjdoEn6n7ZS_l9W7trc4ESNWtWxyBUoxpWFr-grac6KFzue7pVVk-I0RhI1DF5vv7z5W80vYqYkHS1Oh0XjkjzjwnPTPU4Yxsqas-Kh925uvt4imKoO) +diagram](../images/inherit.svg)](http://interactive.blockdiag.com/?compression=deflate&src=eJyFj0FqwzAQRfc5hfAqpYiS7kpoT9BdugyEsTxuplZmjDRqcEvvXinQggzGK8Gb97_4rRc3dATv5ntjTIc9JK-nXlgjfaF5Nk_7zCUQsoKScEajBA1Aut_kU5PaxJqOvH19O5gr6TnfidUE9AgR7xpjX0yXf8Fgo4Ibou0lcXdrK-VLt5Jrc4NlUWxFhiJXoBgXYrqAr6Q5K150NE6VVZPiNIocJfRerv_8yPcudWA-IRCwNgvJcVIJ7jyP7XYPt-fxLx8XCvJkyBTZ4eqUsGp8JE-wMnac4ghhqKw5Kx54b-fmzy_M3cYh) ### Builds -Every Monday and whenever a pull request is merged, images are rebuilt and pushed to [the public container registry](https://hub.docker.com/u/jupyter). +Every Monday and whenever a pull request is merged, images are rebuilt and pushed to [the public container registry](https://quay.io/organization/jupyter). ### Versioning via image tags Whenever a docker image is pushed to the container registry, it is tagged with: -- a `latest` tag +- the `latest` tag - a 12-character git commit SHA like `1ffe43816ba9` - a date formatted like `2023-01-30` - OS version like `ubuntu-22.04` - a set of software version tags like `python-3.10.8` and `lab-3.5.3` ```{warning} -- Tags before `2022-07-05` were sometimes incorrect. Please, do not rely on them. -- Single-platform images have either `aarch64` or `x86_64` tag prefixes, for example, `jupyter/base-notebook:aarch64-python-3.10.5` +- Tags before `2022-07-05` were sometimes incorrect. + Please, do not rely on them. +- Single-platform images have either `aarch64-` or `x86_64-` tag prefixes, for example, `quay.io/jupyter/base-notebook:aarch64-python-3.11.6` ``` For stability and reproducibility, you should either reference a date formatted -tag from a date before the current date (in UTC time) or a git commit SHA older +tag from a date before the current date (in UTC) or a git commit SHA older than the latest git commit SHA in the default branch of the [jupyter/docker-stacks GitHub repository](https://github.com/jupyter/docker-stacks/). @@ -295,15 +328,18 @@ See the [contributing guide](../contributing/stacks.md) for information about ho [almond]: https://almond.sh [almond_b]: https://mybinder.org/v2/gh/almond-sh/examples/master?urlpath=lab%2Ftree%2Fnotebooks%2Findex.ipynb -### GPU accelerated notebooks +### Other GPU-accelerated notebooks -| Flavor | Description | -| ------------------ | ----------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- | -| [GPU-Jupyter][gpu] | Power of your NVIDIA GPU and GPU calculations using Tensorflow and Pytorch in collaborative notebooks. This is done by generating a Dockerfile that consists of the **nvidia/cuda** base image, the well-maintained **docker-stacks** that is integrated as a submodule and GPU-able libraries like **Tensorflow**, **Keras** and **PyTorch** on top of it. | -| [PRP-GPU][prp_gpu] | PRP (Pacific Research Platform) maintained [registry][prp_reg] for jupyter stack based on NVIDIA CUDA-enabled image. Added the PRP image with Pytorch and some other python packages and GUI Desktop notebook based on <https://github.com/jupyterhub/jupyter-remote-desktop-proxy>. | -| [b-data][b-data] | GPU accelerated, multi-arch (`linux/amd64`, `linux/arm64/v8`) docker images for [R][r_cuda], [Python][python_cuda] and [Julia][julia_cuda]. Derived from nvidia/cuda `devel`-flavored images, including TensortRT and TensorRT plugin libraries. With [code-server][code-server] next to JupyterLab. Just Python โ€“ no [Conda][conda]/[Mamba][mamba]. | +| Flavor | Description | +| --------------------------------- | ------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------ | +| [GPU-Jupyter][gpu] | Power of your NVIDIA GPU and GPU calculations using Tensorflow and Pytorch in collaborative notebooks. This is done by generating a Dockerfile that consists of the **nvidia/cuda** base image, the well-maintained **docker-stacks** that is integrated as a submodule, and GPU-able libraries like **Tensorflow**, **Keras** and **PyTorch** on top of it. | +| [myLab TH Lรผbeck Images][gpu_thl] | Images based on the **jupyter/docker-stacks**, built and maintained at the [myLab TH Lรผbeck][gpu_mylab] using build scripts similar to iot-salzburg. Several images include GPU libraries. | +| [PRP-GPU][prp_gpu] | PRP (Pacific Research Platform) maintained [registry][prp_reg] for jupyter stack based on NVIDIA CUDA-enabled image. Added the PRP image with Pytorch and some other Python packages and GUI Desktop notebook based on <https://github.com/jupyterhub/jupyter-remote-desktop-proxy>. | +| [b-data][b-data] | GPU accelerated, multi-arch (`linux/amd64`, `linux/arm64/v8`) docker images for [R][r_cuda], [Python][python_cuda] and [Julia][julia_cuda]. Derived from nvidia/cuda `devel`-flavored images, including TensortRT and TensorRT plugin libraries. With [code-server][code-server] next to JupyterLab. Just Python โ€“ no [Conda][conda]/[Mamba][mamba]. | [gpu]: https://github.com/iot-salzburg/gpu-jupyter +[gpu_thl]: https://hub.docker.com/r/hanseware/jlab-images +[gpu_mylab]: https://mylab.th-luebeck.de [prp_gpu]: https://gitlab.nrp-nautilus.io/prp/jupyter-stack/-/tree/prp [prp_reg]: https://gitlab.nrp-nautilus.io/prp/jupyter-stack/container_registry [b-data]: https://github.com/b-data diff --git a/docs/using/specifics.md b/docs/using/specifics.md index 0eb8471731..4df770f1d2 100644 --- a/docs/using/specifics.md +++ b/docs/using/specifics.md @@ -14,7 +14,7 @@ This page provides details about features specific to one or more images. Every new spark context that is created is put onto an incrementing port (i.e. 4040, 4041, 4042, etc.), and it might be necessary to open multiple ports. ``` - For example: `docker run --detach -p 8888:8888 -p 4040:4040 -p 4041:4041 jupyter/pyspark-notebook`. + For example, `docker run --detach -p 8888:8888 -p 4040:4040 -p 4041:4041 quay.io/jupyter/pyspark-notebook`. #### IPython low-level output capture and forward @@ -39,35 +39,42 @@ ipython profile create ### Build an Image with a Different Version of Spark -You can build a `pyspark-notebook` image (and also the downstream `all-spark-notebook` image) with a different version of Spark by overriding the default value of the following arguments at build time. +You can build a `pyspark-notebook` image with a different `Spark` version by overriding the default value of the following arguments at build time. +`all-spark-notebook` is inherited from `pyspark-notebook`, so you have to first build `pyspark-notebook` and then `all-spark-notebook` to get the same version in `all-spark-notebook`. -- Spark distribution is defined by the combination of Spark, Hadoop and Scala versions and verified by the package checksum, +- Spark distribution is defined by the combination of Spark, Hadoop, and Scala versions, see [Download Apache Spark](https://spark.apache.org/downloads.html) and the [archive repo](https://archive.apache.org/dist/spark/) for more information. - - `spark_version`: The Spark version to install (`3.3.0`). - - `hadoop_version`: The Hadoop version (`3.2`). - - `scala_version`: The Scala version (`2.13`, optional). - - `spark_checksum`: The package checksum (`BFE4540...`). - - `openjdk_version`: The version of the OpenJDK (JRE headless) distribution (`17`). + - `openjdk_version`: The version of the OpenJDK (JRE headless) distribution (`17` by default). - This version needs to match the version supported by the Spark distribution used above. - See [Spark Overview](https://spark.apache.org/docs/latest/#downloading) and [Ubuntu packages](https://packages.ubuntu.com/search?keywords=openjdk). + - `spark_version` (optional): The Spark version to install, for example `3.5.0`. + If not specified (this is the default), latest stable Spark will be installed. + - `hadoop_version`: The Hadoop version (`3` by default). + Note, that _Spark < 3.3_ require to specify `major.minor` Hadoop version (i.e. `3.2`). + - `scala_version` (optional): The Scala version, for example `2.13` (not specified by default). + Starting with _Spark >= 3.2_, the distribution file might contain the Scala version. + - `spark_download_url`: URL to use for Spark downloads. + You may need to use <https://archive.apache.org/dist/spark/> url if you want to download old Spark versions. -- Starting with _Spark >= 3.2_, the distribution file might contain Scala version. +For example, here is how to build a `pyspark-notebook` image with Spark `3.2.0`, Hadoop `3.2`, and OpenJDK `11`. -For example, here is how to build a `pyspark-notebook` image with Spark `3.2.0`, Hadoop `3.2` and OpenJDK `11`. +```{warning} +This recipe is not tested and might be broken. +``` ```bash # From the root of the project # Build the image with different arguments docker build --rm --force-rm \ - -t jupyter/pyspark-notebook:spark-3.2.0 ./pyspark-notebook \ + -t my-pyspark-notebook ./images/pyspark-notebook \ + --build-arg openjdk_version=11 \ --build-arg spark_version=3.2.0 \ --build-arg hadoop_version=3.2 \ - --build-arg spark_checksum=707DDE035926A50B75E53FCA72CADA519F3239B14A96546911CB4916A58DCF69A1D2BFDD2C7DD5899324DBD82B6EEAB9797A7B4ABF86736FFCA4C26D0E0BF0EE \ - --build-arg openjdk_version=11 + --build-arg spark_download_url="https://archive.apache.org/dist/spark/" # Check the newly built image -docker run -it --rm jupyter/pyspark-notebook:spark-3.2.0 pyspark --version +docker run -it --rm my-pyspark-notebook pyspark --version # Welcome to # ____ __ @@ -76,7 +83,12 @@ docker run -it --rm jupyter/pyspark-notebook:spark-3.2.0 pyspark --version # /___/ .__/\_,_/_/ /_/\_\ version 3.2.0 # /_/ -# Using Scala version 2.13.5, OpenJDK 64-Bit Server VM, 11.0.15 +# Using Scala version 2.12.15, OpenJDK 64-Bit Server VM, 11.0.21 +# Branch HEAD +# Compiled by user ubuntu on 2021-10-06T12:46:30Z +# Revision 5d45a415f3a29898d92380380cfd82bfc7f579ea +# Url https://github.com/apache/spark +# Type --help for more information. ``` ### Usage Examples @@ -156,7 +168,7 @@ Connection to Spark Cluster on **[Standalone Mode](https://spark.apache.org/docs 0. Verify that the docker image (check the Dockerfile) and the Spark Cluster, which is being deployed, run the same version of Spark. 1. [Deploy Spark in Standalone Mode](https://spark.apache.org/docs/latest/spark-standalone.html). -2. Run the Docker container with `--net=host` in a location that is network addressable by all of +2. Run the Docker container with `--net=host` in a location that is network-addressable by all of your Spark workers. (This is a [Spark networking requirement](https://spark.apache.org/docs/latest/cluster-overview.html#components).) @@ -169,7 +181,7 @@ Connection to Spark Cluster on **[Standalone Mode](https://spark.apache.org/docs ##### Standalone Mode in Python The **same Python version** needs to be used on the notebook (where the driver is located) and on the Spark workers. -The python version used at the driver and worker side can be adjusted by setting the environment variables `PYSPARK_PYTHON` and/or `PYSPARK_DRIVER_PYTHON`, +The Python version used on the driver and worker side can be adjusted by setting the environment variables `PYSPARK_PYTHON` and/or `PYSPARK_DRIVER_PYTHON`, see [Spark Configuration][spark-conf] for more information. ```python @@ -291,5 +303,5 @@ sess.run(hello) ``` [sparkr]: https://spark.apache.org/docs/latest/sparkr.html -[sparklyr]: https://spark.rstudio.com/ +[sparklyr]: https://spark.posit.co [spark-conf]: https://spark.apache.org/docs/latest/configuration.html diff --git a/docs/using/troubleshooting.md b/docs/using/troubleshooting.md index bcab93f266..f06577e522 100644 --- a/docs/using/troubleshooting.md +++ b/docs/using/troubleshooting.md @@ -1,10 +1,10 @@ # Troubleshooting Common Problems When troubleshooting, you may see unexpected behaviors or receive an error message. -This section provides advice on -how to identify and fix some of the most commonly encountered issues. +This section provides advice on how to identify and fix some of the most commonly encountered issues. -Most of the `docker run` flags used in this document are explained in detail in the [Common Features, Docker Options section](common.md#docker-options) of the documentation. +Most of the `docker run` flags used in this document are explained in detail in the +[Common Features, Docker Options section](common.md#docker-options) of the documentation. ## Permission denied when mounting volumes @@ -14,7 +14,7 @@ If you are running a Docker container while mounting a local volume or host dire docker run -it --rm \ -p 8888:8888 \ -v <my-vol>:<container-dir> \ - jupyter/minimal-notebook:latest + quay.io/jupyter/minimal-notebook:latest ``` you might face permissions issues when trying to access the mounted volume: @@ -29,16 +29,18 @@ touch stagingarea/kale.txt # touch: cannot touch 'stagingarea/kale.txt': Permission denied ``` -In this case, the user of the container (`jovyan`) and the owner of the mounted volume (`root`) have different permission levels and ownership over the container's directories and mounts. +In this case, the user of the container (`jovyan`) and the owner of the mounted volume (`root`) +have different permission levels and ownership over the container's directories and mounts. The following sections cover a few of these scenarios and how to fix them. **Some things to try:** 1. **Change ownership of the volume mount** - You can change the ownership of the volume mount using the `chown` command. In the case of the docker-stacks images, you can set the `CHOWN_EXTRA` and `CHOWN_EXTRA_OPTS` environment variables. + You can change the ownership of the volume mount using the `chown` command. + In the case of the docker-stacks images, you can set the `CHOWN_EXTRA` and `CHOWN_EXTRA_OPTS` environment variables. - For example, to change the ownership of the volume mount to the jovyan user (non-privileged default user in the Docker images): + For example, to change the ownership of the volume mount to the `jovyan` user (non-privileged default user in the Docker images): ```bash # running in detached mode - can also be run in interactive mode @@ -48,12 +50,13 @@ The following sections cover a few of these scenarios and how to fix them. --user root \ -e CHOWN_EXTRA="<container-dir>" \ -e CHOWN_EXTRA_OPTS="-R" \ - jupyter/minimal-notebook + quay.io/jupyter/minimal-notebook ``` where: - - `CHOWN_EXTRA=<some-dir>,<some-other-dir>`: will change the ownership and group of the specified container directory (non-recursive by default). You need to provide full paths starting with `/`. + - `CHOWN_EXTRA=<some-dir>,<some-other-dir>`: will change the ownership and group of the specified container directory (non-recursive by default). + You need to provide full paths starting with `/`. - `CHOWN_EXTRA_OPTS="-R"`: will recursively change the ownership and group of the directory specified in `CHOWN_EXTRA`. - `--user root`: you **must** run the container with the root user to change ownership at runtime. @@ -74,19 +77,20 @@ The following sections cover a few of these scenarios and how to fix them. - If you are mounting your volume inside the `/home/` directory, you can use the `-e CHOWN_HOME=yes` and `CHOWN_HOME_OPTS="-R"` flags instead of the `-e CHOWN_EXTRA` and `-e CHOWN_EXTRA_OPTS` in the example above. - This solution should work in most cases where you have created a docker volume - (i.e. using the [`docker volume create --name <my-volume>`command](https://docs.docker.com/storage/volumes/#create-and-manage-volumes)) and mounted it using the`-v` flag in `docker run`. + (i.e. using the [`docker volume create --name <my-volume>`command](https://docs.docker.com/engine/storage/volumes/#create-and-manage-volumes)) and mounted it using the `-v` flag in `docker run`. ``` 2. **Matching the container's UID/GID with the host's** Docker handles mounting host directories differently from mounting volumes, even though the syntax is essentially the same (i.e. `-v`). - When you initialize a Docker container using the flag `-v`, the host directories are bind-mounted directly into the container. + When you initialize a Docker container using the `-v`flag, the host directories are bind-mounted directly into the container. Therefore, the permissions and ownership are copied over and will be **the same** as the ones in your local host (including user ids) which may result in permissions errors when trying to access directories or create/modify files inside. - Suppose your local user has a `UID` and `GID` of `1234` and `5678`, respectively. To fix the UID discrepancies between your local directories and the container's - directories, you can run the container with an explicit `NB_UID` and `NB_GID` to match that of the local user: + Suppose your local user has a `UID` and `GID` of `1234` and `5678`, respectively. + To fix the UID discrepancies between your local directories and the container's directories, + you can run the container with an explicit `NB_UID` and `NB_GID` to match that of the local user: ```bash docker run -it --rm \ @@ -95,7 +99,7 @@ The following sections cover a few of these scenarios and how to fix them. -e NB_UID=1234 \ -e NB_GID=5678 \ -v "${PWD}"/test:/home/jovyan/work \ - jupyter/minimal-notebook:latest + quay.io/jupyter/minimal-notebook:latest # you should see an output similar to this # Update jovyan's UID:GID to 1234:5678 @@ -108,10 +112,12 @@ The following sections cover a few of these scenarios and how to fix them. - You **must** use `--user root` to ensure that the `UID` and `GID` are updated at runtime. ````{admonition} Additional notes -- The caveat with this approach is that since these changes are applied at runtime, you will need to re-run the same command - with the appropriate flags and environment variables if you need to recreate the container (i.e. after removing/destroying it). +- The caveat with this approach is that since these changes are applied at runtime, + you will need to re-run the same command with the appropriate flags and environment variables + if you need to recreate the container (i.e. after removing/destroying it). - If you pass a numeric UID, it **must** be in the range of 0-2147483647 - - This approach only updates the UID and GID of the **existing `jovyan` user** instead of creating a new user. From the above example: + - This approach only updates the UID and GID of the **existing `jovyan` user** instead of creating a new user. + From the above example: ```bash id # uid=1234(jovyan) gid=5678(jovyan) groups=5678(jovyan),100(users) @@ -122,7 +128,7 @@ The following sections cover a few of these scenarios and how to fix them. If you have also **created a new user**, you might be experiencing any of the following issues: -- `root` is the owner of `/home` or a mounted volume +- the `root` user is the owner of `/home` or a mounted volume - when starting the container, you get an error such as `Failed to change ownership of the home directory.` - getting permission denied when trying to `conda install` packages @@ -142,9 +148,9 @@ If you have also **created a new user**, you might be experiencing any of the fo -e NB_GID=1234 \ -e CHOWN_HOME=yes \ -e CHOWN_HOME_OPTS="-R" \ - -w "/home/${NB_USER}" \ + -w "/home/callisto" \ -v "${PWD}"/test:/home/callisto/work \ - jupyter/minimal-notebook + quay.io/jupyter/minimal-notebook # Updated the jovyan user: # - username: jovyan -> callisto @@ -162,19 +168,21 @@ If you have also **created a new user**, you might be experiencing any of the fo - `-e NB_UID=1234` and `-e NB_GID=1234`: will set the `UID` and `GID` of the new user (`callisto`) to `1234` - `-e CHOWN_HOME_OPTS="-R"` and `-e CHOWN_HOME=yes`: ensure that the new user is the owner of the `/home` directory and subdirectories (setting `CHOWN_HOME_OPTS="-R` will ensure this change is applied recursively) - - `-w "/home/${NB_USER}"` sets the working directory to be the new user's home + - `-w "/home/callisto"` sets the working directory to be the new user's home ```{admonition} Additional notes In the example above, the `-v` flag is used to mount the local volume onto the new user's `/home` directory. - However, if you are mounting a volume elsewhere, you also need to use the `-e CHOWN_EXTRA=<some-dir>` flag to avoid any permission - issues (see the section [Permission denied when mounting volumes](#permission-denied-when-mounting-volumes) on this page). + However, if you are mounting a volume elsewhere, + you also need to use the `-e CHOWN_EXTRA=<some-dir>` flag to avoid any permission issues + (see the section [Permission denied when mounting volumes](#permission-denied-when-mounting-volumes) on this page). ``` 2. **Dynamically assign the user ID and GID** The above case ensures that the `/home` directory is owned by a newly created user with a specific `UID` and `GID`, - but if you want to assign the `UID` and `GID` of the new user dynamically, you can make the following adjustments: + but if you want to assign the `UID` and `GID` of the new user dynamically, + you can make the following adjustments: ```bash docker run -it --rm \ @@ -185,14 +193,14 @@ If you have also **created a new user**, you might be experiencing any of the fo -e NB_GID="$(id -g)" \ -e CHOWN_HOME=yes \ -e CHOWN_HOME_OPTS="-R" \ - -w "/home/${NB_USER}" \ + -w "/home/callisto" \ -v "${PWD}"/test:/home/callisto/work \ - jupyter/minimal-notebook + quay.io/jupyter/minimal-notebook ``` where: - - `"$(id -u)" and "$(id -g)"` will dynamically assign the `UID` and `GID` of the user executing the `docker run` command to the new user (`callisto`) + - `"$(id -u)"` and `"$(id -g)"` will dynamically assign the `UID` and `GID` of the user executing the `docker run` command to the new user (`callisto`) ## Additional tips and troubleshooting commands for permission-related errors @@ -202,8 +210,8 @@ If you have also **created a new user**, you might be experiencing any of the fo -v "${PWD}"/<my-vol>:/home/jovyan/work ``` - This example uses the syntax `$(PWD)`, which is replaced with the full path to the current directory at runtime. The destination - path should also be an absolute path starting with a `/` such as `/home/jovyan/work`. + This example uses the syntax `${PWD}`, which is replaced with the full path to the current directory at runtime. + The destination path should also be an absolute path starting with a `/` such as `/home/jovyan/work`. - You might want to consider using the Docker native `--user <UID>` and `--group-add users` flags instead of `-e NB_UID` and `-e NB_GID`: @@ -214,13 +222,15 @@ If you have also **created a new user**, you might be experiencing any of the fo docker run -it --rm \ -p 8888:8888 \ --user "$(id -u)" --group-add users \ - -v <my-vol>:/home/jovyan/work jupyter/datascience-notebook + -v <my-vol>:/home/jovyan/work quay.io/jupyter/datascience-notebook ``` - This command will launch the container with a specific user UID and add that user to the `users` group to modify the files in the default `/home` and `/opt/conda` directories. + This command will launch the container with a specific user UID and add that user to the `users` group + to modify the files in the default `/home` and `/opt/conda` directories. Further avoiding issues when trying to `conda install` additional packages. -- Use `docker inspect <container_id>` and look for the [`Mounts` section](https://docs.docker.com/storage/volumes/#start-a-container-with-a-volume) to verify that the volume was created and mounted accordingly: +- Use `docker inspect <container_id>` and look for the [`Mounts` section](https://docs.docker.com/engine/storage/volumes/#start-a-container-with-a-volume) + to verify that the volume was created and mounted accordingly: ```json { @@ -241,7 +251,8 @@ If you have also **created a new user**, you might be experiencing any of the fo ## Problems installing conda packages from specific channels -By default, the docker-stacks images have the conda channels priority set to `strict`. This may cause problems when trying to install packages from a channel with lower priority. +By default, the docker-stacks images have the conda channels priority set to `strict`. +This may cause problems when trying to install packages from a channel with lower priority. ```bash conda config --show | grep priority @@ -259,18 +270,19 @@ conda config --show default_channels **Installing packages from alternative channels:** -You can install packages from other conda channels (e.g. bioconda) by disabling the `channel_priority` setting: +You can install packages from other conda channels (e.g. `bioconda`) by disabling the `channel_priority` setting: ```bash # install by disabling channel priority at ะตั€ัƒ command level conda install --no-channel-priority -c bioconda bioconductor-geoquery ``` -Additional details are provided in the [Using alternative channels](../using/common.md#using-alternative-channels) section of the [Common features](common.md) page. +Additional details are provided in the [Using Alternative Channels](../using/common.md#using-alternative-channels) section of the [Common Features](common.md) page. ## Tokens are being rejected -If you are a regular user of VSCode and the Jupyter extension, you might experience either of these issues when using any of the docker-stacks images: +If you are a regular user of VSCode and the Jupyter extension, +you might experience either of these issues when using any of the docker-stacks images: - when clicking on the URL displayed on your command line logs, you face a "This site cannot be reached" page on your web browser - using the produced token and/or URL results in an "Invalid credentials" error on the Jupyter "Token authentication is enabled" page @@ -306,11 +318,11 @@ If you are a regular user of VSCode and the Jupyter extension, you might experie 2. **Turn off Jupyter auto-start in VSCode** - Alternatively - you might want to ensure that the `Jupyter: Auto Start` setting is turned off to avoid this issue in the future. + Alternatively - you might want to ensure that the `Jupyter: Disable Jupyter Auto Start` setting is turned on to avoid this issue in the future. - You can achieve this from the `Preferences > Jupyter` menu in VScode: + You can achieve this from the `Settings > Jupyter` menu in VScode: - ![VSCode Preferences UI - Jupyter: Disable Jupyter Auto Start checkbox unchecked](../_static/using/troubleshooting/vscode-jupyter-settings.png) + ![VSCode Settings UI - Jupyter: Disable Jupyter Auto Start checkbox checked](../_static/using/troubleshooting/vscode-jupyter-settings.png) 3. **Route container to unused local port** @@ -318,12 +330,13 @@ If you are a regular user of VSCode and the Jupyter extension, you might experie You can see an example of mapping to local port `8001`: ```bash - docker run -it --rm -p 8001:8888 jupyter/datascience-notebook + docker run -it --rm -p 8001:8888 quay.io/jupyter/datascience-notebook ``` When the terminal provides the link to access Jupyter: <http://127.0.0.1:8888/lab?token=80d45d241a1ba4c2...>, - change the default port value of `8888` in the url to the port value mapped with the `docker run` command. + change the default port value of `8888` in the URL to the port value mapped with the `docker run` command. In this example, we use 8001, so the edited link would be: <http://127.0.0.1:8001/lab?token=80d45d241a1ba4c2...>. - Note: Port mapping for Jupyter has other applications outside of Docker. For example, it can be used to allow multiple Jupyter instances when using SSH to control cloud devices. + Note: Port mapping for Jupyter has other applications outside of Docker. + For example, it can be used to allow multiple Jupyter instances when using SSH to control cloud devices. diff --git a/examples/README.md b/examples/README.md index e7daff64f9..7bb53eb732 100644 --- a/examples/README.md +++ b/examples/README.md @@ -1,4 +1,4 @@ # Examples -These examples and not tested and might not work. +These examples are not tested and might not work. Please, send PRs if you start using these examples and see some issues. diff --git a/examples/docker-compose/README.md b/examples/docker-compose/README.md index b2ea9efbf7..40300caca7 100644 --- a/examples/docker-compose/README.md +++ b/examples/docker-compose/README.md @@ -12,7 +12,7 @@ See the [installation instructions](https://docs.docker.com/engine/installation/ ## Quickstart -Build and run a `jupyter/minimal-notebook` container on a VirtualBox VM on local desktop. +Build and run a `jupyter/minimal-notebook` image on a VirtualBox VM on local desktop. ```bash # create a Docker Machine-controlled VirtualBox VM @@ -42,7 +42,7 @@ You can customize the docker-stack notebook image to deploy by modifying the `no For example, you can build and deploy a `jupyter/all-spark-notebook` by modifying the Dockerfile like so: ```dockerfile -FROM jupyter/all-spark-notebook:2023-06-01 +FROM quay.io/jupyter/all-spark-notebook # Your RUN commands and so on ``` @@ -86,8 +86,8 @@ NAME=your-notebook PORT=9001 WORK_VOLUME=our-work notebook/up.sh ### How do I run over HTTPS? -To run the notebook server with a self-signed certificate, pass the `--secure` option to the `up.sh` script. -You must also provide a password, which will be used to secure the notebook server. +To run the Jupyter Server with a self-signed certificate, pass the `--secure` option to the `up.sh` script. +You must also provide a password, which will be used to secure the Jupyter Server. You can specify the password by setting the `PASSWORD` environment variable, or by passing it to the `up.sh` script. ```bash diff --git a/examples/docker-compose/notebook/Dockerfile b/examples/docker-compose/notebook/Dockerfile index 6e2668ccf8..056c22fb29 100644 --- a/examples/docker-compose/notebook/Dockerfile +++ b/examples/docker-compose/notebook/Dockerfile @@ -2,7 +2,7 @@ # Distributed under the terms of the Modified BSD License. # Pick your favorite docker-stacks image -FROM jupyter/minimal-notebook:2023-06-01 +FROM quay.io/jupyter/minimal-notebook USER root diff --git a/examples/docker-compose/notebook/letsencrypt-notebook.yml b/examples/docker-compose/notebook/letsencrypt-notebook.yml index 1b7449f763..06bab31966 100644 --- a/examples/docker-compose/notebook/letsencrypt-notebook.yml +++ b/examples/docker-compose/notebook/letsencrypt-notebook.yml @@ -18,9 +18,9 @@ services: USE_HTTPS: "yes" PASSWORD: ${PASSWORD} command: > - start-notebook.sh - --NotebookApp.certfile=/etc/letsencrypt/fullchain.pem - --NotebookApp.keyfile=/etc/letsencrypt/privkey.pem + start-notebook.py + --ServerApp.certfile=/etc/letsencrypt/fullchain.pem + --ServerApp.keyfile=/etc/letsencrypt/privkey.pem volumes: work: diff --git a/examples/make-deploy/Dockerfile b/examples/make-deploy/Dockerfile index 6e2668ccf8..056c22fb29 100644 --- a/examples/make-deploy/Dockerfile +++ b/examples/make-deploy/Dockerfile @@ -2,7 +2,7 @@ # Distributed under the terms of the Modified BSD License. # Pick your favorite docker-stacks image -FROM jupyter/minimal-notebook:2023-06-01 +FROM quay.io/jupyter/minimal-notebook USER root diff --git a/examples/make-deploy/Makefile b/examples/make-deploy/Makefile index e937621c60..aa62dd0bbf 100644 --- a/examples/make-deploy/Makefile +++ b/examples/make-deploy/Makefile @@ -13,7 +13,7 @@ define RUN_NOTEBOOK --name $(NAME) \ -v $(WORK_VOLUME):/home/jovyan/work \ $(DOCKER_ARGS) \ - $(IMAGE) bash -c "$(PRE_CMD) chown jovyan /home/jovyan/work && start-notebook.sh $(ARGS)" > /dev/null + $(IMAGE) bash -c "$(PRE_CMD) chown jovyan /home/jovyan/work && start-notebook.py $(ARGS)" > /dev/null @echo "DONE: Notebook '$(NAME)' listening on $$(docker-machine ip $$(docker-machine active)):$(PORT)" endef diff --git a/examples/make-deploy/README.md b/examples/make-deploy/README.md index d4fc70fe18..0636612c83 100644 --- a/examples/make-deploy/README.md +++ b/examples/make-deploy/README.md @@ -20,7 +20,7 @@ make virtualbox-vm NAME=dev eval $(docker-machine env dev) # pull a docker stack and build a local image from it make image -# start a notebook server in a container +# start a Server in a container make notebook ``` diff --git a/examples/make-deploy/letsencrypt.makefile b/examples/make-deploy/letsencrypt.makefile index cb006388fa..19549657e3 100644 --- a/examples/make-deploy/letsencrypt.makefile +++ b/examples/make-deploy/letsencrypt.makefile @@ -3,7 +3,7 @@ # BE CAREFUL when using Docker engine <1.10 because running a container with # `--rm` option while mounting a docker volume may wipe out the volume. -# See issue: https://github.com/docker/docker/issues/17907 +# See issue: https://github.com/moby/moby/issues/17907 # Use letsencrypt production server by default to get a real cert. # Use CERT_SERVER=--staging to hit the staging server (not a real cert). @@ -52,8 +52,8 @@ letsencrypt-notebook: DOCKER_ARGS:=-e USE_HTTPS=yes \ -e PASSWORD=$(PASSWORD) \ -v $(SECRETS_VOLUME):/etc/letsencrypt letsencrypt-notebook: ARGS:=\ - --NotebookApp.certfile=/etc/letsencrypt/fullchain.pem \ - --NotebookApp.keyfile=/etc/letsencrypt/privkey.pem + --ServerApp.certfile=/etc/letsencrypt/fullchain.pem \ + --ServerApp.keyfile=/etc/letsencrypt/privkey.pem letsencrypt-notebook: check @test -n "$(PASSWORD)" || \ (echo "ERROR: PASSWORD not defined or blank"; exit 1) diff --git a/examples/openshift/README.md b/examples/openshift/README.md index 7895abbc32..65928d4aa8 100644 --- a/examples/openshift/README.md +++ b/examples/openshift/README.md @@ -52,7 +52,7 @@ The output will be similar to: * With parameters: * APPLICATION_NAME=notebook - * NOTEBOOK_IMAGE=jupyter/minimal-notebook:latest + * NOTEBOOK_IMAGE=docker.io/jupyter/minimal-notebook:latest * NOTEBOOK_PASSWORD=ded4d7cada554aa48e0db612e1ed1080 # generated --> Creating resources ... @@ -69,7 +69,7 @@ When no template parameters are provided, the name of the deployed notebook will The image used will be: ```lang-none -jupyter/minimal-notebook:latest +docker.io/jupyter/minimal-notebook:latest ``` A password you can use when accessing the notebook will be auto generated and is displayed in the output from running `oc new-app`. @@ -102,21 +102,12 @@ To override the name for the notebook, the image used, and the password, you can ```bash oc new-app --template jupyter-notebook \ --param APPLICATION_NAME=mynotebook \ - --param NOTEBOOK_IMAGE=jupyter/scipy-notebook:latest \ + --param NOTEBOOK_IMAGE=docker.io/jupyter/scipy-notebook:latest \ --param NOTEBOOK_PASSWORD=mypassword ``` You can deploy any of the Jupyter Project docker-stacks images. -- jupyter/base-notebook -- jupyter/r-notebook -- jupyter/minimal-notebook -- jupyter/scipy-notebook -- jupyter/tensorflow-notebook -- jupyter/datascience-notebook -- jupyter/pyspark-notebook -- jupyter/all-spark-notebook - If you don't care what version of the image is used, add the `:latest` tag at the end of the image name, otherwise use the hash corresponding to the image version you want to use. ## Deleting the Notebook Instance @@ -222,7 +213,7 @@ you can use the name of the image stream for the image name, including any image This can be illustrated by first importing an image into the OpenShift project. ```bash -oc import-image jupyter/datascience-notebook:latest --confirm +oc import-image docker.io/jupyter/datascience-notebook:latest --confirm ``` Then deploy it using the name of the image stream created. diff --git a/examples/openshift/templates.json b/examples/openshift/templates.json index 0efd4ad24d..ebcd6cb2eb 100644 --- a/examples/openshift/templates.json +++ b/examples/openshift/templates.json @@ -18,7 +18,7 @@ }, { "name": "NOTEBOOK_IMAGE", - "value": "jupyter/minimal-notebook:latest", + "value": "docker.io/jupyter/minimal-notebook:latest", "required": true }, { @@ -38,7 +38,7 @@ } }, "data": { - "jupyter_server_config.py": "import os\n\npassword = os.environ.get('JUPYTER_NOTEBOOK_PASSWORD')\n\nif password:\n import notebook.auth\n c.NotebookApp.password = notebook.auth.passwd(password)\n del password\n del os.environ['JUPYTER_NOTEBOOK_PASSWORD']\n\nimage_config_file = '/home/jovyan/.jupyter/jupyter_server_config.py'\n\nif os.path.exists(image_config_file):\n with open(image_config_file) as fp:\n exec(compile(fp.read(), image_config_file, 'exec'), globals())\n" + "jupyter_server_config.py": "import os\n\npassword = os.environ.get('JUPYTER_NOTEBOOK_PASSWORD')\n\nif password:\n from jupyter_server.auth import passwd\n c.ServerApp.password = passwd(password)\n del password\n del os.environ['JUPYTER_NOTEBOOK_PASSWORD']\n\nimage_config_file = '/home/jovyan/.jupyter/jupyter_server_config.py'\n\nif os.path.exists(image_config_file):\n with open(image_config_file) as fp:\n exec(compile(fp.read(), image_config_file, 'exec'), globals())\n" } }, { @@ -80,12 +80,11 @@ "name": "jupyter-notebook", "image": "${NOTEBOOK_IMAGE}", "command": [ - "start-notebook.sh", + "start-notebook.py", "--config=/etc/jupyter/openshift/jupyter_server_config.py", "--no-browser", "--ip=0.0.0.0" ], - "ports": [ { "containerPort": 8888, diff --git a/examples/source-to-image/README.md b/examples/source-to-image/README.md index 1e9d3ba992..6d4bcb82ac 100644 --- a/examples/source-to-image/README.md +++ b/examples/source-to-image/README.md @@ -34,13 +34,13 @@ s2i build \ --scripts-url https://raw.githubusercontent.com/jupyter/docker-stacks/main/examples/source-to-image \ --context-dir docs/source/examples/Notebook \ https://github.com/jupyter/notebook \ - jupyter/minimal-notebook:latest \ + docker.io/jupyter/minimal-notebook:latest \ notebook-examples ``` This example command will pull down the Git repository <https://github.com/jupyter/notebook> and build the image `notebook-examples` using the files contained in the `docs/source/examples/Notebook` directory of that Git repository. -The base image which the files will be combined with is `jupyter/minimal-notebook:latest`, but you can specify any of the Jupyter Project `docker-stacks` images as the base image. +The base image which the files will be combined with is `docker.io/jupyter/minimal-notebook:latest`, but you can specify any of the Jupyter Project `docker-stacks` images as the base image. The resulting image from running the command can be seen by running `docker images` command: @@ -117,7 +117,7 @@ with the extra system packages, and then use that image with the S2I build to co The `run` script in this directory is very simple and just runs the notebook application. ```bash -exec start-notebook.sh "$@" +exec start-notebook.py "$@" ``` ## Integration with OpenShift @@ -147,7 +147,7 @@ oc new-app --template jupyter-notebook-quickstart \ --param APPLICATION_NAME=notebook-examples \ --param GIT_REPOSITORY_URL=https://github.com/jupyter/notebook \ --param CONTEXT_DIR=docs/source/examples/Notebook \ - --param BUILDER_IMAGE=jupyter/minimal-notebook:latest \ + --param BUILDER_IMAGE=docker.io/jupyter/minimal-notebook:latest \ --param NOTEBOOK_PASSWORD=mypassword ``` diff --git a/examples/source-to-image/run b/examples/source-to-image/run index b5b641b8f6..556efdda9d 100755 --- a/examples/source-to-image/run +++ b/examples/source-to-image/run @@ -2,4 +2,4 @@ # Start up the notebook instance. -exec start-notebook.sh "$@" +exec start-notebook.py "$@" diff --git a/examples/source-to-image/templates.json b/examples/source-to-image/templates.json index 3f6817aff2..aa67766820 100644 --- a/examples/source-to-image/templates.json +++ b/examples/source-to-image/templates.json @@ -22,7 +22,7 @@ }, { "name": "BUILDER_IMAGE", - "value": "jupyter/minimal-notebook:latest", + "value": "docker.io/jupyter/minimal-notebook:latest", "required": true }, { @@ -125,7 +125,7 @@ }, { "name": "BUILDER_IMAGE", - "value": "jupyter/minimal-notebook:latest", + "value": "docker.io/jupyter/minimal-notebook:latest", "required": true }, { @@ -221,7 +221,7 @@ } }, "data": { - "jupyter_server_config.py": "import os\n\npassword = os.environ.get('JUPYTER_NOTEBOOK_PASSWORD')\n\nif password:\n import notebook.auth\n c.NotebookApp.password = notebook.auth.passwd(password)\n del password\n del os.environ['JUPYTER_NOTEBOOK_PASSWORD']\n\nimage_config_file = '/home/jovyan/.jupyter/jupyter_server_config.py'\n\nif os.path.exists(image_config_file):\n with open(image_config_file) as fp:\n exec(compile(fp.read(), image_config_file, 'exec'), globals())\n" + "jupyter_server_config.py": "import os\n\npassword = os.environ.get('JUPYTER_NOTEBOOK_PASSWORD')\n\nif password:\n from jupyter_server.auth import passwd\n c.ServerApp.password = passwd(password)\n del password\n del os.environ['JUPYTER_NOTEBOOK_PASSWORD']\n\nimage_config_file = '/home/jovyan/.jupyter/jupyter_server_config.py'\n\nif os.path.exists(image_config_file):\n with open(image_config_file) as fp:\n exec(compile(fp.read(), image_config_file, 'exec'), globals())\n" } }, { @@ -274,12 +274,11 @@ "name": "jupyter-notebook", "image": "${APPLICATION_NAME}:latest", "command": [ - "start-notebook.sh", + "start-notebook.py", "--config=/etc/jupyter/openshift/jupyter_server_config.py", "--no-browser", "--ip=0.0.0.0" ], - "ports": [ { "containerPort": 8888, diff --git a/all-spark-notebook/.dockerignore b/images/all-spark-notebook/.dockerignore similarity index 100% rename from all-spark-notebook/.dockerignore rename to images/all-spark-notebook/.dockerignore diff --git a/all-spark-notebook/Dockerfile b/images/all-spark-notebook/Dockerfile similarity index 87% rename from all-spark-notebook/Dockerfile rename to images/all-spark-notebook/Dockerfile index 46509fac71..13eb550907 100644 --- a/all-spark-notebook/Dockerfile +++ b/images/all-spark-notebook/Dockerfile @@ -1,8 +1,9 @@ # Copyright (c) Jupyter Development Team. # Distributed under the terms of the Modified BSD License. +ARG REGISTRY=quay.io ARG OWNER=jupyter -ARG BASE_CONTAINER=$OWNER/pyspark-notebook -FROM $BASE_CONTAINER +ARG BASE_IMAGE=$REGISTRY/$OWNER/pyspark-notebook +FROM $BASE_IMAGE LABEL maintainer="Jupyter Project <jupyter@googlegroups.com>" @@ -13,7 +14,7 @@ SHELL ["/bin/bash", "-o", "pipefail", "-c"] USER root # RSpark config -ENV R_LIBS_USER "${SPARK_HOME}/R/lib" +ENV R_LIBS_USER="${SPARK_HOME}/R/lib" RUN fix-permissions "${R_LIBS_USER}" # R pre-requisites diff --git a/all-spark-notebook/README.md b/images/all-spark-notebook/README.md similarity index 84% rename from all-spark-notebook/README.md rename to images/all-spark-notebook/README.md index 898aca6a8f..1619e77984 100644 --- a/all-spark-notebook/README.md +++ b/images/all-spark-notebook/README.md @@ -1,10 +1,12 @@ # Jupyter Notebook Python, R, Spark Stack +> **Images hosted on Docker Hub are no longer updated. Please, use [quay.io image](https://quay.io/repository/jupyter/all-spark-notebook)** + [![docker pulls](https://img.shields.io/docker/pulls/jupyter/all-spark-notebook.svg)](https://hub.docker.com/r/jupyter/all-spark-notebook/) [![docker stars](https://img.shields.io/docker/stars/jupyter/all-spark-notebook.svg)](https://hub.docker.com/r/jupyter/all-spark-notebook/) [![image size](https://img.shields.io/docker/image-size/jupyter/all-spark-notebook/latest)](https://hub.docker.com/r/jupyter/all-spark-notebook/ "jupyter/all-spark-notebook image size") -GitHub Actions in the <https://github.com/jupyter/docker-stacks> project builds and pushes this image to Docker Hub. +GitHub Actions in the <https://github.com/jupyter/docker-stacks> project builds and pushes this image to the Registry. Please visit the project documentation site for help to use and contribute to this image and others. diff --git a/base-notebook/.dockerignore b/images/base-notebook/.dockerignore similarity index 100% rename from base-notebook/.dockerignore rename to images/base-notebook/.dockerignore diff --git a/base-notebook/Dockerfile b/images/base-notebook/Dockerfile similarity index 56% rename from base-notebook/Dockerfile rename to images/base-notebook/Dockerfile index 0a097c6c39..0cde5b5d90 100644 --- a/base-notebook/Dockerfile +++ b/images/base-notebook/Dockerfile @@ -1,8 +1,9 @@ # Copyright (c) Jupyter Development Team. # Distributed under the terms of the Modified BSD License. +ARG REGISTRY=quay.io ARG OWNER=jupyter -ARG BASE_CONTAINER=$OWNER/docker-stacks-foundation -FROM $BASE_CONTAINER +ARG BASE_IMAGE=$REGISTRY/$OWNER/docker-stacks-foundation +FROM $BASE_IMAGE LABEL maintainer="Jupyter Project <jupyter@googlegroups.com>" @@ -12,34 +13,37 @@ SHELL ["/bin/bash", "-o", "pipefail", "-c"] USER root -# Install all OS dependencies for notebook server that starts but lacks all -# features (e.g., download as all possible file formats) +# Install all OS dependencies for the Server that starts +# but lacks all features (e.g., download as all possible file formats) RUN apt-get update --yes && \ apt-get install --yes --no-install-recommends \ + # - Add necessary fonts for matplotlib/seaborn + # See https://github.com/jupyter/docker-stacks/pull/380 for details fonts-liberation \ - # - pandoc is used to convert notebooks to html files - # it's not present in aarch64 ubuntu image, so we install it here + # - `pandoc` is used to convert notebooks to html files + # it's not present in the aarch64 Ubuntu image, so we install it here pandoc \ - # - run-one - a wrapper script that runs no more - # than one unique instance of some command with a unique set of arguments, - # we use `run-one-constantly` to support `RESTARTABLE` option + # - `run-one` - a wrapper script that runs no more + # than one unique instance of some command with a unique set of arguments, + # we use `run-one-constantly` to support the `RESTARTABLE` option run-one && \ apt-get clean && rm -rf /var/lib/apt/lists/* USER ${NB_UID} -# Install Jupyter Notebook, Lab, and Hub -# Generate a notebook server config +# Install JupyterHub, JupyterLab, NBClassic and Jupyter Notebook +# Generate a Jupyter Server config # Cleanup temporary files # Correct permissions # Do all this in a single RUN command to avoid duplicating all of the # files across image layers when the permissions change WORKDIR /tmp RUN mamba install --yes \ - 'notebook' \ 'jupyterhub' \ - 'jupyterlab' && \ - jupyter notebook --generate-config && \ + 'jupyterlab' \ + 'nbclassic' \ + 'notebook' && \ + jupyter server --generate-config && \ mamba clean --all -f -y && \ npm cache clean --force && \ jupyter lab clean && \ @@ -51,25 +55,20 @@ ENV JUPYTER_PORT=8888 EXPOSE $JUPYTER_PORT # Configure container startup -CMD ["start-notebook.sh"] +CMD ["start-notebook.py"] # Copy local files as late as possible to avoid cache busting -COPY start-notebook.sh start-singleuser.sh /usr/local/bin/ -# Currently need to have both jupyter_notebook_config and jupyter_server_config to support classic and lab +COPY start-notebook.py start-notebook.sh start-singleuser.py start-singleuser.sh /usr/local/bin/ COPY jupyter_server_config.py docker_healthcheck.py /etc/jupyter/ # Fix permissions on /etc/jupyter as root USER root - -# Legacy for Jupyter Notebook Server, see: [#1205](https://github.com/jupyter/docker-stacks/issues/1205) -RUN sed -re "s/c.ServerApp/c.NotebookApp/g" \ - /etc/jupyter/jupyter_server_config.py > /etc/jupyter/jupyter_notebook_config.py && \ - fix-permissions /etc/jupyter/ +RUN fix-permissions /etc/jupyter/ # HEALTHCHECK documentation: https://docs.docker.com/engine/reference/builder/#healthcheck -# This healtcheck works well for `lab`, `notebook`, `nbclassic`, `server` and `retro` jupyter commands +# This healtcheck works well for `lab`, `notebook`, `nbclassic`, `server`, and `retro` jupyter commands # https://github.com/jupyter/docker-stacks/issues/915#issuecomment-1068528799 -HEALTHCHECK --interval=5s --timeout=3s --start-period=5s --retries=3 \ +HEALTHCHECK --interval=3s --timeout=1s --start-period=3s --retries=3 \ CMD /etc/jupyter/docker_healthcheck.py || exit 1 # Switch back to jovyan to avoid accidental container runs as root diff --git a/base-notebook/README.md b/images/base-notebook/README.md similarity index 82% rename from base-notebook/README.md rename to images/base-notebook/README.md index 618da14d0e..6713599653 100644 --- a/base-notebook/README.md +++ b/images/base-notebook/README.md @@ -1,10 +1,12 @@ # Base Jupyter Notebook Stack +> **Images hosted on Docker Hub are no longer updated. Please, use [quay.io image](https://quay.io/repository/jupyter/base-notebook)** + [![docker pulls](https://img.shields.io/docker/pulls/jupyter/base-notebook.svg)](https://hub.docker.com/r/jupyter/base-notebook/) [![docker stars](https://img.shields.io/docker/stars/jupyter/base-notebook.svg)](https://hub.docker.com/r/jupyter/base-notebook/) [![image size](https://img.shields.io/docker/image-size/jupyter/base-notebook/latest)](https://hub.docker.com/r/jupyter/base-notebook/ "jupyter/base-notebook image size") -GitHub Actions in the <https://github.com/jupyter/docker-stacks> project builds and pushes this image to Docker Hub. +GitHub Actions in the <https://github.com/jupyter/docker-stacks> project builds and pushes this image to the Registry. Please visit the project documentation site for help to use and contribute to this image and others. diff --git a/images/base-notebook/docker_healthcheck.py b/images/base-notebook/docker_healthcheck.py new file mode 100755 index 0000000000..b0db1a81be --- /dev/null +++ b/images/base-notebook/docker_healthcheck.py @@ -0,0 +1,39 @@ +#!/usr/bin/env python3 +# Copyright (c) Jupyter Development Team. +# Distributed under the terms of the Modified BSD License. +import json +import os +import subprocess +from pathlib import Path + +import requests + +# Several operations below deliberately don't check for possible errors +# As this is a health check, it should succeed or raise an exception on error + +# Docker runs health checks using an exec +# It uses the default user configured when running the image: root for the case of a custom NB_USER or jovyan for the case of the default image user. +# We manually change HOME to make `jupyter --runtime-dir` report a correct path +# More information: <https://github.com/jupyter/docker-stacks/pull/2074#issuecomment-1879778409> +result = subprocess.run( + ["jupyter", "--runtime-dir"], + check=True, + capture_output=True, + text=True, + env=dict(os.environ) | {"HOME": "/home/" + os.environ["NB_USER"]}, +) +runtime_dir = Path(result.stdout.rstrip()) + +json_file = next(runtime_dir.glob("*server-*.json")) + +url = json.loads(json_file.read_bytes())["url"] +url = url + "api" + +proxies = { + "http": "", + "https": "", +} + +r = requests.get(url, proxies=proxies, verify=False) # request without SSL verification +r.raise_for_status() +print(r.content) diff --git a/base-notebook/jupyter_server_config.py b/images/base-notebook/jupyter_server_config.py similarity index 68% rename from base-notebook/jupyter_server_config.py rename to images/base-notebook/jupyter_server_config.py index 679f96bee0..c0cca3af80 100644 --- a/base-notebook/jupyter_server_config.py +++ b/images/base-notebook/jupyter_server_config.py @@ -4,6 +4,7 @@ import os import stat import subprocess +from pathlib import Path from jupyter_core.paths import jupyter_data_dir @@ -24,17 +25,16 @@ [req_distinguished_name] """ if "GEN_CERT" in os.environ: - dir_name = jupyter_data_dir() - pem_file = os.path.join(dir_name, "notebook.pem") - os.makedirs(dir_name, exist_ok=True) + dir_name = Path(jupyter_data_dir()) + dir_name.mkdir(parents=True, exist_ok=True) + pem_file = dir_name / "notebook.pem" # Generate an openssl.cnf file to set the distinguished name - cnf_file = os.path.join(os.getenv("CONDA_DIR", "/usr/lib"), "ssl", "openssl.cnf") - if not os.path.isfile(cnf_file): - with open(cnf_file, "w") as fh: - fh.write(OPENSSL_CONFIG) + cnf_file = Path(os.getenv("CONDA_DIR", "/usr/lib")) / "ssl/openssl.cnf" + if not cnf_file.exists(): + cnf_file.write_text(OPENSSL_CONFIG) - # Generate a certificate if one doesn't exist on disk + # Generate a certificate if one doesn't exist on a disk subprocess.check_call( [ "openssl", @@ -50,10 +50,9 @@ ] ) # Restrict access to the file - os.chmod(pem_file, stat.S_IRUSR | stat.S_IWUSR) - c.ServerApp.certfile = pem_file + pem_file.chmod(stat.S_IRUSR | stat.S_IWUSR) + c.ServerApp.certfile = str(pem_file) -# Change default umask for all subprocesses of the notebook server if set in -# the environment +# Change default umask for all subprocesses of the Server if set in the environment if "NB_UMASK" in os.environ: os.umask(int(os.environ["NB_UMASK"], 8)) diff --git a/images/base-notebook/start-notebook.py b/images/base-notebook/start-notebook.py new file mode 100755 index 0000000000..973da5aad1 --- /dev/null +++ b/images/base-notebook/start-notebook.py @@ -0,0 +1,44 @@ +#!/usr/bin/env python +# Copyright (c) Jupyter Development Team. +# Distributed under the terms of the Modified BSD License. +import os +import shlex +import sys + +# If we are in a JupyterHub, we pass on to `start-singleuser.py` instead so it does the right thing +if "JUPYTERHUB_API_TOKEN" in os.environ: + print( + "WARNING: using start-singleuser.py instead of start-notebook.py to start a server associated with JupyterHub." + ) + command = ["/usr/local/bin/start-singleuser.py"] + sys.argv[1:] + os.execvp(command[0], command) + + +# Entrypoint is start.sh +command = [] + +# If we want to survive restarts, launch the command using `run-one-constantly` +if os.environ.get("RESTARTABLE") == "yes": + command.append("run-one-constantly") + +# We always launch a jupyter subcommand from this script +command.append("jupyter") + +# Launch the configured subcommand. +# Note that this should be a single string, so we don't split it. +# We default to `lab`. +jupyter_command = os.environ.get("DOCKER_STACKS_JUPYTER_CMD", "lab") +command.append(jupyter_command) + +# Append any optional NOTEBOOK_ARGS we were passed in. +# This is supposed to be multiple args passed on to the notebook command, +# so we split it correctly with shlex +if "NOTEBOOK_ARGS" in os.environ: + command += shlex.split(os.environ["NOTEBOOK_ARGS"]) + +# Pass through any other args we were passed on the command line +command += sys.argv[1:] + +# Execute the command! +print("Executing: " + " ".join(command)) +os.execvp(command[0], command) diff --git a/images/base-notebook/start-notebook.sh b/images/base-notebook/start-notebook.sh new file mode 100755 index 0000000000..c47ebba334 --- /dev/null +++ b/images/base-notebook/start-notebook.sh @@ -0,0 +1,5 @@ +#!/bin/bash +# Shim to emit warning and call start-notebook.py +echo "WARNING: Use start-notebook.py instead" + +exec /usr/local/bin/start-notebook.py "$@" diff --git a/images/base-notebook/start-singleuser.py b/images/base-notebook/start-singleuser.py new file mode 100755 index 0000000000..c80339f52a --- /dev/null +++ b/images/base-notebook/start-singleuser.py @@ -0,0 +1,26 @@ +#!/usr/bin/env python +# Copyright (c) Jupyter Development Team. +# Distributed under the terms of the Modified BSD License. +import os +import shlex +import sys + +# Entrypoint is start.sh +command = ["jupyterhub-singleuser"] + +# set default ip to 0.0.0.0 +if "--ip=" not in os.environ.get("NOTEBOOK_ARGS", ""): + command.append("--ip=0.0.0.0") + +# Append any optional NOTEBOOK_ARGS we were passed in. +# This is supposed to be multiple args passed on to the notebook command, +# so we split it correctly with shlex +if "NOTEBOOK_ARGS" in os.environ: + command += shlex.split(os.environ["NOTEBOOK_ARGS"]) + +# Pass any other args we have been passed through +command += sys.argv[1:] + +# Execute the command! +print("Executing: " + " ".join(command)) +os.execvp(command[0], command) diff --git a/images/base-notebook/start-singleuser.sh b/images/base-notebook/start-singleuser.sh new file mode 100755 index 0000000000..ecf0e068ae --- /dev/null +++ b/images/base-notebook/start-singleuser.sh @@ -0,0 +1,5 @@ +#!/bin/bash +# Shim to emit warning and call start-singleuser.py +echo "WARNING: Use start-singleuser.py instead" + +exec /usr/local/bin/start-singleuser.py "$@" diff --git a/datascience-notebook/.dockerignore b/images/datascience-notebook/.dockerignore similarity index 100% rename from datascience-notebook/.dockerignore rename to images/datascience-notebook/.dockerignore diff --git a/datascience-notebook/Dockerfile b/images/datascience-notebook/Dockerfile similarity index 92% rename from datascience-notebook/Dockerfile rename to images/datascience-notebook/Dockerfile index 4833fef3a3..ab7a8454db 100644 --- a/datascience-notebook/Dockerfile +++ b/images/datascience-notebook/Dockerfile @@ -1,8 +1,9 @@ # Copyright (c) Jupyter Development Team. # Distributed under the terms of the Modified BSD License. +ARG REGISTRY=quay.io ARG OWNER=jupyter -ARG BASE_CONTAINER=$OWNER/scipy-notebook -FROM $BASE_CONTAINER +ARG BASE_IMAGE=$REGISTRY/$OWNER/scipy-notebook +FROM $BASE_IMAGE LABEL maintainer="Jupyter Project <jupyter@googlegroups.com>" @@ -26,7 +27,7 @@ ENV JULIA_DEPOT_PATH=/opt/julia \ JULIA_PKGDIR=/opt/julia # Setup Julia -RUN /opt/setup-scripts/setup-julia.bash +RUN /opt/setup-scripts/setup_julia.py USER ${NB_UID} diff --git a/datascience-notebook/README.md b/images/datascience-notebook/README.md similarity index 83% rename from datascience-notebook/README.md rename to images/datascience-notebook/README.md index 64ef2007e0..42f1e047f6 100644 --- a/datascience-notebook/README.md +++ b/images/datascience-notebook/README.md @@ -1,10 +1,12 @@ # Jupyter Notebook Data Science Stack +> **Images hosted on Docker Hub are no longer updated. Please, use [quay.io image](https://quay.io/repository/jupyter/datascience-notebook)** + [![docker pulls](https://img.shields.io/docker/pulls/jupyter/datascience-notebook.svg)](https://hub.docker.com/r/jupyter/datascience-notebook/) [![docker stars](https://img.shields.io/docker/stars/jupyter/datascience-notebook.svg)](https://hub.docker.com/r/jupyter/datascience-notebook/) [![image size](https://img.shields.io/docker/image-size/jupyter/datascience-notebook/latest)](https://hub.docker.com/r/jupyter/datascience-notebook/ "jupyter/datascience-notebook image size") -GitHub Actions in the <https://github.com/jupyter/docker-stacks> project builds and pushes this image to Docker Hub. +GitHub Actions in the <https://github.com/jupyter/docker-stacks> project builds and pushes this image to the Registry. Please visit the project documentation site for help to use and contribute to this image and others. diff --git a/docker-stacks-foundation/.dockerignore b/images/docker-stacks-foundation/.dockerignore similarity index 100% rename from docker-stacks-foundation/.dockerignore rename to images/docker-stacks-foundation/.dockerignore diff --git a/images/docker-stacks-foundation/10activate-conda-env.sh b/images/docker-stacks-foundation/10activate-conda-env.sh new file mode 100755 index 0000000000..ed7347f3b6 --- /dev/null +++ b/images/docker-stacks-foundation/10activate-conda-env.sh @@ -0,0 +1,8 @@ +#!/bin/bash +# Copyright (c) Jupyter Development Team. +# Distributed under the terms of the Modified BSD License. + +# This registers the initialization code for the conda shell code +# It also activates default environment in the end, so we don't need to activate it manually +# Documentation: https://docs.conda.io/projects/conda/en/latest/dev-guide/deep-dives/activation.html +eval "$(conda shell.bash hook)" diff --git a/docker-stacks-foundation/Dockerfile b/images/docker-stacks-foundation/Dockerfile similarity index 50% rename from docker-stacks-foundation/Dockerfile rename to images/docker-stacks-foundation/Dockerfile index a8ada5fb37..63fdf5430c 100644 --- a/docker-stacks-foundation/Dockerfile +++ b/images/docker-stacks-foundation/Dockerfile @@ -1,11 +1,11 @@ # Copyright (c) Jupyter Development Team. # Distributed under the terms of the Modified BSD License. -# Ubuntu 22.04 (jammy) -# https://hub.docker.com/_/ubuntu/tags?page=1&name=jammy -ARG ROOT_CONTAINER=ubuntu:22.04 +# Ubuntu 24.04 (noble) +# https://hub.docker.com/_/ubuntu/tags?page=1&name=noble +ARG ROOT_IMAGE=ubuntu:24.04 -FROM $ROOT_CONTAINER +FROM $ROOT_IMAGE LABEL maintainer="Jupyter Project <jupyter@googlegroups.com>" ARG NB_USER="jovyan" @@ -18,26 +18,31 @@ SHELL ["/bin/bash", "-o", "pipefail", "-c"] USER root -# Install all OS dependencies for notebook server that starts but lacks all -# features (e.g., download as all possible file formats) -ENV DEBIAN_FRONTEND noninteractive +# Install all OS dependencies for the Server that starts +# but lacks all features (e.g., download as all possible file formats) +ENV DEBIAN_FRONTEND=noninteractive RUN apt-get update --yes && \ - # - apt-get upgrade is run to patch known vulnerabilities in apt-get packages as - # the ubuntu base image is rebuilt too seldom sometimes (less than once a month) + # - `apt-get upgrade` is run to patch known vulnerabilities in system packages + # as the Ubuntu base image is rebuilt too seldom sometimes (less than once a month) apt-get upgrade --yes && \ apt-get install --yes --no-install-recommends \ # - bzip2 is necessary to extract the micromamba executable. bzip2 \ ca-certificates \ locales \ + # - `netbase` provides /etc/{protocols,rpc,services}, part of POSIX + # and required by various C functions like getservbyname and getprotobyname + # https://github.com/jupyter/docker-stacks/pull/2129 + netbase \ sudo \ - # - tini is installed as a helpful container entrypoint that reaps zombie - # processes and such of the actual executable we want to start, see - # https://github.com/krallin/tini#why-tini for details. + # - `tini` is installed as a helpful container entrypoint, + # that reaps zombie processes and such of the actual executable we want to start + # See https://github.com/krallin/tini#why-tini for details tini \ wget && \ apt-get clean && rm -rf /var/lib/apt/lists/* && \ echo "en_US.UTF-8 UTF-8" > /etc/locale.gen && \ + echo "C.UTF-8 UTF-8" >> /etc/locale.gen && \ locale-gen # Configure environment @@ -46,9 +51,9 @@ ENV CONDA_DIR=/opt/conda \ NB_USER="${NB_USER}" \ NB_UID=${NB_UID} \ NB_GID=${NB_GID} \ - LC_ALL=en_US.UTF-8 \ - LANG=en_US.UTF-8 \ - LANGUAGE=en_US.UTF-8 + LC_ALL=C.UTF-8 \ + LANG=C.UTF-8 \ + LANGUAGE=C.UTF-8 ENV PATH="${CONDA_DIR}/bin:${PATH}" \ HOME="/home/${NB_USER}" @@ -59,36 +64,44 @@ RUN chmod a+rx /usr/local/bin/fix-permissions # Enable prompt color in the skeleton .bashrc before creating the default NB_USER # hadolint ignore=SC2016 RUN sed -i 's/^#force_color_prompt=yes/force_color_prompt=yes/' /etc/skel/.bashrc && \ - # Add call to conda init script see https://stackoverflow.com/a/58081608/4413446 - echo 'eval "$(command conda shell.bash hook 2> /dev/null)"' >> /etc/skel/.bashrc + # More information in: https://github.com/jupyter/docker-stacks/pull/2047 + # and docs: https://docs.conda.io/projects/conda/en/latest/dev-guide/deep-dives/activation.html + echo 'eval "$(conda shell.bash hook)"' >> /etc/skel/.bashrc -# Create NB_USER with name jovyan user with UID=1000 and in the 'users' group +# Delete existing user with UID="${NB_UID}" if it exists +# hadolint ignore=SC2046 +RUN if grep -q "${NB_UID}" /etc/passwd; then \ + userdel --remove $(id -un "${NB_UID}"); \ + fi + +# Create "${NB_USER}" user (`jovyan` by default) with UID="${NB_UID}" (`1000` by default) and in the 'users' group # and make sure these dirs are writable by the `users` group. RUN echo "auth requisite pam_deny.so" >> /etc/pam.d/su && \ sed -i.bak -e 's/^%admin/#%admin/' /etc/sudoers && \ sed -i.bak -e 's/^%sudo/#%sudo/' /etc/sudoers && \ - useradd -l -m -s /bin/bash -N -u "${NB_UID}" "${NB_USER}" -g "${NB_GID}" && \ + # useradd -l -m -s /bin/bash -N -u "${NB_UID}" "${NB_USER}" -g "${NB_GID}" && \ + useradd --no-log-init --create-home --shell /bin/bash --uid "${NB_UID}" --no-user-group "${NB_USER}" && \ mkdir -p "${CONDA_DIR}" && \ chown "${NB_USER}:${NB_GID}" "${CONDA_DIR}" && \ chmod g+w /etc/passwd && \ - fix-permissions "${HOME}" && \ - fix-permissions "${CONDA_DIR}" + fix-permissions "${CONDA_DIR}" && \ + fix-permissions "/home/${NB_USER}" USER ${NB_UID} -# Pin python version here, or set it to "default" +# Pin the Python version here, or set it to "default" ARG PYTHON_VERSION=3.11 # Setup work directory for backward-compatibility RUN mkdir "/home/${NB_USER}/work" && \ fix-permissions "/home/${NB_USER}" -# Download and install Micromamba, and initialize Conda prefix. +# Download and install Micromamba, and initialize the Conda prefix. # <https://github.com/mamba-org/mamba#micromamba> # Similar projects using Micromamba: # - Micromamba-Docker: <https://github.com/mamba-org/micromamba-docker> # - repo2docker: <https://github.com/jupyterhub/repo2docker> -# Install Python, Mamba and jupyter_core +# Install Python, Mamba, and jupyter_core # Cleanup temporary files and remove Micromamba # Correct permissions # Do all this in a single RUN command to avoid duplicating all of the @@ -101,33 +114,42 @@ RUN set -x && \ # Should be simpler, see <https://github.com/mamba-org/mamba/issues/1437> arch="64"; \ fi && \ - wget --progress=dot:giga -O /tmp/micromamba.tar.bz2 \ - "https://micromamba.snakepit.net/api/micromamba/linux-${arch}/latest" && \ - tar -xvjf /tmp/micromamba.tar.bz2 --strip-components=1 bin/micromamba && \ - rm /tmp/micromamba.tar.bz2 && \ + # https://mamba.readthedocs.io/en/latest/installation/micromamba-installation.html#linux-and-macos + wget --progress=dot:giga -O - \ + "https://micro.mamba.pm/api/micromamba/linux-${arch}/latest" | tar -xvj bin/micromamba && \ PYTHON_SPECIFIER="python=${PYTHON_VERSION}" && \ if [[ "${PYTHON_VERSION}" == "default" ]]; then PYTHON_SPECIFIER="python"; fi && \ # Install the packages - ./micromamba install \ + ./bin/micromamba install \ --root-prefix="${CONDA_DIR}" \ --prefix="${CONDA_DIR}" \ --yes \ - "${PYTHON_SPECIFIER}" \ - 'mamba' \ - 'jupyter_core' && \ - rm micromamba && \ + 'jupyter_core' \ + # excluding mamba 2.X due to several breaking changes + # https://github.com/jupyter/docker-stacks/pull/2147 + 'mamba<2.0.0' \ + "${PYTHON_SPECIFIER}" && \ + rm -rf /tmp/bin/ && \ # Pin major.minor version of python - mamba list python | grep '^python ' | tr -s ' ' | cut -d ' ' -f 1,2 >> "${CONDA_DIR}/conda-meta/pinned" && \ + # https://conda.io/projects/conda/en/latest/user-guide/tasks/manage-pkgs.html#preventing-packages-from-updating-pinning + mamba list --full-name 'python' | awk 'END{sub("[^.]*$", "*", $2); print $1 " " $2}' >> "${CONDA_DIR}/conda-meta/pinned" && \ mamba clean --all -f -y && \ fix-permissions "${CONDA_DIR}" && \ fix-permissions "/home/${NB_USER}" -# Configure container startup -ENTRYPOINT ["tini", "-g", "--"] -CMD ["start.sh"] - # Copy local files as late as possible to avoid cache busting -COPY start.sh /usr/local/bin/ +COPY run-hooks.sh start.sh /usr/local/bin/ + +# Configure container entrypoint +ENTRYPOINT ["tini", "-g", "--", "start.sh"] + +USER root + +# Create dirs for startup hooks +RUN mkdir /usr/local/bin/start-notebook.d && \ + mkdir /usr/local/bin/before-notebook.d + +COPY 10activate-conda-env.sh /usr/local/bin/before-notebook.d/ # Switch back to jovyan to avoid accidental container runs as root USER ${NB_UID} diff --git a/docker-stacks-foundation/README.md b/images/docker-stacks-foundation/README.md similarity index 83% rename from docker-stacks-foundation/README.md rename to images/docker-stacks-foundation/README.md index b4e4d2db10..666575211e 100644 --- a/docker-stacks-foundation/README.md +++ b/images/docker-stacks-foundation/README.md @@ -1,10 +1,12 @@ # Foundation Jupyter Stack +> **Images hosted on Docker Hub are no longer updated. Please, use [quay.io image](https://quay.io/repository/jupyter/docker-stacks-foundation)** + [![docker pulls](https://img.shields.io/docker/pulls/jupyter/docker-stacks-foundation.svg)](https://hub.docker.com/r/jupyter/docker-stacks-foundation/) [![docker stars](https://img.shields.io/docker/stars/jupyter/docker-stacks-foundation.svg)](https://hub.docker.com/r/jupyter/docker-stacks-foundation/) [![image size](https://img.shields.io/docker/image-size/jupyter/docker-stacks-foundation/latest)](https://hub.docker.com/r/jupyter/docker-stacks-foundation/ "jupyter/docker-stacks-foundation image size") -GitHub Actions in the <https://github.com/jupyter/docker-stacks> project builds and pushes this image to Docker Hub. +GitHub Actions in the <https://github.com/jupyter/docker-stacks> project builds and pushes this image to the Registry. Please visit the project documentation site for help to use and contribute to this image and others. diff --git a/docker-stacks-foundation/fix-permissions b/images/docker-stacks-foundation/fix-permissions similarity index 70% rename from docker-stacks-foundation/fix-permissions rename to images/docker-stacks-foundation/fix-permissions index d167578bcd..47b6d0e205 100755 --- a/docker-stacks-foundation/fix-permissions +++ b/images/docker-stacks-foundation/fix-permissions @@ -1,16 +1,14 @@ #!/bin/bash -# set permissions on a directory -# after any installation, if a directory needs to be (human) user-writable, -# run this script on it. -# It will make everything in the directory owned by the group ${NB_GID} -# and writable by that group. +# Set permissions on a directory +# After any installation, if a directory needs to be (human) user-writable, run this script on it. +# It will make everything in the directory owned by the group ${NB_GID} and writable by that group. # Deployments that want to set a specific user id can preserve permissions # by adding the `--group-add users` line to `docker run`. -# uses find to avoid touching files that already have the right permissions, -# which would cause massive image explosion +# Uses find to avoid touching files that already have the right permissions, +# which would cause a massive image explosion -# right permissions are: +# Right permissions are: # group=${NB_GID} # AND permissions include group rwX (directory-execute) # AND directories have setuid,setgid bits set diff --git a/docker-stacks-foundation/initial-condarc b/images/docker-stacks-foundation/initial-condarc similarity index 100% rename from docker-stacks-foundation/initial-condarc rename to images/docker-stacks-foundation/initial-condarc diff --git a/images/docker-stacks-foundation/run-hooks.sh b/images/docker-stacks-foundation/run-hooks.sh new file mode 100755 index 0000000000..15df23c35e --- /dev/null +++ b/images/docker-stacks-foundation/run-hooks.sh @@ -0,0 +1,46 @@ +#!/bin/bash +# Copyright (c) Jupyter Development Team. +# Distributed under the terms of the Modified BSD License. + +# The run-hooks.sh script looks for *.sh scripts to source +# and executable files to run within a passed directory + +if [ "$#" -ne 1 ]; then + echo "Should pass exactly one directory" + return 1 +fi + +if [[ ! -d "${1}" ]]; then + echo "Directory ${1} doesn't exist or is not a directory" + return 1 +fi + +echo "Running hooks in: ${1} as uid: $(id -u) gid: $(id -g)" +for f in "${1}/"*; do + # Handling a case when the directory is empty + [ -e "${f}" ] || continue + case "${f}" in + *.sh) + echo "Sourcing shell script: ${f}" + # shellcheck disable=SC1090 + source "${f}" + # shellcheck disable=SC2181 + if [ $? -ne 0 ]; then + echo "${f} has failed, continuing execution" + fi + ;; + *) + if [ -x "${f}" ]; then + echo "Running executable: ${f}" + "${f}" + # shellcheck disable=SC2181 + if [ $? -ne 0 ]; then + echo "${f} has failed, continuing execution" + fi + else + echo "Ignoring non-executable: ${f}" + fi + ;; + esac +done +echo "Done running hooks in: ${1}" diff --git a/docker-stacks-foundation/start.sh b/images/docker-stacks-foundation/start.sh similarity index 77% rename from docker-stacks-foundation/start.sh rename to images/docker-stacks-foundation/start.sh index 7c5859ee75..295ee26359 100755 --- a/docker-stacks-foundation/start.sh +++ b/images/docker-stacks-foundation/start.sh @@ -4,9 +4,9 @@ set -e -# The _log function is used for everything this script wants to log. It will -# always log errors and warnings, but can be silenced for other messages -# by setting JUPYTER_DOCKER_STACKS_QUIET environment variable. +# The _log function is used for everything this script wants to log. +# It will always log errors and warnings but can be silenced for other messages +# by setting the JUPYTER_DOCKER_STACKS_QUIET environment variable. _log () { if [[ "$*" == "ERROR:"* ]] || [[ "$*" == "WARNING:"* ]] || [[ "${JUPYTER_DOCKER_STACKS_QUIET}" == "" ]]; then echo "$@" @@ -14,39 +14,12 @@ _log () { } _log "Entered start.sh with args:" "$@" -# The run-hooks function looks for .sh scripts to source and executable files to -# run within a passed directory. -run-hooks () { - if [[ ! -d "${1}" ]] ; then - return - fi - _log "${0}: running hooks in ${1} as uid / gid: $(id -u) / $(id -g)" - for f in "${1}/"*; do - case "${f}" in - *.sh) - _log "${0}: running script ${f}" - # shellcheck disable=SC1090 - source "${f}" - ;; - *) - if [[ -x "${f}" ]] ; then - _log "${0}: running executable ${f}" - "${f}" - else - _log "${0}: ignoring non-executable ${f}" - fi - ;; - esac - done - _log "${0}: done running hooks in ${1}" -} - # A helper function to unset env vars listed in the value of the env var # JUPYTER_ENV_VARS_TO_UNSET. unset_explicit_env_vars () { if [ -n "${JUPYTER_ENV_VARS_TO_UNSET}" ]; then for env_var_to_unset in $(echo "${JUPYTER_ENV_VARS_TO_UNSET}" | tr ',' ' '); do - echo "Unset ${env_var_to_unset} due to JUPYTER_ENV_VARS_TO_UNSET" + _log "Unset ${env_var_to_unset} due to JUPYTER_ENV_VARS_TO_UNSET" unset "${env_var_to_unset}" done unset JUPYTER_ENV_VARS_TO_UNSET @@ -61,15 +34,27 @@ else cmd=( "$@" ) fi +# Backwards compatibility: `start.sh` is executed by default in ENTRYPOINT +# so it should no longer be specified in CMD +if [ "${_START_SH_EXECUTED}" = "1" ]; then + _log "WARNING: start.sh is the default ENTRYPOINT, do not include it in CMD" + _log "Executing the command:" "${cmd[@]}" + exec "${cmd[@]}" +else + export _START_SH_EXECUTED=1 +fi + + # NOTE: This hook will run as the user the container was started with! -run-hooks /usr/local/bin/start-notebook.d +# shellcheck disable=SC1091 +source /usr/local/bin/run-hooks.sh /usr/local/bin/start-notebook.d # If the container started as the root user, then we have permission to refit # the jovyan user, and ensure file permissions, grant sudo rights, and such # things before we run the command passed to start.sh as the desired user # (NB_USER). # -if [ "$(id -u)" == 0 ] ; then +if [ "$(id -u)" == 0 ]; then # Environment variables: # - NB_USER: the desired username and associated home folder # - NB_UID: the desired user id @@ -77,18 +62,18 @@ if [ "$(id -u)" == 0 ] ; then # - NB_GROUP: a group name we want for the group # - GRANT_SUDO: a boolean ("1" or "yes") to grant the user sudo rights # - CHOWN_HOME: a boolean ("1" or "yes") to chown the user's home folder - # - CHOWN_EXTRA: a comma separated list of paths to chown + # - CHOWN_EXTRA: a comma-separated list of paths to chown # - CHOWN_HOME_OPTS / CHOWN_EXTRA_OPTS: arguments to the chown commands - # Refit the jovyan user to the desired the user (NB_USER) - if id jovyan &> /dev/null ; then + # Refit the jovyan user to the desired user (NB_USER) + if id jovyan &> /dev/null; then if ! usermod --home "/home/${NB_USER}" --login "${NB_USER}" jovyan 2>&1 | grep "no changes" > /dev/null; then _log "Updated the jovyan user:" _log "- username: jovyan -> ${NB_USER}" _log "- home dir: /home/jovyan -> /home/${NB_USER}" fi elif ! id -u "${NB_USER}" &> /dev/null; then - _log "ERROR: Neither the jovyan user or '${NB_USER}' exists. This could be the result of stopping and starting, the container with a different NB_USER environment variable." + _log "ERROR: Neither the jovyan user nor '${NB_USER}' exists. This could be the result of stopping and starting, the container with a different NB_USER environment variable." exit 1 fi # Ensure the desired user (NB_USER) gets its desired user id (NB_UID) and is @@ -101,17 +86,25 @@ if [ "$(id -u)" == 0 ] ; then fi # Recreate the desired user as we want it userdel "${NB_USER}" - useradd --home "/home/${NB_USER}" --uid "${NB_UID}" --gid "${NB_GID}" --groups 100 --no-log-init "${NB_USER}" + useradd --no-log-init --home "/home/${NB_USER}" --shell /bin/bash --uid "${NB_UID}" --gid "${NB_GID}" --groups 100 "${NB_USER}" + fi + # Update the home directory if the desired user (NB_USER) is root and the + # desired user id (NB_UID) is 0 and the desired group id (NB_GID) is 0. + if [ "${NB_USER}" = "root" ] && [ "${NB_UID}" = "$(id -u "${NB_USER}")" ] && [ "${NB_GID}" = "$(id -g "${NB_USER}")" ]; then + sed -i "s|/root|/home/root|g" /etc/passwd + # Do not preserve ownership in rootless mode + CP_OPTS="-a --no-preserve=ownership" fi - # Move or symlink the jovyan home directory to the desired users home + # Move or symlink the jovyan home directory to the desired user's home # directory if it doesn't already exist, and update the current working # directory to the new location if needed. if [[ "${NB_USER}" != "jovyan" ]]; then if [[ ! -e "/home/${NB_USER}" ]]; then _log "Attempting to copy /home/jovyan to /home/${NB_USER}..." mkdir "/home/${NB_USER}" - if cp -a /home/jovyan/. "/home/${NB_USER}/"; then + # shellcheck disable=SC2086 + if cp ${CP_OPTS:--a} /home/jovyan/. "/home/${NB_USER}/"; then _log "Success!" else _log "Failed to copy data from /home/jovyan to /home/${NB_USER}!" @@ -132,7 +125,7 @@ if [ "$(id -u)" == 0 ] ; then fi fi - # Optionally ensure the desired user get filesystem ownership of it's home + # Optionally ensure the desired user gets filesystem ownership of its home # folder and/or additional folders if [[ "${CHOWN_HOME}" == "1" || "${CHOWN_HOME}" == "yes" ]]; then _log "Ensuring /home/${NB_USER} is owned by ${NB_UID}:${NB_GID} ${CHOWN_HOME_OPTS:+(chown options: ${CHOWN_HOME_OPTS})}" @@ -147,27 +140,29 @@ if [ "$(id -u)" == 0 ] ; then done fi - # Update potentially outdated environment variables since image build - export XDG_CACHE_HOME="/home/${NB_USER}/.cache" - # Prepend ${CONDA_DIR}/bin to sudo secure_path sed -r "s#Defaults\s+secure_path\s*=\s*\"?([^\"]+)\"?#Defaults secure_path=\"${CONDA_DIR}/bin:\1\"#" /etc/sudoers | grep secure_path > /etc/sudoers.d/path # Optionally grant passwordless sudo rights for the desired user - if [[ "$GRANT_SUDO" == "1" || "$GRANT_SUDO" == "yes" ]]; then + if [[ "${GRANT_SUDO}" == "1" || "${GRANT_SUDO}" == "yes" ]]; then _log "Granting ${NB_USER} passwordless sudo rights!" echo "${NB_USER} ALL=(ALL) NOPASSWD:ALL" >> /etc/sudoers.d/added-by-start-script fi # NOTE: This hook is run as the root user! - run-hooks /usr/local/bin/before-notebook.d - + # shellcheck disable=SC1091 + source /usr/local/bin/run-hooks.sh /usr/local/bin/before-notebook.d unset_explicit_env_vars + _log "Running as ${NB_USER}:" "${cmd[@]}" - exec sudo --preserve-env --set-home --user "${NB_USER}" \ - PATH="${PATH}" \ - PYTHONPATH="${PYTHONPATH:-}" \ - "${cmd[@]}" + if [ "${NB_USER}" = "root" ] && [ "${NB_UID}" = "$(id -u "${NB_USER}")" ] && [ "${NB_GID}" = "$(id -g "${NB_USER}")" ]; then + HOME="/home/root" exec "${cmd[@]}" + else + exec sudo --preserve-env --set-home --user "${NB_USER}" \ + LD_LIBRARY_PATH="${LD_LIBRARY_PATH}" \ + PATH="${PATH}" \ + PYTHONPATH="${PYTHONPATH:-}" \ + "${cmd[@]}" # Notes on how we ensure that the environment that this container is started # with is preserved (except vars listed in JUPYTER_ENV_VARS_TO_UNSET) when # we transition from running as root to running as NB_USER. @@ -178,7 +173,7 @@ if [ "$(id -u)" == 0 ] ; then # command. The behavior can be inspected with `sudo -V` run as root. # # ref: `man sudo` https://linux.die.net/man/8/sudo - # ref: `man sudoers` https://www.sudo.ws/man/1.8.15/sudoers.man.html + # ref: `man sudoers` https://www.sudo.ws/docs/man/sudoers.man/ # # - We use the `--preserve-env` flag to pass through most environment # variables, but understand that exceptions are caused by the sudoers @@ -187,14 +182,15 @@ if [ "$(id -u)" == 0 ] ; then # - We use the `--set-home` flag to set the HOME variable appropriately. # # - To reduce the default list of variables deleted by sudo, we could have - # used `env_delete` from /etc/sudoers. It has higher priority than the + # used `env_delete` from /etc/sudoers. It has a higher priority than the # `--preserve-env` flag and the `env_keep` configuration. # - # - We preserve PATH and PYTHONPATH explicitly. Note however that sudo + # - We preserve LD_LIBRARY_PATH, PATH and PYTHONPATH explicitly. Note however that sudo # resolves `${cmd[@]}` using the "secure_path" variable we modified # above in /etc/sudoers.d/path. Thus PATH is irrelevant to how the above # sudo command resolves the path of `${cmd[@]}`. The PATH will be relevant # for resolving paths of any subprocesses spawned by `${cmd[@]}`. + fi # The container didn't start as the root user, so we will have to act as the # user we started as. @@ -210,7 +206,7 @@ else # Attempt to ensure the user uid we currently run as has a named entry in # the /etc/passwd file, as it avoids software crashing on hard assumptions # on such entry. Writing to the /etc/passwd was allowed for the root group - # from the Dockerfile during build. + # from the Dockerfile during the build. # # ref: https://github.com/jupyter/docker-stacks/issues/552 if ! whoami &> /dev/null; then @@ -255,8 +251,10 @@ else fi # NOTE: This hook is run as the user we started the container as! - run-hooks /usr/local/bin/before-notebook.d + # shellcheck disable=SC1091 + source /usr/local/bin/run-hooks.sh /usr/local/bin/before-notebook.d unset_explicit_env_vars + _log "Executing the command:" "${cmd[@]}" exec "${cmd[@]}" fi diff --git a/julia-notebook/.dockerignore b/images/julia-notebook/.dockerignore similarity index 100% rename from julia-notebook/.dockerignore rename to images/julia-notebook/.dockerignore diff --git a/julia-notebook/Dockerfile b/images/julia-notebook/Dockerfile similarity index 83% rename from julia-notebook/Dockerfile rename to images/julia-notebook/Dockerfile index cee9aa993f..bacf65baa0 100644 --- a/julia-notebook/Dockerfile +++ b/images/julia-notebook/Dockerfile @@ -1,8 +1,9 @@ # Copyright (c) Jupyter Development Team. # Distributed under the terms of the Modified BSD License. +ARG REGISTRY=quay.io ARG OWNER=jupyter -ARG BASE_CONTAINER=$OWNER/minimal-notebook -FROM $BASE_CONTAINER +ARG BASE_IMAGE=$REGISTRY/$OWNER/minimal-notebook +FROM $BASE_IMAGE LABEL maintainer="Jupyter Project <jupyter@googlegroups.com>" @@ -18,7 +19,7 @@ ENV JULIA_DEPOT_PATH=/opt/julia \ JULIA_PKGDIR=/opt/julia # Setup Julia -RUN /opt/setup-scripts/setup-julia.bash +RUN /opt/setup-scripts/setup_julia.py USER ${NB_UID} diff --git a/julia-notebook/README.md b/images/julia-notebook/README.md similarity index 82% rename from julia-notebook/README.md rename to images/julia-notebook/README.md index dfca913b14..4672067065 100644 --- a/julia-notebook/README.md +++ b/images/julia-notebook/README.md @@ -1,10 +1,12 @@ # Jupyter Notebook Julia Stack +> **Images hosted on Docker Hub are no longer updated. Please, use [quay.io image](https://quay.io/repository/jupyter/julia-notebook)** + [![docker pulls](https://img.shields.io/docker/pulls/jupyter/julia-notebook.svg)](https://hub.docker.com/r/jupyter/julia-notebook/) [![docker stars](https://img.shields.io/docker/stars/jupyter/julia-notebook.svg)](https://hub.docker.com/r/jupyter/julia-notebook/) [![image size](https://img.shields.io/docker/image-size/jupyter/julia-notebook/latest)](https://hub.docker.com/r/jupyter/julia-notebook/ "jupyter/julia-notebook image size") -GitHub Actions in the <https://github.com/jupyter/docker-stacks> project builds and pushes this image to Docker Hub. +GitHub Actions in the <https://github.com/jupyter/docker-stacks> project builds and pushes this image to the Registry. Please visit the project documentation site for help to use and contribute to this image and others. diff --git a/minimal-notebook/.dockerignore b/images/minimal-notebook/.dockerignore similarity index 100% rename from minimal-notebook/.dockerignore rename to images/minimal-notebook/.dockerignore diff --git a/minimal-notebook/Dockerfile b/images/minimal-notebook/Dockerfile similarity index 80% rename from minimal-notebook/Dockerfile rename to images/minimal-notebook/Dockerfile index a998d2a109..e1d7dc70e6 100644 --- a/minimal-notebook/Dockerfile +++ b/images/minimal-notebook/Dockerfile @@ -1,8 +1,9 @@ # Copyright (c) Jupyter Development Team. # Distributed under the terms of the Modified BSD License. +ARG REGISTRY=quay.io ARG OWNER=jupyter -ARG BASE_CONTAINER=$OWNER/base-notebook -FROM $BASE_CONTAINER +ARG BASE_IMAGE=$REGISTRY/$OWNER/base-notebook +FROM $BASE_IMAGE LABEL maintainer="Jupyter Project <jupyter@googlegroups.com>" @@ -12,10 +13,11 @@ SHELL ["/bin/bash", "-o", "pipefail", "-c"] USER root -# Install all OS dependencies for fully functional notebook server +# Install all OS dependencies for a fully functional Server RUN apt-get update --yes && \ apt-get install --yes --no-install-recommends \ # Common useful utilities + curl \ git \ nano-tiny \ tzdata \ @@ -23,10 +25,10 @@ RUN apt-get update --yes && \ vim-tiny \ # git-over-ssh openssh-client \ - # less is needed to run help in R + # `less` is needed to run help in R # see: https://github.com/jupyter/docker-stacks/issues/1588 less \ - # nbconvert dependencies + # `nbconvert` dependencies # https://nbconvert.readthedocs.io/en/latest/install.html#installing-tex texlive-xetex \ texlive-fonts-recommended \ @@ -41,7 +43,7 @@ RUN update-alternatives --install /usr/bin/nano nano /bin/nano-tiny 10 # Switch back to jovyan to avoid accidental container runs as root USER ${NB_UID} -# Add R mimetype option to specify how the plot returns from R to the browser +# Add an R mimetype option to specify how the plot returns from R to the browser COPY --chown=${NB_UID}:${NB_GID} Rprofile.site /opt/conda/lib/R/etc/ # Add setup scripts that may be used by downstream images or inherited images diff --git a/minimal-notebook/README.md b/images/minimal-notebook/README.md similarity index 82% rename from minimal-notebook/README.md rename to images/minimal-notebook/README.md index 40697fa1cd..0d0f44f6dc 100644 --- a/minimal-notebook/README.md +++ b/images/minimal-notebook/README.md @@ -1,10 +1,12 @@ # Minimal Jupyter Notebook Stack +> **Images hosted on Docker Hub are no longer updated. Please, use [quay.io image](https://quay.io/repository/jupyter/minimal-notebook)** + [![docker pulls](https://img.shields.io/docker/pulls/jupyter/minimal-notebook.svg)](https://hub.docker.com/r/jupyter/minimal-notebook/) [![docker stars](https://img.shields.io/docker/stars/jupyter/minimal-notebook.svg)](https://hub.docker.com/r/jupyter/minimal-notebook/) [![image size](https://img.shields.io/docker/image-size/jupyter/minimal-notebook/latest)](https://hub.docker.com/r/jupyter/minimal-notebook/ "jupyter/minimal-notebook image size") -GitHub Actions in the <https://github.com/jupyter/docker-stacks> project builds and pushes this image to Docker Hub. +GitHub Actions in the <https://github.com/jupyter/docker-stacks> project builds and pushes this image to the Registry. Please visit the project documentation site for help to use and contribute to this image and others. diff --git a/minimal-notebook/Rprofile.site b/images/minimal-notebook/Rprofile.site similarity index 100% rename from minimal-notebook/Rprofile.site rename to images/minimal-notebook/Rprofile.site diff --git a/images/minimal-notebook/setup-scripts/activate_notebook_custom_env.py b/images/minimal-notebook/setup-scripts/activate_notebook_custom_env.py new file mode 100755 index 0000000000..4d5da9bcd3 --- /dev/null +++ b/images/minimal-notebook/setup-scripts/activate_notebook_custom_env.py @@ -0,0 +1,24 @@ +#!/usr/bin/env python3 +# Copyright (c) Jupyter Development Team. +# Distributed under the terms of the Modified BSD License. +import json +import os +import sys +from pathlib import Path + +env_name = sys.argv[1] +CONDA_DIR = os.environ["CONDA_DIR"] + +file = Path.home() / f".local/share/jupyter/kernels/{env_name}/kernel.json" +content = json.loads(file.read_text()) +content["env"] = { + "XML_CATALOG_FILES": "", + "PATH": f"{CONDA_DIR}/envs/{env_name}/bin:$PATH", + "CONDA_PREFIX": f"{CONDA_DIR}/envs/{env_name}", + "CONDA_PROMPT_MODIFIER": f"({env_name}) ", + "CONDA_SHLVL": "2", + "CONDA_DEFAULT_ENV": env_name, + "CONDA_PREFIX_1": CONDA_DIR, +} + +file.write_text(json.dumps(content, indent=1)) diff --git a/images/minimal-notebook/setup-scripts/setup-julia-packages.bash b/images/minimal-notebook/setup-scripts/setup-julia-packages.bash new file mode 100755 index 0000000000..14a77407d4 --- /dev/null +++ b/images/minimal-notebook/setup-scripts/setup-julia-packages.bash @@ -0,0 +1,55 @@ +#!/bin/bash +set -exuo pipefail +# Requirements: +# - Run as a non-root user +# - The JULIA_PKGDIR environment variable is set +# - Julia is already set up, with the setup_julia.py command + + +# If we don't specify what CPUs the precompilation should be done for, it's +# *only* done for the target of the host doing the compilation. When the +# container runs on a host that's the same architecture, but a *different* +# generation of CPU than what the build host was, the precompilation is useless +# and Julia takes a long long time to start up. This specific multitarget comes +# from https://github.com/JuliaCI/julia-buildkite/blob/9f354745a1f2bf31a5952462aa1ff2d869507cb8/utilities/build_envs.sh#L20-L82, +# and may need to be updated as new CPU generations come out. +# If the architecture the container runs on is different, +# precompilation may still have to be re-done on first startup - but this +# *should* catch most of the issues. See +# https://github.com/jupyter/docker-stacks/issues/2015 for more information +if [ "$(uname -m)" == "x86_64" ]; then + # See https://github.com/JuliaCI/julia-buildkite/blob/9f354745a1f2bf31a5952462aa1ff2d869507cb8/utilities/build_envs.sh#L23 + # for an explanation of these options + export JULIA_CPU_TARGET="generic;sandybridge,-xsaveopt,clone_all;haswell,-rdrnd,base(1);x86-64-v4,-rdrnd,base(1)" +elif [ "$(uname -m)" == "aarch64" ]; then + # See https://github.com/JuliaCI/julia-buildkite/blob/9f354745a1f2bf31a5952462aa1ff2d869507cb8/utilities/build_envs.sh#L56 + # for an explanation of these options + export JULIA_CPU_TARGET="generic;cortex-a57;thunderx2t99;carmel,clone_all;apple-m1,base(3);neoverse-512tvb,base(3)" +fi + +# Install base Julia packages +julia -e ' +import Pkg; +Pkg.update(); +Pkg.add([ + "HDF5", + "IJulia", + "Pluto" +]); +Pkg.precompile(); +' + +# Move the kernelspec out of ${HOME} to the system share location. +# Avoids problems with runtime UID change not taking effect properly +# on the .local folder in the jovyan home dir. +mv "${HOME}/.local/share/jupyter/kernels/julia"* "${CONDA_DIR}/share/jupyter/kernels/" +chmod -R go+rx "${CONDA_DIR}/share/jupyter" +rm -rf "${HOME}/.local" +fix-permissions "${JULIA_PKGDIR}" "${CONDA_DIR}/share/jupyter" + +# Install jupyter-pluto-proxy to get Pluto to work on JupyterHub +mamba install --yes \ + 'jupyter-pluto-proxy' && \ + mamba clean --all -f -y && \ + fix-permissions "${CONDA_DIR}" && \ + fix-permissions "/home/${NB_USER}" diff --git a/images/minimal-notebook/setup-scripts/setup_julia.py b/images/minimal-notebook/setup-scripts/setup_julia.py new file mode 100755 index 0000000000..114e64c00f --- /dev/null +++ b/images/minimal-notebook/setup-scripts/setup_julia.py @@ -0,0 +1,97 @@ +#!/usr/bin/env python3 +# Copyright (c) Jupyter Development Team. +# Distributed under the terms of the Modified BSD License. + +# Requirements: +# - Run as the root user +# - The JULIA_PKGDIR environment variable is set + +import logging +import os +import platform +import shutil +import subprocess +from pathlib import Path + +import requests + +LOGGER = logging.getLogger(__name__) + + +def unify_aarch64(platform: str) -> str: + """ + Renames arm64->aarch64 to support local builds on on aarch64 Macs + """ + return { + "aarch64": "aarch64", + "arm64": "aarch64", + "x86_64": "x86_64", + }[platform] + + +def get_latest_julia_url() -> tuple[str, str]: + """ + Get the last stable version of Julia + Based on: https://github.com/JuliaLang/www.julialang.org/issues/878#issuecomment-749234813 + """ + LOGGER.info("Downloading Julia versions information") + versions = requests.get( + "https://julialang-s3.julialang.org/bin/versions.json" + ).json() + stable_versions = {k: v for k, v in versions.items() if v["stable"]} + # Compare versions semantically + latest_stable_version = max( + stable_versions, key=lambda ver: [int(sub_ver) for sub_ver in ver.split(".")] + ) + latest_version_files = stable_versions[latest_stable_version]["files"] + triplet = unify_aarch64(platform.machine()) + "-linux-gnu" + file_info = [vf for vf in latest_version_files if vf["triplet"] == triplet][0] + LOGGER.info(f"Latest version: {file_info['version']} url: {file_info['url']}") + return file_info["url"], file_info["version"] + + +def download_julia(julia_url: str) -> None: + """ + Downloads and unpacks julia + The resulting julia directory is "/opt/julia-VERSION/" + """ + LOGGER.info("Downloading and unpacking Julia") + tmp_file = Path("/tmp/julia.tar.gz") + subprocess.check_call( + ["curl", "--progress-bar", "--location", "--output", tmp_file, julia_url] + ) + shutil.unpack_archive(tmp_file, "/opt/") + tmp_file.unlink() + + +def configure_julia(julia_version: str) -> None: + """ + Creates /usr/local/bin/julia symlink + Make Julia aware of conda libraries + Creates a directory for Julia user libraries + """ + LOGGER.info("Configuring Julia") + # Link Julia installed version to /usr/local/bin, so julia launches it + subprocess.check_call( + ["ln", "-fs", f"/opt/julia-{julia_version}/bin/julia", "/usr/local/bin/julia"] + ) + + # Tell Julia where conda libraries are + Path("/etc/julia").mkdir() + Path("/etc/julia/juliarc.jl").write_text( + f'push!(Libdl.DL_LOAD_PATH, "{os.environ["CONDA_DIR"]}/lib")\n' + ) + + # Create JULIA_PKGDIR, where user libraries are installed + JULIA_PKGDIR = Path(os.environ["JULIA_PKGDIR"]) + JULIA_PKGDIR.mkdir() + subprocess.check_call(["chown", os.environ["NB_USER"], JULIA_PKGDIR]) + subprocess.check_call(["fix-permissions", JULIA_PKGDIR]) + + +if __name__ == "__main__": + logging.basicConfig(level=logging.INFO) + + julia_url, julia_version = get_latest_julia_url() + download_julia(julia_url=julia_url) + configure_julia(julia_version=julia_version) diff --git a/pyspark-notebook/.dockerignore b/images/pyspark-notebook/.dockerignore similarity index 100% rename from pyspark-notebook/.dockerignore rename to images/pyspark-notebook/.dockerignore diff --git a/images/pyspark-notebook/Dockerfile b/images/pyspark-notebook/Dockerfile new file mode 100644 index 0000000000..71b1c81ce6 --- /dev/null +++ b/images/pyspark-notebook/Dockerfile @@ -0,0 +1,73 @@ +# Copyright (c) Jupyter Development Team. +# Distributed under the terms of the Modified BSD License. +ARG REGISTRY=quay.io +ARG OWNER=jupyter +ARG BASE_IMAGE=$REGISTRY/$OWNER/scipy-notebook +FROM $BASE_IMAGE + +LABEL maintainer="Jupyter Project <jupyter@googlegroups.com>" + +# Fix: https://github.com/hadolint/hadolint/wiki/DL4006 +# Fix: https://github.com/koalaman/shellcheck/wiki/SC3014 +SHELL ["/bin/bash", "-o", "pipefail", "-c"] + +USER root + +# Spark dependencies +# Default values can be overridden at build time +# (ARGS are in lowercase to distinguish them from ENV) +ARG openjdk_version="17" + +RUN apt-get update --yes && \ + apt-get install --yes --no-install-recommends \ + "openjdk-${openjdk_version}-jre-headless" \ + ca-certificates-java && \ + apt-get clean && rm -rf /var/lib/apt/lists/* + +# If spark_version is not set, latest stable Spark will be installed +ARG spark_version +ARG hadoop_version="3" +# If scala_version is not set, Spark without Scala will be installed +ARG scala_version +# URL to use for Spark downloads +# You need to use https://archive.apache.org/dist/spark/ website if you want to download old Spark versions +# But it seems to be slower, that's why we use the recommended site for download +ARG spark_download_url="https://dlcdn.apache.org/spark/" + +ENV SPARK_HOME=/usr/local/spark +ENV PATH="${PATH}:${SPARK_HOME}/bin" +ENV SPARK_OPTS="--driver-java-options=-Xms1024M --driver-java-options=-Xmx4096M --driver-java-options=-Dlog4j.logLevel=info" + +COPY setup_spark.py /opt/setup-scripts/ + +# Setup Spark +RUN /opt/setup-scripts/setup_spark.py \ + --spark-version="${spark_version}" \ + --hadoop-version="${hadoop_version}" \ + --scala-version="${scala_version}" \ + --spark-download-url="${spark_download_url}" + +# Configure IPython system-wide +COPY ipython_kernel_config.py "/etc/ipython/" +RUN fix-permissions "/etc/ipython/" + +USER ${NB_UID} + +# Install pyarrow +# NOTE: It's important to ensure compatibility between Pandas versions. +# The pandas version in this Dockerfile should match the version +# on which the Pandas API for Spark is built. +# To find the right version: +# 1. Check out the Spark branch you are on: <https://github.com/apache/spark> +# 2. Find the pandas version in the file `dev/infra/Dockerfile`. +RUN mamba install --yes \ + 'grpcio-status' \ + 'grpcio' \ + 'pandas=2.0.3' \ + 'pyarrow' && \ + mamba clean --all -f -y && \ + fix-permissions "${CONDA_DIR}" && \ + fix-permissions "/home/${NB_USER}" + +WORKDIR "${HOME}" +EXPOSE 4040 diff --git a/pyspark-notebook/README.md b/images/pyspark-notebook/README.md similarity index 84% rename from pyspark-notebook/README.md rename to images/pyspark-notebook/README.md index 1be0434393..c1c5e9aac2 100644 --- a/pyspark-notebook/README.md +++ b/images/pyspark-notebook/README.md @@ -1,10 +1,12 @@ # Jupyter Notebook Python, Spark Stack +> **Images hosted on Docker Hub are no longer updated. Please, use [quay.io image](https://quay.io/repository/jupyter/pyspark-notebook)** + [![docker pulls](https://img.shields.io/docker/pulls/jupyter/pyspark-notebook.svg)](https://hub.docker.com/r/jupyter/pyspark-notebook/) [![docker stars](https://img.shields.io/docker/stars/jupyter/pyspark-notebook.svg)](https://hub.docker.com/r/jupyter/pyspark-notebook/) [![image size](https://img.shields.io/docker/image-size/jupyter/pyspark-notebook/latest)](https://hub.docker.com/r/jupyter/pyspark-notebook/ "jupyter/pyspark-notebook image size") -GitHub Actions in the <https://github.com/jupyter/docker-stacks> project builds and pushes this image to Docker Hub. +GitHub Actions in the <https://github.com/jupyter/docker-stacks> project builds and pushes this image to the Registry. Please visit the project documentation site for help to use and contribute to this image and others. diff --git a/pyspark-notebook/ipython_kernel_config.py b/images/pyspark-notebook/ipython_kernel_config.py similarity index 93% rename from pyspark-notebook/ipython_kernel_config.py rename to images/pyspark-notebook/ipython_kernel_config.py index f3efbe19c8..921e6fa8f9 100644 --- a/pyspark-notebook/ipython_kernel_config.py +++ b/images/pyspark-notebook/ipython_kernel_config.py @@ -7,8 +7,7 @@ # Logs are particularly verbose with Spark, that is why we turn them off through this flag. # <https://github.com/jupyter/docker-stacks/issues/1423> -# Attempt to capture and forward low-level output, e.g. produced by Extension -# libraries. -# Default: True +# Attempt to capture and forward low-level output, e.g. produced by Extension libraries. +# Default: True # type:ignore c.IPKernelApp.capture_fd_output = False # noqa: F821 diff --git a/images/pyspark-notebook/setup_spark.py b/images/pyspark-notebook/setup_spark.py new file mode 100755 index 0000000000..a494b8322a --- /dev/null +++ b/images/pyspark-notebook/setup_spark.py @@ -0,0 +1,123 @@ +#!/usr/bin/env python3 +# Copyright (c) Jupyter Development Team. +# Distributed under the terms of the Modified BSD License. + +# Requirements: +# - Run as the root user +# - Required env variable: SPARK_HOME + +import argparse +import logging +import os +import subprocess +from pathlib import Path + +import requests +from bs4 import BeautifulSoup + +LOGGER = logging.getLogger(__name__) + + +def get_all_refs(url: str) -> list[str]: + """ + Get all the references for a given webpage + """ + resp = requests.get(url) + soup = BeautifulSoup(resp.text, "html.parser") + return [a["href"] for a in soup.find_all("a", href=True)] + + +def get_latest_spark_version() -> str: + """ + Returns the last stable version of Spark using spark archive + """ + LOGGER.info("Downloading Spark versions information") + all_refs = get_all_refs("https://archive.apache.org/dist/spark/") + stable_versions = [ + ref.removeprefix("spark-").removesuffix("/") + for ref in all_refs + if ref.startswith("spark-") and "incubating" not in ref and "preview" not in ref + ] + # Compare versions semantically + latest_version = max( + stable_versions, key=lambda ver: [int(sub_ver) for sub_ver in ver.split(".")] + ) + LOGGER.info(f"Latest version: {latest_version}") + return latest_version + + +def download_spark( + spark_version: str, + hadoop_version: str, + scala_version: str, + spark_download_url: Path, +) -> str: + """ + Downloads and unpacks spark + The resulting spark directory name is returned + """ + LOGGER.info("Downloading and unpacking Spark") + spark_dir_name = f"spark-{spark_version}-bin-hadoop{hadoop_version}" + if scala_version: + spark_dir_name += f"-scala{scala_version}" + LOGGER.info(f"Spark directory name: {spark_dir_name}") + spark_url = spark_download_url / f"spark-{spark_version}" / f"{spark_dir_name}.tgz" + + tmp_file = Path("/tmp/spark.tar.gz") + subprocess.check_call( + ["curl", "--progress-bar", "--location", "--output", tmp_file, spark_url] + ) + subprocess.check_call( + [ + "tar", + "xzf", + tmp_file, + "-C", + "/usr/local", + "--owner", + "root", + "--group", + "root", + "--no-same-owner", + ] + ) + tmp_file.unlink() + return spark_dir_name + + +def configure_spark(spark_dir_name: str, spark_home: Path) -> None: + """ + Creates a ${SPARK_HOME} symlink to a versioned spark directory + Creates a 10spark-config.sh symlink to source PYTHONPATH automatically + """ + LOGGER.info("Configuring Spark") + subprocess.check_call(["ln", "-s", f"/usr/local/{spark_dir_name}", spark_home]) + + # Add a link in the before_notebook hook in order to source PYTHONPATH automatically + CONFIG_SCRIPT = "/usr/local/bin/before-notebook.d/10spark-config.sh" + subprocess.check_call( + ["ln", "-s", spark_home / "sbin/spark-config.sh", CONFIG_SCRIPT] + ) + + +if __name__ == "__main__": + logging.basicConfig(level=logging.INFO) + + arg_parser = argparse.ArgumentParser() + arg_parser.add_argument("--spark-version", required=True) + arg_parser.add_argument("--hadoop-version", required=True) + arg_parser.add_argument("--scala-version", required=True) + arg_parser.add_argument("--spark-download-url", type=Path, required=True) + args = arg_parser.parse_args() + + args.spark_version = args.spark_version or get_latest_spark_version() + + spark_dir_name = download_spark( + spark_version=args.spark_version, + hadoop_version=args.hadoop_version, + scala_version=args.scala_version, + spark_download_url=args.spark_download_url, + ) + configure_spark( + spark_dir_name=spark_dir_name, spark_home=Path(os.environ["SPARK_HOME"]) + ) diff --git a/r-notebook/.dockerignore b/images/pytorch-notebook/.dockerignore similarity index 100% rename from r-notebook/.dockerignore rename to images/pytorch-notebook/.dockerignore diff --git a/tensorflow-notebook/Dockerfile b/images/pytorch-notebook/Dockerfile similarity index 60% rename from tensorflow-notebook/Dockerfile rename to images/pytorch-notebook/Dockerfile index 24a46d96fe..d5c359834b 100644 --- a/tensorflow-notebook/Dockerfile +++ b/images/pytorch-notebook/Dockerfile @@ -1,8 +1,9 @@ # Copyright (c) Jupyter Development Team. # Distributed under the terms of the Modified BSD License. +ARG REGISTRY=quay.io ARG OWNER=jupyter -ARG BASE_CONTAINER=$OWNER/scipy-notebook -FROM $BASE_CONTAINER +ARG BASE_IMAGE=$REGISTRY/$OWNER/scipy-notebook +FROM $BASE_IMAGE LABEL maintainer="Jupyter Project <jupyter@googlegroups.com>" @@ -10,8 +11,11 @@ LABEL maintainer="Jupyter Project <jupyter@googlegroups.com>" # Fix: https://github.com/koalaman/shellcheck/wiki/SC3014 SHELL ["/bin/bash", "-o", "pipefail", "-c"] -# Install Tensorflow with pip +# Install PyTorch with pip (https://pytorch.org/get-started/locally/) # hadolint ignore=DL3013 -RUN pip install --no-cache-dir tensorflow && \ +RUN pip install --no-cache-dir --index-url 'https://download.pytorch.org/whl/cpu' \ + 'torch' \ + 'torchaudio' \ + 'torchvision' && \ fix-permissions "${CONDA_DIR}" && \ fix-permissions "/home/${NB_USER}" diff --git a/images/pytorch-notebook/README.md b/images/pytorch-notebook/README.md new file mode 100644 index 0000000000..72cc65c194 --- /dev/null +++ b/images/pytorch-notebook/README.md @@ -0,0 +1,8 @@ +# Jupyter Notebook Deep Learning Stack + +GitHub Actions in the <https://github.com/jupyter/docker-stacks> project builds and pushes this image to the Registry. + +Please visit the project documentation site for help to use and contribute to this image and others. + +- [Jupyter Docker Stacks on ReadTheDocs](https://jupyter-docker-stacks.readthedocs.io/en/latest/index.html) +- [Selecting an Image :: Core Stacks :: jupyter/pytorch-notebook](https://jupyter-docker-stacks.readthedocs.io/en/latest/using/selecting.html#jupyter-pytorch-notebook) diff --git a/images/pytorch-notebook/cuda11/Dockerfile b/images/pytorch-notebook/cuda11/Dockerfile new file mode 100644 index 0000000000..d3af70f612 --- /dev/null +++ b/images/pytorch-notebook/cuda11/Dockerfile @@ -0,0 +1,30 @@ +# Copyright (c) Jupyter Development Team. +# Distributed under the terms of the Modified BSD License. +ARG REGISTRY=quay.io +ARG OWNER=jupyter +ARG BASE_IMAGE=$REGISTRY/$OWNER/scipy-notebook +FROM $BASE_IMAGE + +LABEL maintainer="Jupyter Project <jupyter@googlegroups.com>" + +# Fix: https://github.com/hadolint/hadolint/wiki/DL4006 +# Fix: https://github.com/koalaman/shellcheck/wiki/SC3014 +SHELL ["/bin/bash", "-o", "pipefail", "-c"] + +# Install PyTorch with pip (https://pytorch.org/get-started/locally/) +# hadolint ignore=DL3013 +RUN pip install --no-cache-dir --extra-index-url=https://pypi.nvidia.com --index-url 'https://download.pytorch.org/whl/cu118' \ + 'torch' \ + 'torchaudio' \ + 'torchvision' && \ + fix-permissions "${CONDA_DIR}" && \ + fix-permissions "/home/${NB_USER}" + +# https://docs.nvidia.com/datacenter/cloud-native/container-toolkit/latest/docker-specialized.html#dockerfiles +ENV NVIDIA_VISIBLE_DEVICES="all" \ + NVIDIA_DRIVER_CAPABILITIES="compute,utility" + +# Puts the nvidia-smi binary (system management interface) on path +# with associated library files to execute it +ENV PATH="${PATH}:/usr/local/nvidia/bin" \ + LD_LIBRARY_PATH="${LD_LIBRARY_PATH}:/usr/local/nvidia/lib64" diff --git a/images/pytorch-notebook/cuda12/Dockerfile b/images/pytorch-notebook/cuda12/Dockerfile new file mode 100644 index 0000000000..cdfd089a0c --- /dev/null +++ b/images/pytorch-notebook/cuda12/Dockerfile @@ -0,0 +1,30 @@ +# Copyright (c) Jupyter Development Team. +# Distributed under the terms of the Modified BSD License. +ARG REGISTRY=quay.io +ARG OWNER=jupyter +ARG BASE_IMAGE=$REGISTRY/$OWNER/scipy-notebook +FROM $BASE_IMAGE + +LABEL maintainer="Jupyter Project <jupyter@googlegroups.com>" + +# Fix: https://github.com/hadolint/hadolint/wiki/DL4006 +# Fix: https://github.com/koalaman/shellcheck/wiki/SC3014 +SHELL ["/bin/bash", "-o", "pipefail", "-c"] + +# Install PyTorch with pip (https://pytorch.org/get-started/locally/) +# hadolint ignore=DL3013 +RUN pip install --no-cache-dir --extra-index-url=https://pypi.nvidia.com --index-url 'https://download.pytorch.org/whl/cu121' \ + 'torch' \ + 'torchaudio' \ + 'torchvision' && \ + fix-permissions "${CONDA_DIR}" && \ + fix-permissions "/home/${NB_USER}" + +# https://docs.nvidia.com/datacenter/cloud-native/container-toolkit/latest/docker-specialized.html#dockerfiles +ENV NVIDIA_VISIBLE_DEVICES="all" \ + NVIDIA_DRIVER_CAPABILITIES="compute,utility" + +# Puts the nvidia-smi binary (system management interface) on path +# with associated library files to execute it +ENV PATH="${PATH}:/usr/local/nvidia/bin" \ + LD_LIBRARY_PATH="${LD_LIBRARY_PATH}:/usr/local/nvidia/lib64" diff --git a/scipy-notebook/.dockerignore b/images/r-notebook/.dockerignore similarity index 100% rename from scipy-notebook/.dockerignore rename to images/r-notebook/.dockerignore diff --git a/r-notebook/Dockerfile b/images/r-notebook/Dockerfile similarity index 93% rename from r-notebook/Dockerfile rename to images/r-notebook/Dockerfile index 579aa8f706..39cbebdd4a 100644 --- a/r-notebook/Dockerfile +++ b/images/r-notebook/Dockerfile @@ -1,8 +1,9 @@ # Copyright (c) Jupyter Development Team. # Distributed under the terms of the Modified BSD License. +ARG REGISTRY=quay.io ARG OWNER=jupyter -ARG BASE_CONTAINER=$OWNER/minimal-notebook -FROM $BASE_CONTAINER +ARG BASE_IMAGE=$REGISTRY/$OWNER/minimal-notebook +FROM $BASE_IMAGE LABEL maintainer="Jupyter Project <jupyter@googlegroups.com>" diff --git a/r-notebook/README.md b/images/r-notebook/README.md similarity index 82% rename from r-notebook/README.md rename to images/r-notebook/README.md index 045ed12339..4d5fc89415 100644 --- a/r-notebook/README.md +++ b/images/r-notebook/README.md @@ -1,10 +1,12 @@ # Jupyter Notebook R Stack +> **Images hosted on Docker Hub are no longer updated. Please, use [quay.io image](https://quay.io/repository/jupyter/r-notebook)** + [![docker pulls](https://img.shields.io/docker/pulls/jupyter/r-notebook.svg)](https://hub.docker.com/r/jupyter/r-notebook/) [![docker stars](https://img.shields.io/docker/stars/jupyter/r-notebook.svg)](https://hub.docker.com/r/jupyter/r-notebook/) [![image size](https://img.shields.io/docker/image-size/jupyter/r-notebook/latest)](https://hub.docker.com/r/jupyter/r-notebook/ "jupyter/r-notebook image size") -GitHub Actions in the <https://github.com/jupyter/docker-stacks> project builds and pushes this image to Docker Hub. +GitHub Actions in the <https://github.com/jupyter/docker-stacks> project builds and pushes this image to the Registry. Please visit the project documentation site for help to use and contribute to this image and others. diff --git a/tensorflow-notebook/.dockerignore b/images/scipy-notebook/.dockerignore similarity index 100% rename from tensorflow-notebook/.dockerignore rename to images/scipy-notebook/.dockerignore diff --git a/scipy-notebook/Dockerfile b/images/scipy-notebook/Dockerfile similarity index 79% rename from scipy-notebook/Dockerfile rename to images/scipy-notebook/Dockerfile index c2c3bdb595..2e667f3e15 100644 --- a/scipy-notebook/Dockerfile +++ b/images/scipy-notebook/Dockerfile @@ -1,8 +1,9 @@ # Copyright (c) Jupyter Development Team. # Distributed under the terms of the Modified BSD License. +ARG REGISTRY=quay.io ARG OWNER=jupyter -ARG BASE_CONTAINER=$OWNER/minimal-notebook -FROM $BASE_CONTAINER +ARG BASE_IMAGE=$REGISTRY/$OWNER/minimal-notebook +FROM $BASE_IMAGE LABEL maintainer="Jupyter Project <jupyter@googlegroups.com>" @@ -37,7 +38,7 @@ RUN mamba install --yes \ 'dask' \ 'dill' \ 'h5py' \ - 'ipympl'\ + 'ipympl' \ 'ipywidgets' \ 'jupyterlab-git' \ 'matplotlib-base' \ @@ -55,23 +56,21 @@ RUN mamba install --yes \ 'sqlalchemy' \ 'statsmodels' \ 'sympy' \ - 'widgetsnbextension'\ + 'widgetsnbextension' \ 'xlrd' && \ mamba clean --all -f -y && \ fix-permissions "${CONDA_DIR}" && \ fix-permissions "/home/${NB_USER}" -# Install facets which does not have a pip or conda package at the moment +# Install facets package which does not have a `pip` or `conda-forge` package at the moment WORKDIR /tmp -RUN git clone https://github.com/PAIR-code/facets.git && \ - jupyter nbextension install facets/facets-dist/ --sys-prefix && \ +RUN git clone https://github.com/PAIR-code/facets && \ + jupyter nbclassic-extension install facets/facets-dist/ --sys-prefix && \ rm -rf /tmp/facets && \ fix-permissions "${CONDA_DIR}" && \ fix-permissions "/home/${NB_USER}" -# Import matplotlib the first time to build the font cache. -ENV XDG_CACHE_HOME="/home/${NB_USER}/.cache/" - +# Import matplotlib the first time to build the font cache RUN MPLBACKEND=Agg python -c "import matplotlib.pyplot" && \ fix-permissions "/home/${NB_USER}" diff --git a/scipy-notebook/README.md b/images/scipy-notebook/README.md similarity index 82% rename from scipy-notebook/README.md rename to images/scipy-notebook/README.md index e34693d1e6..68d56ec902 100644 --- a/scipy-notebook/README.md +++ b/images/scipy-notebook/README.md @@ -1,10 +1,12 @@ # Jupyter Notebook Scientific Python Stack +> **Images hosted on Docker Hub are no longer updated. Please, use [quay.io image](https://quay.io/repository/jupyter/scipy-notebook)** + [![docker pulls](https://img.shields.io/docker/pulls/jupyter/scipy-notebook.svg)](https://hub.docker.com/r/jupyter/scipy-notebook/) [![docker stars](https://img.shields.io/docker/stars/jupyter/scipy-notebook.svg)](https://hub.docker.com/r/jupyter/scipy-notebook/) [![image size](https://img.shields.io/docker/image-size/jupyter/scipy-notebook/latest)](https://hub.docker.com/r/jupyter/scipy-notebook/ "jupyter/scipy-notebook image size") -GitHub Actions in the <https://github.com/jupyter/docker-stacks> project builds and pushes this image to Docker Hub. +GitHub Actions in the <https://github.com/jupyter/docker-stacks> project builds and pushes this image to the Registry. Please visit the project documentation site for help to use and contribute to this image and others. diff --git a/images/tensorflow-notebook/.dockerignore b/images/tensorflow-notebook/.dockerignore new file mode 100644 index 0000000000..9dea340f35 --- /dev/null +++ b/images/tensorflow-notebook/.dockerignore @@ -0,0 +1,2 @@ +# Documentation +README.md diff --git a/images/tensorflow-notebook/Dockerfile b/images/tensorflow-notebook/Dockerfile new file mode 100644 index 0000000000..a433b2fa7a --- /dev/null +++ b/images/tensorflow-notebook/Dockerfile @@ -0,0 +1,22 @@ +# Copyright (c) Jupyter Development Team. +# Distributed under the terms of the Modified BSD License. +ARG REGISTRY=quay.io +ARG OWNER=jupyter +ARG BASE_IMAGE=$REGISTRY/$OWNER/scipy-notebook +FROM $BASE_IMAGE + +LABEL maintainer="Jupyter Project <jupyter@googlegroups.com>" + +# Fix: https://github.com/hadolint/hadolint/wiki/DL4006 +# Fix: https://github.com/koalaman/shellcheck/wiki/SC3014 +SHELL ["/bin/bash", "-o", "pipefail", "-c"] + +# Install tensorflow with pip, on x86_64 tensorflow-cpu +RUN [[ $(uname -m) = x86_64 ]] && TF_POSTFIX="-cpu" || TF_POSTFIX="" && \ + pip install --no-cache-dir \ + "jupyter-server-proxy" \ + "tensorflow${TF_POSTFIX}" && \ + fix-permissions "${CONDA_DIR}" && \ + fix-permissions "/home/${NB_USER}" + +COPY --chown="${NB_UID}:${NB_GID}" cuda/20tensorboard-proxy-env.sh /usr/local/bin/before-notebook.d/ diff --git a/tensorflow-notebook/README.md b/images/tensorflow-notebook/README.md similarity index 84% rename from tensorflow-notebook/README.md rename to images/tensorflow-notebook/README.md index 5f896fd6e8..41e7e2b234 100644 --- a/tensorflow-notebook/README.md +++ b/images/tensorflow-notebook/README.md @@ -1,10 +1,12 @@ # Jupyter Notebook Deep Learning Stack +> **Images hosted on Docker Hub are no longer updated. Please, use [quay.io image](https://quay.io/repository/jupyter/tensorflow-notebook)** + [![docker pulls](https://img.shields.io/docker/pulls/jupyter/tensorflow-notebook.svg)](https://hub.docker.com/r/jupyter/tensorflow-notebook/) [![docker stars](https://img.shields.io/docker/stars/jupyter/tensorflow-notebook.svg)](https://hub.docker.com/r/jupyter/tensorflow-notebook/) [![image size](https://img.shields.io/docker/image-size/jupyter/tensorflow-notebook/latest)](https://hub.docker.com/r/jupyter/tensorflow-notebook/ "jupyter/tensorflow-notebook image size") -GitHub Actions in the <https://github.com/jupyter/docker-stacks> project builds and pushes this image to Docker Hub. +GitHub Actions in the <https://github.com/jupyter/docker-stacks> project builds and pushes this image to the Registry. Please visit the project documentation site for help to use and contribute to this image and others. diff --git a/images/tensorflow-notebook/cuda/20tensorboard-proxy-env.sh b/images/tensorflow-notebook/cuda/20tensorboard-proxy-env.sh new file mode 100644 index 0000000000..f31c7488dd --- /dev/null +++ b/images/tensorflow-notebook/cuda/20tensorboard-proxy-env.sh @@ -0,0 +1,10 @@ +#!/bin/bash +# Copyright (c) Jupyter Development Team. +# Distributed under the terms of the Modified BSD License. + +set -e + +# Initialize the TENSORBOARD_PROXY_URL with the appropriate path +# to use jupyter-server-proxy. + +export TENSORBOARD_PROXY_URL="${JUPYTERHUB_SERVICE_PREFIX:-/}proxy/%PORT%/" diff --git a/images/tensorflow-notebook/cuda/Dockerfile b/images/tensorflow-notebook/cuda/Dockerfile new file mode 100644 index 0000000000..5018dee43f --- /dev/null +++ b/images/tensorflow-notebook/cuda/Dockerfile @@ -0,0 +1,36 @@ +# Copyright (c) Jupyter Development Team. +# Distributed under the terms of the Modified BSD License. +ARG REGISTRY=quay.io +ARG OWNER=jupyter +ARG BASE_IMAGE=$REGISTRY/$OWNER/scipy-notebook +FROM $BASE_IMAGE + +LABEL maintainer="Jupyter Project <jupyter@googlegroups.com>" + +# Fix: https://github.com/hadolint/hadolint/wiki/DL4006 +# Fix: https://github.com/koalaman/shellcheck/wiki/SC3014 +SHELL ["/bin/bash", "-o", "pipefail", "-c"] + +# Install TensorFlow, CUDA and cuDNN with pip +RUN pip install --no-cache-dir \ + "jupyter-server-proxy" \ + "tensorflow[and-cuda]" && \ + fix-permissions "${CONDA_DIR}" && \ + fix-permissions "/home/${NB_USER}" + +COPY --chown="${NB_UID}:${NB_GID}" 20tensorboard-proxy-env.sh /usr/local/bin/before-notebook.d/ + +# workaround for https://github.com/tensorflow/tensorflow/issues/63362 +RUN mkdir -p "${CONDA_DIR}/etc/conda/activate.d/" && \ + fix-permissions "${CONDA_DIR}" + +COPY --chown="${NB_UID}:${NB_GID}" nvidia-lib-dirs.sh "${CONDA_DIR}/etc/conda/activate.d/" + +# https://docs.nvidia.com/datacenter/cloud-native/container-toolkit/latest/docker-specialized.html#dockerfiles +ENV NVIDIA_VISIBLE_DEVICES="all" \ + NVIDIA_DRIVER_CAPABILITIES="compute,utility" + +# Puts the nvidia-smi binary (system management interface) on path +# with associated library files to execute it +ENV PATH="${PATH}:/usr/local/nvidia/bin" \ + LD_LIBRARY_PATH="${LD_LIBRARY_PATH}:/usr/local/nvidia/lib64" diff --git a/images/tensorflow-notebook/cuda/nvidia-lib-dirs.sh b/images/tensorflow-notebook/cuda/nvidia-lib-dirs.sh new file mode 100644 index 0000000000..1927bd06ed --- /dev/null +++ b/images/tensorflow-notebook/cuda/nvidia-lib-dirs.sh @@ -0,0 +1,9 @@ +#!/bin/bash +# Copyright (c) Jupyter Development Team. +# Distributed under the terms of the Modified BSD License. + +# This adds NVIDIA Python package libraries to the LD_LIBRARY_PATH. +# Workaround for https://github.com/tensorflow/tensorflow/issues/63362 +NVIDIA_DIR=$(dirname "$(python -c 'import nvidia;print(nvidia.__file__)')") +LD_LIBRARY_PATH=$(echo "${NVIDIA_DIR}"/*/lib/ | sed -r 's/\s+/:/g')${LD_LIBRARY_PATH:+:${LD_LIBRARY_PATH}} +export LD_LIBRARY_PATH diff --git a/make-helx-jupyter-stack.sh b/make-helx-jupyter-stack.sh new file mode 100644 index 0000000000..cd5beaec6e --- /dev/null +++ b/make-helx-jupyter-stack.sh @@ -0,0 +1,8 @@ +#!/bin/bash + +OWNER=containers.renci.org/helxplatform/jupyter/docker-stacks +DOCKER_BUILD_ARGS="--build-arg=NB_GID=0" +DOCKER_BUILD_ARGS+=" --build-arg=ROOT_CONTAINER=nvidia/cuda:12.2.2-cudnn8-runtime-ubuntu22.04" +export OWNER DOCKER_BUILD_ARGS + +make "$@" \ No newline at end of file diff --git a/minimal-notebook/setup-scripts/setup-julia-packages.bash b/minimal-notebook/setup-scripts/setup-julia-packages.bash deleted file mode 100755 index 0be05426d7..0000000000 --- a/minimal-notebook/setup-scripts/setup-julia-packages.bash +++ /dev/null @@ -1,25 +0,0 @@ -#!/bin/bash -set -exuo pipefail -# Requirements: -# - Run as non-root user -# - The JULIA_PKGDIR environment variable is set -# - Julia is already set up, with the setup-julia.bash command - -# Install base Julia packages -julia -e ' -import Pkg; -Pkg.update(); -Pkg.add([ - "HDF5", - "IJulia" -]); -Pkg.precompile(); -' - -# Move the kernelspec out to the system share location. Avoids -# problems with runtime UID change not taking effect properly on the -# .local folder in the jovyan home dir. move kernelspec out of home -mv "${HOME}/.local/share/jupyter/kernels/julia"* "${CONDA_DIR}/share/jupyter/kernels/" -chmod -R go+rx "${CONDA_DIR}/share/jupyter" -rm -rf "${HOME}/.local" -fix-permissions "${JULIA_PKGDIR}" "${CONDA_DIR}/share/jupyter" diff --git a/minimal-notebook/setup-scripts/setup-julia.bash b/minimal-notebook/setup-scripts/setup-julia.bash deleted file mode 100755 index 3aab076e03..0000000000 --- a/minimal-notebook/setup-scripts/setup-julia.bash +++ /dev/null @@ -1,39 +0,0 @@ -#!/bin/bash -set -exuo pipefail -# Requirements: -# - Run as the root user -# - The JULIA_PKGDIR environment variable is set - -# Default julia version to install if env var is not set -# Check https://julialang.org/downloads/ -JULIA_VERSION="${JULIA_VERSION:-1.9.1}" - -# Figure out what architecture we are installing in -JULIA_ARCH=$(uname -m) -JULIA_SHORT_ARCH="${JULIA_ARCH}" -if [ "${JULIA_SHORT_ARCH}" == "x86_64" ]; then - JULIA_SHORT_ARCH="x64" -fi - -# Figure out Julia Installer URL -JULIA_INSTALLER="julia-${JULIA_VERSION}-linux-${JULIA_ARCH}.tar.gz" -JULIA_MAJOR_MINOR=$(echo "${JULIA_VERSION}" | cut -d. -f 1,2) - -# Download and install Julia -cd /tmp -mkdir "/opt/julia-${JULIA_VERSION}" -wget --progress=dot:giga "https://julialang-s3.julialang.org/bin/linux/${JULIA_SHORT_ARCH}/${JULIA_MAJOR_MINOR}/${JULIA_INSTALLER}" -tar xzf "${JULIA_INSTALLER}" -C "/opt/julia-${JULIA_VERSION}" --strip-components=1 -rm "${JULIA_INSTALLER}" - -# Link Julia installed version to /usr/local/bin, so julia launches it -ln -fs /opt/julia-*/bin/julia /usr/local/bin/julia - -# Tell Julia where conda libraries are -mkdir -p /etc/julia -echo "push!(Libdl.DL_LOAD_PATH, \"${CONDA_DIR}/lib\")" >> /etc/julia/juliarc.jl - -# Create JULIA_PKGDIR, where user libraries are installed -mkdir "${JULIA_PKGDIR}" -chown "${NB_USER}" "${JULIA_PKGDIR}" -fix-permissions "${JULIA_PKGDIR}" diff --git a/mypy.ini b/mypy.ini index 4373ba9a7a..3e20bf5bd9 100644 --- a/mypy.ini +++ b/mypy.ini @@ -1,7 +1,7 @@ # Mypy is an optional static type checker for Python that aims to combine # the benefits of dynamic (or "duck") typing and static typing. # -# Documentation: http://www.mypy-lang.org +# Documentation: https://www.mypy-lang.org # Project: https://github.com/python/mypy # Config reference: https://mypy.readthedocs.io/en/stable/config_file.html # @@ -42,5 +42,5 @@ ignore_missing_imports = True [mypy-tensorflow.*] ignore_missing_imports = True -[mypy-urllib3.*] +[mypy-torch.*] ignore_missing_imports = True diff --git a/pyspark-notebook/Dockerfile b/pyspark-notebook/Dockerfile deleted file mode 100644 index ad98110371..0000000000 --- a/pyspark-notebook/Dockerfile +++ /dev/null @@ -1,77 +0,0 @@ -# Copyright (c) Jupyter Development Team. -# Distributed under the terms of the Modified BSD License. -ARG OWNER=jupyter -ARG BASE_CONTAINER=$OWNER/scipy-notebook -FROM $BASE_CONTAINER - -LABEL maintainer="Jupyter Project <jupyter@googlegroups.com>" - -# Fix: https://github.com/hadolint/hadolint/wiki/DL4006 -# Fix: https://github.com/koalaman/shellcheck/wiki/SC3014 -SHELL ["/bin/bash", "-o", "pipefail", "-c"] - -USER root - -# Spark dependencies -# Default values can be overridden at build time -# (ARGS are in lower case to distinguish them from ENV) -ARG spark_version="3.4.1" -ARG hadoop_version="3" -ARG scala_version -ARG spark_checksum="5a21295b4c3d1d3f8fc85375c711c7c23e3eeb3ec9ea91778f149d8d321e3905e2f44cf19c69a28df693cffd536f7316706c78932e7e148d224424150f18b2c5" -ARG openjdk_version="17" - -ENV APACHE_SPARK_VERSION="${spark_version}" \ - HADOOP_VERSION="${hadoop_version}" - -RUN apt-get update --yes && \ - apt-get install --yes --no-install-recommends \ - "openjdk-${openjdk_version}-jre-headless" \ - ca-certificates-java && \ - apt-get clean && rm -rf /var/lib/apt/lists/* - -# Spark installation -WORKDIR /tmp - -# You need to use https://archive.apache.org/dist/ website if you want to download old Spark versions -# But it seems to be slower, that's why we use recommended site for download -RUN if [ -z "${scala_version}" ]; then \ - wget --progress=dot:giga -O "spark.tgz" "https://dlcdn.apache.org/spark/spark-${APACHE_SPARK_VERSION}/spark-${APACHE_SPARK_VERSION}-bin-hadoop${HADOOP_VERSION}.tgz"; \ - else \ - wget --progress=dot:giga -O "spark.tgz" "https://dlcdn.apache.org/spark/spark-${APACHE_SPARK_VERSION}/spark-${APACHE_SPARK_VERSION}-bin-hadoop${HADOOP_VERSION}-scala${scala_version}.tgz"; \ - fi && \ - echo "${spark_checksum} *spark.tgz" | sha512sum -c - && \ - tar xzf "spark.tgz" -C /usr/local --owner root --group root --no-same-owner && \ - rm "spark.tgz" - -# Configure Spark -ENV SPARK_HOME=/usr/local/spark -ENV SPARK_OPTS="--driver-java-options=-Xms1024M --driver-java-options=-Xmx4096M --driver-java-options=-Dlog4j.logLevel=info" \ - PATH="${PATH}:${SPARK_HOME}/bin" - -RUN if [ -z "${scala_version}" ]; then \ - ln -s "spark-${APACHE_SPARK_VERSION}-bin-hadoop${HADOOP_VERSION}" "${SPARK_HOME}"; \ - else \ - ln -s "spark-${APACHE_SPARK_VERSION}-bin-hadoop${HADOOP_VERSION}-scala${scala_version}" "${SPARK_HOME}"; \ - fi && \ - # Add a link in the before_notebook hook in order to source automatically PYTHONPATH && \ - mkdir -p /usr/local/bin/before-notebook.d && \ - ln -s "${SPARK_HOME}/sbin/spark-config.sh" /usr/local/bin/before-notebook.d/spark-config.sh - -# Configure IPython system-wide -COPY ipython_kernel_config.py "/etc/ipython/" -RUN fix-permissions "/etc/ipython/" - -USER ${NB_UID} - -# Install pyarrow -# Temporarily pin pandas to version 1.5.3, see: https://github.com/jupyter/docker-stacks/issues/1924 -RUN mamba install --yes \ - 'pandas>=1.5.3,<2.0.0' \ - 'pyarrow' && \ - mamba clean --all -f -y && \ - fix-permissions "${CONDA_DIR}" && \ - fix-permissions "/home/${NB_USER}" - -WORKDIR "${HOME}" -EXPOSE 4040 diff --git a/requirements-dev.txt b/requirements-dev.txt index 34dba9f394..9caa0df30e 100644 --- a/requirements-dev.txt +++ b/requirements-dev.txt @@ -2,8 +2,10 @@ docker plumbum pre-commit pytest +pytest-retry # `pytest-xdist` is a plugin that provides the `--numprocesses` flag, # allowing us to run `pytest` tests in parallel pytest-xdist +PyYAML requests tabulate diff --git a/tagging/README.md b/tagging/README.md index 743e447e31..5d4ff4a5ab 100644 --- a/tagging/README.md +++ b/tagging/README.md @@ -3,19 +3,21 @@ The main purpose of the source code in this folder is to properly tag all the images and to update [build manifests](https://github.com/jupyter/docker-stacks/wiki). These two processes are closely related, so the source code is widely reused. -A basic example of a tag is a `python` version tag. -For example, an image `jupyter/base-notebook` with `python 3.10.5` will have a tag `jupyter/base-notebook:python-3.10.5`. -This tag (and all the other tags) are pushed to Docker Hub. +A basic example of a tag is a `Python` version tag. +For example, an image `jupyter/base-notebook` with `python 3.10.5` will have a full image name `quay.io/jupyter/base-notebook:python-3.10.5`. +This tag (and all the other tags) are pushed to Quay.io. Manifest is a description of some important part of the image in a `markdown`. For example, we dump all the `conda` packages, including their versions. ## Main principles -- All the images are located in a hierarchical tree. More info on [image relationships](../docs/using/selecting.md#image-relationships). +- All the images are located in a hierarchical tree. + More info on [image relationships](../docs/using/selecting.md#image-relationships). - We have `tagger` and `manifest` classes, which can be run inside docker containers to obtain tags and build manifest pieces. -- These classes are inherited from the parent image to all the children images. -- Because manifests and tags might change from parent to children, `taggers` and `manifests` are reevaluated on each image. So, the values are not inherited. +- These classes are inherited from the parent image to all the child images. +- Because manifests and tags might change from parent to child, `taggers` and `manifests` are reevaluated on each image. + So, the values are not inherited. - To tag an image and create a manifest, run `make hook/base-notebook` (or another image of your choice). ## Source code description @@ -48,7 +50,7 @@ The prefix of commit hash (namely, 12 letters) is used as an image tag to make i ### Tagger -`Tagger` is a class which can be run inside a docker container to calculate some tag for an image. +`Tagger` is a class that can be run inside a docker container to calculate some tag for an image. All the taggers are inherited from `TaggerInterface`: @@ -77,12 +79,12 @@ class SHATagger(TaggerInterface): ``` - `taggers.py` contains all the taggers. -- `tag_image.py` is a python executable which is used to tag the image. +- `tag_image.py` is a Python executable that is used to tag the image. ### Manifest `ManifestHeader` is a build manifest header. -It contains information about `Build datetime`, `Docker image size` and `Git commit` info. +It contains the following sections: `Build timestamp`, `Docker image size`, and `Git commit` info. All the other manifest classes are inherited from `ManifestInterface`: @@ -95,7 +97,7 @@ class ManifestInterface: raise NotImplementedError ``` -- `markdown_piece(container)` method returns a piece of markdown file to be used as a part of the build manifest. +- The `markdown_piece(container)` method returns a piece of markdown file to be used as a part of the build manifest. `AptPackagesManifest` example: @@ -106,14 +108,16 @@ from tagging.manifests import ManifestInterface, quoted_output class AptPackagesManifest(ManifestInterface): @staticmethod def markdown_piece(container) -> str: - return "\n".join( - ["## Apt Packages", "", quoted_output(container, "apt list --installed")] - ) + return f"""\ +## Apt Packages + +{quoted_output(container, "apt list --installed")}""" ``` - `quoted_output` simply runs the command inside a container using `DockerRunner.run_simple_command` and wraps it to triple quotes to create a valid markdown piece. + It also adds the command which was run to the markdown piece. - `manifests.py` contains all the manifests. -- `write_manifest.py` is a python executable which is used to create the build manifest and history line for an image. +- `write_manifest.py` is a Python executable that is used to create the build manifest and history line for an image. ### Images Hierarchy diff --git a/tagging/apply_tags.py b/tagging/apply_tags.py index 04148216f0..9c50cd0e01 100755 --- a/tagging/apply_tags.py +++ b/tagging/apply_tags.py @@ -7,6 +7,9 @@ import plumbum +from tagging.get_platform import unify_aarch64 +from tagging.get_prefix import get_file_prefix_for_platform + docker = plumbum.local["docker"] LOGGER = logging.getLogger(__name__) @@ -14,27 +17,26 @@ def apply_tags( short_image_name: str, + registry: str, owner: str, tags_dir: Path, platform: str, + variant: str, ) -> None: """ - Tags <owner>/<short_image_name>:latest with the tags - reported by all taggers for the given image. + Tags <registry>/<owner>/<short_image_name>:latest with the tags reported by all taggers for this image """ LOGGER.info(f"Tagging image: {short_image_name}") - image = f"{owner}/{short_image_name}:latest" - filename = f"{platform}-{short_image_name}.txt" + file_prefix = get_file_prefix_for_platform(platform, variant) + image = f"{registry}/{owner}/{short_image_name}:latest" + filename = f"{file_prefix}-{short_image_name}.txt" tags = (tags_dir / filename).read_text().splitlines() for tag in tags: LOGGER.info(f"Applying tag: {tag}") docker["tag", image, tag] & plumbum.FG - LOGGER.info("Removing latest tag from the image") - docker["image", "rmi", image] & plumbum.FG - if __name__ == "__main__": logging.basicConfig(level=logging.INFO) @@ -43,7 +45,7 @@ def apply_tags( arg_parser.add_argument( "--short-image-name", required=True, - help="Short image name to apply tags for", + help="Short image name", ) arg_parser.add_argument( "--tags-dir", @@ -55,14 +57,34 @@ def apply_tags( "--platform", required=True, type=str, - choices=["x86_64", "aarch64"], + choices=["x86_64", "aarch64", "arm64"], help="Image platform", ) + arg_parser.add_argument( + "--registry", + required=True, + type=str, + choices=["docker.io", "quay.io"], + help="Image registry", + ) arg_parser.add_argument( "--owner", required=True, help="Owner of the image", ) + arg_parser.add_argument( + "--variant", + required=True, + help="Variant tag prefix", + ) args = arg_parser.parse_args() + args.platform = unify_aarch64(args.platform) - apply_tags(args.short_image_name, args.owner, args.tags_dir, args.platform) + apply_tags( + args.short_image_name, + args.registry, + args.owner, + args.tags_dir, + args.platform, + args.variant, + ) diff --git a/tagging/get_platform.py b/tagging/get_platform.py index 139e3697f3..cda791ab7d 100644 --- a/tagging/get_platform.py +++ b/tagging/get_platform.py @@ -5,10 +5,17 @@ ALL_PLATFORMS = {"x86_64", "aarch64"} -def get_platform() -> str: - machine = platform.machine() +def unify_aarch64(platform: str) -> str: + """ + Renames arm64->aarch64 to support local builds on on aarch64 Macs + """ return { "aarch64": "aarch64", - "arm64": "aarch64", # To support local building on aarch64 Macs + "arm64": "aarch64", "x86_64": "x86_64", - }[machine] + }[platform] + + +def get_platform() -> str: + machine = platform.machine() + return unify_aarch64(machine) diff --git a/tagging/get_prefix.py b/tagging/get_prefix.py new file mode 100644 index 0000000000..6e7c718576 --- /dev/null +++ b/tagging/get_prefix.py @@ -0,0 +1,25 @@ +# Copyright (c) Jupyter Development Team. +# Distributed under the terms of the Modified BSD License. +from tagging.get_platform import get_platform + +DEFAULT_VARIANT = "default" + + +def get_file_prefix_for_platform(platform: str, variant: str) -> str: + return f"{platform}-{variant}" + + +def get_tag_prefix_for_platform(platform: str, variant: str) -> str: + if variant == DEFAULT_VARIANT: + return platform + return f"{platform}-{variant}" + + +def get_file_prefix(variant: str) -> str: + platform = get_platform() + return get_file_prefix_for_platform(platform, variant) + + +def get_tag_prefix(variant: str) -> str: + platform = get_platform() + return get_tag_prefix_for_platform(platform, variant) diff --git a/tagging/images_hierarchy.py b/tagging/images_hierarchy.py index 7a0831987f..8c3e3fd384 100644 --- a/tagging/images_hierarchy.py +++ b/tagging/images_hierarchy.py @@ -13,7 +13,6 @@ ) from tagging.taggers import ( DateTagger, - HadoopVersionTagger, JavaVersionTagger, JuliaVersionTagger, JupyterHubVersionTagger, @@ -21,6 +20,7 @@ JupyterNotebookVersionTagger, PythonMajorMinorVersionTagger, PythonVersionTagger, + PytorchVersionTagger, RVersionTagger, SHATagger, SparkVersionTagger, @@ -72,6 +72,9 @@ class ImageDescription: "tensorflow-notebook": ImageDescription( parent_image="scipy-notebook", taggers=[TensorflowVersionTagger()] ), + "pytorch-notebook": ImageDescription( + parent_image="scipy-notebook", taggers=[PytorchVersionTagger()] + ), "datascience-notebook": ImageDescription( parent_image="scipy-notebook", taggers=[RVersionTagger(), JuliaVersionTagger()], @@ -79,7 +82,7 @@ class ImageDescription: ), "pyspark-notebook": ImageDescription( parent_image="scipy-notebook", - taggers=[SparkVersionTagger(), HadoopVersionTagger(), JavaVersionTagger()], + taggers=[SparkVersionTagger(), JavaVersionTagger()], manifests=[SparkInfoManifest()], ), "all-spark-notebook": ImageDescription( diff --git a/tagging/manifests.py b/tagging/manifests.py index bbf1fda556..f043de05f3 100644 --- a/tagging/manifests.py +++ b/tagging/manifests.py @@ -10,44 +10,54 @@ def quoted_output(container: Container, cmd: str) -> str: - return "\n".join( - [ - "```", - DockerRunner.run_simple_command(container, cmd, print_result=False), - "```", - ] - ) + cmd_output = DockerRunner.run_simple_command(container, cmd, print_result=False) + # For example, `mamba info --quiet` adds redundant empty lines + cmd_output = cmd_output.strip("\n") + # For example, R packages list contains trailing backspaces + cmd_output = "\n".join(line.rstrip() for line in cmd_output.split("\n")) + return f"""\ +`{cmd}`: + +```text +{cmd_output} +```""" class ManifestHeader: """ManifestHeader doesn't fall under common interface, and we run it separately""" @staticmethod - def create_header(short_image_name: str, owner: str, build_timestamp: str) -> str: + def create_header( + short_image_name: str, registry: str, owner: str, build_timestamp: str + ) -> str: commit_hash = GitHelper.commit_hash() commit_hash_tag = GitHelper.commit_hash_tag() commit_message = GitHelper.commit_message() + # Unfortunately, `docker images` doesn't work when specifying `docker.io` as registry + fixed_registry = registry + "/" if registry != "docker.io" else "" + image_size = docker[ - "images", f"{owner}/{short_image_name}:latest", "--format", "{{.Size}}" + "images", + f"{fixed_registry}{owner}/{short_image_name}:latest", + "--format", + "{{.Size}}", ]().rstrip() - return "\n".join( - [ - f"# Build manifest for image: {short_image_name}:{commit_hash_tag}", - "", - "## Build Info", - "", - f"* Build datetime: {build_timestamp}", - f"* Docker image: {owner}/{short_image_name}:{commit_hash_tag}", - f"* Docker image size: {image_size}", - f"* Git commit SHA: [{commit_hash}](https://github.com/jupyter/docker-stacks/commit/{commit_hash})", - "* Git commit message:", - "```", - f"{commit_message}", - "```", - ] - ) + return f"""\ +# Build manifest for image: {short_image_name}:{commit_hash_tag} + +## Build Info + +- Build timestamp: {build_timestamp} +- Docker image: `{registry}/{owner}/{short_image_name}:{commit_hash_tag}` +- Docker image size: {image_size} +- Git commit SHA: [{commit_hash}](https://github.com/jupyter/docker-stacks/commit/{commit_hash}) +- Git commit message: + +```text +{commit_message} +```""" class ManifestInterface: @@ -61,72 +71,51 @@ def markdown_piece(container: Container) -> str: class CondaEnvironmentManifest(ManifestInterface): @staticmethod def markdown_piece(container: Container) -> str: - return "\n".join( - [ - "## Python Packages", - "", - quoted_output(container, "python --version"), - "", - quoted_output(container, "mamba info --quiet"), - "", - quoted_output(container, "mamba list"), - ] - ) + return f"""\ +## Python Packages + +{DockerRunner.run_simple_command(container, "python --version")} + +{quoted_output(container, "mamba info --quiet")} + +{quoted_output(container, "mamba list")}""" class AptPackagesManifest(ManifestInterface): @staticmethod def markdown_piece(container: Container) -> str: - return "\n".join( - [ - "## Apt Packages", - "", - quoted_output(container, "apt list --installed"), - ] - ) + return f"""\ +## Apt Packages + +{quoted_output(container, "apt list --installed")}""" class RPackagesManifest(ManifestInterface): @staticmethod def markdown_piece(container: Container) -> str: - return "\n".join( - [ - "## R Packages", - "", - quoted_output(container, "R --version"), - "", - quoted_output( - container, - "R --silent -e 'installed.packages(.Library)[, c(1,3)]'", - ), - ] - ) + return f"""\ +## R Packages + +{quoted_output(container, "R --version")} + +{quoted_output(container, "R --silent -e 'installed.packages(.Library)[, c(1,3)]'")}""" class JuliaPackagesManifest(ManifestInterface): @staticmethod def markdown_piece(container: Container) -> str: - return "\n".join( - [ - "## Julia Packages", - "", - quoted_output( - container, - "julia -E 'using InteractiveUtils; versioninfo()'", - ), - "", - quoted_output(container, "julia -E 'import Pkg; Pkg.status()'"), - ] - ) + return f"""\ +## Julia Packages + +{quoted_output(container, "julia -E 'using InteractiveUtils; versioninfo()'")} + +{quoted_output(container, "julia -E 'import Pkg; Pkg.status()'")}""" class SparkInfoManifest(ManifestInterface): @staticmethod def markdown_piece(container: Container) -> str: - return "\n".join( - [ - "## Apache Spark", - "", - quoted_output(container, "/usr/local/spark/bin/spark-submit --version"), - ] - ) + return f"""\ +## Apache Spark + +{quoted_output(container, "/usr/local/spark/bin/spark-submit --version")}""" diff --git a/tagging/merge_tags.py b/tagging/merge_tags.py index 58247e5eb1..06b18e7019 100755 --- a/tagging/merge_tags.py +++ b/tagging/merge_tags.py @@ -8,6 +8,7 @@ import plumbum from tagging.get_platform import ALL_PLATFORMS +from tagging.get_prefix import get_file_prefix_for_platform docker = plumbum.local["docker"] @@ -16,6 +17,7 @@ def merge_tags( short_image_name: str, + variant: str, tags_dir: Path, ) -> None: """ @@ -26,9 +28,12 @@ def merge_tags( all_tags: set[str] = set() for platform in ALL_PLATFORMS: - filename = f"{platform}-{short_image_name}.txt" - tags = (tags_dir / filename).read_text().splitlines() - all_tags.update(tag.replace(platform + "-", "") for tag in tags) + file_prefix = get_file_prefix_for_platform(platform, variant) + filename = f"{file_prefix}-{short_image_name}.txt" + file_path = tags_dir / filename + if file_path.exists(): + tags = file_path.read_text().splitlines() + all_tags.update(tag.replace(platform + "-", "") for tag in tags) LOGGER.info(f"Got tags: {all_tags}") @@ -59,7 +64,12 @@ def merge_tags( arg_parser.add_argument( "--short-image-name", required=True, - help="Short image name to apply tags for", + help="Short image name", + ) + arg_parser.add_argument( + "--variant", + required=True, + help="Variant tag prefix", ) arg_parser.add_argument( "--tags-dir", @@ -69,4 +79,4 @@ def merge_tags( ) args = arg_parser.parse_args() - merge_tags(args.short_image_name, args.tags_dir) + merge_tags(args.short_image_name, args.variant, args.tags_dir) diff --git a/tagging/taggers.py b/tagging/taggers.py index 8553711718..a42c9d053c 100644 --- a/tagging/taggers.py +++ b/tagging/taggers.py @@ -12,18 +12,6 @@ def _get_program_version(container: Container, program: str) -> str: return DockerRunner.run_simple_command(container, cmd=f"{program} --version") -def _get_env_variable(container: Container, variable: str) -> str: - env = DockerRunner.run_simple_command( - container, - cmd="env", - print_result=False, - ).split() - for env_entry in env: - if env_entry.startswith(variable): - return env_entry[len(variable) + 1 :] - raise KeyError(variable) - - def _get_pip_package_version(container: Container, package: str) -> str: PIP_VERSION_PREFIX = "Version: " @@ -110,25 +98,33 @@ def tag_value(container: Container) -> str: class TensorflowVersionTagger(TaggerInterface): @staticmethod def tag_value(container: Container) -> str: - return "tensorflow-" + _get_pip_package_version(container, "tensorflow") + try: + return "tensorflow-" + _get_pip_package_version(container, "tensorflow") + except AssertionError: + return "tensorflow-" + _get_pip_package_version(container, "tensorflow-cpu") -class JuliaVersionTagger(TaggerInterface): +class PytorchVersionTagger(TaggerInterface): @staticmethod def tag_value(container: Container) -> str: - return "julia-" + _get_program_version(container, "julia").split()[2] + return "pytorch-" + _get_pip_package_version(container, "torch").split("+")[0] -class SparkVersionTagger(TaggerInterface): +class JuliaVersionTagger(TaggerInterface): @staticmethod def tag_value(container: Container) -> str: - return "spark-" + _get_env_variable(container, "APACHE_SPARK_VERSION") + return "julia-" + _get_program_version(container, "julia").split()[2] -class HadoopVersionTagger(TaggerInterface): +class SparkVersionTagger(TaggerInterface): @staticmethod def tag_value(container: Container) -> str: - return "hadoop-" + _get_env_variable(container, "HADOOP_VERSION") + SPARK_VERSION_LINE_PREFIX = r" /___/ .__/\_,_/_/ /_/\_\ version" + + spark_version = _get_program_version(container, "spark-submit") + version_line = spark_version.split("\n")[4] + assert version_line.startswith(SPARK_VERSION_LINE_PREFIX) + return "spark-" + version_line.split(" ")[-1] class JavaVersionTagger(TaggerInterface): diff --git a/tagging/update_wiki.py b/tagging/update_wiki.py new file mode 100755 index 0000000000..292ee2d8db --- /dev/null +++ b/tagging/update_wiki.py @@ -0,0 +1,130 @@ +#!/usr/bin/env python3 +# Copyright (c) Jupyter Development Team. +# Distributed under the terms of the Modified BSD License. +import argparse +import logging +import shutil +from pathlib import Path + +LOGGER = logging.getLogger(__name__) + + +def update_home_wiki_page(wiki_dir: Path, month: str) -> None: + TABLE_BEGINNING = """\ +| Month | +| ---------------------- | +""" + wiki_home_file = wiki_dir / "Home.md" + wiki_home_content = wiki_home_file.read_text() + month_line = f"| [`{month}`](./{month}) |\n" + if month_line not in wiki_home_content: + assert TABLE_BEGINNING in wiki_home_content + wiki_home_content = wiki_home_content.replace( + TABLE_BEGINNING, TABLE_BEGINNING + month_line + ) + wiki_home_file.write_text(wiki_home_content) + LOGGER.info(f"Updated wiki home page with month: {month}") + + +def update_monthly_wiki_page( + wiki_dir: Path, month: str, build_history_line: str +) -> None: + MONTHLY_PAGE_HEADER = f"""\ +# Images built during {month} + +| Date | Image | Links | +| - | - | - | +""" + monthly_page = wiki_dir / "monthly-files" / (month + ".md") + if not monthly_page.exists(): + monthly_page.write_text(MONTHLY_PAGE_HEADER) + LOGGER.info(f"Created monthly page: {monthly_page.relative_to(wiki_dir)}") + + monthly_page_content = monthly_page.read_text() + assert MONTHLY_PAGE_HEADER in monthly_page_content + monthly_page_content = monthly_page_content.replace( + MONTHLY_PAGE_HEADER, MONTHLY_PAGE_HEADER + build_history_line + "\n" + ) + monthly_page.write_text(monthly_page_content) + LOGGER.info(f"Updated monthly page: {monthly_page.relative_to(wiki_dir)}") + + +def get_manifest_timestamp(manifest_file: Path) -> str: + file_content = manifest_file.read_text() + TIMESTAMP_PREFIX = "Build timestamp: " + TIMESTAMP_LENGTH = 20 + timestamp = file_content[ + file_content.find(TIMESTAMP_PREFIX) + len(TIMESTAMP_PREFIX) : + ][:TIMESTAMP_LENGTH] + # Should be good enough till year 2100 + assert timestamp.startswith("20"), timestamp + assert timestamp.endswith("Z"), timestamp + return timestamp + + +def get_manifest_month(manifest_file: Path) -> str: + return get_manifest_timestamp(manifest_file)[:7] + + +def remove_old_manifests(wiki_dir: Path) -> None: + MAX_NUMBER_OF_MANIFESTS = 4500 + + manifest_files: list[tuple[str, Path]] = [] + for file in (wiki_dir / "manifests").rglob("*.md"): + manifest_files.append((get_manifest_timestamp(file), file)) + + manifest_files.sort(reverse=True) + for _, file in manifest_files[MAX_NUMBER_OF_MANIFESTS:]: + file.unlink() + LOGGER.info(f"Removed manifest: {file.relative_to(wiki_dir)}") + + +def update_wiki(wiki_dir: Path, hist_lines_dir: Path, manifests_dir: Path) -> None: + LOGGER.info("Updating wiki") + + manifest_files = list(manifests_dir.rglob("*.md")) + assert manifest_files, "expected to have some manifest files" + for manifest_file in manifest_files: + month = get_manifest_month(manifest_file) + copy_to = wiki_dir / "manifests" / month / manifest_file.name + copy_to.parent.mkdir(exist_ok=True) + shutil.copy(manifest_file, copy_to) + LOGGER.info(f"Added manifest file: {copy_to.relative_to(wiki_dir)}") + + build_history_line_files = sorted(hist_lines_dir.rglob("*.txt")) + assert build_history_line_files, "expected to have some build history line files" + for build_history_line_file in build_history_line_files: + build_history_line = build_history_line_file.read_text() + assert build_history_line.startswith("| `") + month = build_history_line[3:10] + update_home_wiki_page(wiki_dir, month) + update_monthly_wiki_page(wiki_dir, month, build_history_line) + + remove_old_manifests(wiki_dir) + + +if __name__ == "__main__": + logging.basicConfig(level=logging.INFO) + + arg_parser = argparse.ArgumentParser() + arg_parser.add_argument( + "--wiki-dir", + required=True, + type=Path, + help="Directory of the wiki repo", + ) + arg_parser.add_argument( + "--hist-lines-dir", + required=True, + type=Path, + help="Directory with history lines", + ) + arg_parser.add_argument( + "--manifests-dir", + required=True, + type=Path, + help="Directory with manifest files", + ) + args = arg_parser.parse_args() + + update_wiki(args.wiki_dir, args.hist_lines_dir, args.manifests_dir) diff --git a/tagging/update_wiki_page.py b/tagging/update_wiki_page.py deleted file mode 100755 index 597e8f7ce0..0000000000 --- a/tagging/update_wiki_page.py +++ /dev/null @@ -1,57 +0,0 @@ -#!/usr/bin/env python3 -# Copyright (c) Jupyter Development Team. -# Distributed under the terms of the Modified BSD License. -import argparse -import logging -import shutil -from pathlib import Path - -LOGGER = logging.getLogger(__name__) -TABLE_BEGINNING = "|-|-|-|\n" - - -def update_wiki_page(wiki_dir: Path, hist_line_dir: Path, manifest_dir: Path) -> None: - LOGGER.info("Updating wiki page") - - wiki_home_file = wiki_dir / "Home.md" - wiki_home_content = wiki_home_file.read_text() - build_history_line_files = sorted(hist_line_dir.rglob("*.txt")) - build_history_lines = "\n".join( - hist_line_file.read_text() for hist_line_file in build_history_line_files - ) - wiki_home_content = wiki_home_content.replace( - TABLE_BEGINNING, TABLE_BEGINNING + build_history_lines + "\n" - ) - wiki_home_file.write_text(wiki_home_content) - LOGGER.info("Wiki home file updated") - - for manifest_file in sorted(manifest_dir.rglob("*.md")): - shutil.copy(manifest_file, wiki_dir / "manifests" / manifest_file.name) - LOGGER.info(f"Manifest file added: {manifest_file.name}") - - -if __name__ == "__main__": - logging.basicConfig(level=logging.INFO) - - arg_parser = argparse.ArgumentParser() - arg_parser.add_argument( - "--wiki-dir", - required=True, - type=Path, - help="Directory for wiki repo", - ) - arg_parser.add_argument( - "--hist-line-dir", - required=True, - type=Path, - help="Directory to save history line", - ) - arg_parser.add_argument( - "--manifest-dir", - required=True, - type=Path, - help="Directory to save manifest file", - ) - args = arg_parser.parse_args() - - update_wiki_page(args.wiki_dir, args.hist_line_dir, args.manifest_dir) diff --git a/tagging/write_manifest.py b/tagging/write_manifest.py index f81e021335..cdf51a77d4 100755 --- a/tagging/write_manifest.py +++ b/tagging/write_manifest.py @@ -9,22 +9,23 @@ from docker.models.containers import Container from tagging.docker_runner import DockerRunner -from tagging.get_platform import get_platform +from tagging.get_prefix import get_file_prefix, get_tag_prefix from tagging.get_taggers_and_manifests import get_taggers_and_manifests from tagging.git_helper import GitHelper from tagging.manifests import ManifestHeader, ManifestInterface LOGGER = logging.getLogger(__name__) -# This would actually be manifest creation timestamp +# We use a manifest creation timestamp, which happens right after a build BUILD_TIMESTAMP = datetime.datetime.utcnow().isoformat()[:-7] + "Z" MARKDOWN_LINE_BREAK = "<br />" def write_build_history_line( short_image_name: str, + registry: str, owner: str, - hist_line_dir: Path, + hist_lines_dir: Path, filename: str, all_tags: list[str], ) -> None: @@ -32,25 +33,26 @@ def write_build_history_line( date_column = f"`{BUILD_TIMESTAMP}`" image_column = MARKDOWN_LINE_BREAK.join( - f"`{owner}/{short_image_name}:{tag_value}`" for tag_value in all_tags + f"`{registry}/{owner}/{short_image_name}:{tag_value}`" for tag_value in all_tags ) commit_hash = GitHelper.commit_hash() links_column = MARKDOWN_LINE_BREAK.join( [ f"[Git diff](https://github.com/jupyter/docker-stacks/commit/{commit_hash})", - f"[Dockerfile](https://github.com/jupyter/docker-stacks/blob/{commit_hash}/{short_image_name}/Dockerfile)", + f"[Dockerfile](https://github.com/jupyter/docker-stacks/blob/{commit_hash}/images/{short_image_name}/Dockerfile)", f"[Build manifest](./{filename})", ] ) - build_history_line = "|".join([date_column, image_column, links_column]) + "|" - hist_line_dir.mkdir(parents=True, exist_ok=True) - (hist_line_dir / f"{filename}.txt").write_text(build_history_line) + build_history_line = f"| {date_column} | {image_column} | {links_column} |" + hist_lines_dir.mkdir(parents=True, exist_ok=True) + (hist_lines_dir / f"{filename}.txt").write_text(build_history_line) def write_manifest_file( short_image_name: str, + registry: str, owner: str, - manifest_dir: Path, + manifests_dir: Path, filename: str, manifests: list[ManifestInterface], container: Container, @@ -59,39 +61,47 @@ def write_manifest_file( LOGGER.info(f"Using manifests: {manifest_names}") markdown_pieces = [ - ManifestHeader.create_header(short_image_name, owner, BUILD_TIMESTAMP) + ManifestHeader.create_header(short_image_name, registry, owner, BUILD_TIMESTAMP) ] + [manifest.markdown_piece(container) for manifest in manifests] markdown_content = "\n\n".join(markdown_pieces) + "\n" - manifest_dir.mkdir(parents=True, exist_ok=True) - (manifest_dir / f"{filename}.md").write_text(markdown_content) + manifests_dir.mkdir(parents=True, exist_ok=True) + (manifests_dir / f"{filename}.md").write_text(markdown_content) def write_manifest( short_image_name: str, + registry: str, owner: str, - hist_line_dir: Path, - manifest_dir: Path, + variant: str, + hist_lines_dir: Path, + manifests_dir: Path, ) -> None: LOGGER.info(f"Creating manifests for image: {short_image_name}") taggers, manifests = get_taggers_and_manifests(short_image_name) - image = f"{owner}/{short_image_name}:latest" + image = f"{registry}/{owner}/{short_image_name}:latest" - file_prefix = get_platform() + file_prefix = get_file_prefix(variant) commit_hash_tag = GitHelper.commit_hash_tag() filename = f"{file_prefix}-{short_image_name}-{commit_hash_tag}" with DockerRunner(image) as container: - tags_prefix = get_platform() + tags_prefix = get_tag_prefix(variant) all_tags = [ tags_prefix + "-" + tagger.tag_value(container) for tagger in taggers ] write_build_history_line( - short_image_name, owner, hist_line_dir, filename, all_tags + short_image_name, registry, owner, hist_lines_dir, filename, all_tags ) write_manifest_file( - short_image_name, owner, manifest_dir, filename, manifests, container + short_image_name, + registry, + owner, + manifests_dir, + filename, + manifests, + container, ) @@ -102,29 +112,46 @@ def write_manifest( arg_parser.add_argument( "--short-image-name", required=True, - help="Short image name to create manifests for", + help="Short image name", ) arg_parser.add_argument( - "--hist-line-dir", + "--hist-lines-dir", required=True, type=Path, help="Directory to save history line", ) arg_parser.add_argument( - "--manifest-dir", + "--manifests-dir", required=True, type=Path, help="Directory to save manifest file", ) + arg_parser.add_argument( + "--registry", + required=True, + type=str, + choices=["docker.io", "quay.io"], + help="Image registry", + ) arg_parser.add_argument( "--owner", required=True, help="Owner of the image", ) + arg_parser.add_argument( + "--variant", + required=True, + help="Variant tag prefix", + ) args = arg_parser.parse_args() LOGGER.info(f"Current build timestamp: {BUILD_TIMESTAMP}") write_manifest( - args.short_image_name, args.owner, args.hist_line_dir, args.manifest_dir + args.short_image_name, + args.registry, + args.owner, + args.variant, + args.hist_lines_dir, + args.manifests_dir, ) diff --git a/tagging/write_tags_file.py b/tagging/write_tags_file.py index 55cbfab7ec..21c127cd21 100755 --- a/tagging/write_tags_file.py +++ b/tagging/write_tags_file.py @@ -6,7 +6,7 @@ from pathlib import Path from tagging.docker_runner import DockerRunner -from tagging.get_platform import get_platform +from tagging.get_prefix import get_file_prefix, get_tag_prefix from tagging.get_taggers_and_manifests import get_taggers_and_manifests LOGGER = logging.getLogger(__name__) @@ -14,20 +14,23 @@ def write_tags_file( short_image_name: str, + registry: str, owner: str, + variant: str, tags_dir: Path, ) -> None: """ - Writes tags file for the image <owner>/<short_image_name>:latest + Writes tags file for the image <registry>/<owner>/<short_image_name>:latest """ LOGGER.info(f"Tagging image: {short_image_name}") taggers, _ = get_taggers_and_manifests(short_image_name) - image = f"{owner}/{short_image_name}:latest" - tags_prefix = get_platform() - filename = f"{tags_prefix}-{short_image_name}.txt" + image = f"{registry}/{owner}/{short_image_name}:latest" + file_prefix = get_file_prefix(variant) + filename = f"{file_prefix}-{short_image_name}.txt" - tags = [f"{owner}/{short_image_name}:{tags_prefix}-latest"] + tags_prefix = get_tag_prefix(variant) + tags = [f"{registry}/{owner}/{short_image_name}:{tags_prefix}-latest"] with DockerRunner(image) as container: for tagger in taggers: tagger_name = tagger.__class__.__name__ @@ -35,7 +38,9 @@ def write_tags_file( LOGGER.info( f"Calculated tag, tagger_name: {tagger_name} tag_value: {tag_value}" ) - tags.append(f"{owner}/{short_image_name}:{tags_prefix}-{tag_value}") + tags.append( + f"{registry}/{owner}/{short_image_name}:{tags_prefix}-{tag_value}" + ) tags_dir.mkdir(parents=True, exist_ok=True) (tags_dir / filename).write_text("\n".join(tags)) @@ -47,7 +52,7 @@ def write_tags_file( arg_parser.add_argument( "--short-image-name", required=True, - help="Short image name to write tags for", + help="Short image name", ) arg_parser.add_argument( "--tags-dir", @@ -55,11 +60,29 @@ def write_tags_file( type=Path, help="Directory to save tags file", ) + arg_parser.add_argument( + "--registry", + required=True, + type=str, + choices=["docker.io", "quay.io"], + help="Image registry", + ) arg_parser.add_argument( "--owner", required=True, help="Owner of the image", ) + arg_parser.add_argument( + "--variant", + required=True, + help="Variant tag prefix", + ) args = arg_parser.parse_args() - write_tags_file(args.short_image_name, args.owner, args.tags_dir) + write_tags_file( + args.short_image_name, + args.registry, + args.owner, + args.variant, + args.tags_dir, + ) diff --git a/tests/R_mimetype_check.py b/tests/R_mimetype_check.py index 4cfe7267bf..ba90fa51d6 100644 --- a/tests/R_mimetype_check.py +++ b/tests/R_mimetype_check.py @@ -10,11 +10,15 @@ def check_r_mimetypes(container: TrackedContainer) -> None: """Check if Rscript command can be executed""" LOGGER.info("Test that R command can be executed ...") - Rcommand = 'if (length(getOption("jupyter.plot_mimetypes")) != 5) {stop("missing jupyter.plot_mimetypes")}' + R_MIMETYPES_CHECK_CMD = 'if (length(getOption("jupyter.plot_mimetypes")) != 5) {stop("missing jupyter.plot_mimetypes")}' + command = ["Rscript", "-e", R_MIMETYPES_CHECK_CMD] logs = container.run_and_wait( timeout=10, tty=True, - command=["Rscript", "-e", Rcommand], + command=command, ) LOGGER.debug(f"{logs=}") - assert len(logs) == 0, f"Command {Rcommand=} failed" + # If there is any output after this it means there was an error + assert logs.splitlines()[-1] == "Executing the command: " + " ".join( + command + ), f"Command {R_MIMETYPES_CHECK_CMD=} failed" diff --git a/tests/all-spark-notebook/test_spark_notebooks.py b/tests/all-spark-notebook/test_spark_notebooks.py index b49b1cb2be..7e54e5b04c 100644 --- a/tests/all-spark-notebook/test_spark_notebooks.py +++ b/tests/all-spark-notebook/test_spark_notebooks.py @@ -11,6 +11,7 @@ THIS_DIR = Path(__file__).parent.resolve() +@pytest.mark.flaky(retries=3, delay=1) @pytest.mark.parametrize( "test_file", ["issue_1168", "local_pyspark", "local_sparklyr", "local_sparkR"], @@ -20,7 +21,7 @@ def test_nbconvert(container: TrackedContainer, test_file: str) -> None: host_data_dir = THIS_DIR / "data" cont_data_dir = "/home/jovyan/data" output_dir = "/tmp" - conversion_timeout_ms = 600 + conversion_timeout_ms = 5000 LOGGER.info(f"Test that {test_file} notebook can be executed ...") command = ( "jupyter nbconvert --to markdown " @@ -32,7 +33,7 @@ def test_nbconvert(container: TrackedContainer, test_file: str) -> None: timeout=60, volumes={str(host_data_dir): {"bind": cont_data_dir, "mode": "ro"}}, tty=True, - command=["start.sh", "bash", "-c", command], + command=["bash", "-c", command], ) expected_file = f"{output_dir}/{test_file}.md" diff --git a/tests/base-notebook/test_container_options.py b/tests/base-notebook/test_container_options.py index 3fab83136d..aceeaeb907 100644 --- a/tests/base-notebook/test_container_options.py +++ b/tests/base-notebook/test_container_options.py @@ -12,11 +12,10 @@ def test_cli_args(container: TrackedContainer, http_client: requests.Session) -> None: - """Container should respect notebook server command line args - (e.g., disabling token security)""" + """Image should respect command line args (e.g., disabling token security)""" host_port = find_free_port() running_container = container.run_detached( - command=["start-notebook.sh", "--NotebookApp.token=''"], + command=["start-notebook.py", "--IdentityProvider.token=''"], ports={"8888/tcp": host_port}, ) resp = http_client.get(f"http://localhost:{host_port}") @@ -59,17 +58,17 @@ def test_unsigned_ssl( container: TrackedContainer, http_client: requests.Session ) -> None: """Container should generate a self-signed SSL certificate - and notebook server should use it to enable HTTPS. + and Jupyter Server should use it to enable HTTPS. """ host_port = find_free_port() running_container = container.run_detached( environment=["GEN_CERT=yes"], ports={"8888/tcp": host_port}, ) - # NOTE: The requests.Session backing the http_client fixture does not retry - # properly while the server is booting up. An SSL handshake error seems to - # abort the retry logic. Forcing a long sleep for the moment until I have - # time to dig more. + # NOTE: The requests.Session backing the http_client fixture + # does not retry properly while the server is booting up. + # An SSL handshake error seems to abort the retry logic. + # Forcing a long sleep for the moment until I have time to dig more. time.sleep(1) resp = http_client.get(f"https://localhost:{host_port}", verify=False) resp.raise_for_status() @@ -103,7 +102,7 @@ def test_custom_internal_port( host_port = find_free_port() internal_port = env.get("JUPYTER_PORT", 8888) running_container = container.run_detached( - command=["start-notebook.sh", "--NotebookApp.token=''"], + command=["start-notebook.py", "--IdentityProvider.token=''"], environment=env, ports={internal_port: host_port}, ) diff --git a/tests/base-notebook/test_healthcheck.py b/tests/base-notebook/test_healthcheck.py index 3c779803f2..d5874c9a3c 100644 --- a/tests/base-notebook/test_healthcheck.py +++ b/tests/base-notebook/test_healthcheck.py @@ -22,28 +22,39 @@ (["RESTARTABLE=yes"], None, None), (["JUPYTER_PORT=8171"], None, None), (["JUPYTER_PORT=8117", "DOCKER_STACKS_JUPYTER_CMD=notebook"], None, None), - (None, ["start-notebook.sh", "--NotebookApp.base_url=/test"], None), - (None, ["start-notebook.sh", "--NotebookApp.base_url=/test/"], None), - (["GEN_CERT=1"], ["start-notebook.sh", "--NotebookApp.base_url=/test"], None), + (None, ["start-notebook.sh"], None), + (None, ["start-notebook.py", "--ServerApp.base_url=/test"], None), + (None, ["start-notebook.py", "--ServerApp.base_url=/test/"], None), + (["GEN_CERT=1"], ["start-notebook.py", "--ServerApp.base_url=/test"], None), ( ["GEN_CERT=1", "JUPYTER_PORT=7891"], - ["start-notebook.sh", "--NotebookApp.base_url=/test"], + ["start-notebook.py", "--ServerApp.base_url=/test"], None, ), (["NB_USER=testuser", "CHOWN_HOME=1"], None, "root"), ( ["NB_USER=testuser", "CHOWN_HOME=1"], - ["start-notebook.sh", "--NotebookApp.base_url=/test"], + ["start-notebook.py", "--ServerApp.base_url=/test"], "root", ), ( ["NB_USER=testuser", "CHOWN_HOME=1", "JUPYTER_PORT=8123"], - ["start-notebook.sh", "--NotebookApp.base_url=/test"], + ["start-notebook.py", "--ServerApp.base_url=/test"], + "root", + ), + (["JUPYTER_RUNTIME_DIR=/tmp/jupyter-runtime"], ["start-notebook.sh"], None), + ( + [ + "NB_USER=testuser", + "CHOWN_HOME=1", + "JUPYTER_RUNTIME_DIR=/tmp/jupyter-runtime", + ], + ["start-notebook.sh"], "root", ), ], ) -def test_health( +def test_healthy( container: TrackedContainer, env: Optional[list[str]], cmd: Optional[list[str]], @@ -56,13 +67,11 @@ def test_health( user=user, ) - # sleeping some time to let the server start - time_spent = 0.0 - wait_time = 0.1 - time_limit = 15 - while time_spent < time_limit: - time.sleep(wait_time) - time_spent += wait_time + # giving some time to let the server start + finish_time = time.time() + 10 + sleep_time = 0.1 + while time.time() < finish_time: + time.sleep(sleep_time) if get_health(running_container) == "healthy": return @@ -72,20 +81,25 @@ def test_health( @pytest.mark.parametrize( "env,cmd,user", [ - (["NB_USER=testuser", "CHOWN_HOME=1"], None, None), ( - ["NB_USER=testuser", "CHOWN_HOME=1"], - ["start-notebook.sh", "--NotebookApp.base_url=/test"], + ["HTTPS_PROXY=host.docker.internal", "HTTP_PROXY=host.docker.internal"], + None, None, ), ( - ["NB_USER=testuser", "CHOWN_HOME=1", "JUPYTER_PORT=8123"], - ["start-notebook.sh", "--NotebookApp.base_url=/test"], - None, + [ + "NB_USER=testuser", + "CHOWN_HOME=1", + "JUPYTER_PORT=8123", + "HTTPS_PROXY=host.docker.internal", + "HTTP_PROXY=host.docker.internal", + ], + ["start-notebook.py", "--ServerApp.base_url=/test"], + "root", ), ], ) -def test_not_healthy( +def test_healthy_with_proxy( container: TrackedContainer, env: Optional[list[str]], cmd: Optional[list[str]], @@ -98,14 +112,48 @@ def test_not_healthy( user=user, ) - # sleeping some time to let the server start - time_spent = 0.0 - wait_time = 0.1 - time_limit = 15 - while time_spent < time_limit: - time.sleep(wait_time) - time_spent += wait_time + # giving some time to let the server start + finish_time = time.time() + 10 + sleep_time = 0.1 + while time.time() < finish_time: + time.sleep(sleep_time) + if get_health(running_container) == "healthy": + return + + assert get_health(running_container) == "healthy" + + +@pytest.mark.parametrize( + "env,cmd", + [ + (["NB_USER=testuser", "CHOWN_HOME=1"], None), + ( + ["NB_USER=testuser", "CHOWN_HOME=1"], + ["start-notebook.py", "--ServerApp.base_url=/test"], + ), + ( + ["NB_USER=testuser", "CHOWN_HOME=1", "JUPYTER_PORT=8123"], + ["start-notebook.py", "--ServerApp.base_url=/test"], + ), + ], +) +def test_not_healthy( + container: TrackedContainer, + env: Optional[list[str]], + cmd: Optional[list[str]], +) -> None: + running_container = container.run_detached( + tty=True, + environment=env, + command=cmd, + ) + + # giving some time to let the server start + finish_time = time.time() + 5 + sleep_time = 0.1 + while time.time() < finish_time: + time.sleep(sleep_time) if get_health(running_container) == "healthy": - raise RuntimeError("Container should not be healthy for these testcases.") + raise RuntimeError("Container should not be healthy for this testcase") assert get_health(running_container) != "healthy" diff --git a/tests/base-notebook/test_notebook.py b/tests/base-notebook/test_notebook.py index 29c2bec742..3985990eb0 100644 --- a/tests/base-notebook/test_notebook.py +++ b/tests/base-notebook/test_notebook.py @@ -8,7 +8,7 @@ def test_secured_server( container: TrackedContainer, http_client: requests.Session ) -> None: - """Notebook server should eventually request user login.""" + """Jupyter Server should eventually request user login.""" host_port = find_free_port() container.run_detached(ports={"8888/tcp": host_port}) resp = http_client.get(f"http://localhost:{host_port}") diff --git a/tests/base-notebook/test_pandoc.py b/tests/base-notebook/test_pandoc.py index f5cce32542..3c828a3f0b 100644 --- a/tests/base-notebook/test_pandoc.py +++ b/tests/base-notebook/test_pandoc.py @@ -12,6 +12,6 @@ def test_pandoc(container: TrackedContainer) -> None: logs = container.run_and_wait( timeout=10, tty=True, - command=["start.sh", "bash", "-c", 'echo "**BOLD**" | pandoc'], + command=["bash", "-c", 'echo "**BOLD**" | pandoc'], ) assert "<p><strong>BOLD</strong></p>" in logs diff --git a/tests/base-notebook/test_start_container.py b/tests/base-notebook/test_start_container.py index 14418288a9..729e7cacb2 100644 --- a/tests/base-notebook/test_start_container.py +++ b/tests/base-notebook/test_start_container.py @@ -25,7 +25,7 @@ ["JUPYTERHUB_API_TOKEN=my_token"], "jupyterhub-singleuser", False, - ["WARNING: using start-singleuser.sh"], + ["WARNING: using start-singleuser.py"], ), ], ) @@ -37,24 +37,23 @@ def test_start_notebook( expected_start: bool, expected_warnings: list[str], ) -> None: - """Test the notebook start-notebook script""" + """Test the notebook start-notebook.py script""" LOGGER.info( - f"Test that the start-notebook launches the {expected_command} server from the env {env} ..." + f"Test that the start-notebook.py launches the {expected_command} server from the env {env} ..." ) host_port = find_free_port() running_container = container.run_detached( tty=True, environment=env, - command=["start-notebook.sh"], ports={"8888/tcp": host_port}, ) # sleeping some time to let the server start - time.sleep(1) + time.sleep(2) logs = running_container.logs().decode("utf-8") LOGGER.debug(logs) # checking that the expected command is launched assert ( - f"Executing the command: {expected_command}" in logs + f"Executing: {expected_command}" in logs ), f"Not the expected command ({expected_command}) was launched" # checking errors and warnings in logs assert "ERROR" not in logs, "ERROR(s) found in logs" @@ -77,10 +76,7 @@ def test_tini_entrypoint( https://superuser.com/questions/632979/if-i-know-the-pid-number-of-a-process-how-can-i-get-its-name """ LOGGER.info(f"Test that {command} is launched as PID {pid} ...") - running_container = container.run_detached( - tty=True, - command=["start.sh"], - ) + running_container = container.run_detached(tty=True) # Select the PID 1 and get the corresponding command cmd = running_container.exec_run(f"ps -p {pid} -o comm=") output = cmd.output.decode("utf-8").strip("\n") diff --git a/tests/conftest.py b/tests/conftest.py index f7a538a8fa..a151eb6b83 100644 --- a/tests/conftest.py +++ b/tests/conftest.py @@ -78,8 +78,8 @@ def __init__( self.kwargs: Any = kwargs def run_detached(self, **kwargs: Any) -> Container: - """Runs a docker container using the preconfigured image name - and a mix of the preconfigured container options and those passed + """Runs a docker container using the pre-configured image name + and a mix of the pre-configured container options and those passed to this method. Keeps track of the docker.Container instance spawned to kill it @@ -108,6 +108,7 @@ def run_and_wait( timeout: int, no_warnings: bool = True, no_errors: bool = True, + no_failure: bool = True, **kwargs: Any, ) -> str: running_container = self.run_detached(**kwargs) @@ -119,7 +120,7 @@ def run_and_wait( assert not self.get_warnings(logs) if no_errors: assert not self.get_errors(logs) - assert rv == 0 or rv["StatusCode"] == 0 + assert no_failure == (rv["StatusCode"] == 0) return logs @staticmethod diff --git a/tests/datascience-notebook/test_pluto_datascience.py b/tests/datascience-notebook/test_pluto_datascience.py new file mode 100644 index 0000000000..27c4aaf0d3 --- /dev/null +++ b/tests/datascience-notebook/test_pluto_datascience.py @@ -0,0 +1,13 @@ +# Copyright (c) Jupyter Development Team. +# Distributed under the terms of the Modified BSD License. +import requests + +from tests.conftest import TrackedContainer +from tests.pluto_check import check_pluto_proxy + + +def test_pluto_proxy( + container: TrackedContainer, http_client: requests.Session +) -> None: + """Pluto proxy starts Pluto correctly""" + check_pluto_proxy(container, http_client) diff --git a/tests/docker-stacks-foundation/run-hooks-change/a.sh b/tests/docker-stacks-foundation/run-hooks-change/a.sh new file mode 100644 index 0000000000..61701e2a82 --- /dev/null +++ b/tests/docker-stacks-foundation/run-hooks-change/a.sh @@ -0,0 +1,6 @@ +#!/bin/bash +# Copyright (c) Jupyter Development Team. +# Distributed under the terms of the Modified BSD License. + +export MY_VAR=123 +echo "Inside a.sh MY_VAR variable has ${MY_VAR} value" diff --git a/tests/docker-stacks-foundation/run-hooks-change/b.sh b/tests/docker-stacks-foundation/run-hooks-change/b.sh new file mode 100644 index 0000000000..fdca97484a --- /dev/null +++ b/tests/docker-stacks-foundation/run-hooks-change/b.sh @@ -0,0 +1,8 @@ +#!/bin/bash +# Copyright (c) Jupyter Development Team. +# Distributed under the terms of the Modified BSD License. + +echo "Inside b.sh MY_VAR variable has ${MY_VAR} value" +echo "Changing value of MY_VAR" +export MY_VAR=456 +echo "After change inside b.sh MY_VAR variable has ${MY_VAR} value" diff --git a/tests/docker-stacks-foundation/run-hooks-change/c.sh b/tests/docker-stacks-foundation/run-hooks-change/c.sh new file mode 100644 index 0000000000..ef69df395f --- /dev/null +++ b/tests/docker-stacks-foundation/run-hooks-change/c.sh @@ -0,0 +1,5 @@ +#!/bin/bash +# Copyright (c) Jupyter Development Team. +# Distributed under the terms of the Modified BSD License. + +echo "Inside c.sh MY_VAR variable has ${MY_VAR} value" diff --git a/tests/docker-stacks-foundation/run-hooks-executables/executable.py b/tests/docker-stacks-foundation/run-hooks-executables/executable.py new file mode 100755 index 0000000000..5fb2b9a342 --- /dev/null +++ b/tests/docker-stacks-foundation/run-hooks-executables/executable.py @@ -0,0 +1,5 @@ +#!/usr/bin/env python3 +# Copyright (c) Jupyter Development Team. +# Distributed under the terms of the Modified BSD License. + +print("Executable python file was successfully run") diff --git a/tests/docker-stacks-foundation/run-hooks-executables/non_executable.py b/tests/docker-stacks-foundation/run-hooks-executables/non_executable.py new file mode 100644 index 0000000000..19c8d0b743 --- /dev/null +++ b/tests/docker-stacks-foundation/run-hooks-executables/non_executable.py @@ -0,0 +1,5 @@ +#!/usr/bin/env python3 +# Copyright (c) Jupyter Development Team. +# Distributed under the terms of the Modified BSD License. + +assert False diff --git a/tests/docker-stacks-foundation/run-hooks-executables/run-me.sh b/tests/docker-stacks-foundation/run-hooks-executables/run-me.sh new file mode 100644 index 0000000000..f4dc08aa8c --- /dev/null +++ b/tests/docker-stacks-foundation/run-hooks-executables/run-me.sh @@ -0,0 +1,5 @@ +#!/bin/bash +# Copyright (c) Jupyter Development Team. +# Distributed under the terms of the Modified BSD License. + +export SOME_VAR=123 diff --git a/tests/docker-stacks-foundation/run-hooks-failures/a.sh b/tests/docker-stacks-foundation/run-hooks-failures/a.sh new file mode 100644 index 0000000000..7dabeeb861 --- /dev/null +++ b/tests/docker-stacks-foundation/run-hooks-failures/a.sh @@ -0,0 +1,11 @@ +#!/bin/bash +# Copyright (c) Jupyter Development Team. +# Distributed under the terms of the Modified BSD License. + +echo "Started: a.sh" + +export OTHER_VAR=456 + +run-unknown-command + +echo "Finished: a.sh" diff --git a/tests/docker-stacks-foundation/run-hooks-failures/b.py b/tests/docker-stacks-foundation/run-hooks-failures/b.py new file mode 100755 index 0000000000..cc5b0a718e --- /dev/null +++ b/tests/docker-stacks-foundation/run-hooks-failures/b.py @@ -0,0 +1,12 @@ +#!/usr/bin/env python3 +# Copyright (c) Jupyter Development Team. +# Distributed under the terms of the Modified BSD License. +import os +import sys + +print("Started: b.py") +print(f"OTHER_VAR={os.environ['OTHER_VAR']}") + +sys.exit(1) + +print("Finished: b.py") diff --git a/tests/docker-stacks-foundation/run-hooks-failures/c.sh b/tests/docker-stacks-foundation/run-hooks-failures/c.sh new file mode 100644 index 0000000000..a71e69fc4d --- /dev/null +++ b/tests/docker-stacks-foundation/run-hooks-failures/c.sh @@ -0,0 +1,7 @@ +#!/bin/bash +# Copyright (c) Jupyter Development Team. +# Distributed under the terms of the Modified BSD License. + +echo "Started: c.sh" + +run-unknown-command diff --git a/tests/docker-stacks-foundation/run-hooks-failures/d.sh b/tests/docker-stacks-foundation/run-hooks-failures/d.sh new file mode 100644 index 0000000000..abc646a7e0 --- /dev/null +++ b/tests/docker-stacks-foundation/run-hooks-failures/d.sh @@ -0,0 +1,11 @@ +#!/bin/bash +# Copyright (c) Jupyter Development Team. +# Distributed under the terms of the Modified BSD License. + +set -e + +echo "Started: d.sh" + +run-unknown-command + +echo "Finished: d.sh" diff --git a/tests/docker-stacks-foundation/run-hooks-unset/a.sh b/tests/docker-stacks-foundation/run-hooks-unset/a.sh new file mode 100644 index 0000000000..61701e2a82 --- /dev/null +++ b/tests/docker-stacks-foundation/run-hooks-unset/a.sh @@ -0,0 +1,6 @@ +#!/bin/bash +# Copyright (c) Jupyter Development Team. +# Distributed under the terms of the Modified BSD License. + +export MY_VAR=123 +echo "Inside a.sh MY_VAR variable has ${MY_VAR} value" diff --git a/tests/docker-stacks-foundation/run-hooks-unset/b.sh b/tests/docker-stacks-foundation/run-hooks-unset/b.sh new file mode 100644 index 0000000000..ab64e932b2 --- /dev/null +++ b/tests/docker-stacks-foundation/run-hooks-unset/b.sh @@ -0,0 +1,7 @@ +#!/bin/bash +# Copyright (c) Jupyter Development Team. +# Distributed under the terms of the Modified BSD License. + +echo "Inside b.sh MY_VAR variable has ${MY_VAR} value" +echo "Unsetting MY_VAR" +unset MY_VAR diff --git a/tests/docker-stacks-foundation/run-hooks-unset/c.sh b/tests/docker-stacks-foundation/run-hooks-unset/c.sh new file mode 100644 index 0000000000..ef69df395f --- /dev/null +++ b/tests/docker-stacks-foundation/run-hooks-unset/c.sh @@ -0,0 +1,5 @@ +#!/bin/bash +# Copyright (c) Jupyter Development Team. +# Distributed under the terms of the Modified BSD License. + +echo "Inside c.sh MY_VAR variable has ${MY_VAR} value" diff --git a/tests/base-notebook/test_packages.py b/tests/docker-stacks-foundation/test_packages.py similarity index 78% rename from tests/base-notebook/test_packages.py rename to tests/docker-stacks-foundation/test_packages.py index dbdc631381..598a7de709 100644 --- a/tests/base-notebook/test_packages.py +++ b/tests/docker-stacks-foundation/test_packages.py @@ -4,7 +4,7 @@ """ test_packages ~~~~~~~~~~~~~~~ -This test module tests if R and Python packages installed can be imported. +This test module tests if the R and Python packages installed can be imported. It's a basic test aiming to prove that the package is working properly. The goal is to detect import errors that can be caused by incompatibilities between packages, for example: @@ -15,24 +15,26 @@ This module checks dynamically, through the `CondaPackageHelper`, only the requested packages i.e. packages requested by `mamba install` in the `Dockerfile`s. This means that it does not check dependencies. -This choice is a tradeoff to cover the main requirements while achieving reasonable test duration. -However it could be easily changed (or completed) to cover also dependencies. +This choice is a tradeoff to cover the main requirements while achieving a reasonable test duration. +However, it could be easily changed (or completed) to cover dependencies as well. Use `package_helper.installed_packages()` instead of `package_helper.requested_packages()`. Example: - $ make test/base-notebook + $ make test/docker-stacks-foundation # [...] - # test/test_packages.py::test_python_packages - # tests/base-notebook/test_packages.py::test_python_packages - # ---------------------------------------------------------------------------------------------- live log setup ---------------------------------------------------------------------------------------------- - # 2022-02-17 16:44:36 [ INFO] Starting container jupyter/base-notebook ... (package_helper.py:55) - # 2022-02-17 16:44:36 [ INFO] Running jupyter/base-notebook with args {'detach': True, 'tty': True, 'command': ['start.sh', 'bash', '-c', 'sleep infinity']} ... (conftest.py:95) - # 2022-02-17 16:44:37 [ INFO] Grabbing the list of manually requested packages ... (package_helper.py:83) - # ---------------------------------------------------------------------------------------------- live log call ----------------------------------------------------------------------------------------------- - # 2022-02-17 16:44:38 [ INFO] Testing the import of packages ... (test_packages.py:144) - # 2022-02-17 16:44:38 [ INFO] Trying to import mamba (test_packages.py:146) + # tests/docker-stacks-foundation/test_packages.py::test_python_packages + # -------------------------------- live log setup -------------------------------- + # 2024-01-21 17:46:43 [ INFO] Starting container quay.io/jupyter/docker-stacks-foundation ... (package_helper.py:55) + # 2024-01-21 17:46:43 [ INFO] Running quay.io/jupyter/docker-stacks-foundation with args {'detach': True, 'tty': True, 'command': ['bash', '-c', 'sleep infinity']} ... (conftest.py:99) + # 2024-01-21 17:46:44 [ INFO] Grabbing the list of manually requested packages ... (package_helper.py:83) + # -------------------------------- live log call --------------------------------- + # 2024-01-21 17:46:44 [ INFO] Testing the import of packages ... (test_packages.py:151) + # 2024-01-21 17:46:44 [ INFO] Trying to import mamba (test_packages.py:153) + # 2024-01-21 17:46:44 [ INFO] Trying to import jupyter_core (test_packages.py:153) + PASSED [ 17%] + # ------------------------------ live log teardown ------------------------------- # [...] """ @@ -52,6 +54,7 @@ PACKAGE_MAPPING = { # Python "beautifulsoup4": "bs4", + "jupyter-pluto-proxy": "jupyter_pluto_proxy", "matplotlib-base": "matplotlib", "pytables": "tables", "scikit-image": "skimage", @@ -68,15 +71,18 @@ "bzip2", "ca-certificates", "conda-forge::blas[build=openblas]", + "grpcio-status", + "grpcio", "hdf5", "jupyterlab-git", - "jupyter-pluto-proxy", + "mamba[version='<2.0.0']", "openssl", "pandas[version='>", "protobuf", "python", "r-irkernel", "unixodbc", + "jupyterlab-git", ] @@ -144,7 +150,7 @@ def _check_import_packages( """Test if packages can be imported Note: using a list of packages instead of a fixture for the list of packages - since pytest prevents use of multiple yields + since pytest prevents the use of multiple yields """ failures = {} LOGGER.info("Testing the import of packages ...") diff --git a/tests/docker-stacks-foundation/test_python.py b/tests/docker-stacks-foundation/test_python_version.py similarity index 57% rename from tests/docker-stacks-foundation/test_python.py rename to tests/docker-stacks-foundation/test_python_version.py index fe585b725b..559853abb6 100644 --- a/tests/docker-stacks-foundation/test_python.py +++ b/tests/docker-stacks-foundation/test_python_version.py @@ -17,8 +17,18 @@ def test_python_version(container: TrackedContainer) -> None: tty=True, command=["python", "--version"], ) - assert logs.startswith("Python ") - full_version = logs.split()[1] + python = next(line for line in logs.splitlines() if line.startswith("Python ")) + full_version = python.split()[1] major_minor_version = full_version[: full_version.rfind(".")] assert major_minor_version == EXPECTED_PYTHON_VERSION + + +def test_python_pinned_version(container: TrackedContainer) -> None: + LOGGER.info(f"Checking that pinned python version is {EXPECTED_PYTHON_VERSION}.*") + logs = container.run_and_wait( + timeout=5, + tty=True, + command=["cat", "/opt/conda/conda-meta/pinned"], + ) + assert f"python {EXPECTED_PYTHON_VERSION}.*" in logs diff --git a/tests/docker-stacks-foundation/test_run_hooks.py b/tests/docker-stacks-foundation/test_run_hooks.py new file mode 100644 index 0000000000..87467f9fb3 --- /dev/null +++ b/tests/docker-stacks-foundation/test_run_hooks.py @@ -0,0 +1,147 @@ +# Copyright (c) Jupyter Development Team. +# Distributed under the terms of the Modified BSD License. +import logging +from pathlib import Path + +from tests.conftest import TrackedContainer + +LOGGER = logging.getLogger(__name__) +THIS_DIR = Path(__file__).parent.resolve() + + +def test_run_hooks_zero_args(container: TrackedContainer) -> None: + logs = container.run_and_wait( + timeout=5, + tty=True, + no_failure=False, + command=["bash", "-c", "source /usr/local/bin/run-hooks.sh"], + ) + assert "Should pass exactly one directory" in logs + + +def test_run_hooks_two_args(container: TrackedContainer) -> None: + logs = container.run_and_wait( + timeout=5, + tty=True, + no_failure=False, + command=[ + "bash", + "-c", + "source /usr/local/bin/run-hooks.sh first-arg second-arg", + ], + ) + assert "Should pass exactly one directory" in logs + + +def test_run_hooks_missing_dir(container: TrackedContainer) -> None: + logs = container.run_and_wait( + timeout=5, + tty=True, + no_failure=False, + command=[ + "bash", + "-c", + "source /usr/local/bin/run-hooks.sh /tmp/missing-dir/", + ], + ) + assert "Directory /tmp/missing-dir/ doesn't exist or is not a directory" in logs + + +def test_run_hooks_dir_is_file(container: TrackedContainer) -> None: + logs = container.run_and_wait( + timeout=5, + tty=True, + no_failure=False, + command=[ + "bash", + "-c", + "touch /tmp/some-file && source /usr/local/bin/run-hooks.sh /tmp/some-file", + ], + ) + assert "Directory /tmp/some-file doesn't exist or is not a directory" in logs + + +def test_run_hooks_empty_dir(container: TrackedContainer) -> None: + container.run_and_wait( + timeout=5, + tty=True, + command=[ + "bash", + "-c", + "mkdir /tmp/empty-dir && source /usr/local/bin/run-hooks.sh /tmp/empty-dir/", + ], + ) + + +def run_source_in_dir( + container: TrackedContainer, + subdir: str, + command_suffix: str = "", + no_failure: bool = True, +) -> str: + host_data_dir = THIS_DIR / subdir + cont_data_dir = "/home/jovyan/data" + # https://forums.docker.com/t/all-files-appear-as-executable-in-file-paths-using-bind-mount/99921 + # Unfortunately, Docker treats all files in mounter dir as executable files + # So we make a copy of the mounted dir inside a container + command = ( + "cp -r /home/jovyan/data/ /home/jovyan/data-copy/ &&" + "source /usr/local/bin/run-hooks.sh /home/jovyan/data-copy/" + command_suffix + ) + return container.run_and_wait( + timeout=5, + volumes={str(host_data_dir): {"bind": cont_data_dir, "mode": "ro"}}, + tty=True, + no_failure=no_failure, + command=["bash", "-c", command], + ) + + +def test_run_hooks_executables(container: TrackedContainer) -> None: + logs = run_source_in_dir( + container, + subdir="run-hooks-executables", + command_suffix="&& echo SOME_VAR is ${SOME_VAR}", + ) + + assert "Executable python file was successfully run" in logs + assert "Ignoring non-executable: /home/jovyan/data-copy//non_executable.py" in logs + assert "SOME_VAR is 123" in logs + + +def test_run_hooks_with_failures(container: TrackedContainer) -> None: + logs = run_source_in_dir(container, subdir="run-hooks-failures", no_failure=False) + + for file in ["a.sh", "b.py", "c.sh", "d.sh"]: + assert f"Started: {file}" in logs + + for file in ["a.sh"]: + assert f"Finished: {file}" in logs + for file in ["b.py", "c.sh", "d.sh"]: + assert f"Finished: {file}" not in logs + + for file in ["b.py", "c.sh"]: + assert ( + f"/home/jovyan/data-copy//{file} has failed, continuing execution" in logs + ) + + assert "OTHER_VAR=456" in logs + + +def test_run_hooks_unset(container: TrackedContainer) -> None: + logs = run_source_in_dir(container, subdir="run-hooks-unset") + + assert "Inside a.sh MY_VAR variable has 123 value" in logs + assert "Inside b.sh MY_VAR variable has 123 value" in logs + assert "Unsetting MY_VAR" in logs + assert "Inside c.sh MY_VAR variable has value" in logs + + +def test_run_hooks_change(container: TrackedContainer) -> None: + logs = run_source_in_dir(container, subdir="run-hooks-change") + + assert "Inside a.sh MY_VAR variable has 123 value" in logs + assert "Inside b.sh MY_VAR variable has 123 value" in logs + assert "Changing value of MY_VAR" in logs + assert "After change inside b.sh MY_VAR variable has 456 value" in logs + assert "Inside c.sh MY_VAR variable has 456 value" in logs diff --git a/tests/docker-stacks-foundation/test_units.py b/tests/docker-stacks-foundation/test_units.py index 85d07862e1..cfdbc83dd8 100644 --- a/tests/docker-stacks-foundation/test_units.py +++ b/tests/docker-stacks-foundation/test_units.py @@ -10,7 +10,7 @@ def test_units(container: TrackedContainer) -> None: """Various units tests - Add a py file in the `tests/{somestack}-notebook/units` dir, and it will be automatically tested + Add a py file in the `tests/<somestack>/units` dir, and it will be automatically tested """ short_image_name = container.image_name[container.image_name.rfind("/") + 1 :] LOGGER.info(f"Running unit tests for: {short_image_name}") @@ -34,5 +34,5 @@ def test_units(container: TrackedContainer) -> None: timeout=30, volumes={str(host_data_dir): {"bind": cont_data_dir, "mode": "ro"}}, tty=True, - command=["start.sh", "python", f"{cont_data_dir}/{test_file_name}"], + command=["python", f"{cont_data_dir}/{test_file_name}"], ) diff --git a/tests/docker-stacks-foundation/test_user_options.py b/tests/docker-stacks-foundation/test_user_options.py index affba1802b..ee9614f98f 100644 --- a/tests/docker-stacks-foundation/test_user_options.py +++ b/tests/docker-stacks-foundation/test_user_options.py @@ -18,7 +18,7 @@ def test_uid_change(container: TrackedContainer) -> None: tty=True, user="root", environment=["NB_UID=1010"], - command=["start.sh", "bash", "-c", "id && touch /opt/conda/test-file"], + command=["bash", "-c", "id && touch /opt/conda/test-file"], ) assert "uid=1010(jovyan)" in logs @@ -30,7 +30,7 @@ def test_gid_change(container: TrackedContainer) -> None: tty=True, user="root", environment=["NB_GID=110"], - command=["start.sh", "id"], + command=["id"], ) assert "gid=110(jovyan)" in logs assert "groups=110(jovyan),100(users)" in logs @@ -86,7 +86,7 @@ def test_gid_change(container: TrackedContainer) -> None: def test_chown_extra(container: TrackedContainer) -> None: - """Container should change the UID/GID of a comma separated + """Container should change the UID/GID of a comma-separated CHOWN_EXTRA list of folders.""" logs = container.run_and_wait( timeout=120, # chown is slow so give it some time @@ -99,7 +99,6 @@ def test_chown_extra(container: TrackedContainer) -> None: "CHOWN_EXTRA_OPTS=-R", ], command=[ - "start.sh", "bash", "-c", "stat -c '%n:%u:%g' /home/jovyan/.bashrc /opt/conda/bin/jupyter", @@ -123,7 +122,7 @@ def test_chown_home(container: TrackedContainer) -> None: "NB_UID=1010", "NB_GID=101", ], - command=["start.sh", "bash", "-c", "stat -c '%n:%u:%g' /home/kitten/.bashrc"], + command=["bash", "-c", "stat -c '%n:%u:%g' /home/kitten/.bashrc"], ) assert "/home/kitten/.bashrc:1010:101" in logs @@ -135,7 +134,7 @@ def test_sudo(container: TrackedContainer) -> None: tty=True, user="root", environment=["GRANT_SUDO=yes"], - command=["start.sh", "sudo", "id"], + command=["sudo", "id"], ) assert "uid=0(root)" in logs @@ -147,7 +146,7 @@ def test_sudo_path(container: TrackedContainer) -> None: tty=True, user="root", environment=["GRANT_SUDO=yes"], - command=["start.sh", "sudo", "which", "jupyter"], + command=["sudo", "which", "jupyter"], ) assert logs.rstrip().endswith("/opt/conda/bin/jupyter") @@ -158,7 +157,7 @@ def test_sudo_path_without_grant(container: TrackedContainer) -> None: timeout=10, tty=True, user="root", - command=["start.sh", "which", "jupyter"], + command=["which", "jupyter"], ) assert logs.rstrip().endswith("/opt/conda/bin/jupyter") @@ -236,14 +235,14 @@ def test_container_not_delete_bind_mount( "CHOWN_HOME=yes", ], volumes={d: {"bind": "/home/jovyan/data", "mode": "rw"}}, - command=["start.sh", "ls"], + command=["ls"], ) assert p.read_text() == "some-content" assert len(list(tmp_path.iterdir())) == 1 @pytest.mark.parametrize("enable_root", [False, True]) -def test_jupyter_env_vars_to_unset_as_root( +def test_jupyter_env_vars_to_unset( container: TrackedContainer, enable_root: bool ) -> None: """Environment variables names listed in JUPYTER_ENV_VARS_TO_UNSET @@ -259,10 +258,9 @@ def test_jupyter_env_vars_to_unset_as_root( "SECRET_FRUIT=mango", ], command=[ - "start.sh", "bash", "-c", - "echo I like $FRUIT and ${SECRET_FRUIT:-stuff}, and love ${SECRET_ANIMAL:-to keep secrets}!", + "echo I like ${FRUIT} and ${SECRET_FRUIT:-stuff}, and love ${SECRET_ANIMAL:-to keep secrets}!", ], **root_args, # type: ignore ) @@ -270,7 +268,7 @@ def test_jupyter_env_vars_to_unset_as_root( def test_secure_path(container: TrackedContainer, tmp_path: pathlib.Path) -> None: - """Make sure that the sudo command has conda's python (not system's) on path. + """Make sure that the sudo command has conda's python (not system's) on PATH. See <https://github.com/jupyter/docker-stacks/issues/1053>. """ d = tmp_path / "data" @@ -284,7 +282,65 @@ def test_secure_path(container: TrackedContainer, tmp_path: pathlib.Path) -> Non tty=True, user="root", volumes={p: {"bind": "/usr/bin/python", "mode": "ro"}}, - command=["start.sh", "python", "--version"], + command=["python", "--version"], ) assert "Wrong python" not in logs assert "Python" in logs + + +def test_startsh_multiple_exec(container: TrackedContainer) -> None: + """If start.sh is executed multiple times check that configuration only occurs once.""" + logs = container.run_and_wait( + timeout=10, + no_warnings=False, + tty=True, + user="root", + environment=["GRANT_SUDO=yes"], + command=["start.sh", "sudo", "id"], + ) + assert "uid=0(root)" in logs + warnings = TrackedContainer.get_warnings(logs) + assert len(warnings) == 1 + assert ( + "WARNING: start.sh is the default ENTRYPOINT, do not include it in CMD" + in warnings[0] + ) + + +def test_rootless_triplet_change(container: TrackedContainer) -> None: + """Container should change the username (`NB_USER`), the UID and the GID of the default user.""" + logs = container.run_and_wait( + timeout=10, + tty=True, + user="root", + environment=["NB_USER=root", "NB_UID=0", "NB_GID=0"], + command=["id"], + ) + assert "uid=0(root)" in logs + assert "gid=0(root)" in logs + assert "groups=0(root)" in logs + + +def test_rootless_triplet_home(container: TrackedContainer) -> None: + """Container should change the home directory for triplet NB_USER=root, NB_UID=0, NB_GID=0.""" + logs = container.run_and_wait( + timeout=10, + tty=True, + user="root", + environment=["NB_USER=root", "NB_UID=0", "NB_GID=0"], + command=["bash", "-c", "echo HOME=${HOME} && getent passwd root"], + ) + assert "HOME=/home/root" in logs + assert "root:x:0:0:root:/home/root:/bin/bash" in logs + + +def test_rootless_triplet_sudo(container: TrackedContainer) -> None: + """Container should not be started with sudo for triplet NB_USER=root, NB_UID=0, NB_GID=0.""" + logs = container.run_and_wait( + timeout=10, + tty=True, + user="root", + environment=["NB_USER=root", "NB_UID=0", "NB_GID=0"], + command=["env"], + ) + assert "SUDO" not in logs diff --git a/tests/images_hierarchy.py b/tests/images_hierarchy.py index ff3797c323..0ecdcc8351 100644 --- a/tests/images_hierarchy.py +++ b/tests/images_hierarchy.py @@ -16,6 +16,8 @@ "r-notebook": "minimal-notebook", "julia-notebook": "minimal-notebook", "tensorflow-notebook": "scipy-notebook", + "pytorch-notebook": "scipy-notebook", + "datascience-notebook": "scipy-notebook", "pyspark-notebook": "scipy-notebook", "all-spark-notebook": "pyspark-notebook", } diff --git a/tests/julia-notebook/test_pluto.py b/tests/julia-notebook/test_pluto.py new file mode 100644 index 0000000000..27c4aaf0d3 --- /dev/null +++ b/tests/julia-notebook/test_pluto.py @@ -0,0 +1,13 @@ +# Copyright (c) Jupyter Development Team. +# Distributed under the terms of the Modified BSD License. +import requests + +from tests.conftest import TrackedContainer +from tests.pluto_check import check_pluto_proxy + + +def test_pluto_proxy( + container: TrackedContainer, http_client: requests.Session +) -> None: + """Pluto proxy starts Pluto correctly""" + check_pluto_proxy(container, http_client) diff --git a/tests/minimal-notebook/data/notebook_math.ipynb b/tests/minimal-notebook/data/notebook_math.ipynb index 5b028b1dde..019d645f2d 100644 --- a/tests/minimal-notebook/data/notebook_math.ipynb +++ b/tests/minimal-notebook/data/notebook_math.ipynb @@ -2,7 +2,7 @@ "cells": [ { "cell_type": "markdown", - "id": "a69ceb22", + "id": "0", "metadata": {}, "source": [ "# A simple SymPy example" @@ -10,7 +10,7 @@ }, { "cell_type": "markdown", - "id": "3c43c88e", + "id": "1", "metadata": {}, "source": [ "First we import SymPy and initialize printing:" @@ -19,7 +19,7 @@ { "cell_type": "code", "execution_count": null, - "id": "7b561917", + "id": "2", "metadata": { "jupyter": { "outputs_hidden": false @@ -33,7 +33,7 @@ { "cell_type": "code", "execution_count": null, - "id": "d6454356-d5a6-481f-aaac-9abcc101026a", + "id": "3", "metadata": {}, "outputs": [], "source": [ @@ -42,7 +42,7 @@ }, { "cell_type": "markdown", - "id": "fbe0a2f3", + "id": "4", "metadata": {}, "source": [ "Create a few symbols:" @@ -51,7 +51,7 @@ { "cell_type": "code", "execution_count": null, - "id": "c99d7f17", + "id": "5", "metadata": { "jupyter": { "outputs_hidden": false @@ -64,7 +64,7 @@ }, { "cell_type": "markdown", - "id": "f61dddac", + "id": "6", "metadata": {}, "source": [ "Here is a basic expression:" @@ -73,7 +73,7 @@ { "cell_type": "code", "execution_count": null, - "id": "0cfde73c", + "id": "7", "metadata": { "jupyter": { "outputs_hidden": false @@ -88,7 +88,7 @@ { "cell_type": "code", "execution_count": null, - "id": "cb7eb1ad", + "id": "8", "metadata": { "jupyter": { "outputs_hidden": false @@ -102,7 +102,7 @@ { "cell_type": "code", "execution_count": null, - "id": "07441ea9", + "id": "9", "metadata": { "jupyter": { "outputs_hidden": false diff --git a/tests/minimal-notebook/test_nbconvert.py b/tests/minimal-notebook/test_nbconvert.py index 33954cb00f..9c1c017be7 100644 --- a/tests/minimal-notebook/test_nbconvert.py +++ b/tests/minimal-notebook/test_nbconvert.py @@ -11,15 +11,8 @@ THIS_DIR = Path(__file__).parent.resolve() -@pytest.mark.parametrize( - "test_file, output_format", - [ - ("notebook_math", "pdf"), - ("notebook_math", "html"), - ("notebook_svg", "pdf"), - ("notebook_svg", "html"), - ], -) +@pytest.mark.parametrize("test_file", ["notebook_math", "notebook_svg"]) +@pytest.mark.parametrize("output_format", ["pdf", "html"]) def test_nbconvert( container: TrackedContainer, test_file: str, output_format: str ) -> None: @@ -35,7 +28,7 @@ def test_nbconvert( timeout=30, volumes={str(host_data_dir): {"bind": cont_data_dir, "mode": "ro"}}, tty=True, - command=["start.sh", "bash", "-c", command], + command=["bash", "-c", command], ) expected_file = f"{output_dir}/{test_file}.{output_format}" assert expected_file in logs, f"Expected file {expected_file} not generated" diff --git a/tests/package_helper.py b/tests/package_helper.py index 7524fe0aef..5f2f636c79 100644 --- a/tests/package_helper.py +++ b/tests/package_helper.py @@ -22,13 +22,13 @@ # OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE # SOFTWARE. -import json import logging import re from collections import defaultdict from itertools import chain from typing import Any, Optional +import yaml from docker.models.containers import Container from tabulate import tabulate @@ -55,13 +55,13 @@ def start_container(container: TrackedContainer) -> Container: LOGGER.info(f"Starting container {container.image_name} ...") return container.run_detached( tty=True, - command=["start.sh", "bash", "-c", "sleep infinity"], + command=["bash", "-c", "sleep infinity"], ) @staticmethod def _conda_export_command(from_history: bool) -> list[str]: """Return the mamba export command with or without history""" - cmd = ["mamba", "env", "export", "-n", "base", "--json", "--no-builds"] + cmd = ["mamba", "env", "export", "--no-build"] if from_history: cmd.append("--from-history") return cmd @@ -70,7 +70,7 @@ def installed_packages(self) -> dict[str, set[str]]: """Return the installed packages""" if self.installed is None: LOGGER.info("Grabbing the list of installed packages ...") - self.installed = CondaPackageHelper._packages_from_json( + self.installed = CondaPackageHelper._parse_package_versions( self._execute_command( CondaPackageHelper._conda_export_command(from_history=False) ) @@ -81,7 +81,7 @@ def requested_packages(self) -> dict[str, set[str]]: """Return the requested package (i.e. `mamba install <package>`)""" if self.requested is None: LOGGER.info("Grabbing the list of manually requested packages ...") - self.requested = CondaPackageHelper._packages_from_json( + self.requested = CondaPackageHelper._parse_package_versions( self._execute_command( CondaPackageHelper._conda_export_command(from_history=True) ) @@ -94,12 +94,12 @@ def _execute_command(self, command: list[str]) -> str: return rc.output.decode("utf-8") # type: ignore @staticmethod - def _packages_from_json(env_export: str) -> dict[str, set[str]]: + def _parse_package_versions(env_export: str) -> dict[str, set[str]]: """Extract packages and versions from the lines returned by the list of specifications""" - # dependencies = filter(lambda x: isinstance(x, str), json.loads(env_export).get("dependencies")) - dependencies = json.loads(env_export).get("dependencies") - # Filtering packages installed through pip in this case it's a dict {'pip': ['toree==0.3.0']} - # Since we only manage packages installed through mamba here + dependencies = yaml.safe_load(env_export).get("dependencies") + # Filtering packages installed through pip + # since we only manage packages installed through mamba here + # They are represented by a dict with a key 'pip' dependencies = filter(lambda x: isinstance(x, str), dependencies) packages_dict: dict[str, set[str]] = dict() for split in map(lambda x: re.split("=?=", x), dependencies): diff --git a/tests/pluto_check.py b/tests/pluto_check.py new file mode 100644 index 0000000000..48116db472 --- /dev/null +++ b/tests/pluto_check.py @@ -0,0 +1,30 @@ +# Copyright (c) Jupyter Development Team. +# Distributed under the terms of the Modified BSD License. +import logging +import secrets +import time + +import requests + +from tests.conftest import TrackedContainer, find_free_port + +LOGGER = logging.getLogger(__name__) + + +def check_pluto_proxy( + container: TrackedContainer, http_client: requests.Session +) -> None: + host_port = find_free_port() + token = secrets.token_hex() + container.run_detached( + command=[ + "start-notebook.py", + f"--IdentityProvider.token={token}", + ], + ports={"8888/tcp": host_port}, + ) + # Give the server a bit of time to start + time.sleep(2) + resp = http_client.get(f"http://localhost:{host_port}/pluto?token={token}") + resp.raise_for_status() + assert "Pluto.jl notebooks" in resp.text, "Pluto.jl text not found in /pluto page" diff --git a/tests/pyspark-notebook/units/unit_pandas_version.py b/tests/pyspark-notebook/units/unit_pandas_version.py index 1728effa35..03920db4b4 100644 --- a/tests/pyspark-notebook/units/unit_pandas_version.py +++ b/tests/pyspark-notebook/units/unit_pandas_version.py @@ -2,4 +2,4 @@ # Distributed under the terms of the Modified BSD License. import pandas -assert pandas.__version__ == "1.5.3" +assert pandas.__version__ == "2.0.3" diff --git a/tests/pytorch-notebook/units/unit_pytorch.py b/tests/pytorch-notebook/units/unit_pytorch.py new file mode 100644 index 0000000000..1b739a5924 --- /dev/null +++ b/tests/pytorch-notebook/units/unit_pytorch.py @@ -0,0 +1,5 @@ +# Copyright (c) Jupyter Development Team. +# Distributed under the terms of the Modified BSD License. +import torch + +print(torch.tensor([[1.0, 4.0, 7.0], [4.0, 9.0, 11.0]])) diff --git a/tests/run_command.py b/tests/run_command.py index 40f343d0fb..48e3cc07cd 100644 --- a/tests/run_command.py +++ b/tests/run_command.py @@ -18,5 +18,5 @@ def run_command( return container.run_and_wait( timeout=timeout, tty=True, - command=["start.sh", "bash", "-c", command], + command=["bash", "-c", command], ) diff --git a/tests/run_tests.py b/tests/run_tests.py index f8ab85f7be..90529e7731 100755 --- a/tests/run_tests.py +++ b/tests/run_tests.py @@ -13,11 +13,11 @@ LOGGER = logging.getLogger(__name__) -def test_image(short_image_name: str, owner: str) -> None: +def test_image(short_image_name: str, registry: str, owner: str) -> None: LOGGER.info(f"Testing image: {short_image_name}") test_dirs = get_test_dirs(short_image_name) LOGGER.info(f"Test dirs to be run: {test_dirs}") - with plumbum.local.env(TEST_IMAGE=f"{owner}/{short_image_name}"): + with plumbum.local.env(TEST_IMAGE=f"{registry}/{owner}/{short_image_name}"): ( python3[ "-m", @@ -39,7 +39,14 @@ def test_image(short_image_name: str, owner: str) -> None: arg_parser.add_argument( "--short-image-name", required=True, - help="Short image name to run test on", + help="Short image name", + ) + arg_parser.add_argument( + "--registry", + required=True, + type=str, + choices=["docker.io", "quay.io"], + help="Image registry", ) arg_parser.add_argument( "--owner", @@ -49,4 +56,4 @@ def test_image(short_image_name: str, owner: str) -> None: args = arg_parser.parse_args() - test_image(args.short_image_name, args.owner) + test_image(args.short_image_name, args.registry, args.owner) diff --git a/tests/scipy-notebook/data/matplotlib/matplotlib_1.py b/tests/scipy-notebook/data/matplotlib/matplotlib_1.py index e1a0add681..a7d98fbbfc 100644 --- a/tests/scipy-notebook/data/matplotlib/matplotlib_1.py +++ b/tests/scipy-notebook/data/matplotlib/matplotlib_1.py @@ -1,8 +1,7 @@ +# type: ignore # Matplotlib: Create a simple plot example. # Refs: https://matplotlib.org/stable/gallery/lines_bars_and_markers/simple_plot.html -import os - # Optional test with [Matplotlib Jupyter Integration](https://github.com/matplotlib/ipympl) # %matplotlib widget import matplotlib.pyplot as plt @@ -21,7 +20,8 @@ title="About as simple as it gets, folks", ) ax.grid() + # Note that the test can be run headless by checking if an image is produced -file_path = os.path.join("/tmp", "test.png") +file_path = "/tmp/test.png" fig.savefig(file_path) print(f"File {file_path} saved") diff --git a/tests/scipy-notebook/data/matplotlib/matplotlib_fonts_1.py b/tests/scipy-notebook/data/matplotlib/matplotlib_fonts_1.py index 951a3d87c4..8944f2b8ff 100644 --- a/tests/scipy-notebook/data/matplotlib/matplotlib_fonts_1.py +++ b/tests/scipy-notebook/data/matplotlib/matplotlib_fonts_1.py @@ -1,6 +1,4 @@ # Matplotlib: Test tex fonts -import os - import matplotlib import matplotlib.pyplot as plt @@ -22,6 +20,6 @@ ax.plot(x, y, label="a label") ax.legend(fontsize=15) -file_path = os.path.join("/tmp", "test_fonts.png") +file_path = "/tmp/test_fonts.png" fig.savefig(file_path) print(f"File {file_path} saved") diff --git a/tests/scipy-notebook/test_cython.py b/tests/scipy-notebook/test_cython.py index 839d83a170..092271ba5c 100644 --- a/tests/scipy-notebook/test_cython.py +++ b/tests/scipy-notebook/test_cython.py @@ -16,10 +16,9 @@ def test_cython(container: TrackedContainer) -> None: volumes={str(host_data_dir): {"bind": cont_data_dir, "mode": "ro"}}, tty=True, command=[ - "start.sh", "bash", "-c", - # We copy our data to temporary folder to be able to modify the directory + # We copy our data to a temporary folder to be able to modify the directory f"cp -r {cont_data_dir}/ /tmp/test/ && cd /tmp/test && python3 setup.py build_ext", ], ) diff --git a/tests/scipy-notebook/test_extensions.py b/tests/scipy-notebook/test_extensions.py index b77dc89fec..d90cc62290 100644 --- a/tests/scipy-notebook/test_extensions.py +++ b/tests/scipy-notebook/test_extensions.py @@ -9,7 +9,7 @@ LOGGER = logging.getLogger(__name__) -@pytest.mark.skip(reason="Not yet compliant with JupyterLab 3") +@pytest.mark.skip(reason="Not yet compliant with JupyterLab 4") @pytest.mark.parametrize( "extension", [ @@ -21,7 +21,7 @@ def test_check_extension(container: TrackedContainer, extension: str) -> None: """Basic check of each extension - The list of extensions can be obtained through this command + The list of installed extensions can be obtained through this command: $ jupyter labextension list @@ -30,5 +30,5 @@ def test_check_extension(container: TrackedContainer, extension: str) -> None: container.run_and_wait( timeout=10, tty=True, - command=["start.sh", "jupyter", "labextension", "check", extension], + command=["jupyter", "labextension", "check", extension], ) diff --git a/tests/scipy-notebook/test_matplotlib.py b/tests/scipy-notebook/test_matplotlib.py index 27c6d3c34c..e96bc8c859 100644 --- a/tests/scipy-notebook/test_matplotlib.py +++ b/tests/scipy-notebook/test_matplotlib.py @@ -17,7 +17,7 @@ ( "matplotlib_1.py", "test.png", - "Test that matplotlib is able to plot a graph and write it as an image ...", + "Test that matplotlib can plot a graph and write it as an image ...", ), ( "matplotlib_fonts_1.py", @@ -38,11 +38,10 @@ def test_matplotlib( cont_data_dir = "/home/jovyan/data" output_dir = "/tmp" LOGGER.info(description) - command = "sleep infinity" running_container = container.run_detached( volumes={str(host_data_dir): {"bind": cont_data_dir, "mode": "ro"}}, tty=True, - command=["start.sh", "bash", "-c", command], + command=["bash", "-c", "sleep infinity"], ) command = f"python {cont_data_dir}/{test_file}" cmd = running_container.exec_run(command) diff --git a/tests/scipy-notebook/units/unit_pandas.py b/tests/scipy-notebook/units/unit_pandas.py index 2190a0b5c8..a211de501e 100644 --- a/tests/scipy-notebook/units/unit_pandas.py +++ b/tests/scipy-notebook/units/unit_pandas.py @@ -1,5 +1,6 @@ # Copyright (c) Jupyter Development Team. # Distributed under the terms of the Modified BSD License. +# type: ignore import numpy as np import pandas as pd