Documentation Automation (intel#273)

* init rtd

* update config

* switch to mkdocs

* pre-commit

* test

* add metadata

* add table generation for python and pytorch

* rerun pre-commit

* add comments and exclusionary functionality

* utilize new python base svc

* return isort cfg

* fix pre-commit isort config

* add horizontal frame building

* add get_repo method

* add docs ci

* swap to internal runners

* remove folder filtering on pr

* add container labelling scheme

* add concurrency

* add job read clause

* add reqs install step

* return dockerhub docs

* update mkdocs config

* add readmes to site

* refactor scripts

* re-enable mkdocs hook

* add improved python dep tracking

* correct reqs

* update xpu reqs

* refac 2

* return proxies

* remove import regression

* return depends_on to remove regression

* Update Dockerfile

* Update docker-compose.yaml

* Update Dockerfile

.github/dependabot.yml

@@ -63,3 +63,11 @@ updates:
     package-ecosystem: pip
     schedule:
       interval: weekly
+  - directory: /docs
+    groups:
+      python-requirements:
+        patterns:
+          - "*"
+    package-ecosystem: pip
+    schedule:
+      interval: weekly

.github/linters/.python-lint

@@ -1,2 +1,2 @@
 [MESSAGES CONTROL]
-disable=line-too-long, missing-module-docstring, redefined-outer-name, no-member, used-before-assignment, consider-using-with, bare-except, consider-using-dict-items, import-error, broad-exception-caught, raise-missing-from, not-an-iterable, too-few-public-methods
+disable=line-too-long, missing-module-docstring, redefined-outer-name, no-member, used-before-assignment, consider-using-with, bare-except, consider-using-dict-items, import-error, broad-exception-caught, raise-missing-from, not-an-iterable, too-few-public-methods, unused-argument, unsubscriptable-object, cell-var-from-loop

.github/workflows/docs.yaml

@@ -0,0 +1,46 @@
+# Copyright (c) 2024 Intel Corporation
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#    http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+name: Docs
+on:
+  merge_group: null
+  pull_request: null
+  push:
+    branches:
+    - main
+permissions: read-all
+concurrency:
+  group: ${{ github.workflow }}-${{ github.ref }}
+  cancel-in-progress: true
+jobs:
+  mkdocs:
+    runs-on: k8-runners # ubuntu-latest
+    permissions:
+      contents: read
+      pages: write
+    steps:
+      - uses: actions/checkout@v4
+      - uses: actions/setup-python@v5
+        with:
+          python-version: 3.8
+          cache: pip
+      - name: Install python requirements
+        run: python -m pip install -r docs/requirements.txt
+      - name: Build
+        run: mkdocs build --clean
+      - uses: actions/upload-pages-artifact@v3
+        with:
+          path: site
+      - if: github.ref == format('refs/heads/{0}', github.event.repository.default_branch)
+        uses: actions/deploy-pages@v4

.github/workflows/lint.yaml

@@ -40,6 +40,7 @@ jobs:
           GITHUB_ACTIONS_COMMAND_ARGS: '-ignore SC.*'
           VALIDATE_BASH_EXEC: false
           VALIDATE_CHECKOV: false
+          VALIDATE_HTML: false
           VALIDATE_KUBERNETES_KUBECONFORM: false
           VALIDATE_NATURAL_LANGUAGE: false
           VALIDATE_PYTHON_FLAKE8: false

.github/workflows/test-runner-ci.yaml

@@ -14,13 +14,13 @@
 
 name: Test Runner CI
 on:
-  push:
-    branches:
-    - main
   merge_group: null
   pull_request:
     paths:
     - 'test-runner/**'
+  push:
+    branches:
+    - main
 permissions: read-all
 concurrency:
   group: ${{ github.workflow }}-${{ github.ref }}
@@ -46,7 +46,7 @@ jobs:
       with:
         python-version: ${{ matrix.python }}
     - name: Install requirements
-      run: python -m pip install -U pip tox tox-gh-actions
+      run: python -m pip install -U pip tox-gh-actions
     - name: Tox
       run: python -m tox
       env:

.gitignore

@@ -1,3 +1,4 @@
+.coverage*
 .env
 .pytest*
 .secrets
@@ -6,7 +7,9 @@
 **__pycache__**
 **.ipynb_checkpoints**
 **/.hypothesis
+docs/assets
+docs/repos/
 logs/
 output/
-.coverage*
+site
 venv/

.pre-commit-config.yaml

@@ -1,94 +1,81 @@
----
 ci:
   autofix_commit_msg: "[pre-commit.ci] auto fixes from pre-commit.com hooks"
   autofix_prs: true
   autoupdate_commit_msg: "[pre-commit.ci] pre-commit autoupdate"
   autoupdate_schedule: "weekly"
-  skip:
-    [
-      pylint,
-      shellcheck,
-      markdownlint,
-      hadolint-docker,
-      shfmt,
-      tox
-    ]
+  skip: [pylint, shellcheck, markdownlint, hadolint-docker, shfmt, tox]
 repos:
-  - hooks:
-      - id: check-added-large-files
-      - id: check-ast
-      - id: check-merge-conflict
-      - id: debug-statements
-      - id: end-of-file-fixer
-      - id: forbid-submodules
-      - id: sort-simple-yaml
-      - id: trailing-whitespace
-    repo: https://github.com/pre-commit/pre-commit-hooks
-    rev: v4.6.0
-  - hooks:
-      - id: gitleaks
-    repo: https://github.com/gitleaks/gitleaks
-    rev: v8.18.2
-  - hooks:
-      - args:
-          [
-            --license-filepath=.github/license_template.txt,
-            --use-current-year,
-            --detect-license-in-X-top-lines=40,
-            --skip-license-insertion-comment=Copyright
-          ]
-        files: |
-          (?x)^(
-            .*(py|yaml|yml|sh|Dockerfile)|
-          )$
-        id: insert-license
-    repo: https://github.com/Lucas-C/pre-commit-hooks
-    rev: v1.5.5
-  - hooks:
-      - args: [ "--config", ".github/linters/.hadolint.yaml" ]
-        id: hadolint-docker
-    repo: https://github.com/hadolint/hadolint
-    rev: v2.12.1-beta
-  - hooks:
-      - args: [ "--config", ".github/linters/.markdown-lint.yaml" ]
-        id: markdownlint
-    repo: https://github.com/igorshubovych/markdownlint-cli
-    rev: v0.39.0
-  - hooks:
-      - id: shellcheck
-    repo: https://github.com/koalaman/shellcheck-precommit
-    rev: v0.10.0
-  - hooks:
-      - args: [ "--config-file", ".github/linters/actionlint.yml" ]
-        id: actionlint
-    repo: https://github.com/rhysd/actionlint
-    rev: v1.6.27
-  - hooks:
-      - id: shfmt
-    repo: https://github.com/cisagov/pre-commit-shfmt
-    rev: v0.0.2
-  - hooks:
-      - id: black
-    repo: https://github.com/ambv/black
-    rev: 24.4.0
-  - hooks:
-      - args: [ "--profile=black" ]
-        id: isort
-    repo: https://github.com/pycqa/isort
-    rev: 5.13.2
-  - hooks:
-      - args: [ "--rcfile=.github/linters/.python-lint" ]
-        entry: pylint
-        id: pylint
-        language: system
-        name: pylint
-        types: [ python ]
-      - entry: bash -c "tox -e py310"
-        id: tox
-        files: |
-          (?x)^(
-            ./test-runner/.*
-          )$
-        language: system
-        name: tox
-    repo: local
+- hooks:
+  - id: check-added-large-files
+  - id: check-ast
+  - id: check-merge-conflict
+  - id: debug-statements
+  - id: end-of-file-fixer
+  - id: forbid-submodules
+  - id: sort-simple-yaml
+  - id: trailing-whitespace
+  repo: https://github.com/pre-commit/pre-commit-hooks
+  rev: v4.6.0
+- hooks:
+  - id: gitleaks
+  repo: https://github.com/gitleaks/gitleaks
+  rev: v8.18.2
+- hooks:
+  - args: [--license-filepath=.github/license_template.txt, --use-current-year, --detect-license-in-X-top-lines=40, --skip-license-insertion-comment=Copyright]
+    files: |
+      (?x)^(
+        .*(py|yaml|yml|sh|Dockerfile)|
+      )$
+    id: insert-license
+  repo: https://github.com/Lucas-C/pre-commit-hooks
+  rev: v1.5.5
+- hooks:
+  - args: ["--config", ".github/linters/.hadolint.yaml"]
+    id: hadolint-docker
+  repo: https://github.com/hadolint/hadolint
+  rev: v2.13.0-beta
+- hooks:
+  - args: ["--config", ".github/linters/.markdown-lint.yaml"]
+    id: markdownlint
+  repo: https://github.com/igorshubovych/markdownlint-cli
+  rev: v0.40.0
+- hooks:
+  - id: shellcheck
+  repo: https://github.com/koalaman/shellcheck-precommit
+  rev: v0.10.0
+- hooks:
+  - args: ["--config-file", ".github/linters/actionlint.yml"]
+    id: actionlint
+  repo: https://github.com/rhysd/actionlint
+  rev: v1.6.27
+- hooks:
+  - id: shfmt
+  repo: https://github.com/cisagov/pre-commit-shfmt
+  rev: v0.0.2
+- hooks:
+  - id: black
+  repo: https://github.com/ambv/black
+  rev: 24.4.2
+- hooks:
+  - args: ["--sp", ".github/linters"]
+    id: isort
+  repo: https://github.com/pycqa/isort
+  rev: 5.13.2
+- hooks:
+  - args: ["--rcfile=.github/linters/.python-lint"]
+    entry: pylint
+    id: pylint
+    language: system
+    name: pylint
+    types: [python]
+  - entry: bash -c "python -m tox -e py310"
+    files: ^test-runner/
+    id: tox
+    language: system
+    name: tox
+  - entry: bash -c "mkdocs build --clean"
+    # files: ^docs/
+    id: mkdocs
+    language: system
+    name: mkdocs
+  repo: local

CONTRIBUTING.md

@@ -16,7 +16,7 @@ This project follows the [Contributor Covenant Code of Conduct](CODE_OF_CONDUCT.
 2. Create a new branch for your contribution: `git checkout -b feature/your-feature`.
 3. Install [pre-commit](https://pre-commit.com/), [Docker](https://docs.docker.com/engine/install/), and [Python 3.8+](https://www.python.org/downloads/).
 4. Follow the [Project Setup](README.md#project-setup) steps.
-5. Install the third-party python dependencies necessary for pre-commit with `pip install -r test-runner/dev-requirements.txt && pip install tox`.
+5. Install the third-party python dependencies necessary for pre-commit with `pip install -r test-runner/dev-requirements.txt && pip install -r docs/requirements.txt`.
 6. Make your changes, commit, and sign your changes: `git commit -s -m 'Add your feature'`.
 7. Push to the branch: `git push origin feature/your-feature`.
 8. Submit a pull request.

README.md

@@ -35,7 +35,7 @@ Ensure you have Docker Compose installed on your machine. If you don't have this
 ```bash
 DOCKER_CONFIG=${DOCKER_CONFIG:-$HOME/.docker}
 mkdir -p $DOCKER_CONFIG/cli-plugins
-curl -SL https://github.com/docker/compose/releases/download/v2.25.0/docker-compose-linux-x86_64 -o $DOCKER_CONFIG/cli-plugins/docker-compose
+curl -SL https://github.com/docker/compose/releases/download/v2.26.1/docker-compose-linux-x86_64 -o $DOCKER_CONFIG/cli-plugins/docker-compose
 chmod +x $DOCKER_CONFIG/cli-plugins/docker-compose
 docker compose version
 ```

classical-ml/docker-compose.yaml

@@ -37,7 +37,7 @@ services:
     command: >
       bash -c "python -c 'import sklearnex, sklearn; import xgboost as xgb; print(\"Scikit version:\", sklearn.__version__, \"\\nXGBoost version:\", xgb.__version__)'"
     depends_on:
-      - ${PACKAGE_OPTION:-pip}-base
+      - ${PACKAGE_OPTION:-pip}
     image: ${REGISTRY}/${REPO}:b-${GITHUB_RUN_NUMBER:-0}-${BASE_IMAGE_NAME:-ubuntu}-${BASE_IMAGE_TAG:-22.04}-${PACKAGE_OPTION:-pip}-py${PYTHON_VERSION:-3.10}-scikit-learn-${SCIKIT_VERSION:-2024.3.0}-xgboost-${XGBOOST_VERSION:-2.0.3}-base
     pull_policy: always
   jupyter: