diff --git a/.github/workflows/black.yml b/.github/workflows/black.yml index 3e7fd0864..40790685e 100644 --- a/.github/workflows/black.yml +++ b/.github/workflows/black.yml @@ -9,7 +9,7 @@ jobs: matrix: python-version: [3.9] steps: - - uses: actions/checkout@v2 + - uses: actions/checkout@v4 - name: Extract black version from setup.py run: | diff --git a/.github/workflows/codacy.yml b/.github/workflows/codacy.yml index de9c6eba7..7e1aefbbd 100644 --- a/.github/workflows/codacy.yml +++ b/.github/workflows/codacy.yml @@ -31,4 +31,4 @@ jobs: steps: # Checkout the repository to the GitHub Actions runner - name: Checkout code - uses: actions/checkout@v2 + uses: actions/checkout@v4 diff --git a/.github/workflows/codeql-analysis.yml b/.github/workflows/codeql-analysis.yml index 4a8c4338f..ceab64ca5 100644 --- a/.github/workflows/codeql-analysis.yml +++ b/.github/workflows/codeql-analysis.yml @@ -37,7 +37,7 @@ jobs: steps: - name: Checkout repository - uses: actions/checkout@v2 + uses: actions/checkout@v4 # Initializes the CodeQL tools for scanning. - name: Initialize CodeQL diff --git a/.github/workflows/dependencies/action.yml b/.github/workflows/dependencies/action.yml new file mode 100644 index 000000000..32f862b54 --- /dev/null +++ b/.github/workflows/dependencies/action.yml @@ -0,0 +1,104 @@ +name: 'Install Dependencies' + +description: 'Setup and install dependencies for GaNDLF' + +outputs: + other_modified_files_count: + description: "Which files have changed" + value: ${{ steps.changed-files.outputs.other_modified_files_count }} +# on: +# workflow_call: + +# jobs: +# install_dependencies: +# runs-on: ubuntu-latest +runs: + using: "composite" + steps: + - name: Free space + shell: bash + run: | + df -h + sudo rm -rf /opt/ghc + sudo rm -rf "/usr/local/share/boost" + sudo rm -rf "$AGENT_TOOLSDIRECTORY" + sudo rm -rf "$ANDROID_SDK_ROOT" + df -h + + # - name: Checkout code + # uses: actions/checkout@v4 + + # Use changed-files-specific action to collect file changes. + # The following commented condition applied to a step will run that step only if non-docs files have changed. + # It should be applied to all functionality-related steps. + # if: ${{steps.dependencies.outputs.other_modified_files_count > 0}} + - name: Detect and screen file changes + # shell: bash + id: changed-files-specific + uses: tj-actions/changed-files@v44 + with: + files: | + .github/*.md + .github/ISSUE_TEMPLATE/*.md + .github/workflows/devcontainer.yml + .github/workflows/docker-image.yml + .devcontainer/** + docs/** + mlcube/** + *.md + LICENSE + Dockerfile-* + # run: | + # echo "other_modified_files_count=$(echo ${{ steps.changed-files-specific.outputs.other_modified_files_count }})" >> $GITHUB_OUTPUT + + - name: Summarize docs and non-docs modifications + id: changed-files + shell: bash + run: | + echo "List of docs files that have changed: ${{ steps.changed-files-specific.outputs.all_modified_files }}" + echo "Changed non-docs files: ${{ steps.changed-files-specific.outputs.other_modified_files }}" + echo "Count of non-docs files changed: ${{ steps.changed-files-specific.outputs.other_modified_files_count }}" + echo "If only-modified is triggered: ${{ steps.changed-files-specific.outputs.only_modified }}" + echo "other_modified_files_count=${{ steps.changed-files-specific.outputs.other_modified_files_count }}" >> $GITHUB_OUTPUT + echo "$GITHUB_OUTPUT" + + + ## this did NOT work + # - name: Set output + # shell: bash + # run: echo "other_modified_files_count=${{ steps.changed-files-specific.outputs.other_modified_files_count }}" >> $GITHUB_OUTPUT + + ## this did NOT work + # - name: Check saved output + # shell: bash + # run: echo "GITHUB_OUTPUT:${{ GITHUB_OUTPUT }}" + + # This second step is unnecessary but highly recommended because + # It will cache database and saves time re-downloading it if database isn't stale. + - name: Cache pip + # shell: bash + uses: actions/cache@v3 + with: + path: ~/.cache/pip + key: ${{ runner.os }}-pip-${{ hashFiles('**/setup.py') }} + restore-keys: | + ${{ runner.os }}-pip- + + - name: Set up Python 3.9 + # shell: bash + if: ${{steps.changed-files.outputs.other_modified_files_count > 0}} # Run on any non-docs change + uses: actions/setup-python@v5 + with: + python-version: 3.9 + + - name: Install dependencies and package + shell: bash + if: ${{steps.changed-files.outputs.other_modified_files_count > 0}} # Run on any non-docs change + run: | + sudo apt-get update + sudo apt-get install libvips libvips-tools -y + python -m pip install --upgrade pip==24.0 + python -m pip install wheel + python -m pip install openvino-dev==2023.0.1 mlcube_docker + pip install torch==2.3.1 torchvision==0.18.1 torchaudio==2.3.1 --index-url https://download.pytorch.org/whl/cpu + pip install -e . diff --git a/.github/workflows/devcontainer.yml b/.github/workflows/devcontainer.yml index 730acbff5..4337cd8bd 100644 --- a/.github/workflows/devcontainer.yml +++ b/.github/workflows/devcontainer.yml @@ -19,7 +19,7 @@ jobs: df -h - name: Checkout (GitHub) - uses: actions/checkout@v3 + uses: actions/checkout@v4 # Use changed-files-specific action to collect file changes. # The following commented condition applied to a step will run that step only if non-docs files have changed. diff --git a/.github/workflows/docker-image.yml b/.github/workflows/docker-image.yml index 502467131..d872c57f8 100644 --- a/.github/workflows/docker-image.yml +++ b/.github/workflows/docker-image.yml @@ -35,7 +35,7 @@ jobs: steps: - name: Check out the repository - uses: actions/checkout@v2 + uses: actions/checkout@v4 with: lfs: true submodules: 'recursive' diff --git a/.github/workflows/docs-ci.yml b/.github/workflows/docs-ci.yml index b92a1f625..8b471e9e9 100644 --- a/.github/workflows/docs-ci.yml +++ b/.github/workflows/docs-ci.yml @@ -10,7 +10,7 @@ jobs: deploy: runs-on: ubuntu-latest steps: - - uses: actions/checkout@v2 + - uses: actions/checkout@v4 - uses: actions/setup-python@v4 with: python-version: 3.x diff --git a/.github/workflows/main.yml b/.github/workflows/main.yml index ba5e79b7a..2c286a20b 100644 --- a/.github/workflows/main.yml +++ b/.github/workflows/main.yml @@ -22,7 +22,7 @@ jobs: runs-on: ubuntu-latest steps: - - uses: actions/checkout@v3 + - uses: actions/checkout@v4 - name: Set up Python uses: actions/setup-python@v4 with: diff --git a/.github/workflows/mlcube-test.yml b/.github/workflows/mlcube-test.yml index 23f91b178..fffaa7361 100644 --- a/.github/workflows/mlcube-test.yml +++ b/.github/workflows/mlcube-test.yml @@ -12,68 +12,15 @@ jobs: runs-on: ubuntu-latest steps: - - name: Free space - run: | - df -h - sudo rm -rf /opt/ghc - sudo rm -rf "/usr/local/share/boost" - sudo rm -rf "$AGENT_TOOLSDIRECTORY" - sudo rm -rf "$ANDROID_SDK_ROOT" - df -h - - name: Checkout - uses: actions/checkout@v3 - - # Use changed-files-specific action to collect file changes. - # The following commented condition applied to a step will run that step only if non-docs files have changed. - # It should be applied to all functionality-related steps. - # if: steps.changed-files-specific.outputs.only_modified == 'false' - - name: Detect and screen file changes - id: changed-files-specific - uses: tj-actions/changed-files@v41 - with: - files: | - .github/*.md - .github/ISSUE_TEMPLATE/*.md - .github/workflows/devcontainer.yml - .github/workflows/docker-image.yml - .devcontainer/** - docs/** - mlcube/** - *.md - LICENSE - Dockerfile-* + - name: Checkout code + uses: actions/checkout@v4 - - name: Summarize docs and non-docs modifications - run: | - echo "List of docs files that have changed: ${{ steps.changed-files-specific.outputs.all_modified_files }}" - echo "Changed non-docs files: ${{ steps.changed-files-specific.outputs.other_modified_files }}" + - name: Call reusable workflow to install dependencies + id: dependencies + uses: ./.github/workflows/dependencies - # This second step is unnecessary but highly recommended because - # It will cache database and saves time re-downloading it if database isn't stale. - - name: Cache pip - uses: actions/cache@v3 - with: - path: ~/.cache/pip - key: ${{ runner.os }}-pip-${{ hashFiles('**/setup.py') }} - restore-keys: | - ${{ runner.os }}-pip- - - name: Set up Python 3.9 - if: steps.changed-files-specific.outputs.only_modified == 'false' # Run on any non-docs change - uses: actions/setup-python@v4 - with: - python-version: 3.9 - - name: Install dependencies and package - if: steps.changed-files-specific.outputs.only_modified == 'false' # Run on any non-docs change - run: | - sudo apt-get update - sudo apt-get install libvips libvips-tools -y - python -m pip install --upgrade pip==24.0 - python -m pip install wheel - python -m pip install openvino-dev==2023.0.1 mlcube_docker - pip install torch==2.3.1 torchvision==0.18.1 torchaudio==2.3.1 --index-url https://download.pytorch.org/whl/cpu - pip install -e . - name: Run mlcube deploy tests working-directory: ./testing - if: steps.changed-files-specific.outputs.only_modified == 'false' # Run on any non-docs change + if: ${{steps.dependencies.outputs.other_modified_files_count > 0}} # Run on any non-docs change run: | sh test_deploy.sh diff --git a/.github/workflows/openfl-test.yml b/.github/workflows/openfl-test.yml index 00bf95506..d30ee295b 100644 --- a/.github/workflows/openfl-test.yml +++ b/.github/workflows/openfl-test.yml @@ -17,72 +17,21 @@ jobs: runs-on: ubuntu-latest steps: - - name: Free space - run: | - df -h - sudo rm -rf /opt/ghc - sudo rm -rf "/usr/local/share/boost" - sudo rm -rf "$AGENT_TOOLSDIRECTORY" - sudo rm -rf "$ANDROID_SDK_ROOT" - df -h - - name: Checkout - uses: actions/checkout@v3 - - # Use changed-files-specific action to collect file changes. - # The following commented condition applied to a step will run that step only if non-docs files have changed. - # It should be applied to all functionality-related steps. - # if: steps.changed-files-specific.outputs.only_modified == 'false' - - name: Detect and screen file changes - id: changed-files-specific - uses: tj-actions/changed-files@v41 - with: - files: | - .github/*.md - .github/ISSUE_TEMPLATE/*.md - .github/workflows/devcontainer.yml - .github/workflows/docker-image.yml - .devcontainer/** - docs/** - mlcube/** - *.md - LICENSE - Dockerfile-* + - name: Checkout code + uses: actions/checkout@v4 - - name: Summarize docs and non-docs modifications - run: | - echo "List of docs files that have changed: ${{ steps.changed-files-specific.outputs.all_modified_files }}" - echo "Changed non-docs files: ${{ steps.changed-files-specific.outputs.other_modified_files }}" + - name: Call reusable workflow to install dependencies + id: dependencies + uses: ./.github/workflows/dependencies - # This second step is unnecessary but highly recommended because - # It will cache database and saves time re-downloading it if database isn't stale. - - name: Cache pip - uses: actions/cache@v3 - with: - path: ~/.cache/pip - key: ${{ runner.os }}-pip-${{ hashFiles('**/setup.py') }} - restore-keys: | - ${{ runner.os }}-pip- - - name: Set up Python 3.9 - if: steps.changed-files-specific.outputs.only_modified == 'false' # Run on any non-docs change - uses: actions/setup-python@v4 - with: - python-version: 3.9 - - name: Install dependencies and package - if: steps.changed-files-specific.outputs.only_modified == 'false' # Run on any non-docs change - run: | - sudo apt-get update - sudo apt-get install libvips libvips-tools -y - python -m pip install --upgrade pip==24.0 - python -m pip install wheel - pip install torch==2.3.1 torchvision==0.18.1 torchaudio==2.3.1 --index-url https://download.pytorch.org/whl/cpu - pip install -e . - name: Run generic unit tests to download data and construct CSVs - if: steps.changed-files-specific.outputs.only_modified == 'false' # Run on any non-docs change + if: ${{steps.dependencies.outputs.other_modified_files_count > 0}} # Run on any non-docs change run: | pytest --cov=. --cov-report=xml -k "prepare_data_for_ci" + # openfl tests start here - name: Run OpenFL tests - if: steps.changed-files-specific.outputs.only_modified == 'false' # Run on any non-docs change + if: ${{steps.dependencies.outputs.other_modified_files_count > 0}} # Run on any non-docs change run: | echo "Removing onnx because of protobuf version conflict" pip uninstall onnx -y diff --git a/.github/workflows/ossar-analysis.yml b/.github/workflows/ossar-analysis.yml index b0323a7b2..804ed0dea 100644 --- a/.github/workflows/ossar-analysis.yml +++ b/.github/workflows/ossar-analysis.yml @@ -18,7 +18,7 @@ jobs: steps: - name: Checkout repository - uses: actions/checkout@v2 + uses: actions/checkout@v4 # Ensure a compatible version of dotnet is installed. # The [Microsoft Security Code Analysis CLI](https://aka.ms/mscadocs) is built with dotnet v3.1.201. diff --git a/.github/workflows/publish-nightly.yml b/.github/workflows/publish-nightly.yml index 4d8ff05a3..fb587e476 100644 --- a/.github/workflows/publish-nightly.yml +++ b/.github/workflows/publish-nightly.yml @@ -24,7 +24,7 @@ jobs: runs-on: ubuntu-latest steps: - - uses: actions/checkout@v3 + - uses: actions/checkout@v4 with: ref: master diff --git a/.github/workflows/python-install-check.yml b/.github/workflows/python-install-check.yml index 7ec37528a..7fdb3283c 100644 --- a/.github/workflows/python-install-check.yml +++ b/.github/workflows/python-install-check.yml @@ -9,7 +9,7 @@ jobs: matrix: python-version: ["3.9", "3.10", "3.11"] steps: - - uses: actions/checkout@v2 + - uses: actions/checkout@v4 - name: Set up Python ${{ matrix.python-version }} uses: actions/setup-python@v4 diff --git a/.github/workflows/python-test.yml b/.github/workflows/python-test.yml index 78c10901e..68e09865d 100644 --- a/.github/workflows/python-test.yml +++ b/.github/workflows/python-test.yml @@ -13,103 +13,57 @@ jobs: runs-on: ubuntu-latest steps: - - name: Free space - run: | - df -h - sudo rm -rf /opt/ghc - sudo rm -rf "/usr/local/share/boost" - sudo rm -rf "$AGENT_TOOLSDIRECTORY" - sudo rm -rf "$ANDROID_SDK_ROOT" - df -h - - name: Checkout - uses: actions/checkout@v3 + - name: Checkout code + uses: actions/checkout@v4 - # Use changed-files-specific action to collect file changes. - # The following commented condition applied to a step will run that step only if non-docs files have changed. - # It should be applied to all functionality-related steps. - # if: steps.changed-files-specific.outputs.only_modified == 'false' - - name: Detect and screen file changes - id: changed-files-specific - uses: tj-actions/changed-files@v41 - with: - files: | - .github/*.md - .github/ISSUE_TEMPLATE/*.md - .github/workflows/devcontainer.yml - .github/workflows/docker-image.yml - .devcontainer/** - docs/** - mlcube/** - *.md - LICENSE - Dockerfile-* + - name: Call reusable workflow to install dependencies + id: dependencies + uses: ./.github/workflows/dependencies - - name: Summarize docs and non-docs modifications - run: | - echo "List of docs files that have changed: ${{ steps.changed-files-specific.outputs.all_modified_files }}" - echo "Changed non-docs files: ${{ steps.changed-files-specific.outputs.other_modified_files }}" - - # This second step is unnecessary but highly recommended because - # It will cache database and saves time re-downloading it if database isn't stale. - - name: Cache pip - uses: actions/cache@v3 - with: - path: ~/.cache/pip - key: ${{ runner.os }}-pip-${{ hashFiles('**/setup.py') }} - restore-keys: | - ${{ runner.os }}-pip- - - name: Set up Python 3.9 - if: steps.changed-files-specific.outputs.only_modified == 'false' # Run on any non-docs change - uses: actions/setup-python@v4 - with: - python-version: 3.9 - - name: Install dependencies and package - if: steps.changed-files-specific.outputs.only_modified == 'false' # Run on any non-docs change - run: | - sudo apt-get update - sudo apt-get install libvips libvips-tools -y - python -m pip install --upgrade pip==24.0 - python -m pip install wheel - python -m pip install openvino-dev==2023.0.1 mlcube_docker - pip install torch==2.3.1 torchvision==0.18.1 torchaudio==2.3.1 --index-url https://download.pytorch.org/whl/cpu - pip install -e . - name: Run generic unit tests - if: steps.changed-files-specific.outputs.only_modified == 'false' # Run on any non-docs change + if: ${{steps.dependencies.outputs.other_modified_files_count > 0}} # Run on any non-docs change run: | pytest --cov=. --cov-report=xml -k "generic" - name: Run classification unit tests with histology - if: steps.changed-files-specific.outputs.only_modified == 'false' # Run on any non-docs change + if: ${{steps.dependencies.outputs.other_modified_files_count > 0}} # Run on any non-docs change run: | pytest --cov=. --cov-report=xml --cov-append -k "classification and histology" - name: Run classification unit tests - if: steps.changed-files-specific.outputs.only_modified == 'false' # Run on any non-docs change + if: ${{steps.dependencies.outputs.other_modified_files_count > 0}} # Run on any non-docs change run: | pytest --cov=. --cov-report=xml --cov-append -k "classification and not histology" - name: Run segmentation unit tests - if: steps.changed-files-specific.outputs.only_modified == 'false' # Run on any non-docs change + if: ${{steps.dependencies.outputs.other_modified_files_count > 0}} # Run on any non-docs change run: | pytest --cov=. --cov-report=xml --cov-append -k "segmentation and not transunet" - name: Run regression unit tests - if: steps.changed-files-specific.outputs.only_modified == 'false' # Run on any non-docs change + if: ${{steps.dependencies.outputs.other_modified_files_count > 0}} # Run on any non-docs change run: | pytest --cov=. --cov-report=xml --cov-append -k "regression" - name: Run transunet unit tests - if: steps.changed-files-specific.outputs.only_modified == 'false' # Run on any non-docs change + if: ${{steps.dependencies.outputs.other_modified_files_count > 0}} # Run on any non-docs change run: | pytest --cov=. --cov-report=xml --cov-append -k "transunet" - name: Run entrypoints tests - if: steps.changed-files-specific.outputs.only_modified == 'false' # Run on any non-docs change + if: ${{steps.dependencies.outputs.other_modified_files_count > 0}} # Run on any non-docs change run: | pytest --cov=. --cov-report=xml --cov-append -k "entrypoints" - name: Run test for update_version - if: steps.changed-files-specific.outputs.only_modified == 'false' # Run on any non-docs change + if: ${{steps.dependencies.outputs.other_modified_files_count > 0}} # Run on any non-docs change run: | pytest --cov=. --cov-report=xml --cov-append -k "update_version" - name: Upload coverage - if: steps.changed-files-specific.outputs.only_modified == 'false' # Run on any non-docs change + if: ${{steps.dependencies.outputs.other_modified_files_count > 0}} # Run on any non-docs change uses: codecov/codecov-action@v1 with: token: ${{ secrets.CODECOV_TOKEN }} file: ./coverage.xml - flags: unittests \ No newline at end of file + flags: unittests + + - name: Upload coverage to Codacy + if: github.ref == 'refs/heads/master' # only run when on master + uses: codacy/codacy-coverage-reporter-action@v1.3.0 + with: + project-token: ${{ secrets.CODACY_PROJECT_TOKEN }} + coverage-reports: ./coverage.xml diff --git a/Dockerfile-CPU b/Dockerfile-CPU index d9c87de25..96a27a67c 100644 --- a/Dockerfile-CPU +++ b/Dockerfile-CPU @@ -32,7 +32,7 @@ CMD run # See https://github.com/hexops/dockerfile as a best practices guide. #RUN addgroup --gid 10001 --system nonroot \ # && adduser --uid 10000 --system --ingroup nonroot --home /home/nonroot nonroot -# +# #USER nonroot # Prepare the container for possible model embedding later. diff --git a/Dockerfile-ROCm b/Dockerfile-ROCm index 5d0fb7450..508af28ce 100644 --- a/Dockerfile-ROCm +++ b/Dockerfile-ROCm @@ -36,7 +36,7 @@ CMD run # See https://github.com/hexops/dockerfile as a best practices guide. #RUN addgroup --gid 10001 --system nonroot \ # && adduser --uid 10000 --system --ingroup nonroot --home /home/nonroot nonroot -# +# #USER nonroot # Prepare the container for possible model embedding later. diff --git a/GANDLF/cli/huggingface_hub_handler.py b/GANDLF/cli/huggingface_hub_handler.py new file mode 100644 index 000000000..72e2f35b0 --- /dev/null +++ b/GANDLF/cli/huggingface_hub_handler.py @@ -0,0 +1,142 @@ +from huggingface_hub import HfApi, snapshot_download, ModelCardData, ModelCard +from typing import List, Union +from GANDLF import version +from pathlib import Path +from GANDLF.utils import get_git_hash +import re + + +def validate_model_card(file_path: str): + """ + Validate that the required fields in the model card are not null, empty, or set to 'REQUIRED_FOR_GANDLF'. + The fields must contain valid alphabetic or alphanumeric values. + + Args: + file_path (str): The path to the Markdown file to validate. + + Raises: + AssertionError: If any required field is missing, empty, null, or contains 'REQUIRED_FOR_GANDLF'. + """ + # Read the Markdown file + path = Path(file_path) + with path.open("r") as file: + template_str = file.read() + + # Define required fields and their regex patterns to capture the values + patterns = { + "Developed by": re.compile( + r'\*\*Developed by:\*\*\s*\{\{\s*developers\s*\|\s*default\("(.+?)",\s*true\)\s*\}\}', + re.MULTILINE, + ), + "License": re.compile( + r'\*\*License:\*\*\s*\{\{\s*license\s*\|\s*default\("(.+?)",\s*true\)\s*\}\}', + re.MULTILINE, + ), + "Primary Organization": re.compile( + r'\*\*Primary Organization:\*\*\s*\{\{\s*primary_organization\s*\|\s*default\("(.+?)",\s*true\)\s*\}\}', + re.MULTILINE, + ), + "Commercial use policy": re.compile( + r'\*\*Commercial use policy:\*\*\s*\{\{\s*commercial_use\s*\|\s*default\("(.+?)",\s*true\)\s*\}\}', + re.MULTILINE, + ), + } + + # Iterate through the required fields and validate + for field, pattern in patterns.items(): + match = pattern.search(template_str) + + # Ensure the field is present and does not contain 'REQUIRED_FOR_GANDLF' + assert match, f"Field '{field}' is missing or not found in the file." + + extract_value = match.group(1) + + # Get the field value + value = ( + re.search(r"\[([^\]]+)\]", extract_value).group(1) + if re.search(r"\[([^\]]+)\]", extract_value) + else None + ) + + # Ensure the field is not set to 'REQUIRED_FOR_GANDLF' or empty + assert ( + value != "REQUIRED_FOR_GANDLF" + ), f"The value for '{field}' is set to the default placeholder '[REQUIRED_FOR_GANDLF]'. It must be a valid value." + assert value, f"The value for '{field}' is empty or null." + + # Ensure the value contains only alphabetic or alphanumeric characters + assert re.match( + r"^[a-zA-Z0-9]+$", value + ), f"The value for '{field}' must be alphabetic or alphanumeric, but got: '{value}'" + + print( + "All required fields are valid, non-empty, properly filled, and do not contain '[REQUIRED_FOR_GANDLF]'." + ) + + # Example usage + return template_str + + +def push_to_model_hub( + repo_id: str, + folder_path: str, + hf_template: str, + path_in_repo: Union[str, None] = None, + commit_message: Union[str, None] = None, + commit_description: Union[str, None] = None, + token: Union[str, None] = None, + repo_type: Union[str, None] = None, + revision: Union[str, None] = None, + allow_patterns: Union[List[str], str, None] = None, + ignore_patterns: Union[List[str], str, None] = None, + delete_patterns: Union[List[str], str, None] = None, +): + api = HfApi(token=token) + + try: + repo_id = api.create_repo(repo_id).repo_id + except Exception as e: + print(f"Error: {e}") + + tags = ["v" + version] + + git_hash = get_git_hash() + + if not git_hash == "None": + tags += [git_hash] + + readme_template = validate_model_card(hf_template) + + card_data = ModelCardData(library_name="GaNDLF", tags=tags) + card = ModelCard.from_template(card_data, template_str=readme_template) + + card.save(Path(folder_path, "README.md")) + + api.upload_folder( + repo_id=repo_id, + folder_path=folder_path, + repo_type="model", + revision=revision, + allow_patterns=allow_patterns, + ignore_patterns=ignore_patterns, + delete_patterns=delete_patterns, + ) + print("Model Sucessfully Uploded") + + +def download_from_hub( + repo_id: str, + revision: Union[str, None] = None, + cache_dir: Union[str, None] = None, + local_dir: Union[str, None] = None, + force_download: bool = False, + token: Union[str, None] = None, +): + snapshot_download( + repo_id=repo_id, + revision=revision, + cache_dir=cache_dir, + local_dir=local_dir, + force_download=force_download, + token=token, + ) diff --git a/GANDLF/compute/inference_loop.py b/GANDLF/compute/inference_loop.py index c09b44cf7..6910b5974 100644 --- a/GANDLF/compute/inference_loop.py +++ b/GANDLF/compute/inference_loop.py @@ -15,7 +15,7 @@ from skimage.io import imsave from tqdm import tqdm from torch.cuda.amp import autocast -import tiffslide as openslide +import openslide from GANDLF.data import get_testing_loader from GANDLF.utils import ( best_model_path_end, @@ -89,7 +89,16 @@ def inference_loop( assert file_to_load != None, "The 'best_file' was not found" main_dict = torch.load(file_to_load, map_location=parameters["device"]) - model.load_state_dict(main_dict["model_state_dict"]) + state_dict = main_dict["model_state_dict"] + if parameters.get("differential_privacy"): + # this is required for torch==1.11 and for DP inference + new_state_dict = {} + for key, val in state_dict.items(): + new_key = key.replace("_module.", "") + new_state_dict[new_key] = val # remove `module.` + state_dict = new_state_dict + + model.load_state_dict(state_dict) parameters["previous_parameters"] = main_dict.get("parameters", None) model.eval() elif parameters["model"]["type"].lower() == "openvino": @@ -335,11 +344,13 @@ def inference_loop( ) cv2.imwrite(file_to_write, heatmaps[key]) - os_image_array = os_image.read_region( - (0, 0), - parameters["slide_level"], - (level_width, level_height), - as_array=True, + # this is needed because openslide returns an RGBA image + os_image_array = np.asarray( + os_image.read_region( + (0, 0), + parameters["slide_level"], + (level_width, level_height), + ).convert("RGB") ) blended_image = cv2.addWeighted( os_image_array, diff --git a/GANDLF/compute/training_loop.py b/GANDLF/compute/training_loop.py index 32b52f188..9c56a232d 100644 --- a/GANDLF/compute/training_loop.py +++ b/GANDLF/compute/training_loop.py @@ -31,6 +31,10 @@ from .forward_pass import validate_network from .generic import create_pytorch_objects +from GANDLF.privacy.opacus.model_handling import empty_collate +from GANDLF.privacy.opacus import handle_dynamic_batch_size, prep_for_opacus_training +from opacus.utils.batch_memory_manager import wrap_data_loader + # hides torchio citation request, see https://github.com/fepegar/torchio/issues/235 os.environ["TORCHIO_HIDE_CITATION_PROMPT"] = "1" @@ -91,6 +95,14 @@ def train_network( for batch_idx, (subject) in enumerate( tqdm(train_dataloader, desc="Looping over training data") ): + if params.get("differential_privacy"): + subject, params["batch_size"] = handle_dynamic_batch_size( + subject=subject, params=params + ) + assert not isinstance( + model, torch.nn.DataParallel + ), "Differential privacy is not supported with DataParallel or DistributedDataParallel. Please use a single GPU or DDP with Opacus." + optimizer.zero_grad() image = ( # 5D tensor: (B, C, H, W, D) torch.cat( @@ -212,6 +224,23 @@ def train_network( return average_epoch_train_loss, average_epoch_train_metric +def train_network_wrapper(model, train_dataloader, optimizer, params): + """ + Wrapper Function to handle train_dataloader for benign and DP cases and pass on to train a network for a single epoch + """ + + if params.get("differential_privacy"): + with train_dataloader as memory_safe_data_loader: + epoch_train_loss, epoch_train_metric = train_network( + model, memory_safe_data_loader, optimizer, params + ) + else: + epoch_train_loss, epoch_train_metric = train_network( + model, train_dataloader, optimizer, params + ) + return epoch_train_loss, epoch_train_metric + + def training_loop( training_data: pd.DataFrame, validation_data: pd.DataFrame, @@ -368,6 +397,7 @@ def training_loop( logger_csv_filename=os.path.join(output_dir, "logs_validation.csv"), metrics=metrics_log, mode="valid", + add_epsilon=bool(params.get("differential_privacy")), ) if testingDataDefined: test_logger = Logger( @@ -392,6 +422,36 @@ def training_loop( print("Using device:", device, flush=True) + if params.get("differential_privacy"): + print( + "Using Opacus to make training differentially private with respect to the training data." + ) + + model, optimizer, train_dataloader, privacy_engine = prep_for_opacus_training( + model=model, + optimizer=optimizer, + train_dataloader=train_dataloader, + params=params, + ) + + train_dataloader.collate_fn = empty_collate(train_dataloader.dataset[0]) + + # train_dataloader = BatchMemoryManager( + # data_loader=train_dataloader, + # max_physical_batch_size=MAX_PHYSICAL_BATCH_SIZE, + # optimizer=optimizer, + # ) + batch_size = params["batch_size"] + max_physical_batch_size = params["differential_privacy"].get( + "physical_batch_size" + ) + if max_physical_batch_size and max_physical_batch_size != batch_size: + train_dataloader = wrap_data_loader( + data_loader=train_dataloader, + max_batch_size=max_physical_batch_size, + optimizer=optimizer, + ) + # Iterate for number of epochs for epoch in range(start_epoch, epochs): if params["track_memory_usage"]: @@ -453,6 +513,14 @@ def training_loop( patience += 1 + # if training with differential privacy, print privacy epsilon + if params.get("differential_privacy"): + delta = params["differential_privacy"]["delta"] + this_epsilon = privacy_engine.get_epsilon(delta) + print(f" Epoch Final Privacy: (ε = {this_epsilon:.2f}, δ = {delta})") + # save for logging + epoch_valid_metric["epsilon"] = this_epsilon + # Write the losses to a logger train_logger.write(epoch, epoch_train_loss, epoch_train_metric) valid_logger.write(epoch, epoch_valid_loss, epoch_valid_metric) diff --git a/GANDLF/config_manager.py b/GANDLF/config_manager.py index 99497fbb1..ed26ec8e1 100644 --- a/GANDLF/config_manager.py +++ b/GANDLF/config_manager.py @@ -7,6 +7,7 @@ from .utils import version_check from GANDLF.data.post_process import postprocessing_after_reverse_one_hot_encoding +from GANDLF.privacy.opacus import parse_opacus_params from GANDLF.metrics import surface_distance_ids from importlib.metadata import version @@ -710,6 +711,10 @@ def _parseConfig( temp_dict["type"] = params["optimizer"] params["optimizer"] = temp_dict + # initialize defaults for DP + if params.get("differential_privacy"): + params = parse_opacus_params(params, initialize_key) + # initialize defaults for inference mechanism inference_mechanism = {"grid_aggregator_overlap": "crop", "patch_overlap": 0} initialize_inference_mechanism = False diff --git a/GANDLF/data/inference_dataloader_histopath.py b/GANDLF/data/inference_dataloader_histopath.py index f4380c412..f24e88a67 100644 --- a/GANDLF/data/inference_dataloader_histopath.py +++ b/GANDLF/data/inference_dataloader_histopath.py @@ -1,7 +1,7 @@ import os from typing import Optional import numpy as np -import tiffslide +import openslide from GANDLF.data.patch_miner.opm.utils import get_patch_size_in_microns, tissue_mask from skimage.transform import resize from torch.utils.data.dataset import Dataset @@ -51,7 +51,7 @@ def __init__( self._stride_size = get_patch_size_in_microns(wsi_path, self._stride_size) self._selected_level = selected_level self._mask_level = mask_level - self._os_image = tiffslide.open_slide(os.path.join(self._wsi_path)) + self._os_image = openslide.open_slide(os.path.join(self._wsi_path)) self._points = [] self._basic_preprocessing() @@ -61,11 +61,13 @@ def _basic_preprocessing(self): try: mask_xdim, mask_ydim = self._os_image.level_dimensions[self._mask_level] mask = get_tissue_mask( - self._os_image.read_region( - (0, 0), self._mask_level, (mask_xdim, mask_ydim), as_array=True + # this is needed because openslide returns an RGBA image + np.asarray( + self._os_image.read_region( + (0, 0), self._mask_level, (mask_xdim, mask_ydim) + ).convert("RGB") ) ) - if self._selected_level != self._mask_level: mask = resize(mask, (height, width)) mask = (mask > 0).astype(np.ubyte) @@ -134,9 +136,10 @@ def __getitem__(self, idx): (x_loc, y_loc), self._selected_level, (self._patch_size[0], self._patch_size[1]), - as_array=True, - ) + # as_array=True, openslide-python returns an RGBA PIL image + ).convert("RGB") + patch = np.asarray(patch) # convert the image to ndarray # this is to ensure that channels come at the beginning patch = patch.transpose([2, 0, 1]) # this is to ensure that we always have a z-stack before applying any torchio transforms diff --git a/GANDLF/data/patch_miner/opm/patch.py b/GANDLF/data/patch_miner/opm/patch.py index dc2edb16f..43ba5a09b 100644 --- a/GANDLF/data/patch_miner/opm/patch.py +++ b/GANDLF/data/patch_miner/opm/patch.py @@ -46,7 +46,9 @@ def read_patch(self): return np.asarray( self.slide_object.read_region( (self.coordinates[1], self.coordinates[0]), self.level, self.size - ) + ).convert( + "RGB" + ) # openslide-python returns an RGBA PIL image ) def copy(self): diff --git a/GANDLF/data/patch_miner/opm/patch_manager.py b/GANDLF/data/patch_miner/opm/patch_manager.py index d32e769b5..280f6af47 100644 --- a/GANDLF/data/patch_miner/opm/patch_manager.py +++ b/GANDLF/data/patch_miner/opm/patch_manager.py @@ -7,7 +7,7 @@ from tqdm import tqdm from pathlib import Path import pandas as pd -import tiffslide +import openslide class PatchManager: @@ -41,7 +41,7 @@ def set_subjectID(self, subjectID): def set_slide_path(self, filename): self.img_path = filename self.img_path = convert_to_tiff(self.img_path, self.output_dir, "img") - self.slide_object = tiffslide.open_slide(self.img_path) + self.slide_object = openslide.open_slide(self.img_path) self.slide_dims = self.slide_object.dimensions def set_label_map(self, path): @@ -50,7 +50,7 @@ def set_label_map(self, path): @param path: path to label map. """ self.label_map = convert_to_tiff(path, self.output_dir, "mask") - self.label_map_object = tiffslide.open_slide(self.label_map) + self.label_map_object = openslide.open_slide(self.label_map) assert all( x == y for x, y in zip(self.label_map_object.dimensions, self.slide_dims) diff --git a/GANDLF/data/patch_miner/opm/utils.py b/GANDLF/data/patch_miner/opm/utils.py index 1bee9b1f1..997f13f17 100644 --- a/GANDLF/data/patch_miner/opm/utils.py +++ b/GANDLF/data/patch_miner/opm/utils.py @@ -17,7 +17,7 @@ # import matplotlib.pyplot as plt import yaml -import tiffslide +import openslide # RGB Masking (pen) constants RGB_RED_CHANNEL = 0 @@ -428,7 +428,7 @@ def generate_initial_mask(slide_path: str, scale: int) -> Tuple[np.ndarray, tupl Tuple[np.ndarray, tuple]: The valid mask and the real scale. """ # Open slide and get properties - slide = tiffslide.open_slide(slide_path) + slide = openslide.open_slide(slide_path) slide_dims = slide.dimensions # Call thumbnail for effiency, calculate scale relative to whole slide @@ -505,26 +505,26 @@ def get_patch_size_in_microns( "Using mpp to calculate patch size for dimension {}".format(i) ) # only enter if "m" is present in patch size - input_slide = tiffslide.open_slide(input_slide_path) + input_slide = openslide.open_slide(input_slide_path) metadata = input_slide.properties if i == 0: for _property in [ - tiffslide.PROPERTY_NAME_MPP_X, + openslide.PROPERTY_NAME_MPP_X, "tiff.XResolution", "XResolution", ]: if _property in metadata: - magnification = metadata[_property] + magnification = float(metadata[_property]) magnification_prev = magnification break elif i == 1: for _property in [ - tiffslide.PROPERTY_NAME_MPP_Y, + openslide.PROPERTY_NAME_MPP_Y, "tiff.YResolution", "YResolution", ]: if _property in metadata: - magnification = metadata[_property] + magnification = float(metadata[_property]) break if magnification == -1: # if y-axis data is missing, use x-axis data diff --git a/GANDLF/entrypoints/hf_hub_integration.py b/GANDLF/entrypoints/hf_hub_integration.py new file mode 100644 index 000000000..353f31dfb --- /dev/null +++ b/GANDLF/entrypoints/hf_hub_integration.py @@ -0,0 +1,156 @@ +import click +from GANDLF.entrypoints import append_copyright_to_help +from GANDLF.cli.huggingface_hub_handler import push_to_model_hub, download_from_hub +from pathlib import Path + +huggingfaceDir_ = Path(__file__).parent.absolute() + +huggingfaceDir = huggingfaceDir_.parent + +# Huggingface template by default Path for the Model Deployment +huggingface_file_path = huggingfaceDir / "hugging_face.md" + + +@click.command() +@click.option( + "--upload/--download", + "-u/-d", + required=True, + help="Upload or download to/from a Huggingface Repo", +) +@click.option( + "--repo-id", + "-rid", + required=True, + help="Downloading/Uploading: A user or an organization name and a repo name separated by a /", +) +@click.option( + "--token", + "-tk", + help="Downloading/Uploading: A token to be used for the download/upload", +) +@click.option( + "--revision", + "-rv", + help="Downloading/Uploading: git revision id which can be a branch name, a tag, or a commit hash", +) +@click.option( + "--cache-dir", + "-cdir", + help="Downloading: path to the folder where cached files are stored", + type=click.Path(exists=True, file_okay=False, dir_okay=True), +) +@click.option( + "--local-dir", + "-ldir", + help="Downloading: if provided, the downloaded file will be placed under this directory", + type=click.Path(exists=True, file_okay=False, dir_okay=True), +) +@click.option( + "--force-download", + "-fd", + is_flag=True, + help="Downloading: Whether the file should be downloaded even if it already exists in the local cache", +) +@click.option( + "--folder-path", + "-fp", + help="Uploading: Path to the folder to upload on the local file system", + type=click.Path(exists=True, file_okay=False, dir_okay=True), +) +@click.option( + "--path-in-repo", + "-pir", + help="Uploading: Relative path of the directory in the repo. Will default to the root folder of the repository", +) +@click.option( + "--commit-message", + "-cr", + help='Uploading: The summary / title / first line of the generated commit. Defaults to: f"Upload {path_in_repo} with huggingface_hub"', +) +@click.option( + "--commit-description", + "-cd", + help="Uploading: The description of the generated commit", +) +@click.option( + "--repo-type", + "-rt", + help='Uploading: Set to "dataset" or "space" if uploading to a dataset or space, "model" if uploading to a model. Default is model', +) +@click.option( + "--allow-patterns", + "-ap", + help="Uploading: If provided, only files matching at least one pattern are uploaded.", +) +@click.option( + "--ignore-patterns", + "-ip", + help="Uploading: If provided, files matching any of the patterns are not uploaded.", +) +@click.option( + "--delete-patterns", + "-dp", + help="Uploading: If provided, remote files matching any of the patterns will be deleted from the repo while committing new files. This is useful if you don't know which files have already been uploaded.", +) +@click.option( + "--hf-template", + "-hft", + help="Adding the template path for the model card it is Required during Uploaing a model", + default=huggingface_file_path, + type=click.Path(exists=True, file_okay=True, dir_okay=False), +) +@append_copyright_to_help +def new_way( + upload: bool, + repo_id: str, + token: str, + hf_template: str, + revision: str, + cache_dir: str, + local_dir: str, + force_download: bool, + folder_path: str, + path_in_repo: str, + commit_message: str, + commit_description: str, + repo_type: str, + allow_patterns: str, + ignore_patterns: str, + delete_patterns: str, +): + # """Manages model transfers to and from the Hugging Face Hub""" + # """Manages model transfers to and from the Hugging Face Hub""" + + # # Ensure the hf_template is being passed and loaded correctly + # template_path = Path(hf_template) + + # # Check if file exists and is readable + # if not template_path.exists(): + # raise FileNotFoundError(f"Model card template file '{hf_template}' not found.") + + # with template_path.open('r') as f: + # hf_template = f.read() + + # # Debug print the content to ensure it's being read + # print(f"Template content: {type(hf_template)}...") # Print the first 100 chars as a preview + + if upload: + push_to_model_hub( + repo_id, + folder_path, + hf_template, + path_in_repo, + commit_message, + commit_description, + token, + repo_type, + revision, + allow_patterns, + ignore_patterns, + delete_patterns, + ) + else: + download_from_hub( + repo_id, revision, cache_dir, local_dir, force_download, token + ) diff --git a/GANDLF/entrypoints/run.py b/GANDLF/entrypoints/run.py index 842c47b5d..f202f2b9f 100644 --- a/GANDLF/entrypoints/run.py +++ b/GANDLF/entrypoints/run.py @@ -5,9 +5,7 @@ import argparse import ast -# import traceback from typing import Optional - from deprecated import deprecated import click diff --git a/GANDLF/entrypoints/subcommands.py b/GANDLF/entrypoints/subcommands.py index 814b00b66..354229869 100644 --- a/GANDLF/entrypoints/subcommands.py +++ b/GANDLF/entrypoints/subcommands.py @@ -12,6 +12,7 @@ from GANDLF.entrypoints.generate_metrics import new_way as generate_metrics_command from GANDLF.entrypoints.debug_info import new_way as debug_info_command from GANDLF.entrypoints.split_csv import new_way as split_csv_command +from GANDLF.entrypoints.hf_hub_integration import new_way as hf_command cli_subcommands = { @@ -29,4 +30,5 @@ "generate-metrics": generate_metrics_command, "debug-info": debug_info_command, "split-csv": split_csv_command, + "hf": hf_command, } diff --git a/GANDLF/hugging_face.md b/GANDLF/hugging_face.md new file mode 100644 index 000000000..782a42746 --- /dev/null +++ b/GANDLF/hugging_face.md @@ -0,0 +1,203 @@ +--- +# For reference on model card metadata, see the spec: https://github.com/huggingface/hub-docs/blob/main/modelcard.md?plain=1 +# Doc / guide: https://huggingface.co/docs/hub/model-cards +# Any fields required by GaNDLF are marked with REQUIRED_FOR_GANDLF, and will be checked +{{ card_data }} +--- + +# Model Card for {{ model_id | default("Model ID", true) }} + + + +{{ model_summary | default("", true) }} + +## Model Details + +### Model Description + + + +{{ model_description | default("", true) }} + +- **Developed by:** {{ developers | default("[GANDLF]", true)}} +- **License:** {{ license | default("[GANDLF]", true)}} +- **Funded by [optional]:** {{ funded_by | default("[More Information Needed]", true)}} +- **Shared by [optional]:** {{ shared_by | default("[More Information Needed]", true)}} +- **Model type:** {{ model_type | default("[More Information Needed]", true)}} +- **Language(s) (NLP):** {{ language | default("[More Information Needed]", true)}} +- **Finetuned from model [optional]:** {{ base_model | default("[More Information Needed]", true)}} +- **Primary Organization:** {{ primary_organization | default("[GANDLF]", true)}} +- **Commercial use policy:** {{ commercial_use | default("[GANDLF]", true)}} + +### Model Sources [optional] + + + +- **Repository:** {{ repo | default("[https://github.com/mlcommons/GaNDLF]", true)}} +- **Paper [optional]:** {{ paper | default("[More Information Needed]", true)}} +- **Demo [optional]:** {{ demo | default("[More Information Needed]", true)}} + +## Uses + + + +### Direct Use + + + +{{ direct_use | default("[More Information Needed]", true)}} + +### Downstream Use [optional] + + + +{{ downstream_use | default("[More Information Needed]", true)}} + +### Out-of-Scope Use + + + +{{ out_of_scope_use | default("[More Information Needed]", true)}} + +## Bias, Risks, and Limitations + + + +{{ bias_risks_limitations | default("[More Information Needed]", true)}} + +### Recommendations + + + +{{ bias_recommendations | default("Users (both direct and downstream) should be made aware of the risks, biases and limitations of the model. More information needed for further recommendations.", true)}} + +## How to Get Started with the Model + +Use the code below to get started with the model. + +{{ get_started_code | default("[More Information Needed]", true)}} + +## Training Details + +### Training Data + + + +{{ training_data | default("[More Information Needed]", true)}} + +### Training Procedure + + + +#### Preprocessing [optional] + +{{ preprocessing | default("[More Information Needed]", true)}} + + +#### Training Hyperparameters + +- **Training regime:** {{ training_regime | default("[More Information Needed]", true)}} + +#### Speeds, Sizes, Times [optional] + + + +{{ speeds_sizes_times | default("[More Information Needed]", true)}} + +## Evaluation + + + +### Testing Data, Factors & Metrics + +#### Testing Data + + + +{{ testing_data | default("[More Information Needed]", true)}} + +#### Factors + + + +{{ testing_factors | default("[More Information Needed]", true)}} + +#### Metrics + + + +{{ testing_metrics | default("[More Information Needed]", true)}} + +### Results + +{{ results | default("[More Information Needed]", true)}} + +#### Summary + +{{ results_summary | default("", true) }} + +## Model Examination [optional] + + + +{{ model_examination | default("[More Information Needed]", true)}} + +## Environmental Impact + + + +Carbon emissions can be estimated using the [Machine Learning Impact calculator](https://mlco2.github.io/impact#compute) presented in [Lacoste et al. (2019)](https://arxiv.org/abs/1910.09700). + +- **Hardware Type:** {{ hardware_type | default("[More Information Needed]", true)}} +- **Hours used:** {{ hours_used | default("[More Information Needed]", true)}} +- **Cloud Provider:** {{ cloud_provider | default("[More Information Needed]", true)}} +- **Compute Region:** {{ cloud_region | default("[More Information Needed]", true)}} +- **Carbon Emitted:** {{ co2_emitted | default("[More Information Needed]", true)}} + +## Technical Specifications [optional] + +### Model Architecture and Objective + +{{ model_specs | default("[More Information Needed]", true)}} + +### Compute Infrastructure + +{{ compute_infrastructure | default("[More Information Needed]", true)}} + +#### Hardware + +{{ hardware_requirements | default("[More Information Needed]", true)}} + +#### Software + +{{ software | default("[More Information Needed]", true)}} + +## Citation [optional] + + + +**BibTeX:** + +{{ citation_bibtex | default("[More Information Needed]", true)}} + +**APA:** + +{{ citation_apa | default("[More Information Needed]", true)}} + +## Glossary [optional] + + + +{{ glossary | default("[More Information Needed]", true)}} + +## More Information [optional] + +{{ more_information | default("[More Information Needed]", true)}} + +## Model Card Authors [optional] + +{{ model_card_authors | default("[More Information Needed]", true)}} + +## Model Card Contact + +{{ model_card_contact | default("[More Information Needed]", true)}} diff --git a/GANDLF/logger.py b/GANDLF/logger.py index 2562eb17d..4f1d76e03 100755 --- a/GANDLF/logger.py +++ b/GANDLF/logger.py @@ -12,14 +12,21 @@ class Logger: - def __init__(self, logger_csv_filename: str, metrics: List[str], mode: str) -> None: + def __init__( + self, + logger_csv_filename: str, + metrics: List[str], + mode: str, + add_epsilon: bool = False, + ) -> None: """ - Logger class to log the training and validation metrics to a csv file. - May append to existing file if headers match; elsewise raises an error. + Logger class to log the training and validation metrics to a csv file. May append to existing file if headers match; elsewise raises an error. Args: logger_csv_filename (str): Path to a filename where the csv has to be stored. metrics (Dict[str, float]): The metrics to be logged. + mode (str): The mode of the logger, used as suffix to metric names. Normally may be `train` / `val` / `test` + add_epsilon (bool): Whether to log epsilon values or not (differential privacy measurement) """ self.filename = logger_csv_filename mode = mode.lower() @@ -28,6 +35,8 @@ def __init__(self, logger_csv_filename: str, metrics: List[str], mode: str) -> N new_header = ["epoch_no", f"{mode}_loss"] + [ f"{mode}_{metric}" for metric in metrics ] + if add_epsilon: + new_header.append(f"{self.mode}_epsilon") # TODO: do we really need to support appending to existing files? if os.path.exists(self.filename): diff --git a/GANDLF/models/dynunet_wrapper.py b/GANDLF/models/dynunet_wrapper.py index 3f209a3d8..176596bee 100644 --- a/GANDLF/models/dynunet_wrapper.py +++ b/GANDLF/models/dynunet_wrapper.py @@ -2,6 +2,41 @@ import monai.networks.nets.dynunet as dynunet +def get_kernels_strides(sizes, spacings): + """ + More info: https://github.com/Project-MONAI/tutorials/blob/main/modules/dynunet_pipeline/create_network.py#L19 + + When refering this method for other tasks, please ensure that the patch size for each spatial dimension should + be divisible by the product of all strides in the corresponding dimension. + In addition, the minimal spatial size should have at least one dimension that has twice the size of + the product of all strides. For patch sizes that cannot find suitable strides, an error will be raised. + + """ + input_size = sizes + strides, kernels = [], [] + while True: + spacing_ratio = [sp / min(spacings) for sp in spacings] + stride = [ + 2 if ratio <= 2 and size >= 8 else 1 + for (ratio, size) in zip(spacing_ratio, sizes) + ] + kernel = [3 if ratio <= 2 else 1 for ratio in spacing_ratio] + if all(s == 1 for s in stride): + break + for idx, (i, j) in enumerate(zip(sizes, stride)): + assert ( + i % j == 0 + ), f"Patch size is not supported, please try to modify the size {input_size[idx]} in the spatial dimension {idx}." + sizes = [i / j for i, j in zip(sizes, stride)] + spacings = [i * j for i, j in zip(spacings, stride)] + kernels.append(kernel) + strides.append(stride) + + strides.insert(0, len(spacings) * [1]) + kernels.append(len(spacings) * [3]) + return kernels, strides + + class dynunet_wrapper(ModelBase): """ More info: https://docs.monai.io/en/stable/networks.html#dynunet @@ -26,35 +61,33 @@ class dynunet_wrapper(ModelBase): def __init__(self, parameters: dict): super(dynunet_wrapper, self).__init__(parameters) - # checking for validation - assert ( - "kernel_size" in parameters["model"] - ) == True, "\033[0;31m`kernel_size` key missing in parameters" - assert ( - "strides" in parameters["model"] - ) == True, "\033[0;31m`strides` key missing in parameters" - - # defining some defaults - # if not ("upsample_kernel_size" in parameters["model"]): - # parameters["model"]["upsample_kernel_size"] = parameters["model"][ - # "strides" - # ][1:] + patch_size = parameters.get("patch_size", None) + spacing = parameters.get( + "spacing_for_internal_computations", + [1.0 for i in range(parameters["model"]["dimension"])], + ) + parameters["model"]["kernel_size"] = parameters["model"].get( + "kernel_size", None + ) + parameters["model"]["strides"] = parameters["model"].get("strides", None) + if (parameters["model"]["kernel_size"] is None) or ( + parameters["model"]["strides"] is None + ): + kernel_size, strides = get_kernels_strides(patch_size, spacing) + parameters["model"]["kernel_size"] = kernel_size + parameters["model"]["strides"] = strides parameters["model"]["filters"] = parameters["model"].get("filters", None) parameters["model"]["act_name"] = parameters["model"].get( "act_name", ("leakyrelu", {"inplace": True, "negative_slope": 0.01}) ) - parameters["model"]["deep_supervision"] = parameters["model"].get( - "deep_supervision", True + "deep_supervision", False ) - parameters["model"]["deep_supr_num"] = parameters["model"].get( "deep_supr_num", 1 ) - parameters["model"]["res_block"] = parameters["model"].get("res_block", True) - parameters["model"]["trans_bias"] = parameters["model"].get("trans_bias", False) parameters["model"]["dropout"] = parameters["model"].get("dropout", None) diff --git a/GANDLF/models/imagenet_unet.py b/GANDLF/models/imagenet_unet.py index 940987e1f..f1a203d4a 100644 --- a/GANDLF/models/imagenet_unet.py +++ b/GANDLF/models/imagenet_unet.py @@ -252,6 +252,10 @@ def __init__(self, parameters) -> None: aux_params=classifier_head_parameters, ) + # all BatchNorm should be replaced with InstanceNorm for DP experiments + if "differential_privacy" in parameters: + self.replace_batchnorm(self.model) + if self.n_dimensions == 3: self.model = self.converter(self.model).model diff --git a/GANDLF/privacy/__init__.py b/GANDLF/privacy/__init__.py new file mode 100644 index 000000000..e69de29bb diff --git a/GANDLF/privacy/opacus/__init__.py b/GANDLF/privacy/opacus/__init__.py new file mode 100644 index 000000000..3a7aa2672 --- /dev/null +++ b/GANDLF/privacy/opacus/__init__.py @@ -0,0 +1,3 @@ +from .config_parsing import parse_opacus_params +from .model_handling import opacus_model_fix, prep_for_opacus_training +from .training_utils import handle_dynamic_batch_size diff --git a/GANDLF/privacy/opacus/config_parsing.py b/GANDLF/privacy/opacus/config_parsing.py new file mode 100644 index 000000000..9ae2fd12b --- /dev/null +++ b/GANDLF/privacy/opacus/config_parsing.py @@ -0,0 +1,59 @@ +from typing import Callable + + +def parse_opacus_params(params: dict, initialize_key: Callable) -> dict: + """ + Function to set defaults and augment the parameters related to making a trained model differentially + private with respect to the training data. + + Args: + params (dict): Training parameters. + initialize_key (Callable): Function to fill in value for a missing key. + + Returns: + dict: Updated training parameters. + """ + + if not isinstance(params["differential_privacy"], dict): + print( + "WARNING: Non dictionary value for the key: 'differential_privacy' was used, replacing with default valued dictionary." + ) + params["differential_privacy"] = {} + # these are some defaults + params["differential_privacy"] = initialize_key( + params["differential_privacy"], "noise_multiplier", 10.0 + ) + params["differential_privacy"] = initialize_key( + params["differential_privacy"], "max_grad_norm", 1.0 + ) + params["differential_privacy"] = initialize_key( + params["differential_privacy"], "accountant", "rdp" + ) + params["differential_privacy"] = initialize_key( + params["differential_privacy"], "secure_mode", False + ) + params["differential_privacy"] = initialize_key( + params["differential_privacy"], "allow_opacus_model_fix", True + ) + params["differential_privacy"] = initialize_key( + params["differential_privacy"], "delta", 1e-5 + ) + params["differential_privacy"] = initialize_key( + params["differential_privacy"], "physical_batch_size", params["batch_size"] + ) + + if params["differential_privacy"]["physical_batch_size"] > params["batch_size"]: + print( + f"WARNING: The physical batch size {params['differential_privacy']['physical_batch_size']} is greater" + f"than the batch size {params['batch_size']}, setting the physical batch size to the batch size." + ) + params["differential_privacy"]["physical_batch_size"] = params["batch_size"] + + # these keys need to be parsed as floats, not strings + for key in ["noise_multiplier", "max_grad_norm", "delta", "epsilon"]: + if key in params["differential_privacy"]: + params["differential_privacy"][key] = float( + params["differential_privacy"][key] + ) + + return params diff --git a/GANDLF/privacy/opacus/model_handling.py b/GANDLF/privacy/opacus/model_handling.py new file mode 100644 index 000000000..92d9b55bc --- /dev/null +++ b/GANDLF/privacy/opacus/model_handling.py @@ -0,0 +1,149 @@ +import collections.abc as abc +from functools import partial +from torch.utils.data._utils.collate import default_collate +from torch.utils.data import DataLoader +from typing import Union, Callable, Tuple +import copy + +import numpy as np +import torch +from opacus import PrivacyEngine +from opacus.validators import ModuleValidator + + +def opacus_model_fix(model: torch.nn.Module, params: dict) -> torch.nn.Module: + """ + Function to detect components of the model that are not compatible with Opacus differentially private training, and replacing with compatible components + or raising an exception when a fix cannot be handled by Opacus. + + Args: + model (torch.nn.Module): The model to be trained. + params (dict): Training parameters. + + Returns: + torch.nn.Module: Model, with potentially some components replaced with ones compatible with Opacus. + """ + # use opacus to detect issues with model + opacus_errors_detected = ModuleValidator.validate(model, strict=False) + + if not params["differential_privacy"]["allow_opacus_model_fix"]: + assert ( + opacus_errors_detected == [] + ), f"Training parameters are set to not allow Opacus to try to fix incompatible model components, and the following issues were detected: {opacus_errors_detected}" + elif opacus_errors_detected != []: + print( + f"Allowing Opacus to try and patch the model due to the following issues: {opacus_errors_detected}" + ) + print() + model = ModuleValidator.fix(model) + # If the fix did not work, raise an exception + ModuleValidator.validate(model, strict=True) + return model + + +def prep_for_opacus_training( + model: torch.nn.Module, + optimizer: torch.optim.Optimizer, + train_dataloader: DataLoader, + params: dict, +) -> Tuple[torch.nn.Module, torch.optim.Optimizer, DataLoader, PrivacyEngine]: + """ + Function to prepare the model, optimizer, and dataloader for differentially private training using Opacus. + + Args: + model (torch.nn.Module): The model to be trained. + optimizer (torch.optim.Optimizer): The optimizer to be used for training. + train_dataloader (DataLoader): The dataloader for the training data. + params (dict): Training parameters. + + Returns: + Tuple[torch.nn.Module, torch.optim.Optimizer, DataLoader, PrivacyEngine]: Model, optimizer, dataloader, and privacy engine. + """ + + privacy_engine = PrivacyEngine( + accountant=params["differential_privacy"]["accountant"], + secure_mode=params["differential_privacy"]["secure_mode"], + ) + + if not "epsilon" in params["differential_privacy"]: + model, optimizer, train_dataloader = privacy_engine.make_private( + module=model, + optimizer=optimizer, + data_loader=train_dataloader, + noise_multiplier=params["differential_privacy"]["noise_multiplier"], + max_grad_norm=params["differential_privacy"]["max_grad_norm"], + ) + else: + (model, optimizer, train_dataloader) = privacy_engine.make_private_with_epsilon( + module=model, + optimizer=optimizer, + data_loader=train_dataloader, + max_grad_norm=params["differential_privacy"]["max_grad_norm"], + epochs=params["num_epochs"], + target_epsilon=params["differential_privacy"]["epsilon"], + target_delta=params["differential_privacy"]["delta"], + ) + return model, optimizer, train_dataloader, privacy_engine + + +def build_empty_batch_value( + sample: Union[torch.Tensor, np.ndarray, abc.Mapping, abc.Sequence, int, float, str] +): + """ + Build an empty batch value from a sample. This function is used to create a placeholder for empty batches in an iteration. Inspired from https://github.com/pytorch/pytorch/blob/main/torch/utils/data/_utils/collate.py#L108. The key difference is that pytorch `collate` has to traverse batch of objects AND unite its fields to lists, while this function traverse a single item AND creates an "empty" version of the batch. + + Args: + sample (Union[torch.Tensor, np.ndarray, abc.Mapping, abc.Sequence, int, float, str]): A sample from the dataset. + + Raises: + TypeError: If the data type is not supported. + + Returns: + Union[torch.Tensor, np.ndarray, abc.Mapping, abc.Sequence, int, float, str]: An empty batch value. + """ + if isinstance(sample, torch.Tensor): + # Create an empty tensor with the same shape except for the zeroed batch dimension. + return torch.empty((0,) + sample.shape) + elif isinstance(sample, np.ndarray): + # Create an empty tensor from a numpy array, also with the zeroed batch dimension. + return torch.empty((0,) + sample.shape, dtype=torch.from_numpy(sample).dtype) + elif isinstance(sample, abc.Mapping): + # Recursively handle dictionary-like objects. + return {key: build_empty_batch_value(value) for key, value in sample.items()} + elif isinstance(sample, tuple) and hasattr(sample, "_fields"): # namedtuple + return type(sample)(*(build_empty_batch_value(item) for item in sample)) + elif isinstance(sample, abc.Sequence) and not isinstance(sample, str): + # Handle lists and tuples, but exclude strings. + return [build_empty_batch_value(item) for item in sample] + elif isinstance(sample, (int, float, str)): + # Return an empty list for basic data types. + return [] + else: + raise TypeError(f"Unsupported data type: {type(sample)}") + + +def empty_collate( + item_example: Union[ + torch.Tensor, np.ndarray, abc.Mapping, abc.Sequence, int, float, str + ] +) -> Callable: + """ + Creates a new collate function that behave same as default pytorch one, + but can process the empty batches. + + Args: + item_example (Union[torch.Tensor, np.ndarray, abc.Mapping, abc.Sequence, int, float, str]): An example item from the dataset. + + Returns: + Callable: function that should replace dataloader collate: `dataloader.collate_fn = empty_collate(...)` + """ + + def custom_collate(batch, _empty_batch_value): + if len(batch) > 0: + return default_collate(batch) # default behavior + else: + return copy.copy(_empty_batch_value) + + empty_batch_value = build_empty_batch_value(item_example) + + return partial(custom_collate, _empty_batch_value=empty_batch_value) diff --git a/GANDLF/privacy/opacus/training_utils.py b/GANDLF/privacy/opacus/training_utils.py new file mode 100644 index 000000000..0664ebc45 --- /dev/null +++ b/GANDLF/privacy/opacus/training_utils.py @@ -0,0 +1,106 @@ +from typing import Tuple +import torch +import torchio + + +def handle_nonempty_batch(subject: dict, params: dict) -> Tuple[dict, int]: + """ + Function to detect batch size from the subject an Opacus loader provides in the case of a non-empty batch, and make any changes to the subject dictionary that are needed for GaNDLF to use it. + + Args: + subject (dict): Training data subject dictionary. + params (dict): Training parameters. + + Returns: + Tuple[dict, int]: Modified subject dictionary and batch size. + """ + batch_size = len(subject[params["channel_keys"][0]][torchio.DATA]) + return subject, batch_size + + +def handle_empty_batch(subject: dict, params: dict, feature_shape: list) -> dict: + """ + Function to replace the list of empty arrays an Opacus loader provides in the case of an empty batch with a subject dictionary GANDLF can consume. + + Args: + subject (dict): Training data subject dictionary. + params (dict): Training parameters. + feature_shape (list): Shape of the features. + + Returns: + dict: Modified subject dictionary. + """ + + print("\nConstructing empty batch dictionary.\n") + + subject = { + "subject_id": "empty_batch", + "spacing": None, + "path_to_metadata": None, + "location": None, + } + subject.update( + { + key: {torchio.DATA: torch.zeros(tuple([0] + feature_shape))} + for key in params["channel_keys"] + } + ) + if params["problem_type"] != "segmentation": + subject.update( + { + key: torch.zeros((0, params["model"]["num_classes"])).to(torch.int64) + for key in params["value_keys"] + } + ) + else: + subject.update( + { + "label": { + torchio.DATA: torch.zeros(tuple([0] + feature_shape)).to( + torch.int64 + ) + } + } + ) + + return subject + + +def handle_dynamic_batch_size(subject: dict, params: dict) -> Tuple[dict, int]: + """ + Function to process the subject Opacus loaders provide and prepare to handle their dynamic batch size (including possible empty batches). + + Args: + subject (dict): Training data subject dictionary. + params (dict): Training parameters. + + Raises: + RuntimeError: If the subject is a list object that is not an empty batch. + + Returns: + Tuple[dict, int]: Modified subject dictionary and batch size. + """ + + # The handling performed here is currently to be able to comprehend what + # batch size we are currently working with (which we may later see as not needed) + # and also to handle the previously observed case where Opacus produces + # a subject that is not a dictionary but rather a list of empty arrays + # (due to the empty batch result). The latter case is detected as a subject that + # is a list object. + if isinstance(subject, list): + are_empty = torch.Tensor( + [torch.equal(tensor, torch.Tensor([])) for tensor in subject] + ) + assert torch.all( + are_empty + ), "Detected a list subject that is not an empty batch, which is not expected behavior." + # feature_shape = [params["model"]["num_channels"]]+params["patch_size"] + feature_shape = [params["model"]["num_channels"]] + params["patch_size"] + subject = handle_empty_batch( + subject=subject, params=params, feature_shape=feature_shape + ) + batch_size = 0 + else: + subject, batch_size = handle_nonempty_batch(subject=subject, params=params) + + return subject, batch_size diff --git a/GANDLF/training_manager.py b/GANDLF/training_manager.py index e605af6f6..1fe74f0d3 100644 --- a/GANDLF/training_manager.py +++ b/GANDLF/training_manager.py @@ -5,6 +5,8 @@ from GANDLF.compute import training_loop from GANDLF.utils import get_dataframe, split_data +import yaml + def TrainingManager( dataframe: pd.DataFrame, @@ -158,6 +160,8 @@ def TrainingManager_split( reset (bool): Whether the previous run will be reset or not. """ currentModelConfigPickle = os.path.join(outputDir, "parameters.pkl") + currentModelConfigYaml = os.path.join(outputDir, "config.yaml") + if (not os.path.exists(currentModelConfigPickle)) or reset or resume: with open(currentModelConfigPickle, "wb") as handle: pickle.dump(parameters, handle, protocol=pickle.HIGHEST_PROTOCOL) @@ -170,6 +174,10 @@ def TrainingManager_split( ) parameters = pickle.load(open(currentModelConfigPickle, "rb")) + if (not os.path.exists(currentModelConfigYaml)) or reset or resume: + with open(currentModelConfigYaml, "w") as handle: + yaml.dump(parameters, handle, default_flow_style=False) + training_loop( training_data=dataframe_train, validation_data=dataframe_validation, diff --git a/GANDLF/version.py b/GANDLF/version.py index 9d601767a..5e5047feb 100644 --- a/GANDLF/version.py +++ b/GANDLF/version.py @@ -2,4 +2,4 @@ # -*- coding: UTF-8 -*- # check GaNDLF wiki for versioning and release guidelines: https://github.com/mlcommons/GaNDLF/wiki -__version__ = "0.1.0-dev" +__version__ = "0.1.2-dev" diff --git a/README.md b/README.md index 136333a22..89c69c375 100644 --- a/README.md +++ b/README.md @@ -1,11 +1,14 @@ -# GaNDLF +![GitHub-Mark-Light](https://github.com/mlcommons/GaNDLF/blob/master/docs/images/logo/full.png?raw=true#gh-light-mode-only) + +![GitHub-Mark-Dark](https://github.com/mlcommons/GaNDLF/blob/master/docs/images/logo/full_black.png?raw=true#gh-dark-mode-only)

- Codacy
+
+ Codacy diff --git a/docs/customize.md b/docs/customize.md index 9c33cd523..cfda6e894 100644 --- a/docs/customize.md +++ b/docs/customize.md @@ -136,3 +136,18 @@ This file contains mid-level information regarding various parameters that can b - `q_samples_per_volume`: this determines the number of patches to extract from each volume. A small number of patches ensures a large variability in the queue, but training will be slower. - `q_num_workers`: this determines the number subprocesses to use for data loading; '0' means main process is used, scale this according to available CPU resources. - `q_verbose`: used to debug the queue + +## Differentially Private Training + +GaNDLF supports training differentially private models using [Opacus](https://opacus.ai/). Here are some resources using which one can train private models: + +- TLDR on DP and private training: read [this paper](https://arxiv.org/pdf/1607.00133) and [this blog post](https://medium.com/pytorch/differential-privacy-series-part-1-dp-sgd-algorithm-explained-12512c3959a3). +- All options are present in a new key called `differential_privacy` in the config file. It has the following options: + - `noise_multiplier`: The ratio of the standard deviation of the Gaussian noise to the L2-sensitivity of the function to which the noise is added. + - `max_grad_norm`: The maximum norm of the per-sample gradients. Any gradient with norm higher than this will be clipped to this value. + - `accountant`: Accounting mechanism. Currently supported: `rdp` (RDPAccountant), `gdp` (GaussianAccountant), `prv` (PRVAccountant) + - `secure_mode`: Set to `True` if cryptographically strong DP guarantee is required. `secure_mode=True` uses secure random number generator for noise and shuffling (as opposed to `pseudo-rng` in vanilla PyTorch) and prevents certain floating-point arithmetic-based attacks. + - `allow_opacus_model_fix`: Enabled automated fixing of the model based on Opacus [[ref](https://opacus.ai/api/validator.html)] + - `delta`: Target delta to be achieved. Probability of information being leaked. Use either this or `epsilon`. + - `epsilon`: Target epsilon to be achieved, a metric of privacy loss at differential changes in data. Use either this or `delta`. + - `physical_batch_size`: The batch size to use for DP computation (it is usually set lower than the baseline or non-DP batch size). Defaults to `batch_size`. diff --git a/docs/images/logo/full.png b/docs/images/logo/full.png new file mode 100644 index 000000000..ea558f866 Binary files /dev/null and b/docs/images/logo/full.png differ diff --git a/docs/images/logo/full_black.png b/docs/images/logo/full_black.png new file mode 100644 index 000000000..85fba87bd Binary files /dev/null and b/docs/images/logo/full_black.png differ diff --git a/docs/migration_guide.md b/docs/migration_guide.md index b435e30c0..552f6fba4 100644 --- a/docs/migration_guide.md +++ b/docs/migration_guide.md @@ -15,8 +15,8 @@ The [0.0.20 release](https://github.com/mlcommons/GaNDLF/releases/tag/0.0.20) wa - The main change is the use of the [Version package](https://github.com/keleshev/version) for systematic semantic versioning [[ref](https://github.com/mlcommons/GaNDLF/pull/841)]. - No change is needed if you are using a [stable version](https://docs.mlcommons.org/GaNDLF/setup/#install-from-package-managers). - If you have installed GaNDLF [from source](https://docs.mlcommons.org/GaNDLF/setup/#install-from-sources) or using a [nightly build](https://docs.mlcommons.org/GaNDLF/setup/#install-from-package-managers), you will need to ensure that the `maximum` key under `version` in the configuration file contains the correct version number: - - Either **including** the `-dev` identifier of the current version (e.g., if the current version is `0.1.0-dev`, then the `maximum` key should be `0.1.0-dev`). - - Or **excluding** the `-dev` identifier of the current version, but increasing the version number by one on any level (e.g., if the current version is `0.1.0-dev`, then the `maximum` key should be `0.1.1`). + - Either **including** the `-dev` identifier of the current version (e.g., if the current version is `0.X.Y-dev`, then the `maximum` key should be `0.X.Y-dev`). + - Or **excluding** the `-dev` identifier of the current version, but increasing the version number by one on any level (e.g., if the current version is `0.X.Y-dev`, then the `maximum` key should be `0.X.Y`). ### Use in HPC Environments diff --git a/docs/usage.md b/docs/usage.md index 609b746b2..1f56947c9 100644 --- a/docs/usage.md +++ b/docs/usage.md @@ -24,7 +24,7 @@ Please follow the [installation instructions](./setup.md#installation) to instal ### Anonymize Data -A major reason why one would want to anonymize data is to ensure that trained models do not inadvertently encode protected health information [[1](https://doi.org/10.1145/3436755),[2](https://doi.org/10.1038/s42256-020-0186-1)]. GaNDLF can anonymize one or multiple images using the `gandlf anonymizer` command as follows: +A major reason why one would want to anonymize data is to ensure that trained models do not inadvertently do not encode protect health information [[1](https://doi.org/10.1145/3436755),[2](https://doi.org/10.1038/s42256-020-0186-1)]. GaNDLF can anonymize single images or a collection of images using the `gandlf anonymizer` command. It can be used as follows: ```bash # continue from previous shell @@ -81,7 +81,7 @@ Once these files are present, the patch miner can be run using the following com ### Running preprocessing before training/inference (optional) -Running preprocessing before training/inference is optional, but recommended. It will significantly reduce the computational footprint during training/inference at the expense of larger storage requirements. Use the following command, which will save the processed data in `./experiment_0/output_dir/` with a new data CSV and the corresponding model configuration: +Running preprocessing before training/inference is optional, but recommended. It will significantly reduce the computational footprint during training/inference at the expense of larger storage requirements. To run preprocessing before training/inference you can use the following command, which will save the processed data in `./experiment_0/output_dir/` with a new data CSV and the corresponding model configuration: ```bash # continue from previous shell @@ -108,7 +108,7 @@ N,/full/path/N/0.nii.gz,/full/path/N/1.nii.gz,...,/full/path/N/X.nii.gz,/full/pa **Notes:** - `Channel` can be substituted with `Modality` or `Image` -- `Label` can be substituted with `Mask` or `Segmentation` and is used to specify the annotation file for segmentation models +- `Label` can be substituted with `Mask` or `Segmentation`and is used to specify the annotation file for segmentation models - For classification/regression, add a column called `ValueToPredict`. Currently, we are supporting only a single value prediction per model. - Only a single `Label` or `ValueToPredict` header should be passed - Multiple segmentation classes should be in a single file with unique label numbers. @@ -152,14 +152,14 @@ The following command shows how the script works: (venv_gandlf) $> gandlf construct-csv \ # -h, --help Show help message and exit -i $DATA_DIRECTORY # this is the main data directory - -c _t1.nii.gz,_t1ce.nii.gz,_t2.nii.gz,_flair.nii.gz \ # an example image identifier for 4 structural brain MR sequences for BraTS, and can be changed based on your data. In the simplest case of a single modality, a ".nii.gz" will suffice + -c _t1.nii.gz,_t1ce.nii.gz,_t2.nii.gz,_flair.nii.gz \ # an example image identifier for 4 structural brain MR sequences for BraTS, and can be changed based on your data -l _seg.nii.gz \ # an example label identifier - not needed for regression/classification, and can be changed based on your data -o ./experiment_0/train_data.csv # output CSV to be used for training ``` **Notes**: -- For classification/regression, add a column called `ValueToPredict`. Currently, we support only a single value prediction per model. +- For classification/regression, add a column called `ValueToPredict`. Currently, we are supporting only a single value prediction per model. - `SubjectID` or `PatientName` is used to ensure that the randomized split is done per-subject rather than per-image. - For data arrangement different to what is described above, a customized script will need to be written to generate the CSV, or you can enter the data manually into the CSV. @@ -179,15 +179,13 @@ To split the data CSV into training, validation, and testing CSVs, the `gandlf s ## Customize the Training -Adapting GaNDLF to your needs boils down to modifying a YAML-based configuration file which controls the parameters of training and inference. Below is a list of available samples for users to start as their baseline for further customization: +GaNDLF requires a YAML-based configuration that controls various aspects of the training/inference process. There are multiple samples for users to start as their baseline for further customization. A list of the available samples is presented as follows: +- [Sample showing all the available options](https://github.com/mlcommons/GaNDLF/blob/master/samples/config_all_options.yaml) - [Segmentation example](https://github.com/mlcommons/GaNDLF/blob/master/samples/config_segmentation_brats.yaml) - [Regression example](https://github.com/mlcommons/GaNDLF/blob/master/samples/config_regression.yaml) - [Classification example](https://github.com/mlcommons/GaNDLF/blob/master/samples/config_classification.yaml) -To find **all the parameters** a GaNDLF config may modify, consult the following file: -- [All available options](https://github.com/mlcommons/GaNDLF/blob/master/samples/config_all_options.yaml) - **Notes**: - More details on the configuration options are available in the [customization page](customize.md). @@ -512,3 +510,111 @@ This can be replicated for ROCm for AMD , by following the [instructions to set GaNDLF, and GaNDLF-created models, may be distributed as an [MLCube](https://mlcommons.github.io/mlcube/). This involves distributing an `mlcube.yaml` file. That file can be specified when using the [MLCube runners](https://mlcommons.github.io/mlcube/runners/). The runner will perform many aspects of configuring your container for you. Currently, only the `mlcube_docker` runner is supported. See the [MLCube documentation](https://mlcommons.github.io/mlcube/) for more details. + +## HuggingFace CLI + +This tool allows you to interact with the Hugging Face Hub directly from a terminal. For example, you can create a repository, upload and download files, etc. + +### Download an entire repository +GaNDLF's Hugging Face CLI allows you to download repositories through the command line. This can be done by just specifying the repo id: + +```bash +(main) $> gandlf hf --download --repo-id HuggingFaceH4/zephyr-7b-beta +``` + +Apart from the Repo Id you can also provide other arguments. + +### Revision +To download from a specific revision (commit hash, branch name or tag), use the --revision option: +```bash +(main) $> gandlf hf --download --repo-id distilbert-base-uncased revision --revision v1.1 +``` +### Specify a token +To access private or gated repositories, you must use a token. You can do this using the --token option: + +```bash +(main) $> gandlf hf --download --repo-id distilbert-base-uncased revision --revision v1.1 --token hf_**** +``` + +### Specify cache directory +If not using --local-dir, all files will be downloaded by default to the cache directory defined by the HF_HOME environment variable. You can specify a custom cache using --cache-dir: + +```bash +(main) $> gandlf hf --download --repo-id distilbert-base-uncased revision --revision v1.1 --token hf_**** --cache-dir ./path/to/cache +``` + +### Download to a local folder +The recommended (and default) way to download files from the Hub is to use the cache-system. However, in some cases you want to download files and move them to a specific folder. This is useful to get a workflow closer to what git commands offer. You can do that using the --local-dir option. + +A ./huggingface/ folder is created at the root of your local directory containing metadata about the downloaded files. This prevents re-downloading files if they’re already up-to-date. If the metadata has changed, then the new file version is downloaded. This makes the local-dir optimized for pulling only the latest changes. + +```bash +(main) $> gandlf hf --download --repo-id distilbert-base-uncased revision --revision v1.1 --token hf_**** --cache-dir ./path/to/cache --local-dir ./path/to/dir +``` +### Force Download +To specify if the files should be downloaded even if it already exists in the local cache. + +```bash +(main) $> gandlf hf --download --repo-id distilbert-base-uncased revision --revision v1.1 --token hf_**** --cache-dir ./path/to/cache --local-dir ./path/to/dir --force-download +``` + +### Upload an entire folder +Use the `gandlf hf --upload` upload command to upload files to the Hub directly. + +```bash +(main) $> gandlf hf --upload --repo-id Wauplin/my-cool-model --folder-path ./model --token hf_**** +``` + +### Upload to a Specific Path in Repo +Relative path of the directory in the repo. Will default to the root folder of the repository. +```bash +(main) $> gandlf hf --upload --repo-id Wauplin/my-cool-model --folder-path ./model/data --path-in-repo ./data --token hf_**** +``` + +### Upload multiple files +To upload multiple files from a folder at once without uploading the entire folder, use the --allow-patterns and --ignore-patterns patterns. It can also be combined with the --delete-patterns option to delete files on the repo while uploading new ones. In the example below, we sync the local Space by deleting remote files and uploading all files except the ones in /logs: + +```bash +(main) $> gandlf hf Wauplin/space-example --repo-type=space --exclude="/logs/*" --delete="*" --commit-message="Sync local Space with Hub" +``` + +### Specify a token +To upload files, you must use a token. By default, the token saved locally will be used. If you want to authenticate explicitly, use the --token option: + +```bash +(main) $>gandlf hf --upload Wauplin/my-cool-model --folder-path ./model --token=hf_**** +``` + +### Specify a commit message +Use the --commit-message and --commit-description to set a custom message and description for your commit instead of the default one + +```bash +(main) $>gandlf hf --upload Wauplin/my-cool-model --folder-path ./model --token=hf_**** +--commit-message "Epoch 34/50" --commit-description="Val accuracy: 68%. Check tensorboard for more details." +``` + +### Upload to a dataset or Space +To upload to a dataset or a Space, use the --repo-type option: + +```bash +(main) $>gandlf hf --upload Wauplin/my-cool-model --folder-path ./model --token=hf_**** +--repo-type dataset +``` + +### Huggingface Template For Upload +#### Design and Modify Template +To design the huggingface template use the hugging_face.md file change the medatory field +[REQUIRED_FOR_GANDLF] to it's respective name don't leave it blank other wise it may through error, other field can be modeified by the user as per his convenience + +```bash +# Here the required field change from [REQUIRED_FOR_GANDLF] to [GANDLF] +**Developed by:** {{ developers | default("[GANDLF]", true)}} +``` +#### Mentioned The Huggingface Template +To mentioned the Huggingface Template , use the --hf-template option: + +```bash +(main) $>gandlf hf --upload Wauplin/my-cool-model --folder-path ./model --token=hf_**** +--repo-type dataset --hf-template /hugging_face.md + +``` diff --git a/samples/config_all_options.yaml b/samples/config_all_options.yaml index f460ac252..872d65c44 100644 --- a/samples/config_all_options.yaml +++ b/samples/config_all_options.yaml @@ -1,8 +1,8 @@ # affix version version: { - minimum: 0.1.0-dev, - maximum: 0.1.0-dev # this should NOT be made a variable, but should be tested after every tag is created + minimum: 0.1.2-dev, + maximum: 0.1.2-dev # this should NOT be made a variable, but should be tested after every tag is created } ## Choose the model parameters here model: diff --git a/samples/config_classification.yaml b/samples/config_classification.yaml index 7a8ba7704..e8b720520 100644 --- a/samples/config_classification.yaml +++ b/samples/config_classification.yaml @@ -1,8 +1,8 @@ # affix version version: { - minimum: 0.1.0-dev, - maximum: 0.1.0-dev # this should NOT be made a variable, but should be tested after every tag is created + minimum: 0.1.2-dev, + maximum: 0.1.2-dev # this should NOT be made a variable, but should be tested after every tag is created } # Choose the model parameters here model: diff --git a/samples/config_getting_started_classification_histo2d.yaml b/samples/config_getting_started_classification_histo2d.yaml index 34b5d069f..f824fbd92 100644 --- a/samples/config_getting_started_classification_histo2d.yaml +++ b/samples/config_getting_started_classification_histo2d.yaml @@ -94,6 +94,6 @@ scheduler: track_memory_usage: false verbose: false version: - maximum: 0.1.0-dev + maximum: 0.1.2-dev minimum: 0.0.14 weighted_loss: true diff --git a/samples/config_getting_started_classification_rad3d.yaml b/samples/config_getting_started_classification_rad3d.yaml index e374ee82a..109e001e6 100644 --- a/samples/config_getting_started_classification_rad3d.yaml +++ b/samples/config_getting_started_classification_rad3d.yaml @@ -99,6 +99,6 @@ scheduler: track_memory_usage: false verbose: false version: - maximum: 0.1.0-dev + maximum: 0.1.2-dev minimum: 0.0.14 weighted_loss: true diff --git a/samples/config_getting_started_regression_histo2d.yaml b/samples/config_getting_started_regression_histo2d.yaml index 1b325595d..fa2a41e2f 100644 --- a/samples/config_getting_started_regression_histo2d.yaml +++ b/samples/config_getting_started_regression_histo2d.yaml @@ -59,6 +59,6 @@ scheduler: track_memory_usage: false verbose: false version: - maximum: 0.1.0-dev + maximum: 0.1.2-dev minimum: 0.0.14 weighted_loss: true diff --git a/samples/config_getting_started_regression_rad3d.yaml b/samples/config_getting_started_regression_rad3d.yaml index bb1e3f1e6..8ce80e1d1 100644 --- a/samples/config_getting_started_regression_rad3d.yaml +++ b/samples/config_getting_started_regression_rad3d.yaml @@ -62,6 +62,6 @@ scheduler: track_memory_usage: false verbose: false version: - maximum: 0.1.0-dev + maximum: 0.1.2-dev minimum: 0.0.14 weighted_loss: false diff --git a/samples/config_getting_started_segmentation_histo2d.yaml b/samples/config_getting_started_segmentation_histo2d.yaml index 6640e5a0b..13ca80436 100644 --- a/samples/config_getting_started_segmentation_histo2d.yaml +++ b/samples/config_getting_started_segmentation_histo2d.yaml @@ -66,6 +66,6 @@ scheduler: track_memory_usage: false verbose: true version: - maximum: 0.1.0-dev + maximum: 0.1.2-dev minimum: 0.0.14 weighted_loss: true diff --git a/samples/config_getting_started_segmentation_rad3d.yaml b/samples/config_getting_started_segmentation_rad3d.yaml index 029b18e7a..758163ff6 100644 --- a/samples/config_getting_started_segmentation_rad3d.yaml +++ b/samples/config_getting_started_segmentation_rad3d.yaml @@ -89,6 +89,6 @@ scheduler: track_memory_usage: false verbose: true version: - maximum: 0.1.0-dev - minimum: 0.1.0-dev + maximum: 0.1.2-dev + minimum: 0.1.2-dev weighted_loss: true diff --git a/samples/config_regression.yaml b/samples/config_regression.yaml index d3757afcc..0f4b91737 100644 --- a/samples/config_regression.yaml +++ b/samples/config_regression.yaml @@ -1,8 +1,8 @@ # affix version version: { - minimum: 0.1.0-dev, - maximum: 0.1.0-dev # this should NOT be made a variable, but should be tested after every tag is created + minimum: 0.1.2-dev, + maximum: 0.1.2-dev # this should NOT be made a variable, but should be tested after every tag is created } # Choose the model parameters here model: diff --git a/samples/config_segmentation_brats.yaml b/samples/config_segmentation_brats.yaml index 33ae7378e..c8a5ac005 100644 --- a/samples/config_segmentation_brats.yaml +++ b/samples/config_segmentation_brats.yaml @@ -1,8 +1,8 @@ # affix version version: { - minimum: 0.1.0-dev, - maximum: 0.1.0-dev # this should NOT be made a variable, but should be tested after every tag is created + minimum: 0.1.2-dev, + maximum: 0.1.2-dev # this should NOT be made a variable, but should be tested after every tag is created } # Choose the model parameters here model: diff --git a/samples/config_segmentation_histology.yaml b/samples/config_segmentation_histology.yaml index cedebe7ce..889ee9a98 100644 --- a/samples/config_segmentation_histology.yaml +++ b/samples/config_segmentation_histology.yaml @@ -1,8 +1,8 @@ # affix version version: { - minimum: 0.1.0-dev, - maximum: 0.1.0-dev # this should NOT be made a variable, but should be tested after every tag is created + minimum: 0.1.2-dev, + maximum: 0.1.2-dev # this should NOT be made a variable, but should be tested after every tag is created } # Choose the model parameters here model: diff --git a/setup.py b/setup.py index b82fc9424..908459ccd 100644 --- a/setup.py +++ b/setup.py @@ -6,6 +6,7 @@ import sys, re, os from setuptools import setup, find_packages + try: with open("README.md") as readme_file: readme = readme_file.read() @@ -13,7 +14,6 @@ readme = "No README information found." sys.stderr.write("Warning: Could not open '%s' due %s\n" % ("README.md", error)) - try: filepath = "GANDLF/version.py" version_file = open(filepath) @@ -51,8 +51,7 @@ "scikit-image>=0.19.1", "setuptools", "seaborn", - "pyyaml", - "tiffslide", + "pyyaml==6.0.1", "matplotlib", "gdown==5.1.0", "pytest", @@ -69,7 +68,7 @@ "segmentation-models-pytorch==0.3.3", "ACSConv==0.1.1", # https://github.com/docker/docker-py/issues/3256 - "requests<2.32.0", # 2.32.0 are not compatible with docker 7.0.0; to remove restriction once docker is fixed + "requests>=2.32.2", "docker", "dicom-anonymizer==1.0.12", "twine", @@ -81,6 +80,10 @@ "packaging==24.0", "typer==0.9.0", "colorlog", + "opacus==1.5.2", + "huggingface-hub==0.25.1", + "openslide-bin", + "openslide-python==1.4.1", ] if __name__ == "__main__": diff --git a/testing/config_classification.yaml b/testing/config_classification.yaml index b23ff66f3..79dfb5feb 100644 --- a/testing/config_classification.yaml +++ b/testing/config_classification.yaml @@ -55,7 +55,7 @@ save_output: false scaling_factor: 1 scheduler: triangle version: - maximum: 0.1.0-dev + maximum: 0.1.2-dev minimum: 0.0.14 weighted_loss: True diff --git a/testing/config_regression.yaml b/testing/config_regression.yaml index b9c8dd764..47b9e2aab 100644 --- a/testing/config_regression.yaml +++ b/testing/config_regression.yaml @@ -38,7 +38,7 @@ save_output: false scaling_factor: 1 scheduler: triangle version: - maximum: 0.1.0-dev + maximum: 0.1.2-dev minimum: 0.0.14 weighted_loss: false diff --git a/testing/config_segmentation.yaml b/testing/config_segmentation.yaml index 3a5d48a7e..a275a6b8d 100644 --- a/testing/config_segmentation.yaml +++ b/testing/config_segmentation.yaml @@ -3,7 +3,7 @@ version: { minimum: 0.0.14, - maximum: 0.1.0-dev + maximum: 0.1.2-dev } model: { diff --git a/testing/entrypoints/test_hf_cli.py b/testing/entrypoints/test_hf_cli.py new file mode 100644 index 000000000..d28351022 --- /dev/null +++ b/testing/entrypoints/test_hf_cli.py @@ -0,0 +1,122 @@ +import os.path +import pytest +from click.testing import CliRunner + +from GANDLF.entrypoints.hf_hub_integration import new_way + +from . import CliCase, run_test_case, TmpDire + +# This function is a place where a real logic is executed. +# For tests, we replace it with mock up, and check if this function is called +# with proper args for different cli commands +MOCK_PATH = "GANDLF.entrypoints.hf_hub_integration.download_from_hub" + +# these files would be either created temporarily for test execution, +# or we ensure they do not exist +test_file_system = [TmpDire("./tmp_dir")] + + +test_cases = [ + CliCase( + should_succeed=True, + new_way_lines=[ + # full command + "--download --repo-id distilbert-base-uncased", + # tests short arg aliases + "-d -rid distilbert-base-uncased", + ], + expected_args={ + "repo_id": "distilbert-base-uncased", + "revision": None, + "cache_dir": None, + "local_dir": None, + "force_download": False, + "token": None, + }, + ), + CliCase( + should_succeed=True, + new_way_lines=[ + # full command + "--download --repo-id distilbert-base-uncased --revision 6cdc0aad91f5ae2e6712e91bc7b65d1cf5c05411", + # tests short arg aliases + "-d -rid distilbert-base-uncased -rv 6cdc0aad91f5ae2e6712e91bc7b65d1cf5c05411", + ], + expected_args={ + "repo_id": "distilbert-base-uncased", + "revision": "6cdc0aad91f5ae2e6712e91bc7b65d1cf5c05411", + "cache_dir": None, + "local_dir": None, + "force_download": False, + "token": None, + }, + ), + CliCase( + should_succeed=True, + new_way_lines=[ + # full command + "--download --repo-id distilbert-base-uncased --local-dir tmp_dir", + # tests short arg aliases + "-d -rid distilbert-base-uncased -ldir tmp_dir", + ], + expected_args={ + "repo_id": "distilbert-base-uncased", + "revision": None, + "cache_dir": None, + "local_dir": os.path.normpath("tmp_dir"), + "force_download": False, + "token": None, + }, + ), + CliCase( + should_succeed=False, + new_way_lines=[ + # full command + "--repo-id distilbert-base-uncased ", + # tests short arg aliases + "-rid distilbert-base-uncased -ldir", + ], + expected_args={ + "repo_id": "distilbert-base-uncased", + "revision": None, + "cache_dir": None, + "local_dir": None, + "force_download": False, + "token": None, + }, + ), + CliCase( + should_succeed=False, + new_way_lines=[ + # full command + "--download --repo-id distilbert-base-uncased --local-dir", + # tests short arg aliases + "-d -rid distilbert-base-uncased -ldir", + ], + expected_args={ + "repo_id": "distilbert-base-uncased", + "revision": None, + "cache_dir": None, + "local_dir": os.path.normpath("tmp_dir"), + "force_download": False, + "token": None, + }, + ), +] + + +@pytest.mark.parametrize("case", test_cases) +def test_case(cli_runner: CliRunner, case: CliCase): + """This approach ensures that before passing file_system_config to run_test_case, + you check its value and assign an appropriate default ([]).""" + file_system_config_ = test_file_system if test_file_system is not None else [] + + run_test_case( + cli_runner=cli_runner, + file_system_config=file_system_config_, # Default to empty list if no new_way_lines + case=case, + real_code_function_path=MOCK_PATH, + new_way=new_way, # Pass the real 'new_way' or set as needed + old_way=None, + old_script_name=None, + ) diff --git a/testing/hugging_face.md b/testing/hugging_face.md new file mode 100644 index 000000000..782a42746 --- /dev/null +++ b/testing/hugging_face.md @@ -0,0 +1,203 @@ +--- +# For reference on model card metadata, see the spec: https://github.com/huggingface/hub-docs/blob/main/modelcard.md?plain=1 +# Doc / guide: https://huggingface.co/docs/hub/model-cards +# Any fields required by GaNDLF are marked with REQUIRED_FOR_GANDLF, and will be checked +{{ card_data }} +--- + +# Model Card for {{ model_id | default("Model ID", true) }} + + + +{{ model_summary | default("", true) }} + +## Model Details + +### Model Description + + + +{{ model_description | default("", true) }} + +- **Developed by:** {{ developers | default("[GANDLF]", true)}} +- **License:** {{ license | default("[GANDLF]", true)}} +- **Funded by [optional]:** {{ funded_by | default("[More Information Needed]", true)}} +- **Shared by [optional]:** {{ shared_by | default("[More Information Needed]", true)}} +- **Model type:** {{ model_type | default("[More Information Needed]", true)}} +- **Language(s) (NLP):** {{ language | default("[More Information Needed]", true)}} +- **Finetuned from model [optional]:** {{ base_model | default("[More Information Needed]", true)}} +- **Primary Organization:** {{ primary_organization | default("[GANDLF]", true)}} +- **Commercial use policy:** {{ commercial_use | default("[GANDLF]", true)}} + +### Model Sources [optional] + + + +- **Repository:** {{ repo | default("[https://github.com/mlcommons/GaNDLF]", true)}} +- **Paper [optional]:** {{ paper | default("[More Information Needed]", true)}} +- **Demo [optional]:** {{ demo | default("[More Information Needed]", true)}} + +## Uses + + + +### Direct Use + + + +{{ direct_use | default("[More Information Needed]", true)}} + +### Downstream Use [optional] + + + +{{ downstream_use | default("[More Information Needed]", true)}} + +### Out-of-Scope Use + + + +{{ out_of_scope_use | default("[More Information Needed]", true)}} + +## Bias, Risks, and Limitations + + + +{{ bias_risks_limitations | default("[More Information Needed]", true)}} + +### Recommendations + + + +{{ bias_recommendations | default("Users (both direct and downstream) should be made aware of the risks, biases and limitations of the model. More information needed for further recommendations.", true)}} + +## How to Get Started with the Model + +Use the code below to get started with the model. + +{{ get_started_code | default("[More Information Needed]", true)}} + +## Training Details + +### Training Data + + + +{{ training_data | default("[More Information Needed]", true)}} + +### Training Procedure + + + +#### Preprocessing [optional] + +{{ preprocessing | default("[More Information Needed]", true)}} + + +#### Training Hyperparameters + +- **Training regime:** {{ training_regime | default("[More Information Needed]", true)}} + +#### Speeds, Sizes, Times [optional] + + + +{{ speeds_sizes_times | default("[More Information Needed]", true)}} + +## Evaluation + + + +### Testing Data, Factors & Metrics + +#### Testing Data + + + +{{ testing_data | default("[More Information Needed]", true)}} + +#### Factors + + + +{{ testing_factors | default("[More Information Needed]", true)}} + +#### Metrics + + + +{{ testing_metrics | default("[More Information Needed]", true)}} + +### Results + +{{ results | default("[More Information Needed]", true)}} + +#### Summary + +{{ results_summary | default("", true) }} + +## Model Examination [optional] + + + +{{ model_examination | default("[More Information Needed]", true)}} + +## Environmental Impact + + + +Carbon emissions can be estimated using the [Machine Learning Impact calculator](https://mlco2.github.io/impact#compute) presented in [Lacoste et al. (2019)](https://arxiv.org/abs/1910.09700). + +- **Hardware Type:** {{ hardware_type | default("[More Information Needed]", true)}} +- **Hours used:** {{ hours_used | default("[More Information Needed]", true)}} +- **Cloud Provider:** {{ cloud_provider | default("[More Information Needed]", true)}} +- **Compute Region:** {{ cloud_region | default("[More Information Needed]", true)}} +- **Carbon Emitted:** {{ co2_emitted | default("[More Information Needed]", true)}} + +## Technical Specifications [optional] + +### Model Architecture and Objective + +{{ model_specs | default("[More Information Needed]", true)}} + +### Compute Infrastructure + +{{ compute_infrastructure | default("[More Information Needed]", true)}} + +#### Hardware + +{{ hardware_requirements | default("[More Information Needed]", true)}} + +#### Software + +{{ software | default("[More Information Needed]", true)}} + +## Citation [optional] + + + +**BibTeX:** + +{{ citation_bibtex | default("[More Information Needed]", true)}} + +**APA:** + +{{ citation_apa | default("[More Information Needed]", true)}} + +## Glossary [optional] + + + +{{ glossary | default("[More Information Needed]", true)}} + +## More Information [optional] + +{{ more_information | default("[More Information Needed]", true)}} + +## Model Card Authors [optional] + +{{ model_card_authors | default("[More Information Needed]", true)}} + +## Model Card Contact + +{{ model_card_contact | default("[More Information Needed]", true)}} diff --git a/testing/test_full.py b/testing/test_full.py index b36a8ab64..50b628e76 100644 --- a/testing/test_full.py +++ b/testing/test_full.py @@ -35,6 +35,7 @@ generate_metrics_dict, split_data_and_save_csvs, ) +from GANDLF.cli.huggingface_hub_handler import push_to_model_hub, download_from_hub from GANDLF.schedulers import global_schedulers_dict from GANDLF.optimizers import global_optimizer_dict from GANDLF.models import global_models_dict @@ -46,6 +47,7 @@ ) from GANDLF.anonymize import run_anonymizer from GANDLF.entrypoints.debug_info import _debug_info +from huggingface_hub import HfApi device = "cpu" @@ -276,12 +278,6 @@ def test_train_segmentation_rad_2d(device): ["acs", "soft", "conv3d"] ) - if model == "dynunet": - # More info: https://github.com/Project-MONAI/MONAI/blob/96bfda00c6bd290297f5e3514ea227c6be4d08b4/tests/test_dynunet.py - parameters["model"]["kernel_size"] = (3, 3, 3, 1) - parameters["model"]["strides"] = (1, 1, 1, 1) - parameters["model"]["deep_supervision"] = False - parameters["model"]["architecture"] = model parameters["nested_training"]["testing"] = -5 parameters["nested_training"]["validation"] = -5 @@ -374,12 +370,6 @@ def test_train_segmentation_rad_3d(device): ["acs", "soft", "conv3d"] ) - if model == "dynunet": - # More info: https://github.com/Project-MONAI/MONAI/blob/96bfda00c6bd290297f5e3514ea227c6be4d08b4/tests/test_dynunet.py - parameters["model"]["kernel_size"] = (3, 3, 3, 1) - parameters["model"]["strides"] = (1, 1, 1, 1) - parameters["model"]["deep_supervision"] = False - parameters["model"]["architecture"] = model parameters["nested_training"]["testing"] = -5 parameters["nested_training"]["validation"] = -5 @@ -3193,8 +3183,102 @@ def test_generic_data_split(): print("passed") +def test_upload_download_huggingface(device): + print("52: Starting huggingface upload download tests") + # overwrite previous results + sanitize_outputDir() + output_dir_patches = os.path.join(outputDir, "histo_patches") + if os.path.isdir(output_dir_patches): + shutil.rmtree(output_dir_patches) + Path(output_dir_patches).mkdir(parents=True, exist_ok=True) + output_dir_patches_output = os.path.join(output_dir_patches, "histo_patches_output") + Path(output_dir_patches_output).mkdir(parents=True, exist_ok=True) + + parameters_patch = {} + # extracting minimal number of patches to ensure that the test does not take too long + parameters_patch["num_patches"] = 10 + parameters_patch["read_type"] = "sequential" + # define patches to be extracted in terms of microns + parameters_patch["patch_size"] = ["1000m", "1000m"] + + file_config_temp = write_temp_config_path(parameters_patch) + + patch_extraction( + inputDir + "/train_2d_histo_segmentation.csv", + output_dir_patches_output, + file_config_temp, + ) + + file_for_Training = os.path.join(output_dir_patches_output, "opm_train.csv") + # read and parse csv + parameters = ConfigManager( + testingDir + "/config_segmentation.yaml", version_check_flag=False + ) + training_data, parameters["headers"] = parseTrainingCSV(file_for_Training) + parameters["patch_size"] = patch_size["2D"] + parameters["modality"] = "histo" + parameters["model"]["dimension"] = 2 + parameters["model"]["class_list"] = [0, 255] + parameters["model"]["amp"] = True + parameters["model"]["num_channels"] = 3 + parameters = populate_header_in_parameters(parameters, parameters["headers"]) + parameters["model"]["architecture"] = "resunet" + parameters["nested_training"]["testing"] = 1 + parameters["nested_training"]["validation"] = -2 + parameters["metrics"] = ["dice"] + parameters["model"]["onnx_export"] = True + parameters["model"]["print_summary"] = True + parameters["data_preprocessing"]["resize_image"] = [128, 128] + modelDir = os.path.join(outputDir, "modelDir") + Path(modelDir).mkdir(parents=True, exist_ok=True) + TrainingManager( + dataframe=training_data, + outputDir=modelDir, + parameters=parameters, + device=device, + resume=False, + reset=True, + ) + inference_data, parameters["headers"] = parseTrainingCSV( + inputDir + "/train_2d_histo_segmentation.csv", train=False + ) + + inference_data.drop(index=inference_data.index[-1], axis=0, inplace=True) + InferenceManager( + dataframe=inference_data, + modelDir=modelDir, + parameters=parameters, + device=device, + ) + + # Initialize the Hugging Face API instance + api = HfApi(token="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") + try: + api.create_repo(repo_id="Ritesh43/ndlf_model") + except Exception as e: + print(e) + # Upload the Model to Huggingface Hub + push_to_model_hub( + repo_id="Ritesh43/ndlf_model", + folder_path=modelDir, + hf_template=testingDir + "/hugging_face.md", + token="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", + ) + # Download the Model from Huggingface Hub + download_from_hub(repo_id="Ritesh43/ndlf_model", local_dir=modelDir) + + api.delete_repo(repo_id="Ritesh43/ndlf_model") + + sanitize_outputDir() + # Download the Model from Huggingface Hub + # download_from_hub(repo_id="Ritesh43/Gandlf_new_pr", local_dir=modelDir) + + # sanitize_outputDir() + print("passed") + + def test_generic_logging(capsys): - print("52: Starting test for logging") + print("53: Starting test for logging") log_file = "testing/gandlf.log" logger_setup(log_file) message = "Testing logging" @@ -3242,6 +3326,94 @@ def test_generic_logging(capsys): def test_generic_debug_info(): - print("53: Starting test for logging") + print("54: Starting test for logging") _debug_info(True) print("passed") + + +def test_differential_privacy_epsilon_classification_rad_2d(device): + print("54: Testing complex DP training for 2D classification") + # overwrite previous results + sanitize_outputDir() + # read and initialize parameters for specific data dimension + parameters = parseConfig( + testingDir + "/config_classification.yaml", version_check_flag=False + ) + parameters["modality"] = "rad" + parameters["opt"] = "adam" + parameters["patch_size"] = patch_size["2D"] + parameters["batch_size"] = 32 # needs to be revised + parameters["model"]["dimension"] = 2 + parameters["model"]["amp"] = True + # read and parse csv + training_data, parameters["headers"] = parseTrainingCSV( + inputDir + "/train_2d_rad_classification.csv" + ) + parameters = populate_header_in_parameters(parameters, parameters["headers"]) + parameters["model"]["num_channels"] = 3 + parameters["model"]["norm_type"] = "instance" + parameters["differential_privacy"] = {"epsilon": 25.0, "physical_batch_size": 4} + file_config_temp = os.path.join(outputDir, "config_classification_temp.yaml") + # if found in previous run, discard. + if os.path.exists(file_config_temp): + os.remove(file_config_temp) + + with open(file_config_temp, "w") as file: + yaml.dump(parameters, file) + parameters = parseConfig(file_config_temp, version_check_flag=True) + + TrainingManager( + dataframe=training_data, + outputDir=outputDir, + parameters=parameters, + device=device, + resume=False, + reset=True, + ) + sanitize_outputDir() + + print("passed") + + +def test_differential_privacy_simple_classification_rad_2d(device): + print("55: Testing simple DP") + # overwrite previous results + sanitize_outputDir() + # read and initialize parameters for specific data dimension + parameters = parseConfig( + testingDir + "/config_classification.yaml", version_check_flag=False + ) + parameters["modality"] = "rad" + parameters["opt"] = "adam" + parameters["patch_size"] = patch_size["2D"] + parameters["batch_size"] = 32 # needs to be revised + parameters["model"]["dimension"] = 2 + parameters["model"]["amp"] = False + # read and parse csv + training_data, parameters["headers"] = parseTrainingCSV( + inputDir + "/train_2d_rad_classification.csv" + ) + parameters = populate_header_in_parameters(parameters, parameters["headers"]) + parameters["model"]["num_channels"] = 3 + parameters["model"]["norm_type"] = "instance" + parameters["differential_privacy"] = True + file_config_temp = os.path.join(outputDir, "config_classification_temp.yaml") + # if found in previous run, discard. + if os.path.exists(file_config_temp): + os.remove(file_config_temp) + + with open(file_config_temp, "w") as file: + yaml.dump(parameters, file) + parameters = parseConfig(file_config_temp, version_check_flag=True) + + TrainingManager( + dataframe=training_data, + outputDir=outputDir, + parameters=parameters, + device=device, + resume=False, + reset=True, + ) + sanitize_outputDir() + + print("passed") diff --git a/tutorials/classification_medmnist_notebook/config.yaml b/tutorials/classification_medmnist_notebook/config.yaml index 20d9ef784..f1035dc7d 100644 --- a/tutorials/classification_medmnist_notebook/config.yaml +++ b/tutorials/classification_medmnist_notebook/config.yaml @@ -2,7 +2,7 @@ version: { minimum: 0.0.14, - maximum: 0.1.0-dev # this should NOT be made a variable, but should be tested after every tag is created + maximum: 0.1.2-dev # this should NOT be made a variable, but should be tested after every tag is created } # Choose the model parameters here model: