Merge branch 'develop' into shrivaths/add-zmq-training-gui

talmolab · May 28, 2024 · cd56e1b · cd56e1b
2 parents 3bc55b1 + 74bada8
commit cd56e1b
Show file tree

Hide file tree

Showing 12 changed files with 188 additions and 25 deletions.
diff --git a/.conda/meta.yaml b/.conda/meta.yaml
@@ -91,6 +91,8 @@ requirements:
     - conda-forge::ndx-pose
     - conda-forge::importlib-metadata ==4.11.4
 
-test:
-  imports:
-    - sleap
+# This no longer works so we have moved it to the build workflow
+# https://github.com/talmolab/sleap/pull/1744
+# test:
+#   imports:
+#     - sleap
diff --git a/.conda_mac/build.sh b/.conda_mac/build.sh
@@ -2,11 +2,10 @@
 
 # Install anything that didn't get conda installed via pip.
 # We need to turn pip index back on because Anaconda turns it off for some reason. 
-
 export PIP_NO_INDEX=False
 export PIP_NO_DEPENDENCIES=False
 export PIP_IGNORE_INSTALLED=False
 
-pip install --no-cache-dir -r requirements.txt --no-binary qudida,albumentations
+pip install --no-cache-dir -r requirements.txt
 
 python setup.py install --single-version-externally-managed --record=record.txt
diff --git a/.conda_mac/condarc.yaml b/.conda_mac/condarc.yaml
@@ -1,4 +1,3 @@
-# This file is not used at the moment, but when github actions can be used to build the package, it needs to be listed.
 # https://github.com/github/roadmap/issues/528
 
 channels:

diff --git a/.conda_mac/meta.yaml b/.conda_mac/meta.yaml
@@ -23,7 +23,7 @@ source:
 
 requirements:
   host:  
-    - conda-forge::python ~=3.9
+    - conda-forge::python >=3.9.0, <3.10.0
     - anaconda::numpy >=1.19.5,<1.23.0
     - conda-forge::setuptools
     - conda-forge::packaging
@@ -59,7 +59,7 @@ requirements:
     - conda-forge::ndx-pose
 
   run:
-    - conda-forge::python ~=3.9
+    - conda-forge::python >=3.9.0, <3.10.0
     - conda-forge::attrs >=21.2.0
     - conda-forge::cattrs ==1.1.1
     - conda-forge::h5py
@@ -89,6 +89,6 @@ requirements:
     - conda-forge::albumentations
     - conda-forge::ndx-pose
 
-test:
-  imports:
-    - sleap
+# test:
+#   imports:
+#     - sleap
diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml
@@ -18,34 +18,41 @@ jobs:
         include:
           # Use this condarc as default
           - condarc: .conda/condarc.yaml
+          - pyver: "3.7"
           # Use special condarc if macos
           - os: "macos-14"
             condarc: .conda_mac/condarc.yaml
+            pyver: "3.9"
     steps:
       # Setup
       - uses: actions/checkout@v2
       - name: Cache conda
         uses: actions/cache@v1
         env:
           # Increase this value to reset cache if environment_build.yml has not changed
-          CACHE_NUMBER: 0
+          CACHE_NUMBER: 1
         with:
           path: ~/conda_pkgs_dir
           key: ${{ runner.os }}-conda-${{ env.CACHE_NUMBER }}-${{ hashFiles('environment_build.yml', 'requirements.txt') }}
+
       - name: Setup Miniconda
-        # https://github.com/conda-incubator/setup-miniconda
-        uses: conda-incubator/[email protected]
+        uses: conda-incubator/[email protected]
         with:
-          python-version: 3.7
-          use-only-tar-bz2: true # IMPORTANT: This needs to be set for caching to work properly!
-          environment-file: environment_build.yml
+          miniforge-version: latest
           condarc-file: ${{ matrix.condarc }}
+          python-version: ${{ matrix.pyver }}
+          environment-file: environment_build.yml
           activate-environment: sleap_ci
+          conda-solver: "libmamba"
+          # Increase the cache number if build dependencies change! Otherwise, the cache will not be invalidated.
+          use-only-tar-bz2: true # IMPORTANT: This needs to be set for caching to work properly!
+
       - name: Print environment info
         shell: bash -l {0}
         run: |
           which python
           conda info
+          conda list
 
       # Build pip wheel (Ubuntu)
       - name: Build pip wheel (Ubuntu)
@@ -69,21 +76,92 @@ jobs:
         shell: bash -l {0}
         run: |
           conda build .conda --output-folder build
+          echo "BUILD_PATH=$(pwd)/build" >> "$GITHUB_ENV"
 
-      # Build conda package (Windows)
+      # Build conda package (Windows) 
       - name: Build conda package (Windows)
         if: matrix.os == 'windows-2022'
         shell: powershell
         run: |
           conda build .conda --output-folder build
+          echo "BUILD_PATH=\$(pwd)\build" >> "$env:GITHUB_ENV"
 
       # Build conda package (Mac)
       - name: Build conda package (Mac)
         if: matrix.os == 'macos-14'
         shell: bash -l {0}
         run: |
           conda build .conda_mac --output-folder build
+          echo "BUILD_PATH=$(pwd)/build" >> "$GITHUB_ENV"
+
+      # Test built conda package (Ubuntu and Windows)
+      - name: Test built conda package (Ubuntu and Windows)
+        if: matrix.os != 'macos-14'
+        shell: bash -l {0}
+        run: |
+          echo "Current build path: $BUILD_PATH"
+          conda deactivate
+    
+          echo "Python executable before activating environment:"
+          which python
+          echo "Python version before activating environment:"
+          python --version
+          echo "Conda info before activating environment:"
+          conda info
+
+          echo "Creating and testing conda environment with sleap package..."
+          conda create -y -n sleap_test -c file://$BUILD_PATH -c sleap/label/dev -c conda-forge -c nvidia -c anaconda sleap
+          conda activate sleap_test
 
+          echo "Python executable after activating sleap_test environment:"
+          which python
+          echo "Python version after activating sleap_test environment:"
+          python --version
+          echo "Conda info after activating sleap_test environment:"
+          conda info
+          echo "List of installed conda packages in the sleap_test environment:"
+          conda list
+          echo "List of installed pip packages in the sleap_test environment:"
+          pip list
+
+          echo "Testing sleap package installation..."
+          sleap_version=$(python -c "import sleap; print(sleap.__version__)")
+          echo "Test completed using sleap version: $sleap_version"
+
+      # Test built conda package (Mac)
+      - name: Test built conda package (Mac)
+        if: matrix.os == 'macos-14'
+        shell: bash -l {0}
+        run: |
+          echo "Current build path: $BUILD_PATH"
+          conda deactivate
+    
+          echo "Python executable before activating environment:"
+          which python
+          echo "Python version before activating environment:"
+          python --version
+          echo "Conda info before activating environment:"
+          conda info
+
+          echo "Creating and testing conda environment with sleap package..."
+          conda create -y -n sleap_test -c file://$BUILD_PATH -c conda-forge -c anaconda sleap
+          conda activate sleap_test
+
+          echo "Python executable after activating sleap_test environment:"
+          which python
+          echo "Python version after activating sleap_test environment:"
+          python --version
+          echo "Conda info after activating sleap_test environment:"
+          conda info
+          echo "List of installed conda packages in the sleap_test environment:"
+          conda list
+          echo "List of installed pip packages in the sleap_test environment:"
+          pip list
+
+          echo "Testing sleap package installation..."
+          sleap_version=$(python -c "import sleap; print(sleap.__version__)")
+          echo "Test completed using sleap version: $sleap_version"
+
       # Login to conda (Ubuntu)
       - name: Login to Anaconda (Ubuntu)
         if: matrix.os == 'ubuntu-22.04'

diff --git a/.github/workflows/build_ci.yml b/.github/workflows/build_ci.yml
@@ -1,4 +1,4 @@
-# Run tests using built conda packages and wheels.
+# Run tests using built wheels.
 name: Build CI (no upload)
 
 # Run when changes to pip wheel
@@ -79,7 +79,8 @@ jobs:
     strategy:
       fail-fast: false
       matrix:
-        os: ["ubuntu-22.04", "windows-2022", "macos-14"]
+        os: ["ubuntu-22.04", "windows-2022"]
+        # os: ["ubuntu-22.04", "windows-2022", "macos-14"]  # removing macos-14 for now since the setup-python action only support py>=3.10, which is breaking this CI.
         # https://docs.github.com/en/actions/using-workflows/workflow-syntax-for-github-actions#jobsjob_idstrategymatrixinclude
         include:
           # Default values

diff --git a/.github/workflows/build_manual.yml b/.github/workflows/build_manual.yml
@@ -12,7 +12,6 @@ on:
     branches:
       # - develop
       - fakebranch
-      # - talmo/fix-mac-v140
 
 jobs:
   build:
@@ -83,20 +82,91 @@ jobs:
         shell: bash -l {0}
         run: |
           conda build .conda --output-folder build
+          echo "BUILD_PATH=$(pwd)/build" >> "$GITHUB_ENV"
 
-      # Build conda package (Windows)
+      # Build conda package (Windows) 
       - name: Build conda package (Windows)
         if: matrix.os == 'windows-2022'
         shell: powershell
         run: |
           conda build .conda --output-folder build
+          echo "BUILD_PATH=\$(pwd)\build" >> "$env:GITHUB_ENV"
 
       # Build conda package (Mac)
       - name: Build conda package (Mac)
         if: matrix.os == 'macos-14'
         shell: bash -l {0}
         run: |
           conda build .conda_mac --output-folder build
+          echo "BUILD_PATH=$(pwd)/build" >> "$GITHUB_ENV"
+
+      # Test built conda package (Ubuntu and Windows)
+      - name: Test built conda package (Ubuntu and Windows)
+        if: matrix.os != 'macos-14'
+        shell: bash -l {0}
+        run: |
+          echo "Current build path: $BUILD_PATH"
+          conda deactivate
+    
+          echo "Python executable before activating environment:"
+          which python
+          echo "Python version before activating environment:"
+          python --version
+          echo "Conda info before activating environment:"
+          conda info
+
+          echo "Creating and testing conda environment with sleap package..."
+          conda create -y -n sleap_test -c file://$BUILD_PATH -c sleap/label/dev -c conda-forge -c nvidia -c anaconda sleap
+          conda activate sleap_test
+
+          echo "Python executable after activating sleap_test environment:"
+          which python
+          echo "Python version after activating sleap_test environment:"
+          python --version
+          echo "Conda info after activating sleap_test environment:"
+          conda info
+          echo "List of installed conda packages in the sleap_test environment:"
+          conda list
+          echo "List of installed pip packages in the sleap_test environment:"
+          pip list
+
+          echo "Testing sleap package installation..."
+          sleap_version=$(python -c "import sleap; print(sleap.__version__)")
+          echo "Test completed using sleap version: $sleap_version"
+
+      # Test built conda package (Mac)
+      - name: Test built conda package (Mac)
+        if: matrix.os == 'macos-14'
+        shell: bash -l {0}
+        run: |
+          echo "Current build path: $BUILD_PATH"
+          conda deactivate
+    
+          echo "Python executable before activating environment:"
+          which python
+          echo "Python version before activating environment:"
+          python --version
+          echo "Conda info before activating environment:"
+          conda info
+
+          echo "Creating and testing conda environment with sleap package..."
+          conda create -y -n sleap_test -c file://$BUILD_PATH -c conda-forge -c anaconda sleap
+          conda activate sleap_test
+
+          echo "Python executable after activating sleap_test environment:"
+          which python
+          echo "Python version after activating sleap_test environment:"
+          python --version
+          echo "Conda info after activating sleap_test environment:"
+          conda info
+          echo "List of installed conda packages in the sleap_test environment:"
+          conda list
+          echo "List of installed pip packages in the sleap_test environment:"
+          pip list
+
+          echo "Testing sleap package installation..."
+          sleap_version=$(python -c "import sleap; print(sleap.__version__)")
+          echo "Test completed using sleap version: $sleap_version"
 
       # # Login to conda (Ubuntu)
       # - name: Login to Anaconda (Ubuntu)

diff --git a/requirements.txt b/requirements.txt
@@ -1,7 +1,7 @@
 # This file contains the minimal requirements to be installed via pip when using conda.
 
 # No conda packages for these
-imgstore<0.3.0  # 0.3.3 results in https://github.com/O365/python-o365/issues/591
+imgstore<0.3.0  # 0.3.3 results in https://github.com/O365/python-o365/issues/591 which is from https://github.com/regebro/tzlocal/issues/112 when tzlocal is v3.0
 nixio>=1.5.3  # Constrain put on by @jgrewe from G-Node
 qimage2ndarray  # ==1.9.0
 segmentation-models

diff --git a/sleap/config/pipeline_form.yaml b/sleap/config/pipeline_form.yaml
@@ -382,6 +382,12 @@ inference:
 
   tracking-only:
 
+- name: batch_size
+  label: Batch Size 
+  type: int
+  default: 4
+  range: 1,512
+
 - name: tracking.tracker
   label: Tracker (cross-frame identity) Method
   type: stacked

diff --git a/sleap/gui/learning/dialog.py b/sleap/gui/learning/dialog.py
@@ -605,6 +605,7 @@ def get_selected_frames_to_predict(
 
     def get_items_for_inference(self, pipeline_form_data) -> runners.ItemsForInference:
         predict_frames_choice = pipeline_form_data.get("_predict_frames", "")
+        batch_size = pipeline_form_data.get("batch_size")
 
         frame_selection = self.get_selected_frames_to_predict(pipeline_form_data)
         frame_count = self.count_total_frames_for_selection_option(frame_selection)
@@ -617,6 +618,7 @@ def get_items_for_inference(self, pipeline_form_data) -> runners.ItemsForInferen
                     )
                 ],
                 total_frame_count=frame_count,
+                batch_size=batch_size,
             )
         elif predict_frames_choice.startswith("suggested"):
             items_for_inference = runners.ItemsForInference(
@@ -626,13 +628,15 @@ def get_items_for_inference(self, pipeline_form_data) -> runners.ItemsForInferen
                     )
                 ],
                 total_frame_count=frame_count,
+                batch_size=batch_size,
             )
         else:
             items_for_inference = runners.ItemsForInference.from_video_frames_dict(
                 video_frames_dict=frame_selection,
                 total_frame_count=frame_count,
                 labels_path=self.labels_filename,
                 labels=self.labels,
+                batch_size=batch_size,
             )
         return items_for_inference