Merge pull request #114 from kabilar/main

Add quality metrics pytests

.devcontainer/Dockerfile

@@ -32,7 +32,6 @@ RUN \
     # git clone https://github.com/datajoint/CaImAn.git  && \
     # pip install --no-cache-dir -r ./CaImAn/requirements.txt  && \
     # pip install --no-cache-dir ./CaImAn && \
-    pip install --no-cache-dir suite2p && \
     pip install --no-cache-dir -r /tmp/requirements.txt && \
     # clean up
     # rm -rf /tmp/CaImAn && \

.devcontainer/devcontainer.json

@@ -1,28 +1,29 @@
-// For format details, see https://aka.ms/devcontainer.json.
 {
-	"name": "Remote Demo",
+	"name": "Tutorial environment",
 	"dockerComposeFile": "docker-compose.yaml",
 	"service": "app",
 	"workspaceFolder": "/workspaces/${localWorkspaceFolderBasename}",
-	// Use this environment variable if you need to bind mount your local source code into a new container.
 	"remoteEnv": {
 		"LOCAL_WORKSPACE_FOLDER": "${localWorkspaceFolder}"
 	},
-	// https://containers.dev/features
 	"onCreateCommand": "mkdir -p ${IMAGING_ROOT_DATA_DIR} && pip install -e . && MYSQL_VER=8.0 docker compose down && MYSQL_VER=8.0 docker compose up --build --wait",
 	"postStartCommand": "docker volume prune -f && s3fs ${DJ_PUBLIC_S3_LOCATION} ${IMAGING_ROOT_DATA_DIR} -o nonempty,multipart_size=530,endpoint=us-east-1,url=http://s3.amazonaws.com,public_bucket=1",
 	"hostRequirements": {
-		"cpus": 4,
-		"memory": "8gb",
+		"cpus": 2,
+		"memory": "16gb",
 		"storage": "32gb"
 	},
 	"forwardPorts": [
 		3306
 	],
-	"customizations": {
+	"customizations": {	
+		"settings": {
+			"python.pythonPath": "/usr/local/bin/python"
+	 	},
 		"vscode": {
 			"extensions": [
-				"ms-python.python"
+				"ms-python.python",
+				"ms-toolsai.jupyter"
 			]
 		}
 	}

.devcontainer/docker-compose.yaml

@@ -2,7 +2,7 @@ version: "3"
 services:
   app:
     extends:
-      file: ./local/docker-compose.yaml
+      file: ./local-data/docker-compose.yaml
       service: app
     environment:
       - DJ_PUBLIC_S3_LOCATION=djhub.vathes.datapub.elements:/workflow-calcium-imaging-data-v2

.devcontainer/local/devcontainer.json → .devcontainer/local-data/devcontainer.json

@@ -1,28 +1,29 @@
-// For format details, see https://aka.ms/devcontainer.json.
 {
-	"name": "Local Demo",
+	"name": "Dev environment with local data",
 	"dockerComposeFile": "docker-compose.yaml",
 	"service": "app",
 	"workspaceFolder": "/workspaces/${localWorkspaceFolderBasename}",
-	// Use this environment variable if you need to bind mount your local source code into a new container.
 	"remoteEnv": {
 		"LOCAL_WORKSPACE_FOLDER": "${localWorkspaceFolder}"
 	},
-	// https://containers.dev/features
 	"onCreateCommand": "pip install -e . && MYSQL_VER=8.0 docker compose down && MYSQL_VER=8.0 docker compose up --build --wait",
 	"postStartCommand": "docker volume prune -f",
 	"hostRequirements": {
-		"cpus": 4,
-		"memory": "8gb",
+		"cpus": 2,
+		"memory": "16gb",
 		"storage": "32gb"
 	},
 	"forwardPorts": [
 		3306
 	],
-	"customizations": {
+	"customizations": {	
+		"settings": {
+			"python.pythonPath": "/usr/local/bin/python"
+	 	},
 		"vscode": {
 			"extensions": [
-				"ms-python.python"
+				"ms-python.python",
+				"ms-toolsai.jupyter"
 			]
 		}
 	}

.devcontainer/local/docker-compose.yaml → .devcontainer/local-data/docker-compose.yaml

@@ -1,8 +1,8 @@
 version: "3"
 services:
   app:
-    cpus: 4
-    mem_limit: 8g
+    cpus: 2
+    mem_limit: 16g
     build:
       context: ../..
       dockerfile: ./.devcontainer/Dockerfile

.devcontainer/local-test/devcontainer.json

@@ -0,0 +1,30 @@
+{
+	"name": "Dev environment for local pytests with remote data",
+	"dockerComposeFile": "docker-compose.yaml",
+	"service": "app",
+	"workspaceFolder": "/workspaces/${localWorkspaceFolderBasename}",
+	"remoteEnv": {
+		"LOCAL_WORKSPACE_FOLDER": "${localWorkspaceFolder}"
+	},
+	"onCreateCommand": "pip install -r ./requirements_dev.txt && pip install -e . && pip install -e ../element-calcium-imaging && MYSQL_VER=8.0 docker compose down && MYSQL_VER=8.0 docker compose up --build --wait",
+	"postStartCommand": "docker volume prune -f && s3fs ${DJ_PUBLIC_S3_LOCATION} ${IMAGING_ROOT_DATA_DIR} -o nonempty,multipart_size=530,endpoint=us-east-1,url=http://s3.amazonaws.com,public_bucket=1",
+	"hostRequirements": {
+		"cpus": 2,
+		"memory": "16gb",
+		"storage": "32gb"
+	},
+	"forwardPorts": [
+		3306
+	],
+	"customizations": {
+		"settings": {
+			"python.pythonPath": "/usr/local/bin/python"
+	 	},
+		"vscode": {
+			"extensions": [
+				"ms-python.python",
+				"ms-toolsai.jupyter"
+			]
+		}
+	}
+}

.devcontainer/local-test/docker-compose.yaml

@@ -0,0 +1,15 @@
+version: "3"
+services:
+  app:
+    cpus: 2
+    mem_limit: 16g
+    environment:
+      - DJ_PUBLIC_S3_LOCATION=djhub.vathes.datapub.elements:/workflow-calcium-imaging-data-v2
+    build:
+      context: ../..
+      dockerfile: ./.devcontainer/Dockerfile
+    extra_hosts:
+      - fakeservices.datajoint.io:127.0.0.1
+    volumes:
+      - ../../..:/workspaces:cached
+    privileged: true # only because of dind

.gitignore

@@ -120,4 +120,7 @@ Diagram.ipynb
 # docker
 .env
 
-example_data
+example_data
+
+# vscode
+*.code-workspace

CHANGELOG.md

@@ -3,6 +3,11 @@
 Observes [Semantic Versioning](https://semver.org/spec/v2.0.0.html) standard and
 [Keep a Changelog](https://keepachangelog.com/en/1.0.0/) convention.
 
+## [0.4.0] - 2023-05-15
+
++ Add - Quality metrics
++ Update - Suite2p version
+
 ## [0.3.1] - 2023-03-09
 
 + Update - Simplify dependencies to reduce Docker image size for GitHub Codespaces
@@ -54,6 +59,7 @@ Observes [Semantic Versioning](https://semver.org/spec/v2.0.0.html) standard and
 + Add - Containerization for pytests
 + Comment - Phase previously designated 0.1.0 -> 0.0.0
 
+[0.4.0]: https://github.com/datajoint/workflow-calcium-imaging/releases/tag/0.4.0
 [0.3.1]: https://github.com/datajoint/workflow-calcium-imaging/releases/tag/0.3.1
 [0.3.0]: https://github.com/datajoint/workflow-calcium-imaging/releases/tag/0.3.0
 [0.2.0]: https://github.com/datajoint/workflow-calcium-imaging/releases/tag/0.2.0

README.md

@@ -1,42 +1,60 @@
 # DataJoint Workflow - Functional Calcium Imaging
 
 DataJoint Workflow for functional calcium imaging combines multiple DataJoint Elements to process data acquired with [ScanImage](http://scanimage.vidriotechnologies.com), [Scanbox](https://scanbox.org), `Nikon
-NIS`, or `PrairieView` acquisition software, using [Suite2p](https://github.com/MouseLand/suite2p) or [CaImAn](https://github.com/flatironinstitute/CaImAn) analysis software. DataJoint Elements collectively standardize and automate data collection and analysis for neuroscience experiments. Each Element is a modular pipeline for data storage and processing with corresponding database tables that can be combined with other Elements to assemble a fully functional pipeline.
+NIS`, or `PrairieView` acquisition software, using [Suite2p](https://github.com/MouseLand/suite2p) or [CaImAn](https://github.com/flatironinstitute/CaImAn) analysis software. DataJoint Elements collectively standardize and automate data collection and analysis for neuroscience experiments. Each Element is a modular pipeline for data storage and processing with corresponding database tables that can be combined with other Elements to assemble a fully functional pipeline. This repository also provides a tutorial environment and notebook to learn the pipeline.
 
-To get started, see below for an [interactive tutorial](#interactive-tutorial) on GitHub Codespaces.  More information can be found at the
-[Element documentation page](https://datajoint.com/docs/elements/element-calcium-imaging).
+## Experiment Flowchart
 
-![diagram](https://raw.githubusercontent.com/datajoint/element-calcium-imaging/main/images/diagram_flowchart.svg)
+![flowchart](https://raw.githubusercontent.com/datajoint/element-calcium-imaging/main/images/flowchart.svg)
+
+## Data Pipeline Diagram
+
+![pipeline](https://raw.githubusercontent.com/datajoint/element-calcium-imaging/main/images/pipeline_imaging.svg)
+
+## Getting Started
+
++ [Interactive tutorial on GitHub Codespaces](#interactive-tutorial)
+
++ Install Element Calcium Imaging from PyPI
+
+     ```bash
+     pip install element-calcium-imaging
+     ```
+
++ [Documentation](https://datajoint.com/docs/elements/element-calcium-imaging)
+
+## Support
+
++ If you need help getting started or run into any errors, please contact our team by email at [email protected].
 
 ## Interactive Tutorial
 
-The easiest way to learn about DataJoint Elements is to use the tutorial notebooks within the included interactive environment configured using [DevContainer](https://containers.dev/).
++ The easiest way to learn about DataJoint Elements is to use the tutorial notebooks within the included interactive environment configured using [DevContainer](https://containers.dev/).
 
 ### Launch Environment
 
 Here are some options that provide a great experience:
 
-- Cloud-based Development Environment: (*recommended*)
+- Cloud-based Environment (*recommended*)
   - Launch using [GitHub Codespaces](https://github.com/features/codespaces) using the `+` option which will `Create codespace on main` in the codebase repository on your fork with default options. For more control, see the `...` where you may create `New with options...`.
-  - Build time for a codespace is **~9m**. This is done infrequently and cached for convenience.
-  - Start time for a codespace is **~40s**. This will pull the built codespace from cache when you need it.
+  - Build time for a codespace is several minutes. This is done infrequently and cached for convenience.
+  - Start time for a codespace is less than 1 minute. This will pull the built codespace from cache when you need it.
   - *Tip*: Each month, GitHub renews a [free-tier](https://docs.github.com/en/billing/managing-billing-for-github-codespaces/about-billing-for-github-codespaces#monthly-included-storage-and-core-hours-for-personal-accounts) quota of compute and storage. Typically we run into the storage limits before anything else since Codespaces consume storage while stopped. It is best to delete Codespaces when not actively in use and recreate when needed. We'll soon be creating prebuilds to avoid larger build times. Once any portion of your quota is reached, you will need to wait for it to be reset at the end of your cycle or add billing info to your GitHub account to handle overages.
   - *Tip*: GitHub auto names the codespace but you can rename the codespace so that it is easier to identify later.
-- Local Development Environment:
+
+- Local Environment
   - Note: On Windows, running the tutorial notebook with the example data in a Dev Container is not currently possible due to a s3fs mounting issue.  Please use the `Cloud-based Development Environment` option above.
   - Install [Git](https://git-scm.com/book/en/v2/Getting-Started-Installing-Git)
   - Install [Docker](https://docs.docker.com/get-docker/)
   - Install [VSCode](https://code.visualstudio.com/)
   - Install the VSCode [Dev Containers extension](https://marketplace.visualstudio.com/items?itemName=ms-vscode-remote.remote-containers)
   - `git clone` the codebase repository and open it in VSCode
-  - Use the `Dev Containers extension` to `Reopen in Container` (More info in the `Getting started` included with the extension)
+  - Use the `Dev Containers extension` to `Reopen in Container` (More info is in the `Getting started` included with the extension.)
 
-You will know your environment has finished loading once you either see a terminal open related to `Running postStartCommand` with a final message: `Done` or the `README.md` is opened in `Preview`.
+You will know your environment has finished loading once you either see a terminal open related to `Running postStartCommand` with a final message of `Done` or the `README.md` is opened in `Preview`.
 
 ### Instructions
 
 1. We recommend you start by navigating to the `notebooks` directory on the left panel and go through the `demo_prepare.ipynb` and `demo_run.ipynb` Jupyter notebooks. Execute the cells in the notebooks to begin your walk through of the tutorial.
 
-1. Once you are done, see the options available to you in the menu in the bottom-left corner. For example, in Codespace you will have an option to `Stop Current Codespace` but when running DevContainer on your own machine the equivalent option is `Reopen folder locally`. By default, GitHub will also automatically stop the Codespace after 30 minutes of inactivity.
-
-If you are new to GitHub and run into any errors, please contact us via email at [email protected]. If you are experienced with GitHub, please create an issue on the upstream repository or if you'd like help contribute, feel free to create a pull request. Please include a thorough explanantion of the error and/or proposed solution.
+1. Once you are done, see the options available to you in the menu in the bottom-left corner. For example, in Codespace you will have an option to `Stop Current Codespace` but when running Dev Container on your own machine the equivalent option is `Reopen folder locally`. By default, GitHub will also automatically stop the Codespace after 30 minutes of inactivity.  Once the Codespace is no longer being used, we recommend deleting the Codespace.