Merge pull request #90 from iraikov/feature/miv_file

Initial implementation of MiV HDF5-based data format

.github/workflows/ci-sphinx.yml

@@ -29,7 +29,7 @@ jobs:
             poetry --version
             poetry config virtualenvs.in-project true
             #poetry config virtualenvs.create false
-            poetry install -E docs
+            poetry install -E "docs experiment"
       - name: Sphinx Build Check
         run: |
             source .venv/bin/activate
@@ -45,9 +45,9 @@ jobs:
                 echo Doc build success
                 exit 0
             fi
-      - name: Sphinx Link Check
-        run: |
-            source .venv/bin/activate
-            cd docs
-            make clean
-            make linkcheck
+    #- name: Sphinx Link Check
+        #run: |
+            #source .venv/bin/activate
+            #cd docs
+            #make clean
+            #make linkcheck

.github/workflows/main.yml

@@ -42,7 +42,7 @@ jobs:
       - name: Install dependencies
         run: |
           poetry config virtualenvs.in-project true
-          poetry install
+          poetry install -E "docs experiment"
       # Test MiV-OS using pytest
       - name: Run tests
         run: |
@@ -78,7 +78,7 @@ jobs:
       - name: Install dependencies
         run: |
           poetry config virtualenvs.in-project true
-          poetry install
+          poetry install -E "docs experiment"
       # Set environment variables for coverage test. Coverage test is done using python 3.8
       - name: Run style checks
         run: |

.gitignore

@@ -235,3 +235,6 @@ outcmaes/*
 
 # csv files
 *.csv
+
+# Emacs backup:
+*~

docs/api/io.rst

@@ -1,12 +1,21 @@
 **************************************
-Data Managing Module (:mod:`miv.io`)
+IO Module (:mod:`miv.io`)
 **************************************
 
+Python Data Managing
+====================
+
 .. automodule:: miv.io.data
 
 .. automodule:: miv.io.binary
    :members:
 
+External Datafile (H5)
+======================
+
+.. automodule:: miv.io.file
+   :members:
+
 Serial Communication Helper
 ===========================
 

docs/guide/data_structure/hdf5_based_data_format.md

@@ -0,0 +1,84 @@
+---
+jupytext:
+  text_representation:
+    extension: .md
+    format_name: myst
+    format_version: 0.13
+    jupytext_version: 1.14.1
+kernelspec:
+  display_name: Python 3
+  language: python
+  name: python3
+mystnb:
+  execution_mode: 'off'
+---
+
+# HDF5 Read
+
+```{code-cell} ipython3
+:tags: [hide-cell]
+
+# Import required modules
+import os
+import sys
+
+import matplotlib.cm as cm
+import matplotlib.colors as colors
+import matplotlib.pyplot as plt
+import numpy as np
+from sklearn.decomposition import PCA
+from sklearn.mixture import GaussianMixture
+from viziphant.rasterplot import rasterplot_rates
+
+import miv
+from miv.io import DataManager
+from miv.io import file as miv_file
+from miv.signal.filter import ButterBandpass, FilterCollection, MedianFilter
+from miv.signal.spike import PCADecomposition, SpikeSorting, ThresholdCutoff
+from miv.statistics import firing_rates, signal_to_noise
+from miv.visualization import extract_waveforms, plot_frequency_domain, plot_waveforms
+```
+
+## Read h5
+
+```{code-cell} ipython3
+recording_id = "2022-03-10_16-19-09-Record Node 104-experiment1_spontaneous-recording1"
+input_data, data_container = miv_file.read(
+    "2022-03-10_16-19-09/MiV_data.h5", groups=recording_id
+)
+```
+
+## Data Access
+
+```{code-cell} ipython3
+signal = input_data[f"{recording_id}/signal"]
+timestamps = input_data[f"{recording_id}/timestamps"]
+sampling_rate = input_data[f"{recording_id}/sampling_rate"][0]
+num_channel = signal.shape[-1]
+```
+
+## Processing Example
+
+```{code-cell} ipython3
+# Set up filters, we use butter bandpass and median filters here. More details are here (https://miv-os.readthedocs.io/en/latest/api/signal.html)
+signal_filter = (
+    FilterCollection()
+    .append(ButterBandpass(300, 3000, order=4))
+    .append(MedianFilter(threshold=60, k=20))
+)
+
+# Set threshold for Signal to noise ratio to detect spikes
+spike_detection = ThresholdCutoff(cutoff=5)
+
+# Preprocess
+signal = signal_filter(signal[0], sampling_rate)
+spiketrains = spike_detection(signal, timestamps, sampling_rate)
+
+# Estimate Firing Rate
+stat = firing_rates(spiketrains)["rates"]
+
+# Plot rasterplot using viziphant
+
+plt.figure(figsize=(24, 8))
+a, b, c = rasterplot_rates(spiketrains, ax=plt.gca())
+```

docs/guide/index.rst

@@ -1,6 +1,8 @@
 How-To Guide
 ============
 
+Here we provide number of example scripts using `MiV` tools. Some examples provide additional files or links to published paper for a complete description. Examples are written to serve as a starting template for customized usages.
+
 .. toctree::
    :maxdepth: 2
    :caption: Analysis
@@ -20,9 +22,10 @@ How-To Guide
 
 .. toctree::
    :maxdepth: 2
-   :caption: Sample Data
+   :caption: Data / Data Structure
 
    sample_datasets
+   data_structure/hdf5_based_data_format
 
 Indices and tables
 ==================

miv/io/file/__init__.py

@@ -0,0 +1,17 @@
+__doc__ = """
+HDF5-based file format for heterogeneous numerical data.
+Based on code from and inspired by
+
+- HEPfile: https://github.com/mattbellis/hepfile
+- NeuroH5: https://github.com/iraikov/neuroh5
+
+.. automodule:: miv.io.file.read
+   :members:
+
+.. automodule:: miv.io.file.write
+   :members:
+
+"""
+
+from miv.io.file.read import *
+from miv.io.file.write import *