MRG, ENH: Fix memory on CircleCI

.circleci/config.yml

@@ -113,115 +113,13 @@ jobs:
               python setup.py develop --user
               mkdir -p ~/mne_data
               touch pattern.txt;
-              if [ "$CIRCLE_BRANCH" == "master" ] || [[ $(cat gitlog.txt) == *"[circle full]"* ]]; then
-                echo "Doing a full dev build";
-                echo html_dev > build.txt;
-                python -c "import mne; mne.datasets._download_all_example_data()";
-              elif [ "$CIRCLE_BRANCH" == "maint/0.21" ]; then
-                echo "Doing a full stable build";
-                echo html_stable > build.txt;
-                python -c "import mne; mne.datasets._download_all_example_data()";
-              else
-                echo "Doing a partial build";
-                if ! git remote -v | grep upstream ; then git remote add upstream git://github.com/mne-tools/mne-python.git; fi
-                git fetch upstream
-                FNAMES=$(git diff --name-only $(git merge-base $CIRCLE_BRANCH upstream/master) $CIRCLE_BRANCH);
-                if [[ $(cat gitlog.txt) == *"[circle front]"* ]]; then
-                  FNAMES="tutorials/source-modeling/plot_mne_dspm_source_localization.py tutorials/machine-learning/plot_receptive_field.py examples/connectivity/plot_mne_inverse_label_connectivity.py tutorials/machine-learning/plot_sensors_decoding.py tutorials/stats-source-space/plot_stats_cluster_spatio_temporal.py tutorials/evoked/plot_20_visualize_evoked.py "${FNAMES};
-                  python -c "import mne; print(mne.datasets.testing.data_path(update_path=True))";
-                fi;
-                echo FNAMES="$FNAMES";
-                for FNAME in $FNAMES; do
-                  if [[ `expr match $FNAME "\(tutorials\|examples\)/.*plot_.*\.py"` ]] ; then
-                    echo "Checking example $FNAME ...";
-                    PATTERN=`basename $FNAME`"\\|"$PATTERN;
-                    if [[ $(cat $FNAME | grep -x ".*datasets.*sample.*" | wc -l) -gt 0 ]]; then
-                      python -c "import mne; print(mne.datasets.sample.data_path(update_path=True))";
-                    fi;
-                    if [[ $(cat $FNAME | grep -x ".*datasets.*fetch_fsaverage.*" | wc -l) -gt 0 ]]; then
-                      python -c "import mne; print(mne.datasets.fetch_fsaverage(verbose=True))";
-                    fi;
-                    if [[ $(cat $FNAME | grep -x ".*datasets.*spm_face.*" | wc -l) -gt 0 ]]; then
-                      python -c "import mne; print(mne.datasets.spm_face.data_path(update_path=True))";
-                    fi;
-                    if [[ $(cat $FNAME | grep -x ".*datasets.*somato.*" | wc -l) -gt 0 ]]; then
-                      python -c "import mne; print(mne.datasets.somato.data_path(update_path=True))";
-                    fi;
-                    if [[ $(cat $FNAME | grep -x ".*datasets.*eegbci.*" | wc -l) -gt 0 ]]; then
-                      python -c "import mne; print(mne.datasets.eegbci.load_data(1, [3, 6, 10, 14], update_path=True))";
-                      python -c "import mne; print(mne.datasets.eegbci.load_data(2, [3], update_path=True))";
-                      python -c "import mne; print(mne.datasets.eegbci.load_data(3, [3], update_path=True))";
-                      python -c "import mne; print(mne.datasets.eegbci.load_data(4, [3], update_path=True))";
-                    fi;
-                    if [[ $(cat $FNAME | grep -x ".*datasets.*sleep_physionet.*" | wc -l) -gt 0 ]]; then
-                      python -c "import mne; print(mne.datasets.sleep_physionet.age.fetch_data([0, 1], recording=[1], update_path=True))";
-                    fi;
-                    if [[ $(cat $FNAME | grep -x ".*datasets.*hf_sef.*" | wc -l) -gt 0 ]]; then
-                      python -c "import mne; print(mne.datasets.hf_sef.data_path(update_path=True))";
-                    fi;
-                    if [[ $(cat $FNAME | grep -x ".*brainstorm.*bst_auditory.*" | wc -l) -gt 0 ]]; then
-                      python -c "import mne; print(mne.datasets.brainstorm.bst_auditory.data_path(update_path=True))" --accept-brainstorm-license;
-                    fi;
-                    if [[ $(cat $FNAME | grep -x ".*brainstorm.*bst_resting.*" | wc -l) -gt 0 ]]; then
-                      python -c "import mne; print(mne.datasets.brainstorm.bst_resting.data_path(update_path=True))" --accept-brainstorm-license;
-                    fi;
-                    if [[ $(cat $FNAME | grep -x ".*brainstorm.*bst_raw.*" | wc -l) -gt 0 ]]; then
-                      python -c "import mne; print(mne.datasets.brainstorm.bst_raw.data_path(update_path=True))" --accept-brainstorm-license;
-                    fi;
-                    if [[ $(cat $FNAME | grep -x ".*brainstorm.*bst_phantom_ctf.*" | wc -l) -gt 0 ]]; then
-                      python -c "import mne; print(mne.datasets.brainstorm.bst_phantom_ctf.data_path(update_path=True))" --accept-brainstorm-license;
-                    fi;
-                    if [[ $(cat $FNAME | grep -x ".*brainstorm.*bst_phantom_elekta.*" | wc -l) -gt 0 ]]; then
-                      python -c "import mne; print(mne.datasets.brainstorm.bst_phantom_elekta.data_path(update_path=True))" --accept-brainstorm-license;
-                    fi;
-                    if [[ $(cat $FNAME | grep -x ".*datasets.*hcp_mmp_parcellation.*" | wc -l) -gt 0 ]]; then
-                      python -c "import mne; print(mne.datasets.sample.data_path(update_path=True))";
-                      python -c "import mne; print(mne.datasets.fetch_hcp_mmp_parcellation(subjects_dir=mne.datasets.sample.data_path() + '/subjects'))" --accept-hcpmmp-license;
-                    fi;
-                    if [[ $(cat $FNAME | grep -x ".*datasets.*misc.*" | wc -l) -gt 0 ]]; then
-                      python -c "import mne; print(mne.datasets.misc.data_path(update_path=True))";
-                    fi;
-                    if [[ $(cat $FNAME | grep -x ".*datasets.*testing.*" | wc -l) -gt 0 ]]; then
-                      python -c "import mne; print(mne.datasets.testing.data_path(update_path=True))";
-                    fi;
-                    if [[ $(cat $FNAME | grep -x ".*datasets.*kiloword.*" | wc -l) -gt 0 ]]; then
-                      python -c "import mne; print(mne.datasets.kiloword.data_path(update_path=True))";
-                    fi;
-                    if [[ $(cat $FNAME | grep -x ".*datasets.*mtrf.*" | wc -l) -gt 0 ]]; then
-                      python -c "import mne; print(mne.datasets.mtrf.data_path(update_path=True))";
-                    fi;
-                    if [[ $(cat $FNAME | grep -x ".*datasets.*fieldtrip_cmc.*" | wc -l) -gt 0 ]]; then
-                      python -c "import mne; print(mne.datasets.fieldtrip_cmc.data_path(update_path=True))";
-                    fi;
-                    if [[ $(cat $FNAME | grep -x ".*datasets.*multimodal.*" | wc -l) -gt 0 ]]; then
-                      python -c "import mne; print(mne.datasets.multimodal.data_path(update_path=True))";
-                    fi;
-                    if [[ $(cat $FNAME | grep -x ".*datasets.*fnirs_motor.*" | wc -l) -gt 0 ]]; then
-                      python -c "import mne; print(mne.datasets.fnirs_motor.data_path(update_path=True))";
-                    fi;
-                    if [[ $(cat $FNAME | grep -x ".*datasets.*opm.*" | wc -l) -gt 0 ]]; then
-                      python -c "import mne; print(mne.datasets.opm.data_path(update_path=True))";
-                    fi;
-                    if [[ $(cat $FNAME | grep -x ".*datasets.*phantom_4dbti.*" | wc -l) -gt 0 ]]; then
-                      python -c "import mne; print(mne.datasets.phantom_4dbti.data_path(update_path=True))";
-                    fi;
-                    if [[ $(cat $FNAME | grep -x ".*datasets.*limo.*" | wc -l) -gt 0 ]]; then
-                      python -c "import mne; print(mne.datasets.limo.data_path(subject=1, update_path=True))";
-                    fi;
-                    if [[ $(cat $FNAME | grep -x ".*datasets.*refmeg_noise.*" | wc -l) -gt 0 ]]; then
-                      python -c "import mne; print(mne.datasets.refmeg_noise.data_path(update_path=True))";
-                    fi;
-                  fi;
-                done;
-                echo PATTERN="$PATTERN";
-                if [[ $PATTERN ]]; then
-                  PATTERN="\(${PATTERN::-2}\)";
-                  echo html_dev-pattern > build.txt;
-                else
-                  echo html_dev-noplot > build.txt;
-                fi;
-              fi;
-              echo "$PATTERN" > pattern.txt;
+              ./tools/circleci_download.sh
+
+        - run:
+            name: Get data (again)
+            when: on_fail
+            command: |
+              ./tools/circleci_download.sh
 
         - run:
             name: Verify build type
@@ -234,7 +132,7 @@ jobs:
         - run:
             name: make test-doc
             command: |
-              if [[ $(cat gitlog.txt) == *"[circle front]"* ]] || [[ $(cat build.txt) == "html_dev" ]] || [[ $(cat build.txt) == "html_stable" ]]; then
+              if [[ $(cat gitlog.txt) == *"[circle front]"* ]] || [[ $(cat build.txt) == "html_dev-memory" ]] || [[ $(cat build.txt) == "html_stable-memory" ]]; then
                 make test-doc;
                 mkdir -p doc/_build/test-results/test-doc;
                 cp junit-results.xml doc/_build/test-results/test-doc/junit.xml;
@@ -245,6 +143,16 @@ jobs:
             command: |
               cd doc;
               PATTERN=$(cat ../pattern.txt) make $(cat ../build.txt);
+        - run:
+            name: Show profiling output
+            when: always
+            command: |
+              if compgen -G "doc/*.dat" > /dev/null; then
+                mkdir -p doc/generated
+                mprof plot doc/*.dat --output doc/generated/memory.png
+              else
+                echo "No profile data found in doc/"
+              fi
         - run:
             name: Sanity check system state
             command: |
@@ -254,7 +162,7 @@ jobs:
         - run:
             name: Reduce artifact upload time
             command: |
-              if grep -q html_dev-pattern build.txt || grep -q html_dev-noplot build.txt; then
+              if grep -q html_dev-pattern-memory build.txt || grep -q html_dev-noplot build.txt; then
                 tar czf doc/_build/html/_downloads.tgz doc/_build/html/_downloads
                 rm -Rf doc/_build/html/_downloads
                 rm -f doc/auto_*/*/*.pickle

.gitignore

@@ -61,6 +61,8 @@ pip-log.txt
 tags
 doc/coverages
 doc/samples
+doc/*.dat
+doc/fil-result
 cover
 *.html
 

doc/Makefile

@@ -5,6 +5,7 @@
 SPHINXOPTS    = -nWT --keep-going
 SPHINXBUILD   = sphinx-build
 PAPER         =
+MPROF         = SG_STAMP_STARTS=true mprof run --python sphinx
 
 # Internal variables.
 PAPEROPT_a4     = -D latex_paper_size=a4
@@ -40,13 +41,28 @@ html_stable:
 	@echo
 	@echo "Build finished. The HTML pages are in _build/html_stable."
 
+html_stable-memory:
+	$(MPROF) -b html $(ALLSPHINXOPTS) _build/html_stable
+	@echo
+	@echo "Build finished. The HTML pages are in _build/html_stable."
+
 html_dev:
 	BUILD_DEV_HTML=1 $(SPHINXBUILD) -b html $(ALLSPHINXOPTS) _build/html
 	@echo
 	@echo "Build finished. The HTML pages are in _build/html"
 
+html_dev-memory:
+	BUILD_DEV_HTML=1 $(MPROF) -b html $(ALLSPHINXOPTS) _build/html
+	@echo
+	@echo "Build finished. The HTML pages are in _build/html"
+
 html_dev-pattern:
-	BUILD_DEV_HTML=1 $(SPHINXBUILD) -D plot_gallery=1 -D sphinx_gallery_conf.filename_pattern=$(PATTERN) -b html $(ALLSPHINXOPTS) _build/html
+	BUILD_DEV_HTML=1 $(SPHINXBUILD) -D sphinx_gallery_conf.filename_pattern=$(PATTERN) -D sphinx_gallery_conf.run_stale_examples=True -b html $(ALLSPHINXOPTS) _build/html
+	@echo
+	@echo "Build finished. The HTML pages are in _build/html"
+
+html_dev-pattern-memory:
+	BUILD_DEV_HTML=1 $(MPROF) -D sphinx_gallery_conf.filename_pattern=$(PATTERN) -D sphinx_gallery_conf.run_stale_examples=True -b html $(ALLSPHINXOPTS) _build/html
 	@echo
 	@echo "Build finished. The HTML pages are in _build/html"
 

doc/changes/latest.inc

@@ -23,6 +23,8 @@ Enhancements
 
 - Add ``proj`` argument to :func:`mne.make_fixed_length_epochs` by `Eric Larson`_ (:gh:`8351`)
 
+- Reduce memory usage of volume source spaces by `Eric Larson`_ (:gh:`8379`)
+
 - Speed up :class:`mne.decoding.TimeDelayingRidge` with edge correction using Numba by `Eric Larson`_ (:gh:`8323`)
 
 Bugs
@@ -65,7 +67,7 @@ Bugs
 
 - Pass ``rank`` everyhwere in forward preparation for source imaging. This bug affected sparse solvers when using maxfilter data, by `Alex Gramfort`_ (:gh:`8368`)
 
-- Fix bug in :func:`mne.viz.plot_alignment` where ECoG and sEEG channels were not plotted and fNIRS channels were always plotted in the head coordinate frame by `Eric Larson`_
+- Fix bug in :func:`mne.viz.plot_alignment` where ECoG and sEEG channels were not plotted and fNIRS channels were always plotted in the head coordinate frame by `Eric Larson`_ (:gh:`8393`)
 
 API changes
 ~~~~~~~~~~~

doc/conf.py

@@ -14,18 +14,20 @@
 
 from datetime import date
 from distutils.version import LooseVersion
-import gc
 import os
 import os.path as op
 import sys
+import time
 import warnings
 
 import sphinx_gallery
 from sphinx_gallery.sorting import FileNameSortKey, ExplicitOrder
 from numpydoc import docscrape
 import matplotlib
 import mne
-from mne.utils import linkcode_resolve  # noqa, analysis:ignore
+from mne.viz import Brain
+from mne.utils import (linkcode_resolve, # noqa, analysis:ignore
+                       _assert_no_instances, sizeof_fmt)
 
 if LooseVersion(sphinx_gallery.__version__) < LooseVersion('0.2'):
     raise ImportError('Must have at least version 0.2 of sphinx-gallery, got '
@@ -370,8 +372,8 @@
 ##############################################################################
 # sphinx-gallery
 
-examples_dirs = ['../examples', '../tutorials']
-gallery_dirs = ['auto_examples', 'auto_tutorials']
+examples_dirs = ['../tutorials', '../examples']
+gallery_dirs = ['auto_tutorials', 'auto_examples']
 os.environ['_MNE_BUILDING_DOC'] = 'true'
 
 scrapers = ('matplotlib',)
@@ -441,17 +443,33 @@ def setup(app):
 class Resetter(object):
     """Simple class to make the str(obj) static for Sphinx build env hash."""
 
+    def __init__(self):
+        self.t0 = time.time()
+
     def __repr__(self):
         return '<%s>' % (self.__class__.__name__,)
 
     def __call__(self, gallery_conf, fname):
         import matplotlib.pyplot as plt
+        try:
+            from pyvista import Plotter
+        except ImportError:
+            Plotter = None
         reset_warnings(gallery_conf, fname)
         # in case users have interactive mode turned on in matplotlibrc,
         # turn it off here (otherwise the build can be very slow)
         plt.ioff()
-        gc.collect()
         plt.rcParams['animation.embed_limit'] = 30.
+        _assert_no_instances(Brain, 'running')  # calls gc.collect()
+        if Plotter is not None:
+            _assert_no_instances(Plotter, 'running')
+        # This will overwrite some Sphinx printing but it's useful
+        # for memory timestamps
+        if os.getenv('SG_STAMP_STARTS', '').lower() == 'true':
+            import psutil
+            process = psutil.Process(os.getpid())
+            mem = sizeof_fmt(process.memory_info().rss)
+            print(f'{time.time() - self.t0:6.1f} s : {mem}'.ljust(22))
 
 
 def reset_warnings(gallery_conf, fname):

examples/connectivity/plot_mixed_source_space_connectivity.py

@@ -179,7 +179,7 @@
 #
 # By default matplotlib does not save using the facecolor, even though this was
 # set when the figure was generated. If not set via savefig, the labels, title,
-# and legend will be cut off from the output png file.
-
-# fname_fig = data_path + '/MEG/sample/plot_mixed_connect.png'
-# plt.savefig(fname_fig, facecolor='black')
+# and legend will be cut off from the output png file::
+#
+#     >>> fname_fig = data_path + '/MEG/sample/plot_mixed_connect.png'
+#     >>> plt.savefig(fname_fig, facecolor='black')

examples/decoding/plot_receptive_field_mtrf.py

@@ -150,7 +150,6 @@
 ax.set(title="Topomap of model coefficients\nfor delay %s" % time_plot)
 mne.viz.tight_layout()
 
-
 ###############################################################################
 # Create and fit a stimulus reconstruction model
 # ----------------------------------------------
@@ -259,8 +258,6 @@
           % (time_plot[0], time_plot[1]))
 mne.viz.tight_layout()
 
-plt.show()
-
 ###############################################################################
 # References
 # ----------

examples/inverse/plot_evoked_ers_source_power.py

@@ -33,7 +33,8 @@
 raw_fname = op.join(data_path, 'sub-{}'.format(subject), 'meg',
                     'sub-{}_task-{}_meg.fif'.format(subject, task))
 
-raw = mne.io.read_raw_fif(raw_fname)
+# crop to 5 minutes to save memory
+raw = mne.io.read_raw_fif(raw_fname).crop(0, 300)
 
 # We are interested in the beta band (12-30 Hz)
 raw.load_data().filter(12, 30)
@@ -45,7 +46,7 @@
 # Read epochs
 events = mne.find_events(raw)
 epochs = mne.Epochs(raw, events, event_id=1, tmin=-1.5, tmax=2, picks=picks,
-                    preload=True)
+                    preload=True, decim=3)
 
 # Read forward operator and point to freesurfer subject directory
 fname_fwd = op.join(data_path, 'derivatives', 'sub-{}'.format(subject),
@@ -86,7 +87,8 @@ def _gen_dics(active_win, baseline_win, epochs):
                               tmax=baseline_win[1], decim=20)
     csd_ers = csd_morlet(epochs, freqs, tmin=active_win[0], tmax=active_win[1],
                          decim=20)
-    filters = make_dics(epochs.info, fwd, csd.mean(), pick_ori='max-power')
+    filters = make_dics(epochs.info, fwd, csd.mean(), pick_ori='max-power',
+                        reduce_rank=True)
     stc_base, freqs = apply_dics_csd(csd_baseline.mean(), filters)
     stc_act, freqs = apply_dics_csd(csd_ers.mean(), filters)
     stc_act /= stc_base
@@ -122,11 +124,22 @@ def _gen_mne(active_cov, baseline_cov, common_cov, fwd, info, method='dSPM'):
 ###############################################################################
 # Plot source estimates
 # ---------------------
+# DICS:
 
-brains = list()
-for method, stc in zip(['DICS', 'LCMV', 'dSPM'],
-                       [stc_dics, stc_lcmv, stc_dspm]):
-    title = '%s source power in the 12-30 Hz frequency band' % method
-    brains.append(stc.plot(
-        hemi='rh', subjects_dir=subjects_dir, subject=subject,
-        time_label=title))
+brain_dics = stc_dics.plot(
+    hemi='rh', subjects_dir=subjects_dir, subject=subject,
+    time_label='DICS source power in the 12-30 Hz frequency band')
+
+###############################################################################
+# LCMV:
+
+brain_lcmv = stc_lcmv.plot(
+    hemi='rh', subjects_dir=subjects_dir, subject=subject,
+    time_label='LCMV source power in the 12-30 Hz frequency band')
+
+###############################################################################
+# dSPM:
+
+brain_dspm = stc_dspm.plot(
+    hemi='rh', subjects_dir=subjects_dir, subject=subject,
+    time_label='dSPM source power in the 12-30 Hz frequency band')