diff --git a/.github/workflows/docs_build_and_deploy.yml b/.github/workflows/docs_build_and_deploy.yml index 2dcd468..5e04d15 100644 --- a/.github/workflows/docs_build_and_deploy.yml +++ b/.github/workflows/docs_build_and_deploy.yml @@ -28,6 +28,8 @@ jobs: runs-on: ubuntu-latest steps: - uses: neuroinformatics-unit/actions/build_sphinx_docs@v2 + with: + python-version: 3.12 deploy_sphinx_docs: name: Deploy Sphinx Docs diff --git a/.gitignore b/.gitignore index aedc8d7..d1267e0 100644 --- a/.gitignore +++ b/.gitignore @@ -1,3 +1,6 @@ +docs/source/ephys_at_swc/example_pipelines/auto_examples_ephys/ +sg_execution_times.rst + # Byte-compiled / optimized / DLL files __pycache__/ *.py[cod] diff --git a/docs/requirements.txt b/docs/requirements.txt index 18b6962..8208154 100644 --- a/docs/requirements.txt +++ b/docs/requirements.txt @@ -1,8 +1,11 @@ linkify-it-py +matplotlib myst-parser nbsphinx numpydoc pydata-sphinx-theme + sphinx sphinx-copybutton sphinx-design +sphinx-gallery diff --git a/docs/source/conf.py b/docs/source/conf.py index 0609f0c..6022863 100644 --- a/docs/source/conf.py +++ b/docs/source/conf.py @@ -42,6 +42,7 @@ "myst_parser", "numpydoc", "nbsphinx", + "sphinx_gallery.gen_gallery", ] # Configure the myst parser to enable cool markdown features @@ -74,6 +75,10 @@ # to ensure that include files (partial pages) aren't built, exclude them # https://github.com/sphinx-doc/sphinx/issues/1965#issuecomment-124732907 "**/includes/**", + # exclude .py and .ipynb files in auto_examples generated by sphinx-gallery + # this is to prevent sphinx from complaining about duplicate source files + "ephys_at_swc/example_pipelines/auto_examples_ephys/*.ipynb", + "ephys_at_swc/example_pipelines/auto_examples_ephys/*.py", ] # Ignore certain URLs from being checked @@ -91,6 +96,7 @@ linkcheck_anchors_ignore_for_url = [ "https://gin.g-node.org/G-Node/Info/wiki", "https://gin.g-node.org/G-Node/info/wiki", # ignore both spellings + "https://github.com/stephenlenzi/npix_lse", # ignore private repository ] linkcheck_retries = 2 @@ -148,3 +154,9 @@ # Configure the code block copy button # don't copy line numbers, prompts, or console outputs copybutton_exclude = ".linenos, .gp, .go" + +# Configure sphinx-gallery +sphinx_gallery_conf = { + 'examples_dirs': ["ephys_at_swc/example_pipelines/examples"], # path to your example scripts + 'gallery_dirs': ["ephys_at_swc/example_pipelines/auto_examples_ephys"], # path to where to save gallery generated output +} diff --git a/docs/source/data_analysis/index.md b/docs/source/data_analysis/index.md index ce094e5..e9003f7 100644 --- a/docs/source/data_analysis/index.md +++ b/docs/source/data_analysis/index.md @@ -5,5 +5,6 @@ Guides related to the analysis of neuroscientific data, spanning a wide range of ```{toctree} :maxdepth: 1 +../ephys_at_swc/index HPC-module-SLEAP ``` diff --git a/docs/source/ephys_at_swc/community.md b/docs/source/ephys_at_swc/community.md new file mode 100644 index 0000000..b736a50 --- /dev/null +++ b/docs/source/ephys_at_swc/community.md @@ -0,0 +1,23 @@ +# Community + +This site is maintained by the +[NeuroInformatics Unit](https://neuroinformatics.dev/). +Please don't hesitate to contact us +(in particular Joe Ziminski) on Slack or +[Zulip chat](https://neuroinformatics.zulipchat.com/) +with any questions or feedback. + +For information and advice on electrophysiology from the SWC community, the +best place to go is the `#forum-extracellular-ephys` channel on the SWC Slack. + +Outside the SWC, you can address any questions or issues about +[SpikeInterface](https://github.com/SpikeInterface) +by raising an issue on their +[GitHub repository](https://github.com/SpikeInterface/spikeinterface/issues). +They are very friendly and +happy to answer any questions! + +In addition, the +[Neuropixels](https://neuropixelsgroup.slack.com/) +Slack channel is a great resource, with an active community discussing +extracellular electrophysiology acquisition and analysis. diff --git a/docs/source/ephys_at_swc/example_pipelines/examples/README.rst b/docs/source/ephys_at_swc/example_pipelines/examples/README.rst new file mode 100644 index 0000000..6181b00 --- /dev/null +++ b/docs/source/ephys_at_swc/example_pipelines/examples/README.rst @@ -0,0 +1,33 @@ +:orphan: + +Python Pipelines +================ + +Multiple probes (Cambridge Neurotech) +------------------------------------- + +Dammy Onih +(`Akrami lab `__) +is running a multi-session paradigm with two +`Cambridge Neurotech `__ +probes. In this multi-session task, mice learn a statistical learning paradigm to a +reward-associated auditory stimulus, recording from the hippocampus. +The pipeline uses SpikeInterface for preprocessing, +sorting, and analysis and can be found `here `__. + +The IBL analysis pipeline +------------------------- + +Nate Miska (`Mrsic-Flogel lab `__) +is a member of the +`International Brain Laboratory +(IBL) `_ +running the +`IBL's standardised behavioural task `_ +with acute Neuropixels 1.0 recordings. Details of the +`analysis pipeline code `__ +on the IBL data management system can be found +`here `_. + +Standalone Scripts +------------------ diff --git a/docs/source/ephys_at_swc/example_pipelines/examples/notes-on-docs-structure.txt b/docs/source/ephys_at_swc/example_pipelines/examples/notes-on-docs-structure.txt new file mode 100644 index 0000000..0ad495e --- /dev/null +++ b/docs/source/ephys_at_swc/example_pipelines/examples/notes-on-docs-structure.txt @@ -0,0 +1,5 @@ +Here, the 'Python Examples' page is rendered as a sphinx gallery. The +first section is python examples that link to outside repos. The second +section is the sphinx gallery entries. The 'README.rst' is the +'Python Examples' page, it must be called 'README.rst' for sphinx-related +reasons. diff --git a/docs/source/ephys_at_swc/example_pipelines/examples/sara_mederos.py b/docs/source/ephys_at_swc/example_pipelines/examples/sara_mederos.py new file mode 100644 index 0000000..7efbd41 --- /dev/null +++ b/docs/source/ephys_at_swc/example_pipelines/examples/sara_mederos.py @@ -0,0 +1,112 @@ +""" +NP2.0 in SpikeInterface +======================= + +Sara Mederos +`(Hofer Lab) `__ +performs chronic electrophysiological +recordings from subcortical and cortical areas using 4-shank +Neuropixels 2.0 probes (acquired using +`Open Ephys `__). +Recordings are conducted in freely moving mice during behavioral +paradigms that assess the cognitive control of innate behaviors. +A pipeline used for pre-processing, sorting, and quality metrics +can be found below. +""" + +from spikeinterface import extract_waveforms +from spikeinterface.extractors import read_openephys +from spikeinterface.preprocessing import phase_shift, bandpass_filter, common_reference +from spikeinterface.sorters import run_sorter +from spikeinterface.qualitymetrics import compute_quality_metrics +from pathlib import Path +from probeinterface.plotting import plot_probe +import matplotlib.pyplot as plt +from spikeinterface import curation +from spikeinterface.widgets import plot_timeseries +import numpy as np + + +data_path = Path( + r"/ceph/.../100323/2023-10-03_18-57-09/Record Node 101/experiment1" +) +output_path = Path( + r"/ceph/.../derivatives/100323/" +) + +show_probe = False +show_preprocessing = True + +# This reads OpenEphys 'Binary' format. It determines the +# probe using probeinterface.read_openephys, which reads `settings.xml` +# and requires the NP_PROBE field is filled. +raw_recording = read_openephys(data_path) + +if show_probe: + probe = raw_recording.get_probe() + plot_probe(probe) + plt.show() + +# Run time shift (multiplex correction) and filter +shifted_recording = phase_shift(raw_recording) +filtered_recording = bandpass_filter(shifted_recording, freq_min=300, freq_max=6000) + +# Perform median average filter by shank +channel_group = filtered_recording.get_property("group") +split_channel_ids = [ + filtered_recording.get_channel_ids()[channel_group == idx] + for idx in np.unique(channel_group) +] +preprocessed_recording = common_reference( + filtered_recording, reference="global", operator="median", groups=split_channel_ids +) + +if show_preprocessing: + recs_grouped_by_shank = preprocessed_recording.split_by("group") + for rec in recs_grouped_by_shank: + plot_timeseries( + preprocessed_recording, + order_channel_by_depth=True, + time_range=(3499, 3500), + return_scaled=True, + show_channel_ids=True, + mode="map", + ) + plt.show() + +# Run the sorting +sorting = run_sorter( + "kilosort3", + preprocessed_recording, + singularity_image=True, + output_folder=(output_path / "sorting").as_posix(), + car=False, + freq_min=150, +) + +# Curate the sorting output and extract waveforms. Calculate +# quality metrics from the waveforms. +sorting = sorting.remove_empty_units() + +sorting = curation.remove_excess_spikes(sorting, preprocessed_recording) + +# The way spikeinterface is set up means that quality metrics are +# calculated on the spikeinterface-preprocessed, NOT the kilosort +# preprocessed (i.e. drift-correct data). +# see https://github.com/SpikeInterface/spikeinterface/pull/1954 for details. +waveforms = extract_waveforms( + preprocessed_recording, + sorting, + folder=(output_path / "postprocessing").as_posix(), + ms_before=2, + ms_after=2, + max_spikes_per_unit=500, + return_scaled=True, + sparse=True, + peak_sign="neg", + method="radius", + radius_um=75, +) + +quality_metrics = compute_quality_metrics(waveforms) +quality_metrics.to_csv(output_path / "postprocessing") diff --git a/docs/source/ephys_at_swc/example_pipelines/index.md b/docs/source/ephys_at_swc/example_pipelines/index.md new file mode 100644 index 0000000..6c11472 --- /dev/null +++ b/docs/source/ephys_at_swc/example_pipelines/index.md @@ -0,0 +1,34 @@ +# Examples + +This page shares extracellular electrophysiology pipelines used by researchers across the SWC. + +These examples aim to demonstrate the possibilities for ephys +pipeline building rather than serve as drop-in tools for +data processing. However, please feel free to +contact any listed researchers to ask questions +or seek advice! + +::::{grid} 1 2 2 2 +:gutter: 3 + +:::{grid-item-card} {fas}`chart-simple;sd-text-primary` Python Examples +:link: auto_examples_ephys/index +:link-type: doc +::: + +:::{grid-item-card} {fas}`chart-simple;sd-text-primary` MATLAB Examples +:link: matlab_examples +:link-type: doc +::: + +:::: + + +```{toctree} +:maxdepth: 2 +:hidden: + +auto_examples_ephys/index +matlab_examples + +``` diff --git a/docs/source/ephys_at_swc/example_pipelines/matlab_examples.rst b/docs/source/ephys_at_swc/example_pipelines/matlab_examples.rst new file mode 100644 index 0000000..1781f6f --- /dev/null +++ b/docs/source/ephys_at_swc/example_pipelines/matlab_examples.rst @@ -0,0 +1,31 @@ +.. _matlab_examples: + +:orphan: + +Matlab Pipelines +================= + +Multi-probe Neuropixels 1.0 +--------------------------- + +Andrei Khilkevich +(`Mrsic-Flogel lab `__) +performs acute recordings in a visual change-detection decision-making task. +Two implanted probes (NP 1.0) in a head-fixed preparation are used to +simultaneously record different regions across the whole brain. +Their pipeline is available +`here `__. + +Automated pipeline for multimodal integration +--------------------------------------------- + +Mateo Velez-Fort +(`Margie Lab `__) +investigates the integration of visual +and vestibular information in the visual cortex with the +Margrie lab's 'Translocator' setup. They use +Neuropixels 2.0 (acquired using `SpikeGLX `__ +for acute recordings from the primary visual cortex and other cortical areas as the head-fixed +mouse is physically displaced along a track. +The pipeline is available +`here `__. diff --git a/docs/source/ephys_at_swc/index.md b/docs/source/ephys_at_swc/index.md new file mode 100644 index 0000000..158e796 --- /dev/null +++ b/docs/source/ephys_at_swc/index.md @@ -0,0 +1,46 @@ +# Electrophysiology + +This page serves as an information source for researchers using +extracellular electrophysiology at the SWC. It contains +useful [resources](resources.md) for learning, as well as +[example pipelines](example_pipelines/index) +used by researchers in the building. + +We encourage all types of contributions. If you'd like to contribute a +guide or pipeline, please don't hesitate to get in touch! See the +[community](community.md) +page for more details. + +This page is maintained by the [Neuroinformatics Unit](https://neuroinformatics.dev). +We are focused on building community tools to automate +and standardise electrophysiology analysis. + + +::::{grid} 1 3 3 3 +:gutter: 3 + +:::{grid-item-card} {fas}`chart-simple;sd-text-primary` Resources +:link: resources +:link-type: doc +::: + +:::{grid-item-card} {fas}`chart-simple;sd-text-primary` Example Pipelines +:link: example_pipelines/index +:link-type: doc +::: + +:::{grid-item-card} {fas}`chart-simple;sd-text-primary` Community +:link: community +:link-type: doc +::: + +:::: + +```{toctree} +:maxdepth: 2 +:hidden: + +resources +example_pipelines/index +community +``` diff --git a/docs/source/ephys_at_swc/resources.md b/docs/source/ephys_at_swc/resources.md new file mode 100644 index 0000000..4488848 --- /dev/null +++ b/docs/source/ephys_at_swc/resources.md @@ -0,0 +1,122 @@ +# Resources + +This section includes articles and videos providing +background and technical detail on extracellular +electrophysiology data preprocessing and analysis. + +This section is by no means exhaustive, please +feel free to [get in contact](community.md) to suggest +additions to this page. + +For pipeline building, we recommend +[SpikeInterface](https://github.com/SpikeInterface/spikeinterface), +an open source community toolkit for extracellular electrophysiology, +for preprocessing and spike sorting. To begin +building your pipeline, the +[SpikeInterface](https://spikeinterface.readthedocs.io/en/stable/) +documentation is a good starting point. + +## General Introduction + +Below are a selection of papers that give a history +and overview of the extracellular electrophysiology landscape: + +[Steinmetz NA et al. (2018). Challenges and opportunities for large-scale electrophysiology with Neuropixels probes. *Current Opinion in Neurobiology*.](https://pubmed.ncbi.nlm.nih.gov/29444488/) + +[Buccino AP et al. (2022). Spike sorting: new trends and challenges of the era of high-density probes. *Progress in Biomedical Engineering*.](https://iopscience.iop.org/article/10.1088/2516-1091/ac6b96/meta) + +[Rey HG et al. (2015). Past, present and future of spike sorting techniques. *Brain Research Bulletin*.](https://www.ncbi.nlm.nih.gov/pmc/articles/PMC4674014/) + +[Carlson D et al. (2019). Continuing progress of spike sorting in the era of big data. *Current Opinion in Neurobiology*](https://pubmed.ncbi.nlm.nih.gov/30856552/) + +## Technical Introduction + +This section includes more technical resources on the different stages of +extracellular electrophysiology analysis. + +A particularly useful resource is the +[Neuropixels](https://www.ucl.ac.uk/neuropixels/courses) course, with their videos published online +(e.g. [2023](https://www.ucl.ac.uk/neuropixels/training/2023-neuropixels-course)). +While these are targeted towards Neuropixels users, they are +valuable resources for any researcher +approaching electrophysiology preprocessing and analysis. + +### Preprocessing + +The [IBL white paper](https://figshare.com/articles/online_resource/Spike_sorting_pipeline_for_the_International_Brain_Laboratory/19705522) +contains a clearly written overview of common preprocessing steps. Similarly, +[Bill Karsh's guide](https://billkarsh.github.io/SpikeGLX/help/catgt_tshift/catgt_tshift/) on +SpikeGLX preprocessing tools gives a useful overview. + +[de Cheveigné & Nelken (2019)](https://pubmed.ncbi.nlm.nih.gov/30998899/) +provide a technical treatment of digital filtering, a key step in preprocessing and analysis. + +### Spike Sorting + +[This video on Spike Sorting with Christophe Pouzat](https://www.youtube.com/watch?v=vSydfDvsewY), +provides an excellent overview of the spike-sorting problem. + +This article provides a more detailed introduction to spike sorting and associated quality metrics: + +[Hill DN et al, (2007). Spike Sorting. In *Observed Brain Dynamics by P. P. Mitra and H. Bokil*.](https://neurophysics.ucsd.edu/publications/obd_ch3_2.pdf) + +It is also recommended to check out the +papers of existing spike sorting algorithms. A list of the main +spike sorters can be found +[on the SpikeInterface website](https://spikeinterface.readthedocs.io/en/latest/modules/sorters.html#supported-spike-sorters). + + +### Quality Metrics and Manual Curation + +Assessing the quality of spike-sorting is a key to producing high-quality data. + +These two papers provide a nice introduction to quality metrics for assessing +spike sorting outputs: + +[Hill DN et al. (2011). Quality Metrics to Accompany Spike Sorting of Extracellular Signals. *Journal of Neuroscience*.](https://www.jneurosci.org/content/31/24/8699) + +[Harris KD et al. (2016). Improving data quality in neuronal population recordings. *Nature Neuroscience*.](https://www.ncbi.nlm.nih.gov/pmc/articles/PMC5244825/) + +[Phy](https://github.com/cortex-lab/phy) +is the most popular tool for performing manual curation of spike sorting results. +A great [guide by Steve Lenzi](https://phy.readthedocs.io/en/latest/sorting_user_guide/) takes you +through the key steps for manual curation. + +More recently, advances in the automating curation has been made in the +[Bombcell package](https://github.com/Julie-Fabre/bombcell). + +SpikeInterface also maintains a set of quality metrics, +[explained in detail](https://spikeinterface.readthedocs.io/en/latest/modules/qualitymetrics.html) +in their documentation. + +## SpikeInterface + +Visit the SpikeInterface +[GitHub](https://github.com/SpikeInterface/spikeinterface) +and +[Documentation](https://spikeinterface.readthedocs.io/en/stable/) +to get started. + +## Other Community Tools + +### Analysis + +SpikeInterface +is mainly focused on preprocessing, spike sorting and quality metrics. +[Pynapple](https://github.com/pynapple-org/pynapple), +[Elephant]( https://neuralensemble.org/elephant/), +and [Nemos](https://github.com/flatironinstitute/nemos) +all provide useful toolboxes for analysing data post-sorting. + +The [SpikeForest](https://spikeforest.flatironinstitute.org/) +project is an excellent resource for assessing the performance of +different spike-sorting algorithms across probe types and brain regions. + +### Pipelines + +[The Allen spike-sorting pipeline](https://github.com/AllenInstitute/ecephys_spike_sorting) + +[The IBL sorting pipeline](https://github.com/int-brain-lab/ibl-neuropixel) + +For working with NeuroPixels, [Neuropixels Utils](https://djoshea.github.io/neuropixel-utils/) package +(MATLAB) and [NeuroPyxels](https://github.com/m-beau/NeuroPyxels) (Python). diff --git a/docs/source/index.md b/docs/source/index.md index 109a66f..4e572fa 100644 --- a/docs/source/index.md +++ b/docs/source/index.md @@ -6,9 +6,9 @@ The website is maintained by members of the [Neuroinformatics Unit](https://neur ## Main sections -The guides are organised into three main sections: +The guides are organised into four main sections: -::::{grid} 1 2 2 3 +::::{grid} 1 2 2 2 :gutter: 3 :::{grid-item-card} {fas}`chart-simple;sd-text-primary` Data Analysis