Merge branch 'nwb_schema_2.8.0' into deprecate_image_mask_series

CHANGELOG.md

@@ -4,13 +4,32 @@
 
 ### Enhancements and minor changes
 - Added support for NWB schema 2.8.0:
-  - Fixed support for `data__bounds` field to `SpatialSeries` to set optional boundary range (min, max) for each dimension of data. Removed `SpatialSeries.bounds` field that was not functional. @rly [#1907](https://github.com/NeurodataWithoutBorders/pynwb/pull/1907)
+  - Removed `SpatialSeries.bounds` field that was not functional. This will be fixed in a future release. @rly [#1907](https://github.com/NeurodataWithoutBorders/pynwb/pull/1907), [#1996](https://github.com/NeurodataWithoutBorders/pynwb/pull/1996)
+  - Added support for `NWBFile.was_generated_by` field. @stephprince [#1924](https://github.com/NeurodataWithoutBorders/pynwb/pull/1924)
+  - Added support for `model_number`, `model_name`, and `serial_number` fields to `Device`. @stephprince [#1997](https://github.com/NeurodataWithoutBorders/pynwb/pull/1997)
+  - Deprecated `EventWaveform` neurodata type. @rly [#1940](https://github.com/NeurodataWithoutBorders/pynwb/pull/1940)
 
 ## PyNWB 2.8.3 (Upcoming)
 
+### Enhancements and minor changes
+* Added `NWBHDF5IO.read_nwb` convenience method to simplify reading an NWB file. @h-mayorquin [#1979](https://github.com/NeurodataWithoutBorders/pynwb/pull/1979)
+* Removed unused references to region references and builders in preparation for changes in HDMF 4.0. @rly [#1991](https://github.com/NeurodataWithoutBorders/pynwb/pull/1991)
+
+### Documentation and tutorial enhancements
+- Added documentation example for `SpikeEventSeries`. @stephprince [#1983](https://github.com/NeurodataWithoutBorders/pynwb/pull/1983)
+- Added documentation example for `AnnotationSeries`. @stephprince [#1989](https://github.com/NeurodataWithoutBorders/pynwb/pull/1989)
+
 ### Performance
 - Cache global type map to speed import 3X. @sneakers-the-rat [#1931](https://github.com/NeurodataWithoutBorders/pynwb/pull/1931)
 
+### Bug fixes
+- Fixed bug in how `ElectrodeGroup.__init__` validates its `position` argument. @oruebel [#1770](https://github.com/NeurodataWithoutBorders/pynwb/pull/1770)
+- Changed `SpatialSeries.reference_frame` from required to optional as specified in the schema. @rly [#1986](https://github.com/NeurodataWithoutBorders/pynwb/pull/1986)
+
+### Enhancements and minor changes
+- Made gain an optional argument for PatchClampSeries to match the schema. @stephprince [#1975](https://github.com/NeurodataWithoutBorders/pynwb/pull/1975)
+- Added warning when writing files with `NWBHDF5IO` without the `.nwb` extension. @stephprince [#1978](https://github.com/NeurodataWithoutBorders/pynwb/pull/1978)
+
 ## PyNWB 2.8.2 (September 9, 2024)
 
 ### Enhancements and minor changes

docs/gallery/advanced_io/plot_editing.py

@@ -24,6 +24,9 @@
 
 First, let's create an NWB file with data:
 """
+
+# sphinx_gallery_thumbnail_path = "figures/gallery_thumbnails_editing.png"
+
 from pynwb import NWBHDF5IO, NWBFile, TimeSeries
 from datetime import datetime
 from dateutil.tz import tzlocal

docs/gallery/domain/ecephys.py

@@ -31,7 +31,7 @@
 from dateutil.tz import tzlocal
 
 from pynwb import NWBHDF5IO, NWBFile
-from pynwb.ecephys import LFP, ElectricalSeries
+from pynwb.ecephys import LFP, ElectricalSeries, SpikeEventSeries
 
 #######################
 # Creating and Writing NWB files
@@ -77,10 +77,16 @@
 #
 # The electrodes table references a required :py:class:`~pynwb.ecephys.ElectrodeGroup`, which is used to represent a
 # group of electrodes. Before creating an :py:class:`~pynwb.ecephys.ElectrodeGroup`, you must define a
-# :py:class:`~pynwb.device.Device` object using the method :py:meth:`.NWBFile.create_device`.
-
+# :py:class:`~pynwb.device.Device` object using the method :py:meth:`.NWBFile.create_device`. The fields
+# ``description``, ``manufacturer``, ``model_number``, ``model_name``, and ``serial_number`` are optional, but
+# recommended.
 device = nwbfile.create_device(
-    name="array", description="the best array", manufacturer="Probe Company 9000"
+    name="array",
+    description="A 12-channel array with 4 shanks and 3 channels per shank",
+    manufacturer="Array Technologies",
+    model_number="PRB_1_4_0480_123",
+    model_name="Neurovoxels 0.99",
+    serial_number="1234567890",
 )
 
 #######################
@@ -244,8 +250,8 @@
 ####################
 # .. _units_electrode:
 #
-# Spike Times
-# ^^^^^^^^^^^
+# Sorted spike times
+# ^^^^^^^^^^^^^^^^^^
 #
 # Spike times are stored in the :py:class:`~pynwb.misc.Units` table, which is a subclass of
 # :py:class:`~hdmf.common.table.DynamicTable`. Adding columns to the :py:class:`~pynwb.misc.Units` table is analogous
@@ -272,6 +278,29 @@
 
 nwbfile.units.to_dataframe()
 
+####################
+# Unsorted spike times
+# ^^^^^^^^^^^^^^^^^^^^
+#
+# While the :py:class:`~pynwb.misc.Units` table is used to store spike times and waveform data for
+# spike-sorted, single-unit activity, you may also want to store spike times and waveform snippets of
+# unsorted spiking activity (e.g., multi-unit activity detected via threshold crossings during data acquisition).
+# This information can be stored using :py:class:`~pynwb.ecephys.SpikeEventSeries` objects. 
+
+spike_snippets = np.random.rand(20, 3, 40)  # 20 events, 3 channels, 40 samples per event
+shank0 = nwbfile.create_electrode_table_region(
+    region=[0, 1, 2],
+    description="shank0",
+)
+
+
+spike_events = SpikeEventSeries(name='SpikeEvents_Shank0',
+                                description="events detected with 100uV threshold",
+                                data=spike_snippets,
+                                timestamps=np.arange(20),
+                                electrodes=shank0)
+nwbfile.add_acquisition(spike_events)
+
 #######################
 # Designating electrophysiology data
 # ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
@@ -283,17 +312,17 @@
 # :py:mod:`API documentation <pynwb.ecephys>` and :ref:`basics` for more details on
 # using these objects.
 #
-# For storing spike data, there are two options. Which one you choose depends on what data you have available.
-# If you need to store the complete, continuous raw voltage traces, you should store the traces with
+# For storing unsorted spiking data, there are two options. Which one you choose depends on what data you 
+# have available. If you need to store the complete, continuous raw voltage traces, you should store the traces with
 # :py:class:`~pynwb.ecephys.ElectricalSeries` objects as :ref:`acquisition <basic_timeseries>` data, and use
 # the :py:class:`~pynwb.ecephys.EventDetection` class for identifying the spike events in your raw traces.
 # If you do not want to store the raw voltage traces and only the waveform 'snippets' surrounding spike events,
-# you should use the :py:class:`~pynwb.ecephys.EventWaveform` class, which can store one or more
-# :py:class:`~pynwb.ecephys.SpikeEventSeries` objects.
+# you should store the snippets with :py:class:`~pynwb.ecephys.SpikeEventSeries` objects.
 #
 # The results of spike sorting (or clustering) should be stored in the top-level :py:class:`~pynwb.misc.Units` table.
-# Note that it is not required to store spike waveforms in order to store spike events or mean waveforms--if you only
-# want to store the spike times of clustered units you can use only the Units table.
+# The :py:class:`~pynwb.misc.Units` table can contain simply the spike times of sorted units, or you can also include 
+# individual and mean waveform information in some of the optional, predefined :py:class:`~pynwb.misc.Units` table 
+# columns: ``waveform_mean``, ``waveform_sd``, or ``waveforms``.
 #
 # For local field potential data, there are two options. Again, which one you choose depends on what data you
 # have available. With both options, you should store your traces with :py:class:`~pynwb.ecephys.ElectricalSeries`

docs/gallery/domain/ophys.py

@@ -86,12 +86,17 @@
 #     :align: center
 #
 # Create a :py:class:`~pynwb.device.Device` named ``"Microscope"`` in the :py:class:`~pynwb.file.NWBFile` object. Then
-# create an  :py:class:`~pynwb.ophys.OpticalChannel` named ``"OpticalChannel"``.
+# create an  :py:class:`~pynwb.ophys.OpticalChannel` named ``"OpticalChannel"``. The fields
+# ``description``, ``manufacturer``, ``model_number``, ``model_name``, and ``serial_number`` are optional, but
+# recommended.
 
 device = nwbfile.create_device(
     name="Microscope",
     description="My two-photon microscope",
-    manufacturer="The best microscope manufacturer",
+    manufacturer="Loki Labs",
+    model_number="ABC-123",
+    model_name="Loki 1.0",
+    serial_number="1234567890",
 )
 optical_channel = OpticalChannel(
     name="OpticalChannel",

docs/gallery/domain/plot_behavior.py

@@ -105,17 +105,13 @@
 #
 # For position data ``reference_frame`` indicates the zero-position, e.g.
 # the 0,0 point might be the bottom-left corner of an enclosure, as viewed from the tracking camera.
-# In :py:class:`~pynwb.behavior.SpatialSeries`, the ``data__bounds`` field allows the user to set
-# the boundary range, i.e.,  (min, max), for each dimension of ``data``. The units are the same as in ``data``.
-# This field does not enforce a boundary on the dataset itself.
 
 timestamps = np.linspace(0, 50) / 200
 
 position_spatial_series = SpatialSeries(
     name="SpatialSeries",
     description="Position (x, y) in an open field.",
     data=position_data,
-    data__bounds=[(0,50), (0,50)],
     timestamps=timestamps,
     reference_frame="(0,0) is bottom left corner",
 )

docs/gallery/general/plot_configurator.py

@@ -8,18 +8,18 @@
 Introduction
 -------------
 Users will create a configuration YAML file that outlines the fields (within a neurodata type)
-they want to be validated against a set of allowed terms. 
+they want to be validated against a set of allowed terms.
 After creating the configuration file, users will need to load the
 configuration file with the :py:func:`~pynwb.load_type_config` method.
 With the configuration loaded, every instance of the neurodata
 types defined in the configuration file will have the respective fields wrapped with a
 :py:class:`~hdmf.term_set.TermSetWrapper`.
 This automatic wrapping is what provides the term validation for the field value.
 For greater control on which datasets and attributes are validated
-against which sets of allowed terms, use the 
+against which sets of allowed terms, use the
 :py:class:`~hdmf.term_set.TermSetWrapper` on individual datasets and attributes instead.
-You can follow the 
-`TermSet tutorial in the HDMF documentation 
+You can follow the
+`TermSet tutorial in the HDMF documentation
 <https://hdmf.readthedocs.io/en/stable/tutorials/plot_term_set.html#sphx-glr-tutorials-plot-term-set-py>`_
 for more information.
 
@@ -42,6 +42,8 @@
 3. Each data type will have a list of fields associated with a :py:class:`~hdmf.term_set.TermSet`.
    The user can use the same or unique TermSet instances for each field.
 """
+# sphinx_gallery_thumbnail_path = 'figures/gallery_thumbnails_configurator.png'
+
 try:
     import linkml_runtime  # noqa: F401
 except ImportError as e:

docs/gallery/general/plot_file.py

@@ -91,7 +91,6 @@
       :py:class:`~pynwb.behavior.EyeTracking`.
 
     * **Extracellular electrophysiology:** :py:class:`~pynwb.ecephys.EventDetection`,
-      :py:class:`~pynwb.ecephys.EventWaveform`,
       :py:class:`~pynwb.ecephys.FeatureExtraction`,
       :py:class:`~pynwb.ecephys.FilteredEphys`,
       :py:class:`~pynwb.ecephys.LFP`.
@@ -130,6 +129,7 @@
 from pynwb import NWBHDF5IO, NWBFile, TimeSeries
 from pynwb.behavior import Position, SpatialSeries
 from pynwb.file import Subject
+from pynwb.misc import AnnotationSeries
 
 ####################
 # .. _basics_nwbfile:
@@ -284,6 +284,27 @@
 # or using the method :py:meth:`.NWBFile.get_acquisition`:
 nwbfile.get_acquisition("test_timeseries")
 
+####################
+# Other Types of Time Series
+# ^^^^^^^^^^^^^^^^^^^^^^^^^^
+#
+# As mentioned previously, there are many subtypes of :py:class:`~pynwb.base.TimeSeries` that are used to store
+# different kinds of data. One example is :py:class:`~pynwb.misc.AnnotationSeries`, a subclass of 
+# :py:class:`~pynwb.base.TimeSeries` that stores text-based records about the experiment. Similarly to our
+# :py:class:`~pynwb.base.TimeSeries` example above, we can create an :py:class:`~pynwb.misc.AnnotationSeries` 
+# object with text information about a stimulus and add it to the stimulus group in 
+# the :py:class:`~pynwb.file.NWBFile`. 
+
+annotations = AnnotationSeries(name='airpuffs',
+                               data=['Left Airpuff', 'Right Airpuff', 'Right Airpuff'],
+                               description='Airpuff events delivered to the animal',
+                               timestamps=[1.0, 3.0, 8.0])
+
+nwbfile.add_stimulus(annotations)
+
+####################
+# This approach of creating a :py:class:`~pynwb.base.TimeSeries` object and adding it to the appropriate 
+# :py:class:`~pynwb.file.NWBFile` group can be used for all subtypes of :py:class:`~pynwb.base.TimeSeries` data.
 
 ####################
 # .. _basic_spatialseries:
@@ -570,8 +591,7 @@
 
 ####################
 # .. [#] Some data interface objects have a default name. This default name is the type of the data interface. For
-#    example, the default name for :py:class:`~pynwb.ophys.ImageSegmentation` is "ImageSegmentation" and the default
-#    name for :py:class:`~pynwb.ecephys.EventWaveform` is "EventWaveform".
+#    example, the default name for :py:class:`~pynwb.ophys.ImageSegmentation` is "ImageSegmentation".
 #
 # .. [#] HDF5 is the primary backend supported by NWB.
 #

docs/source/conf.py

@@ -244,7 +244,6 @@ def __call__(self, filename):
 # html_theme = 'default'
 # html_theme = "sphinxdoc"
 html_theme = "sphinx_rtd_theme"
-html_theme_path = [sphinx_rtd_theme.get_html_theme_path()]
 
 # Theme options are theme-specific and customize the look and feel of a theme
 # further.  For a list of options available for each theme, see the
@@ -260,9 +259,6 @@ def __call__(self, filename):
     'css/custom.css',
 ]
 
-# Add any paths that contain custom themes here, relative to this directory.
-# html_theme_path = []
-
 # The name for this set of Sphinx documents.  If None, it defaults to
 # "<project> v<release> documentation".
 # html_title = None

docs/source/export.rst

@@ -53,12 +53,12 @@ on the :py:class:`~pynwb.file.NWBFile` before exporting.
 
 How do I create a copy of an NWB file with different data layouts (e.g., applying compression)?
 ---------------------------------------------------------------------------------------------------------
-Use the `h5repack <https://docs.hdfgroup.org/hdf5/v1_14/_view_tools_edit.html>`_ command line tool from the HDF5 Group.
+Use the `h5repack <https://support.hdfgroup.org/documentation/hdf5/latest/_view_tools_edit.html#secViewToolsEditChange>`_ command line tool from the HDF5 Group.
 
 
 How do I create a copy of an NWB file with different controls over how links are treated and whether copies are deep or shallow?
 ---------------------------------------------------------------------------------------------------------------------------------
-Use the `h5copy <https://docs.hdfgroup.org/hdf5/v1_14/_view_tools_edit.html>`_ command line tool from the HDF5 Group.
+Use the `h5copy <https://support.hdfgroup.org/documentation/hdf5/latest/_view_tools_edit.html#secViewToolsEditCopy>`_ command line tool from the HDF5 Group.
 
 
 How do I generate new object IDs for a newly exported NWB file?
@@ -99,7 +99,7 @@ For example:
            export_io.export(src_io=read_io, nwbfile=nwbfile, write_args={'link_data': False})  # copy linked datasets
            # the written file will contain no links to external datasets
 
-You can also the `h5copy <https://docs.hdfgroup.org/hdf5/v1_14/_view_tools_edit.html>`_ command line tool \
+You can also the `h5copy <https://support.hdfgroup.org/documentation/hdf5/latest/_view_tools_edit.html#secViewToolsEditCopy>`_ command line tool \
 from the HDF5 Group.
 
 

docs/source/overview_software_architecture.rst

@@ -69,7 +69,7 @@ Builder
    * :py:class:`~hdmf.build.builders.GroupBuilder` - represents a collection of objects
    * :py:class:`~hdmf.build.builders.DatasetBuilder` - represents data
    * :py:class:`~hdmf.build.builders.LinkBuilder` - represents soft-links
-   * :py:class:`~hdmf.build.builders.RegionBuilder` - represents a slice into data (Subclass of :py:class:`~hdmf.build.builders.DatasetBuilder`)
+   * :py:class:`~hdmf.build.builders.ReferenceBuilder` - represents a reference to another group or dataset
 
 * **Main Module:** :py:mod:`hdmf.build.builders`
 

pyproject.toml

@@ -34,7 +34,7 @@ classifiers = [
 ]
 dependencies = [
     "h5py>=2.10",
-    "hdmf>=3.14.3",
+    "hdmf>=3.14.5",
     "numpy>=1.18",
     "pandas>=1.1.5",
     "python-dateutil>=2.7.3",

requirements-min.txt

@@ -1,6 +1,6 @@
 # minimum versions of package dependencies for installing PyNWB
 h5py==2.10  # support for selection of datasets with list of indices added in 2.10
-hdmf==3.14.3
+hdmf==3.14.5
 numpy==1.18
 pandas==1.1.5
 python-dateutil==2.7.3

requirements-opt.txt

@@ -1,3 +1,8 @@
 linkml-runtime==1.7.4; python_version >= "3.9"
 schemasheets==0.2.1; python_version >= "3.9"
 oaklib==0.5.32; python_version >= "3.9"
+
+# for streaming tests
+fsspec==2024.10.0
+requests==2.32.3
+aiohttp==3.10.11

requirements.txt

@@ -1,6 +1,6 @@
 # pinned dependencies to reproduce an entire development environment to use PyNWB
 h5py==3.11.0
-hdmf==3.14.3
+hdmf==3.14.5
 numpy==2.1.1; python_version > "3.9"  # numpy 2.1+ is not compatible with py3.9
 numpy==2.0.2; python_version == "3.9"
 pandas==2.2.2

src/pynwb/__init__.py

@@ -397,6 +397,10 @@ def __init__(self, **kwargs):
         if mode in io_modes_that_create_file or manager is not None or extensions is not None:
             load_namespaces = False
 
+        if mode in io_modes_that_create_file and not str(path).endswith('.nwb'):
+            warn(f"The file path provided: {path} does not end in '.nwb'. "
+                 "It is recommended that NWB files using the HDF5 backend use the '.nwb' extension.", UserWarning)
+
         if load_namespaces:
             tm = get_type_map()
             super().load_namespaces(tm, path, file=file_obj, driver=driver, aws_region=aws_region)
@@ -502,6 +506,34 @@ def export(self, **kwargs):
         kwargs['container'] = nwbfile
         super().export(**kwargs)
 
+    @staticmethod
+    @docval({'name': 'path', 'type': (str, Path), 'doc': 'the path to the HDF5 file', 'default': None},
+            {'name': 'file', 'type': [h5py.File, 'S3File'], 'doc': 'a pre-existing h5py.File object', 'default': None},
+            is_method=False)
+    def read_nwb(**kwargs):
+        """
+        Helper factory method for reading an NWB file and return the NWBFile object
+        """
+        # Retrieve the filepath
+        path = popargs('path', kwargs)
+        file = popargs('file', kwargs)
+
+        path = str(path) if path is not None else None
+
+        # Streaming case
+        if path is not None and (path.startswith("s3://") or path.startswith("http")):
+            import fsspec
+            fsspec_file_system = fsspec.filesystem("http")
+            ffspec_file = fsspec_file_system.open(path, "rb")
+
+            open_file = h5py.File(ffspec_file, "r")
+            io = NWBHDF5IO(file=open_file)
+            nwbfile = io.read()
+        else:
+            io = NWBHDF5IO(path=path, file=file, mode="r", load_namespaces=True)
+            nwbfile = io.read()
+
+        return nwbfile
 
 from . import io as __io  # noqa: F401,E402
 from .core import NWBContainer, NWBData  # noqa: F401,E402