Merge pull request #1800 from AllenInstitute/rc/2.5.0

rc/2.5.0

Author: djkapner

CHANGELOG.md

@@ -1,6 +1,10 @@
 # Change Log
 All notable changes to this project will be documented in this file.
 
+## [2.5.0] = 2021-01-29
+-  Adds unfiltered running speed as new data stream
+-  run_demixing gracefully ignores any ROIs that are not in the input trace file
+
 ## [2.4.1] = 2021-01-04
 -  update deprecated call to scipy.spatial.transform.Rotation.as_dcm() to .as_matrix()
 

allensdk/__init__.py

@@ -37,7 +37,7 @@
 
 
 
-__version__ = '2.4.1'
+__version__ = '2.5.0'
 
 
 try:

allensdk/brain_observatory/behavior/behavior_ophys_session.py

@@ -516,11 +516,11 @@ def _get_roi_masks_by_cell_roi_id(self, cell_roi_ids=None):
         table.set_index("cell_roi_id", inplace=True)
         table = table.loc[cell_roi_ids, :]
 
-        full_image_shape = table.iloc[0]["image_mask"].shape
+        full_image_shape = table.iloc[0]["roi_mask"].shape
 
         output = np.zeros((len(cell_roi_ids), full_image_shape[0], full_image_shape[1]), dtype=np.uint8)
         for ii, (_, row) in enumerate(table.iterrows()):
-            output[ii, :, :] = _translate_roi_mask(row["image_mask"], int(row["y"]), int(row["x"]))
+            output[ii, :, :] = _translate_roi_mask(row["roi_mask"], int(row["y"]), int(row["x"]))
 
         # Pixel spacing and units of mask image will match either the
         # max or avg projection image of 2P movie.

allensdk/brain_observatory/behavior/session_apis/abcs/behavior_base.py

@@ -44,9 +44,14 @@ def get_rewards(self) -> pd.DataFrame:
         raise NotImplementedError()
 
     @abc.abstractmethod
-    def get_running_data_df(self) -> pd.DataFrame:
+    def get_running_data_df(self, lowpass=True) -> pd.DataFrame:
         """Get running speed data.
 
+        Parameters
+        ----------
+        lowpass: bool
+            Whether to return running speed with low pass filter applied or without
+
         Returns
         -------
         pd.DataFrame

allensdk/brain_observatory/behavior/session_apis/data_io/behavior_ophys_nwb_api.py

@@ -61,9 +61,10 @@ def save(self, session_object):
         # Add running data to NWB in-memory object:
         unit_dict = {'v_sig': 'V', 'v_in': 'V',
                      'speed': 'cm/s', 'timestamps': 's', 'dx': 'cm'}
-        nwb.add_running_data_df_to_nwbfile(nwbfile,
-                                           session_object.running_data_df,
-                                           unit_dict)
+        nwb.add_running_data_dfs_to_nwbfile(nwbfile,
+                                            session_object.running_data_df,
+                                            session_object.raw_running_data_df,
+                                            unit_dict)
 
         # Add stimulus template data to NWB in-memory object:
         for name, image_data in session_object.stimulus_templates.items():
@@ -148,9 +149,22 @@ def get_ophys_session_id(self) -> int:
     def get_eye_tracking(self) -> int:
         raise NotImplementedError()
 
-    def get_running_data_df(self, **kwargs) -> pd.DataFrame:
+    def get_running_data_df(self, lowpass=True) -> pd.DataFrame:
+        """
+        Gets the running data df
+        Parameters
+        ----------
+        lowpass: bool
+            Whether to return running speed with or without low pass filter applied
 
-        running_speed = self.get_running_speed()
+        Returns
+        -------
+            pd.DataFrame:
+                Dataframe containing various signals used to compute running
+                speed, and the filtered or unfiltered speed.
+        """
+
+        running_speed = self.get_running_speed(lowpass=lowpass)
 
         running_data_df = pd.DataFrame({'speed': running_speed.values},
                                        index=pd.Index(running_speed.timestamps,
@@ -263,6 +277,10 @@ def get_task_parameters(self) -> dict:
     def get_cell_specimen_table(self) -> pd.DataFrame:
         # NOTE: ROI masks are stored in full frame width and height arrays
         df = self.nwbfile.processing['ophys'].data_interfaces['image_segmentation'].plane_segmentations['cell_specimen_table'].to_dataframe()
+
+        # Because pynwb stores this field as "image_mask", it is renamed here
+        df = df.rename(columns={'image_mask': 'roi_mask'})
+
         df.index.rename('cell_roi_id', inplace=True)
         df['cell_specimen_id'] = [None if csid == -1 else csid for csid in df['cell_specimen_id'].values]
 

allensdk/brain_observatory/behavior/session_apis/data_io/ophys_lims_api.py

@@ -458,7 +458,7 @@ def get_raw_cell_specimen_table_dict(self) -> dict:
                 """.format(ophys_cell_seg_run_id)
         initial_cs_table = pd.read_sql(query, self.lims_db.get_connection())
         cell_specimen_table = initial_cs_table.rename(
-            columns={'id': 'cell_roi_id', 'mask_matrix': 'image_mask'})
+            columns={'id': 'cell_roi_id', 'mask_matrix': 'roi_mask'})
         cell_specimen_table.drop(['ophys_experiment_id',
                                   'ophys_cell_segmentation_run_id'],
                                  inplace=True, axis=1)

allensdk/brain_observatory/behavior/session_apis/data_transforms/behavior_ophys_data_xforms.py

@@ -46,7 +46,7 @@ def get_cell_specimen_table(self):
         fov_height = self.get_field_of_view_shape()['height']
 
         # Convert cropped ROI masks to uncropped versions
-        image_mask_list = []
+        roi_mask_list = []
         for cell_roi_id, table_row in cell_specimen_table.iterrows():
             # Deserialize roi data into AllenSDK RoiMask object
             curr_roi = roi.RoiMask(image_w=fov_width, image_h=fov_height,
@@ -55,10 +55,10 @@ def get_cell_specimen_table(self):
             curr_roi.y = table_row['y']
             curr_roi.width = table_row['width']
             curr_roi.height = table_row['height']
-            curr_roi.mask = np.array(table_row['image_mask'])
-            image_mask_list.append(curr_roi.get_mask_plane().astype(np.bool))
+            curr_roi.mask = np.array(table_row['roi_mask'])
+            roi_mask_list.append(curr_roi.get_mask_plane().astype(np.bool))
 
-        cell_specimen_table['image_mask'] = image_mask_list
+        cell_specimen_table['roi_mask'] = roi_mask_list
         cell_specimen_table = cell_specimen_table[sorted(cell_specimen_table.columns)]
 
         cell_specimen_table.index.rename('cell_roi_id', inplace=True)
@@ -340,7 +340,7 @@ def get_corrected_fluorescence_traces(self):
         ophys_timestamps = self.get_ophys_timestamps()
 
         num_trace_timepoints = corrected_fluorescence_traces.shape[1]
-        assert num_trace_timepoints, ophys_timestamps.shape[0]
+        assert num_trace_timepoints == ophys_timestamps.shape[0]
         df = pd.DataFrame(
             {'corrected_fluorescence': list(corrected_fluorescence_traces)},
             index=pd.Index(cell_roi_id_list, name='cell_roi_id'))

allensdk/brain_observatory/behavior/write_nwb/_schemas.py

@@ -18,7 +18,7 @@ class CellSpecimenTable(RaisingSchema):
     height = Dict(String, Int, required=True)
     width = Dict(String, Int, required=True)
     mask_image_plane = Dict(String, Int, required=True)
-    image_mask = Dict(String, List(List(Boolean)), required=True)
+    roi_mask = Dict(String, List(List(Boolean)), required=True)
 
 
 

allensdk/brain_observatory/nwb/__init__.py

@@ -333,37 +333,48 @@ def add_running_speed_to_nwbfile(nwbfile, running_speed, name='speed', unit='cm/
         unit=unit
     )
 
-    running_mod = ProcessingModule('running', 'Running speed processing module')
-    nwbfile.add_processing_module(running_mod)
+    if 'running' in nwbfile.processing:
+        running_mod = nwbfile.processing['running']
+    else:
+        running_mod = ProcessingModule('running', 'Running speed processing module')
+        nwbfile.add_processing_module(running_mod)
 
     running_mod.add_data_interface(running_speed_series)
 
     return nwbfile
 
 
-def add_running_data_df_to_nwbfile(nwbfile, running_data_df, unit_dict, index_key='timestamps'):
-    ''' Adds running speed data to an NWBFile as timeseries in acquisition and processing
+def add_running_data_dfs_to_nwbfile(nwbfile, running_data_df, running_data_df_unfiltered, unit_dict):
+    """Adds both unfiltered (raw) and filtered running speed data to an NWBFile as timeseries in acquisition and processing
 
     Parameters
     ----------
     nwbfile : pynwb.NWBFile
-        File to which runnign speeds will be written
-    running_speed : pandas.DataFrame
-        Contains 'speed' and 'times', 'v_in', 'vsig', 'dx'
-    unit : str, optional
+        File to which running speeds will be written
+    running_data_df : pandas.DataFrame
+        Filtered running data
+        Contains 'speed', 'v_in', 'vsig', 'dx'
+        Note that 'v_in', 'vsig', 'dx' are expected to be the same as in running_data_df_unfiltered
+    running_data_df_unfiltered : pandas.DataFrame
+        Unfiltered (raw) Running data
+        Contains 'speed', 'v_in', 'vsig', 'dx'
+        Note that 'v_in', 'vsig', 'dx' are expected to be the same as in running_data_df
+    unit_dict : dict, optional
         SI units of running speed values
 
     Returns
     -------
     nwbfile : pynwb.NWBFile
 
-    '''
-    assert running_data_df.index.name == index_key
-
+    """
     running_speed = RunningSpeed(timestamps=running_data_df.index.values,
                                  values=running_data_df['speed'].values)
 
+    running_speed_unfiltered = RunningSpeed(timestamps=running_data_df_unfiltered.index.values,
+                                            values=running_data_df_unfiltered['speed'].values)
+
     add_running_speed_to_nwbfile(nwbfile, running_speed, name='speed', unit=unit_dict['speed'])
+    add_running_speed_to_nwbfile(nwbfile, running_speed_unfiltered, name='speed_unfiltered', unit=unit_dict['speed'])
 
     running_mod = nwbfile.processing['running']
     timestamps_ts = running_mod.get_data_interface('speed').timestamps
@@ -883,9 +894,9 @@ def add_cell_specimen_table(nwbfile: NWBFile,
         imaging_plane=imaging_plane)
 
     for col_name in cell_roi_table.columns:
-        # the columns 'image_mask', 'pixel_mask', and 'voxel_mask' are already defined
+        # the columns 'roi_mask', 'pixel_mask', and 'voxel_mask' are already defined
         # in the nwb.ophys::PlaneSegmentation Object
-        if col_name not in ['id', 'mask_matrix', 'image_mask', 'pixel_mask', 'voxel_mask']:
+        if col_name not in ['id', 'mask_matrix', 'roi_mask', 'pixel_mask', 'voxel_mask']:
             # This builds the columns with name of column and description of column
             # both equal to the column name in the cell_roi_table
             plane_segmentation.add_column(col_name,
@@ -895,13 +906,13 @@ def add_cell_specimen_table(nwbfile: NWBFile,
     # go through each roi and add it to the plan segmentation object
     for cell_roi_id, table_row in cell_roi_table.iterrows():
 
-        # NOTE: The 'image_mask' in this cell_roi_table has already been
+        # NOTE: The 'roi_mask' in this cell_roi_table has already been
         # processing by the function from 
         # allensdk.brain_observatory.behavior.session_apis.data_io.ophys_lims_api
         # get_cell_specimen_table() method. As a result, the ROI is stored in
         # an array that is the same shape as the FULL field of view of the
         # experiment (e.g. 512 x 512).
-        mask = table_row.pop('image_mask')
+        mask = table_row.pop('roi_mask')
 
         csid = table_row.pop('cell_specimen_id')
         table_row['cell_specimen_id'] = -1 if csid is None else csid

allensdk/brain_observatory/nwb/nwb_api.py

@@ -45,10 +45,23 @@ def from_path(cls, path, **kwargs):
 
         return cls(path=path, **kwargs)
 
-    def get_running_speed(self) -> RunningSpeed:
-
-        values = self.nwbfile.modules['running'].get_data_interface('speed').data[:]
-        timestamps = self.nwbfile.modules['running'].get_data_interface('speed').timestamps[:]
+    def get_running_speed(self, lowpass=True) -> RunningSpeed:
+        """
+        Gets the running speed
+        Parameters
+        ----------
+        lowpass: bool
+            Whether to return the running speed with lowpass filter applied or without
+
+        Returns
+        -------
+        RunningSpeed:
+            The running speed
+        """
+
+        interface_name = 'speed' if lowpass else 'speed_unfiltered'
+        values = self.nwbfile.modules['running'].get_data_interface(interface_name).data[:]
+        timestamps = self.nwbfile.modules['running'].get_data_interface(interface_name).timestamps[:]
 
         return RunningSpeed(
             timestamps=timestamps,