Address comments on release from Marina

Add project codes to stim processing. Switch trials_id calculation to not require stimulus_block. Remove x,y,heigh,width Update change log Update cells table. Clean up and format cache warning message. Update change log with counts and comment suggestions. Remove incorrect information. Update numbers and image in visual behavior documentation. Change bucket location to test rc notebooks against new data (temporary for testing).
AllenInstitute · Nov 1, 2023 · f1ae5ab · f1ae5ab
1 parent fe87c87
commit f1ae5ab
Show file tree

Hide file tree

Showing 11 changed files with 133,332 additions and 133,189 deletions.
diff --git a/allensdk/brain_observatory/behavior/behavior_project_cache/behavior_project_cache.py b/allensdk/brain_observatory/behavior/behavior_project_cache/behavior_project_cache.py
@@ -2,6 +2,7 @@
 from pathlib import Path
 import pandas as pd
 import numpy as np
+import warnings
 
 from allensdk.api.warehouse_cache.cache import Cache
 from allensdk.brain_observatory.behavior.behavior_ophys_experiment import \
@@ -25,6 +26,10 @@
     .project_cache_base import ProjectCacheBase
 
 
+class UpdatedStimulusPresentationTableWarning(UserWarning):
+    pass
+
+
 class VBOLimsCache(Cache):
     """
     A class that ineherits from the warehouse Cache and provides
@@ -64,7 +69,7 @@ class VBOLimsCache(Cache):
 class VisualBehaviorOphysProjectCache(ProjectCacheBase):
 
     PROJECT_NAME = "visual-behavior-ophys"
-    BUCKET_NAME = "visual-behavior-ophys-data"
+    BUCKET_NAME = "staging.visual-behavior-ophys-data"
 
     def __init__(
             self,
@@ -121,6 +126,23 @@ def __init__(
                                           version=version,
                                           cache=cache)
 
+        warnings.warn(
+            message="\n\tAs of AllenSDK version 2.16.0, the latest Visual "
+                    "Behavior Ophys data has been significantly updated from "
+                    "previous releases. Specifically the user will need to "
+                    "update all processing of the stimulus_presentations "
+                    "tables. These tables now include multiple stimulus types "
+                    "delineated by the columns `stimulus_block` and "
+                    "`stimulus_block_name`.\n\nThe data that was available in "
+                    "previous releases are stored in the "
+                    "block name containing 'change_detection' and can be "
+                    "accessed in the pandas table by using: \n\t"
+                    "`stimulus_presentations["
+                    "stimulus_presentations.stimulus_block_name.str.contains"
+                    "('change_detection')]`",
+            category=UpdatedStimulusPresentationTableWarning
+        )
+
     @classmethod
     def cloud_api_class(cls):
         return BehaviorProjectCloudApi
@@ -346,7 +368,7 @@ def get_natural_movie_template(self, n_workers=None) -> pd.DataFrame:
         frames as presented to the mouse. The DataFrame is indexed with the
         same frame index as shown in the stimulus presentation table.
 
-        The processing of the movie requires signicant processing and its
+        The processing of the movie requires significant processing and its
         return size is very large so take care in requesting this data.
 
         Parameters

diff --git a/...vatory/behavior/behavior_project_cache/project_apis/data_io/behavior_project_cloud_api.py b/...vatory/behavior/behavior_project_cache/project_apis/data_io/behavior_project_cloud_api.py
@@ -65,7 +65,7 @@ def sanitize_data_columns(
 
 
 class BehaviorProjectCloudApi(BehaviorProjectBase, ProjectCloudApiBase):
-    MANIFEST_COMPATIBILITY = ["0.0.0", "2.0.0"]
+    MANIFEST_COMPATIBILITY = ["1.0.0", "2.0.0"]
 
     def __init__(
         self,

diff --git a/...rvatory/behavior/behavior_project_cache/project_apis/data_io/behavior_project_lims_api.py b/...rvatory/behavior/behavior_project_cache/project_apis/data_io/behavior_project_lims_api.py
@@ -402,11 +402,7 @@ def _get_ophys_cells_table(self):
             SELECT
             cr.id as cell_roi_id,
             cr.cell_specimen_id,
-            cr.ophys_experiment_id,
-            cr.x,
-            cr.y,
-            cr.height,
-            cr.width
+            cr.ophys_experiment_id
             FROM cell_rois AS cr
             JOIN ophys_cell_segmentation_runs AS ocsr
                 ON ocsr.id=cr.ophys_cell_segmentation_run_id

diff --git a/allensdk/brain_observatory/behavior/behavior_session.py b/allensdk/brain_observatory/behavior/behavior_session.py
@@ -67,9 +67,6 @@
 from allensdk.brain_observatory.behavior.data_objects.trials.trials import (
     Trials,
 )
-from allensdk.brain_observatory.behavior.stimulus_processing import (
-    compute_trials_id_for_stimulus,
-)
 from allensdk.brain_observatory.sync_dataset import Dataset as SyncDataset
 from allensdk.core import (
     DataObject,
@@ -1068,55 +1065,113 @@ def raw_running_speed(self) -> pd.DataFrame:
 
     @property
     def stimulus_presentations(self) -> pd.DataFrame:
-        """Table whose rows are stimulus presentations (i.e. a given image,
+        """Table whose rows are stimulus presentations (e.g. a given image,
         for a given duration, typically 250 ms) and whose columns are
         presentation characteristics.
 
-        Adds trials_id to the stimulus table if the column is not already
-        present.
+        For VBO, the stimulus presentations tables now contain data from all
+        stimuli not just the active/passive image stimulus. These new stimuli
+        are delineated by stimulus_block and stimulus_block_name. The
+        previously released image behavior stimulus is accessible as the block
+        with a name containing "change_detection". Use the following example
+        code snippet to retrieve the original stimulus block from the pandas
+        table:
+            `stimulus_presentations[
+                stimulus_presentations.stimulus_block_name.str.contains(
+                    'change_detection'
+                )
+            ]`
 
         Returns
         -------
         pd.DataFrame
             Table whose rows are stimulus presentations
-            (i.e. a given image, for a given duration, typically 250 ms)
+            (e.g. a given image, for a given duration, typically 250 ms)
             and whose columns are presentation characteristics.
 
             dataframe columns:
                 stimulus_presentations_id [index]: (int)
                     identifier for a stimulus presentation
                     (presentation of an image)
+                stimulus_block: (int)
+                    Index of the different stimuli presented to the mouse
+                    in order.
+                stimulus_block_name [VBO only]: (int)
+                    Name of the individual stimulus blocks describing briefly
+                    what was shown to the mouse.
+                image_index [VBO only]: (int)
+                    image index (0-7) for a given session,
+                    corresponding to each image name
+                image_name: (string)
+                    Name of the image presented to the mouse if applicable.
+                    This indexes in to the stimulus_templates object.
+                movie_frame_index [VBO only]: (int)
+                    Index into the natural movie presented. use with the
+                    method get_raw_natural_movie or if you need the movie
+                    frames stretched and warped to the monitor use
+                    get_natural_movie_template. (Warning this is a very compute
+                    intensive process).
                 duration: (float)
                     duration of an image presentation (flash)
                     in seconds (stop_time - start_time). NaN if omitted
-                start_frame: (int)
-                    image presentation start frame
-                end_frame: (float)
-                    image presentation end frame
                 start_time: (float)
                     image presentation start time in seconds
                 end_time: (float)
                     image presentation end time in seconds
-                image_index: (int)
-                    image index (0-7) for a given session,
-                    corresponding to each image name
+                start_frame: (int)
+                    image presentation start frame
+                end_frame: (int)
+                    image presentation end frame
+                is_change: (bool)
+                    True if the image stimulus has changed. <NA> if not
+                    an image stimulus block.
+                is_image_novel: (bool)
+                    True if the image stimulus is novel. <NA> if not
+                    an image stimulus block.
                 omitted: (bool)
                     True if no image was shown for this stimulus
-                    presentation
+                    presentation. <NA> if not an image stimulus block.
+                movie_repeat [VBO only]: (int)
+                    Number of times the movie has been repeated during the
+                    natural movie block. -99 if not a movie block.
+                flashes_since_change: (bool)
+                    Number of times the image has been flashed in between when
+                    it has changed.
                 trials_id: (int)
                     Id to match to the table Index of the trials table.
+                active: (bool)
+                    Stimulus block overlaps with the trials.
+                is_sham_change: (bool)
+                    True if the stimulus overlaps with a catch trial.
+                stimulus_index [VBN only]: (int)
+                    0 for gabor_20_deg_250ms, 1 for flash_250ms. -99 otherwise
+                position_x [VBN only]: (float)
+                    X position of gabor grating when shown. NaN if not
+                    applicable.
+                position_y [VBN only]: (float)
+                    Y position of gabor grating when shown. NaN if not
+                    applicable.
+                color [VBN only]: (float)
+                    Value of the color int flash_250ms stimulus. Either -1 or
+                    1. NaN if not applicable.
+                rewarded [VBN only]: (bool)
+                    Merge of trials data. True if the mouse was rewarded during
+                    the stimulus. <NA> if not applicable to stimulus block.
+                orientation [VBN only]: (float)
+                    Orientation of the gabor grating in degrees.
+                temporal_frequency [VBN only]: (float)
+                    Temporal frequency of the gabor grating. NaN if not
+                    applicable.
+                spatial_frequency [VBN only]: (float)
+                    Spatial frequency of the gabor grating. NaN if not
+                    applicable.
+                contrast [VBN only]: float
+                    Contrast value of the gabor grating. NaN if not applicable.
         """
         table = self._stimuli.presentations.value
         table = table.drop(columns=["image_set", "index"], errors="ignore")
         table = table.rename(columns={"stop_time": "end_time"})
 
-        if (
-            "trials_id" not in table.columns
-            and "stimulus_block" in table.columns
-        ):
-            table["trials_id"] = compute_trials_id_for_stimulus(
-                table, self.trials
-            )
         return table
 
     @property
@@ -1220,11 +1275,10 @@ def trials(self) -> pd.DataFrame:
         Returns
         -------
         pd.DataFrame
-            A dataframe containing trial and behavioral response data,
-            by cell specimen id
+            A dataframe containing trial and behavioral response data.
 
             dataframe columns:
-                trials_id: (int)
+                trials_id [index]: (int)
                     trial identifier
                 lick_times: (array of float)
                     array of lick times in seconds during that trial.
@@ -1279,6 +1333,17 @@ def trials(self) -> pd.DataFrame:
                 change_image_name: (string)
                     name of image that is changed to at the change time,
                     on go trials
+                change_time: (float)
+                    Time in the session at which the change occurred.
+                response_latency [VBO only]: (float)
+                    Delay between change and first lick in seconds. NaN if
+                    no change occurred.
+                change_frame: (int)
+                    Frame in the session at which the change occurred. -99 if
+                    no change.
+                change_time_no_display_delay [VBN only]: (float)
+                    Time of the change in seconds, before the display lag is
+                    accounted for and applied.
         """
         return self._trials.data
 

diff --git a/allensdk/brain_observatory/behavior/data_objects/stimuli/presentations.py b/allensdk/brain_observatory/behavior/data_objects/stimuli/presentations.py
@@ -87,6 +87,7 @@ def __init__(
             [
                 "flashes_since_change",
                 "image_index",
+                "movie_repeat",
                 "movie_frame_index",
                 "repeat",
                 "stimulus_index",

diff --git a/allensdk/brain_observatory/behavior/stimulus_processing.py b/allensdk/brain_observatory/behavior/stimulus_processing.py
@@ -742,13 +742,6 @@ def compute_trials_id_for_stimulus(
         index=stim_pres_table.index,
         name="trials_id",
     ).astype("int")
-    # Return input frame if the stimulus_block or active is not available.
-    if (
-        "stimulus_block" not in stim_pres_table.columns
-        or "active" not in stim_pres_table.columns
-    ):
-        return trials_ids
-    active_sorted = stim_pres_table.active
 
     # Find stimulus blocks that start within a trial. Copy the trial_id
     # into our new trials_ids series. For some sessions there are gaps in
@@ -757,12 +750,20 @@ def compute_trials_id_for_stimulus(
     max_trials_stop = trials_table.stop_time.max()
     for idx, trial in trials_table.iterrows():
         stim_mask = (
-            (stim_pres_table.start_time > trial.start_time)
-            & (stim_pres_table.start_time < max_trials_stop)
-            & (~stim_pres_table.image_name.isna())
+                (stim_pres_table.start_time > trial.start_time)
+                & (stim_pres_table.start_time < max_trials_stop)
+                & (~stim_pres_table.image_name.isna())
         )
         trials_ids[stim_mask] = idx
 
+    # Return input frame if the stimulus_block or active is not available.
+    if (
+        "stimulus_block" not in stim_pres_table.columns
+        or "active" not in stim_pres_table.columns
+    ):
+        return trials_ids
+    active_sorted = stim_pres_table.active
+
     # The code below finds all stimulus blocks that contain images/trials
     # and attempts to detect blocks that are identical to copy the associated
     # trials_ids into those blocks. In the parlance of the data this is

diff --git a/allensdk/brain_observatory/ophys/project_constants.py b/allensdk/brain_observatory/ophys/project_constants.py
@@ -8,11 +8,19 @@
 #
 #####################################################################
 
+# Last three codes are projects that have experiments that were pulled into
+# the data release. The mouse in these codes had most of its sessions
+# reassigned to one of the first 4 codes. This is a patch to allow these
+# projects to get the proper stimulus presentation tables with correct
+# stimulus_block_names.
 PROJECT_CODES = [
     "VisualBehavior",
     "VisualBehaviorTask1B",
     "VisualBehaviorMultiscope",
     "VisualBehaviorMultiscope4areasx2d",
+    "DevelopmentMultiscope4areasx2d",
+    "MesoscopeDevelopment",
+    "VisBIntTestDatacube"
 ]
 
 NUM_STRUCTURES_DICT = {