run_x_to_nwb_conversion: Ignore feedback stimulus data by default for…

… ABF files

Not all data from ABF files is real data, some is also the fedback
stimulus data.

Ignore that by default when converting.

The distinction can be made due to the following properties:

> The best way to distinguish are the names. The real data channels are
> always "IN0", "IN1", "IN2" or "IN3". The feedback channels are "IN4"
> (for IN0), "IN5" (for IN1), "IN6" (for IN2) and "IN7" (for IN3). There
> is one set up where the real data channel is called "IN0" and the
> feedback data channel is called "RawOutput".

Source: Private communication from Natalia Goriounova (Mansvelder lab).

ipfx/bin/run_x_to_nwb_conversion.py

@@ -7,7 +7,7 @@
 from ipfx.x_to_nwb.DatConverter import DatConverter
 
 
-def convert(inFileOrFolder, overwrite=False, fileType=None, outputMetadata=False):
+def convert(inFileOrFolder, overwrite=False, fileType=None, outputMetadata=False, outputFeedbackChannel=False):
     """
     Convert the given file to a NeuroDataWithoutBorders file using pynwb
 
@@ -19,6 +19,7 @@ def convert(inFileOrFolder, overwrite=False, fileType=None, outputMetadata=False
     :param overwrite: overwrite output file, defaults to `False`
     :param fileType: file type to be converted, must be passed iff `inFileOrFolder` refers to a folder
     :param outputMetadata: output metadata of the file, helpful for debugging
+    :param outputFeedbackChannel: Output ADC data which stems from stimulus feedback channels (ignored for DAT files)
 
     :return: path of the created NWB file
     """
@@ -51,7 +52,7 @@ def convert(inFileOrFolder, overwrite=False, fileType=None, outputMetadata=False
         if outputMetadata:
             ABFConverter.outputMetadata(inFileOrFolder)
         else:
-            ABFConverter(inFileOrFolder, outFile)
+            ABFConverter(inFileOrFolder, outFile, outputFeedbackChannel=outputFeedbackChannel)
     elif ext == ".dat":
         if outputMetadata:
             DatConverter.outputMetadata(inFileOrFolder)
@@ -77,6 +78,8 @@ def main():
                               "if passing folders)."))
     parser.add_argument("--outputMetadata", action="store_true", default=False,
                         help="Helper for debugging which outputs HTML/TXT files with the metadata contents of the files.")
+    parser.add_argument("--outputFeedbackChannel", action="store_true", default=False,
+                        help="Output ADC data to the NWB file which stems from stimulus feedback channels.")
     parser.add_argument("filesOrFolders", nargs="+",
                         help="List of ABF files/folders to convert.")
 
@@ -91,7 +94,8 @@ def main():
     for fileOrFolder in args.filesOrFolders:
         print(f"Converting {fileOrFolder}")
         convert(fileOrFolder, overwrite=args.overwrite, fileType=args.fileType,
-                outputMetadata=args.outputMetadata)
+                outputMetadata=args.outputMetadata,
+                outputFeedbackChannel=args.outputFeedbackChannel)
 
 
 if __name__ == "__main__":

ipfx/x_to_nwb/ABFConverter.py

@@ -20,14 +20,16 @@
 class ABFConverter:
 
     protocolStorageDir = None
+    adcNamesWithRealData = ("IN 0", "IN 1", "IN 2", "IN 3")
 
-    def __init__(self, inFileOrFolder, outFile):
+    def __init__(self, inFileOrFolder, outFile, outputFeedbackChannel):
         """
         Convert the given ABF file to NWB
 
         Keyword arguments:
-        inFileOrFolder -- input file, or folder with multiple files, in ABF v2 format
-        outFile        -- target filepath (must not exist)
+        inFileOrFolder        -- input file, or folder with multiple files, in ABF v2 format
+        outFile               -- target filepath (must not exist)
+        outputFeedbackChannel -- Output ADC data from feedback channels as well (useful for debugging only)
         """
 
         inFiles = []
@@ -39,6 +41,8 @@ def __init__(self, inFileOrFolder, outFile):
         else:
             raise ValueError(f"{inFileOrFolder} is neither a folder nor a path.")
 
+        self.outputFeedbackChannel = outputFeedbackChannel
+
         self._amplifierSettings = self._getJSONFile(inFileOrFolder)
 
         self.abfs = []
@@ -423,6 +427,15 @@ def _createAcquiredSeries(self, electrodes):
             for sweep in range(abf.sweepCount):
                 cycle_id = createCycleID([file_index, sweep], total=self.totalSeriesCount)
                 for channel in range(abf.channelCount):
+                    adcName = abf.adcNames[channel]
+
+                    if not self.outputFeedbackChannel:
+                        if adcName in adcNamesWithRealData:
+                            pass
+                        else:
+                            # feedback data, skip
+                            continue
+
                     abf.setSweep(sweep, channel=channel, absoluteTime=True)
                     name, counter = createSeriesName("index", counter, total=self.totalSeriesCount)
                     data = createCompressedDataset(abf.sweepY)
@@ -433,7 +446,6 @@ def _createAcquiredSeries(self, electrodes):
                     starting_time = self._calculateStartingTime(abf)
                     stimulus_description = abf.protocol
                     rate = float(abf.dataRate)
-                    adcName = abf.adcNames[channel]
                     description = json.dumps({"cycle_id": cycle_id,
                                               "protocol": abf.protocol,
                                               "protocolPath": abf.protocolPath,

tests/test_x_nwb.py

@@ -68,7 +68,7 @@ def test_file_level_regressions(raw_file):
 
     ref_folder = f"reference_{ext[1:]}_nwb"
 
-    new_file = convert(raw_file, overwrite=True)
+    new_file = convert(raw_file, overwrite=True, outputFeedbackChannel=True)
     ref_file = os.path.join(ref_folder, os.path.basename(new_file))
 
     assert os.path.isfile(ref_file)