src/x_to_nwb/conversion.py: Add generic channel selection options for…

… ABF

Up to know the list of written AD channels into NWB was a mixture of
hardcoded via ABFConverter.adcNamesWithRealData and the possibility to
extend that list either to all using --outputFeedbackChannel or
partially with --realDataChannel.

This approach does not work for arbitrary ABF files.

We now have new options --includeChannel and --discardChannel which
allows total control over the outputted AD channels.

README.md

@@ -11,6 +11,23 @@ writes them to a file in JSON output.
 
 In case you don't have a JSON settings file pass `--no-searchSettingsFile` to avoid warnings.
 
+For historic reasons the ABF conversion uses a hardcoded list of AD channel names
+which will be written into the NWB file, other channels are discarded.
+
+To turn that behaviour off call the conversion script with
+
+```sh
+x-to-nwb --includeChannel "*" 2018_03_20_0000.abf
+```
+
+which outputs all AD channels.
+
+To discard some AD channels (the hardcoded list is ignored in this case) use
+
+```sh
+x-to-nwb --discardChannel ABCD 2018_03_20_0000.abf
+```
+
 #### Required input files
 
 - ABF files acquired with Clampex/pCLAMP.

src/x_to_nwb/ABFConverter.py

@@ -41,18 +41,29 @@
 class ABFConverter:
 
     protocolStorageDir = None
+    # TODO hardcoded channel names should be removed,
+    # together with the --outputFeedbackChannel and --realDataChannel options of run_x_to_nwb_conversion.py
     adcNamesWithRealData = ["IN 0", "IN 1", "IN 2", "IN 3"]
 
-    def __init__(self, inFileOrFolder, outFile, outputFeedbackChannel, compression=True, searchSettingsFile=True):
+    def __init__(
+        self,
+        inFileOrFolder,
+        outFile,
+        compression=True,
+        searchSettingsFile=True,
+        includeChannelList=None,
+        discardChannelList=None,
+    ):
         """
-        Convert the given ABF file to NWB
+        Convert the given ABF file to NWB. By default all ADC channel are written in to the NWB file.
 
         Keyword arguments:
         inFileOrFolder        -- input file, or folder with multiple files, in ABF v2 format
         outFile               -- target filepath (must not exist)
-        outputFeedbackChannel -- Output ADC data from feedback channels as well (useful for debugging only)
         compression           -- Toggle compression for HDF5 datasets
         searchSettingsFile    -- Search the JSON settings file and warn if it could not be found
+        includeChannelList    -- ADC channels to write into the NWB file
+        discardChannelList    -- ADC channels to not write into the NWB file
         """
 
         inFiles = []
@@ -64,7 +75,16 @@ def __init__(self, inFileOrFolder, outFile, outputFeedbackChannel, compression=T
         else:
             raise ValueError(f"{inFileOrFolder} is neither a folder nor a path.")
 
-        self.outputFeedbackChannel = outputFeedbackChannel
+        if includeChannelList is not None and discardChannelList is not None:
+            raise ValueError(
+                f"includeChannelList and discardChannelList are mutually exclusive. Pass only one of them."
+            )
+        elif includeChannelList is None and discardChannelList is None:
+            includeChannelList = list("*")
+
+        self.includeChannelList = includeChannelList
+        self.discardChannelList = discardChannelList
+
         self.compression = compression
         self.searchSettingsFile = searchSettingsFile
 
@@ -224,21 +244,20 @@ def _check(self, abf):
 
     def _reduceChannelList(self, abf):
         """
-        Return a reduced channel list taking into account the feedback channel export setting.
+        Return a reduced channel list taking into account the include and discard ADC channel settings.
         """
 
-        if self.outputFeedbackChannel:
-            return abf.channelList
+        if self.includeChannelList is not None:
 
-        cleanChanneList = []
+            if self.includeChannelList == list("*"):
+                return abf.adcNames
 
-        for channel in range(abf.channelCount):
-            adcName = abf.adcNames[channel]
+            return list(set(abf.adcNames).intersection(self.includeChannelList))
 
-            if adcName in ABFConverter.adcNamesWithRealData:
-                cleanChanneList.append(abf.channelList[channel])
+        elif self.discardChannelList is not None:
+            return list(set(abf.adcNames) - set(abf.adcNames).intersection(self.discardChannelList))
 
-        return cleanChanneList
+        raise ValueError(f"Unexpected include and discard channel settings.")
 
     def _checkAll(self):
         """
@@ -579,17 +598,25 @@ def _createAcquiredSeries(self, electrodes):
             _, jsonSource = self._findSettingsEntry(abf)
             log.debug(f"Using JSON settings for {jsonSource}.")
 
+            channelList = self._reduceChannelList(abf)
+            log.debug(f"Channel lists: original {abf.adcNames}, reduced {channelList}")
+
+            if len(channelList) == 0:
+                warnings.warn(
+                    f"The channel settings {self.includeChannelList} (included) and {self.discardChannelList} (discarded) resulted "
+                    f"in an empty channelList for {abf.abfFilePath} with the unfiltered channels being {abf.adcNames}."
+                )
+                continue
+
             for sweep in range(abf.sweepCount):
                 cycle_id = createCycleID([file_index, sweep], total=self.totalSeriesCount)
+
                 for channel in range(abf.channelCount):
+
                     adcName = abf.adcNames[channel]
 
-                    if not self.outputFeedbackChannel:
-                        if adcName in ABFConverter.adcNamesWithRealData:
-                            pass
-                        else:
-                            # feedback data, skip
-                            continue
+                    if adcName not in channelList:
+                        continue
 
                     abf.setSweep(sweep, channel=channel, absoluteTime=True)
                     name, counter = createSeriesName("index", counter, total=self.totalSeriesCount)

src/x_to_nwb/conversion.py

@@ -14,10 +14,11 @@ def convert(
     overwrite=False,
     fileType=None,
     outputMetadata=False,
-    outputFeedbackChannel=False,
     multipleGroupsPerFile=False,
     compression=True,
     searchSettingsFile=True,
+    includeChannelList="*",
+    discardChannelList=None,
 ):
     """
     Convert the given file to a NeuroDataWithoutBorders file using pynwb
@@ -30,11 +31,12 @@ def convert(
     :param overwrite: overwrite output file, defaults to `False`
     :param fileType: file type to be converted, must be passed iff `inFileOrFolder` refers to a folder
     :param outputMetadata: output metadata of the file, helpful for debugging
-    :param outputFeedbackChannel: Output ADC data which stems from stimulus feedback channels (ignored for DAT files)
     :param multipleGroupsPerFile: Write all Groups in the DAT file into one NWB
                                   file. By default we create one NWB per Group (ignored for ABF files).
     :param searchSettingsFile: Search the JSON amplifier settings file and warn if it could not be found (ignored for DAT files)
     :param compression: Toggle compression for HDF5 datasets
+    :param includeChannelList: ADC channels to write into the NWB file (ignored for DAT files)
+    :param discardChannelList: ADC channels to not write into the NWB file (ignored for DAT files)
 
     :return: path of the created NWB file
     """
@@ -69,9 +71,10 @@ def convert(
             ABFConverter(
                 inFileOrFolder,
                 outFile,
-                outputFeedbackChannel=outputFeedbackChannel,
                 compression=compression,
                 searchSettingsFile=searchSettingsFile,
+                includeChannelList=includeChannelList,
+                discardChannelList=discardChannelList,
             )
     elif ext == ".dat":
         if outputMetadata:
@@ -144,6 +147,22 @@ def convert_cli():
         help="Don't search the JSON file for the amplifier settings.",
     )
 
+    abf_group_channels = abf_group.add_mutually_exclusive_group(required=False)
+    abf_group_channels.add_argument(
+        "--includeChannel",
+        type=str,
+        dest="includeChannelList",
+        action="append",
+        help=f"Name of ADC channels to include in the NWB file. Can not be combined with --outputFeedbackChannel and --realDataChannel as these settings are ignored.",
+    )
+    abf_group_channels.add_argument(
+        "--discardChannel",
+        type=str,
+        dest="discardChannelList",
+        action="append",
+        help=f"Name of ADC channels to not include in the NWB file. Can not be combined with --outputFeedbackChannel and --realDataChannel as these settings are ignored.",
+    )
+
     dat_group.add_argument(
         "--multipleGroupsPerFile",
         action="store_true",
@@ -153,6 +172,19 @@ def convert_cli():
 
     args = parser.parse_args()
 
+    if args.includeChannelList is not None or args.discardChannelList is not None:
+        if args.outputFeedbackChannel or args.realDataChannel:
+            raise ValueError(
+                "--outputFeedbackChannel and --realDataChannel can not be present together with --includeChannel or --discardChannel."
+            )
+
+    elif args.realDataChannel:
+        args.includeChannelList = ABFConverter.adcNamesWithRealData + args.realDataChannel
+    elif args.outputFeedbackChannel:
+        args.includeChannelList = "*"
+    else:
+        args.includeChannelList = ABFConverter.adcNamesWithRealData
+
     if args.log:
         numeric_level = getattr(logging, args.log.upper(), None)
 
@@ -168,20 +200,18 @@ def convert_cli():
 
         ABFConverter.protocolStorageDir = args.protocolDir
 
-    if args.realDataChannel:
-        ABFConverter.adcNamesWithRealData.append(args.realDataChannel)
-
     for fileOrFolder in args.filesOrFolders:
         print(f"Converting {fileOrFolder}")
         convert(
             fileOrFolder,
             overwrite=args.overwrite,
             fileType=args.fileType,
             outputMetadata=args.outputMetadata,
-            outputFeedbackChannel=args.outputFeedbackChannel,
             multipleGroupsPerFile=args.multipleGroupsPerFile,
             compression=args.compression,
             searchSettingsFile=args.searchSettingsFile,
+            includeChannelList=args.includeChannelList,
+            discardChannelList=args.discardChannelList,
         )
 
 

tests/test_x_nwb.py

@@ -56,7 +56,7 @@ def test_file_level_regressions(raw_file):
 
     ref_folder = f"reference_{ext[1:]}_nwb"
 
-    new_file = convert(raw_file, overwrite=True, outputFeedbackChannel=True, multipleGroupsPerFile=True)
+    new_file = convert(raw_file, overwrite=True, multipleGroupsPerFile=True)
     ref_file = os.path.join(ref_folder, os.path.basename(new_file))
 
     assert os.path.isfile(ref_file)

tests/test_x_nwb_helper.py

@@ -108,7 +108,7 @@ def create_files_for_upload(ext):
 
     for f in files:
         print(f"Converting {f}")
-        convert(f, overwrite=True, outputFeedbackChannel=True, multipleGroupsPerFile=True)
+        convert(f, overwrite=True, multipleGroupsPerFile=True)
 
     nwb_folder = basename + "_nwb"
     zip_files(folder, nwb_folder, ".nwb")