diff --git a/.gitignore b/.gitignore index 43202e4b..93112cd3 100644 --- a/.gitignore +++ b/.gitignore @@ -92,6 +92,7 @@ src/nectarchain/user_scripts/**/test **.log **.pdf **.csv +**.png #VScode .vscode/ diff --git a/README.md b/README.md index 1524af1f..12515fff 100644 --- a/README.md +++ b/README.md @@ -1,4 +1,4 @@ -# nectarchain [![Build Status](https://github.com/cta-observatory/nectarchain/actions/workflows/ci.yml/badge.svg?branch=main)](https://github.com/cta-observatory/nectarchain/actions/workflows/ci.yml?query=workflow%3ACI+branch%3Amain) +# nectarchain [![Build Status](https://github.com/cta-observatory/nectarchain/actions/workflows/ci.yml/badge.svg?branch=main)](https://github.com/cta-observatory/nectarchain/actions/workflows/ci.yml?query=workflow%3ACI+branch%3Amain) [![pypi](https://badge.fury.io/py/nectarchain.svg)](https://badge.fury.io/py/nectarchain) [![conda](https://anaconda.org/conda-forge/nectarchain/badges/version.svg)](https://anaconda.org/conda-forge/nectarchain) Repository for the high level analysis of the NectarCAM data. The analysis is heavily based on [ctapipe](https://github.com/cta-observatory/ctapipe), adding custom code for NectarCAM calibration. diff --git a/docs/Makefile b/docs/Makefile index 3f474726..b31d1965 100644 --- a/docs/Makefile +++ b/docs/Makefile @@ -20,5 +20,5 @@ help: @$(SPHINXBUILD) -M $@ "$(SOURCEDIR)" "$(BUILDDIR)" $(SPHINXOPTS) $(O) clean: - rm -rf api + rm -rf _autosummary @$(SPHINXBUILD) -M $@ "$(SOURCEDIR)" "$(BUILDDIR)" $(SPHINXOPTS) $(O) diff --git a/docs/_static/nectarcam.png b/docs/_static/nectarcam.png deleted file mode 100644 index 8f28b617..00000000 Binary files a/docs/_static/nectarcam.png and /dev/null differ diff --git a/docs/_static/nectarcam_logo.webp b/docs/_static/nectarcam_logo.webp new file mode 100644 index 00000000..cb468cf4 Binary files /dev/null and b/docs/_static/nectarcam_logo.webp differ diff --git a/docs/_static/nectarcam_logo_dark.webp b/docs/_static/nectarcam_logo_dark.webp new file mode 100644 index 00000000..340c0ed1 Binary files /dev/null and b/docs/_static/nectarcam_logo_dark.webp differ diff --git a/docs/_templates/custom-class-template.rst b/docs/_templates/custom-class-template.rst new file mode 100644 index 00000000..af1c92dc --- /dev/null +++ b/docs/_templates/custom-class-template.rst @@ -0,0 +1,33 @@ +{{ fullname | escape | underline}} + +.. currentmodule:: {{ module }} + +.. autoclass:: {{ objname }} + :members: + :show-inheritance: + :special-members: __call__, __add__, __mul__ + + {% block methods %} + {% if methods %} + .. rubric:: {{ _('Methods') }} + + .. autosummary:: + :nosignatures: + {% for item in methods %} + {%- if not item.startswith('_') %} + ~{{ name }}.{{ item }} + {%- endif -%} + {%- endfor %} + {% endif %} + {% endblock %} + + {% block attributes %} + {% if attributes %} + .. rubric:: {{ _('Attributes') }} + + .. autosummary:: + {% for item in attributes %} + ~{{ name }}.{{ item }} + {%- endfor %} + {% endif %} + {% endblock %} diff --git a/docs/_templates/custom-module-template.rst b/docs/_templates/custom-module-template.rst new file mode 100644 index 00000000..d066d0e4 --- /dev/null +++ b/docs/_templates/custom-module-template.rst @@ -0,0 +1,66 @@ +{{ fullname | escape | underline}} + +.. automodule:: {{ fullname }} + + {% block attributes %} + {% if attributes %} + .. rubric:: Module attributes + + .. autosummary:: + :toctree: + {% for item in attributes %} + {{ item }} + {%- endfor %} + {% endif %} + {% endblock %} + + {% block functions %} + {% if functions %} + .. rubric:: {{ _('Functions') }} + + .. autosummary:: + :toctree: + :nosignatures: + {% for item in functions %} + {{ item }} + {%- endfor %} + {% endif %} + {% endblock %} + + {% block classes %} + {% if classes %} + .. rubric:: {{ _('Classes') }} + + .. autosummary:: + :toctree: + :template: custom-class-template.rst + :nosignatures: + {% for item in classes %} + {{ item }} + {%- endfor %} + {% endif %} + {% endblock %} + + {% block exceptions %} + {% if exceptions %} + .. rubric:: {{ _('Exceptions') }} + + .. autosummary:: + :toctree: + {% for item in exceptions %} + {{ item }} + {%- endfor %} + {% endif %} + {% endblock %} + +{% block modules %} +{% if modules %} +.. autosummary:: + :toctree: + :template: custom-module-template.rst + :recursive: +{% for item in modules %} + {{ item }} +{%- endfor %} +{% endif %} +{% endblock %} diff --git a/docs/api-reference/data/index.rst b/docs/api-reference/data/index.rst deleted file mode 100644 index 74e2ce05..00000000 --- a/docs/api-reference/data/index.rst +++ /dev/null @@ -1,11 +0,0 @@ -.. _data: - -Data (`data`) -============= -Module containing functions and classes holding NectarCAM data structure, and interaction with DIRAC. - -Reference/API -============= -.. automodule:: nectarchain.data - :members: - :undoc-members: \ No newline at end of file diff --git a/docs/api-reference/display/index.rst b/docs/api-reference/display/index.rst deleted file mode 100644 index 2420ad5e..00000000 --- a/docs/api-reference/display/index.rst +++ /dev/null @@ -1,10 +0,0 @@ -.. _display: - -Display (`display`) -=================== - -Reference/API -============= -.. automodule:: nectarchain.display - :members: - :undoc-members: diff --git a/docs/api-reference/dqm/index.rst b/docs/api-reference/dqm/index.rst deleted file mode 100644 index ac61a127..00000000 --- a/docs/api-reference/dqm/index.rst +++ /dev/null @@ -1,12 +0,0 @@ -.. _dqm: - -Data Quality Monitoring (`dqm`) -=============================== - -Module for the NectarCAM data quality monitoring. - -Reference/API -============= -.. automodule:: nectarchain.dqm - :members: - :undoc-members: \ No newline at end of file diff --git a/docs/api-reference/index.rst b/docs/api-reference/index.rst deleted file mode 100644 index 2303e42c..00000000 --- a/docs/api-reference/index.rst +++ /dev/null @@ -1,8 +0,0 @@ -API Docs -======== - -.. toctree:: - :maxdepth: 1 - :glob: - - */index diff --git a/docs/api-reference/tools/index.rst b/docs/api-reference/tools/index.rst deleted file mode 100644 index 0cf536e7..00000000 --- a/docs/api-reference/tools/index.rst +++ /dev/null @@ -1,12 +0,0 @@ -.. _tools: - -Tools (`tools`) -=============== - -Tools submodule. - -Reference/API -============= -.. automodule:: nectarchain.tools - :members: - :undoc-members: \ No newline at end of file diff --git a/docs/api-reference/utils/index.rst b/docs/api-reference/utils/index.rst deleted file mode 100644 index d115fbce..00000000 --- a/docs/api-reference/utils/index.rst +++ /dev/null @@ -1,12 +0,0 @@ -.. _utils: - -Utilities (`utils`) -=================== - -Utilities submodule. - -Reference/API -============= -.. automodule:: nectarchain.utils - :members: - :undoc-members: \ No newline at end of file diff --git a/docs/api.rst b/docs/api.rst new file mode 100644 index 00000000..4eb1a014 --- /dev/null +++ b/docs/api.rst @@ -0,0 +1,12 @@ +.. + DO NOT DELETE THIS FILE! It contains the all-important `.. autosummary::` directive with `:recursive:` option, without + which API documentation wouldn't get extracted from docstrings by the `sphinx.ext.autosummary` engine. It is hidden + (not declared in any toctree) to remove an unnecessary intermediate page; index.rst instead points directly to the + package page. DO NOT REMOVE THIS FILE! + +.. autosummary:: + :toctree: _autosummary + :template: custom-module-template.rst + :recursive: + + nectarchain diff --git a/docs/conf.py b/docs/conf.py index 486b563a..7ee815f2 100644 --- a/docs/conf.py +++ b/docs/conf.py @@ -42,18 +42,25 @@ # extensions coming with Sphinx (named 'sphinx.ext.*') or your custom # ones. extensions = [ - "sphinx.ext.githubpages", - "sphinx.ext.intersphinx", "sphinx.ext.autodoc", "sphinx.ext.autosummary", + "sphinx.ext.intersphinx", "sphinx.ext.viewcode", - "sphinx_automodapi.automodapi", - "sphinx_automodapi.smart_resolver", - "numpydoc", + "sphinx_autodoc_typehints", # Automatically document param types (less noise in + # class signature) ] -numpydoc_show_class_members = False -autosummary_generate = True +autosummary_generate = True # Turn on sphinx.ext.autosummary +autoclass_content = "both" # Add __init__ doc (ie. params) to class summaries +html_show_sourcelink = ( + False # Remove 'view source code' from top of page (for html, not python) +) +autodoc_inherit_docstrings = False # If no docstring, inherit from base class +set_type_checking_flag = True # Enable 'expensive' imports for sphinx_autodoc_typehints +nbsphinx_allow_errors = True # Continue through Jupyter errors +# autodoc_typehints = "description" # Sphinx-native method. Not as good as +# sphinx_autodoc_typehints +add_module_names = False # Remove namespaces from class/method signatures # The suffix(es) of source filenames. # You can specify multiple suffix as a list of string: @@ -64,17 +71,19 @@ # The master toctree document. master_doc = "index" -templates_path = [] # ["_templates"] +templates_path = ["_templates"] exclude_patterns = ["_build", "Thumbs.db", ".DS_Store"] # intersphinx allows referencing other packages sphinx docs intersphinx_mapping = { "python": ("https://docs.python.org/3.9", None), + "numpy": ("https://numpy.org/doc/stable", None), + "scipy": ("https://docs.scipy.org/doc/scipy", None), "astropy": ("https://docs.astropy.org/en/latest/", None), - "ctapipe": ("https://ctapipe.readthedocs.io/en/v0.19.3/", None), - "matplotlib": ("https://matplotlib.org/", None), + "matplotlib": ("https://matplotlib.org/stable/", None), "traitlets": ("https://traitlets.readthedocs.io/en/stable/", None), + "ctapipe": ("https://ctapipe.readthedocs.io/en/v0.19.3/", None), } # These links are ignored in the checks, necessary due to broken intersphinx for these @@ -95,6 +104,8 @@ ("py:class", "ClassesType"), ] +suppress_warnings = ["autosummary"] + # If true, `todo` and `todoList` produce output, else they produce nothing. todo_include_todos = True @@ -107,8 +118,6 @@ # Output file base name for HTML help builder. htmlhelp_basename = f"{project}doc" -html_logo = "_static/nectarcam.png" - # Add any paths that contain custom static files (such as style sheets) here, # relative to this directory. They are copied after the builtin static files, # so a file named "default.css" will overwrite the builtin "default.css". @@ -123,6 +132,11 @@ html_file_suffix = ".html" html_theme_options = { + "logo": { + "image_light": "_static/nectarcam_logo.webp", + "image_dark": "_static/nectarcam_logo_dark.webp", + "alt_text": "nectarchain", + }, "navigation_with_keys": False, "github_url": f"https://github.com/cta-observatory/{project}", "navbar_start": ["navbar-logo", "version-switcher"], diff --git a/docs/index.rst b/docs/index.rst index 82d05db6..282210fd 100644 --- a/docs/index.rst +++ b/docs/index.rst @@ -14,7 +14,7 @@ Welcome to nectarchain's documentation! :hidden: developer-guide/index - api-reference/index + API reference <_autosummary/nectarchain> Indices and tables diff --git a/pyproject.toml b/pyproject.toml index e5c76f24..f98c6442 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -48,6 +48,7 @@ dev = [ docs = [ "sphinx", + "sphinx-autodoc-typehints", "sphinx-automodapi", "pydata_sphinx_theme", "numpydoc", diff --git a/src/nectarchain/data/container/eventSource.py b/src/nectarchain/data/container/eventSource.py index e8f84414..bf55c8e5 100644 --- a/src/nectarchain/data/container/eventSource.py +++ b/src/nectarchain/data/container/eventSource.py @@ -192,27 +192,29 @@ class LightNectarCAMEventSource(NectarCAMEventSource): LightNectarCAMEventSource is a subclass of NectarCAMEventSource that provides a generator for iterating over NectarCAM events. - This implementation of the NectarCAMEventSource is mucvh lighter than the one within - ctapipe_io_nectarcam, only the fileds interesting - for nectachain are kept. - - Attributes: - input_url (str): The input URL of the data source. - max_events (int): The maximum number of events to process. - tel_id (int): The telescope ID. - nectarcam_service (NectarCAMService): The service container for NectarCAM. - trigger_information (bool): Flag indicating whether to fill trigger information - in the event container. - obs_ids (list): The list of observation IDs. - multi_file (MultiFileReader): The multi-file reader for reading the data source. - r0_r1_calibrator (R0R1Calibrator): The calibrator for R0 to R1 conversion. - calibrate_flatfields_and_pedestals (bool): Flag indicating whether to calibrate - flatfield and pedestal events. - - Methods: - _generator: The generator function that yields NectarCAMDataContainer objects - representing each event. - + This implementation of the NectarCAMEventSource is much lighter than the one within + ctapipe_io_nectarcam, only the fields interesting for nectarchain are kept. + + Parameters + ---------- + input_url : str + The input URL of the data source. + max_events : int + The maximum number of events to process. + tel_id : int + The telescope ID. + nectarcam_service : NectarCAMService + The service container for NectarCAM. + trigger_information : bool + Flag indicating whether to fill trigger information in the event container. + obs_ids : list + The list of observation IDs. + multi_file : MultiFileReader + The multi-file reader for reading the data source. + r0_r1_calibrator : R0R1Calibrator + The calibrator for R0 to R1 conversion. + calibrate_flatfields_and_pedestals : bool + Flag indicating whether to calibrate flatfield and pedestal events. """ def _generator(self): @@ -220,12 +222,15 @@ def _generator(self): The generator function that yields NectarCAMDataContainer objects representing each event. - Yields: - NectarCAMDataContainer: The NectarCAMDataContainer object representing - each event. + Yields + ------ + NectarCAMDataContainer : + The NectarCAMDataContainer object representing each event. + + Raises + ------ + None - Raises: - None """ # container for NectarCAM data array_event = NectarCAMDataContainer() diff --git a/src/nectarchain/data/management.py b/src/nectarchain/data/management.py index e438c292..cd9f0dcb 100644 --- a/src/nectarchain/data/management.py +++ b/src/nectarchain/data/management.py @@ -33,14 +33,19 @@ class DataManagement: @staticmethod def findrun(run_number: int, search_on_GRID=True) -> Tuple[Path, List[Path]]: - """method to find in NECTARCAMDATA the list of `*.fits.fz` files associated to + """method to find in NECTARCAMDATA the list of ``*.fits.fz`` files associated to run_number - Args: - run_number (int): the run number + Parameters + ---------- + run_number: int + the run number + + Returns + ------- + (PosixPath,list): + the path list of `*fits.fz` files - Returns: - (PosixPath,list): the path list of `*fits.fz` files """ basepath = f"{os.environ['NECTARCAMDATA']}/runs/" list = glob.glob( @@ -77,10 +82,12 @@ def findrun(run_number: int, search_on_GRID=True) -> Tuple[Path, List[Path]]: @staticmethod def getRunFromDIRAC(lfns: list): - """method do get run files from GRID-EGI from input lfns + """Method to get run files from the EGI grid from input lfns - Args: - lfns (list): list of lfns path + Parameters + ---------- + lfns: list + list of lfns path """ with KeepLoggingUnchanged(): from DIRAC.Interfaces.API.Dirac import Dirac @@ -111,16 +118,28 @@ def get_GRID_location( """ Method to get run location on GRID from Elog (work in progress!) - Args: - run_number (int): run number - output_lfns (bool, optional): if True, return lfns path of fits.gz files, else return parent directory of run location. Defaults to True. - basepath (str) : the path on GRID where nectarCAM data are stored. Default to /vo.cta.in2p3.fr/nectarcam/. - fromElog (bool,optional): To force to use the method which read the Elog. Default to False. To use the method with DIRAC API. - username (_type_, optional): username for Elog login. Defaults to None. - password (_type_, optional): password for Elog login. Defaults to None. + Parameters + ---------- + run_number: int + Run number + output_lfns: bool, optional + If True, return lfns path of fits.gz files, else return parent directory + of run location. Defaults to True. + basepath: str + The path on GRID where nectarCAM data are stored. Default to + ``/vo.cta.in2p3.fr/nectarcam/``. + fromElog: bool, optional + To force to use the method which read the Elog. Default to False. To use + the method with DIRAC API. + username: _type_, optional + username for Elog login. Defaults to None. + password: _type_, optional + password for Elog login. Defaults to None. + + Returns + ------- + __get_GRID_location_ELog or __get_GRID_location_DIRAC - Returns: - _type_: _description_ """ if fromElog: return __class__.__get_GRID_location_ELog( diff --git a/src/nectarchain/makers/component/PedestalComponent.py b/src/nectarchain/makers/component/PedestalComponent.py index d01bad7e..7489342d 100644 --- a/src/nectarchain/makers/component/PedestalComponent.py +++ b/src/nectarchain/makers/component/PedestalComponent.py @@ -4,10 +4,10 @@ import numpy as np import numpy.ma as ma from ctapipe.containers import EventType +from ctapipe.core.traits import Dict, Float, Integer, Unicode +from ctapipe_io_nectarcam.constants import HIGH_GAIN, LOW_GAIN, N_GAINS from ctapipe_io_nectarcam.containers import NectarCAMDataContainer -from ctapipe_io_nectarcam.constants import N_GAINS, HIGH_GAIN, LOW_GAIN -from ctapipe.core.traits import Integer, Unicode, Float, Dict from ...data.container import NectarCAMPedestalContainer, PedestalFlagBits from ...utils import ComponentUtils from .chargesComponent import ChargesComponent @@ -28,7 +28,8 @@ class PedestalEstimationComponent(NectarCAMComponent): Component that computes calibration pedestal coefficients from raw data. Waveforms can be filtered based on time, standard deviation of the waveforms or charge distribution within the sample. - Use the `events_per_slice' parameter of `NectarCAMComponent' to reduce memory load. + Use the ``events_per_slice`` parameter of ``NectarCAMComponent`` to reduce + memory load. Parameters ---------- @@ -47,8 +48,8 @@ class PedestalEstimationComponent(NectarCAMComponent): Threshold in charge distribution (number of sigmas above mean) beyond which a waveform is excluded from pedestal computation. charge_sigma_low_thr : float - Threshold in charge distribution (number of sigmas below mean) beyond which a waveform - is excluded from pedestal computation. + Threshold in charge distribution (number of sigmas below mean) beyond which a + waveform is excluded from pedestal computation. pixel_mask_nevents_min : int Minimum number of events below which the pixel is flagged as bad pixel_mask_mean_min : float @@ -56,9 +57,12 @@ class PedestalEstimationComponent(NectarCAMComponent): pixel_mask_mean_max : float Maximum value of pedestal mean above which the pixel is flagged as bad pixel_mask_std_sample_min : float - Minimum value of pedestal standard deviation for all samples below which the pixel is flagged as bad + Minimum value of pedestal standard deviation for all samples below which the + pixel is flagged as bad pixel_mask_std_pixel_max : float - Maximum value of pedestal standard deviation in a pixel above which the pixel is flagged as bad + Maximum value of pedestal standard deviation in a pixel above which the pixel is + flagged as bad + """ ucts_tmin = Integer( @@ -75,9 +79,9 @@ class PedestalEstimationComponent(NectarCAMComponent): filter_method = Unicode( None, - help="""The waveforms filter method to be used. -Implemented methods: WaveformsStdFilter (standard deviation of waveforms), - ChargeDistributionFilter (charge distribution).""", + help="The waveforms filter method to be used. Implemented methods: " + "WaveformsStdFilter (standard deviation of waveforms), " + "ChargeDistributionFilter (charge distribution).", read_only=False, allow_none=True, ).tag(config=True) @@ -117,14 +121,18 @@ class PedestalEstimationComponent(NectarCAMComponent): pixel_mask_std_sample_min = Float( 0.5, - # for run 3938 in dark room typical standard deviations in working pixel HG are 2.7, LT 2024 July 3 - help="Minimum value of pedestal standard deviation for all samples below which the pixel is flagged as bad", + # for run 3938 in dark room typical standard deviations in working pixel HG + # are 2.7, LT 2024 July 3 + help="Minimum value of pedestal standard deviation for all samples below " + "which the pixel is flagged as bad", ).tag(config=True) pixel_mask_std_pixel_max = Float( 4, - # for run 3938 in dark room typical standard deviations in working pixel HG are 0.25, LT 2024 July 3 - help="Maximum value of pedestal standard deviation in a pixel above which the pixel is flagged as bad", + # for run 3938 in dark room typical standard deviations in working pixel HG + # are 0.25, LT 2024 July 3 + help="Maximum value of pedestal standard deviation in a pixel above which the " + "pixel is flagged as bad", ).tag(config=True) # I do not understand why but the ChargesComponents traits are not loaded @@ -148,7 +156,7 @@ def __init__(self, subarray, config=None, parent=None, *args, **kwargs): Component that computes calibration pedestal coefficients from raw data. Waveforms can be filtered based on time, standard deviation of the waveforms or charge distribution within the sample. - Use the `events_per_slice' parameter of `NectarCAMComponent' to + Use the ``events_per_slice`` parameter of ``NectarCAMComponent`` to reduce memory load. Parameters @@ -248,10 +256,11 @@ def flag_bad_pixels(self, ped_stats, nevents): Parameters ---------- ped_stats : `dict` - A dictionary containing 3D (n_chan,n_pixels,n_samples) arrays for each statistic + A dictionary containing 3D (n_chan,n_pixels,n_samples) arrays for each + statistic nevents : `np.ndarray` - An array that contains the numbber of events used to calculate the statistics - for each pixel + An array that contains the numbber of events used to calculate the + statistics for each pixel Returns ------- @@ -265,7 +274,8 @@ def flag_bad_pixels(self, ped_stats, nevents): # Flag on number of events log.info( - f"Flag pixels with number of events below the acceptable minimum value {self.pixel_mask_nevents_min}" + f"Flag pixels with number of events below the acceptable minimum value " + f"{self.pixel_mask_nevents_min}" ) flag_nevents = np.int8(nevents < self.pixel_mask_nevents_min) # Bitwise OR @@ -274,7 +284,8 @@ def flag_bad_pixels(self, ped_stats, nevents): # Flag on mean pedestal value # Average over all samples for each channel/pixel log.info( - f"Flag pixels with mean pedestal outside acceptable range {self.pixel_mask_mean_min}-{self.pixel_mask_mean_max}" + f"Flag pixels with mean pedestal outside acceptable range " + f"{self.pixel_mask_mean_min}-{self.pixel_mask_mean_max}" ) ped_mean = np.mean(ped_stats["mean"], axis=2) # Apply thresholds @@ -289,7 +300,9 @@ def flag_bad_pixels(self, ped_stats, nevents): # Flag on standard deviation per sample # all samples in channel/pixel below threshold log.info( - f"Flag pixels with pedestal standard deviation for all samples in channel/pixel below the minimum acceptable value {self.pixel_mask_std_sample_min}" + f"Flag pixels with pedestal standard deviation for all samples in " + f"channel/pixel below the minimum acceptable value " + f"{self.pixel_mask_std_sample_min}" ) flag_sample_std = np.int8( np.all(ped_stats["std"] < self.pixel_mask_std_sample_min, axis=2) @@ -300,7 +313,8 @@ def flag_bad_pixels(self, ped_stats, nevents): # Flag on standard deviation per pixel # Standard deviation of pedestal in channel/pixel above threshold log.info( - f"Flag pixels with pedestal standard deviation in a chennel/pixel above the maximum acceptable value {self.pixel_mask_std_pixel_max}" + f"Flag pixels with pedestal standard deviation in a chennel/pixel above " + f"the maximum acceptable value {self.pixel_mask_std_pixel_max}" ) flag_pixel_std = np.int8( np.std(ped_stats["mean"], axis=2) > self.pixel_mask_std_pixel_max diff --git a/src/nectarchain/makers/component/chargesComponent.py b/src/nectarchain/makers/component/chargesComponent.py index c699a789..0b997499 100644 --- a/src/nectarchain/makers/component/chargesComponent.py +++ b/src/nectarchain/makers/component/chargesComponent.py @@ -1,10 +1,5 @@ -import logging - -logging.basicConfig(format="%(asctime)s %(name)s %(levelname)s %(message)s") -log = logging.getLogger(__name__) -log.handlers = logging.getLogger("__main__").handlers - import copy +import logging import time from argparse import ArgumentError @@ -12,15 +7,15 @@ import numpy.ma as ma from ctapipe.containers import EventType from ctapipe.core.traits import Dict, Unicode -from ctapipe.image.extractor import ( +from ctapipe.image.extractor import FixedWindowSum # noqa: F401 +from ctapipe.image.extractor import FullWaveformSum # noqa: F401 +from ctapipe.image.extractor import GlobalPeakWindowSum # noqa: F401 +from ctapipe.image.extractor import LocalPeakWindowSum # noqa: F401 +from ctapipe.image.extractor import NeighborPeakWindowSum # noqa: F401 +from ctapipe.image.extractor import SlidingWindowMaxSum # noqa: F401 +from ctapipe.image.extractor import TwoPassWindowSum # noqa: F401 +from ctapipe.image.extractor import ( # noqa: F401 BaselineSubtractedNeighborPeakWindowSum, - FixedWindowSum, - FullWaveformSum, - GlobalPeakWindowSum, - LocalPeakWindowSum, - NeighborPeakWindowSum, - SlidingWindowMaxSum, - TwoPassWindowSum, ) from ctapipe.instrument import SubarrayDescription from ctapipe_io_nectarcam import constants @@ -36,6 +31,10 @@ from ..extractor.utils import CtapipeExtractor from .core import ArrayDataComponent +logging.basicConfig(format="%(asctime)s %(name)s %(levelname)s %(message)s") +log = logging.getLogger(__name__) +log.handlers = logging.getLogger("__main__").handlers + __all__ = ["ChargesComponent"] list_ctapipe_charge_extractor = [ @@ -62,14 +61,17 @@ cache=True, ) def make_histo(charge, all_range, mask_broken_pix, _mask, hist_ma_data): - """compute histogram of charge with numba + """ + Compute histogram of charge with numba - Args: + Parameters + ---------- charge (np.ndarray(pixels,nevents)): charge all_range (np.ndarray(nbins)): charge range mask_broken_pix (np.ndarray(pixels)): mask on broxen pixels _mask (np.ndarray(pixels,nbins)): mask hist_ma_data (np.ndarray(pixels,nbins)): histogram + """ # print(f"charge.shape = {charge.shape[0]}") # print(f"_mask.shape = {_mask.shape[0]}") @@ -139,12 +141,15 @@ def _init_trigger_type(self, trigger_type: EventType, **kwargs): """ Initializes the ChargesMaker based on the trigger type. - Args: + Parameters + ---------- trigger_type (EventType): The type of trigger. - **kwargs: Additional keyword arguments. + kwargs: Additional keyword arguments. - Returns: + Returns + ------- None + """ super()._init_trigger_type(trigger_type, **kwargs) name = __class__._get_name_trigger(trigger_type) @@ -208,26 +213,38 @@ def _get_extractor_kwargs_from_method_and_kwargs(method: str, kwargs: dict): @staticmethod def _get_imageExtractor(method: str, subarray: SubarrayDescription, **kwargs): """ - Create an instance of a charge extraction method based on the provided method name and subarray description. - Args: - method (str): The name of the charge extraction method. - subarray (SubarrayDescription): The description of the subarray. - **kwargs (dict): Additional keyword arguments for the charge extraction method. - Returns: - imageExtractor: An instance of the charge extraction method specified by `method` with the provided subarray description and keyword arguments. + Create an instance of a charge extraction method based on the provided method + name and subarray description. + + Parameters + ---------- + method : str + The name of the charge extraction method. + subarray : SubarrayDescription + The description of the subarray. + ``**kwargs`` : dict + Additional keyword arguments for the charge extraction method. + + Returns + ------- + imageExtractor: + An instance of the charge extraction method specified by `method` with the + provided subarray description and keyword arguments. """ if not ( method in list_ctapipe_charge_extractor or method in list_nectarchain_charge_extractor ): raise ArgumentError( - f"method must be in {list_ctapipe_charge_extractor} or {list_nectarchain_charge_extractor}" + f"method must be in {list_ctapipe_charge_extractor} or " + f"{list_nectarchain_charge_extractor}" ) extractor_kwargs = __class__._get_extractor_kwargs_from_method_and_kwargs( method=method, kwargs=kwargs ) log.debug( - f"Extracting charges with method {method} and extractor_kwargs {extractor_kwargs}" + f"Extracting charges with method {method} and extractor_kwargs " + f"{extractor_kwargs}" ) imageExtractor = eval(method)(subarray, **extractor_kwargs) return imageExtractor @@ -236,13 +253,17 @@ def finish(self, *args, **kwargs): """ Create an output container for the specified trigger type and method. - Args: + Parameters + ---------- trigger_type (EventType): The type of trigger. method (str): The name of the charge extraction method. - *args: Additional positional arguments. - **kwargs: Additional keyword arguments. - Returns: + args: Additional positional arguments. + kwargs: Additional keyword arguments. + + Returns + ------- list: A list of ChargesContainer objects. + """ output = ChargesContainers() for i, trigger in enumerate(self.trigger_list): @@ -278,14 +299,23 @@ def sort(chargesContainer: ChargesContainer, method: str = "event_id"): """ Sorts the charges in a ChargesContainer object based on the specified method. - Args: - chargesContainer (ChargesContainer): The ChargesContainer object to be sorted. - method (str, optional): The sorting method. Defaults to 'event_id'. - Returns: - ChargesContainer: A new ChargesContainer object with the charges sorted based on the specified method. - - Raises: - ArgumentError: If the specified method is not valid. + Parameters + ---------- + chargesContainer : ChargesContainer + The ChargesContainer object to be sorted. + method : str, optional + The sorting method. Defaults to 'event_id'. + + Returns + ------- + ChargesContainer: + A new ChargesContainer object with the charges sorted based on the specified + method. + + Raises + ------ + ArgumentError: + If the specified method is not valid. """ output = ChargesContainer( run_number=chargesContainer.run_number, @@ -317,13 +347,21 @@ def sort(chargesContainer: ChargesContainer, method: str = "event_id"): @staticmethod def select_charges_hg(chargesContainer: ChargesContainer, pixel_id: np.ndarray): """ - Selects the charges from the ChargesContainer object for the given pixel_id and returns the result transposed. - - Args: - chargesContainer (ChargesContainer): The ChargesContainer object. - pixel_id (np.ndarray): An array of pixel IDs. - Returns: - np.ndarray: The selected charges from the ChargesContainer object for the given pixel_id, transposed. + Selects the charges from the ChargesContainer object for the given pixel_id and + returns the result transposed. + + Parameters + ---------- + chargesContainer : ChargesContainer + The ChargesContainer object. + pixel_id : np.ndarray + An array of pixel IDs. + + Returns + ------- + res : np.ndarray + The selected charges from the ChargesContainer object for the given + ``pixel_id``, transposed. """ res = __class__.select_container_array_field( container=chargesContainer, pixel_id=pixel_id, field="charges_hg" @@ -334,13 +372,21 @@ def select_charges_hg(chargesContainer: ChargesContainer, pixel_id: np.ndarray): @staticmethod def select_charges_lg(chargesContainer: ChargesContainer, pixel_id: np.ndarray): """ - Selects the charges from the ChargesContainer object for the given pixel_id and returns the result transposed. - - Args: - chargesContainer (ChargesContainer): The ChargesContainer object. - pixel_id (np.ndarray): An array of pixel IDs. - Returns: - np.ndarray: The selected charges from the ChargesContainer object for the given pixel_id, transposed. + Selects the charges from the ChargesContainer object for the given pixel_id and + returns the result transposed. + + Parameters + ---------- + chargesContainer : ChargesContainer + The ChargesContainer object. + pixel_id : np.ndarray + An array of pixel IDs. + + Returns + ------- + res : np.ndarray + The selected charges from the ChargesContainer object for the given + ``pixel_id``, transposed. """ res = __class__.select_container_array_field( container=chargesContainer, pixel_id=pixel_id, field="charges_lg" @@ -350,12 +396,18 @@ def select_charges_lg(chargesContainer: ChargesContainer, pixel_id: np.ndarray): def charges_hg(self, trigger: EventType): """ - Returns the charges for a specific trigger type as a NumPy array of unsigned 16-bit integers. - - Args: - trigger (EventType): The specific trigger type. - Returns: - np.ndarray: The charges for the specific trigger type. + Returns the charges for a specific trigger type as a NumPy array of unsigned + 16-bit integers. + + Parameters + ---------- + trigger : EventType + The specific trigger type. + + Returns + ------- + : np.ndarray + The charges for the specific trigger type. """ return np.array( self.__charges_hg[__class__._get_name_trigger(trigger)], @@ -364,12 +416,18 @@ def charges_hg(self, trigger: EventType): def charges_lg(self, trigger: EventType): """ - Returns the charges for a specific trigger type as a NumPy array of unsigned 16-bit integers. - - Args: - trigger (EventType): The specific trigger type. - Returns: - np.ndarray: The charges for the specific trigger type. + Returns the charges for a specific trigger type as a NumPy array of unsigned + 16-bit integers. + + Parameters + ---------- + trigger : EventType + The specific trigger type. + + Returns + ------- + : np.ndarray + The charges for the specific trigger type. """ return np.array( self.__charges_lg[__class__._get_name_trigger(trigger)], @@ -378,12 +436,18 @@ def charges_lg(self, trigger: EventType): def peak_hg(self, trigger: EventType): """ - Returns the peak charges for a specific trigger type as a NumPy array of unsigned 16-bit integers. - - Args: - trigger (EventType): The specific trigger type. - Returns: - np.ndarray: The peak charges for the specific trigger type. + Returns the peak charges for a specific trigger type as a NumPy array of + unsigned 16-bit integers. + + Parameters + ---------- + trigger : EventType + The specific trigger type. + + Returns + ------- + : np.ndarray + The peak charges for the specific trigger type. """ return np.array( self.__peak_hg[__class__._get_name_trigger(trigger)], @@ -392,12 +456,18 @@ def peak_hg(self, trigger: EventType): def peak_lg(self, trigger: EventType): """ - Returns the peak charges for a specific trigger type as a NumPy array of unsigned 16-bit integers. - - Args: - trigger (EventType): The specific trigger type. - Returns: - np.ndarray: The peak charges for the specific trigger type. + Returns the peak charges for a specific trigger type as a NumPy array of + unsigned 16-bit integers. + + Parameters + ---------- + trigger : EventType + The specific trigger type. + + Returns + ------- + : np.ndarray + The peak charges for the specific trigger type. """ return np.array( self.__peak_lg[__class__._get_name_trigger(trigger)], @@ -423,14 +493,22 @@ def create_from_waveforms( **kwargs, ) -> ChargesContainer: """ - Create a ChargesContainer object from waveforms using the specified charge extraction method. - - Args: - waveformsContainer (WaveformsContainer): The waveforms container object. - method (str, optional): The charge extraction method to use (default is "FullWaveformSum"). - **kwargs: Additional keyword arguments to pass to the charge extraction method. - Returns: - ChargesContainer: The charges container object containing the computed charges and peak times. + Create a ChargesContainer object from waveforms using the specified charge + extraction method. + + Parameters + ---------- + waveformsContainer : WaveformsContainer + The waveforms container object. + method : str, optional + The charge extraction method to use (default is ``FullWaveformSum``). + kwargs + Additional keyword arguments to pass to the charge extraction method. + + Returns + ------- + chargesContainer : ChargesContainer + The charges container object containing the computed charges and peak times. """ chargesContainer = ChargesContainer() for field in waveformsContainer.keys(): @@ -471,18 +549,32 @@ def compute_charges( """ Compute charge from waveforms. - Args: - waveformContainer (WaveformsContainer): The waveforms container object. - channel (int): The channel to compute charges for. - method (str, optional): The charge extraction method to use (default is "FullWaveformSum"). - **kwargs: Additional keyword arguments to pass to the charge extraction method. - Raises: - ArgumentError: If the extraction method is unknown. - ArgumentError: If the channel is unknown. - Returns: - tuple: A tuple containing the computed charges and peak times. + Parameters + ---------- + waveformContainer : WaveformsContainer + The waveforms container object. + channel : int + The channel to compute charges for. + method : str, optional + The charge extraction method to use (default is ``FullWaveformSum``). + kwargs + Additional keyword arguments to pass to the charge extraction method. + + Raises + ------ + ArgumentError + If the extraction method is unknown. + ArgumentError + If the channel is unknown. + + Returns + ------- + : tuple + A tuple containing the computed charges and peak times. """ - # import is here for fix issue with pytest (TypeError : inference is not possible with python <3.9 (Numba conflict bc there is no inference...)) + # import is here for fix issue with pytest + # (TypeError : inference is not possible with python <3.9 (Numba conflict bc + # there is no inference...)) from ..extractor.utils import CtapipeExtractor if tel_id is None: @@ -537,13 +629,22 @@ def histo_hg( """ Computes histogram of high gain charges from a ChargesContainer object. - Args: - chargesContainer (ChargesContainer): A ChargesContainer object that holds information about charges from a specific run. - n_bins (int, optional): The number of bins in the charge histogram. Defaults to 1000. - autoscale (bool, optional): Whether to automatically detect the number of bins based on the pixel data. Defaults to True. - - Returns: - ma.masked_array: A masked array representing the charge histogram, where each row corresponds to an event and each column corresponds to a bin in the histogram. + Parameters + ---------- + chargesContainer : ChargesContainer + A ChargesContainer object that holds information about charges from a + specific run. + n_bins : int, optional + The number of bins in the charge histogram. Defaults to 1000. + autoscale : bool, optional + Whether to automatically detect the number of bins based on the pixel data. + Defaults to True. + + Returns + ------- + : ma.masked_array + A masked array representing the charge histogram, where each row corresponds + to an event and each column corresponds to a bin in the histogram. """ return __class__._histo( chargesContainer=chargesContainer, @@ -559,13 +660,22 @@ def histo_lg( """ Computes histogram of low gain charges from a ChargesContainer object. - Args: - chargesContainer (ChargesContainer): A ChargesContainer object that holds information about charges from a specific run. - n_bins (int, optional): The number of bins in the charge histogram. Defaults to 1000. - autoscale (bool, optional): Whether to automatically detect the number of bins based on the pixel data. Defaults to True. - - Returns: - ma.masked_array: A masked array representing the charge histogram, where each row corresponds to an event and each column corresponds to a bin in the histogram. + Parameters + ---------- + chargesContainer : ChargesContainer + A ChargesContainer object that holds information about charges from a + specific run. + n_bins : int, optional + The number of bins in the charge histogram. Defaults to 1000. + autoscale : bool, optional + Whether to automatically detect the number of bins based on the pixel + data. Defaults to True. + + Returns + ------- + : ma.masked_array + A masked array representing the charge histogram, where each row corresponds + to an event and each column corresponds to a bin in the histogram. """ return __class__._histo( chargesContainer=chargesContainer, @@ -585,14 +695,25 @@ def _histo( Computes histogram of charges for a given field from a ChargesContainer object. Numba is used to compute histograms in a vectorized way. - Args: - chargesContainer (ChargesContainer): A ChargesContainer object that holds information about charges from a specific run. - field (str): The field name for which the histogram is computed. - n_bins (int, optional): The number of bins in the charge histogram. Defaults to 1000. - autoscale (bool, optional): Whether to automatically detect the number of bins based on the pixel data. Defaults to True. - - Returns: - ma.masked_array: A masked array representing the charge histogram, where each row corresponds to an event and each column corresponds to a bin in the histogram. + Parameters + ---------- + chargesContainer : ChargesContainer + A ChargesContainer object that holds information about charges from a + specific run. + field : str + The field name for which the histogram is computed. + n_bins : int, optional + The number of bins in the charge histogram. Defaults to 1000. + autoscale : bool, optional + Whether to automatically detect the number of bins based on the pixel data. + Defaults to True. + + Returns + ------- + : ma.masked_array + A masked array representing the charge histogram, where each row + corresponds to an event and each column corresponds to a bin in the + histogram. """ mask_broken_pix = np.array( (chargesContainer[field] == chargesContainer[field].mean(axis=0)).mean( @@ -601,7 +722,8 @@ def _histo( dtype=bool, ) log.debug( - f"there are {mask_broken_pix.sum()} broken pixels (charge stays at same level for each events)" + f"there are {mask_broken_pix.sum()} broken pixels (charge stays at same " + f"level for each events)" ) if autoscale: diff --git a/src/nectarchain/makers/component/spe/spe_algorithm.py b/src/nectarchain/makers/component/spe/spe_algorithm.py index c1c6607e..0be6345f 100644 --- a/src/nectarchain/makers/component/spe/spe_algorithm.py +++ b/src/nectarchain/makers/component/spe/spe_algorithm.py @@ -1,11 +1,5 @@ -import logging -import sys - -logging.basicConfig(format="%(asctime)s %(name)s %(levelname)s %(message)s") -log = logging.getLogger(__name__) -log.handlers = logging.getLogger("__main__").handlers - import copy +import logging import multiprocessing as mp import os import time @@ -17,9 +11,6 @@ import matplotlib import matplotlib.pyplot as plt import matplotlib.style as mplstyle - -mplstyle.use("fast") - import numpy as np import yaml from astropy.table import QTable @@ -37,6 +28,12 @@ from ..chargesComponent import ChargesComponent from .parameters import Parameter, Parameters +mplstyle.use("fast") + +logging.basicConfig(format="%(asctime)s %(name)s %(levelname)s %(message)s") +log = logging.getLogger(__name__) +log.handlers = logging.getLogger("__main__").handlers + __all__ = [ "SPEHHValgorithm", "SPEHHVStdalgorithm", @@ -172,14 +169,16 @@ def npixels(self): # methods def read_param_from_yaml(self, parameters_file, only_update=False) -> None: """ - Reads parameters from a YAML file and updates the internal parameters of the FlatFieldSPEMaker class. - - Args: - parameters_file (str): The name of the YAML file containing the parameters. - only_update (bool, optional): If True, only the parameters that exist in the YAML file will be updated. Default is False. - - Returns: - None + Reads parameters from a YAML file and updates the internal parameters of the + FlatFieldSPEMaker class. + + Parameters + ---------- + parameters_file : str + The name of the YAML file containing the parameters. + only_update : bool, optional + If True, only the parameters that exist in the YAML file will be updated. + Default is False. """ with open( f"{os.path.dirname(os.path.abspath(__file__))}/{parameters_file}" @@ -212,16 +211,26 @@ def _update_parameters( parameters: Parameters, charge: np.ndarray, counts: np.ndarray, **kwargs ) -> Parameters: """ - Update the parameters of the FlatFieldSPEMaker class based on the input charge and counts data. - - Args: - parameters (Parameters): An instance of the Parameters class that holds the internal parameters of the FlatFieldSPEMaker class. - charge (np.ndarray): An array of charge values. - counts (np.ndarray): An array of corresponding counts values. - **kwargs: Additional keyword arguments. - - Returns: - Parameters: The updated parameters object with the pedestal and mean values and their corresponding limits. + Update the parameters of the FlatFieldSPEMaker class based on the input charge + and counts data. + + Parameters + ---------- + parameters (Parameters): + An instance of the Parameters class that holds the internal parameters of + the FlatFieldSPEMaker class. + charge (np.ndarray): + An array of charge values. + counts (np.ndarray): + An array of corresponding counts values. + kwargs + Additional keyword arguments. + + Returns + ------- + parameters : Parameters + The updated parameters object with the pedestal and mean values and their + corresponding limits. """ try: coeff_ped, coeff_mean = __class__._get_mean_gaussian_fit( @@ -263,16 +272,24 @@ def _get_mean_gaussian_fit( """ Perform a Gaussian fit on the data to determine the pedestal and mean values. - Args: - charge (np.ndarray): An array of charge values. - counts (np.ndarray): An array of corresponding counts. - pixel_id (int): The id of the current pixel. Default to None - **kwargs: Additional keyword arguments. + Parameters + ---------- + charge : np.ndarray + An array of charge values. + counts : np.ndarray + An array of corresponding counts. + pixel_id : int + The id of the current pixel. Default to None + kwargs + Additional keyword arguments. - Returns: - Tuple[np.ndarray, np.ndarray]: A tuple of fit coefficients for the pedestal and mean. + Returns + ------- + Tuple[np.ndarray, np.ndarray] + A tuple of fit coefficients for the pedestal and mean. + + Example usage:: - Example Usage: flat_field_maker = FlatFieldSPEMaker() charge = np.array([1, 2, 3, 4, 5]) counts = np.array([10, 20, 30, 40, 50]) @@ -301,7 +318,8 @@ def _get_mean_gaussian_fit( ax.plot( charge, histo_smoothed, - label=f"smoothed data with savgol filter (windows lenght : {windows_lenght}, order : {order})", + label=f"smoothed data with savgol filter (windows lenght : " + f"{windows_lenght}, order : {order})", ) ax.plot( charge, @@ -328,14 +346,17 @@ def _get_mean_gaussian_fit( ax.set_ylabel("Events", size=15) ax.legend(fontsize=7) os.makedirs( - f"{os.environ.get('NECTARCHAIN_LOG', '/tmp')}/{os.getpid()}/figures/", + f"{os.environ.get('NECTARCHAIN_LOG', '/tmp')}/{os.getpid()}/" + f"figures/", exist_ok=True, ) log.info( - f'figures of initialization of parameters will be accessible at {os.environ.get("NECTARCHAIN_LOG","/tmp")}/{os.getpid()}' + f"figures of initialization of parameters will be accessible at " + f'{os.environ.get("NECTARCHAIN_LOG","/tmp")}/{os.getpid()}' ) fig.savefig( - f"{os.environ.get('NECTARCHAIN_LOG','/tmp')}/{os.getpid()}/figures/initialization_pedestal_pixel{pixel_id}_{os.getpid()}.pdf" + f"{os.environ.get('NECTARCHAIN_LOG','/tmp')}/{os.getpid()}/figures/" + f"initialization_pedestal_pixel{pixel_id}_{os.getpid()}.pdf" ) fig.clf() plt.close(fig) @@ -365,7 +386,8 @@ def _get_mean_gaussian_fit( ax.plot( charge, histo_smoothed, - label=f"smoothed data with savgol filter (windows lenght : {windows_lenght}, order : {order})", + label=f"smoothed data with savgol filter (windows length : " + f"{windows_lenght}, order : {order})", ) ax.plot( charge, @@ -392,11 +414,13 @@ def _get_mean_gaussian_fit( ax.set_ylabel("Events", size=15) ax.legend(fontsize=7) os.makedirs( - f"{os.environ.get('NECTARCHAIN_LOG','/tmp')}/{os.getpid()}/figures/", + f"{os.environ.get('NECTARCHAIN_LOG','/tmp')}/{os.getpid()}/" + f"figures/", exist_ok=True, ) fig.savefig( - f"{os.environ.get('NECTARCHAIN_LOG','/tmp')}/{os.getpid()}/figures/initialization_mean_pixel{pixel_id}_{os.getpid()}.pdf" + f"{os.environ.get('NECTARCHAIN_LOG','/tmp')}/{os.getpid()}/figures/" + f"initialization_mean_pixel{pixel_id}_{os.getpid()}.pdf" ) fig.clf() plt.close(fig) @@ -443,11 +467,17 @@ def __init__( ) -> None: """ Initializes the FlatFieldSingleHHVSPEMaker object. - Args: - charge (np.ma.masked_array or array-like): The charge data. - counts (np.ma.masked_array or array-like): The counts data. - *args: Additional positional arguments. - **kwargs: Additional keyword arguments. + + Parameters + ---------- + charge : np.ma.masked_array or array-like + The charge data. + counts : np.ma.masked_array or array-like + The counts data. + ``*args`` + Additional positional arguments. + ``**kwargs`` + Additional keyword arguments. """ super().__init__(pixels_id=pixels_id, config=config, parent=parent, **kwargs) if isinstance(charge, np.ma.masked_array): @@ -466,12 +496,20 @@ def create_from_chargesContainer( cls, signal: ChargesContainer, config=None, parent=None, **kwargs ): """ - Creates an instance of FlatFieldSingleHHVSPEMaker using charge and counts data from a ChargesContainer object. - Args: - signal (ChargesContainer): The ChargesContainer object. - **kwargs: Additional keyword arguments. - Returns: - FlatFieldSingleHHVSPEMaker: An instance of FlatFieldSingleHHVSPEMaker. + Creates an instance of FlatFieldSingleHHVSPEMaker using charge and counts data + from a ChargesContainer object. + + Parameters + ---------- + signal : ChargesContainer + The ChargesContainer object. + ``**kwargs`` + Additional keyword arguments. + + Returns + ------- + FlatFieldSingleHHVSPEMaker + An instance of FlatFieldSingleHHVSPEMaker. """ histo = ChargesComponent.histo_hg(signal, autoscale=True) return cls( @@ -487,28 +525,28 @@ def create_from_chargesContainer( @property def charge(self): """ - Returns a deep copy of the __charge attribute. + Returns a deep copy of the ``__charge`` attribute. """ return copy.deepcopy(self.__charge) @property def _charge(self): """ - Returns the __charge attribute. + Returns the ``__charge`` attribute. """ return self.__charge @property def counts(self): """ - Returns a deep copy of the __counts attribute. + Returns a deep copy of the ``__counts`` attribute. """ return copy.deepcopy(self.__counts) @property def _counts(self): """ - Returns the __counts attribute. + Returns the ``__counts`` attribute. """ return self.__counts @@ -517,16 +555,17 @@ def _fill_results_table_from_dict( self, dico: dict, pixels_id: np.ndarray, return_fit_array: bool = True ) -> None: """ - Populates the results table with fit values and errors for each pixel based on the dictionary provided as input. - - Args: - dico (dict): A dictionary containing fit values and errors for each pixel. - pixels_id (np.ndarray): An array of pixel IDs. - - Returns: - None + Populates the results table with fit values and errors for each pixel based on + the dictionary provided as input. + + Parameters + ---------- + dico : dict + A dictionary containing fit values and errors for each pixel. + pixels_id : np.ndarray + An array of pixel IDs. """ - ########NEED TO BE OPTIMIZED!!!########### + # ######### NEED TO BE OPTIMIZED!!! ########### chi2_sig = signature(_chi2) if return_fit_array: fit_array = np.empty(len(pixels_id), dtype=np.object_) @@ -540,7 +579,8 @@ def _fill_results_table_from_dict( index = np.argmax(self._results.pixels_id == pixels_id[i]) if len(values) != len(chi2_sig.parameters): e = Exception( - "the size out the minuit output parameters values array does not fit the signature of the minimized cost function" + "the size out the minuit output parameters values array does " + "not fit the signature of the minimized cost function" ) log.error(e, exc_info=True) raise e @@ -560,7 +600,8 @@ def _fill_results_table_from_dict( ) if fit_status["has_reached_call_limit"]: self.log.warning( - f"The minuit fit for pixel {pixels_id[i]} reached the call limit" + f"The minuit fit for pixel {pixels_id[i]} reached the call " + f"limit" ) self._results.likelihood[index] = fit_status["values"] ndof = ( @@ -588,19 +629,37 @@ def _NG_Likelihood_Chi2( ): """ Calculates the chi-square value using the MPE2 function. - Parameters: - pp (float): The pp parameter. - res (float): The res parameter. - mu2 (float): The mu2 parameter. - n (float): The n parameter. - muped (float): The muped parameter. - sigped (float): The sigped parameter. - lum (float): The lum parameter. - charge (np.ndarray): An array of charge values. - counts (np.ndarray): An array of count values. - **kwargs: Additional keyword arguments. - Returns: - float: The chi-square value. + The different parameters are explained in `Caroff et al. (2019) `_. + + .. _CAROFF: https://ui.adsabs.harvard.edu/abs/2019SPIE11119E..1WC + + Parameters + ---------- + pp : float + The pp parameter. + res : float + The res parameter. + mu2 : float + The mu2 parameter. + n : float + The n parameter. + muped : float + The muped parameter. + sigped : float + The sigped parameter. + lum : float + The lum parameter. + charge : np.ndarray + An array of charge values. + counts : np.ndarray + An array of count values. + ``**kwargs`` + Additional keyword arguments. + + Returns + ------- + Lik : float + The chi-square value. """ if not (kwargs.get("ntotalPE", False)): for i in range(1000): @@ -623,13 +682,18 @@ def _make_minuitParameters_array_from_parameters( self, pixels_id: np.ndarray = None, **kwargs ) -> np.ndarray: """ - Create an array of Minuit fit instances based on the parameters and data for each pixel. - - Args: - pixels_id (optional): An array of pixel IDs. If not provided, all pixels will be used. - - Returns: - np.ndarray: An array of Minuit fit instances, one for each pixel. + Create an array of Minuit fit instances based on the parameters and data for + each pixel. + + Parameters + ---------- + pixels_id : optional + An array of pixel IDs. If not provided, all pixels will be used. + + Returns + ------- + minuitParameters_array : np.ndarray: + An array of Minuit fit instances, one for each pixel. """ if pixels_id is None: npix = self.npixels @@ -665,12 +729,17 @@ def run_fit(i: int, tol: float) -> dict: """ Perform a fit on a specific pixel using the Minuit package. - Args: - i (int): The index of the pixel to perform the fit on. - - Returns: - dict: A dictionary containing the fit values and errors for the specified pixel. - The keys are "values_i" and "errors_i", where "i" is the index of the pixel. + Parameters + ---------- + i : int + The index of the pixel to perform the fit on. + + Returns + ------- + : dict + A dictionary containing the fit values and errors for the specified pixel. + The keys are ``values_i`` and ``errors_i``, where ``i`` is the index of the + pixel. """ log = logging.getLogger(__name__) log.setLevel(logging.INFO) @@ -784,7 +853,8 @@ def run( log.error(e, exc_info=True) raise e self.log.info( - f"total time for multiproc with starmap_async execution is {time.time() - t:.2e} sec" + f"total time for multiproc with starmap_async execution is " + f"{time.time() - t:.2e} sec" ) else: @@ -804,7 +874,8 @@ def run( t = time.time() self.display(pixels_id, **kwargs) log.info( - f"time for plotting {len(pixels_id)} pixels : {time.time() - t:.2e} sec" + f"time for plotting {len(pixels_id)} pixels : " + f"{time.time() - t:.2e} sec" ) return output @@ -824,13 +895,12 @@ def plot_single_pyqtgraph( likelihood: float, ) -> tuple: import pyqtgraph as pg - from pyqtgraph.Qt import QtCore, QtGui - # from pyqtgraph.Qt import QtGui + # from pyqtgraph.Qt import QtCore, QtGui + from pyqtgraph.Qt import QtGui - app = pg.mkQApp(name="minimal") - # - ## Create a window + # app = pg.mkQApp(name="minimal") + # Create a window win = pg.GraphicsLayoutWidget(show=False) win.setWindowTitle(f"SPE fit pixel id : {pixel_id}") @@ -855,10 +925,11 @@ def plot_single_pyqtgraph( pedestalWidth, luminosity, ), - name=f"SPE model fit", + name="SPE model fit", ) legend = pg.TextItem( - f"SPE model fit gain : {gain - gain_error:.2f} < {gain:.2f} < {gain + gain_error:.2f} ADC/pe,\n likelihood : {likelihood:.2f}", + f"SPE model fit gain : {gain - gain_error:.2f} < {gain:.2f} < " + f"{gain + gain_error:.2f} ADC/pe,\n likelihood : {likelihood:.2f}", color=(200, 200, 200), ) legend.setPos(pedestal, np.max(counts) / 2) @@ -896,23 +967,42 @@ def plot_single_matplotlib( ) -> tuple: """ Generate a plot of the data and a model fit for a specific pixel. + The different parameters are explained in `Caroff et al. (2019) `_. + + .. _CAROFF: https://ui.adsabs.harvard.edu/abs/2019SPIE11119E..1WC + + Parameters + ---------- + pixel_id: int + The ID of the pixel for which the plot is generated. + charge: np.ndarray + An array of charge values. + counts: np.ndarray + An array of event counts corresponding to the charge values. + pp: float + The value of the ``pp`` parameter. + resolution: float + The value of the ``resolution`` parameter. + gain: float + The value of the ``gain`` parameter. + gain_error: float + The value of the ``gain_error`` parameter. + n: float + The value of the ``n`` parameter. + pedestal: float + The value of the ``pedestal`` parameter. + pedestalWidth: float + The value of the ``pedestalWidth`` parameter. + luminosity: float + The value of the ``luminosity`` parameter. + likelihood: float + The value of the ``likelihood`` parameter. + + Returns + ------- + : tuple + A tuple containing the generated plot figure and the axes of the plot. - Args: - pixel_id (int): The ID of the pixel for which the plot is generated. - charge (np.ndarray): An array of charge values. - counts (np.ndarray): An array of event counts corresponding to the charge values. - pp (float): The value of the `pp` parameter. - resolution (float): The value of the `resolution` parameter. - gain (float): The value of the `gain` parameter. - gain_error (float): The value of the `gain_error` parameter. - n (float): The value of the `n` parameter. - pedestal (float): The value of the `pedestal` parameter. - pedestalWidth (float): The value of the `pedestalWidth` parameter. - luminosity (float): The value of the `luminosity` parameter. - likelihood (float): The value of the `likelihood` parameter. - - Returns: - tuple: A tuple containing the generated plot figure and the axes of the plot. """ fig, ax = plt.subplots(1, 1, figsize=(8, 8)) ax.errorbar(charge, counts, np.sqrt(counts), zorder=0, fmt=".", label="data") @@ -931,7 +1021,8 @@ def plot_single_matplotlib( ), zorder=1, linewidth=2, - label=f"SPE model fit \n gain : {gain - gain_error:.2f} < {gain:.2f} < {gain + gain_error:.2f} ADC/pe,\n likelihood : {likelihood:.2f}", + label=f"SPE model fit \n gain : {gain - gain_error:.2f} < {gain:.2f} < " + f"{gain + gain_error:.2f} ADC/pe,\n likelihood : {likelihood:.2f}", ) ax.set_xlabel("Charge (ADC)", size=15) ax.set_ylabel("Events", size=15) @@ -944,15 +1035,23 @@ def display(self, pixels_id: np.ndarray, package="pyqtgraph", **kwargs) -> None: """ Display and save the plot for each specified pixel ID. - Args: - pixels_id (np.ndarray): An array of pixel IDs. - package (str): the package use to plot, can be matplotlib or pyqtgraph. Default to pyqtgraph - **kwargs: Additional keyword arguments. - figpath (str): The path to save the generated plot figures. Defaults to "/tmp/NectarGain_pid{os.getpid()}". + Parameters + ---------- + pixels_id: np.ndarray + An array of pixel IDs. + package: str + the package used to plot, can be matplotlib or pyqtgraph. + Default to pyqtgraph + kwargs + Additional keyword arguments. + figpath : str + The path to save the generated plot figures. + Defaults to ``/tmp/NectarGain_pid{os.getpid()}``. """ figpath = kwargs.get( "figpath", - f"{os.environ.get('NECTARCHAIN_FIGURES','/tmp')}/NectarGain_pid{os.getpid()}", + f"{os.environ.get('NECTARCHAIN_FIGURES','/tmp')}/" + f"NectarGain_pid{os.getpid()}", ) self.log.debug(f"saving figures in {figpath}") os.makedirs(figpath, exist_ok=True) @@ -1010,7 +1109,9 @@ def display(self, pixels_id: np.ndarray, package="pyqtgraph", **kwargs) -> None: class SPEHHValgorithm(SPEnominalalgorithm): - """class to perform fit of the SPE HHV signal with n and pp free""" + """ + Class to perform fit of the SPE HHV signal with ``n`` and ``pp`` free. + """ parameters_file = Unicode( "parameters_SPEHHV.yaml", @@ -1025,7 +1126,7 @@ class SPEHHValgorithm(SPEnominalalgorithm): class SPEnominalStdalgorithm(SPEnominalalgorithm): - """class to perform fit of the SPE signal with n and pp fixed""" + """Class to perform fit of the SPE signal with ``n`` and ``pp`` fixed""" parameters_file = Unicode( "parameters_SPEnominalStd.yaml", @@ -1045,11 +1146,16 @@ def __init__( """ Initializes a new instance of the FlatFieldSingleHHVStdSPEMaker class. - Args: - charge (np.ndarray): The charge data. - counts (np.ndarray): The counts data. - *args: Additional positional arguments. - **kwargs: Additional keyword arguments. + Parameters + ---------- + charge : np.ndarray + The charge data. + counts : np.ndarray + The counts data. + ``*args`` + Additional positional arguments. + ``**kwargs`` + Additional keyword arguments. """ super().__init__( pixels_id=pixels_id, @@ -1063,7 +1169,8 @@ def __init__( def __fix_parameters(self) -> None: """ - Fixes the values of the n and pp parameters by setting their frozen attribute to True. + Fixes the values of the ``n`` and ``pp`` parameters by setting their frozen + attribute to True. """ self.log.info("updating parameters by fixing pp and n") pp = self._parameters["pp"] @@ -1099,7 +1206,8 @@ class SPECombinedalgorithm(SPEnominalalgorithm): ).tag(config=True) SPE_result = Path( - help="the path of the SPE result container computed with very high voltage data", + help="the path of the SPE result container computed with " + "very high voltage data", ).tag(config=True) same_luminosity = Bool( @@ -1119,11 +1227,16 @@ def __init__( """ Initializes a new instance of the FlatFieldSingleHHVStdSPEMaker class. - Args: - charge (np.ndarray): The charge data. - counts (np.ndarray): The counts data. - *args: Additional positional arguments. - **kwargs: Additional keyword arguments. + Parameters + ---------- + charge : np.ndarray + The charge data. + counts : np.ndarray + The counts data. + ``*args`` + Additional positional arguments. + ``**kwargs`` + Additional keyword arguments. """ super().__init__( pixels_id=pixels_id, @@ -1150,15 +1263,18 @@ def __init__( == 0 ): self.log.warning( - "The intersection between pixels id from the data and those valid from the SPE fit result is empty" + "The intersection between pixels id from the data and those valid from " + "the SPE fit result is empty" ) def __fix_parameters(self) -> None: """ - Fixes the parameters n, pp, res, and possibly luminosity. + Fixes the parameters ``n``, ``pp``, ``res``, and possibly ``luminosity``. - Args: - same_luminosity (bool): Whether to fix the luminosity parameter. + Parameters + ---------- + same_luminosity : bool + Whether to fix the luminosity parameter. """ self.log.info("updating parameters by fixing pp, n and res") pp = self._parameters["pp"] @@ -1174,14 +1290,20 @@ def __fix_parameters(self) -> None: def _make_fit_array_from_parameters(self, pixels_id=None, **kwargs): """ - Generates the fit array from the fixed parameters and the fitted data obtained from a 1400V run. - - Args: - pixels_id (array-like, optional): The pixels to generate the fit array for. Defaults to None. - **kwargs: Arbitrary keyword arguments. - - Returns: - array-like: The fit array. + Generates the fit array from the fixed parameters and the fitted data obtained + from a 1400V run. + + Parameters + ---------- + pixels_id : array-like, optional + The pixels to generate the fit array for. Defaults to None. + ``**kwargs`` + Arbitrary keyword arguments. + + Returns + ------- + : array-like + The fit array. """ return super()._make_fit_array_from_parameters( pixels_id=pixels_id, @@ -1199,18 +1321,28 @@ def _update_parameters( **kwargs, ): """ - Updates the parameters with the fixed values from the fitted data obtained from a 1400V run. - - Args: - parameters (Parameters): The parameters to update. - charge (np.ndarray): The charge values. - counts (np.ndarray): The counts values. - pixel_id (int): The pixel ID. - nectarGainSPEresult (QTable): The fitted data obtained from a 1400V run. - **kwargs: Arbitrary keyword arguments. - - Returns: - dict: The updated parameters. + Updates the parameters with the fixed values from the fitted data obtained from + a 1400V run. + + Parameters + ---------- + parameters : Parameters + The parameters to update. + charge : np.ndarray + The charge values. + counts : np.ndarray + The counts values. + pixel_id : int + The pixel ID. + nectarGainSPEresult : QTable + The fitted data obtained from a 1400V run. + ``**kwargs`` + Arbitrary keyword arguments. + + Returns + ------- + param : dict + The updated parameters. """ param = super(__class__, __class__)._update_parameters( parameters, charge, counts, **kwargs diff --git a/src/nectarchain/makers/core.py b/src/nectarchain/makers/core.py index 5fdf8de8..e154ad35 100644 --- a/src/nectarchain/makers/core.py +++ b/src/nectarchain/makers/core.py @@ -74,14 +74,19 @@ def load_run( """Static method to load from $NECTARCAMDATA directory data for specified run with max_events. - Args:self.__run_number = run_number - run_number (int): run_id - maxevents (int, optional): max of events to be loaded. Defaults to -1, to - load everything. - run_file (optional) : if provided, will load this run file - Returns: - List[ctapipe_io_nectarcam.LightNectarCAMEventSource]: List of EventSource - for each run files. + Parameters + ---------- + run_number : int + run_id + maxevents : int, optional + max of events to be loaded. Defaults to -1, to load everything. + run_file : optional + if provided, will load this run file + + Returns + ------- + List[ctapipe_io_nectarcam.LightNectarCAMEventSource] + List of EventSource for each run files. """ # Load the data from the run file. if run_file is None: diff --git a/src/nectarchain/utils/stats.py b/src/nectarchain/utils/stats.py index ac02288b..5116d048 100644 --- a/src/nectarchain/utils/stats.py +++ b/src/nectarchain/utils/stats.py @@ -193,12 +193,11 @@ class CameraSampleStats(Stats): Examples -------- - Cumulating the rawdata from a run to get the average waveform + Cumulating the rawdata from a run to get the average waveform:: + >>> from nectarchain.utils.stats import CameraSampleStats >>> from ctapipe_io_nectarcam import NectarCAMEventSource - >>> reader = NectarCAMEventSource(input_url='NectarCAM.Run4560.00??.fits.fz') - >>> s = CameraSampleStats() >>> for event in reader: >>> s.add(event.r0.tel[0].waveform, diff --git a/src/nectarchain/utils/utils.py b/src/nectarchain/utils/utils.py index dd53b25b..13ff02cf 100644 --- a/src/nectarchain/utils/utils.py +++ b/src/nectarchain/utils/utils.py @@ -3,6 +3,7 @@ import math import numpy as np +from ctapipe.core.component import Component from iminuit import Minuit from scipy import interpolate, signal from scipy.special import gammainc @@ -12,15 +13,13 @@ log = logging.getLogger(__name__) log.handlers = logging.getLogger("__main__").handlers -from ctapipe.core.component import Component - class ComponentUtils: @staticmethod def is_in_non_abstract_subclasses( component: Component, motherClass="NectarCAMComponent" ): - from nectarchain.makers.component.core import NectarCAMComponent + from nectarchain.makers.component.core import NectarCAMComponent # noqa: F401 # module = importlib.import_module(f'nectarchain.makers.component.core') is_in = False @@ -38,7 +37,7 @@ def get_specific_traits(component: Component): if ComponentUtils.is_in_non_abstract_subclasses( component, "NectarCAMComponent" ) and not (component.SubComponents.default_value is None): - for component_name in component.SubComponents.default_value: #####CPT + for component_name in component.SubComponents.default_value: # CPT _class = getattr( importlib.import_module("nectarchain.makers.component"), component_name, @@ -66,7 +65,8 @@ def get_class_name_from_ComponentName(componentName: str): return _class raise ValueError( - "componentName is not a valid component, this component is not known as a child of NectarCAMComponent" + "componentName is not a valid component, this component is not known as a " + "child of NectarCAMComponent" ) @@ -106,12 +106,12 @@ def make_minuit_par_kwargs(parameters): @staticmethod def set_minuit_parameters_limits_and_errors(m: Minuit, parameters: dict): - """function to set minuit parameter limits and errors with Minuit >2.0 + """Function to set minuit parameter limits and errors with Minuit >2.0 Args: m (Minuit): a Minuit instance - parameters (dict): dict containing parameters names, limits errors - and values + parameters (dict): dict containing parameters names, limits errors and + values. """ for name in parameters["names"]: m.limits[name] = parameters[f"limit_{name}"] @@ -122,7 +122,8 @@ def set_minuit_parameters_limits_and_errors(m: Minuit, parameters: dict): # Useful functions for the fit def gaussian(x, mu, sig): - # return (1./(sig*np.sqrt(2*math.pi)))*np.exp(-np.power(x - mu, 2.) / (2 * np.power(sig, 2.))) + # return (1./(sig*np.sqrt(2*math.pi))) * + # np.exp(-np.power(x - mu, 2.) / (2 * np.power(sig, 2.))) return norm.pdf(x, loc=mu, scale=sig) @@ -304,12 +305,14 @@ def sigma2(n, p, res, mu2): else: return n * SigMax(p, res, mu2) - # The real final model callign all the above for luminosity (lum) + PED, wil return probability of number of Spe + # The real final model callign all the above for luminosity (lum) + PED, wil return + # probability of number of Spe def MPE2(x, pp, res, mu2, n, muped, sigped, lum, **kwargs): log.debug( - f"pp = {pp}, res = {res}, mu2 = {mu2}, n = {n}, muped = {muped}, sigped = {sigped}, lum = {lum}" + f"pp = {pp}, res = {res}, mu2 = {mu2}, n = {n}, muped = {muped}, " + f"sigped = {sigped}, lum = {lum}" ) f = 0 ntotalPE = kwargs.get("ntotalPE", 0) @@ -322,7 +325,8 @@ def MPE2(x, pp, res, mu2, n, muped, sigped, lum, **kwargs): # print(ntotalPE) # about 8 sec, 1 sec by nPEPDF call # for i in range(ntotalPE): - # f = f + ((lum**i)/math.factorial(i)) * np.exp(-lum) * nPEPDF(x,pp,res,mu2,n,muped,sigped,i,int(mu2*ntotalPE+10*mu2)) + # f = f + ((lum**i)/math.factorial(i)) * np.exp(-lum) * + # nPEPDF(x,pp,res,mu2,n,muped,sigped,i,int(mu2*ntotalPE+10*mu2)) f = np.sum( [