diff --git a/pipelines/cpCore.yaml b/pipelines/cpCore.yaml index 341f82aab..db01927e1 100644 --- a/pipelines/cpCore.yaml +++ b/pipelines/cpCore.yaml @@ -44,28 +44,28 @@ tasks: atools.biasMeanByDate: CalibAmpScatterTool atools.biasMeanByDate.prep.panelKey: amplifier atools.biasMeanByDate.prep.dataKey: mjd - atools.biasMeanByDate.prep.quantityKey: BIAS_MEAN + atools.biasMeanByDate.prep.quantityKey: ["BIAS_MEAN"] atools.biasMeanByDate.produce.plot.xAxisLabel: "MJD" atools.biasMeanByDate.produce.plot.yAxisLabel: "Residual bias mean (ADU)" atools.biasStdByDate: CalibAmpScatterTool atools.biasStdByDate.prep.panelKey: amplifier atools.biasStdByDate.prep.dataKey: mjd - atools.biasStdByDate.prep.quantityKey: BIAS_NOISE + atools.biasStdByDate.prep.quantityKey: ["BIAS_NOISE"] atools.biasStdByDate.produce.plot.xAxisLabel: "MJD" atools.biasStdByDate.produce.plot.yAxisLabel: "Residual bias stdev (ADU)" atools.biasROCornerByDate: CalibAmpScatterTool atools.biasROCornerByDate.prep.panelKey: amplifier atools.biasROCornerByDate.prep.dataKey: mjd - atools.biasROCornerByDate.prep.quantityKey: BIAS_AMP_CORNER + atools.biasROCornerByDate.prep.quantityKey: ["BIAS_AMP_CORNER"] atools.biasROCornerByDate.produce.plot.xAxisLabel: "MJD" atools.biasROCornerByDate.produce.plot.yAxisLabel: "Residual bias mean at RO corner (ADU)" atools.biasTestsByDate: CalibAmpScatterTool atools.biasTestsByDate.prep.panelKey: amplifier atools.biasTestsByDate.prep.dataKey: mjd - atools.biasTestsByDate.prep.quantityKey: BIAS_VERIFY_MEAN + atools.biasTestsByDate.prep.quantityKey: ["BIAS_VERIFY_MEAN"] atools.biasTestsByDate.produce.plot.xAxisLabel: "MJD" atools.biasTestsByDate.produce.plot.yAxisLabel: "Bias Test Passing" @@ -118,21 +118,21 @@ tasks: atools.darkMeanByDate: CalibAmpScatterTool atools.darkMeanByDate.prep.panelKey: amplifier atools.darkMeanByDate.prep.dataKey: mjd - atools.darkMeanByDate.prep.quantityKey: DARK_MEAN + atools.darkMeanByDate.prep.quantityKey: ["DARK_MEAN"] atools.darkMeanByDate.produce.plot.xAxisLabel: "MJD" atools.darkMeanByDate.produce.plot.yAxisLabel: "Residual dark mean (ADU)" atools.darkStdByDate: CalibAmpScatterTool atools.darkStdByDate.prep.panelKey: amplifier atools.darkStdByDate.prep.dataKey: mjd - atools.darkStdByDate.prep.quantityKey: DARK_NOISE + atools.darkStdByDate.prep.quantityKey: ["DARK_NOISE"] atools.darkStdByDate.produce.plot.xAxisLabel: "MJD" atools.darkStdByDate.produce.plot.yAxisLabel: "Residual dark std (ADU)" atools.darkTestsByDate: CalibAmpScatterTool atools.darkTestsByDate.prep.panelKey: amplifier atools.darkTestsByDate.prep.dataKey: mjd - atools.darkTestsByDate.prep.quantityKey: DARK_VERIFY_MEAN + atools.darkTestsByDate.prep.quantityKey: ["DARK_VERIFY_MEAN"] atools.darkTestsByDate.produce.plot.xAxisLabel: "MJD" atools.darkTestsByDate.produce.plot.yAxisLabel: "Dark Test Passing" @@ -148,7 +148,7 @@ tasks: atools.flatTestsByDate: CalibAmpScatterTool atools.flatTestsByDate.prep.panelKey: amplifier atools.flatTestsByDate.prep.dataKey: mjd - atools.flatTestsByDate.prep.quantityKey: FLAT_VERIFY_NOISE + atools.flatTestsByDate.prep.quantityKey: ["FLAT_VERIFY_NOISE"] # TODO: DM-43878 # FLAT_DET_VERIFY_SCATTER atools.flatTestsByDate.produce.plot.xAxisLabel: "MJD" @@ -210,7 +210,7 @@ tasks: atools.ptcPlot: CalibAmpScatterTool atools.ptcPlot.prep.panelKey: amplifier atools.ptcPlot.prep.dataKey: PTC_PTC_RAW_MEANS - atools.ptcPlot.prep.quantityKey: PTC_PTC_RAW_VARIANCE + atools.ptcPlot.prep.quantityKey: ["PTC_PTC_RAW_VARIANCE"] atools.ptcPlot.produce.plot.xAxisLabel: "Exposure Pair Flux (ADU)" atools.ptcPlot.produce.plot.yAxisLabel: "Exposure Pair Variance (ADU^2)" python: | diff --git a/python/lsst/analysis/tools/actions/plot/gridPlot.py b/python/lsst/analysis/tools/actions/plot/gridPlot.py index bf8dfe0da..4809baf93 100644 --- a/python/lsst/analysis/tools/actions/plot/gridPlot.py +++ b/python/lsst/analysis/tools/actions/plot/gridPlot.py @@ -84,10 +84,7 @@ class GridPlot(PlotAction): doc="Independent data definitions. The key of this dict is the panel ID. The values are keys of data " "to plot (comma-separated for multiple) where each key may be a subset of a full key.", ) - figsize = ListField[float]( - doc="Figure size.", - default=[8, 8], - ) + figsize = ListField[float](doc="Figure size.", default=[8, 8], length=2) dpi = Field[float]( doc="Dots per inch.", default=150, diff --git a/python/lsst/analysis/tools/atools/calibQuantityProfile.py b/python/lsst/analysis/tools/atools/calibQuantityProfile.py index 80be9a892..9154b1ab3 100644 --- a/python/lsst/analysis/tools/atools/calibQuantityProfile.py +++ b/python/lsst/analysis/tools/atools/calibQuantityProfile.py @@ -20,8 +20,6 @@ # along with this program. If not, see . from __future__ import annotations -from lsst.analysis.tools.interfaces._interfaces import KeyedDataSchema - __all__ = ( "CalibQuantityBaseTool", "CalibQuantityAmpProfileScatterTool", @@ -31,14 +29,34 @@ "CalibPtcCovarScatterTool", ) -from typing import cast +from typing import Optional -from lsst.pex.config import Field +import numpy as np +from lsst.pex.config import Field, ListField from lsst.pex.config.configurableActions import ConfigurableActionField from ..actions.plot.elements import HistElement, ScatterElement from ..actions.plot.gridPlot import GridPanelConfig, GridPlot -from ..interfaces import AnalysisTool, KeyedData, KeyedDataAction, PlotElement, Vector +from ..interfaces import AnalysisTool, KeyedData, KeyedDataAction, KeyedDataSchema, PlotElement, Vector + +_CALIB_AMP_NAME_DICT: dict[int, str] = { + 0: "C00", + 1: "C01", + 2: "C02", + 3: "C03", + 4: "C04", + 5: "C05", + 6: "C06", + 7: "C07", + 8: "C10", + 9: "C11", + 10: "C12", + 11: "C13", + 12: "C14", + 13: "C15", + 14: "C16", + 15: "C17", +} class PrepRepacker(KeyedDataAction): @@ -50,61 +68,23 @@ class PrepRepacker(KeyedDataAction): dataKey = Field[str]( doc="Data selector. Data will be separated into multiple groups in a single panel based on this key.", ) - quantityKey = Field[str]( - doc="Quantity selector. The actual data quantities to be plotted.", + quantityKey = ListField[str]( + doc="Quantity selector. The actual data quantities to be plotted.", minLength=1, optional=False ) def __call__(self, data: KeyedData, **kwargs) -> KeyedData: - repackedData = {} - # Loop over the length of the data vector and repack row by row - for i in range(len(cast(Vector, data[self.panelKey]))): - panelVec = cast(Vector, data[self.panelKey]) - dataVec = cast(Vector, data[self.dataKey]) - quantityVec = cast(Vector, data[self.quantityKey]) - repackedData[f"{panelVec[i]}_{dataVec[i]}_{self.quantityKey}"] = quantityVec[i] - return repackedData - - def getInputSchema(self) -> KeyedDataSchema: - return ( - (self.panelKey, Vector), - (self.dataKey, Vector), - (self.quantityKey, Vector), - ) - - def addInputSchema(self, inputSchema: KeyedDataSchema) -> None: - pass - - -class SingleValueRepacker(KeyedDataAction): - """Prep action to repack data.""" - - panelKey = Field[str]( - doc="Panel selector. Data will be separated into multiple panels based on this key.", - ) - dataKey = Field[str]( - doc="Data selector. Data will be separated into multiple groups in a single panel based on this key.", - ) - quantityKey = Field[str]( - doc="Quantity selector. The actual data quantities to be plotted.", - ) - - def __call__(self, data: KeyedData, **kwargs) -> KeyedData: - repackedData = {} + repackedData: dict[str, Vector] = {} uniquePanelKeys = list(set(data[self.panelKey])) - # Loop over data vector to repack information as it is expected. - for i in range(len(uniquePanelKeys)): - repackedData[f"{uniquePanelKeys[i]}_x"] = [] - repackedData[f"{uniquePanelKeys[i]}"] = [] - - panelVec = cast(Vector, data[self.panelKey]) - dataVec = cast(Vector, data[self.dataKey]) - quantityVec = cast(Vector, data[self.quantityKey]) - - for i in range(len(panelVec)): - repackedData[f"{panelVec[i]}_x"].append(dataVec[i]) - repackedData[f"{panelVec[i]}"].append(quantityVec[i]) + for pKey in uniquePanelKeys: + # Make a boolean array that selects the correct panel data + sel: np.ndarray = data[self.panelKey] == pKey + # Setup the x axis + repackedData[f"{pKey}_x"] = data[self.dataKey][sel] + for qkey in self.quantityKey: + # Setup a y axis series for each quantityKey + repackedData[f"{pKey}_{qkey}"] = data[qkey][sel] return repackedData def getInputSchema(self) -> KeyedDataSchema: @@ -136,6 +116,12 @@ class CalibQuantityBaseTool(AnalysisTool): doc="Plot element.", ) + def _get_xKey_dict(self, yKeys: Optional[dict[int, str]] = None) -> dict[int, str]: + """Generate the dictionary of x axis keys from the y axis ones""" + if yKeys is None: + yKeys = self.produce.plot.valsGroupBy + return {k: f"{v}_x" for k, v in yKeys.items()} + def setDefaults(self): super().setDefaults() @@ -152,24 +138,7 @@ def setDefaults(self): self.produce.plot.numCols = 4 # Values to group by to distinguish between data in differing panels - self.produce.plot.valsGroupBy = { - 0: "C00", - 1: "C01", - 2: "C02", - 3: "C03", - 4: "C04", - 5: "C05", - 6: "C06", - 7: "C07", - 8: "C10", - 9: "C11", - 10: "C12", - 11: "C13", - 12: "C14", - 13: "C15", - 14: "C16", - 15: "C17", - } + self.produce.plot.valsGroupBy = _CALIB_AMP_NAME_DICT def finalize(self): super().finalize() @@ -206,7 +175,7 @@ def setDefaults(self): self.plotElement = ScatterElement() # Repack the input data into a usable format - self.prep = SingleValueRepacker() + self.prep = PrepRepacker() self.produce.plot = GridPlot() self.produce.plot.panels = {} @@ -214,44 +183,9 @@ def setDefaults(self): self.produce.plot.numCols = 4 # Values to use for x-axis data - self.produce.plot.xDataKeys = { - 0: "C00_x", - 1: "C01_x", - 2: "C02_x", - 3: "C03_x", - 4: "C04_x", - 5: "C05_x", - 6: "C06_x", - 7: "C07_x", - 8: "C10_x", - 9: "C11_x", - 10: "C12_x", - 11: "C13_x", - 12: "C14_x", - 13: "C15_x", - 14: "C16_x", - 15: "C17_x", - } - # Values to group by to distinguish between data in differing panels - self.produce.plot.valsGroupBy = { - 0: "C00", - 1: "C01", - 2: "C02", - 3: "C03", - 4: "C04", - 5: "C05", - 6: "C06", - 7: "C07", - 8: "C10", - 9: "C11", - 10: "C12", - 11: "C13", - 12: "C14", - 13: "C15", - 14: "C16", - 15: "C17", - } + self.produce.plot.valsGroupBy = _CALIB_AMP_NAME_DICT + self.produce.plot.xDataKeys = self._get_xKey_dict() self.prep.panelKey = "amplifier" self.prep.dataKey = "mjd" @@ -277,7 +211,7 @@ def setDefaults(self): self.plotElement = ScatterElement() # Repack the input data into a usable format - self.prep = SingleValueRepacker() + self.prep = PrepRepacker() self.produce.plot = GridPlot() self.produce.plot.panels = {} @@ -287,10 +221,7 @@ def setDefaults(self): # Values to group by to distinguish between data in differing panels self.produce.plot.valsGroupBy = {0: "bank1", 1: "bank0"} - self.produce.plot.xDataKeys = { - 0: "bank1_x", - 1: "bank0_x", - } + self.produce.plot.xDataKeys = self._get_xKey_dict() self.prep.panelKey = "amplifier" self.prep.dataKey = "mjd" diff --git a/python/lsst/analysis/tools/interfaces/_actions.py b/python/lsst/analysis/tools/interfaces/_actions.py index 08ff9d276..ce0e4a5db 100644 --- a/python/lsst/analysis/tools/interfaces/_actions.py +++ b/python/lsst/analysis/tools/interfaces/_actions.py @@ -40,7 +40,7 @@ import warnings from abc import abstractmethod from dataclasses import dataclass -from typing import TYPE_CHECKING, Iterable +from typing import TYPE_CHECKING, Any, Generator, GenericAlias, Iterable, Sequence import lsst.pex.config as pexConfig from lsst.pex.config.configurableActions import ConfigurableAction, ConfigurableActionField @@ -108,7 +108,7 @@ def getOutputSchema(self) -> KeyedDataSchema | None: """ return None - def getFormattedInputSchema(self, **kwargs) -> KeyedDataSchema: + def getFormattedInputSchema(self, **kwargs) -> Generator[tuple[Any, GenericAlias], None, None]: """Return input schema, with keys formatted with any arguments supplied by kwargs passed to this method. @@ -119,7 +119,11 @@ def getFormattedInputSchema(self, **kwargs) -> KeyedDataSchema: action, formatted with any input arguments (e.g. band='i') """ for key, typ in self.getInputSchema(): - yield key.format_map(kwargs), typ + if isinstance(key, Sequence) and not isinstance(key, str): + for subkey in key: + yield subkey.format_map(kwargs), typ + else: + yield key.format_map(kwargs), typ def addInputSchema(self, inputSchema: KeyedDataSchema) -> None: """Add the supplied inputSchema argument to the class such that it will