Support multiple quantities plotting in CalibAmpScatterTool

This commit changes the `quantityKey` field in the `CalibAmpScatterTool` class
to be of type `ListField`. This enables multiple quantities to be plotted on a
single scatter plot. In addition, pipeline YAML files for calibration
verification are re-worded to use the new `ListField` type

pipelines/cpCore.yaml

@@ -44,28 +44,28 @@ tasks:
       atools.biasMeanByDate: CalibAmpScatterTool
       atools.biasMeanByDate.prep.panelKey: amplifier
       atools.biasMeanByDate.prep.dataKey: mjd
-      atools.biasMeanByDate.prep.quantityKey: BIAS_MEAN
+      atools.biasMeanByDate.prep.quantityKey: ["BIAS_MEAN"]
       atools.biasMeanByDate.produce.plot.xAxisLabel: "MJD"
       atools.biasMeanByDate.produce.plot.yAxisLabel: "Residual bias mean (ADU)"
 
       atools.biasStdByDate: CalibAmpScatterTool
       atools.biasStdByDate.prep.panelKey: amplifier
       atools.biasStdByDate.prep.dataKey: mjd
-      atools.biasStdByDate.prep.quantityKey: BIAS_NOISE
+      atools.biasStdByDate.prep.quantityKey: ["BIAS_NOISE"]
       atools.biasStdByDate.produce.plot.xAxisLabel: "MJD"
       atools.biasStdByDate.produce.plot.yAxisLabel: "Residual bias stdev (ADU)"
 
       atools.biasROCornerByDate: CalibAmpScatterTool
       atools.biasROCornerByDate.prep.panelKey: amplifier
       atools.biasROCornerByDate.prep.dataKey: mjd
-      atools.biasROCornerByDate.prep.quantityKey: BIAS_AMP_CORNER
+      atools.biasROCornerByDate.prep.quantityKey: ["BIAS_AMP_CORNER"]
       atools.biasROCornerByDate.produce.plot.xAxisLabel: "MJD"
       atools.biasROCornerByDate.produce.plot.yAxisLabel: "Residual bias mean at RO corner (ADU)"
 
       atools.biasTestsByDate: CalibAmpScatterTool
       atools.biasTestsByDate.prep.panelKey: amplifier
       atools.biasTestsByDate.prep.dataKey: mjd
-      atools.biasTestsByDate.prep.quantityKey: BIAS_VERIFY_MEAN
+      atools.biasTestsByDate.prep.quantityKey: ["BIAS_VERIFY_MEAN"]
       atools.biasTestsByDate.produce.plot.xAxisLabel: "MJD"
       atools.biasTestsByDate.produce.plot.yAxisLabel: "Bias Test Passing"
 
@@ -118,21 +118,21 @@ tasks:
       atools.darkMeanByDate: CalibAmpScatterTool
       atools.darkMeanByDate.prep.panelKey: amplifier
       atools.darkMeanByDate.prep.dataKey: mjd
-      atools.darkMeanByDate.prep.quantityKey: DARK_MEAN
+      atools.darkMeanByDate.prep.quantityKey: ["DARK_MEAN"]
       atools.darkMeanByDate.produce.plot.xAxisLabel: "MJD"
       atools.darkMeanByDate.produce.plot.yAxisLabel: "Residual dark mean (ADU)"
 
       atools.darkStdByDate: CalibAmpScatterTool
       atools.darkStdByDate.prep.panelKey: amplifier
       atools.darkStdByDate.prep.dataKey: mjd
-      atools.darkStdByDate.prep.quantityKey: DARK_NOISE
+      atools.darkStdByDate.prep.quantityKey: ["DARK_NOISE"]
       atools.darkStdByDate.produce.plot.xAxisLabel: "MJD"
       atools.darkStdByDate.produce.plot.yAxisLabel: "Residual dark std (ADU)"
 
       atools.darkTestsByDate: CalibAmpScatterTool
       atools.darkTestsByDate.prep.panelKey: amplifier
       atools.darkTestsByDate.prep.dataKey: mjd
-      atools.darkTestsByDate.prep.quantityKey: DARK_VERIFY_MEAN
+      atools.darkTestsByDate.prep.quantityKey: ["DARK_VERIFY_MEAN"]
       atools.darkTestsByDate.produce.plot.xAxisLabel: "MJD"
       atools.darkTestsByDate.produce.plot.yAxisLabel: "Dark Test Passing"
 
@@ -148,7 +148,7 @@ tasks:
       atools.flatTestsByDate: CalibAmpScatterTool
       atools.flatTestsByDate.prep.panelKey: amplifier
       atools.flatTestsByDate.prep.dataKey: mjd
-      atools.flatTestsByDate.prep.quantityKey: FLAT_VERIFY_NOISE
+      atools.flatTestsByDate.prep.quantityKey: ["FLAT_VERIFY_NOISE"]
       # TODO: DM-43878
       # FLAT_DET_VERIFY_SCATTER
       atools.flatTestsByDate.produce.plot.xAxisLabel: "MJD"
@@ -210,7 +210,7 @@ tasks:
       atools.ptcPlot: CalibAmpScatterTool
       atools.ptcPlot.prep.panelKey: amplifier
       atools.ptcPlot.prep.dataKey: PTC_PTC_RAW_MEANS
-      atools.ptcPlot.prep.quantityKey: PTC_PTC_RAW_VARIANCE
+      atools.ptcPlot.prep.quantityKey: ["PTC_PTC_RAW_VARIANCE"]
       atools.ptcPlot.produce.plot.xAxisLabel: "Exposure Pair Flux (ADU)"
       atools.ptcPlot.produce.plot.yAxisLabel: "Exposure Pair Variance (ADU^2)"
       python: |

python/lsst/analysis/tools/actions/plot/gridPlot.py

@@ -84,10 +84,7 @@ class GridPlot(PlotAction):
         doc="Independent data definitions. The key of this dict is the panel ID. The values are keys of data "
         "to plot (comma-separated for multiple) where each key may be a subset of a full key.",
     )
-    figsize = ListField[float](
-        doc="Figure size.",
-        default=[8, 8],
-    )
+    figsize = ListField[float](doc="Figure size.", default=[8, 8], length=2)
     dpi = Field[float](
         doc="Dots per inch.",
         default=150,

python/lsst/analysis/tools/atools/calibQuantityProfile.py

@@ -20,8 +20,6 @@
 # along with this program.  If not, see <https://www.gnu.org/licenses/>.
 from __future__ import annotations
 
-from lsst.analysis.tools.interfaces._interfaces import KeyedDataSchema
-
 __all__ = (
     "CalibQuantityBaseTool",
     "CalibQuantityAmpProfileScatterTool",
@@ -31,14 +29,34 @@
     "CalibPtcCovarScatterTool",
 )
 
-from typing import cast
+from typing import Optional
 
-from lsst.pex.config import Field
+import numpy as np
+from lsst.pex.config import Field, ListField
 from lsst.pex.config.configurableActions import ConfigurableActionField
 
 from ..actions.plot.elements import HistElement, ScatterElement
 from ..actions.plot.gridPlot import GridPanelConfig, GridPlot
-from ..interfaces import AnalysisTool, KeyedData, KeyedDataAction, PlotElement, Vector
+from ..interfaces import AnalysisTool, KeyedData, KeyedDataAction, KeyedDataSchema, PlotElement, Vector
+
+_CALIB_AMP_NAME_DICT: dict[int, str] = {
+    0: "C00",
+    1: "C01",
+    2: "C02",
+    3: "C03",
+    4: "C04",
+    5: "C05",
+    6: "C06",
+    7: "C07",
+    8: "C10",
+    9: "C11",
+    10: "C12",
+    11: "C13",
+    12: "C14",
+    13: "C15",
+    14: "C16",
+    15: "C17",
+}
 
 
 class PrepRepacker(KeyedDataAction):
@@ -50,61 +68,23 @@ class PrepRepacker(KeyedDataAction):
     dataKey = Field[str](
         doc="Data selector. Data will be separated into multiple groups in a single panel based on this key.",
     )
-    quantityKey = Field[str](
-        doc="Quantity selector. The actual data quantities to be plotted.",
+    quantityKey = ListField[str](
+        doc="Quantity selector. The actual data quantities to be plotted.", minLength=1, optional=False
     )
 
     def __call__(self, data: KeyedData, **kwargs) -> KeyedData:
-        repackedData = {}
-        # Loop over the length of the data vector and repack row by row
-        for i in range(len(cast(Vector, data[self.panelKey]))):
-            panelVec = cast(Vector, data[self.panelKey])
-            dataVec = cast(Vector, data[self.dataKey])
-            quantityVec = cast(Vector, data[self.quantityKey])
-            repackedData[f"{panelVec[i]}_{dataVec[i]}_{self.quantityKey}"] = quantityVec[i]
-        return repackedData
-
-    def getInputSchema(self) -> KeyedDataSchema:
-        return (
-            (self.panelKey, Vector),
-            (self.dataKey, Vector),
-            (self.quantityKey, Vector),
-        )
-
-    def addInputSchema(self, inputSchema: KeyedDataSchema) -> None:
-        pass
-
-
-class SingleValueRepacker(KeyedDataAction):
-    """Prep action to repack data."""
-
-    panelKey = Field[str](
-        doc="Panel selector. Data will be separated into multiple panels based on this key.",
-    )
-    dataKey = Field[str](
-        doc="Data selector. Data will be separated into multiple groups in a single panel based on this key.",
-    )
-    quantityKey = Field[str](
-        doc="Quantity selector. The actual data quantities to be plotted.",
-    )
-
-    def __call__(self, data: KeyedData, **kwargs) -> KeyedData:
-        repackedData = {}
+        repackedData: dict[str, Vector] = {}
         uniquePanelKeys = list(set(data[self.panelKey]))
 
-        # Loop over data vector to repack information as it is expected.
-        for i in range(len(uniquePanelKeys)):
-            repackedData[f"{uniquePanelKeys[i]}_x"] = []
-            repackedData[f"{uniquePanelKeys[i]}"] = []
-
-        panelVec = cast(Vector, data[self.panelKey])
-        dataVec = cast(Vector, data[self.dataKey])
-        quantityVec = cast(Vector, data[self.quantityKey])
-
-        for i in range(len(panelVec)):
-            repackedData[f"{panelVec[i]}_x"].append(dataVec[i])
-            repackedData[f"{panelVec[i]}"].append(quantityVec[i])
+        for pKey in uniquePanelKeys:
+            # Make a boolean array that selects the correct panel data
+            sel: np.ndarray = data[self.panelKey] == pKey
 
+            # Setup the x axis
+            repackedData[f"{pKey}_x"] = data[self.dataKey][sel]
+            for qkey in self.quantityKey:
+                # Setup a y axis series for each quantityKey
+                repackedData[f"{pKey}_{qkey}"] = data[qkey][sel]
         return repackedData
 
     def getInputSchema(self) -> KeyedDataSchema:
@@ -136,6 +116,12 @@ class CalibQuantityBaseTool(AnalysisTool):
         doc="Plot element.",
     )
 
+    def _get_xKey_dict(self, yKeys: Optional[dict[int, str]] = None) -> dict[int, str]:
+        """Generate the dictionary of x axis keys from the y axis ones"""
+        if yKeys is None:
+            yKeys = self.produce.plot.valsGroupBy
+        return {k: f"{v}_x" for k, v in yKeys.items()}
+
     def setDefaults(self):
         super().setDefaults()
 
@@ -152,24 +138,7 @@ def setDefaults(self):
         self.produce.plot.numCols = 4
 
         # Values to group by to distinguish between data in differing panels
-        self.produce.plot.valsGroupBy = {
-            0: "C00",
-            1: "C01",
-            2: "C02",
-            3: "C03",
-            4: "C04",
-            5: "C05",
-            6: "C06",
-            7: "C07",
-            8: "C10",
-            9: "C11",
-            10: "C12",
-            11: "C13",
-            12: "C14",
-            13: "C15",
-            14: "C16",
-            15: "C17",
-        }
+        self.produce.plot.valsGroupBy = _CALIB_AMP_NAME_DICT
 
     def finalize(self):
         super().finalize()
@@ -206,52 +175,17 @@ def setDefaults(self):
         self.plotElement = ScatterElement()
 
         # Repack the input data into a usable format
-        self.prep = SingleValueRepacker()
+        self.prep = PrepRepacker()
 
         self.produce.plot = GridPlot()
         self.produce.plot.panels = {}
         self.produce.plot.numRows = 4
         self.produce.plot.numCols = 4
 
         # Values to use for x-axis data
-        self.produce.plot.xDataKeys = {
-            0: "C00_x",
-            1: "C01_x",
-            2: "C02_x",
-            3: "C03_x",
-            4: "C04_x",
-            5: "C05_x",
-            6: "C06_x",
-            7: "C07_x",
-            8: "C10_x",
-            9: "C11_x",
-            10: "C12_x",
-            11: "C13_x",
-            12: "C14_x",
-            13: "C15_x",
-            14: "C16_x",
-            15: "C17_x",
-        }
-
         # Values to group by to distinguish between data in differing panels
-        self.produce.plot.valsGroupBy = {
-            0: "C00",
-            1: "C01",
-            2: "C02",
-            3: "C03",
-            4: "C04",
-            5: "C05",
-            6: "C06",
-            7: "C07",
-            8: "C10",
-            9: "C11",
-            10: "C12",
-            11: "C13",
-            12: "C14",
-            13: "C15",
-            14: "C16",
-            15: "C17",
-        }
+        self.produce.plot.valsGroupBy = _CALIB_AMP_NAME_DICT
+        self.produce.plot.xDataKeys = self._get_xKey_dict()
 
         self.prep.panelKey = "amplifier"
         self.prep.dataKey = "mjd"
@@ -277,7 +211,7 @@ def setDefaults(self):
         self.plotElement = ScatterElement()
 
         # Repack the input data into a usable format
-        self.prep = SingleValueRepacker()
+        self.prep = PrepRepacker()
 
         self.produce.plot = GridPlot()
         self.produce.plot.panels = {}
@@ -287,10 +221,7 @@ def setDefaults(self):
         # Values to group by to distinguish between data in differing panels
         self.produce.plot.valsGroupBy = {0: "bank1", 1: "bank0"}
 
-        self.produce.plot.xDataKeys = {
-            0: "bank1_x",
-            1: "bank0_x",
-        }
+        self.produce.plot.xDataKeys = self._get_xKey_dict()
         self.prep.panelKey = "amplifier"
         self.prep.dataKey = "mjd"
 

python/lsst/analysis/tools/interfaces/_actions.py

@@ -40,7 +40,7 @@
 import warnings
 from abc import abstractmethod
 from dataclasses import dataclass
-from typing import TYPE_CHECKING, Iterable
+from typing import TYPE_CHECKING, Any, Generator, GenericAlias, Iterable, Sequence
 
 import lsst.pex.config as pexConfig
 from lsst.pex.config.configurableActions import ConfigurableAction, ConfigurableActionField
@@ -108,7 +108,7 @@ def getOutputSchema(self) -> KeyedDataSchema | None:
         """
         return None
 
-    def getFormattedInputSchema(self, **kwargs) -> KeyedDataSchema:
+    def getFormattedInputSchema(self, **kwargs) -> Generator[tuple[Any, GenericAlias], None, None]:
         """Return input schema, with keys formatted with any arguments supplied
         by kwargs passed to this method.
 
@@ -119,7 +119,11 @@ def getFormattedInputSchema(self, **kwargs) -> KeyedDataSchema:
             action, formatted with any input arguments (e.g. band='i')
         """
         for key, typ in self.getInputSchema():
-            yield key.format_map(kwargs), typ
+            if isinstance(key, Sequence) and not isinstance(key, str):
+                for subkey in key:
+                    yield subkey.format_map(kwargs), typ
+            else:
+                yield key.format_map(kwargs), typ
 
     def addInputSchema(self, inputSchema: KeyedDataSchema) -> None:
         """Add the supplied inputSchema argument to the class such that it will