Skip to content

Commit

Permalink
start work on supporting multiple quantities in CalibAmpScatterTool
Browse files Browse the repository at this point in the history
  • Loading branch information
@weatherhead99
weatherhead99 committed Jun 12, 2024
1 parent cda0bd4 commit 7255cdc
Show file tree
Hide file tree
Showing 2 changed files with 67 additions and 89 deletions.
18 changes: 9 additions & 9 deletions pipelines/cpCore.yaml
View file
Original file line number Diff line number Diff line change
Expand Up @@ -44,28 +44,28 @@ tasks:
atools.biasMeanByDate: CalibAmpScatterTool
atools.biasMeanByDate.prep.panelKey: amplifier
atools.biasMeanByDate.prep.dataKey: mjd
atools.biasMeanByDate.prep.quantityKey: BIAS_MEAN
atools.biasMeanByDate.prep.quantityKey: ["BIAS_MEAN"]
atools.biasMeanByDate.produce.plot.xAxisLabel: "MJD"
atools.biasMeanByDate.produce.plot.yAxisLabel: "Residual bias mean (ADU)"

atools.biasStdByDate: CalibAmpScatterTool
atools.biasStdByDate.prep.panelKey: amplifier
atools.biasStdByDate.prep.dataKey: mjd
atools.biasStdByDate.prep.quantityKey: BIAS_NOISE
atools.biasStdByDate.prep.quantityKey: ["BIAS_NOISE"]
atools.biasStdByDate.produce.plot.xAxisLabel: "MJD"
atools.biasStdByDate.produce.plot.yAxisLabel: "Residual bias stdev (ADU)"

atools.biasROCornerByDate: CalibAmpScatterTool
atools.biasROCornerByDate.prep.panelKey: amplifier
atools.biasROCornerByDate.prep.dataKey: mjd
atools.biasROCornerByDate.prep.quantityKey: BIAS_AMP_CORNER
atools.biasROCornerByDate.prep.quantityKey: ["BIAS_AMP_CORNER"]
atools.biasROCornerByDate.produce.plot.xAxisLabel: "MJD"
atools.biasROCornerByDate.produce.plot.yAxisLabel: "Residual bias mean at RO corner (ADU)"

atools.biasTestsByDate: CalibAmpScatterTool
atools.biasTestsByDate.prep.panelKey: amplifier
atools.biasTestsByDate.prep.dataKey: mjd
atools.biasTestsByDate.prep.quantityKey: BIAS_VERIFY_MEAN
atools.biasTestsByDate.prep.quantityKey: ["BIAS_VERIFY_MEAN"]
atools.biasTestsByDate.produce.plot.xAxisLabel: "MJD"
atools.biasTestsByDate.produce.plot.yAxisLabel: "Bias Test Passing"

Expand Down Expand Up @@ -118,21 +118,21 @@ tasks:
atools.darkMeanByDate: CalibAmpScatterTool
atools.darkMeanByDate.prep.panelKey: amplifier
atools.darkMeanByDate.prep.dataKey: mjd
atools.darkMeanByDate.prep.quantityKey: DARK_MEAN
atools.darkMeanByDate.prep.quantityKey: ["DARK_MEAN"]
atools.darkMeanByDate.produce.plot.xAxisLabel: "MJD"
atools.darkMeanByDate.produce.plot.yAxisLabel: "Residual dark mean (ADU)"

atools.darkStdByDate: CalibAmpScatterTool
atools.darkStdByDate.prep.panelKey: amplifier
atools.darkStdByDate.prep.dataKey: mjd
atools.darkStdByDate.prep.quantityKey: DARK_NOISE
atools.darkStdByDate.prep.quantityKey: ["DARK_NOISE"]
atools.darkStdByDate.produce.plot.xAxisLabel: "MJD"
atools.darkStdByDate.produce.plot.yAxisLabel: "Residual dark std (ADU)"

atools.darkTestsByDate: CalibAmpScatterTool
atools.darkTestsByDate.prep.panelKey: amplifier
atools.darkTestsByDate.prep.dataKey: mjd
atools.darkTestsByDate.prep.quantityKey: DARK_VERIFY_MEAN
atools.darkTestsByDate.prep.quantityKey: ["DARK_VERIFY_MEAN"]
atools.darkTestsByDate.produce.plot.xAxisLabel: "MJD"
atools.darkTestsByDate.produce.plot.yAxisLabel: "Dark Test Passing"

Expand All @@ -148,7 +148,7 @@ tasks:
atools.flatTestsByDate: CalibAmpScatterTool
atools.flatTestsByDate.prep.panelKey: amplifier
atools.flatTestsByDate.prep.dataKey: mjd
atools.flatTestsByDate.prep.quantityKey: FLAT_VERIFY_NOISE
atools.flatTestsByDate.prep.quantityKey: ["FLAT_VERIFY_NOISE"]
# TODO: DM-43878
# FLAT_DET_VERIFY_SCATTER
atools.flatTestsByDate.produce.plot.xAxisLabel: "MJD"
Expand Down Expand Up @@ -210,7 +210,7 @@ tasks:
atools.ptcPlot: CalibAmpScatterTool
atools.ptcPlot.prep.panelKey: amplifier
atools.ptcPlot.prep.dataKey: PTC_PTC_RAW_MEANS
atools.ptcPlot.prep.quantityKey: PTC_PTC_RAW_VARIANCE
atools.ptcPlot.prep.quantityKey: ["PTC_PTC_RAW_VARIANCE"]
atools.ptcPlot.produce.plot.xAxisLabel: "Exposure Pair Flux (ADU)"
atools.ptcPlot.produce.plot.yAxisLabel: "Exposure Pair Variance (ADU^2)"
python: |
Expand Down
138 changes: 58 additions & 80 deletions python/lsst/analysis/tools/atools/calibQuantityProfile.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -31,15 +31,48 @@
"CalibPtcCovarScatterTool",
)

from typing import cast
from typing import cast, TypeVar, Callable, Any

Check failure on line 34 in python/lsst/analysis/tools/atools/calibQuantityProfile.py

View workflow job for this annotation

GitHub Actions / call-workflow / lint

F401 

'typing.cast' imported but unused

from lsst.pex.config import Field
from lsst.pex.config import Field, ListField
from lsst.pex.config.configurableActions import ConfigurableActionField

from ..actions.plot.elements import HistElement, ScatterElement
from ..actions.plot.gridPlot import GridPanelConfig, GridPlot
from ..interfaces import AnalysisTool, KeyedData, KeyedDataAction, PlotElement, Vector

_CALIB_AMP_NAME_DICT: dict[int, str] = {
0: "C00",
1: "C01",
2: "C02",
3: "C03",
4: "C04",
5: "C05",
6: "C06",
7: "C07",
8: "C10",
9: "C11",
10: "C12",
11: "C13",
12: "C14",
13: "C15",
14: "C16",
15: "C17",
}


RepackerT = TypeVar("RepackerT", bound=[PrepRepacker, SingleValueRepacker])

Check failure on line 63 in python/lsst/analysis/tools/atools/calibQuantityProfile.py

View workflow job for this annotation

GitHub Actions / call-workflow / lint

F821 

undefined name 'PrepRepacker'

Check failure on line 63 in python/lsst/analysis/tools/atools/calibQuantityProfile.py

View workflow job for this annotation

GitHub Actions / call-workflow / lint

F821 

undefined name 'SingleValueRepacker'
RepackerLoopFun = Callable[None, [KeyedData, Any, Any, Any, Any]]
def _repack_loop_helper(obj: RepackerT, repackfun: RepackerLoopFun, data: KeyedData) -> KeyedData:

Check failure on line 65 in python/lsst/analysis/tools/atools/calibQuantityProfile.py

View workflow job for this annotation

GitHub Actions / call-workflow / lint

E302 

expected 2 blank lines, found 0
repackedData: dict[str, Vector] = {}
quantitiesData = [data[_] for _ in obj.quantityKey]

for p, d in zip(data[obj.panelKey], data[obj.dataKey]):
for qName, qD in zip(obj.quantityKey, quantitiesData):
RepackerLoopFun(repackedData, p, d, qName, qD)
return repackedData




class PrepRepacker(KeyedDataAction):

Check failure on line 77 in python/lsst/analysis/tools/atools/calibQuantityProfile.py

View workflow job for this annotation

GitHub Actions / call-workflow / lint

E303 

too many blank lines (4)
"""Prep action to repack data."""
Expand All @@ -50,18 +83,15 @@ class PrepRepacker(KeyedDataAction):
dataKey = Field[str](
doc="Data selector. Data will be separated into multiple groups in a single panel based on this key.",
)
quantityKey = Field[str](
quantityKey = ListField[str](
doc="Quantity selector. The actual data quantities to be plotted.",
)
minLength=1, optional=False)

def __call__(self, data: KeyedData, **kwargs) -> KeyedData:
repackedData = {}
# Loop over the length of the data vector and repack row by row
for i in range(len(cast(Vector, data[self.panelKey]))):
panelVec = cast(Vector, data[self.panelKey])
dataVec = cast(Vector, data[self.dataKey])
quantityVec = cast(Vector, data[self.quantityKey])
repackedData[f"{panelVec[i]}_{dataVec[i]}_{self.quantityKey}"] = quantityVec[i]
def rp_loop(rpData: KeyedData, panel, data, quantityName, quantityData):
qName = f"{panel}_{data}_{quantityName}"
rpData[qName] = quantityData
repackedData = _repack_loop_helper(self, rp_loop, data)

Check failure on line 94 in python/lsst/analysis/tools/atools/calibQuantityProfile.py

View workflow job for this annotation

GitHub Actions / call-workflow / lint

W291 

trailing whitespace
return repackedData

def getInputSchema(self) -> KeyedDataSchema:
Expand All @@ -84,27 +114,27 @@ class SingleValueRepacker(KeyedDataAction):
dataKey = Field[str](
doc="Data selector. Data will be separated into multiple groups in a single panel based on this key.",
)
quantityKey = Field[str](
quantityKey = ListField[str](
doc="Quantity selector. The actual data quantities to be plotted.",
)
minLength=1, optional=False)

def __call__(self, data: KeyedData, **kwargs) -> KeyedData:
repackedData = {}
uniquePanelKeys = list(set(data[self.panelKey]))
uniquePanelKeys: list = list(set(data[self.panelKey]))

# Loop over data vector to repack information as it is expected.
for i in range(len(uniquePanelKeys)):
repackedData[f"{uniquePanelKeys[i]}_x"] = []
repackedData[f"{uniquePanelKeys[i]}"] = []
repackedData: dict[str, Vector] = {}
uniquePanelKeys: list = list(set(data[self.panelKey]))

panelVec = cast(Vector, data[self.panelKey])
dataVec = cast(Vector, data[self.dataKey])
quantityVec = cast(Vector, data[self.quantityKey])
# Loop over data vector to repack information as it is expected.
for uk in uniquePanelKeys:
repackedData[f"{uk}_x"] = []
for q in self.quantityKey:
repackedData[f"{uk}_{q}"] = []

for i in range(len(panelVec)):
repackedData[f"{panelVec[i]}_x"].append(dataVec[i])
repackedData[f"{panelVec[i]}"].append(quantityVec[i])
def rp_loop(rpData: KeyedData, panel, data, quantityName, quantityData):
rpData[f"{panel}_x"].append(data)
rpData[f"{panel}_{quantityName}"] = quantityData

repackedData: KeyedData = _repack_loop_helper(self, rp_loop, data)
return repackedData

def getInputSchema(self) -> KeyedDataSchema:
Expand Down Expand Up @@ -152,24 +182,7 @@ def setDefaults(self):
self.produce.plot.numCols = 4

# Values to group by to distinguish between data in differing panels
self.produce.plot.valsGroupBy = {
0: "C00",
1: "C01",
2: "C02",
3: "C03",
4: "C04",
5: "C05",
6: "C06",
7: "C07",
8: "C10",
9: "C11",
10: "C12",
11: "C13",
12: "C14",
13: "C15",
14: "C16",
15: "C17",
}
self.produce.plot.valsGroupBy = _CALIB_AMP_NAME_DICT

def finalize(self):
super().finalize()
Expand Down Expand Up @@ -214,45 +227,10 @@ def setDefaults(self):
self.produce.plot.numCols = 4

# Values to use for x-axis data
self.produce.plot.xDataKeys = {
0: "C00_x",
1: "C01_x",
2: "C02_x",
3: "C03_x",
4: "C04_x",
5: "C05_x",
6: "C06_x",
7: "C07_x",
8: "C10_x",
9: "C11_x",
10: "C12_x",
11: "C13_x",
12: "C14_x",
13: "C15_x",
14: "C16_x",
15: "C17_x",
}
self.produce.plot.xDataKeys = {k : f"{v}_x" for k,v in _CALIB_AMP_NAME_DICT.items()}

Check failure on line 230 in python/lsst/analysis/tools/atools/calibQuantityProfile.py

View workflow job for this annotation

GitHub Actions / call-workflow / lint

E231 

missing whitespace after ','

# Values to group by to distinguish between data in differing panels
self.produce.plot.valsGroupBy = {
0: "C00",
1: "C01",
2: "C02",
3: "C03",
4: "C04",
5: "C05",
6: "C06",
7: "C07",
8: "C10",
9: "C11",
10: "C12",
11: "C13",
12: "C14",
13: "C15",
14: "C16",
15: "C17",
}

self.produce.plot.valsGroupBy = _CALIB_AMP_NAME_DICT
self.prep.panelKey = "amplifier"
self.prep.dataKey = "mjd"

Expand Down

0 comments on commit 7255cdc

Please sign in to comment.