From 2ea34b4bada7b5e1e21c135e2cdf1a0e9c1aface Mon Sep 17 00:00:00 2001 From: craiglagegit Date: Sat, 11 Feb 2023 10:18:33 -0800 Subject: [PATCH 1/3] Initial push. Still not working --- python/lsst/ip/isr/ptcDataset.py | 25 +++++++++++++++++-------- 1 file changed, 17 insertions(+), 8 deletions(-) diff --git a/python/lsst/ip/isr/ptcDataset.py b/python/lsst/ip/isr/ptcDataset.py index 6ad97cc6f..b021e4cdd 100644 --- a/python/lsst/ip/isr/ptcDataset.py +++ b/python/lsst/ip/isr/ptcDataset.py @@ -207,7 +207,7 @@ def setAmpValues(self, ampName, inputExpIdPair=[(np.nan, np.nan)], expIdMask=[np covArrayModel=[], covSqrtWeights=[], aMatrix=[], bMatrix=[], covArrayModelNoB=[], aMatrixNoB=[], finalVar=[np.nan], finalModelVar=[np.nan], finalMean=[np.nan]): """Function to initialize an amp of a PhotonTransferCurveDataset. - + print(f"In setAmpValues. amp:{ampName}") Notes ----- The parameters are all documented in `init`. @@ -228,11 +228,11 @@ def setAmpValues(self, ampName, inputExpIdPair=[(np.nan, np.nan)], expIdMask=[np if len(aMatrixNoB) == 0: aMatrixNoB = nanMatrix - self.inputExpIdPairs[ampName] = inputExpIdPair - self.expIdMask[ampName] = expIdMask - self.rawExpTimes[ampName] = rawExpTime - self.rawMeans[ampName] = rawMean - self.rawVars[ampName] = rawVar + self.inputExpIdPairs[ampName].append(inputExpIdPair) + self.expIdMask[ampName].append(expIdMask) + self.rawExpTimes[ampName].append(rawExpTime) + self.rawMeans[ampName].append(rawMean) + self.rawVars[ampName].append(rawVar) self.photoCharge[ampName] = photoCharge self.gain[ampName] = gain self.gainErr[ampName] = gainErr @@ -242,8 +242,17 @@ def setAmpValues(self, ampName, inputExpIdPair=[(np.nan, np.nan)], expIdMask=[np self.ptcFitParsError[ampName] = ptcFitParsError self.ptcFitChiSq[ampName] = ptcFitChiSq self.ptcTurnoff[ampName] = ptcTurnoff - self.covariances[ampName] = covArray - self.covariancesSqrtWeights[ampName] = covSqrtWeights + print(f"In ip_isr, type(covArray)={type(covArray)}, shape = {np.array(covArray).shape}") + if len(self.covariances[ampName]) == 0: + self.covariances[ampName] = np.array(covArray) + else: + self.covariances[ampName] = np.concatenate((self.covariances[ampName],np.array(covArray))) + print(f"In ip_isr, type(covSqrtWeights)={type(covSqrtWeights)}, shape = {np.array(covSqrtWeights).shape}") + print(f"In ip_isr, type(self.covariancesSqrtWeights[ampName])={type(self.covariancesSqrtWeights[ampName])}, shape = {np.array(self.covariancesSqrtWeights[ampName]).shape}") + if len(self.covariancesSqrtWeights[ampName]) == 0: + self.covariancesSqrtWeights[ampName] = np.array(covSqrtWeights) + else: + self.covariancesSqrtWeights[ampName] = np.concatenate((self.covariancesSqrtWeights[ampName],np.array(covSqrtWeights))) self.covariancesModel[ampName] = covArrayModel self.covariancesModelNoB[ampName] = covArrayModelNoB self.aMatrix[ampName] = aMatrix From 15e60ee5334a20c2c80c0c887ff9b726f7922831 Mon Sep 17 00:00:00 2001 From: craiglagegit Date: Sun, 12 Feb 2023 03:51:35 -0800 Subject: [PATCH 2/3] Changes to deal with larger arrays. cpExtract now runs, but still fails in cpSolve --- python/lsst/ip/isr/ptcDataset.py | 21 ++++++++++++++------- 1 file changed, 14 insertions(+), 7 deletions(-) diff --git a/python/lsst/ip/isr/ptcDataset.py b/python/lsst/ip/isr/ptcDataset.py index b021e4cdd..957488f57 100644 --- a/python/lsst/ip/isr/ptcDataset.py +++ b/python/lsst/ip/isr/ptcDataset.py @@ -242,19 +242,18 @@ def setAmpValues(self, ampName, inputExpIdPair=[(np.nan, np.nan)], expIdMask=[np self.ptcFitParsError[ampName] = ptcFitParsError self.ptcFitChiSq[ampName] = ptcFitChiSq self.ptcTurnoff[ampName] = ptcTurnoff - print(f"In ip_isr, type(covArray)={type(covArray)}, shape = {np.array(covArray).shape}") if len(self.covariances[ampName]) == 0: self.covariances[ampName] = np.array(covArray) else: self.covariances[ampName] = np.concatenate((self.covariances[ampName],np.array(covArray))) - print(f"In ip_isr, type(covSqrtWeights)={type(covSqrtWeights)}, shape = {np.array(covSqrtWeights).shape}") - print(f"In ip_isr, type(self.covariancesSqrtWeights[ampName])={type(self.covariancesSqrtWeights[ampName])}, shape = {np.array(self.covariancesSqrtWeights[ampName]).shape}") + #print(f"In ip_isr, type(covSqrtWeights)={type(covSqrtWeights)}, shape = {np.array(covSqrtWeights).shape}") + #print(f"In ip_isr, type(self.covariancesSqrtWeights[ampName])={type(self.covariancesSqrtWeights[ampName])}, shape = {np.array(self.covariancesSqrtWeights[ampName]).shape}") if len(self.covariancesSqrtWeights[ampName]) == 0: self.covariancesSqrtWeights[ampName] = np.array(covSqrtWeights) else: self.covariancesSqrtWeights[ampName] = np.concatenate((self.covariancesSqrtWeights[ampName],np.array(covSqrtWeights))) - self.covariancesModel[ampName] = covArrayModel - self.covariancesModelNoB[ampName] = covArrayModelNoB + self.covariancesModel[ampName] = np.full_like(self.covariances[ampName], np.nan) + self.covariancesModelNoB[ampName] = np.full_like(self.covariances[ampName], np.nan) self.aMatrix[ampName] = aMatrix self.bMatrix[ampName] = bMatrix self.aMatrixNoB[ampName] = aMatrixNoB @@ -264,6 +263,7 @@ def setAmpValues(self, ampName, inputExpIdPair=[(np.nan, np.nan)], expIdMask=[np self.finalVars[ampName] = finalVar self.finalModelVars[ampName] = finalModelVar self.finalMeans[ampName] = finalMean + print(f"In ip_isr, amp:{ampName},rawMeans shape = {np.array(self.rawMeans[ampName]).shape}, covArray shape = {np.array(self.covariances[ampName]).shape}") def updateMetadata(self, **kwargs): """Update calibration metadata. @@ -530,12 +530,19 @@ def toTable(self): nPoints = [] for i, ampName in enumerate(self.ampNames): nPoints.append(len(list(self.covariances.values())[i])) + print(f"In toTable, amp:{ampName}, npoint = {len(list(self.covariances.values())[i])}") nSignalPoints = max(nPoints) + print(f"nSignalPoints={nSignalPoints}") nPadPoints = {} for i, ampName in enumerate(self.ampNames): nPadPoints[ampName] = nSignalPoints - len(list(self.covariances.values())[i]) - covMatrixSide = self.covMatrixSide - + print(f"In toTable, amp:{ampName}, nPadPoints = {nPadPoints[ampName]}") + covMatrixSide = self.covMatrixSide + print(f"In toTable, reshape size = {nSignalPoints*covMatrixSide**2}") + print(f"In toTable, cov size = {np.array(self.covariances[ampName]).shape}") + print(f"In toTable, covSQW size = {np.array(self.covariancesSqrtWeights[ampName]).shape}") + print(f"In toTable, covM size = {np.array(self.covariancesModel[ampName]).shape}") + print(f"In toTable, covMNoB size = {np.array(self.covariancesModelNoB[ampName]).shape}") catalog = Table([{'AMPLIFIER_NAME': ampName, 'PTC_FIT_TYPE': self.ptcFitType, 'COV_MATRIX_SIDE': self.covMatrixSide, From 76b5d50da870de41a5c87694bba2d3bc266d1e85 Mon Sep 17 00:00:00 2001 From: craiglagegit Date: Sun, 12 Feb 2023 04:29:05 -0800 Subject: [PATCH 3/3] More fixes and clean-up. Seems to be working now --- python/lsst/ip/isr/ptcDataset.py | 27 ++++++++------------------- 1 file changed, 8 insertions(+), 19 deletions(-) diff --git a/python/lsst/ip/isr/ptcDataset.py b/python/lsst/ip/isr/ptcDataset.py index 957488f57..4466818b4 100644 --- a/python/lsst/ip/isr/ptcDataset.py +++ b/python/lsst/ip/isr/ptcDataset.py @@ -207,7 +207,6 @@ def setAmpValues(self, ampName, inputExpIdPair=[(np.nan, np.nan)], expIdMask=[np covArrayModel=[], covSqrtWeights=[], aMatrix=[], bMatrix=[], covArrayModelNoB=[], aMatrixNoB=[], finalVar=[np.nan], finalModelVar=[np.nan], finalMean=[np.nan]): """Function to initialize an amp of a PhotonTransferCurveDataset. - print(f"In setAmpValues. amp:{ampName}") Notes ----- The parameters are all documented in `init`. @@ -227,12 +226,13 @@ def setAmpValues(self, ampName, inputExpIdPair=[(np.nan, np.nan)], expIdMask=[np bMatrix = nanMatrix if len(aMatrixNoB) == 0: aMatrixNoB = nanMatrix - - self.inputExpIdPairs[ampName].append(inputExpIdPair) - self.expIdMask[ampName].append(expIdMask) - self.rawExpTimes[ampName].append(rawExpTime) - self.rawMeans[ampName].append(rawMean) - self.rawVars[ampName].append(rawVar) + # Modifications here for the subregions. + # New data is appended to the existing data + self.inputExpIdPairs[ampName] += inputExpIdPair + self.expIdMask[ampName] += expIdMask + self.rawExpTimes[ampName] += rawExpTime + self.rawMeans[ampName] += rawMean + self.rawVars[ampName] += rawVar self.photoCharge[ampName] = photoCharge self.gain[ampName] = gain self.gainErr[ampName] = gainErr @@ -246,8 +246,6 @@ def setAmpValues(self, ampName, inputExpIdPair=[(np.nan, np.nan)], expIdMask=[np self.covariances[ampName] = np.array(covArray) else: self.covariances[ampName] = np.concatenate((self.covariances[ampName],np.array(covArray))) - #print(f"In ip_isr, type(covSqrtWeights)={type(covSqrtWeights)}, shape = {np.array(covSqrtWeights).shape}") - #print(f"In ip_isr, type(self.covariancesSqrtWeights[ampName])={type(self.covariancesSqrtWeights[ampName])}, shape = {np.array(self.covariancesSqrtWeights[ampName]).shape}") if len(self.covariancesSqrtWeights[ampName]) == 0: self.covariancesSqrtWeights[ampName] = np.array(covSqrtWeights) else: @@ -263,7 +261,6 @@ def setAmpValues(self, ampName, inputExpIdPair=[(np.nan, np.nan)], expIdMask=[np self.finalVars[ampName] = finalVar self.finalModelVars[ampName] = finalModelVar self.finalMeans[ampName] = finalMean - print(f"In ip_isr, amp:{ampName},rawMeans shape = {np.array(self.rawMeans[ampName]).shape}, covArray shape = {np.array(self.covariances[ampName]).shape}") def updateMetadata(self, **kwargs): """Update calibration metadata. @@ -530,19 +527,11 @@ def toTable(self): nPoints = [] for i, ampName in enumerate(self.ampNames): nPoints.append(len(list(self.covariances.values())[i])) - print(f"In toTable, amp:{ampName}, npoint = {len(list(self.covariances.values())[i])}") nSignalPoints = max(nPoints) - print(f"nSignalPoints={nSignalPoints}") nPadPoints = {} for i, ampName in enumerate(self.ampNames): nPadPoints[ampName] = nSignalPoints - len(list(self.covariances.values())[i]) - print(f"In toTable, amp:{ampName}, nPadPoints = {nPadPoints[ampName]}") - covMatrixSide = self.covMatrixSide - print(f"In toTable, reshape size = {nSignalPoints*covMatrixSide**2}") - print(f"In toTable, cov size = {np.array(self.covariances[ampName]).shape}") - print(f"In toTable, covSQW size = {np.array(self.covariancesSqrtWeights[ampName]).shape}") - print(f"In toTable, covM size = {np.array(self.covariancesModel[ampName]).shape}") - print(f"In toTable, covMNoB size = {np.array(self.covariancesModelNoB[ampName]).shape}") + covMatrixSide = self.covMatrixSide catalog = Table([{'AMPLIFIER_NAME': ampName, 'PTC_FIT_TYPE': self.ptcFitType, 'COV_MATRIX_SIDE': self.covMatrixSide,