Merge branch 'tickets/DM-45925'

pipelines/coaddQualityCore.yaml

@@ -22,7 +22,7 @@ tasks:
       atools.yPerpCModel: YPerpCModel
       atools.skyObjectSky: SkyObjectSkyPlot
       atools.skyObjectFlux: SkyObjectHistPlot
-      atools.psfCModelScatter: PsfCModelScatterPlot
+      atools.psfCModelScatter: PsfCModelScatterPlot  # photometry.py
       atools.shapeSizeDetRadiusVsCmodelMag: SizeMagnitudePlot
       atools.shapeSizeDetRadiusVsCmodelMag.size_type: "determinantRadius"
       atools.shapeSizeDetRadiusVsCmodelMag.mag_x: "cmodel_err"
@@ -50,6 +50,7 @@ tasks:
       atools.cModelDiskSizeVsCmodelDiskMag.produce.plot.yLims: (-4, 3)
       atools.cModelDiskSizeVsCmodelDiskMag.applyContext: CoaddContext
       atools.coaddInputCount: CoaddInputCount
+      atools.coaddPatchCount: CountPatches  #  counts.py
       python: |
         from lsst.analysis.tools.atools import *
         from lsst.analysis.tools.contexts import *

python/lsst/analysis/tools/actions/plot/colorColorFitPlot.py

@@ -26,10 +26,10 @@
 from typing import Mapping, cast
 
 import matplotlib.patheffects as pathEffects
-import matplotlib.pyplot as plt
 import numpy as np
 import scipy.stats
 from lsst.pex.config import Field, ListField, RangeField
+from lsst.utils.plotting import make_figure
 from matplotlib.figure import Figure
 from matplotlib.patches import Rectangle
 from scipy.ndimage import median_filter
@@ -234,23 +234,23 @@ def makePlot(
 
         # TODO: Make a no data fig function and use here.
         if sum(fitPoints) < paramDict["minObjectForFit"]:
-            fig = plt.figure(dpi=120)
+            fig = make_figure(dpi=120)
             ax = fig.add_axes([0.12, 0.25, 0.43, 0.62])
             ax.tick_params(labelsize=7)
             noDataText = (
                 "Number of objects after cuts ({})\nis less than the minimum required\nby "
                 "paramDict[minObjectForFit] ({})".format(sum(fitPoints), int(paramDict["minObjectForFit"]))
             )
-            plt.text(0.5, 0.5, noDataText, ha="center", va="center", fontsize=8)
-            fig = addPlotInfo(plt.gcf(), plotInfo)
+            fig.text(0.5, 0.5, noDataText, ha="center", va="center", fontsize=8)
+            fig = addPlotInfo(fig, plotInfo)
             return fig
 
         # Define new colormaps.
         newBlues = mkColormap(["darkblue", "paleturquoise"])
         newGrays = mkColormap(["lightslategray", "white"])
 
         # Make a figure with three panels.
-        fig = plt.figure(dpi=300)
+        fig = make_figure(dpi=300)
         ax = fig.add_axes([0.12, 0.25, 0.43, 0.62])
         if self.doPlotDistVsColor:
             axLowerRight = fig.add_axes([0.65, 0.11, 0.26, 0.34])
@@ -301,7 +301,7 @@ def makePlot(
 
         # Add colorbars.
         cbAx = fig.add_axes([0.12, 0.07, 0.43, 0.04])
-        plt.colorbar(fitScatter, cax=cbAx, orientation="horizontal")
+        fig.colorbar(fitScatter, cax=cbAx, orientation="horizontal")
         cbKwargs = {
             "color": "k",
             "rotation": "horizontal",
@@ -319,7 +319,7 @@ def makePlot(
         cbText.set_path_effects([pathEffects.Stroke(linewidth=1.5, foreground="w"), pathEffects.Normal()])
         cbAx.set_xticks([np.min(zUsed), np.max(zUsed)], labels=["Less", "More"], fontsize=7)
         cbAxNotUsed = fig.add_axes([0.12, 0.11, 0.43, 0.04])
-        plt.colorbar(notUsedScatter, cax=cbAxNotUsed, orientation="horizontal")
+        fig.colorbar(notUsedScatter, cax=cbAxNotUsed, orientation="horizontal")
         cbText = cbAxNotUsed.text(
             0.5,
             0.5,
@@ -541,7 +541,7 @@ def makePlot(
             axLowerRight.legend(fontsize=4, loc="upper right", handlelength=1.0)
             # Add colorbars.
             cbAx = fig.add_axes([0.915, 0.11, 0.014, 0.34])
-            plt.colorbar(lowerRightPlot, cax=cbAx, orientation="vertical")
+            fig.colorbar(lowerRightPlot, cax=cbAx, orientation="vertical")
             cbKwargs = {
                 "color": "k",
                 "rotation": "vertical",
@@ -580,6 +580,7 @@ def makePlot(
             axLowerRight.set_xlim(meanDists - nSigToPlot * madDists, meanDists + nSigToPlot * madDists)
         axLowerRight.tick_params(labelsize=6)
 
-        fig = addPlotInfo(plt.gcf(), plotInfo)
+        fig.canvas.draw()
+        fig = addPlotInfo(fig, plotInfo)
 
         return fig

python/lsst/analysis/tools/actions/plot/histPlot.py

@@ -26,7 +26,6 @@
 from collections import defaultdict
 from typing import Mapping
 
-import matplotlib.pyplot as plt
 import numpy as np
 from lsst.pex.config import (
     ChoiceField,
@@ -38,6 +37,8 @@
     FieldValidationError,
     ListField,
 )
+from lsst.utils.plotting import make_figure
+from matplotlib import cm
 from matplotlib.figure import Figure
 from matplotlib.gridspec import GridSpec
 from matplotlib.patches import Rectangle
@@ -272,7 +273,7 @@ def makePlot(
         """
 
         # set up figure
-        fig = plt.figure(dpi=300)
+        fig = make_figure(dpi=300)
         hist_fig, side_fig = fig.subfigures(1, 2, wspace=0, width_ratios=[3, 1])
         axs, ncols, nrows = self._makeAxes(hist_fig)
 
@@ -327,7 +328,7 @@ def makePlot(
             hist_fig = addPlotInfo(hist_fig, plotInfo)
 
         # finish up
-        plt.draw()
+        fig.canvas.draw()
         return fig
 
     def _makeAxes(self, fig):
@@ -395,7 +396,7 @@ def _assignColors(self):
             all_colors = custom_cmaps[self.cmap]
         else:
             try:
-                all_colors = getattr(plt.cm, self.cmap).copy().colors
+                all_colors = getattr(cm, self.cmap).copy().colors
             except AttributeError:
                 raise ValueError(f"Unrecognized color map: {self.cmap}")
 
@@ -597,7 +598,7 @@ def _addStatisticsPanel(
         """Add an adjoining panel containing histogram summary statistics."""
         ax = fig.add_subplot(1, 1, 1)
         ax.axis("off")
-        plt.subplots_adjust(left=0.05, right=0.95, bottom=0.0, top=1.0)
+        fig.subplots_adjust(left=0.05, right=0.95, bottom=0.0, top=1.0)
         # empty handle, used to populate the bespoke legend layout
         empty = Rectangle((0, 0), 1, 1, fc="w", fill=False, edgecolor="none", linewidth=0)
 

python/lsst/analysis/tools/actions/plot/plotUtils.py

@@ -24,12 +24,12 @@
 
 from typing import TYPE_CHECKING, Iterable, List, Mapping, Tuple
 
+import esutil
 import matplotlib
-import matplotlib.pyplot as plt
 import numpy as np
 from lsst.geom import Box2D, SpherePoint, degrees
 from lsst.pex.config import Config, Field
-from matplotlib import colors
+from matplotlib import cm, colors
 from matplotlib.collections import PatchCollection
 from matplotlib.patches import Rectangle
 from scipy.stats import binned_statistic_2d
@@ -75,13 +75,20 @@ def generateSummaryStats(data, skymap, plotInfo):
     patchInfoDict = {}
     maxPatchNum = tractInfo.num_patches.x * tractInfo.num_patches.y
     patches = np.arange(0, maxPatchNum, 1)
+
+    # Histogram (group) the patch values, and return an array of
+    # "reverse indices" which is a specially encoded array of where
+    # every patch is in the overall array.
+    if len(data["patch"]) == 0:
+        rev = np.full(maxPatchNum + 2, maxPatchNum + 2)
+    else:
+        _, rev = esutil.stat.histogram(data["patch"], min=0, max=maxPatchNum - 1, rev=True)
+
     for patch in patches:
-        if patch is None:
-            continue
-        # Once the objectTable_tract catalogues are using gen 3 patches
-        # this will go away
-        onPatch = data["patch"] == patch
-        if sum(onPatch) == 0:
+        # Pull out the onPatch indices
+        onPatch = rev[rev[patch] : rev[patch + 1]]
+
+        if len(onPatch) == 0:
             stat = np.nan
         else:
             stat = nanMedian(data[yCol][onPatch])
@@ -434,7 +441,7 @@ def addSummaryPlot(fig, loc, sumStats, label):
             axCorner.annotate(dataId, (cenX, cenY), color="k", fontsize=4, ha="center", va="center")
 
     # Set the bad color to transparent and make a masked array
-    cmapPatch = plt.cm.coolwarm.copy()
+    cmapPatch = cm.coolwarm.copy()
     cmapPatch.set_bad(color="none")
     colors = np.ma.array(colors, mask=np.isnan(colors))
     collection = PatchCollection(patches, cmap=cmapPatch)
@@ -451,7 +458,7 @@ def addSummaryPlot(fig, loc, sumStats, label):
     pos = axCorner.get_position()
     yOffset = (pos.y1 - pos.y0) / 3
     cax = fig.add_axes([pos.x0, pos.y1 + yOffset, pos.x1 - pos.x0, 0.025])
-    plt.colorbar(collection, cax=cax, orientation="horizontal")
+    fig.colorbar(collection, cax=cax, orientation="horizontal")
     cax.text(
         0.5,
         0.48,

python/lsst/analysis/tools/actions/plot/scatterplotWithTwoHists.py

@@ -26,11 +26,11 @@
 from typing import Mapping, NamedTuple, Optional, cast
 
 import matplotlib.colors
-import matplotlib.pyplot as plt
 import numpy as np
 from lsst.pex.config import Field
 from lsst.pex.config.configurableActions import ConfigurableActionField
 from lsst.pex.config.listField import ListField
+from lsst.utils.plotting import make_figure
 from matplotlib import gridspec
 from matplotlib.axes import Axes
 from matplotlib.collections import PolyCollection
@@ -45,10 +45,6 @@
 from ..vector import ConvertFluxToMag, SnSelector
 from .plotUtils import addPlotInfo, addSummaryPlot, generateSummaryStats, mkColormap
 
-# ignore because coolwarm is actually part of module
-cmapPatch = plt.cm.coolwarm.copy()  # type: ignore
-cmapPatch.set_bad(color="none")
-
 
 class ScatterPlotStatsAction(KeyedDataAction):
     """Calculates the statistics needed for the
@@ -358,7 +354,7 @@ def makePlot(
         if "hlineStyle" not in kwargs:
             kwargs["hlineStyle"] = (0, (1, 4))
 
-        fig = plt.figure(dpi=300)
+        fig = make_figure(dpi=300)
         gs = gridspec.GridSpec(4, 4)
 
         # add the various plot elements
@@ -372,8 +368,8 @@ def makePlot(
             label = self.yAxisLabel
             fig = addSummaryPlot(fig, gs[0, -1], sumStats, label)
 
-        plt.draw()
-        plt.subplots_adjust(wspace=0.0, hspace=0.0, bottom=0.22, left=0.21)
+        fig.canvas.draw()
+        fig.subplots_adjust(wspace=0.0, hspace=0.0, bottom=0.22, left=0.21)
         fig = addPlotInfo(fig, plotInfo)
         return fig
 
@@ -814,4 +810,6 @@ def _makeSideHistogram(
         if self.plot2DHist and histIm is not None:
             divider = make_axes_locatable(sideHist)
             cax = divider.append_axes("right", size="8%", pad=0)
-            plt.colorbar(histIm, cax=cax, orientation="vertical", label="Number of Points Per Bin")
+            sideHist.get_figure().colorbar(
+                histIm, cax=cax, orientation="vertical", label="Number of Points Per Bin"
+            )

python/lsst/analysis/tools/actions/plot/skyPlot.py

@@ -26,10 +26,10 @@
 from typing import Mapping, Optional
 
 import matplotlib.patheffects as pathEffects
-import matplotlib.pyplot as plt
 import numpy as np
 from lsst.pex.config import Field, ListField
 from lsst.pex.config.configurableActions import ConfigurableActionField
+from lsst.utils.plotting import make_figure
 from matplotlib.figure import Figure
 from matplotlib.patches import Rectangle
 
@@ -216,7 +216,7 @@ def makePlot(
         :ref:`getting started guide<analysis-tools-getting-started>`.
         """
 
-        fig = plt.figure(dpi=300)
+        fig = make_figure(dpi=300)
         ax = fig.add_subplot(111)
 
         if sumStats is None:
@@ -361,7 +361,7 @@ def makePlot(
                 showExtremeOutliers=self.showExtremeOutliers,
             )
             cax = fig.add_axes([0.87 + i * 0.04, 0.11, 0.04, 0.77])
-            plt.colorbar(plotOut, cax=cax, extend="both")
+            fig.colorbar(plotOut, cax=cax, extend="both")
             colorBarLabel = "{}: {}".format(self.zAxisLabel, label)
             text = cax.text(
                 0.5,
@@ -386,7 +386,7 @@ def makePlot(
         ax.tick_params(labelsize=7)
 
         ax.set_aspect("equal")
-        plt.draw()
+        fig.canvas.draw()
 
         # Find some useful axis limits
         lenXs = [len(xs) for (xs, _, _, _, _) in toPlotList]
@@ -399,8 +399,7 @@ def makePlot(
             ax.invert_xaxis()
 
         # Add useful information to the plot
-        plt.subplots_adjust(wspace=0.0, hspace=0.0, right=0.85)
-        fig = plt.gcf()
+        fig.subplots_adjust(wspace=0.0, hspace=0.0, right=0.85)
         fig = addPlotInfo(fig, plotInfo)
 
         return fig

python/lsst/analysis/tools/actions/vector/selectors.py

@@ -34,6 +34,7 @@
     "GalaxySelector",
     "UnknownSelector",
     "VectorSelector",
+    "FiniteSelector",
     "VisitPlotFlagSelector",
     "ThresholdSelector",
     "BandSelector",
@@ -473,6 +474,18 @@ def __call__(self, data: KeyedData, **kwargs) -> Vector:
         return extendedness == 9
 
 
+class FiniteSelector(VectorAction):
+    """Return a mask of finite values for a vector key"""
+
+    vectorKey = Field[str](doc="Key to make a mask of finite values for.")
+
+    def getInputSchema(self) -> KeyedDataSchema:
+        return ((self.vectorKey, Vector),)
+
+    def __call__(self, data: KeyedData, **kwargs) -> Vector:
+        return cast(Vector, np.isfinite(data[self.vectorKey.format(**kwargs)]))
+
+
 class VectorSelector(VectorAction):
     """Load a boolean vector from KeyedData and return it for use as a
     selector.

python/lsst/analysis/tools/atools/__init__.py

@@ -4,6 +4,7 @@
 from .calibQuantityProfile import *
 from .calibration import *
 from .coaddInputCount import *
+from .counts import *
 from .coveragePlots import *
 from .deblenderMetric import *
 from .deltaSkyCorr import *

python/lsst/analysis/tools/atools/counts.py

@@ -0,0 +1,48 @@
+# This file is part of analysis_tools.
+#
+# Developed for the LSST Data Management System.
+# This product includes software developed by the LSST Project
+# (https://www.lsst.org).
+# See the COPYRIGHT file at the top-level directory of this distribution
+# for details of code ownership.
+#
+# This program is free software: you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation, either version 3 of the License, or
+# (at your option) any later version.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with this program.  If not, see <https://www.gnu.org/licenses/>.
+from __future__ import annotations
+
+__all__ = ("CountPatches",)
+
+from ..actions.scalar.scalarActions import CountUniqueAction
+from ..actions.vector import FiniteSelector, LoadVector
+from ..interfaces import AnalysisTool
+
+
+class CountPatches(AnalysisTool):
+    """An atool to count the patches in a tract.
+    Counts the patches that make it into the
+    objectTable_tract.
+    """
+
+    def setDefaults(self):
+        super().setDefaults()
+        self.prep.selectors.patchSelector = FiniteSelector(vectorKey="{band}_ra")
+        self.process.buildActions.patch = LoadVector(vectorKey="patch")
+
+        self.process.calculateActions.patchCount = CountUniqueAction()
+        self.process.calculateActions.patchCount.vectorKey = "patch"
+
+        self.produce.metric.units = {"patchCount": ""}
+
+        self.produce.metric.newNames = {
+            "patchCount": "{band}_patchCount",
+        }