diff --git a/.github/workflows/python_lint.yml b/.github/workflows/python_lint.yml new file mode 100644 index 00000000..ffed6838 --- /dev/null +++ b/.github/workflows/python_lint.yml @@ -0,0 +1,29 @@ +name: Check python syntax + +on: + pull_request: + +jobs: + ruff: + runs-on: ubuntu-latest + strategy: + matrix: + python-version: ["3.12"] + steps: + - name: Checkout repository + uses: actions/checkout@v4 + + - name: Set Python Version + uses: actions/setup-python@v5 + with: + python-version: ${{ matrix.python-version }} + + - name: Install dependencies + run: | + python -m pip install --upgrade pip + pip install -r requirements-dev.txt + + - name: Python Ruff Lint and Format + run: | + ruff check --output-format=github . + ruff format --check diff --git a/.ruff.toml b/.ruff.toml new file mode 100644 index 00000000..b0a49337 --- /dev/null +++ b/.ruff.toml @@ -0,0 +1,17 @@ +line-length = 79 + +[lint] +# Default `select` is: "E4", "E7", "E9", "F" +# `E` for "pycodestyle" (subset) +# `F` for "Pyflakes" + +# In addition, we also enable: +# `Q` for "flake8-quotes" +# `I` for "isort" +extend-select = ["Q", "I"] + +[lint.per-file-ignores] +"powa/__init__.py" = ["E402"] + +[lint.isort] +no-sections = true diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index c9104572..6b1cc2b5 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -15,6 +15,25 @@ This project relies on a few frameworks / libraries including: For the following steps, we assume you use PoWA web in debug mode (for example by running `./run-powa.py` or using podman dev environment). +### Python syntax and formatting + +Python syntax and formatting must conform to ruff. CI checks new code with ruff. + +If not already available, you can create a virtualenv for developpement purpose: + +```shell +python3 -m venv .venv +source .venv/bin/activate +pip install -r requirements-dev.txt +``` + +You can then do a syntax check and format check by running the following command: + +``` shell +ruff check +ruff format --check +``` + ## Requirements - A recent version of `NodeJS` (16+) and `npm` are required. diff --git a/powa/__init__.py b/powa/__init__.py index 590fece5..07fb40ab 100644 --- a/powa/__init__.py +++ b/powa/__init__.py @@ -1,37 +1,50 @@ from __future__ import print_function + """ Powa main application. """ + import os import re -__VERSION__ = '5.0.0dev' +__VERSION__ = "5.0.0dev" ver_tmp = re.sub("(alpha|beta|dev)[0-9]*", "", __VERSION__) -__VERSION_NUM__ = [int(part) for part in (ver_tmp.split('.'))] +__VERSION_NUM__ = [int(part) for part in (ver_tmp.split("."))] POWA_ROOT = os.path.dirname(__file__) -from tornado.web import Application, URLSpec as U -from powa.options import parse_options +# Import from powa.options must go before tornado.options +from powa.options import parse_options # noqa: I001 from tornado.options import options -from powa import ui_modules, ui_methods +from tornado.web import Application +from tornado.web import URLSpec as U + +from powa import ui_methods, ui_modules +from powa.collector import ( + CollectorDbCatRefreshHandler, + CollectorForceSnapshotHandler, + CollectorReloadHandler, +) +from powa.config import ( + RemoteConfigOverview, + RepositoryConfigOverview, +) +from powa.database import DatabaseOverview, DatabaseSelector from powa.framework import AuthHandler -from powa.user import LoginHandler, LogoutHandler +from powa.function import FunctionOverview +from powa.io import ( + ByBackendTypeIoOverview, + ByContextIoOverview, + ByObjIoOverview, +) from powa.overview import Overview -from powa.server import ServerSelector, ServerOverview -from powa.database import DatabaseSelector, DatabaseOverview -from powa.query import QueryOverview from powa.qual import QualOverview -from powa.function import FunctionOverview -from powa.config import RepositoryConfigOverview, RemoteConfigOverview -from powa.collector import (CollectorReloadHandler, - CollectorForceSnapshotHandler, - CollectorDbCatRefreshHandler) -from powa.wizard import IndexSuggestionHandler -from powa.io import (ByBackendTypeIoOverview, ByObjIoOverview, - ByContextIoOverview) +from powa.query import QueryOverview +from powa.server import ServerOverview, ServerSelector from powa.slru import ByNameSlruOverview +from powa.user import LoginHandler, LogoutHandler +from powa.wizard import IndexSuggestionHandler class IndexHandler(AuthHandler): @@ -52,38 +65,59 @@ def make_app(**kwargs): URLS = [ U(r"%slogin/" % options.url_prefix, LoginHandler, name="login"), U(r"%slogout/" % options.url_prefix, LogoutHandler, name="logout"), - U(r"%sreload_collector/" % options.url_prefix, CollectorReloadHandler, - name="reload_collector"), - U(r"%sforce_snapshot/(\d+)" % options.url_prefix, - CollectorForceSnapshotHandler, name="force_snapshot"), - U(r"%srefresh_db_cat/" % options.url_prefix, - CollectorDbCatRefreshHandler, name="refresh_db_cat"), - U(r"%sserver/select" % options.url_prefix, ServerSelector, - name="server_selector"), - U(r"%sdatabase/select" % options.url_prefix, DatabaseSelector, - name="database_selector"), + U( + r"%sreload_collector/" % options.url_prefix, + CollectorReloadHandler, + name="reload_collector", + ), + U( + r"%sforce_snapshot/(\d+)" % options.url_prefix, + CollectorForceSnapshotHandler, + name="force_snapshot", + ), + U( + r"%srefresh_db_cat/" % options.url_prefix, + CollectorDbCatRefreshHandler, + name="refresh_db_cat", + ), + U( + r"%sserver/select" % options.url_prefix, + ServerSelector, + name="server_selector", + ), + U( + r"%sdatabase/select" % options.url_prefix, + DatabaseSelector, + name="database_selector", + ), U(r"%s" % options.url_prefix, IndexHandler, name="index"), - U(r"%sserver/(\d+)/database/([^\/]+)/suggest/" % options.url_prefix, - IndexSuggestionHandler, name="index_suggestion") + U( + r"%sserver/(\d+)/database/([^\/]+)/suggest/" % options.url_prefix, + IndexSuggestionHandler, + name="index_suggestion", + ), ] - for dashboard in (Overview, - ServerOverview, - DatabaseOverview, - QueryOverview, - QualOverview, - FunctionOverview, - RepositoryConfigOverview, - RemoteConfigOverview, - ByBackendTypeIoOverview, - ByObjIoOverview, - ByContextIoOverview, - ByNameSlruOverview): + for dashboard in ( + Overview, + ServerOverview, + DatabaseOverview, + QueryOverview, + QualOverview, + FunctionOverview, + RepositoryConfigOverview, + RemoteConfigOverview, + ByBackendTypeIoOverview, + ByObjIoOverview, + ByContextIoOverview, + ByNameSlruOverview, + ): URLS.extend(dashboard.url_specs(options.url_prefix)) _cls = Application - if 'legacy_wsgi' in kwargs: + if "legacy_wsgi" in kwargs: from tornado.wsgi import WSGIApplication + _cls = WSGIApplication return _cls( @@ -95,4 +129,5 @@ def make_app(**kwargs): static_url_prefix=("%sstatic/" % options.url_prefix), cookie_secret=options.cookie_secret, template_path=os.path.join(POWA_ROOT, "templates"), - **kwargs) + **kwargs, + ) diff --git a/powa/collector.py b/powa/collector.py index 88524abd..fb977522 100644 --- a/powa/collector.py +++ b/powa/collector.py @@ -2,7 +2,9 @@ Dashboard for the powa-collector summary page, and other infrastructure for the collector handling. """ + from __future__ import absolute_import + import json from powa.dashboards import MetricGroupDef from powa.framework import AuthHandler @@ -23,15 +25,15 @@ def post_process(self, data, server, **kwargs): JOIN {powa}.powa_snapshot_metas m ON m.srvid = s.id WHERE s.id = %(server)s""" - row = self.execute(sql, params={'server': server}) + row = self.execute(sql, params={"server": server}) # unexisting server, bail out - if (len(row) != 1): - data["messages"] = {'alert': ["This server does not exists"]} + if len(row) != 1: + data["messages"] = {"alert": ["This server does not exists"]} return data - status = 'unknown' - if (server == '0'): + status = "unknown" + if server == "0": status = self.execute("""SELECT CASE WHEN count(*) = 1 THEN 'running' ELSE 'stopped' @@ -39,28 +41,30 @@ def post_process(self, data, server, **kwargs): FROM pg_stat_activity WHERE application_name LIKE 'PoWA - %%'""")[0]["status"] else: - raw = self.notify_collector('WORKERS_STATUS', [server], 2) + raw = self.notify_collector("WORKERS_STATUS", [server], 2) status = None # did we receive a valid answer? - if (len(raw) != 0 and "OK" in raw[0]): + if len(raw) != 0 and "OK" in raw[0]: # just keep the first one tmp = raw[0]["OK"] - if (server in tmp): + if server in tmp: status = json.loads(tmp)[server] - if (status is None): - return {'messages': {"warning": ["Could not get status for this " - "instance"]}, - 'data': []} + if status is None: + return { + "messages": { + "warning": ["Could not get status for this " "instance"] + }, + "data": [], + } - msg = 'Collector status for this instance: ' + status + msg = "Collector status for this instance: " + status level = "alert" - if (status == "running"): + if status == "running": level = "success" - return {'messages': {level: [msg]}, - 'data': []} + return {"messages": {level: [msg]}, "data": []} class CollectorReloadHandler(AuthHandler): @@ -69,12 +73,12 @@ class CollectorReloadHandler(AuthHandler): def get(self): res = False - answers = self.notify_collector('RELOAD') + answers = self.notify_collector("RELOAD") # iterate over the results. If at least one OK is received, report # success, otherwise failure for a in answers: - if ("OK" in a): + if "OK" in a: res = True break @@ -85,7 +89,7 @@ class CollectorForceSnapshotHandler(AuthHandler): """Request an immediate snapshot on the given server.""" def get(self, server): - answers = self.notify_collector('FORCE_SNAPSHOT', [server]) + answers = self.notify_collector("FORCE_SNAPSHOT", [server]) self.render_json(answers) @@ -95,11 +99,11 @@ class CollectorDbCatRefreshHandler(AuthHandler): def post(self): payload = json.loads(self.request.body.decode("utf8")) - nb_db = len(payload['dbnames']) - args = [payload['srvid'], str(nb_db)] - if (nb_db > 0): - args.extend(payload['dbnames']); + nb_db = len(payload["dbnames"]) + args = [payload["srvid"], str(nb_db)] + if nb_db > 0: + args.extend(payload["dbnames"]) - answers = self.notify_collector('REFRESH_DB_CAT', args) + answers = self.notify_collector("REFRESH_DB_CAT", args) self.render_json(answers) diff --git a/powa/compat.py b/powa/compat.py index 74dad8af..f161f099 100644 --- a/powa/compat.py +++ b/powa/compat.py @@ -5,29 +5,37 @@ http://pypi.python.org/pypi/six/ """ + from __future__ import absolute_import + +import json import psycopg2 from psycopg2 import extensions -import json # If psycopg2 < 2.5, register json type -psycopg2_version = tuple(psycopg2.__version__.split(' ')[0].split('.')) -if psycopg2_version < ('2', '5'): +psycopg2_version = tuple(psycopg2.__version__.split(" ")[0].split(".")) +if psycopg2_version < ("2", "5"): JSON_OID = 114 newtype = extensions.new_type( - (JSON_OID,), "JSON", lambda data, cursor: json.loads(data)) + (JSON_OID,), "JSON", lambda data, cursor: json.loads(data) + ) extensions.register_type(newtype) + def with_metaclass(meta, *bases): """Create a base class with a metaclass.""" + # This requires a bit of explanation: the basic idea is to make a dummy # metaclass for one level of class instantiation that replaces itself with # the actual metaclass. class metaclass(meta): """The actual metaclass.""" + def __new__(cls, name, _, d): return meta(name, bases, d) - return type.__new__(metaclass, 'temporary_class', (), {}) + + return type.__new__(metaclass, "temporary_class", (), {}) + class classproperty(object): """ diff --git a/powa/config.py b/powa/config.py index 3dd523f6..55be906b 100644 --- a/powa/config.py +++ b/powa/config.py @@ -1,14 +1,20 @@ """ Dashboard for the configuration summary page. """ + from __future__ import absolute_import + +import json +from powa.collector import CollectorServerDetail from powa.dashboards import ( - Dashboard, Grid, - MetricGroupDef, MetricDef, - DashboardPage, ContentWidget) + ContentWidget, + Dashboard, + DashboardPage, + Grid, + MetricDef, + MetricGroupDef, +) from powa.sql.views import get_config_changes -from powa.collector import CollectorServerDetail -import json def get_pgts_query(handler, restrict_database=False): @@ -16,9 +22,10 @@ def get_pgts_query(handler, restrict_database=False): # installed in version 2.0.0 or more. We check for local installation # only, as query will look in local table. If the extension isn't setup # remotely, the check will be pretty quick. - pgts = handler.has_extension_version('0', "pg_track_settings", "2.0.0", - remote_access=False) - if (not pgts): + pgts = handler.has_extension_version( + "0", "pg_track_settings", "2.0.0", remote_access=False + ) + if not pgts: return None return get_config_changes(restrict_database) @@ -73,6 +80,7 @@ class ServersErrors(ContentWidget): """ Detail widget showing the various errors related to servers logged """ + title = "Errors" data_url = r"/config/errors" @@ -100,6 +108,7 @@ class AllCollectorsDetail(ContentWidget): Detail widget showing summarized information for the background worker and the remote collector daemon. """ + title = "Collector Detail" data_url = r"/config/allcollectors" @@ -137,9 +146,9 @@ def get(self): rows = self.execute(sql) self.logger.warn("%r", rows[0]) - if (rows[0]["not_authorized"] == True): + if rows[0]["not_authorized"] is True: collector = None - if (rows[0]["nb_found"] == 0): + if rows[0]["nb_found"] == 0: collector = [] else: collector = rows @@ -211,21 +220,21 @@ def process(self, val, **kwargs): return val def post_process(self, data, **kwargs): - if (len(data["data"])): - raw = self.notify_collector('WORKERS_STATUS', timeout=1) - if (not raw): + if len(data["data"]): + raw = self.notify_collector("WORKERS_STATUS", timeout=1) + if not raw: return data line = None # get the first correct response only, if multiple answers were # returned - while (line is None and len(raw) > 0): + while line is None and len(raw) > 0: tmp = raw.pop(0) - if ("OK" in tmp): + if "OK" in tmp: line = tmp["OK"] # nothing correct, give up - if (line is None or line == {}): + if line is None or line == {}: return data stats = json.loads(line) @@ -260,7 +269,7 @@ class PgSettingsMetricGroup(MetricGroupDef): @property def query(self): - if (self.path_args[0] == '0'): + if self.path_args[0] == "0": return self.__query else: # we'll get the data on the foreign server in post_process @@ -268,15 +277,16 @@ def query(self): def post_process(self, data, server, **kwargs): # For local server we can return data already retrieved - if (server == '0'): + if server == "0": return data values = None # Check first if the info is available locally - if (self.has_extension_version(server, 'pg_track_settings', '2.0.0')): + if self.has_extension_version(server, "pg_track_settings", "2.0.0"): try: - values = self.execute(""" + values = self.execute( + """ SELECT t.name AS setting_name, t.setting AS setting_value, s.unit AS setting_unit, @@ -285,32 +295,40 @@ def post_process(self, data, server, **kwargs): %(srvid)s) t LEFT JOIN pg_catalog.pg_settings s ON s.name = t.name - """, params={'srvid': server}) + """, + params={"srvid": server}, + ) # If no rows were retrieved, it probably means that # pg_tracksettings isn't sampled even if the extension exists. # Reset values so we can try to fetch info from the remote # server. - if (values.rowcount == 0): + if values.rowcount == 0: values = None except Exception: # ignore any error, we'll just fallback on remote check pass - if (values is None): + if values is None: try: values = self.execute(self.__query, srvid=server) except Exception: # ignore any connection or remote execution error pass - if (values is not None): + if values is not None: data = {"data": [self.process(val) for val in values]} else: - data = {"data": [], - "messages": {'alert': ["Could not retrieve PostgreSQL" - + " settings " - + "on remote server"]}} + data = { + "data": [], + "messages": { + "alert": [ + "Could not retrieve PostgreSQL" + + " settings " + + "on remote server" + ] + }, + } return data @@ -332,7 +350,7 @@ class PgStatExtensionsMetricGroup(MetricGroupDef): @property def query(self): - if (self.path_args[0] == '0'): + if self.path_args[0] == "0": return """SELECT pe.extname, pae.name IS NOT NULL AS available, pae.installed_version IS NOT NULL AS installed, pec.enabled AS handled, @@ -369,7 +387,7 @@ def post_process(self, data, server, **kwargs): needed """ # We already have all the data for the local server - if (server == '0'): + if server == "0": return data res = None @@ -381,12 +399,14 @@ def post_process(self, data, server, **kwargs): extnames.append(row["extname"]) try: - res = self.execute(""" + res = self.execute( + """ SELECT name AS extname, installed_version FROM pg_available_extensions - WHERE name = ANY(%(extnames)s)""", srvid=server,params={ - 'extnames': extnames - }) + WHERE name = ANY(%(extnames)s)""", + srvid=server, + params={"extnames": extnames}, + ) except Exception as e: # ignore any connection or remote execution error, but keep the # error message @@ -395,8 +415,12 @@ def post_process(self, data, server, **kwargs): # if we couldn't get any data, send what we have if res is None or len(res) == 0: - data["messages"] = {'alert': ["Could not retrieve extensions" - + " on remote server: %s" % errmsg]} + data["messages"] = { + "alert": [ + "Could not retrieve extensions" + + " on remote server: %s" % errmsg + ] + } return data remote_exts = res @@ -406,24 +430,29 @@ def post_process(self, data, server, **kwargs): found = False for r in remote_exts: - if (r["extname"] == ext["extname"]): + if r["extname"] == ext["extname"]: found = True break - if (not found): + if not found: ext["available"] = False ext["installed"] = None else: ext["available"] = True ext["installed"] = r["installed_version"] is not None - if (ext["handled"] and ext["installed"] is None): + if ext["handled"] and ext["installed"] is None: alerts.append(ext["extname"]) - if (len(alerts) > 0): - data["messages"] = {'alert': - [("%d extensions need to be installed:%s" - % (len(alerts), ' '.join(alerts)))]} + if len(alerts) > 0: + data["messages"] = { + "alert": [ + ( + "%d extensions need to be installed:%s" + % (len(alerts), " ".join(alerts)) + ) + ] + } return data @@ -444,7 +473,7 @@ class PgSupportExtensionsMetricGroup(MetricGroupDef): @property def query(self): - if (self.path_args[0] == '0'): + if self.path_args[0] == "0": return """SELECT pe.extname, pae.name IS NOT NULL AS available, pae.installed_version IS NOT NULL AS installed, coalesce(pae.installed_version , '-') AS extversion @@ -468,7 +497,7 @@ def post_process(self, data, server, **kwargs): Get the missing metadata of the extensions on the remote servers """ # We already have all the data for the local server - if (server == '0'): + if server == "0": return data res = None @@ -480,12 +509,14 @@ def post_process(self, data, server, **kwargs): extnames.append(row["extname"]) try: - res = self.execute(""" + res = self.execute( + """ SELECT name AS extname, installed_version FROM pg_available_extensions - WHERE name = ANY(%(extnames)s)""", srvid=server,params={ - 'extnames': extnames - }) + WHERE name = ANY(%(extnames)s)""", + srvid=server, + params={"extnames": extnames}, + ) except Exception as e: # ignore any connection or remote execution error, but keep the # error message @@ -494,8 +525,12 @@ def post_process(self, data, server, **kwargs): # if we couldn't get any data, send what we have if res is None or len(res) == 0: - data["messages"] = {'alert': ["Could not retrieve extensions" - + " on remote server: %s" % errmsg]} + data["messages"] = { + "alert": [ + "Could not retrieve extensions" + + " on remote server: %s" % errmsg + ] + } return data remote_exts = res @@ -504,14 +539,14 @@ def post_process(self, data, server, **kwargs): found = False for r in remote_exts: - if (r["extname"] == ext["extname"]): + if r["extname"] == ext["extname"]: found = True break - if (not found): + if not found: ext["available"] = False ext["installed"] = None - ext["extversion"] = '-' + ext["extversion"] = "-" else: ext["available"] = True ext["installed"] = r["installed_version"] is not None @@ -574,26 +609,34 @@ class RepositoryConfigOverview(DashboardPage): base_url = r"/config/" datasources = [PowaServersMetricGroup, AllCollectorsDetail, ServersErrors] - title = 'Configuration' + title = "Configuration" def dashboard(self): # This COULD be initialized in the constructor, but tornado < 3 doesn't # call it - if getattr(self, '_dashboard', None) is not None: + if getattr(self, "_dashboard", None) is not None: return self._dashboard self._dashboard = Dashboard( "Server list", - [[AllCollectorsDetail], - [Grid("Servers", - columns=[{ - "name": "server_alias", - "label": "Server", - "url_attr": "url", - "direction": "ascending" - }], - metrics=PowaServersMetricGroup.all())], - [ServersErrors]] + [ + [AllCollectorsDetail], + [ + Grid( + "Servers", + columns=[ + { + "name": "server_alias", + "label": "Server", + "url_attr": "url", + "direction": "ascending", + } + ], + metrics=PowaServersMetricGroup.all(), + ) + ], + [ServersErrors], + ], ) return self._dashboard @@ -604,9 +647,14 @@ class RemoteConfigOverview(DashboardPage): """ base_url = r"/config/(\d+)" - datasources = [PgSettingsMetricGroup, PgStatExtensionsMetricGroup, - PgSupportExtensionsMetricGroup, PgDbModulesMetricGroup, - PgCatalogsMetricGroup, CollectorServerDetail] + datasources = [ + PgSettingsMetricGroup, + PgStatExtensionsMetricGroup, + PgSupportExtensionsMetricGroup, + PgDbModulesMetricGroup, + PgCatalogsMetricGroup, + CollectorServerDetail, + ] params = ["server"] parent = RepositoryConfigOverview # title = 'Remote server configuration' @@ -618,55 +666,74 @@ def breadcrum_title(cls, handler, param): def dashboard(self): # This COULD be initialized in the constructor, but tornado < 3 doesn't # call it - if getattr(self, '_dashboard', None) is not None: + if getattr(self, "_dashboard", None) is not None: return self._dashboard grids = [ - [Grid("Stats Extensions", - columns=[{ - "name": "extname", - "label": "Extension", - }], - metrics=PgStatExtensionsMetricGroup.all() - ), - Grid("Support Extensions", - columns=[{ - "name": "extname", - "label": "Extension", - }], - metrics=PgSupportExtensionsMetricGroup.all() - ) - ]] - - if (self.path_args[0] != '0'): - grids.append([ - Grid("Database modules", - columns=[{ - "name": "db_module", - "label": "DB module", - }], - metrics=PgDbModulesMetricGroup.all() - ), - Grid("Catalogs", - columns=[{ - "name": "datname", - "label": "Database", - }], - metrics=PgCatalogsMetricGroup.all()) - ]) - - grids.append([ - Grid("PostgreSQL settings", - columns=[{ - "name": "setting_name", - "label": "Setting", - }], - metrics=PgSettingsMetricGroup.all() - ) - ]) - - self._dashboard = Dashboard( - "Configuration overview", - grids + [ + Grid( + "Stats Extensions", + columns=[ + { + "name": "extname", + "label": "Extension", + } + ], + metrics=PgStatExtensionsMetricGroup.all(), + ), + Grid( + "Support Extensions", + columns=[ + { + "name": "extname", + "label": "Extension", + } + ], + metrics=PgSupportExtensionsMetricGroup.all(), + ), + ] + ] + + if self.path_args[0] != "0": + grids.append( + [ + Grid( + "Database modules", + columns=[ + { + "name": "db_module", + "label": "DB module", + } + ], + metrics=PgDbModulesMetricGroup.all(), + ), + Grid( + "Catalogs", + columns=[ + { + "name": "datname", + "label": "Database", + } + ], + metrics=PgCatalogsMetricGroup.all(), + ), + ] + ) + + grids.append( + [ + Grid( + "PostgreSQL settings", + columns=[ + { + "name": "setting_name", + "label": "Setting", + } + ], + metrics=PgSettingsMetricGroup.all(), + ) + ] ) + + self._dashboard = Dashboard("Configuration overview", grids) return self._dashboard diff --git a/powa/dashboards.py b/powa/dashboards.py index 3621b648..7736f9f3 100644 --- a/powa/dashboards.py +++ b/powa/dashboards.py @@ -4,16 +4,19 @@ This module provides several classes to define a Dashboard. """ -from powa.json import JSONizable +from operator import attrgetter +from powa.compat import classproperty, with_metaclass from powa.framework import AuthHandler -from powa.compat import with_metaclass, classproperty +from powa.json import JSONizable from powa.ui_modules import MenuEntry from tornado.web import URLSpec -from operator import attrgetter + try: from collections import OrderedDict -except: +except ModuleNotFoundError: from ordereddict import OrderedDict + +import psycopg2 from inspect import isfunction GLOBAL_COUNTER = 0 @@ -37,7 +40,9 @@ def get(self, *args): title = self.dashboard().title % params if self.request.headers.get("Content-Type") == "application/json": - param_dashboard = self.dashboard().parameterized_json(self, **params) + param_dashboard = self.dashboard().parameterized_json( + self, **params + ) param_datasource = [] for datasource in self.datasources: # ugly hack to avoid calling the datasource twice per @@ -48,36 +53,57 @@ def get(self, *args): continue value = datasource.parameterized_json(self, **params) - value['data_url'] = self.reverse_url(datasource.url_name, - *args) + value["data_url"] = self.reverse_url( + datasource.url_name, *args + ) param_datasource.append(value) # tell the frontend how to get the configuration changes, if the # DashboardPage provided it param_timeline = None - if (self.timeline): + if self.timeline: # Dashboards can specify a subset of arguments to use for the # timeline. - if (self.timeline_params): - tl_args = [params[prm] for prm in self.params - if prm in self.timeline_params] + if self.timeline_params: + tl_args = [ + params[prm] + for prm in self.params + if prm in self.timeline_params + ] else: tl_args = args - param_timeline = self.reverse_url(self.timeline.url_name, *tl_args) + param_timeline = self.reverse_url( + self.timeline.url_name, *tl_args + ) last = len(self.breadcrumb) - 1 breadcrumbs = [ { "text": item.title, - "href": self.reverse_url(item.url_name, *item.url_params.values()), - } for i, item in enumerate(reversed(self.breadcrumb)) - ] + [{ - "text": item.children_title, - "children": [{ - "url": self.reverse_url(child.url_name, *child.url_params.values()), - "title": child.title - } for child in item.children] if item.children and i == last else None - } for i, item in enumerate(reversed(self.breadcrumb)) if i == last and item.children + "href": self.reverse_url( + item.url_name, *item.url_params.values() + ), + } + for i, item in enumerate(reversed(self.breadcrumb)) + ] + [ + { + "text": item.children_title, + "children": ( + [ + { + "url": self.reverse_url( + child.url_name, *child.url_params.values() + ), + "title": child.title, + } + for child in item.children + ] + if item.children and i == last + else None + ), + } + for i, item in enumerate(reversed(self.breadcrumb)) + if i == last and item.children ] return self.render_json( @@ -112,16 +138,20 @@ def notify_allowed(self): conn = self.connect() cur = conn.cursor() - powa_roles = ['powa_signal_backend', 'powa_write_all_data', - 'powa_admin'] + powa_roles = [ + "powa_signal_backend", + "powa_write_all_data", + "powa_admin", + ] roles_to_test = [] # pg 14+ introduced predefined roles if conn.server_version >= 140000: - roles_to_test.append('pg_signal_backend') + roles_to_test.append("pg_signal_backend") try: - cur.execute("""WITH s(v) AS ( + cur.execute( + """WITH s(v) AS ( SELECT unnest(%s) UNION ALL SELECT rolname @@ -129,18 +159,20 @@ def notify_allowed(self): WHERE powa_role = ANY (%s) ) SELECT bool_or(pg_has_role(current_user, v, 'USAGE')) - FROM s""", (roles_to_test, powa_roles)) - except psycopg2.Error as e: + FROM s""", + (roles_to_test, powa_roles), + ) + except psycopg2.Error: conn.rollback() return False row = cur.fetchone() # should not happen - if (not row): + if not row: return False - return (row[0] is True) + return row[0] is True @property def breadcrumb(self): @@ -160,10 +192,12 @@ def initialize(self, datasource, params): def get(self, *params): url_params = dict(zip(self.params, params)) - url_query_params = dict(( - (key, value[0].decode('utf8')) - for key, value - in self.request.arguments.items())) + url_query_params = dict( + ( + (key, value[0].decode("utf8")) + for key, value in self.request.arguments.items() + ) + ) url_params.update(url_query_params) url_params = self.add_params(url_params) res = self.query @@ -175,11 +209,12 @@ def get(self, *params): query = res[0] url_params.update(res[1]) data = {"data": []} - if (query is not None): + if query is not None: values = self.execute(query, params=url_params) if values is not None: - data = {"data": [self.process(val, **url_params) - for val in values]} + data = { + "data": [self.process(val, **url_params) for val in values] + } data = self.post_process(data, **url_params) self.render_json(data) @@ -230,6 +265,7 @@ class DataSource(JSONizable): a subclass of RequestHandler used to process this DataSource. """ + datasource_handler_cls = None data_url = None enabled = True @@ -241,13 +277,11 @@ def url_name(cls): """ return "datasource_%s" % cls.__name__ - @classproperty def parameterized_json(cls, handler, **parms): return cls.to_json() - class Metric(JSONizable): """ An indivudal Metric. @@ -311,7 +345,8 @@ def _validate_layout(self): if (12 % len(row)) != 0: raise ValueError( "Each widget row length must be a " - "divisor of 12 (have: %d)" % len(row)) + "divisor of 12 (have: %d)" % len(row) + ) @property def widgets(self): @@ -330,9 +365,11 @@ def set_widgets(self, widgets): def to_json(self): self._validate_layout() - return {'title': self.title, - 'type': 'dashboard', - 'widgets': self.widgets} + return { + "title": self.title, + "type": "dashboard", + "widgets": self.widgets, + } def param_widgets(self, _, **params): param_rows = [] @@ -344,44 +381,46 @@ def param_widgets(self, _, **params): return param_rows def parameterized_json(self, _, **params): - return {'title': self.title % params, - 'type': 'dashboard', - 'widgets': self.param_widgets(_, **params)} + return { + "title": self.title % params, + "type": "dashboard", + "widgets": self.param_widgets(_, **params), + } -class Panel(JSONizable): +class Panel(JSONizable): def __init__(self, title, widget): self.title = title self.widget = widget def to_json(self): - return {"title": self.title, - "widget": self.widget} + return {"title": self.title, "widget": self.widget} def parameterized_json(self, _, **args): - return {"title": self.title % args, - "type": "panel", - "widget": self.widget.parameterized_json(_, **args)} + return { + "title": self.title % args, + "type": "panel", + "widget": self.widget.parameterized_json(_, **args), + } class TabContainer(JSONizable): - def __init__(self, title, tabs=None): self.title = title self.tabs = tabs or [] def to_json(self): - tabs = [] - return {'title': self.title, - 'tabs': self.tabs} + return {"title": self.title, "tabs": self.tabs} def parameterized_json(self, _, **params): tabs = [] for tab in self.tabs: tabs.append(tab.parameterized_json(_, **params)) - return {'title': self.title % params, - 'type': 'tabcontainer', - 'tabs': tabs} + return { + "title": self.title % params, + "type": "tabcontainer", + "tabs": tabs, + } class Widget(JSONizable): @@ -415,19 +454,19 @@ class ContentWidget(Widget, DataSource, AuthHandler): This widget acts as both a Widget and DataSource, since the Data used is simplistic. """ + datasource_handler_cls = ContentHandler def initialize(self, datasource=None, params=None): self.params = params - @classmethod def to_json(cls): return { - 'data_url': cls.data_url, - 'name': cls.__name__, - 'type': cls.__name__, - 'title': cls.title + "data_url": cls.data_url, + "name": cls.__name__, + "type": cls.__name__, + "title": cls.title, } @classmethod @@ -435,7 +474,6 @@ def parameterized_json(cls, _, **params): return cls.to_json() - class Grid(Widget): """ A rich table Widget, backed by a BackGrid component. @@ -464,14 +502,15 @@ def _validate(self): if any(m._group != mg1 for m in self.metrics): raise ValueError( "A grid is not allowed to have metrics from different " - "groups. (title: %s)" % self.title) + "groups. (title: %s)" % self.title + ) def to_json(self): values = self.__dict__.copy() - values['metrics'] = [] - values['type'] = 'grid' + values["metrics"] = [] + values["type"] = "grid" for metric in self.metrics: - values['metrics'].append(metric._fqn()) + values["metrics"].append(metric._fqn()) return values @@ -480,10 +519,15 @@ class Graph(Widget): A widget backed by a Rickshaw graph. """ - def __init__(self, title, grouper=None, url=None, - axistype="time", - metrics=None, - **kwargs): + def __init__( + self, + title, + grouper=None, + url=None, + axistype="time", + metrics=None, + **kwargs, + ): self.title = title self.url = url self.grouper = grouper @@ -503,14 +547,15 @@ def _validate_axis(self, metrics): for metric in metrics: if metric.axis_type != axis_type: raise ValueError( - "Some metrics do not have the same x-axis type!") + "Some metrics do not have the same x-axis type!" + ) def to_json(self): values = self.__dict__.copy() - values['metrics'] = [] - values['type'] = 'graph' + values["metrics"] = [] + values["type"] = "graph" for metric in self.metrics: - values['metrics'].append(metric._fqn()) + values["metrics"].append(metric._fqn()) return values @@ -523,7 +568,7 @@ def __init__(self, *args, **kwargs): self.args = args self.kwargs = kwargs global GLOBAL_COUNTER - self.kwargs.setdefault('_order', GLOBAL_COUNTER) + self.kwargs.setdefault("_order", GLOBAL_COUNTER) GLOBAL_COUNTER += 1 @@ -531,6 +576,7 @@ class MetricDef(Declarative): """ A metric definition. """ + _cls = Metric @@ -542,28 +588,26 @@ class MetaMetricGroup(type, JSONizable): """ def __new__(meta, name, bases, dct): - dct['metrics'] = {} - dct['_stubs'] = {} - if not isinstance(dct.get('name', ''), str): + dct["metrics"] = {} + dct["_stubs"] = {} + if not isinstance(dct.get("name", ""), str): raise ValueError("The metric group name must be of type str") for base in bases: if hasattr(base, "enabled"): dct.setdefault("enabled", base.enabled) - if hasattr(base, '_stubs'): + if hasattr(base, "_stubs"): for key, stub in base._stubs.items(): - dct[key] = stub.__class__(*stub.args, - **stub.kwargs) + dct[key] = stub.__class__(*stub.args, **stub.kwargs) for key, val in list(dct.items()): if isinstance(val, Declarative): - dct['_stubs'][key] = val - val.kwargs['name'] = key + dct["_stubs"][key] = val + val.kwargs["name"] = key dct[key] = val = val._cls(*val.args, **val.kwargs) if isinstance(val, Metric): dct.pop(key) - dct['metrics'][key] = val + dct["metrics"][key] = val return super(MetaMetricGroup, meta).__new__(meta, name, bases, dct) - def __init__(cls, name, bases, dct): for metric in dct.get("metrics").values(): metric.bind(cls) @@ -578,25 +622,28 @@ def __hasattr__(cls, key): return key in cls.metrics - - class MetricGroupDef(with_metaclass(MetaMetricGroup, DataSource)): """ Base class for MetricGroupDef. A MetricGroupDef provides syntactic sugar for instantiating MetricGroups. """ + _inst = None - metrics = {} datasource_handler_cls = MetricGroupHandler @classmethod def to_json(cls): - values = dict(((key, val) for key, val in cls.__dict__.items() - if not key.startswith("_") and not isfunction(val))) - values['type'] = 'metric_group' + values = dict( + ( + (key, val) + for key, val in cls.__dict__.items() + if not key.startswith("_") and not isfunction(val) + ) + ) + values["type"] = "metric_group" values.setdefault("xaxis", "ts") - values['metrics'] = list(cls.metrics.values()) + values["metrics"] = list(cls.metrics.values()) values.pop("query", None) return values @@ -615,8 +662,10 @@ def all(cls, handler=None, **params): if handler is None: return sorted(cls.metrics.values(), key=attrgetter("_order")) - return sorted(cls._get_metrics(handler, **params).values(), - key=attrgetter("_order")) + return sorted( + cls._get_metrics(handler, **params).values(), + key=attrgetter("_order"), + ) @classmethod def split(cls, handler, splits): @@ -681,7 +730,9 @@ class DashboardPage(object): parent = None timeline = None timeline_params = None - docs_stats_url = 'https://powa.readthedocs.io/en/latest/components/stats_extensions/' + docs_stats_url = ( + "https://powa.readthedocs.io/en/latest/components/stats_extensions/" + ) @classmethod def url_specs(cls, url_prefix): @@ -692,21 +743,32 @@ def url_specs(cls, url_prefix): """ url_specs = [] - url_specs.append(URLSpec( - r"%s%s/" % (url_prefix, cls.base_url.strip("/")), - type(cls.__name__, (cls.dashboard_handler_cls, cls), {}), { - "template": cls.template, - "params": cls.params}, - name=cls.__name__)) + url_specs.append( + URLSpec( + r"%s%s/" % (url_prefix, cls.base_url.strip("/")), + type(cls.__name__, (cls.dashboard_handler_cls, cls), {}), + {"template": cls.template, "params": cls.params}, + name=cls.__name__, + ) + ) for datasource in cls.datasources: if datasource.data_url is None: - raise KeyError("A Datasource must have a data_url: %s" % - datasource.__name__) - url_specs.append(URLSpec( - r"%s%s/" % (url_prefix, datasource.data_url.strip("/")), - type(datasource.__name__, (datasource, datasource.datasource_handler_cls), - dict(datasource.__dict__)), - {"datasource": datasource, "params": cls.params}, name=datasource.url_name)) + raise KeyError( + "A Datasource must have a data_url: %s" + % datasource.__name__ + ) + url_specs.append( + URLSpec( + r"%s%s/" % (url_prefix, datasource.data_url.strip("/")), + type( + datasource.__name__, + (datasource, datasource.datasource_handler_cls), + dict(datasource.__dict__), + ), + {"datasource": datasource, "params": cls.params}, + name=datasource.url_name, + ) + ) return url_specs @classmethod @@ -715,18 +777,19 @@ def get_childmenu(cls, handler, params): @classmethod def get_selfmenu(cls, handler, params): - my_params = OrderedDict((key, params.get(key)) - for key in cls.params) + my_params = OrderedDict((key, params.get(key)) for key in cls.params) return MenuEntry(cls.title % params, cls.__name__, my_params) @classmethod def get_breadcrumb(cls, handler, params): - if (getattr(cls, "breadcrum_title", None)): + if getattr(cls, "breadcrum_title", None): title = cls.breadcrum_title(handler, params) else: title = cls.title % params entry = MenuEntry(title, cls.__name__, params) - entry.children_title = cls.title % params if hasattr(cls, "title") else None + entry.children_title = ( + cls.title % params if hasattr(cls, "title") else None + ) entry.children = cls.get_childmenu(handler, params) items = [entry] @@ -736,8 +799,10 @@ def get_breadcrumb(cls, handler, params): else: parent_params = [] - if cls.parent is not None and hasattr(handler, 'parent') and \ - hasattr(cls.parent, 'get_breadcrumb'): - items.extend(cls.parent.get_breadcrumb(handler, - parent_params)) + if ( + cls.parent is not None + and hasattr(handler, "parent") + and hasattr(cls.parent, "get_breadcrumb") + ): + items.extend(cls.parent.get_breadcrumb(handler, parent_params)) return items diff --git a/powa/database.py b/powa/database.py index da04f9e4..b9cbf01c 100644 --- a/powa/database.py +++ b/powa/database.py @@ -1,148 +1,246 @@ """ Module containing the by-database dashboard. """ -from tornado.web import HTTPError -from powa.framework import AuthHandler + +from powa.config import ConfigChangesDatabase from powa.dashboards import ( - Dashboard, Graph, Grid, ContentWidget, - MetricGroupDef, MetricDef, - DashboardPage, TabContainer) - -from powa.sql.views_graph import (powa_getstatdata_sample, - kcache_getstatdata_sample, - powa_getwaitdata_sample, - powa_get_pgsa_sample, - powa_get_all_idx_sample, - powa_get_all_tbl_sample, - powa_get_user_fct_sample, - powa_get_database_sample) -from powa.sql.views_grid import (powa_getstatdata_detailed_db, - powa_getwaitdata_detailed_db, - powa_getuserfuncdata_detailed_db) -from powa.wizard import WizardMetricGroup, Wizard + ContentWidget, + Dashboard, + DashboardPage, + Graph, + Grid, + MetricDef, + MetricGroupDef, + TabContainer, +) +from powa.framework import AuthHandler from powa.server import ServerOverview -from powa.sql.utils import (block_size, sum_per_sec, byte_per_sec, wps, - mulblock, total_read, total_hit, to_epoch) -from powa.config import ConfigChangesDatabase +from powa.sql.utils import ( + block_size, + mulblock, + sum_per_sec, + to_epoch, + total_hit, + total_read, + wps, +) +from powa.sql.views_graph import ( + kcache_getstatdata_sample, + powa_get_all_idx_sample, + powa_get_all_tbl_sample, + powa_get_database_sample, + powa_get_pgsa_sample, + powa_get_user_fct_sample, + powa_getstatdata_sample, + powa_getwaitdata_sample, +) +from powa.sql.views_grid import ( + powa_getstatdata_detailed_db, + powa_getuserfuncdata_detailed_db, + powa_getwaitdata_detailed_db, +) +from powa.wizard import Wizard, WizardMetricGroup +from tornado.web import HTTPError class DatabaseSelector(AuthHandler): """Page allowing to choose a database.""" def get(self): - self.redirect(self.reverse_url( - 'DatabaseOverview', - self.get_argument("server"), - self.get_argument("database"))) + self.redirect( + self.reverse_url( + "DatabaseOverview", + self.get_argument("server"), + self.get_argument("database"), + ) + ) class DatabaseOverviewMetricGroup(MetricGroupDef): """Metric group for the database global graphs.""" + name = "database_overview" xaxis = "ts" data_url = r"/server/(\d+)/metrics/database_overview/([^\/]+)/" - avg_runtime = MetricDef(label="Avg runtime", type="duration", - desc="Average query duration") - calls = MetricDef(label="Queries per sec", type="number", - desc="Number of time the query has been executed, " - "per second") - planload = MetricDef(label="Plantime per sec", type="duration", - desc="Total planning duration") - load = MetricDef(label="Runtime per sec", type="duration", - desc="Total duration of queries executed, per second") - total_blks_hit = MetricDef(label="Total shared buffers hit", - type="sizerate", - desc="Amount of data found in shared buffers") - total_blks_read = MetricDef(label="Total shared buffers miss", - type="sizerate", - desc="Amount of data found in OS cache or" - " read from disk") - wal_records = MetricDef(label="#Wal records", type="integer", - desc="Number of WAL records generated") - wal_fpi = MetricDef(label="#Wal FPI", type="integer", - desc="Number of WAL full-page images generated") - wal_bytes = MetricDef(label="Wal bytes", type="size", - desc="Amount of WAL bytes generated") - - total_sys_hit = MetricDef(label="Total system cache hit", type="sizerate", - desc="Amount of data found in OS cache") - total_disk_read = MetricDef(label="Total disk read", type="sizerate", - desc="Amount of data read from disk") - minflts = MetricDef(label="Soft page faults", type="number", - desc="Memory pages not found in the processor's MMU") - majflts = MetricDef(label="Hard page faults", type="number", - desc="Memory pages not found in memory and loaded" - " from storage") + avg_runtime = MetricDef( + label="Avg runtime", type="duration", desc="Average query duration" + ) + calls = MetricDef( + label="Queries per sec", + type="number", + desc="Number of time the query has been executed, " "per second", + ) + planload = MetricDef( + label="Plantime per sec", + type="duration", + desc="Total planning duration", + ) + load = MetricDef( + label="Runtime per sec", + type="duration", + desc="Total duration of queries executed, per second", + ) + total_blks_hit = MetricDef( + label="Total shared buffers hit", + type="sizerate", + desc="Amount of data found in shared buffers", + ) + total_blks_read = MetricDef( + label="Total shared buffers miss", + type="sizerate", + desc="Amount of data found in OS cache or" " read from disk", + ) + wal_records = MetricDef( + label="#Wal records", + type="integer", + desc="Number of WAL records generated", + ) + wal_fpi = MetricDef( + label="#Wal FPI", + type="integer", + desc="Number of WAL full-page images generated", + ) + wal_bytes = MetricDef( + label="Wal bytes", type="size", desc="Amount of WAL bytes generated" + ) + + total_sys_hit = MetricDef( + label="Total system cache hit", + type="sizerate", + desc="Amount of data found in OS cache", + ) + total_disk_read = MetricDef( + label="Total disk read", + type="sizerate", + desc="Amount of data read from disk", + ) + minflts = MetricDef( + label="Soft page faults", + type="number", + desc="Memory pages not found in the processor's MMU", + ) + majflts = MetricDef( + label="Hard page faults", + type="number", + desc="Memory pages not found in memory and loaded" " from storage", + ) # not maintained on GNU/Linux, and not available on Windows # nswaps = MetricDef(label="Swaps", type="number") # msgsnds = MetricDef(label="IPC messages sent", type="number") # msgrcvs = MetricDef(label="IPC messages received", type="number") # nsignals = MetricDef(label="Signals received", type="number") - nvcsws = MetricDef(label="Voluntary context switches", type="number", - desc="Number of voluntary context switches") - nivcsws = MetricDef(label="Involuntary context switches", type="number", - desc="Number of involuntary context switches") - jit_functions = MetricDef(label="# of JIT functions", type="integer", - desc="Total number of emitted functions") - jit_generation_time = MetricDef(label="JIT generation time", - type="duration", - desc="Total time spent generating code") - jit_inlining_count = MetricDef(label="# of JIT inlining", type="integer", - desc="Number of queries where inlining was" - " done") - jit_inlining_time = MetricDef(label="JIT inlining time", type="duration", - desc="Total time spent inlining code") - jit_optimization_count = MetricDef(label="# of JIT optimization", - type="integer", - desc="Number of queries where" - " optimization was done") - jit_optimization_time = MetricDef(label="JIT optimization time", - type="duration", - desc="Total time spent optimizing code") - jit_emission_count = MetricDef(label="# of JIT emission", type="integer", - desc="Number of queries where emission was" - " done") - jit_emission_time = MetricDef(label="JIT emission time", type="duration", - desc="Total time spent emitting code") - jit_deform_count = MetricDef(label="# of JIT tuple deforming", - type="integer", - desc="Number of queries where tuple deforming" - " was done") - jit_deform_time = MetricDef(label="JIT tuple deforming time", - type="duration", - desc="Total time spent deforming tuple") - jit_expr_time = MetricDef(label="JIT expression generation time", - type="duration", - desc="Total time spent generating expressions") + nvcsws = MetricDef( + label="Voluntary context switches", + type="number", + desc="Number of voluntary context switches", + ) + nivcsws = MetricDef( + label="Involuntary context switches", + type="number", + desc="Number of involuntary context switches", + ) + jit_functions = MetricDef( + label="# of JIT functions", + type="integer", + desc="Total number of emitted functions", + ) + jit_generation_time = MetricDef( + label="JIT generation time", + type="duration", + desc="Total time spent generating code", + ) + jit_inlining_count = MetricDef( + label="# of JIT inlining", + type="integer", + desc="Number of queries where inlining was" " done", + ) + jit_inlining_time = MetricDef( + label="JIT inlining time", + type="duration", + desc="Total time spent inlining code", + ) + jit_optimization_count = MetricDef( + label="# of JIT optimization", + type="integer", + desc="Number of queries where" " optimization was done", + ) + jit_optimization_time = MetricDef( + label="JIT optimization time", + type="duration", + desc="Total time spent optimizing code", + ) + jit_emission_count = MetricDef( + label="# of JIT emission", + type="integer", + desc="Number of queries where emission was" " done", + ) + jit_emission_time = MetricDef( + label="JIT emission time", + type="duration", + desc="Total time spent emitting code", + ) + jit_deform_count = MetricDef( + label="# of JIT tuple deforming", + type="integer", + desc="Number of queries where tuple deforming" " was done", + ) + jit_deform_time = MetricDef( + label="JIT tuple deforming time", + type="duration", + desc="Total time spent deforming tuple", + ) + jit_expr_time = MetricDef( + label="JIT expression generation time", + type="duration", + desc="Total time spent generating expressions", + ) @classmethod def _get_metrics(cls, handler, **params): base = cls.metrics.copy() if not handler.has_extension(params["server"], "pg_stat_kcache"): - for key in ("total_sys_hit", "total_disk_read", "minflts", - "majflts", - # "nswaps", "msgsnds", "msgrcvs", "nsignals", - "nvcsws", "nivcsws"): + for key in ( + "total_sys_hit", + "total_disk_read", + "minflts", + "majflts", + # "nswaps", "msgsnds", "msgrcvs", "nsignals", + "nvcsws", + "nivcsws", + ): base.pop(key) else: base.pop("total_blks_read") - if not handler.has_extension_version(params["server"], - 'pg_stat_statements', '1.8'): + if not handler.has_extension_version( + params["server"], "pg_stat_statements", "1.8" + ): for key in ("planload", "wal_records", "wal_fpi", "wal_bytes"): base.pop(key) - if not handler.has_extension_version(handler.path_args[0], - 'pg_stat_statements', '1.10'): - for key in ("jit_functions", "jit_generation_time", - "jit_inlining_count", "jit_inlining_time", - "jit_optimization_count", "jit_optimization_time", - "jit_emission_count", "jit_emission_time"): + if not handler.has_extension_version( + handler.path_args[0], "pg_stat_statements", "1.10" + ): + for key in ( + "jit_functions", + "jit_generation_time", + "jit_inlining_count", + "jit_inlining_time", + "jit_optimization_count", + "jit_optimization_time", + "jit_emission_count", + "jit_emission_time", + ): base.pop(key) - if not handler.has_extension_version(handler.path_args[0], - 'pg_stat_statements', '1.11'): - for key in ("jit_deform_count", "jit_deform_time", "jit_expr_time"): + if not handler.has_extension_version( + handler.path_args[0], "pg_stat_statements", "1.11" + ): + for key in ( + "jit_deform_count", + "jit_deform_time", + "jit_expr_time", + ): base.pop(key) return base @@ -152,44 +250,57 @@ def query(self): # Fetch the base query for sample, and filter them on the database query = powa_getstatdata_sample("db", ["datname = %(database)s"]) - cols = ["srvid", - to_epoch('ts'), - sum_per_sec('calls', prefix='sub'), - "sum(runtime) / greatest(sum(calls), 1) AS avg_runtime", - sum_per_sec('runtime', prefix='sub', alias='load'), - total_read('sub'), - total_hit('sub') + cols = [ + "srvid", + to_epoch("ts"), + sum_per_sec("calls", prefix="sub"), + "sum(runtime) / greatest(sum(calls), 1) AS avg_runtime", + sum_per_sec("runtime", prefix="sub", alias="load"), + total_read("sub"), + total_hit("sub"), + ] + + if self.has_extension_version( + self.path_args[0], "pg_stat_statements", "1.8" + ): + cols.extend( + [ + sum_per_sec("plantime", prefix="sub", alias="planload"), + sum_per_sec("wal_records", prefix="sub"), + sum_per_sec("wal_fpi", prefix="sub"), + sum_per_sec("wal_bytes", prefix="sub"), ] + ) - if self.has_extension_version(self.path_args[0], - 'pg_stat_statements', '1.8'): - cols.extend([ - sum_per_sec('plantime', prefix='sub', alias='planload'), - sum_per_sec('wal_records', prefix='sub'), - sum_per_sec('wal_fpi', prefix='sub'), - sum_per_sec('wal_bytes', prefix='sub'), - ]) - - if self.has_extension_version(self.path_args[0], - 'pg_stat_statements', '1.10'): - cols.extend([ - sum_per_sec('jit_functions'), - sum_per_sec('jit_generation_time'), - sum_per_sec('jit_inlining_count'), - sum_per_sec('jit_inlining_time'), - sum_per_sec('jit_optimization_count'), - sum_per_sec('jit_optimization_time'), - sum_per_sec('jit_emission_count'), - sum_per_sec('jit_emission_time'), - ]) - - if self.has_extension_version(self.path_args[0], - 'pg_stat_statements', '1.11'): - cols.extend([ - sum_per_sec('jit_deform_count'), - sum_per_sec('jit_deform_time'), - sum_per_sec('jit_generation_time - jit_deform_time', alias='jit_expr_time'), - ]) + if self.has_extension_version( + self.path_args[0], "pg_stat_statements", "1.10" + ): + cols.extend( + [ + sum_per_sec("jit_functions"), + sum_per_sec("jit_generation_time"), + sum_per_sec("jit_inlining_count"), + sum_per_sec("jit_inlining_time"), + sum_per_sec("jit_optimization_count"), + sum_per_sec("jit_optimization_time"), + sum_per_sec("jit_emission_count"), + sum_per_sec("jit_emission_time"), + ] + ) + + if self.has_extension_version( + self.path_args[0], "pg_stat_statements", "1.11" + ): + cols.extend( + [ + sum_per_sec("jit_deform_count"), + sum_per_sec("jit_deform_time"), + sum_per_sec( + "jit_generation_time - jit_deform_time", + alias="jit_expr_time", + ), + ] + ) from_clause = query @@ -197,34 +308,45 @@ def query(self): from_clause = "({query}) AS sub2".format(query=query) # Add system metrics from pg_stat_kcache, - kcache_query = kcache_getstatdata_sample("db", - ["datname = %(database)s"] - ) - - total_sys_hit = "{total_read} - sum(sub.reads)" \ - "/ greatest(extract(epoch FROM sub.mesure_interval), 1)" \ - " AS total_sys_hit""".format( - total_read=total_read('sub', True) - ) - total_disk_read = "sum(sub.reads)" \ - " / greatest(extract(epoch FROM sub.mesure_interval), 1)" \ + kcache_query = kcache_getstatdata_sample( + "db", ["datname = %(database)s"] + ) + + total_sys_hit = ( + "{total_read} - sum(sub.reads)" + "/ greatest(extract(epoch FROM sub.mesure_interval), 1)" + " AS total_sys_hit" + "".format(total_read=total_read("sub", True)) + ) + total_disk_read = ( + "sum(sub.reads)" + " / greatest(extract(epoch FROM sub.mesure_interval), 1)" " AS total_disk_read" - minflts = sum_per_sec('minflts', prefix="sub") - majflts = sum_per_sec('majflts', prefix="sub") + ) + minflts = sum_per_sec("minflts", prefix="sub") + majflts = sum_per_sec("majflts", prefix="sub") # nswaps = sum_per_sec('nswaps', prefix="sub") # msgsnds = sum_per_sec('msgsnds', prefix="sub") # msgrcvs = sum_per_sec('msgrcvs', prefix="sub") # nsignals = sum_per_sec(.nsignals', prefix="sub") - nvcsws = sum_per_sec('nvcsws', prefix="sub") - nivcsws = sum_per_sec('nivcsws', prefix="sub") - - cols.extend([total_sys_hit, total_disk_read, minflts, majflts, - # nswaps, msgsnds, msgrcvs, nsignals, - nvcsws, nivcsws]) + nvcsws = sum_per_sec("nvcsws", prefix="sub") + nivcsws = sum_per_sec("nivcsws", prefix="sub") + + cols.extend( + [ + total_sys_hit, + total_disk_read, + minflts, + majflts, + # nswaps, msgsnds, msgrcvs, nsignals, + nvcsws, + nivcsws, + ] + ) from_clause += """ LEFT JOIN ({kcache_query}) AS kc USING (dbid, ts, srvid)""".format( - kcache_query=kcache_query + kcache_query=kcache_query ) return """SELECT {cols} @@ -235,9 +357,7 @@ def query(self): WHERE sub.calls != 0 GROUP BY sub.srvid, sub.ts, block_size, sub.mesure_interval ORDER BY sub.ts""".format( - cols=', '.join(cols), - from_clause=from_clause, - bs=block_size + cols=", ".join(cols), from_clause=from_clause, bs=block_size ) @@ -245,6 +365,7 @@ class DatabasePGSAOverview(MetricGroupDef): """ Metric group used by pg_stat_activity graphs """ + name = "pgsa" xaxis = "ts" data_url = r"/server/(\d+)/metrics/pgsa_overview/([^\/]+)/" @@ -257,7 +378,9 @@ class DatabasePGSAOverview(MetricGroupDef): nb_active = MetricDef(label="# of active connections") nb_idle_xact = MetricDef(label="# of idle in transaction connections") nb_fastpath = MetricDef(label="# of connections in fastpath function call") - nb_idle_xact_abort = MetricDef(label="# of idle in transaction (aborted) connections") + nb_idle_xact_abort = MetricDef( + label="# of idle in transaction (aborted) connections" + ) nb_disabled = MetricDef(label="# of disabled connections") nb_unknown = MetricDef(label="# of connections in unknown state") nb_parallel_query = MetricDef(label="# of parallel queries") @@ -268,7 +391,7 @@ def _get_metrics(cls, handler, **params): base = cls.metrics.copy() remote_pg_ver = handler.get_pg_version_num(handler.path_args[0]) - if (remote_pg_ver is not None and remote_pg_ver < 130000): + if remote_pg_ver is not None and remote_pg_ver < 130000: for key in ("nb_parallel_query", "nb_parallel_worker"): base.pop(key) return base @@ -277,68 +400,78 @@ def _get_metrics(cls, handler, **params): def query(self): query = powa_get_pgsa_sample(per_db=True) - cols = ["extract(epoch FROM ts) AS ts", - "max(backend_xid_age) AS backend_xid_age", - "max(backend_xmin_age) AS backend_xmin_age", - "max(backend_start_age) FILTER (WHERE datid IS NOT NULL) AS oldest_backend", - "max(xact_start_age) FILTER (WHERE datid IS NOT NULL) AS oldest_xact", - "max(query_start_age) FILTER (WHERE datid IS NOT NULL) AS oldest_query", - "count(*) FILTER (WHERE state = 'idle') AS nb_idle", - "count(*) FILTER (WHERE state = 'active') AS nb_active", - "count(*) FILTER (WHERE state = 'idle in transaction') AS nb_idle_xact", - "count(*) FILTER (WHERE state = 'fastpath function call') AS nb_fastpath", - "count(*) FILTER (WHERE state = 'idle in transaction (aborted)') AS nb_idle_xact_abort", - "count(*) FILTER (WHERE state = 'disabled') AS nb_disabled", - "count(*) FILTER (WHERE state IS NULL) AS nb_unknown", - "count(DISTINCT leader_pid) AS nb_parallel_query", - "count(*) FILTER (WHERE leader_pid IS NOT NULL) AS nb_parallel_worker", - ] + cols = [ + "extract(epoch FROM ts) AS ts", + "max(backend_xid_age) AS backend_xid_age", + "max(backend_xmin_age) AS backend_xmin_age", + "max(backend_start_age) FILTER (WHERE datid IS NOT NULL) AS oldest_backend", + "max(xact_start_age) FILTER (WHERE datid IS NOT NULL) AS oldest_xact", + "max(query_start_age) FILTER (WHERE datid IS NOT NULL) AS oldest_query", + "count(*) FILTER (WHERE state = 'idle') AS nb_idle", + "count(*) FILTER (WHERE state = 'active') AS nb_active", + "count(*) FILTER (WHERE state = 'idle in transaction') AS nb_idle_xact", + "count(*) FILTER (WHERE state = 'fastpath function call') AS nb_fastpath", + "count(*) FILTER (WHERE state = 'idle in transaction (aborted)') AS nb_idle_xact_abort", + "count(*) FILTER (WHERE state = 'disabled') AS nb_disabled", + "count(*) FILTER (WHERE state IS NULL) AS nb_unknown", + "count(DISTINCT leader_pid) AS nb_parallel_query", + "count(*) FILTER (WHERE leader_pid IS NOT NULL) AS nb_parallel_worker", + ] return """SELECT {cols} FROM ({query}) AS sub GROUP BY ts - """.format( - cols=', '.join(cols), - query=query) + """.format(cols=", ".join(cols), query=query) class DatabaseWaitOverviewMetricGroup(MetricGroupDef): """Metric group for the database global wait events graphs.""" + name = "database_waits_overview" xaxis = "ts" data_url = r"/server/(\d+)/metrics/database_waits_overview/([^\/]+)/" # pg 9.6 only metrics - count_lwlocknamed = MetricDef(label="Lightweight Named", - desc="Number of named lightweight lock" - " wait events") - count_lwlocktranche = MetricDef(label="Lightweight Tranche", - desc="Number of lightweight lock tranche" - " wait events") + count_lwlocknamed = MetricDef( + label="Lightweight Named", + desc="Number of named lightweight lock" " wait events", + ) + count_lwlocktranche = MetricDef( + label="Lightweight Tranche", + desc="Number of lightweight lock tranche" " wait events", + ) # pg 10+ metrics - count_lwlock = MetricDef(label="Lightweight Lock", - desc="Number of wait events due to lightweight" - " locks") - count_lock = MetricDef(label="Lock", - desc="Number of wait events due to heavyweight" - " locks") - count_bufferpin = MetricDef(label="Buffer pin", - desc="Number of wait events due to buffer pin") - count_activity = MetricDef(label="Activity", - desc="Number of wait events due to postgres" - " internal processes activity") - count_client = MetricDef(label="Client", - desc="Number of wait events due to client" - " activity") - count_extension = MetricDef(label="Extension", - desc="Number wait events due to third-party" - " extensions") - count_ipc = MetricDef(label="IPC", - desc="Number of wait events due to inter-process" - "communication") - count_timeout = MetricDef(label="Timeout", - desc="Number of wait events due to timeouts") - count_io = MetricDef(label="IO", - desc="Number of wait events due to IO operations") + count_lwlock = MetricDef( + label="Lightweight Lock", + desc="Number of wait events due to lightweight" " locks", + ) + count_lock = MetricDef( + label="Lock", desc="Number of wait events due to heavyweight" " locks" + ) + count_bufferpin = MetricDef( + label="Buffer pin", desc="Number of wait events due to buffer pin" + ) + count_activity = MetricDef( + label="Activity", + desc="Number of wait events due to postgres" + " internal processes activity", + ) + count_client = MetricDef( + label="Client", desc="Number of wait events due to client" " activity" + ) + count_extension = MetricDef( + label="Extension", + desc="Number wait events due to third-party" " extensions", + ) + count_ipc = MetricDef( + label="IPC", + desc="Number of wait events due to inter-process" "communication", + ) + count_timeout = MetricDef( + label="Timeout", desc="Number of wait events due to timeouts" + ) + count_io = MetricDef( + label="IO", desc="Number of wait events due to IO operations" + ) def prepare(self): if not self.has_extension(self.path_args[0], "pg_wait_sampling"): @@ -353,13 +486,24 @@ def query(self): pg_version_num = self.get_pg_version_num(self.path_args[0]) # if we can't connect to the remote server, assume pg10 or above if pg_version_num is not None and pg_version_num < 100000: - cols += [wps("count_lwlocknamed"), wps("count_lwlocktranche"), - wps("count_lock"), wps("count_bufferpin")] + cols += [ + wps("count_lwlocknamed"), + wps("count_lwlocktranche"), + wps("count_lock"), + wps("count_bufferpin"), + ] else: - cols += [wps("count_lwlock"), wps("count_lock"), - wps("count_bufferpin"), wps("count_activity"), - wps("count_client"), wps("count_extension"), - wps("count_ipc"), wps("count_timeout"), wps("count_io")] + cols += [ + wps("count_lwlock"), + wps("count_lock"), + wps("count_bufferpin"), + wps("count_activity"), + wps("count_client"), + wps("count_extension"), + wps("count_ipc"), + wps("count_timeout"), + wps("count_io"), + ] from_clause = "({query}) AS sub".format(query=query) @@ -368,71 +512,107 @@ def query(self): -- WHERE sub.count IS NOT NULL GROUP BY sub.ts, sub.mesure_interval ORDER BY sub.ts""".format( - cols=', '.join(cols), - from_clause=from_clause - ) + cols=", ".join(cols), from_clause=from_clause + ) class DatabaseAllRelMetricGroup(MetricGroupDef): """ Metric group used by "Database objects" graphs. """ + name = "all_relations" xaxis = "ts" data_url = r"/server/(\d+)/metrics/database_all_relations/([^\/]+)/" - idx_size = MetricDef(label="Indexes size", type="size", - desc="Size of all indexes") - tbl_size = MetricDef(label="Tables size", type="size", - desc="Size of all tables") - idx_ratio = MetricDef(label="Index scans ratio", type="percent", - desc="Ratio of index scan / seq scan") - idx_ratio = MetricDef(label="Index scans ratio", type="percent", - desc="Ratio of index scan / seq scan") - idx_scan = MetricDef(label="Index scans", type="number", - desc="Number of index scan per second") - seq_scan = MetricDef(label="Sequential scans", type="number", - desc="Number of sequential scan per second") - n_tup_ins = MetricDef(label="Tuples inserted", type="number", - desc="Number of tuples inserted per second") - n_tup_upd = MetricDef(label="Tuples updated", type="number", - desc="Number of tuples updated per second") - n_tup_hot_upd = MetricDef(label="Tuples HOT updated", type="number", - desc="Number of tuples HOT updated per second") - n_tup_del = MetricDef(label="Tuples deleted", type="number", - desc="Number of tuples deleted per second") - vacuum_count = MetricDef(label="# Vacuum", type="number", - desc="Number of vacuum per second") - autovacuum_count = MetricDef(label="# Autovacuum", type="number", - desc="Number of autovacuum per second") - analyze_count = MetricDef(label="# Analyze", type="number", - desc="Number of analyze per second") - autoanalyze_count = MetricDef(label="# Autoanalyze", type="number", - desc="Number of autoanalyze per second") + idx_size = MetricDef( + label="Indexes size", type="size", desc="Size of all indexes" + ) + tbl_size = MetricDef( + label="Tables size", type="size", desc="Size of all tables" + ) + idx_ratio = MetricDef( + label="Index scans ratio", + type="percent", + desc="Ratio of index scan / seq scan", + ) + idx_ratio = MetricDef( + label="Index scans ratio", + type="percent", + desc="Ratio of index scan / seq scan", + ) + idx_scan = MetricDef( + label="Index scans", + type="number", + desc="Number of index scan per second", + ) + seq_scan = MetricDef( + label="Sequential scans", + type="number", + desc="Number of sequential scan per second", + ) + n_tup_ins = MetricDef( + label="Tuples inserted", + type="number", + desc="Number of tuples inserted per second", + ) + n_tup_upd = MetricDef( + label="Tuples updated", + type="number", + desc="Number of tuples updated per second", + ) + n_tup_hot_upd = MetricDef( + label="Tuples HOT updated", + type="number", + desc="Number of tuples HOT updated per second", + ) + n_tup_del = MetricDef( + label="Tuples deleted", + type="number", + desc="Number of tuples deleted per second", + ) + vacuum_count = MetricDef( + label="# Vacuum", type="number", desc="Number of vacuum per second" + ) + autovacuum_count = MetricDef( + label="# Autovacuum", + type="number", + desc="Number of autovacuum per second", + ) + analyze_count = MetricDef( + label="# Analyze", type="number", desc="Number of analyze per second" + ) + autoanalyze_count = MetricDef( + label="# Autoanalyze", + type="number", + desc="Number of autoanalyze per second", + ) @property def query(self): query1 = powa_get_all_tbl_sample("db") query2 = powa_get_all_idx_sample("db") - cols = ["sub.srvid", - "extract(epoch FROM sub.ts) AS ts", - "sum(tbl_size) AS tbl_size", - "sum(idx_size) AS idx_size", - "CASE WHEN sum(sub.idx_scan + sub.seq_scan) = 0" - " THEN 0" - " ELSE sum(sub.idx_scan) * 100" - " / sum(sub.idx_scan + sub.seq_scan)" - " END AS idx_ratio", - sum_per_sec("idx_scan", prefix="sub"), - sum_per_sec("seq_scan", prefix="sub"), - sum_per_sec("n_tup_ins", prefix="sub"), - sum_per_sec("n_tup_upd", prefix="sub"), - sum_per_sec("n_tup_hot_upd", prefix="sub"), - sum_per_sec("n_tup_del", prefix="sub"), - sum_per_sec("vacuum_count", prefix="sub"), - sum_per_sec("autovacuum_count", prefix="sub"), - sum_per_sec("analyze_count", prefix="sub"), - sum_per_sec("autoanalyze_count", prefix="sub")] + cols = [ + "sub.srvid", + "extract(epoch FROM sub.ts) AS ts", + "sum(tbl_size) AS tbl_size", + "sum(idx_size) AS idx_size", + "CASE WHEN sum(sub.idx_scan + sub.seq_scan) = 0" + " THEN 0" + " ELSE sum(sub.idx_scan) * 100" + " / sum(sub.idx_scan + sub.seq_scan)" + " END AS idx_ratio", + sum_per_sec("idx_scan", prefix="sub"), + sum_per_sec("seq_scan", prefix="sub"), + sum_per_sec("n_tup_ins", prefix="sub"), + sum_per_sec("n_tup_upd", prefix="sub"), + sum_per_sec("n_tup_hot_upd", prefix="sub"), + sum_per_sec("n_tup_del", prefix="sub"), + sum_per_sec("vacuum_count", prefix="sub"), + sum_per_sec("autovacuum_count", prefix="sub"), + sum_per_sec("analyze_count", prefix="sub"), + sum_per_sec("autoanalyze_count", prefix="sub"), + ] return """SELECT {cols} FROM ( @@ -444,9 +624,7 @@ def query(self): AND datname = %(database)s GROUP BY sub.srvid, sub.ts, sub.mesure_interval ORDER BY sub.ts""".format( - cols=', '.join(cols), - query1=query1, - query2=query2 + cols=", ".join(cols), query1=query1, query2=query2 ) @@ -454,15 +632,25 @@ class DatabaseUserFuncMetricGroup(MetricGroupDef): """ Metric group used by "pg_stat_user_functions" graph. """ + name = "user_functions" xaxis = "ts" data_url = r"/server/(\d+)/metrics/database_user_functions/([^\/]+)/" - calls = MetricDef(label="# of calls", type="number", - desc="Number of function calls per second") - total_load = MetricDef(label="Total time per sec", type="number", - desc="Total execution time duration") - self_load = MetricDef(label="Self time per sec", type="number", - desc="Self execution time duration") + calls = MetricDef( + label="# of calls", + type="number", + desc="Number of function calls per second", + ) + total_load = MetricDef( + label="Total time per sec", + type="number", + desc="Total execution time duration", + ) + self_load = MetricDef( + label="Self time per sec", + type="number", + desc="Self execution time duration", + ) @property def query(self): @@ -470,11 +658,13 @@ def query(self): from_clause = query - cols = ["sub.srvid", - "extract(epoch FROM sub.ts) AS ts", - sum_per_sec("calls"), - sum_per_sec("total_time", alias="total_load"), - sum_per_sec("self_time", alias="self_load")] + cols = [ + "sub.srvid", + "extract(epoch FROM sub.ts) AS ts", + sum_per_sec("calls"), + sum_per_sec("total_time", alias="total_load"), + sum_per_sec("self_time", alias="self_load"), + ] return """SELECT {cols} FROM ( @@ -484,8 +674,7 @@ def query(self): AND datname = %(database)s GROUP BY sub.srvid, sub.ts, sub.mesure_interval ORDER BY sub.ts""".format( - cols=', '.join(cols), - from_clause=from_clause + cols=", ".join(cols), from_clause=from_clause ) @@ -493,47 +682,65 @@ class DatabaseDbActivityMetricGroup(MetricGroupDef): """ Metric group used by "Database Activity" graphs. """ + name = "db_activity" xaxis = "ts" data_url = r"/server/(\d+)/metrics/db_activity/([^\/]+)/" - numbackends = MetricDef(label="# of connections", - desc="Total number of connections") - xact_commit = MetricDef(label="# of commits", - desc="Total number of commits per second") - xact_rollback = MetricDef(label="# of rollbacks", - desc="Total number of rollbacks per second") - conflicts = MetricDef(label="# of conflicts", - desc="Total number of conflicts") - deadlocks = MetricDef(label="# of deadlocks", - desc="Total number of deadlocks") - checksum_failures = MetricDef(label="# of checkum_failures", - desc="Total number of checkum_failures") - session_time = MetricDef(label="Session time", - type="duration", - desc="Total time spent by database sessions per " - "second") - active_time = MetricDef(label="Active time", - type="duration", - desc="Total time spent executing SQL statements " - "per second") - idle_in_transaction_time = MetricDef(label="idle in xact time", - type="duration", - desc="Total time spent idling while " - "in a transaction per second") - sessions = MetricDef(label="# sessions", - desc="Total number of sessions established per second") - sessions_abandoned = MetricDef(label="# sessions abandoned", - desc="Number of database sessions that " - "were terminated because connection to " - "the client was lost per second") - sessions_fatal = MetricDef(label="# sessions fatal", - desc="Number of database sessions that " - "were terminated by fatal errors per " - "second") - sessions_killed = MetricDef(label="# sessions killed per second", - desc="Number of database sessions that " - "were terminated by operator intervention " - "per second") + numbackends = MetricDef( + label="# of connections", desc="Total number of connections" + ) + xact_commit = MetricDef( + label="# of commits", desc="Total number of commits per second" + ) + xact_rollback = MetricDef( + label="# of rollbacks", desc="Total number of rollbacks per second" + ) + conflicts = MetricDef( + label="# of conflicts", desc="Total number of conflicts" + ) + deadlocks = MetricDef( + label="# of deadlocks", desc="Total number of deadlocks" + ) + checksum_failures = MetricDef( + label="# of checkum_failures", desc="Total number of checkum_failures" + ) + session_time = MetricDef( + label="Session time", + type="duration", + desc="Total time spent by database sessions per " "second", + ) + active_time = MetricDef( + label="Active time", + type="duration", + desc="Total time spent executing SQL statements " "per second", + ) + idle_in_transaction_time = MetricDef( + label="idle in xact time", + type="duration", + desc="Total time spent idling while " "in a transaction per second", + ) + sessions = MetricDef( + label="# sessions", + desc="Total number of sessions established per second", + ) + sessions_abandoned = MetricDef( + label="# sessions abandoned", + desc="Number of database sessions that " + "were terminated because connection to " + "the client was lost per second", + ) + sessions_fatal = MetricDef( + label="# sessions fatal", + desc="Number of database sessions that " + "were terminated by fatal errors per " + "second", + ) + sessions_killed = MetricDef( + label="# sessions killed per second", + desc="Number of database sessions that " + "were terminated by operator intervention " + "per second", + ) @classmethod def _get_metrics(cls, handler, **params): @@ -542,9 +749,14 @@ def _get_metrics(cls, handler, **params): pg_version_num = handler.get_pg_version_num(handler.path_args[0]) # if we can't connect to the remote server, assume pg14 or above if pg_version_num is not None and pg_version_num < 140000: - for key in ("session_time", "active_time", - "idle_in_transaction_time", "sessions", - "sessions_abandoned", "sessions_fatal", "sessions_killed" + for key in ( + "session_time", + "active_time", + "idle_in_transaction_time", + "sessions", + "sessions_abandoned", + "sessions_fatal", + "sessions_killed", ): base.pop(key) return base @@ -553,34 +765,36 @@ def _get_metrics(cls, handler, **params): def query(self): query = powa_get_database_sample(True) - cols = ["sub.srvid", - "extract(epoch FROM sub.ts) AS ts", - "numbackends", - wps("xact_commit", do_sum=False), - wps("xact_rollback", do_sum=False), - "conflicts", - "deadlocks", - "checksum_failures", - wps("session_time", do_sum=False), - wps("active_time", do_sum=False), - wps("idle_in_transaction_time", do_sum=False), - wps("sessions", do_sum=False), - wps("sessions_abandoned", do_sum=False), - wps("sessions_fatal", do_sum=False), - wps("sessions_killed", do_sum=False), + cols = [ + "sub.srvid", + "extract(epoch FROM sub.ts) AS ts", + "numbackends", + wps("xact_commit", do_sum=False), + wps("xact_rollback", do_sum=False), + "conflicts", + "deadlocks", + "checksum_failures", + wps("session_time", do_sum=False), + wps("active_time", do_sum=False), + wps("idle_in_transaction_time", do_sum=False), + wps("sessions", do_sum=False), + wps("sessions_abandoned", do_sum=False), + wps("sessions_fatal", do_sum=False), + wps("sessions_killed", do_sum=False), ] return """SELECT {cols} FROM ({query}) sub WHERE sub.mesure_interval != '0 s' ORDER BY sub.ts""".format( - cols=', '.join(cols), + cols=", ".join(cols), query=query, ) class ByQueryMetricGroup(MetricGroupDef): """Metric group for indivual query stats (displayed on the grid).""" + name = "all_queries" xaxis = "queryid" axis_type = "category" @@ -607,13 +821,15 @@ class ByQueryMetricGroup(MetricGroupDef): def _get_metrics(cls, handler, **params): base = cls.metrics.copy() - if not handler.has_extension_version(handler.path_args[0], - 'pg_stat_statements', '1.8'): + if not handler.has_extension_version( + handler.path_args[0], "pg_stat_statements", "1.8" + ): for key in ("plantime", "wal_records", "wal_fpi", "wal_bytes"): base.pop(key) - if not handler.has_extension_version(handler.path_args[0], - 'pg_stat_statements', '1.10'): + if not handler.has_extension_version( + handler.path_args[0], "pg_stat_statements", "1.10" + ): base.pop("jit_functions") base.pop("jit_time") @@ -626,66 +842,70 @@ def query(self): inner_query = powa_getstatdata_detailed_db() # Multiply each measure by the size of one block. - cols = ["sub.srvid", - "sub.queryid", - "ps.query", - "sum(sub.calls) AS calls", - "sum(sub.runtime) AS runtime", - mulblock("shared_blks_read", fn="sum"), - mulblock("shared_blks_hit", fn="sum"), - mulblock("shared_blks_dirtied", fn="sum"), - mulblock("shared_blks_written", fn="sum"), - mulblock("temp_blks_read", fn="sum"), - mulblock("temp_blks_written", fn="sum"), - "sum(sub.runtime) / greatest(sum(sub.calls), 1) AS avg_runtime", - "sum(sub.shared_blk_read_time " - + "+ sub.local_blk_read_time " - + "+ sub.temp_blk_read_time) AS blks_read_time", - "sum(sub.shared_blk_write_time " - + "+ sub.local_blk_write_time " - + "+ sub.temp_blk_write_time) AS blks_write_time", + cols = [ + "sub.srvid", + "sub.queryid", + "ps.query", + "sum(sub.calls) AS calls", + "sum(sub.runtime) AS runtime", + mulblock("shared_blks_read", fn="sum"), + mulblock("shared_blks_hit", fn="sum"), + mulblock("shared_blks_dirtied", fn="sum"), + mulblock("shared_blks_written", fn="sum"), + mulblock("temp_blks_read", fn="sum"), + mulblock("temp_blks_written", fn="sum"), + "sum(sub.runtime) / greatest(sum(sub.calls), 1) AS avg_runtime", + "sum(sub.shared_blk_read_time " + + "+ sub.local_blk_read_time " + + "+ sub.temp_blk_read_time) AS blks_read_time", + "sum(sub.shared_blk_write_time " + + "+ sub.local_blk_write_time " + + "+ sub.temp_blk_write_time) AS blks_write_time", + ] + + if self.has_extension_version( + self.path_args[0], "pg_stat_statements", "1.8" + ): + cols.extend( + [ + "sum(sub.plantime) AS plantime", + "sum(sub.wal_records) AS wal_records", + "sum(sub.wal_fpi) AS wal_fpi", + "sum(sub.wal_bytes) AS wal_bytes", ] + ) - if self.has_extension_version(self.path_args[0], 'pg_stat_statements', - '1.8'): - cols.extend([ - "sum(sub.plantime) AS plantime", - "sum(sub.wal_records) AS wal_records", - "sum(sub.wal_fpi) AS wal_fpi", - "sum(sub.wal_bytes) AS wal_bytes" - ]) - - if self.has_extension_version(self.path_args[0], 'pg_stat_statements', - '1.10'): - cols.extend([ + if self.has_extension_version( + self.path_args[0], "pg_stat_statements", "1.10" + ): + cols.extend( + [ "sum(jit_functions) AS jit_functions", "sum(jit_generation_time + jit_inlining_time" + " + jit_optimization_time + jit_emission_time)" - + " AS jit_time" - ]) + + " AS jit_time", + ] + ) from_clause = """( {inner_query} ) AS sub JOIN {{powa}}.powa_statements AS ps USING (srvid, queryid, userid, dbid) - CROSS JOIN {bs}""".format( - inner_query=inner_query, - bs=block_size - ) + CROSS JOIN {bs}""".format(inner_query=inner_query, bs=block_size) return """SELECT {cols} FROM {from_clause} whERE datname = %(database)s GROUP BY srvid, queryid, query, block_size ORDER BY sum(runtime) DESC""".format( - cols=', '.join(cols), - from_clause=from_clause - ) + cols=", ".join(cols), from_clause=from_clause + ) def process(self, val, database=None, **kwargs): val["url"] = self.reverse_url( - "QueryOverview", val["srvid"], database, val["queryid"]) + "QueryOverview", val["srvid"], database, val["queryid"] + ) return val @@ -693,24 +913,28 @@ class ByQueryWaitSamplingMetricGroup(MetricGroupDef): """ Metric group for indivual query wait events stats (displayed on the grid). """ + name = "all_queries_waits" xaxis = "query" axis_type = "category" data_url = r"/server/(\d+)/metrics/database_all_queries_waits/([^\/]+)/" - counts = MetricDef(label="# of events", type="integer", - direction="descending") + counts = MetricDef( + label="# of events", type="integer", direction="descending" + ) @property def query(self): # Working from the waitdata detailed_db base query inner_query = powa_getwaitdata_detailed_db() - cols = ["srvid", - "queryid", - "ps.query", - "event_type", - "event", - "sum(count) AS counts"] + cols = [ + "srvid", + "queryid", + "ps.query", + "event_type", + "event", + "sum(count) AS counts", + ] from_clause = """( {inner_query} @@ -723,13 +947,13 @@ def query(self): WHERE datname = %(database)s GROUP BY srvid, queryid, query, event_type, event ORDER BY sum(count) DESC""".format( - cols=', '.join(cols), - from_clause=from_clause - ) + cols=", ".join(cols), from_clause=from_clause + ) def process(self, val, database=None, **kwargs): val["url"] = self.reverse_url( - "QueryOverview", val["srvid"], database, val["queryid"]) + "QueryOverview", val["srvid"], database, val["queryid"] + ) return val @@ -737,17 +961,21 @@ class ByFuncUserFuncMetricGroup(MetricGroupDef): """ Metric group for indivual function stats (displayed on the grid). """ + name = "all_functions_stats" xaxis = "funcname" axis_type = "category" data_url = r"/server/(\d+)/metrics/database_all_functions_stats/([^\/]+)/" lanname = MetricDef(label="Language", type="string") - calls = MetricDef(label="# of calls", - type="integer", direction="descending") - total_time = MetricDef(label="Cumulated total execution time", - type="duration") - self_time = MetricDef(label="Cumulated self execution time", - type="duration") + calls = MetricDef( + label="# of calls", type="integer", direction="descending" + ) + total_time = MetricDef( + label="Cumulated total execution time", type="duration" + ) + self_time = MetricDef( + label="Cumulated self execution time", type="duration" + ) @property def query(self): @@ -758,12 +986,13 @@ def query(self): def process(self, val, database=None, **kwargs): val["url"] = self.reverse_url( - "FunctionOverview", val["srvid"], database, val["funcid"]) + "FunctionOverview", val["srvid"], database, val["funcid"] + ) return val -class WizardThisDatabase(ContentWidget): - title = 'Apply wizardry to this database' +class WizardThisDatabase(ContentWidget): + title = "Apply wizardry to this database" data_url = r"/server/(\d+)/database/([^\/]+)/wizardthisdatabase/" @@ -773,283 +1002,430 @@ def get(self, database): class DatabaseOverview(DashboardPage): """DatabaseOverview Dashboard.""" + base_url = r"/server/(\d+)/database/([^\/]+)/overview" - datasources = [DatabaseOverviewMetricGroup, ByQueryMetricGroup, - ByQueryWaitSamplingMetricGroup, WizardMetricGroup, - DatabaseWaitOverviewMetricGroup, ConfigChangesDatabase, - DatabaseAllRelMetricGroup, - DatabaseUserFuncMetricGroup, - ByFuncUserFuncMetricGroup, DatabasePGSAOverview, - DatabaseDbActivityMetricGroup] + datasources = [ + DatabaseOverviewMetricGroup, + ByQueryMetricGroup, + ByQueryWaitSamplingMetricGroup, + WizardMetricGroup, + DatabaseWaitOverviewMetricGroup, + ConfigChangesDatabase, + DatabaseAllRelMetricGroup, + DatabaseUserFuncMetricGroup, + ByFuncUserFuncMetricGroup, + DatabasePGSAOverview, + DatabaseDbActivityMetricGroup, + ] params = ["server", "database"] parent = ServerOverview - title = '%(database)s' + title = "%(database)s" timeline = ConfigChangesDatabase def dashboard(self): # This COULD be initialized in the constructor, but tornado < 3 doesn't # call it - if getattr(self, '_dashboard', None) is not None: + if getattr(self, "_dashboard", None) is not None: return self._dashboard - pgss18 = self.has_extension_version(self.path_args[0], - 'pg_stat_statements', '1.8') - pgss110 = self.has_extension_version(self.path_args[0], - 'pg_stat_statements', '1.10') - pgss111 = self.has_extension_version(self.path_args[0], - 'pg_stat_statements', '1.11') + pgss18 = self.has_extension_version( + self.path_args[0], "pg_stat_statements", "1.8" + ) + pgss110 = self.has_extension_version( + self.path_args[0], "pg_stat_statements", "1.10" + ) + pgss111 = self.has_extension_version( + self.path_args[0], "pg_stat_statements", "1.11" + ) self._dashboard = Dashboard("Database overview for %(database)s") - db_metrics = [DatabaseOverviewMetricGroup.avg_runtime, - DatabaseOverviewMetricGroup.load, - DatabaseOverviewMetricGroup.calls] + db_metrics = [ + DatabaseOverviewMetricGroup.avg_runtime, + DatabaseOverviewMetricGroup.load, + DatabaseOverviewMetricGroup.calls, + ] if pgss18: db_metrics.extend([DatabaseOverviewMetricGroup.planload]) - block_graph = Graph("Blocks (On database %(database)s)", - metrics=[DatabaseOverviewMetricGroup. - total_blks_hit], - color_scheme=None) + block_graph = Graph( + "Blocks (On database %(database)s)", + metrics=[DatabaseOverviewMetricGroup.total_blks_hit], + color_scheme=None, + ) - db_graphs = [[Graph("Calls (On database %(database)s)", - metrics=db_metrics), - block_graph]] + db_graphs = [ + [ + Graph("Calls (On database %(database)s)", metrics=db_metrics), + block_graph, + ] + ] - if ("nb_parallel_query" in DatabasePGSAOverview._get_metrics(self)): + if "nb_parallel_query" in DatabasePGSAOverview._get_metrics(self): parallel_metrics = ["nb_parallel_query", "nb_parallel_worker"] else: parallel_metrics = [] - pgsa_metrics = DatabasePGSAOverview.split(self, - [["backend_xid_age", - "backend_xmin_age", - "oldest_backend", - "oldest_xact", - "oldest_query"], - parallel_metrics]) - db_graphs.append([Graph("Global activity (On database %(database)s)", - metrics=pgsa_metrics[0], - renderer="bar", - stack=True)]) - if (len(pgsa_metrics[2]) > 0): - db_graphs[1].append(Graph("Parallel query (On database %(database)s)", - metrics=pgsa_metrics[2])) - db_graphs[1].append(Graph("Backend age (On database %(database)s)", - metrics=pgsa_metrics[1])) + pgsa_metrics = DatabasePGSAOverview.split( + self, + [ + [ + "backend_xid_age", + "backend_xmin_age", + "oldest_backend", + "oldest_xact", + "oldest_query", + ], + parallel_metrics, + ], + ) + db_graphs.append( + [ + Graph( + "Global activity (On database %(database)s)", + metrics=pgsa_metrics[0], + renderer="bar", + stack=True, + ) + ] + ) + if len(pgsa_metrics[2]) > 0: + db_graphs[1].append( + Graph( + "Parallel query (On database %(database)s)", + metrics=pgsa_metrics[2], + ) + ) + db_graphs[1].append( + Graph( + "Backend age (On database %(database)s)", + metrics=pgsa_metrics[1], + ) + ) graphs_dash = [Dashboard("General Overview", db_graphs)] graphs = [TabContainer("All databases", graphs_dash)] if pgss18: # Add WALs graphs - wals_graphs = [[Graph("WAL activity", - metrics=[DatabaseOverviewMetricGroup.wal_records, - DatabaseOverviewMetricGroup.wal_fpi, - DatabaseOverviewMetricGroup.wal_bytes]), - ]] + wals_graphs = [ + [ + Graph( + "WAL activity", + metrics=[ + DatabaseOverviewMetricGroup.wal_records, + DatabaseOverviewMetricGroup.wal_fpi, + DatabaseOverviewMetricGroup.wal_bytes, + ], + ), + ] + ] graphs_dash.append(Dashboard("WALs", wals_graphs)) # Add JIT graphs if pgss110: - jit_tim = [DatabaseOverviewMetricGroup.jit_inlining_time, - DatabaseOverviewMetricGroup.jit_optimization_time, - DatabaseOverviewMetricGroup.jit_emission_time, - ] + jit_tim = [ + DatabaseOverviewMetricGroup.jit_inlining_time, + DatabaseOverviewMetricGroup.jit_optimization_time, + DatabaseOverviewMetricGroup.jit_emission_time, + ] if pgss111: - jit_tim.extend([DatabaseOverviewMetricGroup.jit_deform_time, - DatabaseOverviewMetricGroup.jit_expr_time]) + jit_tim.extend( + [ + DatabaseOverviewMetricGroup.jit_deform_time, + DatabaseOverviewMetricGroup.jit_expr_time, + ] + ) else: jit_tim.append(DatabaseOverviewMetricGroup.jit_generation_time) - jit_cnt = [DatabaseOverviewMetricGroup.jit_functions, - DatabaseOverviewMetricGroup.jit_inlining_count, - DatabaseOverviewMetricGroup.jit_optimization_count, - DatabaseOverviewMetricGroup.jit_emission_count, - ] + jit_cnt = [ + DatabaseOverviewMetricGroup.jit_functions, + DatabaseOverviewMetricGroup.jit_inlining_count, + DatabaseOverviewMetricGroup.jit_optimization_count, + DatabaseOverviewMetricGroup.jit_emission_count, + ] if pgss111: jit_cnt.append(DatabaseOverviewMetricGroup.jit_deform_count) - jit_graphs = [[Graph("JIT timing", metrics=jit_tim, - stack=True)], - [Graph("JIT scheduling", metrics=jit_cnt)]] + jit_graphs = [ + [Graph("JIT timing", metrics=jit_tim, stack=True)], + [Graph("JIT scheduling", metrics=jit_cnt)], + ] graphs_dash.append(Dashboard("JIT", jit_graphs)) # Add pg_stat_database graphs - global_db_graphs = [[Graph("Transactions per second", - metrics=[DatabaseDbActivityMetricGroup.xact_commit, - DatabaseDbActivityMetricGroup.xact_rollback], - renderer="bar", - stack=True), - Graph("Conflicts & deadlocks", - metrics=[DatabaseDbActivityMetricGroup.conflicts, - DatabaseDbActivityMetricGroup.deadlocks])]] - if ("sessions" in DatabaseDbActivityMetricGroup._get_metrics(self)): - global_db_graphs.append([Graph("Cumulated time per second", - metrics=[DatabaseDbActivityMetricGroup.session_time, - DatabaseDbActivityMetricGroup.active_time, - DatabaseDbActivityMetricGroup.idle_in_transaction_time]), - Graph("Sessions per second", - metrics=[DatabaseDbActivityMetricGroup.sessions, - DatabaseDbActivityMetricGroup.sessions_abandoned, - DatabaseDbActivityMetricGroup.sessions_fatal, - DatabaseDbActivityMetricGroup.sessions_killed]) - ]) + global_db_graphs = [ + [ + Graph( + "Transactions per second", + metrics=[ + DatabaseDbActivityMetricGroup.xact_commit, + DatabaseDbActivityMetricGroup.xact_rollback, + ], + renderer="bar", + stack=True, + ), + Graph( + "Conflicts & deadlocks", + metrics=[ + DatabaseDbActivityMetricGroup.conflicts, + DatabaseDbActivityMetricGroup.deadlocks, + ], + ), + ] + ] + if "sessions" in DatabaseDbActivityMetricGroup._get_metrics(self): + global_db_graphs.append( + [ + Graph( + "Cumulated time per second", + metrics=[ + DatabaseDbActivityMetricGroup.session_time, + DatabaseDbActivityMetricGroup.active_time, + DatabaseDbActivityMetricGroup.idle_in_transaction_time, + ], + ), + Graph( + "Sessions per second", + metrics=[ + DatabaseDbActivityMetricGroup.sessions, + DatabaseDbActivityMetricGroup.sessions_abandoned, + DatabaseDbActivityMetricGroup.sessions_fatal, + DatabaseDbActivityMetricGroup.sessions_killed, + ], + ), + ] + ) graphs_dash.append(Dashboard("Database activity", global_db_graphs)) # Add powa_stat_all_relations graphs - all_rel_graphs = [[Graph("Access pattern", - metrics=[DatabaseAllRelMetricGroup.seq_scan, - DatabaseAllRelMetricGroup.idx_scan, - DatabaseAllRelMetricGroup.idx_ratio]), - Graph("DML activity", - metrics=[DatabaseAllRelMetricGroup.n_tup_del, - DatabaseAllRelMetricGroup.n_tup_hot_upd, - DatabaseAllRelMetricGroup.n_tup_upd, - DatabaseAllRelMetricGroup.n_tup_ins]) - ], - [Graph("Vacuum activity", - metrics=[DatabaseAllRelMetricGroup.autoanalyze_count, - DatabaseAllRelMetricGroup.analyze_count, - DatabaseAllRelMetricGroup.autovacuum_count, - DatabaseAllRelMetricGroup.vacuum_count]), - Graph("Object size", - metrics=[DatabaseAllRelMetricGroup.tbl_size, - DatabaseAllRelMetricGroup.idx_size], - renderer="bar", - stack=True, - color_scheme=['#73c03a','#65b9ac'])]] + all_rel_graphs = [ + [ + Graph( + "Access pattern", + metrics=[ + DatabaseAllRelMetricGroup.seq_scan, + DatabaseAllRelMetricGroup.idx_scan, + DatabaseAllRelMetricGroup.idx_ratio, + ], + ), + Graph( + "DML activity", + metrics=[ + DatabaseAllRelMetricGroup.n_tup_del, + DatabaseAllRelMetricGroup.n_tup_hot_upd, + DatabaseAllRelMetricGroup.n_tup_upd, + DatabaseAllRelMetricGroup.n_tup_ins, + ], + ), + ], + [ + Graph( + "Vacuum activity", + metrics=[ + DatabaseAllRelMetricGroup.autoanalyze_count, + DatabaseAllRelMetricGroup.analyze_count, + DatabaseAllRelMetricGroup.autovacuum_count, + DatabaseAllRelMetricGroup.vacuum_count, + ], + ), + Graph( + "Object size", + metrics=[ + DatabaseAllRelMetricGroup.tbl_size, + DatabaseAllRelMetricGroup.idx_size, + ], + renderer="bar", + stack=True, + color_scheme=["#73c03a", "#65b9ac"], + ), + ], + ] graphs_dash.append(Dashboard("Database Objects", all_rel_graphs)) # Add powa_stat_user_functions graphs - user_fct_graph = [Graph("User functions activity", - metrics=DatabaseUserFuncMetricGroup.all(self))] - user_fct_grid = [Grid("User functions activity", - columns=[{ - "name": "func_name", - "label": "Function name", - "url_attr": "url" - }], - metrics=ByFuncUserFuncMetricGroup.all(self))] - graphs_dash.append(Dashboard("User functions", [user_fct_graph, - user_fct_grid])) + user_fct_graph = [ + Graph( + "User functions activity", + metrics=DatabaseUserFuncMetricGroup.all(self), + ) + ] + user_fct_grid = [ + Grid( + "User functions activity", + columns=[ + { + "name": "func_name", + "label": "Function name", + "url_attr": "url", + } + ], + metrics=ByFuncUserFuncMetricGroup.all(self), + ) + ] + graphs_dash.append( + Dashboard("User functions", [user_fct_graph, user_fct_grid]) + ) if self.has_extension(self.path_args[0], "pg_stat_kcache"): - block_graph.metrics.insert(0, DatabaseOverviewMetricGroup. - total_sys_hit) - block_graph.metrics.insert(0, DatabaseOverviewMetricGroup. - total_disk_read) - block_graph.color_scheme = ['#cb513a', '#65b9ac', '#73c03a'] - - sys_graphs = [Graph("System resources (events per sec)", - url=self.docs_stats_url + "pg_stat_kcache.html", - metrics=[DatabaseOverviewMetricGroup.majflts, - DatabaseOverviewMetricGroup.minflts, - # DatabaseOverviewMetricGroup.nswaps, - # DatabaseOverviewMetricGroup.msgsnds, - # DatabaseOverviewMetricGroup.msgrcvs, - # DatabaseOverviewMetricGroup.nsignals, - DatabaseOverviewMetricGroup.nvcsws, - DatabaseOverviewMetricGroup.nivcsws])] + block_graph.metrics.insert( + 0, DatabaseOverviewMetricGroup.total_sys_hit + ) + block_graph.metrics.insert( + 0, DatabaseOverviewMetricGroup.total_disk_read + ) + block_graph.color_scheme = ["#cb513a", "#65b9ac", "#73c03a"] + + sys_graphs = [ + Graph( + "System resources (events per sec)", + url=self.docs_stats_url + "pg_stat_kcache.html", + metrics=[ + DatabaseOverviewMetricGroup.majflts, + DatabaseOverviewMetricGroup.minflts, + # DatabaseOverviewMetricGroup.nswaps, + # DatabaseOverviewMetricGroup.msgsnds, + # DatabaseOverviewMetricGroup.msgrcvs, + # DatabaseOverviewMetricGroup.nsignals, + DatabaseOverviewMetricGroup.nvcsws, + DatabaseOverviewMetricGroup.nivcsws, + ], + ) + ] graphs_dash.append(Dashboard("System resources", [sys_graphs])) else: - block_graph.metrics.insert(0, DatabaseOverviewMetricGroup. - total_blks_read) - block_graph.color_scheme = ['#cb513a', '#73c03a'] + block_graph.metrics.insert( + 0, DatabaseOverviewMetricGroup.total_blks_read + ) + block_graph.color_scheme = ["#cb513a", "#73c03a"] - if (self.has_extension(self.path_args[0], "pg_wait_sampling")): + if self.has_extension(self.path_args[0], "pg_wait_sampling"): metrics = None pg_version_num = self.get_pg_version_num(self.path_args[0]) # if we can't connect to the remote server, assume pg10 or above if pg_version_num is not None and pg_version_num < 100000: - metrics = [DatabaseWaitOverviewMetricGroup.count_lwlocknamed, - DatabaseWaitOverviewMetricGroup.count_lwlocktranche, - DatabaseWaitOverviewMetricGroup.count_lock, - DatabaseWaitOverviewMetricGroup.count_bufferpin] + metrics = [ + DatabaseWaitOverviewMetricGroup.count_lwlocknamed, + DatabaseWaitOverviewMetricGroup.count_lwlocktranche, + DatabaseWaitOverviewMetricGroup.count_lock, + DatabaseWaitOverviewMetricGroup.count_bufferpin, + ] else: - metrics = [DatabaseWaitOverviewMetricGroup.count_lwlock, - DatabaseWaitOverviewMetricGroup.count_lock, - DatabaseWaitOverviewMetricGroup.count_bufferpin, - DatabaseWaitOverviewMetricGroup.count_activity, - DatabaseWaitOverviewMetricGroup.count_client, - DatabaseWaitOverviewMetricGroup.count_extension, - DatabaseWaitOverviewMetricGroup.count_ipc, - DatabaseWaitOverviewMetricGroup.count_timeout, - DatabaseWaitOverviewMetricGroup.count_io] - - graphs_dash.append(Dashboard("Wait Events", - [[Graph("Wait Events (per second)", - url=self.docs_stats_url + "pg_wait_sampling.html", - metrics=metrics)]])) - - toprow = [{ - # query - }] + metrics = [ + DatabaseWaitOverviewMetricGroup.count_lwlock, + DatabaseWaitOverviewMetricGroup.count_lock, + DatabaseWaitOverviewMetricGroup.count_bufferpin, + DatabaseWaitOverviewMetricGroup.count_activity, + DatabaseWaitOverviewMetricGroup.count_client, + DatabaseWaitOverviewMetricGroup.count_extension, + DatabaseWaitOverviewMetricGroup.count_ipc, + DatabaseWaitOverviewMetricGroup.count_timeout, + DatabaseWaitOverviewMetricGroup.count_io, + ] + + graphs_dash.append( + Dashboard( + "Wait Events", + [ + [ + Graph( + "Wait Events (per second)", + url=self.docs_stats_url + + "pg_wait_sampling.html", + metrics=metrics, + ) + ] + ], + ) + ) + + toprow = [ + { + # query + } + ] if pgss18: - toprow.extend([{ - # plan time - }]) - - toprow.extend([{ - 'name': 'Execution', - 'colspan': 3 - }, { - 'name': 'Blocks', - 'colspan': 4, - }, { - 'name': 'Temp blocks', - 'colspan': 2 - }, { - 'name': 'I/O Time', - 'colspan': 2 - }]) + toprow.extend( + [ + { + # plan time + } + ] + ) + + toprow.extend( + [ + {"name": "Execution", "colspan": 3}, + { + "name": "Blocks", + "colspan": 4, + }, + {"name": "Temp blocks", "colspan": 2}, + {"name": "I/O Time", "colspan": 2}, + ] + ) if pgss18: - toprow.extend([{ - 'name': 'WALs', - 'colspan': 3 - }]) + toprow.extend([{"name": "WALs", "colspan": 3}]) if pgss110: - toprow.extend([{ - 'name': 'JIT', - 'colspan': 2 - }]) + toprow.extend([{"name": "JIT", "colspan": 2}]) self._dashboard.widgets.extend( - [graphs, - [Grid("Details for all queries", - toprow=toprow, - columns=[{ - "name": "query", - "label": "Query", - "type": "query", - "url_attr": "url", - "max_length": 70 - }], - metrics=ByQueryMetricGroup.all(self))]]) + [ + graphs, + [ + Grid( + "Details for all queries", + toprow=toprow, + columns=[ + { + "name": "query", + "label": "Query", + "type": "query", + "url_attr": "url", + "max_length": 70, + } + ], + metrics=ByQueryMetricGroup.all(self), + ) + ], + ] + ) if self.has_extension(self.path_args[0], "pg_wait_sampling"): - self._dashboard.widgets.extend([[ - Grid("Wait events for all queries", - url=self.docs_stats_url + "pg_wait_sampling.html", - columns=[{ - "name": "query", - "label": "Query", - "type": "query", - "url_attr": "url", - "max_length": 70 - }, { - "name": "event_type", - "label": "Event Type", - }, { - "name": "event", - "label": "Event", - }], - metrics=ByQueryWaitSamplingMetricGroup.all())]]) + self._dashboard.widgets.extend( + [ + [ + Grid( + "Wait events for all queries", + url=self.docs_stats_url + "pg_wait_sampling.html", + columns=[ + { + "name": "query", + "label": "Query", + "type": "query", + "url_attr": "url", + "max_length": 70, + }, + { + "name": "event_type", + "label": "Event Type", + }, + { + "name": "event", + "label": "Event", + }, + ], + metrics=ByQueryWaitSamplingMetricGroup.all(), + ) + ] + ] + ) self._dashboard.widgets.extend([[Wizard("Index suggestions")]]) return self._dashboard diff --git a/powa/framework.py b/powa/framework.py index 2423ec68..a2c5ade3 100644 --- a/powa/framework.py +++ b/powa/framework.py @@ -1,21 +1,22 @@ """ Utilities for the basis of Powa """ -from collections import defaultdict -from tornado.web import RequestHandler, authenticated, HTTPError -from powa.json import JSONizable -from powa import ui_methods -from powa.json import to_json -import psycopg2 -from psycopg2.extensions import connection as _connection, cursor as _cursor -from psycopg2.extras import RealDictCursor -from tornado.options import options -import pickle + import logging +import pickle +import psycopg2 import random import re import select import time +from collections import defaultdict +from powa import ui_methods +from powa.json import JSONizable, to_json +from psycopg2.extensions import connection as _connection +from psycopg2.extensions import cursor as _cursor +from psycopg2.extras import RealDictCursor +from tornado.options import options +from tornado.web import HTTPError, RequestHandler, authenticated class CustomConnection(_connection): @@ -33,6 +34,7 @@ class CustomConnection(_connection): All you need to do is pass query strings of the form SELECT ... FROM {extension_name}.some_relation ... """ + def initialize(self, logger, srvid, dsn, encoding_query, debug): self._logger = logger self._srvid = srvid or 0 @@ -40,15 +42,15 @@ def initialize(self, logger, srvid, dsn, encoding_query, debug): self._debug = debug if encoding_query is not None: - self.set_client_encoding(encoding_query['client_encoding']) + self.set_client_encoding(encoding_query["client_encoding"]) def cursor(self, *args, **kwargs): - factory = kwargs.get('cursor_factory') + factory = kwargs.get("cursor_factory") if factory is None: - kwargs['cursor_factory'] = CustomCursor + kwargs["cursor_factory"] = CustomCursor elif factory == RealDictCursor: - kwargs['cursor_factory'] = CustomDictCursor + kwargs["cursor_factory"] = CustomDictCursor else: msg = "Unsupported cursor_factory: %s" % factory.__name__ self._logger.error(msg) @@ -85,7 +87,7 @@ def execute(self, query, params=None): def resolve_nsps(query, connection): - if hasattr(connection, '_nsps'): + if hasattr(connection, "_nsps"): return query.format(**connection._nsps) return query @@ -94,7 +96,7 @@ def resolve_nsps(query, connection): def log_query(cls, query, params=None, exception=None): t = round((time.time() - cls.timestamp) * 1000, 2) - fmt = '' + fmt = "" if exception is not None: fmt = "Error during query execution:\n{}\n".format(exception) @@ -102,9 +104,15 @@ def log_query(cls, query, params=None, exception=None): if params is not None: fmt += "\n{params}" - cls.connection._logger.debug(fmt.format(ms=t, query=query, params=params, - dsn=cls.connection._dsn, - srvid=cls.connection._srvid)) + cls.connection._logger.debug( + fmt.format( + ms=t, + query=query, + params=params, + dsn=cls.connection._dsn, + srvid=cls.connection._srvid, + ) + ) class BaseHandler(RequestHandler, JSONizable): @@ -119,15 +127,20 @@ def __init__(self, *args, **kwargs): self._databases = None self._servers = None self._connections = {} - self._ext_versions = defaultdict(lambda : defaultdict(dict)) + self._ext_versions = defaultdict(lambda: defaultdict(dict)) self.url_prefix = options.url_prefix self.logger = logging.getLogger("tornado.application") - if self.application.settings['debug']: + if self.application.settings["debug"]: self.logger.setLevel(logging.DEBUG) def __get_url(self, **connoptions): - url = ' '.join(['%s=%s' % (k, v) - for (k, v) in connoptions.items() if v is not None]) + url = " ".join( + [ + "%s=%s" % (k, v) + for (k, v) in connoptions.items() + if v is not None + ] + ) return url @@ -136,14 +149,14 @@ def __get_safe_dsn(self, **connoptions): Return a simplified dsn that won't leak the password if provided in the options, for logging purpose. """ - dsn = '{user}@{host}:{port}/{database}'.format(**connoptions) + dsn = "{user}@{host}:{port}/{database}".format(**connoptions) return dsn def render_json(self, value): """ Render the object as json response. """ - self.set_header('Content-Type', 'application/json') + self.set_header("Content-Type", "application/json") self.write(to_json(value)) @property @@ -152,11 +165,11 @@ def current_user(self): Return the current_user if he is allowed to connect to his server of choice. """ - raw = self.get_str_cookie('user') + raw = self.get_str_cookie("user") if raw is not None: try: self.connect() - return raw or 'anonymous' + return raw or "anonymous" except Exception: return None @@ -166,7 +179,7 @@ def current_server(self): Return the server connected to if any """ try: - return self.get_secure_cookie('server').decode('utf-8') + return self.get_secure_cookie("server").decode("utf-8") except AttributeError: return None @@ -180,8 +193,8 @@ def current_host(self): return None connoptions = options.servers[server].copy() host = "localhost" - if 'host' in connoptions: - host = connoptions['host'] + if "host" in connoptions: + host = connoptions["host"] return "%s" % (host) @property @@ -194,8 +207,8 @@ def current_port(self): return None connoptions = options.servers[server].copy() port = "5432" - if 'port' in connoptions: - port = connoptions['port'] + if "port" in connoptions: + port = connoptions["port"] return "%s" % (port) @property @@ -209,11 +222,11 @@ def current_connection(self): connoptions = options.servers[server].copy() host = "localhost" port = "5432" - if 'host' in connoptions: - host = connoptions['host'] - if 'port' in connoptions: - port = connoptions['port'] - return "%s:%s" % ( host, port ) + if "host" in connoptions: + host = connoptions["host"] + if "port" in connoptions: + port = connoptions["port"] + return "%s:%s" % (host, port) @property def database(self): @@ -239,19 +252,26 @@ def get_powa_version(self, **kwargs): SELECT regexp_replace(extversion, '(dev|beta\d*)', '') AS version FROM pg_extension WHERE extname = 'powa' - """, **kwargs)[0]['version'] + """, + **kwargs, + )[0]["version"] if version is None: return None - return [int(part) for part in version.split('.')] + return [int(part) for part in version.split(".")] def get_pg_version_num(self, srvid=None, **kwargs): try: - return int(self.execute( - """ + return int( + self.execute( + """ SELECT setting FROM pg_settings WHERE name = 'server_version_num' - """, srvid=srvid, **kwargs)[0]['setting']) + """, + srvid=srvid, + **kwargs, + )[0]["setting"] + ) except Exception: return None @@ -261,29 +281,36 @@ def get_databases(self, srvid): """ if self.current_user: if self._databases is None: - self._databases = [d['datname'] for d in self.execute( - """ + self._databases = [ + d["datname"] + for d in self.execute( + """ SELECT p.datname FROM {powa}.powa_databases p LEFT JOIN pg_database d ON p.oid = d.oid WHERE COALESCE(datallowconn, true) AND srvid = %(srvid)s ORDER BY DATNAME - """, params={'srvid': srvid})] + """, + params={"srvid": srvid}, + ) + ] return self._databases def deparse_srvid(self, srvid): - if (srvid == '0'): + if srvid == "0": return self.current_connection else: - return self.execute(""" + return self.execute( + """ SELECT COALESCE(alias, hostname || ':' || port) AS server FROM {powa}.powa_servers WHERE id = %(srvid)s - """, params={'srvid': int(srvid)} - )[0]['server'] + """, + params={"srvid": int(srvid)}, + )[0]["server"] @property def servers(self, **kwargs): @@ -292,8 +319,10 @@ def servers(self, **kwargs): """ if self.current_user: if self._servers is None: - self._servers = [[s['id'], s['val'], s['alias']] for s in self.execute( - """ + self._servers = [ + [s["id"], s["val"], s["alias"]] + for s in self.execute( + """ SELECT s.id, CASE WHEN s.id = 0 THEN %(default)s ELSE @@ -302,32 +331,42 @@ def servers(self, **kwargs): s.alias FROM {powa}.powa_servers s ORDER BY hostname - """, params={'default': self.current_connection})] + """, + params={"default": self.current_connection}, + ) + ] return self._servers def on_finish(self): for conn in self._connections.values(): conn.close() - def connect(self, srvid=None, server=None, user=None, password=None, - database=None, remote_access=False, **kwargs): + def connect( + self, + srvid=None, + server=None, + user=None, + password=None, + database=None, + remote_access=False, + **kwargs, + ): """ Connect to a specific database. Parameters default values are taken from the cookies and the server configuration file. """ - if (srvid is not None and srvid != "0"): + if srvid is not None and srvid != "0": remote_access = True # Check for global connection restriction first - if (remote_access and not options['allow_ui_connection']): + if remote_access and not options["allow_ui_connection"]: raise Exception("UI connection globally not allowed.") conn_allowed = None - server = server or self.get_str_cookie('server') - user = user or self.get_str_cookie('user') - password = (password or - self.get_str_cookie('password')) + server = server or self.get_str_cookie("server") + user = user or self.get_str_cookie("user") + password = password or self.get_str_cookie("password") if server not in options.servers: raise HTTPError(404, "Server %s not found." % server) @@ -338,44 +377,53 @@ def connect(self, srvid=None, server=None, user=None, password=None, encoding_query = connoptions.pop("query", None) if encoding_query is not None: if not isinstance(encoding_query, dict): - raise Exception('Invalid "query" parameter: %r, ' % - encoding_query) + raise Exception( + 'Invalid "query" parameter: %r, ' % encoding_query + ) for k in encoding_query: if k != "client_encoding": - raise Exception('Invalid "query" parameter: %r", ' - 'unexpected key "%s"'% - (encoding_query, k)) + raise Exception( + 'Invalid "query" parameter: %r", ' + 'unexpected key "%s"' % (encoding_query, k) + ) - if (srvid is not None and srvid != "0"): + if srvid is not None and srvid != "0": tmp = self.connect() cur = tmp.cursor(cursor_factory=RealDictCursor) - cur.execute(""" + cur.execute( + """ SELECT hostname, port, username, password, dbname, allow_ui_connection FROM {powa}.powa_servers WHERE id = %(srvid)s - """, {'srvid': srvid}) + """, + {"srvid": srvid}, + ) row = cur.fetchone() cur.close() - connoptions['host'] = row['hostname'] - connoptions['port'] = row['port'] - connoptions['user'] = row['username'] - connoptions['password'] = row['password'] - connoptions['database'] = row['dbname'] - conn_allowed = row['allow_ui_connection'] + connoptions["host"] = row["hostname"] + connoptions["port"] = row["port"] + connoptions["user"] = row["username"] + connoptions["password"] = row["password"] + connoptions["database"] = row["dbname"] + conn_allowed = row["allow_ui_connection"] else: - if 'user' not in connoptions: - connoptions['user'] = user - if 'password' not in connoptions: - connoptions['password'] = password + if "user" not in connoptions: + connoptions["user"] = user + if "password" not in connoptions: + connoptions["password"] = password # If a non-powa connection is requested, check if we're allowed - if (remote_access): + if remote_access: # authorization check for local connection has not been done yet - if (conn_allowed is None): - tmp = self.connect(remote_access=False, server=server, - user=user, password=password) + if conn_allowed is None: + tmp = self.connect( + remote_access=False, + server=server, + user=user, + password=password, + ) cur = tmp.cursor() cur.execute(""" SELECT allow_ui_connection @@ -385,21 +433,26 @@ def connect(self, srvid=None, server=None, user=None, password=None, cur.close() conn_allowed = row[0] - if (not conn_allowed): + if not conn_allowed: raise Exception("UI connection not allowed for this server.") if database is not None: - connoptions['database'] = database + connoptions["database"] = database url = self.__get_url(**connoptions) if url in self._connections: return self._connections.get(url) - conn = psycopg2.connect(connection_factory=CustomConnection, - **connoptions) - conn.initialize(self.logger, srvid, self.__get_safe_dsn(**connoptions), - encoding_query, - self.application.settings['debug']) + conn = psycopg2.connect( + connection_factory=CustomConnection, **connoptions + ) + conn.initialize( + self.logger, + srvid, + self.__get_safe_dsn(**connoptions), + encoding_query, + self.application.settings["debug"], + ) # Get and cache all extensions schemas, in a dict with the extension # name as the key and the *quoted* schema as the value. @@ -417,8 +470,9 @@ def connect(self, srvid=None, server=None, user=None, password=None, self._connections[url] = conn return self._connections[url] - def __get_extension_version(self, srvid, extname, database=None, - remote_access=True): + def __get_extension_version( + self, srvid, extname, database=None, remote_access=True + ): """ Returns a tuple with all digits of the version of the specific extension on the specific server and database, or None if the extension @@ -432,14 +486,14 @@ def __get_extension_version(self, srvid, extname, database=None, # Check for a cached version first. Note that we do cache the lack of # extension (storing None), so we use an empty string to detect that no # caching happened yet. - remver = self._ext_versions[srvid][database].get(extname, '') + remver = self._ext_versions[srvid][database].get(extname, "") - if remver != '': + if remver != "": return remver # For remote server, check first if powa-collector reported a version # for that extension, but only for default database. - if (srvid != "0" and database is None): + if srvid != "0" and database is None: try: remver = self.execute( """ @@ -447,8 +501,9 @@ def __get_extension_version(self, srvid, extname, database=None, FROM {powa}.powa_extension_config WHERE srvid = %(srvid)s AND extname = %(extname)s - """, params={'srvid': srvid, 'extname': extname} - )[0]['version'] + """, + params={"srvid": srvid, "extname": extname}, + )[0]["version"] except Exception: return None @@ -461,9 +516,12 @@ def __get_extension_version(self, srvid, extname, database=None, FROM pg_catalog.pg_extension WHERE extname = %(extname)s LIMIT 1 - """, srvid=srvid, database=database, - params={"extname": extname}, remote_access=remote_access - )[0]['extversion'] + """, + srvid=srvid, + database=database, + params={"extname": extname}, + remote_access=remote_access, + )[0]["extversion"] except Exception: return None @@ -472,13 +530,13 @@ def __get_extension_version(self, srvid, extname, database=None, return None # Clean up any extraneous characters - remver = re.search(r'[0-9\.]*[0-9]', remver) + remver = re.search(r"[0-9\.]*[0-9]", remver) if remver is None: self._ext_versions[srvid][database][extname] = None return None - remver = tuple(map(int, remver.group(0).split('.'))) + remver = tuple(map(int, remver.group(0).split("."))) self._ext_versions[srvid][database][extname] = remver return remver @@ -495,28 +553,33 @@ def has_extension(self, srvid, extname): the powa-web server. This assumes that "module" is the name of the underlying extension. """ - if (srvid == '0' or srvid == 0): + if srvid == "0" or srvid == 0: # if local server, fallback to the full test, as it won't be more # expensive - return self.has_extension_version(srvid, extname, '0', - remote_access=False) + return self.has_extension_version( + srvid, extname, "0", remote_access=False + ) else: try: # Look for at least an enabled snapshot function. If a module # provides multiple snapshot functions and only a subset is # activated, let's assume that the extension is available. - return self.execute(""" + return self.execute( + """ SELECT COUNT(*) != 0 AS res FROM {powa}.powa_functions WHERE srvid = %(srvid)s AND name = %(extname)s AND enabled - """, params={"srvid": srvid, "extname": extname})[0]['res'] + """, + params={"srvid": srvid, "extname": extname}, + )[0]["res"] except Exception: return False - def has_extension_version(self, srvid, extname, version, database=None, - remote_access=True): + def has_extension_version( + self, srvid, extname, version, database=None, remote_access=True + ): """ Returns whether the version of the specific extension on the specific server and database is at least the given version. @@ -524,13 +587,14 @@ def has_extension_version(self, srvid, extname, version, database=None, if version is None: raise Exception("No version provided!") - remver = self.__get_extension_version(srvid, extname, - remote_access=remote_access) + remver = self.__get_extension_version( + srvid, extname, remote_access=remote_access + ) if remver is None: return False - wanted = tuple(map(int, version.split('.'))) + wanted = tuple(map(int, version.split("."))) return remver >= wanted @@ -547,22 +611,29 @@ def write_error(self, status_code, **kwargs): return super(BaseHandler, self).write_error(status_code, **kwargs) - def execute(self, query, srvid=None, params=None, server=None, - user=None, - database=None, - password=None, - remote_access=False): + def execute( + self, + query, + srvid=None, + params=None, + server=None, + user=None, + database=None, + password=None, + remote_access=False, + ): """ Execute a query against a database, with specific bind parameters. """ if params is None: params = {} - if 'samples' not in params: - params['samples'] = 100 + if "samples" not in params: + params["samples"] = 100 - conn = self.connect(srvid, server, user, password, database, - remote_access) + conn = self.connect( + srvid, server, user, password, database, remote_access + ) cur = conn.cursor(cursor_factory=RealDictCursor) cur.execute("SAVEPOINT powa_web") @@ -602,8 +673,8 @@ def notify_collector(self, command, args=[], timeout=3): # some commands will contain user-provided strings, so we need to # properly escape the arguments. - payload = "%s %s %s" % (command, channel, ' '.join(args)) - cur.execute("NOTIFY powa_collector, %s", (payload, )) + payload = "%s %s %s" % (command, channel, " ".join(args)) + cur.execute("NOTIFY powa_collector, %s", (payload,)) cur.close() conn.commit() @@ -619,18 +690,18 @@ def notify_collector(self, command, args=[], timeout=3): conn.poll() res = [] - while (conn.notifies): + while conn.notifies: notif = conn.notifies.pop(0) - payload = notif.payload.split(' ') + payload = notif.payload.split(" ") received_command = payload.pop(0) # we shouldn't received unexpected messages, but ignore them if any - if (received_command != command): + if received_command != command: continue status = payload.pop(0) - payload = ' '.join(payload) + payload = " ".join(payload) # we should get a single answer, but if multiple are received # append them and let the caller handle it. @@ -652,7 +723,7 @@ def get_pickle_cookie(self, name): def get_str_cookie(self, name, default=None): value = self.get_secure_cookie(name) if value is not None: - return value.decode('utf8') + return value.decode("utf8") return default def set_pickle_cookie(self, name, value): @@ -674,9 +745,9 @@ def to_json(self): "server": self.server, "version": ui_methods.version(None), "year": ui_methods.year(None), - "configUrl": self.reverse_url('RepositoryConfigOverview'), - "logoUrl": self.static_url('img/favicon/favicon-32x32.png'), - "homeUrl": self.reverse_url('Overview'), + "configUrl": self.reverse_url("RepositoryConfigOverview"), + "logoUrl": self.static_url("img/favicon/favicon-32x32.png"), + "homeUrl": self.reverse_url("Overview"), } @@ -689,8 +760,8 @@ class AuthHandler(BaseHandler): def prepare(self): super(AuthHandler, self).prepare() - def to_json(self): - return dict(**super(AuthHandler, self).to_json(), **{ - "logoutUrl": self.reverse_url('logout') - }) + return dict( + **super(AuthHandler, self).to_json(), + **{"logoutUrl": self.reverse_url("logout")}, + ) diff --git a/powa/function.py b/powa/function.py index fc99c510..017c06fe 100644 --- a/powa/function.py +++ b/powa/function.py @@ -2,7 +2,7 @@ Dashboard for the by-function page. """ -from powa.dashboards import (Dashboard, DashboardPage, ContentWidget) +from powa.dashboards import ContentWidget, Dashboard, DashboardPage from powa.database import DatabaseOverview from powa.sql.views_grid import powa_getuserfuncdata_detailed_db @@ -11,19 +11,23 @@ class FunctionDetail(ContentWidget): """ Detail widget showing summarized information for the function. """ + title = "Function Detail" data_url = r"/server/(\d+)/metrics/database/([^\/]+)/function/(\d+)/detail" def get(self, server, database, function): stmt = powa_getuserfuncdata_detailed_db("%(funcid)s") - value = self.execute(stmt, params={ - "server": server, - "database": database, - "funcid": function, - "from": self.get_argument("from"), - "to": self.get_argument("to") - }) + value = self.execute( + stmt, + params={ + "server": server, + "database": database, + "funcid": function, + "from": self.get_argument("from"), + "to": self.get_argument("to"), + }, + ) if value is None or len(value) < 1: self.render_json(None) return @@ -34,19 +38,22 @@ class FunctionOverview(DashboardPage): """ Dashboard page for a function. """ + base_url = r"/server/(\d+)/database/([^\/]+)/function/(\d+)/overview" params = ["server", "database", "function"] datasources = [FunctionDetail] parent = DatabaseOverview - title = 'Function overview' + title = "Function overview" def dashboard(self): # This COULD be initialized in the constructor, but tornado < 3 doesn't # call it - if getattr(self, '_dashboard', None) is not None: + if getattr(self, "_dashboard", None) is not None: return self._dashboard - self._dashboard = Dashboard("Function %(function)s on database %(database)s", - [[FunctionDetail]]) + self._dashboard = Dashboard( + "Function %(function)s on database %(database)s", + [[FunctionDetail]], + ) return self._dashboard diff --git a/powa/io.py b/powa/io.py index 60d262c9..46745e43 100644 --- a/powa/io.py +++ b/powa/io.py @@ -1,17 +1,18 @@ """ Dashboards for the various IO pages. """ -from powa.dashboards import Dashboard, Graph, Grid, DashboardPage -from powa.config import ConfigChangesGlobal -from powa.server import ServerOverview +from powa.config import ConfigChangesGlobal +from powa.dashboards import Dashboard, DashboardPage, Graph, Grid from powa.io_template import TemplateIoGraph, TemplateIoGrid +from powa.server import ServerOverview class TemplateIoOverview(DashboardPage): """ Template dashboard for IO. """ + parent = ServerOverview timeline = ConfigChangesGlobal timeline_params = ["server"] @@ -19,43 +20,59 @@ class TemplateIoOverview(DashboardPage): def dashboard(self): # This COULD be initialized in the constructor, but tornado < 3 doesn't # call it - if getattr(self, '_dashboard', None) is not None: + if getattr(self, "_dashboard", None) is not None: return self._dashboard - io_metrics = self.ds_graph.split(self, - [["reads", "writes", "writebacks", "extends", "fsyncs"], - ["hits", "evictions", "reuses"]] - ) + io_metrics = self.ds_graph.split( + self, + [ + ["reads", "writes", "writebacks", "extends", "fsyncs"], + ["hits", "evictions", "reuses"], + ], + ) graphs = [] - graphs.append([ - Graph("IO blocks", - metrics=io_metrics[1], - ), - Graph("IO timing", - metrics=io_metrics[0], - ), - Graph("IO misc", - metrics=io_metrics[2], - ), - ]) - - graphs.append([ - Grid("IO summary", - columns=[{ - "name": "backend_type", - "label": "Backend Type", - "url_attr": "backend_type_url" - }, { - "name": "obj", - "label": "Object Type", - "url_attr": "obj_url" - }, { - "name": "context", - "label": "Context", - "url_attr": "context_url" - }], - metrics=self.ds_grid.all()) - ]) + graphs.append( + [ + Graph( + "IO blocks", + metrics=io_metrics[1], + ), + Graph( + "IO timing", + metrics=io_metrics[0], + ), + Graph( + "IO misc", + metrics=io_metrics[2], + ), + ] + ) + + graphs.append( + [ + Grid( + "IO summary", + columns=[ + { + "name": "backend_type", + "label": "Backend Type", + "url_attr": "backend_type_url", + }, + { + "name": "obj", + "label": "Object Type", + "url_attr": "obj_url", + }, + { + "name": "context", + "label": "Context", + "url_attr": "context_url", + }, + ], + metrics=self.ds_grid.all(), + ) + ] + ) self._dashboard = Dashboard(self.title, graphs) return self._dashboard @@ -65,6 +82,7 @@ class ByBackendTypeIoGraph(TemplateIoGraph): """ Metric group used by per backend_type graph. """ + name = "backend_type_io_graph" data_url = r"/server/(\d+)/metrics/backend_type_graph/([a-z0-9%]+)/io/" @@ -75,6 +93,7 @@ class ByBackendTypeIoGrid(TemplateIoGrid): """ Metric group used by per backend_type grid. """ + xaxis = "backend_type" name = "backend_type_io_grid" data_url = r"/server/(\d+)/metrics/backend_type_grid/([a-z0-9%]+)/io/" @@ -86,9 +105,10 @@ class ByBackendTypeIoOverview(TemplateIoOverview): """ Per backend-type Dashboard page. """ + base_url = r"/server/(\d+)/metrics/backend_type/([a-z0-9%]+)/io/overview/" params = ["server", "backend_type"] - title = "IO for \"%(backend_type)s\" backend type" + title = 'IO for "%(backend_type)s" backend type' ds_graph = ByBackendTypeIoGraph ds_grid = ByBackendTypeIoGrid @@ -99,6 +119,7 @@ class ByObjIoGraph(TemplateIoGraph): """ Metric group used by per object graph. """ + name = "obj_io_graph" data_url = r"/server/(\d+)/metrics/obj_graph/([a-z0-9%]+)/io/" @@ -109,6 +130,7 @@ class ByObjIoGrid(TemplateIoGrid): """ Metric group used by per object grid. """ + xaxis = "obj" name = "obj_io_grid" data_url = r"/server/(\d+)/metrics/obj_grid/([a-z0-9%]+)/io/" @@ -120,9 +142,10 @@ class ByObjIoOverview(TemplateIoOverview): """ Per object Dashboard page. """ + base_url = r"/server/(\d+)/metrics/obj/([a-z0-9%]+)/io/overview/" params = ["server", "obj"] - title = "IO for \"%(obj)s\" object" + title = 'IO for "%(obj)s" object' ds_graph = ByObjIoGraph ds_grid = ByObjIoGrid @@ -133,6 +156,7 @@ class ByContextIoGraph(TemplateIoGraph): """ Metric group used by per context graph. """ + name = "context_io_graph" data_url = r"/server/(\d+)/metrics/context_graph/([a-z0-9%]+)/io/" @@ -143,6 +167,7 @@ class ByContextIoGrid(TemplateIoGrid): """ Metric group used by per context grid. """ + xaxis = "context" name = "context_io_grid" data_url = r"/server/(\d+)/metrics/context_grid/([a-z0-9%]+)/io/" @@ -154,9 +179,10 @@ class ByContextIoOverview(TemplateIoOverview): """ Per context Dashboard page. """ + base_url = r"/server/(\d+)/metrics/context/([a-z0-9%]+)/io/overview/" params = ["server", "context"] - title = "IO for \"%(context)s\" context" + title = 'IO for "%(context)s" context' ds_graph = ByContextIoGraph ds_grid = ByContextIoGrid diff --git a/powa/io_template.py b/powa/io_template.py index e9139161..fb298652 100644 --- a/powa/io_template.py +++ b/powa/io_template.py @@ -1,8 +1,8 @@ """ Datasource template used for the various IO pages. """ -from powa.dashboards import MetricDef, MetricGroupDef +from powa.dashboards import MetricDef, MetricGroupDef from powa.sql.utils import sum_per_sec from powa.sql.views_graph import powa_get_io_sample from powa.sql.views_grid import powa_getiodata @@ -12,49 +12,71 @@ class TemplateIoGraph(MetricGroupDef): """ Template metric group for IO graph. """ + xaxis = "ts" query_qual = None - reads = MetricDef(label="Reads", - type="sizerate", - desc="Amount of data read per second") - read_time = MetricDef(label="Read time", - type="duration", - desc="Total time spend reading data per second") - writes = MetricDef(label="Write", - type="sizerate", - desc="Amount of data written per second") - write_time = MetricDef(label="Write time", - type="duration", - desc="Total time spend writing data per second") - writebacks = MetricDef(label="Writebacks", - type="sizerate", - desc="Amount of data writeback per second") - writeback_time = MetricDef(label="Writeback time", - type="duration", - desc="Total time spend doing writeback per " - "second") - extends = MetricDef(label="Extends", - type="sizerate", - desc="Amount of data extended per second") - extend_time = MetricDef(label="Extend time", - type="duration", - desc="Total time spend extending relations per " - "second") - hits = MetricDef(label="Hits", type="sizerate", - desc="Amount of data found in shared_buffers per second") - evictions = MetricDef(label="Eviction", type="sizerate", - desc="Amount of data evicted from shared_buffers " - "per second") - reuses = MetricDef(label="Reuses", type="sizerate", - desc="Amount of data reused in shared_buffers per " - "second") - fsyncs = MetricDef(label="Fsyncs", - type="sizerate", - desc="Blocks flushed per second") - fsync_time = MetricDef(label="Fsync time", - type="duration", - desc="Total time spend flushing block per second") + reads = MetricDef( + label="Reads", type="sizerate", desc="Amount of data read per second" + ) + read_time = MetricDef( + label="Read time", + type="duration", + desc="Total time spend reading data per second", + ) + writes = MetricDef( + label="Write", + type="sizerate", + desc="Amount of data written per second", + ) + write_time = MetricDef( + label="Write time", + type="duration", + desc="Total time spend writing data per second", + ) + writebacks = MetricDef( + label="Writebacks", + type="sizerate", + desc="Amount of data writeback per second", + ) + writeback_time = MetricDef( + label="Writeback time", + type="duration", + desc="Total time spend doing writeback per " "second", + ) + extends = MetricDef( + label="Extends", + type="sizerate", + desc="Amount of data extended per second", + ) + extend_time = MetricDef( + label="Extend time", + type="duration", + desc="Total time spend extending relations per " "second", + ) + hits = MetricDef( + label="Hits", + type="sizerate", + desc="Amount of data found in shared_buffers per second", + ) + evictions = MetricDef( + label="Eviction", + type="sizerate", + desc="Amount of data evicted from shared_buffers " "per second", + ) + reuses = MetricDef( + label="Reuses", + type="sizerate", + desc="Amount of data reused in shared_buffers per " "second", + ) + fsyncs = MetricDef( + label="Fsyncs", type="sizerate", desc="Blocks flushed per second" + ) + fsync_time = MetricDef( + label="Fsync time", + type="duration", + desc="Total time spend flushing block per second", + ) @classmethod def _get_metrics(cls, handler, **params): @@ -72,22 +94,23 @@ def query(self): from_clause = query - cols = ["sub.srvid", - "extract(epoch FROM sub.ts) AS ts", - sum_per_sec('reads'), - sum_per_sec('read_time'), - sum_per_sec('writes'), - sum_per_sec('write_time'), - sum_per_sec('writebacks'), - sum_per_sec('writeback_time'), - sum_per_sec('extends'), - sum_per_sec('extend_time'), - sum_per_sec('hits'), - sum_per_sec('evictions'), - sum_per_sec('reuses'), - sum_per_sec('fsyncs'), - sum_per_sec('fsync_time'), - ] + cols = [ + "sub.srvid", + "extract(epoch FROM sub.ts) AS ts", + sum_per_sec("reads"), + sum_per_sec("read_time"), + sum_per_sec("writes"), + sum_per_sec("write_time"), + sum_per_sec("writebacks"), + sum_per_sec("writeback_time"), + sum_per_sec("extends"), + sum_per_sec("extend_time"), + sum_per_sec("hits"), + sum_per_sec("evictions"), + sum_per_sec("reuses"), + sum_per_sec("fsyncs"), + sum_per_sec("fsync_time"), + ] return """SELECT {cols} FROM ( @@ -96,7 +119,7 @@ def query(self): WHERE sub.mesure_interval != '0 s' GROUP BY sub.srvid, sub.ts, sub.mesure_interval ORDER BY sub.ts""".format( - cols=', '.join(cols), + cols=", ".join(cols), from_clause=from_clause, ) @@ -105,48 +128,59 @@ class TemplateIoGrid(MetricGroupDef): """ Template metric group for IO grid. """ + axis_type = "category" query_qual = None - reads = MetricDef(label="Reads", - type="size", - desc="Total amount of data read") - read_time = MetricDef(label="Read time", - type="duration", - desc="Total amount of time reading data") - writes = MetricDef(label="Writes", - type="size", - desc="Total amount of data write") - write_time = MetricDef(label="Write time", - type="duration", - desc="Total amount of time writing data") - writebacks = MetricDef(label="Writebacks", - type="size", - desc="Total amount of data writeback") - writeback_time = MetricDef(label="Writeback time", - type="duration", - desc="Total amount of time doing data writeback") - extends = MetricDef(label="Extends", - type="size", - desc="Total amount of data extension") - extend_time = MetricDef(label="Extend time", - type="duration", - desc="Total amount of time extending data") - hits = MetricDef(label="Hits", - type="size", - desc="Total amount of data hit") - evictions = MetricDef(label="Evictions", - type="size", - desc="Total amount of data evicted") - reuses = MetricDef(label="Reuses", - type="size", - desc="Total amount of data reused") - fsyncs = MetricDef(label="Fsyncs", - type="size", - desc="Total amount of data flushed") - fsync_time = MetricDef(label="Fsync time", - type="duration", - desc="Total amount of time flushing data") + reads = MetricDef( + label="Reads", type="size", desc="Total amount of data read" + ) + read_time = MetricDef( + label="Read time", + type="duration", + desc="Total amount of time reading data", + ) + writes = MetricDef( + label="Writes", type="size", desc="Total amount of data write" + ) + write_time = MetricDef( + label="Write time", + type="duration", + desc="Total amount of time writing data", + ) + writebacks = MetricDef( + label="Writebacks", type="size", desc="Total amount of data writeback" + ) + writeback_time = MetricDef( + label="Writeback time", + type="duration", + desc="Total amount of time doing data writeback", + ) + extends = MetricDef( + label="Extends", type="size", desc="Total amount of data extension" + ) + extend_time = MetricDef( + label="Extend time", + type="duration", + desc="Total amount of time extending data", + ) + hits = MetricDef( + label="Hits", type="size", desc="Total amount of data hit" + ) + evictions = MetricDef( + label="Evictions", type="size", desc="Total amount of data evicted" + ) + reuses = MetricDef( + label="Reuses", type="size", desc="Total amount of data reused" + ) + fsyncs = MetricDef( + label="Fsyncs", type="size", desc="Total amount of data flushed" + ) + fsync_time = MetricDef( + label="Fsync time", + type="duration", + desc="Total amount of time flushing data", + ) @property def query(self): @@ -155,16 +189,16 @@ def query(self): return query def process(self, val, **kwargs): - val["backend_type_url"] = self.reverse_url("ByBackendTypeIoOverview", - val["srvid"], - val["backend_type"]) + val["backend_type_url"] = self.reverse_url( + "ByBackendTypeIoOverview", val["srvid"], val["backend_type"] + ) - val["obj_url"] = self.reverse_url("ByObjIoOverview", - val["srvid"], - val["object"]) + val["obj_url"] = self.reverse_url( + "ByObjIoOverview", val["srvid"], val["object"] + ) - val["context_url"] = self.reverse_url("ByContextIoOverview", - val["srvid"], - val["context"]) + val["context_url"] = self.reverse_url( + "ByContextIoOverview", val["srvid"], val["context"] + ) return val diff --git a/powa/json.py b/powa/json.py index a51e6564..ae21430d 100644 --- a/powa/json.py +++ b/powa/json.py @@ -1,13 +1,16 @@ from __future__ import absolute_import -from json import JSONEncoder as BaseJSONEncoder + from datetime import datetime from decimal import Decimal +from json import JSONEncoder as BaseJSONEncoder + class JSONEncoder(BaseJSONEncoder): """ JSONEncoder used throughout the application. Handle Decimal, datetime and JSONizable objects. """ + def default(self, obj): if isinstance(obj, Decimal): return float(obj) @@ -17,6 +20,7 @@ def default(self, obj): return obj.to_json() return BaseJSONEncoder.default(self, obj) + class JSONizable(object): """ Base class for an object which is serializable to JSON. @@ -29,8 +33,14 @@ def to_json(self): Returns: an object which can be encoded by the BaseJSONEncoder. """ - return dict(((key, val) for key, val in self.__dict__.items() - if not key.startswith("_"))) + return dict( + ( + (key, val) + for key, val in self.__dict__.items() + if not key.startswith("_") + ) + ) + def to_json(object): """ diff --git a/powa/options.py b/powa/options.py index a3d639c6..4bb7c166 100644 --- a/powa/options.py +++ b/powa/options.py @@ -1,8 +1,12 @@ -from tornado.options import (define, parse_config_file, options, - Error, parse_command_line) import os import sys - +from tornado.options import ( + Error, + define, + options, + parse_command_line, + parse_config_file, +) SAMPLE_CONFIG_FILE = """ servers={ @@ -16,23 +20,36 @@ """ CONF_LOCATIONS = [ - '/etc/powa-web.conf', - os.path.expanduser('~/.config/powa-web.conf'), - os.path.expanduser('~/.powa-web.conf'), - './powa-web.conf' + "/etc/powa-web.conf", + os.path.expanduser("~/.config/powa-web.conf"), + os.path.expanduser("~/.powa-web.conf"), + "./powa-web.conf", ] define("cookie_secret", type=str, help="Secret key for cookies") -define("cookie_expires_days", type=int, default=30, - help="Cookie retention in days") -define("port", type=int, default=8888, metavar="port", - help="Listen on ") -define("address", type=str, default="0.0.0.0", metavar="address", - help="Listen on
") +define( + "cookie_expires_days", + type=int, + default=30, + help="Cookie retention in days", +) +define("port", type=int, default=8888, metavar="port", help="Listen on ") +define( + "address", + type=str, + default="0.0.0.0", + metavar="address", + help="Listen on
", +) define("config", type=str, help="path to config file") -define("url_prefix", type=str, help="optional prefix URL", default='/') -define("allow_ui_connection", type=bool, help="Allow UI to connect to databases", default=True) +define("url_prefix", type=str, help="optional prefix URL", default="/") +define( + "allow_ui_connection", + type=bool, + help="Allow UI to connect to databases", + default=True, +) define("certfile", type=str, help="Path to certificate file", default=None) define("keyfile", type=str, help="Path to key file", default=None) @@ -40,7 +57,7 @@ def parse_file(filepath): try: parse_config_file(filepath) - except IOError as e: + except IOError: pass except Error as e: print("Error parsing config file %s:" % filepath) @@ -61,31 +78,39 @@ def parse_options(): print("\t%s" % e) sys.exit(1) - for key in ('servers', 'cookie_secret'): + for key in ("servers", "cookie_secret"): if getattr(options, key, None) is None: - print("You should define a server and cookie_secret in your " - "configuration file.") - print("Place and adapt the following content in one of those " - "locations:""") + print( + "You should define a server and cookie_secret in your " + "configuration file." + ) + print( + "Place and adapt the following content in one of those " + "locations:" + "" + ) print("\n\t".join([""] + CONF_LOCATIONS)) print(SAMPLE_CONFIG_FILE) sys.exit(-1) - if getattr(options, 'url_prefix', '') == '': - setattr(options, 'url_prefix', "/") - elif getattr(options, 'url_prefix', "/") != "/": - prefix = getattr(options, 'url_prefix', "/") - if (prefix[0] != "/"): + if getattr(options, "url_prefix", "") == "": + setattr(options, "url_prefix", "/") + elif getattr(options, "url_prefix", "/") != "/": + prefix = getattr(options, "url_prefix", "/") + if prefix[0] != "/": prefix = "/" + prefix - if (prefix[-1] != '/'): + if prefix[-1] != "/": prefix = prefix + "/" - setattr(options, 'url_prefix', prefix) - define("index_url", type=str, - default="%sserver/" % getattr(options, 'url_prefix', "/")) + setattr(options, "url_prefix", prefix) + define( + "index_url", + type=str, + default="%sserver/" % getattr(options, "url_prefix", "/"), + ) # we used to expect a field named "username", so accept "username" as an # alias for "user" - for key, conf in getattr(options, 'servers', {}).items(): - if 'username' in conf.keys(): - conf['user'] = conf['username'] - del conf['username'] + for key, conf in getattr(options, "servers", {}).items(): + if "username" in conf.keys(): + conf["user"] = conf["username"] + del conf["username"] diff --git a/powa/overview.py b/powa/overview.py index e5f649f4..78f2c005 100644 --- a/powa/overview.py +++ b/powa/overview.py @@ -3,20 +3,19 @@ """ from powa.dashboards import ( - Dashboard, Grid, - MetricGroupDef, MetricDef, - DashboardPage) - -try: - from collections import OrderedDict -except: - from ordereddict import OrderedDict + Dashboard, + DashboardPage, + Grid, + MetricDef, + MetricGroupDef, +) class OverviewMetricGroup(MetricGroupDef): """ Metric group used by the "all servers" grid """ + name = "all_servers" xaxis = "srvid" axis_type = "category" @@ -27,7 +26,6 @@ class OverviewMetricGroup(MetricGroupDef): @property def query(self): - sql = """SELECT id AS srvid, CASE WHEN id = 0 THEN '' @@ -43,7 +41,7 @@ def query(self): LEFT JOIN pg_settings set ON set.name = 'server_version' AND s.id = 0""" - return (sql, {'host': self.current_host, 'port': self.current_port}) + return (sql, {"host": self.current_host, "port": self.current_port}) def process(self, val, **kwargs): val["url"] = self.reverse_url("ServerOverview", val["srvid"]) @@ -54,24 +52,33 @@ class Overview(DashboardPage): """ Overview dashboard page. """ + base_url = r"/server/" datasources = [OverviewMetricGroup] - title = 'All servers' + title = "All servers" def dashboard(self): # This COULD be initialized in the constructor, but tornado < 3 doesn't # call it - if getattr(self, '_dashboard', None) is not None: + if getattr(self, "_dashboard", None) is not None: return self._dashboard - dashes = [[Grid("All servers", - columns=[{ + dashes = [ + [ + Grid( + "All servers", + columns=[ + { "name": "alias", "label": "Instance", "url_attr": "url", - "direction": "descending" - }], - metrics=OverviewMetricGroup.all())]] + "direction": "descending", + } + ], + metrics=OverviewMetricGroup.all(), + ) + ] + ] self._dashboard = Dashboard("All servers", dashes) return self._dashboard @@ -79,6 +86,7 @@ def dashboard(self): @classmethod def get_childmenu(cls, handler, params): from powa.server import ServerOverview + children = [] for s in list(handler.servers): new_params = params.copy() diff --git a/powa/qual.py b/powa/qual.py index e65b61c5..4ce4739b 100644 --- a/powa/qual.py +++ b/powa/qual.py @@ -1,20 +1,26 @@ """ Dashboard for the qual page """ -from tornado.web import HTTPError + from powa.dashboards import ( - Dashboard, Graph, Grid, - MetricGroupDef, MetricDef, - DashboardPage, ContentWidget) -from powa.sql import qual_constants, resolve_quals + ContentWidget, + Dashboard, + DashboardPage, + Grid, + MetricDef, + MetricGroupDef, +) from powa.query import QueryOverview +from powa.sql import qual_constants, resolve_quals from powa.sql.views import qualstat_getstatdata +from tornado.web import HTTPError class QualConstantsMetricGroup(MetricGroupDef): """ Metric group used for the qual charts. """ + name = "QualConstants" data_url = r"/server/(\d+)/metrics/database/([^\/]+)/query/(-?\d+)/qual/(\d+)/constants" xaxis = "rownumber" @@ -23,14 +29,20 @@ class QualConstantsMetricGroup(MetricGroupDef): @property def query(self): - most_used = (qual_constants("%(server)s", "most_used", - """ + most_used = qual_constants( + "%(server)s", + "most_used", + """ datname = %(database)s AND coalesce_range && tstzrange(%(from)s, %(to)s)""", - "%(query)s", "%(qual)s", top=10)) - - correlated = qualstat_getstatdata(extra_where=["qualid = %(qual)s", - "queryid = %(query)s"]) + "%(query)s", + "%(qual)s", + top=10, + ) + + correlated = qualstat_getstatdata( + extra_where=["qualid = %(qual)s", "queryid = %(query)s"] + ) sql = """SELECT sub.*, correlated.occurences as total_occurences FROM ( SELECT * @@ -40,32 +52,33 @@ def query(self): ) AS sub, ( {correlated} ) AS correlated - """.format( - most_used=most_used, - correlated=correlated - ) + """.format(most_used=most_used, correlated=correlated) return sql def add_params(self, params): - params['queryids'] = [int(params['query'])] + params["queryids"] = [int(params["query"])] return params def post_process(self, data, server, database, query, qual, **kwargs): - if not data['data']: + if not data["data"]: return data max_rownumber = 0 total_top10 = 0 total = None - d = {'total_occurences': 0} - for d in data['data']: - max_rownumber = max(max_rownumber, d['rownumber']) - total_top10 += d['occurences'] + d = {"total_occurences": 0} + for d in data["data"]: + max_rownumber = max(max_rownumber, d["rownumber"]) + total_top10 += d["occurences"] else: - total = d['total_occurences'] - data['data'].append({'occurences': total - total_top10, - 'rownumber': max_rownumber + 1, - 'constants': 'Others'}) + total = d["total_occurences"] + data["data"].append( + { + "occurences": total - total_top10, + "rownumber": max_rownumber + 1, + "constants": "Others", + } + ) return data @@ -73,35 +86,45 @@ class QualDetail(ContentWidget): """ Content widget showing detail for a specific qual. """ + title = "Detail for this Qual" - data_url = r"/server/(\d+)/database/([^\/]+)/query/(-?\d+)/qual/(\d+)/detail" + data_url = ( + r"/server/(\d+)/database/([^\/]+)/query/(-?\d+)/qual/(\d+)/detail" + ) def get(self, server, database, query, qual): try: # Check remote access first - remote_conn = self.connect(server, database=database, - remote_access=True) + remote_conn = self.connect( + server, database=database, remote_access=True + ) except Exception as e: - raise HTTPError(501, "Could not connect to remote server: %s" % - str(e)) - stmt = qualstat_getstatdata(extra_select=["queryid = %(query)s" - " AS is_my_query"], - extra_where=["qualid = %(qualid)s", - "occurences > 0"], - extra_groupby=["queryid"]) - quals = list(self.execute( - stmt, - params={"server": server, + raise HTTPError( + 501, "Could not connect to remote server: %s" % str(e) + ) + stmt = qualstat_getstatdata( + extra_select=["queryid = %(query)s" " AS is_my_query"], + extra_where=["qualid = %(qualid)s", "occurences > 0"], + extra_groupby=["queryid"], + ) + quals = list( + self.execute( + stmt, + params={ + "server": server, "query": query, "from": self.get_argument("from"), "to": self.get_argument("to"), "queryids": [query], - "qualid": qual})) + "qualid": qual, + }, + ) + ) my_qual = None for qual in quals: - if qual['is_my_query']: + if qual["is_my_query"]: my_qual = resolve_quals(remote_conn, [qual])[0] if my_qual is None: @@ -113,13 +136,13 @@ def get(self, server, database, query, qual): class OtherQueriesMetricGroup(MetricGroupDef): """Metric group showing other queries for this qual.""" + name = "other_queries" xaxis = "queryid" axis_type = "category" data_url = r"/server/(\d+)/metrics/database/([^\/]+)/query/(-?\d+)/qual/(\d+)/other_queries" query_str = MetricDef(label="Query", type="query", url_attr="url") - @property def query(self): return """ @@ -136,7 +159,8 @@ def query(self): def process(self, val, database=None, **kwargs): val["url"] = self.reverse_url( - "QueryOverview", val["srvid"], database, val["queryid"]) + "QueryOverview", val["srvid"], database, val["queryid"] + ) return val @@ -147,25 +171,40 @@ class QualOverview(DashboardPage): base_url = r"/server/(\d+)/database/([^\/]+)/query/(-?\d+)/qual/(\d+)" params = ["server", "database", "query", "qual"] - datasources = [QualDetail, OtherQueriesMetricGroup, QualConstantsMetricGroup] + datasources = [ + QualDetail, + OtherQueriesMetricGroup, + QualConstantsMetricGroup, + ] parent = QueryOverview - title = 'Predicate Overview' + title = "Predicate Overview" def dashboard(self): # This COULD be initialized in the constructor, but tornado < 3 doesn't # call it - if getattr(self, '_dashboard', None) is not None: + if getattr(self, "_dashboard", None) is not None: return self._dashboard self._dashboard = Dashboard( "Qual %(qual)s", - [[QualDetail], - [Grid("Other queries", - metrics=OtherQueriesMetricGroup.all(), - columns=[])], - [Grid("Most executed values", - metrics=[QualConstantsMetricGroup.occurences], - x_label_attr="constants", - renderer="distribution")]]) + [ + [QualDetail], + [ + Grid( + "Other queries", + metrics=OtherQueriesMetricGroup.all(), + columns=[], + ) + ], + [ + Grid( + "Most executed values", + metrics=[QualConstantsMetricGroup.occurences], + x_label_attr="constants", + renderer="distribution", + ) + ], + ], + ) return self._dashboard diff --git a/powa/query.py b/powa/query.py index 978d2ac4..2c85a12c 100644 --- a/powa/query.py +++ b/powa/query.py @@ -2,321 +2,487 @@ Dashboard for the by-query page. """ -from psycopg2 import Error -from tornado.web import HTTPError - +from powa.config import ConfigChangesQuery from powa.dashboards import ( - Dashboard, TabContainer, Graph, Grid, - MetricGroupDef, MetricDef, - DashboardPage, ContentWidget) + ContentWidget, + Dashboard, + DashboardPage, + Graph, + Grid, + MetricDef, + MetricGroupDef, + TabContainer, +) from powa.database import DatabaseOverview - -from powa.sql import (Plan, format_jumbled_query, resolve_quals, - qualstat_get_figures, get_hypoplans, get_plans, - get_any_sample_query, possible_indexes) -from powa.sql.views import (qualstat_getstatdata, - QUALSTAT_FILTER_RATIO) -from powa.sql.views_graph import (powa_getstatdata_sample, - powa_getwaitdata_sample, - kcache_getstatdata_sample) -from powa.sql.views_grid import (powa_getstatdata_detailed_db, - powa_getwaitdata_detailed_db) -from powa.sql.utils import (to_epoch, sum_per_sec, byte_per_sec, wps, - block_size) -from powa.config import ConfigChangesQuery +from powa.sql import ( + get_any_sample_query, + get_hypoplans, + get_plans, + possible_indexes, + qualstat_get_figures, + resolve_quals, +) +from powa.sql.utils import block_size, byte_per_sec, sum_per_sec, to_epoch, wps +from powa.sql.views import QUALSTAT_FILTER_RATIO, qualstat_getstatdata +from powa.sql.views_graph import ( + kcache_getstatdata_sample, + powa_getstatdata_sample, + powa_getwaitdata_sample, +) +from powa.sql.views_grid import ( + powa_getstatdata_detailed_db, + powa_getwaitdata_detailed_db, +) +from psycopg2 import Error +from tornado.web import HTTPError class QueryOverviewMetricGroup(MetricGroupDef): """ Metric Group for the graphs on the by query page. """ + name = "query_overview" xaxis = "ts" data_url = r"/server/(\d+)/metrics/database/([^\/]+)/query/(-?\d+)" - rows = MetricDef(label="#Rows", - desc="Sum of the number of rows returned by the query" - " per second") - calls = MetricDef(label="#Calls", - desc="Number of time the query has been executed" - " per second") - shared_blks_read = MetricDef(label="Shared read", type="sizerate", - desc="Amount of data found in OS cache or" - " read from disk") - shared_blks_hit = MetricDef(label="Shared hit", type="sizerate", - desc="Amount of data found in shared buffers") - shared_blks_dirtied = MetricDef(label="Shared dirtied", type="sizerate", - desc="Amount of data modified in shared" - " buffers") - shared_blks_written = MetricDef(label="Shared written", type="sizerate", - desc="Amount of shared buffers written to" - " disk") - local_blks_read = MetricDef(label="Local read", type="sizerate", - desc="Amount of local buffers found from OS" - " cache or read from disk") - local_blks_hit = MetricDef(label="Local hit", type="sizerate", - desc="Amount of local buffers found in shared" - " buffers") - local_blks_dirtied = MetricDef(label="Local dirtied", type="sizerate", - desc="Amount of data modified in local" - " buffers") - local_blks_written = MetricDef(label="Local written", type="sizerate", - desc="Amount of local buffers written to" - " disk") - temp_blks_read = MetricDef(label="Temp read", type="sizerate", - desc="Amount of data read from temporary file") - temp_blks_written = MetricDef(label="Temp written", type="sizerate", - desc="Amount of data written to temporary" - " file") - shared_blk_read_time = MetricDef(label="Shared Read time", type="duration", - desc="Time spent reading shared blocks") - shared_blk_write_time = MetricDef(label="Shared Write time", type="duration", - desc="Time spent shared blocks") - local_blk_read_time = MetricDef(label="Local Read time", type="duration", - desc="Time spent reading temp table blocks") - local_blk_write_time = MetricDef(label="Local Write time", type="duration", - desc="Time spent temp table blocks") - temp_blk_read_time = MetricDef(label="Temp Read time", type="duration", - desc="Time spent reading temp file blocks") - temp_blk_write_time = MetricDef(label="Temp Write time", type="duration", - desc="Time spent temp file blocks") - avg_plantime = MetricDef(label="Avg plantime", type="duration", - desc="Average query planning duration") - avg_runtime = MetricDef(label="Avg runtime", type="duration", - desc="Average query duration") - hit_ratio = MetricDef(label="Shared buffers hit ratio", type="percent", - desc="Percentage of data found in shared buffers") - miss_ratio = MetricDef(label="Shared buffers miss ratio", type="percent", - desc="Percentage of data found in OS cache or read" - " from disk") - wal_records = MetricDef(label="#Wal records", type="integer", - desc="Amount of WAL records generated") - wal_fpi = MetricDef(label="#Wal FPI", type="integer", - desc="Amount of WAL full-page images generated") - wal_bytes = MetricDef(label="Wal bytes", type="size", - desc="Amount of WAL bytes generated") - - reads = MetricDef(label="Physical read", type="sizerate", - desc="Amount of data read from disk") - writes = MetricDef(label="Physical writes", type="sizerate", - desc="Amount of data written to disk") - user_time = MetricDef(label="CPU user time / Query time", type="percent", - desc="CPU time spent executing the query") - system_time = MetricDef(label="CPU system time / Query time", - type="percent", - desc="CPU time used by the OS") - other_time = MetricDef(label="CPU other time / Query time", type="percent", - desc="Time spent otherwise") - sys_hit_ratio = MetricDef(label="System cache hit ratio", type="percent", - desc="Percentage of data found in OS cache") - disk_hit_ratio = MetricDef(label="Disk hit ratio", type="percent", - desc="Percentage of data read from disk") - minflts = MetricDef(label="Soft page faults", type="number", - desc="Memory pages not found in the processor's MMU") - majflts = MetricDef(label="Hard page faults", type="number", - desc="Memory pages not found in memory and loaded" - " from storage") + rows = MetricDef( + label="#Rows", + desc="Sum of the number of rows returned by the query" " per second", + ) + calls = MetricDef( + label="#Calls", + desc="Number of time the query has been executed" " per second", + ) + shared_blks_read = MetricDef( + label="Shared read", + type="sizerate", + desc="Amount of data found in OS cache or" " read from disk", + ) + shared_blks_hit = MetricDef( + label="Shared hit", + type="sizerate", + desc="Amount of data found in shared buffers", + ) + shared_blks_dirtied = MetricDef( + label="Shared dirtied", + type="sizerate", + desc="Amount of data modified in shared" " buffers", + ) + shared_blks_written = MetricDef( + label="Shared written", + type="sizerate", + desc="Amount of shared buffers written to" " disk", + ) + local_blks_read = MetricDef( + label="Local read", + type="sizerate", + desc="Amount of local buffers found from OS" + " cache or read from disk", + ) + local_blks_hit = MetricDef( + label="Local hit", + type="sizerate", + desc="Amount of local buffers found in shared" " buffers", + ) + local_blks_dirtied = MetricDef( + label="Local dirtied", + type="sizerate", + desc="Amount of data modified in local" " buffers", + ) + local_blks_written = MetricDef( + label="Local written", + type="sizerate", + desc="Amount of local buffers written to" " disk", + ) + temp_blks_read = MetricDef( + label="Temp read", + type="sizerate", + desc="Amount of data read from temporary file", + ) + temp_blks_written = MetricDef( + label="Temp written", + type="sizerate", + desc="Amount of data written to temporary" " file", + ) + shared_blk_read_time = MetricDef( + label="Shared Read time", + type="duration", + desc="Time spent reading shared blocks", + ) + shared_blk_write_time = MetricDef( + label="Shared Write time", + type="duration", + desc="Time spent shared blocks", + ) + local_blk_read_time = MetricDef( + label="Local Read time", + type="duration", + desc="Time spent reading temp table blocks", + ) + local_blk_write_time = MetricDef( + label="Local Write time", + type="duration", + desc="Time spent temp table blocks", + ) + temp_blk_read_time = MetricDef( + label="Temp Read time", + type="duration", + desc="Time spent reading temp file blocks", + ) + temp_blk_write_time = MetricDef( + label="Temp Write time", + type="duration", + desc="Time spent temp file blocks", + ) + avg_plantime = MetricDef( + label="Avg plantime", + type="duration", + desc="Average query planning duration", + ) + avg_runtime = MetricDef( + label="Avg runtime", type="duration", desc="Average query duration" + ) + hit_ratio = MetricDef( + label="Shared buffers hit ratio", + type="percent", + desc="Percentage of data found in shared buffers", + ) + miss_ratio = MetricDef( + label="Shared buffers miss ratio", + type="percent", + desc="Percentage of data found in OS cache or read" " from disk", + ) + wal_records = MetricDef( + label="#Wal records", + type="integer", + desc="Amount of WAL records generated", + ) + wal_fpi = MetricDef( + label="#Wal FPI", + type="integer", + desc="Amount of WAL full-page images generated", + ) + wal_bytes = MetricDef( + label="Wal bytes", type="size", desc="Amount of WAL bytes generated" + ) + + reads = MetricDef( + label="Physical read", + type="sizerate", + desc="Amount of data read from disk", + ) + writes = MetricDef( + label="Physical writes", + type="sizerate", + desc="Amount of data written to disk", + ) + user_time = MetricDef( + label="CPU user time / Query time", + type="percent", + desc="CPU time spent executing the query", + ) + system_time = MetricDef( + label="CPU system time / Query time", + type="percent", + desc="CPU time used by the OS", + ) + other_time = MetricDef( + label="CPU other time / Query time", + type="percent", + desc="Time spent otherwise", + ) + sys_hit_ratio = MetricDef( + label="System cache hit ratio", + type="percent", + desc="Percentage of data found in OS cache", + ) + disk_hit_ratio = MetricDef( + label="Disk hit ratio", + type="percent", + desc="Percentage of data read from disk", + ) + minflts = MetricDef( + label="Soft page faults", + type="number", + desc="Memory pages not found in the processor's MMU", + ) + majflts = MetricDef( + label="Hard page faults", + type="number", + desc="Memory pages not found in memory and loaded" " from storage", + ) # not maintained on GNU/Linux, and not available on Windows # nswaps = MetricDef(label="Swaps", type="number") # msgsnds = MetricDef(label="IPC messages sent", type="number") # msgrcvs = MetricDef(label="IPC messages received", type="number") # nsignals = MetricDef(label="Signals received", type="number") - nvcsws = MetricDef(label="Voluntary context switches", type="number", - desc="Number of voluntary context switches") - nivcsws = MetricDef(label="Involuntary context switches", type="number", - desc="Number of involuntary context switches") - jit_functions = MetricDef(label="# of JIT functions", type="integer", - desc="Total number of emitted functions") - jit_generation_time = MetricDef(label="JIT generation time", - type="duration", - desc="Total time spent generating code") - jit_inlining_count = MetricDef(label="# of JIT inlining", type="integer", - desc="Number of queries where inlining was" - " done") - jit_inlining_time = MetricDef(label="JIT inlining time", type="duration", - desc="Total time spent inlining code") - jit_optimization_count = MetricDef(label="# of JIT optimization", - type="integer", - desc="Number of queries where" - " optimization was done") - jit_optimization_time = MetricDef(label="JIT optimization time", - type="duration", - desc="Total time spent optimizing code") - jit_emission_count = MetricDef(label="# of JIT emission", type="integer", - desc="Number of queries where emission was" - " done") - jit_emission_time = MetricDef(label="JIT emission time", type="duration", - desc="Total time spent emitting code") - jit_deform_count = MetricDef(label="# of JIT tuple deforming", - type="integer", - desc="Number of queries where tuple deforming" - " was done") - jit_deform_time = MetricDef(label="JIT tuple deforming time", - type="duration", - desc="Total time spent deforming tuple") - jit_expr_time = MetricDef(label="JIT expression generation time", - type="duration", - desc="Total time spent generating expressions") + nvcsws = MetricDef( + label="Voluntary context switches", + type="number", + desc="Number of voluntary context switches", + ) + nivcsws = MetricDef( + label="Involuntary context switches", + type="number", + desc="Number of involuntary context switches", + ) + jit_functions = MetricDef( + label="# of JIT functions", + type="integer", + desc="Total number of emitted functions", + ) + jit_generation_time = MetricDef( + label="JIT generation time", + type="duration", + desc="Total time spent generating code", + ) + jit_inlining_count = MetricDef( + label="# of JIT inlining", + type="integer", + desc="Number of queries where inlining was" " done", + ) + jit_inlining_time = MetricDef( + label="JIT inlining time", + type="duration", + desc="Total time spent inlining code", + ) + jit_optimization_count = MetricDef( + label="# of JIT optimization", + type="integer", + desc="Number of queries where" " optimization was done", + ) + jit_optimization_time = MetricDef( + label="JIT optimization time", + type="duration", + desc="Total time spent optimizing code", + ) + jit_emission_count = MetricDef( + label="# of JIT emission", + type="integer", + desc="Number of queries where emission was" " done", + ) + jit_emission_time = MetricDef( + label="JIT emission time", + type="duration", + desc="Total time spent emitting code", + ) + jit_deform_count = MetricDef( + label="# of JIT tuple deforming", + type="integer", + desc="Number of queries where tuple deforming" " was done", + ) + jit_deform_time = MetricDef( + label="JIT tuple deforming time", + type="duration", + desc="Total time spent deforming tuple", + ) + jit_expr_time = MetricDef( + label="JIT expression generation time", + type="duration", + desc="Total time spent generating expressions", + ) @classmethod def _get_metrics(cls, handler, **params): base = cls.metrics.copy() if not handler.has_extension(params["server"], "pg_stat_kcache"): - for key in ("reads", "writes", "user_time", "system_time", - "other_time", "sys_hit_ratio", "disk_hit_ratio", - "minflts", "majflts", - # "nswaps", "msgsnds", "msgrcvs", "nsignals", - "nvcsws", "nivcsws"): + for key in ( + "reads", + "writes", + "user_time", + "system_time", + "other_time", + "sys_hit_ratio", + "disk_hit_ratio", + "minflts", + "majflts", + # "nswaps", "msgsnds", "msgrcvs", "nsignals", + "nvcsws", + "nivcsws", + ): base.pop(key) else: base.pop("miss_ratio") - if not handler.has_extension_version(params["server"], - 'pg_stat_statements', '1.8'): + if not handler.has_extension_version( + params["server"], "pg_stat_statements", "1.8" + ): for key in ("avg_plantime", "wal_records", "wal_fpi", "wal_bytes"): base.pop(key) - if not handler.has_extension_version(handler.path_args[0], - 'pg_stat_statements', '1.10'): - for key in ("jit_functions", "jit_generation_time", - "jit_inlining_count", "jit_inlining_time", - "jit_optimization_count", "jit_optimization_time", - "jit_emission_count", "jit_emission_time"): + if not handler.has_extension_version( + handler.path_args[0], "pg_stat_statements", "1.10" + ): + for key in ( + "jit_functions", + "jit_generation_time", + "jit_inlining_count", + "jit_inlining_time", + "jit_optimization_count", + "jit_optimization_time", + "jit_emission_count", + "jit_emission_time", + ): base.pop(key) - if not handler.has_extension_version(handler.path_args[0], - 'pg_stat_statements', '1.11'): - for key in ("jit_deform_count", "jit_deform_time", "jit_expr_time"): + if not handler.has_extension_version( + handler.path_args[0], "pg_stat_statements", "1.11" + ): + for key in ( + "jit_deform_count", + "jit_deform_time", + "jit_expr_time", + ): base.pop(key) return base @property def query(self): - query = powa_getstatdata_sample("query", ["datname = %(database)s", - "queryid = %(query)s"]) + query = powa_getstatdata_sample( + "query", ["datname = %(database)s", "queryid = %(query)s"] + ) total_blocks = "(sum(shared_blks_read) + sum(shared_blks_hit))" - cols = [to_epoch('ts'), - sum_per_sec('rows'), - sum_per_sec('calls'), - - "CASE WHEN {total_blocks} = 0 THEN 0 ELSE" - " sum(shared_blks_hit)::numeric * 100 / {total_blocks}" - "END AS hit_ratio".format(total_blocks=total_blocks), - - byte_per_sec("shared_blks_read"), - byte_per_sec("shared_blks_hit"), - byte_per_sec("shared_blks_dirtied"), - byte_per_sec("shared_blks_written"), - byte_per_sec("local_blks_read"), - byte_per_sec("local_blks_hit"), - byte_per_sec("local_blks_dirtied"), - byte_per_sec("local_blks_written"), - byte_per_sec("temp_blks_read"), - byte_per_sec("temp_blks_written"), - sum_per_sec("shared_blk_read_time"), - sum_per_sec("shared_blk_write_time"), - sum_per_sec("local_blk_read_time"), - sum_per_sec("local_blk_write_time"), - sum_per_sec("temp_blk_read_time"), - sum_per_sec("temp_blk_write_time"), - "sum(runtime) / greatest(sum(calls), 1) AS avg_runtime" + cols = [ + to_epoch("ts"), + sum_per_sec("rows"), + sum_per_sec("calls"), + "CASE WHEN {total_blocks} = 0 THEN 0 ELSE" + " sum(shared_blks_hit)::numeric * 100 / {total_blocks}" + "END AS hit_ratio".format(total_blocks=total_blocks), + byte_per_sec("shared_blks_read"), + byte_per_sec("shared_blks_hit"), + byte_per_sec("shared_blks_dirtied"), + byte_per_sec("shared_blks_written"), + byte_per_sec("local_blks_read"), + byte_per_sec("local_blks_hit"), + byte_per_sec("local_blks_dirtied"), + byte_per_sec("local_blks_written"), + byte_per_sec("temp_blks_read"), + byte_per_sec("temp_blks_written"), + sum_per_sec("shared_blk_read_time"), + sum_per_sec("shared_blk_write_time"), + sum_per_sec("local_blk_read_time"), + sum_per_sec("local_blk_write_time"), + sum_per_sec("temp_blk_read_time"), + sum_per_sec("temp_blk_write_time"), + "sum(runtime) / greatest(sum(calls), 1) AS avg_runtime", + ] + + if self.has_extension_version( + self.path_args[0], "pg_stat_statements", "1.8" + ): + cols.extend( + [ + "sum(plantime) / greatest(sum(calls), 1) AS avg_plantime", + sum_per_sec("wal_records"), + sum_per_sec("wal_fpi"), + sum_per_sec("wal_bytes"), ] + ) - if self.has_extension_version(self.path_args[0], - 'pg_stat_statements', '1.8'): - cols.extend([ - "sum(plantime) / greatest(sum(calls), 1) AS avg_plantime", - sum_per_sec("wal_records"), - sum_per_sec("wal_fpi"), - sum_per_sec("wal_bytes") - ]) - - if self.has_extension_version(self.path_args[0], - 'pg_stat_statements', '1.10'): - cols.extend([ - sum_per_sec('jit_functions'), - sum_per_sec('jit_generation_time'), - sum_per_sec('jit_inlining_count'), - sum_per_sec('jit_inlining_time'), - sum_per_sec('jit_optimization_count'), - sum_per_sec('jit_optimization_time'), - sum_per_sec('jit_emission_count'), - sum_per_sec('jit_emission_time'), - ]) - - if self.has_extension_version(self.path_args[0], - 'pg_stat_statements', '1.11'): - cols.extend([ - sum_per_sec('jit_deform_count'), - sum_per_sec('jit_deform_time'), - sum_per_sec('jit_generation_time - jit_deform_time', alias='jit_expr_time'), - ]) + if self.has_extension_version( + self.path_args[0], "pg_stat_statements", "1.10" + ): + cols.extend( + [ + sum_per_sec("jit_functions"), + sum_per_sec("jit_generation_time"), + sum_per_sec("jit_inlining_count"), + sum_per_sec("jit_inlining_time"), + sum_per_sec("jit_optimization_count"), + sum_per_sec("jit_optimization_time"), + sum_per_sec("jit_emission_count"), + sum_per_sec("jit_emission_time"), + ] + ) + + if self.has_extension_version( + self.path_args[0], "pg_stat_statements", "1.11" + ): + cols.extend( + [ + sum_per_sec("jit_deform_count"), + sum_per_sec("jit_deform_time"), + sum_per_sec( + "jit_generation_time - jit_deform_time", + alias="jit_expr_time", + ), + ] + ) from_clause = query if self.has_extension(self.path_args[0], "pg_stat_kcache"): # Add system metrics from pg_stat_kcache, # and detailed hit ratio. - kcache_query = kcache_getstatdata_sample("query", - ["datname = %(database)s", - "queryid = %(query)s"]) + kcache_query = kcache_getstatdata_sample( + "query", ["datname = %(database)s", "queryid = %(query)s"] + ) - sys_hits = "greatest(sum(shared_blks_read) - sum(sub.reads), 0)" \ - " * block_size" - sys_hit_ratio = "{sys_hits}::numeric * 100 / ({total_blocks}" \ + sys_hits = ( + "greatest(sum(shared_blks_read) - sum(sub.reads), 0)" + " * block_size" + ) + sys_hit_ratio = ( + "{sys_hits}::numeric * 100 / ({total_blocks}" " * block_size)".format( - sys_hits=sys_hits, - total_blocks=total_blocks + sys_hits=sys_hits, total_blocks=total_blocks ) - disk_hit_ratio = "sum(sub.reads) * 100 / " \ + ) + disk_hit_ratio = ( + "sum(sub.reads) * 100 / " "({total_blocks} * block_size)".format( total_blocks=total_blocks ) + ) total_time = "greatest(sum(runtime), 1)" - other_time = "sum(runtime) - ((" \ - "(sum(user_time) + sum(system_time))" \ + other_time = ( + "sum(runtime) - ((" + "(sum(user_time) + sum(system_time))" ") * 1000)" + ) # Rusage can return values > real time due to sampling bias # aligned to kernel ticks. As such, we have to clamp values to 100% def total_time_percent(col, alias=None, noalias=False): val = "least(100, ({col} * 100) / {total_time})".format( - col=col, - total_time=total_time - ) + col=col, total_time=total_time + ) if not noalias: val += " as " + alias return val - cols.extend([ - sum_per_sec("reads"), - sum_per_sec("writes"), - sum_per_sec("minflts"), - sum_per_sec("majflts"), - # sum_per_sec("nswaps"), - # sum_per_sec("msgsnds"), - # sum_per_sec("msgrcvs"), - # sum_per_sec("nsignals"), - sum_per_sec("nvcsws"), - sum_per_sec("nivcsws"), - total_time_percent("sum(user_time) * 1000", "user_time"), - total_time_percent("sum(system_time) * 1000", "system_time"), - "greatest({other}, 0) AS other_time".format( - other=total_time_percent(other_time, noalias=True) + cols.extend( + [ + sum_per_sec("reads"), + sum_per_sec("writes"), + sum_per_sec("minflts"), + sum_per_sec("majflts"), + # sum_per_sec("nswaps"), + # sum_per_sec("msgsnds"), + # sum_per_sec("msgrcvs"), + # sum_per_sec("nsignals"), + sum_per_sec("nvcsws"), + sum_per_sec("nivcsws"), + total_time_percent("sum(user_time) * 1000", "user_time"), + total_time_percent( + "sum(system_time) * 1000", "system_time" + ), + "greatest({other}, 0) AS other_time".format( + other=total_time_percent(other_time, noalias=True) + ), + "CASE WHEN {tb} = 0 THEN 0 ELSE {dhr} END AS disk_hit_ratio".format( + tb=total_blocks, dhr=disk_hit_ratio ), - "CASE WHEN {tb} = 0 THEN 0 ELSE {dhr} END AS disk_hit_ratio" - .format( - tb=total_blocks, - dhr=disk_hit_ratio - ), - "CASE WHEN {tb} = 0 THEN 0 ELSE {shr} END AS sys_hit_ratio" - .format( - tb=total_blocks, - shr=sys_hit_ratio - )]) + "CASE WHEN {tb} = 0 THEN 0 ELSE {shr} END AS sys_hit_ratio".format( + tb=total_blocks, shr=sys_hit_ratio + ), + ] + ) from_clause = """SELECT * FROM ( @@ -326,15 +492,16 @@ def total_time_percent(col, alias=None, noalias=False): {kcache_query} ) AS kc USING (ts, srvid, queryid, userid, dbid)""".format( - from_clause=from_clause, - kcache_query=kcache_query - ) + from_clause=from_clause, kcache_query=kcache_query + ) else: - cols.extend([ - """CASE WHEN {tb} = 0 THEN 0 + cols.extend( + [ + """CASE WHEN {tb} = 0 THEN 0 ELSE sum(shared_blks_read)::numeric * 100 / {tb} END AS miss_ratio""".format(tb=total_blocks) - ]) + ] + ) return """SELECT {cols} FROM ( @@ -344,10 +511,8 @@ def total_time_percent(col, alias=None, noalias=False): WHERE calls != 0 GROUP BY ts, block_size, mesure_interval ORDER BY ts""".format( - cols=', '.join(cols), - from_clause=from_clause, - bs=block_size - ) + cols=", ".join(cols), from_clause=from_clause, bs=block_size + ) class QueryIndexes(ContentWidget): @@ -364,64 +529,87 @@ def get(self, srvid, database, query): try: # Check remote access first - remote_conn = self.connect(srvid, database=database, - remote_access=True) + remote_conn = self.connect( + srvid, database=database, remote_access=True + ) except Exception as e: - raise HTTPError(501, "Could not connect to remote server: %s" % - str(e)) + raise HTTPError( + 501, "Could not connect to remote server: %s" % str(e) + ) extra_join = """, LATERAL unnest(quals) AS qual""" - base_query = qualstat_getstatdata(extra_join=extra_join, - extra_where=["queryid = " + query], - extra_having=[ - "bool_or(eval_type = 'f')", - "sum(execution_count) > 1000", - "sum(occurences) > 0", - QUALSTAT_FILTER_RATIO + " > 0.5" - ] - ) - optimizable = list(self.execute(base_query, - params={'server': srvid, - 'query': query, - 'from': '-infinity', - 'to': 'infinity'})) - optimizable = resolve_quals(remote_conn, optimizable, 'quals') + base_query = qualstat_getstatdata( + extra_join=extra_join, + extra_where=["queryid = " + query], + extra_having=[ + "bool_or(eval_type = 'f')", + "sum(execution_count) > 1000", + "sum(occurences) > 0", + QUALSTAT_FILTER_RATIO + " > 0.5", + ], + ) + optimizable = list( + self.execute( + base_query, + params={ + "server": srvid, + "query": query, + "from": "-infinity", + "to": "infinity", + }, + ) + ) + optimizable = resolve_quals(remote_conn, optimizable, "quals") hypoplan = None indexes = {} for qual in optimizable: indexes[qual.where_clause] = possible_indexes(qual) - hypo_version = self.has_extension_version(srvid, "hypopg", "0.0.3", - database=database) + hypo_version = self.has_extension_version( + srvid, "hypopg", "0.0.3", database=database + ) if indexes and hypo_version: # identify indexes # create them - allindexes = [ind for indcollection in indexes.values() - for ind in indcollection] + allindexes = [ + ind + for indcollection in indexes.values() + for ind in indcollection + ] for ind in allindexes: (sql, params) = ind.hypo_ddl if sql is not None: try: - ind.name = self.execute(sql, params=params, - srvid=srvid, - database=database, - remote_access=True - )[0]['indexname'] + ind.name = self.execute( + sql, + params=params, + srvid=srvid, + database=database, + remote_access=True, + )[0]["indexname"] except Error as e: - self.flash("Could not create hypothetical index: %s" % - str(e)) + self.flash( + "Could not create hypothetical index: %s" % str(e) + ) # Build the query and fetch the plans - querystr = get_any_sample_query(self, srvid, database, query, - self.get_argument("from"), - self.get_argument("to")) + querystr = get_any_sample_query( + self, + srvid, + database, + query, + self.get_argument("from"), + self.get_argument("to"), + ) try: - hypoplan = get_hypoplans(remote_conn.cursor(), - querystr, allindexes) + hypoplan = get_hypoplans( + remote_conn.cursor(), querystr, allindexes + ) except Error as e: # TODO: offer the possibility to fill in parameters from the UI - self.flash("We couldn't get plans for this query, presumably " - "because some parameters are missing: %s" % - str(e)) + self.flash( + "We couldn't get plans for this query, presumably " + "because some parameters are missing: %s" % str(e) + ) self.render_json(dict(indexes=indexes, hypoplan=hypoplan)) @@ -430,20 +618,27 @@ class QueryExplains(ContentWidget): """ Content widget showing explain plans for various const values. """ + title = "Example Values" - data_url = r"/server/(\d+)/metrics/database/([^\/]+)/query/(-?\d+)/explains" + data_url = ( + r"/server/(\d+)/metrics/database/([^\/]+)/query/(-?\d+)/explains" + ) def get(self, server, database, query): if not self.has_extension(server, "pg_qualstats"): raise HTTPError(501, "PG qualstats is not installed") plans = [] - row = qualstat_get_figures(self, server, database, - self.get_argument("from"), - self.get_argument("to"), - queries=[query]) + row = qualstat_get_figures( + self, + server, + database, + self.get_argument("from"), + self.get_argument("to"), + queries=[query], + ) if row is not None: - plans = get_plans(self, server, database, row['query'], row) + plans = get_plans(self, server, database, row["query"], row) if len(plans) == 0: self.render_json(None) @@ -456,41 +651,52 @@ class WaitsQueryOverviewMetricGroup(MetricGroupDef): """ Metric Group for the wait event graph on the by query page. """ + name = "waits_query_overview" xaxis = "ts" data_url = r"/server/(\d+)/metrics/database/([^\/]+)/query/(-?\d+)/wait_events_sampled" # pg 9.6 only metrics - count_lwlocknamed = MetricDef(label="Lightweight Named", - desc="Number of named lightweight lock" - " wait events") - count_lwlocktranche = MetricDef(label="Lightweight Tranche", - desc="Number of lightweight lock tranche" - " wait events") + count_lwlocknamed = MetricDef( + label="Lightweight Named", + desc="Number of named lightweight lock" " wait events", + ) + count_lwlocktranche = MetricDef( + label="Lightweight Tranche", + desc="Number of lightweight lock tranche" " wait events", + ) # pg 10+ metrics - count_lwlock = MetricDef(label="Lightweight Lock", - desc="Number of wait events due to lightweight" - " locks") - count_lock = MetricDef(label="Lock", - desc="Number of wait events due to heavyweight" - " locks") - count_bufferpin = MetricDef(label="Buffer pin", - desc="Number of wait events due to buffer pin") - count_activity = MetricDef(label="Activity", - desc="Number of wait events due to postgres" - " internal processes activity") - count_client = MetricDef(label="Client", - desc="Number of wait events due to client" - " activity") - count_extension = MetricDef(label="Extension", - desc="Number wait events due to third-party" - " extensions") - count_ipc = MetricDef(label="IPC", - desc="Number of wait events due to inter-process" - "communication") - count_timeout = MetricDef(label="Timeout", - desc="Number of wait events due to timeouts") - count_io = MetricDef(label="IO", - desc="Number of wait events due to IO operations") + count_lwlock = MetricDef( + label="Lightweight Lock", + desc="Number of wait events due to lightweight" " locks", + ) + count_lock = MetricDef( + label="Lock", desc="Number of wait events due to heavyweight" " locks" + ) + count_bufferpin = MetricDef( + label="Buffer pin", desc="Number of wait events due to buffer pin" + ) + count_activity = MetricDef( + label="Activity", + desc="Number of wait events due to postgres" + " internal processes activity", + ) + count_client = MetricDef( + label="Client", desc="Number of wait events due to client" " activity" + ) + count_extension = MetricDef( + label="Extension", + desc="Number wait events due to third-party" " extensions", + ) + count_ipc = MetricDef( + label="IPC", + desc="Number of wait events due to inter-process" "communication", + ) + count_timeout = MetricDef( + label="Timeout", desc="Number of wait events due to timeouts" + ) + count_io = MetricDef( + label="IO", desc="Number of wait events due to IO operations" + ) def prepare(self): if not self.has_extension(self.path_args[0], "pg_wait_sampling"): @@ -498,28 +704,32 @@ def prepare(self): @property def query(self): - - query = powa_getwaitdata_sample("query", ["datname = %(database)s", - "queryid = %(query)s"]) + query = powa_getwaitdata_sample( + "query", ["datname = %(database)s", "queryid = %(query)s"] + ) cols = [to_epoch("ts")] pg_version_num = self.get_pg_version_num(self.path_args[0]) # if we can't connect to the remote server, assume pg10 or above if pg_version_num is not None and pg_version_num < 100000: - cols += [wps("count_lwlocknamed", do_sum=False), - wps("count_lwlocktranche", do_sum=False), - wps("count_lock", do_sum=False), - wps("count_bufferpin", do_sum=False)] + cols += [ + wps("count_lwlocknamed", do_sum=False), + wps("count_lwlocktranche", do_sum=False), + wps("count_lock", do_sum=False), + wps("count_bufferpin", do_sum=False), + ] else: - cols += [wps("count_lwlock", do_sum=False), - wps("count_lock", do_sum=False), - wps("count_bufferpin", do_sum=False), - wps("count_activity", do_sum=False), - wps("count_client", do_sum=False), - wps("count_extension", do_sum=False), - wps("count_ipc", do_sum=False), - wps("count_timeout", do_sum=False), - wps("count_io", do_sum=False)] + cols += [ + wps("count_lwlock", do_sum=False), + wps("count_lock", do_sum=False), + wps("count_bufferpin", do_sum=False), + wps("count_activity", do_sum=False), + wps("count_client", do_sum=False), + wps("count_extension", do_sum=False), + wps("count_ipc", do_sum=False), + wps("count_timeout", do_sum=False), + wps("count_io", do_sum=False), + ] from_clause = query @@ -528,22 +738,23 @@ def query(self): {from_clause} ) AS sub -- WHERE count IS NOT NULL - ORDER BY ts""".format( - cols=', '.join(cols), - from_clause=from_clause - ) + ORDER BY ts""".format(cols=", ".join(cols), from_clause=from_clause) class WaitSamplingList(MetricGroupDef): """ Datasource used for the wait events grid. """ + name = "query_wait_events" xaxis = "event" axis_type = "category" - data_url = r"/server/(\d+)/metrics/database/([^\/]+)/query/(-?\d+)/wait_events" - counts = MetricDef(label="# of events", type="integer", - direction="descending") + data_url = ( + r"/server/(\d+)/metrics/database/([^\/]+)/query/(-?\d+)/wait_events" + ) + counts = MetricDef( + label="# of events", type="integer", direction="descending" + ) def prepare(self): if not self.has_extension(self.path_args[0], "pg_wait_sampling"): @@ -554,11 +765,13 @@ def query(self): # Working from the waitdata detailed_db base query inner_query = powa_getwaitdata_detailed_db() - cols = ["queryid", - "ps.query", - "event_type", - "event", - "sum(count) AS counts"] + cols = [ + "queryid", + "ps.query", + "event_type", + "event", + "sum(count) AS counts", + ] from_clause = """( {inner_query} @@ -571,21 +784,25 @@ def query(self): WHERE datname = %(database)s AND queryid = %(query)s GROUP BY queryid, query, event_type, event ORDER BY sum(count) DESC""".format( - cols=', '.join(cols), - from_clause=from_clause - ) + cols=", ".join(cols), from_clause=from_clause + ) class QualList(MetricGroupDef): """ Datasource used for the Qual table. """ + name = "query_quals" xaxis = "relname" axis_type = "category" data_url = r"/server/(\d+)/metrics/database/([^\/]+)/query/(-?\d+)/quals" - filter_ratio = MetricDef(label="Avg filter_ratio (excluding index)", type="percent") - execution_count = MetricDef(label="Execution count (excluding index)", type="integer") + filter_ratio = MetricDef( + label="Avg filter_ratio (excluding index)", type="percent" + ) + execution_count = MetricDef( + label="Execution count (excluding index)", type="integer" + ) def prepare(self): if not self.has_extension(self.path_args[0], "pg_qualstats"): @@ -598,16 +815,19 @@ def query(self): def post_process(self, data, server, database, query, **kwargs): try: - remote_conn = self.connect(server, database=database, - remote_access=True) + remote_conn = self.connect( + server, database=database, remote_access=True + ) except Exception as e: - raise HTTPError(501, "Could not connect to remote server: %s" % - str(e)) + raise HTTPError( + 501, "Could not connect to remote server: %s" % str(e) + ) data["data"] = resolve_quals(remote_conn, data["data"]) for qual in data["data"]: - qual.url = self.reverse_url('QualOverview', server, database, - query, qual.qualid) + qual.url = self.reverse_url( + "QualOverview", server, database, query, qual.qualid + ) return data @@ -615,31 +835,30 @@ class QueryDetail(ContentWidget): """ Detail widget showing summarized information for the query. """ + title = "Query Detail" data_url = r"/server/(\d+)/metrics/database/([^\/]+)/query/(-?\d+)/detail" def get(self, srvid, database, query): - stmt = powa_getstatdata_detailed_db(srvid, ["datname = %(database)s", - "queryid = %(query)s"]) + stmt = powa_getstatdata_detailed_db( + srvid, ["datname = %(database)s", "queryid = %(query)s"] + ) from_clause = """{{powa}}.powa_statements AS ps LEFT JOIN ( {stmt} ) AS sub USING (queryid, dbid, userid) - CROSS JOIN {block_size}""".format( - stmt=stmt, - block_size=block_size - ) + CROSS JOIN {block_size}""".format(stmt=stmt, block_size=block_size) rblk = "(sum(shared_blks_read) * block_size)" wblk = "(sum(shared_blks_hit) * block_size)" cols = [ - "query", - "sum(calls) AS calls", - "sum(runtime) AS runtime", - rblk + " AS shared_blks_read", - wblk + " AS shared_blks_hit", - rblk + " + " + wblk + " AS total_blks" + "query", + "sum(calls) AS calls", + "sum(runtime) AS runtime", + rblk + " AS shared_blks_read", + wblk + " AS shared_blks_hit", + rblk + " + " + wblk + " AS total_blks", ] stmt = """SELECT {cols} @@ -647,17 +866,19 @@ def get(self, srvid, database, query): WHERE sub.queryid = %(query)s AND sub.srvid = %(server)s GROUP BY query, block_size""".format( - cols=', '.join(cols), - from_clause=from_clause - ) - - value = self.execute(stmt, params={ - "server": srvid, - "query": query, - "database": database, - "from": self.get_argument("from"), - "to": self.get_argument("to") - }) + cols=", ".join(cols), from_clause=from_clause + ) + + value = self.execute( + stmt, + params={ + "server": srvid, + "query": query, + "database": database, + "from": self.get_argument("from"), + "to": self.get_argument("to"), + }, + ) if value is None or len(value) < 1: self.render_json(None) return @@ -668,151 +889,231 @@ class QueryOverview(DashboardPage): """ Dashboard page for a query. """ + base_url = r"/server/(\d+)/database/([^\/]+)/query/(-?\d+)/overview" params = ["server", "database", "query"] - datasources = [QueryOverviewMetricGroup, WaitsQueryOverviewMetricGroup, - QueryDetail, QueryExplains, QueryIndexes, WaitSamplingList, - QualList, ConfigChangesQuery] + datasources = [ + QueryOverviewMetricGroup, + WaitsQueryOverviewMetricGroup, + QueryDetail, + QueryExplains, + QueryIndexes, + WaitSamplingList, + QualList, + ConfigChangesQuery, + ] parent = DatabaseOverview - title = 'Query Overview' + title = "Query Overview" timeline = ConfigChangesQuery def dashboard(self): # This COULD be initialized in the constructor, but tornado < 3 doesn't # call it - if getattr(self, '_dashboard', None) is not None: + if getattr(self, "_dashboard", None) is not None: return self._dashboard - pgss18 = self.has_extension_version(self.path_args[0], - 'pg_stat_statements', '1.8') - pgss110 = self.has_extension_version(self.path_args[0], - 'pg_stat_statements', '1.10') - pgss111 = self.has_extension_version(self.path_args[0], - 'pg_stat_statements', '1.11') - - hit_ratio_graph = Graph("Hit ratio", - metrics=[QueryOverviewMetricGroup.hit_ratio], - renderer="bar", - stack=True, - color_scheme=['#73c03a','#65b9ac','#cb513a']) - - gen_metrics = [QueryOverviewMetricGroup.avg_runtime, - QueryOverviewMetricGroup.rows, - QueryOverviewMetricGroup.calls] + pgss18 = self.has_extension_version( + self.path_args[0], "pg_stat_statements", "1.8" + ) + pgss110 = self.has_extension_version( + self.path_args[0], "pg_stat_statements", "1.10" + ) + pgss111 = self.has_extension_version( + self.path_args[0], "pg_stat_statements", "1.11" + ) + + hit_ratio_graph = Graph( + "Hit ratio", + metrics=[QueryOverviewMetricGroup.hit_ratio], + renderer="bar", + stack=True, + color_scheme=["#73c03a", "#65b9ac", "#cb513a"], + ) + + gen_metrics = [ + QueryOverviewMetricGroup.avg_runtime, + QueryOverviewMetricGroup.rows, + QueryOverviewMetricGroup.calls, + ] if pgss18: gen_metrics.extend([QueryOverviewMetricGroup.avg_plantime]) dashes = [] - dashes.append(Dashboard("Query detail", - [[Graph("General", - metrics=gen_metrics)]])) + dashes.append( + Dashboard( + "Query detail", [[Graph("General", metrics=gen_metrics)]] + ) + ) if pgss18: # Add WALs graphs - wals_graphs = [[Graph("WAL activity", - metrics=[QueryOverviewMetricGroup.wal_records, - QueryOverviewMetricGroup.wal_fpi, - QueryOverviewMetricGroup.wal_bytes]), - ]] + wals_graphs = [ + [ + Graph( + "WAL activity", + metrics=[ + QueryOverviewMetricGroup.wal_records, + QueryOverviewMetricGroup.wal_fpi, + QueryOverviewMetricGroup.wal_bytes, + ], + ), + ] + ] dashes.append(Dashboard("WALs", wals_graphs)) # Add JIT graphs if pgss110: - jit_tim = [QueryOverviewMetricGroup.jit_inlining_time, - QueryOverviewMetricGroup.jit_optimization_time, - QueryOverviewMetricGroup.jit_emission_time, - ] + jit_tim = [ + QueryOverviewMetricGroup.jit_inlining_time, + QueryOverviewMetricGroup.jit_optimization_time, + QueryOverviewMetricGroup.jit_emission_time, + ] if pgss111: - jit_tim.extend([QueryOverviewMetricGroup.jit_deform_time, - QueryOverviewMetricGroup.jit_expr_time]) + jit_tim.extend( + [ + QueryOverviewMetricGroup.jit_deform_time, + QueryOverviewMetricGroup.jit_expr_time, + ] + ) else: jit_tim.append(QueryOverviewMetricGroup.jit_generation_time) - jit_cnt = [QueryOverviewMetricGroup.jit_functions, - QueryOverviewMetricGroup.jit_inlining_count, - QueryOverviewMetricGroup.jit_optimization_count, - QueryOverviewMetricGroup.jit_emission_count, - ] + jit_cnt = [ + QueryOverviewMetricGroup.jit_functions, + QueryOverviewMetricGroup.jit_inlining_count, + QueryOverviewMetricGroup.jit_optimization_count, + QueryOverviewMetricGroup.jit_emission_count, + ] if pgss111: jit_cnt.append(QueryOverviewMetricGroup.jit_deform_count) - jit_graphs = [[Graph("JIT timing", metrics=jit_tim, - stack=True)], - [Graph("JIT scheduling", metrics=jit_cnt)]] + jit_graphs = [ + [Graph("JIT timing", metrics=jit_tim, stack=True)], + [Graph("JIT scheduling", metrics=jit_cnt)], + ] dashes.append(Dashboard("JIT", jit_graphs)) - dashes.append(Dashboard( - "PG Cache", - [[Graph("Shared block (in Bps)", - metrics=[QueryOverviewMetricGroup.shared_blks_read, - QueryOverviewMetricGroup.shared_blks_hit, - QueryOverviewMetricGroup.shared_blks_dirtied, - QueryOverviewMetricGroup.shared_blks_written]), - Graph("Local block (in Bps)", - metrics=[QueryOverviewMetricGroup.local_blks_read, - QueryOverviewMetricGroup.local_blks_hit, - QueryOverviewMetricGroup.local_blks_dirtied, - QueryOverviewMetricGroup.local_blks_written]), - Graph("Temp block (in Bps)", - metrics=[QueryOverviewMetricGroup.temp_blks_read, - QueryOverviewMetricGroup.temp_blks_written])]])) - - io_time_metrics=[QueryOverviewMetricGroup.shared_blk_read_time, - QueryOverviewMetricGroup.shared_blk_write_time, - ] + dashes.append( + Dashboard( + "PG Cache", + [ + [ + Graph( + "Shared block (in Bps)", + metrics=[ + QueryOverviewMetricGroup.shared_blks_read, + QueryOverviewMetricGroup.shared_blks_hit, + QueryOverviewMetricGroup.shared_blks_dirtied, + QueryOverviewMetricGroup.shared_blks_written, + ], + ), + Graph( + "Local block (in Bps)", + metrics=[ + QueryOverviewMetricGroup.local_blks_read, + QueryOverviewMetricGroup.local_blks_hit, + QueryOverviewMetricGroup.local_blks_dirtied, + QueryOverviewMetricGroup.local_blks_written, + ], + ), + Graph( + "Temp block (in Bps)", + metrics=[ + QueryOverviewMetricGroup.temp_blks_read, + QueryOverviewMetricGroup.temp_blks_written, + ], + ), + ] + ], + ) + ) + + io_time_metrics = [ + QueryOverviewMetricGroup.shared_blk_read_time, + QueryOverviewMetricGroup.shared_blk_write_time, + ] # if we can't connect to the remote server, assume pg16 or below if pgss111: - io_time_metrics.extend([ - QueryOverviewMetricGroup.local_blk_read_time, - QueryOverviewMetricGroup.local_blk_write_time, - QueryOverviewMetricGroup.temp_blk_read_time, - QueryOverviewMetricGroup.temp_blk_write_time, - ]) - iodash = Dashboard("IO", - [[hit_ratio_graph, - Graph("Read / Write time", - url=self.docs_stats_url + "pg_stat_statements.html", - metrics=io_time_metrics - )]]) + io_time_metrics.extend( + [ + QueryOverviewMetricGroup.local_blk_read_time, + QueryOverviewMetricGroup.local_blk_write_time, + QueryOverviewMetricGroup.temp_blk_read_time, + QueryOverviewMetricGroup.temp_blk_write_time, + ] + ) + iodash = Dashboard( + "IO", + [ + [ + hit_ratio_graph, + Graph( + "Read / Write time", + url=self.docs_stats_url + "pg_stat_statements.html", + metrics=io_time_metrics, + ), + ] + ], + ) dashes.append(iodash) if self.has_extension(self.path_args[0], "pg_stat_kcache"): - iodash.widgets.extend([[ - Graph("Physical block (in Bps)", - url=self.docs_stats_url + "pg_stat_kcache.html", - metrics=[QueryOverviewMetricGroup.reads, - QueryOverviewMetricGroup.writes]), - Graph("CPU Time repartition", - url=self.docs_stats_url + "pg_stat_kcache.html", - metrics=[QueryOverviewMetricGroup.user_time, - QueryOverviewMetricGroup.system_time, - QueryOverviewMetricGroup.other_time], - renderer="bar", - stack=True, - color_scheme=['#73c03a','#cb513a','#65b9ac'])]]) + iodash.widgets.extend( + [ + [ + Graph( + "Physical block (in Bps)", + url=self.docs_stats_url + "pg_stat_kcache.html", + metrics=[ + QueryOverviewMetricGroup.reads, + QueryOverviewMetricGroup.writes, + ], + ), + Graph( + "CPU Time repartition", + url=self.docs_stats_url + "pg_stat_kcache.html", + metrics=[ + QueryOverviewMetricGroup.user_time, + QueryOverviewMetricGroup.system_time, + QueryOverviewMetricGroup.other_time, + ], + renderer="bar", + stack=True, + color_scheme=["#73c03a", "#cb513a", "#65b9ac"], + ), + ] + ] + ) hit_ratio_graph.metrics.append( - QueryOverviewMetricGroup.sys_hit_ratio) + QueryOverviewMetricGroup.sys_hit_ratio + ) hit_ratio_graph.metrics.append( - QueryOverviewMetricGroup.disk_hit_ratio) - - sys_graphs = [Graph("System resources (events per sec)", - url=self.docs_stats_url + "pg_stat_kcache.html", - metrics=[QueryOverviewMetricGroup.majflts, - QueryOverviewMetricGroup.minflts, - # QueryOverviewMetricGroup.nswaps, - # QueryOverviewMetricGroup.msgsnds, - # QueryOverviewMetricGroup.msgrcvs, - # QueryOverviewMetricGroup.nsignals, - QueryOverviewMetricGroup.nvcsws, - QueryOverviewMetricGroup.nivcsws])] + QueryOverviewMetricGroup.disk_hit_ratio + ) + + sys_graphs = [ + Graph( + "System resources (events per sec)", + url=self.docs_stats_url + "pg_stat_kcache.html", + metrics=[ + QueryOverviewMetricGroup.majflts, + QueryOverviewMetricGroup.minflts, + # QueryOverviewMetricGroup.nswaps, + # QueryOverviewMetricGroup.msgsnds, + # QueryOverviewMetricGroup.msgrcvs, + # QueryOverviewMetricGroup.nsignals, + QueryOverviewMetricGroup.nvcsws, + QueryOverviewMetricGroup.nivcsws, + ], + ) + ] dashes.append(Dashboard("System resources", [sys_graphs])) else: - hit_ratio_graph.metrics.append( - QueryOverviewMetricGroup.miss_ratio) + hit_ratio_graph.metrics.append(QueryOverviewMetricGroup.miss_ratio) if self.has_extension(self.path_args[0], "pg_wait_sampling"): # Get the metrics depending on the pg server version @@ -820,51 +1121,90 @@ def dashboard(self): pg_version_num = self.get_pg_version_num(self.path_args[0]) # if we can't connect to the remote server, assume pg10 or above if pg_version_num is not None and pg_version_num < 100000: - metrics=[WaitsQueryOverviewMetricGroup.count_lwlocknamed, - WaitsQueryOverviewMetricGroup.count_lwlocktranche, - WaitsQueryOverviewMetricGroup.count_lock, - WaitsQueryOverviewMetricGroup.count_bufferpin] + metrics = [ + WaitsQueryOverviewMetricGroup.count_lwlocknamed, + WaitsQueryOverviewMetricGroup.count_lwlocktranche, + WaitsQueryOverviewMetricGroup.count_lock, + WaitsQueryOverviewMetricGroup.count_bufferpin, + ] else: - metrics=[WaitsQueryOverviewMetricGroup.count_lwlock, - WaitsQueryOverviewMetricGroup.count_lock, - WaitsQueryOverviewMetricGroup.count_bufferpin, - WaitsQueryOverviewMetricGroup.count_activity, - WaitsQueryOverviewMetricGroup.count_client, - WaitsQueryOverviewMetricGroup.count_extension, - WaitsQueryOverviewMetricGroup.count_ipc, - WaitsQueryOverviewMetricGroup.count_timeout, - WaitsQueryOverviewMetricGroup.count_io] - dashes.append(Dashboard("Wait Events", - [[Graph("Wait Events (per second)", - url=self.docs_stats_url + "pg_wait_sampling.html", - metrics=metrics), - Grid("Wait events summary", - url=self.docs_stats_url + "pg_wait_sampling.html", - columns=[{ - "name": "event_type", - "label": "Event Type", - }, { - "name": "event", - "label": "Event", - }], - metrics=WaitSamplingList.all())]])) + metrics = [ + WaitsQueryOverviewMetricGroup.count_lwlock, + WaitsQueryOverviewMetricGroup.count_lock, + WaitsQueryOverviewMetricGroup.count_bufferpin, + WaitsQueryOverviewMetricGroup.count_activity, + WaitsQueryOverviewMetricGroup.count_client, + WaitsQueryOverviewMetricGroup.count_extension, + WaitsQueryOverviewMetricGroup.count_ipc, + WaitsQueryOverviewMetricGroup.count_timeout, + WaitsQueryOverviewMetricGroup.count_io, + ] + dashes.append( + Dashboard( + "Wait Events", + [ + [ + Graph( + "Wait Events (per second)", + url=self.docs_stats_url + + "pg_wait_sampling.html", + metrics=metrics, + ), + Grid( + "Wait events summary", + url=self.docs_stats_url + + "pg_wait_sampling.html", + columns=[ + { + "name": "event_type", + "label": "Event Type", + }, + { + "name": "event", + "label": "Event", + }, + ], + metrics=WaitSamplingList.all(), + ), + ] + ], + ) + ) if self.has_extension(self.path_args[0], "pg_qualstats"): - dashes.append(Dashboard("Predicates", - [[ - Grid("Predicates used by this query", - columns=[{ - "name": "where_clause", - "label": "Predicate", - "type": "query", - "max_length": 60, - "url_attr": "url" - }], - metrics=QualList.all())], - [QueryIndexes], - [QueryExplains]])) - self._dashboard = Dashboard("Query %(query)s on database %(database)s", - [[QueryDetail], [ - TabContainer("Query %(query)s on database %(database)s", - dashes)]]) + dashes.append( + Dashboard( + "Predicates", + [ + [ + Grid( + "Predicates used by this query", + columns=[ + { + "name": "where_clause", + "label": "Predicate", + "type": "query", + "max_length": 60, + "url_attr": "url", + } + ], + metrics=QualList.all(), + ) + ], + [QueryIndexes], + [QueryExplains], + ], + ) + ) + self._dashboard = Dashboard( + "Query %(query)s on database %(database)s", + [ + [QueryDetail], + [ + TabContainer( + "Query %(query)s on database %(database)s", dashes + ) + ], + ], + ) return self._dashboard diff --git a/powa/server.py b/powa/server.py index eeb65277..ff664798 100644 --- a/powa/server.py +++ b/powa/server.py @@ -2,57 +2,72 @@ Index page presenting an overview of the cluster stats. """ -from tornado.web import HTTPError -from powa.framework import AuthHandler -from powa.dashboards import ( - Dashboard, Graph, Grid, - MetricGroupDef, MetricDef, - DashboardPage, TabContainer) from powa.config import ConfigChangesGlobal -from powa.overview import Overview - -from powa.sql.views_graph import (powa_getstatdata_sample, - kcache_getstatdata_sample, - powa_getwaitdata_sample, - powa_get_pgsa_sample, - powa_get_archiver_sample, - powa_get_bgwriter_sample, - powa_get_checkpointer_sample, - powa_get_replication_sample, - powa_get_all_idx_sample, - powa_get_all_tbl_sample, - powa_get_user_fct_sample, - powa_get_database_sample, - powa_get_database_conflicts_sample, - powa_get_io_sample, - powa_get_slru_sample, - powa_get_subscription_sample, - powa_get_wal_sample, - powa_get_wal_receiver_sample) -from powa.sql.views_grid import (powa_getstatdata_db, - powa_getwaitdata_db, - powa_getuserfuncdata_db, - powa_getiodata, - powa_getslrudata) -from powa.sql.utils import (sum_per_sec, byte_per_sec, wps, total_read, - total_hit, block_size, to_epoch, get_ts, mulblock) - +from powa.dashboards import ( + Dashboard, + DashboardPage, + Graph, + Grid, + MetricDef, + MetricGroupDef, + TabContainer, +) +from powa.framework import AuthHandler from powa.io_template import TemplateIoGraph, TemplateIoGrid +from powa.overview import Overview +from powa.sql.utils import ( + block_size, + byte_per_sec, + get_ts, + mulblock, + sum_per_sec, + to_epoch, + total_hit, + total_read, + wps, +) +from powa.sql.views_graph import ( + kcache_getstatdata_sample, + powa_get_all_idx_sample, + powa_get_all_tbl_sample, + powa_get_archiver_sample, + powa_get_bgwriter_sample, + powa_get_checkpointer_sample, + powa_get_database_conflicts_sample, + powa_get_database_sample, + powa_get_pgsa_sample, + powa_get_replication_sample, + powa_get_slru_sample, + powa_get_subscription_sample, + powa_get_user_fct_sample, + powa_get_wal_receiver_sample, + powa_get_wal_sample, + powa_getstatdata_sample, + powa_getwaitdata_sample, +) +from powa.sql.views_grid import ( + powa_getslrudata, + powa_getstatdata_db, + powa_getuserfuncdata_db, + powa_getwaitdata_db, +) +from tornado.web import HTTPError class ServerSelector(AuthHandler): """Page allowing to choose a server.""" def get(self): - self.redirect(self.reverse_url( - 'ServerOverview', - self.get_argument("srvid"))) + self.redirect( + self.reverse_url("ServerOverview", self.get_argument("srvid")) + ) class ByDatabaseMetricGroup(MetricGroupDef): """ Metric group used by the "by database" grid """ + name = "by_database" xaxis = "datname" data_url = r"/server/(\d+)/metrics/by_databases/" @@ -79,13 +94,15 @@ class ByDatabaseMetricGroup(MetricGroupDef): def _get_metrics(cls, handler, **params): base = cls.metrics.copy() - if not handler.has_extension_version(handler.path_args[0], - 'pg_stat_statements', '1.8'): + if not handler.has_extension_version( + handler.path_args[0], "pg_stat_statements", "1.8" + ): for key in ("plantime", "wal_records", "wal_fpi", "wal_bytes"): base.pop(key) - if not handler.has_extension_version(handler.path_args[0], - 'pg_stat_statements', '1.10'): + if not handler.has_extension_version( + handler.path_args[0], "pg_stat_statements", "1.10" + ): base.pop("jit_functions") base.pop("jit_time") @@ -97,9 +114,7 @@ def query(self): inner_query = powa_getstatdata_db("%(server)s") from_clause = """({inner_query}) AS sub JOIN {{powa}}.powa_databases pd ON pd.oid = sub.dbid - AND pd.srvid = sub.srvid""".format( - inner_query=inner_query - ) + AND pd.srvid = sub.srvid""".format(inner_query=inner_query) cols = [ "pd.srvid", @@ -107,47 +122,52 @@ def query(self): "sum(calls) AS calls", "sum(runtime) AS runtime", "round(cast(sum(runtime) AS numeric) / greatest(sum(calls), 1), 2) AS avg_runtime", - mulblock('shared_blks_read', fn="sum"), - mulblock('shared_blks_hit', fn="sum"), - mulblock('shared_blks_dirtied', fn="sum"), - mulblock('shared_blks_written', fn="sum"), - mulblock('temp_blks_read', fn="sum"), - mulblock('temp_blks_written', fn="sum"), + mulblock("shared_blks_read", fn="sum"), + mulblock("shared_blks_hit", fn="sum"), + mulblock("shared_blks_dirtied", fn="sum"), + mulblock("shared_blks_written", fn="sum"), + mulblock("temp_blks_read", fn="sum"), + mulblock("temp_blks_written", fn="sum"), "round(cast(sum(shared_blk_read_time + shared_blk_write_time" - + " + local_blk_read_time + local_blk_write_time" - + " + temp_blk_read_time + temp_blk_write_time" - + ") AS numeric), 2) AS io_time" - ] + + " + local_blk_read_time + local_blk_write_time" + + " + temp_blk_read_time + temp_blk_write_time" + + ") AS numeric), 2) AS io_time", + ] + + if self.has_extension_version( + self.path_args[0], "pg_stat_statements", "1.8" + ): + cols.extend( + [ + "sum(plantime) AS plantime", + "sum(wal_records) AS wal_records", + "sum(wal_fpi) AS wal_fpi," "sum(wal_bytes) AS wal_bytes", + ] + ) - if self.has_extension_version(self.path_args[0], 'pg_stat_statements', - '1.8'): - cols.extend([ - "sum(plantime) AS plantime", - "sum(wal_records) AS wal_records", - "sum(wal_fpi) AS wal_fpi," - "sum(wal_bytes) AS wal_bytes"]) - - if self.has_extension_version(self.path_args[0], 'pg_stat_statements', - '1.10'): - cols.extend([ + if self.has_extension_version( + self.path_args[0], "pg_stat_statements", "1.10" + ): + cols.extend( + [ "sum(jit_functions) AS jit_functions", "sum(jit_generation_time + jit_inlining_time" + " + jit_optimization_time + jit_emission_time)" - + " AS jit_time" - ]) + + " AS jit_time", + ] + ) return """SELECT {cols} FROM {from_clause} CROSS JOIN {bs} GROUP BY pd.srvid, pd.datname, block_size""".format( - cols=', '.join(cols), - from_clause=from_clause, - bs=bs + cols=", ".join(cols), from_clause=from_clause, bs=bs ) def process(self, val, **kwargs): - val["url"] = self.reverse_url("DatabaseOverview", val["srvid"], - val["datname"]) + val["url"] = self.reverse_url( + "DatabaseOverview", val["srvid"], val["datname"] + ) return val @@ -155,12 +175,14 @@ class ByDatabaseWaitSamplingMetricGroup(MetricGroupDef): """ Metric group used by the "wait sampling by database" grid """ + name = "wait_sampling_by_database" xaxis = "datname" data_url = r"/server/(\d+)/metrics/wait_event_by_databases/" axis_type = "category" - counts = MetricDef(label="# of events", - type="integer", direction="descending") + counts = MetricDef( + label="# of events", type="integer", direction="descending" + ) @property def query(self): @@ -168,21 +190,18 @@ def query(self): from_clause = """({inner_query}) AS sub JOIN {{powa}}.powa_databases pd ON pd.oid = sub.dbid - AND pd.srvid = sub.srvid""".format( - inner_query=inner_query - ) + AND pd.srvid = sub.srvid""".format(inner_query=inner_query) return """SELECT pd.srvid, pd.datname, sub.event_type, sub.event, sum(sub.count) AS counts FROM {from_clause} GROUP BY pd.srvid, pd.datname, sub.event_type, sub.event - ORDER BY sum(sub.count) DESC""".format( - from_clause=from_clause - ) + ORDER BY sum(sub.count) DESC""".format(from_clause=from_clause) def process(self, val, **kwargs): - val["url"] = self.reverse_url("DatabaseOverview", - val["srvid"], val["datname"]) + val["url"] = self.reverse_url( + "DatabaseOverview", val["srvid"], val["datname"] + ) return val @@ -190,16 +209,20 @@ class ByDatabaseUserFuncMetricGroup(MetricGroupDef): """ Metric group used by the pg_stat_user_functions grid """ + name = "user_functions_by_database" xaxis = "datname" data_url = r"/server/(\d+)/metrics/user_functions_by_databases/" axis_type = "category" - calls = MetricDef(label="# of calls", - type="integer", direction="descending") - total_time = MetricDef(label="Cumulated total execution time", - type="duration") - self_time = MetricDef(label="Cumulated self execution time", - type="duration") + calls = MetricDef( + label="# of calls", type="integer", direction="descending" + ) + total_time = MetricDef( + label="Cumulated total execution time", type="duration" + ) + self_time = MetricDef( + label="Cumulated self execution time", type="duration" + ) @property def query(self): @@ -208,8 +231,9 @@ def query(self): return query + " ORDER BY calls DESC" def process(self, val, **kwargs): - val["url"] = self.reverse_url("DatabaseOverview", - val["srvid"], val["datname"]) + val["url"] = self.reverse_url( + "DatabaseOverview", val["srvid"], val["datname"] + ) return val @@ -217,108 +241,189 @@ class GlobalDatabasesMetricGroup(MetricGroupDef): """ Metric group used by summarized graphs. """ + name = "all_databases" xaxis = "ts" data_url = r"/server/(\d+)/metrics/databases_globals/" - avg_runtime = MetricDef(label="Avg runtime", type="duration", - desc="Average query duration") - calls = MetricDef(label="Queries per sec", type="number", - desc="Number of time the query has been executed") - planload = MetricDef(label="Plantime per sec", type="duration", - desc="Total planning duration") - load = MetricDef(label="Runtime per sec", type="duration", - desc="Total duration of queries executed") - total_blks_hit = MetricDef(label="Total hit", type="sizerate", - desc="Amount of data found in shared buffers") - total_blks_read = MetricDef(label="Total read", type="sizerate", - desc="Amount of data found in OS cache or" - " read from disk") - wal_records = MetricDef(label="#Wal records", type="integer", - desc="Number of WAL records generated") - wal_fpi = MetricDef(label="#Wal FPI", type="integer", - desc="Number of WAL full-page images generated") - wal_bytes = MetricDef(label="Wal bytes", type="size", - desc="Amount of WAL bytes generated") - - total_sys_hit = MetricDef(label="Total system cache hit", type="sizerate", - desc="Amount of data found in OS cache") - total_disk_read = MetricDef(label="Total disk read", type="sizerate", - desc="Amount of data read from disk") - minflts = MetricDef(label="Soft page faults", type="number", - desc="Memory pages not found in the processor's MMU") - majflts = MetricDef(label="Hard page faults", type="number", - desc="Memory pages not found in memory and loaded" - " from storage") + avg_runtime = MetricDef( + label="Avg runtime", type="duration", desc="Average query duration" + ) + calls = MetricDef( + label="Queries per sec", + type="number", + desc="Number of time the query has been executed", + ) + planload = MetricDef( + label="Plantime per sec", + type="duration", + desc="Total planning duration", + ) + load = MetricDef( + label="Runtime per sec", + type="duration", + desc="Total duration of queries executed", + ) + total_blks_hit = MetricDef( + label="Total hit", + type="sizerate", + desc="Amount of data found in shared buffers", + ) + total_blks_read = MetricDef( + label="Total read", + type="sizerate", + desc="Amount of data found in OS cache or" " read from disk", + ) + wal_records = MetricDef( + label="#Wal records", + type="integer", + desc="Number of WAL records generated", + ) + wal_fpi = MetricDef( + label="#Wal FPI", + type="integer", + desc="Number of WAL full-page images generated", + ) + wal_bytes = MetricDef( + label="Wal bytes", type="size", desc="Amount of WAL bytes generated" + ) + + total_sys_hit = MetricDef( + label="Total system cache hit", + type="sizerate", + desc="Amount of data found in OS cache", + ) + total_disk_read = MetricDef( + label="Total disk read", + type="sizerate", + desc="Amount of data read from disk", + ) + minflts = MetricDef( + label="Soft page faults", + type="number", + desc="Memory pages not found in the processor's MMU", + ) + majflts = MetricDef( + label="Hard page faults", + type="number", + desc="Memory pages not found in memory and loaded" " from storage", + ) # not maintained on GNU/Linux, and not available on Windows # nswaps = MetricDef(label="Swaps", type="number") # msgsnds = MetricDef(label="IPC messages sent", type="number") # msgrcvs = MetricDef(label="IPC messages received", type="number") # nsignals = MetricDef(label="Signals received", type="number") - nvcsws = MetricDef(label="Voluntary context switches", type="number", - desc="Number of voluntary context switches") - nivcsws = MetricDef(label="Involuntary context switches", type="number", - desc="Number of involuntary context switches") - jit_functions = MetricDef(label="# of JIT functions", type="integer", - desc="Total number of emitted functions") - jit_generation_time = MetricDef(label="JIT generation time", - type="duration", - desc="Total time spent generating code") - jit_inlining_count = MetricDef(label="# of JIT inlining", type="integer", - desc="Number of queries where inlining was" - " done") - jit_inlining_time = MetricDef(label="JIT inlining time", type="duration", - desc="Total time spent inlining code") - jit_optimization_count = MetricDef(label="# of JIT optimization", - type="integer", - desc="Number of queries where" - " optimization was done") - jit_optimization_time = MetricDef(label="JIT optimization time", - type="duration", - desc="Total time spent optimizing code") - jit_emission_count = MetricDef(label="# of JIT emission", type="integer", - desc="Number of queries where emission was" - " done") - jit_emission_time = MetricDef(label="JIT emission time", type="duration", - desc="Total time spent emitting code") - jit_deform_count = MetricDef(label="# of JIT tuple deforming", - type="integer", - desc="Number of queries where tuple deforming" - " was done") - jit_deform_time = MetricDef(label="JIT tuple deforming time", - type="duration", - desc="Total time spent deforming tuple") - jit_expr_time = MetricDef(label="JIT expression generation time", - type="duration", - desc="Total time spent generating expressions") + nvcsws = MetricDef( + label="Voluntary context switches", + type="number", + desc="Number of voluntary context switches", + ) + nivcsws = MetricDef( + label="Involuntary context switches", + type="number", + desc="Number of involuntary context switches", + ) + jit_functions = MetricDef( + label="# of JIT functions", + type="integer", + desc="Total number of emitted functions", + ) + jit_generation_time = MetricDef( + label="JIT generation time", + type="duration", + desc="Total time spent generating code", + ) + jit_inlining_count = MetricDef( + label="# of JIT inlining", + type="integer", + desc="Number of queries where inlining was" " done", + ) + jit_inlining_time = MetricDef( + label="JIT inlining time", + type="duration", + desc="Total time spent inlining code", + ) + jit_optimization_count = MetricDef( + label="# of JIT optimization", + type="integer", + desc="Number of queries where" " optimization was done", + ) + jit_optimization_time = MetricDef( + label="JIT optimization time", + type="duration", + desc="Total time spent optimizing code", + ) + jit_emission_count = MetricDef( + label="# of JIT emission", + type="integer", + desc="Number of queries where emission was" " done", + ) + jit_emission_time = MetricDef( + label="JIT emission time", + type="duration", + desc="Total time spent emitting code", + ) + jit_deform_count = MetricDef( + label="# of JIT tuple deforming", + type="integer", + desc="Number of queries where tuple deforming" " was done", + ) + jit_deform_time = MetricDef( + label="JIT tuple deforming time", + type="duration", + desc="Total time spent deforming tuple", + ) + jit_expr_time = MetricDef( + label="JIT expression generation time", + type="duration", + desc="Total time spent generating expressions", + ) @classmethod def _get_metrics(cls, handler, **params): base = cls.metrics.copy() if not handler.has_extension(params["server"], "pg_stat_kcache"): - for key in ("total_sys_hit", "total_disk_read", "minflts", - "majflts", - # "nswaps", "msgsnds", "msgrcvs", "nsignals", - "nvcsws", "nivcsws"): + for key in ( + "total_sys_hit", + "total_disk_read", + "minflts", + "majflts", + # "nswaps", "msgsnds", "msgrcvs", "nsignals", + "nvcsws", + "nivcsws", + ): base.pop(key) else: base.pop("total_blks_read") - if not handler.has_extension_version(params["server"], - 'pg_stat_statements', '1.8'): + if not handler.has_extension_version( + params["server"], "pg_stat_statements", "1.8" + ): for key in ("planload", "wal_records", "wal_fpi", "wal_bytes"): base.pop(key) - if not handler.has_extension_version(handler.path_args[0], - 'pg_stat_statements', '1.10'): - for key in ("jit_functions", "jit_generation_time", - "jit_inlining_count", "jit_inlining_time", - "jit_optimization_count", "jit_optimization_time", - "jit_emission_count", "jit_emission_time"): + if not handler.has_extension_version( + handler.path_args[0], "pg_stat_statements", "1.10" + ): + for key in ( + "jit_functions", + "jit_generation_time", + "jit_inlining_count", + "jit_inlining_time", + "jit_optimization_count", + "jit_optimization_time", + "jit_emission_count", + "jit_emission_time", + ): base.pop(key) - if not handler.has_extension_version(handler.path_args[0], - 'pg_stat_statements', '1.11'): - for key in ("jit_deform_count", "jit_deform_time", "jit_expr_time"): + if not handler.has_extension_version( + handler.path_args[0], "pg_stat_statements", "1.11" + ): + for key in ( + "jit_deform_count", + "jit_deform_time", + "jit_expr_time", + ): base.pop(key) return base @@ -331,42 +436,54 @@ def query(self): cols = [ "sub.srvid", "extract(epoch FROM ts) AS ts", - sum_per_sec('calls'), + sum_per_sec("calls"), "sum(runtime) / greatest(sum(calls), 1) AS avg_runtime", - sum_per_sec('runtime', alias='load'), + sum_per_sec("runtime", alias="load"), total_read("sub"), - total_hit("sub") + total_hit("sub"), ] - if self.has_extension_version(self.path_args[0], - 'pg_stat_statements', '1.8'): - cols.extend([ - sum_per_sec('plantime', alias='planload'), - sum_per_sec('wal_records'), - sum_per_sec('wal_fpi'), - sum_per_sec('wal_bytes') - ]) - - if self.has_extension_version(self.path_args[0], - 'pg_stat_statements', '1.10'): - cols.extend([ - sum_per_sec('jit_functions'), - sum_per_sec('jit_generation_time'), - sum_per_sec('jit_inlining_count'), - sum_per_sec('jit_inlining_time'), - sum_per_sec('jit_optimization_count'), - sum_per_sec('jit_optimization_time'), - sum_per_sec('jit_emission_count'), - sum_per_sec('jit_emission_time'), - ]) - - if self.has_extension_version(self.path_args[0], - 'pg_stat_statements', '1.11'): - cols.extend([ - sum_per_sec('jit_deform_count'), - sum_per_sec('jit_deform_time'), - sum_per_sec('jit_generation_time - jit_deform_time', alias='jit_expr_time'), - ]) + if self.has_extension_version( + self.path_args[0], "pg_stat_statements", "1.8" + ): + cols.extend( + [ + sum_per_sec("plantime", alias="planload"), + sum_per_sec("wal_records"), + sum_per_sec("wal_fpi"), + sum_per_sec("wal_bytes"), + ] + ) + + if self.has_extension_version( + self.path_args[0], "pg_stat_statements", "1.10" + ): + cols.extend( + [ + sum_per_sec("jit_functions"), + sum_per_sec("jit_generation_time"), + sum_per_sec("jit_inlining_count"), + sum_per_sec("jit_inlining_time"), + sum_per_sec("jit_optimization_count"), + sum_per_sec("jit_optimization_time"), + sum_per_sec("jit_emission_count"), + sum_per_sec("jit_emission_time"), + ] + ) + + if self.has_extension_version( + self.path_args[0], "pg_stat_statements", "1.11" + ): + cols.extend( + [ + sum_per_sec("jit_deform_count"), + sum_per_sec("jit_deform_time"), + sum_per_sec( + "jit_generation_time - jit_deform_time", + alias="jit_expr_time", + ), + ] + ) from_clause = query @@ -376,31 +493,39 @@ def query(self): # Add system metrics from pg_stat_kcache, kcache_query = kcache_getstatdata_sample("db") - total_sys_hit = "{total_read} - sum(sub.reads)" \ - "/ {ts}" \ - " AS total_sys_hit""".format( - total_read=total_read('sub', True), - ts=get_ts() - ) - total_disk_read = "sum(sub.reads)" \ - " / " + get_ts() + \ - " AS total_disk_read" - minflts = sum_per_sec('minflts', prefix="sub") - majflts = sum_per_sec('majflts', prefix="sub") + total_sys_hit = ( + "{total_read} - sum(sub.reads)" + "/ {ts}" + " AS total_sys_hit" + "".format(total_read=total_read("sub", True), ts=get_ts()) + ) + total_disk_read = ( + "sum(sub.reads)" " / " + get_ts() + " AS total_disk_read" + ) + minflts = sum_per_sec("minflts", prefix="sub") + majflts = sum_per_sec("majflts", prefix="sub") # nswaps = sum_per_sec('nswaps', prefix="sub") # msgsnds = sum_per_sec('msgsnds', prefix="sub") # msgrcvs = sum_per_sec('msgrcvs', prefix="sub") # nsignals = sum_per_sec(.nsignals', prefix="sub") - nvcsws = sum_per_sec('nvcsws', prefix="sub") - nivcsws = sum_per_sec('nivcsws', prefix="sub") - - cols.extend([total_sys_hit, total_disk_read, minflts, majflts, - # nswaps, msgsnds, msgrcvs, nsignals, - nvcsws, nivcsws]) + nvcsws = sum_per_sec("nvcsws", prefix="sub") + nivcsws = sum_per_sec("nivcsws", prefix="sub") + + cols.extend( + [ + total_sys_hit, + total_disk_read, + minflts, + majflts, + # nswaps, msgsnds, msgrcvs, nsignals, + nvcsws, + nivcsws, + ] + ) from_clause += """ LEFT JOIN ({kcache_query}) AS kc USING (dbid, ts, srvid)""".format( - kcache_query=kcache_query + kcache_query=kcache_query ) return """SELECT {cols} @@ -411,49 +536,58 @@ def query(self): WHERE sub.calls != 0 GROUP BY sub.srvid, sub.ts, block_size, sub.mesure_interval ORDER BY sub.ts""".format( - cols=', '.join(cols), - from_clause=from_clause, - bs=bs + cols=", ".join(cols), from_clause=from_clause, bs=bs ) class GlobalWaitsMetricGroup(MetricGroupDef): """Metric group for global wait events graphs.""" + name = "all_databases_waits" xaxis = "ts" data_url = r"/server/(\d+)/metrics/databases_waits/" # pg 9.6 only metrics - count_lwlocknamed = MetricDef(label="Lightweight Named", - desc="Number of named lightweight lock" - " wait events") - count_lwlocktranche = MetricDef(label="Lightweight Tranche", - desc="Number of lightweight lock tranche" - " wait events") + count_lwlocknamed = MetricDef( + label="Lightweight Named", + desc="Number of named lightweight lock" " wait events", + ) + count_lwlocktranche = MetricDef( + label="Lightweight Tranche", + desc="Number of lightweight lock tranche" " wait events", + ) # pg 10+ metrics - count_lwlock = MetricDef(label="Lightweight Lock", - desc="Number of wait events due to lightweight" - " locks") - count_lock = MetricDef(label="Lock", - desc="Number of wait events due to heavyweight" - " locks") - count_bufferpin = MetricDef(label="Buffer pin", - desc="Number of wait events due to buffer pin") - count_activity = MetricDef(label="Activity", - desc="Number of wait events due to postgres" - " internal processes activity") - count_client = MetricDef(label="Client", - desc="Number of wait events due to client" - " activity") - count_extension = MetricDef(label="Extension", - desc="Number wait events due to third-party" - " extensions") - count_ipc = MetricDef(label="IPC", - desc="Number of wait events due to inter-process" - "communication") - count_timeout = MetricDef(label="Timeout", - desc="Number of wait events due to timeouts") - count_io = MetricDef(label="IO", - desc="Number of wait events due to IO operations") + count_lwlock = MetricDef( + label="Lightweight Lock", + desc="Number of wait events due to lightweight" " locks", + ) + count_lock = MetricDef( + label="Lock", desc="Number of wait events due to heavyweight" " locks" + ) + count_bufferpin = MetricDef( + label="Buffer pin", desc="Number of wait events due to buffer pin" + ) + count_activity = MetricDef( + label="Activity", + desc="Number of wait events due to postgres" + " internal processes activity", + ) + count_client = MetricDef( + label="Client", desc="Number of wait events due to client" " activity" + ) + count_extension = MetricDef( + label="Extension", + desc="Number wait events due to third-party" " extensions", + ) + count_ipc = MetricDef( + label="IPC", + desc="Number of wait events due to inter-process" "communication", + ) + count_timeout = MetricDef( + label="Timeout", desc="Number of wait events due to timeouts" + ) + count_io = MetricDef( + label="IO", desc="Number of wait events due to IO operations" + ) def prepare(self): if not self.has_extension(self.path_args[0], "pg_wait_sampling"): @@ -463,18 +597,29 @@ def prepare(self): def query(self): query = powa_getwaitdata_sample("db") - cols = [to_epoch('ts', 'sub')] + cols = [to_epoch("ts", "sub")] pg_version_num = self.get_pg_version_num(self.path_args[0]) # if we can't connect to the remote server, assume pg10 or above if pg_version_num is not None and pg_version_num < 100000: - cols += [wps("count_lwlocknamed"), wps("count_lwlocktranche"), - wps("count_lock"), wps("count_bufferpin")] + cols += [ + wps("count_lwlocknamed"), + wps("count_lwlocktranche"), + wps("count_lock"), + wps("count_bufferpin"), + ] else: - cols += [wps("count_lwlock"), wps("count_lock"), - wps("count_bufferpin"), wps("count_activity"), - wps("count_client"), wps("count_extension"), - wps("count_ipc"), wps("count_timeout"), wps("count_io")] + cols += [ + wps("count_lwlock"), + wps("count_lock"), + wps("count_bufferpin"), + wps("count_activity"), + wps("count_client"), + wps("count_extension"), + wps("count_ipc"), + wps("count_timeout"), + wps("count_io"), + ] from_clause = "({query}) AS sub".format(query=query) @@ -483,30 +628,34 @@ def query(self): -- WHERE sub.count IS NOT NULL GROUP BY sub.ts, sub.mesure_interval ORDER BY sub.ts""".format( - cols=', '.join(cols), - from_clause=from_clause - ) + cols=", ".join(cols), from_clause=from_clause + ) class GlobalPGSAMetricGroup(MetricGroupDef): """ Metric group used by pg_stat_activity graphs """ + name = "pgsa" xaxis = "ts" data_url = r"/server/(\d+)/metrics/pgsa/" backend_xid_age = MetricDef(label="Backend xid age") backend_xmin_age = MetricDef(label="Backend xmin age") - oldest_backend = MetricDef(label="Oldest backend", type="duration", - desc="Age of the oldest backend, excluding" - " replication backends") + oldest_backend = MetricDef( + label="Oldest backend", + type="duration", + desc="Age of the oldest backend, excluding" " replication backends", + ) oldest_xact = MetricDef(label="Oldest transaction", type="duration") oldest_query = MetricDef(label="Oldest query", type="duration") nb_idle = MetricDef(label="# of idle connections") nb_active = MetricDef(label="# of active connections") nb_idle_xact = MetricDef(label="# of idle in transaction connections") nb_fastpath = MetricDef(label="# of connections in fastpath function call") - nb_idle_xact_abort = MetricDef(label="# of idle in transaction (aborted) connections") + nb_idle_xact_abort = MetricDef( + label="# of idle in transaction (aborted) connections" + ) nb_disabled = MetricDef(label="# of disabled connections") nb_unknown = MetricDef(label="# of connections in unknown state") nb_parallel_query = MetricDef(label="# of parallel queries") @@ -517,7 +666,7 @@ def _get_metrics(cls, handler, **params): base = cls.metrics.copy() remote_pg_ver = handler.get_pg_version_num(handler.path_args[0]) - if (remote_pg_ver is not None and remote_pg_ver < 130000): + if remote_pg_ver is not None and remote_pg_ver < 130000: for key in ("nb_parallel_query", "nb_parallel_worker"): base.pop(key) return base @@ -526,92 +675,104 @@ def _get_metrics(cls, handler, **params): def query(self): query = powa_get_pgsa_sample() - cols = ["extract(epoch FROM ts) AS ts", - "max(backend_xid_age) AS backend_xid_age", - "max(backend_xmin_age) AS backend_xmin_age", - "max(backend_start_age) FILTER (WHERE datid IS NOT NULL) AS oldest_backend", - "max(xact_start_age) FILTER (WHERE datid IS NOT NULL) AS oldest_xact", - "max(query_start_age) FILTER (WHERE datid IS NOT NULL) AS oldest_query", - "count(*) FILTER (WHERE state = 'idle') AS nb_idle", - "count(*) FILTER (WHERE state = 'active') AS nb_active", - "count(*) FILTER (WHERE state = 'idle in transaction') AS nb_idle_xact", - "count(*) FILTER (WHERE state = 'fastpath function call') AS nb_fastpath", - "count(*) FILTER (WHERE state = 'idle in transaction (aborted)') AS nb_idle_xact_abort", - "count(*) FILTER (WHERE state = 'disabled') AS nb_disabled", - "count(*) FILTER (WHERE state IS NULL) AS nb_unknown", - "count(DISTINCT leader_pid) AS nb_parallel_query", - "count(*) FILTER (WHERE leader_pid IS NOT NULL) AS nb_parallel_worker", - ] + cols = [ + "extract(epoch FROM ts) AS ts", + "max(backend_xid_age) AS backend_xid_age", + "max(backend_xmin_age) AS backend_xmin_age", + "max(backend_start_age) FILTER (WHERE datid IS NOT NULL) AS oldest_backend", + "max(xact_start_age) FILTER (WHERE datid IS NOT NULL) AS oldest_xact", + "max(query_start_age) FILTER (WHERE datid IS NOT NULL) AS oldest_query", + "count(*) FILTER (WHERE state = 'idle') AS nb_idle", + "count(*) FILTER (WHERE state = 'active') AS nb_active", + "count(*) FILTER (WHERE state = 'idle in transaction') AS nb_idle_xact", + "count(*) FILTER (WHERE state = 'fastpath function call') AS nb_fastpath", + "count(*) FILTER (WHERE state = 'idle in transaction (aborted)') AS nb_idle_xact_abort", + "count(*) FILTER (WHERE state = 'disabled') AS nb_disabled", + "count(*) FILTER (WHERE state IS NULL) AS nb_unknown", + "count(DISTINCT leader_pid) AS nb_parallel_query", + "count(*) FILTER (WHERE leader_pid IS NOT NULL) AS nb_parallel_worker", + ] return """SELECT {cols} FROM ({query}) AS sub GROUP BY ts - """.format( - cols=', '.join(cols), - query=query) + """.format(cols=", ".join(cols), query=query) class GlobalArchiverMetricGroup(MetricGroupDef): """ Metric group used by pg_stat_archiver graphs. """ + name = "archiver" xaxis = "ts" data_url = r"/server/(\d+)/metrics/archiver/" - nb_arch = MetricDef(label="# of archived WAL per second", - type="number", - desc="Number of WAL archived per second") - nb_to_arch = MetricDef(label="# of WAL to archive", - type="number", - desc="Number of WAL that needs to be archived") + nb_arch = MetricDef( + label="# of archived WAL per second", + type="number", + desc="Number of WAL archived per second", + ) + nb_to_arch = MetricDef( + label="# of WAL to archive", + type="number", + desc="Number of WAL that needs to be archived", + ) @property def query(self): query = powa_get_archiver_sample() cols = [ - "extract(epoch FROM ts) AS ts", - "round(nb_arch::numeric / " + get_ts() + ", 2) AS nb_arch", - "nb_to_arch" + "extract(epoch FROM ts) AS ts", + "round(nb_arch::numeric / " + get_ts() + ", 2) AS nb_arch", + "nb_to_arch", ] return """SELECT {cols} FROM ( {from_clause} - ) AS sub""".format( - cols=', '.join(cols), - from_clause=query) + ) AS sub""".format(cols=", ".join(cols), from_clause=query) + class GlobalBgwriterMetricGroup(MetricGroupDef): """ Metric group used by bgwriter graphs. """ + name = "bgwriter" xaxis = "ts" data_url = r"/server/(\d+)/metrics/bgwriter/" - buffers_clean = MetricDef(label="Buffers clean", - type="sizerate", - desc="Number of buffers written by the" - " background writer") - maxwritten_clean = MetricDef(label="Maxwritten clean", - type="number", - desc="Number of times the background writer" - " stopped a cleaning scan because it had" - " written too many buffers") - buffers_backend = MetricDef(label="Buffers backend", - type="sizerate", - desc="Number of buffers written directly by a" - " backend") - buffers_backend_fsync = MetricDef(label="Buffers backend fsync", - type="number", - desc="Number of times a backend had to" - " execute its own fsync call" - " (normally the background writer handles" - " those even when the backend does its" - " own write") - buffers_alloc = MetricDef(label="Buffers alloc", - type="sizerate", - desc="Number of buffers allocated") + buffers_clean = MetricDef( + label="Buffers clean", + type="sizerate", + desc="Number of buffers written by the" " background writer", + ) + maxwritten_clean = MetricDef( + label="Maxwritten clean", + type="number", + desc="Number of times the background writer" + " stopped a cleaning scan because it had" + " written too many buffers", + ) + buffers_backend = MetricDef( + label="Buffers backend", + type="sizerate", + desc="Number of buffers written directly by a" " backend", + ) + buffers_backend_fsync = MetricDef( + label="Buffers backend fsync", + type="number", + desc="Number of times a backend had to" + " execute its own fsync call" + " (normally the background writer handles" + " those even when the backend does its" + " own write", + ) + buffers_alloc = MetricDef( + label="Buffers alloc", + type="sizerate", + desc="Number of buffers allocated", + ) @property def query(self): @@ -620,14 +781,15 @@ def query(self): from_clause = query - cols = ["sub.srvid", - "extract(epoch FROM sub.ts) AS ts", - byte_per_sec("buffers_clean", prefix="sub"), - sum_per_sec("maxwritten_clean", prefix="sub"), - byte_per_sec("buffers_backend", prefix="sub"), - sum_per_sec("buffers_backend_fsync", prefix="sub"), - byte_per_sec("buffers_alloc", prefix="sub") - ] + cols = [ + "sub.srvid", + "extract(epoch FROM sub.ts) AS ts", + byte_per_sec("buffers_clean", prefix="sub"), + sum_per_sec("maxwritten_clean", prefix="sub"), + byte_per_sec("buffers_backend", prefix="sub"), + sum_per_sec("buffers_backend_fsync", prefix="sub"), + byte_per_sec("buffers_alloc", prefix="sub"), + ] return """SELECT {cols} FROM ( @@ -637,42 +799,49 @@ def query(self): WHERE sub.mesure_interval != '0 s' GROUP BY sub.srvid, sub.ts, block_size, sub.mesure_interval ORDER BY sub.ts""".format( - cols=', '.join(cols), - from_clause=from_clause, - bs=bs + cols=", ".join(cols), from_clause=from_clause, bs=bs ) + class GlobalCheckpointerMetricGroup(MetricGroupDef): """ Metric group used by bgwriter graphs. """ + name = "checkpointer" xaxis = "ts" data_url = r"/server/(\d+)/metrics/checkpointer/" - num_timed = MetricDef(label="# of scheduled checkpoints", - type="number", - desc="Number of scheduled checkpoints that" - " have been performed") - num_requested = MetricDef(label="# of requested checkpoints", - type="number", - desc="Number of requested checkpoints that" - " have been performed") - write_time = MetricDef(label="Write time", - type="duration", - desc="Total amount of time that has been" - " spent in the portion of checkpoint" - " processing where files are written to" - " disk, in milliseconds") - sync_time = MetricDef(label="Sync time", - type="duration", - desc="Total amount of time that has been" - " spent in the portion of checkpoint" - " processing where files are synchronized" - " to disk, in milliseconds") - buffers_written = MetricDef(label="Buffers checkpoint", - type="sizerate", - desc="Number of buffers written during" - " checkpoints") + num_timed = MetricDef( + label="# of scheduled checkpoints", + type="number", + desc="Number of scheduled checkpoints that" " have been performed", + ) + num_requested = MetricDef( + label="# of requested checkpoints", + type="number", + desc="Number of requested checkpoints that" " have been performed", + ) + write_time = MetricDef( + label="Write time", + type="duration", + desc="Total amount of time that has been" + " spent in the portion of checkpoint" + " processing where files are written to" + " disk, in milliseconds", + ) + sync_time = MetricDef( + label="Sync time", + type="duration", + desc="Total amount of time that has been" + " spent in the portion of checkpoint" + " processing where files are synchronized" + " to disk, in milliseconds", + ) + buffers_written = MetricDef( + label="Buffers checkpoint", + type="sizerate", + desc="Number of buffers written during" " checkpoints", + ) @property def query(self): @@ -681,14 +850,15 @@ def query(self): from_clause = query - cols = ["sub.srvid", - "extract(epoch FROM sub.ts) AS ts", - "sum(sub.num_timed) AS num_timed", - "sum(sub.num_requested) AS num_requested", - sum_per_sec("write_time", prefix="sub"), - sum_per_sec("sync_time", prefix="sub"), - byte_per_sec("buffers_written", prefix="sub"), - ] + cols = [ + "sub.srvid", + "extract(epoch FROM sub.ts) AS ts", + "sum(sub.num_timed) AS num_timed", + "sum(sub.num_requested) AS num_requested", + sum_per_sec("write_time", prefix="sub"), + sum_per_sec("sync_time", prefix="sub"), + byte_per_sec("buffers_written", prefix="sub"), + ] return """SELECT {cols} FROM ( @@ -698,55 +868,78 @@ def query(self): WHERE sub.mesure_interval != '0 s' GROUP BY sub.srvid, sub.ts, block_size, sub.mesure_interval ORDER BY sub.ts""".format( - cols=', '.join(cols), - from_clause=from_clause, - bs=bs + cols=", ".join(cols), from_clause=from_clause, bs=bs ) + class GlobalIoMetricGroup(TemplateIoGraph): """ Metric group used by pg_stat_io graphs. """ + name = "io" xaxis = "ts" data_url = r"/server/(\d+)/metrics/io/" + class ByAllIoMetricGroup(TemplateIoGrid): """ Metric group used by the pg_stat_io grid, with full detail (by backend_type, object and context). """ + name = "io_by_all" xaxis = "backend_type" data_url = r"/server/(\d+)/metrics/io_by_all/" + class GlobalSlruMetricGroup(MetricGroupDef): """ Metric group used by pg_stat_slru graph. """ + pass name = "slru" xaxis = "name" data_url = r"/server/(\d+)/metrics/slru/" - blks_zeroed = MetricDef(label="Zeroed", type="sizerate", - desc="Number of blocks zeroed during" - " initializations") - blks_hit = MetricDef(label="Hit", type="sizerate", - desc="Number of times disk blocks were found already" - " in the SLRU, so that a read was not necessary" - " (this only includes hits in the SLRU, not the" - " operating system's file system cache)") - blks_read = MetricDef(label="Read", type="sizerate", - desc="Number of disk blocks read for this SLRU") - blks_written = MetricDef(label="Written", type="sizerate", - desc="Number of disk blocks written for this SLRU") - blks_exists = MetricDef(label="Exists", type="sizerate", - desc="Number of blocks checked for existence for" - " this SLRU") - flushes = MetricDef(label="Flushes", type="number", - desc="Number of flushes of dirty data for this SLRU") - truncates = MetricDef(label="Truncates", type="number", - desc="Number of truncates for this SLRU") + blks_zeroed = MetricDef( + label="Zeroed", + type="sizerate", + desc="Number of blocks zeroed during" " initializations", + ) + blks_hit = MetricDef( + label="Hit", + type="sizerate", + desc="Number of times disk blocks were found already" + " in the SLRU, so that a read was not necessary" + " (this only includes hits in the SLRU, not the" + " operating system's file system cache)", + ) + blks_read = MetricDef( + label="Read", + type="sizerate", + desc="Number of disk blocks read for this SLRU", + ) + blks_written = MetricDef( + label="Written", + type="sizerate", + desc="Number of disk blocks written for this SLRU", + ) + blks_exists = MetricDef( + label="Exists", + type="sizerate", + desc="Number of blocks checked for existence for" " this SLRU", + ) + flushes = MetricDef( + label="Flushes", + type="number", + desc="Number of flushes of dirty data for this SLRU", + ) + truncates = MetricDef( + label="Truncates", + type="number", + desc="Number of truncates for this SLRU", + ) @classmethod def _get_metrics(cls, handler, **params): @@ -764,16 +957,17 @@ def query(self): from_clause = query - cols = ["sub.srvid", - "extract(epoch FROM sub.ts) AS ts", - sum_per_sec('blks_zeroed'), - sum_per_sec('blks_hit'), - sum_per_sec('blks_read'), - sum_per_sec('blks_written'), - sum_per_sec('blks_exists'), - sum_per_sec('flushes'), - sum_per_sec('truncates'), - ] + cols = [ + "sub.srvid", + "extract(epoch FROM sub.ts) AS ts", + sum_per_sec("blks_zeroed"), + sum_per_sec("blks_hit"), + sum_per_sec("blks_read"), + sum_per_sec("blks_written"), + sum_per_sec("blks_exists"), + sum_per_sec("flushes"), + sum_per_sec("truncates"), + ] return """SELECT {cols} FROM ( @@ -782,38 +976,57 @@ def query(self): WHERE sub.mesure_interval != '0 s' GROUP BY sub.srvid, sub.ts, sub.mesure_interval ORDER BY sub.ts""".format( - cols=', '.join(cols), + cols=", ".join(cols), from_clause=from_clause, ) + class ByAllSlruMetricGroup(MetricGroupDef): """ Metric group used by pg_stat_slru grid. """ + name = "slru_by_all" xaxis = "ts" data_url = r"/server/(\d+)/metrics/slru_by_all/" - blks_zeroed = MetricDef(label="Zeroed", type="size", - desc="Total number of blocks zeroed during" - " initializations") - blks_hit = MetricDef(label="Hit", type="size", - desc="Total number of times disk blocks were found" - " already in the SLRU, so that a read was not" - " necessary (this only includes hits in the SLRU," - " not the operating system's file system cache)") - blks_read = MetricDef(label="Read", type="size", - desc="Total number of disk blocks read for this SLRU") - blks_written = MetricDef(label="Written", type="size", - desc="Total number of disk blocks written for this" - " SLRU") - blks_exists = MetricDef(label="Exists", type="size", - desc="Total number of blocks checked for existence" - " for this SLRU") - flushes = MetricDef(label="Flushes", type="number", - desc="Total number of flushes of dirty data for this" - " SLRU") - truncates = MetricDef(label="Truncates", type="number", - desc="Total number of truncates for this SLRU") + blks_zeroed = MetricDef( + label="Zeroed", + type="size", + desc="Total number of blocks zeroed during" " initializations", + ) + blks_hit = MetricDef( + label="Hit", + type="size", + desc="Total number of times disk blocks were found" + " already in the SLRU, so that a read was not" + " necessary (this only includes hits in the SLRU," + " not the operating system's file system cache)", + ) + blks_read = MetricDef( + label="Read", + type="size", + desc="Total number of disk blocks read for this SLRU", + ) + blks_written = MetricDef( + label="Written", + type="size", + desc="Total number of disk blocks written for this" " SLRU", + ) + blks_exists = MetricDef( + label="Exists", + type="size", + desc="Total number of blocks checked for existence" " for this SLRU", + ) + flushes = MetricDef( + label="Flushes", + type="number", + desc="Total number of flushes of dirty data for this" " SLRU", + ) + truncates = MetricDef( + label="Truncates", + type="number", + desc="Total number of truncates for this SLRU", + ) @property def query(self): @@ -822,32 +1035,47 @@ def query(self): return query def process(self, val, **kwargs): - val["url"] = self.reverse_url("ByNameSlruOverview", val["srvid"], - val["name"]) + val["url"] = self.reverse_url( + "ByNameSlruOverview", val["srvid"], val["name"] + ) return val + class GlobalSubMetricGroup(MetricGroupDef): """ Metric group used by pg_stat_subscription(_stats) graphs. """ + pass name = "subscriptions" xaxis = "name" data_url = r"/server/(\d+)/metrics/subscriptions/" - last_msg_lag = MetricDef(label="Last message latency", type="duration", - desc="Time spent transmitting the last message" - " received from origin WAL sender") - report_lag = MetricDef(label="Report lag", type="duration", - desc="Time elapsed since since last reporting of" - " WAL location to origin WAL sender") - apply_error_count = MetricDef(label="# apply error", type="number", - desc="Total number of times an error" - " occurred while applying changes") - sync_error_count = MetricDef(label="# sync error", type="number", - desc="Total number of times an error" - " occurred during the inital table" - " synchronization") + last_msg_lag = MetricDef( + label="Last message latency", + type="duration", + desc="Time spent transmitting the last message" + " received from origin WAL sender", + ) + report_lag = MetricDef( + label="Report lag", + type="duration", + desc="Time elapsed since since last reporting of" + " WAL location to origin WAL sender", + ) + apply_error_count = MetricDef( + label="# apply error", + type="number", + desc="Total number of times an error" + " occurred while applying changes", + ) + sync_error_count = MetricDef( + label="# sync error", + type="number", + desc="Total number of times an error" + " occurred during the inital table" + " synchronization", + ) @classmethod def _get_metrics(cls, handler, **params): @@ -876,13 +1104,14 @@ def query(self): from_clause = query - cols = ["sub.srvid", - "extract(epoch FROM sub.ts) AS ts", - "max(last_msg_lag) AS last_msg_lag", - "max(report_lag) AS report_lag", - "sum(apply_error_count) AS apply_error_count", - "sum(sync_error_count) AS sync_error_count", - ] + cols = [ + "sub.srvid", + "extract(epoch FROM sub.ts) AS ts", + "max(last_msg_lag) AS last_msg_lag", + "max(report_lag) AS report_lag", + "sum(apply_error_count) AS apply_error_count", + "sum(sync_error_count) AS sync_error_count", + ] return """SELECT {cols} FROM ( @@ -891,38 +1120,64 @@ def query(self): WHERE sub.mesure_interval != '0 s' GROUP BY sub.srvid, sub.ts, sub.mesure_interval ORDER BY sub.ts""".format( - cols=', '.join(cols), + cols=", ".join(cols), from_clause=from_clause, ) + class GlobalWalMetricGroup(MetricGroupDef): """ Metric group used by pg_stat_wal graph. """ + name = "wal" xaxis = "name" data_url = r"/server/(\d+)/metrics/wal/" - wal_records = MetricDef(label="# records", type="number", - desc="Total number of WAL records generated") - wal_fpi = MetricDef(label="# fpi", type="number", - desc="Total number of WAL full page images generated") - wal_bytes = MetricDef(label="Generated", type="sizerate", - desc="Total amount of WAL generated in bytes") - wal_buffers_full = MetricDef(label="# buffers full", type="number", - desc="Number of times WAL data was written to" - " disk because WAL buffers became full") - wal_write = MetricDef(label="# writes", type="number", - desc="Number of times WAL buffers were written out" - " to disk via XLogWrite request") - wal_sync = MetricDef(label="# sync", type="number", - desc="Number of times WAL files were synced to disk" - " via issue_xlog_fsync request") - wal_write_time = MetricDef(label="Write time", type="duration", - desc="Total amount of time spent writing WAL buffers" - " to disk via XLogWrite request") - wal_sync_time = MetricDef(label="Sync time", type="duration", - desc="Total amount of time spent syncing WAL files to" - " disk via issue_xlog_fsync request") + wal_records = MetricDef( + label="# records", + type="number", + desc="Total number of WAL records generated", + ) + wal_fpi = MetricDef( + label="# fpi", + type="number", + desc="Total number of WAL full page images generated", + ) + wal_bytes = MetricDef( + label="Generated", + type="sizerate", + desc="Total amount of WAL generated in bytes", + ) + wal_buffers_full = MetricDef( + label="# buffers full", + type="number", + desc="Number of times WAL data was written to" + " disk because WAL buffers became full", + ) + wal_write = MetricDef( + label="# writes", + type="number", + desc="Number of times WAL buffers were written out" + " to disk via XLogWrite request", + ) + wal_sync = MetricDef( + label="# sync", + type="number", + desc="Number of times WAL files were synced to disk" + " via issue_xlog_fsync request", + ) + wal_write_time = MetricDef( + label="Write time", + type="duration", + desc="Total amount of time spent writing WAL buffers" + " to disk via XLogWrite request", + ) + wal_sync_time = MetricDef( + label="Sync time", + type="duration", + desc="Total amount of time spent syncing WAL files to" + " disk via issue_xlog_fsync request", + ) @classmethod def _get_metrics(cls, handler, **params): @@ -940,17 +1195,18 @@ def query(self): from_clause = query - cols = ["sub.srvid", - "extract(epoch FROM sub.ts) AS ts", - sum_per_sec('wal_records'), - sum_per_sec('wal_fpi'), - sum_per_sec('wal_bytes'), - sum_per_sec('wal_buffers_full'), - sum_per_sec('wal_write'), - sum_per_sec('wal_sync'), - sum_per_sec('wal_write_time'), - sum_per_sec('wal_sync_time'), - ] + cols = [ + "sub.srvid", + "extract(epoch FROM sub.ts) AS ts", + sum_per_sec("wal_records"), + sum_per_sec("wal_fpi"), + sum_per_sec("wal_bytes"), + sum_per_sec("wal_buffers_full"), + sum_per_sec("wal_write"), + sum_per_sec("wal_sync"), + sum_per_sec("wal_write_time"), + sum_per_sec("wal_sync_time"), + ] return """SELECT {cols} FROM ( @@ -959,7 +1215,7 @@ def query(self): WHERE sub.mesure_interval != '0 s' GROUP BY sub.srvid, sub.ts, sub.mesure_interval ORDER BY sub.ts""".format( - cols=', '.join(cols), + cols=", ".join(cols), from_clause=from_clause, ) @@ -968,27 +1224,44 @@ class GlobalWalReceiverMetricGroup(MetricGroupDef): """ Metric group used by pg_stat_wal_receiver graphs. """ + name = "wal_receiver" xaxis = "ts" data_url = r"/server/(\d+)/metrics/wal_receiver/" - write_delta = MetricDef(label="Write delta", type="size", - desc="Total amount of data received from the" - " primary but not written yet") - flush_delta = MetricDef(label="Flush delta", type="size", - desc="Total amount of data received and written" - " but not flushed yet") - last_msg_lag = MetricDef(label="Last message latency", type="duration", - desc="Time spent transmitting the last message" - " received from origin WAL sender") - report_delta = MetricDef(label="Report delta", type="size", - desc="Total amount of data not yet reported to" - " origin WAL sender") - report_lag = MetricDef(label="Report lag", type="duration", - desc="Time elapsed since since last reporting of" - " WAL location to origin WAL sender") - received_bytes = MetricDef(label="WAL receiver bandwidth", type="sizerate", - desc="Amount of data received from original WAL" - " sender") + write_delta = MetricDef( + label="Write delta", + type="size", + desc="Total amount of data received from the" + " primary but not written yet", + ) + flush_delta = MetricDef( + label="Flush delta", + type="size", + desc="Total amount of data received and written" + " but not flushed yet", + ) + last_msg_lag = MetricDef( + label="Last message latency", + type="duration", + desc="Time spent transmitting the last message" + " received from origin WAL sender", + ) + report_delta = MetricDef( + label="Report delta", + type="size", + desc="Total amount of data not yet reported to" " origin WAL sender", + ) + report_lag = MetricDef( + label="Report lag", + type="duration", + desc="Time elapsed since since last reporting of" + " WAL location to origin WAL sender", + ) + received_bytes = MetricDef( + label="WAL receiver bandwidth", + type="sizerate", + desc="Amount of data received from original WAL" " sender", + ) @classmethod def _get_metrics(cls, handler, **params): @@ -1006,15 +1279,16 @@ def query(self): from_clause = query - cols = ["sub.srvid", - "extract(epoch FROM sub.ts) AS ts", - "last_received_lsn - written_lsn AS write_delta", - "written_lsn - flushed_lsn AS flush_delta", - "extract(epoch FROM ((last_msg_receipt_time - last_msg_send_time) * 1000)) AS last_msg_lag", - "last_received_lsn - latest_end_lsn AS report_delta", - "greatest(extract(epoch FROM ((ts - latest_end_time) * 1000)), 0) AS report_lag", - "received_bytes", - ] + cols = [ + "sub.srvid", + "extract(epoch FROM sub.ts) AS ts", + "last_received_lsn - written_lsn AS write_delta", + "written_lsn - flushed_lsn AS flush_delta", + "extract(epoch FROM ((last_msg_receipt_time - last_msg_send_time) * 1000)) AS last_msg_lag", + "last_received_lsn - latest_end_lsn AS report_delta", + "greatest(extract(epoch FROM ((ts - latest_end_time) * 1000)), 0) AS report_lag", + "received_bytes", + ] return """SELECT {cols} FROM ( @@ -1023,59 +1297,81 @@ def query(self): WHERE sub.mesure_interval != '0 s' --GROUP BY sub.srvid, sub.ts, sub.mesure_interval ORDER BY sub.ts""".format( - cols=', '.join(cols), + cols=", ".join(cols), from_clause=from_clause, ) + class GlobalReplicationMetricGroup(MetricGroupDef): """ Metric group used by pg_stat_replication graphs. """ + name = "replication" xaxis = "ts" data_url = r"/server/(\d+)/metrics/replication/" - sent_lsn = MetricDef(label="Sent delta", - type="size", - desc="Maximum amount of data not sent to the replica") - write_lsn = MetricDef(label="Write delta", - type="size", - desc="Maximum amount of data sent to the replica but" - " not written locally") - flush_lsn = MetricDef(label="Flush delta", - type="size", - desc="Maximum amount of data written on the replica" - " but not flushed locally") - replay_lsn = MetricDef(label="Replay delta", - type="size", - desc="Maximum amount of data flushed on the replica" - " but not replayed locally") - write_lag = MetricDef(label="Write lag", - type="duration", - desc="Maximum write lag in s") - flush_lag = MetricDef(label="Flush lag", - type="duration", - desc="Maximum flush lag in s") - replay_lag = MetricDef(label="Replay lag", - type="duration", - desc="Maximum replay lag in s") - nb_async = MetricDef(label="# of async replication connections", - type="number", - desc="Number of asynchronous replication connections") - nb_sync = MetricDef(label="# of sync replication connections", - type="number", - desc="Number of synchronous replication connections") - nb_physical_act = MetricDef(label="# physical slots (active)", - type="number", - desc="Number of active physical replication slots") - nb_physical_not_act = MetricDef(label="# physical slots (disconnected)", - type="number", - desc="Number of disconnected physical replication slots") - nb_logical_act = MetricDef(label="# logical slots (active)", - type="number", - desc="Number of active logical replication slots") - nb_logical_not_act = MetricDef(label="# logical slots (disconnected)", - type="number", - desc="Number of inactive logical replication slots") + sent_lsn = MetricDef( + label="Sent delta", + type="size", + desc="Maximum amount of data not sent to the replica", + ) + write_lsn = MetricDef( + label="Write delta", + type="size", + desc="Maximum amount of data sent to the replica but" + " not written locally", + ) + flush_lsn = MetricDef( + label="Flush delta", + type="size", + desc="Maximum amount of data written on the replica" + " but not flushed locally", + ) + replay_lsn = MetricDef( + label="Replay delta", + type="size", + desc="Maximum amount of data flushed on the replica" + " but not replayed locally", + ) + write_lag = MetricDef( + label="Write lag", type="duration", desc="Maximum write lag in s" + ) + flush_lag = MetricDef( + label="Flush lag", type="duration", desc="Maximum flush lag in s" + ) + replay_lag = MetricDef( + label="Replay lag", type="duration", desc="Maximum replay lag in s" + ) + nb_async = MetricDef( + label="# of async replication connections", + type="number", + desc="Number of asynchronous replication connections", + ) + nb_sync = MetricDef( + label="# of sync replication connections", + type="number", + desc="Number of synchronous replication connections", + ) + nb_physical_act = MetricDef( + label="# physical slots (active)", + type="number", + desc="Number of active physical replication slots", + ) + nb_physical_not_act = MetricDef( + label="# physical slots (disconnected)", + type="number", + desc="Number of disconnected physical replication slots", + ) + nb_logical_act = MetricDef( + label="# logical slots (active)", + type="number", + desc="Number of active logical replication slots", + ) + nb_logical_not_act = MetricDef( + label="# logical slots (disconnected)", + type="number", + desc="Number of inactive logical replication slots", + ) @property def query(self): @@ -1084,26 +1380,26 @@ def query(self): from_clause = query cols = [ - "extract(epoch FROM sub.ts) AS ts", - # the datasource retrieves the current lsn first and then the - # rest of the counters, so it's entirely possible to get a - # slightly negative number here. If that happens it just means - # that there was some activity happening and everything is - # working as expected. - "greatest(rep_current_lsn - sent_lsn, 0) AS sent_lsn", - "sent_lsn - write_lsn AS write_lsn", - "write_lsn - flush_lsn AS flush_lsn", - "flush_lsn - replay_lsn AS replay_lsn", - "coalesce(extract(epoch from write_lag), 0) AS write_lag", - "coalesce(extract(epoch from flush_lag - write_lag), 0) AS flush_lag", - "coalesce(extract(epoch from replay_lag - flush_lag), 0) AS replay_lag", - "nb_async", - "nb_repl - nb_async AS nb_sync", - "nb_physical_act", - "nb_physical_not_act", - "nb_logical_act", - "nb_logical_not_act", - ] + "extract(epoch FROM sub.ts) AS ts", + # the datasource retrieves the current lsn first and then the + # rest of the counters, so it's entirely possible to get a + # slightly negative number here. If that happens it just means + # that there was some activity happening and everything is + # working as expected. + "greatest(rep_current_lsn - sent_lsn, 0) AS sent_lsn", + "sent_lsn - write_lsn AS write_lsn", + "write_lsn - flush_lsn AS flush_lsn", + "flush_lsn - replay_lsn AS replay_lsn", + "coalesce(extract(epoch from write_lag), 0) AS write_lag", + "coalesce(extract(epoch from flush_lag - write_lag), 0) AS flush_lag", + "coalesce(extract(epoch from replay_lag - flush_lag), 0) AS replay_lag", + "nb_async", + "nb_repl - nb_async AS nb_sync", + "nb_physical_act", + "nb_physical_not_act", + "nb_logical_act", + "nb_logical_not_act", + ] return """SELECT {cols} FROM ( @@ -1111,7 +1407,7 @@ def query(self): ) AS sub WHERE sub.mesure_interval != '0 s' ORDER BY sub.ts""".format( - cols=', '.join(cols), + cols=", ".join(cols), from_clause=from_clause, ) @@ -1120,46 +1416,64 @@ class GlobalDbActivityMetricGroup(MetricGroupDef): """ Metric group used by "Database Activity" graphs. """ + name = "all_db_activity" xaxis = "ts" data_url = r"/server/(\d+)/metrics/all_db_activity/" - numbackends = MetricDef(label="# of connections", - desc="Total number of connections") - xact_commit = MetricDef(label="# of commits", - desc="Total number of commits per second") - xact_rollback = MetricDef(label="# of rollbacks", - desc="Total number of rollbacks per second") - conflicts = MetricDef(label="# of conflicts", - desc="Total number of conflicts") - deadlocks = MetricDef(label="# of deadlocks", - desc="Total number of deadlocks") - checksum_failures = MetricDef(label="# of checkum_failures", - desc="Total number of checkum_failures") - session_time = MetricDef(label="Session time", - type="duration", - desc="Total time spent by database sessions per " - "second") - active_time = MetricDef(label="Active time", - type="duration", - desc="Total time spent executing SQL statements " - "per second") - idle_in_transaction_time = MetricDef(label="idle in xact time", - type="duration", - desc="Total time spent idling while " - "in a transaction per second") - sessions = MetricDef(label="# sessions", - desc="Total number of sessions established per second") - sessions_abandoned = MetricDef(label="# sessions abandoned", - desc="Number of database sessions that " - "were terminated because connection to " - "the client was lost per second") - sessions_fatal = MetricDef(label="# sessions fatal", - desc="Number of database sessions that " - "were terminated by fatal errors") - sessions_killed = MetricDef(label="# sessions killed per second", - desc="Number of database sessions that " - "were terminated by operator intervention " - "per second") + numbackends = MetricDef( + label="# of connections", desc="Total number of connections" + ) + xact_commit = MetricDef( + label="# of commits", desc="Total number of commits per second" + ) + xact_rollback = MetricDef( + label="# of rollbacks", desc="Total number of rollbacks per second" + ) + conflicts = MetricDef( + label="# of conflicts", desc="Total number of conflicts" + ) + deadlocks = MetricDef( + label="# of deadlocks", desc="Total number of deadlocks" + ) + checksum_failures = MetricDef( + label="# of checkum_failures", desc="Total number of checkum_failures" + ) + session_time = MetricDef( + label="Session time", + type="duration", + desc="Total time spent by database sessions per " "second", + ) + active_time = MetricDef( + label="Active time", + type="duration", + desc="Total time spent executing SQL statements " "per second", + ) + idle_in_transaction_time = MetricDef( + label="idle in xact time", + type="duration", + desc="Total time spent idling while " "in a transaction per second", + ) + sessions = MetricDef( + label="# sessions", + desc="Total number of sessions established per second", + ) + sessions_abandoned = MetricDef( + label="# sessions abandoned", + desc="Number of database sessions that " + "were terminated because connection to " + "the client was lost per second", + ) + sessions_fatal = MetricDef( + label="# sessions fatal", + desc="Number of database sessions that " + "were terminated by fatal errors", + ) + sessions_killed = MetricDef( + label="# sessions killed per second", + desc="Number of database sessions that " + "were terminated by operator intervention " + "per second", + ) @classmethod def _get_metrics(cls, handler, **params): @@ -1168,9 +1482,14 @@ def _get_metrics(cls, handler, **params): pg_version_num = handler.get_pg_version_num(handler.path_args[0]) # if we can't connect to the remote server, assume pg14 or above if pg_version_num is not None and pg_version_num < 140000: - for key in ("session_time", "active_time", - "idle_in_transaction_time", "sessions", - "sessions_abandoned", "sessions_fatal", "sessions_killed" + for key in ( + "session_time", + "active_time", + "idle_in_transaction_time", + "sessions", + "sessions_abandoned", + "sessions_fatal", + "sessions_killed", ): base.pop(key) return base @@ -1179,28 +1498,29 @@ def _get_metrics(cls, handler, **params): def query(self): query = powa_get_database_sample() - cols = ["sub.srvid", - "extract(epoch FROM sub.ts) AS ts", - "numbackends", - wps("xact_commit", do_sum=False), - wps("xact_rollback", do_sum=False), - "conflicts", - "deadlocks", - "checksum_failures", - wps("session_time", do_sum=False), - wps("active_time", do_sum=False), - wps("idle_in_transaction_time", do_sum=False), - wps("sessions", do_sum=False), - wps("sessions_abandoned", do_sum=False), - wps("sessions_fatal", do_sum=False), - wps("sessions_killed", do_sum=False), + cols = [ + "sub.srvid", + "extract(epoch FROM sub.ts) AS ts", + "numbackends", + wps("xact_commit", do_sum=False), + wps("xact_rollback", do_sum=False), + "conflicts", + "deadlocks", + "checksum_failures", + wps("session_time", do_sum=False), + wps("active_time", do_sum=False), + wps("idle_in_transaction_time", do_sum=False), + wps("sessions", do_sum=False), + wps("sessions_abandoned", do_sum=False), + wps("sessions_fatal", do_sum=False), + wps("sessions_killed", do_sum=False), ] return """SELECT {cols} FROM ({query}) sub WHERE sub.mesure_interval != '0 s' ORDER BY sub.ts""".format( - cols=', '.join(cols), + cols=", ".join(cols), query=query, ) @@ -1209,28 +1529,41 @@ class GlobalDbActivityConflMetricGroup(MetricGroupDef): """ Metric group used by the "Recovery conflicts" graph. """ + name = "all_db_activity_conf" xaxis = "ts" data_url = r"/server/(\d+)/metrics/all_db_activity_conf/" - confl_tablespace = MetricDef(label="Tablespaces", - desc="Total number of queries that have been" - " been canceled due to drop tablespaces") - confl_lock = MetricDef(label="Locks", - desc="Total number of queries that have been " - "canceled due to lock timeouts") - confl_snapshot = MetricDef(label="Snapshots", - desc="Total number of queries that have been" - " canceled due to old snapshots") - confl_bufferpin = MetricDef(label="Pinned buffers", - desc="Total number of queries that have been" - " canceled due to pinned buffers") - confl_deadlock = MetricDef(label="Deadlocks", - desc="Total number of queries that have been" - " canceled due to deadlocks") - confl_active_logicalslot = MetricDef(label="Logical slots", - desc="Total number of uses of logical slots that" - " have canceled due to old snapshots or too" - " low wal_level on the primary") + confl_tablespace = MetricDef( + label="Tablespaces", + desc="Total number of queries that have been" + " been canceled due to drop tablespaces", + ) + confl_lock = MetricDef( + label="Locks", + desc="Total number of queries that have been " + "canceled due to lock timeouts", + ) + confl_snapshot = MetricDef( + label="Snapshots", + desc="Total number of queries that have been" + " canceled due to old snapshots", + ) + confl_bufferpin = MetricDef( + label="Pinned buffers", + desc="Total number of queries that have been" + " canceled due to pinned buffers", + ) + confl_deadlock = MetricDef( + label="Deadlocks", + desc="Total number of queries that have been" + " canceled due to deadlocks", + ) + confl_active_logicalslot = MetricDef( + label="Logical slots", + desc="Total number of uses of logical slots that" + " have canceled due to old snapshots or too" + " low wal_level on the primary", + ) @classmethod def _get_metrics(cls, handler, **params): @@ -1239,28 +1572,29 @@ def _get_metrics(cls, handler, **params): pg_version_num = handler.get_pg_version_num(handler.path_args[0]) # if we can't connect to the remote server, assume pg15 or below if pg_version_num is None or pg_version_num < 160000: - base.pop('confl_active_logicalslot') + base.pop("confl_active_logicalslot") return base @property def query(self): query = powa_get_database_conflicts_sample() - cols = ["sub.srvid", - "extract(epoch FROM sub.ts) AS ts", - "confl_tablespace", - "confl_lock", - "confl_snapshot", - "confl_bufferpin", - "confl_deadlock", - "confl_active_logicalslot", + cols = [ + "sub.srvid", + "extract(epoch FROM sub.ts) AS ts", + "confl_tablespace", + "confl_lock", + "confl_snapshot", + "confl_bufferpin", + "confl_deadlock", + "confl_active_logicalslot", ] return """SELECT {cols} FROM ({query}) sub WHERE sub.mesure_interval != '0 s' ORDER BY sub.ts""".format( - cols=', '.join(cols), + cols=", ".join(cols), query=query, ) @@ -1269,60 +1603,94 @@ class GlobalAllRelMetricGroup(MetricGroupDef): """ Metric group used by "Database objects" graphs. """ + name = "all_relations" xaxis = "ts" data_url = r"/server/(\d+)/metrics/all_relations/" - idx_size = MetricDef(label="Indexes size", type="size", - desc="Size of all indexes") - tbl_size = MetricDef(label="Tables size", type="size", - desc="Size of all tables") - idx_ratio = MetricDef(label="Index scans ratio", type="percent", - desc="Ratio of index scan / seq scan") - idx_scan = MetricDef(label="Index scans", type="number", - desc="Number of index scan per second") - seq_scan = MetricDef(label="Sequential scans", type="number", - desc="Number of sequential scan per second") - n_tup_ins = MetricDef(label="Tuples inserted", type="number", - desc="Number of tuples inserted per second") - n_tup_upd = MetricDef(label="Tuples updated", type="number", - desc="Number of tuples updated per second") - n_tup_hot_upd = MetricDef(label="Tuples HOT updated", type="number", - desc="Number of tuples HOT updated per second") - n_tup_del = MetricDef(label="Tuples deleted", type="number", - desc="Number of tuples deleted per second") - vacuum_count = MetricDef(label="# Vacuum", type="number", - desc="Number of vacuum per second") - autovacuum_count = MetricDef(label="# Autovacuum", type="number", - desc="Number of autovacuum per second") - analyze_count = MetricDef(label="# Analyze", type="number", - desc="Number of analyze per second") - autoanalyze_count = MetricDef(label="# Autoanalyze", type="number", - desc="Number of autoanalyze per second") + idx_size = MetricDef( + label="Indexes size", type="size", desc="Size of all indexes" + ) + tbl_size = MetricDef( + label="Tables size", type="size", desc="Size of all tables" + ) + idx_ratio = MetricDef( + label="Index scans ratio", + type="percent", + desc="Ratio of index scan / seq scan", + ) + idx_scan = MetricDef( + label="Index scans", + type="number", + desc="Number of index scan per second", + ) + seq_scan = MetricDef( + label="Sequential scans", + type="number", + desc="Number of sequential scan per second", + ) + n_tup_ins = MetricDef( + label="Tuples inserted", + type="number", + desc="Number of tuples inserted per second", + ) + n_tup_upd = MetricDef( + label="Tuples updated", + type="number", + desc="Number of tuples updated per second", + ) + n_tup_hot_upd = MetricDef( + label="Tuples HOT updated", + type="number", + desc="Number of tuples HOT updated per second", + ) + n_tup_del = MetricDef( + label="Tuples deleted", + type="number", + desc="Number of tuples deleted per second", + ) + vacuum_count = MetricDef( + label="# Vacuum", type="number", desc="Number of vacuum per second" + ) + autovacuum_count = MetricDef( + label="# Autovacuum", + type="number", + desc="Number of autovacuum per second", + ) + analyze_count = MetricDef( + label="# Analyze", type="number", desc="Number of analyze per second" + ) + autoanalyze_count = MetricDef( + label="# Autoanalyze", + type="number", + desc="Number of autoanalyze per second", + ) @property def query(self): query1 = powa_get_all_tbl_sample("db") query2 = powa_get_all_idx_sample("db") - cols = ["sub.srvid", - "extract(epoch FROM sub.ts) AS ts", - "sum(tbl_size) AS tbl_size", - "sum(idx_size) AS idx_size", - "CASE WHEN sum(sub.idx_scan + sub.seq_scan) = 0" - " THEN 0" - " ELSE sum(sub.idx_scan) * 100" - " / sum(sub.idx_scan + sub.seq_scan)" - " END AS idx_ratio", - sum_per_sec("idx_scan", prefix="sub"), - sum_per_sec("seq_scan", prefix="sub"), - sum_per_sec("n_tup_ins", prefix="sub"), - sum_per_sec("n_tup_upd", prefix="sub"), - sum_per_sec("n_tup_hot_upd", prefix="sub"), - sum_per_sec("n_tup_del", prefix="sub"), - sum_per_sec("vacuum_count", prefix="sub"), - sum_per_sec("autovacuum_count", prefix="sub"), - sum_per_sec("analyze_count", prefix="sub"), - sum_per_sec("autoanalyze_count", prefix="sub")] + cols = [ + "sub.srvid", + "extract(epoch FROM sub.ts) AS ts", + "sum(tbl_size) AS tbl_size", + "sum(idx_size) AS idx_size", + "CASE WHEN sum(sub.idx_scan + sub.seq_scan) = 0" + " THEN 0" + " ELSE sum(sub.idx_scan) * 100" + " / sum(sub.idx_scan + sub.seq_scan)" + " END AS idx_ratio", + sum_per_sec("idx_scan", prefix="sub"), + sum_per_sec("seq_scan", prefix="sub"), + sum_per_sec("n_tup_ins", prefix="sub"), + sum_per_sec("n_tup_upd", prefix="sub"), + sum_per_sec("n_tup_hot_upd", prefix="sub"), + sum_per_sec("n_tup_del", prefix="sub"), + sum_per_sec("vacuum_count", prefix="sub"), + sum_per_sec("autovacuum_count", prefix="sub"), + sum_per_sec("analyze_count", prefix="sub"), + sum_per_sec("autoanalyze_count", prefix="sub"), + ] return """SELECT {cols} FROM ( @@ -1333,9 +1701,7 @@ def query(self): WHERE sub.mesure_interval != '0 s' GROUP BY sub.srvid, sub.ts, sub.mesure_interval ORDER BY sub.ts""".format( - cols=', '.join(cols), - query1=query1, - query2=query2 + cols=", ".join(cols), query1=query1, query2=query2 ) @@ -1343,15 +1709,25 @@ class GlobalUserFctMetricGroup(MetricGroupDef): """ Metric group used by "pg_stat_user_function" graph. """ + name = "user_functions" xaxis = "ts" data_url = r"/server/(\d+)/metrics/user_functions/" - calls = MetricDef(label="# of calls", type="number", - desc="Number of function calls per second") - total_load = MetricDef(label="Total time per sec", type="number", - desc="Total execution time duration") - self_load = MetricDef(label="Self time per sec", type="number", - desc="Self execution time duration") + calls = MetricDef( + label="# of calls", + type="number", + desc="Number of function calls per second", + ) + total_load = MetricDef( + label="Total time per sec", + type="number", + desc="Total execution time duration", + ) + self_load = MetricDef( + label="Self time per sec", + type="number", + desc="Self execution time duration", + ) @property def query(self): @@ -1359,11 +1735,13 @@ def query(self): from_clause = query - cols = ["sub.srvid", - "extract(epoch FROM sub.ts) AS ts", - sum_per_sec("calls"), - sum_per_sec("total_time", alias="total_load"), - sum_per_sec("self_time", alias="self_load")] + cols = [ + "sub.srvid", + "extract(epoch FROM sub.ts) AS ts", + sum_per_sec("calls"), + sum_per_sec("total_time", alias="total_load"), + sum_per_sec("self_time", alias="self_load"), + ] return """SELECT {cols} FROM ( @@ -1372,8 +1750,7 @@ def query(self): WHERE sub.mesure_interval != '0 s' GROUP BY sub.srvid, sub.ts, sub.mesure_interval ORDER BY sub.ts""".format( - cols=', '.join(cols), - from_clause=from_clause + cols=", ".join(cols), from_clause=from_clause ) @@ -1381,20 +1758,32 @@ class ServerOverview(DashboardPage): """ ServerOverview dashboard page. """ + base_url = r"/server/(\d+)/overview/" - datasources = [GlobalDatabasesMetricGroup, ByDatabaseMetricGroup, - ByDatabaseWaitSamplingMetricGroup, GlobalWaitsMetricGroup, - GlobalBgwriterMetricGroup, GlobalCheckpointerMetricGroup, - GlobalAllRelMetricGroup, - GlobalUserFctMetricGroup, ByDatabaseUserFuncMetricGroup, - ConfigChangesGlobal, GlobalPGSAMetricGroup, - GlobalArchiverMetricGroup, GlobalReplicationMetricGroup, - GlobalDbActivityMetricGroup, - GlobalDbActivityConflMetricGroup, GlobalIoMetricGroup, - ByAllIoMetricGroup, - GlobalSlruMetricGroup, ByAllSlruMetricGroup, - GlobalWalMetricGroup, GlobalWalReceiverMetricGroup, - GlobalSubMetricGroup] + datasources = [ + GlobalDatabasesMetricGroup, + ByDatabaseMetricGroup, + ByDatabaseWaitSamplingMetricGroup, + GlobalWaitsMetricGroup, + GlobalBgwriterMetricGroup, + GlobalCheckpointerMetricGroup, + GlobalAllRelMetricGroup, + GlobalUserFctMetricGroup, + ByDatabaseUserFuncMetricGroup, + ConfigChangesGlobal, + GlobalPGSAMetricGroup, + GlobalArchiverMetricGroup, + GlobalReplicationMetricGroup, + GlobalDbActivityMetricGroup, + GlobalDbActivityConflMetricGroup, + GlobalIoMetricGroup, + ByAllIoMetricGroup, + GlobalSlruMetricGroup, + ByAllSlruMetricGroup, + GlobalWalMetricGroup, + GlobalWalReceiverMetricGroup, + GlobalSubMetricGroup, + ] params = ["server"] parent = Overview title = "All databases" @@ -1403,53 +1792,81 @@ class ServerOverview(DashboardPage): def dashboard(self): # This COULD be initialized in the constructor, but tornado < 3 doesn't # call it - if getattr(self, '_dashboard', None) is not None: + if getattr(self, "_dashboard", None) is not None: return self._dashboard pg_version_num = self.get_pg_version_num(self.path_args[0]) - pgss18 = self.has_extension_version(self.path_args[0], - 'pg_stat_statements', '1.8') - pgss110 = self.has_extension_version(self.path_args[0], - 'pg_stat_statements', '1.10') - pgss111 = self.has_extension_version(self.path_args[0], - 'pg_stat_statements', '1.11') - - all_db_metrics = [GlobalDatabasesMetricGroup.avg_runtime, - GlobalDatabasesMetricGroup.load, - GlobalDatabasesMetricGroup.calls] + pgss18 = self.has_extension_version( + self.path_args[0], "pg_stat_statements", "1.8" + ) + pgss110 = self.has_extension_version( + self.path_args[0], "pg_stat_statements", "1.10" + ) + pgss111 = self.has_extension_version( + self.path_args[0], "pg_stat_statements", "1.11" + ) + + all_db_metrics = [ + GlobalDatabasesMetricGroup.avg_runtime, + GlobalDatabasesMetricGroup.load, + GlobalDatabasesMetricGroup.calls, + ] if pgss18: all_db_metrics.extend([GlobalDatabasesMetricGroup.planload]) - block_graph = Graph("Block access in Bps", - metrics=[GlobalDatabasesMetricGroup. - total_blks_hit], - color_scheme=None) + block_graph = Graph( + "Block access in Bps", + metrics=[GlobalDatabasesMetricGroup.total_blks_hit], + color_scheme=None, + ) - all_db_graphs = [[Graph("Query runtime per second (all databases)", - metrics=all_db_metrics), - block_graph]] + all_db_graphs = [ + [ + Graph( + "Query runtime per second (all databases)", + metrics=all_db_metrics, + ), + block_graph, + ] + ] - if ("nb_parallel_query" in GlobalPGSAMetricGroup._get_metrics(self)): + if "nb_parallel_query" in GlobalPGSAMetricGroup._get_metrics(self): parallel_metrics = ["nb_parallel_query", "nb_parallel_worker"] else: parallel_metrics = [] - pgsa_metrics = GlobalPGSAMetricGroup.split(self, - [["backend_xid_age", - "backend_xmin_age", - "oldest_backend", - "oldest_xact", - "oldest_query"], - parallel_metrics]) - all_db_graphs.append([Graph("Global activity (all databases)", - metrics=pgsa_metrics[0], - renderer="bar", - stack=True)]) - if (len(pgsa_metrics[2]) > 0): - all_db_graphs[1].append(Graph("Parallel query (all databases)", - metrics=pgsa_metrics[2])) - all_db_graphs[1].append(Graph("Backend age (all databases)", - metrics=pgsa_metrics[1])) + pgsa_metrics = GlobalPGSAMetricGroup.split( + self, + [ + [ + "backend_xid_age", + "backend_xmin_age", + "oldest_backend", + "oldest_xact", + "oldest_query", + ], + parallel_metrics, + ], + ) + all_db_graphs.append( + [ + Graph( + "Global activity (all databases)", + metrics=pgsa_metrics[0], + renderer="bar", + stack=True, + ) + ] + ) + if len(pgsa_metrics[2]) > 0: + all_db_graphs[1].append( + Graph( + "Parallel query (all databases)", metrics=pgsa_metrics[2] + ) + ) + all_db_graphs[1].append( + Graph("Backend age (all databases)", metrics=pgsa_metrics[1]) + ) graphs_dash = [Dashboard("General Overview", all_db_graphs)] graphs = [TabContainer("All databases", graphs_dash)] @@ -1457,367 +1874,577 @@ def dashboard(self): # Add WALs graphs # if we can't connect to the remote server, assume pg14 or above - if (pg_version_num is None or pg_version_num >= 140000): - wal_metrics = GlobalWalMetricGroup.split(self, - [["wal_write_time", - "wal_sync_time"]]) - wals_graphs = [[Graph("WAL activity", - url="https://www.postgresql.org/docs/current/monitoring-stats.html#MONITORING-PG-STAT-WAL-VIEW", - metrics=wal_metrics[0]), - Graph("WAL timing", - url="https://www.postgresql.org/docs/current/monitoring-stats.html#MONITORING-PG-STAT-WAL-VIEW", - metrics=wal_metrics[1]), - ]] + if pg_version_num is None or pg_version_num >= 140000: + wal_metrics = GlobalWalMetricGroup.split( + self, [["wal_write_time", "wal_sync_time"]] + ) + wals_graphs = [ + [ + Graph( + "WAL activity", + url="https://www.postgresql.org/docs/current/monitoring-stats.html#MONITORING-PG-STAT-WAL-VIEW", + metrics=wal_metrics[0], + ), + Graph( + "WAL timing", + url="https://www.postgresql.org/docs/current/monitoring-stats.html#MONITORING-PG-STAT-WAL-VIEW", + metrics=wal_metrics[1], + ), + ] + ] graphs_dash.append(Dashboard("WALs", wals_graphs)) elif pgss18: - wals_graphs = [[Graph("WAL activity", - metrics=[GlobalDatabasesMetricGroup.wal_records, - GlobalDatabasesMetricGroup.wal_fpi, - GlobalDatabasesMetricGroup.wal_bytes]), - ]] + wals_graphs = [ + [ + Graph( + "WAL activity", + metrics=[ + GlobalDatabasesMetricGroup.wal_records, + GlobalDatabasesMetricGroup.wal_fpi, + GlobalDatabasesMetricGroup.wal_bytes, + ], + ), + ] + ] graphs_dash.append(Dashboard("WALs", wals_graphs)) # Add JIT graphs if pgss110: - jit_tim = [GlobalDatabasesMetricGroup.jit_inlining_time, - GlobalDatabasesMetricGroup.jit_optimization_time, - GlobalDatabasesMetricGroup.jit_emission_time, - ] + jit_tim = [ + GlobalDatabasesMetricGroup.jit_inlining_time, + GlobalDatabasesMetricGroup.jit_optimization_time, + GlobalDatabasesMetricGroup.jit_emission_time, + ] if pgss111: - jit_tim.extend([GlobalDatabasesMetricGroup.jit_deform_time, - GlobalDatabasesMetricGroup.jit_expr_time]) + jit_tim.extend( + [ + GlobalDatabasesMetricGroup.jit_deform_time, + GlobalDatabasesMetricGroup.jit_expr_time, + ] + ) else: jit_tim.append(GlobalDatabasesMetricGroup.jit_generation_time) - jit_cnt = [GlobalDatabasesMetricGroup.jit_functions, - GlobalDatabasesMetricGroup.jit_inlining_count, - GlobalDatabasesMetricGroup.jit_optimization_count, - GlobalDatabasesMetricGroup.jit_emission_count, - ] + jit_cnt = [ + GlobalDatabasesMetricGroup.jit_functions, + GlobalDatabasesMetricGroup.jit_inlining_count, + GlobalDatabasesMetricGroup.jit_optimization_count, + GlobalDatabasesMetricGroup.jit_emission_count, + ] if pgss111: jit_cnt.append(GlobalDatabasesMetricGroup.jit_deform_count) - jit_graphs = [[Graph("JIT timing", metrics=jit_tim, - stack=True)], - [Graph("JIT scheduling", metrics=jit_cnt)]] + jit_graphs = [ + [Graph("JIT timing", metrics=jit_tim, stack=True)], + [Graph("JIT scheduling", metrics=jit_cnt)], + ] graphs_dash.append(Dashboard("JIT", jit_graphs)) # Add pg_stat_bgwriter / pg_stat_checkpointer graphs - bgw_graphs = [[Graph("Checkpointer scheduling", - metrics=[GlobalCheckpointerMetricGroup.num_timed, - GlobalCheckpointerMetricGroup.num_requested]), - Graph("Checkpointer activity", - metrics=[GlobalCheckpointerMetricGroup.write_time, - GlobalCheckpointerMetricGroup.sync_time, - GlobalCheckpointerMetricGroup.buffers_written])], - [Graph("Background writer", - metrics=[GlobalBgwriterMetricGroup.buffers_clean, - GlobalBgwriterMetricGroup.maxwritten_clean, - GlobalBgwriterMetricGroup.buffers_alloc]), - Graph("Backends", - metrics=[GlobalBgwriterMetricGroup.buffers_backend, - GlobalBgwriterMetricGroup.buffers_backend_fsync - ]) - ]] - graphs_dash.append(Dashboard("Background Writer / Checkpointer", bgw_graphs)) + bgw_graphs = [ + [ + Graph( + "Checkpointer scheduling", + metrics=[ + GlobalCheckpointerMetricGroup.num_timed, + GlobalCheckpointerMetricGroup.num_requested, + ], + ), + Graph( + "Checkpointer activity", + metrics=[ + GlobalCheckpointerMetricGroup.write_time, + GlobalCheckpointerMetricGroup.sync_time, + GlobalCheckpointerMetricGroup.buffers_written, + ], + ), + ], + [ + Graph( + "Background writer", + metrics=[ + GlobalBgwriterMetricGroup.buffers_clean, + GlobalBgwriterMetricGroup.maxwritten_clean, + GlobalBgwriterMetricGroup.buffers_alloc, + ], + ), + Graph( + "Backends", + metrics=[ + GlobalBgwriterMetricGroup.buffers_backend, + GlobalBgwriterMetricGroup.buffers_backend_fsync, + ], + ), + ], + ] + graphs_dash.append( + Dashboard("Background Writer / Checkpointer", bgw_graphs) + ) # Add archiver / replication graphs - sub_metrics = GlobalSubMetricGroup.split(self, - [["last_msg_lag", "report_lag"]]) + sub_metrics = GlobalSubMetricGroup.split( + self, [["last_msg_lag", "report_lag"]] + ) sub_graphs = [] - sub_graphs.append(Graph("Subscriptions message & report", - url="https://www.postgresql.org/docs/current/monitoring-stats.html#MONITORING-PG-STAT-SUBSCRIPTION", - metrics=sub_metrics[1])) + sub_graphs.append( + Graph( + "Subscriptions message & report", + url="https://www.postgresql.org/docs/current/monitoring-stats.html#MONITORING-PG-STAT-SUBSCRIPTION", + metrics=sub_metrics[1], + ) + ) if len(sub_metrics[0]) > 0: - sub_graphs.append(Graph("Subscriptions errors", - url="https://www.postgresql.org/docs/current/monitoring-stats.html#MONITORING-PG-STAT-SUBSCRIPTION-STATS", - metrics=sub_metrics[0])) - - arch_graphs = [[Graph("Archiver", - metrics=GlobalArchiverMetricGroup.all(self)), - Graph("Replication connections", - metrics=[GlobalReplicationMetricGroup.nb_async, - GlobalReplicationMetricGroup.nb_sync], - renderer="bar", - stack=True), - Graph("Replication slots", - metrics=[GlobalReplicationMetricGroup.nb_physical_act, - GlobalReplicationMetricGroup.nb_physical_not_act, - GlobalReplicationMetricGroup.nb_logical_act, - GlobalReplicationMetricGroup.nb_logical_not_act], - renderer="bar", - stack=True), - ], - [Graph("Replication delta in B", - metrics=[GlobalReplicationMetricGroup.sent_lsn, - GlobalReplicationMetricGroup.write_lsn, - GlobalReplicationMetricGroup.flush_lsn, - GlobalReplicationMetricGroup.replay_lsn, - ], - renderer="bar", - stack=True), - Graph("Replication lag in s", - metrics=[GlobalReplicationMetricGroup.write_lag, - GlobalReplicationMetricGroup.flush_lag, - GlobalReplicationMetricGroup.replay_lag, - ], - renderer="bar", - stack=True), - Graph("Recovery conflicts", - stack=True, - metrics=GlobalDbActivityConflMetricGroup.all(self) - # metrics=[GlobalDbActivityConflMetricGroup.confl_tablespace] - ) - ], - [Graph("WAL receiver bandwidth", - metrics=[GlobalWalReceiverMetricGroup.received_bytes]), - Graph("WAL receiver Delta", - url="https://www.postgresql.org/docs/current/monitoring-stats.html#MONITORING-PG-STAT-WAL-RECEIVER-VIEW", - metrics=[GlobalWalReceiverMetricGroup.write_delta, - GlobalWalReceiverMetricGroup.flush_delta], - renderer="bar", - stack=True), - ], - [ - Graph("WAL receiver message", - url="https://www.postgresql.org/docs/current/monitoring-stats.html#MONITORING-PG-STAT-WAL-RECEIVER-VIEW", - metrics=[GlobalWalReceiverMetricGroup.last_msg_lag], - renderer="bar", - stack=True), - Graph("WAL receiver report", - url="https://www.postgresql.org/docs/current/monitoring-stats.html#MONITORING-PG-STAT-WAL-RECEIVER-VIEW", - metrics=[GlobalWalReceiverMetricGroup.report_delta, - GlobalWalReceiverMetricGroup.report_lag], - renderer="bar", - stack=True), - ], - sub_graphs] + sub_graphs.append( + Graph( + "Subscriptions errors", + url="https://www.postgresql.org/docs/current/monitoring-stats.html#MONITORING-PG-STAT-SUBSCRIPTION-STATS", + metrics=sub_metrics[0], + ) + ) + + arch_graphs = [ + [ + Graph("Archiver", metrics=GlobalArchiverMetricGroup.all(self)), + Graph( + "Replication connections", + metrics=[ + GlobalReplicationMetricGroup.nb_async, + GlobalReplicationMetricGroup.nb_sync, + ], + renderer="bar", + stack=True, + ), + Graph( + "Replication slots", + metrics=[ + GlobalReplicationMetricGroup.nb_physical_act, + GlobalReplicationMetricGroup.nb_physical_not_act, + GlobalReplicationMetricGroup.nb_logical_act, + GlobalReplicationMetricGroup.nb_logical_not_act, + ], + renderer="bar", + stack=True, + ), + ], + [ + Graph( + "Replication delta in B", + metrics=[ + GlobalReplicationMetricGroup.sent_lsn, + GlobalReplicationMetricGroup.write_lsn, + GlobalReplicationMetricGroup.flush_lsn, + GlobalReplicationMetricGroup.replay_lsn, + ], + renderer="bar", + stack=True, + ), + Graph( + "Replication lag in s", + metrics=[ + GlobalReplicationMetricGroup.write_lag, + GlobalReplicationMetricGroup.flush_lag, + GlobalReplicationMetricGroup.replay_lag, + ], + renderer="bar", + stack=True, + ), + Graph( + "Recovery conflicts", + stack=True, + metrics=GlobalDbActivityConflMetricGroup.all(self), + # metrics=[GlobalDbActivityConflMetricGroup.confl_tablespace] + ), + ], + [ + Graph( + "WAL receiver bandwidth", + metrics=[GlobalWalReceiverMetricGroup.received_bytes], + ), + Graph( + "WAL receiver Delta", + url="https://www.postgresql.org/docs/current/monitoring-stats.html#MONITORING-PG-STAT-WAL-RECEIVER-VIEW", + metrics=[ + GlobalWalReceiverMetricGroup.write_delta, + GlobalWalReceiverMetricGroup.flush_delta, + ], + renderer="bar", + stack=True, + ), + ], + [ + Graph( + "WAL receiver message", + url="https://www.postgresql.org/docs/current/monitoring-stats.html#MONITORING-PG-STAT-WAL-RECEIVER-VIEW", + metrics=[GlobalWalReceiverMetricGroup.last_msg_lag], + renderer="bar", + stack=True, + ), + Graph( + "WAL receiver report", + url="https://www.postgresql.org/docs/current/monitoring-stats.html#MONITORING-PG-STAT-WAL-RECEIVER-VIEW", + metrics=[ + GlobalWalReceiverMetricGroup.report_delta, + GlobalWalReceiverMetricGroup.report_lag, + ], + renderer="bar", + stack=True, + ), + ], + sub_graphs, + ] graphs_dash.append(Dashboard("Archiver / Replication", arch_graphs)) # Add pg_stat_database graphs - global_db_graphs = [[Graph("Transactions per second", - metrics=[GlobalDbActivityMetricGroup.xact_commit, - GlobalDbActivityMetricGroup.xact_rollback], - renderer="bar", - stack=True), - Graph("Conflicts & deadlocks", - metrics=[GlobalDbActivityMetricGroup.conflicts, - GlobalDbActivityMetricGroup.deadlocks])]] - if ("sessions" in GlobalDbActivityMetricGroup._get_metrics(self)): - global_db_graphs.append([Graph("Cumulated time per second", - metrics=[GlobalDbActivityMetricGroup.session_time, - GlobalDbActivityMetricGroup.active_time, - GlobalDbActivityMetricGroup.idle_in_transaction_time]), - Graph("Sessions per second", - metrics=[GlobalDbActivityMetricGroup.sessions, - GlobalDbActivityMetricGroup.sessions_abandoned, - GlobalDbActivityMetricGroup.sessions_fatal, - GlobalDbActivityMetricGroup.sessions_killed]) - ]) + global_db_graphs = [ + [ + Graph( + "Transactions per second", + metrics=[ + GlobalDbActivityMetricGroup.xact_commit, + GlobalDbActivityMetricGroup.xact_rollback, + ], + renderer="bar", + stack=True, + ), + Graph( + "Conflicts & deadlocks", + metrics=[ + GlobalDbActivityMetricGroup.conflicts, + GlobalDbActivityMetricGroup.deadlocks, + ], + ), + ] + ] + if "sessions" in GlobalDbActivityMetricGroup._get_metrics(self): + global_db_graphs.append( + [ + Graph( + "Cumulated time per second", + metrics=[ + GlobalDbActivityMetricGroup.session_time, + GlobalDbActivityMetricGroup.active_time, + GlobalDbActivityMetricGroup.idle_in_transaction_time, + ], + ), + Graph( + "Sessions per second", + metrics=[ + GlobalDbActivityMetricGroup.sessions, + GlobalDbActivityMetricGroup.sessions_abandoned, + GlobalDbActivityMetricGroup.sessions_fatal, + GlobalDbActivityMetricGroup.sessions_killed, + ], + ), + ] + ) graphs_dash.append(Dashboard("Database activity", global_db_graphs)) # Add powa_stat_all_relations graphs - all_rel_graphs = [[Graph("Access pattern", - metrics=[GlobalAllRelMetricGroup.seq_scan, - GlobalAllRelMetricGroup.idx_scan, - GlobalAllRelMetricGroup.idx_ratio]), - Graph("DML activity", - metrics=[GlobalAllRelMetricGroup.n_tup_del, - GlobalAllRelMetricGroup.n_tup_hot_upd, - GlobalAllRelMetricGroup.n_tup_upd, - GlobalAllRelMetricGroup.n_tup_ins]) - ], - [Graph("Vacuum activity", - metrics=[GlobalAllRelMetricGroup.autoanalyze_count, - GlobalAllRelMetricGroup.analyze_count, - GlobalAllRelMetricGroup.autovacuum_count, - GlobalAllRelMetricGroup.vacuum_count]), - Graph("Object size", - metrics=[GlobalAllRelMetricGroup.tbl_size, - GlobalAllRelMetricGroup.idx_size], - renderer="bar", - stack=True, - color_scheme=['#73c03a','#65b9ac'])]] + all_rel_graphs = [ + [ + Graph( + "Access pattern", + metrics=[ + GlobalAllRelMetricGroup.seq_scan, + GlobalAllRelMetricGroup.idx_scan, + GlobalAllRelMetricGroup.idx_ratio, + ], + ), + Graph( + "DML activity", + metrics=[ + GlobalAllRelMetricGroup.n_tup_del, + GlobalAllRelMetricGroup.n_tup_hot_upd, + GlobalAllRelMetricGroup.n_tup_upd, + GlobalAllRelMetricGroup.n_tup_ins, + ], + ), + ], + [ + Graph( + "Vacuum activity", + metrics=[ + GlobalAllRelMetricGroup.autoanalyze_count, + GlobalAllRelMetricGroup.analyze_count, + GlobalAllRelMetricGroup.autovacuum_count, + GlobalAllRelMetricGroup.vacuum_count, + ], + ), + Graph( + "Object size", + metrics=[ + GlobalAllRelMetricGroup.tbl_size, + GlobalAllRelMetricGroup.idx_size, + ], + renderer="bar", + stack=True, + color_scheme=["#73c03a", "#65b9ac"], + ), + ], + ] graphs_dash.append(Dashboard("Database Objects", all_rel_graphs)) # Add powa_stat_user_functions graphs - user_fct_graph = [Graph("User functions activity", - metrics=GlobalUserFctMetricGroup.all(self))] - user_fct_grid = [Grid("User functions activity for all databases", - columns=[{ - "name": "datname", - "label": "Database", - "url_attr": "url" - }], - metrics=ByDatabaseUserFuncMetricGroup.all(self))] - graphs_dash.append(Dashboard("User functions", [user_fct_graph, - user_fct_grid])) + user_fct_graph = [ + Graph( + "User functions activity", + metrics=GlobalUserFctMetricGroup.all(self), + ) + ] + user_fct_grid = [ + Grid( + "User functions activity for all databases", + columns=[ + {"name": "datname", "label": "Database", "url_attr": "url"} + ], + metrics=ByDatabaseUserFuncMetricGroup.all(self), + ) + ] + graphs_dash.append( + Dashboard("User functions", [user_fct_graph, user_fct_grid]) + ) sys_graphs = [] if self.has_extension(self.path_args[0], "pg_stat_kcache"): - block_graph.metrics.insert(0, GlobalDatabasesMetricGroup. - total_sys_hit) - block_graph.metrics.insert(0, GlobalDatabasesMetricGroup. - total_disk_read) - block_graph.color_scheme = ['#cb513a', '#65b9ac', '#73c03a'] - - sys_graphs.append([Graph("System resources (events per sec)", - url=self.docs_stats_url + "pg_stat_kcache.html", - metrics=[GlobalDatabasesMetricGroup.majflts, - GlobalDatabasesMetricGroup.minflts, - # GlobalDatabasesMetricGroup.nswaps, - # GlobalDatabasesMetricGroup.msgsnds, - # GlobalDatabasesMetricGroup.msgrcvs, - # GlobalDatabasesMetricGroup.nsignals, - GlobalDatabasesMetricGroup.nvcsws, - GlobalDatabasesMetricGroup.nivcsws])]) + block_graph.metrics.insert( + 0, GlobalDatabasesMetricGroup.total_sys_hit + ) + block_graph.metrics.insert( + 0, GlobalDatabasesMetricGroup.total_disk_read + ) + block_graph.color_scheme = ["#cb513a", "#65b9ac", "#73c03a"] + + sys_graphs.append( + [ + Graph( + "System resources (events per sec)", + url=self.docs_stats_url + "pg_stat_kcache.html", + metrics=[ + GlobalDatabasesMetricGroup.majflts, + GlobalDatabasesMetricGroup.minflts, + # GlobalDatabasesMetricGroup.nswaps, + # GlobalDatabasesMetricGroup.msgsnds, + # GlobalDatabasesMetricGroup.msgrcvs, + # GlobalDatabasesMetricGroup.nsignals, + GlobalDatabasesMetricGroup.nvcsws, + GlobalDatabasesMetricGroup.nivcsws, + ], + ) + ] + ) else: - block_graph.metrics.insert(0, GlobalDatabasesMetricGroup. - total_blks_read) - block_graph.color_scheme = ['#cb513a', '#73c03a'] + block_graph.metrics.insert( + 0, GlobalDatabasesMetricGroup.total_blks_read + ) + block_graph.color_scheme = ["#cb513a", "#73c03a"] # Add pg_stat_io graphs - if ("reads" in GlobalIoMetricGroup._get_metrics(self)): - io_metrics = GlobalIoMetricGroup.split(self, - [["reads", "writes", "writebacks", "extends", "fsyncs"], - ["hits", "evictions", "reuses"]] + if "reads" in GlobalIoMetricGroup._get_metrics(self): + io_metrics = GlobalIoMetricGroup.split( + self, + [ + ["reads", "writes", "writebacks", "extends", "fsyncs"], + ["hits", "evictions", "reuses"], + ], + ) + sys_graphs.append( + [ + Graph( + "IO blocks", + metrics=io_metrics[1], + ), + Graph( + "IO timing", + metrics=io_metrics[0], + ), + Graph( + "IO misc", + metrics=io_metrics[2], + ), + ] + ) + + sys_graphs.append( + [ + Grid( + "IO summary", + columns=[ + { + "name": "backend_type", + "label": "Backend Type", + "url_attr": "backend_type_url", + }, + { + "name": "object", + "label": "Object Type", + "url_attr": "obj_url", + }, + { + "name": "context", + "label": "Context", + "url_attr": "context_url", + }, + ], + metrics=ByAllIoMetricGroup.all(), ) - sys_graphs.append([ - Graph("IO blocks", - metrics=io_metrics[1], - ), - Graph("IO timing", - metrics=io_metrics[0], - ), - Graph("IO misc", - metrics=io_metrics[2], - ), - ]) - - sys_graphs.append([ - Grid("IO summary", - columns=[{ - "name": "backend_type", - "label": "Backend Type", - "url_attr": "backend_type_url" - }, { - "name": "object", - "label": "Object Type", - "url_attr": "obj_url" - }, { - "name": "context", - "label": "Context", - "url_attr": "context_url" - }], - metrics=ByAllIoMetricGroup.all()) - ]) - - if (len(sys_graphs) != 0): + ] + ) + + if len(sys_graphs) != 0: graphs_dash.append(Dashboard("System resources", sys_graphs)) # if we can't connect to the remote server, assume pg13 or above - if (pg_version_num is None or pg_version_num >= 130000): - graphs_dash.append(Dashboard("SLRU", - [[Graph("All SLRUs (per second)", - metrics=GlobalSlruMetricGroup.all(self))], - [Grid("All SLRUs", - columns=[{ - "name": "name", - "label": "SLRU name", - "url_attr": "url" - }], - metrics=ByAllSlruMetricGroup.all(self))]])) - - if (self.has_extension(self.path_args[0], "pg_wait_sampling")): - metrics=None + if pg_version_num is None or pg_version_num >= 130000: + graphs_dash.append( + Dashboard( + "SLRU", + [ + [ + Graph( + "All SLRUs (per second)", + metrics=GlobalSlruMetricGroup.all(self), + ) + ], + [ + Grid( + "All SLRUs", + columns=[ + { + "name": "name", + "label": "SLRU name", + "url_attr": "url", + } + ], + metrics=ByAllSlruMetricGroup.all(self), + ) + ], + ], + ) + ) + + if self.has_extension(self.path_args[0], "pg_wait_sampling"): + metrics = None # if we can't connect to the remote server, assume pg10 or above if pg_version_num is not None and pg_version_num < 100000: - metrics = [GlobalWaitsMetricGroup.count_lwlocknamed, - GlobalWaitsMetricGroup.count_lwlocktranche, - GlobalWaitsMetricGroup.count_lock, - GlobalWaitsMetricGroup.count_bufferpin] + metrics = [ + GlobalWaitsMetricGroup.count_lwlocknamed, + GlobalWaitsMetricGroup.count_lwlocktranche, + GlobalWaitsMetricGroup.count_lock, + GlobalWaitsMetricGroup.count_bufferpin, + ] else: - metrics = [GlobalWaitsMetricGroup.count_lwlock, - GlobalWaitsMetricGroup.count_lock, - GlobalWaitsMetricGroup.count_bufferpin, - GlobalWaitsMetricGroup.count_activity, - GlobalWaitsMetricGroup.count_client, - GlobalWaitsMetricGroup.count_extension, - GlobalWaitsMetricGroup.count_ipc, - GlobalWaitsMetricGroup.count_timeout, - GlobalWaitsMetricGroup.count_io] - - graphs_dash.append(Dashboard("Wait Events", - [[Graph("Wait Events (per second)", - url=self.docs_stats_url + "pg_wait_sampling.html", - metrics=metrics)]])) + metrics = [ + GlobalWaitsMetricGroup.count_lwlock, + GlobalWaitsMetricGroup.count_lock, + GlobalWaitsMetricGroup.count_bufferpin, + GlobalWaitsMetricGroup.count_activity, + GlobalWaitsMetricGroup.count_client, + GlobalWaitsMetricGroup.count_extension, + GlobalWaitsMetricGroup.count_ipc, + GlobalWaitsMetricGroup.count_timeout, + GlobalWaitsMetricGroup.count_io, + ] + + graphs_dash.append( + Dashboard( + "Wait Events", + [ + [ + Graph( + "Wait Events (per second)", + url=self.docs_stats_url + + "pg_wait_sampling.html", + metrics=metrics, + ) + ] + ], + ) + ) - toprow = [{ - # database - }] + toprow = [ + { + # database + } + ] if pgss18: - toprow.extend([{ - # plan time - }]) - - toprow.extend([{ - 'name': 'Execution', - 'colspan': 3 - }, { - 'name': 'Blocks', - 'colspan': 4, - }, { - 'name': 'Temp blocks', - 'colspan': 2 - }, { - 'name': 'I/O', - }]) + toprow.extend( + [ + { + # plan time + } + ] + ) + + toprow.extend( + [ + {"name": "Execution", "colspan": 3}, + { + "name": "Blocks", + "colspan": 4, + }, + {"name": "Temp blocks", "colspan": 2}, + { + "name": "I/O", + }, + ] + ) if pgss18: - toprow.extend([{ - 'name': 'WAL', - 'colspan': 3 - }]) + toprow.extend([{"name": "WAL", "colspan": 3}]) if pgss110: - toprow.extend([{ - 'name': 'JIT', - 'colspan': 2 - }]) - - dashes = [graphs, - [Grid("Details for all databases", - toprow=toprow, - columns=[{ + toprow.extend([{"name": "JIT", "colspan": 2}]) + + dashes = [ + graphs, + [ + Grid( + "Details for all databases", + toprow=toprow, + columns=[ + { "name": "datname", "label": "Database", - "url_attr": "url" - }], - metrics=ByDatabaseMetricGroup.all(self))]] + "url_attr": "url", + } + ], + metrics=ByDatabaseMetricGroup.all(self), + ) + ], + ] if self.has_extension(self.path_args[0], "pg_wait_sampling"): - dashes.append([Grid("Wait events for all databases", - url=self.docs_stats_url + "pg_wait_sampling.html", - columns=[{ - "name": "datname", - "label": "Database", - "url_attr": "url" - }, { - "name": "event_type", - "label": "Event Type", - }, { - "name": "event", - "label": "Event", - }], - metrics=ByDatabaseWaitSamplingMetricGroup. - all())]) + dashes.append( + [ + Grid( + "Wait events for all databases", + url=self.docs_stats_url + "pg_wait_sampling.html", + columns=[ + { + "name": "datname", + "label": "Database", + "url_attr": "url", + }, + { + "name": "event_type", + "label": "Event Type", + }, + { + "name": "event", + "label": "Event", + }, + ], + metrics=ByDatabaseWaitSamplingMetricGroup.all(), + ) + ] + ) self._dashboard = Dashboard("All databases", dashes) return self._dashboard @@ -1829,6 +2456,7 @@ def breadcrum_title(cls, handler, param): @classmethod def get_childmenu(cls, handler, params): from powa.database import DatabaseOverview + children = [] for d in list(handler.get_databases(params["server"])): new_params = params.copy() diff --git a/powa/slru.py b/powa/slru.py index 2c8f0afa..f5b168e7 100644 --- a/powa/slru.py +++ b/powa/slru.py @@ -1,40 +1,67 @@ """ Dashboards for the SLRU page. """ -from powa.dashboards import (Dashboard, Graph, Grid, DashboardPage, - MetricDef, MetricGroupDef) + from powa.config import ConfigChangesGlobal -from powa.server import ServerOverview, ByAllSlruMetricGroup -from powa.sql.views_graph import powa_get_slru_sample -from powa.sql.views_grid import powa_getslrudata +from powa.dashboards import ( + Dashboard, + DashboardPage, + Graph, + Grid, + MetricDef, + MetricGroupDef, +) +from powa.server import ByAllSlruMetricGroup, ServerOverview from powa.sql.utils import sum_per_sec +from powa.sql.views_graph import powa_get_slru_sample + class NameSlruMetricGroup(MetricGroupDef): """ Metric group used by pg_stat_slru graph. """ + name = "slru_name" xaxis = "name" data_url = r"/server/(\d+)/metrics/slru/([a-zA-Z]+)" - blks_zeroed = MetricDef(label="Zeroed", type="sizerate", - desc="Number of blocks zeroed during" - " initializations") - blks_hit = MetricDef(label="Hit", type="sizerate", - desc="Number of times disk blocks were found already" - " in the SLRU, so that a read was not necessary" - " (this only includes hits in the SLRU, not the" - " operating system's file system cache)") - blks_read = MetricDef(label="Read", type="sizerate", - desc="Number of disk blocks read for this SLRU") - blks_written = MetricDef(label="Written", type="sizerate", - desc="Number of disk blocks written for this SLRU") - blks_exists = MetricDef(label="Exists", type="sizerate", - desc="Number of blocks checked for existence for" - " this SLRU") - flushes = MetricDef(label="Flushes", type="number", - desc="Number of flushes of dirty data for this SLRU") - truncates = MetricDef(label="Truncates", type="number", - desc="Number of truncates for this SLRU") + blks_zeroed = MetricDef( + label="Zeroed", + type="sizerate", + desc="Number of blocks zeroed during" " initializations", + ) + blks_hit = MetricDef( + label="Hit", + type="sizerate", + desc="Number of times disk blocks were found already" + " in the SLRU, so that a read was not necessary" + " (this only includes hits in the SLRU, not the" + " operating system's file system cache)", + ) + blks_read = MetricDef( + label="Read", + type="sizerate", + desc="Number of disk blocks read for this SLRU", + ) + blks_written = MetricDef( + label="Written", + type="sizerate", + desc="Number of disk blocks written for this SLRU", + ) + blks_exists = MetricDef( + label="Exists", + type="sizerate", + desc="Number of blocks checked for existence for" " this SLRU", + ) + flushes = MetricDef( + label="Flushes", + type="number", + desc="Number of flushes of dirty data for this SLRU", + ) + truncates = MetricDef( + label="Truncates", + type="number", + desc="Number of truncates for this SLRU", + ) @classmethod def _get_metrics(cls, handler, **params): @@ -52,16 +79,17 @@ def query(self): from_clause = query - cols = ["sub.srvid", - "extract(epoch FROM sub.ts) AS ts", - sum_per_sec('blks_zeroed'), - sum_per_sec('blks_hit'), - sum_per_sec('blks_read'), - sum_per_sec('blks_written'), - sum_per_sec('blks_exists'), - sum_per_sec('flushes'), - sum_per_sec('truncates'), - ] + cols = [ + "sub.srvid", + "extract(epoch FROM sub.ts) AS ts", + sum_per_sec("blks_zeroed"), + sum_per_sec("blks_hit"), + sum_per_sec("blks_read"), + sum_per_sec("blks_written"), + sum_per_sec("blks_exists"), + sum_per_sec("flushes"), + sum_per_sec("truncates"), + ] return """SELECT {cols} FROM ( @@ -70,10 +98,11 @@ def query(self): WHERE sub.mesure_interval != '0 s' GROUP BY sub.srvid, sub.ts, sub.mesure_interval ORDER BY sub.ts""".format( - cols=', '.join(cols), + cols=", ".join(cols), from_clause=from_clause, ) + class ByAllSlruMetricGroup2(ByAllSlruMetricGroup): """ Metric group used by pg_stat_slru grid. @@ -85,38 +114,51 @@ class ByAllSlruMetricGroup2(ByAllSlruMetricGroup): make it work. It's not a big problem since this is a different page, so we won't end up running the same query twice. """ + name = "slru_by_name2" data_url = r"/server/(\d+)/metrics/slru_by_name2/([a-zA-Z]+)" + class ByNameSlruOverview(DashboardPage): """ Per SLRU Dashboard page. """ + base_url = r"/server/(\d+)/metrics/slru/([a-zA-Z]+)/overview/" datasources = [NameSlruMetricGroup, ByAllSlruMetricGroup2] params = ["server", "slru"] parent = ServerOverview - title = "Activity for \"%(slru)s\" SLRU" + title = 'Activity for "%(slru)s" SLRU' timeline = ConfigChangesGlobal timeline_params = ["server"] def dashboard(self): # This COULD be initialized in the constructor, but tornado < 3 doesn't # call it - if getattr(self, '_dashboard', None) is not None: + if getattr(self, "_dashboard", None) is not None: return self._dashboard graphs = [ - [Graph("%(slru)s SLRU (per second)", - metrics=NameSlruMetricGroup.all(self))], - [Grid("All SLRUs", - columns=[{ - "name": "name", - "label": "SLRU name", - "url_attr": "url" - }], - metrics=ByAllSlruMetricGroup2.all(self))] - ] + [ + Graph( + "%(slru)s SLRU (per second)", + metrics=NameSlruMetricGroup.all(self), + ) + ], + [ + Grid( + "All SLRUs", + columns=[ + { + "name": "name", + "label": "SLRU name", + "url_attr": "url", + } + ], + metrics=ByAllSlruMetricGroup2.all(self), + ) + ], + ] self._dashboard = Dashboard(self.title, graphs) return self._dashboard diff --git a/powa/sql/__init__.py b/powa/sql/__init__.py index 8d402aca..d0de42e4 100644 --- a/powa/sql/__init__.py +++ b/powa/sql/__init__.py @@ -1,10 +1,11 @@ """ Utilities for commonly used SQL constructs. """ + import re -from collections import namedtuple, defaultdict -from powa.json import JSONizable import sys +from collections import defaultdict, namedtuple +from powa.json import JSONizable TOTAL_MEASURE_INTERVAL = """ extract( epoch from @@ -15,9 +16,9 @@ def unprepare(sql): - if sql.startswith('PREPARE'): - sql = re.sub('PREPARE.*AS', '', sql) - sql = re.sub('\$\d+', '?', sql) + if sql.startswith("PREPARE"): + sql = re.sub("PREPARE.*AS", "", sql) + sql = re.sub("\$\d+", "?", sql) return sql @@ -100,16 +101,21 @@ def format_jumbled_query(sql, params): class ResolvedQual(JSONizable): - - def __init__(self, nspname, relname, attname, - opname, amops, - n_distinct=0, - most_common_values=None, - null_frac=None, - example_values=None, - eval_type=None, - relid=None, - attnum=None): + def __init__( + self, + nspname, + relname, + attname, + opname, + amops, + n_distinct=0, + most_common_values=None, + null_frac=None, + example_values=None, + eval_type=None, + relid=None, + attnum=None, + ): self.nspname = nspname self.relname = relname self.attname = attname @@ -143,17 +149,20 @@ def to_json(self): class ComposedQual(JSONizable): - - def __init__(self, nspname=None, relname=None, - avg_filter=None, - filter_ratio=None, - occurences=None, - execution_count=None, - table_liverows=None, - qualid=None, - relid=None, - queries=None, - queryids=None): + def __init__( + self, + nspname=None, + relname=None, + avg_filter=None, + filter_ratio=None, + occurences=None, + execution_count=None, + table_liverows=None, + qualid=None, + relid=None, + queries=None, + queryids=None, + ): super(ComposedQual, self).__init__() self.qualid = qualid self.relname = relname @@ -170,8 +179,9 @@ def __init__(self, nspname=None, relname=None, def append(self, element): if not isinstance(element, ResolvedQual): - raise ValueError(("ComposedQual elements must be instances of ", - "ResolvedQual")) + raise ValueError( + ("ComposedQual elements must be instances of ", "ResolvedQual") + ) self._quals.append(element) def __iter__(self): @@ -186,8 +196,8 @@ def where_clause(self): def to_json(self): base = super(ComposedQual, self).to_json() - base['quals'] = self._quals - base['where_clause'] = self.where_clause + base["quals"] = self._quals + base["where_clause"] = self.where_clause return base @@ -212,70 +222,81 @@ def resolve_quals(conn, quallist, attribute="quals"): if not isinstance(values, list): values = [values] for v in values: - operator_to_look.add(v['opno']) + operator_to_look.add(v["opno"]) attname_to_look.add((v["relid"], v["attnum"])) if operator_to_look: cur = conn.cursor() - cur.execute( - RESOLVE_OPNAME, - {"oid_list": tuple(operator_to_look)}) + cur.execute(RESOLVE_OPNAME, {"oid_list": tuple(operator_to_look)}) operators = cur.fetchone()[0] cur.close() if attname_to_look: cur = conn.cursor() - cur.execute( - RESOLVE_ATTNAME, - {"att_list": tuple(attname_to_look)}) + cur.execute(RESOLVE_ATTNAME, {"att_list": tuple(attname_to_look)}) attnames = cur.fetchone()[0] new_qual_list = [] for row in quallist: row = dict(row) newqual = ComposedQual( - occurences=row['occurences'], - execution_count=row['execution_count'], - avg_filter=row['avg_filter'], - filter_ratio=row['filter_ratio'], - qualid=row['qualid'], - queries=row.get('queries'), - queryids=row.get('queryids') + occurences=row["occurences"], + execution_count=row["execution_count"], + avg_filter=row["avg_filter"], + filter_ratio=row["filter_ratio"], + qualid=row["qualid"], + queries=row.get("queries"), + queryids=row.get("queryids"), ) new_qual_list.append(newqual) - values = [v for v in row[attribute] if v['relid'] != '0'] + values = [v for v in row[attribute] if v["relid"] != "0"] if not isinstance(values, list): values = [values] for v in values: attname = attnames["{}.{}".format(v["relid"], v["attnum"])] if newqual.relname is not None: - if newqual.relname != attname['relname']: - raise ValueError("All individual qual parts should be on the " - "same relation") + if newqual.relname != attname["relname"]: + raise ValueError( + "All individual qual parts should be on the " + "same relation" + ) else: newqual.relname = attname["relname"] newqual.nspname = attname["nspname"] newqual.relid = v["relid"] newqual.table_liverows = attname["table_liverows"] - newqual.append(ResolvedQual( - nspname=attname['nspname'], - relname=attname['relname'], - attname=attname['attname'], - opname=operators[v["opno"]]["name"], - amops=operators[v["opno"]]["amop_names"], - n_distinct=attname["n_distinct"], - most_common_values=attname["most_common_values"], - null_frac=attname["null_frac"], - eval_type=v["eval_type"], - relid=v["relid"], - attnum=v["attnum"])) + newqual.append( + ResolvedQual( + nspname=attname["nspname"], + relname=attname["relname"], + attname=attname["attname"], + opname=operators[v["opno"]]["name"], + amops=operators[v["opno"]]["amop_names"], + n_distinct=attname["n_distinct"], + most_common_values=attname["most_common_values"], + null_frac=attname["null_frac"], + eval_type=v["eval_type"], + relid=v["relid"], + attnum=v["attnum"], + ) + ) return new_qual_list Plan = namedtuple( "Plan", - ("title", "values", "query", "plan", "filter_ratio", "exec_count", "occurences")) - - -def qual_constants(srvid, type, filter_clause, queries=None, quals=None, - top=1): + ( + "title", + "values", + "query", + "plan", + "filter_ratio", + "exec_count", + "occurences", + ), +) + + +def qual_constants( + srvid, type, filter_clause, queries=None, quals=None, top=1 +): """ filter_clause is a plain string corresponding to the list of predicates to apply in the main query. @@ -284,13 +305,17 @@ def qual_constants(srvid, type, filter_clause, queries=None, quals=None, filter on. """ orders = { - 'most_executed': "8 DESC", - 'least_filtering': "9", - 'most_filtering': "9 DESC", - 'most_used': '6 DESC' + "most_executed": "8 DESC", + "least_filtering": "9", + "most_filtering": "9 DESC", + "most_used": "6 DESC", } - if type not in ('most_executed', 'most_filtering', - 'least_filtering', 'most_used'): + if type not in ( + "most_executed", + "most_filtering", + "least_filtering", + "most_used", + ): return query_subfilter = "" @@ -361,7 +386,8 @@ def qual_constants(srvid, type, filter_clause, queries=None, quals=None, qual_filter=qual_filter, order=orders[type], top_value=top, - srvid=srvid) + srvid=srvid, + ) query = "SELECT * FROM " + base @@ -374,29 +400,41 @@ def quote_ident(name): def get_plans(cls, server, database, query, all_vals): plans = [] - for key in ('most filtering', 'least filtering', 'most executed', - 'most used'): + for key in ( + "most filtering", + "least filtering", + "most executed", + "most used", + ): vals = all_vals[key] - query = format_jumbled_query(query, vals['constants']) + query = format_jumbled_query(query, vals["constants"]) plan = "N/A" try: sqlQuery = "EXPLAIN {}".format(query) - result = cls.execute(sqlQuery, - srvid=server, - database=database, - remote_access=True) - plan = "\n".join(v['QUERY PLAN'] for v in result) + result = cls.execute( + sqlQuery, srvid=server, database=database, remote_access=True + ) + plan = "\n".join(v["QUERY PLAN"] for v in result) except Exception as e: plan = "ERROR: %r" % e pass - plans.append(Plan(key, vals['constants'], query, - plan, vals["filter_ratio"], vals['execution_count'], - vals['occurences'])) + plans.append( + Plan( + key, + vals["constants"], + query, + plan, + vals["filter_ratio"], + vals["execution_count"], + vals["occurences"], + ) + ) return plans -def get_unjumbled_query(ctrl, srvid, database, queryid, _from, _to, - kind='most executed'): +def get_unjumbled_query( + ctrl, srvid, database, queryid, _from, _to, kind="most executed" +): """ From a queryid, build a query string ready to be executed. @@ -408,15 +446,21 @@ def get_unjumbled_query(ctrl, srvid, database, queryid, _from, _to, been found and/or the SELECT clause has been normalized """ - rs = list(ctrl.execute(""" + rs = list( + ctrl.execute( + """ SELECT query FROM {powa}.powa_statements WHERE srvid= %(srvid)s AND queryid = %(queryid)s LIMIT 1 - """, params={"srvid": srvid, "queryid": queryid}))[0] - normalized_query = rs['query'] - values = qualstat_get_figures(ctrl, srvid, database, _from, _to, - queries=[queryid]) + """, + params={"srvid": srvid, "queryid": queryid}, + ) + )[0] + normalized_query = rs["query"] + values = qualstat_get_figures( + ctrl, srvid, database, _from, _to, queries=[queryid] + ) if values is None: unprepared = unprepare(normalized_query) @@ -424,8 +468,9 @@ def get_unjumbled_query(ctrl, srvid, database, queryid, _from, _to, return None # Try to inject values - sql = format_jumbled_query(normalized_query, - values[kind].get('constants', [])) + sql = format_jumbled_query( + normalized_query, values[kind].get("constants", []) + ) return sql @@ -443,7 +488,8 @@ def get_any_sample_query(ctrl, srvid, database, queryid, _from, _to): has_pgqs = ctrl.has_extension_version(srvid, "pg_qualstats", "0.0.7") example_query = None if has_pgqs: - rows = ctrl.execute(""" + rows = ctrl.execute( + """ WITH s(max, v) AS ( SELECT (SELECT setting::int @@ -455,34 +501,40 @@ def get_any_sample_query(ctrl, srvid, database, queryid, _from, _to): FROM s WHERE v NOT ILIKE '%%EXPLAIN%%' LIMIT 1 - """, params={"queryid": queryid}, srvid=srvid, remote_access=True) + """, + params={"queryid": queryid}, + srvid=srvid, + remote_access=True, + ) if rows is not None and len(rows) > 0: # Ignore the query if it looks like it was truncated - if rows[0]['len'] < (rows[0]['max'] - 1): - example_query = rows[0]['v'] + if rows[0]["len"] < (rows[0]["max"] - 1): + example_query = rows[0]["v"] if example_query is not None: unprepared = unprepare(example_query) if example_query == unprepared: return example_query - return get_unjumbled_query(ctrl, srvid, database, queryid, - _from, _to, 'most executed') + return get_unjumbled_query( + ctrl, srvid, database, queryid, _from, _to, "most executed" + ) -def qualstat_get_figures(cls, srvid, database, tsfrom, tsto, - queries=None, quals=None): +def qualstat_get_figures( + cls, srvid, database, tsfrom, tsto, queries=None, quals=None +): condition = """datname = %(database)s AND coalesce_range && tstzrange(%(from)s, %(to)s)""" if queries is not None: - queries_str = ','.join(str(q) for q in queries) + queries_str = ",".join(str(q) for q in queries) cols = [ - 'most_filtering.quals', - 'most_filtering.query', + "most_filtering.quals", + "most_filtering.query", 'to_json(most_filtering) as "most filtering"', 'to_json(least_filtering) as "least filtering"', 'to_json(most_executed) as "most executed"', - 'to_json(most_used) as "most used"' + 'to_json(most_used) as "most used"', ] sql = """SELECT {cols} @@ -490,22 +542,28 @@ def qualstat_get_figures(cls, srvid, database, tsfrom, tsto, JOIN ({least_filtering}) AS least_filtering USING (rownumber) JOIN ({most_executed}) AS most_executed USING (rownumber) JOIN ({most_used}) AS most_used USING (rownumber)""".format( - cols=', '.join(cols), - most_filtering=qual_constants(srvid, "least_filtering", - condition, queries_str, quals), - least_filtering=qual_constants(srvid, "least_filtering", - condition, queries_str, quals), - most_executed=qual_constants(srvid, "most_executed", - condition, queries_str, quals), - most_used=qual_constants(srvid, "most_used", - condition, queries_str, quals) - ) + cols=", ".join(cols), + most_filtering=qual_constants( + srvid, "least_filtering", condition, queries_str, quals + ), + least_filtering=qual_constants( + srvid, "least_filtering", condition, queries_str, quals + ), + most_executed=qual_constants( + srvid, "most_executed", condition, queries_str, quals + ), + most_used=qual_constants( + srvid, "most_used", condition, queries_str, quals + ), + ) - params = {"server": srvid, - "database": database, - "from": tsfrom, - "to": tsto, - "queryids": queries} + params = { + "server": srvid, + "database": database, + "from": tsfrom, + "to": tsto, + "queryids": queries, + } quals = cls.execute(sql, params=params) if len(quals) == 0: @@ -517,10 +575,9 @@ def qualstat_get_figures(cls, srvid, database, tsfrom, tsto, class HypoPlan(JSONizable): - - def __init__(self, baseplan, basecost, - hypoplan, hypocost, - query, indexes=None): + def __init__( + self, baseplan, basecost, hypoplan, hypocost, query, indexes=None + ): self.baseplan = baseplan self.basecost = basecost self.hypoplan = hypoplan @@ -530,18 +587,18 @@ def __init__(self, baseplan, basecost, @property def gain_percent(self): - return round(100 - float(self.hypocost) * 100 / float(self.basecost), 2) + return round( + 100 - float(self.hypocost) * 100 / float(self.basecost), 2 + ) def to_json(self): base = super(HypoPlan, self).to_json() - base['gain_percent'] = self.gain_percent + base["gain_percent"] = self.gain_percent return base class HypoIndex(JSONizable): - - def __init__(self, nspname, relname, amname, - composed_qual=None): + def __init__(self, nspname, relname, amname, composed_qual=None): self.nspname = nspname self.relname = relname self.qual = composed_qual @@ -552,23 +609,23 @@ def __init__(self, nspname, relname, amname, self._update_ddl() def _update_ddl(self): - if 'btree' == self.amname: + if "btree" == self.amname: attrs = [] for qual in self.qual: if qual.attname not in attrs: attrs.append(qual.attname) # Qual resolution is responsible for quoting all identifiers super(HypoIndex, self).__setattr__( - '_ddl', + "_ddl", """CREATE INDEX ON {nsp}.{rel}({attrs})""".format( - nsp=self.nspname, - rel=self.relname, - attrs=",".join(attrs))) + nsp=self.nspname, rel=self.relname, attrs=",".join(attrs) + ), + ) def __setattr(self, name, value): super(HypoIndex, self).__setattr__(name, value) # Only btree is supported right now - if name in ('amname', 'nspname', 'relname', 'composed_qual'): + if name in ("amname", "nspname", "relname", "composed_qual"): self._update_ddl() @property @@ -579,14 +636,16 @@ def ddl(self): def hypo_ddl(self): ddl = self.ddl if ddl is not None: - return ("SELECT indexname" - + " FROM {hypopg}.hypopg_create_index(%(sql)s)", - {'sql': ddl}) + return ( + "SELECT indexname" + + " FROM {hypopg}.hypopg_create_index(%(sql)s)", + {"sql": ddl}, + ) return (None, None) def to_json(self): base = super(HypoIndex, self).to_json() - base['ddl'] = self.ddl + base["ddl"] = self.ddl return base @@ -600,16 +659,14 @@ def sorter(qual): else: # - attnum to guarantee stable sort return sys.maxsize - attnum + for qual in sorted(composed_qual, key=sorter): for am in qual.amops.keys(): by_am[am].append(qual) indexes = [] for am, quals in by_am.items(): base = quals[0] - indexes.append(HypoIndex(base.nspname, - base.relname, - am, - quals)) + indexes.append(HypoIndex(base.nspname, base.relname, am, quals)) return indexes @@ -667,5 +724,6 @@ def get_hypoplans(cur, query, indexes=None): for ind in indexes: if ind.name is not None and ind.name in hypoplan: used_indexes.append(ind) - return HypoPlan(baseplan, basecost, hypoplan, hypocost, query, - used_indexes) + return HypoPlan( + baseplan, basecost, hypoplan, hypocost, query, used_indexes + ) diff --git a/powa/sql/utils.py b/powa/sql/utils.py index ca52e449..3faa33c0 100644 --- a/powa/sql/utils.py +++ b/powa/sql/utils.py @@ -1,5 +1,7 @@ -block_size = "(SELECT cast(current_setting('block_size') AS numeric)" \ - " AS block_size) AS bs" +block_size = ( + "(SELECT cast(current_setting('block_size') AS numeric)" + " AS block_size) AS bs" +) def mulblock(col, alias=None, fn=None): @@ -16,26 +18,26 @@ def mulblock(col, alias=None, fn=None): def total_measure_interval(col): - sql = "extract(epoch FROM " \ - + " CASE WHEN min({col}) = '0 second' THEN '1 second'" \ - + " ELSE min({col})" \ - + "END)" + sql = ( + "extract(epoch FROM " + + " CASE WHEN min({col}) = '0 second' THEN '1 second'" + + " ELSE min({col})" + + "END)" + ) return sql.format(col=col) def diff(var, alias=None): alias = alias or var - return "max({var}) - min({var}) AS {alias}".format( - var=var, - alias=alias) + return "max({var}) - min({var}) AS {alias}".format(var=var, alias=alias) + def diffblk(var, blksize=8192, alias=None): alias = alias or var return "(max({var}) - min({var})) * {blksize} AS {alias}".format( - var=var, - blksize=blksize, - alias=alias) + var=var, blksize=blksize, alias=alias + ) def get_ts(): @@ -47,13 +49,10 @@ def sum_per_sec(col, prefix=None, alias=None): if prefix is not None: prefix = prefix + "." else: - prefix = '' + prefix = "" return "sum({prefix}{col}) / {ts} AS {alias}".format( - prefix=prefix, - col=col, - ts=get_ts(), - alias=alias + prefix=prefix, col=col, ts=get_ts(), alias=alias ) @@ -62,13 +61,10 @@ def byte_per_sec(col, prefix=None, alias=None): if prefix is not None: prefix = prefix + "." else: - prefix = '' + prefix = "" return "sum({prefix}{col}) * block_size / {ts} AS {alias}".format( - prefix=prefix, - col=col, - ts=get_ts(), - alias=alias + prefix=prefix, col=col, ts=get_ts(), alias=alias ) @@ -78,9 +74,8 @@ def wps(col, do_sum=True): field = "sum(" + field + ")" return "({field} / {ts}) AS {col}".format( - field=field, - col=col, - ts=get_ts()) + field=field, col=col, ts=get_ts() + ) def to_epoch(col, prefix=None): @@ -94,23 +89,27 @@ def to_epoch(col, prefix=None): def total_read(prefix, noalias=False): if noalias: - alias = '' + alias = "" else: - alias = ' AS total_blks_read' + alias = " AS total_blks_read" - sql = "sum({prefix}.shared_blks_hit" \ - + "+ {prefix}.local_blks_read" \ - + "+ {prefix}.temp_blks_read" \ - ") * block_size / {total_measure_interval}{alias}" + sql = ( + "sum({prefix}.shared_blks_hit" + + "+ {prefix}.local_blks_read" + + "+ {prefix}.temp_blks_read" + ") * block_size / {total_measure_interval}{alias}" + ) return sql.format( prefix=prefix, - total_measure_interval=total_measure_interval('mesure_interval'), - alias=alias + total_measure_interval=total_measure_interval("mesure_interval"), + alias=alias, ) def total_hit(c): - return "sum(shared_blks_hit + local_blks_hit) * block_size /" \ - + total_measure_interval('mesure_interval') \ - + " AS total_blks_hit" + return ( + "sum(shared_blks_hit + local_blks_hit) * block_size /" + + total_measure_interval("mesure_interval") + + " AS total_blks_hit" + ) diff --git a/powa/sql/views.py b/powa/sql/views.py index a26e2625..0fcaec02 100644 --- a/powa/sql/views.py +++ b/powa/sql/views.py @@ -2,6 +2,7 @@ Functions to generate the queries used in components that are neither graphs or grid. """ + QUALSTAT_FILTER_RATIO = """CASE WHEN sum(execution_count) = 0 THEN 0 ELSE sum(nbfiltered) / sum(execution_count)::numeric * 100 @@ -10,12 +11,7 @@ def qualstat_base_statdata(eval_type=None): if eval_type is not None: - base_cols = ["srvid", - "qualid," - "queryid", - "dbid", - "userid" - ] + base_cols = ["srvid", "qualid," "queryid", "dbid", "userid"] pqnh = """( SELECT {outer_cols} @@ -26,11 +22,11 @@ def qualstat_base_statdata(eval_type=None): WHERE (qual).eval_type = '{eval_type}' GROUP BY {base_cols} )""".format( - outer_cols=', '.join(base_cols + ["array_agg(qual) AS quals"]), - inner_cols=', '.join(base_cols + ["unnest(quals) AS qual"]), - base_cols=', '.join(base_cols), - eval_type=eval_type - ) + outer_cols=", ".join(base_cols + ["array_agg(qual) AS quals"]), + inner_cols=", ".join(base_cols + ["unnest(quals) AS qual"]), + base_cols=", ".join(base_cols), + eval_type=eval_type, + ) else: pqnh = "{powa}.powa_qualstats_quals" @@ -58,32 +54,38 @@ def qualstat_base_statdata(eval_type=None): return base_query -def qualstat_getstatdata(eval_type=None, extra_from='', extra_join='', - extra_select=[], extra_where=[], extra_groupby=[], - extra_having=[]): +def qualstat_getstatdata( + eval_type=None, + extra_from="", + extra_join="", + extra_select=[], + extra_where=[], + extra_groupby=[], + extra_having=[], +): base_query = qualstat_base_statdata(eval_type) # Reformat extra_select, extra_where, extra_groupby and extra_having to be # plain additional SQL clauses. if len(extra_select) > 0: - extra_select = ', ' + ', '.join(extra_select) + extra_select = ", " + ", ".join(extra_select) else: - extra_select = '' + extra_select = "" if len(extra_where) > 0: - extra_where = ' AND ' + ' AND '.join(extra_where) + extra_where = " AND " + " AND ".join(extra_where) else: - extra_where = '' + extra_where = "" if len(extra_groupby) > 0: - extra_groupby = ', ' + ', '.join(extra_groupby) + extra_groupby = ", " + ", ".join(extra_groupby) else: - extra_groupby = '' + extra_groupby = "" if len(extra_having) > 0: - extra_having = " HAVING " + ' AND '.join(extra_having) + extra_having = " HAVING " + " AND ".join(extra_having) else: - extra_having = '' + extra_having = "" return """SELECT ps.srvid, qualid, ps.queryid, query, ps.dbid, @@ -102,14 +104,14 @@ def qualstat_getstatdata(eval_type=None, extra_from='', extra_join='', GROUP BY ps.srvid, qualid, ps.queryid, ps.dbid, ps.query, quals {extra_groupby} {extra_having}""".format( - filter_ratio=QUALSTAT_FILTER_RATIO, - extra_select=extra_select, - base_query=base_query, - extra_join=extra_join, - extra_where=extra_where, - extra_groupby=extra_groupby, - extra_having=extra_having - ) + filter_ratio=QUALSTAT_FILTER_RATIO, + extra_select=extra_select, + base_query=base_query, + extra_join=extra_join, + extra_where=extra_where, + extra_groupby=extra_groupby, + extra_having=extra_having, + ) TEXTUAL_INDEX_QUERY = """ @@ -163,7 +165,7 @@ def qualstat_getstatdata(eval_type=None, extra_from='', extra_join='', def get_config_changes(restrict_database=False): restrict_db = "" - if (restrict_database): + if restrict_database: restrict_db = "AND (d.datname = %(database)s OR h.setdatabase = 0)" sql = """SELECT * FROM diff --git a/powa/sql/views_graph.py b/powa/sql/views_graph.py index e9677e73..9dc2a6d1 100644 --- a/powa/sql/views_graph.py +++ b/powa/sql/views_graph.py @@ -1,6 +1,8 @@ """ Functions to generate the queries used in the various graphs components """ + + def wal_to_num(walname): """ Extracts the sequence number from the given WAL file name, similarly to @@ -9,10 +11,11 @@ def wal_to_num(walname): return """(('x' || substr({walname}, 9, 8))::bit(32)::bigint * '4294967296'::bigint / 16777216 + ('x' || substr({walname}, 17, 8))::bit(32)::bigint)""".format( - walname=walname) + walname=walname + ) -class Biggest(object): +class Biggest(object): def __init__(self, base_columns, order_by): if type(base_columns) is str: base_columns = [base_columns] @@ -25,22 +28,23 @@ def __init__(self, base_columns, order_by): def __call__(self, var, minval=0, label=None): label = label or var - sql = "greatest(lead({var})" \ - " OVER (PARTITION BY {partitionby} ORDER BY {orderby})" \ - " - {var}," \ - " {minval})" \ - " AS {alias}".format( - var=var, - orderby=', '.join(self.order_by), - partitionby=', '.join(self.base_columns), - minval=minval, - alias=label - ) + sql = ( + "greatest(lead({var})" + " OVER (PARTITION BY {partitionby} ORDER BY {orderby})" + " - {var}," + " {minval})" + " AS {alias}".format( + var=var, + orderby=", ".join(self.order_by), + partitionby=", ".join(self.base_columns), + minval=minval, + alias=label, + ) + ) return sql class Biggestsum(object): - def __init__(self, base_columns, order_by): if type(base_columns) is str: base_columns = [base_columns] @@ -53,17 +57,19 @@ def __init__(self, base_columns, order_by): def __call__(self, var, minval=0, label=None): label = label or var - sql = "greatest(lead(sum({var}))"\ - " OVER (PARTITION BY {partitionby} ORDER BY {orderby})" \ - " - sum({var})," \ - " {minval})" \ - " AS {alias}".format( - var=var, - orderby=', '.join(self.order_by), - partitionby=', '.join(self.base_columns), - minval=minval, - alias=label - ) + sql = ( + "greatest(lead(sum({var}))" + " OVER (PARTITION BY {partitionby} ORDER BY {orderby})" + " - sum({var})," + " {minval})" + " AS {alias}".format( + var=var, + orderby=", ".join(self.order_by), + partitionby=", ".join(self.base_columns), + minval=minval, + alias=label, + ) + ) return sql @@ -158,15 +164,15 @@ def powa_getstatdata_sample(mode, predicates=[]): base_query = BASE_QUERY_SAMPLE base_columns = ["srvid", "dbid", "queryid", "userid"] - biggest = Biggest(base_columns, 'ts') - biggestsum = Biggestsum(base_columns, 'ts') + biggest = Biggest(base_columns, "ts") + biggestsum = Biggestsum(base_columns, "ts") - where = ' AND '.join(predicates) - if where != '': - where = ' AND ' + where + where = " AND ".join(predicates) + if where != "": + where = " AND " + where cols = base_columns + [ - 'ts', + "ts", biggest("ts", "'0 s'", "mesure_interval"), biggestsum("calls"), biggestsum("total_plan_time", label="plantime"), @@ -208,10 +214,10 @@ def powa_getstatdata_sample(mode, predicates=[]): WHERE srvid = %(server)s {where} GROUP BY {base_columns}, ts""".format( - cols=', '.join(cols), + cols=", ".join(cols), base_query=base_query, where=where, - base_columns=', '.join(base_columns) + base_columns=", ".join(base_columns), ) @@ -319,48 +325,55 @@ def kcache_getstatdata_sample(mode, predicates=[]): """ predicates is an optional array of plain-text predicates. """ - if (mode == "db"): + if mode == "db": base_query = BASE_QUERY_KCACHE_SAMPLE_DB base_columns = ["d.oid AS dbid", "srvid, datname"] groupby_columns = "d.oid, srvid, datname" - elif (mode == "query"): + elif mode == "query": base_query = BASE_QUERY_KCACHE_SAMPLE - base_columns = ["d.oid AS dbid", "d.srvid", "datname", "queryid", - "userid"] + base_columns = [ + "d.oid AS dbid", + "d.srvid", + "datname", + "queryid", + "userid", + ] groupby_columns = "d.oid, d.srvid, datname, queryid, userid" biggestsum = Biggestsum(groupby_columns, "ts") - where = ' AND '.join(predicates) - if where != '': - where = ' AND ' + where + where = " AND ".join(predicates) + if where != "": + where = " AND " + where - base_columns.extend([ - "ts", - biggestsum("reads"), - biggestsum("writes"), - biggestsum("user_time"), - biggestsum("system_time"), - biggestsum("minflts"), - biggestsum("majflts"), - # not maintained on GNU/Linux, and not available on Windows - # biggestsum("nswaps"), - # biggestsum("msgsnds"), - # biggestsum("msgrcvs"), - # biggestsum("nsignals"), - biggestsum("nvcsws"), - biggestsum("nivcsws"), - ]) + base_columns.extend( + [ + "ts", + biggestsum("reads"), + biggestsum("writes"), + biggestsum("user_time"), + biggestsum("system_time"), + biggestsum("minflts"), + biggestsum("majflts"), + # not maintained on GNU/Linux, and not available on Windows + # biggestsum("nswaps"), + # biggestsum("msgsnds"), + # biggestsum("msgrcvs"), + # biggestsum("nsignals"), + biggestsum("nvcsws"), + biggestsum("nivcsws"), + ] + ) return """SELECT {base_columns} FROM {base_query} WHERE d.srvid = %(server)s {where} GROUP BY {groupby_columns}, ts""".format( - base_columns=', '.join(base_columns), + base_columns=", ".join(base_columns), groupby_columns=groupby_columns, where=where, - base_query=base_query + base_query=base_query, ) @@ -495,7 +508,7 @@ def kcache_getstatdata_sample(mode, predicates=[]): # knowing that the first 3 xid are reserved and can never be assigned, and that # the highest transaction id is 2^32. def BASE_QUERY_PGSA_SAMPLE(per_db=False): - if (per_db): + if per_db: extra = """JOIN {powa}.powa_catalog_databases d ON d.oid = pgsa_history.datid WHERE d.datname = %(database)s""" @@ -631,7 +644,7 @@ def BASE_QUERY_PGSA_SAMPLE(per_db=False): def BASE_QUERY_DATABASE_SAMPLE(per_db=False): - if (per_db): + if per_db: extra = """JOIN {powa}.powa_catalog_databases d ON d.oid = psd_history.datid WHERE d.datname = %(database)s""" @@ -692,7 +705,7 @@ def BASE_QUERY_DATABASE_SAMPLE(per_db=False): def BASE_QUERY_DATABASE_CONFLICTS_SAMPLE(per_db=False): - if (per_db): + if per_db: extra = """JOIN {powa}.powa_catalog_databases d ON d.oid = psd_history.datid WHERE d.datname = %(database)s""" @@ -891,7 +904,7 @@ def BASE_QUERY_DATABASE_CONFLICTS_SAMPLE(per_db=False): def BASE_QUERY_SUBSCRIPTION_SAMPLE(subname=None): - if (subname is not None): + if subname is not None: extra = "AND subname = %(subscription)s" else: extra = "" @@ -1256,12 +1269,12 @@ def powa_getwaitdata_sample(mode, predicates=[]): base_query = BASE_QUERY_WAIT_SAMPLE base_columns = ["srvid", "dbid", "queryid"] - biggest = Biggest(base_columns, 'ts') - biggestsum = Biggestsum(base_columns, 'ts') + biggest = Biggest(base_columns, "ts") + biggestsum = Biggestsum(base_columns, "ts") - where = ' AND '.join(predicates) - if where != '': - where = ' AND ' + where + where = " AND ".join(predicates) + if where != "": + where = " AND " + where all_cols = base_columns + [ "ts", @@ -1286,10 +1299,10 @@ def powa_getwaitdata_sample(mode, predicates=[]): WHERE srvid = %(server)s {where} GROUP BY {base_columns}, ts""".format( - all_cols=', '.join(all_cols), - base_columns=', '.join(base_columns), + all_cols=", ".join(all_cols), + base_columns=", ".join(base_columns), base_query=base_query, - where=where + where=where, ) @@ -1314,26 +1327,24 @@ def txid_age(field): def ts_get_sec(field): alias = field + "_age" return """extract(epoch FROM (ts - {f})) * 1000 AS {a}""".format( - f=field, - a=alias) + f=field, a=alias + ) all_cols = base_columns + [ "ts", "datid", - txid_age("backend_xid"), # backend_xid_age - txid_age("backend_xmin"), # backend_xmin_age - ts_get_sec("backend_start"), # backend_start_age - ts_get_sec("xact_start"), # xact_start_age - ts_get_sec("query_start"), # query_start_age + txid_age("backend_xid"), # backend_xid_age + txid_age("backend_xmin"), # backend_xmin_age + ts_get_sec("backend_start"), # backend_start_age + ts_get_sec("xact_start"), # xact_start_age + ts_get_sec("query_start"), # query_start_age "state", "leader_pid", ] return """SELECT {all_cols} FROM {base_query}""".format( - all_cols=', '.join(all_cols), - base_columns=', '.join(base_columns), - base_query=base_query + all_cols=", ".join(all_cols), base_query=base_query ) @@ -1341,8 +1352,7 @@ def powa_get_archiver_sample(): base_query = BASE_QUERY_ARCHIVER_SAMPLE base_columns = ["srvid"] - biggest = Biggest(base_columns, 'ts') - biggestsum = Biggestsum(base_columns, 'ts') + biggest = Biggest(base_columns, "ts") all_cols = base_columns + [ "ts", @@ -1352,15 +1362,14 @@ def powa_get_archiver_sample(): biggest(wal_to_num("last_archived_wal"), label="nb_arch"), "last_failed_time", wal_to_num("current_wal") - + " - " + wal_to_num("last_archived_wal") - + " - 1 AS nb_to_arch", + + " - " + + wal_to_num("last_archived_wal") + + " - 1 AS nb_to_arch", ] return """SELECT {all_cols} FROM {base_query}""".format( - all_cols=', '.join(all_cols), - base_columns=', '.join(base_columns), - base_query=base_query + all_cols=", ".join(all_cols), base_query=base_query ) @@ -1368,8 +1377,8 @@ def powa_get_bgwriter_sample(): base_query = BASE_QUERY_BGWRITER_SAMPLE base_columns = ["srvid"] - biggest = Biggest(base_columns, 'ts') - biggestsum = Biggestsum(base_columns, 'ts') + biggest = Biggest(base_columns, "ts") + biggestsum = Biggestsum(base_columns, "ts") all_cols = base_columns + [ "ts", @@ -1384,9 +1393,9 @@ def powa_get_bgwriter_sample(): return """SELECT {all_cols} FROM {base_query} GROUP BY {base_columns}, ts""".format( - all_cols=', '.join(all_cols), - base_columns=', '.join(base_columns), - base_query=base_query + all_cols=", ".join(all_cols), + base_columns=", ".join(base_columns), + base_query=base_query, ) @@ -1394,8 +1403,8 @@ def powa_get_checkpointer_sample(): base_query = BASE_QUERY_CHECKPOINTER_SAMPLE base_columns = ["srvid"] - biggest = Biggest(base_columns, 'ts') - biggestsum = Biggestsum(base_columns, 'ts') + biggest = Biggest(base_columns, "ts") + biggestsum = Biggestsum(base_columns, "ts") all_cols = base_columns + [ "ts", @@ -1410,9 +1419,9 @@ def powa_get_checkpointer_sample(): return """SELECT {all_cols} FROM {base_query} GROUP BY {base_columns}, ts""".format( - all_cols=', '.join(all_cols), - base_columns=', '.join(base_columns), - base_query=base_query + all_cols=", ".join(all_cols), + base_columns=", ".join(base_columns), + base_query=base_query, ) @@ -1420,76 +1429,73 @@ def powa_get_database_sample(per_db=False): base_query = BASE_QUERY_DATABASE_SAMPLE(per_db) base_columns = ["srvid"] - biggest = Biggest(base_columns, 'ts') + biggest = Biggest(base_columns, "ts") all_cols = base_columns + [ - "ts", - biggest("ts", "'0 s'", "mesure_interval"), - biggest("numbackends"), - biggest("xact_commit"), - biggest("xact_rollback"), - biggest("blks_read"), - biggest("blks_hit"), - biggest("tup_returned"), - biggest("tup_fetched"), - biggest("tup_inserted"), - biggest("tup_updated"), - biggest("tup_deleted"), - biggest("conflicts"), - biggest("temp_files"), - biggest("temp_bytes"), - biggest("deadlocks"), - biggest("checksum_failures"), - "checksum_last_failure", - biggest("blk_read_time"), - biggest("blk_write_time"), - biggest("session_time"), - biggest("active_time"), - biggest("idle_in_transaction_time"), - biggest("sessions"), - biggest("sessions_abandoned"), - biggest("sessions_fatal"), - biggest("sessions_killed"), - "stats_reset", - ] + "ts", + biggest("ts", "'0 s'", "mesure_interval"), + biggest("numbackends"), + biggest("xact_commit"), + biggest("xact_rollback"), + biggest("blks_read"), + biggest("blks_hit"), + biggest("tup_returned"), + biggest("tup_fetched"), + biggest("tup_inserted"), + biggest("tup_updated"), + biggest("tup_deleted"), + biggest("conflicts"), + biggest("temp_files"), + biggest("temp_bytes"), + biggest("deadlocks"), + biggest("checksum_failures"), + "checksum_last_failure", + biggest("blk_read_time"), + biggest("blk_write_time"), + biggest("session_time"), + biggest("active_time"), + biggest("idle_in_transaction_time"), + biggest("sessions"), + biggest("sessions_abandoned"), + biggest("sessions_fatal"), + biggest("sessions_killed"), + "stats_reset", + ] return """SELECT {all_cols} FROM {base_query}""".format( - all_cols=', '.join(all_cols), - base_query=base_query - ) + all_cols=", ".join(all_cols), base_query=base_query + ) def powa_get_database_conflicts_sample(per_db=False): base_query = BASE_QUERY_DATABASE_CONFLICTS_SAMPLE(per_db) base_columns = ["srvid"] - biggest = Biggest(base_columns, 'ts') + biggest = Biggest(base_columns, "ts") all_cols = base_columns + [ - "ts", - biggest("ts", "'0 s'", "mesure_interval"), - biggest("confl_tablespace"), - biggest("confl_lock"), - biggest("confl_snapshot"), - biggest("confl_bufferpin"), - biggest("confl_deadlock"), - biggest("confl_active_logicalslot"), - ] + "ts", + biggest("ts", "'0 s'", "mesure_interval"), + biggest("confl_tablespace"), + biggest("confl_lock"), + biggest("confl_snapshot"), + biggest("confl_bufferpin"), + biggest("confl_deadlock"), + biggest("confl_active_logicalslot"), + ] return """SELECT {all_cols} FROM {base_query}""".format( - all_cols=', '.join(all_cols), - base_query=base_query - ) + all_cols=", ".join(all_cols), base_query=base_query + ) def powa_get_replication_sample(): base_query = BASE_QUERY_REPLICATION_SAMPLE base_columns = ["srvid"] - biggest = Biggest(base_columns, 'ts') - biggestsum = Biggestsum(base_columns, 'ts') + biggest = Biggest(base_columns, "ts") all_cols = base_columns + [ "ts", @@ -1513,9 +1519,7 @@ def powa_get_replication_sample(): return """SELECT {all_cols} FROM {base_query}""".format( - all_cols=', '.join(all_cols), - base_columns=', '.join(base_columns), - base_query=base_query + all_cols=", ".join(all_cols), base_query=base_query ) @@ -1523,13 +1527,12 @@ def powa_get_io_sample(qual=None): base_query = BASE_QUERY_IO_SAMPLE base_columns = ["srvid", "backend_type", "object", "context"] - biggest = Biggest(base_columns, 'ts') - biggestsum = Biggestsum(base_columns, 'ts') + biggest = Biggest(base_columns, "ts") - if (qual is not None): - qual = ' AND %s' % qual + if qual is not None: + qual = " AND %s" % qual else: - qual = '' + qual = "" all_cols = base_columns + [ "ts", @@ -1552,9 +1555,7 @@ def powa_get_io_sample(qual=None): return """SELECT {all_cols} FROM {base_query} {qual}""".format( - all_cols=', '.join(all_cols), - base_query=base_query, - qual=qual + all_cols=", ".join(all_cols), base_query=base_query, qual=qual ) @@ -1562,13 +1563,12 @@ def powa_get_slru_sample(qual=None): base_query = BASE_QUERY_SLRU_SAMPLE base_columns = ["srvid", "name"] - biggest =Biggest(base_columns, 'ts') - biggestsum = Biggestsum(base_columns, 'ts') + biggest = Biggest(base_columns, "ts") - if (qual is not None): - qual = ' AND %s' % qual + if qual is not None: + qual = " AND %s" % qual else: - qual = '' + qual = "" all_cols = base_columns + [ "ts", @@ -1585,7 +1585,7 @@ def powa_get_slru_sample(qual=None): return """SELECT {all_cols} FROM {base_query} {qual}""".format( - all_cols=', '.join(all_cols), + all_cols=", ".join(all_cols), base_query=base_query, qual=qual, ) @@ -1595,8 +1595,7 @@ def powa_get_subscription_sample(subname=None): base_query = BASE_QUERY_SUBSCRIPTION_SAMPLE(subname) base_columns = ["srvid", "subname"] - biggest =Biggest(base_columns, 'ts') - biggestsum = Biggestsum(base_columns, 'ts') + biggest = Biggest(base_columns, "ts") all_cols = base_columns + [ "ts", @@ -1609,7 +1608,7 @@ def powa_get_subscription_sample(subname=None): return """SELECT {all_cols} FROM {base_query}""".format( - all_cols=', '.join(all_cols), + all_cols=", ".join(all_cols), base_query=base_query, ) @@ -1618,8 +1617,8 @@ def powa_get_wal_sample(): base_query = BASE_QUERY_WAL_SAMPLE base_columns = ["srvid"] - biggest = Biggest(base_columns, 'ts') - biggestsum = Biggestsum(base_columns, 'ts') + biggest = Biggest(base_columns, "ts") + biggestsum = Biggestsum(base_columns, "ts") all_cols = base_columns + [ "ts", @@ -1637,9 +1636,9 @@ def powa_get_wal_sample(): return """SELECT {all_cols} FROM {base_query} GROUP BY {base_columns}, ts""".format( - all_cols=', '.join(all_cols), - base_columns=', '.join(base_columns), - base_query=base_query + all_cols=", ".join(all_cols), + base_columns=", ".join(base_columns), + base_query=base_query, ) @@ -1647,7 +1646,7 @@ def powa_get_wal_receiver_sample(): base_query = BASE_QUERY_WAL_RECEIVER_SAMPLE base_columns = ["srvid", "slot_name", "sender_host", "sender_port"] - biggest = Biggest(base_columns, 'ts') + biggest = Biggest(base_columns, "ts") all_cols = base_columns + [ "ts", @@ -1673,14 +1672,11 @@ def powa_get_wal_receiver_sample(): return """SELECT {all_cols} FROM {base_query}""".format( - all_cols=', '.join(all_cols), - base_columns=', '.join(base_columns), - base_query=base_query + all_cols=", ".join(all_cols), base_query=base_query ) def powa_get_all_idx_sample(mode): - if mode == "db": base_query = BASE_QUERY_ALL_IDXS_SAMPLE_DB base_columns = ["srvid", "dbid", "datname"] @@ -1688,8 +1684,8 @@ def powa_get_all_idx_sample(mode): base_query = BASE_QUERY_ALL_IDXS_SAMPLE base_columns = ["srvid", "dbid", "datname", "relid"] - biggest = Biggest(base_columns, 'ts') - biggestsum = Biggestsum(base_columns, 'ts') + biggest = Biggest(base_columns, "ts") + biggestsum = Biggestsum(base_columns, "ts") all_cols = base_columns + [ "ts", @@ -1706,14 +1702,13 @@ def powa_get_all_idx_sample(mode): FROM {base_query} WHERE srvid = %(server)s GROUP BY {base_columns}, ts""".format( - all_cols=', '.join(all_cols), - base_columns=', '.join(base_columns), - base_query=base_query + all_cols=", ".join(all_cols), + base_columns=", ".join(base_columns), + base_query=base_query, ) def powa_get_all_tbl_sample(mode): - if mode == "db": base_query = BASE_QUERY_ALL_TBLS_SAMPLE_DB base_columns = ["srvid", "dbid", "datname"] @@ -1721,8 +1716,8 @@ def powa_get_all_tbl_sample(mode): base_query = BASE_QUERY_ALL_TBLS_SAMPLE base_columns = ["srvid", "dbid", "datname", "relid"] - biggest = Biggest(base_columns, 'ts') - biggestsum = Biggestsum(base_columns, 'ts') + biggest = Biggest(base_columns, "ts") + biggestsum = Biggestsum(base_columns, "ts") all_cols = base_columns + [ "ts", @@ -1744,14 +1739,13 @@ def powa_get_all_tbl_sample(mode): FROM {base_query} WHERE srvid = %(server)s GROUP BY {base_columns}, ts""".format( - all_cols=', '.join(all_cols), - base_columns=', '.join(base_columns), - base_query=base_query + all_cols=", ".join(all_cols), + base_columns=", ".join(base_columns), + base_query=base_query, ) def powa_get_user_fct_sample(mode): - if mode == "db": base_query = BASE_QUERY_USER_FCTS_SAMPLE_DB base_columns = ["srvid", "dbid", "datname"] @@ -1759,8 +1753,8 @@ def powa_get_user_fct_sample(mode): base_query = BASE_QUERY_USER_FCTS_SAMPLE base_columns = ["srvid", "dbid", "datname", "funcid"] - biggest = Biggest(base_columns, 'ts') - biggestsum = Biggestsum(base_columns, 'ts') + biggest = Biggest(base_columns, "ts") + biggestsum = Biggestsum(base_columns, "ts") all_cols = base_columns + [ "ts", @@ -1774,7 +1768,7 @@ def powa_get_user_fct_sample(mode): FROM {base_query} WHERE srvid = %(server)s GROUP BY {base_columns}, ts""".format( - all_cols=', '.join(all_cols), - base_columns=', '.join(base_columns), - base_query=base_query + all_cols=", ".join(all_cols), + base_columns=", ".join(base_columns), + base_query=base_query, ) diff --git a/powa/sql/views_grid.py b/powa/sql/views_grid.py index 7b4cb4f1..36aea6ba 100644 --- a/powa/sql/views_grid.py +++ b/powa/sql/views_grid.py @@ -1,6 +1,7 @@ """ Functions to generate the queries used in the various grid components """ + from powa.sql.utils import block_size, diff, diffblk @@ -316,10 +317,10 @@ def powa_getiodata(qual=None): """ base_query = powa_base_io() - if (qual is not None): - qual = ' WHERE %s' % qual + if qual is not None: + qual = " WHERE %s" % qual else: - qual = '' + qual = "" base_cols = [ "srvid", @@ -348,8 +349,8 @@ def powa_getiodata(qual=None): FROM {base_query} {qual} GROUP BY srvid, {base_cols}, op_bytes""".format( - cols=', '.join(cols), - base_cols=', '.join(base_cols), + cols=", ".join(cols), + base_cols=", ".join(base_cols), base_query=base_query, qual=qual, ) @@ -432,10 +433,10 @@ def powa_getslrudata(qual=None): """ base_query = powa_base_slru() - if (qual is not None): - qual = ' WHERE %s' % qual + if qual is not None: + qual = " WHERE %s" % qual else: - qual = '' + qual = "" base_cols = [ "srvid", @@ -457,8 +458,8 @@ def powa_getslrudata(qual=None): CROSS JOIN {block_size} {qual} GROUP BY {base_cols}, block_size""".format( - cols=', '.join(cols), - base_cols=', '.join(base_cols), + cols=", ".join(cols), + base_cols=", ".join(base_cols), base_query=base_query, block_size=block_size, qual=qual, @@ -510,9 +511,9 @@ def powa_getstatdata_detailed_db(srvid="%(server)s", predicates=[]): base_query = powa_base_statdata_detailed_db() diffs = get_diffs_forstatdata() - where = ' AND '.join(predicates) - if where != '': - where = ' AND ' + where + where = " AND ".join(predicates) + if where != "": + where = " AND " + where cols = [ "srvid", @@ -529,10 +530,7 @@ def powa_getstatdata_detailed_db(srvid="%(server)s", predicates=[]): {where} GROUP BY srvid, queryid, dbid, toplevel, userid, datname HAVING max(calls) - min(calls) > 0""".format( - cols=', '.join(cols), - base_query=base_query, - srvid=srvid, - where=where + cols=", ".join(cols), base_query=base_query, srvid=srvid, where=where ) @@ -546,19 +544,14 @@ def powa_getstatdata_db(srvid): base_query = powa_base_statdata_db() diffs = get_diffs_forstatdata() - cols = [ - "srvid", - "dbid" - ] + diffs + cols = ["srvid", "dbid"] + diffs return """SELECT {cols} FROM {base_query} WHERE srvid = {srvid} GROUP BY srvid, dbid HAVING max(calls) - min(calls) > 0""".format( - cols=', '.join(cols), - base_query=base_query, - srvid=srvid + cols=", ".join(cols), base_query=base_query, srvid=srvid ) @@ -771,8 +764,7 @@ def powa_getwaitdata_detailed_db(): FROM {base_query} GROUP BY srvid, queryid, dbid, datname, event_type, event HAVING max(count) - min(count) > 0""".format( - count=diff('count'), - base_query=base_query + count=diff("count"), base_query=base_query ) @@ -789,8 +781,7 @@ def powa_getwaitdata_db(): FROM {base_query} GROUP BY srvid, dbid, event_type, event HAVING max(count) - min(count) > 0""".format( - count=diff('count'), - base_query=base_query + count=diff("count"), base_query=base_query ) @@ -970,37 +961,37 @@ def powa_getuserfuncdata_detailed_db(funcid=None): base_query = powa_base_userfuncdata_detailed_db() cols = [ - "d.srvid", - "h.dbid", - "d.datname", - "funcid", - "coalesce(regprocedure, '<' || funcid || '>') AS func_name", - "lanname", - diff("calls"), - diff("total_time"), - diff("self_time"), - ] - if (funcid): + "d.srvid", + "h.dbid", + "d.datname", + "funcid", + "coalesce(regprocedure, '<' || funcid || '>') AS func_name", + "lanname", + diff("calls"), + diff("total_time"), + diff("self_time"), + ] + if funcid: cols.extend(["prosrc", "last_refresh"]) groupby = [ - "d.srvid", - "h.dbid", - "d.datname", - "funcid", - "regprocedure", - "lanname" - ] - if (funcid): + "d.srvid", + "h.dbid", + "d.datname", + "funcid", + "regprocedure", + "lanname", + ] + if funcid: groupby.extend(["prosrc", "last_refresh"]) - if (funcid): + if funcid: join_db = """LEFT JOIN {powa}.powa_catalog_databases pcd ON pcd.srvid = d.srvid AND pcd.oid = h.dbid""" and_funcid = "AND funcid = {funcid}".format(funcid=funcid) else: - join_db = '' - and_funcid = '' + join_db = "" + and_funcid = "" return """SELECT {cols} FROM {base_query} @@ -1009,11 +1000,11 @@ def powa_getuserfuncdata_detailed_db(funcid=None): {and_funcid} GROUP BY {groupby} HAVING max(calls) - min(calls) > 0""".format( - cols=', '.join(cols), + cols=", ".join(cols), base_query=base_query, join_db=join_db, and_funcid=and_funcid, - groupby=', '.join(groupby) + groupby=", ".join(groupby), ) @@ -1027,18 +1018,17 @@ def powa_getuserfuncdata_db(): base_query = powa_base_userfuncdata_db() cols = [ - "d.srvid", - "d.datname", - "h.dbid", - diff("calls"), - diff("total_time"), - diff("self_time"), - ] + "d.srvid", + "d.datname", + "h.dbid", + diff("calls"), + diff("total_time"), + diff("self_time"), + ] return """SELECT {cols} FROM {base_query} GROUP BY d.srvid, d.datname, h.dbid, d.oid HAVING max(calls) - min(calls) > 0""".format( - cols=', '.join(cols), - base_query=base_query + cols=", ".join(cols), base_query=base_query ) diff --git a/powa/ui_methods.py b/powa/ui_methods.py index a756fefa..1054f703 100644 --- a/powa/ui_methods.py +++ b/powa/ui_methods.py @@ -1,9 +1,9 @@ """ Set of helper functions available from the templates. """ -import os -import json +import json +import os from datetime import datetime from powa import __VERSION__ from powa.json import JSONEncoder @@ -40,19 +40,23 @@ def field(_, **kwargs): Returns: a form field formatted with the given attributes. """ - kwargs.setdefault('tag', 'input') - kwargs.setdefault('type', 'text') - kwargs.setdefault('class', 'form-control') - attrs = " ".join('%s="%s"' % (key, value) for key, value in kwargs.items() - if key not in ('tag', 'label')) - kwargs['attrs'] = attrs + kwargs.setdefault("tag", "input") + kwargs.setdefault("type", "text") + kwargs.setdefault("class", "form-control") + attrs = " ".join( + '%s="%s"' % (key, value) + for key, value in kwargs.items() + if key not in ("tag", "label") + ) + kwargs["attrs"] = attrs def render(content): """ Render the field itself. """ - kwargs['content'] = content.decode('utf8') - return """ + kwargs["content"] = content.decode("utf8") + return ( + """ -""" % kwargs +""" + % kwargs + ) return render @@ -89,7 +95,7 @@ def flashed_messages(self): def sanitycheck_messages(self): - messages = {'error': []} + messages = {"error": []} # Check if now collector is running sql = """SELECT @@ -109,7 +115,7 @@ def sanitycheck_messages(self): JOIN pg_stat_activity a ON a.application_name LIKE n.val""" rows = self.execute(sql) - if (rows is None): + if rows is None: messages["error"].append("No collector is running!") sql = """SELECT @@ -126,9 +132,9 @@ def sanitycheck_messages(self): ) m ON m.srvid = s.id""" rows = self.execute(sql) - if (rows is not None and len(rows) > 0): + if rows is not None and len(rows) > 0: for r in rows: - messages["error"].append("%s: %s" % (r['alias'], r['error'])) + messages["error"].append("%s: %s" % (r["alias"], r["error"])) return messages return {} @@ -140,12 +146,15 @@ def to_json(_, value): """ return JSONEncoder().encode(value) + def manifest(self, entrypoint): fn = os.path.realpath(__file__ + "../../static/dist/.vite/manifest.json") try: f = open(fn) except FileNotFoundError: - raise HTTPError(500, "manifest.json doesn't exist, did you run `npm run build`?") + raise HTTPError( + 500, "manifest.json doesn't exist, did you run `npm run build`?" + ) else: with f: entrypoints = json.load(f) diff --git a/powa/ui_modules.py b/powa/ui_modules.py index 23058a5b..38e7c7fd 100644 --- a/powa/ui_modules.py +++ b/powa/ui_modules.py @@ -1,9 +1,12 @@ -from tornado.web import UIModule - - class MenuEntry(object): - - def __init__(self, title, url_name, url_params=None, children_title=None, children=None): + def __init__( + self, + title, + url_name, + url_params=None, + children_title=None, + children=None, + ): self.title = title self.url_name = url_name self.url_params = url_params or {} diff --git a/powa/user.py b/powa/user.py index 1941d031..b018ebcc 100644 --- a/powa/user.py +++ b/powa/user.py @@ -1,19 +1,19 @@ from __future__ import unicode_literals -from tornado.options import options -from powa.framework import BaseHandler + from powa import __VERSION_NUM__ +from powa.framework import BaseHandler +from tornado.options import options class LoginHandler(BaseHandler): - def get(self): self._status_code = 403 return self.render("login.html", title="Login") def post(self, *args, **kwargs): - user = self.get_argument('user') - password = self.get_argument('password') - server = self.get_argument('server') + user = self.get_argument("user") + password = self.get_argument("password") + server = self.get_argument("server") expires_days = options.cookie_expires_days if expires_days == 0: expires_days = None @@ -22,33 +22,38 @@ def post(self, *args, **kwargs): self.connect(user=user, password=password, server=server) except Exception as e: self.flash("Auth failed", "alert") - self.logger.error('Error: %r', e) + self.logger.error("Error: %r", e) self.get() return # Check that the database is correctly installed - version = self.get_powa_version(user=user, - password=password, - server=server) + version = self.get_powa_version( + user=user, password=password, server=server + ) if version is None: - self.flash('PoWA is not installed on your target database. ' - 'You should check your installation.', 'alert') + self.flash( + "PoWA is not installed on your target database. " + "You should check your installation.", + "alert", + ) self.redirect(self.url_prefix) # Major.Minor version should be the same if version[0:2] != __VERSION_NUM__[0:2]: self.flash( - 'Unable to connect: powa-archivist version %s.X does not match powa-web version %s.X' % - ('.'.join(str(x) for x in version[0:2]), - '.'.join(str(x) for x in __VERSION_NUM__[0:2])), - 'alert') + "Unable to connect: powa-archivist version %s.X does not match powa-web version %s.X" + % ( + ".".join(str(x) for x in version[0:2]), + ".".join(str(x) for x in __VERSION_NUM__[0:2]), + ), + "alert", + ) self.redirect(self.url_prefix) - self.set_secure_cookie('user', user, expires_days=expires_days) - self.set_secure_cookie('password', password, expires_days=expires_days) - self.set_secure_cookie('server', server, expires_days=expires_days) - self.redirect(self.get_argument('next', self.url_prefix)) + self.set_secure_cookie("user", user, expires_days=expires_days) + self.set_secure_cookie("password", password, expires_days=expires_days) + self.set_secure_cookie("server", server, expires_days=expires_days) + self.redirect(self.get_argument("next", self.url_prefix)) class LogoutHandler(BaseHandler): - def get(self): self.clear_all_cookies() return self.redirect(self.url_prefix) diff --git a/powa/wizard.py b/powa/wizard.py index b83aa3f4..91eed724 100644 --- a/powa/wizard.py +++ b/powa/wizard.py @@ -1,14 +1,18 @@ """ Global optimization widget """ + from __future__ import absolute_import -from powa.framework import AuthHandler -from powa.dashboards import ( - Widget, MetricGroupDef) -from powa.sql import (resolve_quals, get_any_sample_query, - get_hypoplans, HypoIndex) import json +from powa.dashboards import MetricGroupDef, Widget +from powa.framework import AuthHandler +from powa.sql import ( + HypoIndex, + get_any_sample_query, + get_hypoplans, + resolve_quals, +) from powa.sql.views import qualstat_getstatdata from psycopg2 import Error from psycopg2.extras import RealDictCursor @@ -16,41 +20,44 @@ class IndexSuggestionHandler(AuthHandler): - def post(self, srvid, database): try: # Check remote access first - remote_conn = self.connect(srvid, database=database, - remote_access=True) + remote_conn = self.connect( + srvid, database=database, remote_access=True + ) remote_cur = remote_conn.cursor() except Exception as e: - raise HTTPError(501, "Could not connect to remote server: %s" % - str(e)) + raise HTTPError( + 501, "Could not connect to remote server: %s" % str(e) + ) payload = json.loads(self.request.body.decode("utf8")) - from_date = payload['from_date'] - to_date = payload['to_date'] + from_date = payload["from_date"] + to_date = payload["to_date"] indexes = [] - for ind in payload['indexes']: - hypoind = HypoIndex(ind['nspname'], - ind['relname'], - ind['ams']) - hypoind._ddl = ind['ddl'] + for ind in payload["indexes"]: + hypoind = HypoIndex(ind["nspname"], ind["relname"], ind["ams"]) + hypoind._ddl = ind["ddl"] indexes.append(hypoind) - queryids = payload['queryids'] + queryids = payload["queryids"] powa_conn = self.connect(database="powa") cur = powa_conn.cursor(cursor_factory=RealDictCursor) - cur.execute(""" + cur.execute( + """ SELECT DISTINCT query, ps.queryid FROM {powa}.powa_statements ps WHERE srvid = %(srvid)s AND queryid IN %(queryids)s - """, ({'srvid': srvid, 'queryids': tuple(queryids)})) + """, + ({"srvid": srvid, "queryids": tuple(queryids)}), + ) queries = cur.fetchall() cur.close() # Create all possible indexes for this qual - hypo_version = self.has_extension_version(srvid, "hypopg", "0.0.3", - database=database) + hypo_version = self.has_extension_version( + srvid, "hypopg", "0.0.3", database=database + ) hypoplans = {} indbyname = {} inderrors = {} @@ -59,9 +66,12 @@ def post(self, srvid, database): # create them for ind in indexes: try: - remote_cur.execute("""SELECT * + remote_cur.execute( + """SELECT * FROM hypopg_create_index(%(ddl)s) - """, {'ddl': ind.ddl}) + """, + {"ddl": ind.ddl}, + ) indname = remote_cur.fetchone()[1] indbyname[indname] = ind except Error as e: @@ -72,14 +82,14 @@ def post(self, srvid, database): continue # Build the query and fetch the plans for row in queries: - querystr = get_any_sample_query(self, srvid, database, - row['queryid'], - from_date, - to_date) + querystr = get_any_sample_query( + self, srvid, database, row["queryid"], from_date, to_date + ) if querystr: try: - hypoplans[row['queryid']] = get_hypoplans( - remote_cur, querystr, indbyname.values()) + hypoplans[row["queryid"]] = get_hypoplans( + remote_cur, querystr, indbyname.values() + ) except Exception: # TODO: stop ignoring the error continue @@ -93,6 +103,7 @@ def post(self, srvid, database): class WizardMetricGroup(MetricGroupDef): """Metric group for the wizard.""" + name = "wizard" xaxis = "quals" axis_type = "category" @@ -101,7 +112,7 @@ class WizardMetricGroup(MetricGroupDef): @property def query(self): - pq = qualstat_getstatdata(eval_type='f') + pq = qualstat_getstatdata(eval_type="f") cols = [ # queryid in pg11+ is int64, so the value can exceed javascript's @@ -116,7 +127,7 @@ def query(self): "execution_count", "array_agg(query) AS queries", "avg_filter", - "filter_ratio" + "filter_ratio", ] query = """SELECT {cols} @@ -133,7 +144,7 @@ def query(self): sub.quals::jsonb, sub.avg_filter, sub.filter_ratio ORDER BY sub.occurences DESC LIMIT 200""".format( - cols=', '.join(cols), + cols=", ".join(cols), pq=pq, ) return query @@ -145,36 +156,36 @@ def post_process(self, data, server, database, **kwargs): class Wizard(Widget): - def __init__(self, title): self.title = title def parameterized_json(self, handler, **parms): values = self.__dict__.copy() - values['metrics'] = [] - values['type'] = 'wizard' - values['datasource'] = 'wizard' + values["metrics"] = [] + values["type"] = "wizard" + values["datasource"] = "wizard" # First check that we can connect on the remote server, otherwise we # won't be able to do anything try: - remote_conn = handler.connect(parms["server"], - database=parms["database"], - remote_access=True) + handler.connect( + parms["server"], database=parms["database"], remote_access=True + ) except Exception as e: - values['has_remote_conn'] = False - values['conn_error'] = str(e) + values["has_remote_conn"] = False + values["conn_error"] = str(e) return values - values['has_remote_conn'] = True - - hypover = handler.has_extension_version(parms["server"], - "hypopg", "0.0.3", - database=parms["database"]) - qsver = handler.has_extension_version(parms["server"], "pg_qualstats", - "0.0.7") - values['has_hypopg'] = hypover - values['has_qualstats'] = qsver - values['server'] = parms["server"] - values['database'] = parms["database"] + values["has_remote_conn"] = True + + hypover = handler.has_extension_version( + parms["server"], "hypopg", "0.0.3", database=parms["database"] + ) + qsver = handler.has_extension_version( + parms["server"], "pg_qualstats", "0.0.7" + ) + values["has_hypopg"] = hypover + values["has_qualstats"] = qsver + values["server"] = parms["server"] + values["database"] = parms["database"] return values diff --git a/requirements-dev.txt b/requirements-dev.txt new file mode 100644 index 00000000..25d0388c --- /dev/null +++ b/requirements-dev.txt @@ -0,0 +1,2 @@ +-r requirements.txt +ruff --only-binary=ruff diff --git a/run_powa.py b/run_powa.py index 1f8017e6..7a1d0e17 100755 --- a/run_powa.py +++ b/run_powa.py @@ -1,13 +1,14 @@ #!/usr/bin/env python -from powa import make_app +import logging import tornado.ioloop +from powa import make_app from tornado.options import options -import logging if __name__ == "__main__": application = make_app(debug=True, gzip=True, compress_response=True) application.listen(8888) logger = logging.getLogger("tornado.application") - logger.info("Starting powa-web on http://127.0.0.1:8888%s", - options.url_prefix) + logger.info( + "Starting powa-web on http://127.0.0.1:8888%s", options.url_prefix + ) tornado.ioloop.IOLoop.instance().start() diff --git a/setup.py b/setup.py index 2206404b..1656b7d8 100755 --- a/setup.py +++ b/setup.py @@ -1,36 +1,33 @@ -from setuptools import setup, find_packages import sys +from setuptools import find_packages, setup __VERSION__ = None -with open('powa/__init__.py') as f: +with open("powa/__init__.py") as f: for line in f: - if line.startswith('__VERSION__'): - __VERSION__ = line.split('=')[1].replace("'", '').strip() + if line.startswith("__VERSION__"): + __VERSION__ = line.split("=")[1].replace("'", "").strip() -requires = [ - 'tornado>=2.0', - 'psycopg2' -] +requires = ["tornado>=2.0", "psycopg2"] # include ordereddict for python2.6 if sys.version_info < (2, 7, 0): - requires.append('ordereddict') + requires.append("ordereddict") setup( - name='powa-web', + name="powa-web", version=__VERSION__, - author='powa-team', - license='Postgresql', + author="powa-team", + license="Postgresql", packages=find_packages(), install_requires=requires, include_package_data=True, url="https://powa.readthedocs.io/", description="A User Interface for the PoWA project", long_description="See https://powa.readthedocs.io/", - scripts=['powa-web'], + scripts=["powa-web"], classifiers=[ "Development Status :: 5 - Production/Stable", "Intended Audience :: System Administrators", @@ -40,5 +37,6 @@ "Operating System :: OS Independent", "Programming Language :: Python :: 2", "Programming Language :: Python :: 3", - "Topic :: Database :: Front-Ends"] + "Topic :: Database :: Front-Ends", + ], )