diff --git a/.github/workflows/python_lint.yml b/.github/workflows/python_lint.yml index 8103e7e8..4c962daf 100644 --- a/.github/workflows/python_lint.yml +++ b/.github/workflows/python_lint.yml @@ -8,7 +8,7 @@ jobs: runs-on: ubuntu-latest strategy: matrix: - python-version: ["3.12"] + python-version: ["3.8", "3.12"] steps: - name: Checkout repository uses: actions/checkout@v4 @@ -31,3 +31,29 @@ jobs: - name: Check manifest run: | check-manifest + + python36: + # This job will detect InvalidSyntax error on Python 3.6. + # A success doesn't necessarily mean that PoWA works. + runs-on: ubuntu-20.04 + steps: + - name: Checkout repository + uses: actions/checkout@v4 + + - name: Set Python Version + uses: actions/setup-python@v5 + with: + python-version: 3.6 + + - name: Install dependencies + run: | + python -m pip install --upgrade pip + pip install -r requirements.txt + + - name: Configure PoWA + run: | + cp powa-web.conf-dist powa-web.conf + + - name: Check for syntax errors + run: | + timeout 1 python run_powa.py || code=$?; if [[ $code -ne 124 && $code -ne 0 ]]; then exit $code; fi diff --git a/.ruff.toml b/.ruff.toml index 60912519..fa871987 100644 --- a/.ruff.toml +++ b/.ruff.toml @@ -9,7 +9,8 @@ extend-include = ["powa-web", "powa/powa.wsgi"] # In addition, we also enable: # `Q` for "flake8-quotes" # `I` for "isort" -extend-select = ["Q", "I"] +# `UP` for "pyupgrade" +extend-select = ["Q", "I", "UP"] [lint.per-file-ignores] "powa/__init__.py" = ["E402"] diff --git a/powa/__init__.py b/powa/__init__.py index 17ff427a..0824d2a1 100644 --- a/powa/__init__.py +++ b/powa/__init__.py @@ -1,5 +1,3 @@ -from __future__ import print_function - """ Powa main application. """ @@ -63,36 +61,36 @@ def make_app(**kwargs): parse_options() URLS = [ - U(r"%slogin/" % options.url_prefix, LoginHandler, name="login"), - U(r"%slogout/" % options.url_prefix, LogoutHandler, name="logout"), + U(rf"{options.url_prefix}login/", LoginHandler, name="login"), + U(rf"{options.url_prefix}logout/", LogoutHandler, name="logout"), U( - r"%sreload_collector/" % options.url_prefix, + rf"{options.url_prefix}reload_collector/", CollectorReloadHandler, name="reload_collector", ), U( - r"%sforce_snapshot/(\d+)" % options.url_prefix, + rf"{options.url_prefix}force_snapshot/(\d+)", CollectorForceSnapshotHandler, name="force_snapshot", ), U( - r"%srefresh_db_cat/" % options.url_prefix, + rf"{options.url_prefix}refresh_db_cat/", CollectorDbCatRefreshHandler, name="refresh_db_cat", ), U( - r"%sserver/select" % options.url_prefix, + rf"{options.url_prefix}server/select", ServerSelector, name="server_selector", ), U( - r"%sdatabase/select" % options.url_prefix, + rf"{options.url_prefix}database/select", DatabaseSelector, name="database_selector", ), - U(r"%s" % options.url_prefix, IndexHandler, name="index"), + U(rf"{options.url_prefix}", IndexHandler, name="index"), U( - r"%sserver/(\d+)/database/([^\/]+)/suggest/" % options.url_prefix, + rf"{options.url_prefix}server/(\d+)/database/([^\/]+)/suggest/", IndexSuggestionHandler, name="index_suggestion", ), @@ -124,9 +122,9 @@ def make_app(**kwargs): URLS, ui_modules=ui_modules, ui_methods=ui_methods, - login_url=("%slogin/" % options.url_prefix), + login_url=(f"{options.url_prefix}login/"), static_path=os.path.join(POWA_ROOT, "static"), - static_url_prefix=("%sstatic/" % options.url_prefix), + static_url_prefix=(f"{options.url_prefix}static/"), cookie_secret=options.cookie_secret, template_path=os.path.join(POWA_ROOT, "templates"), **kwargs, diff --git a/powa/collector.py b/powa/collector.py index 54858a64..84f56462 100644 --- a/powa/collector.py +++ b/powa/collector.py @@ -3,8 +3,6 @@ collector handling. """ -from __future__ import absolute_import - import json from powa.dashboards import MetricGroupDef from powa.framework import AuthHandler diff --git a/powa/compat.py b/powa/compat.py index f161f099..6e3c9ba7 100644 --- a/powa/compat.py +++ b/powa/compat.py @@ -6,8 +6,6 @@ """ -from __future__ import absolute_import - import json import psycopg2 from psycopg2 import extensions @@ -37,7 +35,7 @@ def __new__(cls, name, _, d): return type.__new__(metaclass, "temporary_class", (), {}) -class classproperty(object): +class classproperty: """ A descriptor similar to property, but using the class. """ diff --git a/powa/config.py b/powa/config.py index 55be906b..e058aeb1 100644 --- a/powa/config.py +++ b/powa/config.py @@ -2,8 +2,6 @@ Dashboard for the configuration summary page. """ -from __future__ import absolute_import - import json from powa.collector import CollectorServerDetail from powa.dashboards import ( @@ -418,7 +416,7 @@ def post_process(self, data, server, **kwargs): data["messages"] = { "alert": [ "Could not retrieve extensions" - + " on remote server: %s" % errmsg + + f" on remote server: {errmsg}" ] } return data @@ -448,8 +446,8 @@ def post_process(self, data, server, **kwargs): data["messages"] = { "alert": [ ( - "%d extensions need to be installed:%s" - % (len(alerts), " ".join(alerts)) + f"{len(alerts)} extensions need " + f"to be installed:{' '.join(alerts)}" ) ] } @@ -528,7 +526,7 @@ def post_process(self, data, server, **kwargs): data["messages"] = { "alert": [ "Could not retrieve extensions" - + " on remote server: %s" % errmsg + + f" on remote server: {errmsg}" ] } return data diff --git a/powa/dashboards.py b/powa/dashboards.py index 1a7e6c77..7d8fbe1e 100644 --- a/powa/dashboards.py +++ b/powa/dashboards.py @@ -275,7 +275,7 @@ def url_name(cls): """ Returns the default url_name for this data source. """ - return "datasource_%s" % cls.__name__ + return f"datasource_{cls.__name__}" @classproperty def parameterized_json(cls, handler, **parms): @@ -310,14 +310,14 @@ def bind(self, group): """ if self._group is not None: - raise ValueError("Already bound to %s" % self._group) + raise ValueError(f"Already bound to {self._group}") self._group = group def _fqn(self): """ Return the fully qualified name of this metric. """ - return "%s.%s" % (self._group.name, self.name) + return f"{self._group.name}.{self.name}" class Dashboard(JSONizable): @@ -345,7 +345,7 @@ def _validate_layout(self): if (12 % len(row)) != 0: raise ValueError( "Each widget row length must be a " - "divisor of 12 (have: %d)" % len(row) + f"divisor of 12 (have: {len(row)})" ) @property @@ -502,7 +502,7 @@ def _validate(self): if any(m._group != mg1 for m in self.metrics): raise ValueError( "A grid is not allowed to have metrics from different " - "groups. (title: %s)" % self.title + f"groups. (title: {self.title})" ) def to_json(self): @@ -559,7 +559,7 @@ def to_json(self): return values -class Declarative(object): +class Declarative: """ Base class for declarative classes. """ @@ -606,12 +606,12 @@ def __new__(meta, name, bases, dct): if isinstance(val, Metric): dct.pop(key) dct["metrics"][key] = val - return super(MetaMetricGroup, meta).__new__(meta, name, bases, dct) + return super().__new__(meta, name, bases, dct) def __init__(cls, name, bases, dct): for metric in dct.get("metrics").values(): metric.bind(cls) - super(MetaMetricGroup, cls).__init__(name, bases, dct) + super().__init__(name, bases, dct) def __getattr__(cls, key): if key not in cls.metrics: @@ -705,7 +705,7 @@ def metrics(self): return self._metrics -class DashboardPage(object): +class DashboardPage: """ A Dashboard page ties together a set of datasources, and a dashboard. @@ -745,7 +745,7 @@ def url_specs(cls, url_prefix): url_specs = [] url_specs.append( URLSpec( - r"%s%s/" % (url_prefix, cls.base_url.strip("/")), + r"{}{}/".format(url_prefix, cls.base_url.strip("/")), type(cls.__name__, (cls.dashboard_handler_cls, cls), {}), {"template": cls.template, "params": cls.params}, name=cls.__name__, @@ -754,12 +754,13 @@ def url_specs(cls, url_prefix): for datasource in cls.datasources: if datasource.data_url is None: raise KeyError( - "A Datasource must have a data_url: %s" - % datasource.__name__ + f"A Datasource must have a data_url: {datasource.__name__}" ) url_specs.append( URLSpec( - r"%s%s/" % (url_prefix, datasource.data_url.strip("/")), + r"{}{}/".format( + url_prefix, datasource.data_url.strip("/") + ), type( datasource.__name__, (datasource, datasource.datasource_handler_cls), diff --git a/powa/database.py b/powa/database.py index 368a7954..3d461f90 100644 --- a/powa/database.py +++ b/powa/database.py @@ -305,7 +305,7 @@ def query(self): from_clause = query if self.has_extension(self.path_args[0], "pg_stat_kcache"): - from_clause = "({query}) AS sub2".format(query=query) + from_clause = f"({query}) AS sub2" # Add system metrics from pg_stat_kcache, kcache_query = kcache_getstatdata_sample( @@ -344,10 +344,8 @@ def query(self): ] ) - from_clause += """ - LEFT JOIN ({kcache_query}) AS kc USING (dbid, ts, srvid)""".format( - kcache_query=kcache_query - ) + from_clause += f""" + LEFT JOIN ({kcache_query}) AS kc USING (dbid, ts, srvid)""" return """SELECT {cols} FROM ( @@ -505,7 +503,7 @@ def query(self): wps("count_io"), ] - from_clause = "({query}) AS sub".format(query=query) + from_clause = f"({query}) AS sub" return """SELECT {cols} FROM {from_clause} @@ -887,12 +885,12 @@ def query(self): ] ) - from_clause = """( + from_clause = f"""( {inner_query} ) AS sub JOIN {{powa}}.powa_statements AS ps USING (srvid, queryid, userid, dbid) - CROSS JOIN {bs}""".format(inner_query=inner_query, bs=block_size) + CROSS JOIN {block_size}""" return """SELECT {cols} FROM {from_clause} @@ -936,11 +934,11 @@ def query(self): "sum(count) AS counts", ] - from_clause = """( + from_clause = f"""( {inner_query} ) AS sub JOIN {{powa}}.powa_statements AS ps - USING (srvid, queryid, dbid)""".format(inner_query=inner_query) + USING (srvid, queryid, dbid)""" return """SELECT {cols} FROM {from_clause} diff --git a/powa/framework.py b/powa/framework.py index a2c5ade3..3dc99024 100644 --- a/powa/framework.py +++ b/powa/framework.py @@ -52,7 +52,7 @@ def cursor(self, *args, **kwargs): elif factory == RealDictCursor: kwargs["cursor_factory"] = CustomDictCursor else: - msg = "Unsupported cursor_factory: %s" % factory.__name__ + msg = f"Unsupported cursor_factory: {factory.__name__}" self._logger.error(msg) raise Exception(msg) @@ -65,7 +65,7 @@ def execute(self, query, params=None): self.timestamp = time.time() try: - return super(CustomDictCursor, self).execute(query, params) + return super().execute(query, params) except Exception as e: log_query(self, query, params, e) raise e @@ -79,7 +79,7 @@ def execute(self, query, params=None): self.timestamp = time.time() try: - return super(CustomCursor, self).execute(query, params) + return super().execute(query, params) except Exception as e: log_query(self, query, params, e) finally: @@ -98,7 +98,7 @@ def log_query(cls, query, params=None, exception=None): fmt = "" if exception is not None: - fmt = "Error during query execution:\n{}\n".format(exception) + fmt = f"Error during query execution:\n{exception}\n" fmt += "query on {dsn} (srvid {srvid}): {ms} ms\n{query}" if params is not None: @@ -122,7 +122,7 @@ class BaseHandler(RequestHandler, JSONizable): """ def __init__(self, *args, **kwargs): - super(BaseHandler, self).__init__(*args, **kwargs) + super().__init__(*args, **kwargs) self.flashed_messages = {} self._databases = None self._servers = None @@ -135,11 +135,7 @@ def __init__(self, *args, **kwargs): def __get_url(self, **connoptions): url = " ".join( - [ - "%s=%s" % (k, v) - for (k, v) in connoptions.items() - if v is not None - ] + [f"{k}={v}" for (k, v) in connoptions.items() if v is not None] ) return url @@ -195,7 +191,7 @@ def current_host(self): host = "localhost" if "host" in connoptions: host = connoptions["host"] - return "%s" % (host) + return f"{host}" @property def current_port(self): @@ -209,7 +205,7 @@ def current_port(self): port = "5432" if "port" in connoptions: port = connoptions["port"] - return "%s" % (port) + return f"{port}" @property def current_connection(self): @@ -226,7 +222,7 @@ def current_connection(self): host = connoptions["host"] if "port" in connoptions: port = connoptions["port"] - return "%s:%s" % (host, port) + return f"{host}:{port}" @property def database(self): @@ -368,7 +364,7 @@ def connect( user = user or self.get_str_cookie("user") password = password or self.get_str_cookie("password") if server not in options.servers: - raise HTTPError(404, "Server %s not found." % server) + raise HTTPError(404, f"Server {server} not found.") connoptions = options.servers[server].copy() @@ -378,14 +374,14 @@ def connect( if encoding_query is not None: if not isinstance(encoding_query, dict): raise Exception( - 'Invalid "query" parameter: %r, ' % encoding_query + f'Invalid "query" parameter: {encoding_query!r}, ' ) for k in encoding_query: if k != "client_encoding": raise Exception( - 'Invalid "query" parameter: %r", ' - 'unexpected key "%s"' % (encoding_query, k) + f'Invalid "query" parameter: {encoding_query!r}", ' + f'unexpected key "{k}"' ) if srvid is not None and srvid != "0": @@ -609,7 +605,7 @@ def write_error(self, status_code, **kwargs): self._status_code = status_code self.render("xhr.html", content=kwargs["exc_info"][1].log_message) return - super(BaseHandler, self).write_error(status_code, **kwargs) + super().write_error(status_code, **kwargs) def execute( self, @@ -668,12 +664,12 @@ def notify_collector(self, command, args=[], timeout=3): cur.execute("UNLISTEN *") random.seed() - channel = "r%d" % random.randint(1, 99999) - cur.execute("LISTEN %s" % channel) + channel = f"r{random.randint(1, 99999)}" + cur.execute(f"LISTEN {channel}") # some commands will contain user-provided strings, so we need to # properly escape the arguments. - payload = "%s %s %s" % (command, channel, " ".join(args)) + payload = "{} {} {}".format(command, channel, " ".join(args)) cur.execute("NOTIFY powa_collector, %s", (payload,)) cur.close() @@ -758,10 +754,10 @@ class AuthHandler(BaseHandler): @authenticated def prepare(self): - super(AuthHandler, self).prepare() + super().prepare() def to_json(self): return dict( - **super(AuthHandler, self).to_json(), + **super().to_json(), **{"logoutUrl": self.reverse_url("logout")}, ) diff --git a/powa/json.py b/powa/json.py index ae21430d..a32284bc 100644 --- a/powa/json.py +++ b/powa/json.py @@ -1,5 +1,3 @@ -from __future__ import absolute_import - from datetime import datetime from decimal import Decimal from json import JSONEncoder as BaseJSONEncoder @@ -21,7 +19,7 @@ def default(self, obj): return BaseJSONEncoder.default(self, obj) -class JSONizable(object): +class JSONizable: """ Base class for an object which is serializable to JSON. """ diff --git a/powa/options.py b/powa/options.py index 4bb7c166..b1fccb8d 100644 --- a/powa/options.py +++ b/powa/options.py @@ -57,11 +57,11 @@ def parse_file(filepath): try: parse_config_file(filepath) - except IOError: + except OSError: pass except Error as e: - print("Error parsing config file %s:" % filepath) - print("\t%s" % e) + print(f"Error parsing config file {filepath}:") + print(f"\t{e}") sys.exit(-1) @@ -75,7 +75,7 @@ def parse_options(): parse_file(options.config) except Error as e: print("Error parsing command line options:") - print("\t%s" % e) + print(f"\t{e}") sys.exit(1) for key in ("servers", "cookie_secret"): @@ -105,7 +105,7 @@ def parse_options(): define( "index_url", type=str, - default="%sserver/" % getattr(options, "url_prefix", "/"), + default="{}server/".format(getattr(options, "url_prefix", "/")), ) # we used to expect a field named "username", so accept "username" as an diff --git a/powa/powa.wsgi b/powa/powa.wsgi index bb6c7f63..6806a2f1 100644 --- a/powa/powa.wsgi +++ b/powa/powa.wsgi @@ -14,9 +14,9 @@ else: # of tornado don't check for the existence of isatty if not hasattr(sys.stderr, "isatty"): - class StdErrWrapper(object): + class StdErrWrapper: def __init__(self, wrapped): - super(StdErrWrapper, self).__setattr__("wrapped", wrapped) + super().__setattr__("wrapped", wrapped) def isatty(self): if hasattr(self.wrapped, "isatty"): diff --git a/powa/qual.py b/powa/qual.py index 0ef26360..c62951c1 100644 --- a/powa/qual.py +++ b/powa/qual.py @@ -43,7 +43,7 @@ def query(self): correlated = qualstat_getstatdata( extra_where=["qualid = %(qual)s", "queryid = %(query)s"] ) - sql = """SELECT sub.*, correlated.occurences as total_occurences + sql = f"""SELECT sub.*, correlated.occurences as total_occurences FROM ( SELECT * FROM ( @@ -52,7 +52,7 @@ def query(self): ) AS sub, ( {correlated} ) AS correlated - """.format(most_used=most_used, correlated=correlated) + """ return sql @@ -100,7 +100,7 @@ def get(self, server, database, query, qual): ) except Exception as e: raise HTTPError( - 501, "Could not connect to remote server: %s" % str(e) + 501, f"Could not connect to remote server: {str(e)}" ) stmt = qualstat_getstatdata( extra_select=["queryid = %(query)s AS is_my_query"], diff --git a/powa/query.py b/powa/query.py index 7ed1746c..fc925424 100644 --- a/powa/query.py +++ b/powa/query.py @@ -349,9 +349,9 @@ def query(self): to_epoch("ts"), sum_per_sec("rows"), sum_per_sec("calls"), - "CASE WHEN {total_blocks} = 0 THEN 0 ELSE" - " sum(shared_blks_hit)::numeric * 100 / {total_blocks}" - "END AS hit_ratio".format(total_blocks=total_blocks), + f"CASE WHEN {total_blocks} = 0 THEN 0 ELSE" + f" sum(shared_blks_hit)::numeric * 100 / {total_blocks}" + "END AS hit_ratio", byte_per_sec("shared_blks_read"), byte_per_sec("shared_blks_hit"), byte_per_sec("shared_blks_dirtied"), @@ -426,16 +426,10 @@ def query(self): " * block_size" ) sys_hit_ratio = ( - "{sys_hits}::numeric * 100 / ({total_blocks}" - " * block_size)".format( - sys_hits=sys_hits, total_blocks=total_blocks - ) + f"{sys_hits}::numeric * 100 / ({total_blocks}" " * block_size)" ) disk_hit_ratio = ( - "sum(sub.reads) * 100 / " - "({total_blocks} * block_size)".format( - total_blocks=total_blocks - ) + "sum(sub.reads) * 100 / " f"({total_blocks} * block_size)" ) total_time = "greatest(sum(runtime), 1)" other_time = ( @@ -447,9 +441,7 @@ def query(self): # Rusage can return values > real time due to sampling bias # aligned to kernel ticks. As such, we have to clamp values to 100% def total_time_percent(col, alias=None, noalias=False): - val = "least(100, ({col} * 100) / {total_time})".format( - col=col, total_time=total_time - ) + val = f"least(100, ({col} * 100) / {total_time})" if not noalias: val += " as " + alias @@ -472,34 +464,26 @@ def total_time_percent(col, alias=None, noalias=False): total_time_percent( "sum(system_time) * 1000", "system_time" ), - "greatest({other}, 0) AS other_time".format( - other=total_time_percent(other_time, noalias=True) - ), - "CASE WHEN {tb} = 0 THEN 0 ELSE {dhr} END AS disk_hit_ratio".format( - tb=total_blocks, dhr=disk_hit_ratio - ), - "CASE WHEN {tb} = 0 THEN 0 ELSE {shr} END AS sys_hit_ratio".format( - tb=total_blocks, shr=sys_hit_ratio - ), + f"greatest({total_time_percent(other_time, noalias=True)}, 0) AS other_time", + f"CASE WHEN {total_blocks} = 0 THEN 0 ELSE {disk_hit_ratio} END AS disk_hit_ratio", + f"CASE WHEN {total_blocks} = 0 THEN 0 ELSE {sys_hit_ratio} END AS sys_hit_ratio", ] ) - from_clause = """SELECT * + from_clause = f"""SELECT * FROM ( {from_clause} ) sub2 LEFT JOIN ( {kcache_query} ) AS kc - USING (ts, srvid, queryid, userid, dbid)""".format( - from_clause=from_clause, kcache_query=kcache_query - ) + USING (ts, srvid, queryid, userid, dbid)""" else: cols.extend( [ - """CASE WHEN {tb} = 0 THEN 0 - ELSE sum(shared_blks_read)::numeric * 100 / {tb} - END AS miss_ratio""".format(tb=total_blocks) + f"""CASE WHEN {total_blocks} = 0 THEN 0 + ELSE sum(shared_blks_read)::numeric * 100 / {total_blocks} + END AS miss_ratio""" ] ) @@ -534,7 +518,7 @@ def get(self, srvid, database, query): ) except Exception as e: raise HTTPError( - 501, "Could not connect to remote server: %s" % str(e) + 501, f"Could not connect to remote server: {str(e)}" ) extra_join = """, @@ -589,7 +573,7 @@ def get(self, srvid, database, query): )[0]["indexname"] except Error as e: self.flash( - "Could not create hypothetical index: %s" % str(e) + f"Could not create hypothetical index: {str(e)}" ) # Build the query and fetch the plans querystr = get_any_sample_query( @@ -608,7 +592,7 @@ def get(self, srvid, database, query): # TODO: offer the possibility to fill in parameters from the UI self.flash( "We couldn't get plans for this query, presumably " - "because some parameters are missing: %s" % str(e) + f"because some parameters are missing: {str(e)}" ) self.render_json(dict(indexes=indexes, hypoplan=hypoplan)) @@ -773,11 +757,11 @@ def query(self): "sum(count) AS counts", ] - from_clause = """( + from_clause = f"""( {inner_query} ) AS sub JOIN {{powa}}.powa_statements AS ps - USING (srvid, queryid, dbid)""".format(inner_query=inner_query) + USING (srvid, queryid, dbid)""" return """SELECT {cols} FROM {from_clause} @@ -820,7 +804,7 @@ def post_process(self, data, server, database, query, **kwargs): ) except Exception as e: raise HTTPError( - 501, "Could not connect to remote server: %s" % str(e) + 501, f"Could not connect to remote server: {str(e)}" ) data["data"] = resolve_quals(remote_conn, data["data"]) @@ -844,11 +828,11 @@ def get(self, srvid, database, query): srvid, ["datname = %(database)s", "queryid = %(query)s"] ) - from_clause = """{{powa}}.powa_statements AS ps + from_clause = f"""{{powa}}.powa_statements AS ps LEFT JOIN ( {stmt} ) AS sub USING (queryid, dbid, userid) - CROSS JOIN {block_size}""".format(stmt=stmt, block_size=block_size) + CROSS JOIN {block_size}""" rblk = "(sum(shared_blks_read) * block_size)" wblk = "(sum(shared_blks_hit) * block_size)" diff --git a/powa/server.py b/powa/server.py index 6a90b0d2..e1e8ebba 100644 --- a/powa/server.py +++ b/powa/server.py @@ -112,9 +112,9 @@ def _get_metrics(cls, handler, **params): def query(self): bs = block_size inner_query = powa_getstatdata_db("%(server)s") - from_clause = """({inner_query}) AS sub + from_clause = f"""({inner_query}) AS sub JOIN {{powa}}.powa_databases pd ON pd.oid = sub.dbid - AND pd.srvid = sub.srvid""".format(inner_query=inner_query) + AND pd.srvid = sub.srvid""" cols = [ "pd.srvid", @@ -189,15 +189,15 @@ class ByDatabaseWaitSamplingMetricGroup(MetricGroupDef): def query(self): inner_query = powa_getwaitdata_db() - from_clause = """({inner_query}) AS sub + from_clause = f"""({inner_query}) AS sub JOIN {{powa}}.powa_databases pd ON pd.oid = sub.dbid - AND pd.srvid = sub.srvid""".format(inner_query=inner_query) + AND pd.srvid = sub.srvid""" - return """SELECT pd.srvid, pd.datname, sub.event_type, sub.event, + return f"""SELECT pd.srvid, pd.datname, sub.event_type, sub.event, sum(sub.count) AS counts FROM {from_clause} GROUP BY pd.srvid, pd.datname, sub.event_type, sub.event - ORDER BY sum(sub.count) DESC""".format(from_clause=from_clause) + ORDER BY sum(sub.count) DESC""" def process(self, val, **kwargs): val["url"] = self.reverse_url( @@ -489,7 +489,7 @@ def query(self): from_clause = query if self.has_extension(self.path_args[0], "pg_stat_kcache"): - from_clause = "({query}) AS sub2".format(query=query) + from_clause = f"({query}) AS sub2" # Add system metrics from pg_stat_kcache, kcache_query = kcache_getstatdata_sample("db") @@ -524,10 +524,8 @@ def query(self): ] ) - from_clause += """ - LEFT JOIN ({kcache_query}) AS kc USING (dbid, ts, srvid)""".format( - kcache_query=kcache_query - ) + from_clause += f""" + LEFT JOIN ({kcache_query}) AS kc USING (dbid, ts, srvid)""" return """SELECT {cols} FROM ( @@ -622,7 +620,7 @@ def query(self): wps("count_io"), ] - from_clause = "({query}) AS sub".format(query=query) + from_clause = f"({query}) AS sub" return """SELECT {cols} FROM {from_clause} diff --git a/powa/sql/__init__.py b/powa/sql/__init__.py index d0de42e4..de2e6ab6 100644 --- a/powa/sql/__init__.py +++ b/powa/sql/__init__.py @@ -130,19 +130,19 @@ def __init__( self.attnum = attnum def __str__(self): - return "{}.{} {} ?".format(self.relname, self.attname, self.opname) + return f"{self.relname}.{self.attname} {self.opname} ?" @property def distinct_values(self): if self.n_distinct == 0: return None elif self.n_distinct > 0: - return "{}".format(self.n_distinct) + return f"{self.n_distinct}" else: - return "{} %".format(abs(self.n_distinct) * 100) + return f"{abs(self.n_distinct) * 100} %" def to_json(self): - base = super(ResolvedQual, self).to_json() + base = super().to_json() base["label"] = str(self) base["distinct_values"] = self.distinct_values return base @@ -163,7 +163,7 @@ def __init__( queries=None, queryids=None, ): - super(ComposedQual, self).__init__() + super().__init__() self.qualid = qualid self.relname = relname self.nspname = nspname @@ -192,10 +192,10 @@ def __str__(self): @property def where_clause(self): - return "WHERE {}".format(self) + return f"WHERE {self}" def to_json(self): - base = super(ComposedQual, self).to_json() + base = super().to_json() base["quals"] = self._quals base["where_clause"] = self.where_clause return base @@ -324,14 +324,14 @@ def qual_constants( qual_filter = "" if queries is not None: - query_subfilter = "AND queryid IN ({})".format(queries) - query_filter = "AND s.queryid IN ({})".format(queries) + query_subfilter = f"AND queryid IN ({queries})" + query_filter = f"AND s.queryid IN ({queries})" if quals is not None: - qual_subfilter = "AND qualid IN ({})".format(quals) - qual_filter = "AND qnc.qualid IN ({})".format(quals) + qual_subfilter = f"AND qualid IN ({quals})" + qual_filter = f"AND qnc.qualid IN ({quals})" - base = """ + base = f""" ( WITH sample AS ( SELECT s.srvid, query, s.queryid, qn.qualid, quals as quals, @@ -358,14 +358,14 @@ def qual_constants( {qual_subfilter} ) qnc ON qnc.srvid = s.srvid AND qn.qualid = qnc.qualid AND qn.queryid = qnc.queryid, LATERAL - unnest({qual_type}) as t(constants,occurences, execution_count, nbfiltered) - WHERE {filter} + unnest({type}) as t(constants,occurences, execution_count, nbfiltered) + WHERE {filter_clause} {query_filter} {qual_filter} AND s.srvid = {srvid} GROUP BY s.srvid, qn.qualid, quals, constants, s.queryid, query - ORDER BY {order} - LIMIT {top_value} + ORDER BY {orders[type]} + LIMIT {top} ) SELECT srvid, query, queryid, qualid, quals, constants as constants, occurences as occurences, @@ -375,19 +375,9 @@ def qual_constants( row_number() OVER (ORDER BY execution_count desc NULLS LAST) AS rownumber FROM sample ORDER BY rownumber - LIMIT {top_value} - ) {qual_type} - """.format( - qual_type=type, - filter=filter_clause, - query_subfilter=query_subfilter, - query_filter=query_filter, - qual_subfilter=qual_subfilter, - qual_filter=qual_filter, - order=orders[type], - top_value=top, - srvid=srvid, - ) + LIMIT {top} + ) {type} + """ query = "SELECT * FROM " + base @@ -410,13 +400,13 @@ def get_plans(cls, server, database, query, all_vals): query = format_jumbled_query(query, vals["constants"]) plan = "N/A" try: - sqlQuery = "EXPLAIN {}".format(query) + sqlQuery = f"EXPLAIN {query}" result = cls.execute( sqlQuery, srvid=server, database=database, remote_access=True ) plan = "\n".join(v["QUERY PLAN"] for v in result) except Exception as e: - plan = "ERROR: %r" % e + plan = f"ERROR: {e!r}" pass plans.append( Plan( @@ -592,7 +582,7 @@ def gain_percent(self): ) def to_json(self): - base = super(HypoPlan, self).to_json() + base = super().to_json() base["gain_percent"] = self.gain_percent return base @@ -615,7 +605,7 @@ def _update_ddl(self): if qual.attname not in attrs: attrs.append(qual.attname) # Qual resolution is responsible for quoting all identifiers - super(HypoIndex, self).__setattr__( + super().__setattr__( "_ddl", """CREATE INDEX ON {nsp}.{rel}({attrs})""".format( nsp=self.nspname, rel=self.relname, attrs=",".join(attrs) @@ -623,7 +613,7 @@ def _update_ddl(self): ) def __setattr(self, name, value): - super(HypoIndex, self).__setattr__(name, value) + super().__setattr__(name, value) # Only btree is supported right now if name in ("amname", "nspname", "relname", "composed_qual"): self._update_ddl() @@ -644,7 +634,7 @@ def hypo_ddl(self): return (None, None) def to_json(self): - base = super(HypoIndex, self).to_json() + base = super().to_json() base["ddl"] = self.ddl return base @@ -692,7 +682,7 @@ def get_hypoplans(cur, query, indexes=None): cur.execute("SET hypopg.enabled = off") try: cur.execute("SAVEPOINT hypo") - cur.execute("EXPLAIN {}".format(query)) + cur.execute(f"EXPLAIN {query}") baseplan = "\n".join(v[0] for v in cur.fetchall()) cur.execute("RELEASE hypo") except Exception as e: @@ -702,7 +692,7 @@ def get_hypoplans(cur, query, indexes=None): cur.execute("SET hypopg.enabled = on") try: cur.execute("SAVEPOINT hypo") - cur.execute("EXPLAIN {}".format(query)) + cur.execute(f"EXPLAIN {query}") hypoplan = "\n".join(v[0] for v in cur.fetchall()) cur.execute("RELEASE hypo") except Exception as e: diff --git a/powa/sql/utils.py b/powa/sql/utils.py index 3faa33c0..5e3af060 100644 --- a/powa/sql/utils.py +++ b/powa/sql/utils.py @@ -30,14 +30,12 @@ def total_measure_interval(col): def diff(var, alias=None): alias = alias or var - return "max({var}) - min({var}) AS {alias}".format(var=var, alias=alias) + return f"max({var}) - min({var}) AS {alias}" def diffblk(var, blksize=8192, alias=None): alias = alias or var - return "(max({var}) - min({var})) * {blksize} AS {alias}".format( - var=var, blksize=blksize, alias=alias - ) + return f"(max({var}) - min({var})) * {blksize} AS {alias}" def get_ts(): @@ -51,9 +49,7 @@ def sum_per_sec(col, prefix=None, alias=None): else: prefix = "" - return "sum({prefix}{col}) / {ts} AS {alias}".format( - prefix=prefix, col=col, ts=get_ts(), alias=alias - ) + return f"sum({prefix}{col}) / {get_ts()} AS {alias}" def byte_per_sec(col, prefix=None, alias=None): @@ -63,9 +59,7 @@ def byte_per_sec(col, prefix=None, alias=None): else: prefix = "" - return "sum({prefix}{col}) * block_size / {ts} AS {alias}".format( - prefix=prefix, col=col, ts=get_ts(), alias=alias - ) + return f"sum({prefix}{col}) * block_size / {get_ts()} AS {alias}" def wps(col, do_sum=True): @@ -73,18 +67,16 @@ def wps(col, do_sum=True): if do_sum: field = "sum(" + field + ")" - return "({field} / {ts}) AS {col}".format( - field=field, col=col, ts=get_ts() - ) + return f"({field} / {get_ts()}) AS {col}" def to_epoch(col, prefix=None): if prefix is not None: - qn = "{prefix}.{col}".format(prefix=prefix, col=col) + qn = f"{prefix}.{col}" else: qn = col - return "extract(epoch FROM {qn}) AS {col}".format(qn=qn, col=col) + return f"extract(epoch FROM {qn}) AS {col}" def total_read(prefix, noalias=False): diff --git a/powa/sql/views.py b/powa/sql/views.py index e37f39f2..b700426d 100644 --- a/powa/sql/views.py +++ b/powa/sql/views.py @@ -30,7 +30,7 @@ def qualstat_base_statdata(eval_type=None): else: pqnh = "{powa}.powa_qualstats_quals" - base_query = """ + base_query = f""" ( SELECT srvid, qualid, queryid, dbid, userid, (unnested.records).* FROM ( @@ -49,7 +49,7 @@ def qualstat_base_statdata(eval_type=None): WHERE pqnc.ts <@ tstzrange(%(from)s, %(to)s, '[]') AND pqnc.srvid = %(server)s ) h - JOIN {pqnh} AS pqnh USING (srvid, queryid, qualid)""".format(pqnh=pqnh) + JOIN {pqnh} AS pqnh USING (srvid, queryid, qualid)""" return base_query @@ -87,13 +87,13 @@ def qualstat_getstatdata( else: extra_having = "" - return """SELECT + return f"""SELECT ps.srvid, qualid, ps.queryid, query, ps.dbid, to_json(quals) AS quals, sum(execution_count) AS execution_count, sum(occurences) AS occurences, (sum(nbfiltered) / sum(occurences)) AS avg_filter, - {filter_ratio} AS filter_ratio + {QUALSTAT_FILTER_RATIO} AS filter_ratio {extra_select} FROM {base_query} @@ -103,15 +103,7 @@ def qualstat_getstatdata( {extra_where} GROUP BY ps.srvid, qualid, ps.queryid, ps.dbid, ps.query, quals {extra_groupby} - {extra_having}""".format( - filter_ratio=QUALSTAT_FILTER_RATIO, - extra_select=extra_select, - base_query=base_query, - extra_join=extra_join, - extra_where=extra_where, - extra_groupby=extra_groupby, - extra_having=extra_having, - ) + {extra_having}""" TEXTUAL_INDEX_QUERY = """ @@ -168,7 +160,7 @@ def get_config_changes(restrict_database=False): if restrict_database: restrict_db = "AND (d.datname = %(database)s OR h.setdatabase = 0)" - sql = """SELECT * FROM + sql = f"""SELECT * FROM ( WITH src AS ( select ts, name, @@ -234,6 +226,6 @@ def get_config_changes(restrict_database=False): WHERE r.srvid = %(server)s AND r.ts>= %(from)s AND r.ts <= %(to)s -ORDER BY ts""".format(restrict_db=restrict_db) +ORDER BY ts""" return sql diff --git a/powa/sql/views_graph.py b/powa/sql/views_graph.py index 9dc2a6d1..e31e4616 100644 --- a/powa/sql/views_graph.py +++ b/powa/sql/views_graph.py @@ -8,14 +8,12 @@ def wal_to_num(walname): Extracts the sequence number from the given WAL file name, similarly to pg_split_walfile_name(). It's assuming a 16MB wal_segment_size. """ - return """(('x' || substr({walname}, 9, 8))::bit(32)::bigint + return f"""(('x' || substr({walname}, 9, 8))::bit(32)::bigint * '4294967296'::bigint / 16777216 - + ('x' || substr({walname}, 17, 8))::bit(32)::bigint)""".format( - walname=walname - ) + + ('x' || substr({walname}, 17, 8))::bit(32)::bigint)""" -class Biggest(object): +class Biggest: def __init__(self, base_columns, order_by): if type(base_columns) is str: base_columns = [base_columns] @@ -44,7 +42,7 @@ def __call__(self, var, minval=0, label=None): return sql -class Biggestsum(object): +class Biggestsum: def __init__(self, base_columns, order_by): if type(base_columns) is str: base_columns = [base_columns] @@ -516,7 +514,7 @@ def BASE_QUERY_PGSA_SAMPLE(per_db=False): extra = "" # We use dense_rank() as we need ALL the records for a specific ts - return """ + return f""" (SELECT pgsa_history.srvid, dense_rank() OVER (ORDER BY pgsa_history.ts) AS number, count(*) OVER () AS total, @@ -548,7 +546,7 @@ def BASE_QUERY_PGSA_SAMPLE(per_db=False): {extra} ) AS pgsa WHERE number %% ( int8larger((total)/(%(samples)s+1),1) ) = 0 -""".format(extra=extra) +""" BASE_QUERY_ARCHIVER_SAMPLE = """ @@ -651,7 +649,7 @@ def BASE_QUERY_DATABASE_SAMPLE(per_db=False): else: extra = "" - return """ + return f""" (SELECT psd_history.srvid, row_number() OVER (ORDER BY psd_history.ts) AS number, count(*) OVER () AS total, @@ -701,7 +699,7 @@ def BASE_QUERY_DATABASE_SAMPLE(per_db=False): GROUP BY psd_history.srvid, psd_history.ts ) AS psd WHERE number %% ( int8larger((total)/(%(samples)s+1),1) ) = 0 -""".format(extra=extra) +""" def BASE_QUERY_DATABASE_CONFLICTS_SAMPLE(per_db=False): @@ -712,7 +710,7 @@ def BASE_QUERY_DATABASE_CONFLICTS_SAMPLE(per_db=False): else: extra = "" - return """ + return f""" (SELECT psdc_history.srvid, row_number() OVER (ORDER BY psdc_history.ts) AS number, count(*) OVER () AS total, @@ -742,7 +740,7 @@ def BASE_QUERY_DATABASE_CONFLICTS_SAMPLE(per_db=False): GROUP BY psdc_history.srvid, psdc_history.ts ) AS psdc WHERE number %% ( int8larger((total)/(%(samples)s+1),1) ) = 0 -""".format(extra=extra) +""" # We use dense_rank() as we need ALL the records for a specific ts. Caller @@ -911,7 +909,7 @@ def BASE_QUERY_SUBSCRIPTION_SAMPLE(subname=None): # We use dense_rank() as we need ALL the records for a specific ts. Caller # will group the data as needed. - return """ + return f""" (SELECT srvid, dense_rank() OVER (ORDER BY ts) AS number, count(*) OVER (PARTITION BY ts) AS num_per_window, @@ -962,7 +960,7 @@ def BASE_QUERY_SUBSCRIPTION_SAMPLE(subname=None): ) AS subscription_history ) AS io WHERE number %% ( int8larger((total / num_per_window)/(%(samples)s+1),1) ) = 0 -""".format(extra=extra) +""" BASE_QUERY_WAL_SAMPLE = """ @@ -1314,7 +1312,7 @@ def txid_age(field): ref = "cur_txid" alias = field + "_age" - return """CASE + return f"""CASE WHEN {ref}::text::bigint - {field}::text::bigint < -100000 THEN ({ref}::text::bigint - 3) + ((4::bigint * 1024 * 1024 * 1024) - {field}::text::bigint) @@ -1322,13 +1320,11 @@ def txid_age(field): THEN 0 ELSE {ref}::text::bigint - {field}::text::bigint - END AS {alias}""".format(field=field, ref=ref, alias=alias) + END AS {alias}""" def ts_get_sec(field): alias = field + "_age" - return """extract(epoch FROM (ts - {f})) * 1000 AS {a}""".format( - f=field, a=alias - ) + return f"""extract(epoch FROM (ts - {field})) * 1000 AS {alias}""" all_cols = base_columns + [ "ts", @@ -1530,7 +1526,7 @@ def powa_get_io_sample(qual=None): biggest = Biggest(base_columns, "ts") if qual is not None: - qual = " AND %s" % qual + qual = f" AND {qual}" else: qual = "" @@ -1566,7 +1562,7 @@ def powa_get_slru_sample(qual=None): biggest = Biggest(base_columns, "ts") if qual is not None: - qual = " AND %s" % qual + qual = f" AND {qual}" else: qual = "" diff --git a/powa/sql/views_grid.py b/powa/sql/views_grid.py index 36aea6ba..864f1062 100644 --- a/powa/sql/views_grid.py +++ b/powa/sql/views_grid.py @@ -318,7 +318,7 @@ def powa_getiodata(qual=None): base_query = powa_base_io() if qual is not None: - qual = " WHERE %s" % qual + qual = f" WHERE {qual}" else: qual = "" @@ -434,7 +434,7 @@ def powa_getslrudata(qual=None): base_query = powa_base_slru() if qual is not None: - qual = " WHERE %s" % qual + qual = f" WHERE {qual}" else: qual = "" @@ -988,7 +988,7 @@ def powa_getuserfuncdata_detailed_db(funcid=None): if funcid: join_db = """LEFT JOIN {powa}.powa_catalog_databases pcd ON pcd.srvid = d.srvid AND pcd.oid = h.dbid""" - and_funcid = "AND funcid = {funcid}".format(funcid=funcid) + and_funcid = f"AND funcid = {funcid}" else: join_db = "" and_funcid = "" diff --git a/powa/ui_methods.py b/powa/ui_methods.py index 1054f703..cddd16e3 100644 --- a/powa/ui_methods.py +++ b/powa/ui_methods.py @@ -44,7 +44,7 @@ def field(_, **kwargs): kwargs.setdefault("type", "text") kwargs.setdefault("class", "form-control") attrs = " ".join( - '%s="%s"' % (key, value) + f'{key}="{value}"' for key, value in kwargs.items() if key not in ("tag", "label") ) @@ -55,18 +55,15 @@ def render(content): Render the field itself. """ kwargs["content"] = content.decode("utf8") - return ( - """ + return """ - -""" - % kwargs - ) +""".format(**kwargs) return render @@ -134,7 +131,7 @@ def sanitycheck_messages(self): if rows is not None and len(rows) > 0: for r in rows: - messages["error"].append("%s: %s" % (r["alias"], r["error"])) + messages["error"].append("{}: {}".format(r["alias"], r["error"])) return messages return {} diff --git a/powa/ui_modules.py b/powa/ui_modules.py index 38e7c7fd..c6fe4dab 100644 --- a/powa/ui_modules.py +++ b/powa/ui_modules.py @@ -1,4 +1,4 @@ -class MenuEntry(object): +class MenuEntry: def __init__( self, title, diff --git a/powa/user.py b/powa/user.py index b018ebcc..5396a45e 100644 --- a/powa/user.py +++ b/powa/user.py @@ -1,5 +1,3 @@ -from __future__ import unicode_literals - from powa import __VERSION_NUM__ from powa.framework import BaseHandler from tornado.options import options @@ -39,8 +37,7 @@ def post(self, *args, **kwargs): # Major.Minor version should be the same if version[0:2] != __VERSION_NUM__[0:2]: self.flash( - "Unable to connect: powa-archivist version %s.X does not match powa-web version %s.X" - % ( + "Unable to connect: powa-archivist version {}.X does not match powa-web version {}.X".format( ".".join(str(x) for x in version[0:2]), ".".join(str(x) for x in __VERSION_NUM__[0:2]), ), diff --git a/powa/wizard.py b/powa/wizard.py index 91eed724..5df921b5 100644 --- a/powa/wizard.py +++ b/powa/wizard.py @@ -2,8 +2,6 @@ Global optimization widget """ -from __future__ import absolute_import - import json from powa.dashboards import MetricGroupDef, Widget from powa.framework import AuthHandler @@ -29,7 +27,7 @@ def post(self, srvid, database): remote_cur = remote_conn.cursor() except Exception as e: raise HTTPError( - 501, "Could not connect to remote server: %s" % str(e) + 501, f"Could not connect to remote server: {str(e)}" ) payload = json.loads(self.request.body.decode("utf8")) diff --git a/setup.py b/setup.py index c5f4a61c..dd618d76 100755 --- a/setup.py +++ b/setup.py @@ -1,4 +1,3 @@ -import sys from setuptools import find_packages, setup __VERSION__ = None @@ -11,11 +10,6 @@ requires = ["tornado>=2.0", "psycopg2"] -# include ordereddict for python2.6 -if sys.version_info < (2, 7, 0): - requires.append("ordereddict") - - setup( name="powa-web", version=__VERSION__, @@ -35,8 +29,15 @@ "License :: Other/Proprietary License", "License :: OSI Approved :: BSD License", "Operating System :: OS Independent", - "Programming Language :: Python :: 2", "Programming Language :: Python :: 3", + "Programming Language :: Python :: 3.6", + "Programming Language :: Python :: 3.7", + "Programming Language :: Python :: 3.8", + "Programming Language :: Python :: 3.9", + "Programming Language :: Python :: 3.10", + "Programming Language :: Python :: 3.11", + "Programming Language :: Python :: 3 :: Only", "Topic :: Database :: Front-Ends", ], + python_requires=">=3.6, <4", )