diff --git a/.gitignore b/.gitignore index c89013a11..a344568c0 100644 --- a/.gitignore +++ b/.gitignore @@ -12,6 +12,7 @@ htmlcov .tox geckodriver.log coverage.xml +venv .direnv/ .envrc venv diff --git a/Makefile b/Makefile index 4d2db27af..8c1b2d0ab 100644 --- a/Makefile +++ b/Makefile @@ -7,6 +7,12 @@ example: ## Run the example application --noinput --username="$(USER)" --email="$(USER)@mailinator.com" python example/manage.py runserver +example_async: + python example/manage.py migrate --noinput + -DJANGO_SUPERUSER_PASSWORD=p python example/manage.py createsuperuser \ + --noinput --username="$(USER)" --email="$(USER)@mailinator.com" + daphne example.asgi:application + example_test: ## Run the test suite for the example application python example/manage.py test example diff --git a/debug_toolbar/migrations/0001_initial.py b/debug_toolbar/migrations/0001_initial.py new file mode 100644 index 000000000..e4d30fede --- /dev/null +++ b/debug_toolbar/migrations/0001_initial.py @@ -0,0 +1,24 @@ +from django.db import migrations, models + + +class Migration(migrations.Migration): + initial = True + + operations = [ + migrations.CreateModel( + name="HistoryEntry", + fields=[ + ( + "request_id", + models.UUIDField(primary_key=True, serialize=False), + ), + ("data", models.JSONField(default=dict)), + ("created_at", models.DateTimeField(auto_now_add=True)), + ], + options={ + "verbose_name": "history entry", + "verbose_name_plural": "history entries", + "ordering": ["-created_at"], + }, + ), + ] diff --git a/debug_toolbar/migrations/__init__.py b/debug_toolbar/migrations/__init__.py new file mode 100644 index 000000000..e69de29bb diff --git a/debug_toolbar/models.py b/debug_toolbar/models.py new file mode 100644 index 000000000..686ac4cfa --- /dev/null +++ b/debug_toolbar/models.py @@ -0,0 +1,16 @@ +from django.db import models +from django.utils.translation import gettext_lazy as _ + + +class HistoryEntry(models.Model): + request_id = models.UUIDField(primary_key=True) + data = models.JSONField(default=dict) + created_at = models.DateTimeField(auto_now_add=True) + + class Meta: + verbose_name = _("history entry") + verbose_name_plural = _("history entries") + ordering = ["-created_at"] + + def __str__(self): + return str(self.request_id) diff --git a/debug_toolbar/panels/__init__.py b/debug_toolbar/panels/__init__.py index 217708ec2..a53ba6652 100644 --- a/debug_toolbar/panels/__init__.py +++ b/debug_toolbar/panels/__init__.py @@ -1,5 +1,6 @@ from django.core.handlers.asgi import ASGIRequest from django.template.loader import render_to_string +from django.utils.functional import classproperty from debug_toolbar import settings as dt_settings from debug_toolbar.utils import get_name_from_obj @@ -15,19 +16,27 @@ class Panel: def __init__(self, toolbar, get_response): self.toolbar = toolbar self.get_response = get_response + self.from_store = False # Private panel properties - @property - def panel_id(self): - return self.__class__.__name__ + @classproperty + def panel_id(cls): + return cls.__name__ @property def enabled(self) -> bool: - # check if the panel is async compatible + # Check if the panel is async compatible if not self.is_async and isinstance(self.toolbar.request, ASGIRequest): return False + if self.from_store: + # If the toolbar was loaded from the store the existence of + # recorded data indicates whether it was enabled or not. + # We can't use the remainder of the logic since we don't have + # a request to work off of. + return bool(self.get_stats()) + # The user's cookies should override the default value cookie_value = self.toolbar.request.COOKIES.get("djdt" + self.panel_id) if cookie_value is not None: @@ -175,9 +184,16 @@ def record_stats(self, stats): """ Store data gathered by the panel. ``stats`` is a :class:`dict`. - Each call to ``record_stats`` updates the statistics dictionary. + Each call to ``record_stats`` updates the store's data for + the panel. + + To support backwards compatibility, it will also update the + panel's statistics dictionary. """ self.toolbar.stats.setdefault(self.panel_id, {}).update(stats) + self.toolbar.store.save_panel( + self.toolbar.request_id, self.panel_id, self.toolbar.stats[self.panel_id] + ) def get_stats(self): """ @@ -261,6 +277,15 @@ def generate_server_timing(self, request, response): Does not return a value. """ + def load_stats_from_store(self, data): + """ + Instantiate the panel from serialized data. + + Return the panel instance. + """ + self.toolbar.stats.setdefault(self.panel_id, {}).update(data) + self.from_store = True + @classmethod def run_checks(cls): """ diff --git a/debug_toolbar/panels/alerts.py b/debug_toolbar/panels/alerts.py index c8e59002c..030ca1ee1 100644 --- a/debug_toolbar/panels/alerts.py +++ b/debug_toolbar/panels/alerts.py @@ -141,11 +141,9 @@ def check_invalid_file_form_configuration(self, html_content): return self.alerts def generate_stats(self, request, response): - if not is_processable_html_response(response): - return - - html_content = response.content.decode(response.charset) - self.check_invalid_file_form_configuration(html_content) + if is_processable_html_response(response): + html_content = response.content.decode(response.charset) + self.check_invalid_file_form_configuration(html_content) # Further alert checks can go here diff --git a/debug_toolbar/panels/cache.py b/debug_toolbar/panels/cache.py index 1b15b446f..d3242d9d9 100644 --- a/debug_toolbar/panels/cache.py +++ b/debug_toolbar/panels/cache.py @@ -171,16 +171,17 @@ def _record_call(self, cache, name, original_method, args, kwargs): @property def nav_subtitle(self): - cache_calls = len(self.calls) + stats = self.get_stats() + cache_calls = len(stats.get("calls")) return ngettext( "%(cache_calls)d call in %(time).2fms", "%(cache_calls)d calls in %(time).2fms", cache_calls, - ) % {"cache_calls": cache_calls, "time": self.total_time} + ) % {"cache_calls": cache_calls, "time": stats.get("total_time")} @property def title(self): - count = len(getattr(settings, "CACHES", ["default"])) + count = self.get_stats().get("total_caches") return ngettext( "Cache calls from %(count)d backend", "Cache calls from %(count)d backends", @@ -216,6 +217,7 @@ def generate_stats(self, request, response): "hits": self.hits, "misses": self.misses, "counts": self.counts, + "total_caches": len(getattr(settings, "CACHES", ["default"])), } ) diff --git a/debug_toolbar/panels/history/__init__.py b/debug_toolbar/panels/history/__init__.py index 52ceb7984..193ced242 100644 --- a/debug_toolbar/panels/history/__init__.py +++ b/debug_toolbar/panels/history/__init__.py @@ -1,3 +1,3 @@ from debug_toolbar.panels.history.panel import HistoryPanel -__all__ = ["HistoryPanel"] +__all__ = [HistoryPanel.panel_id] diff --git a/debug_toolbar/panels/history/forms.py b/debug_toolbar/panels/history/forms.py index 952b2409d..2aec18c34 100644 --- a/debug_toolbar/panels/history/forms.py +++ b/debug_toolbar/panels/history/forms.py @@ -5,8 +5,8 @@ class HistoryStoreForm(forms.Form): """ Validate params - store_id: The key for the store instance to be fetched. + request_id: The key for the store instance to be fetched. """ - store_id = forms.CharField(widget=forms.HiddenInput()) + request_id = forms.CharField(widget=forms.HiddenInput()) exclude_history = forms.BooleanField(widget=forms.HiddenInput(), required=False) diff --git a/debug_toolbar/panels/history/panel.py b/debug_toolbar/panels/history/panel.py index 56a891848..d2f3f8ab6 100644 --- a/debug_toolbar/panels/history/panel.py +++ b/debug_toolbar/panels/history/panel.py @@ -24,9 +24,9 @@ class HistoryPanel(Panel): def get_headers(self, request): headers = super().get_headers(request) observe_request = self.toolbar.get_observe_request() - store_id = self.toolbar.store_id - if store_id and observe_request(request): - headers["djdt-store-id"] = store_id + request_id = self.toolbar.request_id + if request_id and observe_request(request): + headers["djdt-request-id"] = request_id return headers @property @@ -87,23 +87,25 @@ def content(self): Fetch every store for the toolbar and include it in the template. """ - stores = {} - for id, toolbar in reversed(self.toolbar._store.items()): - stores[id] = { - "toolbar": toolbar, + toolbar_history = {} + for request_id in reversed(self.toolbar.store.request_ids()): + toolbar_history[request_id] = { + "history_stats": self.toolbar.store.panel( + request_id, HistoryPanel.panel_id + ), "form": HistoryStoreForm( - initial={"store_id": id, "exclude_history": True} + initial={"request_id": request_id, "exclude_history": True} ), } return render_to_string( self.template, { - "current_store_id": self.toolbar.store_id, - "stores": stores, + "current_request_id": self.toolbar.request_id, + "toolbar_history": toolbar_history, "refresh_form": HistoryStoreForm( initial={ - "store_id": self.toolbar.store_id, + "request_id": self.toolbar.request_id, "exclude_history": True, } ), diff --git a/debug_toolbar/panels/history/views.py b/debug_toolbar/panels/history/views.py index fb6e28c93..2dd6de820 100644 --- a/debug_toolbar/panels/history/views.py +++ b/debug_toolbar/panels/history/views.py @@ -4,6 +4,7 @@ from debug_toolbar._compat import login_not_required from debug_toolbar.decorators import render_with_toolbar_language, require_show_toolbar from debug_toolbar.panels.history.forms import HistoryStoreForm +from debug_toolbar.store import get_store from debug_toolbar.toolbar import DebugToolbar @@ -15,12 +16,12 @@ def history_sidebar(request): form = HistoryStoreForm(request.GET) if form.is_valid(): - store_id = form.cleaned_data["store_id"] - toolbar = DebugToolbar.fetch(store_id) + request_id = form.cleaned_data["request_id"] + toolbar = DebugToolbar.fetch(request_id) exclude_history = form.cleaned_data["exclude_history"] context = {} if toolbar is None: - # When the store_id has been popped already due to + # When the request_id has been popped already due to # RESULTS_CACHE_SIZE return JsonResponse(context) for panel in toolbar.panels: @@ -36,7 +37,7 @@ def history_sidebar(request): ), } return JsonResponse(context) - return HttpResponseBadRequest("Form errors") + return HttpResponseBadRequest(f"Form errors: {form.errors}") @login_not_required @@ -49,19 +50,22 @@ def history_refresh(request): if form.is_valid(): requests = [] # Convert to list to handle mutations happening in parallel - for id, toolbar in list(DebugToolbar._store.items()): + for request_id in get_store().request_ids(): + toolbar = DebugToolbar.fetch(request_id) requests.append( { - "id": id, + "id": request_id, "content": render_to_string( "debug_toolbar/panels/history_tr.html", { - "id": id, - "store_context": { - "toolbar": toolbar, + "request_id": request_id, + "history_context": { + "history_stats": toolbar.store.panel( + request_id, "HistoryPanel" + ), "form": HistoryStoreForm( initial={ - "store_id": id, + "request_id": request_id, "exclude_history": True, } ), diff --git a/debug_toolbar/panels/profiling.py b/debug_toolbar/panels/profiling.py index 4613a3cad..263e7a004 100644 --- a/debug_toolbar/panels/profiling.py +++ b/debug_toolbar/panels/profiling.py @@ -25,6 +25,7 @@ def __init__( self.id = id self.parent_ids = parent_ids or [] self.hsv = hsv + self.has_subfuncs = False def parent_classes(self): return self.parent_classes @@ -130,6 +131,21 @@ def cumtime_per_call(self): def indent(self): return 16 * self.depth + def serialize(self): + return { + "has_subfuncs": self.has_subfuncs, + "id": self.id, + "parent_ids": self.parent_ids, + "is_project_func": self.is_project_func(), + "indent": self.indent(), + "func_std_string": self.func_std_string(), + "cumtime": self.cumtime(), + "cumtime_per_call": self.cumtime_per_call(), + "tottime": self.tottime(), + "tottime_per_call": self.tottime_per_call(), + "count": self.count(), + } + class ProfilingPanel(Panel): """ @@ -148,7 +164,6 @@ def process_request(self, request): def add_node(self, func_list, func, max_depth, cum_time): func_list.append(func) - func.has_subfuncs = False if func.depth < max_depth: for subfunc in func.subfuncs(): # Always include the user's code @@ -182,4 +197,4 @@ def generate_stats(self, request, response): dt_settings.get_config()["PROFILER_MAX_DEPTH"], cum_time_threshold, ) - self.record_stats({"func_list": func_list}) + self.record_stats({"func_list": [func.serialize() for func in func_list]}) diff --git a/debug_toolbar/panels/settings.py b/debug_toolbar/panels/settings.py index ff32bd2c0..68ab44c0b 100644 --- a/debug_toolbar/panels/settings.py +++ b/debug_toolbar/panels/settings.py @@ -1,4 +1,4 @@ -from django.conf import settings +from django.utils.encoding import force_str from django.utils.translation import gettext_lazy as _ from django.views.debug import get_default_exception_reporter_filter @@ -19,7 +19,16 @@ class SettingsPanel(Panel): nav_title = _("Settings") def title(self): - return _("Settings from %s") % settings.SETTINGS_MODULE + return _("Settings from %s") % self.get_stats()["settings"].get( + "SETTINGS_MODULE" + ) def generate_stats(self, request, response): - self.record_stats({"settings": dict(sorted(get_safe_settings().items()))}) + self.record_stats( + { + "settings": { + key: force_str(value) + for key, value in sorted(get_safe_settings().items()) + } + } + ) diff --git a/debug_toolbar/panels/sql/__init__.py b/debug_toolbar/panels/sql/__init__.py index 46c68a3c6..9da548f7f 100644 --- a/debug_toolbar/panels/sql/__init__.py +++ b/debug_toolbar/panels/sql/__init__.py @@ -1,3 +1,3 @@ from debug_toolbar.panels.sql.panel import SQLPanel -__all__ = ["SQLPanel"] +__all__ = [SQLPanel.panel_id] diff --git a/debug_toolbar/panels/sql/forms.py b/debug_toolbar/panels/sql/forms.py index 1fa90ace4..44906924d 100644 --- a/debug_toolbar/panels/sql/forms.py +++ b/debug_toolbar/panels/sql/forms.py @@ -4,25 +4,22 @@ from django.core.exceptions import ValidationError from django.db import connections from django.utils.functional import cached_property +from django.utils.translation import gettext_lazy as _ from debug_toolbar.panels.sql.utils import is_select_query, reformat_sql +from debug_toolbar.toolbar import DebugToolbar class SQLSelectForm(forms.Form): """ Validate params - sql: The sql statement with interpolated params - raw_sql: The sql statement with placeholders - params: JSON encoded parameter values - duration: time for SQL to execute passed in from toolbar just for redisplay + request_id: The identifier for the request + query_id: The identifier for the query """ - sql = forms.CharField() - raw_sql = forms.CharField() - params = forms.CharField() - alias = forms.CharField(required=False, initial="default") - duration = forms.FloatField() + request_id = forms.CharField() + djdt_query_id = forms.CharField() def clean_raw_sql(self): value = self.cleaned_data["raw_sql"] @@ -48,12 +45,91 @@ def clean_alias(self): return value + def clean(self): + from debug_toolbar.panels.sql import SQLPanel + + cleaned_data = super().clean() + toolbar = DebugToolbar.fetch( + self.cleaned_data["request_id"], panel_id=SQLPanel.panel_id + ) + if toolbar is None: + raise ValidationError(_("Data for this panel isn't available anymore.")) + + panel = toolbar.get_panel_by_id(SQLPanel.panel_id) + # Find the query for this form submission + query = None + for q in panel.get_stats()["queries"]: + if q["djdt_query_id"] != self.cleaned_data["djdt_query_id"]: + continue + else: + query = q + break + if not query: + raise ValidationError(_("Invalid query id.")) + cleaned_data["query"] = query + return cleaned_data + + def select(self): + query = self.cleaned_data["query"] + sql = query["raw_sql"] + params = json.loads(query["params"]) + with self.cursor as cursor: + cursor.execute(sql, params) + headers = [d[0] for d in cursor.description] + result = cursor.fetchall() + return result, headers + + def explain(self): + query = self.cleaned_data["query"] + sql = query["raw_sql"] + params = json.loads(query["params"]) + vendor = query["vendor"] + with self.cursor as cursor: + if vendor == "sqlite": + # SQLite's EXPLAIN dumps the low-level opcodes generated for a query; + # EXPLAIN QUERY PLAN dumps a more human-readable summary + # See https://www.sqlite.org/lang_explain.html for details + cursor.execute(f"EXPLAIN QUERY PLAN {sql}", params) + elif vendor == "postgresql": + cursor.execute(f"EXPLAIN ANALYZE {sql}", params) + else: + cursor.execute(f"EXPLAIN {sql}", params) + headers = [d[0] for d in cursor.description] + result = cursor.fetchall() + return result, headers + + def profile(self): + query = self.cleaned_data["query"] + sql = query["raw_sql"] + params = json.loads(query["params"]) + with self.cursor as cursor: + cursor.execute("SET PROFILING=1") # Enable profiling + cursor.execute(sql, params) # Execute SELECT + cursor.execute("SET PROFILING=0") # Disable profiling + # The Query ID should always be 1 here but I'll subselect to get + # the last one just in case... + cursor.execute( + """ + SELECT * + FROM information_schema.profiling + WHERE query_id = ( + SELECT query_id + FROM information_schema.profiling + ORDER BY query_id DESC + LIMIT 1 + ) + """ + ) + headers = [d[0] for d in cursor.description] + result = cursor.fetchall() + return result, headers + def reformat_sql(self): - return reformat_sql(self.cleaned_data["sql"], with_toggle=False) + return reformat_sql(self.cleaned_data["query"]["sql"], with_toggle=False) @property def connection(self): - return connections[self.cleaned_data["alias"]] + return connections[self.cleaned_data["query"]["alias"]] @cached_property def cursor(self): diff --git a/debug_toolbar/panels/sql/panel.py b/debug_toolbar/panels/sql/panel.py index 206686352..6bfd10500 100644 --- a/debug_toolbar/panels/sql/panel.py +++ b/debug_toolbar/panels/sql/panel.py @@ -1,10 +1,11 @@ import uuid from collections import defaultdict -from copy import copy from asgiref.sync import sync_to_async from django.db import connections +from django.template.loader import render_to_string from django.urls import path +from django.utils.functional import cached_property from django.utils.translation import gettext_lazy as _, ngettext from debug_toolbar import settings as dt_settings @@ -86,7 +87,7 @@ def _similar_query_key(query): def _duplicate_query_key(query): - raw_params = () if query["raw_params"] is None else tuple(query["raw_params"]) + raw_params = () if query["params"] is None else tuple(query["params"]) # repr() avoids problems because of unhashable types # (e.g. lists) when used as dictionary keys. # https://github.com/django-commons/django-debug-toolbar/issues/1091 @@ -146,6 +147,7 @@ def current_transaction_id(self, alias): return trans_id def record(self, **kwargs): + kwargs["djdt_query_id"] = uuid.uuid4().hex self._queries.append(kwargs) alias = kwargs["alias"] if alias not in self._databases: @@ -164,19 +166,20 @@ def record(self, **kwargs): @property def nav_subtitle(self): - query_count = len(self._queries) + stats = self.get_stats() + query_count = len(stats.get("queries", [])) return ngettext( "%(query_count)d query in %(sql_time).2fms", "%(query_count)d queries in %(sql_time).2fms", query_count, ) % { "query_count": query_count, - "sql_time": self._sql_time, + "sql_time": stats.get("sql_time"), } @property def title(self): - count = len(self._databases) + count = len(self.get_stats().get("databases")) return ngettext( "SQL queries from %(count)d connection", "SQL queries from %(count)d connections", @@ -211,8 +214,6 @@ def disable_instrumentation(self): connection._djdt_logger = None def generate_stats(self, request, response): - colors = contrasting_color_generator() - trace_colors = defaultdict(lambda: next(colors)) similar_query_groups = defaultdict(list) duplicate_query_groups = defaultdict(list) @@ -269,14 +270,6 @@ def generate_stats(self, request, response): query["trans_status"] = get_transaction_status_display( query["vendor"], query["trans_status"] ) - - query["form"] = SignedDataForm( - auto_id=None, initial=SQLSelectForm(initial=copy(query)).initial - ) - - if query["sql"]: - query["sql"] = reformat_sql(query["sql"], with_toggle=True) - query["is_slow"] = query["duration"] > sql_warning_threshold query["is_select"] = is_select_query(query["raw_sql"]) @@ -288,9 +281,6 @@ def generate_stats(self, request, response): query["start_offset"] = width_ratio_tally query["end_offset"] = query["width_ratio"] + query["start_offset"] width_ratio_tally += query["width_ratio"] - query["stacktrace"] = render_stacktrace(query["stacktrace"]) - - query["trace_color"] = trace_colors[query["stacktrace"]] last_by_alias[alias] = query @@ -323,3 +313,38 @@ def generate_server_timing(self, request, response): title = "SQL {} queries".format(len(stats.get("queries", []))) value = stats.get("sql_time", 0) self.record_server_timing("sql_time", title, value) + + def record_stats(self, stats): + """ + Store data gathered by the panel. ``stats`` is a :class:`dict`. + + Each call to ``record_stats`` updates the statistics dictionary. + """ + for query in stats.get("queries", []): + query["params"] + return super().record_stats(stats) + + # Cache the content property since it manipulates the queries in the stats + # This allows the caller to treat content as idempotent + @cached_property + def content(self): + if self.has_content: + stats = self.get_stats() + colors = contrasting_color_generator() + trace_colors = defaultdict(lambda: next(colors)) + + for query in stats.get("queries", []): + query["sql"] = reformat_sql(query["sql"], with_toggle=True) + query["form"] = SignedDataForm( + auto_id=None, + initial=SQLSelectForm( + initial={ + "djdt_query_id": query["djdt_query_id"], + "request_id": self.toolbar.request_id, + } + ).initial, + ) + query["stacktrace"] = render_stacktrace(query["stacktrace"]) + query["trace_color"] = trace_colors[query["stacktrace"]] + + return render_to_string(self.template, stats) diff --git a/debug_toolbar/panels/sql/tracking.py b/debug_toolbar/panels/sql/tracking.py index 477106fdd..45e0c0c17 100644 --- a/debug_toolbar/panels/sql/tracking.py +++ b/debug_toolbar/panels/sql/tracking.py @@ -184,7 +184,6 @@ def _record(self, method, sql, params): "duration": duration, "raw_sql": sql, "params": _params, - "raw_params": params, "stacktrace": get_stack_trace(skip=2), "template_info": template_info, } diff --git a/debug_toolbar/panels/sql/views.py b/debug_toolbar/panels/sql/views.py index b3ad6debb..4da009eba 100644 --- a/debug_toolbar/panels/sql/views.py +++ b/debug_toolbar/panels/sql/views.py @@ -6,6 +6,7 @@ from debug_toolbar.decorators import render_with_toolbar_language, require_show_toolbar from debug_toolbar.forms import SignedDataForm from debug_toolbar.panels.sql.forms import SQLSelectForm +from debug_toolbar.panels.sql.utils import reformat_sql def get_signed_data(request): @@ -29,19 +30,14 @@ def sql_select(request): form = SQLSelectForm(verified_data) if form.is_valid(): - sql = form.cleaned_data["raw_sql"] - params = form.cleaned_data["params"] - with form.cursor as cursor: - cursor.execute(sql, params) - headers = [d[0] for d in cursor.description] - result = cursor.fetchall() - + query = form.cleaned_data["query"] + result, headers = form.select() context = { "result": result, - "sql": form.reformat_sql(), - "duration": form.cleaned_data["duration"], + "sql": reformat_sql(query["sql"], with_toggle=False), + "duration": query["duration"], "headers": headers, - "alias": form.cleaned_data["alias"], + "alias": query["alias"], } content = render_to_string("debug_toolbar/panels/sql_select.html", context) return JsonResponse({"content": content}) @@ -60,28 +56,14 @@ def sql_explain(request): form = SQLSelectForm(verified_data) if form.is_valid(): - sql = form.cleaned_data["raw_sql"] - params = form.cleaned_data["params"] - vendor = form.connection.vendor - with form.cursor as cursor: - if vendor == "sqlite": - # SQLite's EXPLAIN dumps the low-level opcodes generated for a query; - # EXPLAIN QUERY PLAN dumps a more human-readable summary - # See https://www.sqlite.org/lang_explain.html for details - cursor.execute(f"EXPLAIN QUERY PLAN {sql}", params) - elif vendor == "postgresql": - cursor.execute(f"EXPLAIN ANALYZE {sql}", params) - else: - cursor.execute(f"EXPLAIN {sql}", params) - headers = [d[0] for d in cursor.description] - result = cursor.fetchall() - + query = form.cleaned_data["query"] + result, headers = form.explain() context = { "result": result, - "sql": form.reformat_sql(), - "duration": form.cleaned_data["duration"], + "sql": reformat_sql(query["sql"], with_toggle=False), + "duration": query["duration"], "headers": headers, - "alias": form.cleaned_data["alias"], + "alias": query["alias"], } content = render_to_string("debug_toolbar/panels/sql_explain.html", context) return JsonResponse({"content": content}) @@ -100,45 +82,24 @@ def sql_profile(request): form = SQLSelectForm(verified_data) if form.is_valid(): - sql = form.cleaned_data["raw_sql"] - params = form.cleaned_data["params"] + query = form.cleaned_data["query"] result = None headers = None result_error = None - with form.cursor as cursor: - try: - cursor.execute("SET PROFILING=1") # Enable profiling - cursor.execute(sql, params) # Execute SELECT - cursor.execute("SET PROFILING=0") # Disable profiling - # The Query ID should always be 1 here but I'll subselect to get - # the last one just in case... - cursor.execute( - """ - SELECT * - FROM information_schema.profiling - WHERE query_id = ( - SELECT query_id - FROM information_schema.profiling - ORDER BY query_id DESC - LIMIT 1 - ) - """ - ) - headers = [d[0] for d in cursor.description] - result = cursor.fetchall() - except Exception: - result_error = ( - "Profiling is either not available or not supported by your " - "database." - ) + try: + result, headers = form.profile() + except Exception: + result_error = ( + "Profiling is either not available or not supported by your database." + ) context = { "result": result, "result_error": result_error, "sql": form.reformat_sql(), - "duration": form.cleaned_data["duration"], + "duration": query["duration"], "headers": headers, - "alias": form.cleaned_data["alias"], + "alias": query["alias"], } content = render_to_string("debug_toolbar/panels/sql_profile.html", context) return JsonResponse({"content": content}) diff --git a/debug_toolbar/panels/staticfiles.py b/debug_toolbar/panels/staticfiles.py index 9f1970ef6..544caab81 100644 --- a/debug_toolbar/panels/staticfiles.py +++ b/debug_toolbar/panels/staticfiles.py @@ -63,9 +63,10 @@ class StaticFilesPanel(panels.Panel): @property def title(self): + stats = self.get_stats() return _("Static files (%(num_found)s found, %(num_used)s used)") % { - "num_found": self.num_found, - "num_used": self.num_used, + "num_found": stats.get("num_found"), + "num_used": stats.get("num_used"), } def __init__(self, *args, **kwargs): @@ -97,16 +98,11 @@ def disable_instrumentation(self): record_static_file_signal.disconnect(self._store_static_files_signal_handler) request_id_context_var.reset(self.ctx_token) - @property - def num_used(self): - stats = self.get_stats() - return stats and stats["num_used"] - nav_title = _("Static files") @property def nav_subtitle(self): - num_used = self.num_used + num_used = self.get_stats().get("num_used") return ngettext( "%(num_used)s file used", "%(num_used)s files used", num_used ) % {"num_used": num_used} diff --git a/debug_toolbar/panels/templates/__init__.py b/debug_toolbar/panels/templates/__init__.py index a1d509b9e..5cd78bbb3 100644 --- a/debug_toolbar/panels/templates/__init__.py +++ b/debug_toolbar/panels/templates/__init__.py @@ -1,3 +1,3 @@ from debug_toolbar.panels.templates.panel import TemplatesPanel -__all__ = ["TemplatesPanel"] +__all__ = [TemplatesPanel.panel_id] diff --git a/debug_toolbar/panels/templates/panel.py b/debug_toolbar/panels/templates/panel.py index e9a5b4e83..6dbd02ee0 100644 --- a/debug_toolbar/panels/templates/panel.py +++ b/debug_toolbar/panels/templates/panel.py @@ -10,6 +10,7 @@ from django.test.signals import template_rendered from django.test.utils import instrumented_test_render from django.urls import path +from django.utils.encoding import force_str from django.utils.translation import gettext_lazy as _ from debug_toolbar.panels import Panel @@ -104,15 +105,16 @@ def _store_template_info(self, sender, **kwargs): @property def title(self): - num_templates = len(self.templates) + num_templates = len(self.get_stats()["templates"]) return _("Templates (%(num_templates)s rendered)") % { "num_templates": num_templates } @property def nav_subtitle(self): - if self.templates: - return self.templates[0]["template"].name + templates = self.get_stats()["templates"] + if templates: + return templates[0]["template"]["name"] return "" template = "debug_toolbar/panels/templates.html" @@ -196,7 +198,11 @@ def generate_stats(self, request, response): else: template.origin_name = _("No origin") template.origin_hash = "" - info["template"] = template + info["template"] = { + "name": template.name, + "origin_name": template.origin_name, + "origin_hash": template.origin_hash, + } # Clean up context for better readability if self.toolbar.config["SHOW_TEMPLATE_CONTEXT"]: if "context_list" not in template_data: @@ -208,7 +214,14 @@ def generate_stats(self, request, response): # Fetch context_processors/template_dirs from any template if self.templates: - context_processors = self.templates[0]["context_processors"] + context_processors = ( + { + key: force_str(value) + for key, value in self.templates[0]["context_processors"].items() + } + if self.templates[0]["context_processors"] + else None + ) template = self.templates[0]["template"] # django templates have the 'engine' attribute, while jinja # templates use 'backend' diff --git a/debug_toolbar/panels/timer.py b/debug_toolbar/panels/timer.py index 554798e7d..6ef9f0d7c 100644 --- a/debug_toolbar/panels/timer.py +++ b/debug_toolbar/panels/timer.py @@ -19,11 +19,11 @@ class TimerPanel(Panel): def nav_subtitle(self): stats = self.get_stats() - if hasattr(self, "_start_rusage"): - utime = self._end_rusage.ru_utime - self._start_rusage.ru_utime - stime = self._end_rusage.ru_stime - self._start_rusage.ru_stime + if stats.get("utime"): + utime = stats.get("utime") + stime = stats.get("stime") return _("CPU: %(cum)0.2fms (%(total)0.2fms)") % { - "cum": (utime + stime) * 1000.0, + "cum": (utime + stime), "total": stats["total_time"], } elif "total_time" in stats: @@ -64,27 +64,44 @@ def process_request(self, request): self._start_rusage = resource.getrusage(resource.RUSAGE_SELF) return super().process_request(request) + def serialize_rusage(self, data): + fields_to_serialize = [ + "ru_utime", + "ru_stime", + "ru_nvcsw", + "ru_nivcsw", + "ru_minflt", + "ru_majflt", + ] + return {field: getattr(data, field) for field in fields_to_serialize} + def generate_stats(self, request, response): stats = {} if hasattr(self, "_start_time"): stats["total_time"] = (perf_counter() - self._start_time) * 1000 - if hasattr(self, "_start_rusage"): + if self.has_content: self._end_rusage = resource.getrusage(resource.RUSAGE_SELF) - stats["utime"] = 1000 * self._elapsed_ru("ru_utime") - stats["stime"] = 1000 * self._elapsed_ru("ru_stime") + start = self.serialize_rusage(self._start_rusage) + end = self.serialize_rusage(self._end_rusage) + stats.update( + { + "utime": 1000 * self._elapsed_ru(start, end, "ru_utime"), + "stime": 1000 * self._elapsed_ru(start, end, "ru_stime"), + "vcsw": self._elapsed_ru(start, end, "ru_nvcsw"), + "ivcsw": self._elapsed_ru(start, end, "ru_nivcsw"), + "minflt": self._elapsed_ru(start, end, "ru_minflt"), + "majflt": self._elapsed_ru(start, end, "ru_majflt"), + } + ) stats["total"] = stats["utime"] + stats["stime"] - stats["vcsw"] = self._elapsed_ru("ru_nvcsw") - stats["ivcsw"] = self._elapsed_ru("ru_nivcsw") - stats["minflt"] = self._elapsed_ru("ru_minflt") - stats["majflt"] = self._elapsed_ru("ru_majflt") # these are documented as not meaningful under Linux. If you're # running BSD feel free to enable them, and add any others that I # hadn't gotten to before I noticed that I was getting nothing but # zeroes and that the docs agreed. :-( # - # stats['blkin'] = self._elapsed_ru('ru_inblock') - # stats['blkout'] = self._elapsed_ru('ru_oublock') - # stats['swap'] = self._elapsed_ru('ru_nswap') + # stats['blkin'] = self._elapsed_ru(start, end, 'ru_inblock') + # stats['blkout'] = self._elapsed_ru(start, end, 'ru_oublock') + # stats['swap'] = self._elapsed_ru(start, end, 'ru_nswap') # stats['rss'] = self._end_rusage.ru_maxrss # stats['srss'] = self._end_rusage.ru_ixrss # stats['urss'] = self._end_rusage.ru_idrss @@ -102,5 +119,6 @@ def generate_server_timing(self, request, response): "total_time", "Elapsed time", stats.get("total_time", 0) ) - def _elapsed_ru(self, name): - return getattr(self._end_rusage, name) - getattr(self._start_rusage, name) + @staticmethod + def _elapsed_ru(start, end, name): + return end.get(name) - start.get(name) diff --git a/debug_toolbar/panels/versions.py b/debug_toolbar/panels/versions.py index 77915e78b..491bdcfc9 100644 --- a/debug_toolbar/panels/versions.py +++ b/debug_toolbar/panels/versions.py @@ -16,7 +16,7 @@ class VersionsPanel(Panel): @property def nav_subtitle(self): - return f"Django {django.get_version()}" + return "Django %s" % self.get_stats()["django_version"] title = _("Versions") @@ -29,7 +29,11 @@ def generate_stats(self, request, response): ] versions += list(self.gen_app_versions()) self.record_stats( - {"versions": sorted(versions, key=lambda v: v[0]), "paths": sys.path} + { + "django_version": django.get_version(), + "versions": sorted(versions, key=lambda v: v[0]), + "paths": sys.path, + } ) def gen_app_versions(self): diff --git a/debug_toolbar/settings.py b/debug_toolbar/settings.py index 59d538a0b..db49d7b32 100644 --- a/debug_toolbar/settings.py +++ b/debug_toolbar/settings.py @@ -1,3 +1,4 @@ +import os import sys import warnings from functools import cache @@ -43,7 +44,8 @@ "SQL_WARNING_THRESHOLD": 500, # milliseconds "OBSERVE_REQUEST_CALLBACK": "debug_toolbar.toolbar.observe_request", "TOOLBAR_LANGUAGE": None, - "IS_RUNNING_TESTS": "test" in sys.argv, + "TOOLBAR_STORE_CLASS": "debug_toolbar.store.MemoryStore", + "IS_RUNNING_TESTS": "test" in sys.argv or "PYTEST_VERSION" in os.environ, "UPDATE_ON_FETCH": False, } diff --git a/debug_toolbar/static/debug_toolbar/js/history.js b/debug_toolbar/static/debug_toolbar/js/history.js index d10156660..c2644eca7 100644 --- a/debug_toolbar/static/debug_toolbar/js/history.js +++ b/debug_toolbar/static/debug_toolbar/js/history.js @@ -21,14 +21,17 @@ function refreshHistory() { const formTarget = djDebug.querySelector(".refreshHistory"); const container = document.getElementById("djdtHistoryRequests"); const oldIds = new Set( - pluckData(container.querySelectorAll("tr[data-store-id]"), "storeId") + pluckData( + container.querySelectorAll("tr[data-request-id]"), + "requestId" + ) ); ajaxForm(formTarget) .then((data) => { // Remove existing rows first then re-populate with new data for (const node of container.querySelectorAll( - "tr[data-store-id]" + "tr[data-request-id]" )) { node.remove(); } @@ -39,8 +42,8 @@ function refreshHistory() { .then(() => { const allIds = new Set( pluckData( - container.querySelectorAll("tr[data-store-id]"), - "storeId" + container.querySelectorAll("tr[data-request-id]"), + "requestId" ) ); const newIds = difference(allIds, oldIds); @@ -54,13 +57,13 @@ function refreshHistory() { .then((refreshInfo) => { for (const newId of refreshInfo.newIds) { const row = container.querySelector( - `tr[data-store-id="${newId}"]` + `tr[data-request-id="${newId}"]` ); row.classList.add("flash-new"); } setTimeout(() => { for (const row of container.querySelectorAll( - "tr[data-store-id]" + "tr[data-request-id]" )) { row.classList.remove("flash-new"); } @@ -68,9 +71,9 @@ function refreshHistory() { }); } -function switchHistory(newStoreId) { +function switchHistory(newRequestId) { const formTarget = djDebug.querySelector( - `.switchHistory[data-store-id='${newStoreId}']` + `.switchHistory[data-request-id='${newRequestId}']` ); const tbody = formTarget.closest("tbody"); @@ -84,16 +87,16 @@ function switchHistory(newStoreId) { if (Object.keys(data).length === 0) { const container = document.getElementById("djdtHistoryRequests"); container.querySelector( - `button[data-store-id="${newStoreId}"]` + `button[data-request-id="${newRequestId}"]` ).innerHTML = "Switch [EXPIRED]"; } - replaceToolbarState(newStoreId, data); + replaceToolbarState(newRequestId, data); }); } $$.on(djDebug, "click", ".switchHistory", function (event) { event.preventDefault(); - switchHistory(this.dataset.storeId); + switchHistory(this.dataset.requestId); }); $$.on(djDebug, "click", ".refreshHistory", (event) => { diff --git a/debug_toolbar/static/debug_toolbar/js/toolbar.js b/debug_toolbar/static/debug_toolbar/js/toolbar.js index 19658f76e..609842209 100644 --- a/debug_toolbar/static/debug_toolbar/js/toolbar.js +++ b/debug_toolbar/static/debug_toolbar/js/toolbar.js @@ -39,13 +39,13 @@ const djdt = { const inner = current.querySelector( ".djDebugPanelContent .djdt-scroll" ); - const storeId = djDebug.dataset.storeId; - if (storeId && inner.children.length === 0) { + const requestId = djDebug.dataset.requestId; + if (requestId && inner.children.length === 0) { const url = new URL( djDebug.dataset.renderPanelUrl, window.location ); - url.searchParams.append("store_id", storeId); + url.searchParams.append("request_id", requestId); url.searchParams.append("panel_id", panelId); ajax(url).then((data) => { inner.previousElementSibling.remove(); // Remove AJAX loader @@ -296,12 +296,12 @@ const djdt = { document.getElementById("djDebug").dataset.sidebarUrl; const slowjax = debounce(ajax, 200); - function handleAjaxResponse(storeId) { - const encodedStoreId = encodeURIComponent(storeId); - const dest = `${sidebarUrl}?store_id=${encodedStoreId}`; + function handleAjaxResponse(requestId) { + const encodedRequestId = encodeURIComponent(requestId); + const dest = `${sidebarUrl}?request_id=${encodedRequestId}`; slowjax(dest).then((data) => { if (djdt.needUpdateOnFetch) { - replaceToolbarState(encodedStoreId, data); + replaceToolbarState(encodedRequestId, data); } }); } @@ -314,9 +314,11 @@ const djdt = { // when the header can't be fetched. While it doesn't impede execution // it's worrisome to developers. if ( - this.getAllResponseHeaders().indexOf("djdt-store-id") >= 0 + this.getAllResponseHeaders().indexOf("djdt-request-id") >= 0 ) { - handleAjaxResponse(this.getResponseHeader("djdt-store-id")); + handleAjaxResponse( + this.getResponseHeader("djdt-request-id") + ); } }); origOpen.apply(this, args); @@ -330,10 +332,10 @@ const djdt = { // https://github.com/django-commons/django-debug-toolbar/pull/2100 const promise = origFetch.apply(this, args); return promise.then((response) => { - if (response.headers.get("djdt-store-id") !== null) { + if (response.headers.get("djdt-request-id") !== null) { try { handleAjaxResponse( - response.headers.get("djdt-store-id") + response.headers.get("djdt-request-id") ); } catch (err) { throw new Error( diff --git a/debug_toolbar/static/debug_toolbar/js/utils.js b/debug_toolbar/static/debug_toolbar/js/utils.js index 0cfa80474..9b34f86f8 100644 --- a/debug_toolbar/static/debug_toolbar/js/utils.js +++ b/debug_toolbar/static/debug_toolbar/js/utils.js @@ -109,10 +109,10 @@ function ajaxForm(element) { return ajax(url, ajaxData); } -function replaceToolbarState(newStoreId, data) { +function replaceToolbarState(newRequestId, data) { const djDebug = document.getElementById("djDebug"); - djDebug.setAttribute("data-store-id", newStoreId); - // Check if response is empty, it could be due to an expired storeId. + djDebug.setAttribute("data-request-id", newRequestId); + // Check if response is empty, it could be due to an expired requestId. for (const panelId of Object.keys(data)) { const panel = document.getElementById(panelId); if (panel) { diff --git a/debug_toolbar/store.py b/debug_toolbar/store.py new file mode 100644 index 000000000..76526fcff --- /dev/null +++ b/debug_toolbar/store.py @@ -0,0 +1,227 @@ +import contextlib +import json +import logging +from collections import defaultdict, deque +from collections.abc import Iterable +from typing import Any + +from django.core.serializers.json import DjangoJSONEncoder +from django.db import transaction +from django.utils.encoding import force_str +from django.utils.module_loading import import_string + +from debug_toolbar import settings as dt_settings +from debug_toolbar.models import HistoryEntry + +logger = logging.getLogger(__name__) + + +class DebugToolbarJSONEncoder(DjangoJSONEncoder): + def default(self, o): + try: + return super().default(o) + except (TypeError, ValueError): + logger.debug("The debug toolbar can't serialize %s into JSON" % o) + return force_str(o) + + +def serialize(data: Any) -> str: + # If this starts throwing an exceptions, consider + # Subclassing DjangoJSONEncoder and using force_str to + # make it JSON serializable. + return json.dumps(data, cls=DebugToolbarJSONEncoder) + + +def deserialize(data: str) -> Any: + return json.loads(data) + + +class BaseStore: + @classmethod + def request_ids(cls) -> Iterable: + """The stored request ids""" + raise NotImplementedError + + @classmethod + def exists(cls, request_id: str) -> bool: + """Does the given request_id exist in the store""" + raise NotImplementedError + + @classmethod + def set(cls, request_id: str): + """Set a request_id in the store""" + raise NotImplementedError + + @classmethod + def clear(cls): + """Remove all requests from the request store""" + raise NotImplementedError + + @classmethod + def delete(cls, request_id: str): + """Delete the store for the given request_id""" + raise NotImplementedError + + @classmethod + def save_panel(cls, request_id: str, panel_id: str, data: Any = None): + """Save the panel data for the given request_id""" + raise NotImplementedError + + @classmethod + def panel(cls, request_id: str, panel_id: str) -> Any: + """Fetch the panel data for the given request_id""" + raise NotImplementedError + + +class MemoryStore(BaseStore): + # ids is the collection of storage ids that have been used. + # Use a dequeue to support O(1) appends and pops + # from either direction. + _request_ids: deque = deque() + _request_store: dict[str, dict] = defaultdict(dict) + + @classmethod + def request_ids(cls) -> Iterable: + """The stored request ids""" + return cls._request_ids + + @classmethod + def exists(cls, request_id: str) -> bool: + """Does the given request_id exist in the request store""" + return request_id in cls._request_ids + + @classmethod + def set(cls, request_id: str): + """Set a request_id in the request store""" + if request_id not in cls._request_ids: + cls._request_ids.append(request_id) + for _ in range( + len(cls._request_ids) - dt_settings.get_config()["RESULTS_CACHE_SIZE"] + ): + removed_id = cls._request_ids.popleft() + cls._request_store.pop(removed_id, None) + + @classmethod + def clear(cls): + """Remove all requests from the request store""" + cls._request_ids.clear() + cls._request_store.clear() + + @classmethod + def delete(cls, request_id: str): + """Delete the stored request for the given request_id""" + cls._request_store.pop(request_id, None) + # Suppress when request_id doesn't exist in the collection of ids. + with contextlib.suppress(ValueError): + cls._request_ids.remove(request_id) + + @classmethod + def save_panel(cls, request_id: str, panel_id: str, data: Any = None): + """Save the panel data for the given request_id""" + cls.set(request_id) + cls._request_store[request_id][panel_id] = serialize(data) + + @classmethod + def panel(cls, request_id: str, panel_id: str) -> Any: + """Fetch the panel data for the given request_id""" + try: + data = cls._request_store[request_id][panel_id] + except KeyError: + return {} + else: + return deserialize(data) + + @classmethod + def panels(cls, request_id: str) -> Any: + """Fetch all the panel data for the given request_id""" + try: + panel_mapping = cls._request_store[request_id] + except KeyError: + return {} + for panel, data in panel_mapping.items(): + yield panel, deserialize(data) + + +class DatabaseStore(BaseStore): + @classmethod + def _cleanup_old_entries(cls): + """ + Enforce the cache size limit - keeping only the most recently used entries + up to RESULTS_CACHE_SIZE. + """ + # Determine which entries to keep + keep_ids = cls.request_ids() + + # Delete all entries not in the keep list + if keep_ids: + HistoryEntry.objects.exclude(request_id__in=keep_ids).delete() + + @classmethod + def request_ids(cls): + """Return all stored request ids within the cache size limit""" + cache_size = dt_settings.get_config()["RESULTS_CACHE_SIZE"] + return list( + HistoryEntry.objects.all()[:cache_size].values_list("request_id", flat=True) + ) + + @classmethod + def exists(cls, request_id: str) -> bool: + """Check if the given request_id exists in the store""" + return HistoryEntry.objects.filter(request_id=request_id).exists() + + @classmethod + def set(cls, request_id: str): + """Set a request_id in the store and clean up old entries""" + with transaction.atomic(): + # Create the entry if it doesn't exist (ignore otherwise) + _, created = HistoryEntry.objects.get_or_create(request_id=request_id) + + # Only enforce cache size limit when new entries are created + if created: + cls._cleanup_old_entries() + + @classmethod + def clear(cls): + """Remove all requests from the store""" + HistoryEntry.objects.all().delete() + + @classmethod + def delete(cls, request_id: str): + """Delete the stored request for the given request_id""" + HistoryEntry.objects.filter(request_id=request_id).delete() + + @classmethod + def save_panel(cls, request_id: str, panel_id: str, data: Any = None): + """Save the panel data for the given request_id""" + with transaction.atomic(): + obj, _ = HistoryEntry.objects.get_or_create(request_id=request_id) + store_data = obj.data + store_data[panel_id] = serialize(data) + obj.data = store_data + obj.save() + + @classmethod + def panel(cls, request_id: str, panel_id: str) -> Any: + """Fetch the panel data for the given request_id""" + try: + data = HistoryEntry.objects.get(request_id=request_id).data + panel_data = data.get(panel_id) + if panel_data is None: + return {} + return deserialize(panel_data) + except HistoryEntry.DoesNotExist: + return {} + + @classmethod + def panels(cls, request_id: str) -> Any: + """Fetch all panel data for the given request_id""" + try: + data = HistoryEntry.objects.get(request_id=request_id).data + for panel_id, panel_data in data.items(): + yield panel_id, deserialize(panel_data) + except HistoryEntry.DoesNotExist: + return {} + + +def get_store() -> BaseStore: + return import_string(dt_settings.get_config()["TOOLBAR_STORE_CLASS"]) diff --git a/debug_toolbar/templates/debug_toolbar/base.html b/debug_toolbar/templates/debug_toolbar/base.html index 607863104..b7562ecba 100644 --- a/debug_toolbar/templates/debug_toolbar/base.html +++ b/debug_toolbar/templates/debug_toolbar/base.html @@ -8,7 +8,7 @@ {% endblock js %}
{{ store_context.toolbar.stats.HistoryPanel.request_method|escape }}
+{{ history_context.history_stats.request_method|escape }}
{{ store_context.toolbar.stats.HistoryPanel.request_url|truncatechars:100|escape }}
+{{ history_context.history_stats.request_url|truncatechars:100|escape }}
{{ key|pprint }} |
{{ value|pprint }} |
@@ -39,12 +39,12 @@
- {{ store_context.toolbar.stats.HistoryPanel.status_code|escape }} +{{ history_context.history_stats.status_code|escape }} |