diff --git a/.gitignore b/.gitignore index c89013a11..a344568c0 100644 --- a/.gitignore +++ b/.gitignore @@ -12,6 +12,7 @@ htmlcov .tox geckodriver.log coverage.xml +venv .direnv/ .envrc venv diff --git a/Makefile b/Makefile index 4d2db27af..8c1b2d0ab 100644 --- a/Makefile +++ b/Makefile @@ -7,6 +7,12 @@ example: ## Run the example application --noinput --username="$(USER)" --email="$(USER)@mailinator.com" python example/manage.py runserver +example_async: + python example/manage.py migrate --noinput + -DJANGO_SUPERUSER_PASSWORD=p python example/manage.py createsuperuser \ + --noinput --username="$(USER)" --email="$(USER)@mailinator.com" + daphne example.asgi:application + example_test: ## Run the test suite for the example application python example/manage.py test example diff --git a/debug_toolbar/migrations/0001_initial.py b/debug_toolbar/migrations/0001_initial.py new file mode 100644 index 000000000..e4d30fede --- /dev/null +++ b/debug_toolbar/migrations/0001_initial.py @@ -0,0 +1,24 @@ +from django.db import migrations, models + + +class Migration(migrations.Migration): + initial = True + + operations = [ + migrations.CreateModel( + name="HistoryEntry", + fields=[ + ( + "request_id", + models.UUIDField(primary_key=True, serialize=False), + ), + ("data", models.JSONField(default=dict)), + ("created_at", models.DateTimeField(auto_now_add=True)), + ], + options={ + "verbose_name": "history entry", + "verbose_name_plural": "history entries", + "ordering": ["-created_at"], + }, + ), + ] diff --git a/debug_toolbar/migrations/__init__.py b/debug_toolbar/migrations/__init__.py new file mode 100644 index 000000000..e69de29bb diff --git a/debug_toolbar/models.py b/debug_toolbar/models.py new file mode 100644 index 000000000..686ac4cfa --- /dev/null +++ b/debug_toolbar/models.py @@ -0,0 +1,16 @@ +from django.db import models +from django.utils.translation import gettext_lazy as _ + + +class HistoryEntry(models.Model): + request_id = models.UUIDField(primary_key=True) + data = models.JSONField(default=dict) + created_at = models.DateTimeField(auto_now_add=True) + + class Meta: + verbose_name = _("history entry") + verbose_name_plural = _("history entries") + ordering = ["-created_at"] + + def __str__(self): + return str(self.request_id) diff --git a/debug_toolbar/panels/__init__.py b/debug_toolbar/panels/__init__.py index 217708ec2..a53ba6652 100644 --- a/debug_toolbar/panels/__init__.py +++ b/debug_toolbar/panels/__init__.py @@ -1,5 +1,6 @@ from django.core.handlers.asgi import ASGIRequest from django.template.loader import render_to_string +from django.utils.functional import classproperty from debug_toolbar import settings as dt_settings from debug_toolbar.utils import get_name_from_obj @@ -15,19 +16,27 @@ class Panel: def __init__(self, toolbar, get_response): self.toolbar = toolbar self.get_response = get_response + self.from_store = False # Private panel properties - @property - def panel_id(self): - return self.__class__.__name__ + @classproperty + def panel_id(cls): + return cls.__name__ @property def enabled(self) -> bool: - # check if the panel is async compatible + # Check if the panel is async compatible if not self.is_async and isinstance(self.toolbar.request, ASGIRequest): return False + if self.from_store: + # If the toolbar was loaded from the store the existence of + # recorded data indicates whether it was enabled or not. + # We can't use the remainder of the logic since we don't have + # a request to work off of. + return bool(self.get_stats()) + # The user's cookies should override the default value cookie_value = self.toolbar.request.COOKIES.get("djdt" + self.panel_id) if cookie_value is not None: @@ -175,9 +184,16 @@ def record_stats(self, stats): """ Store data gathered by the panel. ``stats`` is a :class:`dict`. - Each call to ``record_stats`` updates the statistics dictionary. + Each call to ``record_stats`` updates the store's data for + the panel. + + To support backwards compatibility, it will also update the + panel's statistics dictionary. """ self.toolbar.stats.setdefault(self.panel_id, {}).update(stats) + self.toolbar.store.save_panel( + self.toolbar.request_id, self.panel_id, self.toolbar.stats[self.panel_id] + ) def get_stats(self): """ @@ -261,6 +277,15 @@ def generate_server_timing(self, request, response): Does not return a value. """ + def load_stats_from_store(self, data): + """ + Instantiate the panel from serialized data. + + Return the panel instance. + """ + self.toolbar.stats.setdefault(self.panel_id, {}).update(data) + self.from_store = True + @classmethod def run_checks(cls): """ diff --git a/debug_toolbar/panels/alerts.py b/debug_toolbar/panels/alerts.py index c8e59002c..030ca1ee1 100644 --- a/debug_toolbar/panels/alerts.py +++ b/debug_toolbar/panels/alerts.py @@ -141,11 +141,9 @@ def check_invalid_file_form_configuration(self, html_content): return self.alerts def generate_stats(self, request, response): - if not is_processable_html_response(response): - return - - html_content = response.content.decode(response.charset) - self.check_invalid_file_form_configuration(html_content) + if is_processable_html_response(response): + html_content = response.content.decode(response.charset) + self.check_invalid_file_form_configuration(html_content) # Further alert checks can go here diff --git a/debug_toolbar/panels/cache.py b/debug_toolbar/panels/cache.py index 1b15b446f..d3242d9d9 100644 --- a/debug_toolbar/panels/cache.py +++ b/debug_toolbar/panels/cache.py @@ -171,16 +171,17 @@ def _record_call(self, cache, name, original_method, args, kwargs): @property def nav_subtitle(self): - cache_calls = len(self.calls) + stats = self.get_stats() + cache_calls = len(stats.get("calls")) return ngettext( "%(cache_calls)d call in %(time).2fms", "%(cache_calls)d calls in %(time).2fms", cache_calls, - ) % {"cache_calls": cache_calls, "time": self.total_time} + ) % {"cache_calls": cache_calls, "time": stats.get("total_time")} @property def title(self): - count = len(getattr(settings, "CACHES", ["default"])) + count = self.get_stats().get("total_caches") return ngettext( "Cache calls from %(count)d backend", "Cache calls from %(count)d backends", @@ -216,6 +217,7 @@ def generate_stats(self, request, response): "hits": self.hits, "misses": self.misses, "counts": self.counts, + "total_caches": len(getattr(settings, "CACHES", ["default"])), } ) diff --git a/debug_toolbar/panels/history/__init__.py b/debug_toolbar/panels/history/__init__.py index 52ceb7984..193ced242 100644 --- a/debug_toolbar/panels/history/__init__.py +++ b/debug_toolbar/panels/history/__init__.py @@ -1,3 +1,3 @@ from debug_toolbar.panels.history.panel import HistoryPanel -__all__ = ["HistoryPanel"] +__all__ = [HistoryPanel.panel_id] diff --git a/debug_toolbar/panels/history/forms.py b/debug_toolbar/panels/history/forms.py index 952b2409d..2aec18c34 100644 --- a/debug_toolbar/panels/history/forms.py +++ b/debug_toolbar/panels/history/forms.py @@ -5,8 +5,8 @@ class HistoryStoreForm(forms.Form): """ Validate params - store_id: The key for the store instance to be fetched. + request_id: The key for the store instance to be fetched. """ - store_id = forms.CharField(widget=forms.HiddenInput()) + request_id = forms.CharField(widget=forms.HiddenInput()) exclude_history = forms.BooleanField(widget=forms.HiddenInput(), required=False) diff --git a/debug_toolbar/panels/history/panel.py b/debug_toolbar/panels/history/panel.py index 56a891848..d2f3f8ab6 100644 --- a/debug_toolbar/panels/history/panel.py +++ b/debug_toolbar/panels/history/panel.py @@ -24,9 +24,9 @@ class HistoryPanel(Panel): def get_headers(self, request): headers = super().get_headers(request) observe_request = self.toolbar.get_observe_request() - store_id = self.toolbar.store_id - if store_id and observe_request(request): - headers["djdt-store-id"] = store_id + request_id = self.toolbar.request_id + if request_id and observe_request(request): + headers["djdt-request-id"] = request_id return headers @property @@ -87,23 +87,25 @@ def content(self): Fetch every store for the toolbar and include it in the template. """ - stores = {} - for id, toolbar in reversed(self.toolbar._store.items()): - stores[id] = { - "toolbar": toolbar, + toolbar_history = {} + for request_id in reversed(self.toolbar.store.request_ids()): + toolbar_history[request_id] = { + "history_stats": self.toolbar.store.panel( + request_id, HistoryPanel.panel_id + ), "form": HistoryStoreForm( - initial={"store_id": id, "exclude_history": True} + initial={"request_id": request_id, "exclude_history": True} ), } return render_to_string( self.template, { - "current_store_id": self.toolbar.store_id, - "stores": stores, + "current_request_id": self.toolbar.request_id, + "toolbar_history": toolbar_history, "refresh_form": HistoryStoreForm( initial={ - "store_id": self.toolbar.store_id, + "request_id": self.toolbar.request_id, "exclude_history": True, } ), diff --git a/debug_toolbar/panels/history/views.py b/debug_toolbar/panels/history/views.py index fb6e28c93..2dd6de820 100644 --- a/debug_toolbar/panels/history/views.py +++ b/debug_toolbar/panels/history/views.py @@ -4,6 +4,7 @@ from debug_toolbar._compat import login_not_required from debug_toolbar.decorators import render_with_toolbar_language, require_show_toolbar from debug_toolbar.panels.history.forms import HistoryStoreForm +from debug_toolbar.store import get_store from debug_toolbar.toolbar import DebugToolbar @@ -15,12 +16,12 @@ def history_sidebar(request): form = HistoryStoreForm(request.GET) if form.is_valid(): - store_id = form.cleaned_data["store_id"] - toolbar = DebugToolbar.fetch(store_id) + request_id = form.cleaned_data["request_id"] + toolbar = DebugToolbar.fetch(request_id) exclude_history = form.cleaned_data["exclude_history"] context = {} if toolbar is None: - # When the store_id has been popped already due to + # When the request_id has been popped already due to # RESULTS_CACHE_SIZE return JsonResponse(context) for panel in toolbar.panels: @@ -36,7 +37,7 @@ def history_sidebar(request): ), } return JsonResponse(context) - return HttpResponseBadRequest("Form errors") + return HttpResponseBadRequest(f"Form errors: {form.errors}") @login_not_required @@ -49,19 +50,22 @@ def history_refresh(request): if form.is_valid(): requests = [] # Convert to list to handle mutations happening in parallel - for id, toolbar in list(DebugToolbar._store.items()): + for request_id in get_store().request_ids(): + toolbar = DebugToolbar.fetch(request_id) requests.append( { - "id": id, + "id": request_id, "content": render_to_string( "debug_toolbar/panels/history_tr.html", { - "id": id, - "store_context": { - "toolbar": toolbar, + "request_id": request_id, + "history_context": { + "history_stats": toolbar.store.panel( + request_id, "HistoryPanel" + ), "form": HistoryStoreForm( initial={ - "store_id": id, + "request_id": request_id, "exclude_history": True, } ), diff --git a/debug_toolbar/panels/profiling.py b/debug_toolbar/panels/profiling.py index 4613a3cad..263e7a004 100644 --- a/debug_toolbar/panels/profiling.py +++ b/debug_toolbar/panels/profiling.py @@ -25,6 +25,7 @@ def __init__( self.id = id self.parent_ids = parent_ids or [] self.hsv = hsv + self.has_subfuncs = False def parent_classes(self): return self.parent_classes @@ -130,6 +131,21 @@ def cumtime_per_call(self): def indent(self): return 16 * self.depth + def serialize(self): + return { + "has_subfuncs": self.has_subfuncs, + "id": self.id, + "parent_ids": self.parent_ids, + "is_project_func": self.is_project_func(), + "indent": self.indent(), + "func_std_string": self.func_std_string(), + "cumtime": self.cumtime(), + "cumtime_per_call": self.cumtime_per_call(), + "tottime": self.tottime(), + "tottime_per_call": self.tottime_per_call(), + "count": self.count(), + } + class ProfilingPanel(Panel): """ @@ -148,7 +164,6 @@ def process_request(self, request): def add_node(self, func_list, func, max_depth, cum_time): func_list.append(func) - func.has_subfuncs = False if func.depth < max_depth: for subfunc in func.subfuncs(): # Always include the user's code @@ -182,4 +197,4 @@ def generate_stats(self, request, response): dt_settings.get_config()["PROFILER_MAX_DEPTH"], cum_time_threshold, ) - self.record_stats({"func_list": func_list}) + self.record_stats({"func_list": [func.serialize() for func in func_list]}) diff --git a/debug_toolbar/panels/settings.py b/debug_toolbar/panels/settings.py index ff32bd2c0..68ab44c0b 100644 --- a/debug_toolbar/panels/settings.py +++ b/debug_toolbar/panels/settings.py @@ -1,4 +1,4 @@ -from django.conf import settings +from django.utils.encoding import force_str from django.utils.translation import gettext_lazy as _ from django.views.debug import get_default_exception_reporter_filter @@ -19,7 +19,16 @@ class SettingsPanel(Panel): nav_title = _("Settings") def title(self): - return _("Settings from %s") % settings.SETTINGS_MODULE + return _("Settings from %s") % self.get_stats()["settings"].get( + "SETTINGS_MODULE" + ) def generate_stats(self, request, response): - self.record_stats({"settings": dict(sorted(get_safe_settings().items()))}) + self.record_stats( + { + "settings": { + key: force_str(value) + for key, value in sorted(get_safe_settings().items()) + } + } + ) diff --git a/debug_toolbar/panels/sql/__init__.py b/debug_toolbar/panels/sql/__init__.py index 46c68a3c6..9da548f7f 100644 --- a/debug_toolbar/panels/sql/__init__.py +++ b/debug_toolbar/panels/sql/__init__.py @@ -1,3 +1,3 @@ from debug_toolbar.panels.sql.panel import SQLPanel -__all__ = ["SQLPanel"] +__all__ = [SQLPanel.panel_id] diff --git a/debug_toolbar/panels/sql/forms.py b/debug_toolbar/panels/sql/forms.py index 1fa90ace4..44906924d 100644 --- a/debug_toolbar/panels/sql/forms.py +++ b/debug_toolbar/panels/sql/forms.py @@ -4,25 +4,22 @@ from django.core.exceptions import ValidationError from django.db import connections from django.utils.functional import cached_property +from django.utils.translation import gettext_lazy as _ from debug_toolbar.panels.sql.utils import is_select_query, reformat_sql +from debug_toolbar.toolbar import DebugToolbar class SQLSelectForm(forms.Form): """ Validate params - sql: The sql statement with interpolated params - raw_sql: The sql statement with placeholders - params: JSON encoded parameter values - duration: time for SQL to execute passed in from toolbar just for redisplay + request_id: The identifier for the request + query_id: The identifier for the query """ - sql = forms.CharField() - raw_sql = forms.CharField() - params = forms.CharField() - alias = forms.CharField(required=False, initial="default") - duration = forms.FloatField() + request_id = forms.CharField() + djdt_query_id = forms.CharField() def clean_raw_sql(self): value = self.cleaned_data["raw_sql"] @@ -48,12 +45,91 @@ def clean_alias(self): return value + def clean(self): + from debug_toolbar.panels.sql import SQLPanel + + cleaned_data = super().clean() + toolbar = DebugToolbar.fetch( + self.cleaned_data["request_id"], panel_id=SQLPanel.panel_id + ) + if toolbar is None: + raise ValidationError(_("Data for this panel isn't available anymore.")) + + panel = toolbar.get_panel_by_id(SQLPanel.panel_id) + # Find the query for this form submission + query = None + for q in panel.get_stats()["queries"]: + if q["djdt_query_id"] != self.cleaned_data["djdt_query_id"]: + continue + else: + query = q + break + if not query: + raise ValidationError(_("Invalid query id.")) + cleaned_data["query"] = query + return cleaned_data + + def select(self): + query = self.cleaned_data["query"] + sql = query["raw_sql"] + params = json.loads(query["params"]) + with self.cursor as cursor: + cursor.execute(sql, params) + headers = [d[0] for d in cursor.description] + result = cursor.fetchall() + return result, headers + + def explain(self): + query = self.cleaned_data["query"] + sql = query["raw_sql"] + params = json.loads(query["params"]) + vendor = query["vendor"] + with self.cursor as cursor: + if vendor == "sqlite": + # SQLite's EXPLAIN dumps the low-level opcodes generated for a query; + # EXPLAIN QUERY PLAN dumps a more human-readable summary + # See https://www.sqlite.org/lang_explain.html for details + cursor.execute(f"EXPLAIN QUERY PLAN {sql}", params) + elif vendor == "postgresql": + cursor.execute(f"EXPLAIN ANALYZE {sql}", params) + else: + cursor.execute(f"EXPLAIN {sql}", params) + headers = [d[0] for d in cursor.description] + result = cursor.fetchall() + return result, headers + + def profile(self): + query = self.cleaned_data["query"] + sql = query["raw_sql"] + params = json.loads(query["params"]) + with self.cursor as cursor: + cursor.execute("SET PROFILING=1") # Enable profiling + cursor.execute(sql, params) # Execute SELECT + cursor.execute("SET PROFILING=0") # Disable profiling + # The Query ID should always be 1 here but I'll subselect to get + # the last one just in case... + cursor.execute( + """ + SELECT * + FROM information_schema.profiling + WHERE query_id = ( + SELECT query_id + FROM information_schema.profiling + ORDER BY query_id DESC + LIMIT 1 + ) + """ + ) + headers = [d[0] for d in cursor.description] + result = cursor.fetchall() + return result, headers + def reformat_sql(self): - return reformat_sql(self.cleaned_data["sql"], with_toggle=False) + return reformat_sql(self.cleaned_data["query"]["sql"], with_toggle=False) @property def connection(self): - return connections[self.cleaned_data["alias"]] + return connections[self.cleaned_data["query"]["alias"]] @cached_property def cursor(self): diff --git a/debug_toolbar/panels/sql/panel.py b/debug_toolbar/panels/sql/panel.py index 206686352..6bfd10500 100644 --- a/debug_toolbar/panels/sql/panel.py +++ b/debug_toolbar/panels/sql/panel.py @@ -1,10 +1,11 @@ import uuid from collections import defaultdict -from copy import copy from asgiref.sync import sync_to_async from django.db import connections +from django.template.loader import render_to_string from django.urls import path +from django.utils.functional import cached_property from django.utils.translation import gettext_lazy as _, ngettext from debug_toolbar import settings as dt_settings @@ -86,7 +87,7 @@ def _similar_query_key(query): def _duplicate_query_key(query): - raw_params = () if query["raw_params"] is None else tuple(query["raw_params"]) + raw_params = () if query["params"] is None else tuple(query["params"]) # repr() avoids problems because of unhashable types # (e.g. lists) when used as dictionary keys. # https://github.com/django-commons/django-debug-toolbar/issues/1091 @@ -146,6 +147,7 @@ def current_transaction_id(self, alias): return trans_id def record(self, **kwargs): + kwargs["djdt_query_id"] = uuid.uuid4().hex self._queries.append(kwargs) alias = kwargs["alias"] if alias not in self._databases: @@ -164,19 +166,20 @@ def record(self, **kwargs): @property def nav_subtitle(self): - query_count = len(self._queries) + stats = self.get_stats() + query_count = len(stats.get("queries", [])) return ngettext( "%(query_count)d query in %(sql_time).2fms", "%(query_count)d queries in %(sql_time).2fms", query_count, ) % { "query_count": query_count, - "sql_time": self._sql_time, + "sql_time": stats.get("sql_time"), } @property def title(self): - count = len(self._databases) + count = len(self.get_stats().get("databases")) return ngettext( "SQL queries from %(count)d connection", "SQL queries from %(count)d connections", @@ -211,8 +214,6 @@ def disable_instrumentation(self): connection._djdt_logger = None def generate_stats(self, request, response): - colors = contrasting_color_generator() - trace_colors = defaultdict(lambda: next(colors)) similar_query_groups = defaultdict(list) duplicate_query_groups = defaultdict(list) @@ -269,14 +270,6 @@ def generate_stats(self, request, response): query["trans_status"] = get_transaction_status_display( query["vendor"], query["trans_status"] ) - - query["form"] = SignedDataForm( - auto_id=None, initial=SQLSelectForm(initial=copy(query)).initial - ) - - if query["sql"]: - query["sql"] = reformat_sql(query["sql"], with_toggle=True) - query["is_slow"] = query["duration"] > sql_warning_threshold query["is_select"] = is_select_query(query["raw_sql"]) @@ -288,9 +281,6 @@ def generate_stats(self, request, response): query["start_offset"] = width_ratio_tally query["end_offset"] = query["width_ratio"] + query["start_offset"] width_ratio_tally += query["width_ratio"] - query["stacktrace"] = render_stacktrace(query["stacktrace"]) - - query["trace_color"] = trace_colors[query["stacktrace"]] last_by_alias[alias] = query @@ -323,3 +313,38 @@ def generate_server_timing(self, request, response): title = "SQL {} queries".format(len(stats.get("queries", []))) value = stats.get("sql_time", 0) self.record_server_timing("sql_time", title, value) + + def record_stats(self, stats): + """ + Store data gathered by the panel. ``stats`` is a :class:`dict`. + + Each call to ``record_stats`` updates the statistics dictionary. + """ + for query in stats.get("queries", []): + query["params"] + return super().record_stats(stats) + + # Cache the content property since it manipulates the queries in the stats + # This allows the caller to treat content as idempotent + @cached_property + def content(self): + if self.has_content: + stats = self.get_stats() + colors = contrasting_color_generator() + trace_colors = defaultdict(lambda: next(colors)) + + for query in stats.get("queries", []): + query["sql"] = reformat_sql(query["sql"], with_toggle=True) + query["form"] = SignedDataForm( + auto_id=None, + initial=SQLSelectForm( + initial={ + "djdt_query_id": query["djdt_query_id"], + "request_id": self.toolbar.request_id, + } + ).initial, + ) + query["stacktrace"] = render_stacktrace(query["stacktrace"]) + query["trace_color"] = trace_colors[query["stacktrace"]] + + return render_to_string(self.template, stats) diff --git a/debug_toolbar/panels/sql/tracking.py b/debug_toolbar/panels/sql/tracking.py index 477106fdd..45e0c0c17 100644 --- a/debug_toolbar/panels/sql/tracking.py +++ b/debug_toolbar/panels/sql/tracking.py @@ -184,7 +184,6 @@ def _record(self, method, sql, params): "duration": duration, "raw_sql": sql, "params": _params, - "raw_params": params, "stacktrace": get_stack_trace(skip=2), "template_info": template_info, } diff --git a/debug_toolbar/panels/sql/views.py b/debug_toolbar/panels/sql/views.py index b3ad6debb..4da009eba 100644 --- a/debug_toolbar/panels/sql/views.py +++ b/debug_toolbar/panels/sql/views.py @@ -6,6 +6,7 @@ from debug_toolbar.decorators import render_with_toolbar_language, require_show_toolbar from debug_toolbar.forms import SignedDataForm from debug_toolbar.panels.sql.forms import SQLSelectForm +from debug_toolbar.panels.sql.utils import reformat_sql def get_signed_data(request): @@ -29,19 +30,14 @@ def sql_select(request): form = SQLSelectForm(verified_data) if form.is_valid(): - sql = form.cleaned_data["raw_sql"] - params = form.cleaned_data["params"] - with form.cursor as cursor: - cursor.execute(sql, params) - headers = [d[0] for d in cursor.description] - result = cursor.fetchall() - + query = form.cleaned_data["query"] + result, headers = form.select() context = { "result": result, - "sql": form.reformat_sql(), - "duration": form.cleaned_data["duration"], + "sql": reformat_sql(query["sql"], with_toggle=False), + "duration": query["duration"], "headers": headers, - "alias": form.cleaned_data["alias"], + "alias": query["alias"], } content = render_to_string("debug_toolbar/panels/sql_select.html", context) return JsonResponse({"content": content}) @@ -60,28 +56,14 @@ def sql_explain(request): form = SQLSelectForm(verified_data) if form.is_valid(): - sql = form.cleaned_data["raw_sql"] - params = form.cleaned_data["params"] - vendor = form.connection.vendor - with form.cursor as cursor: - if vendor == "sqlite": - # SQLite's EXPLAIN dumps the low-level opcodes generated for a query; - # EXPLAIN QUERY PLAN dumps a more human-readable summary - # See https://www.sqlite.org/lang_explain.html for details - cursor.execute(f"EXPLAIN QUERY PLAN {sql}", params) - elif vendor == "postgresql": - cursor.execute(f"EXPLAIN ANALYZE {sql}", params) - else: - cursor.execute(f"EXPLAIN {sql}", params) - headers = [d[0] for d in cursor.description] - result = cursor.fetchall() - + query = form.cleaned_data["query"] + result, headers = form.explain() context = { "result": result, - "sql": form.reformat_sql(), - "duration": form.cleaned_data["duration"], + "sql": reformat_sql(query["sql"], with_toggle=False), + "duration": query["duration"], "headers": headers, - "alias": form.cleaned_data["alias"], + "alias": query["alias"], } content = render_to_string("debug_toolbar/panels/sql_explain.html", context) return JsonResponse({"content": content}) @@ -100,45 +82,24 @@ def sql_profile(request): form = SQLSelectForm(verified_data) if form.is_valid(): - sql = form.cleaned_data["raw_sql"] - params = form.cleaned_data["params"] + query = form.cleaned_data["query"] result = None headers = None result_error = None - with form.cursor as cursor: - try: - cursor.execute("SET PROFILING=1") # Enable profiling - cursor.execute(sql, params) # Execute SELECT - cursor.execute("SET PROFILING=0") # Disable profiling - # The Query ID should always be 1 here but I'll subselect to get - # the last one just in case... - cursor.execute( - """ - SELECT * - FROM information_schema.profiling - WHERE query_id = ( - SELECT query_id - FROM information_schema.profiling - ORDER BY query_id DESC - LIMIT 1 - ) - """ - ) - headers = [d[0] for d in cursor.description] - result = cursor.fetchall() - except Exception: - result_error = ( - "Profiling is either not available or not supported by your " - "database." - ) + try: + result, headers = form.profile() + except Exception: + result_error = ( + "Profiling is either not available or not supported by your database." + ) context = { "result": result, "result_error": result_error, "sql": form.reformat_sql(), - "duration": form.cleaned_data["duration"], + "duration": query["duration"], "headers": headers, - "alias": form.cleaned_data["alias"], + "alias": query["alias"], } content = render_to_string("debug_toolbar/panels/sql_profile.html", context) return JsonResponse({"content": content}) diff --git a/debug_toolbar/panels/staticfiles.py b/debug_toolbar/panels/staticfiles.py index 9f1970ef6..544caab81 100644 --- a/debug_toolbar/panels/staticfiles.py +++ b/debug_toolbar/panels/staticfiles.py @@ -63,9 +63,10 @@ class StaticFilesPanel(panels.Panel): @property def title(self): + stats = self.get_stats() return _("Static files (%(num_found)s found, %(num_used)s used)") % { - "num_found": self.num_found, - "num_used": self.num_used, + "num_found": stats.get("num_found"), + "num_used": stats.get("num_used"), } def __init__(self, *args, **kwargs): @@ -97,16 +98,11 @@ def disable_instrumentation(self): record_static_file_signal.disconnect(self._store_static_files_signal_handler) request_id_context_var.reset(self.ctx_token) - @property - def num_used(self): - stats = self.get_stats() - return stats and stats["num_used"] - nav_title = _("Static files") @property def nav_subtitle(self): - num_used = self.num_used + num_used = self.get_stats().get("num_used") return ngettext( "%(num_used)s file used", "%(num_used)s files used", num_used ) % {"num_used": num_used} diff --git a/debug_toolbar/panels/templates/__init__.py b/debug_toolbar/panels/templates/__init__.py index a1d509b9e..5cd78bbb3 100644 --- a/debug_toolbar/panels/templates/__init__.py +++ b/debug_toolbar/panels/templates/__init__.py @@ -1,3 +1,3 @@ from debug_toolbar.panels.templates.panel import TemplatesPanel -__all__ = ["TemplatesPanel"] +__all__ = [TemplatesPanel.panel_id] diff --git a/debug_toolbar/panels/templates/panel.py b/debug_toolbar/panels/templates/panel.py index e9a5b4e83..6dbd02ee0 100644 --- a/debug_toolbar/panels/templates/panel.py +++ b/debug_toolbar/panels/templates/panel.py @@ -10,6 +10,7 @@ from django.test.signals import template_rendered from django.test.utils import instrumented_test_render from django.urls import path +from django.utils.encoding import force_str from django.utils.translation import gettext_lazy as _ from debug_toolbar.panels import Panel @@ -104,15 +105,16 @@ def _store_template_info(self, sender, **kwargs): @property def title(self): - num_templates = len(self.templates) + num_templates = len(self.get_stats()["templates"]) return _("Templates (%(num_templates)s rendered)") % { "num_templates": num_templates } @property def nav_subtitle(self): - if self.templates: - return self.templates[0]["template"].name + templates = self.get_stats()["templates"] + if templates: + return templates[0]["template"]["name"] return "" template = "debug_toolbar/panels/templates.html" @@ -196,7 +198,11 @@ def generate_stats(self, request, response): else: template.origin_name = _("No origin") template.origin_hash = "" - info["template"] = template + info["template"] = { + "name": template.name, + "origin_name": template.origin_name, + "origin_hash": template.origin_hash, + } # Clean up context for better readability if self.toolbar.config["SHOW_TEMPLATE_CONTEXT"]: if "context_list" not in template_data: @@ -208,7 +214,14 @@ def generate_stats(self, request, response): # Fetch context_processors/template_dirs from any template if self.templates: - context_processors = self.templates[0]["context_processors"] + context_processors = ( + { + key: force_str(value) + for key, value in self.templates[0]["context_processors"].items() + } + if self.templates[0]["context_processors"] + else None + ) template = self.templates[0]["template"] # django templates have the 'engine' attribute, while jinja # templates use 'backend' diff --git a/debug_toolbar/panels/timer.py b/debug_toolbar/panels/timer.py index 554798e7d..6ef9f0d7c 100644 --- a/debug_toolbar/panels/timer.py +++ b/debug_toolbar/panels/timer.py @@ -19,11 +19,11 @@ class TimerPanel(Panel): def nav_subtitle(self): stats = self.get_stats() - if hasattr(self, "_start_rusage"): - utime = self._end_rusage.ru_utime - self._start_rusage.ru_utime - stime = self._end_rusage.ru_stime - self._start_rusage.ru_stime + if stats.get("utime"): + utime = stats.get("utime") + stime = stats.get("stime") return _("CPU: %(cum)0.2fms (%(total)0.2fms)") % { - "cum": (utime + stime) * 1000.0, + "cum": (utime + stime), "total": stats["total_time"], } elif "total_time" in stats: @@ -64,27 +64,44 @@ def process_request(self, request): self._start_rusage = resource.getrusage(resource.RUSAGE_SELF) return super().process_request(request) + def serialize_rusage(self, data): + fields_to_serialize = [ + "ru_utime", + "ru_stime", + "ru_nvcsw", + "ru_nivcsw", + "ru_minflt", + "ru_majflt", + ] + return {field: getattr(data, field) for field in fields_to_serialize} + def generate_stats(self, request, response): stats = {} if hasattr(self, "_start_time"): stats["total_time"] = (perf_counter() - self._start_time) * 1000 - if hasattr(self, "_start_rusage"): + if self.has_content: self._end_rusage = resource.getrusage(resource.RUSAGE_SELF) - stats["utime"] = 1000 * self._elapsed_ru("ru_utime") - stats["stime"] = 1000 * self._elapsed_ru("ru_stime") + start = self.serialize_rusage(self._start_rusage) + end = self.serialize_rusage(self._end_rusage) + stats.update( + { + "utime": 1000 * self._elapsed_ru(start, end, "ru_utime"), + "stime": 1000 * self._elapsed_ru(start, end, "ru_stime"), + "vcsw": self._elapsed_ru(start, end, "ru_nvcsw"), + "ivcsw": self._elapsed_ru(start, end, "ru_nivcsw"), + "minflt": self._elapsed_ru(start, end, "ru_minflt"), + "majflt": self._elapsed_ru(start, end, "ru_majflt"), + } + ) stats["total"] = stats["utime"] + stats["stime"] - stats["vcsw"] = self._elapsed_ru("ru_nvcsw") - stats["ivcsw"] = self._elapsed_ru("ru_nivcsw") - stats["minflt"] = self._elapsed_ru("ru_minflt") - stats["majflt"] = self._elapsed_ru("ru_majflt") # these are documented as not meaningful under Linux. If you're # running BSD feel free to enable them, and add any others that I # hadn't gotten to before I noticed that I was getting nothing but # zeroes and that the docs agreed. :-( # - # stats['blkin'] = self._elapsed_ru('ru_inblock') - # stats['blkout'] = self._elapsed_ru('ru_oublock') - # stats['swap'] = self._elapsed_ru('ru_nswap') + # stats['blkin'] = self._elapsed_ru(start, end, 'ru_inblock') + # stats['blkout'] = self._elapsed_ru(start, end, 'ru_oublock') + # stats['swap'] = self._elapsed_ru(start, end, 'ru_nswap') # stats['rss'] = self._end_rusage.ru_maxrss # stats['srss'] = self._end_rusage.ru_ixrss # stats['urss'] = self._end_rusage.ru_idrss @@ -102,5 +119,6 @@ def generate_server_timing(self, request, response): "total_time", "Elapsed time", stats.get("total_time", 0) ) - def _elapsed_ru(self, name): - return getattr(self._end_rusage, name) - getattr(self._start_rusage, name) + @staticmethod + def _elapsed_ru(start, end, name): + return end.get(name) - start.get(name) diff --git a/debug_toolbar/panels/versions.py b/debug_toolbar/panels/versions.py index 77915e78b..491bdcfc9 100644 --- a/debug_toolbar/panels/versions.py +++ b/debug_toolbar/panels/versions.py @@ -16,7 +16,7 @@ class VersionsPanel(Panel): @property def nav_subtitle(self): - return f"Django {django.get_version()}" + return "Django %s" % self.get_stats()["django_version"] title = _("Versions") @@ -29,7 +29,11 @@ def generate_stats(self, request, response): ] versions += list(self.gen_app_versions()) self.record_stats( - {"versions": sorted(versions, key=lambda v: v[0]), "paths": sys.path} + { + "django_version": django.get_version(), + "versions": sorted(versions, key=lambda v: v[0]), + "paths": sys.path, + } ) def gen_app_versions(self): diff --git a/debug_toolbar/settings.py b/debug_toolbar/settings.py index 59d538a0b..db49d7b32 100644 --- a/debug_toolbar/settings.py +++ b/debug_toolbar/settings.py @@ -1,3 +1,4 @@ +import os import sys import warnings from functools import cache @@ -43,7 +44,8 @@ "SQL_WARNING_THRESHOLD": 500, # milliseconds "OBSERVE_REQUEST_CALLBACK": "debug_toolbar.toolbar.observe_request", "TOOLBAR_LANGUAGE": None, - "IS_RUNNING_TESTS": "test" in sys.argv, + "TOOLBAR_STORE_CLASS": "debug_toolbar.store.MemoryStore", + "IS_RUNNING_TESTS": "test" in sys.argv or "PYTEST_VERSION" in os.environ, "UPDATE_ON_FETCH": False, } diff --git a/debug_toolbar/static/debug_toolbar/js/history.js b/debug_toolbar/static/debug_toolbar/js/history.js index d10156660..c2644eca7 100644 --- a/debug_toolbar/static/debug_toolbar/js/history.js +++ b/debug_toolbar/static/debug_toolbar/js/history.js @@ -21,14 +21,17 @@ function refreshHistory() { const formTarget = djDebug.querySelector(".refreshHistory"); const container = document.getElementById("djdtHistoryRequests"); const oldIds = new Set( - pluckData(container.querySelectorAll("tr[data-store-id]"), "storeId") + pluckData( + container.querySelectorAll("tr[data-request-id]"), + "requestId" + ) ); ajaxForm(formTarget) .then((data) => { // Remove existing rows first then re-populate with new data for (const node of container.querySelectorAll( - "tr[data-store-id]" + "tr[data-request-id]" )) { node.remove(); } @@ -39,8 +42,8 @@ function refreshHistory() { .then(() => { const allIds = new Set( pluckData( - container.querySelectorAll("tr[data-store-id]"), - "storeId" + container.querySelectorAll("tr[data-request-id]"), + "requestId" ) ); const newIds = difference(allIds, oldIds); @@ -54,13 +57,13 @@ function refreshHistory() { .then((refreshInfo) => { for (const newId of refreshInfo.newIds) { const row = container.querySelector( - `tr[data-store-id="${newId}"]` + `tr[data-request-id="${newId}"]` ); row.classList.add("flash-new"); } setTimeout(() => { for (const row of container.querySelectorAll( - "tr[data-store-id]" + "tr[data-request-id]" )) { row.classList.remove("flash-new"); } @@ -68,9 +71,9 @@ function refreshHistory() { }); } -function switchHistory(newStoreId) { +function switchHistory(newRequestId) { const formTarget = djDebug.querySelector( - `.switchHistory[data-store-id='${newStoreId}']` + `.switchHistory[data-request-id='${newRequestId}']` ); const tbody = formTarget.closest("tbody"); @@ -84,16 +87,16 @@ function switchHistory(newStoreId) { if (Object.keys(data).length === 0) { const container = document.getElementById("djdtHistoryRequests"); container.querySelector( - `button[data-store-id="${newStoreId}"]` + `button[data-request-id="${newRequestId}"]` ).innerHTML = "Switch [EXPIRED]"; } - replaceToolbarState(newStoreId, data); + replaceToolbarState(newRequestId, data); }); } $$.on(djDebug, "click", ".switchHistory", function (event) { event.preventDefault(); - switchHistory(this.dataset.storeId); + switchHistory(this.dataset.requestId); }); $$.on(djDebug, "click", ".refreshHistory", (event) => { diff --git a/debug_toolbar/static/debug_toolbar/js/toolbar.js b/debug_toolbar/static/debug_toolbar/js/toolbar.js index 19658f76e..609842209 100644 --- a/debug_toolbar/static/debug_toolbar/js/toolbar.js +++ b/debug_toolbar/static/debug_toolbar/js/toolbar.js @@ -39,13 +39,13 @@ const djdt = { const inner = current.querySelector( ".djDebugPanelContent .djdt-scroll" ); - const storeId = djDebug.dataset.storeId; - if (storeId && inner.children.length === 0) { + const requestId = djDebug.dataset.requestId; + if (requestId && inner.children.length === 0) { const url = new URL( djDebug.dataset.renderPanelUrl, window.location ); - url.searchParams.append("store_id", storeId); + url.searchParams.append("request_id", requestId); url.searchParams.append("panel_id", panelId); ajax(url).then((data) => { inner.previousElementSibling.remove(); // Remove AJAX loader @@ -296,12 +296,12 @@ const djdt = { document.getElementById("djDebug").dataset.sidebarUrl; const slowjax = debounce(ajax, 200); - function handleAjaxResponse(storeId) { - const encodedStoreId = encodeURIComponent(storeId); - const dest = `${sidebarUrl}?store_id=${encodedStoreId}`; + function handleAjaxResponse(requestId) { + const encodedRequestId = encodeURIComponent(requestId); + const dest = `${sidebarUrl}?request_id=${encodedRequestId}`; slowjax(dest).then((data) => { if (djdt.needUpdateOnFetch) { - replaceToolbarState(encodedStoreId, data); + replaceToolbarState(encodedRequestId, data); } }); } @@ -314,9 +314,11 @@ const djdt = { // when the header can't be fetched. While it doesn't impede execution // it's worrisome to developers. if ( - this.getAllResponseHeaders().indexOf("djdt-store-id") >= 0 + this.getAllResponseHeaders().indexOf("djdt-request-id") >= 0 ) { - handleAjaxResponse(this.getResponseHeader("djdt-store-id")); + handleAjaxResponse( + this.getResponseHeader("djdt-request-id") + ); } }); origOpen.apply(this, args); @@ -330,10 +332,10 @@ const djdt = { // https://github.com/django-commons/django-debug-toolbar/pull/2100 const promise = origFetch.apply(this, args); return promise.then((response) => { - if (response.headers.get("djdt-store-id") !== null) { + if (response.headers.get("djdt-request-id") !== null) { try { handleAjaxResponse( - response.headers.get("djdt-store-id") + response.headers.get("djdt-request-id") ); } catch (err) { throw new Error( diff --git a/debug_toolbar/static/debug_toolbar/js/utils.js b/debug_toolbar/static/debug_toolbar/js/utils.js index 0cfa80474..9b34f86f8 100644 --- a/debug_toolbar/static/debug_toolbar/js/utils.js +++ b/debug_toolbar/static/debug_toolbar/js/utils.js @@ -109,10 +109,10 @@ function ajaxForm(element) { return ajax(url, ajaxData); } -function replaceToolbarState(newStoreId, data) { +function replaceToolbarState(newRequestId, data) { const djDebug = document.getElementById("djDebug"); - djDebug.setAttribute("data-store-id", newStoreId); - // Check if response is empty, it could be due to an expired storeId. + djDebug.setAttribute("data-request-id", newRequestId); + // Check if response is empty, it could be due to an expired requestId. for (const panelId of Object.keys(data)) { const panel = document.getElementById(panelId); if (panel) { diff --git a/debug_toolbar/store.py b/debug_toolbar/store.py new file mode 100644 index 000000000..76526fcff --- /dev/null +++ b/debug_toolbar/store.py @@ -0,0 +1,227 @@ +import contextlib +import json +import logging +from collections import defaultdict, deque +from collections.abc import Iterable +from typing import Any + +from django.core.serializers.json import DjangoJSONEncoder +from django.db import transaction +from django.utils.encoding import force_str +from django.utils.module_loading import import_string + +from debug_toolbar import settings as dt_settings +from debug_toolbar.models import HistoryEntry + +logger = logging.getLogger(__name__) + + +class DebugToolbarJSONEncoder(DjangoJSONEncoder): + def default(self, o): + try: + return super().default(o) + except (TypeError, ValueError): + logger.debug("The debug toolbar can't serialize %s into JSON" % o) + return force_str(o) + + +def serialize(data: Any) -> str: + # If this starts throwing an exceptions, consider + # Subclassing DjangoJSONEncoder and using force_str to + # make it JSON serializable. + return json.dumps(data, cls=DebugToolbarJSONEncoder) + + +def deserialize(data: str) -> Any: + return json.loads(data) + + +class BaseStore: + @classmethod + def request_ids(cls) -> Iterable: + """The stored request ids""" + raise NotImplementedError + + @classmethod + def exists(cls, request_id: str) -> bool: + """Does the given request_id exist in the store""" + raise NotImplementedError + + @classmethod + def set(cls, request_id: str): + """Set a request_id in the store""" + raise NotImplementedError + + @classmethod + def clear(cls): + """Remove all requests from the request store""" + raise NotImplementedError + + @classmethod + def delete(cls, request_id: str): + """Delete the store for the given request_id""" + raise NotImplementedError + + @classmethod + def save_panel(cls, request_id: str, panel_id: str, data: Any = None): + """Save the panel data for the given request_id""" + raise NotImplementedError + + @classmethod + def panel(cls, request_id: str, panel_id: str) -> Any: + """Fetch the panel data for the given request_id""" + raise NotImplementedError + + +class MemoryStore(BaseStore): + # ids is the collection of storage ids that have been used. + # Use a dequeue to support O(1) appends and pops + # from either direction. + _request_ids: deque = deque() + _request_store: dict[str, dict] = defaultdict(dict) + + @classmethod + def request_ids(cls) -> Iterable: + """The stored request ids""" + return cls._request_ids + + @classmethod + def exists(cls, request_id: str) -> bool: + """Does the given request_id exist in the request store""" + return request_id in cls._request_ids + + @classmethod + def set(cls, request_id: str): + """Set a request_id in the request store""" + if request_id not in cls._request_ids: + cls._request_ids.append(request_id) + for _ in range( + len(cls._request_ids) - dt_settings.get_config()["RESULTS_CACHE_SIZE"] + ): + removed_id = cls._request_ids.popleft() + cls._request_store.pop(removed_id, None) + + @classmethod + def clear(cls): + """Remove all requests from the request store""" + cls._request_ids.clear() + cls._request_store.clear() + + @classmethod + def delete(cls, request_id: str): + """Delete the stored request for the given request_id""" + cls._request_store.pop(request_id, None) + # Suppress when request_id doesn't exist in the collection of ids. + with contextlib.suppress(ValueError): + cls._request_ids.remove(request_id) + + @classmethod + def save_panel(cls, request_id: str, panel_id: str, data: Any = None): + """Save the panel data for the given request_id""" + cls.set(request_id) + cls._request_store[request_id][panel_id] = serialize(data) + + @classmethod + def panel(cls, request_id: str, panel_id: str) -> Any: + """Fetch the panel data for the given request_id""" + try: + data = cls._request_store[request_id][panel_id] + except KeyError: + return {} + else: + return deserialize(data) + + @classmethod + def panels(cls, request_id: str) -> Any: + """Fetch all the panel data for the given request_id""" + try: + panel_mapping = cls._request_store[request_id] + except KeyError: + return {} + for panel, data in panel_mapping.items(): + yield panel, deserialize(data) + + +class DatabaseStore(BaseStore): + @classmethod + def _cleanup_old_entries(cls): + """ + Enforce the cache size limit - keeping only the most recently used entries + up to RESULTS_CACHE_SIZE. + """ + # Determine which entries to keep + keep_ids = cls.request_ids() + + # Delete all entries not in the keep list + if keep_ids: + HistoryEntry.objects.exclude(request_id__in=keep_ids).delete() + + @classmethod + def request_ids(cls): + """Return all stored request ids within the cache size limit""" + cache_size = dt_settings.get_config()["RESULTS_CACHE_SIZE"] + return list( + HistoryEntry.objects.all()[:cache_size].values_list("request_id", flat=True) + ) + + @classmethod + def exists(cls, request_id: str) -> bool: + """Check if the given request_id exists in the store""" + return HistoryEntry.objects.filter(request_id=request_id).exists() + + @classmethod + def set(cls, request_id: str): + """Set a request_id in the store and clean up old entries""" + with transaction.atomic(): + # Create the entry if it doesn't exist (ignore otherwise) + _, created = HistoryEntry.objects.get_or_create(request_id=request_id) + + # Only enforce cache size limit when new entries are created + if created: + cls._cleanup_old_entries() + + @classmethod + def clear(cls): + """Remove all requests from the store""" + HistoryEntry.objects.all().delete() + + @classmethod + def delete(cls, request_id: str): + """Delete the stored request for the given request_id""" + HistoryEntry.objects.filter(request_id=request_id).delete() + + @classmethod + def save_panel(cls, request_id: str, panel_id: str, data: Any = None): + """Save the panel data for the given request_id""" + with transaction.atomic(): + obj, _ = HistoryEntry.objects.get_or_create(request_id=request_id) + store_data = obj.data + store_data[panel_id] = serialize(data) + obj.data = store_data + obj.save() + + @classmethod + def panel(cls, request_id: str, panel_id: str) -> Any: + """Fetch the panel data for the given request_id""" + try: + data = HistoryEntry.objects.get(request_id=request_id).data + panel_data = data.get(panel_id) + if panel_data is None: + return {} + return deserialize(panel_data) + except HistoryEntry.DoesNotExist: + return {} + + @classmethod + def panels(cls, request_id: str) -> Any: + """Fetch all panel data for the given request_id""" + try: + data = HistoryEntry.objects.get(request_id=request_id).data + for panel_id, panel_data in data.items(): + yield panel_id, deserialize(panel_data) + except HistoryEntry.DoesNotExist: + return {} + + +def get_store() -> BaseStore: + return import_string(dt_settings.get_config()["TOOLBAR_STORE_CLASS"]) diff --git a/debug_toolbar/templates/debug_toolbar/base.html b/debug_toolbar/templates/debug_toolbar/base.html index 607863104..b7562ecba 100644 --- a/debug_toolbar/templates/debug_toolbar/base.html +++ b/debug_toolbar/templates/debug_toolbar/base.html @@ -8,7 +8,7 @@ {% endblock js %}
- {% for id, store_context in stores.items %} + {% for request_id, history_context in toolbar_history.items %} {% include "debug_toolbar/panels/history_tr.html" %} {% endfor %} diff --git a/debug_toolbar/templates/debug_toolbar/panels/history_tr.html b/debug_toolbar/templates/debug_toolbar/panels/history_tr.html index 1642b4a47..decce3836 100644 --- a/debug_toolbar/templates/debug_toolbar/panels/history_tr.html +++ b/debug_toolbar/templates/debug_toolbar/panels/history_tr.html @@ -1,17 +1,17 @@ {% load i18n %} - + - {{ store_context.toolbar.stats.HistoryPanel.time|escape }} + {{ history_context.history_stats.time|escape }} -

{{ store_context.toolbar.stats.HistoryPanel.request_method|escape }}

+

{{ history_context.history_stats.request_method|escape }}

-

{{ store_context.toolbar.stats.HistoryPanel.request_url|truncatechars:100|escape }}

+

{{ history_context.history_stats.request_url|truncatechars:100|escape }}

- -
+ +
@@ -24,7 +24,7 @@ - {% for key, value in store_context.toolbar.stats.HistoryPanel.data.items %} + {% for key, value in history_context.history_stats.data.items %} @@ -39,12 +39,12 @@ diff --git a/debug_toolbar/toolbar.py b/debug_toolbar/toolbar.py index 04e5894c5..6ebc74234 100644 --- a/debug_toolbar/toolbar.py +++ b/debug_toolbar/toolbar.py @@ -2,15 +2,14 @@ The main DebugToolbar class that loads and renders the Toolbar. """ +import logging import re import uuid -from collections import OrderedDict from functools import cache from django.apps import apps from django.conf import settings from django.core.exceptions import ImproperlyConfigured -from django.core.handlers.asgi import ASGIRequest from django.dispatch import Signal from django.template import TemplateSyntaxError from django.template.loader import render_to_string @@ -20,14 +19,17 @@ from django.utils.translation import get_language, override as lang_override from debug_toolbar import APP_NAME, settings as dt_settings -from debug_toolbar.panels import Panel +from debug_toolbar.store import get_store + +logger = logging.getLogger(__name__) class DebugToolbar: # for internal testing use only _created = Signal() + store = None - def __init__(self, request, get_response): + def __init__(self, request, get_response, request_id=None): self.request = request self.config = dt_settings.get_config().copy() panels = [] @@ -37,16 +39,11 @@ def __init__(self, request, get_response): if panel.enabled: get_response = panel.process_request self.process_request = get_response - # Use OrderedDict for the _panels attribute so that items can be efficiently - # removed using FIFO order in the DebugToolbar.store() method. The .popitem() - # method of Python's built-in dict only supports LIFO removal. - self._panels = OrderedDict[str, Panel]() - while panels: - panel = panels.pop() - self._panels[panel.panel_id] = panel + self._panels = {panel.panel_id: panel for panel in reversed(panels)} self.stats = {} self.server_timing_stats = {} - self.store_id = None + self.request_id = request_id + self.init_store() self._created.send(request, toolbar=self) # Manage panels @@ -88,7 +85,7 @@ def render_toolbar(self): Renders the overall Toolbar with panels inside. """ if not self.should_render_panels(): - self.store() + self.init_store() try: context = {"toolbar": self} lang = self.config["TOOLBAR_LANGUAGE"] or get_language() @@ -109,37 +106,24 @@ def should_render_panels(self): If False, the panels will be loaded via Ajax. """ - if (render_panels := self.config["RENDER_PANELS"]) is None: - # If wsgi.multiprocess is true then it is either being served - # from ASGI or multithreaded third-party WSGI server eg gunicorn. - # we need to make special check for ASGI for supporting - # async context based requests. - if isinstance(self.request, ASGIRequest): - render_panels = False - else: - # The wsgi.multiprocess case of being True isn't supported until the - # toolbar has resolved the following issue: - # This type of set up is most likely - # https://github.com/django-commons/django-debug-toolbar/issues/1430 - render_panels = self.request.META.get("wsgi.multiprocess", True) - return render_panels + return self.config["RENDER_PANELS"] or False # Handle storing toolbars in memory and fetching them later on - _store = OrderedDict() + def init_store(self): + # Store already initialized. + if self.store is None: + self.store = get_store() - def store(self): - # Store already exists. - if self.store_id: + if self.request_id: return - self.store_id = uuid.uuid4().hex - self._store[self.store_id] = self - for _ in range(self.config["RESULTS_CACHE_SIZE"], len(self._store)): - self._store.popitem(last=False) + self.request_id = uuid.uuid4().hex + self.store.set(self.request_id) @classmethod - def fetch(cls, store_id): - return cls._store.get(store_id) + def fetch(cls, request_id, panel_id=None): + if get_store().exists(request_id): + return StoredDebugToolbar.from_store(request_id, panel_id=panel_id) # Manually implement class-level caching of panel classes and url patterns # because it's more obvious than going through an abstraction. @@ -208,6 +192,41 @@ def observe_request(request): return True +def from_store_get_response(request): + logger.warning( + "get_response was called for debug toolbar after being loaded from the store. No request exists in this scenario as the request is not stored, only the panel's data." + ) + return None + + +class StoredDebugToolbar(DebugToolbar): + def __init__(self, request, get_response, request_id=None): + self.request = None + self.config = dt_settings.get_config().copy() + self.process_request = get_response + self.stats = {} + self.server_timing_stats = {} + self.request_id = request_id + self.init_store() + + @classmethod + def from_store(cls, request_id, panel_id=None): + toolbar = StoredDebugToolbar( + None, from_store_get_response, request_id=request_id + ) + toolbar._panels = {} + + for panel_class in reversed(cls.get_panel_classes()): + panel = panel_class(toolbar, from_store_get_response) + if panel_id and panel.panel_id != panel_id: + continue + data = toolbar.store.panel(toolbar.request_id, panel.panel_id) + if data: + panel.load_stats_from_store(data) + toolbar._panels[panel.panel_id] = panel + return toolbar + + def debug_toolbar_urls(prefix="__debug__"): """ Return a URL pattern for serving toolbar in debug mode. diff --git a/debug_toolbar/views.py b/debug_toolbar/views.py index b9a410db5..279a6cb9a 100644 --- a/debug_toolbar/views.py +++ b/debug_toolbar/views.py @@ -12,7 +12,7 @@ @render_with_toolbar_language def render_panel(request): """Render the contents of a panel""" - toolbar = DebugToolbar.fetch(request.GET["store_id"]) + toolbar = DebugToolbar.fetch(request.GET["request_id"], request.GET["panel_id"]) if toolbar is None: content = _( "Data for this panel isn't available anymore. " diff --git a/docs/changes.rst b/docs/changes.rst index bf1998de8..b322bcd40 100644 --- a/docs/changes.rst +++ b/docs/changes.rst @@ -4,6 +4,31 @@ Change log Pending ------- +* Added support for checking if pytest as the test runner when determining + if tests are running. +* Defines the ``BaseStore`` interface for request storage mechanisms. +* Added the setting ``TOOLBAR_STORE_CLASS`` to configure the request + storage mechanism. Defaults to ``debug_toolbar.store.MemoryStore``. +* Rename ``store_id`` properties to ``request_id`` and ``Toolbar.store`` to + ``Toolbar.init_store``. +* Support ``Panel`` instances with stored stats via + ``Panel.load_stats_from_store``. +* Swapped ``Toolbar._store`` for the ``get_store()`` class. +* Created a ``StoredDebugToolbar`` that support creating an instance of the + toolbar representing an old request. It should only be used for fetching + panels' contents. +* Drop ``raw_params`` from query data. +* Queries now have a unique ``djdt_query_id``. The SQL forms now reference + this id and avoid passing SQL to be executed. +* Move the formatting logic of SQL queries to just before rendering in + ``SQLPanel.content``. +* Make ``Panel.panel_id`` a class member. +* Update all panels to utilize data from ``Panel.get_stats()`` to load content + to render. Specifically for ``Panel.title`` and ``Panel.nav_title``. +* Extend example app to contain an async version. +* Added ``debug_toolbar.store.DatabaseStore`` for persistent debug data + storage. + 5.2.0 (2025-04-29) ------------------ @@ -205,7 +230,6 @@ Please see everything under 5.0.0-alpha as well. 4.1.0 (2023-05-15) ------------------ - * Improved SQL statement formatting performance. Additionally, fixed the indentation of ``CASE`` statements and stopped simplifying ``.count()`` queries. diff --git a/docs/configuration.rst b/docs/configuration.rst index d9e7ff342..d02a54c01 100644 --- a/docs/configuration.rst +++ b/docs/configuration.rst @@ -77,7 +77,7 @@ Toolbar options * ``IS_RUNNING_TESTS`` - Default: ``"test" in sys.argv`` + Default: ``"test" in sys.argv or "PYTEST_VERSION" in os.environ`` This setting whether the application is running tests. If this resolves to ``True``, the toolbar will prevent you from running tests. This should only @@ -109,7 +109,8 @@ Toolbar options Default: ``25`` - The toolbar keeps up to this many results in memory. + The toolbar keeps up to this many results in memory or persistent storage. + .. _ROOT_TAG_EXTRA_ATTRS: @@ -178,6 +179,33 @@ Toolbar options toolbar should update on AJAX requests or not. The default implementation always returns ``True``. +.. _TOOLBAR_STORE_CLASS: + +* ``TOOLBAR_STORE_CLASS`` + + Default: ``"debug_toolbar.store.MemoryStore"`` + + The path to the class to be used for storing the toolbar's data per request. + + Available store classes: + + * ``debug_toolbar.store.MemoryStore`` - Stores data in memory + * ``debug_toolbar.store.DatabaseStore`` - Stores data in the database + + The DatabaseStore provides persistence and automatically cleans up old + entries based on the ``RESULTS_CACHE_SIZE`` setting. + + Note: For full functionality, DatabaseStore requires migrations for + the debug_toolbar app: + + .. code-block:: bash + + python manage.py migrate debug_toolbar + + For the DatabaseStore to work properly, you need to run migrations for the + debug_toolbar app. The migrations create the necessary database table to store + toolbar data. + .. _TOOLBAR_LANGUAGE: * ``TOOLBAR_LANGUAGE`` @@ -378,6 +406,14 @@ Here's what a slightly customized toolbar configuration might look like:: 'SQL_WARNING_THRESHOLD': 100, # milliseconds } +Here's an example of using a persistent store to keep debug data between server +restarts:: + + DEBUG_TOOLBAR_CONFIG = { + 'TOOLBAR_STORE_CLASS': 'debug_toolbar.store.DatabaseStore', + 'RESULTS_CACHE_SIZE': 100, # Store up to 100 requests + } + Theming support --------------- The debug toolbar uses CSS variables to define fonts and colors. This allows diff --git a/docs/installation.rst b/docs/installation.rst index 61187570d..b89a2f563 100644 --- a/docs/installation.rst +++ b/docs/installation.rst @@ -165,7 +165,7 @@ can do this by adding another setting: .. code-block:: python - TESTING = "test" in sys.argv + TESTING = "test" in sys.argv or "PYTEST_VERSION" in os.environ if not TESTING: INSTALLED_APPS = [ diff --git a/example/asgi.py b/example/asgi.py index 9d7c78703..7c5c501f6 100644 --- a/example/asgi.py +++ b/example/asgi.py @@ -1,9 +1,16 @@ -"""ASGI config for example project.""" +""" +ASGI config for example_async project. + +It exposes the ASGI callable as a module-level variable named ``application``. + +For more information on this file, see +https://docs.djangoproject.com/en/stable/howto/deployment/asgi/ +""" import os from django.core.asgi import get_asgi_application -os.environ.setdefault("DJANGO_SETTINGS_MODULE", "example.settings") +os.environ.setdefault("DJANGO_SETTINGS_MODULE", "example.async_.settings") application = get_asgi_application() diff --git a/example/async_/__init__.py b/example/async_/__init__.py new file mode 100644 index 000000000..e69de29bb diff --git a/example/async_/settings.py b/example/async_/settings.py new file mode 100644 index 000000000..f3bef673a --- /dev/null +++ b/example/async_/settings.py @@ -0,0 +1,5 @@ +"""Django settings for example project.""" + +from ..settings import * # noqa: F403 + +ROOT_URLCONF = "example.async_.urls" diff --git a/example/async_/urls.py b/example/async_/urls.py new file mode 100644 index 000000000..ad19cbc83 --- /dev/null +++ b/example/async_/urls.py @@ -0,0 +1,9 @@ +from django.urls import path + +from example.async_ import views +from example.urls import urlpatterns as sync_urlpatterns + +urlpatterns = [ + path("async/db/", views.async_db_view, name="async_db_view"), + *sync_urlpatterns, +] diff --git a/example/async_/views.py b/example/async_/views.py new file mode 100644 index 000000000..7326e0d0b --- /dev/null +++ b/example/async_/views.py @@ -0,0 +1,9 @@ +from django.contrib.auth.models import User +from django.http import JsonResponse + + +async def async_db_view(request): + names = [] + async for user in User.objects.all(): + names.append(user.username) + return JsonResponse({"names": names}) diff --git a/example/settings.py b/example/settings.py index 06b70f7fa..ffaa09fe5 100644 --- a/example/settings.py +++ b/example/settings.py @@ -29,6 +29,7 @@ MIDDLEWARE = [ "django.middleware.security.SecurityMiddleware", + "whitenoise.middleware.WhiteNoiseMiddleware", "django.contrib.sessions.middleware.SessionMiddleware", "django.middleware.common.CommonMiddleware", "django.middleware.csrf.CsrfViewMiddleware", diff --git a/example/templates/index.html b/example/templates/index.html index 4b25aefca..a10c2b5ac 100644 --- a/example/templates/index.html +++ b/example/templates/index.html @@ -25,9 +25,14 @@

Index of Tests

+ {% comment %} + + {% endcomment %} + diff --git a/requirements_dev.txt b/requirements_dev.txt index 6915226fd..90e490192 100644 --- a/requirements_dev.txt +++ b/requirements_dev.txt @@ -4,6 +4,10 @@ Django sqlparse Jinja2 +# Django Async +daphne +whitenoise # To avoid dealing with static files + # Testing coverage[toml] diff --git a/tests/base.py b/tests/base.py index 3f40261fe..c2a6f379e 100644 --- a/tests/base.py +++ b/tests/base.py @@ -14,6 +14,7 @@ ) from debug_toolbar.panels import Panel +from debug_toolbar.store import get_store from debug_toolbar.toolbar import DebugToolbar data_contextvar = contextvars.ContextVar("djdt_toolbar_test_client") @@ -127,6 +128,5 @@ def setUp(self): # The HistoryPanel keeps track of previous stores in memory. # This bleeds into other tests and violates their idempotency. # Clear the store before each test. - for key in list(DebugToolbar._store.keys()): - del DebugToolbar._store[key] + get_store().clear() super().setUp() diff --git a/tests/panels/test_alerts.py b/tests/panels/test_alerts.py index 5c926f275..40ad8cf67 100644 --- a/tests/panels/test_alerts.py +++ b/tests/panels/test_alerts.py @@ -109,4 +109,4 @@ def _render(): response = StreamingHttpResponse(_render()) self.panel.generate_stats(self.request, response) - self.assertEqual(self.panel.get_stats(), {}) + self.assertEqual(self.panel.get_stats(), {"alerts": []}) diff --git a/tests/panels/test_cache.py b/tests/panels/test_cache.py index aacf521cb..a016f81f0 100644 --- a/tests/panels/test_cache.py +++ b/tests/panels/test_cache.py @@ -1,10 +1,12 @@ from django.core import cache +from debug_toolbar.panels.cache import CachePanel + from ..base import BaseTestCase class CachePanelTestCase(BaseTestCase): - panel_id = "CachePanel" + panel_id = CachePanel.panel_id def test_recording(self): self.assertEqual(len(self.panel.calls), 0) diff --git a/tests/panels/test_history.py b/tests/panels/test_history.py index 4c5244934..29e062da0 100644 --- a/tests/panels/test_history.py +++ b/tests/panels/test_history.py @@ -4,6 +4,8 @@ from django.test import RequestFactory, override_settings from django.urls import resolve, reverse +from debug_toolbar.panels.history import HistoryPanel +from debug_toolbar.store import get_store from debug_toolbar.toolbar import DebugToolbar from ..base import BaseTestCase, IntegrationTestCase @@ -12,7 +14,7 @@ class HistoryPanelTestCase(BaseTestCase): - panel_id = "HistoryPanel" + panel_id = HistoryPanel.panel_id def test_disabled(self): config = {"DISABLE_PANELS": {"debug_toolbar.panels.history.HistoryPanel"}} @@ -78,20 +80,21 @@ class HistoryViewsTestCase(IntegrationTestCase): "AlertsPanel", "CachePanel", "SignalsPanel", - "ProfilingPanel", } def test_history_panel_integration_content(self): """Verify the history panel's content renders properly..""" - self.assertEqual(len(DebugToolbar._store), 0) + store = get_store() + self.assertEqual(len(list(store.request_ids())), 0) data = {"foo": "bar"} self.client.get("/json_view/", data, content_type="application/json") # Check the history panel's stats to verify the toolbar rendered properly. - self.assertEqual(len(DebugToolbar._store), 1) - toolbar = list(DebugToolbar._store.values())[0] - content = toolbar.get_panel_by_id("HistoryPanel").content + request_ids = list(store.request_ids()) + self.assertEqual(len(request_ids), 1) + toolbar = DebugToolbar.fetch(request_ids[0]) + content = toolbar.get_panel_by_id(HistoryPanel.panel_id).content self.assertIn("bar", content) self.assertIn('name="exclude_history" value="True"', content) @@ -101,23 +104,28 @@ def test_history_sidebar_invalid(self): def test_history_headers(self): """Validate the headers injected from the history panel.""" + DebugToolbar.get_observe_request.cache_clear() response = self.client.get("/json_view/") - store_id = list(DebugToolbar._store)[0] - self.assertEqual(response.headers["djdt-store-id"], store_id) + request_id = list(get_store().request_ids())[0] + self.assertEqual(response.headers["djdt-request-id"], request_id) - @override_settings( - DEBUG_TOOLBAR_CONFIG={"OBSERVE_REQUEST_CALLBACK": lambda request: False} - ) def test_history_headers_unobserved(self): """Validate the headers aren't injected from the history panel.""" - response = self.client.get("/json_view/") - self.assertNotIn("djdt-store-id", response.headers) + with self.settings( + DEBUG_TOOLBAR_CONFIG={"OBSERVE_REQUEST_CALLBACK": lambda request: False} + ): + DebugToolbar.get_observe_request.cache_clear() + response = self.client.get("/json_view/") + self.assertNotIn("djdt-request-id", response.headers) + # Clear it again to avoid conflicting with another test + # Specifically, DebugToolbarLiveTestCase.test_ajax_refresh + DebugToolbar.get_observe_request.cache_clear() def test_history_sidebar(self): """Validate the history sidebar view.""" self.client.get("/json_view/") - store_id = list(DebugToolbar._store)[0] - data = {"store_id": store_id, "exclude_history": True} + request_id = list(get_store().request_ids())[0] + data = {"request_id": request_id, "exclude_history": True} response = self.client.get(reverse("djdt:history_sidebar"), data=data) self.assertEqual(response.status_code, 200) self.assertEqual( @@ -129,10 +137,9 @@ def test_history_sidebar_includes_history(self): """Validate the history sidebar view.""" self.client.get("/json_view/") panel_keys = copy.copy(self.PANEL_KEYS) - panel_keys.add("HistoryPanel") - panel_keys.add("RedirectsPanel") - store_id = list(DebugToolbar._store)[0] - data = {"store_id": store_id} + panel_keys.add(HistoryPanel.panel_id) + request_id = list(get_store().request_ids())[0] + data = {"request_id": request_id} response = self.client.get(reverse("djdt:history_sidebar"), data=data) self.assertEqual(response.status_code, 200) self.assertEqual( @@ -141,32 +148,34 @@ def test_history_sidebar_includes_history(self): ) @override_settings( - DEBUG_TOOLBAR_CONFIG={"RESULTS_CACHE_SIZE": 1, "RENDER_PANELS": False} + DEBUG_TOOLBAR_CONFIG={"RENDER_PANELS": False, "RESULTS_CACHE_SIZE": 1} ) - def test_history_sidebar_expired_store_id(self): + def test_history_sidebar_expired_request_id(self): """Validate the history sidebar view.""" self.client.get("/json_view/") - store_id = list(DebugToolbar._store)[0] - data = {"store_id": store_id, "exclude_history": True} + request_id = list(get_store().request_ids())[0] + data = {"request_id": request_id, "exclude_history": True} response = self.client.get(reverse("djdt:history_sidebar"), data=data) self.assertEqual(response.status_code, 200) self.assertEqual( set(response.json()), self.PANEL_KEYS, ) + # Make enough requests to unset the original self.client.get("/json_view/") - # Querying old store_id should return in empty response - data = {"store_id": store_id, "exclude_history": True} + # Querying old request_id should return in empty response + data = {"request_id": request_id, "exclude_history": True} response = self.client.get(reverse("djdt:history_sidebar"), data=data) self.assertEqual(response.status_code, 200) self.assertEqual(response.json(), {}) - # Querying with latest store_id - latest_store_id = list(DebugToolbar._store)[0] - data = {"store_id": latest_store_id, "exclude_history": True} + # Querying with latest request_id + latest_request_id = list(get_store().request_ids())[0] + data = {"request_id": latest_request_id, "exclude_history": True} response = self.client.get(reverse("djdt:history_sidebar"), data=data) self.assertEqual(response.status_code, 200) + self.assertEqual( set(response.json()), self.PANEL_KEYS, @@ -180,15 +189,15 @@ def test_history_refresh(self): ) response = self.client.get( - reverse("djdt:history_refresh"), data={"store_id": "foo"} + reverse("djdt:history_refresh"), data={"request_id": "foo"} ) self.assertEqual(response.status_code, 200) data = response.json() self.assertEqual(len(data["requests"]), 2) - store_ids = list(DebugToolbar._store) - self.assertIn(html.escape(store_ids[0]), data["requests"][0]["content"]) - self.assertIn(html.escape(store_ids[1]), data["requests"][1]["content"]) + request_ids = list(get_store().request_ids()) + self.assertIn(html.escape(request_ids[0]), data["requests"][0]["content"]) + self.assertIn(html.escape(request_ids[1]), data["requests"][1]["content"]) for val in ["foo", "bar"]: self.assertIn(val, data["requests"][0]["content"]) diff --git a/tests/panels/test_profiling.py b/tests/panels/test_profiling.py index 88ec57dd6..931a5dbf6 100644 --- a/tests/panels/test_profiling.py +++ b/tests/panels/test_profiling.py @@ -6,6 +6,8 @@ from django.http import HttpResponse from django.test.utils import override_settings +from debug_toolbar.panels.profiling import ProfilingPanel + from ..base import BaseTestCase, IntegrationTestCase from ..views import listcomp_view, regular_view @@ -14,7 +16,7 @@ DEBUG_TOOLBAR_PANELS=["debug_toolbar.panels.profiling.ProfilingPanel"] ) class ProfilingPanelTestCase(BaseTestCase): - panel_id = "ProfilingPanel" + panel_id = ProfilingPanel.panel_id def test_regular_view(self): self._get_response = lambda request: regular_view(request, "profiling") diff --git a/tests/panels/test_redirects.py b/tests/panels/test_redirects.py index 7d6d5ac06..c81c7eaba 100644 --- a/tests/panels/test_redirects.py +++ b/tests/panels/test_redirects.py @@ -4,11 +4,13 @@ from django.http import HttpResponse from django.test import AsyncRequestFactory +from debug_toolbar.panels.redirects import RedirectsPanel + from ..base import BaseTestCase class RedirectsPanelTestCase(BaseTestCase): - panel_id = "RedirectsPanel" + panel_id = RedirectsPanel.panel_id def test_regular_response(self): not_redirect = HttpResponse() diff --git a/tests/panels/test_request.py b/tests/panels/test_request.py index 2eb7ba610..cfbbc65e4 100644 --- a/tests/panels/test_request.py +++ b/tests/panels/test_request.py @@ -1,13 +1,15 @@ from django.http import QueryDict from django.test import RequestFactory +from debug_toolbar.panels.request import RequestPanel + from ..base import BaseTestCase rf = RequestFactory() class RequestPanelTestCase(BaseTestCase): - panel_id = "RequestPanel" + panel_id = RequestPanel.panel_id def test_non_ascii_session(self): self.request.session = {"où": "où"} @@ -52,7 +54,7 @@ def test_query_dict_for_request_in_method_get(self): def test_dict_for_request_in_method_get(self): """ Test verifies the correctness of the statistics generation method - in the case when the GET request is class Dict + in the case when the GET request is class dict """ self.request.GET = {"foo": "bar"} response = self.panel.process_request(self.request) @@ -78,7 +80,7 @@ def test_query_dict_for_request_in_method_post(self): def test_dict_for_request_in_method_post(self): """ Test verifies the correctness of the statistics generation method - in the case when the POST request is class Dict + in the case when the POST request is class dict """ self.request.POST = {"foo": "bar"} response = self.panel.process_request(self.request) diff --git a/tests/panels/test_sql.py b/tests/panels/test_sql.py index a411abb5d..e238bd0d8 100644 --- a/tests/panels/test_sql.py +++ b/tests/panels/test_sql.py @@ -15,6 +15,7 @@ from django.test.utils import override_settings import debug_toolbar.panels.sql.tracking as sql_tracking +from debug_toolbar.panels.sql import SQLPanel try: import psycopg @@ -47,7 +48,7 @@ async def concurrent_async_sql_call(*, use_iterator=False): class SQLPanelTestCase(BaseTestCase): - panel_id = "SQLPanel" + panel_id = SQLPanel.panel_id def test_disabled(self): config = {"DISABLE_PANELS": {"debug_toolbar.panels.sql.SQLPanel"}} @@ -357,7 +358,7 @@ def test_binary_param_force_text(self): self.assertIn( "SELECT * FROM" " tests_binary WHERE field =", - self.panel._queries[0]["sql"], + self.panel.content, ) @unittest.skipUnless(connection.vendor != "sqlite", "Test invalid for SQLite") @@ -429,8 +430,6 @@ def test_insert_content(self): """ list(User.objects.filter(username="café")) response = self.panel.process_request(self.request) - # ensure the panel does not have content yet. - self.assertNotIn("café", self.panel.content) self.panel.generate_stats(self.request, response) # ensure the panel renders correctly. content = self.panel.content @@ -559,20 +558,29 @@ def test_prettify_sql(self): list(User.objects.filter(username__istartswith="spam")) response = self.panel.process_request(self.request) self.panel.generate_stats(self.request, response) + # The content formats the sql and prettifies it + self.assertTrue(self.panel.content) pretty_sql = self.panel._queries[-1]["sql"] self.assertEqual(len(self.panel._queries), 1) - # Reset the queries - self.panel._queries = [] + # Recreate the panel to reset the queries. Content being a cached_property + # which doesn't have a way to reset it. + self.panel.disable_instrumentation() + self.panel = SQLPanel(self.panel.toolbar, self.panel.get_response) + self.panel.enable_instrumentation() # Run it again, but with prettify off. Verify that it's different. with override_settings(DEBUG_TOOLBAR_CONFIG={"PRETTIFY_SQL": False}): list(User.objects.filter(username__istartswith="spam")) response = self.panel.process_request(self.request) self.panel.generate_stats(self.request, response) + # The content formats the sql and prettifies it + self.assertTrue(self.panel.content) self.assertEqual(len(self.panel._queries), 1) - self.assertNotEqual(pretty_sql, self.panel._queries[-1]["sql"]) + self.assertNotIn(pretty_sql, self.panel.content) - self.panel._queries = [] + self.panel.disable_instrumentation() + self.panel = SQLPanel(self.panel.toolbar, self.panel.get_response) + self.panel.enable_instrumentation() # Run it again, but with prettify back on. # This is so we don't have to check what PRETTIFY_SQL does exactly, # but we know it's doing something. @@ -580,8 +588,10 @@ def test_prettify_sql(self): list(User.objects.filter(username__istartswith="spam")) response = self.panel.process_request(self.request) self.panel.generate_stats(self.request, response) + # The content formats the sql and prettifies it + self.assertTrue(self.panel.content) self.assertEqual(len(self.panel._queries), 1) - self.assertEqual(pretty_sql, self.panel._queries[-1]["sql"]) + self.assertIn(pretty_sql, self.panel.content) def test_simplification(self): """ @@ -593,6 +603,8 @@ def test_simplification(self): list(User.objects.values_list("id")) response = self.panel.process_request(self.request) self.panel.generate_stats(self.request, response) + # The content formats the sql which injects the ellipsis character + self.assertTrue(self.panel.content) self.assertEqual(len(self.panel._queries), 3) self.assertNotIn("\u2022", self.panel._queries[0]["sql"]) self.assertNotIn("\u2022", self.panel._queries[1]["sql"]) @@ -618,6 +630,8 @@ def test_top_level_simplification(self): ) response = self.panel.process_request(self.request) self.panel.generate_stats(self.request, response) + # The content formats the sql which injects the ellipsis character + self.assertTrue(self.panel.content) if connection.vendor != "mysql": self.assertEqual(len(self.panel._queries), 4) else: @@ -738,7 +752,7 @@ def test_explain_with_union(self): class SQLPanelMultiDBTestCase(BaseMultiDBTestCase): - panel_id = "SQLPanel" + panel_id = SQLPanel.panel_id def test_aliases(self): self.assertFalse(self.panel._queries) diff --git a/tests/panels/test_staticfiles.py b/tests/panels/test_staticfiles.py index 2306c8365..81afe6afa 100644 --- a/tests/panels/test_staticfiles.py +++ b/tests/panels/test_staticfiles.py @@ -5,13 +5,13 @@ from django.shortcuts import render from django.test import AsyncRequestFactory, RequestFactory -from debug_toolbar.panels.staticfiles import URLMixin +from debug_toolbar.panels.staticfiles import StaticFilesPanel, URLMixin from ..base import BaseTestCase class StaticFilesPanelTestCase(BaseTestCase): - panel_id = "StaticFilesPanel" + panel_id = StaticFilesPanel.panel_id def test_default_case(self): response = self.panel.process_request(self.request) @@ -23,7 +23,7 @@ def test_default_case(self): self.assertIn( "django.contrib.staticfiles.finders.FileSystemFinder (2 files)", content ) - self.assertEqual(self.panel.num_used, 0) + self.assertEqual(self.panel.get_stats()["num_used"], 0) self.assertNotEqual(self.panel.num_found, 0) expected_apps = ["django.contrib.admin", "debug_toolbar"] if settings.USE_GIS: @@ -42,7 +42,7 @@ async def get_response(request): async_request = AsyncRequestFactory().get("/") response = await self.panel.process_request(async_request) self.panel.generate_stats(self.request, response) - self.assertEqual(self.panel.num_used, 1) + self.assertEqual(self.panel.get_stats()["num_used"], 1) def test_insert_content(self): """ @@ -76,7 +76,7 @@ def get_response(request): request = RequestFactory().get("/") response = self.panel.process_request(request) self.panel.generate_stats(self.request, response) - self.assertEqual(self.panel.num_used, 1) + self.assertEqual(self.panel.get_stats()["num_used"], 1) self.assertIn('"/static/additional_static/base.css"', self.panel.content) def test_storage_state_preservation(self): diff --git a/tests/panels/test_template.py b/tests/panels/test_template.py index 44ac4ff0d..f79914024 100644 --- a/tests/panels/test_template.py +++ b/tests/panels/test_template.py @@ -6,17 +6,20 @@ from django.test import override_settings from django.utils.functional import SimpleLazyObject +from debug_toolbar.panels.sql import SQLPanel +from debug_toolbar.panels.templates import TemplatesPanel + from ..base import BaseTestCase, IntegrationTestCase from ..forms import TemplateReprForm from ..models import NonAsciiRepr class TemplatesPanelTestCase(BaseTestCase): - panel_id = "TemplatesPanel" + panel_id = TemplatesPanel.panel_id def setUp(self): super().setUp() - self.sql_panel = self.toolbar.get_panel_by_id("SQLPanel") + self.sql_panel = self.toolbar.get_panel_by_id(SQLPanel.panel_id) self.sql_panel.enable_instrumentation() def tearDown(self): diff --git a/tests/panels/test_versions.py b/tests/panels/test_versions.py index 27ccba92b..b484c043a 100644 --- a/tests/panels/test_versions.py +++ b/tests/panels/test_versions.py @@ -1,5 +1,7 @@ from collections import namedtuple +from debug_toolbar.panels.versions import VersionsPanel + from ..base import BaseTestCase version_info_t = namedtuple( @@ -8,7 +10,7 @@ class VersionsPanelTestCase(BaseTestCase): - panel_id = "VersionsPanel" + panel_id = VersionsPanel.panel_id def test_app_version_from_get_version_fn(self): class FakeApp: diff --git a/tests/settings.py b/tests/settings.py index 12561fb11..e10338cb4 100644 --- a/tests/settings.py +++ b/tests/settings.py @@ -133,6 +133,7 @@ DEBUG_TOOLBAR_CONFIG = { # Django's test client sets wsgi.multiprocess to True inappropriately "RENDER_PANELS": False, + "RESULTS_CACHE_SIZE": 3, # IS_RUNNING_TESTS must be False even though we're running tests because we're running the toolbar's own tests. "IS_RUNNING_TESTS": False, } diff --git a/tests/test_csp_rendering.py b/tests/test_csp_rendering.py index 144e65ba0..902d4f5c3 100644 --- a/tests/test_csp_rendering.py +++ b/tests/test_csp_rendering.py @@ -5,10 +5,11 @@ from django.conf import settings from django.http.response import HttpResponse -from django.test.utils import ContextList, override_settings +from django.test.utils import override_settings from html5lib.constants import E from html5lib.html5parser import HTMLParser +from debug_toolbar.store import get_store from debug_toolbar.toolbar import DebugToolbar from .base import IntegrationTestCase @@ -82,8 +83,7 @@ def test_exists(self): self.assertContains(response, "djDebug") namespaces = get_namespaces(element=html_root) - toolbar = list(DebugToolbar._store.values())[-1] - nonce = str(toolbar.csp_nonce) + nonce = response.context["request"].csp_nonce self._fail_if_missing( root=html_root, path=".//link", namespaces=namespaces, nonce=nonce ) @@ -127,8 +127,7 @@ def test_redirects_exists(self): self.assertContains(response, "djDebug") namespaces = get_namespaces(element=html_root) - context: ContextList = response.context # pyright: ignore[reportAttributeAccessIssue] - nonce = str(context["toolbar"].csp_nonce) + nonce = response.context["request"].csp_nonce self._fail_if_missing( root=html_root, path=".//link", namespaces=namespaces, nonce=nonce ) @@ -137,12 +136,14 @@ def test_redirects_exists(self): ) def test_panel_content_nonce_exists(self): + store = get_store() for middleware in [MIDDLEWARE_CSP_BEFORE, MIDDLEWARE_CSP_LAST]: with self.settings(MIDDLEWARE=middleware): response = cast(HttpResponse, self.client.get(path="/csp_view/")) self.assertEqual(response.status_code, 200) - toolbar = list(DebugToolbar._store.values())[-1] + request_ids = list(store.request_ids()) + toolbar = DebugToolbar.fetch(request_ids[-1]) panels_to_check = ["HistoryPanel", "TimerPanel"] for panel in panels_to_check: content = toolbar.get_panel_by_id(panel).content diff --git a/tests/test_integration.py b/tests/test_integration.py index a431ba29f..d32f2ad21 100644 --- a/tests/test_integration.py +++ b/tests/test_integration.py @@ -11,12 +11,19 @@ from django.db import connection from django.http import HttpResponse from django.template.loader import get_template -from django.test import AsyncRequestFactory, RequestFactory +from django.test import RequestFactory from django.test.utils import override_settings from debug_toolbar.forms import SignedDataForm from debug_toolbar.middleware import DebugToolbarMiddleware, show_toolbar from debug_toolbar.panels import Panel +from debug_toolbar.panels.cache import CachePanel +from debug_toolbar.panels.history import HistoryPanel +from debug_toolbar.panels.request import RequestPanel +from debug_toolbar.panels.sql import SQLPanel +from debug_toolbar.panels.templates import TemplatesPanel +from debug_toolbar.panels.versions import VersionsPanel +from debug_toolbar.store import get_store from debug_toolbar.toolbar import DebugToolbar from .base import BaseTestCase, IntegrationTestCase @@ -36,13 +43,13 @@ rf = RequestFactory() -def toolbar_store_id(): +def toolbar_request_id(): def get_response(request): return HttpResponse() toolbar = DebugToolbar(rf.get("/"), get_response) - toolbar.store() - return toolbar.store_id + toolbar.init_store() + return toolbar.request_id class BuggyPanel(Panel): @@ -107,45 +114,12 @@ def test_should_render_panels_RENDER_PANELS(self): toolbar.config["RENDER_PANELS"] = True self.assertTrue(toolbar.should_render_panels()) toolbar.config["RENDER_PANELS"] = None - self.assertTrue(toolbar.should_render_panels()) - - def test_should_render_panels_multiprocess(self): - """ - The toolbar should render the panels on each request when wsgi.multiprocess - is True or missing. - """ - request = rf.get("/") - request.META["wsgi.multiprocess"] = True - toolbar = DebugToolbar(request, self.get_response) - toolbar.config["RENDER_PANELS"] = None - self.assertTrue(toolbar.should_render_panels()) - - request.META["wsgi.multiprocess"] = False - self.assertFalse(toolbar.should_render_panels()) - - request.META.pop("wsgi.multiprocess") - self.assertTrue(toolbar.should_render_panels()) - - def test_should_render_panels_asgi(self): - """ - The toolbar not should render the panels on each request when wsgi.multiprocess - is True or missing in case of async context rather than multithreaded - wsgi. - """ - async_request = AsyncRequestFactory().get("/") - # by default ASGIRequest will have wsgi.multiprocess set to True - # but we are still assigning this to true cause this could change - # and we specifically need to check that method returns false even with - # wsgi.multiprocess set to true - async_request.META["wsgi.multiprocess"] = True - toolbar = DebugToolbar(async_request, self.get_response) - toolbar.config["RENDER_PANELS"] = None self.assertFalse(toolbar.should_render_panels()) def _resolve_stats(self, path): # takes stats from Request panel request = rf.get(path) - panel = self.toolbar.get_panel_by_id("RequestPanel") + panel = self.toolbar.get_panel_by_id(RequestPanel.panel_id) response = panel.process_request(request) panel.generate_stats(request, response) return panel.get_stats() @@ -196,9 +170,13 @@ def test_cache_page(self): # may run earlier and cause fewer cache calls. cache.clear() response = self.client.get("/cached_view/") - self.assertEqual(len(response.toolbar.get_panel_by_id("CachePanel").calls), 3) + self.assertEqual( + len(response.toolbar.get_panel_by_id(CachePanel.panel_id).calls), 3 + ) response = self.client.get("/cached_view/") - self.assertEqual(len(response.toolbar.get_panel_by_id("CachePanel").calls), 2) + self.assertEqual( + len(response.toolbar.get_panel_by_id(CachePanel.panel_id).calls), 2 + ) @override_settings(ROOT_URLCONF="tests.urls_use_package_urls") def test_include_package_urls(self): @@ -207,16 +185,24 @@ def test_include_package_urls(self): # may run earlier and cause fewer cache calls. cache.clear() response = self.client.get("/cached_view/") - self.assertEqual(len(response.toolbar.get_panel_by_id("CachePanel").calls), 3) + self.assertEqual( + len(response.toolbar.get_panel_by_id(CachePanel.panel_id).calls), 3 + ) response = self.client.get("/cached_view/") - self.assertEqual(len(response.toolbar.get_panel_by_id("CachePanel").calls), 2) + self.assertEqual( + len(response.toolbar.get_panel_by_id(CachePanel.panel_id).calls), 2 + ) def test_low_level_cache_view(self): """Test cases when low level caching API is used within a request.""" response = self.client.get("/cached_low_level_view/") - self.assertEqual(len(response.toolbar.get_panel_by_id("CachePanel").calls), 2) + self.assertEqual( + len(response.toolbar.get_panel_by_id(CachePanel.panel_id).calls), 2 + ) response = self.client.get("/cached_low_level_view/") - self.assertEqual(len(response.toolbar.get_panel_by_id("CachePanel").calls), 1) + self.assertEqual( + len(response.toolbar.get_panel_by_id(CachePanel.panel_id).calls), 1 + ) def test_cache_disable_instrumentation(self): """ @@ -228,7 +214,9 @@ def test_cache_disable_instrumentation(self): response = self.client.get("/execute_sql/") self.assertEqual(cache.get("UseCacheAfterToolbar.before"), 1) self.assertEqual(cache.get("UseCacheAfterToolbar.after"), 1) - self.assertEqual(len(response.toolbar.get_panel_by_id("CachePanel").calls), 0) + self.assertEqual( + len(response.toolbar.get_panel_by_id(CachePanel.panel_id).calls), 0 + ) def test_is_toolbar_request(self): request = rf.get("/__debug__/render_panel/") @@ -275,7 +263,7 @@ def test_is_toolbar_request_with_script_prefix(self): def test_data_gone(self): response = self.client.get( - "/__debug__/render_panel/?store_id=GONE&panel_id=RequestPanel" + "/__debug__/render_panel/?request_id=GONE&panel_id=RequestPanel" ) self.assertIn("Please reload the page and retry.", response.json()["content"]) @@ -330,9 +318,13 @@ def test_html5_validation(self): raise self.failureException(msg) def test_render_panel_checks_show_toolbar(self): - url = "/__debug__/render_panel/" - data = {"store_id": toolbar_store_id(), "panel_id": "VersionsPanel"} + request_id = toolbar_request_id() + get_store().save_panel( + request_id, VersionsPanel.panel_id, {"value": "Test data"} + ) + data = {"request_id": request_id, "panel_id": VersionsPanel.panel_id} + url = "/__debug__/render_panel/" response = self.client.get(url, data) self.assertEqual(response.status_code, 200) response = self.client.get( @@ -349,18 +341,20 @@ def test_render_panel_checks_show_toolbar(self): def test_middleware_render_toolbar_json(self): """Verify the toolbar is rendered and data is stored for a json request.""" - self.assertEqual(len(DebugToolbar._store), 0) + store = get_store() + self.assertEqual(len(list(store.request_ids())), 0) data = {"foo": "bar"} response = self.client.get("/json_view/", data, content_type="application/json") self.assertEqual(response.status_code, 200) self.assertEqual(response.content.decode("utf-8"), '{"foo": "bar"}') # Check the history panel's stats to verify the toolbar rendered properly. - self.assertEqual(len(DebugToolbar._store), 1) - toolbar = list(DebugToolbar._store.values())[0] + request_ids = list(store.request_ids()) + self.assertEqual(len(request_ids), 1) + toolbar = DebugToolbar.fetch(request_ids[0]) self.assertEqual( - toolbar.get_panel_by_id("HistoryPanel").get_stats()["data"], - {"foo": ["bar"]}, + toolbar.get_panel_by_id(HistoryPanel.panel_id).get_stats()["data"], + {"foo": "bar"}, ) def test_template_source_checks_show_toolbar(self): @@ -406,15 +400,19 @@ def test_template_source_errors(self): self.assertContains(response, "Template Does Not Exist: does_not_exist.html") def test_sql_select_checks_show_toolbar(self): + self.client.get("/execute_sql/") + request_ids = list(get_store().request_ids()) + request_id = request_ids[-1] + toolbar = DebugToolbar.fetch(request_id, SQLPanel.panel_id) + panel = toolbar.get_panel_by_id(SQLPanel.panel_id) + djdt_query_id = panel.get_stats()["queries"][-1]["djdt_query_id"] + url = "/__debug__/sql_select/" data = { "signed": SignedDataForm.sign( { - "sql": "SELECT * FROM auth_user", - "raw_sql": "SELECT * FROM auth_user", - "params": "{}", - "alias": "default", - "duration": "0", + "request_id": request_id, + "djdt_query_id": djdt_query_id, } ) } @@ -434,15 +432,19 @@ def test_sql_select_checks_show_toolbar(self): self.assertEqual(response.status_code, 404) def test_sql_explain_checks_show_toolbar(self): + self.client.get("/execute_sql/") + request_ids = list(get_store().request_ids()) + request_id = request_ids[-1] + toolbar = DebugToolbar.fetch(request_id, SQLPanel.panel_id) + panel = toolbar.get_panel_by_id(SQLPanel.panel_id) + djdt_query_id = panel.get_stats()["queries"][-1]["djdt_query_id"] + url = "/__debug__/sql_explain/" data = { "signed": SignedDataForm.sign( { - "sql": "SELECT * FROM auth_user", - "raw_sql": "SELECT * FROM auth_user", - "params": "{}", - "alias": "default", - "duration": "0", + "request_id": request_id, + "djdt_query_id": djdt_query_id, } ) } @@ -468,15 +470,19 @@ def test_sql_explain_postgres_union_query(self): """ Confirm select queries that start with a parenthesis can be explained. """ + self.client.get("/execute_union_sql/") + request_ids = list(get_store().request_ids()) + request_id = request_ids[-1] + toolbar = DebugToolbar.fetch(request_id, SQLPanel.panel_id) + panel = toolbar.get_panel_by_id(SQLPanel.panel_id) + djdt_query_id = panel.get_stats()["queries"][-1]["djdt_query_id"] + url = "/__debug__/sql_explain/" data = { "signed": SignedDataForm.sign( { - "sql": "(SELECT * FROM auth_user) UNION (SELECT * from auth_user)", - "raw_sql": "(SELECT * FROM auth_user) UNION (SELECT * from auth_user)", - "params": "{}", - "alias": "default", - "duration": "0", + "request_id": request_id, + "djdt_query_id": djdt_query_id, } ) } @@ -488,19 +494,19 @@ def test_sql_explain_postgres_union_query(self): connection.vendor == "postgresql", "Test valid only on PostgreSQL" ) def test_sql_explain_postgres_json_field(self): + self.client.get("/execute_json_sql/") + request_ids = list(get_store().request_ids()) + request_id = request_ids[-1] + toolbar = DebugToolbar.fetch(request_id, SQLPanel.panel_id) + panel = toolbar.get_panel_by_id(SQLPanel.panel_id) + djdt_query_id = panel.get_stats()["queries"][-1]["djdt_query_id"] + url = "/__debug__/sql_explain/" - base_query = ( - 'SELECT * FROM "tests_postgresjson" WHERE "tests_postgresjson"."field" @>' - ) - query = base_query + """ '{"foo": "bar"}'""" data = { "signed": SignedDataForm.sign( { - "sql": query, - "raw_sql": base_query + " %s", - "params": '["{\\"foo\\": \\"bar\\"}"]', - "alias": "default", - "duration": "0", + "request_id": request_id, + "djdt_query_id": djdt_query_id, } ) } @@ -519,15 +525,19 @@ def test_sql_explain_postgres_json_field(self): self.assertEqual(response.status_code, 404) def test_sql_profile_checks_show_toolbar(self): + self.client.get("/execute_sql/") + request_ids = list(get_store().request_ids()) + request_id = request_ids[-1] + toolbar = DebugToolbar.fetch(request_id, SQLPanel.panel_id) + panel = toolbar.get_panel_by_id(SQLPanel.panel_id) + djdt_query_id = panel.get_stats()["queries"][-1]["djdt_query_id"] + url = "/__debug__/sql_profile/" data = { "signed": SignedDataForm.sign( { - "sql": "SELECT * FROM auth_user", - "raw_sql": "SELECT * FROM auth_user", - "params": "{}", - "alias": "default", - "duration": "0", + "request_id": request_id, + "djdt_query_id": djdt_query_id, } ) } @@ -556,7 +566,7 @@ def test_render_panels_in_request(self): response = self.client.get(url) self.assertIn(b'id="djDebug"', response.content) # Verify the store id is not included. - self.assertNotIn(b"data-store-id", response.content) + self.assertNotIn(b"data-request-id", response.content) # Verify the history panel was disabled self.assertIn( b'' - ) - query = base_query + """ '{"foo": "bar"}'""" data = { "signed": SignedDataForm.sign( { - "sql": query, - "raw_sql": base_query + " %s", - "params": '["{\\"foo\\": \\"bar\\"}"]', - "alias": "default", - "duration": "0", + "request_id": request_id, + "djdt_query_id": djdt_query_id, } ) } @@ -405,15 +427,19 @@ async def test_sql_explain_postgres_json_field(self): self.assertEqual(response.status_code, 404) async def test_sql_profile_checks_show_toolbar(self): + await self.async_client.get("/execute_sql/") + request_ids = list(get_store().request_ids()) + request_id = request_ids[-1] + toolbar = DebugToolbar.fetch(request_id, SQLPanel.panel_id) + panel = toolbar.get_panel_by_id(SQLPanel.panel_id) + djdt_query_id = panel.get_stats()["queries"][-1]["djdt_query_id"] + url = "/__debug__/sql_profile/" data = { "signed": SignedDataForm.sign( { - "sql": "SELECT * FROM auth_user", - "raw_sql": "SELECT * FROM auth_user", - "params": "{}", - "alias": "default", - "duration": "0", + "request_id": request_id, + "djdt_query_id": djdt_query_id, } ) } @@ -434,7 +460,7 @@ async def test_render_panels_in_request(self): response = await self.async_client.get(url) self.assertIn(b'id="djDebug"', response.content) # Verify the store id is not included. - self.assertNotIn(b"data-store-id", response.content) + self.assertNotIn(b"data-request-id", response.content) # Verify the history panel was disabled self.assertIn( b' None: + cls.store = store.MemoryStore + + def tearDown(self) -> None: + self.store.clear() + + def test_ids(self): + self.store.set("foo") + self.store.set("bar") + self.assertEqual(list(self.store.request_ids()), ["foo", "bar"]) + + def test_exists(self): + self.assertFalse(self.store.exists("missing")) + self.store.set("exists") + self.assertTrue(self.store.exists("exists")) + + def test_set(self): + self.store.set("foo") + self.assertEqual(list(self.store.request_ids()), ["foo"]) + + def test_set_max_size(self): + with self.settings(DEBUG_TOOLBAR_CONFIG={"RESULTS_CACHE_SIZE": 1}): + self.store.save_panel("foo", "foo.panel", "foo.value") + self.store.save_panel("bar", "bar.panel", {"a": 1}) + self.assertEqual(list(self.store.request_ids()), ["bar"]) + self.assertEqual(self.store.panel("foo", "foo.panel"), {}) + self.assertEqual(self.store.panel("bar", "bar.panel"), {"a": 1}) + + def test_clear(self): + self.store.save_panel("bar", "bar.panel", {"a": 1}) + self.store.clear() + self.assertEqual(list(self.store.request_ids()), []) + self.assertEqual(self.store.panel("bar", "bar.panel"), {}) + + def test_delete(self): + self.store.save_panel("bar", "bar.panel", {"a": 1}) + self.store.delete("bar") + self.assertEqual(list(self.store.request_ids()), []) + self.assertEqual(self.store.panel("bar", "bar.panel"), {}) + # Make sure it doesn't error + self.store.delete("bar") + + def test_save_panel(self): + self.store.save_panel("bar", "bar.panel", {"a": 1}) + self.assertEqual(list(self.store.request_ids()), ["bar"]) + self.assertEqual(self.store.panel("bar", "bar.panel"), {"a": 1}) + + def test_panel(self): + self.assertEqual(self.store.panel("missing", "missing"), {}) + self.store.save_panel("bar", "bar.panel", {"a": 1}) + self.assertEqual(self.store.panel("bar", "bar.panel"), {"a": 1}) + + +class StubStore(store.BaseStore): + pass + + +class GetStoreTestCase(TestCase): + def test_get_store(self): + self.assertIs(store.get_store(), store.MemoryStore) + + @override_settings( + DEBUG_TOOLBAR_CONFIG={"TOOLBAR_STORE_CLASS": "tests.test_store.StubStore"} + ) + def test_get_store_with_setting(self): + self.assertIs(store.get_store(), StubStore) + + +class DatabaseStoreTestCase(TestCase): + @classmethod + def setUpTestData(cls) -> None: + cls.store = store.DatabaseStore + + def tearDown(self) -> None: + self.store.clear() + + def test_ids(self): + id1 = str(uuid.uuid4()) + id2 = str(uuid.uuid4()) + self.store.set(id1) + self.store.set(id2) + # Convert the UUIDs to strings for comparison + request_ids = {str(id) for id in self.store.request_ids()} + self.assertEqual(request_ids, {id1, id2}) + + def test_exists(self): + missing_id = str(uuid.uuid4()) + self.assertFalse(self.store.exists(missing_id)) + id1 = str(uuid.uuid4()) + self.store.set(id1) + self.assertTrue(self.store.exists(id1)) + + def test_set(self): + id1 = str(uuid.uuid4()) + self.store.set(id1) + self.assertTrue(self.store.exists(id1)) + + def test_set_max_size(self): + with self.settings(DEBUG_TOOLBAR_CONFIG={"RESULTS_CACHE_SIZE": 1}): + # Clear any existing entries first + self.store.clear() + + # Add first entry + id1 = str(uuid.uuid4()) + self.store.set(id1) + + # Verify it exists + self.assertTrue(self.store.exists(id1)) + + # Add second entry, which should push out the first one due to size limit=1 + id2 = str(uuid.uuid4()) + self.store.set(id2) + + # Verify only the bar entry exists now + # Convert the UUIDs to strings for comparison + request_ids = {str(id) for id in self.store.request_ids()} + self.assertEqual(request_ids, {id2}) + self.assertFalse(self.store.exists(id1)) + + def test_clear(self): + id1 = str(uuid.uuid4()) + self.store.save_panel(id1, "bar.panel", {"a": 1}) + self.store.clear() + self.assertEqual(list(self.store.request_ids()), []) + self.assertEqual(self.store.panel(id1, "bar.panel"), {}) + + def test_delete(self): + id1 = str(uuid.uuid4()) + self.store.save_panel(id1, "bar.panel", {"a": 1}) + self.store.delete(id1) + self.assertEqual(list(self.store.request_ids()), []) + self.assertEqual(self.store.panel(id1, "bar.panel"), {}) + # Make sure it doesn't error + self.store.delete(id1) + + def test_save_panel(self): + id1 = str(uuid.uuid4()) + self.store.save_panel(id1, "bar.panel", {"a": 1}) + self.assertTrue(self.store.exists(id1)) + self.assertEqual(self.store.panel(id1, "bar.panel"), {"a": 1}) + + def test_update_panel(self): + id1 = str(uuid.uuid4()) + self.store.save_panel(id1, "test.panel", {"original": True}) + self.assertEqual(self.store.panel(id1, "test.panel"), {"original": True}) + + # Update the panel + self.store.save_panel(id1, "test.panel", {"updated": True}) + self.assertEqual(self.store.panel(id1, "test.panel"), {"updated": True}) + + def test_panels_nonexistent_request(self): + missing_id = str(uuid.uuid4()) + panels = dict(self.store.panels(missing_id)) + self.assertEqual(panels, {}) + + def test_panel(self): + id1 = str(uuid.uuid4()) + missing_id = str(uuid.uuid4()) + self.assertEqual(self.store.panel(missing_id, "missing"), {}) + self.store.save_panel(id1, "bar.panel", {"a": 1}) + self.assertEqual(self.store.panel(id1, "bar.panel"), {"a": 1}) + + def test_panels(self): + id1 = str(uuid.uuid4()) + self.store.save_panel(id1, "panel1", {"a": 1}) + self.store.save_panel(id1, "panel2", {"b": 2}) + panels = dict(self.store.panels(id1)) + self.assertEqual(len(panels), 2) + self.assertEqual(panels["panel1"], {"a": 1}) + self.assertEqual(panels["panel2"], {"b": 2}) + + def test_cleanup_old_entries(self): + # Create multiple entries + ids = [str(uuid.uuid4()) for _ in range(5)] + for id in ids: + self.store.save_panel(id, "test.panel", {"test": True}) + + # Set a small cache size + with self.settings(DEBUG_TOOLBAR_CONFIG={"RESULTS_CACHE_SIZE": 2}): + # Trigger cleanup + self.store._cleanup_old_entries() + + # Check that only the most recent 2 entries remain + self.assertEqual(len(list(self.store.request_ids())), 2) diff --git a/tests/urls.py b/tests/urls.py index 124e55892..32355d110 100644 --- a/tests/urls.py +++ b/tests/urls.py @@ -17,7 +17,11 @@ path("non_ascii_request/", views.regular_view, {"title": NonAsciiRepr()}), path("new_user/", views.new_user), path("execute_sql/", views.execute_sql), + path("execute_json_sql/", views.execute_json_sql), + path("execute_union_sql/", views.execute_union_sql), path("async_execute_sql/", views.async_execute_sql), + path("async_execute_json_sql/", views.async_execute_json_sql), + path("async_execute_union_sql/", views.async_execute_union_sql), path("async_execute_sql_concurrently/", views.async_execute_sql_concurrently), path("cached_view/", views.cached_view), path("cached_low_level_view/", views.cached_low_level_view), diff --git a/tests/views.py b/tests/views.py index b6e3252af..fa8f0cf22 100644 --- a/tests/views.py +++ b/tests/views.py @@ -8,12 +8,41 @@ from django.template.response import TemplateResponse from django.views.decorators.cache import cache_page +from tests.models import PostgresJSON + def execute_sql(request): list(User.objects.all()) return render(request, "base.html") +def execute_json_sql(request): + list(PostgresJSON.objects.filter(field__contains={"foo": "bar"})) + return render(request, "base.html") + + +async def async_execute_json_sql(request): + list_store = [] + # make async query with filter, which is compatible with async for. + async for obj in PostgresJSON.objects.filter(field__contains={"foo": "bar"}): + list_store.append(obj) + return render(request, "base.html") + + +def execute_union_sql(request): + list(User.objects.all().union(User.objects.all(), all=True)) + return render(request, "base.html") + + +async def async_execute_union_sql(request): + list_store = [] + # make async query with filter, which is compatible with async for. + users = User.objects.all().union(User.objects.all(), all=True) + async for user in users: + list_store.append(user) + return render(request, "base.html") + + async def async_execute_sql(request): """ Some query API can be executed asynchronously but some requires
{{ key|pprint }} {{ value|pprint }} -

{{ store_context.toolbar.stats.HistoryPanel.status_code|escape }}

+

{{ history_context.history_stats.status_code|escape }}

- {{ store_context.form.as_div }} - + {{ history_context.form.as_div }} +