Skip to content

Commit

Permalink
Merge pull request #5081 from open-formulieren/bug/5065-attachments-i…
Browse files Browse the repository at this point in the history 
…n-a-repeating-group-are-not-processed-in-the-json-dump-registration-backend

Process edit grid components in the json dump registration backend
  • Loading branch information
@viktorvanwijk
viktorvanwijk authored Feb 6, 2025
2 parents 9894ae6 + 2c638d2 commit 6d3db22
Show file tree
Hide file tree
Showing 4 changed files with 633 additions and 78 deletions.
245 changes: 167 additions & 78 deletions src/openforms/registrations/contrib/json_dump/plugin.py
View file
Original file line number Diff line number Diff line change
@@ -1,16 +1,20 @@
import base64
import json
from collections import defaultdict
from typing import cast

from django.core.exceptions import SuspiciousOperation
from django.core.serializers.json import DjangoJSONEncoder
from django.db.models import F, TextField, Value
from django.db.models.functions import Coalesce, NullIf
from django.utils.translation import gettext_lazy as _

from zgw_consumers.client import build_client

from openforms.formio.constants import DataSrcOptions
from openforms.formio.service import rewrite_formio_components
from openforms.formio.typing import (
Component,
FileComponent,
RadioComponent,
SelectBoxesComponent,
Expand Down Expand Up @@ -114,27 +118,43 @@ def post_process(
) -> None:
"""Post-process the values and schema.
File components need special treatment, as we send the content of the file
encoded with base64, instead of the output from the serializer. Also, Radio,
Select, and SelectBoxes components need to be updated if their data source is
set to another form variable.
- Update the configuration wrapper. This is necessary to update the options for
Select, SelectBoxes, and Radio components that get their options from another form
variable.
- Get all attachments of this submission, and group them by the data path
- Process each component
:param values: JSONObject
:param schema: JSONObject
:param submission: Submission
:param values: Mapping from key to value of the data to be sent.
:param schema: JSON schema describing ``values``.
:param submission: The corresponding submission instance.
"""
state = submission.load_submission_value_variables_state()

# Update config wrapper. This is necessary to update the options for Select,
# SelectBoxes, and Radio components that get their options from another form
# variable.
# Update config wrapper
data = DataContainer(state=state)
configuration_wrapper = rewrite_formio_components(
submission.total_configuration_wrapper,
submission=submission,
data=data.data,
)

# Create attachment mapping from key or component data path to attachment list
attachments = submission.attachments.annotate(
data_path=Coalesce(
NullIf(
F("_component_data_path"),
Value(""),
),
# fall back to variable/component key if no explicit data path is set
F("submission_variable__key"),
output_field=TextField(),
)
)
attachments_dict = defaultdict(list)
for attachment in attachments:
key = attachment.data_path # pyright: ignore[reportAttributeAccessIssue]
attachments_dict[key].append(attachment)

for key in values.keys():
variable = state.variables.get(key)
if (
Expand All @@ -149,86 +169,154 @@ def post_process(
component = configuration_wrapper[key]
assert component is not None

match component:
case {"type": "file", "multiple": True}:
attachment_list, base_schema = get_attachments_and_base_schema(
cast(FileComponent, component), submission
)
values[key] = attachment_list # type: ignore
schema["properties"][key] = to_multiple(base_schema) # type: ignore

case {"type": "file"}: # multiple is False or missing
attachment_list, base_schema = get_attachments_and_base_schema(
cast(FileComponent, component), submission
)

assert (n_attachments := len(attachment_list)) <= 1 # sanity check
if n_attachments == 0:
value = None
base_schema = {"title": component["label"], "type": "null"}
else:
value = attachment_list[0]
values[key] = value
schema["properties"][key] = base_schema # type: ignore

case {"type": "radio", "openForms": {"dataSrc": DataSrcOptions.variable}}:
component = cast(RadioComponent, component)
choices = [options["value"] for options in component["values"]]
choices.append("") # Take into account an unfilled field
process_component(
component, values, schema, attachments_dict, configuration_wrapper
)


def process_component(
component: Component,
values: JSONObject,
schema: JSONObject,
attachments: dict[str, list[SubmissionFileAttachment]],
configuration_wrapper,
key_prefix: str = "",
) -> None:
"""Process a component.
The following components need extra attention:
- File components: we send the content of the file encoded with base64, instead of
the output from the serializer.
- Radio, Select, and SelectBoxes components: need to be updated if their data source
is set to another form variable.
- Edit grid components: layout component with other components as children, which
(potentially) need to be processed.
:param component: Component
:param values: Mapping from key to value of the data to be sent.
:param schema: JSON schema describing ``values``.
:param attachments: Mapping from component submission data path to list of
attachments corresponding to that component.
:param configuration_wrapper: Updated total configuration wrapper. This is required
for edit grid components, which need to fetch their children from it.
:param key_prefix: If the component is part of an edit grid component, this key
prefix includes the parent key and the index of the component as it appears in the
submitted data list of that edit grid component.
"""
key = component["key"]

match component:
case {"type": "file", "multiple": True}:
attachment_list, base_schema = get_attachments_and_base_schema(
cast(FileComponent, component), attachments, key_prefix
)
values[key] = attachment_list # type: ignore
schema["properties"][key] = to_multiple(base_schema) # type: ignore

case {"type": "file"}: # multiple is False or missing
attachment_list, base_schema = get_attachments_and_base_schema(
cast(FileComponent, component), attachments, key_prefix
)

assert (n_attachments := len(attachment_list)) <= 1 # sanity check
if n_attachments == 0:
value = None
base_schema = {"title": component["label"], "type": "null"}
else:
value = attachment_list[0]
values[key] = value
schema["properties"][key] = base_schema # type: ignore

case {"type": "radio", "openForms": {"dataSrc": DataSrcOptions.variable}}:
component = cast(RadioComponent, component)
choices = [options["value"] for options in component["values"]]
choices.append("") # Take into account an unfilled field
schema["properties"][key]["enum"] = choices # type: ignore

case {"type": "select", "openForms": {"dataSrc": DataSrcOptions.variable}}:
component = cast(SelectComponent, component)
choices = [options["value"] for options in component["data"]["values"]] # type: ignore[reportTypedDictNotRequiredAccess]
choices.append("") # Take into account an unfilled field

if component.get("multiple", False):
schema["properties"][key]["items"]["enum"] = choices # type: ignore
else:
schema["properties"][key]["enum"] = choices # type: ignore

case {"type": "select", "openForms": {"dataSrc": DataSrcOptions.variable}}:
component = cast(SelectComponent, component)
choices = [options["value"] for options in component["data"]["values"]] # type: ignore[reportTypedDictNotRequiredAccess]
choices.append("") # Take into account an unfilled field

if component.get("multiple", False):
schema["properties"][key]["items"]["enum"] = choices # type: ignore
else:
schema["properties"][key]["enum"] = choices # type: ignore

case {"type": "selectboxes"}:
component = cast(SelectBoxesComponent, component)
data_src = component.get("openForms", {}).get("dataSrc")

if data_src == DataSrcOptions.variable:
properties = {
options["value"]: {"type": "boolean"}
for options in component["values"]
}
base_schema = {
"properties": properties,
"required": list(properties.keys()),
"additionalProperties": False,
}
schema["properties"][key].update(base_schema) # type: ignore

# If the select boxes component was hidden, the submitted data of this
# component is an empty dict, so set the required to an empty list.
if not values[key]:
schema["properties"][key]["required"] = [] # type: ignore
case _:
pass
case {"type": "selectboxes"}:
component = cast(SelectBoxesComponent, component)
data_src = component.get("openForms", {}).get("dataSrc")

if data_src == DataSrcOptions.variable:
properties = {
options["value"]: {"type": "boolean"}
for options in component["values"]
}
base_schema = {
"properties": properties,
"required": list(properties.keys()),
"additionalProperties": False,
}
schema["properties"][key].update(base_schema) # type: ignore

# If the select boxes component was hidden, the submitted data of this
# component is an empty dict, so set the required to an empty list.
if not values[key]:
schema["properties"][key]["required"] = [] # type: ignore

case {"type": "editgrid"}:
# Note: the schema actually only needs to be processed once for each child
# component, but will be processed for each submitted repeating group entry
# for implementation simplicity.
edit_grid_schema: JSONObject = schema["properties"][key]["items"] # type: ignore

for index, edit_grid_values in enumerate(
cast(list[JSONObject], values[key])
):

for child_key in edit_grid_values.keys():
process_component(
component=configuration_wrapper[child_key],
values=edit_grid_values,
schema=edit_grid_schema,
attachments=attachments,
configuration_wrapper=configuration_wrapper,
key_prefix=(
f"{key_prefix}.{key}.{index}"
if key_prefix
else f"{key}.{index}"
),
)

case _:
pass


def get_attachments_and_base_schema(
component: FileComponent, submission: Submission
component: FileComponent,
attachments: dict[str, list[SubmissionFileAttachment]],
key_prefix: str = "",
) -> tuple[list[JSONObject], JSONObject]:
"""Return list of encoded attachments and the base schema.
:param component: FileComponent
:param submission: Submission
:return encoded_attachments: list[JSONObject]
:return base_schema: JSONObject
:param attachments: Mapping from component submission data path to list of
attachments corresponding to that component.
:param key_prefix: If the file component is part of an edit grid component, this key
prefix includes the parent key and the index of the component as it appears in the
submitted data list of the edit grid component.
:return encoded_attachments: List of encoded attachments for this file component.
:return base_schema: JSON schema describing the entries of ``encoded_attachments``.
"""
key = f"{key_prefix}.{component['key']}" if key_prefix else component["key"]

encoded_attachments: list[JSONObject] = [
{
"file_name": attachment.original_name,
"content": encode_attachment(attachment),
}
for attachment in submission.attachments
if attachment.form_key == component["key"]
for attachment in attachments.get(key, [])
]

base_schema: JSONObject = {
Expand All @@ -251,10 +339,11 @@ def get_attachments_and_base_schema(


def encode_attachment(attachment: SubmissionFileAttachment) -> str:
"""Encode an attachment using base64
"""Encode an attachment using base64.
:param attachment: Attachment to encode.
:param attachment: Attachment to encode
:returns: Encoded base64 data as a string
:returns: Encoded base64 data as a string.
"""
with attachment.content.open("rb") as f:
f.seek(0)
Expand Down
81 changes: 81 additions & 0 deletions ...ackendTests.test_components_with_form_variable_as_data_source_in_edit_grid_component.yaml
View file
Original file line number Diff line number Diff line change
@@ -0,0 +1,81 @@
interactions:
- request:
body: '"{\"values\": {\"repeatingGroup\": [{\"radio\": \"A\", \"select\": \"B\",
\"selectBoxes\": {\"A\": false, \"B\": false, \"C\": true}}]}, \"values_schema\":
{\"$schema\": \"https://json-schema.org/draft/2020-12/schema\", \"type\": \"object\",
\"properties\": {\"repeatingGroup\": {\"title\": \"Repeating Group\", \"type\":
\"array\", \"items\": {\"type\": \"object\", \"properties\": {\"radio\": {\"title\":
\"Radio\", \"type\": \"string\", \"enum\": [\"A\", \"B\", \"C\", \"\"]}, \"select\":
{\"type\": \"string\", \"title\": \"Select\", \"enum\": [\"A\", \"B\", \"C\",
\"\"]}, \"selectBoxes\": {\"title\": \"Select Boxes\", \"type\": \"object\",
\"additionalProperties\": false, \"properties\": {\"A\": {\"type\": \"boolean\"},
\"B\": {\"type\": \"boolean\"}, \"C\": {\"type\": \"boolean\"}}, \"required\":
[\"A\", \"B\", \"C\"]}}, \"required\": [\"radio\", \"select\", \"selectBoxes\"],
\"additionalProperties\": false}}}, \"required\": [\"repeatingGroup\"], \"additionalProperties\":
false}, \"metadata\": {}, \"metadata_schema\": {\"$schema\": \"https://json-schema.org/draft/2020-12/schema\",
\"type\": \"object\", \"properties\": {}, \"required\": [], \"additionalProperties\":
false}}"'
headers:
Accept:
- '*/*'
Accept-Encoding:
- gzip, deflate, br
Authorization:
- Bearer eyJhbGciOiJIUzI1NiIsInR5cCI6IkpXVCJ9.eyJpc3MiOiIiLCJpYXQiOjE3Mzg4NTA2NjYsImNsaWVudF9pZCI6IiIsInVzZXJfaWQiOiIiLCJ1c2VyX3JlcHJlc2VudGF0aW9uIjoiIn0.Rl0rTl_HiE1CgWXQMqR9Yfq9u31k6sN7Gj-Z3wSnX4Y
Connection:
- keep-alive
Content-Length:
- '1192'
Content-Type:
- application/json
User-Agent:
- python-requests/2.32.2
method: POST
uri: http://localhost/json_plugin
response:
body:
string: "{\n \"data\": {\n \"metadata\": {},\n \"metadata_schema\": {\n
\ \"$schema\": \"https://json-schema.org/draft/2020-12/schema\",\n \"additionalProperties\":
false,\n \"properties\": {},\n \"required\": [],\n \"type\":
\"object\"\n },\n \"values\": {\n \"repeatingGroup\": [\n {\n
\ \"radio\": \"A\",\n \"select\": \"B\",\n \"selectBoxes\":
{\n \"A\": false,\n \"B\": false,\n \"C\":
true\n }\n }\n ]\n },\n \"values_schema\": {\n
\ \"$schema\": \"https://json-schema.org/draft/2020-12/schema\",\n \"additionalProperties\":
false,\n \"properties\": {\n \"repeatingGroup\": {\n \"items\":
{\n \"additionalProperties\": false,\n \"properties\":
{\n \"radio\": {\n \"enum\": [\n \"A\",\n
\ \"B\",\n \"C\",\n \"\"\n
\ ],\n \"title\": \"Radio\",\n \"type\":
\"string\"\n },\n \"select\": {\n \"enum\":
[\n \"A\",\n \"B\",\n \"C\",\n
\ \"\"\n ],\n \"title\": \"Select\",\n
\ \"type\": \"string\"\n },\n \"selectBoxes\":
{\n \"additionalProperties\": false,\n \"properties\":
{\n \"A\": {\n \"type\": \"boolean\"\n
\ },\n \"B\": {\n \"type\":
\"boolean\"\n },\n \"C\": {\n \"type\":
\"boolean\"\n }\n },\n \"required\":
[\n \"A\",\n \"B\",\n \"C\"\n
\ ],\n \"title\": \"Select Boxes\",\n \"type\":
\"object\"\n }\n },\n \"required\": [\n
\ \"radio\",\n \"select\",\n \"selectBoxes\"\n
\ ],\n \"type\": \"object\"\n },\n \"title\":
\"Repeating Group\",\n \"type\": \"array\"\n }\n },\n
\ \"required\": [\n \"repeatingGroup\"\n ],\n \"type\":
\"object\"\n }\n },\n \"message\": \"Data received\"\n}\n"
headers:
Connection:
- close
Content-Length:
- '2191'
Content-Type:
- application/json
Date:
- Thu, 06 Feb 2025 14:04:26 GMT
Server:
- Werkzeug/3.1.3 Python/3.12.8
status:
code: 201
message: CREATED
version: 1
Loading

0 comments on commit 6d3db22

Please sign in to comment.