Merge branch 'master' into frontend-release

.pre-commit-config.yaml

@@ -42,12 +42,15 @@ repos:
             "click==8.0.4",
             "enlighten~=1.11",
             # backend
-            "django-cors-headers~=3.14",
-            "django-crispy-forms~=1.14",
-            "django-elasticsearch-dsl~=7.2",
-            "django-environ~=0.10",
-            "django-user-agents~=0.4",
-            "django-widget-tweaks~=1.4",
+            "Django~=4.2.3",
+            "django-cors-headers~=3.14.0",
+            "django-crispy-forms~=1.14.0",
+            "django-elasticsearch-dsl~=7.3.0",
+            "django-bulk-sync~=3.3.0",
+            "django-environ~=0.10.0",
+            "django-stubs-ext~=4.2.2",
+            "django-user-agents~=0.4.0",
+            "django-widget-tweaks~=1.4.12",
             "google-api-python-client~=2.86",
             "jsonschema~=4.20.0",
             "Levenshtein~=0.21.1",

MPCAutofill/accounts/forms.py

@@ -3,7 +3,7 @@
 from django.contrib.auth.models import User
 
 
-class SignUpForm(UserCreationForm):
+class SignUpForm(UserCreationForm[User]):
     # def __init__(self, request, *args, **kwargs):
     #     super().__init__(*args, **kwargs)
     #     self.request = request

MPCAutofill/cardpicker/integrations/mtg.py

@@ -146,7 +146,7 @@ def get_base_url() -> str:
     def retrieve_card_list(cls, url: str) -> str:
         deck_id = url.split("/")[-1]
         response = requests.get(
-            f"https://api.moxfield.com/v2/decks/all/{deck_id}", headers={"user-agent": "Mozilla/5.0"}
+            f"https://api.moxfield.com/v2/decks/all/{deck_id}", headers={"x-requested-by": "mpcautofill"}
         )
 
         response_json = json.loads(response.content.decode("utf-8"))

MPCAutofill/cardpicker/search/search_functions.py

@@ -17,7 +17,7 @@
 
 from django.conf import settings
 from django.core.paginator import Paginator
-from django.db.models import Q
+from django.db.models import Q, QuerySet
 from django.http import HttpRequest
 from django.utils import timezone
 
@@ -343,10 +343,10 @@ def retrieve_card_identifiers(self, search_settings: SearchSettings) -> list[str
         """
         This is the core search function for MPC Autofill - queries Elasticsearch for `self` given `search_settings`
         and returns the list of corresponding `Card` identifiers.
+        Expects that the search index exists. Since this function is called many times, it makes sense to check this
+        once at the call site rather than in the body of this function.
         """
 
-        if not Index(CardSearch.Index.name).exists():
-            raise SearchExceptions.IndexNotFoundException(CardSearch.__name__)
         query_parsed = to_searchable(self.query)
 
         # set up search - match the query and use the AND operator
@@ -379,12 +379,7 @@ def retrieve_card_identifiers(self, search_settings: SearchSettings) -> list[str
         hits_iterable = s.params(preserve_order=True).scan()
 
         source_order = search_settings.get_source_order()
-        if search_settings.fuzzy_search:
-            hits = sorted(hits_iterable, key=lambda x: (source_order[x.source], distance(x.searchq, query_parsed)))
-        else:
-            hits = sorted(hits_iterable, key=lambda x: source_order[x.source])
-
-        return [x.identifier for x in hits]
+        return [result.identifier for result in sorted(hits_iterable, key=lambda result: source_order[result.source])]
 
 
 def get_schema_directory() -> Path:
@@ -472,12 +467,12 @@ def parse_json_body_as_search_data(json_body: dict[str, Any]) -> tuple[SearchSet
     return SearchSettings.from_json_body(json_body), SearchQuery.list_from_json_body(json_body)
 
 
-def get_new_cards_paginator(source: Source) -> Paginator:
+def get_new_cards_paginator(source: Source) -> Paginator[QuerySet[Card]]:
     now = timezone.now()
     cards = Card.objects.filter(
         source=source, date__lt=now, date__gte=now - dt.timedelta(days=NEW_CARDS_DAYS)
     ).order_by("-date")
-    return Paginator(cards, NEW_CARDS_PAGE_SIZE)
+    return Paginator(cards, NEW_CARDS_PAGE_SIZE)  # type: ignore  # TODO: `_SupportsPagination`
 
 
 # endregion

MPCAutofill/cardpicker/sources/api.py

@@ -88,7 +88,12 @@ def unpack_name(self, tags: Tags) -> tuple[pycountry.Languages, str, set[str], s
         language_code, name = image_name_results.groups()
         language = pycountry.languages.get(alpha_2=language_code) if language_code else None
         name_with_no_tags, extracted_tags = tags.extract_name_and_tags(name)
-        return language, sanitisation.fix_whitespace(name_with_no_tags), extracted_tags, extension
+        return (
+            language,
+            sanitisation.fix_whitespace(name_with_no_tags),
+            extracted_tags | self.folder.get_tags(tags=tags),
+            extension,
+        )
 
     def get_language(self, tags: Tags) -> pycountry.Languages:
         language, _, _, _ = self.unpack_name(tags=tags)

MPCAutofill/cardpicker/tags.py

@@ -68,7 +68,18 @@ def extract_name_and_tags(self, name: Optional[str]) -> tuple[str, set[str]]:
                             if start > 0 and end > 0:
                                 name_with_no_tags = name_with_no_tags[0:start] + name_with_no_tags[end:]
 
-        name_with_no_tags = name_with_no_tags.replace("( )", "").replace("()", "").replace("[ ]", "").replace("[]", "")
+        artifacts: list[tuple[str, str]] = [  # remove these extra bits from the name
+            ("( )", ""),
+            ("()", ""),
+            ("[ ]", ""),
+            ("[]", ""),
+            ("[, ", "["),
+            (", ]", "]"),
+            ("(, ", "("),
+            (", )", ")"),
+        ]
+        for artifact, replacement in artifacts:
+            name_with_no_tags = name_with_no_tags.replace(artifact, replacement)
         return name_with_no_tags, tag_set
 
 

MPCAutofill/cardpicker/tests/test_sources.py

@@ -51,6 +51,7 @@ class TestAPI:
     IMAGE_K = Image(
         id="K", name="Image K [Grandchild Tag].png", size=1, created_time=DEFAULT_DATE, height=1, folder=FOLDER_A
     )
+    IMAGE_L = Image(id="L", name="Image L [NSFW].png", size=1, created_time=DEFAULT_DATE, height=1, folder=FOLDER_D)
     IMAGE_FRENCH = Image(
         id="french", name="French.png", size=1, created_time=DEFAULT_DATE, height=1, folder=FOLDER_FRENCH
     )
@@ -106,19 +107,34 @@ def test_folder_tags(self, django_settings, tag_in_data, extended_tag, full_art_
         assert folder.get_tags(tags=tags) == expected_tags
 
     @pytest.mark.parametrize(
-        "folder, expected_name",
+        "folder, expected_language, expected_name, expected_tags",
         [
-            (FOLDER_A, "Folder A"),
-            (FOLDER_B, "Folder B"),
-            (FOLDER_C, "Folder C"),
-            (FOLDER_G, "Folder G (Some more words)"),
-            (FOLDER_H, "Folder H [Some more words]"),
+            (FOLDER_A, None, "Folder A", set()),
+            (FOLDER_B, None, "Folder B", set()),
+            (FOLDER_C, None, "Folder C", {"NSFW"}),
+            (FOLDER_G, None, "Folder G (Some more words)", {"Tag in Data"}),
+            (FOLDER_H, None, "Folder H [Some more words]", {"Tag in Data"}),
         ],
     )
-    def test_folder_name(self, django_settings, tag_in_data, extended_tag, full_art_tag, folder, expected_name):
+    def test_folder_name(
+        self,
+        django_settings,
+        tag_in_data,
+        extended_tag,
+        full_art_tag,
+        folder,
+        expected_language,
+        expected_name,
+        expected_tags,
+    ):
         tags = Tags()
-        _, name, _ = folder.unpack_name(tags=tags)
+        language, name, extracted_tags = folder.unpack_name(tags=tags)
+        if expected_language is None:
+            assert language is None
+        else:
+            assert language.alpha_2.lower() == expected_language.lower()
         assert name == expected_name
+        assert extracted_tags == expected_tags
 
     @pytest.mark.parametrize(
         "image, expected_language",
@@ -154,25 +170,43 @@ def test_image_tags(self, django_settings, grandchild_tag, extended_tag, full_ar
         assert image.get_tags(tags=tags) == expected_tags
 
     @pytest.mark.parametrize(
-        "image, expected_name",
+        "image, expected_language, expected_name, expected_tags, expected_extension",
         [
-            (IMAGE_A, "Image A"),
-            (IMAGE_B, "Image B"),
-            (IMAGE_C, "Image C"),
-            (IMAGE_D, "Image D"),
-            (IMAGE_E, "Image E [invalid tag"),
-            (IMAGE_F, "Image F"),
-            (IMAGE_G, "Image G (John Doe)"),
-            (IMAGE_H, "Image H [A, B] (John Doe)"),
-            (IMAGE_I, "Image A.I"),
-            (IMAGE_NSFW, "NSFW"),
-            (IMAGE_DOUBLE_NSFW, "NSFW"),
+            (IMAGE_A, None, "Image A", set(), "png"),
+            (IMAGE_B, None, "Image B", {"NSFW"}, "png"),
+            (IMAGE_C, None, "Image C", {"NSFW"}, "png"),  # tag inherited from parent
+            (IMAGE_D, None, "Image D", {"NSFW", "Full Art"}, "png"),
+            (IMAGE_E, None, "Image E [invalid tag", set(), "png"),
+            (IMAGE_F, None, "Image F", {"NSFW", "Tag in Data"}, "png"),
+            (IMAGE_G, None, "Image G (John Doe)", {"NSFW"}, "png"),
+            (IMAGE_H, None, "Image H [A, B] (John Doe)", {"NSFW"}, "png"),
+            (IMAGE_I, None, "Image A.I", set(), "png"),
+            (IMAGE_L, None, "Image L", {"NSFW", "Tag in Data"}, "png"),  # first tag from folder, second from image
+            (IMAGE_NSFW, None, "NSFW", {"NSFW"}, "png"),
+            (IMAGE_DOUBLE_NSFW, None, "NSFW", {"NSFW"}, "png"),
         ],
     )
-    def test_image_name(self, django_settings, tag_in_data, extended_tag, full_art_tag, image, expected_name):
+    def test_unpack_name(
+        self,
+        django_settings,
+        tag_in_data,
+        extended_tag,
+        full_art_tag,
+        image,
+        expected_language,
+        expected_name,
+        expected_tags,
+        expected_extension,
+    ):
         tags = Tags()
-        _, name, _, _ = image.unpack_name(tags=tags)
+        language, name, extracted_tags, extension = image.unpack_name(tags=tags)
+        if expected_language is None:
+            assert language is None
+        else:
+            assert language.alpha_2.lower() == expected_language.lower()
         assert name == expected_name
+        assert extracted_tags == expected_tags
+        assert extension == expected_extension
 
 
 # endregion

MPCAutofill/cardpicker/views.py

@@ -8,6 +8,7 @@
 import pycountry
 import sentry_sdk
 from blog.models import BlogPost
+from elasticsearch_dsl.index import Index
 from jsonschema import ValidationError, validate
 
 from django.conf import settings
@@ -18,6 +19,7 @@
 from django.views.decorators.csrf import csrf_exempt
 
 from cardpicker.constants import CARDS_PAGE_SIZE, DEFAULT_LANGUAGE, NSFW
+from cardpicker.documents import CardSearch
 from cardpicker.forms import InputCSV, InputLink, InputText, InputXML
 from cardpicker.integrations.integrations import get_configured_game_integration
 from cardpicker.integrations.patreon import get_patreon_campaign_details, get_patrons
@@ -285,7 +287,7 @@ def input_csv(request: HttpRequest) -> HttpResponse:
             drive_order, fuzzy_search = retrieve_search_settings(request)
             if len(drive_order) == 0:
                 return redirect("index")
-            csvfile = request.FILES["file"].read()
+            csvfile = request.FILES["file"].read()  # type: ignore  # TODO: revisit this and type it properly
 
             # parse the csv file to obtain the order dict and quantity in this order
             order = MPCOrder()
@@ -311,7 +313,7 @@ def input_xml(request: HttpRequest) -> HttpResponse:
                 drive_order, fuzzy_search = retrieve_search_settings(request)
                 if len(drive_order) == 0:
                     return redirect("index")
-                xmlfile = request.FILES["file"].read()
+                xmlfile = request.FILES["file"].read()  # type: ignore  # TODO: revisit this and type it properly
 
                 # parse the XML file to obtain the order dict and quantity in this order
                 order = MPCOrder()
@@ -462,6 +464,8 @@ def post_search_results(request: HttpRequest) -> HttpResponse:
 
     if not ping_elasticsearch():
         raise SearchExceptions.ElasticsearchOfflineException()
+    if not Index(CardSearch.Index.name).exists():
+        raise SearchExceptions.IndexNotFoundException(CardSearch.__name__)
 
     results: dict[str, dict[str, list[str]]] = defaultdict(dict)
     for query in queries:

desktop-tool/src/driver.py

@@ -399,23 +399,16 @@ def authenticate(self) -> None:
         action = self.action
         self.driver.get(f"{self.target_site.value.login_url}")
         self.set_state(States.defining_order, "Awaiting user sign-in")
-        input(
+        print(
             textwrap.dedent(
                 f"""
                 The specified inputs require you to sign into your {bold(self.target_site.name)} account.
-                Please sign in, then return to the console window and press Enter.
+                The tool will automatically resume once you've signed in.
                 """
             )
         )
         while not self.is_user_authenticated():
-            input(
-                textwrap.dedent(
-                    """
-                    It looks like you're not signed in.
-                    Please sign in, then return to the console window and press Enter.
-                    """
-                )
-            )
+            time.sleep(1)
         print("Successfully signed in!")
         self.set_state(States.defining_order, action)
         self.driver.get(f"{self.target_site.value.starting_url}")

frontend/src/common/constants.ts

@@ -10,6 +10,7 @@ export const Token: CardType = "TOKEN";
 
 export const SelectedImageSeparator = "@";
 export const CardTypeSeparator = ":";
+export const FaceSeparator = "//";
 
 export const CardTypePrefixes: { [prefix: string]: CardType } = {
   "": Card,
@@ -26,7 +27,6 @@ export const ReversedCardTypePrefixes = Object.fromEntries(
 
 export const Front: Faces = "front";
 export const Back: Faces = "back";
-export const FaceSeparator = "|";
 
 export const ToggleButtonHeight = 38; // pixels
 export const NavbarHeight = 50; // pixels - aligns with the natural height of the navbar