Merge branch 'master' into frontend-release

MPCAutofill/cardpicker/admin.py

@@ -7,21 +7,25 @@
 @admin.register(Tag)
 class AdminTag(admin.ModelAdmin[Tag]):
     list_display = ("name",)
+    search_fields = ("name",)
 
 
 @admin.register(Card)
 class AdminCard(admin.ModelAdmin[Card]):
     list_display = ("identifier", "name", "source", "dpi", "date", "tags")
+    search_fields = ("identifier", "name")
 
 
 @admin.register(DFCPair)
 class AdminDFCPair(admin.ModelAdmin[DFCPair]):
     list_display = ("front", "back")
+    search_fields = ("front",)
 
 
 @admin.register(Source)
 class AdminSource(admin.ModelAdmin[Source]):
     list_display = ("name", "identifier", "contribution", "description")
+    search_fields = ("name", "identifier")
 
     def contribution(self, obj: Source) -> str:
         qty_all, qty_cards, qty_cardbacks, qty_tokens, avgdpi = obj.count()

MPCAutofill/cardpicker/search/search_functions.py

@@ -268,7 +268,7 @@ def retrieve_cardback_identifiers(self) -> list[str]:
         """
 
         cardbacks: list[str]
-        order_by = ["-priority", "source__name", "name"]
+        order_by = ["-priority", "source__ordinal", "source__name", "name"]
         if self.filter_cardbacks:
             # afaik, `~Q(pk__in=[])` is the best way to have an always-true filter
             language_filter = (

MPCAutofill/cardpicker/sources/update_database.py

@@ -20,7 +20,10 @@
 
 
 def add_images_in_folder_to_list(source_type: Type[SourceType], folder: Folder, images: deque[Image]) -> None:
-    images.extend(source_type.get_all_images_inside_folder(folder))
+    try:
+        images.extend(source_type.get_all_images_inside_folder(folder))
+    except Exception as e:
+        print(f"Uncaught exception while adding images in folder to list: **{e}**")
 
 
 def explore_folder(source: Source, source_type: Type[SourceType], root_folder: Folder) -> list[Image]:
@@ -59,6 +62,7 @@ def transform_images_into_objects(source: Source, images: list[Image], tags: Tag
     card_count = 0
     cardback_count = 0
     token_count = 0
+    errors: list[str] = []  # report on all exceptions at the end
 
     for image in images:
         try:
@@ -115,12 +119,23 @@ def transform_images_into_objects(source: Source, images: list[Image], tags: Tag
                 )
             )
         except AssertionError as e:
-            print(f"Skipping image **{image.name}** (identifier **{image.id}**) for the following reason: **{e}**")
+            errors.append(
+                f"Assertion error while processing **{image.name}** (identifier **{image.id}**) will not be indexed "
+                f"for the following reason: **{e}**"
+            )
+        except Exception as e:
+            errors.append(
+                f"Uncaught exception while processing image **{image.name}** (identifier **{image.id}**): **{e}**"
+            )
     print(
         f" and done! Generated {TEXT_BOLD}{card_count:,}{TEXT_END} card/s, {TEXT_BOLD}{cardback_count:,}{TEXT_END} "
         f"cardback/s, and {TEXT_BOLD}{token_count:,}{TEXT_END} token/s in "
         f"{TEXT_BOLD}{(time.time() - t0):.2f}{TEXT_END} seconds."
     )
+    if errors:
+        print("The following cards failed to process:", flush=True)
+        for error in errors:
+            print(f"* {error}", flush=True)
 
     return cards
 

desktop-tool/requirements.txt

@@ -6,7 +6,7 @@ defusedxml~=0.7.1
 enlighten~=1.11.2
 fpdf2~=2.7.4
 InquirerPy~=0.3.4
-pillow==10.0.1
+pillow==10.2.0
 pre-commit
 pyinstaller~=5.13.0
 pytest~=7.3

docker/django/Dockerfile

@@ -19,12 +19,6 @@ RUN apt-get update && \
 RUN rm -rf /etc/cron.*/* && \
     chmod u+s /usr/sbin/cron
 
-# Configure apt with node 18
-RUN curl -sL https://deb.nodesource.com/setup_18.x | bash -
-
-# Install node and npm
-RUN apt-get install -y nodejs
-
 # Copy requirements.txt
 COPY MPCAutofill/requirements.txt /MPCAutofill/
 WORKDIR /MPCAutofill
@@ -33,15 +27,20 @@ WORKDIR /MPCAutofill
 RUN pip3 install gunicorn wheel
 RUN pip3 install -r requirements.txt
 
-# Copy frontend files from repository
-COPY MPCAutofill/cardpicker/frontend /MPCAutofill/cardpicker/frontend
-COPY MPCAutofill/package.json /MPCAutofill/package.json
-COPY MPCAutofill/package-lock.json /MPCAutofill/package-lock.json
-
 # Copy relevant files from repository
 COPY docker /MPCAutofill/docker
+COPY common /MPCAutofill/common
 COPY MPCAutofill /MPCAutofill/MPCAutofill
 
+# Handle environment variables - write out `.env` with the variables passed from the
+# compose file (and overwrite `.env` as copied from the host machine in the process)
+RUN touch /MPCAutofill/MPCAutofill/MPCAutofill/.env
+RUN echo "DATABASE_HOST=$DATABASE_HOST" > /MPCAutofill/MPCAutofill/MPCAutofill/.env  # using > to overwrite the file
+RUN echo "ELASTICSEARCH_HOST=$ELASTICSEARCH_HOST" >> /MPCAutofill/MPCAutofill/MPCAutofill/.env
+RUN echo "DEBUG=$DEBUG" >> /MPCAutofill/MPCAutofill/MPCAutofill/.env
+RUN echo "ALLOWED_HOSTS=$ALLOWED_HOSTS" >> /MPCAutofill/MPCAutofill/MPCAutofill/.env
+RUN echo "GAME=$GAME" >> /MPCAutofill/MPCAutofill/MPCAutofill/.env
+
 # Make sure that all scripts are executable, and in case we
 # checked out under Windows with CRLF, convert line endings
 RUN chmod +x docker/django/*.sh && \

docker/django/entrypoint.sh

@@ -18,7 +18,6 @@ until curl --silent --output /dev/null http://elasticsearch:9200/_cat/health?h=s
 done
 
 # Gather static files
-npm install && npm run build
 python3 manage.py collectstatic --noinput
 
 # Check if we are running for the first time

docker/docker-compose.prod.yml

@@ -7,6 +7,8 @@
 # Many code templates and inspirations are taken from blog post:
 # https://testdriven.io/blog/dockerizing-django-with-postgres-gunicorn-and-nginx/
 
+# Run this file with compose by specifying this file and the base compose file:
+# docker compose -f docker-compose.prod.yml -f docker-compose.yml up
 version: "3.4"
 
 services:
@@ -22,6 +24,14 @@ services:
     depends_on:
       - postgres
       - elasticsearch
+    environment:
+      # Do not change these variables
+      - DATABASE_HOST=postgres
+      - ELASTICSEARCH_HOST=elasticsearch
+      - DEBUG=False
+      - ALLOWED_HOSTS=django-api
+      # These variables may be customised
+      - GAME=MTG
 
   # nginx serving the frontend
   nginx:
@@ -31,7 +41,7 @@ services:
       context: ..
       dockerfile: ./docker/nginx/Dockerfile
     ports:
-      - "${FRONTEND_PORT}:80"
+      - "80:80" # TODO: support ports other than 80
 
   postgres:
     expose:

docker/docker-compose.yml

@@ -28,9 +28,9 @@ services:
       - discovery.type=single-node
       - logger.level=WARN
     volumes:
-      - elaticsearch_data:/usr/share/elasticsearch/data
+      - elasticsearch_data:/usr/share/elasticsearch/data
 
 # Persistent storage for containers
 volumes:
   postgres_data:
-  elaticsearch_data:
+  elasticsearch_data:

docker/nginx/Dockerfile

@@ -1,16 +1,15 @@
 # Docker to serve the static Next.js frontend via nginx
 FROM nginx:1.21-alpine
 
-# Copy nginx config
-RUN rm /etc/nginx/conf.d/default.conf
-COPY docker/nginx/nginx.conf /etc/nginx/conf.d
-
 # Copy common and frontend files
 COPY frontend /frontend
 RUN rm -rf /frontend/out
 COPY common /common
 WORKDIR /frontend
 
+# Point frontend at the correct URL to communicate with the backend
+RUN echo "NEXT_PUBLIC_BACKEND_URL=http://localhost:80" > /frontend/.env.local
+
 # install npm
 RUN apk add --update npm
 
@@ -21,3 +20,7 @@ RUN npx next build
 
 # Copy our static site files into the directory that the nginx docker container consumes files from
 RUN cp -r out/* /usr/share/nginx/html
+
+# Copy nginx config
+RUN rm /etc/nginx/conf.d/default.conf
+COPY docker/nginx/nginx.conf /etc/nginx/conf.d

docker/nginx/nginx.conf

@@ -1,8 +1,15 @@
+upstream django-api {
+    server django:8000;
+}
+
 server {
     listen 80 default_server;
     listen [::]:80 default_server;
     root /usr/share/nginx/html;
     index index.html;
+    location /2/ {
+        proxy_pass http://django-api;
+    }
     location / {
         # remove .html extension. retrieved from https://stackoverflow.com/a/38238001
         if ($request_uri ~ ^/(.*)\.html(\?|$)) {