Merge branch 'master' into frontend-release

MPCAutofill/cardpicker/sources/update_database.py

@@ -1,3 +1,4 @@
+import socket
 import time
 from collections import deque
 from concurrent.futures import ThreadPoolExecutor
@@ -161,6 +162,8 @@ def update_database(source_key: Optional[str] = None) -> None:
     If `source_key` is specified, only update that source; otherwise, update all sources.
     """
 
+    # try to work around https://github.com/googleapis/google-api-python-client/issues/2186
+    socket.setdefaulttimeout(15 * 60)
     tags = Tags()
     if source_key:
         try:

MPCAutofill/cardpicker/tests/__snapshots__/test_integrations.ambr

@@ -80,6 +80,7 @@
     '1 Delver of Secrets // Insectile Aberration': 1,
     '3 Past in Flames': 1,
     '4 Brainstorm': 1,
+    't:Innistrad Checklist': 1,
   })
 # ---
 # name: TestMTGIntegration.test_valid_url[https://www.mtggoldfish.com/deck/5149750]

frontend/.env.dist

@@ -1 +1,2 @@
 NEXT_PUBLIC_BACKEND_URL=
+NEXT_PUBLIC_IMAGE_CDN_URL=

frontend/package.json

@@ -70,7 +70,6 @@
     "eslint-plugin-simple-import-sort": "^10.0.0",
     "file-loader": "^6.2.0",
     "jest": "^29.5.0",
-    "jest-each": "^29.5.0",
     "jest-environment-jsdom": "^29.5.0",
     "jest-styled-components": "^7.1.1",
     "jsdom": "^21.1.1",

frontend/src/common/constants.ts

@@ -79,3 +79,11 @@ export const GoogleDriveImageAPIURL =
 
 export const SearchResultsEndpointPageSize = 300;
 export const CardEndpointPageSize = 1000;
+
+export enum CSVHeaders {
+  quantity = "Quantity",
+  frontQuery = "Front",
+  frontSelectedImage = "Front ID",
+  backQuery = "Back",
+  backSelectedImage = "Back ID",
+}

frontend/src/common/processing.test.ts

@@ -1,5 +1,3 @@
-import each from "jest-each";
-
 import {
   Card,
   Cardback,
@@ -8,6 +6,7 @@ import {
   Token,
 } from "@/common/constants";
 import {
+  parseCSVFileAsLines,
   processLine,
   processPrefix,
   processQuery,
@@ -350,18 +349,30 @@ test("a line specifying the selected image ID for both faces is processed correc
 });
 
 describe("URLs are sanitised correctly", () => {
-  each([
+  test.each([
     "http://127.0.0.1:8000",
     "http://127.0.0.1:8000/",
     "https://127.0.0.1:8000",
     "127.0.0.1:8000",
     "127.0.0.1:8000/",
     "127.0.0.1:8000/path",
-  ]).test("%s", (text) => {
+  ])("%s", (text) => {
     expect(standardiseURL(text)).toBe(
       "http" + (text.includes("http://") ? "" : "s") + "://127.0.0.1:8000"
     );
   });
 });
 
+test.each([
+  "Quantity,Front,Front ID,Back,Back ID\n2, opt, xyz, char, abcd",
+  "Quantity, Front, Front ID, Back, Back ID\n2, opt, xyz, char, abcd",
+  "Quantity,   Front, Front ID,   Back,  Back ID   \n    2,   opt,  xyz,  char, abcd  ",
+])("CSV is parsed correctly", () => {
+  const csv =
+    "Quantity, Front, Front ID, Back, Back ID\n2, opt, xyz, char, abcd";
+  expect(parseCSVFileAsLines(csv)).toStrictEqual([
+    `2 opt${SelectedImageSeparator}xyz ${FaceSeparator} char${SelectedImageSeparator}abcd`,
+  ]);
+});
+
 // # endregion

frontend/src/common/processing.ts

@@ -3,12 +3,15 @@
  */
 
 import { toByteArray } from "base64-js";
+// @ts-ignore
+import { parse } from "lil-csv";
 
 import {
   Card,
   Cardback,
   CardTypePrefixes,
   CardTypeSeparator,
+  CSVHeaders,
   FaceSeparator,
   ProjectMaxSize,
   ReversedCardTypePrefixes,
@@ -17,6 +20,7 @@ import {
 } from "@/common/constants";
 import {
   CardDocument,
+  CSVRow,
   DFCPairs,
   ProcessedLine,
   ProjectMember,
@@ -304,3 +308,31 @@ export const formatPlaceholderText = (placeholders: {
   }
   return placeholderTextByCardType.join(separator + separator);
 };
+
+export const parseCSVRowAsLine = (rawRow: CSVRow): string => {
+  const row = Object.fromEntries(
+    Object.entries(rawRow).map(([key, value]) => [key.trim(), value?.trim()])
+  );
+  let formattedLine = `${row[CSVHeaders.quantity] ?? ""} ${
+    row[CSVHeaders.frontQuery] ?? ""
+  }`;
+  if ((row[CSVHeaders.frontSelectedImage] ?? "").length > 0) {
+    formattedLine += `${SelectedImageSeparator}${
+      row[CSVHeaders.frontSelectedImage]
+    }`;
+  }
+  if ((row[CSVHeaders.backQuery] ?? "").length > 0) {
+    formattedLine += ` ${FaceSeparator} ${row[CSVHeaders.backQuery]}`;
+    if ((row[CSVHeaders.backSelectedImage] ?? "").length > 0) {
+      formattedLine += `${SelectedImageSeparator}${
+        row[CSVHeaders.backSelectedImage]
+      }`;
+    }
+  }
+  return formattedLine;
+};
+
+export const parseCSVFileAsLines = (fileContents: string): Array<string> => {
+  const rows: Array<CSVRow> = parse(fileContents);
+  return rows.map(parseCSVRowAsLine);
+};

frontend/src/common/types.ts

@@ -2,6 +2,7 @@ import { createAsyncThunk } from "@reduxjs/toolkit";
 import { TypedUseSelectorHook, useDispatch, useSelector } from "react-redux";
 
 import type { AppDispatch, RootState } from "@/app/store";
+import { CSVHeaders } from "@/common/constants";
 import { SearchQuery } from "@/common/schema_types";
 export type {
   FilterSettings,
@@ -266,3 +267,7 @@ export interface Tag {
   parent: string | null;
   children: Array<Tag>;
 }
+
+export type CSVRow = {
+  [column in CSVHeaders]?: string;
+};

frontend/src/features/card/card.tsx

@@ -115,11 +115,21 @@ function CardImage({
 
   //# region computed constants
 
+  // TODO: always point at image server once it's stable
+  const imageCDNURL = process.env.NEXT_PUBLIC_IMAGE_CDN_URL;
+  const smallThumbnailURL =
+    imageCDNURL != null
+      ? `${imageCDNURL}/images/google_drive/small/${maybeCardDocument?.identifier}.jpg`
+      : maybeCardDocument?.small_thumbnail_url;
+  const mediumThumbnailURL =
+    imageCDNURL != null
+      ? `${imageCDNURL}/images/google_drive/large/${maybeCardDocument?.identifier}.jpg`
+      : maybeCardDocument?.medium_thumbnail_url;
   const imageSrc: string | undefined =
     imageState !== "errored"
       ? small
-        ? maybeCardDocument?.small_thumbnail_url
-        : maybeCardDocument?.medium_thumbnail_url
+        ? smallThumbnailURL
+        : mediumThumbnailURL
       : small
       ? "/error_404.png"
       : "/error_404_med.png";

frontend/src/features/export/finishedMyProjectModal.tsx

@@ -224,7 +224,7 @@ export function FinishedMyProjectModal({ show, handleClose }: ExitModal) {
               style={{ backgroundColor: "#d99a07" }}
               className="text-center"
             >
-              <b>Are you on macOS or Linux</b> There&apos;s another step before
+              <b>Are you on macOS or Linux?</b> There&apos;s another step before
               you can double-click it - check out the wiki (linked above) for
               details.
             </Alert>

frontend/src/features/import/importCSV.test.tsx

@@ -182,3 +182,26 @@ test("importing a more complex CSV into an empty project", async () => {
   await expectCardGridSlotState(3, Back, cardDocument2.name, 1, 2);
   await expectCardbackSlotState(cardDocument2.name, 1, 2);
 });
+
+test("CSV header has spaces", async () => {
+  server.use(
+    cardDocumentsThreeResults,
+    cardbacksTwoOtherResults,
+    sourceDocumentsOneResult,
+    searchResultsThreeResults,
+    ...defaultHandlers
+  );
+  renderWithProviders(<App />, { preloadedState });
+
+  // import a card
+  await importCSV(
+    `Quantity, Front , Front ID
+    ,my search query,${cardDocument3.identifier}`
+  );
+
+  // a card slot should have been created
+  await expectCardSlotToExist(1);
+  await expectCardGridSlotState(1, Front, cardDocument3.name, 3, 3);
+  await expectCardGridSlotState(1, Back, cardDocument2.name, 1, 2);
+  await expectCardbackSlotState(cardDocument2.name, 1, 2);
+});

frontend/src/features/import/importCSV.tsx

@@ -6,18 +6,17 @@
  * for repeatability.
  */
 
-// @ts-ignore // TODO: put a PR into this repo adding types
-import { parse } from "lil-csv";
 import React, { useState } from "react";
 import Button from "react-bootstrap/Button";
 import Dropdown from "react-bootstrap/Dropdown";
 import Modal from "react-bootstrap/Modal";
 
 import { useGetDFCPairsQuery } from "@/app/api";
-import { FaceSeparator, SelectedImageSeparator } from "@/common/constants";
+import { CSVHeaders } from "@/common/constants";
 import { TextFileDropzone } from "@/common/dropzone";
 import {
   convertLinesIntoSlotProjectMembers,
+  parseCSVFileAsLines,
   processLines,
 } from "@/common/processing";
 import { useAppDispatch, useAppSelector } from "@/common/types";
@@ -27,14 +26,6 @@ import { addMembers, selectProjectSize } from "@/features/project/projectSlice";
 import { selectFuzzySearch } from "@/features/searchSettings/searchSettingsSlice";
 import { setError } from "@/features/toasts/toastsSlice";
 
-const CSVHeaders: { [key: string]: string } = {
-  quantity: "Quantity",
-  frontQuery: "Front",
-  frontSelectedImage: "Front ID",
-  backQuery: "Back",
-  backSelectedImage: "Back ID",
-};
-
 function CSVFormat() {
   /**
    * Instruct the user on how to format their CSV files.
@@ -136,33 +127,9 @@ export function ImportCSV() {
       return;
     }
 
-    const rows = parse(fileContents, {
-      header: Object.fromEntries(
-        Object.entries(CSVHeaders).map(([key, value]) => [value, key])
-      ),
-    });
-    const formatCSVRowAsLine = (x: {
-      quantity: string | null;
-      frontQuery: string | null;
-      frontSelectedImage: string | null;
-      backQuery: string | null;
-      backSelectedImage: string | null;
-    }): string => {
-      let formattedLine = `${x.quantity ?? ""} ${x.frontQuery ?? ""}`;
-      if ((x.frontSelectedImage ?? "").length > 0) {
-        formattedLine += `${SelectedImageSeparator}${x.frontSelectedImage}`;
-      }
-      if ((x.backQuery ?? "").length > 0) {
-        formattedLine += ` ${FaceSeparator} ${x.backQuery}`;
-        if ((x.backSelectedImage ?? "").length > 0) {
-          formattedLine += `${SelectedImageSeparator}${x.backSelectedImage}`;
-        }
-      }
-      return formattedLine;
-    };
-
+    const lines = parseCSVFileAsLines(fileContents);
     const processedLines = processLines(
-      rows.map(formatCSVRowAsLine),
+      lines,
       dfcPairsQuery.data ?? {},
       fuzzySearch
     );

frontend/src/features/import/importText.tsx

@@ -4,7 +4,7 @@
  * A freeform text area is exposed and the cards are processed when the user hits Submit.
  */
 
-import React, { FormEvent, useState } from "react";
+import React, { FormEvent, useRef, useState } from "react";
 import { Accordion } from "react-bootstrap";
 import Button from "react-bootstrap/Button";
 import Dropdown from "react-bootstrap/Dropdown";
@@ -47,6 +47,7 @@ export function ImportText() {
 
   const [showTextModal, setShowTextModal] = useState<boolean>(false);
   const [textModalValue, setTextModalValue] = useState<string>("");
+  const focusRef = useRef<HTMLTextAreaElement>(null);
 
   //# endregion
 
@@ -96,6 +97,11 @@ export function ImportText() {
       <Modal
         scrollable
         show={showTextModal}
+        onEntered={() => {
+          if (focusRef.current) {
+            focusRef.current.focus();
+          }
+        }}
         onHide={handleCloseTextModal}
         onExited={() => setTextModalValue("")}
         data-testid="import-text"
@@ -175,6 +181,7 @@ export function ImportText() {
           <Form id="importTextForm" onSubmit={handleSubmitTextModal}>
             <Form.Group className="mb-3">
               <Form.Control
+                ref={focusRef}
                 as="textarea"
                 rows={12}
                 placeholder={placeholderText}

frontend/src/features/import/importURL.tsx

@@ -5,7 +5,7 @@
  * to process the URL when the user hits Submit.
  */
 
-import React, { FormEvent, useCallback, useState } from "react";
+import React, { FormEvent, useCallback, useRef, useState } from "react";
 import Button from "react-bootstrap/Button";
 import Dropdown from "react-bootstrap/Dropdown";
 import Form from "react-bootstrap/Form";
@@ -44,6 +44,7 @@ export function ImportURL() {
   const [URLModalValue, setURLModalValue] = useState<string>("");
   const [showURLModal, setShowURLModal] = useState<boolean>(false);
   const [loading, setLoading] = useState<boolean>(false);
+  const focusRef = useRef<HTMLInputElement>(null);
 
   //# endregion
 
@@ -130,6 +131,11 @@ export function ImportURL() {
       <Modal
         scrollable
         show={loading || showURLModal}
+        onEntered={() => {
+          if (focusRef.current) {
+            focusRef.current.focus();
+          }
+        }}
         onHide={handleCloseURLModal}
         onExited={() => setURLModalValue("")}
       >
@@ -164,6 +170,7 @@ export function ImportURL() {
           <Form id="importURLForm" onSubmit={handleSubmitURLModal}>
             <Form.Group className="mb-3">
               <Form.Control
+                ref={focusRef}
                 type="url"
                 required={true}
                 placeholder="https://"

frontend/src/pages/index.tsx

@@ -120,6 +120,21 @@ export default function Index() {
         />
       </Head>
       <br />
+      {process.env.NEXT_PUBLIC_IMAGE_CDN_URL != null && (
+        <Alert variant="info">
+          Howdy! I&apos;m testing an experimental feature for image loading at
+          the moment.
+          <br />
+          If you noticed any issues, please create an issue on{" "}
+          <a
+            href="https://github.com/chilli-axe/mpc-autofill/issues"
+            target="_blank"
+          >
+            the GitHub repo
+          </a>
+          . Thanks for your patience!
+        </Alert>
+      )}
       <DynamicLogo />
       <br />
       <JumpIntoEditorButton />