From 716114e39449956c81f2d590eaf586a7b41668c3 Mon Sep 17 00:00:00 2001 From: Georges-Antoine Assi Date: Sat, 17 Aug 2024 12:35:45 -0400 Subject: [PATCH] parallel upload --- backend/endpoints/responses/rom.py | 5 -- backend/endpoints/rom.py | 58 +++++-------------- frontend/src/__generated__/index.ts | 1 - .../models/UploadRomsResponse.ts | 10 ---- .../common/Game/Dialog/UploadRom.vue | 38 +++++++----- .../components/common/UploadInProgress.vue | 46 +++++++-------- frontend/src/services/api/rom.ts | 54 +++++++++-------- frontend/src/stores/upload.ts | 58 ++++++++++--------- 8 files changed, 122 insertions(+), 148 deletions(-) delete mode 100644 frontend/src/__generated__/models/UploadRomsResponse.ts diff --git a/backend/endpoints/responses/rom.py b/backend/endpoints/responses/rom.py index a8180174c..fcf79e84f 100644 --- a/backend/endpoints/responses/rom.py +++ b/backend/endpoints/responses/rom.py @@ -186,11 +186,6 @@ class UserNotesSchema(TypedDict): note_raw_markdown: str -class UploadRomsResponse(TypedDict): - uploaded_files: list[str] - skipped_files: list[str] - - class CustomStreamingResponse(StreamingResponse): def __init__(self, *args, **kwargs) -> None: self.emit_body = kwargs.pop("emit_body", None) diff --git a/backend/endpoints/rom.py b/backend/endpoints/rom.py index 15f702056..65d94a97e 100644 --- a/backend/endpoints/rom.py +++ b/backend/endpoints/rom.py @@ -18,12 +18,11 @@ DetailedRomSchema, RomUserSchema, SimpleRomSchema, - UploadRomsResponse, ) from exceptions.endpoint_exceptions import RomNotFoundInDatabaseException from exceptions.fs_exceptions import RomAlreadyExistsException from fastapi import HTTPException, Query, Request, UploadFile, status -from fastapi.responses import JSONResponse, Response +from fastapi.responses import Response from handler.database import db_platform_handler, db_rom_handler from handler.filesystem import fs_resource_handler, fs_rom_handler from handler.filesystem.base_handler import CoverSize @@ -39,22 +38,18 @@ @protected_route(router.post, "/roms", ["roms.write"]) -async def add_roms( - request: Request, -) -> UploadRomsResponse: - """Upload roms endpoint (one or more at the same time) +async def add_rom(request: Request): + """Upload single rom endpoint Args: request (Request): Fastapi Request object Raises: HTTPException: No files were uploaded - - Returns: - UploadRomsResponse: Standard message response """ platform_id = int(request.headers.get("x-upload-platform", None)) - if not platform_id: + filename = request.headers.get("x-upload-filename", None) + if not platform_id or not filename: raise HTTPException( status_code=status.HTTP_400_BAD_REQUEST, detail="No platform ID provided", @@ -64,59 +59,38 @@ async def add_roms( roms_path = fs_rom_handler.build_upload_file_path(platform_fs_slug) log.info(f"Uploading roms to {platform_fs_slug}") + file_location = f"{roms_path}/{filename}" parser = StreamingFormDataParser(headers=request.headers) parser.register("x-upload-platform", NullTarget()) - parser.register("x-upload-filenames", NullTarget()) - - skipped_files = [] - uploaded_files = [] - - filenames = request.headers.get("x-upload-filenames", "").split(",") - for name in filenames: - file_location = f"{roms_path}/{name}" - if await Path(file_location).exists(): - log.warning(f" - Skipping {name} since the file already exists") - skipped_files.append(name) - continue - else: - uploaded_files.append(name) - - parser.register(name, FileTarget(f"{roms_path}/{name}")) + parser.register(filename, FileTarget(file_location)) - if not filenames: - log.error("No files were uploaded") + if await Path(file_location).exists(): + log.warning(f" - Skipping {filename} since the file already exists") raise HTTPException( status_code=status.HTTP_400_BAD_REQUEST, - detail="No files were uploaded", + detail=f"File {filename} already exists", ) - async def cleanup_partial_files(): - for name in filenames: - file_location = f"{roms_path}/{name}" - if await Path(file_location).exists(): - await Path(file_location).unlink() + async def cleanup_partial_file(): + if await Path(file_location).exists(): + await Path(file_location).unlink() try: async for chunk in request.stream(): parser.data_received(chunk) except ClientDisconnect: log.error("Client disconnected during upload") - await cleanup_partial_files() + await cleanup_partial_file() except Exception as exc: log.error("Error uploading files", exc_info=exc) - await cleanup_partial_files() + await cleanup_partial_file() raise HTTPException( status_code=status.HTTP_500_INTERNAL_SERVER_ERROR, detail="There was an error uploading the file(s)", ) - return JSONResponse( - { - "uploaded_files": uploaded_files, - "skipped_files": skipped_files, - } - ) + return Response(status_code=status.HTTP_201_CREATED) @protected_route(router.get, "/roms", ["roms.read"]) diff --git a/frontend/src/__generated__/index.ts b/frontend/src/__generated__/index.ts index 0d6c8a583..4ba37aea8 100644 --- a/frontend/src/__generated__/index.ts +++ b/frontend/src/__generated__/index.ts @@ -46,7 +46,6 @@ export type { TokenResponse } from './models/TokenResponse'; export type { UploadedSavesResponse } from './models/UploadedSavesResponse'; export type { UploadedScreenshotsResponse } from './models/UploadedScreenshotsResponse'; export type { UploadedStatesResponse } from './models/UploadedStatesResponse'; -export type { UploadRomsResponse } from './models/UploadRomsResponse'; export type { UserNotesSchema } from './models/UserNotesSchema'; export type { UserSchema } from './models/UserSchema'; export type { ValidationError } from './models/ValidationError'; diff --git a/frontend/src/__generated__/models/UploadRomsResponse.ts b/frontend/src/__generated__/models/UploadRomsResponse.ts deleted file mode 100644 index a8d0cc439..000000000 --- a/frontend/src/__generated__/models/UploadRomsResponse.ts +++ /dev/null @@ -1,10 +0,0 @@ -/* generated using openapi-typescript-codegen -- do no edit */ -/* istanbul ignore file */ -/* tslint:disable */ -/* eslint-disable */ - -export type UploadRomsResponse = { - uploaded_files: Array; - skipped_files: Array; -}; - diff --git a/frontend/src/components/common/Game/Dialog/UploadRom.vue b/frontend/src/components/common/Game/Dialog/UploadRom.vue index 084962250..43e73c53b 100644 --- a/frontend/src/components/common/Game/Dialog/UploadRom.vue +++ b/frontend/src/components/common/Game/Dialog/UploadRom.vue @@ -17,7 +17,7 @@ import { useDisplay } from "vuetify"; // Props const { xs, mdAndUp, smAndUp } = useDisplay(); const show = ref(false); -const romsToUpload = ref([]); +const filesToUpload = ref([]); const scanningStore = storeScanning(); const selectedPlatform = ref(null); const supportedPlatforms = ref(); @@ -66,7 +66,6 @@ emitter?.on("showUploadRomDialog", (platformWhereUpload) => { async function uploadRoms() { if (!selectedPlatform.value) return; show.value = false; - scanningStore.set(true); if (selectedPlatform.value.id == -1) { await platformApi @@ -98,15 +97,18 @@ async function uploadRoms() { await romApi .uploadRoms({ - romsToUpload: romsToUpload.value, + filesToUpload: filesToUpload.value, platformId: platformId, }) - .then(({ data }) => { + .then((responses: PromiseSettledResult[]) => { uploadStore.clear(); - const { uploaded_files, skipped_files } = data; + const successfulUploads = responses.filter( + (d) => d.status == "fulfilled" + ); + const failedUploads = responses.filter((d) => d.status == "rejected"); - if (uploaded_files.length == 0) { + if (successfulUploads.length == 0) { return emitter?.emit("snackbarShow", { msg: `All files skipped, nothing to upload.`, icon: "mdi-close-circle", @@ -116,12 +118,14 @@ async function uploadRoms() { } emitter?.emit("snackbarShow", { - msg: `${uploaded_files.length} files uploaded successfully (and ${skipped_files.length} skipped). Starting scan...`, + msg: `${successfulUploads.length} files uploaded successfully (and ${failedUploads.length} skipped/failed). Starting scan...`, icon: "mdi-check-bold", color: "green", timeout: 3000, }); + scanningStore.set(true); + if (!socket.connected) socket.connect(); setTimeout(() => { socket.emit("scan", { @@ -141,7 +145,7 @@ async function uploadRoms() { timeout: 4000, }); }); - romsToUpload.value = []; + filesToUpload.value = []; selectedPlatform.value = null; } @@ -151,17 +155,19 @@ function triggerFileInput() { } function removeRomFromList(romName: string) { - romsToUpload.value = romsToUpload.value.filter((rom) => rom.name !== romName); + filesToUpload.value = filesToUpload.value.filter( + (rom) => rom.name !== romName + ); } function closeDialog() { show.value = false; - romsToUpload.value = []; + filesToUpload.value = []; selectedPlatform.value = null; } function updateDataTablePages() { - pageCount.value = Math.ceil(romsToUpload.value.length / itemsPerPage.value); + pageCount.value = Math.ceil(filesToUpload.value.length / itemsPerPage.value); } watch(itemsPerPage, async () => { updateDataTablePages(); @@ -233,7 +239,7 @@ watch(itemsPerPage, async () => { {