Skip to content

Commit

Permalink
parallel upload
Browse files Browse the repository at this point in the history
  • Loading branch information
@gantoine
gantoine committed Aug 17, 2024
1 parent 83b4f32 commit 716114e
Show file tree
Hide file tree
Showing 8 changed files with 122 additions and 148 deletions.
5 changes: 0 additions & 5 deletions backend/endpoints/responses/rom.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -186,11 +186,6 @@ class UserNotesSchema(TypedDict):
note_raw_markdown: str


class UploadRomsResponse(TypedDict):
uploaded_files: list[str]
skipped_files: list[str]


class CustomStreamingResponse(StreamingResponse):
def __init__(self, *args, **kwargs) -> None:
self.emit_body = kwargs.pop("emit_body", None)
Expand Down
58 changes: 16 additions & 42 deletions backend/endpoints/rom.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -18,12 +18,11 @@
DetailedRomSchema,
RomUserSchema,
SimpleRomSchema,
UploadRomsResponse,
)
from exceptions.endpoint_exceptions import RomNotFoundInDatabaseException
from exceptions.fs_exceptions import RomAlreadyExistsException
from fastapi import HTTPException, Query, Request, UploadFile, status
from fastapi.responses import JSONResponse, Response
from fastapi.responses import Response
from handler.database import db_platform_handler, db_rom_handler
from handler.filesystem import fs_resource_handler, fs_rom_handler
from handler.filesystem.base_handler import CoverSize
Expand All @@ -39,22 +38,18 @@


@protected_route(router.post, "/roms", ["roms.write"])
async def add_roms(
request: Request,
) -> UploadRomsResponse:
"""Upload roms endpoint (one or more at the same time)
async def add_rom(request: Request):
"""Upload single rom endpoint
Args:
request (Request): Fastapi Request object
Raises:
HTTPException: No files were uploaded
Returns:
UploadRomsResponse: Standard message response
"""
platform_id = int(request.headers.get("x-upload-platform", None))
if not platform_id:
filename = request.headers.get("x-upload-filename", None)
if not platform_id or not filename:
raise HTTPException(
status_code=status.HTTP_400_BAD_REQUEST,
detail="No platform ID provided",
Expand All @@ -64,59 +59,38 @@ async def add_roms(
roms_path = fs_rom_handler.build_upload_file_path(platform_fs_slug)
log.info(f"Uploading roms to {platform_fs_slug}")

file_location = f"{roms_path}/{filename}"
parser = StreamingFormDataParser(headers=request.headers)
parser.register("x-upload-platform", NullTarget())
parser.register("x-upload-filenames", NullTarget())

skipped_files = []
uploaded_files = []

filenames = request.headers.get("x-upload-filenames", "").split(",")
for name in filenames:
file_location = f"{roms_path}/{name}"
if await Path(file_location).exists():
log.warning(f" - Skipping {name} since the file already exists")
skipped_files.append(name)
continue
else:
uploaded_files.append(name)

parser.register(name, FileTarget(f"{roms_path}/{name}"))
parser.register(filename, FileTarget(file_location))

if not filenames:
log.error("No files were uploaded")
if await Path(file_location).exists():
log.warning(f" - Skipping {filename} since the file already exists")
raise HTTPException(
status_code=status.HTTP_400_BAD_REQUEST,
detail="No files were uploaded",
detail=f"File {filename} already exists",
)

async def cleanup_partial_files():
for name in filenames:
file_location = f"{roms_path}/{name}"
if await Path(file_location).exists():
await Path(file_location).unlink()
async def cleanup_partial_file():
if await Path(file_location).exists():
await Path(file_location).unlink()

try:
async for chunk in request.stream():
parser.data_received(chunk)
except ClientDisconnect:
log.error("Client disconnected during upload")
await cleanup_partial_files()
await cleanup_partial_file()
except Exception as exc:
log.error("Error uploading files", exc_info=exc)
await cleanup_partial_files()
await cleanup_partial_file()

raise HTTPException(

Check failure on line 88 in backend/endpoints/rom.py

View check run for this annotation

Trunk.io / Trunk Check

ruff(B904) 

[new] Within an `except` clause, raise exceptions with `raise ... from err` or `raise ... from None` to distinguish them from errors in exception handling

Check failure on line 88 in backend/endpoints/rom.py

View workflow job for this annotation

GitHub Actions / Trunk Check

ruff(B904) 

[new] Within an `except` clause, raise exceptions with `raise ... from err` or `raise ... from None` to distinguish them from errors in exception handling
status_code=status.HTTP_500_INTERNAL_SERVER_ERROR,
detail="There was an error uploading the file(s)",
)

return JSONResponse(
{
"uploaded_files": uploaded_files,
"skipped_files": skipped_files,
}
)
return Response(status_code=status.HTTP_201_CREATED)


@protected_route(router.get, "/roms", ["roms.read"])
Expand Down
1 change: 0 additions & 1 deletion frontend/src/__generated__/index.ts
View file

Some generated files are not rendered by default. Learn more about how customized files appear on GitHub.

10 changes: 0 additions & 10 deletions frontend/src/__generated__/models/UploadRomsResponse.ts
View file

This file was deleted.

38 changes: 22 additions & 16 deletions frontend/src/components/common/Game/Dialog/UploadRom.vue
View file
Original file line number Diff line number Diff line change
Expand Up @@ -17,7 +17,7 @@ import { useDisplay } from "vuetify";
// Props
const { xs, mdAndUp, smAndUp } = useDisplay();
const show = ref(false);
const romsToUpload = ref<File[]>([]);
const filesToUpload = ref<File[]>([]);
const scanningStore = storeScanning();
const selectedPlatform = ref<Platform | null>(null);
const supportedPlatforms = ref<Platform[]>();
Expand Down Expand Up @@ -66,7 +66,6 @@ emitter?.on("showUploadRomDialog", (platformWhereUpload) => {
async function uploadRoms() {
if (!selectedPlatform.value) return;
show.value = false;
scanningStore.set(true);
if (selectedPlatform.value.id == -1) {
await platformApi
Expand Down Expand Up @@ -98,15 +97,18 @@ async function uploadRoms() {
await romApi
.uploadRoms({
romsToUpload: romsToUpload.value,
filesToUpload: filesToUpload.value,
platformId: platformId,
})
.then(({ data }) => {
.then((responses: PromiseSettledResult<unknown>[]) => {
uploadStore.clear();
const { uploaded_files, skipped_files } = data;
const successfulUploads = responses.filter(
(d) => d.status == "fulfilled"
);
const failedUploads = responses.filter((d) => d.status == "rejected");
if (uploaded_files.length == 0) {
if (successfulUploads.length == 0) {
return emitter?.emit("snackbarShow", {
msg: `All files skipped, nothing to upload.`,
icon: "mdi-close-circle",
Expand All @@ -116,12 +118,14 @@ async function uploadRoms() {
}
emitter?.emit("snackbarShow", {
msg: `${uploaded_files.length} files uploaded successfully (and ${skipped_files.length} skipped). Starting scan...`,
msg: `${successfulUploads.length} files uploaded successfully (and ${failedUploads.length} skipped/failed). Starting scan...`,
icon: "mdi-check-bold",
color: "green",
timeout: 3000,
});
scanningStore.set(true);
if (!socket.connected) socket.connect();
setTimeout(() => {
socket.emit("scan", {
Expand All @@ -141,7 +145,7 @@ async function uploadRoms() {
timeout: 4000,
});
});
romsToUpload.value = [];
filesToUpload.value = [];
selectedPlatform.value = null;
}
Expand All @@ -151,17 +155,19 @@ function triggerFileInput() {
}
function removeRomFromList(romName: string) {
romsToUpload.value = romsToUpload.value.filter((rom) => rom.name !== romName);
filesToUpload.value = filesToUpload.value.filter(
(rom) => rom.name !== romName
);
}
function closeDialog() {
show.value = false;
romsToUpload.value = [];
filesToUpload.value = [];
selectedPlatform.value = null;
}
function updateDataTablePages() {
pageCount.value = Math.ceil(romsToUpload.value.length / itemsPerPage.value);
pageCount.value = Math.ceil(filesToUpload.value.length / itemsPerPage.value);
}
watch(itemsPerPage, async () => {
updateDataTablePages();
Expand Down Expand Up @@ -233,7 +239,7 @@ watch(itemsPerPage, async () => {
</v-btn>
<v-file-input
id="file-input"
v-model="romsToUpload"
v-model="filesToUpload"
@update:model-value="updateDataTablePages"
class="file-input"
multiple
Expand All @@ -244,9 +250,9 @@ watch(itemsPerPage, async () => {
</template>
<template #content>
<v-data-table
v-if="romsToUpload.length > 0"
v-if="filesToUpload.length > 0"
:item-value="(item) => item.name"
:items="romsToUpload"
:items="filesToUpload"
:width="mdAndUp ? '60vw' : '95vw'"
:items-per-page="itemsPerPage"
:items-per-page-options="PER_PAGE_OPTIONS"
Expand Down Expand Up @@ -315,9 +321,9 @@ watch(itemsPerPage, async () => {
<v-btn class="bg-terciary" @click="closeDialog"> Cancel </v-btn>
<v-btn
class="bg-terciary text-romm-green"
:disabled="romsToUpload.length == 0 || selectedPlatform == null"
:disabled="filesToUpload.length == 0 || selectedPlatform == null"
:variant="
romsToUpload.length == 0 || selectedPlatform == null
filesToUpload.length == 0 || selectedPlatform == null
? 'plain'
: 'flat'
"
Expand Down
46 changes: 23 additions & 23 deletions frontend/src/components/common/UploadInProgress.vue
View file
Original file line number Diff line number Diff line change
Expand Up @@ -7,12 +7,11 @@ import { storeToRefs } from "pinia";
const { xs } = useDisplay();
const uploadStore = storeUpload();
const { filenames, progress, total, loaded, rate, finished } =
storeToRefs(uploadStore);
const { files } = storeToRefs(uploadStore);
const show = ref(false);
watch(filenames, (fns) => {
show.value = fns.length > 0;
watch(files, (newList) => {
show.value = newList.length > 0;
});
</script>

Expand All @@ -28,38 +27,39 @@ watch(filenames, (fns) => {
color="tooltip"
>
<v-list>
<v-list-item-title class="py-2 px-4 d-flex justify-space-between">
Uploading {{ filenames.length }} files...
<v-icon icon="mdi-loading mdi-spin" color="white" class="mx-2"/>
</v-list-item-title>
<v-list-item class="py-1 px-4">
<v-list-item-title v-for="filename in filenames" class="d-flex justify-space-between">
<div class="upload-speeds">
• {{ filename }}
</div>
<v-list-item
v-for="file in files"
class="py-2 px-4"
:disabled="file.finished"
>
<v-list-item-title class="d-flex justify-space-between">
{{ file.filename }}
<v-icon
:icon="file.finished ? `mdi-check` : `mdi-loading mdi-spin`"
:color="file.finished ? `green` : `white`"
class="mx-2"
/>
</v-list-item-title>
</v-list-item>
<v-list-item class="py-0 px-4">
<template v-if="progress > 0 && !finished">
<template v-if="file.progress > 0 && !file.finished">
<v-progress-linear
v-model="progress"
v-model="file.progress"
height="4"
color="white"
class="mt-1"
/>
<div class="upload-speeds d-flex justify-space-between mt-1">
<div>{{ formatBytes(rate) }}/s</div>
<div>{{ formatBytes(file.rate) }}/s</div>
<div>
{{ formatBytes(loaded) }} /
{{ formatBytes(total) }}
{{ formatBytes(file.loaded) }} /
{{ formatBytes(file.total) }}
</div>
</div>
</template>
<template v-if="finished">
<template v-if="file.finished">
<div class="upload-speeds d-flex justify-space-between mt-1">
<div />
<div>
{{ formatBytes(total) }}
{{ formatBytes(file.total) }}
</div>
</div>
</template>
Expand All @@ -74,6 +74,6 @@ watch(filenames, (fns) => {
}
.upload-speeds {
font-size: 12px;
font-size: 10px;
}
</style>
Loading

0 comments on commit 716114e

Please sign in to comment.