Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Better performance for large collections #1128

Merged
merged 2 commits into from
Aug 28, 2024
Merged

Better performance for large collections #1128

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
2 commits
Select commit Hold shift + click to select a range
e481691
Better performance for large collections
gantoine Aug 28, 2024
307d906
Merge branch 'master' into large-lib-performance-fixes
gantoine Aug 28, 2024
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
53 changes: 20 additions & 33 deletions backend/handler/database/platforms_handler.py
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -1,78 +1,65 @@
import functools

from decorators.database import begin_session
from models.platform import Platform
from models.rom import Rom
from sqlalchemy import Select, delete, or_, select
from sqlalchemy.orm import Query, Session, selectinload
from sqlalchemy.orm import Session

from .base_handler import DBBaseHandler


def with_roms(func):
@functools.wraps(func)
def wrapper(*args, **kwargs):
session = kwargs.get("session")
if session is None:
raise ValueError("session is required")

kwargs["query"] = select(Platform).options(
selectinload(Platform.roms).load_only(Rom.id)
)
return func(*args, **kwargs)

return wrapper


class DBPlatformsHandler(DBBaseHandler):
@begin_session
@with_roms
def add_platform(
self, platform: Platform, query: Query = None, session: Session = None
self,
platform: Platform,
session: Session,
) -> Platform:
platform = session.merge(platform)
session.flush()

return session.scalar(query.filter_by(id=platform.id).limit(1))
new_platform = session.scalar(
select(Platform).filter_by(id=platform.id).limit(1)
)
if not new_platform:
raise ValueError("Could not find newlyewly created platform")

return new_platform

@begin_session
@with_roms
def get_platform(
self, id: int, *, query: Query = None, session: Session = None
) -> Platform | None:
return session.scalar(query.filter_by(id=id).limit(1))
def get_platform(self, id: int, *, session: Session) -> Platform | None:
return session.scalar(select(Platform).filter_by(id=id).limit(1))

@begin_session
def get_platforms(self, *, session: Session = None) -> Select[tuple[Platform]]:
def get_platforms(self, *, session: Session) -> Select[tuple[Platform]]:
return (
session.scalars(select(Platform).order_by(Platform.name.asc())) # type: ignore[attr-defined]
.unique()
.all()
)

@begin_session
@with_roms
def get_platform_by_fs_slug(
self, fs_slug: str, query: Query = None, session: Session = None
self, fs_slug: str, session: Session
) -> Platform | None:
return session.scalar(query.filter_by(fs_slug=fs_slug).limit(1))
return session.scalar(select(Platform).filter_by(fs_slug=fs_slug).limit(1))

@begin_session
def delete_platform(self, id: int, session: Session = None) -> int:
def delete_platform(self, id: int, session: Session) -> None:
# Remove all roms from that platforms first
session.execute(
delete(Rom)
.where(Rom.platform_id == id)
.execution_options(synchronize_session="evaluate")
)
return session.execute(

session.execute(
delete(Platform)
.where(Platform.id == id)
.execution_options(synchronize_session="evaluate")
)

@begin_session
def purge_platforms(self, fs_platforms: list[str], session: Session = None) -> int:
def purge_platforms(self, fs_platforms: list[str], session: Session) -> int:
return session.execute(
delete(Platform)
.where(or_(Platform.fs_slug.not_in(fs_platforms), Platform.slug.is_(None))) # type: ignore[attr-defined]
Expand Down
144 changes: 90 additions & 54 deletions frontend/src/stores/roms.ts
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -23,8 +23,8 @@ export default defineStore("roms", {
currentRom: null as DetailedRom | null,
allRoms: [] as SimpleRom[],
_grouped: [] as SimpleRom[],
_filteredIDs: [] as number[],
_selectedIDs: [] as number[],
_filteredIDs: new Set<number>(),
_selectedIDs: new Set<number>(),
recentRoms: [] as SimpleRom[],
lastSelectedIndex: -1,
selecting: false,
Expand All @@ -34,9 +34,9 @@ export default defineStore("roms", {

getters: {
filteredRoms: (state) =>
state._grouped.filter((rom) => state._filteredIDs.includes(rom.id)),
state._grouped.filter((rom) => state._filteredIDs.has(rom.id)),
selectedRoms: (state) =>
state._grouped.filter((rom) => state._selectedIDs.includes(rom.id)),
state._grouped.filter((rom) => state._selectedIDs.has(rom.id)),
},

actions: {
Expand Down Expand Up @@ -115,22 +115,18 @@ export default defineStore("roms", {
return rom.id === value.id;
});
});
this._filteredIDs = this._filteredIDs.filter((value) => {
return !roms.find((rom) => {
return rom.id === value;
});
});
roms.forEach((rom) => this._filteredIDs.delete(rom.id));
},
reset() {
this.allRoms = [];
this._grouped = [];
this._filteredIDs = [];
this._selectedIDs = [];
this._filteredIDs = new Set<number>();
this._selectedIDs = new Set<number>();
this.lastSelectedIndex = -1;
},
// Filter roms by gallery filter store state
setFiltered(roms: SimpleRom[], galleryFilter: GalleryFilterStore) {
this._filteredIDs = roms.map((rom) => rom.id);
this._filteredIDs = new Set(roms.map((rom) => rom.id));
if (galleryFilter.filterSearch) {
this._filterSearch(galleryFilter.filterSearch);
}
Expand All @@ -157,68 +153,108 @@ export default defineStore("roms", {
}
},
_filterSearch(searchFilter: string) {
this._filteredIDs = this.filteredRoms
.filter(
(rom) =>
rom.name?.toLowerCase().includes(searchFilter.toLowerCase()) ||
rom.file_name?.toLowerCase().includes(searchFilter.toLowerCase()),
)
.map((roms) => roms.id);
const bySearch = new Set(
this.filteredRoms
.filter(
(rom) =>
rom.name?.toLowerCase().includes(searchFilter.toLowerCase()) ||
rom.file_name?.toLowerCase().includes(searchFilter.toLowerCase()),
)
.map((roms) => roms.id),
);

// @ts-expect-error intersection is recently defined on Set
this._filteredIDs = bySearch.intersection(this._filteredIDs);
},
_filterUnmatched() {
this._filteredIDs = this.filteredRoms
.filter((rom) => !rom.igdb_id && !rom.moby_id)
.map((roms) => roms.id);
const byUnmatched = new Set(
this.filteredRoms
.filter((rom) => !rom.igdb_id && !rom.moby_id)
.map((roms) => roms.id),
);

// @ts-expect-error intersection is recently defined on Set
this._filteredIDs = byUnmatched.intersection(this._filteredIDs);
},
_filterFavourites() {
this._filteredIDs = this.filteredRoms
.filter((rom) => collectionStore.favCollection?.roms?.includes(rom.id))
.map((roms) => roms.id);
const byFavourites = new Set(
this.filteredRoms
.filter((rom) =>
collectionStore.favCollection?.roms?.includes(rom.id),
)
.map((roms) => roms.id),
);

// @ts-expect-error intersection is recently defined on Set
this._filteredIDs = byFavourites.intersection(this._filteredIDs);
},
_filterDuplicates() {
this._filteredIDs = this.filteredRoms
.filter((rom) => rom.sibling_roms?.length)
.map((rom) => rom.id);
const byDuplicates = new Set(
this.filteredRoms
.filter((rom) => rom.sibling_roms?.length)
.map((rom) => rom.id),
);

// @ts-expect-error intersection is recently defined on Set
this._filteredIDs = byDuplicates.intersection(this._filteredIDs);
},
_filterGenre(genreToFilter: string) {
this._filteredIDs = this.filteredRoms
.filter((rom) => rom.genres.some((genre) => genre === genreToFilter))
.map((rom) => rom.id);
const byGenre = new Set(
this.filteredRoms
.filter((rom) => rom.genres.some((genre) => genre === genreToFilter))
.map((rom) => rom.id),
);

// @ts-expect-error intersection is recently defined on Set
this._filteredIDs = byGenre.intersection(this._filteredIDs);
},
_filterFranchise(franchiseToFilter: string) {
this._filteredIDs = this.filteredRoms
.filter((rom) =>
rom.franchises.some((franchise) => franchise === franchiseToFilter),
)
.map((rom) => rom.id);
const byFranchise = new Set(
this.filteredRoms
.filter((rom) =>
rom.franchises.some((franchise) => franchise === franchiseToFilter),
)
.map((rom) => rom.id),
);

// @ts-expect-error intersection is recently defined on Set
this._filteredIDs = byFranchise.intersection(this._filteredIDs);
},
_filterCollection(collectionToFilter: string) {
this._filteredIDs = this.filteredRoms
.filter((rom) =>
rom.collections.some(
(collection) => collection === collectionToFilter,
),
)
.map((rom) => rom.id);
const byCollection = new Set(
this.filteredRoms
.filter((rom) =>
rom.collections.some(
(collection) => collection === collectionToFilter,
),
)
.map((rom) => rom.id),
);

// @ts-expect-error intersection is recently defined on Set
this._filteredIDs = byCollection.intersection(this._filteredIDs);
},
_filterCompany(companyToFilter: string) {
this._filteredIDs = this.filteredRoms
.filter((rom) =>
rom.companies.some((company) => company === companyToFilter),
)
.map((rom) => rom.id);
const byCompany = new Set(
this.filteredRoms
.filter((rom) =>
rom.companies.some((company) => company === companyToFilter),
)
.map((rom) => rom.id),
);

// @ts-expect-error intersection is recently defined on Set
this._filteredIDs = byCompany.intersection(this._filteredIDs);
},
// Selected roms
setSelection(roms: SimpleRom[]) {
this._selectedIDs = roms.map((rom) => rom.id);
this._selectedIDs = new Set(roms.map((rom) => rom.id));
},
addToSelection(rom: SimpleRom) {
this._selectedIDs.push(rom.id);
this._selectedIDs.add(rom.id);
},
removeFromSelection(rom: SimpleRom) {
this._selectedIDs = this._selectedIDs.filter((id) => {
return id !== rom.id;
});
this._selectedIDs.delete(rom.id);
},
updateLastSelected(index: number) {
this.lastSelectedIndex = index;
Expand All @@ -227,7 +263,7 @@ export default defineStore("roms", {
this.selecting = !this.selecting;
},
resetSelection() {
this._selectedIDs = [];
this._selectedIDs = new Set<number>();
this.lastSelectedIndex = -1;
},
isSimpleRom(rom: SimpleRom | SearchRomSchema): rom is SimpleRom {
Expand Down
Loading