Skip to content

Commit

Permalink
Rollback db
Browse files Browse the repository at this point in the history
  • Loading branch information
@ChenglongMa
ChenglongMa committed Apr 27, 2024
1 parent bc105b9 commit 126e70c
Show file tree
Hide file tree
Showing 10 changed files with 152 additions and 77 deletions.
11 changes: 6 additions & 5 deletions package.json
View file
Original file line number Diff line number Diff line change
Expand Up @@ -35,13 +35,14 @@
"homepage": "https://chenglongma.com/zoplicate/",
"dependencies": {
"dexie": "^4.0.4",
"indexeddbshim": "^13.0.0",
"zotero-plugin-toolkit": "^2.3.29"
},
"devDependencies": {
"@jest/globals": "^29.7.0",
"@types/node": "^20.11.30",
"@typescript-eslint/eslint-plugin": "^7.4.0",
"@typescript-eslint/parser": "^7.4.0",
"@types/node": "^20.12.7",
"@typescript-eslint/eslint-plugin": "^7.7.1",
"@typescript-eslint/parser": "^7.7.1",
"chokidar": "^3.6.0",
"compressing": "^1.10.0",
"esbuild": "^0.20.2",
Expand All @@ -50,11 +51,11 @@
"fake-indexeddb": "^5.0.2",
"jest": "^29.7.0",
"prettier": "^3.2.5",
"release-it": "^17.1.1",
"release-it": "^17.2.1",
"replace-in-file": "^7.1.0",
"ts-jest": "^29.1.2",
"ts-node": "^10.9.2",
"typescript": "^5.4.3",
"typescript": "^5.4.5",
"zotero-types": "1.3.24"
},
"eslintConfig": {
Expand Down
3 changes: 3 additions & 0 deletions src/addon.ts
View file
Original file line number Diff line number Diff line change
Expand Up @@ -9,6 +9,7 @@ class Addon {
alive: boolean;
// Env type, see build.mjs
env: "development" | "production";
database: "SQLite" | "IndexedDB";
ztoolkit: ZToolkit;
locale?: {
current: any;
Expand Down Expand Up @@ -36,6 +37,7 @@ class Addon {
this.data = {
alive: true,
env: __env__,
database: "SQLite",
ztoolkit: createZToolkit(),
dialogs: {},
needResetDuplicateSearch: {},
Expand All @@ -52,6 +54,7 @@ class Addon {
this.data = {
alive: true,
env: __env__,
database: "SQLite",
ztoolkit: createZToolkit(),
dialogs: {},
needResetDuplicateSearch: {},
Expand Down
35 changes: 31 additions & 4 deletions src/hooks.ts
View file
Original file line number Diff line number Diff line change
Expand Up @@ -8,11 +8,13 @@ import { BulkDuplicates } from "./modules/bulkDuplicates";
import { fetchDuplicates, registerButtonsInDuplicatePane } from "./modules/duplicates";
import menus from "./modules/menus";
// import "./modules/zduplicates.js";
import { IndexedDB } from "./modules/db";
import database from "./modules/db";
import { NonDuplicates, registerNonDuplicatesSection } from "./modules/nonDuplicates";
import { patchGetSearchObject, patchItemSaveData } from "./modules/patcher";
import { containsRegularItem, isInDuplicatesPane, refreshItemTree } from "./utils/zotero";
import { registerDuplicateStats } from "./modules/duplicateStats";
import { waitUtilAsync } from "./utils/wait";
import Dexie from "dexie";

async function onStartup() {
await Promise.all([Zotero.initializationPromise, Zotero.unlockPromise, Zotero.uiReadyPromise]);
Expand All @@ -27,10 +29,34 @@ async function onStartup() {
await onMainWindowLoad(window);
}

function handleError(e:any) {
switch (e.name) {
case "AbortError":
if (e.inner) {
return handleError(e.inner);
}
ztoolkit.log("Abort error " + e.message);
break;
case "QuotaExceededError":
ztoolkit.log("QuotaExceededError " + e.message);
break;
default:
ztoolkit.log(e);
break;
}
}

async function onMainWindowLoad(win: Window): Promise<void> {
await waitUtilAsync(() => document.readyState === "complete");
// Dexie.dependencies.indexedDB = window.indexedDB;
// Dexie.dependencies.IDBKeyRange = window.IDBKeyRange;
addon.data.ztoolkit = createZToolkit();
const db = IndexedDB.getInstance();
const db = database.getDatabase();
ztoolkit.log("onMainWindowLoad before db init", window.IDBKeyRange);
await db.init();
db.insertNonDuplicatePair(1, 2, 1);//.catch((e) => handleError(e));
ztoolkit.log("insert done");

NonDuplicates.getInstance().init(db);
registerNonDuplicatesSection(db);
registerStyleSheets();
Expand All @@ -42,18 +68,19 @@ async function onMainWindowLoad(win: Window): Promise<void> {
patchItemSaveData();
await registerDuplicateStats();
registerButtonsInDuplicatePane(win);
ztoolkit.log("onMainWindowLoad done");
}

async function onMainWindowUnload(win: Window): Promise<void> {
ztoolkit.unregisterAll();
addon.data.dialogs.dialog?.window?.close();
await IndexedDB.getInstance().close();
await database.getDatabase().close();
}

async function onShutdown() {
ztoolkit.unregisterAll();
addon.data.dialogs.dialog?.window?.close();
await IndexedDB.getInstance().close();
await database.getDatabase().close();
// Remove addon object
addon.data.alive = false;
delete Zotero[config.addonInstance];
Expand Down
2 changes: 2 additions & 0 deletions src/modules/bulkDuplicates.ts
View file
Original file line number Diff line number Diff line change
Expand Up @@ -187,6 +187,8 @@ export class BulkDuplicates {

ZoteroPane.collectionsView &&
ZoteroPane.collectionsView.onSelect.addListener(async () => {
ztoolkit.log(`Main window loaded`, win.indexedDB);

const groupBox = win.document.getElementById("zotero-item-pane-groupbox") as Element;
if (isInDuplicatesPane()) {
ztoolkit.UI.appendElement(msgVBox, groupBox);
Expand Down
113 changes: 72 additions & 41 deletions src/modules/db.ts
View file
Original file line number Diff line number Diff line change
@@ -1,16 +1,16 @@
import { config } from "../../package.json";
import Dexie, { Table } from "dexie";
import Dexie, { DexieOptions, Table } from "dexie";

export interface IDatabase {
init(): Promise<void>;

get db(): Dexie | typeof Zotero.DBConnection;

insertNonDuplicatePair(itemID: number, itemID2: number): Promise<void>;
insertNonDuplicatePair(itemID: number, itemID2: number, libraryID?: number): Promise<void>;

insertNonDuplicatePairs(rows: { itemID: number; itemID2: number }[]): Promise<void>;
insertNonDuplicatePairs(rows: { itemID: number; itemID2: number }[], libraryID?: number): Promise<void>;

insertNonDuplicates(itemIDs: number[]): Promise<void>;
insertNonDuplicates(itemIDs: number[], libraryID?: number): Promise<void>;

deleteNonDuplicatePair(itemID: number, itemID2: number): Promise<void>;

Expand All @@ -22,7 +22,12 @@ export interface IDatabase {

existsNonDuplicates(itemIDs: number[]): Promise<boolean>;

getNonDuplicates(itemID?: number): Promise<{ itemID: number; itemID2: number }[]>;
getNonDuplicates({ itemID, libraryID }: { itemID?: number; libraryID?: number }): Promise<
{
itemID: number;
itemID2: number;
}[]
>;

close(): Promise<void>;
}
Expand All @@ -34,36 +39,36 @@ export interface INonDuplicatePair {
libraryID: number;
}

export class IndexedDB extends Dexie implements IDatabase {
class DexieDB extends Dexie implements IDatabase {
nonDuplicates!: Table<INonDuplicatePair>;

private static _instance: IndexedDB;
private static _instance: DexieDB;

private constructor(databaseName: string = config.addonName) {
super(databaseName);
private constructor(databaseName: string = config.addonName, options?: DexieOptions) {
super(databaseName, options);
this.version(1).stores({
nonDuplicates: "++id, &[itemID+itemID2], itemID, itemID2, libraryID",
});
}

public static getInstance(databaseName: string = config.addonName): IndexedDB {
if (!IndexedDB._instance) {
IndexedDB._instance = new IndexedDB(databaseName);
public static getInstance(databaseName: string = config.addonName, options?: DexieOptions): DexieDB {
if (!DexieDB._instance) {
DexieDB._instance = new DexieDB(databaseName, options);
}
return IndexedDB._instance;
return DexieDB._instance;
}

get db(): Dexie | typeof Zotero.DBConnection {
return this;
}

async init() {
// await this._db.open();
// await this.open();
}

async insertNonDuplicatePair(itemID: number, itemID2: number, libraryID?: number) {
if (await this.existsNonDuplicatePair(itemID, itemID2)) {
// ztoolkit.log("Pair already exists: ", itemID, itemID2);
ztoolkit.log("Pair already exists: ", itemID, itemID2);
return;
}
libraryID = libraryID ?? Zotero.Items.get(itemID).libraryID;
Expand All @@ -79,14 +84,7 @@ export class IndexedDB extends Dexie implements IDatabase {
await this.nonDuplicates.bulkPut(rows.map(({ itemID, itemID2 }) => ({ itemID, itemID2, libraryID })));
}

private getUniquePairs(
itemIDs: number[],
libraryID?: number,
): {
itemID: number;
itemID2: number;
libraryID: number;
}[] {
private getUniquePairs(itemIDs: number[], libraryID?: number): INonDuplicatePair[] {
libraryID = libraryID ?? Zotero.Items.get(itemIDs[0]).libraryID;
const rows: { itemID: number; itemID2: number; libraryID: number }[] = [];
const seenPairs: Set<string> = new Set();
Expand All @@ -95,16 +93,21 @@ export class IndexedDB extends Dexie implements IDatabase {
for (let j = i + 1; j < itemIDs.length; j++) {
const pairKey = `${itemIDs[i]}-${itemIDs[j]}`;
if (!seenPairs.has(pairKey)) {
rows.push({ itemID: itemIDs[i], itemID2: itemIDs[j], libraryID });
const itemID = Math.min(itemIDs[i], itemIDs[j]);
const itemID2 = Math.max(itemIDs[i], itemIDs[j]);
rows.push({ itemID, itemID2, libraryID });
seenPairs.add(pairKey);
}
}
}
ztoolkit.log("Unique pairs:", rows);
return rows;
}

async insertNonDuplicates(itemIDs: number[], libraryID?: number): Promise<void> {
await this.nonDuplicates.bulkPut(this.getUniquePairs(itemIDs, libraryID));
const records = this.getUniquePairs(itemIDs, libraryID);
await this.nonDuplicates.bulkPut(records);
ztoolkit.log("Inserted non-duplicates done");
}

async deleteNonDuplicatePair(itemID: number, itemID2: number) {
Expand Down Expand Up @@ -146,7 +149,12 @@ export class IndexedDB extends Dexie implements IDatabase {
return result.length === rows.length;
}

async getNonDuplicates(itemID?: number): Promise<{ itemID: number; itemID2: number }[]> {
async getNonDuplicates({ itemID, libraryID }: { itemID?: number; libraryID?: number }): Promise<
{
itemID: number;
itemID2: number;
}[]
> {
if (itemID !== undefined && itemID !== null) {
return this.nonDuplicates.where("itemID").equals(itemID).or("itemID2").equals(itemID).toArray();
} else {
Expand All @@ -159,10 +167,10 @@ export class IndexedDB extends Dexie implements IDatabase {
}
}

export class SQLiteDB implements IDatabase {
class SQLiteDB implements IDatabase {
private static _instance: SQLiteDB;
private readonly _db: typeof Zotero.DBConnection;
private tables = {
private readonly tables = {
nonDuplicates: "nonDuplicates",
};

Expand Down Expand Up @@ -190,34 +198,42 @@ export class SQLiteDB implements IDatabase {
await this._db.queryAsync(
`CREATE TABLE IF NOT EXISTS ${this.tables.nonDuplicates}
(
itemID INTEGER,
itemID2 INTEGER,
itemID INTEGER,
itemID2 INTEGER,
libraryID INTEGER,
PRIMARY KEY (itemID, itemID2)
);`,
);
}

async insertNonDuplicatePair(itemID: number, itemID2: number) {
private buildRow(itemID: number, itemID2: number, libraryID: number) {
return itemID > itemID2 ? [itemID2, itemID, libraryID] : [itemID, itemID2, libraryID];
}

async insertNonDuplicatePair(itemID: number, itemID2: number, libraryID?: number) {
libraryID = libraryID ?? Zotero.Items.get(itemID).libraryID;
const row = this.buildRow(itemID, itemID2, libraryID);
await this._db.queryAsync(
`INSERT OR IGNORE INTO ${this.tables.nonDuplicates} (itemID, itemID2)
VALUES (?, ?);`,
[itemID, itemID2].sort(),
`INSERT OR IGNORE INTO ${this.tables.nonDuplicates} (itemID, itemID2, libraryID)
VALUES (?, ?, ?);`,
row,
);
}

async insertNonDuplicatePairs(rows: { itemID: number; itemID2: number }[]) {
const placeholders = rows.map(() => "(?, ?)").join(",");
const values = rows.flatMap(({ itemID, itemID2 }) => [itemID, itemID2].sort());
async insertNonDuplicatePairs(rows: { itemID: number; itemID2: number }[], libraryID?: number) {
libraryID = libraryID ?? Zotero.Items.get(rows[0].itemID).libraryID;
const placeholders = rows.map(() => "(?, ?, ?)").join(",");
const values = rows.flatMap(({ itemID, itemID2 }) => this.buildRow(itemID, itemID2, libraryID));
await this._db.queryAsync(
`INSERT OR IGNORE INTO ${this.tables.nonDuplicates} (itemID, itemID2)
`INSERT OR IGNORE INTO ${this.tables.nonDuplicates} (itemID, itemID2, libraryID)
VALUES ${placeholders};`,
values,
);
}

async insertNonDuplicates(itemIDs: number[]) {
async insertNonDuplicates(itemIDs: number[], libraryID?: number) {
const rows = itemIDs.flatMap((itemID, i) => itemIDs.slice(i + 1).map((itemID2) => ({ itemID, itemID2 })));
await this.insertNonDuplicatePairs(rows);
await this.insertNonDuplicatePairs(rows, libraryID);
}

async deleteNonDuplicatePair(itemID: number, itemID2: number) {
Expand Down Expand Up @@ -269,7 +285,7 @@ export class SQLiteDB implements IDatabase {
return result[0].count === rows.length;
}

async getNonDuplicates(itemID: number | undefined = undefined) {
async getNonDuplicates({ itemID, libraryID }: { itemID?: number; libraryID?: number }) {
const params: number[] = [];
let query = `SELECT itemID, itemID2
FROM ${this.tables.nonDuplicates}`;
Expand All @@ -279,6 +295,11 @@ export class SQLiteDB implements IDatabase {
params.push(itemID, itemID);
}

if (libraryID !== undefined && libraryID !== null) {
query += ` WHERE libraryID = ?`;
params.push(libraryID);
}

const rows: { itemID: number; itemID2: number }[] = await this._db.queryAsync(query, params);
return rows;
}
Expand All @@ -287,3 +308,13 @@ export class SQLiteDB implements IDatabase {
await this._db.closeDatabase(permanent);
}
}

export default {
getDatabase: (dbType: "IndexedDB" | "SQLite" = addon.data.database): IDatabase => {
if (dbType === "IndexedDB") {
return DexieDB.getInstance();
} else {
return SQLiteDB.getInstance();
}
},
};
4 changes: 2 additions & 2 deletions src/modules/menus.ts
View file
Original file line number Diff line number Diff line change
Expand Up @@ -3,7 +3,7 @@ import { config } from "../../package.json";
import { showingDuplicateStats } from "../utils/prefs";
import { fetchAllDuplicates, fetchDuplicates } from "./duplicates";
import { MenuManager } from "zotero-plugin-toolkit/dist/managers/menu";
import { IndexedDB } from "./db";
import database from "./db";
import { isInDuplicatesPane } from "../utils/zotero";
import MenuPopup = XUL.MenuPopup;
import { toggleNonDuplicates } from "./nonDuplicates";
Expand Down Expand Up @@ -77,7 +77,7 @@ function registerItemsViewMenu(menuManager: MenuManager, win: Window) {
`${config.addonRef}-menuitem-not-duplicate`,
) as HTMLElement;
const itemIDs = selectedItems.map((item) => item.id);
showingIsDuplicate = await IndexedDB.getInstance().existsNonDuplicates(itemIDs);
showingIsDuplicate = await database.getDatabase().existsNonDuplicates(itemIDs);
if (showingIsDuplicate) {
isDuplicateMenuItem.removeAttribute("hidden");
notDuplicateMenuItem.setAttribute("hidden", "true");
Expand Down
Loading

0 comments on commit 126e70c

Please sign in to comment.