solved relative paths saving, and file updating

files_map.json

@@ -1,52 +1,52 @@
 {
   "page": {
     "efa8d9de-c3bf-420d-8e1d-f8238df144d2": {
-      "path": "content/blog/node-file-writing-methods-createwritestream-vs-writefilesync.md",
+      "path": "/blog/node-file-writing-methods-createwritestream-vs-writefilesync.md",
       "lastEditedTime": "2024-08-26T18:24:00.000Z"
     },
     "57701aab-8b5c-4798-9e6d-5b3f9603520b": {
-      "path": "content/blog/avoiding-mistakes-in-nextjs-using-the-typescript-plugin.md",
+      "path": "/blog/avoiding-mistakes-in-nextjs-using-the-typescript-plugin.md",
       "lastEditedTime": "2024-08-12T08:07:00.000Z"
     },
     "c94502e0-2678-4269-baa0-036954ba5649": {
-      "path": "content/authors/fmoretti.md",
+      "path": "/authors/fmoretti.md",
       "lastEditedTime": "2024-08-16T18:36:00.000Z"
     }
   },
   "database": {
     "c974ccd9-c70c-4abd-8a5b-d4f5a294e5dd": {
-      "path": "content",
+      "path": "",
       "lastEditedTime": "2024-08-14T08:54:00.000Z"
     },
     "bbad12b6-7bcf-4390-bb50-3b177f17a9f1": {
-      "path": "content/blog",
+      "path": "/blog",
       "lastEditedTime": "2024-08-25T09:07:00.000Z"
     },
     "2eae1488-3b02-4a1a-9929-02f3130ee537": {
-      "path": "content/authors",
-      "lastEditedTime": "2024-08-26T18:52:00.000Z"
+      "path": "/authors",
+      "lastEditedTime": "2024-08-29T08:44:00.000Z"
     }
   },
   "image": {
     "7c023a96-c86e-4633-b496-1885ed0a92ae": {
-      "path": "public/assets/blog/node-file-writing-methods-createwritestream-vs-writefilesync.md.7c023a96-c86e-4633-b496-1885ed0a92ae.png",
+      "path": "blog/node-file-writing-methods-createwritestream-vs-writefilesync.md.7c023a96-c86e-4633-b496-1885ed0a92ae.png",
       "lastEditedTime": "2024-08-24T15:28:00.000Z"
     },
     "efa8d9de-c3bf-420d-8e1d-f8238df144d2": {
-      "path": "public/assets/blog/node-file-writing-methods-createwritestream-vs-writefilesync.md.efa8d9de-c3bf-420d-8e1d-f8238df144d2.jpg",
+      "path": "blog/node-file-writing-methods-createwritestream-vs-writefilesync.md.efa8d9de-c3bf-420d-8e1d-f8238df144d2.jpg",
       "lastEditedTime": "2024-08-26T18:24:00.000Z"
     },
     "57701aab-8b5c-4798-9e6d-5b3f9603520b": {
-      "path": "public/assets/blog/avoiding-mistakes-in-nextjs-using-the-typescript-plugin.md.57701aab-8b5c-4798-9e6d-5b3f9603520b.png",
+      "path": "blog/avoiding-mistakes-in-nextjs-using-the-typescript-plugin.md.57701aab-8b5c-4798-9e6d-5b3f9603520b.png",
       "lastEditedTime": "2024-08-12T08:07:00.000Z"
     },
     "c94502e0-2678-4269-baa0-036954ba5649": {
-      "path": "public/assets/authors/fmoretti.md.c94502e0-2678-4269-baa0-036954ba5649.png",
+      "path": "authors/fmoretti.md.c94502e0-2678-4269-baa0-036954ba5649.png",
       "lastEditedTime": "2024-08-16T18:36:00.000Z"
     },
     "2eae1488-3b02-4a1a-9929-02f3130ee537": {
-      "path": "public/assets/authors.2eae1488-3b02-4a1a-9929-02f3130ee537.jpg",
-      "lastEditedTime": "2024-08-26T18:52:00.000Z"
+      "path": "authors.2eae1488-3b02-4a1a-9929-02f3130ee537.jpg",
+      "lastEditedTime": "2024-08-29T08:44:00.000Z"
     }
   }
 }

packages/download-notion/src/FilesManager.ts

@@ -1,6 +1,13 @@
 import fs from "fs-extra"
 
-import { FileRecord, FileType, FilesMap, ObjectsDirectories } from "./FilesMap"
+import {
+  FileRecord,
+  FileType,
+  FilesMap,
+  FilesMapData,
+  ObjectsDirectories,
+  recordMapWithPathPrefix,
+} from "./FilesMap"
 import { NotionObject } from "./NotionObject"
 import { info, verbose } from "./log"
 
@@ -9,46 +16,167 @@ type ExtendedFileRecord = FileRecord & {
 }
 
 export class FilesManager {
-  public filesMap: FilesMap
-  protected objectsDirectories: ObjectsDirectories
-
-  public constructor(
-    // TODO: Should be structured props and intirialsFilesMap should be optional
-    initialFilesMap: FilesMap | undefined,
+  // This class holds directories in which each file is located and relative paths to them
+  public filesMap: FilesMap // Relative paths to files directories
+  protected objectsDirectories: ObjectsDirectories // Files directories
+
+  public constructor({
+    objectsDirectories,
+    initialFilesMap,
+  }: {
     objectsDirectories: ObjectsDirectories
-  ) {
+    initialFilesMap?: FilesMap
+  }) {
     this.filesMap = initialFilesMap || new FilesMap()
     // TODO: If directories changed, cleanup all files in directories changed here
     this.objectsDirectories = objectsDirectories
   }
 
-  public async cleanOldFiles(newFilesMap: FilesMap): Promise<void> {
-    if (!this.filesMap) {
-      info("No files map found, skipping cleanup")
-      return
+  // TODO: Rethink if this method should be in this class. FilesManager shouldn't know about processing. Maybe name isNewObject.
+  public shouldProcessObject(notionObject: NotionObject): boolean {
+    if (
+      !this.filesMap.exists(
+        // TODO: Make this FilesMa structure more generic when we want to store more than images
+        notionObject.object == "block" ? "image" : notionObject.object,
+        notionObject.id
+      )
+    ) {
+      return true // Process new pages
     }
-    info("Cleaning up old files")
+    const existingRecord = this.filesMap.get(
+      notionObject.object == "block" ? "image" : notionObject.object,
+      notionObject.id
+    )
+    // If new file date is older than old file date, the state is inconcistent and we throw an error
+    if (
+      new Date(notionObject.lastEditedTime).getTime() >
+      new Date(existingRecord.lastEditedTime).getTime()
+    ) {
+      return true
+    }
+
+    return false
+  }
+
+  public exists(type: FileType, id: string): boolean {
+    return this.filesMap.exists(type, id)
+  }
+
+  public get(
+    relativeTo: "directory" | "root",
+    type: FileType,
+    id: string
+  ): FileRecord {
+    const recordFromDirectory = this.filesMap.get(type, id)
+
+    if (relativeTo === "root") {
+      return this.filesMap.recordToRootRelativePath(
+        recordFromDirectory,
+        this.objectsDirectories[type]
+      )
+    } else {
+      return recordFromDirectory
+    }
+  }
 
-    const oldFiles = this.getFileRecords(this.filesMap, "page")
-    const newFiles = this.getFileRecords(newFilesMap, "page")
-    const oldImages = this.getFileRecords(this.filesMap, "image")
-    const newImages = this.getFileRecords(newFilesMap, "image")
+  public set(
+    relativeTo: "directory" | "root",
+    type: FileType,
+    id: string,
+    record: FileRecord
+  ): void {
+    const recordToSet =
+      relativeTo === "root"
+        ? this.filesMap.recordToDirectoriesRelativePath(
+            record,
+            this.objectsDirectories[type]
+          )
+        : record
+    this.filesMap.set(type, id, recordToSet)
+  }
+
+  public delete(type: FileType, id: string): void {
+    this.filesMap.delete(type, id)
+  }
+
+  public getAllOfType(
+    relativeTo: "directory" | "root",
+    type: FileType
+  ): Record<string, FileRecord> {
+    const records = this.filesMap.getAllOfType(type)
+    if (relativeTo === "root") {
+      return recordMapWithPathPrefix(records, this.objectsDirectories[type])
+    } else {
+      return records
+    }
+  }
+
+  public getAll(relativeTo: "directory" | "root"): FilesMapData {
+    const filesMapData = this.filesMap.getAll()
+
+    if (relativeTo === "root") {
+      return {
+        page: recordMapWithPathPrefix(
+          filesMapData.page,
+          this.objectsDirectories.page
+        ),
+        database: recordMapWithPathPrefix(
+          filesMapData.database,
+          this.objectsDirectories.database
+        ),
+        image: recordMapWithPathPrefix(
+          filesMapData.image,
+          this.objectsDirectories.image
+        ),
+      }
+    } else {
+      return filesMapData
+    }
+  }
+}
+
+export class FilesCleaner {
+  private oldFilesManager: FilesManager
+  private newFilesManager: FilesManager
+
+  constructor({
+    oldFilesManager,
+    newFilesManager,
+  }: {
+    oldFilesManager: FilesManager
+    newFilesManager: FilesManager
+  }) {
+    this.oldFilesManager = oldFilesManager
+    this.newFilesManager = newFilesManager
+  }
+
+  public async cleanupOldFiles(): Promise<void> {
+    info("Cleaning up old files")
+    const oldFiles = this.getFileRecords(this.oldFilesManager, "page")
+    const newFiles = this.getFileRecords(this.newFilesManager, "page")
+    const oldImages = this.getFileRecords(this.oldFilesManager, "image")
+    const newImages = this.getFileRecords(this.newFilesManager, "image")
 
     const filesToRemove = this.getFilesToRemove(oldFiles, newFiles)
     const imagesToRemove = this.getFilesToRemove(oldImages, newImages)
 
-    const filesCleaner = new FilesCleaner()
-    await filesCleaner.cleanupOldFiles([...filesToRemove, ...imagesToRemove])
+    for (const p of [...filesToRemove, ...imagesToRemove]) {
+      verbose(`Removing file: ${p}`)
+      await fs.rm(p)
+    }
   }
 
   private getFileRecords(
-    filesMap: FilesMap,
+    filesManager: FilesManager,
     type: "page" | "image"
   ): ExtendedFileRecord[] {
-    return Object.entries(filesMap.getAllOfType(type)).map(([id, record]) => ({
-      id,
-      ...record,
-    }))
+    // Root path is needed so that fiels can be removed
+    return Object.entries(filesManager.getAllOfType("root", type)).map(
+      ([id, record]) => ({
+        id,
+        ...record,
+      })
+    )
   }
 
   private getFilesToRemove(
@@ -85,9 +213,10 @@ export class FilesManager {
       return "moved"
     }
 
-    // If records were updated,it should not be removed
+    // If records were updated, it should not be removed
     if (
-      new Date(newRecord.lastEditedTime) >= new Date(oldRecord.lastEditedTime)
+      new Date(newRecord.lastEditedTime).getTime() >=
+      new Date(oldRecord.lastEditedTime).getTime()
     ) {
       return null
     }
@@ -102,65 +231,4 @@ export class FilesManager {
     // We should never get here
     throw new Error("get removal reason failed")
   }
-
-  public shouldProcessObject(notionObject: NotionObject): boolean {
-    if (!this.filesMap) {
-      return true // Process all pages if there's no initial files map
-    }
-
-    if (
-      !this.filesMap.exists(
-        // TODO: Make this FilesMa structure more generic when we want to store more than images
-        notionObject.object == "block" ? "image" : notionObject.object,
-        notionObject.id
-      )
-    ) {
-      return true // Process new pages
-    }
-    const existingRecord = this.filesMap.get(
-      notionObject.object == "block" ? "image" : notionObject.object,
-      notionObject.id
-    )
-    // If new file date is older than old file date, the state is inconcistent and we throw an error
-    if (
-      new Date(notionObject.lastEditedTime) >
-      new Date(existingRecord.lastEditedTime)
-    ) {
-      return true
-    }
-
-    return false
-  }
-
-  public get(
-    relativeTo: "directory" | "root",
-    type: FileType,
-    id: string
-  ): FileRecord {
-    if (!this.filesMap) {
-      throw new Error(
-        "Trying to get file record from files map that does not exist"
-      )
-    }
-
-    const recordFromRoot = this.filesMap.get(type, id)
-
-    if (relativeTo === "directory") {
-      return this.filesMap.recordToDirectoriesRelativePath(
-        recordFromRoot,
-        this.objectsDirectories[type]
-      )
-    } else {
-      return recordFromRoot
-    }
-  }
-}
-
-class FilesCleaner {
-  public async cleanupOldFiles(filesToRemove: string[]): Promise<void> {
-    for (const p of filesToRemove) {
-      verbose(`Removing file: ${p}`)
-      await fs.rm(p)
-    }
-  }
 }