plugins system

FranciscoMoretti · Oct 14, 2024 · 7bdb9be · 7bdb9be
1 parent 61787fb
commit 7bdb9be
Show file tree

Hide file tree

Showing 19 changed files with 174 additions and 137 deletions.
diff --git a/downloader.config.ts b/downloader.config.ts
@@ -1,4 +1,40 @@
-import { Config } from "./packages/notion-downloader/src/index"
+/* This file is only used when testing docu-notion itself, not when it is used as a library.
+  E.g., if you run `npm run pull-test-tagged`, docu-notion will read this file and use it to configure itself,
+  using these example plugins.
+ */ import {
+  Config,
+  IPlugin,
+  NotionBlock,
+} from "./packages/notion-downloader/src/index"
+
+// This is an example of a plugin that needs customization by the end user.
+// It uses a closure to supply the plugin with the customization parameter.
+function dummyBlockModifier(customParameter: string): IPlugin {
+  return {
+    name: "dummyBlockModifier",
+
+    notionBlockModifications: [
+      {
+        modify: (block: NotionBlock) => {
+          console.log(
+            `dummyBlockModifier has customParameter:${customParameter}.`
+          )
+        },
+      },
+    ],
+  }
+}
+
+const dummyMarkdownModifier: IPlugin = {
+  name: "dummyMarkdownModifier",
+
+  regexMarkdownModifications: [
+    {
+      regex: /aaa(.*)aaa/,
+      replacementPattern: "bbb$1bbb",
+    },
+  ],
+}
 
 const config: Config = {
   conversion: {
@@ -11,10 +47,16 @@ const config: Config = {
       markdown: "pages",
       assets: "public",
     },
+    plugins: [
+      // here we're adding a plugin that needs a parameter for customization
+      "video",
+      // here's we're adding a plugin that doesn't take any customization
+      dummyMarkdownModifier,
+    ],
   },
   rootDbAsFolder: true,
   rootObjectType: "page",
-  rootId: "74fe3069cc484ee5b94fb76bd67732ae",
+  rootId: "11a047149aef80ffb78ef8afd3325647",
   cache: {
     cleanCache: false,
     cacheStrategy: "cache",

diff --git a/packages/notion-downloader/package.json b/packages/notion-downloader/package.json
@@ -60,6 +60,7 @@
     "chalk": "5.2.0",
     "commander": "^10.0.0",
     "cosmiconfig": "9.0.0",
+    "cosmiconfig-typescript-loader": "^5.0.0",
     "diff": "^5.1.0",
     "dotenv": "^16.4.5",
     "fast-glob": "^3.3.2",
@@ -73,8 +74,8 @@
     "node-fetch": "^3.3.0",
     "notion-cache-client": "workspace:*",
     "notion-client": "^4",
-    "notion-tree": "workspace:*",
     "notion-to-md": "3.1.1",
+    "notion-tree": "workspace:*",
     "ora": "^6.1.2",
     "path": "^0.12.7",
     "prompts": "^2.4.2",

diff --git a/packages/notion-downloader/src/config/plugins.ts b/packages/notion-downloader/src/config/plugins.ts
@@ -10,11 +10,6 @@ import { defaultPlugins } from "./defaultPlugins"
 import { NotionToMdPlugin } from "./pluginSchema"
 import { PluginsConfig } from "./schema"
 
-// TODO: Remove IPluginsConfig
-export type IPluginsConfig = {
-  plugins: IPlugin[]
-}
-
 function loadOfficialPlugin(pluginName: string): NotionToMdPlugin {
   if (!(pluginName in standardPluginsDict)) {
     throw new Error(`Official plugin "${pluginName}" not found`)

diff --git a/packages/notion-downloader/src/latex.spec.ts b/packages/notion-downloader/src/latex.spec.ts
@@ -3,13 +3,12 @@ import { BlockObjectResponse } from "@notionhq/client/build/src/api-endpoints"
 import { NotionToMarkdown } from "notion-to-md"
 import { expect, test } from "vitest"
 
-import { IPluginsConfig } from "./config/configuration"
-import defaultConfig from "./config/default.plugin.config"
+import { defaultPlugins } from "./config/defaultPlugins"
 import { defaultPullOptions, parsePathFileOptions } from "./config/schema"
 import { FilesManager } from "./files/FilesManager"
 import { NotionPage } from "./notionObjects/NotionPage"
 import { convertInternalUrl } from "./plugins/internalLinks"
-import { IPluginContext } from "./plugins/pluginTypes"
+import { IPlugin, IPluginContext } from "./plugins/pluginTypes"
 import { getMarkdownFromNotionBlocks } from "./transformMarkdown"
 
 test("Latex Rendering", async () => {
@@ -25,7 +24,7 @@ test("Latex Rendering", async () => {
     auth: "",
   })
 
-  const config: IPluginsConfig = defaultConfig
+  const plugins: IPlugin[] = defaultPlugins
 
   const context: IPluginContext = {
     getBlockChildren: (id: string) => {
@@ -95,7 +94,7 @@ test("Latex Rendering", async () => {
     },
   ]
 
-  expect(await getMarkdownFromNotionBlocks(context, config, blocks)).toContain(
+  expect(await getMarkdownFromNotionBlocks(context, plugins, blocks)).toContain(
     "$x$"
   )
 })
diff --git a/packages/notion-downloader/src/plugins/CalloutTransformer.spec.ts b/packages/notion-downloader/src/plugins/CalloutTransformer.spec.ts
@@ -38,79 +38,83 @@ beforeEach(() => {
 test("smoketest callout", async () => {
   const config = { plugins: [standardCalloutTransformer] }
   block.callout.icon.emoji = "ℹ️"
-  let results = await blocksToMarkdown(config, [
-    block as unknown as NotionBlock,
-  ])
+  let results = await blocksToMarkdown(
+    [standardCalloutTransformer],
+    [block as unknown as NotionBlock]
+  )
   expect(results).toContain("\n:::note\n\nThis is the callout\n\n:::\n")
   block.callout.icon.emoji = "❗"
-  results = await blocksToMarkdown(config, [block as unknown as NotionBlock])
+  results = await blocksToMarkdown(
+    [standardCalloutTransformer],
+    [block as unknown as NotionBlock]
+  )
   expect(results).toContain(":::info")
 })
 
 test("external link inside callout, bold preserved", async () => {
-  const config = {
-    plugins: [
+  const results = await blocksToMarkdown(
+    [
       standardCalloutTransformer,
       standardInternalLinkConversion,
       standardExternalLinkConversion,
     ],
-  }
-  const results = await blocksToMarkdown(config, [
-    {
-      type: "callout",
-      callout: {
-        rich_text: [
-          {
-            type: "text",
-            text: { content: "Callouts inline ", link: null },
-            annotations: {
-              bold: false,
-              italic: false,
-              strikethrough: false,
-              underline: false,
-              code: false,
-              color: "default",
-            },
-            plain_text: "Callouts inline ",
-            href: null,
-          },
-          {
-            type: "text",
-            text: {
-              content: "great page",
-              link: { url: `https://github.com` },
+    [
+      {
+        type: "callout",
+        callout: {
+          rich_text: [
+            {
+              type: "text",
+              text: { content: "Callouts inline ", link: null },
+              annotations: {
+                bold: false,
+                italic: false,
+                strikethrough: false,
+                underline: false,
+                code: false,
+                color: "default",
+              },
+              plain_text: "Callouts inline ",
+              href: null,
             },
-            annotations: {
-              bold: true,
-              italic: false,
-              strikethrough: false,
-              underline: false,
-              code: false,
-              color: "default",
+            {
+              type: "text",
+              text: {
+                content: "great page",
+                link: { url: `https://github.com` },
+              },
+              annotations: {
+                bold: true,
+                italic: false,
+                strikethrough: false,
+                underline: false,
+                code: false,
+                color: "default",
+              },
+              plain_text: "great page",
+              href: `https://github.com`,
             },
-            plain_text: "great page",
-            href: `https://github.com`,
-          },
-          {
-            type: "text",
-            text: { content: ".", link: null },
-            annotations: {
-              bold: false,
-              italic: false,
-              strikethrough: false,
-              underline: false,
-              code: false,
-              color: "default",
+            {
+              type: "text",
+              text: { content: ".", link: null },
+              annotations: {
+                bold: false,
+                italic: false,
+                strikethrough: false,
+                underline: false,
+                code: false,
+                color: "default",
+              },
+              plain_text: ".",
+              href: null,
             },
-            plain_text: ".",
-            href: null,
-          },
-        ],
-        icon: { type: "emoji", emoji: "⚠️" },
-        color: "gray_background",
-      },
-    } as unknown as NotionBlock,
-  ])
+          ],
+          icon: { type: "emoji", emoji: "⚠️" },
+          color: "gray_background",
+        },
+      } as unknown as NotionBlock,
+    ]
+  )
   expect(results.trim()).toBe(
     `:::caution
 
@@ -121,20 +125,17 @@ Callouts inline [**great page**](https://github.com).
 })
 
 test("internal link inside callout, bold preserved", async () => {
-  const config = {
-    plugins: [
-      standardCalloutTransformer,
-      standardInternalLinkConversion,
-      standardExternalLinkConversion,
-    ],
-  }
   const slugTargetPage = makeSamplePageObject({
     slug: "hello-world",
     name: "Hello World",
     id: "123",
   })
   const results = await blocksToMarkdown(
-    config,
+    [
+      standardCalloutTransformer,
+      standardInternalLinkConversion,
+      standardExternalLinkConversion,
+    ],
     [
       {
         type: "callout",

diff --git a/packages/notion-downloader/src/plugins/ColumnTransformer.spec.ts b/packages/notion-downloader/src/plugins/ColumnTransformer.spec.ts
@@ -28,7 +28,7 @@ const columnBlock = {
 } as unknown as NotionBlock
 async function getResults(children: NotionBlock[]) {
   return await blocksToMarkdown(
-    { plugins: [standardColumnTransformer] },
+    [standardColumnTransformer],
     [columnBlock],
     undefined,
     children,

diff --git a/packages/notion-downloader/src/plugins/EscapeHtmlBlockModifier.spec.ts b/packages/notion-downloader/src/plugins/EscapeHtmlBlockModifier.spec.ts
@@ -113,8 +113,8 @@ beforeEach(() => {
 })
 
 test("smoketest ", async () => {
-  const config = { plugins: [standardEscapeHtmlBlockModifier] }
-  let results = await blocksToMarkdown(config, blocks)
+  const plugins = [standardEscapeHtmlBlockModifier]
+  let results = await blocksToMarkdown(plugins, blocks)
   // shouldn't escape inside a code block
   expect(results).toContain("This is code: if(1 < 3)")
   // should escape outside a code block

diff --git a/packages/notion-downloader/src/plugins/HeadingTranformer.spec.ts b/packages/notion-downloader/src/plugins/HeadingTranformer.spec.ts
@@ -7,8 +7,8 @@ import { blocksToMarkdown } from "./pluginTestRun"
 test("Adds anchor to headings", async () => {
   //setLogLevel("verbose");
   const headingBlockId = "86f746f4-1c79-4ba1-a2f6-a1d59c2f9d23"
-  const config = { plugins: [standardHeadingTransformer] }
-  const result = await blocksToMarkdown(config, [
+  const plugins = [standardHeadingTransformer]
+  const result = await blocksToMarkdown(plugins, [
     {
       object: "block",
       id: headingBlockId,

diff --git a/packages/notion-downloader/src/plugins/VideoTransformer.spec.ts b/packages/notion-downloader/src/plugins/VideoTransformer.spec.ts
@@ -6,8 +6,8 @@ import { standardVideoTransformer } from "./VideoTransformer"
 import { blocksToMarkdown } from "./pluginTestRun"
 
 test("youtube embedded", async () => {
-  const config = { plugins: [standardVideoTransformer] }
-  const result = await blocksToMarkdown(config, [
+  const plugins = [standardVideoTransformer]
+  const result = await blocksToMarkdown(plugins, [
     {
       object: "block",
       type: "video",
@@ -36,8 +36,8 @@ test("youtube embedded", async () => {
 
 test("vimeo embedded", async () => {
   setLogLevel("verbose")
-  const config = { plugins: [standardVideoTransformer] }
-  const result = await blocksToMarkdown(config, [
+  const plugins = [standardVideoTransformer]
+  const result = await blocksToMarkdown(plugins, [
     {
       object: "block",
       type: "video",
@@ -56,8 +56,8 @@ test("vimeo embedded", async () => {
 
 test("video link, not embedded", async () => {
   setLogLevel("verbose")
-  const config = { plugins: [standardVideoTransformer] }
-  const result = await blocksToMarkdown(config, [
+  const plugins = [standardVideoTransformer]
+  const result = await blocksToMarkdown(plugins, [
     {
       object: "block",
       type: "paragraph",
@@ -90,8 +90,8 @@ test("video link, not embedded", async () => {
 
 test("direct upload to to Notion (embedded)", async () => {
   setLogLevel("verbose")
-  const config = { plugins: [standardVideoTransformer] }
-  const result = await blocksToMarkdown(config, [
+  const plugins = [standardVideoTransformer]
+  const result = await blocksToMarkdown(plugins, [
     {
       object: "block",
       id: "12f7db3b-4412-4be9-a3f7-6ac423fee94a",