using a plugin system in the config

packages/notion-downloader/src/config/defaultPlugins.ts

@@ -0,0 +1,6 @@
+import { standardInternalLinkConversion } from "../plugins/internalLinks"
+import { IPlugin } from "../plugins/pluginTypes"
+
+// TODO: Create a section in the documentation that explains all plugins.
+
+export const defaultPlugins: IPlugin[] = [standardInternalLinkConversion]

packages/notion-downloader/src/config/pluginSchema.ts

@@ -0,0 +1,48 @@
+import { z } from "zod"
+
+import { IPlugin } from "../plugins/pluginTypes"
+
+const NotionBlockModification = z.object({
+  modify: z.function().args(z.any()).returns(z.void()),
+})
+
+const NotionToMarkdownTransform = z.object({
+  type: z.string(),
+  getStringFromBlock: z
+    .function()
+    .args(z.any(), z.any())
+    .returns(z.union([z.string(), z.promise(z.string())])),
+})
+
+const LinkModifier = z.object({
+  match: z.instanceof(RegExp),
+  convert: z.function().args(z.any(), z.string()).returns(z.string()),
+})
+
+const RegexMarkdownModification = z.object({
+  regex: z.instanceof(RegExp),
+  replacementPattern: z.string().optional(),
+  getReplacement: z
+    .function()
+    .args(z.any(), z.any())
+    .returns(z.promise(z.string()))
+    .optional(),
+  includeCodeBlocks: z.boolean().optional(),
+  imports: z.array(z.string()).optional(),
+})
+
+export const NotionToMdPlugin = z.object({
+  name: z.string(),
+  notionBlockModifications: z.array(NotionBlockModification).optional(),
+  notionToMarkdownTransforms: z.array(NotionToMarkdownTransform).optional(),
+  linkModifier: LinkModifier.optional(),
+  regexMarkdownModifications: z.array(RegexMarkdownModification).optional(),
+  init: z.function().args(z.any()).returns(z.promise(z.void())).optional(),
+})
+
+export type NotionToMdPlugin = z.infer<typeof NotionToMdPlugin>
+
+// Asserting that IPlugin satisfies NotionToMdPlugin
+// NOTE: We maintain a zod type for verification and a typescript type for the interface
+//       this is because some of the arguments are too complex for a zod type, like `NotionBlock`
+export const _: NotionToMdPlugin = {} as IPlugin

packages/notion-downloader/src/config/plugins.ts

@@ -0,0 +1,58 @@
+import { exit } from "process"
+import * as Cosmic from "cosmiconfig"
+import { TypeScriptLoader } from "cosmiconfig-typescript-loader"
+
+import { error, verbose } from "../log"
+import { IPlugin } from "../plugins/pluginTypes"
+import { standardPluginsDict } from "../plugins/standardPlugins"
+import { handleError } from "../utils/handle-error"
+import { defaultPlugins } from "./defaultPlugins"
+import { NotionToMdPlugin } from "./pluginSchema"
+import { PluginsConfig } from "./schema"
+
+// TODO: Remove IPluginsConfig
+export type IPluginsConfig = {
+  plugins: IPlugin[]
+}
+
+function loadOfficialPlugin(pluginName: string): NotionToMdPlugin {
+  if (!(pluginName in standardPluginsDict)) {
+    throw new Error(`Official plugin "${pluginName}" not found`)
+  }
+  return NotionToMdPlugin.parse(standardPluginsDict[pluginName])
+}
+
+async function initializePlugin(
+  plugin: NotionToMdPlugin
+): Promise<NotionToMdPlugin> {
+  if (plugin.init) {
+    verbose(`Initializing plugin ${plugin.name}...`)
+    await plugin.init(plugin)
+  }
+  return plugin
+}
+
+export async function loadPlugins(
+  configPlugins: PluginsConfig
+): Promise<IPlugin[]> {
+  let resultingPlugins: IPlugin[] = []
+
+  try {
+    const loadedPlugins: IPlugin[] = await Promise.all(
+      configPlugins.map(async (plugin) => {
+        if (typeof plugin === "string") {
+          return await initializePlugin(loadOfficialPlugin(plugin))
+        } else {
+          return await initializePlugin(plugin)
+        }
+      })
+    )
+
+    resultingPlugins = defaultPlugins.concat(loadedPlugins)
+  } catch (e: any) {
+    handleError(e.message)
+  }
+
+  verbose(`Active plugins: [${resultingPlugins.map((p) => p.name).join(", ")}]`)
+  return resultingPlugins
+}

packages/notion-downloader/src/config/schema.ts

@@ -2,6 +2,8 @@ import { ObjectType, PageOrDatabase } from "notion-cache-client"
 import { cacheOptionsSchema } from "notion-tree"
 import { coerce, z } from "zod"
 
+import { NotionToMdPlugin } from "./pluginSchema"
+
 export const AssetType = z.enum(["image", "file", "video", "pdf", "audio"])
 export type AssetType = z.infer<typeof AssetType>
 
@@ -161,6 +163,9 @@ const filterSchema = z.object({
 
 export type Filter = z.infer<typeof filterSchema>
 
+export const PluginsConfig = z.array(z.union([z.string(), NotionToMdPlugin]))
+export type PluginsConfig = z.infer<typeof PluginsConfig>
+
 export const conversionSchema = z.object({
   skip: z.boolean().default(false),
   overwrite: z.boolean().default(false),
@@ -177,6 +182,7 @@ export const conversionSchema = z.object({
   namingStrategy: namingStrategyOptionsSchema.default(
     AllNamingSchemaName.enum.default
   ),
+  plugins: PluginsConfig.default([]),
 })
 
 export const rootObjectTypeSchema = z.enum([

packages/notion-downloader/src/files/FilesManager.ts

@@ -1,15 +1,15 @@
-import { ObjectType, ObjectType } from "notion-cache-client"
+import { ObjectType } from "notion-cache-client"
 
 import { FilepathGroup, mapToAssetType } from "../config/schema"
 import { NotionObject } from "../notionObjects/NotionObject"
 import { iNotionAssetObject } from "../notionObjects/objectTypes"
 import { FileRecord, FileRecordType, FilesMap, FilesMapData } from "./FilesMap"
+import { convertMarkdownPath } from "./markdownPathUtils"
 import {
   recordMapWithPrefix,
   recordWithPrefix,
   toMapDataWithPrefix,
 } from "./recordPrefixUtils"
-import { convertMarkdownPath } from "./markdownPathUtils"
 
 type PathType = "base" | "output" | "markdown"
 
@@ -72,7 +72,10 @@ export class FilesManager {
     if (pathType === "output") {
       return recordWithPrefix(recordFromDirectory, this.outputDirectories[type])
     } else if (pathType === "markdown") {
-      const record =  recordWithPrefix(recordFromDirectory, this.markdownPrefixes[type])
+      const record = recordWithPrefix(
+        recordFromDirectory,
+        this.markdownPrefixes[type]
+      )
       return {
         ...record,
         path: convertMarkdownPath(record.path),

packages/notion-downloader/src/index.ts

@@ -12,6 +12,10 @@ process.on("SIGTERM", () => process.exit(0))
 
 export { NotionPullOptionsInput as Config }
 
+// TODO: Cleanup unnecesasry exports
+export { IPlugin } from "./plugins/pluginTypes"
+export { NotionBlock } from "./types"
+
 async function main() {
   const packageInfo = await getPackageInfo()
 

packages/notion-downloader/src/notionPull.ts

@@ -13,7 +13,7 @@ import {
 import { NotionToMarkdown } from "notion-to-md"
 import { NotionObjectTree, downloadNotionObjectTree } from "notion-tree"
 
-import { IPluginsConfig, loadConfigAsync } from "./config/configuration"
+import { loadPlugins } from "./config/plugins"
 import {
   FilepathGroup,
   NotionPullOptions,
@@ -37,7 +37,7 @@ import { endGroup, error, group, info } from "./log"
 import { NotionPage } from "./notionObjects/NotionPage"
 import { filterTree } from "./objectTree/filterTree"
 import { convertInternalUrl } from "./plugins/internalLinks"
-import { IPluginContext } from "./plugins/pluginTypes"
+import { IPlugin, IPluginContext } from "./plugins/pluginTypes"
 import {
   readOrDownloadNewAssets,
   saveNewAssets,
@@ -253,13 +253,11 @@ export async function notionPull(options: NotionPullOptions): Promise<void> {
   info(`Found ${objectsTree.getPages().length} pages`)
   info(`Found ${pagesToOutput.length} to output`)
 
-  const pluginsConfig = await loadConfigAsync()
   const notionToMarkdown = new NotionToMarkdown({
     notionClient: cachedNotionClient,
   })
   await outputPages(
     options,
-    pluginsConfig,
     pagesToOutput,
     cachedNotionClient,
     notionToMarkdown,
@@ -399,7 +397,6 @@ function getPagesToOutput(
 
 async function outputPages(
   options: NotionPullOptions,
-  config: IPluginsConfig,
   pages: Array<NotionPage>,
   client: Client,
   notionToMarkdown: NotionToMarkdown,
@@ -412,14 +409,15 @@ async function outputPages(
     notionToMarkdown,
     filesManager
   )
+  const plugins = await loadPlugins(options.conversion.plugins)
 
   for (const page of pages) {
     const mdPathWithRoot = filesManager.get(
       "output",
       ObjectType.enum.page,
       page.id
     )?.path
-    const markdown = await getMarkdownForPage(config, context, page)
+    const markdown = await getMarkdownForPage(plugins, context, page)
     writePage(markdown, mdPathWithRoot)
   }
 

packages/notion-downloader/src/plugins/externalLinks.ts

@@ -19,7 +19,7 @@ export const standardExternalLinkConversion: IPlugin = {
       if (label === "bookmark") {
         const replacement = `[${url}](${url})`
         warning(
-          `[standardExternalLinkConversion] Found Notion "Bookmark" link. In Notion this would show as an embed. The best docu-notion can do at the moment is replace "Bookmark" with the actual URL: ${replacement}`
+          `[standardExternalLinkConversion] Found Notion "Bookmark" link. In Notion this would show as an embed. We replace "Bookmark" with the actual URL: ${replacement}`
         )
         return replacement
       }

packages/notion-downloader/src/plugins/pluginTestRun.ts

@@ -2,7 +2,7 @@ import { Client } from "@notionhq/client"
 import { PageObjectResponse } from "@notionhq/client/build/src/api-endpoints"
 import { NotionToMarkdown } from "notion-to-md"
 
-import { IPluginsConfig } from "../config/configuration"
+import { IPluginsConfig } from "../config/plugins"
 import { defaultPullOptions, parsePathFileOptions } from "../config/schema"
 import { FilesManager } from "../files/FilesManager"
 import { FilesMap } from "../files/FilesMap"

packages/notion-downloader/src/plugins/pluginTypes.ts

@@ -3,8 +3,8 @@ import { ListBlockChildrenResponseResult } from "notion-to-md/build/types"
 
 import { NotionPullOptions } from "../config/schema"
 import { FilesManager } from "../files/FilesManager"
-import { ICounts, NotionBlock } from "../index"
 import { NotionPage } from "../notionObjects/NotionPage"
+import { ICounts, NotionBlock } from "../types"
 
 type linkConversionFunction = (
   context: IPluginContext,

packages/notion-downloader/src/plugins/standardPlugins.ts

@@ -0,0 +1,40 @@
+import { standardCalloutTransformer } from "./CalloutTransformer"
+import { standardColumnListTransformer } from "./ColumnListTransformer"
+import { standardColumnTransformer } from "./ColumnTransformer"
+import { standardEscapeHtmlBlockModifier } from "./EscapeHtmlBlockModifier"
+import { standardHeadingTransformer } from "./HeadingTransformer"
+import { standardTableTransformer } from "./TableTransformer"
+import { standardVideoTransformer } from "./VideoTransformer"
+import { gifEmbed, imgurGifEmbed } from "./embedTweaks"
+import { standardExternalLinkConversion } from "./externalLinks"
+import { standardInternalLinkConversion } from "./internalLinks"
+import { IPlugin } from "./pluginTypes"
+
+// TODO: Create a section in the documentation that explains all plugins.
+
+const standardPlugins = [
+  // Notion "Block" JSON modifiers
+  standardEscapeHtmlBlockModifier,
+  standardHeadingTransformer, // does operations on both the Notion JSON and then later, on the notion to markdown transform
+
+  // Notion to Markdown transformers. Most things get transformed correctly by the notion-to-markdown library,
+  // but some things need special handling.
+  standardColumnTransformer, // STandard column transformer uses notion unofficial API
+  standardColumnListTransformer,
+  standardCalloutTransformer,
+  standardTableTransformer,
+
+  standardVideoTransformer,
+
+  // Link modifiers, which are special because they can read metadata from all the pages in order to figure out the correct url
+  standardInternalLinkConversion,
+  standardExternalLinkConversion,
+
+  // Regexps plus javascript `import`s that operate on the Markdown output
+  imgurGifEmbed,
+  gifEmbed,
+]
+
+export const standardPluginsDict: Record<string, IPlugin> = Object.fromEntries(
+  standardPlugins.map((plugin) => [plugin.name, plugin])
+)