diff --git a/downloader.config.ts b/downloader.config.ts index 16fca6f..1fced5d 100644 --- a/downloader.config.ts +++ b/downloader.config.ts @@ -1,4 +1,40 @@ -import { Config } from "./packages/notion-downloader/src/index" +/* This file is only used when testing docu-notion itself, not when it is used as a library. + E.g., if you run `npm run pull-test-tagged`, docu-notion will read this file and use it to configure itself, + using these example plugins. + */ import { + Config, + IPlugin, + NotionBlock, +} from "./packages/notion-downloader/src/index" + +// This is an example of a plugin that needs customization by the end user. +// It uses a closure to supply the plugin with the customization parameter. +function dummyBlockModifier(customParameter: string): IPlugin { + return { + name: "dummyBlockModifier", + + notionBlockModifications: [ + { + modify: (block: NotionBlock) => { + console.log( + `dummyBlockModifier has customParameter:${customParameter}.` + ) + }, + }, + ], + } +} + +const dummyMarkdownModifier: IPlugin = { + name: "dummyMarkdownModifier", + + regexMarkdownModifications: [ + { + regex: /aaa(.*)aaa/, + replacementPattern: "bbb$1bbb", + }, + ], +} const config: Config = { conversion: { @@ -11,10 +47,16 @@ const config: Config = { markdown: "pages", assets: "public", }, + plugins: [ + // here we're adding a plugin that needs a parameter for customization + "video", + // here's we're adding a plugin that doesn't take any customization + dummyMarkdownModifier, + ], }, rootDbAsFolder: true, rootObjectType: "page", - rootId: "74fe3069cc484ee5b94fb76bd67732ae", + rootId: "11a047149aef80ffb78ef8afd3325647", cache: { cleanCache: false, cacheStrategy: "cache", diff --git a/packages/notion-downloader/package.json b/packages/notion-downloader/package.json index f1bf421..37c5a3f 100644 --- a/packages/notion-downloader/package.json +++ b/packages/notion-downloader/package.json @@ -60,6 +60,7 @@ "chalk": "5.2.0", "commander": "^10.0.0", "cosmiconfig": "9.0.0", + "cosmiconfig-typescript-loader": "^5.0.0", "diff": "^5.1.0", "dotenv": "^16.4.5", "fast-glob": "^3.3.2", @@ -73,8 +74,8 @@ "node-fetch": "^3.3.0", "notion-cache-client": "workspace:*", "notion-client": "^4", - "notion-tree": "workspace:*", "notion-to-md": "3.1.1", + "notion-tree": "workspace:*", "ora": "^6.1.2", "path": "^0.12.7", "prompts": "^2.4.2", diff --git a/packages/notion-downloader/src/config/plugins.ts b/packages/notion-downloader/src/config/plugins.ts index 7207881..71873a5 100644 --- a/packages/notion-downloader/src/config/plugins.ts +++ b/packages/notion-downloader/src/config/plugins.ts @@ -10,11 +10,6 @@ import { defaultPlugins } from "./defaultPlugins" import { NotionToMdPlugin } from "./pluginSchema" import { PluginsConfig } from "./schema" -// TODO: Remove IPluginsConfig -export type IPluginsConfig = { - plugins: IPlugin[] -} - function loadOfficialPlugin(pluginName: string): NotionToMdPlugin { if (!(pluginName in standardPluginsDict)) { throw new Error(`Official plugin "${pluginName}" not found`) diff --git a/packages/notion-downloader/src/latex.spec.ts b/packages/notion-downloader/src/latex.spec.ts index 358b0de..f3471be 100644 --- a/packages/notion-downloader/src/latex.spec.ts +++ b/packages/notion-downloader/src/latex.spec.ts @@ -3,13 +3,12 @@ import { BlockObjectResponse } from "@notionhq/client/build/src/api-endpoints" import { NotionToMarkdown } from "notion-to-md" import { expect, test } from "vitest" -import { IPluginsConfig } from "./config/configuration" -import defaultConfig from "./config/default.plugin.config" +import { defaultPlugins } from "./config/defaultPlugins" import { defaultPullOptions, parsePathFileOptions } from "./config/schema" import { FilesManager } from "./files/FilesManager" import { NotionPage } from "./notionObjects/NotionPage" import { convertInternalUrl } from "./plugins/internalLinks" -import { IPluginContext } from "./plugins/pluginTypes" +import { IPlugin, IPluginContext } from "./plugins/pluginTypes" import { getMarkdownFromNotionBlocks } from "./transformMarkdown" test("Latex Rendering", async () => { @@ -25,7 +24,7 @@ test("Latex Rendering", async () => { auth: "", }) - const config: IPluginsConfig = defaultConfig + const plugins: IPlugin[] = defaultPlugins const context: IPluginContext = { getBlockChildren: (id: string) => { @@ -95,7 +94,7 @@ test("Latex Rendering", async () => { }, ] - expect(await getMarkdownFromNotionBlocks(context, config, blocks)).toContain( + expect(await getMarkdownFromNotionBlocks(context, plugins, blocks)).toContain( "$x$" ) }) diff --git a/packages/notion-downloader/src/plugins/CalloutTransformer.spec.ts b/packages/notion-downloader/src/plugins/CalloutTransformer.spec.ts index a824a32..769e2c0 100644 --- a/packages/notion-downloader/src/plugins/CalloutTransformer.spec.ts +++ b/packages/notion-downloader/src/plugins/CalloutTransformer.spec.ts @@ -38,79 +38,83 @@ beforeEach(() => { test("smoketest callout", async () => { const config = { plugins: [standardCalloutTransformer] } block.callout.icon.emoji = "ℹ️" - let results = await blocksToMarkdown(config, [ - block as unknown as NotionBlock, - ]) + let results = await blocksToMarkdown( + [standardCalloutTransformer], + [block as unknown as NotionBlock] + ) expect(results).toContain("\n:::note\n\nThis is the callout\n\n:::\n") block.callout.icon.emoji = "❗" - results = await blocksToMarkdown(config, [block as unknown as NotionBlock]) + results = await blocksToMarkdown( + [standardCalloutTransformer], + [block as unknown as NotionBlock] + ) expect(results).toContain(":::info") }) test("external link inside callout, bold preserved", async () => { - const config = { - plugins: [ + const results = await blocksToMarkdown( + [ standardCalloutTransformer, standardInternalLinkConversion, standardExternalLinkConversion, ], - } - const results = await blocksToMarkdown(config, [ - { - type: "callout", - callout: { - rich_text: [ - { - type: "text", - text: { content: "Callouts inline ", link: null }, - annotations: { - bold: false, - italic: false, - strikethrough: false, - underline: false, - code: false, - color: "default", - }, - plain_text: "Callouts inline ", - href: null, - }, - { - type: "text", - text: { - content: "great page", - link: { url: `https://github.com` }, + [ + { + type: "callout", + callout: { + rich_text: [ + { + type: "text", + text: { content: "Callouts inline ", link: null }, + annotations: { + bold: false, + italic: false, + strikethrough: false, + underline: false, + code: false, + color: "default", + }, + plain_text: "Callouts inline ", + href: null, }, - annotations: { - bold: true, - italic: false, - strikethrough: false, - underline: false, - code: false, - color: "default", + { + type: "text", + text: { + content: "great page", + link: { url: `https://github.com` }, + }, + annotations: { + bold: true, + italic: false, + strikethrough: false, + underline: false, + code: false, + color: "default", + }, + plain_text: "great page", + href: `https://github.com`, }, - plain_text: "great page", - href: `https://github.com`, - }, - { - type: "text", - text: { content: ".", link: null }, - annotations: { - bold: false, - italic: false, - strikethrough: false, - underline: false, - code: false, - color: "default", + { + type: "text", + text: { content: ".", link: null }, + annotations: { + bold: false, + italic: false, + strikethrough: false, + underline: false, + code: false, + color: "default", + }, + plain_text: ".", + href: null, }, - plain_text: ".", - href: null, - }, - ], - icon: { type: "emoji", emoji: "⚠️" }, - color: "gray_background", - }, - } as unknown as NotionBlock, - ]) + ], + icon: { type: "emoji", emoji: "⚠️" }, + color: "gray_background", + }, + } as unknown as NotionBlock, + ] + ) expect(results.trim()).toBe( `:::caution @@ -121,20 +125,17 @@ Callouts inline [**great page**](https://github.com). }) test("internal link inside callout, bold preserved", async () => { - const config = { - plugins: [ - standardCalloutTransformer, - standardInternalLinkConversion, - standardExternalLinkConversion, - ], - } const slugTargetPage = makeSamplePageObject({ slug: "hello-world", name: "Hello World", id: "123", }) const results = await blocksToMarkdown( - config, + [ + standardCalloutTransformer, + standardInternalLinkConversion, + standardExternalLinkConversion, + ], [ { type: "callout", diff --git a/packages/notion-downloader/src/plugins/ColumnTransformer.spec.ts b/packages/notion-downloader/src/plugins/ColumnTransformer.spec.ts index ebc14be..6e7d83c 100644 --- a/packages/notion-downloader/src/plugins/ColumnTransformer.spec.ts +++ b/packages/notion-downloader/src/plugins/ColumnTransformer.spec.ts @@ -28,7 +28,7 @@ const columnBlock = { } as unknown as NotionBlock async function getResults(children: NotionBlock[]) { return await blocksToMarkdown( - { plugins: [standardColumnTransformer] }, + [standardColumnTransformer], [columnBlock], undefined, children, diff --git a/packages/notion-downloader/src/plugins/EscapeHtmlBlockModifier.spec.ts b/packages/notion-downloader/src/plugins/EscapeHtmlBlockModifier.spec.ts index 82be385..cfd48f0 100644 --- a/packages/notion-downloader/src/plugins/EscapeHtmlBlockModifier.spec.ts +++ b/packages/notion-downloader/src/plugins/EscapeHtmlBlockModifier.spec.ts @@ -113,8 +113,8 @@ beforeEach(() => { }) test("smoketest ", async () => { - const config = { plugins: [standardEscapeHtmlBlockModifier] } - let results = await blocksToMarkdown(config, blocks) + const plugins = [standardEscapeHtmlBlockModifier] + let results = await blocksToMarkdown(plugins, blocks) // shouldn't escape inside a code block expect(results).toContain("This is code: if(1 < 3)") // should escape outside a code block diff --git a/packages/notion-downloader/src/plugins/HeadingTranformer.spec.ts b/packages/notion-downloader/src/plugins/HeadingTranformer.spec.ts index 883f332..92519c9 100644 --- a/packages/notion-downloader/src/plugins/HeadingTranformer.spec.ts +++ b/packages/notion-downloader/src/plugins/HeadingTranformer.spec.ts @@ -7,8 +7,8 @@ import { blocksToMarkdown } from "./pluginTestRun" test("Adds anchor to headings", async () => { //setLogLevel("verbose"); const headingBlockId = "86f746f4-1c79-4ba1-a2f6-a1d59c2f9d23" - const config = { plugins: [standardHeadingTransformer] } - const result = await blocksToMarkdown(config, [ + const plugins = [standardHeadingTransformer] + const result = await blocksToMarkdown(plugins, [ { object: "block", id: headingBlockId, diff --git a/packages/notion-downloader/src/plugins/VideoTransformer.spec.ts b/packages/notion-downloader/src/plugins/VideoTransformer.spec.ts index c5c2238..444f709 100644 --- a/packages/notion-downloader/src/plugins/VideoTransformer.spec.ts +++ b/packages/notion-downloader/src/plugins/VideoTransformer.spec.ts @@ -6,8 +6,8 @@ import { standardVideoTransformer } from "./VideoTransformer" import { blocksToMarkdown } from "./pluginTestRun" test("youtube embedded", async () => { - const config = { plugins: [standardVideoTransformer] } - const result = await blocksToMarkdown(config, [ + const plugins = [standardVideoTransformer] + const result = await blocksToMarkdown(plugins, [ { object: "block", type: "video", @@ -36,8 +36,8 @@ test("youtube embedded", async () => { test("vimeo embedded", async () => { setLogLevel("verbose") - const config = { plugins: [standardVideoTransformer] } - const result = await blocksToMarkdown(config, [ + const plugins = [standardVideoTransformer] + const result = await blocksToMarkdown(plugins, [ { object: "block", type: "video", @@ -56,8 +56,8 @@ test("vimeo embedded", async () => { test("video link, not embedded", async () => { setLogLevel("verbose") - const config = { plugins: [standardVideoTransformer] } - const result = await blocksToMarkdown(config, [ + const plugins = [standardVideoTransformer] + const result = await blocksToMarkdown(plugins, [ { object: "block", type: "paragraph", @@ -90,8 +90,8 @@ test("video link, not embedded", async () => { test("direct upload to to Notion (embedded)", async () => { setLogLevel("verbose") - const config = { plugins: [standardVideoTransformer] } - const result = await blocksToMarkdown(config, [ + const plugins = [standardVideoTransformer] + const result = await blocksToMarkdown(plugins, [ { object: "block", id: "12f7db3b-4412-4be9-a3f7-6ac423fee94a", diff --git a/packages/notion-downloader/src/plugins/embedTweaks.spec.ts b/packages/notion-downloader/src/plugins/embedTweaks.spec.ts index e355d21..4e437dc 100644 --- a/packages/notion-downloader/src/plugins/embedTweaks.spec.ts +++ b/packages/notion-downloader/src/plugins/embedTweaks.spec.ts @@ -8,8 +8,8 @@ import { IPlugin } from "./pluginTypes" test("imgur", async () => { setLogLevel("verbose") - const config = { plugins: [imgurGifEmbed] } - const result = await blocksToMarkdown(config, [ + const plugins = [imgurGifEmbed] + const result = await blocksToMarkdown(plugins, [ { object: "block", id: "e36710d8-98ad-40dc-b41b-b376ebdd6894", @@ -22,8 +22,8 @@ test("imgur", async () => { test("gif", async () => { setLogLevel("verbose") - const config = { plugins: [gifEmbed] } - const result = await blocksToMarkdown(config, [ + const plugins = [gifEmbed] + const result = await blocksToMarkdown(plugins, [ { object: "block", id: "e36710d8-98ad-40dc-b41b-b376ebdd6894", @@ -50,8 +50,8 @@ test("tweaks are not applied inside code blocks", async () => { }, ], } - const config = { plugins: [p] } - const result = await blocksToMarkdown(config, [ + const plugins = [p] + const result = await blocksToMarkdown(plugins, [ { type: "code", code: { @@ -117,8 +117,8 @@ test("simplest possible", async () => { }, ], } - const config = { plugins: [p] } - const result = await blocksToMarkdown(config, [ + const plugins = [p] + const result = await blocksToMarkdown(plugins, [ { type: "paragraph", paragraph: { @@ -156,8 +156,8 @@ test("use match in output", async () => { }, ], } - const config = { plugins: [p] } - const result = await blocksToMarkdown(config, [ + const plugins = [p] + const result = await blocksToMarkdown(plugins, [ { type: "paragraph", paragraph: { diff --git a/packages/notion-downloader/src/plugins/externalLinks.spec.ts b/packages/notion-downloader/src/plugins/externalLinks.spec.ts index 08d9efd..1368d83 100644 --- a/packages/notion-downloader/src/plugins/externalLinks.spec.ts +++ b/packages/notion-downloader/src/plugins/externalLinks.spec.ts @@ -136,8 +136,5 @@ test("inline links to external site", async () => { }) async function getMarkdown(block: Record) { - const config = { - plugins: [standardExternalLinkConversion], - } - return await oneBlockToMarkdown(config, block) + return await oneBlockToMarkdown([standardExternalLinkConversion], block) } diff --git a/packages/notion-downloader/src/plugins/internalLinks.spec.ts b/packages/notion-downloader/src/plugins/internalLinks.spec.ts index 4a04e3a..778979d 100644 --- a/packages/notion-downloader/src/plugins/internalLinks.spec.ts +++ b/packages/notion-downloader/src/plugins/internalLinks.spec.ts @@ -739,12 +739,14 @@ async function getMarkdown( targetPage?: NotionPage, targetPage2?: NotionPage ) { - const config = { - plugins: [ + return await oneBlockToMarkdown( + [ standardCalloutTransformer, standardInternalLinkConversion, standardExternalLinkConversion, ], - } - return await oneBlockToMarkdown(config, block, targetPage, targetPage2) + block, + targetPage, + targetPage2 + ) } diff --git a/packages/notion-downloader/src/plugins/internalLinks.ts b/packages/notion-downloader/src/plugins/internalLinks.ts index f5562a3..fc2775e 100644 --- a/packages/notion-downloader/src/plugins/internalLinks.ts +++ b/packages/notion-downloader/src/plugins/internalLinks.ts @@ -99,9 +99,6 @@ function convertLinkHref( convertedLink = removePathExtension(convertedLink) } - // Encode spaces in the path - convertedLink = convertedLink.split("/").map(encodeURIComponent).join("/") - /***************************** NOTE: as of this writing, the official Notion API completely drops links to headings, unless they are part of a inline link. diff --git a/packages/notion-downloader/src/plugins/mermaidLinkPlugin.spec.ts b/packages/notion-downloader/src/plugins/mermaidLinkPlugin.spec.ts index 09c2086..41563bf 100644 --- a/packages/notion-downloader/src/plugins/mermaidLinkPlugin.spec.ts +++ b/packages/notion-downloader/src/plugins/mermaidLinkPlugin.spec.ts @@ -65,13 +65,14 @@ test("raw url inside a mermaid codeblock gets converted to path using slug of th ], } - const config = { - plugins: [ + const results = await oneBlockToMarkdown( + [ standardInternalLinkConversion, standardExternalLinkConversion, mermaidLinks, ], - } - const results = await oneBlockToMarkdown(config, input, targetPage) + input, + targetPage + ) expect(results.trim()).toContain(`click A "/slug-of-target"`) }) diff --git a/packages/notion-downloader/src/plugins/pluginTestRun.ts b/packages/notion-downloader/src/plugins/pluginTestRun.ts index 20888ce..7241571 100644 --- a/packages/notion-downloader/src/plugins/pluginTestRun.ts +++ b/packages/notion-downloader/src/plugins/pluginTestRun.ts @@ -2,7 +2,6 @@ import { Client } from "@notionhq/client" import { PageObjectResponse } from "@notionhq/client/build/src/api-endpoints" import { NotionToMarkdown } from "notion-to-md" -import { IPluginsConfig } from "../config/plugins" import { defaultPullOptions, parsePathFileOptions } from "../config/schema" import { FilesManager } from "../files/FilesManager" import { FilesMap } from "../files/FilesMap" @@ -13,10 +12,10 @@ import { NotionPage } from "../notionObjects/NotionPage" import { getMarkdownFromNotionBlocks } from "../transformMarkdown" import { NotionBlock } from "../types" import { convertInternalUrl } from "./internalLinks" -import { IPluginContext } from "./pluginTypes" +import { IPlugin, IPluginContext } from "./pluginTypes" export async function blocksToMarkdown( - config: IPluginsConfig, + plugins: IPlugin[], blocks: NotionBlock[], pages?: NotionPage[], // Notes on children: @@ -108,7 +107,7 @@ export async function blocksToMarkdown( // console.log(pages[0].matchesLinkId); // console.log(pluginContext.pages[0].matchesLinkId); } - const r = await getMarkdownFromNotionBlocks(pluginContext, config, blocks) + const r = await getMarkdownFromNotionBlocks(pluginContext, plugins, blocks) //console.log("blocksToMarkdown", r); return r } @@ -243,7 +242,7 @@ export function makeSamplePageObject(options: { } export async function oneBlockToMarkdown( - config: IPluginsConfig, + plugins: IPlugin[], block: Record, targetPage?: NotionPage, targetPage2?: NotionPage @@ -283,7 +282,7 @@ export async function oneBlockToMarkdown( name: "Dummy2", }) return await blocksToMarkdown( - config, + plugins, [fullBlock as NotionBlock], targetPage ? [dummyPage1, targetPage, targetPage2 ?? dummyPage2] : undefined ) diff --git a/packages/notion-downloader/src/transformMarkdown.ts b/packages/notion-downloader/src/transformMarkdown.ts index 1916fed..c74e434 100644 --- a/packages/notion-downloader/src/transformMarkdown.ts +++ b/packages/notion-downloader/src/transformMarkdown.ts @@ -1,6 +1,5 @@ import chalk from "chalk" -import { IPluginsConfig } from "./config/plugins" import { error, info, logDebug, logDebugFn, verbose, warning } from "./log" import { getFileUrl } from "./notionObjects/NotionFile" import { NotionPage } from "./notionObjects/NotionPage" diff --git a/packages/notion-downloader/src/utils/get-package-info.ts b/packages/notion-downloader/src/utils/get-package-info.ts index dd3d156..426f854 100644 --- a/packages/notion-downloader/src/utils/get-package-info.ts +++ b/packages/notion-downloader/src/utils/get-package-info.ts @@ -4,17 +4,11 @@ import fs from "fs-extra" import { type PackageJson } from "type-fest" export function getPackageInfo() { - let packageJsonPath: string - if (typeof __dirname !== "undefined") { - // CommonJS - packageJsonPath = path.resolve(__dirname, "..", "package.json") - } else { - // ESM - // TODO: Fix, this gives a warning for CJS. Maybe its a sign to drop support for CJS? - const __filename = fileURLToPath(import.meta.url) - const __dirname = path.dirname(__filename) - packageJsonPath = path.resolve(__dirname, "..", "..", "package.json") - } + // TODO Maybe drop CJS support + // ESM + const __filename = fileURLToPath(import.meta.url) + const __dirname = path.dirname(__filename) + const packageJsonPath = path.resolve(__dirname, "..", "package.json") return fs.readJSONSync(packageJsonPath) as PackageJson } diff --git a/packages/notion-downloader/test/markdownPathUtils.test.ts b/packages/notion-downloader/test/markdownPathUtils.test.ts index 00c6b52..37c6b0e 100644 --- a/packages/notion-downloader/test/markdownPathUtils.test.ts +++ b/packages/notion-downloader/test/markdownPathUtils.test.ts @@ -67,5 +67,14 @@ describe("markdownPathUtils", () => { "[Link Text](path/to/file%20with%20spaces.md#section-id)" ) }) + test("Links with sapces in different parts of the path", () => { + const convertedPath = convertMarkdownPath( + "folder with spaces/to/file with spaces.md" + ) + const markdownLink = `[Link Text](${convertedPath}#section-id)` + expect(markdownLink).toBe( + "[Link Text](folder%20with%20spaces/to/file%20with%20spaces.md#section-id)" + ) + }) }) }) diff --git a/packages/notion-downloader/test/utils/get-config.test.ts b/packages/notion-downloader/test/utils/get-config.test.ts index 6ba73c0..f9f618f 100644 --- a/packages/notion-downloader/test/utils/get-config.test.ts +++ b/packages/notion-downloader/test/utils/get-config.test.ts @@ -55,6 +55,7 @@ test("get config", async () => { markdownPrefixes: "", layoutStrategy: "hierarchical", namingStrategy: "default", + plugins: [], }, }) })