Merge main into release

README.md

@@ -31,29 +31,29 @@ Currently, docu-notion expects that each page has only one of the following: sub
 
 ## 6. Pull your pages
 
-First, determine the id of your root page by clicking "Share" and looking at the the url it gives you. E.g.
+First, determine the id of your root page by clicking "Share" and looking at the url it gives you. E.g.
 https://www.notion.so/hattonjohn/My-Docs-0456aa5842946bdbea3a4f37c97a0e5
 means that the id is "0456aa5842946PRETEND4f37c97a0e5".
 
 Determine where you want the markdown files and images to land. The following works well for Docusaurus instances:
 
 ```
-npx docu-notion -n secret_PRETEND123456789PRETEND123456789PRETEND6789 -r 0456aa5842946PRETEND4f37c97a0e5"
+npx @sillsdev/docu-notion -n secret_PRETEND123456789PRETEND123456789PRETEND6789 -r 0456aa5842946PRETEND4f37c97a0e5"
 ```
 
 Likely, you will want to store these codes in your environment variables and then use them like this:
 
 ```
 (windows)
-npx docu-notion -n %MY_NOTION_TOKEN% -r %MY_NOTION_DOCS_ROOT_PAGE_ID%
+npx @sillsdev/docu-notion -n %MY_NOTION_TOKEN% -r %MY_NOTION_DOCS_ROOT_PAGE_ID%
 ```
 
 ```
 (linux / mac)
-npx docu-notion -n $MY_NOTION_TOKEN -r $MY_NOTION_DOCS_ROOT_PAGE_ID
+npx @sillsdev/docu-notion -n $MY_NOTION_TOKEN -r $MY_NOTION_DOCS_ROOT_PAGE_ID
 ```
 
-NOTE: In the above, we are using `npx` to use the latest `docu-notion`. A more conservative approach would be to `npm i cross-var docu-notion` and then create a script in your package.json like this:
+NOTE: In the above, we are using `npx` to use the latest `docu-notion`. A more conservative approach would be to `npm i cross-var @sillsdev/docu-notion` and then create a script in your package.json like this:
 
 ```
  "scripts": {
@@ -133,3 +133,16 @@ Options:
 # Plugins
 
 If your project needs some processing that docu-notion doesn't already provide, you can provide a plugin that does it. See the [plugin readme](src/plugins/README.md).
+
+# Callouts ➜ Admonitions
+
+To map Notion callouts to Docusaurus admonitions, ensure the icon is for the type you want.
+
+- ℹ️ ➜ note
+- 📝➜ note
+- 💡➜ tip
+- ❗➜ info
+- ⚠️➜ caution
+- 🔥➜ danger
+
+The default admonition type, if no matching icon is found, is "note".

package.json

@@ -11,6 +11,7 @@
     "// typescript check": "",
     "tsc": "tsc",
     "// test out with a private sample notion db": "",
+    "large-site-test": "npm run ts -- -n $SIL_BLOOM_DOCS_NOTION_TOKEN -r $SIL_BLOOM_DOCS_NOTION_ROOT_PAGE  --locales en,fr",
     "pull-test-tagged": "npm run ts -- -n $DOCU_NOTION_INTEGRATION_TOKEN -r $DOCU_NOTION_TEST_ROOT_PAGE_ID  --log-level debug --status-tag test",
     "pull-sample-site": "npm run ts -- -n $DOCU_NOTION_INTEGRATION_TOKEN -r $DOCU_NOTION_SAMPLE_ROOT_PAGE  --log-level debug",
     "// test with a semi-stable/public site:": "",

src/plugins/ColumnTransformer.ts

@@ -2,6 +2,7 @@ import { NotionAPI } from "notion-client";
 import { NotionToMarkdown } from "notion-to-md";
 import { ListBlockChildrenResponseResult } from "notion-to-md/build/types";
 import { IGetBlockChildrenFn, IPlugin } from "./pluginTypes";
+import { executeWithRateLimitAndRetries } from "../pull";
 
 export const standardColumnTransformer: IPlugin = {
   name: "standardColumnTransformer",
@@ -61,8 +62,13 @@ async function getColumnWidth(
 ): Promise<string> {
   const unofficialNotionClient = new NotionAPI();
   const blockId = block.id;
-  // Yes, it is odd to call 'getPage' for a block, but that's how we access the format info.
-  const recordMap = await unofficialNotionClient.getPage(blockId);
+  const recordMap = await executeWithRateLimitAndRetries(
+    `unofficialNotionClient.getPage(${blockId}) in getColumnWidth()`,
+    () => {
+      // Yes, it is odd to call 'getPage' for a block, but that's how we access the format info.
+      return unofficialNotionClient.getPage(blockId);
+    }
+  );
   const blockResult = recordMap.block[blockId];
 
   // ENHANCE: could we use https://github.com/NotionX/react-notion-x/tree/master/packages/notion-types

src/plugins/internalLinks.spec.ts

@@ -374,6 +374,36 @@ test("does not interfere with mailto links", async () => {
   });
   expect(results.trim()).toBe(`[mailme](mailto:[email protected])`);
 });
+
+test("does not interfere with https links", async () => {
+  const results = await getMarkdown({
+    type: "paragraph",
+    paragraph: {
+      rich_text: [
+        {
+          type: "text",
+          text: {
+            content: "google",
+            link: { url: `https://www.google.com` },
+          },
+          annotations: {
+            bold: false,
+            italic: false,
+            strikethrough: false,
+            underline: false,
+            code: false,
+            color: "default",
+          },
+          plain_text: "google",
+          href: `https://www.google.com`,
+        },
+      ],
+      color: "default",
+    },
+  });
+  expect(results.trim()).toBe(`[google](https://www.google.com)`);
+});
+
 test("links to other notion pages that are not in this site give PROBLEM LINK", async () => {
   const results = await getMarkdown({
     type: "paragraph",

src/pull.ts

@@ -256,11 +256,49 @@ const notionLimiter = new RateLimiter({
 let notionClient: Client;
 
 async function getPageMetadata(id: string): Promise<GetPageResponse> {
+  return await executeWithRateLimitAndRetries(`pages.retrieve(${id})`, () => {
+    return notionClient.pages.retrieve({
+      page_id: id,
+    });
+  });
+}
+
+// While everything works fine locally, on Github Actions we are getting a lot of timeouts, so
+// we're trying this extra retry-able wrapper.
+export async function executeWithRateLimitAndRetries<T>(
+  label: string,
+  asyncFunction: () => Promise<T>
+): Promise<T> {
   await rateLimit();
+  const kRetries = 10;
+  let lastException = undefined;
+  for (let i = 0; i < kRetries; i++) {
+    try {
+      return await asyncFunction();
+    } catch (e: any) {
+      lastException = e;
+      if (
+        e?.code === "notionhq_client_request_timeout" ||
+        e.message.includes("timeout") ||
+        e.message.includes("Timeout") ||
+        e.message.includes("limit") ||
+        e.message.includes("Limit")
+      ) {
+        const secondsToWait = i + 1;
+        info(
+          `While doing "${label}", got error "${
+            e.message as string
+          }". Will retry after  ${secondsToWait}s...`
+        );
+        await new Promise(resolve => setTimeout(resolve, 1000 * secondsToWait));
+      } else {
+        throw e;
+      }
+    }
+  }
 
-  return await notionClient.pages.retrieve({
-    page_id: id,
-  });
+  error(`Error: could not complete "${label}" after ${kRetries} retries.`);
+  throw lastException;
 }
 
 async function rateLimit() {
@@ -275,18 +313,19 @@ async function getBlockChildren(id: string): Promise<NotionBlock[]> {
   // the first response we get, then keep adding to its array of blocks
   // with each subsequent response
   let overallResult: ListBlockChildrenResponse | undefined = undefined;
-  let start_cursor = undefined;
+  let start_cursor: string | undefined | null = undefined;
 
   // Note: there is a now a collectPaginatedAPI() in the notion client, so
   // we could switch to using that (I don't know if it does rate limiting?)
   do {
-    await rateLimit();
-
     const response: ListBlockChildrenResponse =
-      await notionClient.blocks.children.list({
-        start_cursor: start_cursor,
-        block_id: id,
+      await executeWithRateLimitAndRetries(`getBlockChildren(${id})`, () => {
+        return notionClient.blocks.children.list({
+          start_cursor: start_cursor as string | undefined,
+          block_id: id,
+        });
       });
+
     if (!overallResult) {
       overallResult = response;
     } else {

src/transform.ts

@@ -133,7 +133,6 @@ async function doTransformsOnMarkdown(
             matchAsThePluginWouldExpectIt
           );
         } else if (mod.replacementPattern) {
-          console.log(`mod.replacementPattern.replace("$1", ${match[2]}`);
           replacement = mod.replacementPattern.replace("$1", match[2]);
         }
         if (replacement !== undefined) {