fix(ai): only index nodes with unique canonical slugs (#1991)

fern-api · Jan 13, 2025 · f39bf28 · f39bf28
1 parent 54d214c
commit f39bf28
Show file tree

Hide file tree

Showing 2 changed files with 25 additions and 8 deletions.
diff --git a/packages/fdr-sdk/src/navigation/NodeCollector.ts b/packages/fdr-sdk/src/navigation/NodeCollector.ts
@@ -2,6 +2,7 @@ import { EMPTY_ARRAY } from "@fern-api/ui-core-utils";
 import { once } from "es-toolkit/function";
 import { FernNavigation } from "./..";
 import { pruneVersionNode } from "./utils/pruneVersionNode";
+import { NavigationNodeWithMetadata } from "./versions";
 
 interface NavigationNodeWithMetadataAndParents {
   node: FernNavigation.NavigationNodeWithMetadata;
@@ -134,7 +135,6 @@ export class NodeCollector {
       this.#setNode(node.slug, node, parents);
     } else {
       if (FernNavigation.isPage(existing.node)) {
-        // eslint-disable-next-line no-console
         console.warn(`Duplicate slug found: ${node.slug}`, node.title);
       }
       this.orphanedNodes.push(node);
@@ -236,6 +236,29 @@ export class NodeCollector {
     return this.#getIndexablePageSlugs();
   }
 
+  #getIndexablePageNodesWithAuth = once((): NavigationNodeWithMetadata[] => {
+    const slugRecord: Record<string, NavigationNodeWithMetadata> = {};
+
+    [...this.slugToNode.values()]
+      .filter(({ node }) => FernNavigation.isPage(node))
+      .filter(({ node }) => !node.hidden)
+      .filter(({ node }) =>
+        FernNavigation.hasMarkdown(node) ? !node.noindex : true
+      )
+      .forEach((node) => {
+        const canonicalSlug = node.node.canonicalSlug ?? node.node.slug;
+        // Only keep the first node we see for each canonical slug
+        if (!(canonicalSlug in slugRecord)) {
+          slugRecord[canonicalSlug] = node.node;
+        }
+      });
+
+    return Object.values(slugRecord);
+  });
+  get indexablePageNodesWithAuth(): NavigationNodeWithMetadata[] {
+    return this.#getIndexablePageNodesWithAuth();
+  }
+
   public getVersionNodes = (): FernNavigation.VersionNode[] => {
     return this.versionNodes;
   };

diff --git a/packages/fern-docs/search-server/src/turbopuffer/records/create-turbopuffer-records.ts b/packages/fern-docs/search-server/src/turbopuffer/records/create-turbopuffer-records.ts
@@ -5,7 +5,6 @@ import {
   RootNode,
   getPageId,
   hasMarkdown,
-  isPage,
 } from "@fern-api/fdr-sdk/navigation";
 import { flatten } from "es-toolkit/array";
 import { FernTurbopufferRecordWithoutVector } from "../types";
@@ -35,12 +34,7 @@ export async function createTurbopufferRecords({
 > {
   const collector = NodeCollector.collect(root);
 
-  const pageNodes = Array.from(collector.slugMap.values())
-    .filter(isPage)
-    // exclude hidden pages
-    .filter((node) => node.hidden !== true)
-    // exclude pages that are noindexed
-    .filter((node) => (hasMarkdown(node) ? node.noindex !== true : true));
+  const pageNodes = collector.indexablePageNodesWithAuth;
 
   const markdownNodes = pageNodes.filter(hasMarkdown);
   // const apiLeafNodes = pageNodes.filter(isApiLeaf);