Skip to content

Commit 12db481

Browse files
StrongMonkeyStrongMonkey
committed
Include js files
Signed-off-by: Daishan Peng <[email protected]>
1 parent d9ae701 commit 12db481

File tree

5 files changed

+961
-621
lines changed

5 files changed

+961
-621
lines changed

‎.gitignore

Lines changed: 0 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -1,3 +1,2 @@
11
node_modules
2-
dist
32
.env

‎dist/index.js

Lines changed: 145 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,145 @@
1+
"use strict";
2+
var __createBinding = (this && this.__createBinding) || (Object.create ? (function(o, m, k, k2) {
3+
if (k2 === undefined) k2 = k;
4+
var desc = Object.getOwnPropertyDescriptor(m, k);
5+
if (!desc || ("get" in desc ? !m.__esModule : desc.writable || desc.configurable)) {
6+
desc = { enumerable: true, get: function() { return m[k]; } };
7+
}
8+
Object.defineProperty(o, k2, desc);
9+
}) : (function(o, m, k, k2) {
10+
if (k2 === undefined) k2 = k;
11+
o[k2] = m[k];
12+
}));
13+
var __setModuleDefault = (this && this.__setModuleDefault) || (Object.create ? (function(o, v) {
14+
Object.defineProperty(o, "default", { enumerable: true, value: v });
15+
}) : function(o, v) {
16+
o["default"] = v;
17+
});
18+
var __importStar = (this && this.__importStar) || function (mod) {
19+
if (mod && mod.__esModule) return mod;
20+
var result = {};
21+
if (mod != null) for (var k in mod) if (k !== "default" && Object.prototype.hasOwnProperty.call(mod, k)) __createBinding(result, mod, k);
22+
__setModuleDefault(result, mod);
23+
return result;
24+
};
25+
var __awaiter = (this && this.__awaiter) || function (thisArg, _arguments, P, generator) {
26+
function adopt(value) { return value instanceof P ? value : new P(function (resolve) { resolve(value); }); }
27+
return new (P || (P = Promise))(function (resolve, reject) {
28+
function fulfilled(value) { try { step(generator.next(value)); } catch (e) { reject(e); } }
29+
function rejected(value) { try { step(generator["throw"](value)); } catch (e) { reject(e); } }
30+
function step(result) { result.done ? resolve(result.value) : adopt(result.value).then(fulfilled, rejected); }
31+
step((generator = generator.apply(thisArg, _arguments || [])).next());
32+
});
33+
};
34+
var __importDefault = (this && this.__importDefault) || function (mod) {
35+
return (mod && mod.__esModule) ? mod : { "default": mod };
36+
};
37+
Object.defineProperty(exports, "__esModule", { value: true });
38+
const client_1 = require("@notionhq/client");
39+
const dotenv_1 = __importDefault(require("dotenv"));
40+
const promises_1 = require("fs/promises");
41+
const path_1 = __importDefault(require("path"));
42+
const page_1 = require("./page");
43+
const fs = __importStar(require("node:fs"));
44+
dotenv_1.default.config();
45+
function main() {
46+
var _a;
47+
return __awaiter(this, void 0, void 0, function* () {
48+
const notion = new client_1.Client({
49+
auth: process.env.NOTION_TOKEN,
50+
});
51+
// Function to write a page to a file
52+
function writePageToFile(page, directory) {
53+
return __awaiter(this, void 0, void 0, function* () {
54+
const pageId = page.id;
55+
const pageContent = yield (0, page_1.getPageContent)(notion, pageId);
56+
const fileDir = path_1.default.join(directory, pageId.toString());
57+
yield (0, promises_1.mkdir)(fileDir, { recursive: true });
58+
const filePath = getPath(directory, page);
59+
fs.writeFileSync(filePath, pageContent, "utf8");
60+
});
61+
}
62+
function getPath(directory, page) {
63+
var _a, _b, _c, _d, _e, _f;
64+
const pageId = page.id;
65+
const fileDir = path_1.default.join(directory, pageId.toString());
66+
let title = (_f = (_e = (_d = (((_b = (_a = page.properties) === null || _a === void 0 ? void 0 : _a.title) !== null && _b !== void 0 ? _b : (_c = page.properties) === null || _c === void 0 ? void 0 : _c.Name))) === null || _d === void 0 ? void 0 : _d.title[0]) === null || _e === void 0 ? void 0 : _e.plain_text) === null || _f === void 0 ? void 0 : _f.trim().replaceAll(/\//g, "-");
67+
if (!title) {
68+
title = pageId.toString();
69+
}
70+
return path_1.default.join(fileDir, title + ".md");
71+
}
72+
// Function to fetch all pages
73+
function fetchAllPages() {
74+
var _a;
75+
return __awaiter(this, void 0, void 0, function* () {
76+
let pages = [];
77+
let cursor = undefined;
78+
while (true) {
79+
const response = yield notion.search({
80+
filter: {
81+
property: "object",
82+
value: "page",
83+
},
84+
start_cursor: cursor,
85+
});
86+
pages = pages.concat(response.results);
87+
if (!response.has_more) {
88+
break;
89+
}
90+
cursor = (_a = response.next_cursor) !== null && _a !== void 0 ? _a : undefined;
91+
}
92+
return pages;
93+
});
94+
}
95+
// Fetch all pages
96+
const pages = yield fetchAllPages();
97+
let metadata = new Map();
98+
const outputDir = path_1.default.join(process.env.WORKSPACE_DIR, 'knowledge', 'integrations', 'notion');
99+
yield (0, promises_1.mkdir)(outputDir, { recursive: true });
100+
const metadataPath = path_1.default.join(outputDir, 'metadata.json');
101+
if (fs.existsSync(metadataPath)) {
102+
metadata = new Map(Object.entries(JSON.parse(fs.readFileSync(metadataPath, 'utf8').toString())));
103+
}
104+
let updatedPages = 0;
105+
for (const page of pages) {
106+
if (metadata.has(page.id)) {
107+
const entry = metadata.get(page.id);
108+
if ((entry === null || entry === void 0 ? void 0 : entry.updatedAt) === page.last_edited_time && fs.existsSync(getPath(outputDir, page))) {
109+
continue;
110+
}
111+
if (entry === null || entry === void 0 ? void 0 : entry.sync) {
112+
updatedPages++;
113+
yield writePageToFile(page, outputDir);
114+
}
115+
metadata.set(page.id, {
116+
url: page.url,
117+
filename: path_1.default.basename(getPath(outputDir, page)),
118+
updatedAt: page.last_edited_time,
119+
sync: (_a = entry === null || entry === void 0 ? void 0 : entry.sync) !== null && _a !== void 0 ? _a : false,
120+
});
121+
}
122+
metadata.set(page.id, {
123+
url: page.url,
124+
filename: path_1.default.basename(getPath(outputDir, page)),
125+
updatedAt: page.last_edited_time,
126+
sync: false,
127+
});
128+
}
129+
for (const [key, _] of metadata) {
130+
if (!pages.find((page) => page.id === key)) {
131+
fs.rmSync(path_1.default.join(outputDir, key), { recursive: true });
132+
console.log(`Removed ${key} from ${outputDir}`);
133+
metadata.delete(key);
134+
}
135+
}
136+
yield (0, promises_1.writeFile)(metadataPath, JSON.stringify(Object.fromEntries(metadata)), 'utf8');
137+
console.log(`Finished writing ${updatedPages} pages to ${outputDir}`);
138+
});
139+
}
140+
main()
141+
.then(() => process.exit(0))
142+
.catch((err) => {
143+
console.error(err);
144+
process.exit(1);
145+
});

‎dist/page.js

Lines changed: 197 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,197 @@
1+
"use strict";
2+
var __awaiter = (this && this.__awaiter) || function (thisArg, _arguments, P, generator) {
3+
function adopt(value) { return value instanceof P ? value : new P(function (resolve) { resolve(value); }); }
4+
return new (P || (P = Promise))(function (resolve, reject) {
5+
function fulfilled(value) { try { step(generator.next(value)); } catch (e) { reject(e); } }
6+
function rejected(value) { try { step(generator["throw"](value)); } catch (e) { reject(e); } }
7+
function step(result) { result.done ? resolve(result.value) : adopt(result.value).then(fulfilled, rejected); }
8+
step((generator = generator.apply(thisArg, _arguments || [])).next());
9+
});
10+
};
11+
Object.defineProperty(exports, "__esModule", { value: true });
12+
exports.richTextArrayToString = exports.getPageContent = void 0;
13+
function getPageContent(client, id, indentation = 0) {
14+
return __awaiter(this, void 0, void 0, function* () {
15+
const blocks = yield client.blocks.children.list({ block_id: id });
16+
let result = '';
17+
for (let b of blocks.results) {
18+
let block = b;
19+
// Tables are complicated, so we handle them completely separately
20+
if (block.type === "table") {
21+
result += yield printTable(client, b);
22+
continue;
23+
}
24+
result += yield printBlock(client, b, indentation);
25+
if (block.has_children && block.type !== "child_page" && block.type !== "synced_block") {
26+
result += yield getPageContent(client, b.id, indentation + 2);
27+
}
28+
}
29+
return result;
30+
});
31+
}
32+
exports.getPageContent = getPageContent;
33+
function printBlock(client, b, indentation) {
34+
return __awaiter(this, void 0, void 0, function* () {
35+
let result = "";
36+
if (indentation > 0) {
37+
result += " ".repeat(indentation);
38+
}
39+
switch (b.type) {
40+
case "bookmark":
41+
if (b.bookmark.caption !== null && richTextArrayToString(b.bookmark.caption) !== "") {
42+
result += `Bookmark: ${b.bookmark.url} (${richTextArrayToString(b.bookmark.caption)})`;
43+
}
44+
else {
45+
result += `Bookmark: ${b.bookmark.url}`;
46+
}
47+
break;
48+
case "bulleted_list_item":
49+
result += `- ${richTextArrayToString(b.bulleted_list_item.rich_text)}`;
50+
break;
51+
case "callout":
52+
result += `> ${richTextArrayToString(b.callout.rich_text)}`;
53+
break;
54+
case "child_database":
55+
result += `Child Database: ${b.child_database.title}`;
56+
break;
57+
case "child_page":
58+
result += `Child Page: ${b.child_page.title}`;
59+
break;
60+
case "code":
61+
if (b.code.language !== null) {
62+
result += "```" + b.code.language + "\n";
63+
}
64+
else {
65+
result += "```\n";
66+
}
67+
result += richTextArrayToString(b.code.rich_text);
68+
result += "\n```";
69+
if (b.code.caption !== null && richTextArrayToString(b.code.caption) !== "") {
70+
result += `\n(${richTextArrayToString(b.code.caption)})`;
71+
}
72+
break;
73+
case "divider":
74+
result += "-------------------------------------";
75+
break;
76+
case "embed":
77+
result += `Embed: ${b.embed.url}`;
78+
break;
79+
case "equation":
80+
result += `Equation: ${b.equation.expression}`;
81+
break;
82+
case "file":
83+
result += fileToString("File", b.file);
84+
break;
85+
case "heading_1":
86+
result += `# ${richTextArrayToString(b.heading_1.rich_text)}`;
87+
break;
88+
case "heading_2":
89+
result += `## ${richTextArrayToString(b.heading_2.rich_text)}`;
90+
break;
91+
case "heading_3":
92+
result += `### ${richTextArrayToString(b.heading_3.rich_text)}`;
93+
break;
94+
case "image":
95+
result += fileToString("Image", b.image);
96+
break;
97+
case "link_preview":
98+
result += b.link_preview.url;
99+
break;
100+
case "numbered_list_item":
101+
result += `1. ${richTextArrayToString(b.numbered_list_item.rich_text)}`;
102+
break;
103+
case "paragraph":
104+
result += richTextArrayToString(b.paragraph.rich_text);
105+
break;
106+
case "pdf":
107+
result += fileToString("PDF", b.pdf);
108+
break;
109+
case "quote":
110+
result += "\"\"\"\n";
111+
result += richTextArrayToString(b.quote.rich_text);
112+
result += "\n\"\"\"";
113+
break;
114+
case "synced_block":
115+
if (b.synced_block.synced_from !== null) {
116+
yield getPageContent(client, b.synced_block.synced_from.block_id, indentation);
117+
}
118+
break;
119+
case "to_do":
120+
if (b.to_do.checked) {
121+
result += `[x] ${richTextArrayToString(b.to_do.rich_text)}`;
122+
}
123+
else {
124+
result += `[ ] ${richTextArrayToString(b.to_do.rich_text)}`;
125+
}
126+
break;
127+
case "toggle":
128+
result += `> ${richTextArrayToString(b.toggle.rich_text)}`;
129+
break;
130+
case "video":
131+
result += fileToString("Video", b.video);
132+
break;
133+
}
134+
return result.replace("\n", "\n" + " ".repeat(indentation));
135+
});
136+
}
137+
function richTextArrayToString(richTextArray) {
138+
let result = "";
139+
for (let r of richTextArray) {
140+
result += r.plain_text + " ";
141+
}
142+
return result;
143+
}
144+
exports.richTextArrayToString = richTextArrayToString;
145+
function fileToString(prefix, file) {
146+
let result = "";
147+
if (file.type === "file") {
148+
result = `${prefix}: ${file.file.url} (expires ${file.file.expiry_time})`;
149+
}
150+
else if (file.type === "external") {
151+
result = `External ${prefix}: ${file.external.url}`;
152+
}
153+
if (file.caption !== null && richTextArrayToString(file.caption) !== "") {
154+
result += ` (${richTextArrayToString(file.caption)})`;
155+
}
156+
return result;
157+
}
158+
function printTable(client, table) {
159+
return __awaiter(this, void 0, void 0, function* () {
160+
let result = "";
161+
const children = yield client.blocks.children.list({ block_id: table.id });
162+
if (table.table.has_column_header && children.results.length > 0) {
163+
result += printTableRow(children.results[0].table_row, table.table.has_row_header, true);
164+
for (let i = 1; i < children.results.length; i++) {
165+
result += printTableRow(children.results[i].table_row, table.table.has_row_header, false);
166+
}
167+
}
168+
else {
169+
for (let r of children.results) {
170+
result += printTableRow(r.table_row, table.table.has_row_header, false);
171+
}
172+
}
173+
return result;
174+
});
175+
}
176+
function printTableRow(row, boldFirst, boldAll) {
177+
let result = "|";
178+
if (boldAll) {
179+
for (let c of row.cells) {
180+
result += ` **${richTextArrayToString(c)}** |`;
181+
}
182+
let len = result.length;
183+
result += "\n|" + "-".repeat(len - 2) + "|";
184+
}
185+
else if (boldFirst && row.cells.length > 0) {
186+
result += ` **${richTextArrayToString(row.cells[0])}** |`;
187+
for (let i = 1; i < row.cells.length; i++) {
188+
result += ` ${richTextArrayToString(row.cells[i])} |`;
189+
}
190+
}
191+
else {
192+
for (let c of row.cells) {
193+
result += ` ${richTextArrayToString(c)} |`;
194+
}
195+
}
196+
return result;
197+
}

0 commit comments

Comments
 (0)