From 5d00de3033c5d3a4797e66c74366c035d53fee64 Mon Sep 17 00:00:00 2001 From: Andrew Polk Date: Tue, 2 Apr 2024 09:52:23 -0700 Subject: [PATCH] fix: Download images using axios instead of node-fetch --- .gitignore | 2 ++ package-lock.json | 90 +++++++++++++++++++++++++++++++++-------------- package.json | 11 +++--- src/images.ts | 11 +++--- 4 files changed, 77 insertions(+), 37 deletions(-) diff --git a/.gitignore b/.gitignore index f0b4aec..2f13fd2 100644 --- a/.gitignore +++ b/.gitignore @@ -6,3 +6,5 @@ node_modules/ version.json docs/ *.orig + +css/docu-notion-styles.css diff --git a/package-lock.json b/package-lock.json index 6726cee..d140551 100644 --- a/package-lock.json +++ b/package-lock.json @@ -8,15 +8,15 @@ "license": "MIT", "dependencies": { "@notionhq/client": "2.2.3", + "axios": "^1.6.8", "chalk": "^4.1.2", "commander": "^9.2.0", "cosmiconfig": "^8.0.0", "cosmiconfig-typescript-loader": "^4.3.0", - "file-type": "16.5.1", + "file-type": "16.5.3", "fs-extra": "^10.1.0", "limiter": "^2.1.0", "markdown-table": "^2.0.0", - "node-fetch": "2.6.6", "notion-client": "^4", "notion-to-md": "3.1.1", "path": "^0.12.7", @@ -873,10 +873,9 @@ } }, "node_modules/@tokenizer/token": { - "version": "0.1.1", - "resolved": "https://registry.npmjs.org/@tokenizer/token/-/token-0.1.1.tgz", - "integrity": "sha512-XO6INPbZCxdprl+9qa/AAbFFOMzzwqYxpjPgLICrMD6C2FCw6qfJOPcBk6JqqPLSaZ/Qx87qn4rpPmPMwaAK6w==", - "license": "MIT" + "version": "0.3.0", + "resolved": "https://registry.npmjs.org/@tokenizer/token/-/token-0.3.0.tgz", + "integrity": "sha512-OvjF+z51L3ov0OyAU0duzsYuvO01PH7x4t6DJx+guahgTnBHkhJdG7soQeTSFLWN3efnHyibZ4Z8l2EuWwJN3A==" }, "node_modules/@tootallnate/once": { "version": "2.0.0", @@ -1511,6 +1510,29 @@ "node": ">= 4.0.0" } }, + "node_modules/axios": { + "version": "1.6.8", + "resolved": "https://registry.npmjs.org/axios/-/axios-1.6.8.tgz", + "integrity": "sha512-v/ZHtJDU39mDpyBoFVkETcd/uNdxrWRrg3bKpOKzXFA6Bvqopts6ALSMU3y6ijYxbw2B+wPrIv46egTzJXCLGQ==", + "dependencies": { + "follow-redirects": "^1.15.6", + "form-data": "^4.0.0", + "proxy-from-env": "^1.1.0" + } + }, + "node_modules/axios/node_modules/form-data": { + "version": "4.0.0", + "resolved": "https://registry.npmjs.org/form-data/-/form-data-4.0.0.tgz", + "integrity": "sha512-ETEklSGi5t0QMZuiXoA/Q6vcnxcLQP5vdugSpuAyi6SVGi2clPPp+xgEhuMaHC+zGgn31Kd235W35f7Hykkaww==", + "dependencies": { + "asynckit": "^0.4.0", + "combined-stream": "^1.0.8", + "mime-types": "^2.1.12" + }, + "engines": { + "node": ">= 6" + } + }, "node_modules/babel-code-frame": { "version": "6.26.0", "resolved": "https://registry.npmjs.org/babel-code-frame/-/babel-code-frame-6.26.0.tgz", @@ -4516,17 +4538,16 @@ } }, "node_modules/file-type": { - "version": "16.5.1", - "resolved": "https://registry.npmjs.org/file-type/-/file-type-16.5.1.tgz", - "integrity": "sha512-Pi1G43smrCy82Q3be3sfKaeS5uHdfj905dP88YqhroG6TYbVY2ljTdDXeXqa6Cn5nOk6znOjWM2uZptA8vH/qQ==", - "license": "MIT", + "version": "16.5.3", + "resolved": "https://registry.npmjs.org/file-type/-/file-type-16.5.3.tgz", + "integrity": "sha512-uVsl7iFhHSOY4bEONLlTK47iAHtNsFHWP5YE4xJfZ4rnX7S1Q3wce09XgqSC7E/xh8Ncv/be1lNoyprlUH/x6A==", "dependencies": { "readable-web-to-node-stream": "^3.0.0", - "strtok3": "^6.0.3", - "token-types": "^2.0.0" + "strtok3": "^6.2.4", + "token-types": "^4.1.1" }, "engines": { - "node": ">=8" + "node": ">=10" }, "funding": { "url": "https://github.com/sindresorhus/file-type?sponsor=1" @@ -4667,6 +4688,25 @@ "dev": true, "license": "ISC" }, + "node_modules/follow-redirects": { + "version": "1.15.6", + "resolved": "https://registry.npmjs.org/follow-redirects/-/follow-redirects-1.15.6.tgz", + "integrity": "sha512-wWN62YITEaOpSK584EZXJafH1AGpO8RVgElfkuXbTOrPX4fIfOyEpW/CsiNd8JdYrAoOvafRTOEnvsO++qCqFA==", + "funding": [ + { + "type": "individual", + "url": "https://github.com/sponsors/RubenVerborgh" + } + ], + "engines": { + "node": ">=4.0" + }, + "peerDependenciesMeta": { + "debug": { + "optional": true + } + } + }, "node_modules/form-data": { "version": "3.0.1", "resolved": "https://registry.npmjs.org/form-data/-/form-data-3.0.1.tgz", @@ -10055,7 +10095,6 @@ "version": "4.1.0", "resolved": "https://registry.npmjs.org/peek-readable/-/peek-readable-4.1.0.tgz", "integrity": "sha512-ZI3LnwUv5nOGbQzD9c2iDG6toheuXSZP5esSHBjopsXH4dg19soufvpUGA3uohi5anFtGb2lhAVdHzH6R/Evvg==", - "license": "MIT", "engines": { "node": ">=8" }, @@ -10352,6 +10391,11 @@ "dev": true, "license": "ISC" }, + "node_modules/proxy-from-env": { + "version": "1.1.0", + "resolved": "https://registry.npmjs.org/proxy-from-env/-/proxy-from-env-1.1.0.tgz", + "integrity": "sha512-D+zkORCbA9f1tdWRK0RaCR3GPv50cMxcrz4X8k5LTSUD1Dkw47mKJEZQNunItRTkWwgtaUSo1RVFRIG9ZXiFYg==" + }, "node_modules/pseudomap": { "version": "1.0.2", "resolved": "https://registry.npmjs.org/pseudomap/-/pseudomap-1.0.2.tgz", @@ -11643,7 +11687,6 @@ "version": "6.3.0", "resolved": "https://registry.npmjs.org/strtok3/-/strtok3-6.3.0.tgz", "integrity": "sha512-fZtbhtvI9I48xDSywd/somNqgUHl2L2cstmXCCif0itOf96jeW18MBSyrLuNicYQVkvpOxkZtkzujiTJ9LW5Jw==", - "license": "MIT", "dependencies": { "@tokenizer/token": "^0.3.0", "peek-readable": "^4.1.0" @@ -11656,12 +11699,6 @@ "url": "https://github.com/sponsors/Borewit" } }, - "node_modules/strtok3/node_modules/@tokenizer/token": { - "version": "0.3.0", - "resolved": "https://registry.npmjs.org/@tokenizer/token/-/token-0.3.0.tgz", - "integrity": "sha512-OvjF+z51L3ov0OyAU0duzsYuvO01PH7x4t6DJx+guahgTnBHkhJdG7soQeTSFLWN3efnHyibZ4Z8l2EuWwJN3A==", - "license": "MIT" - }, "node_modules/supports-color": { "version": "7.2.0", "resolved": "https://registry.npmjs.org/supports-color/-/supports-color-7.2.0.tgz", @@ -11920,16 +11957,15 @@ } }, "node_modules/token-types": { - "version": "2.1.1", - "resolved": "https://registry.npmjs.org/token-types/-/token-types-2.1.1.tgz", - "integrity": "sha512-wnQcqlreS6VjthyHO3Y/kpK/emflxDBNhlNUPfh7wE39KnuDdOituXomIbyI79vBtF0Ninpkh72mcuRHo+RG3Q==", - "license": "MIT", + "version": "4.2.1", + "resolved": "https://registry.npmjs.org/token-types/-/token-types-4.2.1.tgz", + "integrity": "sha512-6udB24Q737UD/SDsKAHI9FCRP7Bqc9D/MQUV02ORQg5iskjtLJlZJNdN4kKtcdtwCeWIwIHDGaUsTsCCAa8sFQ==", "dependencies": { - "@tokenizer/token": "^0.1.1", + "@tokenizer/token": "^0.3.0", "ieee754": "^1.2.1" }, "engines": { - "node": ">=0.1.98" + "node": ">=10" }, "funding": { "type": "github", diff --git a/package.json b/package.json index 493d205..c0305ba 100644 --- a/package.json +++ b/package.json @@ -12,7 +12,7 @@ "tsc": "tsc", "// test out with a private sample notion db": "", "large-site-test": "npm run ts -- -n $SIL_BLOOM_DOCS_NOTION_TOKEN -r $SIL_BLOOM_DOCS_NOTION_ROOT_PAGE --locales en,fr --log-level debug", - "pull-test-tagged": "npm run ts -- -n $DOCU_NOTION_INTEGRATION_TOKEN -r $DOCU_NOTION_TEST_ROOT_PAGE_ID --log-level debug --status-tag test", + "pull-test-tagged": "npm run ts -- -n $DOCU_NOTION_INTEGRATION_TOKEN -r $DOCU_NOTION_TEST_ROOT_PAGE_ID --log-level info --status-tag test", "pull-test-css": "npm run ts -- --css-output-directory ./test/css -n $DOCU_NOTION_INTEGRATION_TOKEN -r $DOCU_NOTION_TEST_ROOT_PAGE_ID --log-level debug --status-tag test", "pull-sample-site": "npm run ts -- -n $DOCU_NOTION_INTEGRATION_TOKEN -r $DOCU_NOTION_SAMPLE_ROOT_PAGE --log-level debug", "// test with a semi-stable/public site:": "", @@ -20,26 +20,25 @@ "pull-sample-with-paths": "npm run ts -- -n $DOCU_NOTION_INTEGRATION_TOKEN -r $DOCU_NOTION_SAMPLE_ROOT_PAGE -m ./sample --img-output-path ./sample_img" }, "//file-type": "have to use this version before they switched to ESM, which gives a compile error related to require()", - "//node-fetch@2.6.6file-type": "have to use this version before they switched to ESM, which gives a compile error related to require()", "//chalk@4": "also ESM related problem", "//notion-client@4": "also ESM related problem", "//note: ts-node": "really is a runtime dependency", "dependencies": { "@notionhq/client": "2.2.3", + "axios": "^1.6.8", "chalk": "^4.1.2", "commander": "^9.2.0", "cosmiconfig": "^8.0.0", "cosmiconfig-typescript-loader": "^4.3.0", - "file-type": "16.5.1", + "file-type": "16.5.3", "fs-extra": "^10.1.0", "limiter": "^2.1.0", "markdown-table": "^2.0.0", - "node-fetch": "2.6.6", "notion-client": "^4", "notion-to-md": "3.1.1", "path": "^0.12.7", - "ts-node": "^10.2.1", - "sanitize-filename": "^1.6.3" + "sanitize-filename": "^1.6.3", + "ts-node": "^10.2.1" }, "devDependencies": { "@types/fs-extra": "^9.0.13", diff --git a/src/images.ts b/src/images.ts index 2514390..e5c2d0e 100644 --- a/src/images.ts +++ b/src/images.ts @@ -1,6 +1,6 @@ import * as fs from "fs-extra"; import FileType, { FileTypeResult } from "file-type"; -import fetch from "node-fetch"; +import axios from "axios"; import * as Path from "path"; import { makeImagePersistencePlan } from "./MakeImagePersistencePlan"; import { warning, logDebug, verbose, info } from "./log"; @@ -150,9 +150,12 @@ async function processImageBlock( } async function readPrimaryImage(imageSet: ImageSet) { - const response = await fetch(imageSet.primaryUrl); - const arrayBuffer = await response.arrayBuffer(); - imageSet.primaryBuffer = Buffer.from(arrayBuffer); + // In Mar 2024, we started having a problem getting a particular gif from imgur using + // node-fetch. Switching to axios resolved it. I don't know why. + const response = await axios.get(imageSet.primaryUrl, { + responseType: "arraybuffer", + }); + imageSet.primaryBuffer = Buffer.from(response.data, "utf-8"); imageSet.fileType = await FileType.fromBuffer(imageSet.primaryBuffer); }