Better handling of asset fetches (a failed asset shouldn’t fail the d…

…ocument)

src/DataSource/HostedWordPressApi.js

@@ -2,7 +2,7 @@ import { DataSource } from "../DataSource.js";
 
 class HostedWordPressApi extends DataSource {
 	static TYPE = "wordpressapi-hosted";
-	static TYPE_FRIENDLY = "Hosted WordPress";
+	static TYPE_FRIENDLY = "WordPress.com";
 
 	static #getHostname(url) {
 		try {

src/Fetcher.js

@@ -22,7 +22,7 @@ const xmlParser = new XMLParser({
 class Fetcher {
 	static USER_AGENT = "Eleventy Import v1.0.0";
 
-	static getFilenameFromSrc(src, fileExtensionFallback) {
+	static getFilenameFromSrc(src, contentType = "") {
 		let {pathname} = new URL(src);
 		let hash = this.createHash(src);
 
@@ -35,6 +35,7 @@ class Fetcher {
 			return `${filenameWithoutExtension}-${hash}.${extension}`;
 		}
 
+		let [, fileExtensionFallback] = contentType.split("/");
 		// No known file extension
 		return `${filename.slice(0, MAXIMUM_URL_FILENAME_SIZE)}-${hash}${fileExtensionFallback ? `.${fileExtensionFallback}` : ""}`;
 	}
@@ -89,67 +90,77 @@ class Fetcher {
 
 	async fetchAsset(url, outputFolder, urlPath = "assets") {
 		// TODO move this upstream as a Fetch `alias` feature.
-		let result = await this.fetch(url, {
+		return this.fetch(url, {
 			type: "buffer",
 			returnType: "response",
 		},
 		{
 			verbose: true,
 			showErrors: true
-		});
+		}).then(result => {
+			let filename = Fetcher.getFilenameFromSrc(url, result.headers?.["content-type"]);
+			let assetUrlLocation = path.join(urlPath, filename);
+			let fullOutputLocation = path.join(outputFolder, assetUrlLocation);
+			let urlValue = `/${assetUrlLocation}`;
+
+			if(this.writtenAssetFiles.has(fullOutputLocation)) {
+				return urlValue;
+			}
 
-		let [, extension] = result.headers?.["content-type"]?.split("/");
-		let filename = Fetcher.getFilenameFromSrc(url, extension);
-		let assetUrlLocation = path.join(urlPath, filename);
-		let fullOutputLocation = path.join(outputFolder, assetUrlLocation);
-		let urlValue = `/${assetUrlLocation}`;
+			this.writtenAssetFiles.add(fullOutputLocation);
 
-		if(this.writtenAssetFiles.has(fullOutputLocation)) {
-			return urlValue;
-		}
+			if(this.safeMode && fs.existsSync(fullOutputLocation)) {
+				if(this.isVerbose) {
+					Logger.skipping("asset", fullOutputLocation, url);
+				}
+				return urlValue;
+			}
 
-		this.writtenAssetFiles.add(fullOutputLocation);
+			if(this.#directoryManager) {
+				this.#directoryManager.createDirectoryForPath(fullOutputLocation);
+			}
 
-		if(this.safeMode && fs.existsSync(fullOutputLocation)) {
 			if(this.isVerbose) {
-				Logger.skipping("asset", fullOutputLocation, url);
+				Logger.importing("asset", fullOutputLocation, url, {
+					size: result.body.length,
+					dryRun: this.dryRun
+				});
 			}
-			return urlValue;
-		}
 
-		if(this.#directoryManager) {
-			this.#directoryManager.createDirectoryForPath(fullOutputLocation);
-		}
+			if(!this.dryRun) {
+				this.counts.assets++;
 
-		if(this.isVerbose) {
-			Logger.importing("asset", fullOutputLocation, url, {
-				size: result.body.length,
-				dryRun: this.dryRun
-			});
-		}
-
-		if(!this.dryRun) {
-			this.counts.assets++;
-
-			fs.writeFileSync(fullOutputLocation, result.body);
-		}
+				fs.writeFileSync(fullOutputLocation, result.body);
+			}
 
-		return urlValue;
+			return urlValue;
+		}, error => {
+			// Logging the error happens in .fetch() upstream
+			// Fetching the asset failed but we don’t want to fail the upstream document promise
+			return "";
+		});
 	}
 
 	async fetch(url, options = {}, verbosity = {}) {
 		let { verbose, showErrors } = Object.assign({
-			showErrors: true,
-			verbose: true, // whether to log the fetch request
+			verbose: true, // whether to log the initial fetch request
+			showErrors: true, // whether to show if a request has an error.
 		}, verbosity);
 
 		let opts = Object.assign({
 			duration: this.#cacheDuration,
 			type: "text",
-			verbose: false, // we’re handling our own logging here
+			verbose: false, // don’t use Fetch logging—we’re handling it ourself
 			fetchOptions: {},
 		}, options);
 
+		if(!opts.fetchOptions.headers) {
+			opts.fetchOptions.headers = {};
+		}
+		Object.assign(opts.fetchOptions.headers, {
+			"user-agent": Fetcher.USER_AGENT
+		});
+
 		if(!this.fetchedUrls.has(url) && this.isVerbose && verbose) {
 			let logAdds = [];
 			if(Boolean(options?.fetchOptions?.headers?.Authorization)) {
@@ -164,31 +175,21 @@ class Fetcher {
 
 		this.fetchedUrls.add(url);
 
-		if(!opts.fetchOptions.headers) {
-			opts.fetchOptions.headers = {};
-		}
-		Object.assign(opts.fetchOptions.headers, {
-			"user-agent": Fetcher.USER_AGENT
-		});
-
-		try {
-			let result = await EleventyFetch(url, opts);
-
+		return EleventyFetch(url, opts).then(result => {
 			if(opts.type === "xml") {
 				return xmlParser.parse(result);
 			}
 
 			return result;
-		} catch(e) {
+		}, error => {
 			this.errors.add(url);
 
-			// if(this.isVerbose && showErrors) {
-			if(showErrors) {
-				Logger.log(kleur.red(`Error fetching`), url, kleur.red(e.message));
+			if(this.isVerbose && showErrors) {
+				Logger.log(kleur.red(`Error fetching`), url, kleur.red(error.message));
 			}
 
-			return Promise.reject(e);
-		}
+			return Promise.reject(error);
+		});
 	}
 }
 

src/Importer.js

@@ -169,7 +169,7 @@ class Importer {
 			entries = entries.slice(0, MAX_IMPORT_SIZE);
 		}
 
-		let promises = await Promise.all(entries.map(async entry => {
+		let promises = await Promise.allSettled(entries.map(async entry => {
 			if(Importer.isHtml(entry)) {
 				entry.content = await this.htmlTransformer.transform(entry.content, entry.url);
 
@@ -185,7 +185,11 @@ class Importer {
 			return entry;
 		}));
 
-		return promises.sort((a, b) => {
+		return promises.filter(entry => {
+			return entry.status !== "rejected";
+		}).map(entry => {
+			return entry.value;
+		}).sort((a, b) => {
 			if(a.date < b.date) {
 				return 1;
 			}

src/Logger.js

@@ -10,20 +10,23 @@ class Logger {
 		let { size, dryRun } = options;
 
 		let extras = [];
+		let prefix = "";
 		if(label === "Skipping") {
-			extras.push("overwrites disabled");
+			prefix = " (no --overwrite)";
 		} else {
 			if(size) {
 				extras.push(filesize(size, {
 					spacer: ""
 				}));
 			}
+
 			if(dryRun) {
-				extras.push("dry run");
+				prefix = " (dry run)";
 			}
 		}
+
 		let extrasStr = extras.length ? `(${extras.join(", ")}) ` : "";
-		this.log(kleur.gray(`${label} ${type}`), local, kleur.gray(`${extrasStr}from`), remote);
+		this.log(kleur.gray(`${label} ${type}${prefix}`), local, kleur.gray(`${extrasStr}from`), remote);
 	}
 
 	static importing(type, local, remote, options = {}) {