Download images concurrently

misenhower · Nov 18, 2024 · 34c2d65 · 34c2d65
1 parent 6ba815a
commit 34c2d65
Show file tree

Hide file tree

Showing 5 changed files with 55 additions and 2 deletions.
diff --git a/app/data/ImageProcessor.mjs b/app/data/ImageProcessor.mjs
@@ -1,10 +1,13 @@
 import fs from 'fs/promises';
 import path from 'path';
 import mkdirp from 'mkdirp';
+import pQueue from 'p-queue';
 import prefixedConsole from '../common/prefixedConsole.mjs';
 import { normalizeSplatnetResourcePath } from '../common/util.mjs';
 import { exists } from '../common/fs.mjs';
 
+const queue = new pQueue({ concurrency: 4 });
+
 export default class ImageProcessor
 {
   destinationDirectory = 'dist';
@@ -15,17 +18,27 @@ export default class ImageProcessor
     this.siteUrl = process.env.SITE_URL;
   }
 
-  async process(url) {
+  async process(url, defer = true) {
     // Normalize the path
     let destination = this.normalize(url);
 
     // Download the image if necessary
-    await this.maybeDownload(url, destination);
+    let job = () => this.maybeDownload(url, destination);
+    // defer ? queue.add(job) : await job();
+    if (defer) {
+      queue.add(job);
+    } else {
+      await job();
+    }
 
     // Return the new public URL
     return [destination, this.publicUrl(destination)];
   }
 
+  static onIdle() {
+    return queue.onIdle();
+  }
+
   normalize(url) {
     return normalizeSplatnetResourcePath(url);
   }

diff --git a/app/data/index.mjs b/app/data/index.mjs
@@ -7,6 +7,7 @@ import CoopUpdater from './updaters/CoopUpdater.mjs';
 import FestivalUpdater from './updaters/FestivalUpdater.mjs';
 import XRankUpdater from './updaters/XRankUpdater.mjs';
 import StagesUpdater from './updaters/StagesUpdater.mjs';
+import ImageProcessor from './ImageProcessor.mjs';
 
 function updaters() {
   return [
@@ -53,6 +54,7 @@ export async function update(config = 'default') {
   }
 
   if (canSync()) {
+    await ImageProcessor.onIdle();
     await (new S3Syncer).upload();
   }
 

diff --git a/app/data/updaters/DataUpdater.mjs b/app/data/updaters/DataUpdater.mjs
@@ -166,6 +166,8 @@ export default class DataUpdater
       jsonpath.apply(data, expression, url => mapping[url]);
     }
 
+    await ImageProcessor.onIdle();
+
     return images;
   }
 

diff --git a/package-lock.json b/package-lock.json
diff --git a/package.json b/package.json
@@ -41,6 +41,7 @@
     "masto": "^6.7.0",
     "mkdirp": "^1.0.4",
     "nxapi": "^1.4.0",
+    "p-queue": "^8.0.1",
     "pinia": "^2.0.22",
     "puppeteer-core": "^23.8.0",
     "s3-sync-client": "^4.3.1",
-Original file line number
+Diff line change
@@ Expand Up / @@ -166,6 +166,8 @@ export default class DataUpdater @@
           jsonpath.apply(data, expression, url => mapping[url]);
         }
+        await ImageProcessor.onIdle();
         return images;
       }
@@ Expand Down @@