Retrieve localizations in parallel

app/data/updaters/DataUpdater.mjs

@@ -4,14 +4,14 @@ import { Console } from 'node:console';
 import mkdirp from 'mkdirp';
 import jsonpath from 'jsonpath';
 import ical from 'ical-generator';
+import pFilter from 'p-filter';
 import prefixedConsole from '../../common/prefixedConsole.mjs';
 import SplatNet3Client from '../../splatnet/SplatNet3Client.mjs';
 import ImageProcessor from '../ImageProcessor.mjs';
 import NsoClient from '../../splatnet/NsoClient.mjs';
 import { locales, regionalLocales, defaultLocale } from '../../../src/common/i18n.mjs';
 import { LocalizationProcessor } from '../LocalizationProcessor.mjs';
 import { deriveId, getDateParts, getTopOfCurrentHour } from '../../common/util.mjs';
-
 export default class DataUpdater
 {
   name = null;
@@ -135,16 +135,18 @@ export default class DataUpdater
     }
 
     // Retrieve data for missing languages
-    for (let locale of this.locales.filter(l => l !== initialLocale)) {
-      processor = new LocalizationProcessor(locale, this.localizations);
-
-      if (await processor.hasMissingLocalizations(data)) {
-        this.console.info(`Retrieving localized data for ${locale.code}`);
+    let processors = this.locales.filter(l => l !== initialLocale)
+      .map(l => new LocalizationProcessor(l, this.localizations));
+    let missing = await pFilter(processors, p => p.hasMissingLocalizations(data));
 
-        let regionalData = await this.getData(locale);
+    if (missing.length > 0) {
+      await Promise.all(missing.map(async (processor) => {
+        let regionalData = await this.getData(processor.locale);
         this.deriveIds(regionalData);
         await processor.updateLocalizations(regionalData);
-      }
+
+        this.console.info(`Retrieved localized data for: ${processor.locale.code}`);
+      }));
     }
   }
 

app/splatnet/SplatNet3Client.mjs

@@ -1,9 +1,13 @@
 import fs from 'fs/promises';
+import pLimit from 'p-limit';
 import ValueCache from '../common/ValueCache.mjs';
 import prefixedConsole from '../common/prefixedConsole.mjs';
 
 export const SPLATNET3_WEB_SERVICE_ID = '4834290508791808';
 
+// Concurrent request limit
+const limit = pLimit(5);
+
 export default class SplatNet3Client
 {
   baseUrl = 'https://api.lp1.av5ja.srv.nintendo.net';
@@ -108,17 +112,18 @@ export default class SplatNet3Client
 
   async getGraphQL(body = {}) {
     await this._maybeStartSession();
+    let webViewVersion = await this._webViewVersion();
 
-    let response = await fetch(this.baseUrl + '/api/graphql', {
+    let response = await limit(() => fetch(this.baseUrl + '/api/graphql', {
       method: 'POST',
       headers: {
         'Authorization': `Bearer ${this.bulletToken.bulletToken}`,
-        'X-Web-View-Ver': await this._webViewVersion(),
+        'X-Web-View-Ver': webViewVersion,
         'Content-Type': 'application/json',
         'Accept-Language': this.acceptLanguage,
       },
       body: JSON.stringify(body),
-    });
+    }));
 
     if (!response.ok) {
       throw new Error(`Invalid GraphQL response code: ${response.status}`);

package.json

@@ -41,6 +41,8 @@
     "masto": "^6.7.0",
     "mkdirp": "^1.0.4",
     "nxapi": "^1.4.0",
+    "p-filter": "^4.1.0",
+    "p-limit": "^6.1.0",
     "p-queue": "^8.0.1",
     "pinia": "^2.0.22",
     "puppeteer-core": "^23.8.0",