From 5bd5c2b5e935359af17c5504aa503e837cd6c200 Mon Sep 17 00:00:00 2001 From: Shawn Date: Wed, 2 Oct 2024 13:18:38 -0400 Subject: [PATCH 01/21] locally reallocate instead of relying on api, update documentation --- package-lock.json | 21 +++++++++++++++++++-- package.json | 3 ++- readme.md | 24 ++++++++++++++++++++---- src/api.ts | 23 +++++++++++++++++++++++ 4 files changed, 64 insertions(+), 7 deletions(-) diff --git a/package-lock.json b/package-lock.json index 7c4ac8f..22e9d09 100644 --- a/package-lock.json +++ b/package-lock.json @@ -1,16 +1,17 @@ { "name": "kelpie", - "version": "0.4.4", + "version": "0.5.0", "lockfileVersion": 3, "requires": true, "packages": { "": { "name": "kelpie", - "version": "0.4.4", + "version": "0.5.0", "license": "MIT", "dependencies": { "@aws-sdk/client-s3": "^3.556.0", "@aws-sdk/lib-storage": "^3.556.0", + "@saladtechnologies-oss/salad-cloud-imds-sdk": "^0.9.0-alpha.3", "chokidar": "^3.6.0", "pino": "^9.0.0" }, @@ -3669,6 +3670,14 @@ "node": ">= 8" } }, + "node_modules/@saladtechnologies-oss/salad-cloud-imds-sdk": { + "version": "0.9.0-alpha.3", + "resolved": "https://registry.npmjs.org/@saladtechnologies-oss/salad-cloud-imds-sdk/-/salad-cloud-imds-sdk-0.9.0-alpha.3.tgz", + "integrity": "sha512-Pm++a4XKT5QZ0xO0FqVTDICm3upnEZP7D24nEqg24bTeHAiUuyi9c8+EWLLqsEw20x1KGJUTnBFM47eMiJpkcw==", + "dependencies": { + "zod": "3.22.0" + } + }, "node_modules/@smithy/abort-controller": { "version": "2.2.0", "resolved": "https://registry.npmjs.org/@smithy/abort-controller/-/abort-controller-2.2.0.tgz", @@ -7807,6 +7816,14 @@ "engines": { "node": ">=10" } + }, + "node_modules/zod": { + "version": "3.22.0", + "resolved": "https://registry.npmjs.org/zod/-/zod-3.22.0.tgz", + "integrity": "sha512-y5KZY/ssf5n7hCGDGGtcJO/EBJEm5Pa+QQvFBeyMOtnFYOSflalxIFFvdaYevPhePcmcKC4aTbFkCcXN7D0O8Q==", + "funding": { + "url": "https://github.com/sponsors/colinhacks" + } } } } diff --git a/package.json b/package.json index c28e6b5..b60a594 100644 --- a/package.json +++ b/package.json @@ -1,6 +1,6 @@ { "name": "kelpie", - "version": "0.4.4", + "version": "0.5.0", "description": "A worker binary to coordinate long running jobs on salad. Works with Kelpie API", "main": "dist/index.js", "scripts": { @@ -21,6 +21,7 @@ "dependencies": { "@aws-sdk/client-s3": "^3.556.0", "@aws-sdk/lib-storage": "^3.556.0", + "@saladtechnologies-oss/salad-cloud-imds-sdk": "^0.9.0-alpha.3", "chokidar": "^3.6.0", "pino": "^9.0.0" }, diff --git a/readme.md b/readme.md index ab9bb3b..db8d8ee 100644 --- a/readme.md +++ b/readme.md @@ -1,4 +1,5 @@ ![](./kelpie.png) + # 🐕 Kelpie (beta) Kelpie shepherds long-running jobs through to completion on interruptible hardware, coordinating with the [Kelpie API](https://github.com/SaladTechnologies/kelpie-api) @@ -7,6 +8,7 @@ Kelpie shepherds long-running jobs through to completion on interruptible hardwa - [Who is it for?](#who-is-it-for) - [How it Works](#how-it-works) - [Adding the kelpie Worker To Your Container Image](#adding-the-kelpie-worker-to-your-container-image) + - [Environment Variables](#environment-variables) - [Deploying Your Container Group](#deploying-your-container-group) - [What it DOES NOT do](#what-it-does-not-do) - [API Authorization](#api-authorization) @@ -34,7 +36,7 @@ If you'd like to join the Kelpie beta, and are an existing Salad customer, just ![Kelpie Diagram](./kelpie-architecture.png) -Kelpie is a standalone binary that runs in your container image. It coordinates with the Kelpie API to download your input data, upload your output data, and sync progress checkpoints to your s3-compatible storage. You submit jobs to the [Kelpie API](https://kelpie.saladexamples.com/docs), and those jobs get assigned to salad worker nodes that have the Kelpie binary installed. +Kelpie is a standalone binary that runs in your container image. It coordinates with the Kelpie API to download your input data, upload your output data, and sync progress checkpoints to your s3-compatible storage. You submit jobs to the [Kelpie API](https://kelpie.saladexamples.com/docs), and those jobs get assigned to salad worker nodes that have the Kelpie binary installed. If you define [scaling rules](https://kelpie.saladexamples.com/docs#/default/post_CreateScalingRule), the Kelpie API will handle starting and stopping your container group, and scaling it up and down in response to job volume. @@ -76,9 +78,23 @@ Additionally, your script must support the following things: Upload your docker image to the container registry of your choice. Salad supports public and private registries, including Docker Hub, AWS ECR, and GitHub Container Registry, among others. +## Environment Variables + +| Variable | Description | Default Value | Required | +| ------------------------ | -------------------------------------------------------------------------------- | -------------------------- | -------- | +| KELPIE_API_URL | The URL for the Kelpie API | None | Yes | +| KELPIE_API_KEY | The API key for authenticating with the Kelpie API | None | Yes | +| SALAD_MACHINE_ID | The ID of the Salad machine | *set dynamically by Salad* | No | +| SALAD_CONTAINER_GROUP_ID | The ID of the Salad container group | "" (empty string) | No | +| MAX_RETRIES | The maximum number of retries for API operations | "3" | No | +| MAX_JOB_FAILURES | The maximum number of job failures allowed before an instance should reallocate. | "3" | No | +| KELPIE_LOG_LEVEL | The log level for kelpie | "info" | No | + +Additionally, Kelpie will respect AWS environment variables, such as `AWS_ACCESS_KEY_ID`, `AWS_SECRET_ACCESS_KEY`, etc. These are used to authenticate with your s3-compatible storage. + ## Deploying Your Container Group -You can deploy your container group using the [Salad API](https://docs.salad.com/api-reference/container_groups/create-a-container-group), or via the [Salad Portal](https://portal.salad.com/). You will need to add the kelpie salad user (currently shawn.rushefsky@salad.com) to your organization to enable the scaling features of kelpie. Kelpie uses the Salad API to start, stop, and scale your container group in response to job volume. +You can deploy your container group using the [Salad API](https://docs.salad.com/api-reference/container_groups/create-a-container-group), or via the [Salad Portal](https://portal.salad.com/). You will need to add the kelpie salad user (currently ) to your organization to enable the scaling features of kelpie. Kelpie uses the Salad API to start, stop, and scale your container group in response to job volume. In your container group configuration, you will provide the docker image url, the hardware configuration needed by your job, and the environment variables detailed above. You do not need to enable Container Gateway, or Job Queues, and you do not need to configure probes. While Salad does offer built-in logging, it is still recommended to connect an external logging service for more advanced features. @@ -92,7 +108,8 @@ Once your container group is deployed, and you've verified that the node starts 4. kelpie does not create or delete your container groups. If configured with scaling rules, kelpie can start, stop, and scale your container group in response to job volume. ## API Authorization -There are live swagger docs that should be considered more accurate and up to date than this readme: https://kelpie.saladexamples.com/docs + +There are live swagger docs that should be considered more accurate and up to date than this readme: Your kelpie api key is used by you to submit work, and also by kelpie workers to pull and process work. @@ -364,7 +381,6 @@ All query parameters for this endpoint are optional. } ``` - ## Job Lifecycle 1. When kelpie starts on a new node, it starts polling for available work from `/work`. In these requests, it includes some information about what salad node you're on, including the machine id and container group id. This ensures we only hand out work to the correct container group, and that we do not hand out to a machine where that job has previously failed. diff --git a/src/api.ts b/src/api.ts index ca47d1b..76f4436 100644 --- a/src/api.ts +++ b/src/api.ts @@ -2,6 +2,7 @@ import assert from "assert"; import { log as baseLogger } from "./logger"; import { Logger } from "pino"; import { Task } from "./types"; +import { SaladCloudImdsSdk } from "@saladtechnologies-oss/salad-cloud-imds-sdk"; let { KELPIE_API_URL, @@ -9,6 +10,7 @@ let { SALAD_MACHINE_ID = "", SALAD_CONTAINER_GROUP_ID = "", MAX_RETRIES = "3", + MAX_JOB_FAILURES = "3", } = process.env; assert(KELPIE_API_URL, "KELPIE_API_URL is required"); @@ -19,12 +21,15 @@ if (KELPIE_API_URL.endsWith("/")) { } const maxRetries = parseInt(MAX_RETRIES, 10); +const maxJobFailures = parseInt(MAX_JOB_FAILURES, 10); const headers = { "Content-Type": "application/json", "X-Kelpie-Key": KELPIE_API_KEY, }; +const imds = new SaladCloudImdsSdk({}); + async function sleep(ms: number): Promise { return new Promise((resolve) => setTimeout(resolve, ms)); } @@ -98,6 +103,7 @@ export async function sendHeartbeat( return { status }; } +let numFailures = 0; export async function reportFailed(jobId: string, log: Logger): Promise { log.info(`Reporting job failed`); await fetchUpToNTimes( @@ -113,6 +119,23 @@ export async function reportFailed(jobId: string, log: Logger): Promise { maxRetries, log ); + numFailures++; + if (numFailures >= maxJobFailures) { + await reallocateMe(log); + } +} + +export async function reallocateMe(log: Logger): Promise { + try { + log.info("Reallocating container via IMDS"); + await imds.metadata.reallocateContainer({ + reason: "Kelpie: Max Job Failures Exceeded", + }); + } catch (e: any) { + log.error(`Failed to reallocate container via IMDS: ${e.message}`); + log.error("Exiting process"); + process.exit(1); + } } export async function reportCompleted( From a87c98b711ab23c6a6df95ae4be04ea2f013f494 Mon Sep 17 00:00:00 2001 From: Shawn Date: Fri, 11 Oct 2024 09:18:41 -0400 Subject: [PATCH 02/21] customize exit behavior --- readme.md | 19 ++++++++++--------- src/index.ts | 32 +++++++++++++++++++++++++++++++- 2 files changed, 41 insertions(+), 10 deletions(-) diff --git a/readme.md b/readme.md index db8d8ee..ffb24ed 100644 --- a/readme.md +++ b/readme.md @@ -80,15 +80,16 @@ Upload your docker image to the container registry of your choice. Salad support ## Environment Variables -| Variable | Description | Default Value | Required | -| ------------------------ | -------------------------------------------------------------------------------- | -------------------------- | -------- | -| KELPIE_API_URL | The URL for the Kelpie API | None | Yes | -| KELPIE_API_KEY | The API key for authenticating with the Kelpie API | None | Yes | -| SALAD_MACHINE_ID | The ID of the Salad machine | *set dynamically by Salad* | No | -| SALAD_CONTAINER_GROUP_ID | The ID of the Salad container group | "" (empty string) | No | -| MAX_RETRIES | The maximum number of retries for API operations | "3" | No | -| MAX_JOB_FAILURES | The maximum number of job failures allowed before an instance should reallocate. | "3" | No | -| KELPIE_LOG_LEVEL | The log level for kelpie | "info" | No | +| Variable | Description | Default Value | Required | +| ------------------------ | ------------------------------------------------------------------------------------------------------------ | -------------------------- | -------- | +| KELPIE_API_URL | The URL for the Kelpie API | None | Yes | +| KELPIE_API_KEY | The API key for authenticating with the Kelpie API | None | Yes | +| SALAD_MACHINE_ID | The ID of the Salad machine | *set dynamically by Salad* | No | +| SALAD_CONTAINER_GROUP_ID | The ID of the Salad container group | "" (empty string) | No | +| MAX_RETRIES | The maximum number of retries for API operations | "3" | No | +| MAX_JOB_FAILURES | The maximum number of job failures allowed before an instance should reallocate. | "3" | No | +| MAX_TIME_WITH_NO_WORK_S | The maximum time to wait for work before exiting. May be exceeded by up to 10 seconds (1 heartbeat interval) | "-1" (Never) | No | +| KELPIE_LOG_LEVEL | The log level for kelpie | "info" | No | Additionally, Kelpie will respect AWS environment variables, such as `AWS_ACCESS_KEY_ID`, `AWS_SECRET_ACCESS_KEY`, etc. These are used to authenticate with your s3-compatible storage. diff --git a/src/index.ts b/src/index.ts index 3ff11af..adf4c15 100644 --- a/src/index.ts +++ b/src/index.ts @@ -5,6 +5,7 @@ import { HeartbeatManager, reportFailed, reportCompleted, + reallocateMe, } from "./api"; import { DirectoryWatcher, recursivelyClearFilesInDirectory } from "./files"; import { @@ -27,12 +28,21 @@ const { INPUT_DIR = "/input", OUTPUT_DIR = "/output", CHECKPOINT_DIR = "/checkpoint", + + // Default to -1, which means no timeout + MAX_TIME_WITH_NO_WORK_S = "-1", + + // There are backend implications to this, so we aren't documenting it yet. + HEARTBEAT_INTERVAL_S = "10", } = process.env; mkdirSync(INPUT_DIR, { recursive: true }); mkdirSync(OUTPUT_DIR, { recursive: true }); mkdirSync(CHECKPOINT_DIR, { recursive: true }); +const maxTimeWithNoWorkMs = parseInt(MAX_TIME_WITH_NO_WORK_S, 10) * 1000; +const heartbeatIntervalMs = parseInt(HEARTBEAT_INTERVAL_S, 10) * 1000; + const commandExecutor = new CommandExecutor(); async function clearAllDirectories(dirsToClear: string[]): Promise { @@ -101,6 +111,7 @@ async function main() { Array.from(new Set([INPUT_DIR, OUTPUT_DIR, CHECKPOINT_DIR])) ); + let lastWorkReceived = Date.now(); while (keepAlive) { let work; try { @@ -112,10 +123,29 @@ async function main() { } if (!work) { + if ( + maxTimeWithNoWorkMs > 0 && + Date.now() - lastWorkReceived > maxTimeWithNoWorkMs + ) { + baseLogger.info( + `No work received for ${ + maxTimeWithNoWorkMs / 1000 + } seconds, exiting...` + ); + keepAlive = false; + /** + * A common reason to have no work for too long is that the instance is + * banned from a particular workload. In this case, we should reallocate + * the instance to get a new machine id. + */ + await reallocateMe(baseLogger); + break; + } baseLogger.info("No work available, sleeping for 10 seconds..."); - await sleep(10000); + await sleep(heartbeatIntervalMs); continue; } + lastWorkReceived = Date.now(); const log = baseLogger.child({ job_id: work.id }); log.info(`Received work: ${work.id}`); From c6c7bab637c39fbff5bfd124e3df907924000361 Mon Sep 17 00:00:00 2001 From: Shawn Date: Mon, 4 Nov 2024 11:39:26 -0500 Subject: [PATCH 03/21] keep a state file --- readme.md | 2 +- src/files.ts | 2 +- src/index.ts | 12 +++++--- src/s3.ts | 32 +++++++++++++++----- src/state.ts | 84 ++++++++++++++++++++++++++++++++++++++++++++++++++++ 5 files changed, 119 insertions(+), 13 deletions(-) create mode 100644 src/state.ts diff --git a/readme.md b/readme.md index ffb24ed..f55e735 100644 --- a/readme.md +++ b/readme.md @@ -88,7 +88,7 @@ Upload your docker image to the container registry of your choice. Salad support | SALAD_CONTAINER_GROUP_ID | The ID of the Salad container group | "" (empty string) | No | | MAX_RETRIES | The maximum number of retries for API operations | "3" | No | | MAX_JOB_FAILURES | The maximum number of job failures allowed before an instance should reallocate. | "3" | No | -| MAX_TIME_WITH_NO_WORK_S | The maximum time to wait for work before exiting. May be exceeded by up to 10 seconds (1 heartbeat interval) | "-1" (Never) | No | +| MAX_TIME_WITH_NO_WORK_S | The maximum time to wait for work before exiting. May be exceeded by up to 10 seconds (1 heartbeat interval) | "0" (Never) | No | | KELPIE_LOG_LEVEL | The log level for kelpie | "info" | No | Additionally, Kelpie will respect AWS environment variables, such as `AWS_ACCESS_KEY_ID`, `AWS_SECRET_ACCESS_KEY`, etc. These are used to authenticate with your s3-compatible storage. diff --git a/src/files.ts b/src/files.ts index e0c364e..1ec357e 100644 --- a/src/files.ts +++ b/src/files.ts @@ -139,7 +139,7 @@ export async function recursivelyClearFilesInDirectory( await fsPromises.rm(directory, { recursive: true, force: true }); await fsPromises.mkdir(directory, { recursive: true }); log.info("Directory cleared successfully"); - } catch (err) { + } catch (err: any) { log.error("Error clearing directory: ", err); } } diff --git a/src/index.ts b/src/index.ts index adf4c15..818ebfa 100644 --- a/src/index.ts +++ b/src/index.ts @@ -23,14 +23,15 @@ import fs from "fs/promises"; import { log as baseLogger } from "./logger"; import { Logger } from "pino"; import { SyncConfig, Task } from "./types"; +import state from "./state"; const { INPUT_DIR = "/input", OUTPUT_DIR = "/output", CHECKPOINT_DIR = "/checkpoint", - // Default to -1, which means no timeout - MAX_TIME_WITH_NO_WORK_S = "-1", + // Default to 0, which means no timeout + MAX_TIME_WITH_NO_WORK_S = "0", // There are backend implications to this, so we aren't documenting it yet. HEARTBEAT_INTERVAL_S = "10", @@ -107,6 +108,7 @@ const filesBeingSynced = new Set(); async function main() { baseLogger.info(`Kelpie v${version} started`); + await state.saveState(baseLogger); await clearAllDirectories( Array.from(new Set([INPUT_DIR, OUTPUT_DIR, CHECKPOINT_DIR])) ); @@ -118,7 +120,7 @@ async function main() { work = await getWork(); } catch (e: any) { baseLogger.error("Error fetching work: ", e); - await sleep(10000); + await sleep(heartbeatIntervalMs); continue; } @@ -217,7 +219,7 @@ async function main() { directoryWatchers.push(dirWatcher); } } - } else { + } else if (work.input_bucket && work.input_prefix) { // Download required files if (work.input_bucket && work.input_prefix) { try { @@ -306,6 +308,8 @@ async function main() { ); directoryWatchers.push(outputWatcher); } + } else { + log.info("No storage configuration provided, skipping file sync"); } /** diff --git a/src/s3.ts b/src/s3.ts index c7faa09..7a64850 100644 --- a/src/s3.ts +++ b/src/s3.ts @@ -13,6 +13,7 @@ import fsPromises from "fs/promises"; import { createGzip, createGunzip } from "zlib"; import { Logger } from "pino"; import { SyncConfig } from "./types"; +import state from "./state"; const { AWS_REGION, AWS_DEFAULT_REGION } = process.env; @@ -52,8 +53,6 @@ function getDataRatioString( return sizeString; } -const ongoingUploads: Set = new Set(); - export async function uploadFile( localFilePath: string, bucketName: string, @@ -62,11 +61,11 @@ export async function uploadFile( log: Logger ): Promise { try { - if (ongoingUploads.has(localFilePath)) { + if (state.hasUpload(localFilePath)) { log.info(`Upload of ${localFilePath} already in progress`); return; } - ongoingUploads.add(localFilePath); + await state.startUpload(localFilePath, log); log.info(`Uploading ${localFilePath} to s3://${bucketName}/${key}`); // Create a stream from the local file const fileStream = fs.createReadStream(localFilePath); @@ -116,7 +115,7 @@ export async function uploadFile( } catch (err) { log.error("Error uploading file: ", err); } - ongoingUploads.delete(localFilePath); + await state.finishDownload(localFilePath, log); } export async function downloadFile( @@ -128,6 +127,11 @@ export async function downloadFile( ): Promise { try { const start = Date.now(); + if (state.hasDownload(key)) { + log.info(`Download of ${key} already in progress`); + return; + } + state.startDownload(localFilePath, log); const isGzipped = decompress && key.endsWith(".gz"); if (isGzipped) { @@ -148,7 +152,11 @@ export async function downloadFile( if (isGzipped) { log.debug(`Decompressing ${key} file with gunzip`); const unzipStream = createGunzip(); - unzipStream.on("error", (err) => reject(err)); + unzipStream.on("error", async (err) => { + log.error(`Error decompressing ${key} file: ${err}`); + state.finishDownload(localFilePath, log); + reject(err); + }); stream = stream.pipe(unzipStream); } @@ -156,7 +164,11 @@ export async function downloadFile( const writeStream = fs.createWriteStream(localFilePath); stream .pipe(writeStream) - .on("error", (err: any) => reject(err)) + .on("error", (err: any) => { + log.error(`Error writing to ${localFilePath}: ${err}`); + state.finishDownload(localFilePath, log); + reject(err); + }) .on("close", () => { const end = Date.now(); @@ -171,12 +183,14 @@ export async function downloadFile( durString = `${(durMs / 60000).toFixed(2)}m`; } log.info(`${localFilePath} downloaded in ${durString}`); + state.finishDownload(localFilePath, log); resolve(); }); } }); } catch (err: any) { log.error("Error downloading file: ", err); + await state.finishDownload(localFilePath, log); throw err; } } @@ -304,6 +318,10 @@ export async function uploadDirectory({ log.info(`Filtering files with pattern: ${pattern}`); fileList = fileList.filter((key) => pattern.test(key)); } + if (fileList.length === 0) { + log.info("No files found to upload"); + return; + } log.info(`Found ${fileList.length} files to upload`); for (let i = 0; i < fileList.length; i += batchSize) { const batch = fileList.slice(i, i + batchSize); diff --git a/src/state.ts b/src/state.ts new file mode 100644 index 0000000..5fcac26 --- /dev/null +++ b/src/state.ts @@ -0,0 +1,84 @@ +import path from "path"; +import fs from "fs/promises"; +import { Logger } from "pino"; + +const { KELPIE_STATE_FILE = "./kelpie-state.json" } = process.env; + +const kelpieStateFile = path.resolve(KELPIE_STATE_FILE); + +const state = { + start: new Date().toISOString(), + uploads: new Set(), + downloads: new Set(), +}; + +export async function startDownload( + filePath: string, + log: Logger +): Promise { + state.downloads.add(filePath); + await saveState(log); +} + +export async function finishDownload( + filePath: string, + log: Logger +): Promise { + state.downloads.delete(filePath); + await saveState(log); +} + +export function hasDownload(filePath: string): boolean { + return state.downloads.has(filePath); +} + +export async function startUpload( + filePath: string, + log: Logger +): Promise { + state.uploads.add(filePath); + await saveState(log); +} + +export async function finishUpload( + filePath: string, + log: Logger +): Promise { + state.uploads.delete(filePath); + await saveState(log); +} + +export function hasUpload(filePath: string): boolean { + return state.uploads.has(filePath); +} + +function getJSONState(): any { + const { uploads, downloads, start } = state; + return JSON.stringify( + { + start, + uploads: Array.from(uploads), + downloads: Array.from(downloads), + }, + null, + 2 + ); +} + +export async function saveState(log: Logger): Promise { + try { + await fs.writeFile(kelpieStateFile, getJSONState()); + } catch (err) { + log.error("Error saving state: ", err); + } +} + +export default { + startDownload, + finishDownload, + hasDownload, + startUpload, + finishUpload, + hasUpload, + saveState, +}; From a82b60999b070208cd5e0090ea274c91cde4860f Mon Sep 17 00:00:00 2001 From: Shawn Date: Mon, 4 Nov 2024 11:42:41 -0500 Subject: [PATCH 04/21] update imds sdk --- package-lock.json | 8 ++++---- package.json | 2 +- 2 files changed, 5 insertions(+), 5 deletions(-) diff --git a/package-lock.json b/package-lock.json index 22e9d09..e822a1f 100644 --- a/package-lock.json +++ b/package-lock.json @@ -11,7 +11,7 @@ "dependencies": { "@aws-sdk/client-s3": "^3.556.0", "@aws-sdk/lib-storage": "^3.556.0", - "@saladtechnologies-oss/salad-cloud-imds-sdk": "^0.9.0-alpha.3", + "@saladtechnologies-oss/salad-cloud-imds-sdk": "^0.9.0-alpha.4", "chokidar": "^3.6.0", "pino": "^9.0.0" }, @@ -3671,9 +3671,9 @@ } }, "node_modules/@saladtechnologies-oss/salad-cloud-imds-sdk": { - "version": "0.9.0-alpha.3", - "resolved": "https://registry.npmjs.org/@saladtechnologies-oss/salad-cloud-imds-sdk/-/salad-cloud-imds-sdk-0.9.0-alpha.3.tgz", - "integrity": "sha512-Pm++a4XKT5QZ0xO0FqVTDICm3upnEZP7D24nEqg24bTeHAiUuyi9c8+EWLLqsEw20x1KGJUTnBFM47eMiJpkcw==", + "version": "0.9.0-alpha.4", + "resolved": "https://registry.npmjs.org/@saladtechnologies-oss/salad-cloud-imds-sdk/-/salad-cloud-imds-sdk-0.9.0-alpha.4.tgz", + "integrity": "sha512-4S95xspjhXHJ6xaXmR/ujLU5mLG5AII4Pe2rs0TgIqeUL9RijbCQ9w5l6mFa+/g4h9UxeItK1fqf7oC7A6yhcw==", "dependencies": { "zod": "3.22.0" } diff --git a/package.json b/package.json index b60a594..9384288 100644 --- a/package.json +++ b/package.json @@ -21,7 +21,7 @@ "dependencies": { "@aws-sdk/client-s3": "^3.556.0", "@aws-sdk/lib-storage": "^3.556.0", - "@saladtechnologies-oss/salad-cloud-imds-sdk": "^0.9.0-alpha.3", + "@saladtechnologies-oss/salad-cloud-imds-sdk": "^0.9.0-alpha.4", "chokidar": "^3.6.0", "pino": "^9.0.0" }, From b1d047b8422572cd45957550308d20540a5de5aa Mon Sep 17 00:00:00 2001 From: Shawn Date: Mon, 4 Nov 2024 11:50:11 -0500 Subject: [PATCH 05/21] pass filename to application --- src/index.ts | 8 +++++++- src/state.ts | 5 +++-- 2 files changed, 10 insertions(+), 3 deletions(-) diff --git a/src/index.ts b/src/index.ts index 818ebfa..cbe51f8 100644 --- a/src/index.ts +++ b/src/index.ts @@ -319,7 +319,13 @@ async function main() { const exitCode = await commandExecutor.execute( work.command, work.arguments, - { ...work.environment, INPUT_DIR, OUTPUT_DIR, CHECKPOINT_DIR } + { + ...work.environment, + INPUT_DIR, + OUTPUT_DIR, + CHECKPOINT_DIR, + KELPIE_STATE_FILE: state.filename, + } ); /** * Once the script updates, we can stop watching the directories. diff --git a/src/state.ts b/src/state.ts index 5fcac26..c92e45a 100644 --- a/src/state.ts +++ b/src/state.ts @@ -4,7 +4,7 @@ import { Logger } from "pino"; const { KELPIE_STATE_FILE = "./kelpie-state.json" } = process.env; -const kelpieStateFile = path.resolve(KELPIE_STATE_FILE); +export const filename = path.resolve(KELPIE_STATE_FILE); const state = { start: new Date().toISOString(), @@ -67,7 +67,7 @@ function getJSONState(): any { export async function saveState(log: Logger): Promise { try { - await fs.writeFile(kelpieStateFile, getJSONState()); + await fs.writeFile(filename, getJSONState()); } catch (err) { log.error("Error saving state: ", err); } @@ -81,4 +81,5 @@ export default { finishUpload, hasUpload, saveState, + filename, }; From f28fa129c56da1e60134142295cfe1839f79e801 Mon Sep 17 00:00:00 2001 From: Shawn Date: Tue, 5 Nov 2024 07:59:08 -0500 Subject: [PATCH 06/21] pass kelpie job id in environment --- src/index.ts | 1 + 1 file changed, 1 insertion(+) diff --git a/src/index.ts b/src/index.ts index cbe51f8..2aafb53 100644 --- a/src/index.ts +++ b/src/index.ts @@ -325,6 +325,7 @@ async function main() { OUTPUT_DIR, CHECKPOINT_DIR, KELPIE_STATE_FILE: state.filename, + KELPIE_JOB_ID: work.id, } ); /** From 4af7973d88f0be85b8632de9cdb58501f5f16cb3 Mon Sep 17 00:00:00 2001 From: Shawn Date: Tue, 19 Nov 2024 11:36:47 -0500 Subject: [PATCH 07/21] update dependencies, remove s3 request timeout --- .nvmrc | 2 +- package-lock.json | 1676 +++++++++++++++++++++++++-------------------- package.json | 5 +- src/s3.ts | 9 +- 4 files changed, 938 insertions(+), 754 deletions(-) diff --git a/.nvmrc b/.nvmrc index eb800ed..a81deba 100644 --- a/.nvmrc +++ b/.nvmrc @@ -1 +1 @@ -v18.19.0 +v20.12.2 diff --git a/package-lock.json b/package-lock.json index e822a1f..868be44 100644 --- a/package-lock.json +++ b/package-lock.json @@ -12,6 +12,7 @@ "@aws-sdk/client-s3": "^3.556.0", "@aws-sdk/lib-storage": "^3.556.0", "@saladtechnologies-oss/salad-cloud-imds-sdk": "^0.9.0-alpha.4", + "@smithy/node-http-handler": "^3.3.1", "chokidar": "^3.6.0", "pino": "^9.0.0" }, @@ -22,7 +23,7 @@ "@types/chokidar": "^2.1.3", "@types/node": "^20.12.7", "@types/pino": "^7.0.5", - "pkg": "^5.8.1", + "@yao-pkg/pkg": "^6.1.1", "typescript": "^5.4.5" } }, @@ -175,18 +176,6 @@ "node": ">=16.0.0" } }, - "node_modules/@aws-sdk/client-s3/node_modules/@smithy/abort-controller": { - "version": "3.1.4", - "resolved": "https://registry.npmjs.org/@smithy/abort-controller/-/abort-controller-3.1.4.tgz", - "integrity": "sha512-VupaALAQlXViW3/enTf/f5l5JZYSAxoJL7f0nanhNNKnww6DGCg1oYIuNP78KDugnkwthBO6iEcym16HhWV8RQ==", - "dependencies": { - "@smithy/types": "^3.4.2", - "tslib": "^2.6.2" - }, - "engines": { - "node": ">=16.0.0" - } - }, "node_modules/@aws-sdk/client-s3/node_modules/@smithy/fetch-http-handler": { "version": "3.2.8", "resolved": "https://registry.npmjs.org/@smithy/fetch-http-handler/-/fetch-http-handler-3.2.8.tgz", @@ -265,21 +254,6 @@ "node": ">=16.0.0" } }, - "node_modules/@aws-sdk/client-s3/node_modules/@smithy/node-http-handler": { - "version": "3.2.3", - "resolved": "https://registry.npmjs.org/@smithy/node-http-handler/-/node-http-handler-3.2.3.tgz", - "integrity": "sha512-/gcm5DJ3k1b1zEInzBGAZC8ntJ+jwrz1NcSIu+9dSXd1FfG0G6QgkDI40tt8/WYUbHtLyo8fEqtm2v29koWo/w==", - "dependencies": { - "@smithy/abort-controller": "^3.1.4", - "@smithy/protocol-http": "^4.1.3", - "@smithy/querystring-builder": "^3.0.6", - "@smithy/types": "^3.4.2", - "tslib": "^2.6.2" - }, - "engines": { - "node": ">=16.0.0" - } - }, "node_modules/@aws-sdk/client-s3/node_modules/@smithy/property-provider": { "version": "3.1.6", "resolved": "https://registry.npmjs.org/@smithy/property-provider/-/property-provider-3.1.6.tgz", @@ -567,18 +541,6 @@ "@aws-sdk/client-sts": "^3.654.0" } }, - "node_modules/@aws-sdk/client-sso-oidc/node_modules/@smithy/abort-controller": { - "version": "3.1.4", - "resolved": "https://registry.npmjs.org/@smithy/abort-controller/-/abort-controller-3.1.4.tgz", - "integrity": "sha512-VupaALAQlXViW3/enTf/f5l5JZYSAxoJL7f0nanhNNKnww6DGCg1oYIuNP78KDugnkwthBO6iEcym16HhWV8RQ==", - "dependencies": { - "@smithy/types": "^3.4.2", - "tslib": "^2.6.2" - }, - "engines": { - "node": ">=16.0.0" - } - }, "node_modules/@aws-sdk/client-sso-oidc/node_modules/@smithy/fetch-http-handler": { "version": "3.2.8", "resolved": "https://registry.npmjs.org/@smithy/fetch-http-handler/-/fetch-http-handler-3.2.8.tgz", @@ -657,21 +619,6 @@ "node": ">=16.0.0" } }, - "node_modules/@aws-sdk/client-sso-oidc/node_modules/@smithy/node-http-handler": { - "version": "3.2.3", - "resolved": "https://registry.npmjs.org/@smithy/node-http-handler/-/node-http-handler-3.2.3.tgz", - "integrity": "sha512-/gcm5DJ3k1b1zEInzBGAZC8ntJ+jwrz1NcSIu+9dSXd1FfG0G6QgkDI40tt8/WYUbHtLyo8fEqtm2v29koWo/w==", - "dependencies": { - "@smithy/abort-controller": "^3.1.4", - "@smithy/protocol-http": "^4.1.3", - "@smithy/querystring-builder": "^3.0.6", - "@smithy/types": "^3.4.2", - "tslib": "^2.6.2" - }, - "engines": { - "node": ">=16.0.0" - } - }, "node_modules/@aws-sdk/client-sso-oidc/node_modules/@smithy/property-provider": { "version": "3.1.6", "resolved": "https://registry.npmjs.org/@smithy/property-provider/-/property-provider-3.1.6.tgz", @@ -859,18 +806,6 @@ "node": ">=16.0.0" } }, - "node_modules/@aws-sdk/client-sso/node_modules/@smithy/abort-controller": { - "version": "3.1.4", - "resolved": "https://registry.npmjs.org/@smithy/abort-controller/-/abort-controller-3.1.4.tgz", - "integrity": "sha512-VupaALAQlXViW3/enTf/f5l5JZYSAxoJL7f0nanhNNKnww6DGCg1oYIuNP78KDugnkwthBO6iEcym16HhWV8RQ==", - "dependencies": { - "@smithy/types": "^3.4.2", - "tslib": "^2.6.2" - }, - "engines": { - "node": ">=16.0.0" - } - }, "node_modules/@aws-sdk/client-sso/node_modules/@smithy/fetch-http-handler": { "version": "3.2.8", "resolved": "https://registry.npmjs.org/@smithy/fetch-http-handler/-/fetch-http-handler-3.2.8.tgz", @@ -949,21 +884,6 @@ "node": ">=16.0.0" } }, - "node_modules/@aws-sdk/client-sso/node_modules/@smithy/node-http-handler": { - "version": "3.2.3", - "resolved": "https://registry.npmjs.org/@smithy/node-http-handler/-/node-http-handler-3.2.3.tgz", - "integrity": "sha512-/gcm5DJ3k1b1zEInzBGAZC8ntJ+jwrz1NcSIu+9dSXd1FfG0G6QgkDI40tt8/WYUbHtLyo8fEqtm2v29koWo/w==", - "dependencies": { - "@smithy/abort-controller": "^3.1.4", - "@smithy/protocol-http": "^4.1.3", - "@smithy/querystring-builder": "^3.0.6", - "@smithy/types": "^3.4.2", - "tslib": "^2.6.2" - }, - "engines": { - "node": ">=16.0.0" - } - }, "node_modules/@aws-sdk/client-sso/node_modules/@smithy/property-provider": { "version": "3.1.6", "resolved": "https://registry.npmjs.org/@smithy/property-provider/-/property-provider-3.1.6.tgz", @@ -1201,18 +1121,6 @@ "node": ">=16.0.0" } }, - "node_modules/@aws-sdk/client-sts/node_modules/@smithy/abort-controller": { - "version": "3.1.4", - "resolved": "https://registry.npmjs.org/@smithy/abort-controller/-/abort-controller-3.1.4.tgz", - "integrity": "sha512-VupaALAQlXViW3/enTf/f5l5JZYSAxoJL7f0nanhNNKnww6DGCg1oYIuNP78KDugnkwthBO6iEcym16HhWV8RQ==", - "dependencies": { - "@smithy/types": "^3.4.2", - "tslib": "^2.6.2" - }, - "engines": { - "node": ">=16.0.0" - } - }, "node_modules/@aws-sdk/client-sts/node_modules/@smithy/fetch-http-handler": { "version": "3.2.8", "resolved": "https://registry.npmjs.org/@smithy/fetch-http-handler/-/fetch-http-handler-3.2.8.tgz", @@ -1291,21 +1199,6 @@ "node": ">=16.0.0" } }, - "node_modules/@aws-sdk/client-sts/node_modules/@smithy/node-http-handler": { - "version": "3.2.3", - "resolved": "https://registry.npmjs.org/@smithy/node-http-handler/-/node-http-handler-3.2.3.tgz", - "integrity": "sha512-/gcm5DJ3k1b1zEInzBGAZC8ntJ+jwrz1NcSIu+9dSXd1FfG0G6QgkDI40tt8/WYUbHtLyo8fEqtm2v29koWo/w==", - "dependencies": { - "@smithy/abort-controller": "^3.1.4", - "@smithy/protocol-http": "^4.1.3", - "@smithy/querystring-builder": "^3.0.6", - "@smithy/types": "^3.4.2", - "tslib": "^2.6.2" - }, - "engines": { - "node": ">=16.0.0" - } - }, "node_modules/@aws-sdk/client-sts/node_modules/@smithy/property-provider": { "version": "3.1.6", "resolved": "https://registry.npmjs.org/@smithy/property-provider/-/property-provider-3.1.6.tgz", @@ -1513,18 +1406,6 @@ "node": ">=16.0.0" } }, - "node_modules/@aws-sdk/core/node_modules/@smithy/abort-controller": { - "version": "3.1.4", - "resolved": "https://registry.npmjs.org/@smithy/abort-controller/-/abort-controller-3.1.4.tgz", - "integrity": "sha512-VupaALAQlXViW3/enTf/f5l5JZYSAxoJL7f0nanhNNKnww6DGCg1oYIuNP78KDugnkwthBO6iEcym16HhWV8RQ==", - "dependencies": { - "@smithy/types": "^3.4.2", - "tslib": "^2.6.2" - }, - "engines": { - "node": ">=16.0.0" - } - }, "node_modules/@aws-sdk/core/node_modules/@smithy/fetch-http-handler": { "version": "3.2.8", "resolved": "https://registry.npmjs.org/@smithy/fetch-http-handler/-/fetch-http-handler-3.2.8.tgz", @@ -1603,21 +1484,6 @@ "node": ">=16.0.0" } }, - "node_modules/@aws-sdk/core/node_modules/@smithy/node-http-handler": { - "version": "3.2.3", - "resolved": "https://registry.npmjs.org/@smithy/node-http-handler/-/node-http-handler-3.2.3.tgz", - "integrity": "sha512-/gcm5DJ3k1b1zEInzBGAZC8ntJ+jwrz1NcSIu+9dSXd1FfG0G6QgkDI40tt8/WYUbHtLyo8fEqtm2v29koWo/w==", - "dependencies": { - "@smithy/abort-controller": "^3.1.4", - "@smithy/protocol-http": "^4.1.3", - "@smithy/querystring-builder": "^3.0.6", - "@smithy/types": "^3.4.2", - "tslib": "^2.6.2" - }, - "engines": { - "node": ">=16.0.0" - } - }, "node_modules/@aws-sdk/core/node_modules/@smithy/property-provider": { "version": "3.1.6", "resolved": "https://registry.npmjs.org/@smithy/property-provider/-/property-provider-3.1.6.tgz", @@ -1861,18 +1727,6 @@ "node": ">=16.0.0" } }, - "node_modules/@aws-sdk/credential-provider-http/node_modules/@smithy/abort-controller": { - "version": "3.1.4", - "resolved": "https://registry.npmjs.org/@smithy/abort-controller/-/abort-controller-3.1.4.tgz", - "integrity": "sha512-VupaALAQlXViW3/enTf/f5l5JZYSAxoJL7f0nanhNNKnww6DGCg1oYIuNP78KDugnkwthBO6iEcym16HhWV8RQ==", - "dependencies": { - "@smithy/types": "^3.4.2", - "tslib": "^2.6.2" - }, - "engines": { - "node": ">=16.0.0" - } - }, "node_modules/@aws-sdk/credential-provider-http/node_modules/@smithy/fetch-http-handler": { "version": "3.2.8", "resolved": "https://registry.npmjs.org/@smithy/fetch-http-handler/-/fetch-http-handler-3.2.8.tgz", @@ -1951,21 +1805,6 @@ "node": ">=16.0.0" } }, - "node_modules/@aws-sdk/credential-provider-http/node_modules/@smithy/node-http-handler": { - "version": "3.2.3", - "resolved": "https://registry.npmjs.org/@smithy/node-http-handler/-/node-http-handler-3.2.3.tgz", - "integrity": "sha512-/gcm5DJ3k1b1zEInzBGAZC8ntJ+jwrz1NcSIu+9dSXd1FfG0G6QgkDI40tt8/WYUbHtLyo8fEqtm2v29koWo/w==", - "dependencies": { - "@smithy/abort-controller": "^3.1.4", - "@smithy/protocol-http": "^4.1.3", - "@smithy/querystring-builder": "^3.0.6", - "@smithy/types": "^3.4.2", - "tslib": "^2.6.2" - }, - "engines": { - "node": ">=16.0.0" - } - }, "node_modules/@aws-sdk/credential-provider-http/node_modules/@smithy/property-provider": { "version": "3.1.6", "resolved": "https://registry.npmjs.org/@smithy/property-provider/-/property-provider-3.1.6.tgz", @@ -2820,18 +2659,6 @@ "node": ">=16.0.0" } }, - "node_modules/@aws-sdk/middleware-sdk-s3/node_modules/@smithy/abort-controller": { - "version": "3.1.4", - "resolved": "https://registry.npmjs.org/@smithy/abort-controller/-/abort-controller-3.1.4.tgz", - "integrity": "sha512-VupaALAQlXViW3/enTf/f5l5JZYSAxoJL7f0nanhNNKnww6DGCg1oYIuNP78KDugnkwthBO6iEcym16HhWV8RQ==", - "dependencies": { - "@smithy/types": "^3.4.2", - "tslib": "^2.6.2" - }, - "engines": { - "node": ">=16.0.0" - } - }, "node_modules/@aws-sdk/middleware-sdk-s3/node_modules/@smithy/fetch-http-handler": { "version": "3.2.8", "resolved": "https://registry.npmjs.org/@smithy/fetch-http-handler/-/fetch-http-handler-3.2.8.tgz", @@ -2910,21 +2737,6 @@ "node": ">=16.0.0" } }, - "node_modules/@aws-sdk/middleware-sdk-s3/node_modules/@smithy/node-http-handler": { - "version": "3.2.3", - "resolved": "https://registry.npmjs.org/@smithy/node-http-handler/-/node-http-handler-3.2.3.tgz", - "integrity": "sha512-/gcm5DJ3k1b1zEInzBGAZC8ntJ+jwrz1NcSIu+9dSXd1FfG0G6QgkDI40tt8/WYUbHtLyo8fEqtm2v29koWo/w==", - "dependencies": { - "@smithy/abort-controller": "^3.1.4", - "@smithy/protocol-http": "^4.1.3", - "@smithy/querystring-builder": "^3.0.6", - "@smithy/types": "^3.4.2", - "tslib": "^2.6.2" - }, - "engines": { - "node": ">=16.0.0" - } - }, "node_modules/@aws-sdk/middleware-sdk-s3/node_modules/@smithy/property-provider": { "version": "3.1.6", "resolved": "https://registry.npmjs.org/@smithy/property-provider/-/property-provider-3.1.6.tgz", @@ -3530,42 +3342,47 @@ } }, "node_modules/@babel/generator": { - "version": "7.18.2", - "resolved": "https://registry.npmjs.org/@babel/generator/-/generator-7.18.2.tgz", - "integrity": "sha512-W1lG5vUwFvfMd8HVXqdfbuG7RuaSrTCCD8cl8fP8wOivdbtbIg2Db3IWUcgvfxKbbn6ZBGYRW/Zk1MIwK49mgw==", + "version": "7.26.2", + "resolved": "https://registry.npmjs.org/@babel/generator/-/generator-7.26.2.tgz", + "integrity": "sha512-zevQbhbau95nkoxSq3f/DC/SC+EEOUZd3DYqfSkMhY2/wfSeaHV1Ew4vk8e+x8lja31IbyuUa2uQ3JONqKbysw==", "dev": true, "dependencies": { - "@babel/types": "^7.18.2", - "@jridgewell/gen-mapping": "^0.3.0", - "jsesc": "^2.5.1" + "@babel/parser": "^7.26.2", + "@babel/types": "^7.26.0", + "@jridgewell/gen-mapping": "^0.3.5", + "@jridgewell/trace-mapping": "^0.3.25", + "jsesc": "^3.0.2" }, "engines": { "node": ">=6.9.0" } }, "node_modules/@babel/helper-string-parser": { - "version": "7.24.1", - "resolved": "https://registry.npmjs.org/@babel/helper-string-parser/-/helper-string-parser-7.24.1.tgz", - "integrity": "sha512-2ofRCjnnA9y+wk8b9IAREroeUP02KHp431N2mhKniy2yKIDKpbrHv9eXwm8cBeWQYcJmzv5qKCu65P47eCF7CQ==", + "version": "7.25.9", + "resolved": "https://registry.npmjs.org/@babel/helper-string-parser/-/helper-string-parser-7.25.9.tgz", + "integrity": "sha512-4A/SCr/2KLd5jrtOMFzaKjVtAei3+2r/NChoBNoZ3EyP/+GlhoaEGoWOZUmFmoITP7zOJyHIMm+DYRd8o3PvHA==", "dev": true, "engines": { "node": ">=6.9.0" } }, "node_modules/@babel/helper-validator-identifier": { - "version": "7.22.20", - "resolved": "https://registry.npmjs.org/@babel/helper-validator-identifier/-/helper-validator-identifier-7.22.20.tgz", - "integrity": "sha512-Y4OZ+ytlatR8AI+8KZfKuL5urKp7qey08ha31L8b3BwewJAoJamTzyvxPR/5D+KkdJCGPq/+8TukHBlY10FX9A==", + "version": "7.25.9", + "resolved": "https://registry.npmjs.org/@babel/helper-validator-identifier/-/helper-validator-identifier-7.25.9.tgz", + "integrity": "sha512-Ed61U6XJc3CVRfkERJWDz4dJwKe7iLmmJsbOGu9wSloNSFttHV0I8g6UAgb7qnK5ly5bGLPd4oXZlxCdANBOWQ==", "dev": true, "engines": { "node": ">=6.9.0" } }, "node_modules/@babel/parser": { - "version": "7.18.4", - "resolved": "https://registry.npmjs.org/@babel/parser/-/parser-7.18.4.tgz", - "integrity": "sha512-FDge0dFazETFcxGw/EXzOkN8uJp0PC7Qbm+Pe9T+av2zlBpOgunFHkQPPn+eRuClU73JF+98D531UgayY89tow==", + "version": "7.26.2", + "resolved": "https://registry.npmjs.org/@babel/parser/-/parser-7.26.2.tgz", + "integrity": "sha512-DWMCZH9WA4Maitz2q21SRKHo9QXZxkDsbNZoVD62gusNtNBBqDg9i7uOhASfTfIGNzW+O+r7+jAlM8dwphcJKQ==", "dev": true, + "dependencies": { + "@babel/types": "^7.26.0" + }, "bin": { "parser": "bin/babel-parser.js" }, @@ -3574,19 +3391,47 @@ } }, "node_modules/@babel/types": { - "version": "7.19.0", - "resolved": "https://registry.npmjs.org/@babel/types/-/types-7.19.0.tgz", - "integrity": "sha512-YuGopBq3ke25BVSiS6fgF49Ul9gH1x70Bcr6bqRLjWCkcX8Hre1/5+z+IiWOIerRMSSEfGZVB9z9kyq7wVs9YA==", + "version": "7.26.0", + "resolved": "https://registry.npmjs.org/@babel/types/-/types-7.26.0.tgz", + "integrity": "sha512-Z/yiTPj+lDVnF7lWeKCIJzaIkI0vYO87dMpZ4bg4TDrFe4XXLFWL1TbXU27gBP3QccxV9mZICCrnjnYlJjXHOA==", "dev": true, "dependencies": { - "@babel/helper-string-parser": "^7.18.10", - "@babel/helper-validator-identifier": "^7.18.6", - "to-fast-properties": "^2.0.0" + "@babel/helper-string-parser": "^7.25.9", + "@babel/helper-validator-identifier": "^7.25.9" }, "engines": { "node": ">=6.9.0" } }, + "node_modules/@isaacs/cliui": { + "version": "8.0.2", + "resolved": "https://registry.npmjs.org/@isaacs/cliui/-/cliui-8.0.2.tgz", + "integrity": "sha512-O8jcjabXaleOG9DQ0+ARXWZBTfnP4WNAqzuiJK7ll44AmxGKv/J2M4TPjxjY3znBCfvBXFzucm1twdyFybFqEA==", + "dev": true, + "dependencies": { + "string-width": "^5.1.2", + "string-width-cjs": "npm:string-width@^4.2.0", + "strip-ansi": "^7.0.1", + "strip-ansi-cjs": "npm:strip-ansi@^6.0.1", + "wrap-ansi": "^8.1.0", + "wrap-ansi-cjs": "npm:wrap-ansi@^7.0.0" + }, + "engines": { + "node": ">=12" + } + }, + "node_modules/@isaacs/fs-minipass": { + "version": "4.0.1", + "resolved": "https://registry.npmjs.org/@isaacs/fs-minipass/-/fs-minipass-4.0.1.tgz", + "integrity": "sha512-wgm9Ehl2jpeqP3zw/7mo3kRHFp5MEDhqAdwy1fTGkHAwnkGOVsgpvQhL8B5n1qlb01jV3n/bI0ZfZp5lWA1k4w==", + "dev": true, + "dependencies": { + "minipass": "^7.0.4" + }, + "engines": { + "node": ">=18.0.0" + } + }, "node_modules/@jridgewell/gen-mapping": { "version": "0.3.5", "resolved": "https://registry.npmjs.org/@jridgewell/gen-mapping/-/gen-mapping-0.3.5.tgz", @@ -3620,9 +3465,9 @@ } }, "node_modules/@jridgewell/sourcemap-codec": { - "version": "1.4.15", - "resolved": "https://registry.npmjs.org/@jridgewell/sourcemap-codec/-/sourcemap-codec-1.4.15.tgz", - "integrity": "sha512-eF2rxCRulEKXHTRiDrDy6erMYWqNw4LPdQ8UQA4huuxaQsVeRPFl2oM8oDGxMFhJUWZf9McpLtJasDDZb/Bpeg==", + "version": "1.5.0", + "resolved": "https://registry.npmjs.org/@jridgewell/sourcemap-codec/-/sourcemap-codec-1.5.0.tgz", + "integrity": "sha512-gv3ZRaISU3fjPAgNsriBRqGWQL6quFx04YMPW/zD8XMLsU32mhCCbfbO6KZFLjvYpCZ8zyDEgqsgf+PwPaM7GQ==", "dev": true }, "node_modules/@jridgewell/trace-mapping": { @@ -3635,39 +3480,14 @@ "@jridgewell/sourcemap-codec": "^1.4.14" } }, - "node_modules/@nodelib/fs.scandir": { - "version": "2.1.5", - "resolved": "https://registry.npmjs.org/@nodelib/fs.scandir/-/fs.scandir-2.1.5.tgz", - "integrity": "sha512-vq24Bq3ym5HEQm2NKCr3yXDwjc7vTsEThRDnkp2DK9p1uqLR+DHurm/NOTo0KG7HYHU7eppKZj3MyqYuMBf62g==", + "node_modules/@pkgjs/parseargs": { + "version": "0.11.0", + "resolved": "https://registry.npmjs.org/@pkgjs/parseargs/-/parseargs-0.11.0.tgz", + "integrity": "sha512-+1VkjdD0QBLPodGrJUeqarH8VAIvQODIbwh9XpP5Syisf7YoQgsJKPNFoqqLQlu+VQ/tVSshMR6loPMn8U+dPg==", "dev": true, - "dependencies": { - "@nodelib/fs.stat": "2.0.5", - "run-parallel": "^1.1.9" - }, - "engines": { - "node": ">= 8" - } - }, - "node_modules/@nodelib/fs.stat": { - "version": "2.0.5", - "resolved": "https://registry.npmjs.org/@nodelib/fs.stat/-/fs.stat-2.0.5.tgz", - "integrity": "sha512-RkhPPp2zrqDAQA/2jNhnztcPAlv64XdhIp7a7454A5ovI7Bukxgt7MX7udwAu3zg1DcpPU0rz3VV1SeaqvY4+A==", - "dev": true, - "engines": { - "node": ">= 8" - } - }, - "node_modules/@nodelib/fs.walk": { - "version": "1.2.8", - "resolved": "https://registry.npmjs.org/@nodelib/fs.walk/-/fs.walk-1.2.8.tgz", - "integrity": "sha512-oGB+UxlgWcgQkgwo8GcEGwemoTFt3FIO9ababBmaGwXIoBKZ+GTy0pP185beGg7Llih/NSHSV2XAs1lnznocSg==", - "dev": true, - "dependencies": { - "@nodelib/fs.scandir": "2.1.5", - "fastq": "^1.6.0" - }, + "optional": true, "engines": { - "node": ">= 8" + "node": ">=14" } }, "node_modules/@saladtechnologies-oss/salad-cloud-imds-sdk": { @@ -3851,18 +3671,6 @@ "node": ">=16.0.0" } }, - "node_modules/@smithy/core/node_modules/@smithy/abort-controller": { - "version": "3.1.4", - "resolved": "https://registry.npmjs.org/@smithy/abort-controller/-/abort-controller-3.1.4.tgz", - "integrity": "sha512-VupaALAQlXViW3/enTf/f5l5JZYSAxoJL7f0nanhNNKnww6DGCg1oYIuNP78KDugnkwthBO6iEcym16HhWV8RQ==", - "dependencies": { - "@smithy/types": "^3.4.2", - "tslib": "^2.6.2" - }, - "engines": { - "node": ">=16.0.0" - } - }, "node_modules/@smithy/core/node_modules/@smithy/fetch-http-handler": { "version": "3.2.8", "resolved": "https://registry.npmjs.org/@smithy/fetch-http-handler/-/fetch-http-handler-3.2.8.tgz", @@ -3941,21 +3749,6 @@ "node": ">=16.0.0" } }, - "node_modules/@smithy/core/node_modules/@smithy/node-http-handler": { - "version": "3.2.3", - "resolved": "https://registry.npmjs.org/@smithy/node-http-handler/-/node-http-handler-3.2.3.tgz", - "integrity": "sha512-/gcm5DJ3k1b1zEInzBGAZC8ntJ+jwrz1NcSIu+9dSXd1FfG0G6QgkDI40tt8/WYUbHtLyo8fEqtm2v29koWo/w==", - "dependencies": { - "@smithy/abort-controller": "^3.1.4", - "@smithy/protocol-http": "^4.1.3", - "@smithy/querystring-builder": "^3.0.6", - "@smithy/types": "^3.4.2", - "tslib": "^2.6.2" - }, - "engines": { - "node": ">=16.0.0" - } - }, "node_modules/@smithy/core/node_modules/@smithy/property-provider": { "version": "3.1.6", "resolved": "https://registry.npmjs.org/@smithy/property-provider/-/property-provider-3.1.6.tgz", @@ -4669,18 +4462,6 @@ "node": ">=16.0.0" } }, - "node_modules/@smithy/middleware-retry/node_modules/@smithy/abort-controller": { - "version": "3.1.4", - "resolved": "https://registry.npmjs.org/@smithy/abort-controller/-/abort-controller-3.1.4.tgz", - "integrity": "sha512-VupaALAQlXViW3/enTf/f5l5JZYSAxoJL7f0nanhNNKnww6DGCg1oYIuNP78KDugnkwthBO6iEcym16HhWV8RQ==", - "dependencies": { - "@smithy/types": "^3.4.2", - "tslib": "^2.6.2" - }, - "engines": { - "node": ">=16.0.0" - } - }, "node_modules/@smithy/middleware-retry/node_modules/@smithy/fetch-http-handler": { "version": "3.2.8", "resolved": "https://registry.npmjs.org/@smithy/fetch-http-handler/-/fetch-http-handler-3.2.8.tgz", @@ -4759,21 +4540,6 @@ "node": ">=16.0.0" } }, - "node_modules/@smithy/middleware-retry/node_modules/@smithy/node-http-handler": { - "version": "3.2.3", - "resolved": "https://registry.npmjs.org/@smithy/node-http-handler/-/node-http-handler-3.2.3.tgz", - "integrity": "sha512-/gcm5DJ3k1b1zEInzBGAZC8ntJ+jwrz1NcSIu+9dSXd1FfG0G6QgkDI40tt8/WYUbHtLyo8fEqtm2v29koWo/w==", - "dependencies": { - "@smithy/abort-controller": "^3.1.4", - "@smithy/protocol-http": "^4.1.3", - "@smithy/querystring-builder": "^3.0.6", - "@smithy/types": "^3.4.2", - "tslib": "^2.6.2" - }, - "engines": { - "node": ">=16.0.0" - } - }, "node_modules/@smithy/middleware-retry/node_modules/@smithy/property-provider": { "version": "3.1.6", "resolved": "https://registry.npmjs.org/@smithy/property-provider/-/property-provider-3.1.6.tgz", @@ -5000,30 +4766,89 @@ } }, "node_modules/@smithy/node-http-handler": { - "version": "2.5.0", - "resolved": "https://registry.npmjs.org/@smithy/node-http-handler/-/node-http-handler-2.5.0.tgz", - "integrity": "sha512-mVGyPBzkkGQsPoxQUbxlEfRjrj6FPyA3u3u2VXGr9hT8wilsoQdZdvKpMBFMB8Crfhv5dNkKHIW0Yyuc7eABqA==", + "version": "3.3.1", + "resolved": "https://registry.npmjs.org/@smithy/node-http-handler/-/node-http-handler-3.3.1.tgz", + "integrity": "sha512-fr+UAOMGWh6bn4YSEezBCpJn9Ukp9oR4D32sCjCo7U81evE11YePOQ58ogzyfgmjIO79YeOdfXXqr0jyhPQeMg==", "dependencies": { - "@smithy/abort-controller": "^2.2.0", - "@smithy/protocol-http": "^3.3.0", - "@smithy/querystring-builder": "^2.2.0", - "@smithy/types": "^2.12.0", + "@smithy/abort-controller": "^3.1.8", + "@smithy/protocol-http": "^4.1.7", + "@smithy/querystring-builder": "^3.0.10", + "@smithy/types": "^3.7.1", "tslib": "^2.6.2" }, "engines": { - "node": ">=14.0.0" + "node": ">=16.0.0" } }, - "node_modules/@smithy/property-provider": { - "version": "2.2.0", - "resolved": "https://registry.npmjs.org/@smithy/property-provider/-/property-provider-2.2.0.tgz", - "integrity": "sha512-+xiil2lFhtTRzXkx8F053AV46QnIw6e7MV8od5Mi68E1ICOjCeCHw2XfLnDEUHnT9WGUIkwcqavXjfwuJbGlpg==", + "node_modules/@smithy/node-http-handler/node_modules/@smithy/abort-controller": { + "version": "3.1.8", + "resolved": "https://registry.npmjs.org/@smithy/abort-controller/-/abort-controller-3.1.8.tgz", + "integrity": "sha512-+3DOBcUn5/rVjlxGvUPKc416SExarAQ+Qe0bqk30YSUjbepwpS7QN0cyKUSifvLJhdMZ0WPzPP5ymut0oonrpQ==", "dependencies": { - "@smithy/types": "^2.12.0", + "@smithy/types": "^3.7.1", "tslib": "^2.6.2" }, "engines": { - "node": ">=14.0.0" + "node": ">=16.0.0" + } + }, + "node_modules/@smithy/node-http-handler/node_modules/@smithy/protocol-http": { + "version": "4.1.7", + "resolved": "https://registry.npmjs.org/@smithy/protocol-http/-/protocol-http-4.1.7.tgz", + "integrity": "sha512-FP2LepWD0eJeOTm0SjssPcgqAlDFzOmRXqXmGhfIM52G7Lrox/pcpQf6RP4F21k0+O12zaqQt5fCDOeBtqY6Cg==", + "dependencies": { + "@smithy/types": "^3.7.1", + "tslib": "^2.6.2" + }, + "engines": { + "node": ">=16.0.0" + } + }, + "node_modules/@smithy/node-http-handler/node_modules/@smithy/querystring-builder": { + "version": "3.0.10", + "resolved": "https://registry.npmjs.org/@smithy/querystring-builder/-/querystring-builder-3.0.10.tgz", + "integrity": "sha512-nT9CQF3EIJtIUepXQuBFb8dxJi3WVZS3XfuDksxSCSn+/CzZowRLdhDn+2acbBv8R6eaJqPupoI/aRFIImNVPQ==", + "dependencies": { + "@smithy/types": "^3.7.1", + "@smithy/util-uri-escape": "^3.0.0", + "tslib": "^2.6.2" + }, + "engines": { + "node": ">=16.0.0" + } + }, + "node_modules/@smithy/node-http-handler/node_modules/@smithy/types": { + "version": "3.7.1", + "resolved": "https://registry.npmjs.org/@smithy/types/-/types-3.7.1.tgz", + "integrity": "sha512-XKLcLXZY7sUQgvvWyeaL/qwNPp6V3dWcUjqrQKjSb+tzYiCy340R/c64LV5j+Tnb2GhmunEX0eou+L+m2hJNYA==", + "dependencies": { + "tslib": "^2.6.2" + }, + "engines": { + "node": ">=16.0.0" + } + }, + "node_modules/@smithy/node-http-handler/node_modules/@smithy/util-uri-escape": { + "version": "3.0.0", + "resolved": "https://registry.npmjs.org/@smithy/util-uri-escape/-/util-uri-escape-3.0.0.tgz", + "integrity": "sha512-LqR7qYLgZTD7nWLBecUi4aqolw8Mhza9ArpNEQ881MJJIU2sE5iHCK6TdyqqzcDLy0OPe10IY4T8ctVdtynubg==", + "dependencies": { + "tslib": "^2.6.2" + }, + "engines": { + "node": ">=16.0.0" + } + }, + "node_modules/@smithy/property-provider": { + "version": "2.2.0", + "resolved": "https://registry.npmjs.org/@smithy/property-provider/-/property-provider-2.2.0.tgz", + "integrity": "sha512-+xiil2lFhtTRzXkx8F053AV46QnIw6e7MV8od5Mi68E1ICOjCeCHw2XfLnDEUHnT9WGUIkwcqavXjfwuJbGlpg==", + "dependencies": { + "@smithy/types": "^2.12.0", + "tslib": "^2.6.2" + }, + "engines": { + "node": ">=14.0.0" } }, "node_modules/@smithy/protocol-http": { @@ -5314,18 +5139,6 @@ "node": ">= 10.0.0" } }, - "node_modules/@smithy/util-defaults-mode-browser/node_modules/@smithy/abort-controller": { - "version": "3.1.4", - "resolved": "https://registry.npmjs.org/@smithy/abort-controller/-/abort-controller-3.1.4.tgz", - "integrity": "sha512-VupaALAQlXViW3/enTf/f5l5JZYSAxoJL7f0nanhNNKnww6DGCg1oYIuNP78KDugnkwthBO6iEcym16HhWV8RQ==", - "dependencies": { - "@smithy/types": "^3.4.2", - "tslib": "^2.6.2" - }, - "engines": { - "node": ">=16.0.0" - } - }, "node_modules/@smithy/util-defaults-mode-browser/node_modules/@smithy/fetch-http-handler": { "version": "3.2.8", "resolved": "https://registry.npmjs.org/@smithy/fetch-http-handler/-/fetch-http-handler-3.2.8.tgz", @@ -5404,21 +5217,6 @@ "node": ">=16.0.0" } }, - "node_modules/@smithy/util-defaults-mode-browser/node_modules/@smithy/node-http-handler": { - "version": "3.2.3", - "resolved": "https://registry.npmjs.org/@smithy/node-http-handler/-/node-http-handler-3.2.3.tgz", - "integrity": "sha512-/gcm5DJ3k1b1zEInzBGAZC8ntJ+jwrz1NcSIu+9dSXd1FfG0G6QgkDI40tt8/WYUbHtLyo8fEqtm2v29koWo/w==", - "dependencies": { - "@smithy/abort-controller": "^3.1.4", - "@smithy/protocol-http": "^4.1.3", - "@smithy/querystring-builder": "^3.0.6", - "@smithy/types": "^3.4.2", - "tslib": "^2.6.2" - }, - "engines": { - "node": ">=16.0.0" - } - }, "node_modules/@smithy/util-defaults-mode-browser/node_modules/@smithy/property-provider": { "version": "3.1.6", "resolved": "https://registry.npmjs.org/@smithy/property-provider/-/property-provider-3.1.6.tgz", @@ -5623,18 +5421,6 @@ "node": ">= 10.0.0" } }, - "node_modules/@smithy/util-defaults-mode-node/node_modules/@smithy/abort-controller": { - "version": "3.1.4", - "resolved": "https://registry.npmjs.org/@smithy/abort-controller/-/abort-controller-3.1.4.tgz", - "integrity": "sha512-VupaALAQlXViW3/enTf/f5l5JZYSAxoJL7f0nanhNNKnww6DGCg1oYIuNP78KDugnkwthBO6iEcym16HhWV8RQ==", - "dependencies": { - "@smithy/types": "^3.4.2", - "tslib": "^2.6.2" - }, - "engines": { - "node": ">=16.0.0" - } - }, "node_modules/@smithy/util-defaults-mode-node/node_modules/@smithy/fetch-http-handler": { "version": "3.2.8", "resolved": "https://registry.npmjs.org/@smithy/fetch-http-handler/-/fetch-http-handler-3.2.8.tgz", @@ -5713,21 +5499,6 @@ "node": ">=16.0.0" } }, - "node_modules/@smithy/util-defaults-mode-node/node_modules/@smithy/node-http-handler": { - "version": "3.2.3", - "resolved": "https://registry.npmjs.org/@smithy/node-http-handler/-/node-http-handler-3.2.3.tgz", - "integrity": "sha512-/gcm5DJ3k1b1zEInzBGAZC8ntJ+jwrz1NcSIu+9dSXd1FfG0G6QgkDI40tt8/WYUbHtLyo8fEqtm2v29koWo/w==", - "dependencies": { - "@smithy/abort-controller": "^3.1.4", - "@smithy/protocol-http": "^4.1.3", - "@smithy/querystring-builder": "^3.0.6", - "@smithy/types": "^3.4.2", - "tslib": "^2.6.2" - }, - "engines": { - "node": ">=16.0.0" - } - }, "node_modules/@smithy/util-defaults-mode-node/node_modules/@smithy/property-provider": { "version": "3.1.6", "resolved": "https://registry.npmjs.org/@smithy/property-provider/-/property-provider-3.1.6.tgz", @@ -6042,6 +5813,21 @@ "node": ">=14.0.0" } }, + "node_modules/@smithy/util-stream/node_modules/@smithy/node-http-handler": { + "version": "2.5.0", + "resolved": "https://registry.npmjs.org/@smithy/node-http-handler/-/node-http-handler-2.5.0.tgz", + "integrity": "sha512-mVGyPBzkkGQsPoxQUbxlEfRjrj6FPyA3u3u2VXGr9hT8wilsoQdZdvKpMBFMB8Crfhv5dNkKHIW0Yyuc7eABqA==", + "dependencies": { + "@smithy/abort-controller": "^2.2.0", + "@smithy/protocol-http": "^3.3.0", + "@smithy/querystring-builder": "^2.2.0", + "@smithy/types": "^2.12.0", + "tslib": "^2.6.2" + }, + "engines": { + "node": ">=14.0.0" + } + }, "node_modules/@smithy/util-uri-escape": { "version": "2.2.0", "resolved": "https://registry.npmjs.org/@smithy/util-uri-escape/-/util-uri-escape-2.2.0.tgz", @@ -6130,6 +5916,65 @@ "pino": "*" } }, + "node_modules/@yao-pkg/pkg": { + "version": "6.1.1", + "resolved": "https://registry.npmjs.org/@yao-pkg/pkg/-/pkg-6.1.1.tgz", + "integrity": "sha512-AqVKon68/8V81vfGhmRrqUCcMZFzQNWFbBowcKNq82KND7vp17FlDwheRst4wn8bjNA2lwRTFhuQJWHA5EfETw==", + "dev": true, + "dependencies": { + "@babel/generator": "^7.23.0", + "@babel/parser": "^7.23.0", + "@babel/types": "^7.23.0", + "@yao-pkg/pkg-fetch": "3.5.17", + "into-stream": "^6.0.0", + "minimist": "^1.2.6", + "multistream": "^4.1.0", + "picocolors": "^1.1.0", + "picomatch": "^4.0.2", + "prebuild-install": "^7.1.1", + "resolve": "^1.22.0", + "stream-meter": "^1.0.4", + "tar": "^7.4.3", + "tinyglobby": "^0.2.9", + "unzipper": "^0.12.3" + }, + "bin": { + "pkg": "lib-es5/bin.js" + }, + "engines": { + "node": ">=18.0.0" + } + }, + "node_modules/@yao-pkg/pkg-fetch": { + "version": "3.5.17", + "resolved": "https://registry.npmjs.org/@yao-pkg/pkg-fetch/-/pkg-fetch-3.5.17.tgz", + "integrity": "sha512-2gD2K8JUwHwvFFZbwVXwmm90P0U3s8Kqiym4w7t2enTajH28LMhpXaqh/x+SzKeNwvPGoaRUhV0h2nPtWTDoDA==", + "dev": true, + "dependencies": { + "https-proxy-agent": "^5.0.0", + "node-fetch": "^2.6.6", + "picocolors": "^1.1.0", + "progress": "^2.0.3", + "semver": "^7.3.5", + "tar-fs": "^2.1.1", + "yargs": "^16.2.0" + }, + "bin": { + "pkg-fetch": "lib-es5/bin.js" + } + }, + "node_modules/@yao-pkg/pkg/node_modules/picomatch": { + "version": "4.0.2", + "resolved": "https://registry.npmjs.org/picomatch/-/picomatch-4.0.2.tgz", + "integrity": "sha512-M7BAV6Rlcy5u+m6oPhAPFgJTzAioX/6B0DxyvDlo9l8+T3nLKbrczg2WLUyzd45L8RqfUMyGPzekbMvX2Ldkwg==", + "dev": true, + "engines": { + "node": ">=12" + }, + "funding": { + "url": "https://github.com/sponsors/jonschlinkert" + } + }, "node_modules/abort-controller": { "version": "3.0.0", "resolved": "https://registry.npmjs.org/abort-controller/-/abort-controller-3.0.0.tgz", @@ -6154,24 +5999,24 @@ } }, "node_modules/ansi-regex": { - "version": "5.0.1", - "resolved": "https://registry.npmjs.org/ansi-regex/-/ansi-regex-5.0.1.tgz", - "integrity": "sha512-quJQXlTSUGL2LH9SUXo8VwsY4soanhgo6LNSm84E1LBcE8s3O0wpdiRzyR9z/ZZJMlMWv37qOOb9pdJlMUEKFQ==", + "version": "6.1.0", + "resolved": "https://registry.npmjs.org/ansi-regex/-/ansi-regex-6.1.0.tgz", + "integrity": "sha512-7HSX4QQb4CspciLpVFwyRe79O3xsIZDDLER21kERQ71oaPodF8jL725AgJMFAYbooIqolJoRLuM81SpeUkpkvA==", "dev": true, "engines": { - "node": ">=8" + "node": ">=12" + }, + "funding": { + "url": "https://github.com/chalk/ansi-regex?sponsor=1" } }, "node_modules/ansi-styles": { - "version": "4.3.0", - "resolved": "https://registry.npmjs.org/ansi-styles/-/ansi-styles-4.3.0.tgz", - "integrity": "sha512-zbB9rCJAT1rbjiVDb2hqKFHNYLxgtk8NURxZ3IZwD3F6NtxbXZQCnnSi1Lkx+IDohdPlFp222wVALIheZJQSEg==", + "version": "6.2.1", + "resolved": "https://registry.npmjs.org/ansi-styles/-/ansi-styles-6.2.1.tgz", + "integrity": "sha512-bN798gFfQX+viw3R7yrGWRqnrN2oRkEkUjjl4JNn4E8GxxbjtG3FbrEIIY3l8/hrwUwIeCZvi4QuOTP4MErVug==", "dev": true, - "dependencies": { - "color-convert": "^2.0.1" - }, "engines": { - "node": ">=8" + "node": ">=12" }, "funding": { "url": "https://github.com/chalk/ansi-styles?sponsor=1" @@ -6189,24 +6034,6 @@ "node": ">= 8" } }, - "node_modules/array-union": { - "version": "2.1.0", - "resolved": "https://registry.npmjs.org/array-union/-/array-union-2.1.0.tgz", - "integrity": "sha512-HGyxoOTYUyCM6stUe6EJgnd4EoewAI7zMdfqO+kGjnlZmBDz/cR5pf8r/cR4Wq60sL/p0IkcjUEEPwS3GFrIyw==", - "dev": true, - "engines": { - "node": ">=8" - } - }, - "node_modules/at-least-node": { - "version": "1.0.0", - "resolved": "https://registry.npmjs.org/at-least-node/-/at-least-node-1.0.0.tgz", - "integrity": "sha512-+q/t7Ekv1EDY2l6Gda6LLiX14rU9TV20Wa3ofeQmwPFZbOMo9DXrLbOjFaaclkXKWidIaopwAObQDqwWtGUjqg==", - "dev": true, - "engines": { - "node": ">= 4.0.0" - } - }, "node_modules/atomic-sleep": { "version": "1.0.0", "resolved": "https://registry.npmjs.org/atomic-sleep/-/atomic-sleep-1.0.0.tgz", @@ -6215,6 +6042,12 @@ "node": ">=8.0.0" } }, + "node_modules/balanced-match": { + "version": "1.0.2", + "resolved": "https://registry.npmjs.org/balanced-match/-/balanced-match-1.0.2.tgz", + "integrity": "sha512-3oSeUO0TMV67hN1AmbXsK4yaqU7tjiHlbxRDZOpH0KW9+CeX4bRAaX0Anxt0tx2MrpRpWwQaPwIlISEJhYU5Pw==", + "dev": true + }, "node_modules/base64-js": { "version": "1.5.1", "resolved": "https://registry.npmjs.org/base64-js/-/base64-js-1.5.1.tgz", @@ -6256,11 +6089,26 @@ "readable-stream": "^3.4.0" } }, + "node_modules/bluebird": { + "version": "3.7.2", + "resolved": "https://registry.npmjs.org/bluebird/-/bluebird-3.7.2.tgz", + "integrity": "sha512-XpNj6GDQzdfW+r2Wnn7xiSAd7TM3jzkxGXBGTtWKuSXv1xUV+azxAm8jdWZN06QTQk+2N2XB9jRDkvbmQmcRtg==", + "dev": true + }, "node_modules/bowser": { "version": "2.11.0", "resolved": "https://registry.npmjs.org/bowser/-/bowser-2.11.0.tgz", "integrity": "sha512-AlcaJBi/pqqJBIQ8U9Mcpc9i8Aqxn88Skv5d+xBX006BY5u8N3mGLHa5Lgppa7L/HfwgwLgZ6NYs+Ag6uUmJRA==" }, + "node_modules/brace-expansion": { + "version": "2.0.1", + "resolved": "https://registry.npmjs.org/brace-expansion/-/brace-expansion-2.0.1.tgz", + "integrity": "sha512-XnAIvQ8eM+kC6aULx6wuQiwVsnzsi9d3WxzV3FpWTGA19F621kwdbsAcFKXgKUHZWsy+mY6iL1sHTxWEFCytDA==", + "dev": true, + "dependencies": { + "balanced-match": "^1.0.0" + } + }, "node_modules/braces": { "version": "3.0.3", "resolved": "https://registry.npmjs.org/braces/-/braces-3.0.3.tgz", @@ -6281,22 +6129,6 @@ "ieee754": "^1.1.4" } }, - "node_modules/chalk": { - "version": "4.1.2", - "resolved": "https://registry.npmjs.org/chalk/-/chalk-4.1.2.tgz", - "integrity": "sha512-oKnbhFyRIXpUuez8iBMmyEa4nbj4IOQyuhc/wy9kY7/WVPcwIO9VA668Pu8RkO7+0G76SLROeyw9CpQ061i4mA==", - "dev": true, - "dependencies": { - "ansi-styles": "^4.1.0", - "supports-color": "^7.1.0" - }, - "engines": { - "node": ">=10" - }, - "funding": { - "url": "https://github.com/chalk/chalk?sponsor=1" - } - }, "node_modules/chokidar": { "version": "3.6.0", "resolved": "https://registry.npmjs.org/chokidar/-/chokidar-3.6.0.tgz", @@ -6321,10 +6153,13 @@ } }, "node_modules/chownr": { - "version": "1.1.4", - "resolved": "https://registry.npmjs.org/chownr/-/chownr-1.1.4.tgz", - "integrity": "sha512-jJ0bqzaylmJtVnNgzTeSOs8DPavpbYgEr/b0YL8/2GO3xJEhInFmhKMUnEJQjZumK7KXGFhUy89PrsJWlakBVg==", - "dev": true + "version": "3.0.0", + "resolved": "https://registry.npmjs.org/chownr/-/chownr-3.0.0.tgz", + "integrity": "sha512-+IxzY9BZOQd/XuYPRmrvEVjF/nqj5kgT4kEq7VofrDoM1MxoRjEWkrCC3EtLi59TVawxTAn+orJwFQcrqEN1+g==", + "dev": true, + "engines": { + "node": ">=18" + } }, "node_modules/cliui": { "version": "7.0.4", @@ -6337,65 +6172,152 @@ "wrap-ansi": "^7.0.0" } }, - "node_modules/color-convert": { - "version": "2.0.1", - "resolved": "https://registry.npmjs.org/color-convert/-/color-convert-2.0.1.tgz", - "integrity": "sha512-RRECPsj7iu/xb5oKYcsFHSppFNnsj/52OVTRKb4zP5onXwVF3zVmmToNcOfGC+CRDpfK/U584fMg38ZHCaElKQ==", + "node_modules/cliui/node_modules/ansi-regex": { + "version": "5.0.1", + "resolved": "https://registry.npmjs.org/ansi-regex/-/ansi-regex-5.0.1.tgz", + "integrity": "sha512-quJQXlTSUGL2LH9SUXo8VwsY4soanhgo6LNSm84E1LBcE8s3O0wpdiRzyR9z/ZZJMlMWv37qOOb9pdJlMUEKFQ==", + "dev": true, + "engines": { + "node": ">=8" + } + }, + "node_modules/cliui/node_modules/ansi-styles": { + "version": "4.3.0", + "resolved": "https://registry.npmjs.org/ansi-styles/-/ansi-styles-4.3.0.tgz", + "integrity": "sha512-zbB9rCJAT1rbjiVDb2hqKFHNYLxgtk8NURxZ3IZwD3F6NtxbXZQCnnSi1Lkx+IDohdPlFp222wVALIheZJQSEg==", "dev": true, "dependencies": { - "color-name": "~1.1.4" + "color-convert": "^2.0.1" }, "engines": { - "node": ">=7.0.0" + "node": ">=8" + }, + "funding": { + "url": "https://github.com/chalk/ansi-styles?sponsor=1" } }, - "node_modules/color-name": { - "version": "1.1.4", - "resolved": "https://registry.npmjs.org/color-name/-/color-name-1.1.4.tgz", - "integrity": "sha512-dOy+3AuW3a2wNbZHIuMZpTcgjGuLU/uBL/ubcZF9OXbDo8ff4O8yVp5Bf0efS8uEoYo5q4Fx7dY9OgQGXgAsQA==", - "dev": true - }, - "node_modules/core-util-is": { - "version": "1.0.3", - "resolved": "https://registry.npmjs.org/core-util-is/-/core-util-is-1.0.3.tgz", - "integrity": "sha512-ZQBvi1DcpJ4GDqanjucZ2Hj3wEO5pZDS89BWbkcrvdxksJorwUDDZamX9ldFkp9aw2lmBDLgkObEA4DWNJ9FYQ==", + "node_modules/cliui/node_modules/emoji-regex": { + "version": "8.0.0", + "resolved": "https://registry.npmjs.org/emoji-regex/-/emoji-regex-8.0.0.tgz", + "integrity": "sha512-MSjYzcWNOA0ewAHpz0MxpYFvwg6yjy1NG3xteoqz644VCo/RPgnr1/GGt+ic3iJTzQ8Eu3TdM14SawnVUmGE6A==", "dev": true }, - "node_modules/debug": { - "version": "4.3.4", - "resolved": "https://registry.npmjs.org/debug/-/debug-4.3.4.tgz", - "integrity": "sha512-PRWFHuSU3eDtQJPvnNY7Jcket1j0t5OuOsFzPPzsekD52Zl8qUfFIPEiswXqIvHWGVHOgX+7G/vCNNhehwxfkQ==", + "node_modules/cliui/node_modules/string-width": { + "version": "4.2.3", + "resolved": "https://registry.npmjs.org/string-width/-/string-width-4.2.3.tgz", + "integrity": "sha512-wKyQRQpjJ0sIp62ErSZdGsjMJWsap5oRNihHhu6G7JVO/9jIB6UyevL+tXuOqrng8j/cxKTWyWUwvSTriiZz/g==", "dev": true, "dependencies": { - "ms": "2.1.2" + "emoji-regex": "^8.0.0", + "is-fullwidth-code-point": "^3.0.0", + "strip-ansi": "^6.0.1" }, "engines": { - "node": ">=6.0" - }, - "peerDependenciesMeta": { - "supports-color": { - "optional": true - } + "node": ">=8" } }, - "node_modules/decompress-response": { - "version": "6.0.0", - "resolved": "https://registry.npmjs.org/decompress-response/-/decompress-response-6.0.0.tgz", - "integrity": "sha512-aW35yZM6Bb/4oJlZncMH2LCoZtJXTRxES17vE3hoRiowU2kWHaJKFkSBDnDR+cm9J+9QhXmREyIfv0pji9ejCQ==", + "node_modules/cliui/node_modules/strip-ansi": { + "version": "6.0.1", + "resolved": "https://registry.npmjs.org/strip-ansi/-/strip-ansi-6.0.1.tgz", + "integrity": "sha512-Y38VPSHcqkFrCpFnQ9vuSXmquuv5oXOKpGeT6aGrr3o3Gc9AlVa6JBfUSOCnbxGGZF+/0ooI7KrPuUSztUdU5A==", "dev": true, "dependencies": { - "mimic-response": "^3.1.0" + "ansi-regex": "^5.0.1" }, "engines": { - "node": ">=10" - }, - "funding": { - "url": "https://github.com/sponsors/sindresorhus" + "node": ">=8" } }, - "node_modules/deep-extend": { - "version": "0.6.0", - "resolved": "https://registry.npmjs.org/deep-extend/-/deep-extend-0.6.0.tgz", + "node_modules/cliui/node_modules/wrap-ansi": { + "version": "7.0.0", + "resolved": "https://registry.npmjs.org/wrap-ansi/-/wrap-ansi-7.0.0.tgz", + "integrity": "sha512-YVGIj2kamLSTxw6NsZjoBxfSwsn0ycdesmc4p+Q21c5zPuZ1pl+NfxVdxPtdHvmNVOQ6XSYG4AUtyt/Fi7D16Q==", + "dev": true, + "dependencies": { + "ansi-styles": "^4.0.0", + "string-width": "^4.1.0", + "strip-ansi": "^6.0.0" + }, + "engines": { + "node": ">=10" + }, + "funding": { + "url": "https://github.com/chalk/wrap-ansi?sponsor=1" + } + }, + "node_modules/color-convert": { + "version": "2.0.1", + "resolved": "https://registry.npmjs.org/color-convert/-/color-convert-2.0.1.tgz", + "integrity": "sha512-RRECPsj7iu/xb5oKYcsFHSppFNnsj/52OVTRKb4zP5onXwVF3zVmmToNcOfGC+CRDpfK/U584fMg38ZHCaElKQ==", + "dev": true, + "dependencies": { + "color-name": "~1.1.4" + }, + "engines": { + "node": ">=7.0.0" + } + }, + "node_modules/color-name": { + "version": "1.1.4", + "resolved": "https://registry.npmjs.org/color-name/-/color-name-1.1.4.tgz", + "integrity": "sha512-dOy+3AuW3a2wNbZHIuMZpTcgjGuLU/uBL/ubcZF9OXbDo8ff4O8yVp5Bf0efS8uEoYo5q4Fx7dY9OgQGXgAsQA==", + "dev": true + }, + "node_modules/core-util-is": { + "version": "1.0.3", + "resolved": "https://registry.npmjs.org/core-util-is/-/core-util-is-1.0.3.tgz", + "integrity": "sha512-ZQBvi1DcpJ4GDqanjucZ2Hj3wEO5pZDS89BWbkcrvdxksJorwUDDZamX9ldFkp9aw2lmBDLgkObEA4DWNJ9FYQ==", + "dev": true + }, + "node_modules/cross-spawn": { + "version": "7.0.6", + "resolved": "https://registry.npmjs.org/cross-spawn/-/cross-spawn-7.0.6.tgz", + "integrity": "sha512-uV2QOWP2nWzsy2aMp8aRibhi9dlzF5Hgh5SHaB9OiTGEyDTiJJyx0uy51QXdyWbtAHNua4XJzUKca3OzKUd3vA==", + "dev": true, + "dependencies": { + "path-key": "^3.1.0", + "shebang-command": "^2.0.0", + "which": "^2.0.1" + }, + "engines": { + "node": ">= 8" + } + }, + "node_modules/debug": { + "version": "4.3.7", + "resolved": "https://registry.npmjs.org/debug/-/debug-4.3.7.tgz", + "integrity": "sha512-Er2nc/H7RrMXZBFCEim6TCmMk02Z8vLC2Rbi1KEBggpo0fS6l0S1nnapwmIi3yW/+GOJap1Krg4w0Hg80oCqgQ==", + "dev": true, + "dependencies": { + "ms": "^2.1.3" + }, + "engines": { + "node": ">=6.0" + }, + "peerDependenciesMeta": { + "supports-color": { + "optional": true + } + } + }, + "node_modules/decompress-response": { + "version": "6.0.0", + "resolved": "https://registry.npmjs.org/decompress-response/-/decompress-response-6.0.0.tgz", + "integrity": "sha512-aW35yZM6Bb/4oJlZncMH2LCoZtJXTRxES17vE3hoRiowU2kWHaJKFkSBDnDR+cm9J+9QhXmREyIfv0pji9ejCQ==", + "dev": true, + "dependencies": { + "mimic-response": "^3.1.0" + }, + "engines": { + "node": ">=10" + }, + "funding": { + "url": "https://github.com/sponsors/sindresorhus" + } + }, + "node_modules/deep-extend": { + "version": "0.6.0", + "resolved": "https://registry.npmjs.org/deep-extend/-/deep-extend-0.6.0.tgz", "integrity": "sha512-LOHxIOaPYdHlJRtCQfDIVZtfw/ufM8+rVj649RIHzcm/vGwQRXFt6OPqIFWsm2XEMrNIEtWR64sY1LEKD2vAOA==", "dev": true, "engines": { @@ -6411,22 +6333,55 @@ "node": ">=8" } }, - "node_modules/dir-glob": { - "version": "3.0.1", - "resolved": "https://registry.npmjs.org/dir-glob/-/dir-glob-3.0.1.tgz", - "integrity": "sha512-WkrWp9GR4KXfKGYzOLmTuGVi1UWFfws377n9cc55/tb6DuqyF6pcQ5AbiHEshaDpY9v6oaSr2XCDidGmMwdzIA==", + "node_modules/duplexer2": { + "version": "0.1.4", + "resolved": "https://registry.npmjs.org/duplexer2/-/duplexer2-0.1.4.tgz", + "integrity": "sha512-asLFVfWWtJ90ZyOUHMqk7/S2w2guQKxUI2itj3d92ADHhxUSbCMGi1f1cBcJ7xM1To+pE/Khbwo1yuNbMEPKeA==", "dev": true, "dependencies": { - "path-type": "^4.0.0" - }, - "engines": { - "node": ">=8" + "readable-stream": "^2.0.2" + } + }, + "node_modules/duplexer2/node_modules/readable-stream": { + "version": "2.3.8", + "resolved": "https://registry.npmjs.org/readable-stream/-/readable-stream-2.3.8.tgz", + "integrity": "sha512-8p0AUk4XODgIewSi0l8Epjs+EVnWiK7NoDIEGU0HhE7+ZyY8D1IMY7odu5lRrFXGg71L15KG8QrPmum45RTtdA==", + "dev": true, + "dependencies": { + "core-util-is": "~1.0.0", + "inherits": "~2.0.3", + "isarray": "~1.0.0", + "process-nextick-args": "~2.0.0", + "safe-buffer": "~5.1.1", + "string_decoder": "~1.1.1", + "util-deprecate": "~1.0.1" } }, + "node_modules/duplexer2/node_modules/safe-buffer": { + "version": "5.1.2", + "resolved": "https://registry.npmjs.org/safe-buffer/-/safe-buffer-5.1.2.tgz", + "integrity": "sha512-Gd2UZBJDkXlY7GbJxfsE8/nvKkUEU1G38c1siN6QP6a9PT9MmHB8GnpscSmMJSoF8LOIrt8ud/wPtojys4G6+g==", + "dev": true + }, + "node_modules/duplexer2/node_modules/string_decoder": { + "version": "1.1.1", + "resolved": "https://registry.npmjs.org/string_decoder/-/string_decoder-1.1.1.tgz", + "integrity": "sha512-n/ShnvDi6FHbbVfviro+WojiFzv+s8MPMHBczVePfUpDJLwoLT0ht1l4YwBCbi8pJAveEEdnkHyPyTP/mzRfwg==", + "dev": true, + "dependencies": { + "safe-buffer": "~5.1.0" + } + }, + "node_modules/eastasianwidth": { + "version": "0.2.0", + "resolved": "https://registry.npmjs.org/eastasianwidth/-/eastasianwidth-0.2.0.tgz", + "integrity": "sha512-I88TYZWc9XiYHRQ4/3c5rjjfgkjhLyW2luGIheGERbNQ6OY7yTybanSpDXZa8y7VUP9YmDcYa+eyq4ca7iLqWA==", + "dev": true + }, "node_modules/emoji-regex": { - "version": "8.0.0", - "resolved": "https://registry.npmjs.org/emoji-regex/-/emoji-regex-8.0.0.tgz", - "integrity": "sha512-MSjYzcWNOA0ewAHpz0MxpYFvwg6yjy1NG3xteoqz644VCo/RPgnr1/GGt+ic3iJTzQ8Eu3TdM14SawnVUmGE6A==", + "version": "9.2.2", + "resolved": "https://registry.npmjs.org/emoji-regex/-/emoji-regex-9.2.2.tgz", + "integrity": "sha512-L18DaJsXSUk2+42pv8mLs5jJT2hqFkFE4j21wOmgbUqsZ2hL72NsUU785g9RXgo3s0ZNgVl42TiHp3ZtOv/Vyg==", "dev": true }, "node_modules/end-of-stream": { @@ -6439,9 +6394,9 @@ } }, "node_modules/escalade": { - "version": "3.1.2", - "resolved": "https://registry.npmjs.org/escalade/-/escalade-3.1.2.tgz", - "integrity": "sha512-ErCHMCae19vR8vQGe50xIsVomy19rg6gFu3+r3jkEO46suLMWBksvVyoGgQV+jOfl84ZSOSlmv6Gxa89PmTGmA==", + "version": "3.2.0", + "resolved": "https://registry.npmjs.org/escalade/-/escalade-3.2.0.tgz", + "integrity": "sha512-WUj2qlxaQtO4g6Pq5c29GTcWGDyd8itL8zTlipgECz3JesAiiOKotd8JU6otB3PACgG6xkJUyVhboMS+bje/jA==", "dev": true, "engines": { "node": ">=6" @@ -6472,22 +6427,6 @@ "node": ">=6" } }, - "node_modules/fast-glob": { - "version": "3.3.2", - "resolved": "https://registry.npmjs.org/fast-glob/-/fast-glob-3.3.2.tgz", - "integrity": "sha512-oX2ruAFQwf/Orj8m737Y5adxDQO0LAB7/S5MnxCdTNDd4p6BsyIVsv9JQsATbTSq8KHRpLwIHbVlUNatxd+1Ow==", - "dev": true, - "dependencies": { - "@nodelib/fs.stat": "^2.0.2", - "@nodelib/fs.walk": "^1.2.3", - "glob-parent": "^5.1.2", - "merge2": "^1.3.0", - "micromatch": "^4.0.4" - }, - "engines": { - "node": ">=8.6.0" - } - }, "node_modules/fast-redact": { "version": "3.5.0", "resolved": "https://registry.npmjs.org/fast-redact/-/fast-redact-3.5.0.tgz", @@ -6517,15 +6456,6 @@ "fxparser": "src/cli/cli.js" } }, - "node_modules/fastq": { - "version": "1.17.1", - "resolved": "https://registry.npmjs.org/fastq/-/fastq-1.17.1.tgz", - "integrity": "sha512-sRVD3lWVIXWg6By68ZN7vho9a1pQcN/WBFaAAsDDFzlJjvoGx0P8z7V1t72grFJfJhu3YPZBuu25f7Kaw2jN1w==", - "dev": true, - "dependencies": { - "reusify": "^1.0.4" - } - }, "node_modules/fill-range": { "version": "7.1.1", "resolved": "https://registry.npmjs.org/fill-range/-/fill-range-7.1.1.tgz", @@ -6537,6 +6467,22 @@ "node": ">=8" } }, + "node_modules/foreground-child": { + "version": "3.3.0", + "resolved": "https://registry.npmjs.org/foreground-child/-/foreground-child-3.3.0.tgz", + "integrity": "sha512-Ld2g8rrAyMYFXBhEqMz8ZAHBi4J4uS1i/CxGMDnjyFWddMXLVcDp051DZfu+t7+ab7Wv6SMqpWmyFIj5UbfFvg==", + "dev": true, + "dependencies": { + "cross-spawn": "^7.0.0", + "signal-exit": "^4.0.1" + }, + "engines": { + "node": ">=14" + }, + "funding": { + "url": "https://github.com/sponsors/isaacs" + } + }, "node_modules/from2": { "version": "2.3.0", "resolved": "https://registry.npmjs.org/from2/-/from2-2.3.0.tgz", @@ -6584,18 +6530,17 @@ "dev": true }, "node_modules/fs-extra": { - "version": "9.1.0", - "resolved": "https://registry.npmjs.org/fs-extra/-/fs-extra-9.1.0.tgz", - "integrity": "sha512-hcg3ZmepS30/7BSFqRvoo3DOMQu7IjqxO5nCDt+zM9XWjb33Wg7ziNT+Qvqbuc3+gWpzO02JubVyk2G4Zvo1OQ==", + "version": "11.2.0", + "resolved": "https://registry.npmjs.org/fs-extra/-/fs-extra-11.2.0.tgz", + "integrity": "sha512-PmDi3uwK5nFuXh7XDTlVnS17xJS7vW36is2+w3xcv8SVxiB4NyATf4ctkVY5bkSjX0Y4nbvZCq1/EjtEyr9ktw==", "dev": true, "dependencies": { - "at-least-node": "^1.0.0", "graceful-fs": "^4.2.0", "jsonfile": "^6.0.1", "universalify": "^2.0.0" }, "engines": { - "node": ">=10" + "node": ">=14.14" } }, "node_modules/fsevents": { @@ -6635,6 +6580,26 @@ "integrity": "sha512-SyHy3T1v2NUXn29OsWdxmK6RwHD+vkj3v8en8AOBZ1wBQ/hCAQ5bAQTD02kW4W9tUp/3Qh6J8r9EvntiyCmOOw==", "dev": true }, + "node_modules/glob": { + "version": "10.4.5", + "resolved": "https://registry.npmjs.org/glob/-/glob-10.4.5.tgz", + "integrity": "sha512-7Bv8RF0k6xjo7d4A/PxYLbUCfb6c+Vpd2/mB2yRDlew7Jb5hEXiCD9ibfO7wpk8i4sevK6DFny9h7EYbM3/sHg==", + "dev": true, + "dependencies": { + "foreground-child": "^3.1.0", + "jackspeak": "^3.1.2", + "minimatch": "^9.0.4", + "minipass": "^7.1.2", + "package-json-from-dist": "^1.0.0", + "path-scurry": "^1.11.1" + }, + "bin": { + "glob": "dist/esm/bin.mjs" + }, + "funding": { + "url": "https://github.com/sponsors/isaacs" + } + }, "node_modules/glob-parent": { "version": "5.1.2", "resolved": "https://registry.npmjs.org/glob-parent/-/glob-parent-5.1.2.tgz", @@ -6646,50 +6611,12 @@ "node": ">= 6" } }, - "node_modules/globby": { - "version": "11.1.0", - "resolved": "https://registry.npmjs.org/globby/-/globby-11.1.0.tgz", - "integrity": "sha512-jhIXaOzy1sb8IyocaruWSn1TjmnBVs8Ayhcy83rmxNJ8q2uWKCAj3CnJY+KpGSXCueAPc0i05kVvVKtP1t9S3g==", - "dev": true, - "dependencies": { - "array-union": "^2.1.0", - "dir-glob": "^3.0.1", - "fast-glob": "^3.2.9", - "ignore": "^5.2.0", - "merge2": "^1.4.1", - "slash": "^3.0.0" - }, - "engines": { - "node": ">=10" - }, - "funding": { - "url": "https://github.com/sponsors/sindresorhus" - } - }, "node_modules/graceful-fs": { "version": "4.2.11", "resolved": "https://registry.npmjs.org/graceful-fs/-/graceful-fs-4.2.11.tgz", "integrity": "sha512-RbJ5/jmFcNNCcDV5o9eTnBLJ/HszWV0P73bc+Ff4nS/rJj+YaS6IGyiOL0VoBYX+l1Wrl3k63h/KrH+nhJ0XvQ==", "dev": true }, - "node_modules/has": { - "version": "1.0.4", - "resolved": "https://registry.npmjs.org/has/-/has-1.0.4.tgz", - "integrity": "sha512-qdSAmqLF6209RFj4VVItywPMbm3vWylknmB3nvNiUIs72xAimcM8nVYxYr7ncvZq5qzk9MKIZR8ijqD/1QuYjQ==", - "dev": true, - "engines": { - "node": ">= 0.4.0" - } - }, - "node_modules/has-flag": { - "version": "4.0.0", - "resolved": "https://registry.npmjs.org/has-flag/-/has-flag-4.0.0.tgz", - "integrity": "sha512-EykJT/Q1KjTWctppgIAgfSO0tKVuZUjhgMr17kqTumMl6Afv3EISleU7qZUzoXDFTAHTDC4NOoG/ZxU3EvlMPQ==", - "dev": true, - "engines": { - "node": ">=8" - } - }, "node_modules/hasown": { "version": "2.0.2", "resolved": "https://registry.npmjs.org/hasown/-/hasown-2.0.2.tgz", @@ -6734,15 +6661,6 @@ } ] }, - "node_modules/ignore": { - "version": "5.3.1", - "resolved": "https://registry.npmjs.org/ignore/-/ignore-5.3.1.tgz", - "integrity": "sha512-5Fytz/IraMjqpwfd34ke28PTVMjZjJG2MPn5t7OE4eUCUNf8BAa7b5WUS9/Qvr6mwOQS7Mk6vdsMno5he+T8Xw==", - "dev": true, - "engines": { - "node": ">= 4" - } - }, "node_modules/inherits": { "version": "2.0.4", "resolved": "https://registry.npmjs.org/inherits/-/inherits-2.0.4.tgz", @@ -6782,12 +6700,15 @@ } }, "node_modules/is-core-module": { - "version": "2.9.0", - "resolved": "https://registry.npmjs.org/is-core-module/-/is-core-module-2.9.0.tgz", - "integrity": "sha512-+5FPy5PnwmO3lvfMb0AsoPaBG+5KHUI0wYFXOtYPnVVVspTFUuMZNfNaNVRt3FZadstu2c8x23vykRW/NBoU6A==", + "version": "2.15.1", + "resolved": "https://registry.npmjs.org/is-core-module/-/is-core-module-2.15.1.tgz", + "integrity": "sha512-z0vtXSwucUJtANQWldhbtbt7BnL0vxiFjIdDLAatwhDYty2bad6s+rijD6Ri4YuYJubLzIJLUidCh09e1djEVQ==", "dev": true, "dependencies": { - "has": "^1.0.3" + "hasown": "^2.0.2" + }, + "engines": { + "node": ">= 0.4" }, "funding": { "url": "https://github.com/sponsors/ljharb" @@ -6835,16 +6756,37 @@ "integrity": "sha512-VLghIWNM6ELQzo7zwmcg0NmTVyWKYjvIeM83yjp0wRDTmUnrM678fQbcKBo6n2CJEF0szoG//ytg+TKla89ALQ==", "dev": true }, + "node_modules/isexe": { + "version": "2.0.0", + "resolved": "https://registry.npmjs.org/isexe/-/isexe-2.0.0.tgz", + "integrity": "sha512-RHxMLp9lnKHGHRng9QFhRCMbYAcVpn69smSGcq3f36xjgVVWThj4qqLbTLlq7Ssj8B+fIQ1EuCEGI2lKsyQeIw==", + "dev": true + }, + "node_modules/jackspeak": { + "version": "3.4.3", + "resolved": "https://registry.npmjs.org/jackspeak/-/jackspeak-3.4.3.tgz", + "integrity": "sha512-OGlZQpz2yfahA/Rd1Y8Cd9SIEsqvXkLVoSw/cgwhnhFMDbsQFeZYoJJ7bIZBS9BcamUW96asq/npPWugM+RQBw==", + "dev": true, + "dependencies": { + "@isaacs/cliui": "^8.0.2" + }, + "funding": { + "url": "https://github.com/sponsors/isaacs" + }, + "optionalDependencies": { + "@pkgjs/parseargs": "^0.11.0" + } + }, "node_modules/jsesc": { - "version": "2.5.2", - "resolved": "https://registry.npmjs.org/jsesc/-/jsesc-2.5.2.tgz", - "integrity": "sha512-OYu7XEzjkCQ3C5Ps3QIZsQfNpqoJyZZA99wd9aWd05NCtC5pWOkShK2mkL6HXQR6/Cy2lbNdPlZBpuQHXE63gA==", + "version": "3.0.2", + "resolved": "https://registry.npmjs.org/jsesc/-/jsesc-3.0.2.tgz", + "integrity": "sha512-xKqzzWXDttJuOcawBt4KnKHHIf5oQ/Cxax+0PWFG+DFDgHNAdi+TXECADI+RYiFUMmx8792xsMbbgXj4CwnP4g==", "dev": true, "bin": { "jsesc": "bin/jsesc" }, "engines": { - "node": ">=4" + "node": ">=6" } }, "node_modules/jsonfile": { @@ -6860,58 +6802,82 @@ } }, "node_modules/lru-cache": { - "version": "6.0.0", - "resolved": "https://registry.npmjs.org/lru-cache/-/lru-cache-6.0.0.tgz", - "integrity": "sha512-Jo6dJ04CmSjuznwJSS3pUeWmd/H0ffTlkXXgwZi+eq1UCmqQwCh+eLsYOYCwY991i2Fah4h1BEMCx4qThGbsiA==", + "version": "10.4.3", + "resolved": "https://registry.npmjs.org/lru-cache/-/lru-cache-10.4.3.tgz", + "integrity": "sha512-JNAzZcXrCt42VGLuYz0zfAzDfAvJWW6AfYlDBQyDV5DClI2m5sAmK+OIO7s59XfsRsWHp02jAJrRadPRGTt6SQ==", + "dev": true + }, + "node_modules/mimic-response": { + "version": "3.1.0", + "resolved": "https://registry.npmjs.org/mimic-response/-/mimic-response-3.1.0.tgz", + "integrity": "sha512-z0yWI+4FDrrweS8Zmt4Ej5HdJmky15+L2e6Wgn3+iK5fWzb6T3fhNFq2+MeTRb064c6Wr4N/wv0DzQTjNzHNGQ==", + "dev": true, + "engines": { + "node": ">=10" + }, + "funding": { + "url": "https://github.com/sponsors/sindresorhus" + } + }, + "node_modules/minimatch": { + "version": "9.0.5", + "resolved": "https://registry.npmjs.org/minimatch/-/minimatch-9.0.5.tgz", + "integrity": "sha512-G6T0ZX48xgozx7587koeX9Ys2NYy6Gmv//P89sEte9V9whIapMNF4idKxnW2QtCcLiTWlb/wfCabAtAFWhhBow==", "dev": true, "dependencies": { - "yallist": "^4.0.0" + "brace-expansion": "^2.0.1" }, "engines": { - "node": ">=10" + "node": ">=16 || 14 >=14.17" + }, + "funding": { + "url": "https://github.com/sponsors/isaacs" + } + }, + "node_modules/minimist": { + "version": "1.2.8", + "resolved": "https://registry.npmjs.org/minimist/-/minimist-1.2.8.tgz", + "integrity": "sha512-2yyAR8qBkN3YuheJanUpWC5U3bb5osDywNB8RzDVlDwDHbocAJveqqj1u8+SVD7jkWT4yvsHCpWqqWqAxb0zCA==", + "dev": true, + "funding": { + "url": "https://github.com/sponsors/ljharb" } }, - "node_modules/merge2": { - "version": "1.4.1", - "resolved": "https://registry.npmjs.org/merge2/-/merge2-1.4.1.tgz", - "integrity": "sha512-8q7VEgMJW4J8tcfVPy8g09NcQwZdbwFEqhe/WZkoIzjn/3TGDwtOCYtXGxA3O8tPzpczCCDgv+P2P5y00ZJOOg==", + "node_modules/minipass": { + "version": "7.1.2", + "resolved": "https://registry.npmjs.org/minipass/-/minipass-7.1.2.tgz", + "integrity": "sha512-qOOzS1cBTWYF4BH8fVePDBOO9iptMnGUEZwNc/cMWnTV2nVLZ7VoNWEPHkYczZA0pdoA7dl6e7FL659nX9S2aw==", "dev": true, "engines": { - "node": ">= 8" + "node": ">=16 || 14 >=14.17" } }, - "node_modules/micromatch": { - "version": "4.0.8", - "resolved": "https://registry.npmjs.org/micromatch/-/micromatch-4.0.8.tgz", - "integrity": "sha512-PXwfBhYu0hBCPw8Dn0E+WDYb7af3dSLVWKi3HGv84IdF4TyFoC0ysxFd0Goxw7nSv4T/PzEJQxsYsEiFCKo2BA==", + "node_modules/minizlib": { + "version": "3.0.1", + "resolved": "https://registry.npmjs.org/minizlib/-/minizlib-3.0.1.tgz", + "integrity": "sha512-umcy022ILvb5/3Djuu8LWeqUa8D68JaBzlttKeMWen48SjabqS3iY5w/vzeMzMUNhLDifyhbOwKDSznB1vvrwg==", "dev": true, "dependencies": { - "braces": "^3.0.3", - "picomatch": "^2.3.1" + "minipass": "^7.0.4", + "rimraf": "^5.0.5" }, "engines": { - "node": ">=8.6" + "node": ">= 18" } }, - "node_modules/mimic-response": { - "version": "3.1.0", - "resolved": "https://registry.npmjs.org/mimic-response/-/mimic-response-3.1.0.tgz", - "integrity": "sha512-z0yWI+4FDrrweS8Zmt4Ej5HdJmky15+L2e6Wgn3+iK5fWzb6T3fhNFq2+MeTRb064c6Wr4N/wv0DzQTjNzHNGQ==", + "node_modules/mkdirp": { + "version": "3.0.1", + "resolved": "https://registry.npmjs.org/mkdirp/-/mkdirp-3.0.1.tgz", + "integrity": "sha512-+NsyUUAZDmo6YVHzL/stxSu3t9YS1iljliy3BSDrXJ/dkn1KYdmtZODGGjLcc9XLgVVpH4KshHB8XmZgMhaBXg==", "dev": true, + "bin": { + "mkdirp": "dist/cjs/src/bin.js" + }, "engines": { "node": ">=10" }, "funding": { - "url": "https://github.com/sponsors/sindresorhus" - } - }, - "node_modules/minimist": { - "version": "1.2.8", - "resolved": "https://registry.npmjs.org/minimist/-/minimist-1.2.8.tgz", - "integrity": "sha512-2yyAR8qBkN3YuheJanUpWC5U3bb5osDywNB8RzDVlDwDHbocAJveqqj1u8+SVD7jkWT4yvsHCpWqqWqAxb0zCA==", - "dev": true, - "funding": { - "url": "https://github.com/sponsors/ljharb" + "url": "https://github.com/sponsors/isaacs" } }, "node_modules/mkdirp-classic": { @@ -6921,9 +6887,9 @@ "dev": true }, "node_modules/ms": { - "version": "2.1.2", - "resolved": "https://registry.npmjs.org/ms/-/ms-2.1.2.tgz", - "integrity": "sha512-sGkPx+VjMtmA6MX27oA4FBFELFCZZ4S4XqeGOXCv68tT+jb3vk/RyaKWP0PTKyWtmLSM0b+adUTEvbs1PEaH2w==", + "version": "2.1.3", + "resolved": "https://registry.npmjs.org/ms/-/ms-2.1.3.tgz", + "integrity": "sha512-6FlzubTLZG3J2a/NVCAleEhjzq5oxgHyaCU9yYXvcLsvoVaHJq/s5xXI6/XXP6tz7R9xAOtHnSO/tXtF3WRTlA==", "dev": true }, "node_modules/multistream": { @@ -6957,9 +6923,9 @@ "dev": true }, "node_modules/node-abi": { - "version": "3.59.0", - "resolved": "https://registry.npmjs.org/node-abi/-/node-abi-3.59.0.tgz", - "integrity": "sha512-HyyfzvTLCE8b1SX2nWimlra8cibEsypcSu/Az4SXMhWhtuctkwAX7qsEYNjUOIoYtPV884oN3wtYTN+iZKBtvw==", + "version": "3.71.0", + "resolved": "https://registry.npmjs.org/node-abi/-/node-abi-3.71.0.tgz", + "integrity": "sha512-SZ40vRiy/+wRTf21hxkkEjPJZpARzUMVcJoQse2EF8qkUWbbO2z7vd5oA/H6bVH6SZQ5STGcu0KRDS7biNRfxw==", "dev": true, "dependencies": { "semver": "^7.3.5" @@ -6988,6 +6954,12 @@ } } }, + "node_modules/node-int64": { + "version": "0.4.0", + "resolved": "https://registry.npmjs.org/node-int64/-/node-int64-0.4.0.tgz", + "integrity": "sha512-O5lz91xSOeoXP6DulyHfllpq+Eg00MWitZIbtPfoSEvqIHdl5gfcY6hYzDWnj0qD5tz52PI08u9qUvSVeUBeHw==", + "dev": true + }, "node_modules/normalize-path": { "version": "3.0.0", "resolved": "https://registry.npmjs.org/normalize-path/-/normalize-path-3.0.0.tgz", @@ -7022,21 +6994,49 @@ "node": ">=8" } }, + "node_modules/package-json-from-dist": { + "version": "1.0.1", + "resolved": "https://registry.npmjs.org/package-json-from-dist/-/package-json-from-dist-1.0.1.tgz", + "integrity": "sha512-UEZIS3/by4OC8vL3P2dTXRETpebLI2NiI5vIrjaD/5UtrkFX/tNbwjTSRAGC/+7CAo2pIcBaRgWmcBBHcsaCIw==", + "dev": true + }, + "node_modules/path-key": { + "version": "3.1.1", + "resolved": "https://registry.npmjs.org/path-key/-/path-key-3.1.1.tgz", + "integrity": "sha512-ojmeN0qd+y0jszEtoY48r0Peq5dwMEkIlCOu6Q5f41lfkswXuKtYrhgoTpLnyIcHm24Uhqx+5Tqm2InSwLhE6Q==", + "dev": true, + "engines": { + "node": ">=8" + } + }, "node_modules/path-parse": { "version": "1.0.7", "resolved": "https://registry.npmjs.org/path-parse/-/path-parse-1.0.7.tgz", "integrity": "sha512-LDJzPVEEEPR+y48z93A0Ed0yXb8pAByGWo/k5YYdYgpY2/2EsOsksJrq7lOHxryrVOn1ejG6oAp8ahvOIQD8sw==", "dev": true }, - "node_modules/path-type": { - "version": "4.0.0", - "resolved": "https://registry.npmjs.org/path-type/-/path-type-4.0.0.tgz", - "integrity": "sha512-gDKb8aZMDeD/tZWs9P6+q0J9Mwkdl6xMV8TjnGP3qJVJ06bdMgkbBlLU8IdfOsIsFz2BW1rNVT3XuNEl8zPAvw==", + "node_modules/path-scurry": { + "version": "1.11.1", + "resolved": "https://registry.npmjs.org/path-scurry/-/path-scurry-1.11.1.tgz", + "integrity": "sha512-Xa4Nw17FS9ApQFJ9umLiJS4orGjm7ZzwUrwamcGQuHSzDyth9boKDaycYdDcZDuqYATXw4HFXgaqWTctW/v1HA==", "dev": true, + "dependencies": { + "lru-cache": "^10.2.0", + "minipass": "^5.0.0 || ^6.0.2 || ^7.0.0" + }, "engines": { - "node": ">=8" + "node": ">=16 || 14 >=14.18" + }, + "funding": { + "url": "https://github.com/sponsors/isaacs" } }, + "node_modules/picocolors": { + "version": "1.1.1", + "resolved": "https://registry.npmjs.org/picocolors/-/picocolors-1.1.1.tgz", + "integrity": "sha512-xceH2snhtb5M9liqDsmEw56le376mTZkEX/jEb/RxNFyegNul7eNslCXP9FDj/Lcu0X8KEyMceP2ntpaHrDEVA==", + "dev": true + }, "node_modules/picomatch": { "version": "2.3.1", "resolved": "https://registry.npmjs.org/picomatch/-/picomatch-2.3.1.tgz", @@ -7121,62 +7121,10 @@ "resolved": "https://registry.npmjs.org/pino-std-serializers/-/pino-std-serializers-6.2.2.tgz", "integrity": "sha512-cHjPPsE+vhj/tnhCy/wiMh3M3z3h/j15zHQX+S9GkTBgqJuTuJzYJ4gUyACLhDaJ7kk9ba9iRDmbH2tJU03OiA==" }, - "node_modules/pkg": { - "version": "5.8.1", - "resolved": "https://registry.npmjs.org/pkg/-/pkg-5.8.1.tgz", - "integrity": "sha512-CjBWtFStCfIiT4Bde9QpJy0KeH19jCfwZRJqHFDFXfhUklCx8JoFmMj3wgnEYIwGmZVNkhsStPHEOnrtrQhEXA==", - "dev": true, - "dependencies": { - "@babel/generator": "7.18.2", - "@babel/parser": "7.18.4", - "@babel/types": "7.19.0", - "chalk": "^4.1.2", - "fs-extra": "^9.1.0", - "globby": "^11.1.0", - "into-stream": "^6.0.0", - "is-core-module": "2.9.0", - "minimist": "^1.2.6", - "multistream": "^4.1.0", - "pkg-fetch": "3.4.2", - "prebuild-install": "7.1.1", - "resolve": "^1.22.0", - "stream-meter": "^1.0.4" - }, - "bin": { - "pkg": "lib-es5/bin.js" - }, - "peerDependencies": { - "node-notifier": ">=9.0.1" - }, - "peerDependenciesMeta": { - "node-notifier": { - "optional": true - } - } - }, - "node_modules/pkg-fetch": { - "version": "3.4.2", - "resolved": "https://registry.npmjs.org/pkg-fetch/-/pkg-fetch-3.4.2.tgz", - "integrity": "sha512-0+uijmzYcnhC0hStDjm/cl2VYdrmVVBpe7Q8k9YBojxmR5tG8mvR9/nooQq3QSXiQqORDVOTY3XqMEqJVIzkHA==", - "dev": true, - "dependencies": { - "chalk": "^4.1.2", - "fs-extra": "^9.1.0", - "https-proxy-agent": "^5.0.0", - "node-fetch": "^2.6.6", - "progress": "^2.0.3", - "semver": "^7.3.5", - "tar-fs": "^2.1.1", - "yargs": "^16.2.0" - }, - "bin": { - "pkg-fetch": "lib-es5/bin.js" - } - }, - "node_modules/prebuild-install": { - "version": "7.1.1", - "resolved": "https://registry.npmjs.org/prebuild-install/-/prebuild-install-7.1.1.tgz", - "integrity": "sha512-jAXscXWMcCK8GgCoHOfIr0ODh5ai8mj63L2nWrjuAgXE6tDyYGnx4/8o/rCgU+B4JSyZBKbeZqzhtwtC3ovxjw==", + "node_modules/prebuild-install": { + "version": "7.1.2", + "resolved": "https://registry.npmjs.org/prebuild-install/-/prebuild-install-7.1.2.tgz", + "integrity": "sha512-UnNke3IQb6sgarcZIDU3gbMeTp/9SSU1DAIkil7PrqG1vZlBtY5msYccSKSHDqa3hNg436IXK+SNImReuA1wEQ==", "dev": true, "dependencies": { "detect-libc": "^2.0.0", @@ -7228,35 +7176,15 @@ } }, "node_modules/pump": { - "version": "3.0.0", - "resolved": "https://registry.npmjs.org/pump/-/pump-3.0.0.tgz", - "integrity": "sha512-LwZy+p3SFs1Pytd/jYct4wpv49HiYCqd9Rlc5ZVdk0V+8Yzv6jR5Blk3TRmPL1ft69TxP0IMZGJ+WPFU2BFhww==", + "version": "3.0.2", + "resolved": "https://registry.npmjs.org/pump/-/pump-3.0.2.tgz", + "integrity": "sha512-tUPXtzlGM8FE3P0ZL6DVs/3P58k9nk8/jZeQCurTJylQA8qFYzHFfhBJkuqyE0FifOsQ0uKWekiZ5g8wtr28cw==", "dev": true, "dependencies": { "end-of-stream": "^1.1.0", "once": "^1.3.1" } }, - "node_modules/queue-microtask": { - "version": "1.2.3", - "resolved": "https://registry.npmjs.org/queue-microtask/-/queue-microtask-1.2.3.tgz", - "integrity": "sha512-NuaNSa6flKT5JaSYQzJok04JzTL1CA6aGhv5rfLW3PgqA+M2ChpZQnAC8h8i4ZFkBS8X5RqkDBHA7r4hej3K9A==", - "dev": true, - "funding": [ - { - "type": "github", - "url": "https://github.com/sponsors/feross" - }, - { - "type": "patreon", - "url": "https://www.patreon.com/feross" - }, - { - "type": "consulting", - "url": "https://feross.org/support" - } - ] - }, "node_modules/quick-format-unescaped": { "version": "4.0.4", "resolved": "https://registry.npmjs.org/quick-format-unescaped/-/quick-format-unescaped-4.0.4.tgz", @@ -7335,49 +7263,19 @@ "url": "https://github.com/sponsors/ljharb" } }, - "node_modules/resolve/node_modules/is-core-module": { - "version": "2.13.1", - "resolved": "https://registry.npmjs.org/is-core-module/-/is-core-module-2.13.1.tgz", - "integrity": "sha512-hHrIjvZsftOsvKSn2TRYl63zvxsgE0K+0mYMoH6gD4omR5IWB2KynivBQczo3+wF1cCkjzvptnI9Q0sPU66ilw==", + "node_modules/rimraf": { + "version": "5.0.10", + "resolved": "https://registry.npmjs.org/rimraf/-/rimraf-5.0.10.tgz", + "integrity": "sha512-l0OE8wL34P4nJH/H2ffoaniAokM2qSmrtXHmlpvYr5AVVX8msAyW0l8NVJFDxlSK4u3Uh/f41cQheDVdnYijwQ==", "dev": true, "dependencies": { - "hasown": "^2.0.0" + "glob": "^10.3.7" + }, + "bin": { + "rimraf": "dist/esm/bin.mjs" }, "funding": { - "url": "https://github.com/sponsors/ljharb" - } - }, - "node_modules/reusify": { - "version": "1.0.4", - "resolved": "https://registry.npmjs.org/reusify/-/reusify-1.0.4.tgz", - "integrity": "sha512-U9nH88a3fc/ekCF1l0/UP1IosiuIjyTh7hBvXVMHYgVcfGvt897Xguj2UOLDeI5BG2m7/uwyaLVT6fbtCwTyzw==", - "dev": true, - "engines": { - "iojs": ">=1.0.0", - "node": ">=0.10.0" - } - }, - "node_modules/run-parallel": { - "version": "1.2.0", - "resolved": "https://registry.npmjs.org/run-parallel/-/run-parallel-1.2.0.tgz", - "integrity": "sha512-5l4VyZR86LZ/lDxZTR6jqL8AFE2S0IFLMP26AbjsLVADxHdhB/c0GUsH+y39UfCi3dzz8OlQuPmnaJOMoDHQBA==", - "dev": true, - "funding": [ - { - "type": "github", - "url": "https://github.com/sponsors/feross" - }, - { - "type": "patreon", - "url": "https://www.patreon.com/feross" - }, - { - "type": "consulting", - "url": "https://feross.org/support" - } - ], - "dependencies": { - "queue-microtask": "^1.2.2" + "url": "https://github.com/sponsors/isaacs" } }, "node_modules/safe-buffer": { @@ -7408,13 +7306,10 @@ } }, "node_modules/semver": { - "version": "7.6.0", - "resolved": "https://registry.npmjs.org/semver/-/semver-7.6.0.tgz", - "integrity": "sha512-EnwXhrlwXMk9gKu5/flx5sv/an57AkRplG3hTK68W7FRDN+k+OWBj65M7719OkA82XLBxrcX0KSHj+X5COhOVg==", + "version": "7.6.3", + "resolved": "https://registry.npmjs.org/semver/-/semver-7.6.3.tgz", + "integrity": "sha512-oVekP1cKtI+CTDvHWYFUcMtsK/00wmAEfyqKfNdARm8u1wNVhSgaX7A8d4UuIlUI5e84iEwOhs7ZPYRmzU9U6A==", "dev": true, - "dependencies": { - "lru-cache": "^6.0.0" - }, "bin": { "semver": "bin/semver.js" }, @@ -7422,6 +7317,39 @@ "node": ">=10" } }, + "node_modules/shebang-command": { + "version": "2.0.0", + "resolved": "https://registry.npmjs.org/shebang-command/-/shebang-command-2.0.0.tgz", + "integrity": "sha512-kHxr2zZpYtdmrN1qDjrrX/Z1rR1kG8Dx+gkpK1G4eXmvXswmcE1hTWBWYUzlraYw1/yZp6YuDY77YtvbN0dmDA==", + "dev": true, + "dependencies": { + "shebang-regex": "^3.0.0" + }, + "engines": { + "node": ">=8" + } + }, + "node_modules/shebang-regex": { + "version": "3.0.0", + "resolved": "https://registry.npmjs.org/shebang-regex/-/shebang-regex-3.0.0.tgz", + "integrity": "sha512-7++dFhtcx3353uBaq8DDR4NuxBetBzC7ZQOhmTQInHEd6bSrXdiEyzCvG07Z44UYdLShWUyXt5M/yhz8ekcb1A==", + "dev": true, + "engines": { + "node": ">=8" + } + }, + "node_modules/signal-exit": { + "version": "4.1.0", + "resolved": "https://registry.npmjs.org/signal-exit/-/signal-exit-4.1.0.tgz", + "integrity": "sha512-bzyZ1e88w9O1iNJbKnOlvYTrWPDl46O1bG0D3XInv+9tkPrxrN8jUUTiFlDkkmKWgn1M6CfIA13SuGqOa9Korw==", + "dev": true, + "engines": { + "node": ">=14" + }, + "funding": { + "url": "https://github.com/sponsors/isaacs" + } + }, "node_modules/simple-concat": { "version": "1.0.1", "resolved": "https://registry.npmjs.org/simple-concat/-/simple-concat-1.0.1.tgz", @@ -7467,15 +7395,6 @@ "simple-concat": "^1.0.0" } }, - "node_modules/slash": { - "version": "3.0.0", - "resolved": "https://registry.npmjs.org/slash/-/slash-3.0.0.tgz", - "integrity": "sha512-g9Q1haeby36OSStwb4ntCGGGaKsaVSjQ68fBxoQcutl5fS1vuY18H3wSt3jFyFtrkx+Kz0V1G85A4MyAdDMi2Q==", - "dev": true, - "engines": { - "node": ">=8" - } - }, "node_modules/sonic-boom": { "version": "3.8.1", "resolved": "https://registry.npmjs.org/sonic-boom/-/sonic-boom-3.8.1.tgz", @@ -7549,6 +7468,24 @@ } }, "node_modules/string-width": { + "version": "5.1.2", + "resolved": "https://registry.npmjs.org/string-width/-/string-width-5.1.2.tgz", + "integrity": "sha512-HnLOCR3vjcY8beoNLtcjZ5/nxn2afmME6lhrDrebokqMap+XbeW8n9TXpPDOqdGK5qcI3oT0GKTW6wC7EMiVqA==", + "dev": true, + "dependencies": { + "eastasianwidth": "^0.2.0", + "emoji-regex": "^9.2.2", + "strip-ansi": "^7.0.1" + }, + "engines": { + "node": ">=12" + }, + "funding": { + "url": "https://github.com/sponsors/sindresorhus" + } + }, + "node_modules/string-width-cjs": { + "name": "string-width", "version": "4.2.3", "resolved": "https://registry.npmjs.org/string-width/-/string-width-4.2.3.tgz", "integrity": "sha512-wKyQRQpjJ0sIp62ErSZdGsjMJWsap5oRNihHhu6G7JVO/9jIB6UyevL+tXuOqrng8j/cxKTWyWUwvSTriiZz/g==", @@ -7562,7 +7499,50 @@ "node": ">=8" } }, + "node_modules/string-width-cjs/node_modules/ansi-regex": { + "version": "5.0.1", + "resolved": "https://registry.npmjs.org/ansi-regex/-/ansi-regex-5.0.1.tgz", + "integrity": "sha512-quJQXlTSUGL2LH9SUXo8VwsY4soanhgo6LNSm84E1LBcE8s3O0wpdiRzyR9z/ZZJMlMWv37qOOb9pdJlMUEKFQ==", + "dev": true, + "engines": { + "node": ">=8" + } + }, + "node_modules/string-width-cjs/node_modules/emoji-regex": { + "version": "8.0.0", + "resolved": "https://registry.npmjs.org/emoji-regex/-/emoji-regex-8.0.0.tgz", + "integrity": "sha512-MSjYzcWNOA0ewAHpz0MxpYFvwg6yjy1NG3xteoqz644VCo/RPgnr1/GGt+ic3iJTzQ8Eu3TdM14SawnVUmGE6A==", + "dev": true + }, + "node_modules/string-width-cjs/node_modules/strip-ansi": { + "version": "6.0.1", + "resolved": "https://registry.npmjs.org/strip-ansi/-/strip-ansi-6.0.1.tgz", + "integrity": "sha512-Y38VPSHcqkFrCpFnQ9vuSXmquuv5oXOKpGeT6aGrr3o3Gc9AlVa6JBfUSOCnbxGGZF+/0ooI7KrPuUSztUdU5A==", + "dev": true, + "dependencies": { + "ansi-regex": "^5.0.1" + }, + "engines": { + "node": ">=8" + } + }, "node_modules/strip-ansi": { + "version": "7.1.0", + "resolved": "https://registry.npmjs.org/strip-ansi/-/strip-ansi-7.1.0.tgz", + "integrity": "sha512-iq6eVVI64nQQTRYq2KtEg2d2uU7LElhTJwsH4YzIHZshxlgZms/wIc4VoDQTlG/IvVIrBKG06CrZnp0qv7hkcQ==", + "dev": true, + "dependencies": { + "ansi-regex": "^6.0.1" + }, + "engines": { + "node": ">=12" + }, + "funding": { + "url": "https://github.com/chalk/strip-ansi?sponsor=1" + } + }, + "node_modules/strip-ansi-cjs": { + "name": "strip-ansi", "version": "6.0.1", "resolved": "https://registry.npmjs.org/strip-ansi/-/strip-ansi-6.0.1.tgz", "integrity": "sha512-Y38VPSHcqkFrCpFnQ9vuSXmquuv5oXOKpGeT6aGrr3o3Gc9AlVa6JBfUSOCnbxGGZF+/0ooI7KrPuUSztUdU5A==", @@ -7574,6 +7554,15 @@ "node": ">=8" } }, + "node_modules/strip-ansi-cjs/node_modules/ansi-regex": { + "version": "5.0.1", + "resolved": "https://registry.npmjs.org/ansi-regex/-/ansi-regex-5.0.1.tgz", + "integrity": "sha512-quJQXlTSUGL2LH9SUXo8VwsY4soanhgo6LNSm84E1LBcE8s3O0wpdiRzyR9z/ZZJMlMWv37qOOb9pdJlMUEKFQ==", + "dev": true, + "engines": { + "node": ">=8" + } + }, "node_modules/strip-json-comments": { "version": "2.0.1", "resolved": "https://registry.npmjs.org/strip-json-comments/-/strip-json-comments-2.0.1.tgz", @@ -7588,18 +7577,6 @@ "resolved": "https://registry.npmjs.org/strnum/-/strnum-1.0.5.tgz", "integrity": "sha512-J8bbNyKKXl5qYcR36TIO8W3mVGVHrmmxsd5PAItGkmyzwJvybiw2IVq5nqd0i4LSNSkB/sx9VHllbfFdr9k1JA==" }, - "node_modules/supports-color": { - "version": "7.2.0", - "resolved": "https://registry.npmjs.org/supports-color/-/supports-color-7.2.0.tgz", - "integrity": "sha512-qpCAvRl9stuOHveKsn7HncJRvv501qIacKzQlO/+Lwxc9+0q2wLyv4Dfvt80/DPn2pqOBsJdDiogXGR9+OvwRw==", - "dev": true, - "dependencies": { - "has-flag": "^4.0.0" - }, - "engines": { - "node": ">=8" - } - }, "node_modules/supports-preserve-symlinks-flag": { "version": "1.0.0", "resolved": "https://registry.npmjs.org/supports-preserve-symlinks-flag/-/supports-preserve-symlinks-flag-1.0.0.tgz", @@ -7612,6 +7589,23 @@ "url": "https://github.com/sponsors/ljharb" } }, + "node_modules/tar": { + "version": "7.4.3", + "resolved": "https://registry.npmjs.org/tar/-/tar-7.4.3.tgz", + "integrity": "sha512-5S7Va8hKfV7W5U6g3aYxXmlPoZVAwUMy9AOKyF2fVuZa2UD3qZjg578OrLRt8PcNN1PleVaL/5/yYATNL0ICUw==", + "dev": true, + "dependencies": { + "@isaacs/fs-minipass": "^4.0.0", + "chownr": "^3.0.0", + "minipass": "^7.1.2", + "minizlib": "^3.0.1", + "mkdirp": "^3.0.1", + "yallist": "^5.0.0" + }, + "engines": { + "node": ">=18" + } + }, "node_modules/tar-fs": { "version": "2.1.1", "resolved": "https://registry.npmjs.org/tar-fs/-/tar-fs-2.1.1.tgz", @@ -7624,6 +7618,12 @@ "tar-stream": "^2.1.4" } }, + "node_modules/tar-fs/node_modules/chownr": { + "version": "1.1.4", + "resolved": "https://registry.npmjs.org/chownr/-/chownr-1.1.4.tgz", + "integrity": "sha512-jJ0bqzaylmJtVnNgzTeSOs8DPavpbYgEr/b0YL8/2GO3xJEhInFmhKMUnEJQjZumK7KXGFhUy89PrsJWlakBVg==", + "dev": true + }, "node_modules/tar-stream": { "version": "2.2.0", "resolved": "https://registry.npmjs.org/tar-stream/-/tar-stream-2.2.0.tgz", @@ -7648,13 +7648,43 @@ "real-require": "^0.2.0" } }, - "node_modules/to-fast-properties": { - "version": "2.0.0", - "resolved": "https://registry.npmjs.org/to-fast-properties/-/to-fast-properties-2.0.0.tgz", - "integrity": "sha512-/OaKK0xYrs3DmxRYqL/yDc+FxFUVYhDlXMhRmv3z915w2HF1tnN1omB354j8VUGO/hbRzyD6Y3sA7v7GS/ceog==", + "node_modules/tinyglobby": { + "version": "0.2.10", + "resolved": "https://registry.npmjs.org/tinyglobby/-/tinyglobby-0.2.10.tgz", + "integrity": "sha512-Zc+8eJlFMvgatPZTl6A9L/yht8QqdmUNtURHaKZLmKBE12hNPSrqNkUp2cs3M/UKmNVVAMFQYSjYIVHDjW5zew==", "dev": true, + "dependencies": { + "fdir": "^6.4.2", + "picomatch": "^4.0.2" + }, "engines": { - "node": ">=4" + "node": ">=12.0.0" + } + }, + "node_modules/tinyglobby/node_modules/fdir": { + "version": "6.4.2", + "resolved": "https://registry.npmjs.org/fdir/-/fdir-6.4.2.tgz", + "integrity": "sha512-KnhMXsKSPZlAhp7+IjUkRZKPb4fUyccpDrdFXbi4QL1qkmFh9kVY09Yox+n4MaOb3lHZ1Tv829C3oaaXoMYPDQ==", + "dev": true, + "peerDependencies": { + "picomatch": "^3 || ^4" + }, + "peerDependenciesMeta": { + "picomatch": { + "optional": true + } + } + }, + "node_modules/tinyglobby/node_modules/picomatch": { + "version": "4.0.2", + "resolved": "https://registry.npmjs.org/picomatch/-/picomatch-4.0.2.tgz", + "integrity": "sha512-M7BAV6Rlcy5u+m6oPhAPFgJTzAioX/6B0DxyvDlo9l8+T3nLKbrczg2WLUyzd45L8RqfUMyGPzekbMvX2Ldkwg==", + "dev": true, + "engines": { + "node": ">=12" + }, + "funding": { + "url": "https://github.com/sponsors/jonschlinkert" } }, "node_modules/to-regex-range": { @@ -7719,6 +7749,19 @@ "node": ">= 10.0.0" } }, + "node_modules/unzipper": { + "version": "0.12.3", + "resolved": "https://registry.npmjs.org/unzipper/-/unzipper-0.12.3.tgz", + "integrity": "sha512-PZ8hTS+AqcGxsaQntl3IRBw65QrBI6lxzqDEL7IAo/XCEqRTKGfOX56Vea5TH9SZczRVxuzk1re04z/YjuYCJA==", + "dev": true, + "dependencies": { + "bluebird": "~3.7.2", + "duplexer2": "~0.1.4", + "fs-extra": "^11.2.0", + "graceful-fs": "^4.2.2", + "node-int64": "^0.4.0" + } + }, "node_modules/util-deprecate": { "version": "1.0.2", "resolved": "https://registry.npmjs.org/util-deprecate/-/util-deprecate-1.0.2.tgz", @@ -7752,7 +7795,40 @@ "webidl-conversions": "^3.0.0" } }, + "node_modules/which": { + "version": "2.0.2", + "resolved": "https://registry.npmjs.org/which/-/which-2.0.2.tgz", + "integrity": "sha512-BLI3Tl1TW3Pvl70l3yq3Y64i+awpwXqsGBYWkkqMtnbXgrMD+yj7rhW0kuEDxzJaYXGjEW5ogapKNMEKNMjibA==", + "dev": true, + "dependencies": { + "isexe": "^2.0.0" + }, + "bin": { + "node-which": "bin/node-which" + }, + "engines": { + "node": ">= 8" + } + }, "node_modules/wrap-ansi": { + "version": "8.1.0", + "resolved": "https://registry.npmjs.org/wrap-ansi/-/wrap-ansi-8.1.0.tgz", + "integrity": "sha512-si7QWI6zUMq56bESFvagtmzMdGOtoxfR+Sez11Mobfc7tm+VkUckk9bW2UeffTGVUbOksxmSw0AA2gs8g71NCQ==", + "dev": true, + "dependencies": { + "ansi-styles": "^6.1.0", + "string-width": "^5.0.1", + "strip-ansi": "^7.0.1" + }, + "engines": { + "node": ">=12" + }, + "funding": { + "url": "https://github.com/chalk/wrap-ansi?sponsor=1" + } + }, + "node_modules/wrap-ansi-cjs": { + "name": "wrap-ansi", "version": "7.0.0", "resolved": "https://registry.npmjs.org/wrap-ansi/-/wrap-ansi-7.0.0.tgz", "integrity": "sha512-YVGIj2kamLSTxw6NsZjoBxfSwsn0ycdesmc4p+Q21c5zPuZ1pl+NfxVdxPtdHvmNVOQ6XSYG4AUtyt/Fi7D16Q==", @@ -7769,6 +7845,62 @@ "url": "https://github.com/chalk/wrap-ansi?sponsor=1" } }, + "node_modules/wrap-ansi-cjs/node_modules/ansi-regex": { + "version": "5.0.1", + "resolved": "https://registry.npmjs.org/ansi-regex/-/ansi-regex-5.0.1.tgz", + "integrity": "sha512-quJQXlTSUGL2LH9SUXo8VwsY4soanhgo6LNSm84E1LBcE8s3O0wpdiRzyR9z/ZZJMlMWv37qOOb9pdJlMUEKFQ==", + "dev": true, + "engines": { + "node": ">=8" + } + }, + "node_modules/wrap-ansi-cjs/node_modules/ansi-styles": { + "version": "4.3.0", + "resolved": "https://registry.npmjs.org/ansi-styles/-/ansi-styles-4.3.0.tgz", + "integrity": "sha512-zbB9rCJAT1rbjiVDb2hqKFHNYLxgtk8NURxZ3IZwD3F6NtxbXZQCnnSi1Lkx+IDohdPlFp222wVALIheZJQSEg==", + "dev": true, + "dependencies": { + "color-convert": "^2.0.1" + }, + "engines": { + "node": ">=8" + }, + "funding": { + "url": "https://github.com/chalk/ansi-styles?sponsor=1" + } + }, + "node_modules/wrap-ansi-cjs/node_modules/emoji-regex": { + "version": "8.0.0", + "resolved": "https://registry.npmjs.org/emoji-regex/-/emoji-regex-8.0.0.tgz", + "integrity": "sha512-MSjYzcWNOA0ewAHpz0MxpYFvwg6yjy1NG3xteoqz644VCo/RPgnr1/GGt+ic3iJTzQ8Eu3TdM14SawnVUmGE6A==", + "dev": true + }, + "node_modules/wrap-ansi-cjs/node_modules/string-width": { + "version": "4.2.3", + "resolved": "https://registry.npmjs.org/string-width/-/string-width-4.2.3.tgz", + "integrity": "sha512-wKyQRQpjJ0sIp62ErSZdGsjMJWsap5oRNihHhu6G7JVO/9jIB6UyevL+tXuOqrng8j/cxKTWyWUwvSTriiZz/g==", + "dev": true, + "dependencies": { + "emoji-regex": "^8.0.0", + "is-fullwidth-code-point": "^3.0.0", + "strip-ansi": "^6.0.1" + }, + "engines": { + "node": ">=8" + } + }, + "node_modules/wrap-ansi-cjs/node_modules/strip-ansi": { + "version": "6.0.1", + "resolved": "https://registry.npmjs.org/strip-ansi/-/strip-ansi-6.0.1.tgz", + "integrity": "sha512-Y38VPSHcqkFrCpFnQ9vuSXmquuv5oXOKpGeT6aGrr3o3Gc9AlVa6JBfUSOCnbxGGZF+/0ooI7KrPuUSztUdU5A==", + "dev": true, + "dependencies": { + "ansi-regex": "^5.0.1" + }, + "engines": { + "node": ">=8" + } + }, "node_modules/wrappy": { "version": "1.0.2", "resolved": "https://registry.npmjs.org/wrappy/-/wrappy-1.0.2.tgz", @@ -7785,10 +7917,13 @@ } }, "node_modules/yallist": { - "version": "4.0.0", - "resolved": "https://registry.npmjs.org/yallist/-/yallist-4.0.0.tgz", - "integrity": "sha512-3wdGidZyq5PB084XLES5TpOSRA3wjXAlIWMhum2kRcv/41Sn2emQ0dycQW4uZXLejwKvg6EsvbdlVL+FYEct7A==", - "dev": true + "version": "5.0.0", + "resolved": "https://registry.npmjs.org/yallist/-/yallist-5.0.0.tgz", + "integrity": "sha512-YgvUTfwqyc7UXVMrB+SImsVYSmTS8X/tSrtdNZMImM+n7+QTriRXyXim0mBrTXNeqzVF0KWGgHPeiyViFFrNDw==", + "dev": true, + "engines": { + "node": ">=18" + } }, "node_modules/yargs": { "version": "16.2.0", @@ -7817,6 +7952,47 @@ "node": ">=10" } }, + "node_modules/yargs/node_modules/ansi-regex": { + "version": "5.0.1", + "resolved": "https://registry.npmjs.org/ansi-regex/-/ansi-regex-5.0.1.tgz", + "integrity": "sha512-quJQXlTSUGL2LH9SUXo8VwsY4soanhgo6LNSm84E1LBcE8s3O0wpdiRzyR9z/ZZJMlMWv37qOOb9pdJlMUEKFQ==", + "dev": true, + "engines": { + "node": ">=8" + } + }, + "node_modules/yargs/node_modules/emoji-regex": { + "version": "8.0.0", + "resolved": "https://registry.npmjs.org/emoji-regex/-/emoji-regex-8.0.0.tgz", + "integrity": "sha512-MSjYzcWNOA0ewAHpz0MxpYFvwg6yjy1NG3xteoqz644VCo/RPgnr1/GGt+ic3iJTzQ8Eu3TdM14SawnVUmGE6A==", + "dev": true + }, + "node_modules/yargs/node_modules/string-width": { + "version": "4.2.3", + "resolved": "https://registry.npmjs.org/string-width/-/string-width-4.2.3.tgz", + "integrity": "sha512-wKyQRQpjJ0sIp62ErSZdGsjMJWsap5oRNihHhu6G7JVO/9jIB6UyevL+tXuOqrng8j/cxKTWyWUwvSTriiZz/g==", + "dev": true, + "dependencies": { + "emoji-regex": "^8.0.0", + "is-fullwidth-code-point": "^3.0.0", + "strip-ansi": "^6.0.1" + }, + "engines": { + "node": ">=8" + } + }, + "node_modules/yargs/node_modules/strip-ansi": { + "version": "6.0.1", + "resolved": "https://registry.npmjs.org/strip-ansi/-/strip-ansi-6.0.1.tgz", + "integrity": "sha512-Y38VPSHcqkFrCpFnQ9vuSXmquuv5oXOKpGeT6aGrr3o3Gc9AlVa6JBfUSOCnbxGGZF+/0ooI7KrPuUSztUdU5A==", + "dev": true, + "dependencies": { + "ansi-regex": "^5.0.1" + }, + "engines": { + "node": ">=8" + } + }, "node_modules/zod": { "version": "3.22.0", "resolved": "https://registry.npmjs.org/zod/-/zod-3.22.0.tgz", diff --git a/package.json b/package.json index 9384288..23c9902 100644 --- a/package.json +++ b/package.json @@ -7,7 +7,7 @@ "test": "echo \"Error: no test specified\" && exit 1", "build": "tsc", "start": "node dist/index.js", - "build-binary": "tsc && pkg -t node18-linux-x64 --out-path bin ." + "build-binary": "tsc && pkg -t node20-linux-x64 --out-path bin ." }, "author": "Shawn Rushefsky", "license": "MIT", @@ -15,13 +15,14 @@ "@types/chokidar": "^2.1.3", "@types/node": "^20.12.7", "@types/pino": "^7.0.5", - "pkg": "^5.8.1", + "@yao-pkg/pkg": "^6.1.1", "typescript": "^5.4.5" }, "dependencies": { "@aws-sdk/client-s3": "^3.556.0", "@aws-sdk/lib-storage": "^3.556.0", "@saladtechnologies-oss/salad-cloud-imds-sdk": "^0.9.0-alpha.4", + "@smithy/node-http-handler": "^3.3.1", "chokidar": "^3.6.0", "pino": "^9.0.0" }, diff --git a/src/s3.ts b/src/s3.ts index 7a64850..f040ba9 100644 --- a/src/s3.ts +++ b/src/s3.ts @@ -6,6 +6,7 @@ import { DeleteObjectCommand, } from "@aws-sdk/client-s3"; import { Progress, Upload } from "@aws-sdk/lib-storage"; +import { NodeHttpHandler } from "@smithy/node-http-handler"; import fs from "fs"; import { Readable } from "stream"; import path from "path"; @@ -17,7 +18,13 @@ import state from "./state"; const { AWS_REGION, AWS_DEFAULT_REGION } = process.env; -const s3Client = new S3Client({ region: AWS_REGION || AWS_DEFAULT_REGION }); +const s3Client = new S3Client({ + region: AWS_REGION || AWS_DEFAULT_REGION, + requestHandler: new NodeHttpHandler({ + requestTimeout: 0, + connectionTimeout: 10000, + }), +}); function getDataRatioString( loaded: number | undefined, From 9d2cc487fdae572a4c5818804d30ec55ae77cd22 Mon Sep 17 00:00:00 2001 From: Shawn Date: Tue, 19 Nov 2024 11:38:54 -0500 Subject: [PATCH 08/21] update default to match api --- src/api.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/api.ts b/src/api.ts index 76f4436..83e1a66 100644 --- a/src/api.ts +++ b/src/api.ts @@ -10,7 +10,7 @@ let { SALAD_MACHINE_ID = "", SALAD_CONTAINER_GROUP_ID = "", MAX_RETRIES = "3", - MAX_JOB_FAILURES = "3", + MAX_JOB_FAILURES = "5", } = process.env; assert(KELPIE_API_URL, "KELPIE_API_URL is required"); From afebcfc0295dff355669042fa8daa193cf8ddc69 Mon Sep 17 00:00:00 2001 From: Shawn Date: Thu, 21 Nov 2024 13:54:54 -0500 Subject: [PATCH 09/21] see if we can get builds on prs --- .github/workflows/pr-build.yml | 44 ++++++++++++++++++++++++++++++++++ 1 file changed, 44 insertions(+) create mode 100644 .github/workflows/pr-build.yml diff --git a/.github/workflows/pr-build.yml b/.github/workflows/pr-build.yml new file mode 100644 index 0000000..c1974eb --- /dev/null +++ b/.github/workflows/pr-build.yml @@ -0,0 +1,44 @@ +name: Build Release Candidate + +on: + pull_request: + branches: + - main + +jobs: + build: + runs-on: ubuntu-latest + + steps: + - uses: actions/checkout@v4 + + - name: Set up Node.js + uses: actions/setup-node@v4 + with: + node-version: "20" + + - name: Install dependencies + run: npm install + + - name: Build + run: npm run build-binary + + - name: Archive production artifacts + id: archive + uses: actions/upload-artifact@v4 + with: + path: bin/kelpie + name: kelpie-${{ github.sha }} + + - name: Comment on PR with link + uses: actions/github-script@v7 + with: + github-token: ${{ secrets.GITHUB_TOKEN }} + script: | + github.issues.createComment({ + issue_number: context.issue.number, + owner: context.repo.owner, + repo: context.repo.repo, + body: `:rocket: [Download the latest release candidate](${{ steps.archive.outputs.artifact-url }}) :rocket:` + }) + From a9235006ae12ffb2e85d5e3664aa501678e46e2c Mon Sep 17 00:00:00 2001 From: Shawn Date: Thu, 21 Nov 2024 13:57:37 -0500 Subject: [PATCH 10/21] syntax error --- .github/workflows/pr-build.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/pr-build.yml b/.github/workflows/pr-build.yml index c1974eb..d983fd6 100644 --- a/.github/workflows/pr-build.yml +++ b/.github/workflows/pr-build.yml @@ -35,7 +35,7 @@ jobs: with: github-token: ${{ secrets.GITHUB_TOKEN }} script: | - github.issues.createComment({ + github.rest.issues.createComment({ issue_number: context.issue.number, owner: context.repo.owner, repo: context.repo.repo, From 8012f3d90047c7a5e2b3d2f00fd2b5cb3b87f16c Mon Sep 17 00:00:00 2001 From: Shawn Date: Thu, 21 Nov 2024 14:57:27 -0500 Subject: [PATCH 11/21] github action for creating releases on merge-to-main --- .github/workflows/create-release.yml | 75 ++++++++++++++++++++++++++++ 1 file changed, 75 insertions(+) create mode 100644 .github/workflows/create-release.yml diff --git a/.github/workflows/create-release.yml b/.github/workflows/create-release.yml new file mode 100644 index 0000000..cdf9214 --- /dev/null +++ b/.github/workflows/create-release.yml @@ -0,0 +1,75 @@ +name: Create Release + +on: + push: + branches: + - main + +jobs: + build-and-release: + runs-on: ubuntu-latest + + steps: + - uses: actions/checkout@v4 + + - name: Set up Node.js + uses: actions/setup-node@v4 + with: + node-version: "20" + + - name: Install dependencies + run: npm install + + - name: Build + run: npm run build-binary + + - name: Get version from package.json + id: version + run: echo ::set-output name=version::$(node -p "require('./package.json').version") + + - name: Get the PR that was merged into main + id: pr + uses: actions/github-script@v7 + with: + github-token: ${{ secrets.GITHUB_TOKEN }} + script: | + const { data: pulls } = await github.rest.pulls.list({ + owner: context.repo.owner, + repo: context.repo.repo, + state: 'closed', + base: 'main', + sort: 'updated', + direction: 'desc', + per_page: 1 + }); + const pr = pulls[0]; + // Set the title and body as outputs + return { + title: pr.title, + body: pr.body + }; + + - name: Make outputs of previous step available + id: pr-output + run: | + echo "::set-output name=title:$(echo ${{ steps.pr.outputs.result }} | jq -r '.title')" + echo "::set-output name=body::$(echo ${{ steps.pr.outputs.result }} | jq -r '.body')" + + - name: Create a release + uses: actions/create-release@v1 + env: + GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} + with: + tag_name: ${{ steps.version.outputs.version }} + release_name: ${{ steps.pr-output.outputs.title }} + body: ${{ steps.pr-output.outputs.body }} + draft: false + prerelease: false + + - name: Upload release artifacts + env: + GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} + run: | + gh release upload ${{ steps.version.outputs.version }} ./bin/kelpie#Linux_x64 --clobber + + From 1d4631afd32efc75e4ed8be86ea297e86b972126 Mon Sep 17 00:00:00 2001 From: Shawn Date: Fri, 22 Nov 2024 09:40:49 -0500 Subject: [PATCH 12/21] don't make releases with absolutely every change --- .github/workflows/create-release.yml | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/.github/workflows/create-release.yml b/.github/workflows/create-release.yml index cdf9214..f07eb64 100644 --- a/.github/workflows/create-release.yml +++ b/.github/workflows/create-release.yml @@ -4,6 +4,11 @@ on: push: branches: - main + paths-ignore: + - ".github/**" + - "**.md" + - "**.png" + - ".gitignore" jobs: build-and-release: From bb5e3bd0f37c1fcab8d1dce0bf2f01e3b7a2973f Mon Sep 17 00:00:00 2001 From: Shawn Date: Fri, 22 Nov 2024 16:01:09 -0500 Subject: [PATCH 13/21] don't reallocate when there are in-progress uploads --- src/index.ts | 15 ++++++++++++++- src/state.ts | 20 ++++++++++++++++++++ 2 files changed, 34 insertions(+), 1 deletion(-) diff --git a/src/index.ts b/src/index.ts index 2aafb53..7303f68 100644 --- a/src/index.ts +++ b/src/index.ts @@ -138,7 +138,20 @@ async function main() { /** * A common reason to have no work for too long is that the instance is * banned from a particular workload. In this case, we should reallocate - * the instance to get a new machine id. + * the instance to get a new machine id. However, we want to make sure any uploads + * that are currently in progress complete. + */ + if (state.getState().uploads.size > 0) { + await state.waitForUploads(baseLogger); + /** + * Once we've finished our uploads, we check one more time for work, + * just in case. + */ + continue; + } + + /** + * If there are no uploads in progress, we can reallocate the instance. */ await reallocateMe(baseLogger); break; diff --git a/src/state.ts b/src/state.ts index c92e45a..5c9a6c7 100644 --- a/src/state.ts +++ b/src/state.ts @@ -73,6 +73,24 @@ export async function saveState(log: Logger): Promise { } } +export function getState() { + return state; +} + +async function sleep(ms: number): Promise { + return new Promise((resolve) => setTimeout(resolve, ms)); +} + +export async function waitForUploads( + log: Logger, + intervalMs: number = 1000 +): Promise { + while (state.uploads.size > 0) { + log.info("Waiting for uploads to finish..."); + await sleep(intervalMs); + } +} + export default { startDownload, finishDownload, @@ -81,5 +99,7 @@ export default { finishUpload, hasUpload, saveState, + getState, + waitForUploads, filename, }; From fa83145ef1064daf1ed5cd765bea1c678e5b94d9 Mon Sep 17 00:00:00 2001 From: Shawn Date: Fri, 22 Nov 2024 16:21:55 -0500 Subject: [PATCH 14/21] comments --- src/s3.ts | 51 +++++++++++++++++++++++++++++++++++++++++++++++++-- 1 file changed, 49 insertions(+), 2 deletions(-) diff --git a/src/s3.ts b/src/s3.ts index f040ba9..b087e09 100644 --- a/src/s3.ts +++ b/src/s3.ts @@ -150,11 +150,15 @@ export async function downloadFile( Key: key, }; - // Perform the download + // Get the object from S3, which returns a stream const data = await s3Client.send(new GetObjectCommand(downloadParams)); return new Promise((resolve, reject) => { if (data.Body instanceof Readable) { + /** + * S3 gives us a stream back from our request, which can pipe through + * the various steps it needs to take to get to the final file. + */ let stream: Readable = data.Body; if (isGzipped) { log.debug(`Decompressing ${key} file with gunzip`); @@ -164,10 +168,17 @@ export async function downloadFile( state.finishDownload(localFilePath, log); reject(err); }); + /** + * If we've determined that the file is gzipped, we'll pipe the stream + * directly through the gunzip stream to decompress it. + */ stream = stream.pipe(unzipStream); } - // Loop through body chunks and write to file + /** + * At this point we have a decompressed stream that we can pipe directly + * to the file system to write the file. + */ const writeStream = fs.createWriteStream(localFilePath); stream .pipe(writeStream) @@ -259,6 +270,24 @@ async function processBatch( }); } +/** + * Downloads all files with a given prefix from an S3 bucket to a local directory, + * and with an additional regular expression filter. + * + * This function downloads files in batches to improve performance. The number of files + * downloaded in parallel can be controlled with the `batchSize` parameter. + * + * If the `decompress` parameter is set to true, the function will decompress the files + * while downloading them. + * + * @param options.bucket The name of the S3 bucket to download from + * @param options.prefix The prefix of the files to download + * @param options.outputDir The local directory to download the files to + * @param options.batchSize The number of files to download in parallel + * @param options.decompress Whether to decompress the files after downloading + * @param options.log The logger to use for output + * @param options.pattern A regular expression to filter the files to download. + */ export async function downloadAllFilesFromPrefix({ bucket, prefix, @@ -301,6 +330,24 @@ export async function downloadAllFilesFromPrefix({ } } +/** + * Uploads all files from a local directory to an S3 bucket with a given prefix. + * The function uploads files in batches to improve performance. The number of files + * uploaded in parallel can be controlled with the `batchSize` parameter. Files are themselves + * uploaded in parallel using the `Upload` class from the `@aws-sdk/lib-storage` package. The + * intention is to fully utilize the available bandwidth and improve the overall upload speed. + * + * If the `compress` parameter is set to true, the function will compress the files + * while uploading them. + * + * The `pattern` parameter can be used to filter the files to upload using a regular + * expression. + * + * + * + * @param param0 + * @returns + */ export async function uploadDirectory({ directory, bucket, From 088052bea080516e6e059c7efa1886c4144dcddc Mon Sep 17 00:00:00 2001 From: Shawn Date: Fri, 22 Nov 2024 16:32:55 -0500 Subject: [PATCH 15/21] update version in readme --- readme.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/readme.md b/readme.md index f55e735..9144cf4 100644 --- a/readme.md +++ b/readme.md @@ -52,7 +52,7 @@ You can find a working example [here](https://github.com/SaladTechnologies/kelpi FROM yourimage:yourtag # Add the kelpie binary to your container image -ADD https://github.com/SaladTechnologies/kelpie/releases/download/0.4.3/kelpie /kelpie +ADD https://github.com/SaladTechnologies/kelpie/releases/download/0.5.0/kelpie /kelpie RUN chmod +x /kelpie # Use kelpie as the "main" command. Kelpie will then execute your From f7903e27518e64284b44fec01d7db0895815cb62 Mon Sep 17 00:00:00 2001 From: Shawn Date: Fri, 22 Nov 2024 16:34:39 -0500 Subject: [PATCH 16/21] don't need to build for docs changes --- .github/workflows/pr-build.yml | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/.github/workflows/pr-build.yml b/.github/workflows/pr-build.yml index d983fd6..d04820d 100644 --- a/.github/workflows/pr-build.yml +++ b/.github/workflows/pr-build.yml @@ -4,6 +4,10 @@ on: pull_request: branches: - main + paths-ignore: + - "**.md" + - "**.png" + jobs: build: From cb5cc0d259017542bfcc6cfd385162f7933eca44 Mon Sep 17 00:00:00 2001 From: Shawn Date: Fri, 22 Nov 2024 16:41:33 -0500 Subject: [PATCH 17/21] readme update --- readme.md | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/readme.md b/readme.md index 9144cf4..ec7c726 100644 --- a/readme.md +++ b/readme.md @@ -6,6 +6,7 @@ Kelpie shepherds long-running jobs through to completion on interruptible hardwa - [🐕 Kelpie (beta)](#-kelpie-beta) - [Who is it for?](#who-is-it-for) + - [What it is](#what-it-is) - [How it Works](#how-it-works) - [Adding the kelpie Worker To Your Container Image](#adding-the-kelpie-worker-to-your-container-image) - [Environment Variables](#environment-variables) @@ -30,7 +31,11 @@ Kelpie shepherds long-running jobs through to completion on interruptible hardwa Kelpie is for anyone who wants to run long running compute-intensive jobs on [Salad](https://salad.com/), the world's largest distributed GPU cloud. Whether that's [LoRA training](https://blog.salad.com/cost-effective-stable-diffusion-fine-tuning-on-salad/), Monte Carlo simulations, Molecular Dynamics simulations, or anything else, Kelpie can help you run your jobs to completion, even if they take days or weeks. You bring your own docker container that contains your script and dependencies, add the Kelpie binary to it, and deploy. -If you'd like to join the Kelpie beta, and are an existing Salad customer, just reach out to your point of contact via email, discord, or slack. If you're interested in Kelpie and are new to Salad, [sign up for a demo](https://salad.com/get-a-demo), and mention you're interested in using Kelpie. +If you'd like to join the Kelpie beta, and are an existing Salad customer, just reach out to your point of contact via email, discord, or slack. If you're interested in Kelpie and are new to Salad, reach out to support at [cloud@salad.com](cloud@salad.com), and mention you're interested in using Kelpie. + +## What it is + +Kelpie is a job queue that is particularly focused on the challenges of running extremely long tasks on interruptible hardware. It is designed to be simple to instrument, and to be able to integrate with any containerized workload. It executes scripts in a container according to a job definition, and handles downloading input data, uploading output data, and syncing progress checkpoints to your s3-compatible storage. It also provides a mechanism for scaling your container group in response to job volume. ## How it Works From f1ca44f16783c7dccb2aea08dd9ce8e06135ef0e Mon Sep 17 00:00:00 2001 From: Shawn Date: Tue, 3 Dec 2024 15:09:55 -0500 Subject: [PATCH 18/21] expand state format --- src/api.ts | 3 ++ src/index.ts | 47 ++++++++++++++++++++++---- src/s3.ts | 51 +++++++++++++++++++++------- src/state.ts | 95 ++++++++++++++++++++++++++++++++++++++++++++-------- 4 files changed, 164 insertions(+), 32 deletions(-) diff --git a/src/api.ts b/src/api.ts index 83e1a66..5f91b6b 100644 --- a/src/api.ts +++ b/src/api.ts @@ -3,6 +3,7 @@ import { log as baseLogger } from "./logger"; import { Logger } from "pino"; import { Task } from "./types"; import { SaladCloudImdsSdk } from "@saladtechnologies-oss/salad-cloud-imds-sdk"; +import state from "./state"; let { KELPIE_API_URL, @@ -106,6 +107,7 @@ export async function sendHeartbeat( let numFailures = 0; export async function reportFailed(jobId: string, log: Logger): Promise { log.info(`Reporting job failed`); + state.finishJob(jobId, "failed", log); await fetchUpToNTimes( `${KELPIE_API_URL}/jobs/${jobId}/failed`, { @@ -143,6 +145,7 @@ export async function reportCompleted( log: Logger ): Promise { log.info("Reporting job completed"); + state.finishJob(jobId, "completed", log); await fetchUpToNTimes( `${KELPIE_API_URL}/jobs/${jobId}/completed`, { diff --git a/src/index.ts b/src/index.ts index 7303f68..851db2e 100644 --- a/src/index.ts +++ b/src/index.ts @@ -64,6 +64,7 @@ async function uploadAndCompleteJob( ): Promise { try { await uploadDirectory({ + jobId: work.id, directory: dirToUpload, bucket: work.output_bucket!, prefix: work.output_prefix!, @@ -141,11 +142,23 @@ async function main() { * the instance to get a new machine id. However, we want to make sure any uploads * that are currently in progress complete. */ - if (state.getState().uploads.size > 0) { - await state.waitForUploads(baseLogger); + const currentState = state.getState(); + let uploadsInProgress = false; + await Promise.all( + currentState.jobs.map(async (job) => { + if (job.activeUploads.size) { + uploadsInProgress = true; + baseLogger.info( + `Waiting for uploads to finish before reallocation...` + ); + await state.waitForUploads(job.id, baseLogger); + } + }) + ); + + if (uploadsInProgress) { /** - * Once we've finished our uploads, we check one more time for work, - * just in case. + * If there were uploads in progress, we should check for work one more time */ continue; } @@ -163,6 +176,7 @@ async function main() { lastWorkReceived = Date.now(); const log = baseLogger.child({ job_id: work.id }); log.info(`Received work: ${work.id}`); + state.startJob(work.id, log); log.info("Starting heartbeat manager..."); const heartbeatManager = new HeartbeatManager(work.id, log); @@ -182,7 +196,12 @@ async function main() { if (work.sync) { if (work.sync.before && work.sync.before.length) { for (const syncConfig of work.sync.before) { - await downloadSyncConfig(syncConfig, !!work.compression, log); + await downloadSyncConfig( + work.id, + syncConfig, + !!work.compression, + log + ); } } @@ -206,6 +225,7 @@ async function main() { ) { filesBeingSynced.add(localFilePath); await uploadFile( + work.id, localFilePath, syncConfig.bucket, syncConfig.prefix + relativeFilename, @@ -237,6 +257,7 @@ async function main() { if (work.input_bucket && work.input_prefix) { try { await downloadAllFilesFromPrefix({ + jobId: work.id, bucket: work.input_bucket, prefix: work.input_prefix, outputDir: INPUT_DIR, @@ -254,6 +275,7 @@ async function main() { if (work.checkpoint_bucket && work.checkpoint_prefix) { try { await downloadAllFilesFromPrefix({ + jobId: work.id, bucket: work.checkpoint_bucket, prefix: work.checkpoint_prefix, outputDir: CHECKPOINT_DIR, @@ -276,6 +298,7 @@ async function main() { ); if (eventType === "add" || eventType === "change") { await uploadFile( + work.id, localFilePath, work.checkpoint_bucket!, work.checkpoint_prefix + relativeFilename, @@ -310,6 +333,7 @@ async function main() { const relativeFilename = path.relative(OUTPUT_DIR, localFilePath); if (eventType === "add") { await uploadFile( + work.id, localFilePath, work.output_bucket!, work.output_prefix + relativeFilename, @@ -341,6 +365,12 @@ async function main() { KELPIE_JOB_ID: work.id, } ); + /** + * Once the command exits, we can update the job's status in the state. + * In the event the exitCode is null, we will default to -2, which is + * an error code that is not used by any system commands. + */ + state.jobExited(work.id, exitCode ?? -2, log); /** * Once the script updates, we can stop watching the directories. * This will stop the event-driven file sync behavior that is @@ -419,7 +449,12 @@ async function main() { */ Promise.all( modifiedOutputs.map(async (syncConfig) => { - await uploadSyncConfig(syncConfig, !!work.compression, log); + await uploadSyncConfig( + work.id, + syncConfig, + !!work.compression, + log + ); }) ) .then(async () => { diff --git a/src/s3.ts b/src/s3.ts index b087e09..809d6a6 100644 --- a/src/s3.ts +++ b/src/s3.ts @@ -61,6 +61,7 @@ function getDataRatioString( } export async function uploadFile( + jobId: string, localFilePath: string, bucketName: string, key: string, @@ -68,11 +69,11 @@ export async function uploadFile( log: Logger ): Promise { try { - if (state.hasUpload(localFilePath)) { + if (state.hasUpload(jobId, localFilePath)) { log.info(`Upload of ${localFilePath} already in progress`); return; } - await state.startUpload(localFilePath, log); + await state.startUpload(jobId, localFilePath, log); log.info(`Uploading ${localFilePath} to s3://${bucketName}/${key}`); // Create a stream from the local file const fileStream = fs.createReadStream(localFilePath); @@ -122,10 +123,11 @@ export async function uploadFile( } catch (err) { log.error("Error uploading file: ", err); } - await state.finishDownload(localFilePath, log); + await state.finishDownload(jobId, localFilePath, log); } export async function downloadFile( + jobId: string, bucketName: string, key: string, localFilePath: string, @@ -134,11 +136,11 @@ export async function downloadFile( ): Promise { try { const start = Date.now(); - if (state.hasDownload(key)) { + if (state.hasDownload(jobId, key)) { log.info(`Download of ${key} already in progress`); return; } - state.startDownload(localFilePath, log); + state.startDownload(jobId, localFilePath, log); const isGzipped = decompress && key.endsWith(".gz"); if (isGzipped) { @@ -165,7 +167,7 @@ export async function downloadFile( const unzipStream = createGunzip(); unzipStream.on("error", async (err) => { log.error(`Error decompressing ${key} file: ${err}`); - state.finishDownload(localFilePath, log); + state.finishDownload(jobId, localFilePath, log); reject(err); }); /** @@ -184,7 +186,7 @@ export async function downloadFile( .pipe(writeStream) .on("error", (err: any) => { log.error(`Error writing to ${localFilePath}: ${err}`); - state.finishDownload(localFilePath, log); + state.finishDownload(jobId, localFilePath, log); reject(err); }) .on("close", () => { @@ -201,14 +203,14 @@ export async function downloadFile( durString = `${(durMs / 60000).toFixed(2)}m`; } log.info(`${localFilePath} downloaded in ${durString}`); - state.finishDownload(localFilePath, log); + state.finishDownload(jobId, localFilePath, log); resolve(); }); } }); } catch (err: any) { log.error("Error downloading file: ", err); - await state.finishDownload(localFilePath, log); + await state.finishDownload(jobId, localFilePath, log); throw err; } } @@ -247,6 +249,7 @@ async function listAllS3Objects( } async function processBatch( + jobId: string, batch: string[], bucket: string, prefix: string, @@ -259,7 +262,7 @@ async function processBatch( const localFilePath = path.join(outputDir, filename); const dir = path.dirname(localFilePath); await fsPromises.mkdir(dir, { recursive: true }); - return downloadFile(bucket, key, localFilePath, decompress, log); + return downloadFile(jobId, bucket, key, localFilePath, decompress, log); }); const results = await Promise.allSettled(downloadPromises); @@ -280,6 +283,7 @@ async function processBatch( * If the `decompress` parameter is set to true, the function will decompress the files * while downloading them. * + * @param options.jobId The ID of the job * @param options.bucket The name of the S3 bucket to download from * @param options.prefix The prefix of the files to download * @param options.outputDir The local directory to download the files to @@ -289,6 +293,7 @@ async function processBatch( * @param options.pattern A regular expression to filter the files to download. */ export async function downloadAllFilesFromPrefix({ + jobId, bucket, prefix, outputDir, @@ -297,6 +302,7 @@ export async function downloadAllFilesFromPrefix({ log, pattern, }: { + jobId: string; bucket: string; prefix: string; outputDir: string; @@ -319,7 +325,15 @@ export async function downloadAllFilesFromPrefix({ // Download files in batches for (let i = 0; i < allKeys.length; i += batchSize) { const batch = allKeys.slice(i, i + batchSize); - await processBatch(batch, bucket, prefix, outputDir, decompress, log); + await processBatch( + jobId, + batch, + bucket, + prefix, + outputDir, + decompress, + log + ); } log.info( @@ -349,6 +363,7 @@ export async function downloadAllFilesFromPrefix({ * @returns */ export async function uploadDirectory({ + jobId, directory, bucket, prefix, @@ -357,6 +372,7 @@ export async function uploadDirectory({ log, pattern, }: { + jobId: string; directory: string; bucket: string; prefix: string; @@ -383,7 +399,14 @@ export async function uploadDirectory({ batch.map(async (filePath) => { const localFilePath = path.join(directory, filePath); const key = prefix + filePath; - return await uploadFile(localFilePath, bucket, key, compress, log); + return await uploadFile( + jobId, + localFilePath, + bucket, + key, + compress, + log + ); }) ); } @@ -433,6 +456,7 @@ export async function deleteFile( } export async function downloadSyncConfig( + jobId: string, config: SyncConfig, compression: boolean, log: Logger @@ -440,6 +464,7 @@ export async function downloadSyncConfig( const { bucket, prefix, local_path, direction, pattern } = config; if (direction === "download") { await downloadAllFilesFromPrefix({ + jobId, bucket, prefix, outputDir: local_path, @@ -452,6 +477,7 @@ export async function downloadSyncConfig( } export async function uploadSyncConfig( + jobId: string, config: SyncConfig, compression: boolean, log: Logger @@ -459,6 +485,7 @@ export async function uploadSyncConfig( const { local_path, bucket, prefix, direction, pattern } = config; if (direction === "upload") { await uploadDirectory({ + jobId, directory: local_path, bucket, prefix, diff --git a/src/state.ts b/src/state.ts index 5c9a6c7..c8136c8 100644 --- a/src/state.ts +++ b/src/state.ts @@ -6,59 +6,82 @@ const { KELPIE_STATE_FILE = "./kelpie-state.json" } = process.env; export const filename = path.resolve(KELPIE_STATE_FILE); +export type JobState = { + id: string; // Job ID + status: "running" | "completed" | "failed"; + start: string; // ISO 8601 date string when this worker started the job + exitTime?: string; // ISO 8601 date string when the job exited + exitCode?: number; // Exit code of the job + end?: string; // ISO 8601 date string when this worker finished the job, including all uploads + activeUploads: Set; + activeDownloads: Set; +}; + const state = { start: new Date().toISOString(), - uploads: new Set(), - downloads: new Set(), + jobs: [] as JobState[], }; export async function startDownload( + jobId: string, filePath: string, log: Logger ): Promise { - state.downloads.add(filePath); + state.jobs.find((job) => job.id === jobId)?.activeDownloads.add(filePath); await saveState(log); } export async function finishDownload( + jobId: string, filePath: string, log: Logger ): Promise { - state.downloads.delete(filePath); + state.jobs.find((job) => job.id === jobId)?.activeDownloads.delete(filePath); await saveState(log); } -export function hasDownload(filePath: string): boolean { - return state.downloads.has(filePath); +export function hasDownload(jobId: string, filePath: string): boolean { + return ( + state.jobs.find((job) => job.id === jobId)?.activeDownloads.has(filePath) ?? + false + ); } export async function startUpload( + jobId: string, filePath: string, log: Logger ): Promise { - state.uploads.add(filePath); + state.jobs.find((job) => job.id === jobId)?.activeUploads.add(filePath); await saveState(log); } export async function finishUpload( + jobId: string, filePath: string, log: Logger ): Promise { - state.uploads.delete(filePath); + state.jobs.find((job) => job.id === jobId)?.activeUploads.delete(filePath); await saveState(log); } -export function hasUpload(filePath: string): boolean { - return state.uploads.has(filePath); +export function hasUpload(jobId: string, filePath: string): boolean { + return ( + state.jobs.find((job) => job.id === jobId)?.activeUploads.has(filePath) ?? + false + ); } function getJSONState(): any { - const { uploads, downloads, start } = state; + const { jobs, start } = state; return JSON.stringify( { start, - uploads: Array.from(uploads), - downloads: Array.from(downloads), + jobs: jobs.map((job) => ({ + ...job, + activeDownloads: Array.from(job.activeDownloads).sort(), + activeUploads: Array.from(job.activeUploads).sort(), + })), }, null, 2 @@ -82,15 +105,56 @@ async function sleep(ms: number): Promise { } export async function waitForUploads( + jobId: string, log: Logger, intervalMs: number = 1000 ): Promise { - while (state.uploads.size > 0) { + while (state.jobs.find((job) => job.id === jobId)?.activeUploads.size) { log.info("Waiting for uploads to finish..."); await sleep(intervalMs); } } +export async function startJob(jobId: string, log: Logger): Promise { + state.jobs.push({ + id: jobId, + status: "running", + start: new Date().toISOString(), + activeUploads: new Set(), + activeDownloads: new Set(), + }); + + await saveState(log); +} + +export async function jobExited( + jobId: string, + exitCode: number, + log: Logger +): Promise { + const job = state.jobs.find((job) => job.id === jobId); + if (job) { + job.exitTime = new Date().toISOString(); + job.exitCode = exitCode; + } + + await saveState(log); +} + +export async function finishJob( + jobId: string, + status: "completed" | "failed", + log: Logger +): Promise { + const job = state.jobs.find((job) => job.id === jobId); + if (job) { + job.status = status; + job.end = new Date().toISOString(); + } + + await saveState(log); +} + export default { startDownload, finishDownload, @@ -101,5 +165,8 @@ export default { saveState, getState, waitForUploads, + startJob, + jobExited, + finishJob, filename, }; From 9e5314711e8f4b169a4362a4a0fb7b2216eae944 Mon Sep 17 00:00:00 2001 From: Shawn Date: Tue, 3 Dec 2024 17:28:09 -0500 Subject: [PATCH 19/21] debug logs --- src/s3.ts | 2 ++ src/state.ts | 3 ++- 2 files changed, 4 insertions(+), 1 deletion(-) diff --git a/src/s3.ts b/src/s3.ts index 809d6a6..beb3726 100644 --- a/src/s3.ts +++ b/src/s3.ts @@ -71,6 +71,7 @@ export async function uploadFile( try { if (state.hasUpload(jobId, localFilePath)) { log.info(`Upload of ${localFilePath} already in progress`); + log.debug(state.getJSONState()); return; } await state.startUpload(jobId, localFilePath, log); @@ -124,6 +125,7 @@ export async function uploadFile( log.error("Error uploading file: ", err); } await state.finishDownload(jobId, localFilePath, log); + log.debug(state.getJSONState()); } export async function downloadFile( diff --git a/src/state.ts b/src/state.ts index c8136c8..fe85679 100644 --- a/src/state.ts +++ b/src/state.ts @@ -72,7 +72,7 @@ export function hasUpload(jobId: string, filePath: string): boolean { ); } -function getJSONState(): any { +export function getJSONState(): any { const { jobs, start } = state; return JSON.stringify( { @@ -164,6 +164,7 @@ export default { hasUpload, saveState, getState, + getJSONState, waitForUploads, startJob, jobExited, From 08aac7970eaa118dca6350108ccfbc6132d9e451 Mon Sep 17 00:00:00 2001 From: Shawn Date: Wed, 4 Dec 2024 11:58:33 -0500 Subject: [PATCH 20/21] wow, really dumb problem --- src/s3.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/s3.ts b/src/s3.ts index beb3726..e585dff 100644 --- a/src/s3.ts +++ b/src/s3.ts @@ -124,7 +124,7 @@ export async function uploadFile( } catch (err) { log.error("Error uploading file: ", err); } - await state.finishDownload(jobId, localFilePath, log); + await state.finishUpload(jobId, localFilePath, log); log.debug(state.getJSONState()); } From b40755d5111db07134372014af7d9c190f3bb6f3 Mon Sep 17 00:00:00 2001 From: Shawn Date: Thu, 5 Dec 2024 14:29:48 -0500 Subject: [PATCH 21/21] still need to complete a job that has no io --- src/index.ts | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/src/index.ts b/src/index.ts index 851db2e..fada1f8 100644 --- a/src/index.ts +++ b/src/index.ts @@ -479,6 +479,12 @@ async function main() { modifiedOutputs.map((syncConfig) => syncConfig.local_path) ); }); + } else { + /** + * If there's no IO to process at all, we can just report the job as completed. + */ + await heartbeatManager.stopHeartbeat(); + await reportCompleted(work.id, log); } } else { await reportFailed(work.id, log);