diff --git a/package-lock.json b/package-lock.json index 32d58b8..90e9acb 100644 --- a/package-lock.json +++ b/package-lock.json @@ -2841,6 +2841,12 @@ "node": ">=6.9.0" } }, + "node_modules/@balena/dockerignore": { + "version": "1.0.2", + "resolved": "https://registry.npmjs.org/@balena/dockerignore/-/dockerignore-1.0.2.tgz", + "integrity": "sha512-wMue2Sy4GAVTk6Ic4tJVcnfdau+gx2EnG7S+uAEe+TWJFqE4YoWN4/H8MSLj4eYJKxGg26lZwboEniNiNwZQ6Q==", + "dev": true + }, "node_modules/@envelop/core": { "version": "3.0.6", "resolved": "https://registry.npmjs.org/@envelop/core/-/core-3.0.6.tgz", @@ -3358,6 +3364,51 @@ "xtend": "^4.0.0" } }, + "node_modules/@ipld/car": { + "version": "5.2.0", + "resolved": "https://registry.npmjs.org/@ipld/car/-/car-5.2.0.tgz", + "integrity": "sha512-Y4DiyVoPaeGxY6gKV/0A/73SlIIuDu7fl25NdlrO6BYhyTN6v59KqcilmMXbiBA/zcf7cZr1GZVPHRyG2+nmAw==", + "dependencies": { + "@ipld/dag-cbor": "^9.0.0", + "cborg": "^1.9.0", + "multiformats": "^11.0.0", + "varint": "^6.0.0" + }, + "engines": { + "node": ">=16.0.0", + "npm": ">=7.0.0" + } + }, + "node_modules/@ipld/car/node_modules/cborg": { + "version": "1.10.2", + "resolved": "https://registry.npmjs.org/cborg/-/cborg-1.10.2.tgz", + "integrity": "sha512-b3tFPA9pUr2zCUiCfRd2+wok2/LBSNUMKOuRRok+WlvvAgEt/PlbgPTsZUcwCOs53IJvLgTp0eotwtosE6njug==", + "bin": { + "cborg": "cli.js" + } + }, + "node_modules/@ipld/car/node_modules/multiformats": { + "version": "11.0.2", + "resolved": "https://registry.npmjs.org/multiformats/-/multiformats-11.0.2.tgz", + "integrity": "sha512-b5mYMkOkARIuVZCpvijFj9a6m5wMVLC7cf/jIPd5D/ARDOfLC5+IFkbgDXQgcU2goIsTD/O9NY4DI/Mt4OGvlg==", + "engines": { + "node": ">=16.0.0", + "npm": ">=7.0.0" + } + }, + "node_modules/@ipld/dag-cbor": { + "version": "9.0.4", + "resolved": "https://registry.npmjs.org/@ipld/dag-cbor/-/dag-cbor-9.0.4.tgz", + "integrity": "sha512-HBNVngk/47pKNLTAelN6ORWgKkjJtQj96Xb+jIBtRShJGCsXgghj1TzTynTTIp1dZxwPe5rVIL6yjZmvdyP2Wg==", + "dependencies": { + "cborg": "^2.0.1", + "multiformats": "^12.0.1" + }, + "engines": { + "node": ">=16.0.0", + "npm": ">=7.0.0" + } + }, "node_modules/@ipld/dag-json": { "version": "10.1.3", "resolved": "https://registry.npmjs.org/@ipld/dag-json/-/dag-json-10.1.3.tgz", @@ -4142,6 +4193,25 @@ "resolved": "https://registry.npmjs.org/@types/retry/-/retry-0.12.1.tgz", "integrity": "sha512-xoDlM2S4ortawSWORYqsdU+2rxdh4LRW9ytc3zmT37RIKQh6IHyKwwtKhKis9ah8ol07DCkZxPt8BBvPjC6v4g==" }, + "node_modules/@types/ssh2": { + "version": "0.5.52", + "resolved": "https://registry.npmjs.org/@types/ssh2/-/ssh2-0.5.52.tgz", + "integrity": "sha512-lbLLlXxdCZOSJMCInKH2+9V/77ET2J6NPQHpFI0kda61Dd1KglJs+fPQBchizmzYSOJBgdTajhPqBO1xxLywvg==", + "dev": true, + "dependencies": { + "@types/node": "*", + "@types/ssh2-streams": "*" + } + }, + "node_modules/@types/ssh2-streams": { + "version": "0.1.9", + "resolved": "https://registry.npmjs.org/@types/ssh2-streams/-/ssh2-streams-0.1.9.tgz", + "integrity": "sha512-I2J9jKqfmvXLR5GomDiCoHrEJ58hAOmFrekfFqmCFd+A6gaEStvWnPykoWUwld1PNg4G5ag1LwdA+Lz1doRJqg==", + "dev": true, + "dependencies": { + "@types/node": "*" + } + }, "node_modules/@types/ws": { "version": "8.5.5", "resolved": "https://registry.npmjs.org/@types/ws/-/ws-8.5.5.tgz", @@ -4539,6 +4609,15 @@ "integrity": "sha512-PCVAQswWemu6UdxsDFFX/+gVeYqKAod3D3UVm91jHwynguOwAvYPhx8nNlM++NqRcK6CxxpUafjmhIdKiHibqg==", "dev": true }, + "node_modules/asn1": { + "version": "0.2.6", + "resolved": "https://registry.npmjs.org/asn1/-/asn1-0.2.6.tgz", + "integrity": "sha512-ix/FxPn0MDjeyJ7i/yoHGFt/EX6LyNbxSEhPPXODPL+KB0VPk86UYfL0lMdy+KCnv+fmvIzySwaK5COwqVbWTQ==", + "dev": true, + "dependencies": { + "safer-buffer": "~2.1.0" + } + }, "node_modules/asn1.js": { "version": "5.4.1", "resolved": "https://registry.npmjs.org/asn1.js/-/asn1.js-5.4.1.tgz", @@ -4595,6 +4674,12 @@ "integrity": "sha512-csOlWGAcRFJaI6m+F2WKdnMKr4HhdhFVBk0H/QbJFMCr+uO2kwohwXQPxw/9OCxp05r5ghVBFSyioixx3gfkNQ==", "dev": true }, + "node_modules/async-lock": { + "version": "1.4.0", + "resolved": "https://registry.npmjs.org/async-lock/-/async-lock-1.4.0.tgz", + "integrity": "sha512-coglx5yIWuetakm3/1dsX9hxCNox22h7+V80RQOu2XUUMidtArxKoZoOtHUPuR84SycKTXzgGzAUR5hJxujyJQ==", + "dev": true + }, "node_modules/at-least-node": { "version": "1.0.0", "resolved": "https://registry.npmjs.org/at-least-node/-/at-least-node-1.0.0.tgz", @@ -5178,6 +5263,12 @@ "follow-redirects": "^1.14.4" } }, + "node_modules/b4a": { + "version": "1.6.4", + "resolved": "https://registry.npmjs.org/b4a/-/b4a-1.6.4.tgz", + "integrity": "sha512-fpWrvyVHEKyeEvbKZTVOeZF3VSKKWtJxFIxX/jaVPf+cLbGUSitjb49pHLqPV2BUNNZ0LcoeEGfE/YCpyDYHIw==", + "dev": true + }, "node_modules/balanced-match": { "version": "1.0.2", "resolved": "https://registry.npmjs.org/balanced-match/-/balanced-match-1.0.2.tgz", @@ -5204,6 +5295,15 @@ } ] }, + "node_modules/bcrypt-pbkdf": { + "version": "1.0.2", + "resolved": "https://registry.npmjs.org/bcrypt-pbkdf/-/bcrypt-pbkdf-1.0.2.tgz", + "integrity": "sha512-qeFIXtP4MSoi6NLqO12WfqARWWuCKi2Rn/9hJLEmtB5yTNr9DqFWkJRCf2qShWzPeAMRnOgCrq0sg/KLv5ES9w==", + "dev": true, + "dependencies": { + "tweetnacl": "^0.14.3" + } + }, "node_modules/big-integer": { "version": "1.6.51", "resolved": "https://registry.npmjs.org/big-integer/-/big-integer-1.6.51.tgz", @@ -5476,6 +5576,16 @@ "node": ">=0.2.0" } }, + "node_modules/buildcheck": { + "version": "0.0.6", + "resolved": "https://registry.npmjs.org/buildcheck/-/buildcheck-0.0.6.tgz", + "integrity": "sha512-8f9ZJCUXyT1M35Jx7MkBgmBMo3oHTTBIPLiY9xyL0pl3T5RwcPEY8cUHr5LBNfu/fk6c2T4DJZuVM/8ZZT2D2A==", + "dev": true, + "optional": true, + "engines": { + "node": ">=10.0.0" + } + }, "node_modules/builtin-modules": { "version": "3.2.0", "resolved": "https://registry.npmjs.org/builtin-modules/-/builtin-modules-3.2.0.tgz", @@ -5500,6 +5610,15 @@ "node": ">=10.16.0" } }, + "node_modules/byline": { + "version": "5.0.0", + "resolved": "https://registry.npmjs.org/byline/-/byline-5.0.0.tgz", + "integrity": "sha512-s6webAy+R4SR8XVuJWt2V2rGvhnrhxN+9S15GNuTK3wKPOXFF6RNc+8ug2XhH+2s4f+uudG4kUVYmYOQWL2g0Q==", + "dev": true, + "engines": { + "node": ">=0.10.0" + } + }, "node_modules/bytes": { "version": "3.1.2", "resolved": "https://registry.npmjs.org/bytes/-/bytes-3.1.2.tgz", @@ -5560,6 +5679,40 @@ } ] }, + "node_modules/cardex": { + "version": "2.3.1", + "resolved": "https://registry.npmjs.org/cardex/-/cardex-2.3.1.tgz", + "integrity": "sha512-850vVrRGg48z3aPuEmPvDjQBs+Bp4shSEF8Smmr412NvhCJ7ka2kBQzruoBF6FrotexGxftvmILRH3gUTZQmpQ==", + "dependencies": { + "@ipld/car": "^5.1.0", + "multiformats": "^11.0.2", + "sade": "^1.8.1", + "uint8arrays": "^4.0.3", + "varint": "^6.0.0" + }, + "bin": { + "cardex": "src/bin.js" + } + }, + "node_modules/cardex/node_modules/multiformats": { + "version": "11.0.2", + "resolved": "https://registry.npmjs.org/multiformats/-/multiformats-11.0.2.tgz", + "integrity": "sha512-b5mYMkOkARIuVZCpvijFj9a6m5wMVLC7cf/jIPd5D/ARDOfLC5+IFkbgDXQgcU2goIsTD/O9NY4DI/Mt4OGvlg==", + "engines": { + "node": ">=16.0.0", + "npm": ">=7.0.0" + } + }, + "node_modules/carstream": { + "version": "1.1.0", + "resolved": "https://registry.npmjs.org/carstream/-/carstream-1.1.0.tgz", + "integrity": "sha512-tbf8FOnGX1+0kOe77nm9MG53REiqQopDwzwbXYVxUcsKOAHG2KSD++qy95v1vrtRt1Q6L0Sb01it7QwJ+Yt1sQ==", + "dependencies": { + "@ipld/dag-cbor": "^9.0.3", + "multiformats": "^12.0.1", + "uint8arraylist": "^2.4.3" + } + }, "node_modules/cborg": { "version": "2.0.4", "resolved": "https://registry.npmjs.org/cborg/-/cborg-2.0.4.tgz", @@ -7190,6 +7343,21 @@ "integrity": "sha512-ZQBvi1DcpJ4GDqanjucZ2Hj3wEO5pZDS89BWbkcrvdxksJorwUDDZamX9ldFkp9aw2lmBDLgkObEA4DWNJ9FYQ==", "devOptional": true }, + "node_modules/cpu-features": { + "version": "0.0.9", + "resolved": "https://registry.npmjs.org/cpu-features/-/cpu-features-0.0.9.tgz", + "integrity": "sha512-AKjgn2rP2yJyfbepsmLfiYcmtNn/2eUvocUyM/09yB0YDiz39HteK/5/T4Onf0pmdYDMgkBoGvRLvEguzyL7wQ==", + "dev": true, + "hasInstallScript": true, + "optional": true, + "dependencies": { + "buildcheck": "~0.0.6", + "nan": "^2.17.0" + }, + "engines": { + "node": ">=10.0.0" + } + }, "node_modules/crc-32": { "version": "1.2.2", "resolved": "https://registry.npmjs.org/crc-32/-/crc-32-1.2.2.tgz", @@ -7402,6 +7570,88 @@ "node": "^14.15.0 || ^16.10.0 || >=18.0.0" } }, + "node_modules/docker-compose": { + "version": "0.24.2", + "resolved": "https://registry.npmjs.org/docker-compose/-/docker-compose-0.24.2.tgz", + "integrity": "sha512-2/WLvA7UZ6A2LDLQrYW0idKipmNBWhtfvrn2yzjC5PnHDzuFVj1zAZN6MJxVMKP0zZH8uzAK6OwVZYHGuyCmTw==", + "dev": true, + "dependencies": { + "yaml": "^2.2.2" + }, + "engines": { + "node": ">= 6.0.0" + } + }, + "node_modules/docker-compose/node_modules/yaml": { + "version": "2.3.1", + "resolved": "https://registry.npmjs.org/yaml/-/yaml-2.3.1.tgz", + "integrity": "sha512-2eHWfjaoXgTBC2jNM1LRef62VQa0umtvRiDSk6HSzW7RvS5YtkabJrwYLLEKWBc8a5U2PTSCs+dJjUTJdlHsWQ==", + "dev": true, + "engines": { + "node": ">= 14" + } + }, + "node_modules/docker-modem": { + "version": "3.0.8", + "resolved": "https://registry.npmjs.org/docker-modem/-/docker-modem-3.0.8.tgz", + "integrity": "sha512-f0ReSURdM3pcKPNS30mxOHSbaFLcknGmQjwSfmbcdOw1XWKXVhukM3NJHhr7NpY9BIyyWQb0EBo3KQvvuU5egQ==", + "dev": true, + "dependencies": { + "debug": "^4.1.1", + "readable-stream": "^3.5.0", + "split-ca": "^1.0.1", + "ssh2": "^1.11.0" + }, + "engines": { + "node": ">= 8.0" + } + }, + "node_modules/docker-modem/node_modules/readable-stream": { + "version": "3.6.2", + "resolved": "https://registry.npmjs.org/readable-stream/-/readable-stream-3.6.2.tgz", + "integrity": "sha512-9u/sniCrY3D5WdsERHzHE4G2YCXqoG5FTHUiCC4SIbr6XcLZBY05ya9EKjYek9O5xOAwjGq+1JdGBAS7Q9ScoA==", + "dev": true, + "dependencies": { + "inherits": "^2.0.3", + "string_decoder": "^1.1.1", + "util-deprecate": "^1.0.1" + }, + "engines": { + "node": ">= 6" + } + }, + "node_modules/dockerode": { + "version": "3.3.5", + "resolved": "https://registry.npmjs.org/dockerode/-/dockerode-3.3.5.tgz", + "integrity": "sha512-/0YNa3ZDNeLr/tSckmD69+Gq+qVNhvKfAHNeZJBnp7EOP6RGKV8ORrJHkUn20So5wU+xxT7+1n5u8PjHbfjbSA==", + "dev": true, + "dependencies": { + "@balena/dockerignore": "^1.0.2", + "docker-modem": "^3.0.0", + "tar-fs": "~2.0.1" + }, + "engines": { + "node": ">= 8.0" + } + }, + "node_modules/dockerode/node_modules/chownr": { + "version": "1.1.4", + "resolved": "https://registry.npmjs.org/chownr/-/chownr-1.1.4.tgz", + "integrity": "sha512-jJ0bqzaylmJtVnNgzTeSOs8DPavpbYgEr/b0YL8/2GO3xJEhInFmhKMUnEJQjZumK7KXGFhUy89PrsJWlakBVg==", + "dev": true + }, + "node_modules/dockerode/node_modules/tar-fs": { + "version": "2.0.1", + "resolved": "https://registry.npmjs.org/tar-fs/-/tar-fs-2.0.1.tgz", + "integrity": "sha512-6tzWDMeroL87uF/+lin46k+Q+46rAJ0SyPGz7OW7wTgblI273hsBqk2C1j0/xNadNLKDTUL9BukSjB7cwgmlPA==", + "dev": true, + "dependencies": { + "chownr": "^1.1.1", + "mkdirp-classic": "^0.5.2", + "pump": "^3.0.0", + "tar-stream": "^2.0.0" + } + }, "node_modules/dot-prop": { "version": "6.0.1", "resolved": "https://registry.npmjs.org/dot-prop/-/dot-prop-6.0.1.tgz", @@ -7666,6 +7916,12 @@ "integrity": "sha512-f3qQ9oQy9j2AhBe/H9VC91wLmKBCCU/gDOnKNAYG5hswO7BLKj09Hc5HYNz9cGI++xlpDCIgDaitVs03ATR84Q==", "dev": true }, + "node_modules/fast-fifo": { + "version": "1.3.0", + "resolved": "https://registry.npmjs.org/fast-fifo/-/fast-fifo-1.3.0.tgz", + "integrity": "sha512-IgfweLvEpwyA4WgiQe9Nx6VV2QkML2NkvZnk1oKnIzXgXdWxuhF7zw4DvLTPZJn6PIUneiAXPF24QmoEqHTjyw==", + "dev": true + }, "node_modules/fast-jwt": { "version": "1.7.2", "resolved": "https://registry.npmjs.org/fast-jwt/-/fast-jwt-1.7.2.tgz", @@ -9201,6 +9457,12 @@ "node": ">=10" } }, + "node_modules/mkdirp-classic": { + "version": "0.5.3", + "resolved": "https://registry.npmjs.org/mkdirp-classic/-/mkdirp-classic-0.5.3.tgz", + "integrity": "sha512-gKLcREMhtuZRwRAfqP3RFW+TK4JqApVBtOIftVgjuABpAtpxhPGaDcfvbhNvD0B8iD1oUr/txX35NjcaY6Ns/A==", + "dev": true + }, "node_modules/mlly": { "version": "1.4.0", "resolved": "https://registry.npmjs.org/mlly/-/mlly-1.4.0.tgz", @@ -9360,6 +9622,13 @@ "integrity": "sha512-nnbWWOkoWyUsTjKrhgD0dcz22mdkSnpYqbEjIm2nhwhuxlSkpywJmBo8h0ZqJdkp73mb90SssHkN4rsRaBAfAA==", "dev": true }, + "node_modules/nan": { + "version": "2.17.0", + "resolved": "https://registry.npmjs.org/nan/-/nan-2.17.0.tgz", + "integrity": "sha512-2ZTgtl0nJsO0KQCjEpxcIr5D+Yv90plTitZt9JBfQvVJDS5seMl3FOvsh3+9CoYWXf/1l5OaZzzF6nDm4cagaQ==", + "dev": true, + "optional": true + }, "node_modules/nanoid": { "version": "3.3.6", "resolved": "https://registry.npmjs.org/nanoid/-/nanoid-3.3.6.tgz", @@ -9392,6 +9661,26 @@ "node": ">= 0.6" } }, + "node_modules/node-fetch": { + "version": "2.6.13", + "resolved": "https://registry.npmjs.org/node-fetch/-/node-fetch-2.6.13.tgz", + "integrity": "sha512-StxNAxh15zr77QvvkmveSQ8uCQ4+v5FkvNTj0OESmiHu+VRi/gXArXtkWMElOsOUNLtUEvI4yS+rdtOHZTwlQA==", + "dev": true, + "dependencies": { + "whatwg-url": "^5.0.0" + }, + "engines": { + "node": "4.x || >=6.0.0" + }, + "peerDependencies": { + "encoding": "^0.1.0" + }, + "peerDependenciesMeta": { + "encoding": { + "optional": true + } + } + }, "node_modules/node-releases": { "version": "2.0.13", "resolved": "https://registry.npmjs.org/node-releases/-/node-releases-2.0.13.tgz", @@ -9824,6 +10113,38 @@ "read": "^1.0.4" } }, + "node_modules/proper-lockfile": { + "version": "4.1.2", + "resolved": "https://registry.npmjs.org/proper-lockfile/-/proper-lockfile-4.1.2.tgz", + "integrity": "sha512-TjNPblN4BwAWMXU8s9AEz4JmQxnD1NNL7bNOY/AKUzyamc379FWASUhc/K1pL2noVb+XmZKLL68cjzLsiOAMaA==", + "dev": true, + "dependencies": { + "graceful-fs": "^4.2.4", + "retry": "^0.12.0", + "signal-exit": "^3.0.2" + } + }, + "node_modules/proper-lockfile/node_modules/retry": { + "version": "0.12.0", + "resolved": "https://registry.npmjs.org/retry/-/retry-0.12.0.tgz", + "integrity": "sha512-9LkiTwjUh6rT555DtE9rTX+BKByPfrMzEAtnlEtdEwr3Nkffwiihqe2bWADg+OQRjt9gl6ICdmB/ZFDCGAtSow==", + "dev": true, + "engines": { + "node": ">= 4" + } + }, + "node_modules/properties-reader": { + "version": "2.2.0", + "resolved": "https://registry.npmjs.org/properties-reader/-/properties-reader-2.2.0.tgz", + "integrity": "sha512-CgVcr8MwGoBKK24r9TwHfZkLLaNFHQ6y4wgT9w/XzdpacOOi5ciH4hcuLechSDAwXsfrGQtI2JTutY2djOx2Ow==", + "dev": true, + "dependencies": { + "mkdirp": "^1.0.4" + }, + "engines": { + "node": ">=10" + } + }, "node_modules/proxy-addr": { "version": "2.0.7", "resolved": "https://registry.npmjs.org/proxy-addr/-/proxy-addr-2.0.7.tgz", @@ -9896,6 +10217,12 @@ "node": ">=0.4.x" } }, + "node_modules/queue-tick": { + "version": "1.0.1", + "resolved": "https://registry.npmjs.org/queue-tick/-/queue-tick-1.0.1.tgz", + "integrity": "sha512-kJt5qhMxoszgU/62PLP1CJytzd2NKetjSRnyuj31fDd3Rlcz3fzlFdFLD1SItunPwyqEOkca6GbV612BWfaBag==", + "dev": true + }, "node_modules/range-parser": { "version": "1.2.1", "resolved": "https://registry.npmjs.org/range-parser/-/range-parser-1.2.1.tgz", @@ -10370,6 +10697,12 @@ "node": ">=0.10.0" } }, + "node_modules/split-ca": { + "version": "1.0.1", + "resolved": "https://registry.npmjs.org/split-ca/-/split-ca-1.0.1.tgz", + "integrity": "sha512-Q5thBSxp5t8WPTTJQS59LrGqOZqOsrhDGDVm8azCqIBjSBd7nd9o2PM+mDulQQkh8h//4U6hFZnc/mul8t5pWQ==", + "dev": true + }, "node_modules/split2": { "version": "3.2.2", "resolved": "https://registry.npmjs.org/split2/-/split2-3.2.2.tgz", @@ -10399,6 +10732,34 @@ "integrity": "sha512-UUFYD2oWbNwULH6WoVtLUOw8ch586B+HUqcsAjjjeoBQAM1bD4wZRXu01koaxyd8UeYpybWqW4h+lO1Okv40Tg==", "devOptional": true }, + "node_modules/ssh-remote-port-forward": { + "version": "1.0.4", + "resolved": "https://registry.npmjs.org/ssh-remote-port-forward/-/ssh-remote-port-forward-1.0.4.tgz", + "integrity": "sha512-x0LV1eVDwjf1gmG7TTnfqIzf+3VPRz7vrNIjX6oYLbeCrf/PeVY6hkT68Mg+q02qXxQhrLjB0jfgvhevoCRmLQ==", + "dev": true, + "dependencies": { + "@types/ssh2": "^0.5.48", + "ssh2": "^1.4.0" + } + }, + "node_modules/ssh2": { + "version": "1.14.0", + "resolved": "https://registry.npmjs.org/ssh2/-/ssh2-1.14.0.tgz", + "integrity": "sha512-AqzD1UCqit8tbOKoj6ztDDi1ffJZ2rV2SwlgrVVrHPkV5vWqGJOVp5pmtj18PunkPJAuKQsnInyKV+/Nb2bUnA==", + "dev": true, + "hasInstallScript": true, + "dependencies": { + "asn1": "^0.2.6", + "bcrypt-pbkdf": "^1.0.2" + }, + "engines": { + "node": ">=10.16.0" + }, + "optionalDependencies": { + "cpu-features": "~0.0.8", + "nan": "^2.17.0" + } + }, "node_modules/sst": { "version": "2.23.1", "resolved": "https://registry.npmjs.org/sst/-/sst-2.23.1.tgz", @@ -10625,6 +10986,16 @@ "node": ">=10.0.0" } }, + "node_modules/streamx": { + "version": "2.15.1", + "resolved": "https://registry.npmjs.org/streamx/-/streamx-2.15.1.tgz", + "integrity": "sha512-fQMzy2O/Q47rgwErk/eGeLu/roaFWV0jVsogDmrszM9uIw8L5OA+t+V93MgYlufNptfjmYR1tOMWhei/Eh7TQA==", + "dev": true, + "dependencies": { + "fast-fifo": "^1.1.0", + "queue-tick": "^1.0.1" + } + }, "node_modules/string_decoder": { "version": "1.1.1", "resolved": "https://registry.npmjs.org/string_decoder/-/string_decoder-1.1.1.tgz", @@ -10832,6 +11203,28 @@ "node": ">=10" } }, + "node_modules/tar-fs": { + "version": "3.0.4", + "resolved": "https://registry.npmjs.org/tar-fs/-/tar-fs-3.0.4.tgz", + "integrity": "sha512-5AFQU8b9qLfZCX9zp2duONhPmZv0hGYiBPJsyUdqMjzq/mqVpy/rEUSeHk1+YitmxugaptgBh5oDGU3VsAJq4w==", + "dev": true, + "dependencies": { + "mkdirp-classic": "^0.5.2", + "pump": "^3.0.0", + "tar-stream": "^3.1.5" + } + }, + "node_modules/tar-fs/node_modules/tar-stream": { + "version": "3.1.6", + "resolved": "https://registry.npmjs.org/tar-stream/-/tar-stream-3.1.6.tgz", + "integrity": "sha512-B/UyjYwPpMBv+PaFSWAmtYjwdrlEaZQEhMIBFNC5oEG8lpiW8XjcSdmEaClj28ArfKScKHs2nshz3k2le6crsg==", + "dev": true, + "dependencies": { + "b4a": "^1.6.4", + "fast-fifo": "^1.2.0", + "streamx": "^2.15.0" + } + }, "node_modules/tar-stream": { "version": "2.2.0", "resolved": "https://registry.npmjs.org/tar-stream/-/tar-stream-2.2.0.tgz", @@ -10868,6 +11261,40 @@ "integrity": "sha512-3wdGidZyq5PB084XLES5TpOSRA3wjXAlIWMhum2kRcv/41Sn2emQ0dycQW4uZXLejwKvg6EsvbdlVL+FYEct7A==", "devOptional": true }, + "node_modules/testcontainers": { + "version": "10.2.1", + "resolved": "https://registry.npmjs.org/testcontainers/-/testcontainers-10.2.1.tgz", + "integrity": "sha512-R9LUMUEkKGSL2M4cP466Jah+Vi+ZLFlvrT4BENjEKJKNzubATOmDk26RHe8DHeFT+hnMD6fvVii+McXr0UTO7g==", + "dev": true, + "dependencies": { + "@balena/dockerignore": "^1.0.2", + "archiver": "^5.3.1", + "async-lock": "^1.4.0", + "byline": "^5.0.0", + "debug": "^4.3.4", + "docker-compose": "^0.24.2", + "dockerode": "^3.3.5", + "get-port": "^5.1.1", + "node-fetch": "^2.6.12", + "proper-lockfile": "^4.1.2", + "properties-reader": "^2.2.0", + "ssh-remote-port-forward": "^1.0.4", + "tar-fs": "^3.0.4", + "tmp": "^0.2.1" + } + }, + "node_modules/testcontainers/node_modules/get-port": { + "version": "5.1.1", + "resolved": "https://registry.npmjs.org/get-port/-/get-port-5.1.1.tgz", + "integrity": "sha512-g/Q1aTSDOxFpchXC4i8ZWvxA1lnPqx/JHqcpIw0/LX9T8x/GBbi6YnlN5nhaKIFkT8oFsscUKgDJYxfwfS6QsQ==", + "dev": true, + "engines": { + "node": ">=8" + }, + "funding": { + "url": "https://github.com/sponsors/sindresorhus" + } + }, "node_modules/tinybench": { "version": "2.5.0", "resolved": "https://registry.npmjs.org/tinybench/-/tinybench-2.5.0.tgz", @@ -10892,6 +11319,75 @@ "node": ">=14.0.0" } }, + "node_modules/tmp": { + "version": "0.2.1", + "resolved": "https://registry.npmjs.org/tmp/-/tmp-0.2.1.tgz", + "integrity": "sha512-76SUhtfqR2Ijn+xllcI5P1oyannHNHByD80W1q447gU3mp9G9PSpGdWmjUOHRDPiHYacIk66W7ubDTuPF3BEtQ==", + "dev": true, + "dependencies": { + "rimraf": "^3.0.0" + }, + "engines": { + "node": ">=8.17.0" + } + }, + "node_modules/tmp/node_modules/brace-expansion": { + "version": "1.1.11", + "resolved": "https://registry.npmjs.org/brace-expansion/-/brace-expansion-1.1.11.tgz", + "integrity": "sha512-iCuPHDFgrHX7H2vEI/5xpz07zSHB00TpugqhmYtVmMO6518mCuRMoOYFldEBl0g187ufozdaHgWKcYFb61qGiA==", + "dev": true, + "dependencies": { + "balanced-match": "^1.0.0", + "concat-map": "0.0.1" + } + }, + "node_modules/tmp/node_modules/glob": { + "version": "7.2.3", + "resolved": "https://registry.npmjs.org/glob/-/glob-7.2.3.tgz", + "integrity": "sha512-nFR0zLpU2YCaRxwoCJvL6UvCH2JFyFVIvwTLsIf21AuHlMskA1hhTdk+LlYJtOlYt9v6dvszD2BGRqBL+iQK9Q==", + "dev": true, + "dependencies": { + "fs.realpath": "^1.0.0", + "inflight": "^1.0.4", + "inherits": "2", + "minimatch": "^3.1.1", + "once": "^1.3.0", + "path-is-absolute": "^1.0.0" + }, + "engines": { + "node": "*" + }, + "funding": { + "url": "https://github.com/sponsors/isaacs" + } + }, + "node_modules/tmp/node_modules/minimatch": { + "version": "3.1.2", + "resolved": "https://registry.npmjs.org/minimatch/-/minimatch-3.1.2.tgz", + "integrity": "sha512-J7p63hRiAjw1NDEww1W7i37+ByIrOWO5XQQAzZ3VOcL0PNybwpfmV/N05zFAzwQ9USyEcX6t3UO+K5aqBQOIHw==", + "dev": true, + "dependencies": { + "brace-expansion": "^1.1.7" + }, + "engines": { + "node": "*" + } + }, + "node_modules/tmp/node_modules/rimraf": { + "version": "3.0.2", + "resolved": "https://registry.npmjs.org/rimraf/-/rimraf-3.0.2.tgz", + "integrity": "sha512-JZkJMZkAGFFPP2YqXZXPbMlMBgsxzE8ILs4lMIX/2o0L9UBw9O/Y3o6wFw/i9YLapcUJWwqbi3kdxIPdC62TIA==", + "dev": true, + "dependencies": { + "glob": "^7.1.3" + }, + "bin": { + "rimraf": "bin.js" + }, + "funding": { + "url": "https://github.com/sponsors/isaacs" + } + }, "node_modules/to-fast-properties": { "version": "2.0.0", "resolved": "https://registry.npmjs.org/to-fast-properties/-/to-fast-properties-2.0.0.tgz", @@ -10922,6 +11418,12 @@ "node": ">=0.6" } }, + "node_modules/tr46": { + "version": "0.0.3", + "resolved": "https://registry.npmjs.org/tr46/-/tr46-0.0.3.tgz", + "integrity": "sha512-N3WMsuqV66lT30CrXNbEjx4GEwlow3v6rr4mCcv6prnfwhS01rkgyFdjPNBYd9br7LpXV1+Emh01fHnq2Gdgrw==", + "dev": true + }, "node_modules/traverse": { "version": "0.3.9", "resolved": "https://registry.npmjs.org/traverse/-/traverse-0.3.9.tgz", @@ -10945,6 +11447,12 @@ "resolved": "https://registry.npmjs.org/tslib/-/tslib-2.6.1.tgz", "integrity": "sha512-t0hLfiEKfMUoqhG+U1oid7Pva4bbDPHYfJNiB7BiIjRkj1pyC++4N3huJfqY6aRH6VTB0rvtzQwjM4K6qpfOig==" }, + "node_modules/tweetnacl": { + "version": "0.14.5", + "resolved": "https://registry.npmjs.org/tweetnacl/-/tweetnacl-0.14.5.tgz", + "integrity": "sha512-KXXFFdAbFXY4geFIwoyNK+f5Z1b7swfXABfL7HXCmoIWMKU3dmS26672A4EeQtDzLKy7SXmfBu51JolvEKwtGA==", + "dev": true + }, "node_modules/type-detect": { "version": "4.0.8", "resolved": "https://registry.npmjs.org/type-detect/-/type-detect-4.0.8.tgz", @@ -11004,6 +11512,26 @@ "integrity": "sha512-RsPyTbqORDNDxqAdQPQBpgqhWle1VcTSou/FraClYlHf6TZnQcGslpLcAphNR+sQW4q5lLWLbOsRlh9j24baQg==", "dev": true }, + "node_modules/uint8arraylist": { + "version": "2.4.3", + "resolved": "https://registry.npmjs.org/uint8arraylist/-/uint8arraylist-2.4.3.tgz", + "integrity": "sha512-oEVZr4/GrH87K0kjNce6z8pSCzLEPqHNLNR5sj8cJOySrTP8Vb/pMIbZKLJGhQKxm1TiZ31atNrpn820Pyqpow==", + "dependencies": { + "uint8arrays": "^4.0.2" + }, + "engines": { + "node": ">=16.0.0", + "npm": ">=7.0.0" + } + }, + "node_modules/uint8arrays": { + "version": "4.0.6", + "resolved": "https://registry.npmjs.org/uint8arrays/-/uint8arrays-4.0.6.tgz", + "integrity": "sha512-4ZesjQhqOU2Ip6GPReIwN60wRxIupavL8T0Iy36BBHr2qyMrNxsPJvr7vpS4eFt8F8kSguWUPad6ZM9izs/vyw==", + "dependencies": { + "multiformats": "^12.0.1" + } + }, "node_modules/ultron": { "version": "1.1.1", "resolved": "https://registry.npmjs.org/ultron/-/ultron-1.1.1.tgz", @@ -11212,6 +11740,11 @@ "node": ">=12" } }, + "node_modules/varint": { + "version": "6.0.0", + "resolved": "https://registry.npmjs.org/varint/-/varint-6.0.0.tgz", + "integrity": "sha512-cXEIW6cfr15lFv563k4GuVuW/fiwjknytD37jIOLSdSWuOI6WnO/oKwmP2FQTU2l01LP8/M5TSAJpzUaGe3uWg==" + }, "node_modules/vary": { "version": "1.1.2", "resolved": "https://registry.npmjs.org/vary/-/vary-1.1.2.tgz", @@ -11407,6 +11940,12 @@ "tslib": "^2.4.0" } }, + "node_modules/webidl-conversions": { + "version": "3.0.1", + "resolved": "https://registry.npmjs.org/webidl-conversions/-/webidl-conversions-3.0.1.tgz", + "integrity": "sha512-2JAn3z8AR6rjK8Sm8orRC0h/bcl/DqL7tRPdGZ4I1CjdF+EaMLmYxBHyXuKL849eucPFhvBoxMsflfOb8kxaeQ==", + "dev": true + }, "node_modules/websocket-stream": { "version": "5.5.2", "resolved": "https://registry.npmjs.org/websocket-stream/-/websocket-stream-5.5.2.tgz", @@ -11438,6 +11977,16 @@ "ultron": "~1.1.0" } }, + "node_modules/whatwg-url": { + "version": "5.0.0", + "resolved": "https://registry.npmjs.org/whatwg-url/-/whatwg-url-5.0.0.tgz", + "integrity": "sha512-saE57nupxk6v3HY35+jzBwYa0rKSy0XR8JSxZPwgLr7ys0IBzhGviA1/TUGJLmSVqs8pb9AnvICXEuOHLprYTw==", + "dev": true, + "dependencies": { + "tr46": "~0.0.3", + "webidl-conversions": "^3.0.0" + } + }, "node_modules/which": { "version": "1.3.1", "resolved": "https://registry.npmjs.org/which/-/which-1.3.1.tgz", @@ -11775,14 +12324,37 @@ "version": "0.0.0", "dependencies": { "@aws-sdk/client-s3": "^3.383.0", - "multiformats": "^12.0.1" + "cardex": "^2.3.1", + "carstream": "^1.1.0", + "multiformats": "^12.0.1", + "uint8arraylist": "^2.4.3" }, "devDependencies": { "@types/aws-lambda": "^8.10.119", "@types/node": "^20.4.7", + "nanoid": "^4.0.2", "sst": "^2.23.1", + "testcontainers": "^10.2.1", "vitest": "^0.34.1" } + }, + "packages/functions/node_modules/nanoid": { + "version": "4.0.2", + "resolved": "https://registry.npmjs.org/nanoid/-/nanoid-4.0.2.tgz", + "integrity": "sha512-7ZtY5KTCNheRGfEFxnedV5zFiORN1+Y1N6zvPTnHQd8ENUvfaDBeuJDZb2bN/oXwXxu3qkTXDzy57W5vAmDTBw==", + "dev": true, + "funding": [ + { + "type": "github", + "url": "https://github.com/sponsors/ai" + } + ], + "bin": { + "nanoid": "bin/nanoid.js" + }, + "engines": { + "node": "^14 || ^16 || >=18" + } } } } diff --git a/packages/functions/package.json b/packages/functions/package.json index c082be0..2109586 100644 --- a/packages/functions/package.json +++ b/packages/functions/package.json @@ -3,17 +3,22 @@ "version": "0.0.0", "type": "module", "scripts": { - "test": "sst bind vitest", + "test": "vitest run", "typecheck": "tsc -noEmit" }, "devDependencies": { "@types/aws-lambda": "^8.10.119", "@types/node": "^20.4.7", + "nanoid": "^4.0.2", "sst": "^2.23.1", + "testcontainers": "^10.2.1", "vitest": "^0.34.1" }, "dependencies": { "@aws-sdk/client-s3": "^3.383.0", - "multiformats": "^12.0.1" + "cardex": "^2.3.1", + "carstream": "^1.1.0", + "multiformats": "^12.0.1", + "uint8arraylist": "^2.4.3" } } diff --git a/packages/functions/src/copy.ts b/packages/functions/src/copy.ts new file mode 100644 index 0000000..edb28f2 --- /dev/null +++ b/packages/functions/src/copy.ts @@ -0,0 +1,263 @@ +import crypto from 'node:crypto' +import { ApiHandler } from 'sst/node/api' +import { AbortMultipartUploadCommand, CompleteMultipartUploadCommand, CompletedPart, CreateMultipartUploadCommand, GetObjectCommand, HeadObjectCommand, PutObjectCommand, S3Client, UploadPartCommand } from '@aws-sdk/client-s3' +import { Readable } from 'node:stream' +import * as Link from 'multiformats/link' +import { UnknownLink } from 'multiformats/link' +import * as Digest from 'multiformats/hashes/digest' +import { sha256 } from 'multiformats/hashes/sha2' +import { base64pad } from 'multiformats/bases/base64' +import { Uint8ArrayList } from 'uint8arraylist' +import { CARReaderStream } from 'carstream' +import { MultihashIndexSortedWriter } from 'cardex/multihash-index-sorted' +import { mustGetEnv, errorResponse } from './lib/util' + +const CAR_CODEC = 0x0202 +const MAX_PUT_SIZE = 1024 * 1024 * 1024 * 5 +const TARGET_PART_SIZE = 1024 * 1024 * 100 + +type ShardLink = Link.Link + +interface ObjectID { + region: string + bucket: string + key: string + endpoint?: string + credentials?: { + accessKeyId: string, + secretAccessKey: string + } +} + +interface ContentAddressedObjectID< + Data extends unknown = unknown, + Format extends number = number, + Alg extends number = number, + V extends Link.Version = 1 +> extends ObjectID { + cid: Link.Link +} + +interface ShardObjectID extends ContentAddressedObjectID {} + +interface ShardSource extends ContentAddressedObjectID { + size: number + body: ReadableStream +} + +interface PartSource { + uploadID: string + partNumber: number + body: Uint8Array +} + +export const handler = ApiHandler(event => _handler.call(null, new Request(`http://localhost/?${event.rawQueryString}`), process.env)) + +export const _handler = async (request: Request, env: Record) => { + try { + const { searchParams } = new URL(request.url) + + const srcRegion = searchParams.get('region') + if (!srcRegion) return errorResponse('Missing "region" search parameter', 400) + if (!['us-east-2', 'us-west-2'].includes(srcRegion)) return errorResponse('Invalid region', 400) + + const srcBucketName = searchParams.get('bucket') + if (!srcBucketName) return errorResponse('Missing "bucket" search parameter', 400) + if (!srcBucketName.startsWith('dotstorage')) return errorResponse('Invalid bucket', 400) + + const srcKey = searchParams.get('key') + if (!srcKey) return errorResponse('Missing "key" search parameter', 400) + if (!srcKey.endsWith('.car')) return errorResponse('Only keys for CARs supported', 400) + + const shardstr = searchParams.get('shard') + if (!shardstr) return errorResponse('Missing "shard" search parameter', 400) + const shard: ShardLink = Link.parse(shardstr) + if (shard.code !== CAR_CODEC) return errorResponse('Not a CAR file hash', 400) + + const rootstr = searchParams.get('root') + if (!rootstr) return errorResponse('Missing "root" search parameter', 400) + const root: UnknownLink = Link.parse(rootstr).toV1() + + const src = { + cid: shard, + region: srcRegion, + bucket: srcBucketName, + key: srcKey + } + + const dest = { + endpoint: mustGetEnv(env, 'DEST_ENDPOINT'), + region: mustGetEnv(env, 'DEST_REGION'), + credentials: { + accessKeyId: mustGetEnv(env, 'DEST_ACCESS_KEY_ID'), + secretAccessKey: mustGetEnv(env, 'DEST_SECRET_ACCESS_KEY') + } + } + + return await copy(src, { + bucket: mustGetEnv(env, 'CARPARK_BUCKET'), + key: `${shard}/${shard}.car`, + ...dest + }, { + bucket: mustGetEnv(env, 'SATNAV_BUCKET'), + key: `${shard}/${shard}.car.idx`, + ...dest + }, { + bucket: mustGetEnv(env, 'DUDEWHERE_BUCKET'), + key: `${root}/${shard}`, + ...dest + }) + } catch (err: any) { + console.error(err) + return errorResponse(err.message, 500) + } +} + +export const copy = async (src: ShardObjectID, dest: ObjectID, indexDest: ObjectID, linkDest: ObjectID, options?: { maxPutSize?: number }) => { + try { + console.log(`HeadObject ${dest.region}/${dest.bucket}/${dest.key}`) + await s3Client(dest).send(new HeadObjectCommand({ Bucket: dest.bucket, Key: dest.key })) + return { statusCode: 200, body: JSON.stringify({ ok: true }) } // already exists 🙌 + } catch (err: any) { + if (err.$metadata?.httpStatusCode !== 404) { + console.error(err) + return errorResponse('Failed to determine if object exists at destination', 500) + } + } + + console.log(`GetObject ${src.region}/${src.bucket}/${src.key}`) + const getCmd = new GetObjectCommand({ Bucket: src.bucket, Key: src.key }) + const getRes = await s3Client(src).send(getCmd) + if (!getRes.Body) return errorResponse('Object not found', 404) + if (!getRes.ContentLength) return errorResponse('Object has no size', 404) + + const [srcReadable0, srcReadable1] = getRes.Body.transformToWebStream().tee() + await Promise.all([ + writeCAR({ + ...src, + size: getRes.ContentLength, + body: srcReadable0 + }, dest, options), + writeCARIndex({ + ...src, + size: getRes.ContentLength, + body: srcReadable1 + }, indexDest), + s3Client(linkDest).send(new PutObjectCommand({ + Bucket: linkDest.bucket, + Key: linkDest.key, + Body: new Uint8Array() + })) + ]) + + return { statusCode: 200, body: JSON.stringify({ ok: true }) } +} + +const writeCAR = async (src: ShardSource, dest: ObjectID, options?: { maxPutSize?: number }) => { + const maxPutSize = options?.maxPutSize ?? MAX_PUT_SIZE + // for small files, just do a regular put with ChecksumSHA256 + if (src.size < maxPutSize) { + console.log(`PutObject ${src.region}/${src.bucket}/${src.key} => ${dest.region}/${dest.bucket}/${dest.key}`) + return s3Client(dest).send(new PutObjectCommand({ + Bucket: dest.bucket, + Key: dest.key, + // @ts-expect-error + Body: Readable.fromWeb(src.body), + ContentLength: src.size, + ChecksumSHA256: base64pad.encode(src.cid.multihash.digest).slice(1) + })) + } + + const hasher = crypto.createHash('sha256') + const buffer = new Uint8ArrayList() + const parts: CompletedPart[] = [] + let uploadID: string + + await src.body.pipeTo(new WritableStream({ + async start () { + console.log(`CreateMultipartUpload ${src.region}/${src.bucket}/${src.key} => ${dest.region}/${dest.bucket}/${dest.key}`) + const res = await s3Client(dest).send(new CreateMultipartUploadCommand({ Bucket: dest.bucket, Key: dest.key })) + if (!res.UploadId) throw new Error('missing multipart upload ID') + uploadID = res.UploadId + }, + async write (chunk) { + buffer.append(chunk) + hasher.update(chunk) + if (buffer.length >= TARGET_PART_SIZE) { + const part = await uploadPart({ uploadID, partNumber: parts.length, body: buffer.subarray() }, dest) + parts.push(part) + buffer.consume(buffer.length) + } + }, + async close () { + if (buffer.length) { + const part = await uploadPart({ uploadID, partNumber: parts.length, body: buffer.subarray() }, dest) + parts.push(part) + } + + const digest = Digest.create(sha256.code, hasher.digest()) + if (Link.create(CAR_CODEC, digest).toString() !== src.cid.toString()) { + console.log(`AbortMultipartUpload ${src.region}/${src.bucket}/${src.key} => ${dest.region}/${dest.bucket}/${dest.key}`) + await s3Client(dest).send(new AbortMultipartUploadCommand({ Bucket: dest.bucket, Key: dest.key, UploadId: uploadID })) + throw new Error('integrity check failed') + } + + console.log(`CompleteMultipartUpload ${src.region}/${src.bucket}/${src.key} => ${dest.region}/${dest.bucket}/${dest.key}`) + await s3Client(dest).send(new CompleteMultipartUploadCommand({ + Bucket: dest.bucket, + Key: dest.key, + UploadId: uploadID, + MultipartUpload: { Parts: parts } + })) + } + })) +} + +const uploadPart = async (src: PartSource, dest: ObjectID): Promise => { + console.log(`UploadPart ${src.uploadID} (#${src.partNumber}) => ${dest.region}/${dest.bucket}/${dest.key}`) + const digest = await sha256.digest(src.body) + const checksum = base64pad.encode(digest.digest).slice(1) + const res = await s3Client(dest).send(new UploadPartCommand({ + UploadId: src.uploadID, + PartNumber: src.partNumber, + Bucket: dest.bucket, + Key: dest.key, + Body: src.body, + ContentLength: src.body.length, + ChecksumSHA256: checksum + })) + return { ETag: res.ETag, PartNumber: src.partNumber, ChecksumSHA256: checksum } +} + +const writeCARIndex = async (src: ShardSource, dest: ObjectID) => { + const { readable, writable } = new TransformStream() + const writer = MultihashIndexSortedWriter.createWriter({ writer: writable.getWriter() }) + const chunks: Uint8Array[] = [] + await Promise.all([ + src.body + .pipeThrough(new CARReaderStream()) + .pipeTo(new WritableStream({ + async write (block) { + await writer.add(block.cid, block.offset) + }, + async close () { + await writer.close() + } + })), + readable.pipeTo(new WritableStream({ write: chunk => { chunks.push(chunk) } })) + ]) + await s3Client(dest).send(new PutObjectCommand({ + Bucket: dest.bucket, + Key: dest.key, + Body: Uint8ArrayList.fromUint8Arrays(chunks).subarray() + })) +} + +const s3Client = ({ region, endpoint, credentials }: { + region: string + endpoint?: string + credentials?: { + accessKeyId: string, + secretAccessKey: string + } +}) => new S3Client({ region, endpoint, credentials }) diff --git a/packages/functions/src/lambda.ts b/packages/functions/src/hash.ts similarity index 92% rename from packages/functions/src/lambda.ts rename to packages/functions/src/hash.ts index 7405bb7..4665f9e 100644 --- a/packages/functions/src/lambda.ts +++ b/packages/functions/src/hash.ts @@ -4,6 +4,7 @@ import { GetObjectCommand, S3Client } from '@aws-sdk/client-s3' import * as Link from 'multiformats/link' import * as Digest from 'multiformats/hashes/digest' import { sha256 } from 'multiformats/hashes/sha2' +import { errorResponse } from './lib/util' const CAR_CODEC = 0x0202 @@ -37,8 +38,3 @@ export const handler = ApiHandler(async event => { return { statusCode: 200, body: `{"ok":true,"cid":{"/":"${cid}"}}` } }) - -const errorResponse = (message: string, statusCode = 500) => ({ - statusCode, - body: JSON.stringify({ ok: false, error: message }) -}) diff --git a/packages/functions/src/lib/util.ts b/packages/functions/src/lib/util.ts new file mode 100644 index 0000000..cbabdfc --- /dev/null +++ b/packages/functions/src/lib/util.ts @@ -0,0 +1,12 @@ +export const notNully = (obj: Record, k: string, msg = 'unexpected null value') => { + const v = obj[k] + if (!v) throw new Error(`${msg}: ${k}`) + return v +} + +export const mustGetEnv = (env: Record, k: string) => notNully(env, k, 'missing enviornment variable') + +export const errorResponse = (message: string, statusCode = 500) => ({ + statusCode, + body: JSON.stringify({ ok: false, error: message }) +}) diff --git a/packages/functions/test/copy.test.ts b/packages/functions/test/copy.test.ts new file mode 100644 index 0000000..943197e --- /dev/null +++ b/packages/functions/test/copy.test.ts @@ -0,0 +1,116 @@ +import { expect, test, beforeAll, beforeEach, afterAll, afterEach, Nullable } from 'vitest' +import fs from 'node:fs' +import { PutObjectCommand, S3Client } from '@aws-sdk/client-s3' +import { UnknownLink, Link } from 'multiformats' +import { TestAWSService, createS3, createS3Bucket, keyExists } from './helpers/aws' +import { generateTestCAR } from './helpers/car' +import { copy } from '../src/copy' + +let s3: TestAWSService +let srcBucket: string +let carparkBucket: string +let satnavBucket: string +let dudewhereBucket: string +let srcCAR: Nullable<{ cid: Link, root: UnknownLink, size: number, path: string }> + +beforeAll(async () => { + s3 = await createS3() +}) + +beforeEach(async () => { + srcBucket = await createS3Bucket(s3.client, 'src') + carparkBucket = await createS3Bucket(s3.client, 'carpark') + satnavBucket = await createS3Bucket(s3.client, 'satnav') + dudewhereBucket = await createS3Bucket(s3.client, 'dudewhere') + srcCAR = null +}) + +afterEach(async () => { + if (srcCAR) await fs.promises.rm(srcCAR.path) +}) + +afterAll(async () => { + await s3.container.stop() +}) + +test('copy a small CAR', async () => { + srcCAR = await generateTestCAR(5 * 1024 * 1024) + const srcKey = `complete/${srcCAR.root}.car` + + await s3.client.send(new PutObjectCommand({ + Bucket: srcBucket, + Key: srcKey, + ContentLength: srcCAR.size, + Body: fs.createReadStream(srcCAR.path) + })) + + const carparkKey = `${srcCAR.cid}/${srcCAR.cid}.car` + const satnavKey = `${srcCAR.cid}/${srcCAR.cid}.car.idx` + const dudewhereKey = `${srcCAR.root}/${srcCAR.cid}` + + const res = await copy({ + ...s3, + cid: srcCAR.cid, + bucket: srcBucket, + key: srcKey + }, { + ...s3, + bucket: carparkBucket, + key: carparkKey + }, { + ...s3, + bucket: satnavBucket, + key: satnavKey + }, { + ...s3, + bucket: dudewhereBucket, + key: dudewhereKey + }) + expect(res.statusCode).toBe(200) + + await expect(keyExists(s3.client, carparkBucket, carparkKey)).resolves.toBe(true) + await expect(keyExists(s3.client, satnavBucket, satnavKey)).resolves.toBe(true) + await expect(keyExists(s3.client, dudewhereBucket, dudewhereKey)).resolves.toBe(true) +}) + +test('copy a large CAR with multipart', async () => { + srcCAR = await generateTestCAR(500 * 1024 * 1024) + const srcKey = `complete/${srcCAR.root}.car` + + await s3.client.send(new PutObjectCommand({ + Bucket: srcBucket, + Key: srcKey, + ContentLength: srcCAR.size, + Body: fs.createReadStream(srcCAR.path) + })) + + const carparkKey = `${srcCAR.cid}/${srcCAR.cid}.car` + const satnavKey = `${srcCAR.cid}/${srcCAR.cid}.car.idx` + const dudewhereKey = `${srcCAR.root}/${srcCAR.cid}` + + console.time('copy') + const res = await copy({ + ...s3, + cid: srcCAR.cid, + bucket: srcBucket, + key: srcKey + }, { + ...s3, + bucket: carparkBucket, + key: carparkKey + }, { + ...s3, + bucket: satnavBucket, + key: satnavKey + }, { + ...s3, + bucket: dudewhereBucket, + key: dudewhereKey + }, { maxPutSize: 1024 * 1024 * 50 }) + expect(res.statusCode).toBe(200) + console.timeEnd('copy') + + await expect(keyExists(s3.client, carparkBucket, carparkKey)).resolves.toBe(true) + await expect(keyExists(s3.client, satnavBucket, satnavKey)).resolves.toBe(true) + await expect(keyExists(s3.client, dudewhereBucket, dudewhereKey)).resolves.toBe(true) +}, { timeout: 60_000 }) diff --git a/packages/functions/test/helpers/aws.ts b/packages/functions/test/helpers/aws.ts new file mode 100644 index 0000000..028ba86 --- /dev/null +++ b/packages/functions/test/helpers/aws.ts @@ -0,0 +1,54 @@ +import { GenericContainer, StartedTestContainer } from 'testcontainers' +import { customAlphabet } from 'nanoid' +import { S3Client, CreateBucketCommand, HeadObjectCommand } from '@aws-sdk/client-s3' + +const id = customAlphabet('1234567890abcdefghijklmnopqrstuvwxyz', 10) +const credentials = { accessKeyId: 'minioadmin', secretAccessKey: 'minioadmin' } + +export interface TestAWSService { + client: T + container: StartedTestContainer + region: string + endpoint: string + credentials: { accessKeyId: string, secretAccessKey: string } +} + +export const createS3 = async (opts: { port?: number, region?: string } = {}): Promise> => { + console.log('Creating local S3...') + const region = opts.region || 'us-west-2' + const port = opts.port || 9000 + + const container = await new GenericContainer('quay.io/minio/minio') + .withCommand(['server', '/data']) + .withExposedPorts(port) + .start() + + const clientOpts = { + endpoint: `http://127.0.0.1:${container.getMappedPort(port)}`, + forcePathStyle: true, + region, + credentials + } + + return { container, client: new S3Client(clientOpts), ...clientOpts } +} + +export const createS3Bucket = async (s3: S3Client, pfx = '') => { + const name = (pfx ? `${pfx}-` : '') + id() + console.log(`Creating S3 bucket "${name}"...`) + await s3.send(new CreateBucketCommand({ Bucket: name })) + return name +} + +export const keyExists = async (s3: S3Client, bucket: string, key: string) => { + try { + await s3.send(new HeadObjectCommand({ Bucket: bucket, Key: key })) + return true + } catch (err: any) { + if (err.$metadata?.httpStatusCode !== 404) { + console.error(err) + throw new Error('failed head request', { cause: err }) + } + return false + } +} diff --git a/packages/functions/test/helpers/car.ts b/packages/functions/test/helpers/car.ts new file mode 100644 index 0000000..3ef847f --- /dev/null +++ b/packages/functions/test/helpers/car.ts @@ -0,0 +1,72 @@ +import crypto from 'node:crypto' +import fs from 'node:fs' +import os from 'node:os' +import path from 'node:path' +import { Readable, Writable } from 'node:stream' +import { CARWriterStream } from 'carstream' +import * as Link from 'multiformats/link' +import { sha256 } from 'multiformats/hashes/sha2' +import * as raw from 'multiformats/codecs/raw' +import * as Block from 'multiformats/block' +import * as Digest from 'multiformats/hashes/digest' +import { customAlphabet } from 'nanoid' + +const id = customAlphabet('1234567890abcdefghijklmnopqrstuvwxyz', 10) + +export const generateTestCAR = async (targetSize: number) => { + const car = await testCAR(targetSize) + const carPath = path.join(os.tmpdir(), `${id()}.car`) + await car.body.pipeTo(Writable.toWeb(fs.createWriteStream(carPath))) + const cid = await carHash(carPath) + const size = await carSize(carPath) + console.log(`generated test CAR: ${cid}`) + console.log(` root: ${car.root}`) + console.log(` size: ${size} bytes`) + console.log(` path: ${carPath}`) + return { cid, root: car.root, size, path: carPath } +} + +const randomBlock = async () => { + const bytes = crypto.randomBytes(randomInt(1, 1024 * 1024 * 2)) + const cid = Link.create(raw.code, await sha256.digest(bytes)) + // @ts-expect-error + const block = Block.createUnsafe({ bytes, cid, codec: raw }) + return block +} + +const testCAR = async (targetSize: number) => { + const root = await randomBlock() + let total = root.bytes.length + + const src = new ReadableStream({ + async pull (controller) { + const block = await randomBlock() + total += block.bytes.length + controller.enqueue(block) + if (total >= targetSize) { + controller.close() + } + } + }) + + return { root: root.cid, body: src.pipeThrough(new CARWriterStream([root.cid])) } +} + +const carHash = async (path: string) => { + const hasher = crypto.createHash('sha256') + await Readable.toWeb(fs.createReadStream(path)) + .pipeTo(new WritableStream({ write: chunk => { hasher.update(chunk) } })) + const digest = Digest.create(sha256.code, hasher.digest()) + return Link.create(0x0202, digest) +} + +const carSize = async (path: string) => { + const stat = await fs.promises.stat(path) + return stat.size +} + +const randomInt = (min: number, max: number) => { + min = Math.ceil(min) + max = Math.floor(max) + return Math.floor(Math.random() * (max - min) + min) +} diff --git a/stacks/api.ts b/stacks/api.ts index 0664a16..3a1a800 100644 --- a/stacks/api.ts +++ b/stacks/api.ts @@ -1,6 +1,13 @@ -import { StackContext, Function } from "sst/constructs"; +import { StackContext, Function, Config } from 'sst/constructs' +import { mustGetEnv } from '../packages/functions/src/lib/util' + +export function API ({ stack }: StackContext) { + const DEST_ENDPOINT = mustGetEnv(process.env, 'DEST_ENDPOINT') + const DEST_REGION = mustGetEnv(process.env, 'DEST_REGION') + const CARPARK_BUCKET = mustGetEnv(process.env, 'CARPARK_BUCKET') + const SATNAV_BUCKET = mustGetEnv(process.env, 'SATNAV_BUCKET') + const DUDEWHERE_BUCKET = mustGetEnv(process.env, 'DUDEWHERE_BUCKET') -export function API({ stack }: StackContext) { stack.setDefaultFunctionProps({ memorySize: '1 GB', runtime: 'nodejs18.x', @@ -8,12 +15,33 @@ export function API({ stack }: StackContext) { timeout: '15 minutes' }) - const fun = new Function(stack, 'fn', { - handler: 'packages/functions/src/lambda.handler', + const hashFunction = new Function(stack, 'hash', { + handler: 'packages/functions/src/hash.handler', url: { cors: true, authorizer: 'none' } }) - fun.attachPermissions(['s3:GetObject']) + hashFunction.attachPermissions(['s3:GetObject']) + + const accessKeyID = new Config.Secret(stack, 'DEST_ACCESS_KEY_ID') + const secretAccessKey = new Config.Secret(stack, 'DEST_SECRET_ACCESS_KEY') + + const copyFunction = new Function(stack, 'copy', { + handler: 'packages/functions/src/copy.handler', + url: { cors: true, authorizer: 'none' }, + environment: { + DEST_ENDPOINT, + DEST_REGION, + CARPARK_BUCKET, + SATNAV_BUCKET, + DUDEWHERE_BUCKET + }, + bind: [accessKeyID, secretAccessKey] + }) + + copyFunction.attachPermissions(['s3:GetObject']) - stack.addOutputs({ url: fun.url }) + stack.addOutputs({ + hashFunctionURL: hashFunction.url, + copyFunctionURL: copyFunction.url + }) }