From e598fe1742260e497b36238dc6c7a26f4190a28a Mon Sep 17 00:00:00 2001 From: Kikorono Date: Mon, 14 Mar 2022 14:17:41 -0500 Subject: [PATCH 01/32] Adding support for archive formats 6 and 7 --- src/archive.ts | 106 ++++++++++++++++++++------------- src/config/archive-flags.ts | 6 ++ src/db/archive-index.entity.ts | 8 ++- src/db/index-service.ts | 8 ++- src/dev.ts | 2 +- 5 files changed, 83 insertions(+), 47 deletions(-) create mode 100644 src/config/archive-flags.ts diff --git a/src/archive.ts b/src/archive.ts index a0d3634..51f5d45 100644 --- a/src/archive.ts +++ b/src/archive.ts @@ -6,6 +6,7 @@ import { ArchiveFormat, FileState, FlatFile, Group } from './index'; import { ArchiveIndexEntity } from './db'; import { FileBreadcrumb, IndexedFile } from './indexed-file'; import { ArchiveConfig } from './config'; +import { archiveFlags } from './config/archive-flags'; export class Archive extends IndexedFile { @@ -50,25 +51,24 @@ export class Archive extends IndexedFile { const archiveData = this._data; const format = this.index.format = archiveData.get('byte', 'unsigned'); - const filesNamed = (archiveData.get('byte', 'unsigned') & 0x01) !== 0; - const groupCount = this.index.groupCount = archiveData.get('short', 'unsigned'); + const mainDataType = format >= ArchiveFormat.smart ? 'smart_int' : 'short'; + this.index.version = format >= ArchiveFormat.versioned ? archiveData.get('int') : 0; + const flags = archiveFlags(archiveData.get('byte', 'unsigned')); + const groupCount = archiveData.get(mainDataType, 'unsigned'); logger.info(`${groupCount} groups were found within the ${this.name} archive.`); - if(filesNamed !== this.config.filesNamed) { - logger.warn(`Archive file name flag mismatch; expected ${this.config.filesNamed} ` + - `but received ${filesNamed}!`); - } - - const groupIndices: number[] = new Array(groupCount); + const groupKeys: number[] = new Array(groupCount); + const groupChildCounts: Map = new Map(); let accumulator = 0; + // Group index keys for(let i = 0; i < groupCount; i++) { - const delta = archiveData.get('short', 'unsigned'); - groupIndices[i] = accumulator += delta; + const delta = archiveData.get(mainDataType, 'unsigned'); + groupKeys[i] = accumulator += delta; const group = new Group(this.indexService.validateGroup({ - numericKey: groupIndices[i], - name: String(groupIndices[i]), + numericKey: groupKeys[i], + name: String(groupKeys[i]), archive: this }), { store: this.store, @@ -76,45 +76,69 @@ export class Archive extends IndexedFile { }); group.setState(FileState.encoded); - this.set(groupIndices[i], group); + this.set(groupKeys[i], group); } - if(filesNamed) { - for(const groupIndex of groupIndices) { + // Group names + if(flags.groupNames) { + for(const groupIndex of groupKeys) { const group = this.get(groupIndex); group.nameHash = group.index.nameHash = archiveData.get('int'); group.name = group.index.name = this.store.findFileName(group.nameHash, String(group.nameHash)); } } - /* read the crc values */ - for(const groupIndex of groupIndices) { - const group = this.get(groupIndex); + // Compressed file data CRC32 checksums + for(const key of groupKeys) { + const group = this.get(key); group.crc32 = archiveData.get('int'); } - /* read the version numbers */ - for(const groupIndex of groupIndices) { - const group = this.get(groupIndex); - group.version = archiveData.get('int'); + // Decompressed file data CRC32 checksums + if(flags.decompressedCrcs) { + for(const key of groupKeys) { + const decompressedCrc32 = archiveData.get('int'); + // @TODO assign to group (requires changing existing 'crc32' to 'compressedCrc`) + } } - /* read the child count */ - const groupChildCounts: Map = new Map(); + // File data whirlpool digests + if(flags.whirlpoolDigests) { + for(const key of groupKeys) { + const whirlpoolDigest = new ByteBuffer(512); + archiveData.getBytes(whirlpoolDigest, 512); + // @TODO assign to group + } + } + + // Group file sizes + if(flags.groupSizes) { + for(const key of groupKeys) { + const compressedSize = archiveData.get('int'); + const decompressedSize = archiveData.get('int'); + // @TODO assign to group (requires changing existing 'size' to 'compressedSize') + } + } + + // Group version numbers + for(const key of groupKeys) { + const group = this.get(key); + group.version = archiveData.get('int'); + } - for(const groupIndex of groupIndices) { - // group file count - groupChildCounts.set(groupIndex, archiveData.get('short', 'unsigned')); + // Group file counts + for(const key of groupKeys) { + groupChildCounts.set(key, archiveData.get(mainDataType, 'unsigned')); } - /* read the file groupIndices */ - for(const groupIndex of groupIndices) { - const group = this.get(groupIndex) as Group; - const fileCount = groupChildCounts.get(groupIndex); + // Grouped file index keys + for(const key of groupKeys) { + const group = this.get(key) as Group; + const fileCount = groupChildCounts.get(key); accumulator = 0; for(let i = 0; i < fileCount; i++) { - const delta = archiveData.get('short', 'unsigned'); + const delta = archiveData.get(mainDataType, 'unsigned'); const childFileIndex = accumulator += delta; group.set(childFileIndex, new FlatFile(this.indexService.validateFile({ numericKey: childFileIndex, @@ -128,17 +152,17 @@ export class Archive extends IndexedFile { } } - /* read the child name hashes */ - if(filesNamed) { - for(const groupIndex of groupIndices) { - const fileGroup = this.get(groupIndex) as Group; + // Grouped file names + if(flags) { + for(const key of groupKeys) { + const fileGroup = this.get(key) as Group; - for(const [ , childFile ] of fileGroup.files) { + for(const [ , file ] of fileGroup.files) { const nameHash = archiveData.get('int'); - if(childFile) { - childFile.nameHash = childFile.index.nameHash = nameHash; - childFile.name = childFile.index.name = - this.store.findFileName(childFile.nameHash, String(childFile.nameHash)); + if(file) { + file.nameHash = file.index.nameHash = nameHash; + file.name = file.index.name = + this.store.findFileName(file.nameHash, String(file.nameHash)); } } } diff --git a/src/config/archive-flags.ts b/src/config/archive-flags.ts new file mode 100644 index 0000000..4d3958b --- /dev/null +++ b/src/config/archive-flags.ts @@ -0,0 +1,6 @@ +export const archiveFlags = (flags: number) => ({ + groupNames: (flags & 0x01) !== 0, + whirlpoolDigests: (flags & 0x02) !== 0, + groupSizes: (flags & 0x04) !== 0, + decompressedCrcs: (flags & 0x08) !== 0, +}); diff --git a/src/db/archive-index.entity.ts b/src/db/archive-index.entity.ts index 3ef8d8c..ebea8fc 100644 --- a/src/db/archive-index.entity.ts +++ b/src/db/archive-index.entity.ts @@ -4,6 +4,7 @@ import { IndexEntity } from './index-entity'; import { StoreIndexEntity } from './store-index.entity'; import { GroupIndexEntity } from './group-index.entity'; import { FileState } from '../file-state'; +import { ArchiveFormat } from '../config'; @Entity('archive_index') @@ -16,8 +17,11 @@ export class ArchiveIndexEntity extends IndexEntity { @Column('integer', { name: 'group_count', nullable: false, default: 0 }) groupCount: number = 0; - @Column('integer', { name: 'format', nullable: false, default: 5 }) - format: number = 5; + @Column('integer', { name: 'format', nullable: false, default: ArchiveFormat.original }) + format: number = ArchiveFormat.original; + + @Column('integer', { nullable: false, default: 0 }) + version: number = 0; @Column('text', { name: 'data_state', nullable: false }) state: FileState; diff --git a/src/db/index-service.ts b/src/db/index-service.ts index c5425a0..b7654d4 100644 --- a/src/db/index-service.ts +++ b/src/db/index-service.ts @@ -130,6 +130,7 @@ export class IndexService { this.updateEntityIndex(archive); archiveIndex.format = archiveIndex.format || ArchiveFormat.original; + archiveIndex.version = archiveIndex.version || 0; archiveIndex.gameBuild = this.store.gameBuild; archiveIndex.state = archive.state || FileState.unloaded; archiveIndex.groupCount = archive.groups?.size ?? 0; @@ -149,9 +150,10 @@ export class IndexService { let affected; if(existingIndex) { - const { name, size, sha256, crc32, data, state } = archiveIndex; + const { name, size, version, sha256, crc32, data, state } = archiveIndex; existingIndex.name = name; existingIndex.size = size; + existingIndex.version = version; existingIndex.sha256 = sha256; existingIndex.crc32 = crc32; existingIndex.data = data; @@ -372,10 +374,10 @@ export class IndexService { file.nameHash = -1; } - if(index instanceof GroupIndexEntity || index instanceof FileIndexEntity) { + if(index instanceof ArchiveIndexEntity || index instanceof GroupIndexEntity || index instanceof FileIndexEntity) { index.version = file.version; - if(file.archive?.config?.versioned && file.modified) { + if(file.modified) { index.version = index.version ? index.version + 1 : 1; } } diff --git a/src/dev.ts b/src/dev.ts index 244f61f..1d1245c 100644 --- a/src/dev.ts +++ b/src/dev.ts @@ -2,5 +2,5 @@ import { Store } from './index'; import { join } from 'path'; -const store = Store.create(435); +const store = Store.create('435'); From 83d210633b9155f95eadc7956de845baf6352961 Mon Sep 17 00:00:00 2001 From: Kikorono Date: Mon, 14 Mar 2022 14:41:10 -0500 Subject: [PATCH 02/32] Fixing bugs with the new formats --- src/archive.ts | 4 +-- src/db/index-service.ts | 14 +++++++--- src/scripts/unpacker.ts | 62 +++++++++++++++++++---------------------- 3 files changed, 41 insertions(+), 39 deletions(-) diff --git a/src/archive.ts b/src/archive.ts index 51f5d45..d95a909 100644 --- a/src/archive.ts +++ b/src/archive.ts @@ -128,7 +128,7 @@ export class Archive extends IndexedFile { // Group file counts for(const key of groupKeys) { - groupChildCounts.set(key, archiveData.get(mainDataType, 'unsigned')); + groupChildCounts.set(key, archiveData.get('short', 'unsigned')); } // Grouped file index keys @@ -153,7 +153,7 @@ export class Archive extends IndexedFile { } // Grouped file names - if(flags) { + if(flags.groupNames) { for(const key of groupKeys) { const fileGroup = this.get(key) as Group; diff --git a/src/db/index-service.ts b/src/db/index-service.ts index b7654d4..c5ea9d7 100644 --- a/src/db/index-service.ts +++ b/src/db/index-service.ts @@ -33,8 +33,8 @@ export class IndexService { database: join(indexPath, `index_${this.store.gameBuild}.sqlite3`), entities: [ StoreIndexEntity, ArchiveIndexEntity, GroupIndexEntity, FileIndexEntity ], synchronize: true, - // logging: [ 'error', 'warn' ], - logging: 'all', + logging: [ 'error', 'warn' ], + // logging: 'all', name: 'index-service' }); } @@ -240,10 +240,14 @@ export class IndexService { const groupIndexes = groups.filter(group => this.entityModified(group)) .map(group => this.validateGroup(group)); + groupIndexes.forEach(i => delete i.files); + if(!groupIndexes.length) { logger.info(`No groups were modified.`); } else { - await this.groupRepo.upsert(groupIndexes, []); + await this.groupRepo.save(groupIndexes, { + chunk: CHUNK_SIZE, reload: false + }); } } @@ -324,7 +328,9 @@ export class IndexService { if(!flatFileIndexes.length) { logger.info(`No flat files were modified.`); } else { - await this.fileRepo.upsert(flatFileIndexes, []); + await this.fileRepo.save(flatFileIndexes, { + chunk: CHUNK_SIZE, reload: false + }); } } diff --git a/src/scripts/unpacker.ts b/src/scripts/unpacker.ts index c5947db..3783fdc 100644 --- a/src/scripts/unpacker.ts +++ b/src/scripts/unpacker.ts @@ -40,52 +40,48 @@ async function unpackFiles(store: Store, args: UnpackOptions): Promise { const { archive: archiveName, debug } = args; - try { - store.loadPackedStore(); + store.loadPackedStore(); - if(archiveName === 'main') { - logger.info(`Unpacking JS5 file store with arguments:`, argDebugString); + if(archiveName === 'main') { + logger.info(`Unpacking JS5 file store with arguments:`, argDebugString); - store.decode(true); + store.decode(true); - store.encode(true); - store.compress(true); + store.encode(true); + store.compress(true); - if(!debug) { - store.write(); - } else { - logger.info(`Flat file store writing is disabled in debug mode.`); - } - - logger.info(`Decoding completed.`); - - await store.saveIndexData(true, true, true); + if(!debug) { + store.write(); } else { - logger.info(`Unpacking JS5 archive with arguments:`, argDebugString); + logger.info(`Flat file store writing is disabled in debug mode.`); + } - const a = store.find(archiveName); + logger.info(`Decoding completed.`); - if(!a) { - throw new Error(`Archive ${ a } was not found.`); - } + await store.saveIndexData(true, true, true); + } else { + logger.info(`Unpacking JS5 archive with arguments:`, argDebugString); - a.decode(true); + const a = store.find(archiveName); - a.encode(true); - a.compress(true); + if(!a) { + throw new Error(`Archive ${ a } was not found.`); + } - if(!debug) { - a.write(); - } else { - logger.info(`Archive writing is disabled in debug mode.`); - } + a.decode(true); - logger.info(`Decoding completed.`); + a.encode(true); + a.compress(true); - await a.saveIndexData(true, true); + if(!debug) { + a.write(); + } else { + logger.info(`Archive writing is disabled in debug mode.`); } - } catch(error) { - logger.error(error); + + logger.info(`Decoding completed.`); + + await a.saveIndexData(true, true); } } From cc2f61e80a80abe3640f667a97e01c696233c8f3 Mon Sep 17 00:00:00 2001 From: Kikorono Date: Mon, 14 Mar 2022 16:42:23 -0500 Subject: [PATCH 03/32] Adding support for archives 16-28 (until game build 582) --- README.md | 27 ++++++++++++--- config/archives.json5 | 79 ++++++++++++++++++++++++++++++++++++++++++- src/archive.ts | 21 ------------ src/store.ts | 2 +- 4 files changed, 102 insertions(+), 27 deletions(-) diff --git a/README.md b/README.md index 3325c10..4759d30 100644 --- a/README.md +++ b/README.md @@ -7,15 +7,13 @@ Node tools for managing and indexing the JS5 file store used with RuneJS. ## CLI Tools -_@todo work in progress_ - `index` - `unpack` ## Archives -_@todo work in progress_ -### JS5 // Game Version 400+ +### JS5 File Store // Game Builds 400-550 | Key | Archive Name | Content Description | File Format | Build | |----------------------------------:|:-----------------|:----------------------------------|:-----------:|:-----------------:| @@ -35,6 +33,27 @@ _@todo work in progress_ | idx **11** | midi_jingles | Shorter Midi Jingles | .mid | _400_+ | | | | | | | | idx **12** | clientscripts | Client Script (CS2) Files | .cs2 | _435_+ | +| | | | | | | idx **13** | fontmetrics | Game Font Metrics | | _443_+ | -| idx **14** | vorbis | Vorbis Sound Files | .ogg | _451_+ | +| | | | | | +| idx **14** | vorbis | Vorbis Sound Files | | _451_+ | | idx **15** | midi_instruments | Midi Song Instruments | | _451_+ | +| | | | | | +| idx **16** | config_loc | Location Object Configs | | _489_+ | +| idx **17** | config_enum | Enum Configs | | _489_+ | +| idx **18** | config_npc | NPC Configs | | _489_+ | +| idx **19** | config_obj | Item Object Configs | | _489_+ | +| idx **20** | config_seq | Animation Sequence Configs | | _489_+ | +| idx **21** | config_spot | Graphical Spot Animation Configs | | _489_+ | +| idx **22** | config_var_bit | VarBit Configs | | _489_+ | +| | | | | | +| idx **23** | worldmapdata | In-Game World Map Data | | _493_+ | +| | | | | | +| idx **24** | quickchat | Quickchat Data | | _498_+ | +| idx **25** | quickchat_global | Global Quickchat Data | | _498_+ | +| | | | | | +| idx **26** | materials | Materials | | _500_+ | +| | | | | | +| idx **27** | config_particle | Particle Configs | | _523_+ | +| | | | | | +| idx **28** | defaults | Defaults | | _537_+ | diff --git a/config/archives.json5 b/config/archives.json5 index ff5fe86..6401e8e 100644 --- a/config/archives.json5 +++ b/config/archives.json5 @@ -117,7 +117,6 @@ index: 14, compression: 'gzip', versioned: true, - contentType: '.ogg', build: 451 }, midi_instruments: { @@ -125,5 +124,83 @@ compression: 'gzip', versioned: false, build: 451 + }, + config_loc: { + index: 16, + compression: 'gzip', + versioned: false, + build: 489 + }, + config_enum: { + index: 17, + compression: 'gzip', + versioned: false, + build: 489 + }, + config_npc: { + index: 18, + compression: 'gzip', + versioned: false, + build: 489 + }, + config_obj: { + index: 19, + compression: 'gzip', + versioned: false, + build: 489 + }, + config_seq: { + index: 20, + compression: 'gzip', + versioned: false, + build: 489 + }, + config_spot: { + index: 21, + compression: 'gzip', + versioned: false, + build: 489 + }, + config_var_bit: { + index: 22, + compression: 'gzip', + versioned: false, + build: 489 + }, + worldmapdata: { + index: 23, + compression: 'gzip', + versioned: false, + build: 493 + }, + quickchat: { + index: 24, + compression: 'gzip', + versioned: false, + build: 498 + }, + quickchat_global: { + index: 25, + compression: 'gzip', + versioned: false, + build: 498 + }, + materials: { + index: 26, + compression: 'gzip', + versioned: false, + build: 500 + }, + config_particle: { + index: 27, + compression: 'gzip', + versioned: false, + build: 523 + }, + defaults: { + index: 28, + compression: 'gzip', + versioned: false, + build: 537 } } diff --git a/src/archive.ts b/src/archive.ts index d95a909..ca0832d 100644 --- a/src/archive.ts +++ b/src/archive.ts @@ -169,35 +169,14 @@ export class Archive extends IndexedFile { } if(decodeGroups) { - let successes = 0; - let failures = 0; - for(const [ , group ] of this.groups) { try { group.decode(); - - if(group.data?.length && group.state === FileState.raw) { - successes++; - } else { - failures++; - } } catch(error) { logger.error(error); - failures++; } } - if(successes) { - logger.info(`${groupCount} groups(s) were found, ` + - `${successes} decompressed successfully.`); - } else { - logger.info(`${groupCount} groups(s) were found.`); - } - - if(failures) { - logger.error(`${failures} groups(s) failed to decompress.`); - } - if(this.missingEncryptionKeys) { logger.error(`Missing ${this.missingEncryptionKeys} XTEA decryption key(s).`); } diff --git a/src/store.ts b/src/store.ts index d6526e7..a1656fc 100644 --- a/src/store.ts +++ b/src/store.ts @@ -98,7 +98,7 @@ export class Store { revision = Number(gameBuild); } if(revision < config.build) { - logger.info(`Skipping archive ${name} as it is not available in this game build.`); + // logger.info(`Skipping archive ${name} as it is not available in this game build.`); continue; } } From 2e0721a51968b1f41b834c0b3ac7fb6a99196e5e Mon Sep 17 00:00:00 2001 From: Kikorono Date: Mon, 14 Mar 2022 16:59:07 -0500 Subject: [PATCH 04/32] Adding archive 29 support and fixing the readme --- README.md | 12 ++---------- config/archives.json5 | 6 ++++++ 2 files changed, 8 insertions(+), 10 deletions(-) diff --git a/README.md b/README.md index 4759d30..7ea16f8 100644 --- a/README.md +++ b/README.md @@ -13,7 +13,7 @@ Node tools for managing and indexing the JS5 file store used with RuneJS. ## Archives -### JS5 File Store // Game Builds 400-550 +### JS5 File Store // Game Builds 400-604 | Key | Archive Name | Content Description | File Format | Build | |----------------------------------:|:-----------------|:----------------------------------|:-----------:|:-----------------:| @@ -31,14 +31,10 @@ Node tools for managing and indexing the JS5 file store used with RuneJS. | idx **9** | textures | Game Textures | | _400_+ | | idx **10** | binary | Miscellaneous Binary Files | | _400_+ | | idx **11** | midi_jingles | Shorter Midi Jingles | .mid | _400_+ | -| | | | | | | idx **12** | clientscripts | Client Script (CS2) Files | .cs2 | _435_+ | -| | | | | | | idx **13** | fontmetrics | Game Font Metrics | | _443_+ | -| | | | | | | idx **14** | vorbis | Vorbis Sound Files | | _451_+ | | idx **15** | midi_instruments | Midi Song Instruments | | _451_+ | -| | | | | | | idx **16** | config_loc | Location Object Configs | | _489_+ | | idx **17** | config_enum | Enum Configs | | _489_+ | | idx **18** | config_npc | NPC Configs | | _489_+ | @@ -46,14 +42,10 @@ Node tools for managing and indexing the JS5 file store used with RuneJS. | idx **20** | config_seq | Animation Sequence Configs | | _489_+ | | idx **21** | config_spot | Graphical Spot Animation Configs | | _489_+ | | idx **22** | config_var_bit | VarBit Configs | | _489_+ | -| | | | | | | idx **23** | worldmapdata | In-Game World Map Data | | _493_+ | -| | | | | | | idx **24** | quickchat | Quickchat Data | | _498_+ | | idx **25** | quickchat_global | Global Quickchat Data | | _498_+ | -| | | | | | | idx **26** | materials | Materials | | _500_+ | -| | | | | | | idx **27** | config_particle | Particle Configs | | _523_+ | -| | | | | | | idx **28** | defaults | Defaults | | _537_+ | +| idx **29** | billboards | Billboards | | _582_+ | diff --git a/config/archives.json5 b/config/archives.json5 index 6401e8e..226ef3f 100644 --- a/config/archives.json5 +++ b/config/archives.json5 @@ -202,5 +202,11 @@ compression: 'gzip', versioned: false, build: 537 + }, + billboards: { + index: 29, + compression: 'gzip', + versioned: false, + build: 582 } } From fde2f98bdd3a7665608f4f485e28de803f259ac3 Mon Sep 17 00:00:00 2001 From: Kikorono Date: Sun, 20 Mar 2022 13:07:11 -0500 Subject: [PATCH 05/32] Adding support for configurable config group names --- package.json | 4 ++-- src/db/index-service.ts | 5 +++-- src/group.ts | 8 ++++++++ 3 files changed, 13 insertions(+), 4 deletions(-) diff --git a/package.json b/package.json index 27a502a..c27d16b 100644 --- a/package.json +++ b/package.json @@ -18,9 +18,9 @@ "start": "ts-node-dev src/dev.ts", "lint": "eslint --ext .ts src", "lint:fix": "eslint --ext .ts src --fix", - "unpack": "ts-node-dev src/scripts/unpacker.ts", + "unpack": "ts-node-dev --max-old-space-size=2048 src/scripts/unpacker.ts", "unpacker": "npm run unpack", - "index": "ts-node-dev src/scripts/indexer.ts", + "index": "ts-node-dev --max-old-space-size=2048 src/scripts/indexer.ts", "indexer": "npm run index", "copy-documents": "copyfiles package.json README.md .npmignore LICENSE lib", "package": "rimraf lib && npm i && npm run build && npm run copy-documents && cd lib && npm publish --dry-run", diff --git a/src/db/index-service.ts b/src/db/index-service.ts index c5ea9d7..d573d70 100644 --- a/src/db/index-service.ts +++ b/src/db/index-service.ts @@ -328,8 +328,9 @@ export class IndexService { if(!flatFileIndexes.length) { logger.info(`No flat files were modified.`); } else { + logger.info(`${flatFileIndexes.length} flat files were modified.`); await this.fileRepo.save(flatFileIndexes, { - chunk: CHUNK_SIZE, reload: false + chunk: CHUNK_SIZE, reload: false, transaction: false, listeners: false }); } } @@ -355,7 +356,7 @@ export class IndexService { if((index instanceof ArchiveIndexEntity || index instanceof GroupIndexEntity) && (file instanceof Archive || file instanceof Group)) { if(file.state !== index.state) { - return true; + // return true; } } diff --git a/src/group.ts b/src/group.ts index 39590fb..4c9331d 100644 --- a/src/group.ts +++ b/src/group.ts @@ -34,6 +34,14 @@ export class Group extends IndexedFile { if(isSet(index.nameHash)) { this.nameHash = index.nameHash; } + + if(this.archive.config.groupNames) { + const nameEntries = Object.entries(this.archive.config.groupNames); + const namedEntry = nameEntries.find(entry => entry[1] === this.numericKey) || null; + if(namedEntry) { + this.name = namedEntry[0]; + } + } } public override decode(): ByteBuffer | null { From 07557b7bb0aeda2419fbf01c6cd0ba9ef6538bd2 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Mon, 13 Jun 2022 17:30:29 +0000 Subject: [PATCH 06/32] Bump minimist from 1.2.5 to 1.2.6 Bumps [minimist](https://github.com/substack/minimist) from 1.2.5 to 1.2.6. - [Release notes](https://github.com/substack/minimist/releases) - [Commits](https://github.com/substack/minimist/compare/1.2.5...1.2.6) --- updated-dependencies: - dependency-name: minimist dependency-type: indirect ... Signed-off-by: dependabot[bot] --- package-lock.json | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/package-lock.json b/package-lock.json index 8202072..6ff18f1 100644 --- a/package-lock.json +++ b/package-lock.json @@ -2156,9 +2156,9 @@ } }, "node_modules/minimist": { - "version": "1.2.5", - "resolved": "https://registry.npmjs.org/minimist/-/minimist-1.2.5.tgz", - "integrity": "sha512-FM9nNUYrRBAELZQT3xeZQ7fmMOBg6nWNmJKTcgsJeaLstP/UODVpGsr5OhXhhXg6f+qtJ8uiZ+PUxkDWcgIXLw==" + "version": "1.2.6", + "resolved": "https://registry.npmjs.org/minimist/-/minimist-1.2.6.tgz", + "integrity": "sha512-Jsjnk4bw3YJqYzbdyBiNsPWHPfO++UGG749Cxs6peCu5Xg4nrena6OVxOYxrQTqww0Jmwt+Ref8rggumkTLz9Q==" }, "node_modules/mkdirp": { "version": "1.0.4", @@ -5215,9 +5215,9 @@ } }, "minimist": { - "version": "1.2.5", - "resolved": "https://registry.npmjs.org/minimist/-/minimist-1.2.5.tgz", - "integrity": "sha512-FM9nNUYrRBAELZQT3xeZQ7fmMOBg6nWNmJKTcgsJeaLstP/UODVpGsr5OhXhhXg6f+qtJ8uiZ+PUxkDWcgIXLw==" + "version": "1.2.6", + "resolved": "https://registry.npmjs.org/minimist/-/minimist-1.2.6.tgz", + "integrity": "sha512-Jsjnk4bw3YJqYzbdyBiNsPWHPfO++UGG749Cxs6peCu5Xg4nrena6OVxOYxrQTqww0Jmwt+Ref8rggumkTLz9Q==" }, "mkdirp": { "version": "1.0.4", From 0e58044f89d1e66a5b18ad40ff98ade663f7f628 Mon Sep 17 00:00:00 2001 From: Kikorono Date: Tue, 12 Jul 2022 18:01:49 -0500 Subject: [PATCH 07/32] Working to implement a simpler file-system/store approach --- package.json | 8 - src/config/archive-config.ts | 2 +- src/db/index-service.ts | 1 + src/file-system/archive.ts | 57 +++ src/file-system/db/index-database.ts | 61 +++ src/file-system/db/index.entity.ts | 88 ++++ src/file-system/file-base.ts | 45 ++ src/file-system/file-error.ts | 3 + src/file-system/file-store.ts | 118 ++++++ src/file-system/file-type.ts | 5 + src/file-system/flat-file.ts | 23 ++ src/file-system/group.ts | 60 +++ src/file-system/js5.ts | 590 +++++++++++++++++++++++++++ 13 files changed, 1052 insertions(+), 9 deletions(-) create mode 100644 src/file-system/archive.ts create mode 100644 src/file-system/db/index-database.ts create mode 100644 src/file-system/db/index.entity.ts create mode 100644 src/file-system/file-base.ts create mode 100644 src/file-system/file-error.ts create mode 100644 src/file-system/file-store.ts create mode 100644 src/file-system/file-type.ts create mode 100644 src/file-system/flat-file.ts create mode 100644 src/file-system/group.ts create mode 100644 src/file-system/js5.ts diff --git a/package.json b/package.json index c27d16b..dbb51ac 100644 --- a/package.json +++ b/package.json @@ -77,13 +77,5 @@ "source-map-support": "^0.5.21", "ts-node-dev": "^1.1.8", "typescript": "^4.5.5" - }, - "eslintConfig": { - "extends": [ - "@runejs/eslint-config" - ], - "parserOptions": { - "project": "./tsconfig.json" - } } } diff --git a/src/config/archive-config.ts b/src/config/archive-config.ts index 611e6d8..d2a5694 100644 --- a/src/config/archive-config.ts +++ b/src/config/archive-config.ts @@ -7,7 +7,7 @@ export interface ArchiveConfig { name: string; versioned?: boolean; compression?: CompressionMethod; - encryption?: EncryptionMethod | [ EncryptionMethod, string ]; + encryption?: [ EncryptionMethod, string ]; contentType?: string; filesNamed?: boolean; flatten?: boolean; diff --git a/src/db/index-service.ts b/src/db/index-service.ts index d573d70..7d32e3e 100644 --- a/src/db/index-service.ts +++ b/src/db/index-service.ts @@ -170,6 +170,7 @@ export class IndexService { } else { delete archiveIndex.groups; + archiveIndex.data = archive.data.toNodeBuffer(); const result = await this.archiveRepo.insert(archiveIndex); affected = result?.identifiers?.length || 0; } diff --git a/src/file-system/archive.ts b/src/file-system/archive.ts new file mode 100644 index 0000000..611a5d6 --- /dev/null +++ b/src/file-system/archive.ts @@ -0,0 +1,57 @@ +import { FileBase } from './file-base'; +import { FileStore } from './file-store'; +import { FileType } from './file-type'; +import { Group } from './group'; + + +export class Archive extends FileBase { + + readonly children: Map; + + constructor( + fileStore: FileStore, + key: number, + ) { + super(fileStore, key, -1, 'ARCHIVE'); + this.children = new Map(); + } + + js5Unpack(): Buffer | null { + return this.fileStore.js5.unpack(this); + } + + js5Decompress(): Buffer | null { + return this.fileStore.js5.decompress(this); + } + + async js5Decode(): Promise { + await this.fileStore.js5.decodeArchive(this); + } + + js5Pack(): Buffer | null { + return this.fileStore.js5.pack(this); + } + + js5Compress(): Buffer | null { + return this.fileStore.js5.compress(this); + } + + js5Encode(): Buffer | null { + return this.fileStore.js5.encodeArchive(this); + } + + getChild(groupIndex: number): Group { + return this.children.get(String(groupIndex)) || null; + } + + setChild(groupIndex: number, group: Group): void { + this.children.set(String(groupIndex), group); + } + + findChild(groupName: string): Group { + return Array.from(this.children.values()).find( + group => group?.index?.name === groupName + ) || null; + } + +} diff --git a/src/file-system/db/index-database.ts b/src/file-system/db/index-database.ts new file mode 100644 index 0000000..3f956d3 --- /dev/null +++ b/src/file-system/db/index-database.ts @@ -0,0 +1,61 @@ +import { Connection, createConnection, LoggerOptions, Repository } from 'typeorm'; +import { join } from 'path'; +import { existsSync, mkdirSync } from 'graceful-fs'; +import { IndexEntity } from './index.entity'; +import { FileType } from '../file-type'; + + +export class IndexDatabase { + + private readonly gameBuild: string; + private readonly databasePath: string; + private readonly loggerOptions: LoggerOptions; + + private _connection: Connection; + private _repository: Repository; + + constructor(gameBuild: string, databasePath: string, loggerOptions: LoggerOptions = 'all') { + this.gameBuild = gameBuild; + this.databasePath = databasePath; + // [ 'error', 'warn' ], 'all', etc... + this.loggerOptions = loggerOptions; + } + + async openConnection(): Promise { + if(!existsSync(this.databasePath)) { + mkdirSync(this.databasePath, { recursive: true }); + } + + this._connection = await createConnection({ + type: 'better-sqlite3', + database: join(this.databasePath, `${this.gameBuild}.index.sqlite3`), + entities: [ IndexEntity ], + synchronize: true, + logging: this.loggerOptions, + name: 'index-repository' + }); + + this._repository = this._connection.getRepository(IndexEntity); + + return this._connection; + } + + async getIndex(fileType: FileType, key: number, parentKey: number): Promise { + return await this.repository.findOne({ + where: { fileType, key, parentKey } + }) || null; + } + + get connection(): Connection { + return this._connection; + } + + get repository(): Repository { + return this._repository; + } + + get loaded(): boolean { + return !!this._connection; + } + +} diff --git a/src/file-system/db/index.entity.ts b/src/file-system/db/index.entity.ts new file mode 100644 index 0000000..f97fdca --- /dev/null +++ b/src/file-system/db/index.entity.ts @@ -0,0 +1,88 @@ +import { Column, CreateDateColumn, Entity, Index, PrimaryColumn, UpdateDateColumn } from 'typeorm'; +import { FileType } from '../file-type'; +import { CompressionMethod } from '@runejs/common/compress'; +import { FileError } from '../file-error'; + + +@Entity('file') +@Index('index_identifier', [ + 'fileType', 'gameBuild', 'key', 'parentKey' +], { unique: true }) +export class IndexEntity { + + @PrimaryColumn('text', { name: 'file_type', nullable: false, unique: false }) + fileType: FileType; + + @PrimaryColumn('text', { name: 'game_build', nullable: false, unique: false }) + gameBuild: string; + + @PrimaryColumn('integer', { nullable: false, unique: false }) + key: number; + + @PrimaryColumn('integer', { name: 'parent_key', nullable: false, unique: false, default: -1 }) + parentKey: number = -1; + + @Column('text', { nullable: true, default: null }) + name: string = null; + + @Column('integer', { name: 'name_hash', nullable: true, default: -1 }) + nameHash: number = -1; + + @Column('integer', { nullable: false, default: 0 }) + version: number = 0; + + @Column('integer', { name: 'childCount', nullable: false, default: 0 }) + childCount: number = 0; + + @Column('integer', { nullable: false, default: -1 }) + checksum: number = -1; + + @Column('text', { name: 'sha_digest', nullable: true, default: null }) + shaDigest: string = null; + + @Column('blob', { name: 'whirlpool_digest', nullable: true, default: null }) + whirlpoolDigest: Buffer = null; + + @Column('integer', { name: 'file_size', nullable: false, default: 0 }) + fileSize: number = 0; + + @Column('blob', { name: 'data', nullable: true, default: null }) + data: Buffer = null; + + @Column('text', { name: 'compression_method', nullable: true, default: null }) + compressionMethod: CompressionMethod = null; + + @Column('integer', { name: 'compressed_checksum', nullable: false, default: -1 }) + compressedChecksum: number = -1; + + @Column('text', { name: 'compressed_sha_digest', nullable: true, default: null }) + compressedShaDigest: string = null; + + @Column('integer', { name: 'compressed_file_size', nullable: false, default: 0 }) + compressedFileSize: number = 0; + + @Column('blob', { name: 'compressed_data', nullable: true, default: null }) + compressedData: Buffer = null; + + @Column('boolean', { nullable: true, default: false }) + encrypted: boolean = false; + + @Column('integer', { name: 'stripe_count', nullable: false, default: 1 }) + stripeCount: number = 1; + + @Column('text', { nullable: true, default: null }) + stripes: string | null = null; + + @Column('integer', { nullable: true, default: null }) + archiveFormat: number = null; + + @Column('text', { name: 'file_error', nullable: true, default: null }) + fileError: FileError = null; + + @CreateDateColumn() + created?: Date; + + @UpdateDateColumn() + updated?: Date; + +} diff --git a/src/file-system/file-base.ts b/src/file-system/file-base.ts new file mode 100644 index 0000000..305be70 --- /dev/null +++ b/src/file-system/file-base.ts @@ -0,0 +1,45 @@ +import { IndexEntity } from './db/index.entity'; +import { FileStore } from './file-store'; +import { FileType } from './file-type'; + + +export class FileBase { + + readonly fileStore: FileStore; + + index: IndexEntity; + + constructor( + fileStore: FileStore, + key: number, + parentKey: number, + fileType: FileType, + ) { + this.fileStore = fileStore; + this.index = new IndexEntity(); + this.index.fileType = fileType; + this.index.key = key; + this.index.parentKey = parentKey; + } + + async loadIndex(): Promise { + const indexEntity = await this.fileStore.database.getIndex( + this.index.fileType, this.index.key, this.index.parentKey + ); + + if (indexEntity) { + this.index = indexEntity; + } + + return this.index; + } + + get stripes(): number[] { + if (!this.index?.stripes) { + return []; + } + + return this.index.stripes.split(',').map(n => Number(n)); + } + +} diff --git a/src/file-system/file-error.ts b/src/file-system/file-error.ts new file mode 100644 index 0000000..c91a863 --- /dev/null +++ b/src/file-system/file-error.ts @@ -0,0 +1,3 @@ +export type FileError = + 'FILE_MISSING' | + 'MISSING_ENCRYPTION_KEYS'; diff --git a/src/file-system/file-store.ts b/src/file-system/file-store.ts new file mode 100644 index 0000000..9cec826 --- /dev/null +++ b/src/file-system/file-store.ts @@ -0,0 +1,118 @@ +import { join } from 'path'; +import JSON5 from 'json5'; +import { existsSync, readFileSync } from 'graceful-fs'; +import { logger } from '@runejs/common'; +import { IndexDatabase } from './db/index-database'; +import { ArchiveConfig } from '../config'; +import { JS5 } from './js5'; + + +export class FileStore { + + readonly gameBuild: string; + readonly fileStorePath: string; + readonly fileNameHashes: Map; + readonly js5: JS5; + + private _archiveConfig: { [key: string]: ArchiveConfig }; + private _database: IndexDatabase; + + constructor(gameBuild: string, storePath: string = './') { + this.gameBuild = gameBuild; + this.fileStorePath = storePath; + this.fileNameHashes = new Map(); + this.loadArchiveConfig(); + this.loadFileNames(); + this.js5 = new JS5(this); + } + + hashFileName(fileName: string): number { + if (!fileName) { + return 0; + } + + let hash = 0; + for (let i = 0; i < fileName.length; i++) { + hash = fileName.charCodeAt(i) + ((hash << 5) - hash); + } + + const nameHash = hash | 0; + + this.fileNameHashes.set(nameHash, fileName); + + return nameHash; + } + + findFileName(nameHash: string | number | undefined, defaultName?: string | undefined): string | undefined { + if (!this.fileNameHashes.size) { + this.loadFileNames(); + } + + if (nameHash === undefined || nameHash === null) { + return defaultName; + } + + if (typeof nameHash === 'string') { + nameHash = Number(nameHash); + } + + if (isNaN(nameHash) || nameHash === -1 || nameHash === 0) { + return defaultName; + } + + return this.fileNameHashes.get(nameHash) || defaultName; + } + + loadFileNames(): void { + const configPath = join(this.fileStorePath, 'config', 'name-hashes.json'); + if (!existsSync(configPath)) { + logger.error(`Error loading file names: ${configPath} was not found.`); + return; + } + + const nameTable = JSON.parse( + readFileSync(configPath, 'utf-8') + ) as { [key: string]: string }; + + Object.keys(nameTable).forEach( + nameHash => this.fileNameHashes.set(Number(nameHash), nameTable[nameHash]) + ); + + if(!this.fileNameHashes.size) { + logger.error(`Error reading file name lookup table. ` + + `Please ensure that the ${configPath} file exists and is valid.`); + } + } + + loadArchiveConfig(): void { + const configPath = join(this.fileStorePath, 'config', 'archives.json5'); + if (!existsSync(configPath)) { + logger.error(`Error loading file store: ${configPath} was not found.`); + return; + } + + this._archiveConfig = JSON5.parse( + readFileSync(configPath, 'utf-8') + ) as { [key: string]: ArchiveConfig }; + + if (!Object.values(this._archiveConfig)?.length) { + throw new Error(`Error reading archive configuration file. ` + + `Please ensure that the ${configPath} file exists and is valid.`); + } + } + + async openDatabase(): Promise { + this._database = new IndexDatabase(this.gameBuild, join(this.fileStorePath, 'indexes')); + await this._database.openConnection(); + return this._database; + } + + get archiveConfig(): { [key: string]: ArchiveConfig } { + return this._archiveConfig; + } + + get database(): IndexDatabase { + return this._database; + } + +} diff --git a/src/file-system/file-type.ts b/src/file-system/file-type.ts new file mode 100644 index 0000000..0e89e7a --- /dev/null +++ b/src/file-system/file-type.ts @@ -0,0 +1,5 @@ +export type FileType = + 'FILE' | + 'GROUP' | + 'ARCHIVE' | + 'STORE'; diff --git a/src/file-system/flat-file.ts b/src/file-system/flat-file.ts new file mode 100644 index 0000000..a33a273 --- /dev/null +++ b/src/file-system/flat-file.ts @@ -0,0 +1,23 @@ +import { FileBase } from './file-base'; +import { FileStore } from './file-store'; +import { FileType } from './file-type'; +import { Archive } from './archive'; +import { Group } from './group'; + + +export class FlatFile extends FileBase { + + readonly archive: Archive; + readonly group: Group; + + constructor( + fileStore: FileStore, + key: number, + group: Group, + ) { + super(fileStore, key, group.index.key, 'FILE'); + this.archive = group.archive; + this.group = group; + } + +} diff --git a/src/file-system/group.ts b/src/file-system/group.ts new file mode 100644 index 0000000..6836388 --- /dev/null +++ b/src/file-system/group.ts @@ -0,0 +1,60 @@ +import { FileBase } from './file-base'; +import { FileStore } from './file-store'; +import { Archive } from './archive'; +import { FlatFile } from './flat-file'; + + +export class Group extends FileBase { + + readonly archive: Archive; + readonly children: Map; + + constructor( + fileStore: FileStore, + key: number, + archive: Archive, + ) { + super(fileStore, key, archive.index.key, 'GROUP'); + this.archive = archive; + this.children = new Map(); + } + + js5Unpack(): Buffer | null { + return this.fileStore.js5.unpack(this); + } + + js5Decompress(): Buffer | null { + return this.fileStore.js5.decompress(this); + } + + async js5Decode(): Promise { + await this.fileStore.js5.decodeGroup(this); + } + + js5Pack(): Buffer | null { + return this.fileStore.js5.pack(this); + } + + js5Compress(): Buffer | null { + return this.fileStore.js5.compress(this); + } + + js5Encode(): Buffer | null { + return this.fileStore.js5.encodeGroup(this); + } + + getChild(fileIndex: number): FlatFile { + return this.children.get(String(fileIndex)) || null; + } + + setChild(fileIndex: number, file: FlatFile): void { + this.children.set(String(fileIndex), file); + } + + findChild(fileName: string): FlatFile { + return Array.from(this.children.values()).find( + file => file?.index?.name === fileName + ) || null; + } + +} diff --git a/src/file-system/js5.ts b/src/file-system/js5.ts new file mode 100644 index 0000000..4b2502b --- /dev/null +++ b/src/file-system/js5.ts @@ -0,0 +1,590 @@ +import { ByteBuffer, logger } from '@runejs/common'; +import { Bzip2, getCompressionMethod, Gzip } from '@runejs/common/compress'; +import { Xtea, XteaKeys } from '@runejs/common/encrypt'; +import { Group } from './group'; +import { Archive } from './archive'; +import { FileStore } from './file-store'; +import { ArchiveFormat } from '../config'; +import { archiveFlags } from '../config/archive-flags'; +import { FlatFile } from './flat-file'; + + +export class JS5 { + + readonly fileStore: FileStore; + + private mainIndex: ByteBuffer; + private archiveIndexes: Map; + private mainArchiveData: ByteBuffer; + + constructor(fileStore: FileStore) { + this.fileStore = fileStore; + } + + unpack(file: Group | Archive): Buffer | null { + const fileDetails = file.index; + const fileKey = fileDetails.key; + const archiveKey: number = file instanceof Archive ? 255 : file.archive.index.key; + const archiveName: string = file instanceof Archive ? 'main' : file.archive.index.name; + + const indexChannel: ByteBuffer = archiveKey !== 255 ? + this.archiveIndexes.get(String(archiveKey)) : this.mainIndex; + + if (archiveKey === 255 && fileKey === 255) { + return null; + } + + const indexDataLength = 6; + const dataChannel = this.mainArchiveData; + + indexChannel.readerIndex = 0; + dataChannel.readerIndex = 0; + + let pointer = fileKey * indexDataLength; + + if (pointer < 0 || pointer >= indexChannel.length) { + logger.error(`File ${fileKey} was not found within the ${archiveName} archive index file.`); + return null; + } + + const fileIndexData = new ByteBuffer(indexDataLength); + indexChannel.copy(fileIndexData, 0, pointer, pointer + indexDataLength); + + if (fileIndexData.readable !== indexDataLength) { + logger.error(`Error extracting JS5 file ${fileKey}: the end of the data stream was reached.`); + return null; + } + + fileDetails.fileSize = fileIndexData.get('int24', 'unsigned'); + fileDetails.stripeCount = fileIndexData.get('int24', 'unsigned'); + + if (fileDetails.fileSize <= 0) { + logger.warn(`Extracted JS5 file ${fileKey} has a recorded size of 0, no file data will be extracted.`); + return null; + } + + const data = new ByteBuffer(fileDetails.fileSize); + const stripeDataLength = 512; + const stripeLength = 520; + + let stripe = 0; + let remaining = fileDetails.fileSize; + pointer = fileDetails.stripeCount * stripeLength; + + do { + const temp = new ByteBuffer(stripeLength); + dataChannel.copy(temp, 0, pointer, pointer + stripeLength); + + if(temp.readable !== stripeLength) { + logger.error(`Error reading stripe for packed file ${fileKey}, the end of the data stream was reached.`); + return null; + } + + const stripeFileIndex = temp.get('short', 'unsigned'); + const currentStripe = temp.get('short', 'unsigned'); + const nextStripe = temp.get('int24', 'unsigned'); + const stripeArchiveIndex = temp.get('byte', 'unsigned'); + const stripeData = new ByteBuffer(stripeDataLength); + temp.copy(stripeData, 0, temp.readerIndex, temp.readerIndex + stripeDataLength); + + if(remaining > stripeDataLength) { + stripeData.copy(data, data.writerIndex, 0, stripeDataLength); + data.writerIndex = (data.writerIndex + stripeDataLength); + remaining -= stripeDataLength; + + if(stripeArchiveIndex !== archiveKey) { + logger.error(`Archive index mismatch, expected archive ${archiveKey} but found archive ${stripeFileIndex}`); + return null; + } + + if(stripeFileIndex !== fileKey) { + logger.error(`File index mismatch, expected ${fileKey} but found ${stripeFileIndex}.`); + return null; + } + + if(currentStripe !== stripe++) { + logger.error(`Error extracting JS5 file ${fileKey}, file data is corrupted.`); + return null; + } + + pointer = nextStripe * stripeLength; + } else { + stripeData.copy(data, data.writerIndex, 0, remaining); + data.writerIndex = (data.writerIndex + remaining); + remaining = 0; + } + } while (remaining > 0); + + if (data?.length) { + fileDetails.compressedData = data.toNodeBuffer(); + } else { + fileDetails.compressedData = null; + fileDetails.fileError = 'FILE_MISSING'; + } + + return fileDetails.compressedData; + } + + readCompressedFileHeader(file: Group | Archive): { + compressedLength: number; + readerIndex: number; + } { + const fileDetails = file.index; + + if (!fileDetails.compressedData?.length) { + return { compressedLength: 0, readerIndex: 0 }; + } + + const compressedData = new ByteBuffer(fileDetails.compressedData); + + fileDetails.compressionMethod = getCompressionMethod( + compressedData.get('byte', 'unsigned')); + + const compressedLength = compressedData.get('int', 'unsigned'); + const readerIndex = compressedData.readerIndex; + + return { compressedLength, readerIndex }; + } + + decrypt(file: Group | Archive): Buffer { + const fileDetails = file.index; + const fileName = fileDetails.name; + + if (!fileDetails.compressedData?.length) { + logger.error(`Error decrypting file ${fileName || fileDetails.key}, file data not found.`, + `Please ensure that the file has been unpacked from an existing JS5 file store using JS5.unpack(file);`); + return null; + } + + if (!fileDetails.encrypted) { + return fileDetails.compressedData; + } + + // @todo move to JS5.decodeArchive + // const archiveName = file instanceof Archive ? 'main' : file.archive.index.name; + // const archiveConfig = this.fileStore.archiveConfig[archiveName]; + // + // if (archiveConfig.encryption) { + // const [ encryption, pattern ] = archiveConfig.encryption; + // const patternRegex = new RegExp(pattern); + // + // // Only XTEA encryption is supported at this time + // if(encryption !== 'xtea' || !patternRegex.test(fileName)) { + // // FileBase name does not match the pattern, data should be unencrypted + // return fileDetails.compressedData; + // } + // } else { + // return fileDetails.compressedData; + // } + + const gameBuild = this.fileStore.gameBuild; + let keySets: XteaKeys[] = []; + + const loadedKeys = this.getEncryptionKeys(fileName); + if (loadedKeys) { + if(!Array.isArray(loadedKeys)) { + keySets = [ loadedKeys ]; + } else { + keySets = loadedKeys; + } + } + + const { compressedLength, readerIndex } = this.readCompressedFileHeader(file); + const encryptedData = new ByteBuffer(fileDetails.compressedData); + const keySet = keySets.find(keySet => keySet.gameBuild === gameBuild); + + if (Xtea.validKeys(keySet?.key)) { + const dataCopy = encryptedData.clone(); + dataCopy.readerIndex = readerIndex; + + let lengthOffset = readerIndex; + if(dataCopy.length - (compressedLength + readerIndex + 4) >= 2) { + lengthOffset += 2; + } + + const decryptedData = Xtea.decrypt(dataCopy, keySet.key, dataCopy.length - lengthOffset); + + if (decryptedData?.length) { + decryptedData.copy(dataCopy, readerIndex, 0); + dataCopy.readerIndex = readerIndex; + fileDetails.compressedData = dataCopy.toNodeBuffer(); + fileDetails.encrypted = false; + } else { + logger.warn(`Invalid XTEA decryption keys found for file ` + + `${fileName || fileDetails.key} using game build ${ gameBuild }.`); + fileDetails.fileError = 'MISSING_ENCRYPTION_KEYS'; + } + } else { + logger.warn(`No XTEA decryption keys found for file ` + + `${fileName || fileDetails.key} using game build ${gameBuild}.`); + fileDetails.fileError = 'MISSING_ENCRYPTION_KEYS'; + } + + return fileDetails.compressedData; + } + + decompress(file: Group | Archive): Buffer | null { + const fileDetails = file.index; + + if (!fileDetails.compressedData?.length) { + return null; + } + + if (fileDetails.encrypted && !this.decrypt(file)) { + return null; + } + + const { compressedLength, readerIndex } = this.readCompressedFileHeader(file); + + // JS5.decrypt will set compressedData to the new decrypted data after completion + const compressedData = new ByteBuffer(fileDetails.compressedData); + compressedData.readerIndex = readerIndex; + let data: ByteBuffer; + + if (fileDetails.compressionMethod === 'none') { + // Uncompressed file + data = new ByteBuffer(compressedLength); + compressedData.copy(data, 0, compressedData.readerIndex, compressedLength); + compressedData.readerIndex = (compressedData.readerIndex + compressedLength); + } else { + // BZIP or GZIP compressed file + const decompressedLength = compressedData.get('int', 'unsigned'); + if (decompressedLength < 0) { + const errorPrefix = `Unable to decompress file ${fileDetails.name || fileDetails.key}:`; + if (fileDetails.fileError === 'FILE_MISSING') { + logger.error(`${errorPrefix} Missing file data.`); + } else { + logger.error(`${errorPrefix} Missing or invalid XTEA key.`); + fileDetails.fileError = 'MISSING_ENCRYPTION_KEYS'; + } + } else { + const decompressedData = new ByteBuffer(fileDetails.compressionMethod === 'bzip' ? + decompressedLength : (compressedData.length - compressedData.readerIndex + 2)); + + compressedData.copy(decompressedData, 0, compressedData.readerIndex); + + try { + data = fileDetails.compressionMethod === 'bzip' ? + Bzip2.decompress(decompressedData) : Gzip.decompress(decompressedData); + + compressedData.readerIndex = compressedData.readerIndex + compressedLength; + + if (data.length !== decompressedLength) { + logger.error(`Compression length mismatch.`); + data = null; + } + } catch (error) { + logger.error(`Error decompressing file ${fileDetails.name || fileDetails.key}:`, + error?.message ?? error); + data = null; + } + } + } + + // Read the file footer, if it has one + if(compressedData.readable >= 2) { + fileDetails.version = compressedData.get('short', 'unsigned'); + } + + if (data?.length) { + fileDetails.data = data?.toNodeBuffer(); + } + + return fileDetails.data; + } + + async decodeArchive(archive: Archive): Promise { + const archiveDetails = archive.index; + + if (archiveDetails.key === 255) { + return; + } + + const archiveName = archiveDetails.name; + + logger.info(`Decoding archive ${archiveName}...`); + + if (!archiveDetails.data) { + this.decompress(archive); + + if (!archiveDetails.data) { + logger.error(`Unable to decode archive ${archiveName}.`); + return; + } + } + + // logger.info(`Archive ${archiveName} checksum: ${this.crc32}`); + + const archiveData = new ByteBuffer(archiveDetails.data); + const format = archiveDetails.archiveFormat = archiveData.get('byte', 'unsigned'); + const mainDataType = format >= ArchiveFormat.smart ? 'smart_int' : 'short'; + archiveDetails.version = format >= ArchiveFormat.versioned ? archiveData.get('int') : 0; + const flags = archiveFlags(archiveData.get('byte', 'unsigned')); + const groupCount = archiveData.get(mainDataType, 'unsigned'); + const groups: Group[] = new Array(groupCount); + let missingEncryptionKeys = 0; + let accumulator = 0; + + logger.info(`${groupCount} groups were found within the ${archiveName} archive.`); + + // Group index keys + for (let i = 0; i < groupCount; i++) { + const delta = archiveData.get(mainDataType, 'unsigned'); + const groupKey = accumulator += delta; + const group = groups[i] = new Group(this.fileStore, groupKey, archive); + archive.setChild(groupKey, group); + } + + // Load group database indexes, or create them if they don't yet exist + // @todo batch load all archive groups at once + for (const group of groups) { + await group.loadIndex(); + } + + // Group name hashes + if (flags.groupNames) { + for (const group of groups) { + group.index.nameHash = archiveData.get('int'); + group.index.name = this.fileStore.findFileName( + group.index.nameHash, + group.index.name || String(group.index.nameHash) || String(group.index.key) + ); + } + } + + // Compressed file data CRC32 checksums + for (const group of groups) { + group.index.compressedChecksum = archiveData.get('int'); + } + + // Decompressed file data CRC32 checksums + if (flags.decompressedCrcs) { + for (const group of groups) { + group.index.checksum = archiveData.get('int'); + } + } + + // File data whirlpool digests + if (flags.whirlpoolDigests) { + for (const group of groups) { + group.index.whirlpoolDigest = Buffer.alloc(512); + archiveData.getBytes(group.index.whirlpoolDigest, 512); + } + } + + // Group file sizes + if (flags.groupSizes) { + for (const group of groups) { + group.index.compressedFileSize = archiveData.get('int'); + group.index.fileSize = archiveData.get('int'); + } + } + + // Group version numbers + for (const group of groups) { + group.index.version = archiveData.get('int'); + } + + // Group child file counts + for (let i = 0; i < groupCount; i++) { + groups[i].index.childCount = archiveData.get('short', 'unsigned'); + } + + // Grouped file index keys + for (const group of groups) { + const fileCount = group.index.childCount || 0; + accumulator = 0; + + for (let i = 0; i < fileCount; i++) { + const delta = archiveData.get(mainDataType, 'unsigned'); + const childFileIndex = accumulator += delta; + const flatFile = new FlatFile(this.fileStore, childFileIndex, group); + group.setChild(childFileIndex, flatFile); + } + } + + // Load flat file database indexes, or create them if they don't yet exist + // @todo batch load all grouped files at once + for (const group of groups) { + for (const [ , flatFile ] of group.children) { + await flatFile.loadIndex(); + } + } + + // Grouped file names + if (flags.groupNames) { + for (const group of groups) { + for (const [ , flatFile ] of group.children) { + flatFile.index.nameHash = archiveData.get('int'); + flatFile.index.name = this.fileStore.findFileName( + flatFile.index.nameHash, + flatFile.index.name || String(flatFile.index.nameHash) || String(flatFile.index.key) + ); + } + } + } + } + + async decodeGroup(group: Group): Promise { + const groupDetails = group.index; + const { key: groupKey, name: groupName } = groupDetails; + const files = group.children; + + if (!groupDetails.data) { + this.decompress(group); + + if (!groupDetails.data) { + logger.error(`Unable to decode group ${groupName || groupKey}.`); + return; + } + } + + const data = new ByteBuffer(groupDetails.data); + + if(groupDetails.childCount === 1) { + return; + } + + data.readerIndex = (data.length - 1); // EOF - 1 byte + + groupDetails.stripeCount = data.get('byte', 'unsigned'); + + data.readerIndex = (data.length - 1 - groupDetails.stripeCount * + groupDetails.childCount * 4); // Stripe data footer + + if (data.readerIndex < 0) { + logger.error(`Invalid reader index of ${data.readerIndex} for group ` + + `${groupName || groupKey}.`); + return; + } + + const fileSizeMap = new Map(); + const fileStripeMap = new Map(); + const fileDataMap = new Map(); + + for (const [ flatFileKey, ] of files) { + fileSizeMap.set(flatFileKey, 0); + fileStripeMap.set(flatFileKey, new Array(groupDetails.stripeCount)); + } + + for (let stripe = 0; stripe < groupDetails.stripeCount; stripe++) { + let currentLength = 0; + + for (const [ flatFileKey, ] of files) { + const delta = data.get('int'); + currentLength += delta; + + const fileStripes = fileStripeMap.get(flatFileKey); + const size = fileSizeMap.get(flatFileKey) + currentLength; + + fileStripes[stripe] = currentLength; + fileSizeMap.set(flatFileKey, size + currentLength); + } + } + + for (const [ flatFileKey, file ] of files) { + file.index.fileSize = fileSizeMap.get(flatFileKey); + file.index.stripeCount = groupDetails.stripeCount; + file.index.stripes = fileStripeMap.get(flatFileKey).join(','); + fileDataMap.set(flatFileKey, new ByteBuffer(file.index.fileSize)); + } + + data.readerIndex = 0; + + for (let stripe = 0; stripe < groupDetails.stripeCount; stripe++) { + for (const [ fileIndex, ] of files) { + let stripeLength = fileStripeMap.get(fileIndex)[stripe]; + let sourceEnd: number = data.readerIndex + stripeLength; + + if (data.readerIndex + stripeLength >= data.length) { + sourceEnd = data.length; + stripeLength = (data.readerIndex + stripeLength) - data.length; + } + + const stripeData = data.getSlice(data.readerIndex, stripeLength); + const fileData = fileDataMap.get(fileIndex); + + fileData.putBytes(stripeData); + + data.readerIndex = sourceEnd; + } + } + + for (const [ fileIndex, file ] of files) { + file.index.data = fileDataMap.get(fileIndex).toNodeBuffer(); + } + } + + pack(file: Group | Archive): Buffer | null { + return null; // @todo stub + } + + encrypt(file: Group | Archive): Buffer | null { + return null; // @todo stub + } + + compress(file: Group | Archive): Buffer | null { + const fileDetails = file.index; + + if (!fileDetails.data?.length) { + return null; + } + + const decompressedData = new ByteBuffer(fileDetails.data); + let data: ByteBuffer; + + if (fileDetails.compressionMethod === 'none') { + // uncompressed files + data = new ByteBuffer(decompressedData.length + 5); + + // indicate that no file compression is applied + data.put(0); + + // write the uncompressed file length + data.put(decompressedData.length, 'int'); + + // write the uncompressed file data + data.putBytes(decompressedData); + } else { + // compressed Bzip2 or Gzip file + + const compressedData: ByteBuffer = fileDetails.compressionMethod === 'bzip' ? + Bzip2.compress(decompressedData) : Gzip.compress(decompressedData); + + const compressedLength: number = compressedData.length; + + data = new ByteBuffer(compressedData.length + 9); + + // indicate which type of file compression was used (1 or 2) + data.put(fileDetails.compressionMethod === 'bzip' ? 1 : 2); + + // write the compressed file length + data.put(compressedLength, 'int'); + + // write the uncompressed file length + data.put(decompressedData.length, 'int'); + + // write the compressed file data + data.putBytes(compressedData); + } + + if (data?.length) { + fileDetails.compressedData = data.toNodeBuffer(); + } + + return fileDetails.compressedData; + } + + encodeArchive(archive: Archive): Buffer { + return null; // @todo stub + } + + encodeGroup(group: Group): Buffer { + return null; // @todo stub + } + + getEncryptionKeys(fileName: string): any[] { + return []; // @todo stub + } + +} From 25206368ebbd8f02ddea0ef7763953a416623a37 Mon Sep 17 00:00:00 2001 From: Kikorono Date: Tue, 12 Jul 2022 18:26:27 -0500 Subject: [PATCH 08/32] Implementing js5 main index encoding and changing some variable names --- src/file-system/archive.ts | 17 +-- src/file-system/file-store.ts | 39 +++++ src/file-system/group.ts | 16 +- src/file-system/js5.ts | 266 ++++++++++++++++++++++------------ 4 files changed, 225 insertions(+), 113 deletions(-) diff --git a/src/file-system/archive.ts b/src/file-system/archive.ts index 611a5d6..a030c38 100644 --- a/src/file-system/archive.ts +++ b/src/file-system/archive.ts @@ -1,19 +1,18 @@ import { FileBase } from './file-base'; import { FileStore } from './file-store'; -import { FileType } from './file-type'; import { Group } from './group'; export class Archive extends FileBase { - readonly children: Map; + readonly groups: Map; constructor( fileStore: FileStore, key: number, ) { super(fileStore, key, -1, 'ARCHIVE'); - this.children = new Map(); + this.groups = new Map(); } js5Unpack(): Buffer | null { @@ -40,16 +39,16 @@ export class Archive extends FileBase { return this.fileStore.js5.encodeArchive(this); } - getChild(groupIndex: number): Group { - return this.children.get(String(groupIndex)) || null; + getGroup(groupIndex: number): Group | null { + return this.groups.get(String(groupIndex)) || null; } - setChild(groupIndex: number, group: Group): void { - this.children.set(String(groupIndex), group); + setGroup(groupIndex: number, group: Group): void { + this.groups.set(String(groupIndex), group); } - findChild(groupName: string): Group { - return Array.from(this.children.values()).find( + findGroup(groupName: string): Group | null { + return Array.from(this.groups.values()).find( group => group?.index?.name === groupName ) || null; } diff --git a/src/file-system/file-store.ts b/src/file-system/file-store.ts index 9cec826..0702739 100644 --- a/src/file-system/file-store.ts +++ b/src/file-system/file-store.ts @@ -5,6 +5,7 @@ import { logger } from '@runejs/common'; import { IndexDatabase } from './db/index-database'; import { ArchiveConfig } from '../config'; import { JS5 } from './js5'; +import { Archive } from './archive'; export class FileStore { @@ -13,6 +14,7 @@ export class FileStore { readonly fileStorePath: string; readonly fileNameHashes: Map; readonly js5: JS5; + readonly archives: Map; private _archiveConfig: { [key: string]: ArchiveConfig }; private _database: IndexDatabase; @@ -21,11 +23,33 @@ export class FileStore { this.gameBuild = gameBuild; this.fileStorePath = storePath; this.fileNameHashes = new Map(); + this.archives = new Map(); this.loadArchiveConfig(); this.loadFileNames(); this.js5 = new JS5(this); } + async load(): Promise { + const archiveNames = Object.keys(this._archiveConfig); + + for (const archiveName of archiveNames) { + const archiveConfig = this._archiveConfig[archiveName]; + if (!this.archives.has(archiveConfig.index)) { + const archive = new Archive(this, archiveConfig.index); + this.archives.set(archiveConfig.index, archive); + await archive.loadIndex(); + } + } + } + + js5Load(): void { + this.js5.loadJS5Store(); + } + + js5EncodeMainIndex(): Buffer { + return this.js5.encodeMainIndex().toNodeBuffer(); + } + hashFileName(fileName: string): number { if (!fileName) { return 0; @@ -86,6 +110,7 @@ export class FileStore { loadArchiveConfig(): void { const configPath = join(this.fileStorePath, 'config', 'archives.json5'); + if (!existsSync(configPath)) { logger.error(`Error loading file store: ${configPath} was not found.`); return; @@ -107,6 +132,20 @@ export class FileStore { return this._database; } + getArchive(archiveKey: number): Archive | null { + return this.archives.get(archiveKey) || null; + } + + setArchive(archiveKey: number, archive: Archive): void { + this.archives.set(archiveKey, archive); + } + + findArchive(archiveName: string): Archive | null { + return Array.from(this.archives.values()).find( + a => a?.index?.name === archiveName + ) || null; + } + get archiveConfig(): { [key: string]: ArchiveConfig } { return this._archiveConfig; } diff --git a/src/file-system/group.ts b/src/file-system/group.ts index 6836388..a36dde8 100644 --- a/src/file-system/group.ts +++ b/src/file-system/group.ts @@ -7,7 +7,7 @@ import { FlatFile } from './flat-file'; export class Group extends FileBase { readonly archive: Archive; - readonly children: Map; + readonly files: Map; constructor( fileStore: FileStore, @@ -16,7 +16,7 @@ export class Group extends FileBase { ) { super(fileStore, key, archive.index.key, 'GROUP'); this.archive = archive; - this.children = new Map(); + this.files = new Map(); } js5Unpack(): Buffer | null { @@ -43,16 +43,16 @@ export class Group extends FileBase { return this.fileStore.js5.encodeGroup(this); } - getChild(fileIndex: number): FlatFile { - return this.children.get(String(fileIndex)) || null; + getFile(fileIndex: number): FlatFile | null { + return this.files.get(String(fileIndex)) || null; } - setChild(fileIndex: number, file: FlatFile): void { - this.children.set(String(fileIndex), file); + setFile(fileIndex: number, file: FlatFile): void { + this.files.set(String(fileIndex), file); } - findChild(fileName: string): FlatFile { - return Array.from(this.children.values()).find( + findFile(fileName: string): FlatFile | null { + return Array.from(this.files.values()).find( file => file?.index?.name === fileName ) || null; } diff --git a/src/file-system/js5.ts b/src/file-system/js5.ts index 4b2502b..28b9255 100644 --- a/src/file-system/js5.ts +++ b/src/file-system/js5.ts @@ -7,6 +7,8 @@ import { FileStore } from './file-store'; import { ArchiveFormat } from '../config'; import { archiveFlags } from '../config/archive-flags'; import { FlatFile } from './flat-file'; +import { join } from 'path'; +import { existsSync, readdirSync, readFileSync, statSync } from 'graceful-fs'; export class JS5 { @@ -21,6 +23,60 @@ export class JS5 { this.fileStore = fileStore; } + loadJS5Store(): void { + const js5StorePath = join(this.fileStore.fileStorePath, 'js5'); + + if (!existsSync(js5StorePath)) { + throw new Error(`${js5StorePath} could not be found.`); + } + + const stats = statSync(js5StorePath); + if (!stats?.isDirectory()) { + throw new Error(`${js5StorePath} is not a valid directory.`); + } + + const storeFileNames = readdirSync(js5StorePath); + const dataFile = 'main_file_cache.dat2'; + const mainIndexFile = 'main_file_cache.idx255'; + + if (storeFileNames.indexOf(dataFile) === -1) { + throw new Error(`The main ${dataFile} data file could not be found.`); + } + + if (storeFileNames.indexOf(mainIndexFile) === -1) { + throw new Error(`The main ${mainIndexFile} index file could not be found.`); + } + + const indexFilePrefix = 'main_file_cache.idx'; + const dataFilePath = join(js5StorePath, dataFile); + const mainIndexFilePath = join(js5StorePath, mainIndexFile); + + this.mainArchiveData = new ByteBuffer(readFileSync(dataFilePath)); + this.mainIndex = new ByteBuffer(readFileSync(mainIndexFilePath)); + this.archiveIndexes = new Map(); + + for (const fileName of storeFileNames) { + if (!fileName?.length || fileName === mainIndexFile || fileName === dataFile) { + continue; + } + + if (!fileName.startsWith(indexFilePrefix)) { + continue; + } + + const index = fileName.substring(fileName.indexOf('.idx') + 4); + const numericIndex = Number(index); + + if (isNaN(numericIndex)) { + logger.error(`Index file ${fileName} does not have a valid extension.`); + } + + this.archiveIndexes.set(index, new ByteBuffer(readFileSync(join(js5StorePath, fileName)))); + } + + logger.info(`JS5 store loaded for game build ${this.fileStore.gameBuild}.`); + } + unpack(file: Group | Archive): Buffer | null { const fileDetails = file.index; const fileKey = fileDetails.key; @@ -293,6 +349,96 @@ export class JS5 { return fileDetails.data; } + async decodeGroup(group: Group): Promise { + const groupDetails = group.index; + const { key: groupKey, name: groupName } = groupDetails; + const files = group.files; + + if (!groupDetails.data) { + this.decompress(group); + + if (!groupDetails.data) { + logger.error(`Unable to decode group ${groupName || groupKey}.`); + return; + } + } + + const data = new ByteBuffer(groupDetails.data); + + if(groupDetails.childCount === 1) { + return; + } + + data.readerIndex = (data.length - 1); // EOF - 1 byte + + groupDetails.stripeCount = data.get('byte', 'unsigned'); + + data.readerIndex = (data.length - 1 - groupDetails.stripeCount * + groupDetails.childCount * 4); // Stripe data footer + + if (data.readerIndex < 0) { + logger.error(`Invalid reader index of ${data.readerIndex} for group ` + + `${groupName || groupKey}.`); + return; + } + + const fileSizeMap = new Map(); + const fileStripeMap = new Map(); + const fileDataMap = new Map(); + + for (const [ flatFileKey, ] of files) { + fileSizeMap.set(flatFileKey, 0); + fileStripeMap.set(flatFileKey, new Array(groupDetails.stripeCount)); + } + + for (let stripe = 0; stripe < groupDetails.stripeCount; stripe++) { + let currentLength = 0; + + for (const [ flatFileKey, ] of files) { + const delta = data.get('int'); + currentLength += delta; + + const fileStripes = fileStripeMap.get(flatFileKey); + const size = fileSizeMap.get(flatFileKey) + currentLength; + + fileStripes[stripe] = currentLength; + fileSizeMap.set(flatFileKey, size + currentLength); + } + } + + for (const [ flatFileKey, file ] of files) { + file.index.fileSize = fileSizeMap.get(flatFileKey); + file.index.stripeCount = groupDetails.stripeCount; + file.index.stripes = fileStripeMap.get(flatFileKey).join(','); + fileDataMap.set(flatFileKey, new ByteBuffer(file.index.fileSize)); + } + + data.readerIndex = 0; + + for (let stripe = 0; stripe < groupDetails.stripeCount; stripe++) { + for (const [ fileIndex, ] of files) { + let stripeLength = fileStripeMap.get(fileIndex)[stripe]; + let sourceEnd: number = data.readerIndex + stripeLength; + + if (data.readerIndex + stripeLength >= data.length) { + sourceEnd = data.length; + stripeLength = (data.readerIndex + stripeLength) - data.length; + } + + const stripeData = data.getSlice(data.readerIndex, stripeLength); + const fileData = fileDataMap.get(fileIndex); + + fileData.putBytes(stripeData); + + data.readerIndex = sourceEnd; + } + } + + for (const [ fileIndex, file ] of files) { + file.index.data = fileDataMap.get(fileIndex).toNodeBuffer(); + } + } + async decodeArchive(archive: Archive): Promise { const archiveDetails = archive.index; @@ -332,7 +478,7 @@ export class JS5 { const delta = archiveData.get(mainDataType, 'unsigned'); const groupKey = accumulator += delta; const group = groups[i] = new Group(this.fileStore, groupKey, archive); - archive.setChild(groupKey, group); + archive.setGroup(groupKey, group); } // Load group database indexes, or create them if they don't yet exist @@ -399,14 +545,14 @@ export class JS5 { const delta = archiveData.get(mainDataType, 'unsigned'); const childFileIndex = accumulator += delta; const flatFile = new FlatFile(this.fileStore, childFileIndex, group); - group.setChild(childFileIndex, flatFile); + group.setFile(childFileIndex, flatFile); } } // Load flat file database indexes, or create them if they don't yet exist // @todo batch load all grouped files at once for (const group of groups) { - for (const [ , flatFile ] of group.children) { + for (const [ , flatFile ] of group.files) { await flatFile.loadIndex(); } } @@ -414,7 +560,7 @@ export class JS5 { // Grouped file names if (flags.groupNames) { for (const group of groups) { - for (const [ , flatFile ] of group.children) { + for (const [ , flatFile ] of group.files) { flatFile.index.nameHash = archiveData.get('int'); flatFile.index.name = this.fileStore.findFileName( flatFile.index.nameHash, @@ -425,96 +571,6 @@ export class JS5 { } } - async decodeGroup(group: Group): Promise { - const groupDetails = group.index; - const { key: groupKey, name: groupName } = groupDetails; - const files = group.children; - - if (!groupDetails.data) { - this.decompress(group); - - if (!groupDetails.data) { - logger.error(`Unable to decode group ${groupName || groupKey}.`); - return; - } - } - - const data = new ByteBuffer(groupDetails.data); - - if(groupDetails.childCount === 1) { - return; - } - - data.readerIndex = (data.length - 1); // EOF - 1 byte - - groupDetails.stripeCount = data.get('byte', 'unsigned'); - - data.readerIndex = (data.length - 1 - groupDetails.stripeCount * - groupDetails.childCount * 4); // Stripe data footer - - if (data.readerIndex < 0) { - logger.error(`Invalid reader index of ${data.readerIndex} for group ` + - `${groupName || groupKey}.`); - return; - } - - const fileSizeMap = new Map(); - const fileStripeMap = new Map(); - const fileDataMap = new Map(); - - for (const [ flatFileKey, ] of files) { - fileSizeMap.set(flatFileKey, 0); - fileStripeMap.set(flatFileKey, new Array(groupDetails.stripeCount)); - } - - for (let stripe = 0; stripe < groupDetails.stripeCount; stripe++) { - let currentLength = 0; - - for (const [ flatFileKey, ] of files) { - const delta = data.get('int'); - currentLength += delta; - - const fileStripes = fileStripeMap.get(flatFileKey); - const size = fileSizeMap.get(flatFileKey) + currentLength; - - fileStripes[stripe] = currentLength; - fileSizeMap.set(flatFileKey, size + currentLength); - } - } - - for (const [ flatFileKey, file ] of files) { - file.index.fileSize = fileSizeMap.get(flatFileKey); - file.index.stripeCount = groupDetails.stripeCount; - file.index.stripes = fileStripeMap.get(flatFileKey).join(','); - fileDataMap.set(flatFileKey, new ByteBuffer(file.index.fileSize)); - } - - data.readerIndex = 0; - - for (let stripe = 0; stripe < groupDetails.stripeCount; stripe++) { - for (const [ fileIndex, ] of files) { - let stripeLength = fileStripeMap.get(fileIndex)[stripe]; - let sourceEnd: number = data.readerIndex + stripeLength; - - if (data.readerIndex + stripeLength >= data.length) { - sourceEnd = data.length; - stripeLength = (data.readerIndex + stripeLength) - data.length; - } - - const stripeData = data.getSlice(data.readerIndex, stripeLength); - const fileData = fileDataMap.get(fileIndex); - - fileData.putBytes(stripeData); - - data.readerIndex = sourceEnd; - } - } - - for (const [ fileIndex, file ] of files) { - file.index.data = fileDataMap.get(fileIndex).toNodeBuffer(); - } - } - pack(file: Group | Archive): Buffer | null { return null; // @todo stub } @@ -575,15 +631,33 @@ export class JS5 { return fileDetails.compressedData; } - encodeArchive(archive: Archive): Buffer { + encodeGroup(group: Group): Buffer { return null; // @todo stub } - encodeGroup(group: Group): Buffer { + encodeArchive(archive: Archive): Buffer { return null; // @todo stub } + encodeMainIndex(): ByteBuffer { + const archiveCount = this.fileStore.archives.size; + const fileSize = 4 * archiveCount; + + const data = new ByteBuffer(fileSize + 31); + + data.put(0); + data.put(fileSize, 'int'); + + for (let archiveIndex = 0; archiveIndex < archiveCount; archiveIndex++) { + data.put(this.fileStore.archives.get(archiveIndex).index.checksum, 'int'); + } + + this.mainIndex = data; + return this.mainIndex; + } + getEncryptionKeys(fileName: string): any[] { + // @todo pull key files from openrs2.org return []; // @todo stub } From df2a68871e870151c1a0112eba1769b1f7a0bc55 Mon Sep 17 00:00:00 2001 From: Kikorono Date: Tue, 12 Jul 2022 18:32:06 -0500 Subject: [PATCH 09/32] Removing old filestore-system files and reorganizing the new package --- package.json | 12 +- src/archive.ts | 451 ------------------ src/{file-system => config}/file-error.ts | 0 src/{file-system => config}/file-type.ts | 0 src/db/archive-index.entity.ts | 38 -- src/db/file-index.entity.ts | 56 --- src/db/group-index.entity.ts | 54 --- src/{file-system => }/db/index-database.ts | 2 +- src/db/index-entity.ts | 31 -- src/db/index-service.ts | 445 ----------------- src/{file-system => }/db/index.entity.ts | 4 +- src/db/index.ts | 6 - src/db/store-index.entity.ts | 32 -- src/dev.ts | 6 - src/file-state.ts | 54 --- src/file-system/file-base.ts | 4 +- src/file-system/file-store.ts | 2 +- src/file-system/flat-file.ts | 2 +- src/flat-file.ts | 124 ----- src/group.ts | 344 -------------- src/index.ts | 10 - src/indexed-file.ts | 502 -------------------- src/scripts/dev.ts | 1 + src/store.ts | 526 --------------------- src/util/index.ts | 2 - 25 files changed, 14 insertions(+), 2694 deletions(-) delete mode 100644 src/archive.ts rename src/{file-system => config}/file-error.ts (100%) rename src/{file-system => config}/file-type.ts (100%) delete mode 100644 src/db/archive-index.entity.ts delete mode 100644 src/db/file-index.entity.ts delete mode 100644 src/db/group-index.entity.ts rename src/{file-system => }/db/index-database.ts (97%) delete mode 100644 src/db/index-entity.ts delete mode 100644 src/db/index-service.ts rename src/{file-system => }/db/index.entity.ts (96%) delete mode 100644 src/db/index.ts delete mode 100644 src/db/store-index.entity.ts delete mode 100644 src/dev.ts delete mode 100644 src/file-state.ts delete mode 100644 src/flat-file.ts delete mode 100644 src/group.ts delete mode 100644 src/index.ts delete mode 100644 src/indexed-file.ts create mode 100644 src/scripts/dev.ts delete mode 100644 src/store.ts delete mode 100644 src/util/index.ts diff --git a/package.json b/package.json index dbb51ac..1955dda 100644 --- a/package.json +++ b/package.json @@ -1,6 +1,6 @@ { - "name": "@runejs/store", - "version": "1.0.0-beta.1", + "name": "@runejs/filestore", + "version": "1.0.0-next.0", "description": "Tools for managing and indexing the asset file store used with RuneJS.", "main": "./index.js", "types": "./index.d.ts", @@ -15,7 +15,7 @@ }, "scripts": { "build": "tsc", - "start": "ts-node-dev src/dev.ts", + "start": "ts-node-dev src/scripts/dev.ts", "lint": "eslint --ext .ts src", "lint:fix": "eslint --ext .ts src --fix", "unpack": "ts-node-dev --max-old-space-size=2048 src/scripts/unpacker.ts", @@ -30,7 +30,7 @@ }, "repository": { "type": "git", - "url": "git+ssh://git@github.com/runejs/store.git" + "url": "git+ssh://git@github.com/runejs/filestore.git" }, "keywords": [ "runejs", @@ -44,9 +44,9 @@ "author": "Kikorono", "license": "GPL-3.0", "bugs": { - "url": "https://github.com/runejs/store/issues" + "url": "https://github.com/runejs/filestore/issues" }, - "homepage": "https://github.com/runejs/store#readme", + "homepage": "https://github.com/runejs/filestore#readme", "peerDependencies": { "@runejs/common": "2.0.2-beta.2", "graceful-fs": ">=4.2.0", diff --git a/src/archive.ts b/src/archive.ts deleted file mode 100644 index ca0832d..0000000 --- a/src/archive.ts +++ /dev/null @@ -1,451 +0,0 @@ -import { join } from 'path'; -import { existsSync, mkdirSync, rmSync } from 'graceful-fs'; -import { ByteBuffer, logger } from '@runejs/common'; - -import { ArchiveFormat, FileState, FlatFile, Group } from './index'; -import { ArchiveIndexEntity } from './db'; -import { FileBreadcrumb, IndexedFile } from './indexed-file'; -import { ArchiveConfig } from './config'; -import { archiveFlags } from './config/archive-flags'; - - -export class Archive extends IndexedFile { - - public readonly config: ArchiveConfig; - public readonly groups: Map; - - private _missingEncryptionKeys: number; - - public constructor(index: ArchiveIndexEntity, config: ArchiveConfig, breadcrumb?: Partial) { - super(index, breadcrumb); - - this.groups = new Map(); - - config.filesNamed = config.filesNamed || false; - config.versioned = config.versioned || false; - - this.config = config; - this.encryption = this.config.encryption || 'none'; - this.compression = this.config.compression || 'none'; - } - - public override decode(decodeGroups: boolean = true): ByteBuffer | null { - logger.info(`Decoding archive ${this.name}...`); - - this._missingEncryptionKeys = 0; - - this.unpack(); - - logger.info(`Archive ${this.name} checksum: ${this.crc32}`); - - if(this.numericKey === 255) { - return this.data; - } - - this.decompress(); - - if(!this._data?.length) { - logger.error(`Error decompressing file data.`); - return null; - } - - const archiveData = this._data; - const format = this.index.format = archiveData.get('byte', 'unsigned'); - const mainDataType = format >= ArchiveFormat.smart ? 'smart_int' : 'short'; - this.index.version = format >= ArchiveFormat.versioned ? archiveData.get('int') : 0; - const flags = archiveFlags(archiveData.get('byte', 'unsigned')); - const groupCount = archiveData.get(mainDataType, 'unsigned'); - - logger.info(`${groupCount} groups were found within the ${this.name} archive.`); - - const groupKeys: number[] = new Array(groupCount); - const groupChildCounts: Map = new Map(); - let accumulator = 0; - - // Group index keys - for(let i = 0; i < groupCount; i++) { - const delta = archiveData.get(mainDataType, 'unsigned'); - groupKeys[i] = accumulator += delta; - const group = new Group(this.indexService.validateGroup({ - numericKey: groupKeys[i], - name: String(groupKeys[i]), - archive: this - }), { - store: this.store, - archive: this - }); - - group.setState(FileState.encoded); - this.set(groupKeys[i], group); - } - - // Group names - if(flags.groupNames) { - for(const groupIndex of groupKeys) { - const group = this.get(groupIndex); - group.nameHash = group.index.nameHash = archiveData.get('int'); - group.name = group.index.name = this.store.findFileName(group.nameHash, String(group.nameHash)); - } - } - - // Compressed file data CRC32 checksums - for(const key of groupKeys) { - const group = this.get(key); - group.crc32 = archiveData.get('int'); - } - - // Decompressed file data CRC32 checksums - if(flags.decompressedCrcs) { - for(const key of groupKeys) { - const decompressedCrc32 = archiveData.get('int'); - // @TODO assign to group (requires changing existing 'crc32' to 'compressedCrc`) - } - } - - // File data whirlpool digests - if(flags.whirlpoolDigests) { - for(const key of groupKeys) { - const whirlpoolDigest = new ByteBuffer(512); - archiveData.getBytes(whirlpoolDigest, 512); - // @TODO assign to group - } - } - - // Group file sizes - if(flags.groupSizes) { - for(const key of groupKeys) { - const compressedSize = archiveData.get('int'); - const decompressedSize = archiveData.get('int'); - // @TODO assign to group (requires changing existing 'size' to 'compressedSize') - } - } - - // Group version numbers - for(const key of groupKeys) { - const group = this.get(key); - group.version = archiveData.get('int'); - } - - // Group file counts - for(const key of groupKeys) { - groupChildCounts.set(key, archiveData.get('short', 'unsigned')); - } - - // Grouped file index keys - for(const key of groupKeys) { - const group = this.get(key) as Group; - const fileCount = groupChildCounts.get(key); - - accumulator = 0; - for(let i = 0; i < fileCount; i++) { - const delta = archiveData.get(mainDataType, 'unsigned'); - const childFileIndex = accumulator += delta; - group.set(childFileIndex, new FlatFile(this.indexService.validateFile({ - numericKey: childFileIndex, - name: String(childFileIndex), - group, archive: this - }), { - store: this.store, - archive: this, - group: group - })); - } - } - - // Grouped file names - if(flags.groupNames) { - for(const key of groupKeys) { - const fileGroup = this.get(key) as Group; - - for(const [ , file ] of fileGroup.files) { - const nameHash = archiveData.get('int'); - if(file) { - file.nameHash = file.index.nameHash = nameHash; - file.name = file.index.name = - this.store.findFileName(file.nameHash, String(file.nameHash)); - } - } - } - } - - if(decodeGroups) { - for(const [ , group ] of this.groups) { - try { - group.decode(); - } catch(error) { - logger.error(error); - } - } - - if(this.missingEncryptionKeys) { - logger.error(`Missing ${this.missingEncryptionKeys} XTEA decryption key(s).`); - } - } else { - logger.info(`${groupCount} groups(s) were found.`); - } - - this.setData(this._data, FileState.raw); - return this._data ?? null; - } - - public override encode(encodeGroups: boolean = true): ByteBuffer | null { - if(this.numericKey === 255) { - return this.store.encode(); - } - - const groups = this.groups; - const groupCount = groups.size; - - // @TODO add sizes of all files instead of using a set amount here - const buffer = new ByteBuffer(1000 * 1000); - - // Write index file header - buffer.put(this.index.format ?? ArchiveFormat.original); - buffer.put(this.config.filesNamed ? 1 : 0); - buffer.put(groupCount, 'short'); - - // Write file indexes - let writtenFileIndex = 0; - for(const [ , group ] of groups) { - const val = group.numericKey; - buffer.put(val - writtenFileIndex, 'short'); - writtenFileIndex = val; - } - - // Write name hashes (if applicable) - if(this.config.filesNamed) { - for(const [ , file ] of groups) { - buffer.put(file.nameHash ?? -1, 'int'); - } - } - - // Write file crc values - for(const [ , file ] of groups) { - buffer.put(file.crc32 ?? -1, 'int'); - } - - // Write file version numbers - for(const [ , ] of groups) { - buffer.put(0, 'int'); - } - - // Write file group child counts - for(const [ , group ] of groups) { - buffer.put(group.files.size ?? 1, 'short'); - } - - // Write group file indices - for(const [ , group ] of groups) { - if(group.files.size > 1) { - writtenFileIndex = 0; - - for(const [ , file ] of group.files) { - const i = file.numericKey; - buffer.put(i - writtenFileIndex, 'short'); - writtenFileIndex = i; - } - } else { - buffer.put(0, 'short'); - } - } - - // Write group file name hashes (if applicable) - if(this.config.filesNamed) { - for(const [ , group ] of groups) { - if(group.files.size > 1) { - for(const [ , file ] of group.files) { - buffer.put(file.nameHash ?? -1, 'int'); - } - } else { - buffer.put(0, 'int'); - } - } - } - - const indexData = buffer?.flipWriter(); - - if(indexData?.length) { - this.setData(indexData, FileState.encoded); - this.sha256 = this.index.sha256 = this.generateSha256(); - } - - if(encodeGroups) { - this.groups.forEach(group => group.encode()); - } - - return this.data ?? null; - } - - public override compress(compressGroups: boolean = true): ByteBuffer | null { - if(compressGroups) { - this.groups.forEach(group => group.compress()); - } - return super.compress(); - } - - public override async read(compress: boolean = false, readDiskFiles: boolean = true): Promise { - logger.info(`Reading archive ${this.name}...`); - - // Read in all groups within the archive - const groupIndexes = await this.index.groups; - for(const groupIndex of groupIndexes) { - const group = new Group(groupIndex, { - store: this.store, - archive: this - }); - - this.groups.set(group.key, group); - await group.read(false, readDiskFiles); - } - - if(compress) { - // Then compress them, if needed - for(const [ , group ] of this.groups) { - group.compress(); - } - } - - logger.info(`${this.groups.size} groups(s) were loaded from the ${this.name} archive.`); - - this.encode(); - - if(compress) { - return this.compress(); - } else { - return this._data; - } - } - - public override write(): void { - if(!this.groups.size) { - logger.error(`Error writing archive ${this.name || this.key}: Archive is empty.`); - return; - } - - const start = Date.now(); - logger.info(`Writing archive ${this.name || this.key}...`); - - const archivePath = this.outputPath; - - if(existsSync(archivePath)) { - rmSync(archivePath, { recursive: true, force: true }); - } - - mkdirSync(archivePath, { recursive: true }); - - Array.from(this.groups.values()).forEach(group => group.write()); - - const end = Date.now(); - logger.info(`Archive ${this.name || this.key} written in ${(end - start) / 1000} seconds.`) - } - - public async saveIndexData(saveGroups: boolean = true, saveFiles: boolean = true): Promise { - if(!this.groups.size) { - return; - } - - logger.info(`Saving archive ${this.name} to index...`); - - await this.indexService.saveArchiveIndex(this); - - if(saveGroups) { - await this.saveGroupIndexes(saveFiles); - } - - logger.info(`Archive ${this.name} indexing complete.`); - } - - public async saveGroupIndexes(saveFlatFiles: boolean = true): Promise { - const groups = Array.from(this.groups.values()); - - if(groups?.length) { - logger.info(`Saving archive ${ this.name } group indexes...`); - await this.indexService.saveGroupIndexes(groups); - } - - if(saveFlatFiles) { - await this.saveFlatFileIndexes(); - } - } - - public async saveFlatFileIndexes(): Promise { - if(this.config.flatten) { - return; - } - - const groups = Array.from(this.groups.values()); - const flatFiles = groups.filter(group => { - if(!group?.files?.size || group?.index?.flatFile) { - return false; - } - return group.files.size > 1; - }).map(group => Array.from(group.files.values())) - .reduce((a, v) => a.concat(v), []); - - if(flatFiles?.length) { - logger.info(`Saving archive ${ this.name } flat file indexes...`); - await this.indexService.saveFileIndexes(flatFiles); - } - } - - public has(groupKey: string): boolean; - public has(groupKey: number): boolean; - public has(groupKey: string | number): boolean; - public has(groupKey: string | number): boolean { - return this.groups.has(String(groupKey)); - } - - public get(groupKey: string): Group | null; - public get(groupKey: number): Group | null; - public get(groupKey: string | number): Group | null; - public get(groupKey: string | number): Group | null { - return this.groups.get(String(groupKey)) ?? null; - } - - public set(groupKey: string, group: Group): void; - public set(groupKey: number, group: Group): void; - public set(groupKey: string | number, group: Group): void; - public set(groupKey: string | number, group: Group): void { - this.groups.set(String(groupKey), group); - } - - public find(groupName: string): Archive | Group | FlatFile | null { - const children = Array.from(this.groups.values()); - return children.find(child => child?.name === groupName) ?? null; - } - - public incrementMissingEncryptionKeys(): void { - this._missingEncryptionKeys++; - } - - public get missingEncryptionKeys(): number { - return this._missingEncryptionKeys; - } - - public override get path(): string { - if(!this.store?.path) { - throw new Error(`Error generating archive path; Store path not provided for archive ${this.key}.`); - } - if(!this.name) { - throw new Error(`Error generating archive path; Name not provided for archive ${this.key}.`); - } - - return join(this.store.path, 'unpacked', this.name); - } - - public override get outputPath(): string { - if(!this.store?.outputPath) { - throw new Error(`Error generating archive output path; Store output path not provided for archive ${this.key}.`); - } - if(!this.name) { - throw new Error(`Error generating archive output path; Name not provided for archive ${this.key}.`); - } - - return join(this.store.outputPath, this.name); - } - - public get versioned(): boolean { - return this.config.versioned; - } - -} diff --git a/src/file-system/file-error.ts b/src/config/file-error.ts similarity index 100% rename from src/file-system/file-error.ts rename to src/config/file-error.ts diff --git a/src/file-system/file-type.ts b/src/config/file-type.ts similarity index 100% rename from src/file-system/file-type.ts rename to src/config/file-type.ts diff --git a/src/db/archive-index.entity.ts b/src/db/archive-index.entity.ts deleted file mode 100644 index ebea8fc..0000000 --- a/src/db/archive-index.entity.ts +++ /dev/null @@ -1,38 +0,0 @@ -import { Column, Entity, Index, JoinColumn, ManyToOne, OneToMany, PrimaryColumn } from 'typeorm'; - -import { IndexEntity } from './index-entity'; -import { StoreIndexEntity } from './store-index.entity'; -import { GroupIndexEntity } from './group-index.entity'; -import { FileState } from '../file-state'; -import { ArchiveFormat } from '../config'; - - -@Entity('archive_index') -@Index('archive_identifier', [ 'key', 'gameBuild' ], { unique: true }) -export class ArchiveIndexEntity extends IndexEntity { - - @PrimaryColumn('text', { name: 'game_build', nullable: false, unique: false }) - gameBuild: string; - - @Column('integer', { name: 'group_count', nullable: false, default: 0 }) - groupCount: number = 0; - - @Column('integer', { name: 'format', nullable: false, default: ArchiveFormat.original }) - format: number = ArchiveFormat.original; - - @Column('integer', { nullable: false, default: 0 }) - version: number = 0; - - @Column('text', { name: 'data_state', nullable: false }) - state: FileState; - - @ManyToOne(() => StoreIndexEntity, async store => store.archives, - { primary: true, onDelete: 'CASCADE' }) - @JoinColumn({ name: 'game_build', referencedColumnName: 'gameBuild' }) - store: StoreIndexEntity; - - @OneToMany(() => GroupIndexEntity, group => group.archive, - { cascade: true, lazy: true }) - groups: Promise | GroupIndexEntity[]; - -} diff --git a/src/db/file-index.entity.ts b/src/db/file-index.entity.ts deleted file mode 100644 index 90ae874..0000000 --- a/src/db/file-index.entity.ts +++ /dev/null @@ -1,56 +0,0 @@ -import { Column, Entity, Index, JoinColumn, ManyToOne, PrimaryColumn } from 'typeorm'; - -import { IndexEntity } from './index-entity'; -import { StoreIndexEntity } from './store-index.entity'; -import { ArchiveIndexEntity } from './archive-index.entity'; -import { GroupIndexEntity } from './group-index.entity'; - - -@Entity('file_index') -@Index('file_identifier', [ 'key', 'gameBuild', 'archiveKey', 'groupKey' ], { unique: true }) -export class FileIndexEntity extends IndexEntity { - - @PrimaryColumn('text', { name: 'game_build', nullable: false, unique: false }) - gameBuild: string; - - @PrimaryColumn('integer', { name: 'archive_key', unique: false, nullable: false }) - archiveKey: number; - - @PrimaryColumn('integer', { name: 'group_key', unique: false, nullable: false }) - groupKey: number; - - @Column('integer', { name: 'name_hash', nullable: true, default: 0 }) - nameHash: number = 0; - - @Column('integer', { nullable: false, default: 0 }) - version: number = 0; - - @Column('integer', { name: 'stripe_count', nullable: false, default: 1 }) - stripeCount: number = 1; - - @Column('text', { nullable: true, default: null }) - stripes: string | null = null; - - @ManyToOne(() => StoreIndexEntity, async store => store.files, - { primary: true, onDelete: 'CASCADE' }) - @JoinColumn({ name: 'game_build', referencedColumnName: 'gameBuild' }) - store: StoreIndexEntity; - - @ManyToOne(() => ArchiveIndexEntity, async archive => archive.groups, - { primary: true, onDelete: 'CASCADE' }) - @JoinColumn([ - { name: 'archive_key', referencedColumnName: 'key' }, - { name: 'game_build', referencedColumnName: 'gameBuild' } - ]) - archive: ArchiveIndexEntity; - - @ManyToOne(() => GroupIndexEntity, async group => group.files, - { primary: true, onDelete: 'CASCADE' }) - @JoinColumn([ - { name: 'archive_key', referencedColumnName: 'archiveKey' }, - { name: 'group_key', referencedColumnName: 'key' }, - { name: 'game_build', referencedColumnName: 'gameBuild' } - ]) - group: GroupIndexEntity; - -} diff --git a/src/db/group-index.entity.ts b/src/db/group-index.entity.ts deleted file mode 100644 index db87aca..0000000 --- a/src/db/group-index.entity.ts +++ /dev/null @@ -1,54 +0,0 @@ -import { Column, Entity, Index, JoinColumn, ManyToOne, OneToMany, PrimaryColumn } from 'typeorm'; -import { FileIndexEntity } from './file-index.entity'; -import { ArchiveIndexEntity } from './archive-index.entity'; -import { IndexEntity } from './index-entity'; -import { StoreIndexEntity } from './store-index.entity'; -import { FileState } from '../file-state'; - - -@Entity('group_index') -@Index('group_identifier', [ 'key', 'gameBuild', 'archiveKey' ], { unique: true }) -export class GroupIndexEntity extends IndexEntity { - - @PrimaryColumn('text', { name: 'game_build', nullable: false, unique: false }) - gameBuild: string; - - @PrimaryColumn('integer', { name: 'archive_key', unique: false, nullable: false }) - archiveKey: number; - - @Column('boolean', { name: 'flat', nullable: false, default: false }) - flatFile: boolean = false; - - @Column('integer', { name: 'stripe_count', nullable: false, default: 1 }) - stripeCount: number = 1; - - @Column('text', { nullable: true, default: null }) - stripes: string | null = null; - - @Column('integer', { name: 'name_hash', nullable: true, default: 0 }) - nameHash: number = 0; - - @Column('integer', { nullable: false, default: 0 }) - version: number = 0; - - @Column('text', { name: 'data_state', nullable: false }) - state: FileState; - - @ManyToOne(() => StoreIndexEntity, async store => store.groups, - { primary: true, onDelete: 'CASCADE' }) - @JoinColumn({ name: 'game_build', referencedColumnName: 'gameBuild' }) - store: StoreIndexEntity; - - @ManyToOne(() => ArchiveIndexEntity, async archive => archive.groups, - { primary: true, onDelete: 'CASCADE' }) - @JoinColumn([ - { name: 'archive_key', referencedColumnName: 'key' }, - { name: 'game_build', referencedColumnName: 'gameBuild' } - ]) - archive: ArchiveIndexEntity; - - @OneToMany(() => FileIndexEntity, fileIndex => fileIndex.group, - { cascade: true, lazy: true }) - files: Promise | FileIndexEntity[]; - -} diff --git a/src/file-system/db/index-database.ts b/src/db/index-database.ts similarity index 97% rename from src/file-system/db/index-database.ts rename to src/db/index-database.ts index 3f956d3..da6e29d 100644 --- a/src/file-system/db/index-database.ts +++ b/src/db/index-database.ts @@ -2,7 +2,7 @@ import { Connection, createConnection, LoggerOptions, Repository } from 'typeorm import { join } from 'path'; import { existsSync, mkdirSync } from 'graceful-fs'; import { IndexEntity } from './index.entity'; -import { FileType } from '../file-type'; +import { FileType } from '../config/file-type'; export class IndexDatabase { diff --git a/src/db/index-entity.ts b/src/db/index-entity.ts deleted file mode 100644 index 2f40472..0000000 --- a/src/db/index-entity.ts +++ /dev/null @@ -1,31 +0,0 @@ -import { Column, CreateDateColumn, PrimaryColumn, UpdateDateColumn } from 'typeorm'; -import { FileState } from '../index'; - - -export abstract class IndexEntity { - - @PrimaryColumn('integer', { nullable: false, unique: false }) - key: number; - - @Column('text', { nullable: true, default: null }) - name: string | null = null; - - @Column('integer', { nullable: false, default: 0 }) - size: number = 0; - - @Column('integer', { nullable: true, default: null }) - crc32: number | null = null; - - @Column('text', { nullable: true, default: null }) - sha256: string | null = null; - - @Column('blob', { name: 'data', nullable: true, default: null }) - data: Buffer | null = null; - - @CreateDateColumn() - created: Date; - - @UpdateDateColumn() - updated: Date; - -} diff --git a/src/db/index-service.ts b/src/db/index-service.ts deleted file mode 100644 index 7d32e3e..0000000 --- a/src/db/index-service.ts +++ /dev/null @@ -1,445 +0,0 @@ -import { join } from 'path'; -import { existsSync, mkdirSync } from 'graceful-fs'; -import { Connection, createConnection, Repository } from 'typeorm'; - -import { logger } from '@runejs/common'; - -import { Archive, ArchiveFormat, FileState, FlatFile, Group, IndexedFile, IndexEntity, Store } from '../index'; -import { ArchiveIndexEntity, FileIndexEntity, GroupIndexEntity, StoreIndexEntity } from './index'; - - -const CHUNK_SIZE = 300; // 250 - - -export class IndexService { - - public readonly store: Store; - - private connection: Connection; - - public constructor(store: Store) { - this.store = store; - } - - public async load(): Promise { - const indexPath = join(this.store.path, 'indexes'); - - if(!existsSync(indexPath)) { - mkdirSync(indexPath, { recursive: true }); - } - - this.connection = await createConnection({ - type: 'better-sqlite3', - database: join(indexPath, `index_${this.store.gameBuild}.sqlite3`), - entities: [ StoreIndexEntity, ArchiveIndexEntity, GroupIndexEntity, FileIndexEntity ], - synchronize: true, - logging: [ 'error', 'warn' ], - // logging: 'all', - name: 'index-service' - }); - } - - public async getStoreIndex(): Promise { - return await this.storeRepo.findOne({ - where: { - gameBuild: this.store.gameBuild - } - }) || null; - } - - public async saveStoreIndex(): Promise { - let storeIndex = await this.getStoreIndex(); - let update = true; - - if(!storeIndex) { - storeIndex = new StoreIndexEntity(); - update = false; - } - - if(!storeIndex.gameBuild) { - storeIndex.gameBuild = this.store.gameBuild; - } - - if(!this.connection.isConnected) { - logger.error(`The index database connection was closed prematurely.`); - return null; - } - - storeIndex.data = this.store.data?.toNodeBuffer() || null; - - delete storeIndex.archives; - delete storeIndex.groups; - delete storeIndex.files; - - let savedIndex: StoreIndexEntity; - - if(update) { - const updateResult = await this.storeRepo.update({ - gameBuild: this.store.gameBuild - }, storeIndex); - - if(!updateResult?.affected) { - logger.error(`Main store entity update failed.`); - return null; - } - - savedIndex = await this.getStoreIndex(); - } else { - savedIndex = await this.storeRepo.save(storeIndex); - } - - if(savedIndex?.gameBuild !== this.store.gameBuild) { - logger.error(`Error saving store index ${this.store.gameBuild}.`); - return null; - } - - logger.info(`Store index ${this.store.gameBuild} saved.`); - - return savedIndex; - } - - public async getArchiveIndex(archive: Archive): Promise; - public async getArchiveIndex(archiveKey: number): Promise; - public async getArchiveIndex(archive: Archive | number): Promise { - const key = typeof archive === 'number' ? archive : archive.numericKey; - return await this.archiveRepo.findOne({ - where: { - key, gameBuild: this.store.gameBuild - }, - relations: [ 'groups' ] - }) || null; - } - - public async getArchiveIndexes(): Promise { - return await this.archiveRepo.find({ - where: { - gameBuild: this.store.gameBuild - }, - order: { - key: 'ASC' - } - }) || []; - } - - public validateArchive(archive: Archive | Partial): ArchiveIndexEntity { - const archiveIndex: ArchiveIndexEntity = archive.index ? archive.index : new ArchiveIndexEntity(); - if(!archive.index) { - archive.index = archiveIndex; - } - - this.updateEntityIndex(archive); - - archiveIndex.format = archiveIndex.format || ArchiveFormat.original; - archiveIndex.version = archiveIndex.version || 0; - archiveIndex.gameBuild = this.store.gameBuild; - archiveIndex.state = archive.state || FileState.unloaded; - archiveIndex.groupCount = archive.groups?.size ?? 0; - - return archiveIndex; - } - - public async saveArchiveIndex(archive: Archive): Promise { - const archiveIndex = archive.index; - const existingIndex = await this.archiveRepo.findOne({ - where: { - key: archiveIndex.key, - gameBuild: this.store.gameBuild - } - }); - - let affected; - - if(existingIndex) { - const { name, size, version, sha256, crc32, data, state } = archiveIndex; - existingIndex.name = name; - existingIndex.size = size; - existingIndex.version = version; - existingIndex.sha256 = sha256; - existingIndex.crc32 = crc32; - existingIndex.data = data; - existingIndex.state = state; - - delete existingIndex.groups; - - const result = await this.archiveRepo.update({ - key: archiveIndex.key, - gameBuild: this.store.gameBuild - }, existingIndex); - - affected = result?.affected || 0; - } else { - delete archiveIndex.groups; - - archiveIndex.data = archive.data.toNodeBuffer(); - const result = await this.archiveRepo.insert(archiveIndex); - affected = result?.identifiers?.length || 0; - } - - if(!affected) { - logger.error(`Error updating archive ${archiveIndex.name} database index.`); - } else { - logger.info(`Archive ${archiveIndex.name} database index saved.`); - } - - return await this.getArchiveIndex(archiveIndex.key); - } - - public async getGroupIndex(group: Group): Promise; - public async getGroupIndex(groupKey: number, archiveKey: number): Promise; - public async getGroupIndex(group: Group | number, archive?: number): Promise { - const key = typeof group === 'number' ? group : group.numericKey; - const archiveKey = typeof group === 'number' ? archive : group.archive.numericKey; - - return await this.groupRepo.findOne({ - where: { - key, archiveKey, - gameBuild: this.store.gameBuild - } - }) || null; - } - - public async getGroupIndexes(archive: ArchiveIndexEntity): Promise { - return await this.groupRepo.find({ - where: { - archiveKey: archive.key, - gameBuild: this.store.gameBuild - }, - order: { - key: 'ASC' - } - }) || []; - } - - public validateGroup(group: Group | Partial): GroupIndexEntity { - const { stripes, archive, files } = group; - - const groupIndex: GroupIndexEntity = group.index ? group.index : new GroupIndexEntity(); - if(!group.index) { - group.index = groupIndex; - } - - this.updateEntityIndex(group); - - groupIndex.gameBuild = this.store.gameBuild; - groupIndex.archiveKey = archive.numericKey; - groupIndex.state = group.state; - groupIndex.stripes = stripes?.length ? stripes.join(',') : null; - groupIndex.stripeCount = stripes?.length || 1; - groupIndex.flatFile = (files?.size === 1 || archive.config.flatten); - - return groupIndex; - } - - public async saveGroupIndex(group: Group): Promise { - if(!this.entityModified(group)) { - return; - } - await this.groupRepo.upsert(this.validateGroup(group), []); - } - - public async saveGroupIndexes(groups: Group[]): Promise { - const groupIndexes = groups.filter(group => this.entityModified(group)) - .map(group => this.validateGroup(group)); - - groupIndexes.forEach(i => delete i.files); - - if(!groupIndexes.length) { - logger.info(`No groups were modified.`); - } else { - await this.groupRepo.save(groupIndexes, { - chunk: CHUNK_SIZE, reload: false - }); - } - } - - public async getFileIndex(file: FlatFile): Promise { - return await this.fileRepo.findOne({ - where: { - key: file.numericKey, - groupKey: file.group.numericKey, - archiveKey: file.archive.numericKey, - gameBuild: this.store.gameBuild - } - }) || null; - } - - public async getFileIndexes(archiveOrGroup: ArchiveIndexEntity | GroupIndexEntity): Promise { - if(archiveOrGroup instanceof ArchiveIndexEntity) { - // Return all files for the specified archive - - return await this.fileRepo.find({ - where: { - archiveKey: archiveOrGroup.key, - gameBuild: this.store.gameBuild - }, - order: { - groupKey: 'ASC', - key: 'ASC' - } - }) || []; - } else { - // Return all files for the specified group - - return await this.fileRepo.find({ - where: { - groupKey: archiveOrGroup.key, - archiveKey: archiveOrGroup.archiveKey, - gameBuild: this.store.gameBuild - }, - order: { - key: 'ASC' - } - }) || []; - } - } - - public validateFile(file: FlatFile | Partial): FileIndexEntity { - const { size, stripes, group, archive } = file; - - const fileIndex: FileIndexEntity = file.index ? file.index : new FileIndexEntity(); - if(!file.index) { - file.index = fileIndex; - } - - this.updateEntityIndex(file); - - fileIndex.gameBuild = this.store.gameBuild; - fileIndex.store = this.store.index; - fileIndex.archiveKey = archive.numericKey; - fileIndex.archive = archive.index; - fileIndex.groupKey = group.numericKey; - fileIndex.group = group.index; - fileIndex.stripes = stripes?.join(',') || String(size); - fileIndex.stripeCount = fileIndex.stripes?.length || 1; - - return fileIndex; - } - - public async saveFileIndex(file: FlatFile): Promise { - if(!this.entityModified(file)) { - return; - } - await this.fileRepo.upsert(this.validateFile(file), []); - } - - public async saveFileIndexes(files: FlatFile[]): Promise { - const flatFileIndexes = files.filter(file => this.entityModified(file)) - .map(file => this.validateFile(file)); - - if(!flatFileIndexes.length) { - logger.info(`No flat files were modified.`); - } else { - logger.info(`${flatFileIndexes.length} flat files were modified.`); - await this.fileRepo.save(flatFileIndexes, { - chunk: CHUNK_SIZE, reload: false, transaction: false, listeners: false - }); - } - } - - public entityModified(file: IndexedFile | Partial>): boolean { - const index = file.index; - - if(file.numericKey !== index.key || file.name !== index.name) { - return true; - } - - if((index instanceof GroupIndexEntity || index instanceof FileIndexEntity) && - (file instanceof Group || file instanceof FlatFile)) { - if(file.nameHash !== index.nameHash || file.version !== index.version) { - return true; - } - - if(file.archive.numericKey !== index.archiveKey) { - return true; - } - } - - if((index instanceof ArchiveIndexEntity || index instanceof GroupIndexEntity) && - (file instanceof Archive || file instanceof Group)) { - if(file.state !== index.state) { - // return true; - } - } - - if(index instanceof FileIndexEntity && file instanceof FlatFile) { - if(file.group.numericKey !== index.groupKey) { - return true; - } - } - - return file.size !== index.size || file.crc32 !== index.crc32 || file.sha256 !== index.sha256; - } - - public updateEntityIndex(file: IndexedFile | Partial>): T { - const index = file.index; - - if(!file.name && file.hasNameHash) { - file.name = file.hasNameHash ? - this.store.findFileName(file.nameHash, String(file.nameHash)) : file.key; - } else if(!file.hasNameHash && file.named) { - file.nameHash = this.store.hashFileName(file.name); - } else { - file.nameHash = -1; - } - - if(index instanceof ArchiveIndexEntity || index instanceof GroupIndexEntity || index instanceof FileIndexEntity) { - index.version = file.version; - - if(file.modified) { - index.version = index.version ? index.version + 1 : 1; - } - } - - if(index.key === undefined || index.key === null) { - index.key = file.numericKey; - } - - if(index.name !== file.name) { - index.name = file.name; - } - - let dataModified = false; - - if(index.size !== file.size) { - index.size = file.size; - dataModified = true; - } - - if(index.crc32 !== file.crc32) { - index.crc32 = file.crc32; - dataModified = true; - } - - if(index.sha256 !== file.sha256) { - index.sha256 = file.sha256; - dataModified = true; - } - - if(dataModified || !index.data?.length) { - index.data = file.data?.length ? Buffer.from(file.data) : null; - } - - return index; - } - - public get loaded(): boolean { - return !!this.connection; - } - - public get storeRepo(): Repository { - return this.connection.getRepository(StoreIndexEntity); - } - - public get archiveRepo(): Repository { - return this.connection.getRepository(ArchiveIndexEntity); - } - - public get groupRepo(): Repository { - return this.connection.getRepository(GroupIndexEntity); - } - - public get fileRepo(): Repository { - return this.connection.getRepository(FileIndexEntity); - } - -} diff --git a/src/file-system/db/index.entity.ts b/src/db/index.entity.ts similarity index 96% rename from src/file-system/db/index.entity.ts rename to src/db/index.entity.ts index f97fdca..ca130cc 100644 --- a/src/file-system/db/index.entity.ts +++ b/src/db/index.entity.ts @@ -1,7 +1,7 @@ import { Column, CreateDateColumn, Entity, Index, PrimaryColumn, UpdateDateColumn } from 'typeorm'; -import { FileType } from '../file-type'; +import { FileType } from '../config/file-type'; import { CompressionMethod } from '@runejs/common/compress'; -import { FileError } from '../file-error'; +import { FileError } from '../config/file-error'; @Entity('file') diff --git a/src/db/index.ts b/src/db/index.ts deleted file mode 100644 index 5c19b78..0000000 --- a/src/db/index.ts +++ /dev/null @@ -1,6 +0,0 @@ -export * from './store-index.entity'; -export * from './archive-index.entity'; -export * from './file-index.entity'; -export * from './group-index.entity'; -export * from './index-entity'; -export * from './index-service'; diff --git a/src/db/store-index.entity.ts b/src/db/store-index.entity.ts deleted file mode 100644 index 94541ba..0000000 --- a/src/db/store-index.entity.ts +++ /dev/null @@ -1,32 +0,0 @@ -import { Column, CreateDateColumn, Entity, OneToMany, PrimaryColumn, UpdateDateColumn } from 'typeorm'; - -import { ArchiveIndexEntity } from './archive-index.entity'; -import { GroupIndexEntity } from './group-index.entity'; -import { FileIndexEntity } from './file-index.entity'; - - -@Entity('store_index') -export class StoreIndexEntity { - - @PrimaryColumn('text', { name: 'game_build', nullable: false, unique: true }) - gameBuild: string; - - @OneToMany(() => ArchiveIndexEntity, archive => archive.store, { lazy: true }) - archives: Promise | ArchiveIndexEntity[]; - - @OneToMany(() => GroupIndexEntity, group => group.store, { lazy: true }) - groups: Promise | GroupIndexEntity[]; - - @OneToMany(() => FileIndexEntity, file => file.store, { lazy: true }) - files: Promise | FileIndexEntity[]; - - @Column('blob', { name: 'data', nullable: true, default: null }) - data: Buffer | null = null; - - @CreateDateColumn() - created: Date; - - @UpdateDateColumn() - updated: Date; - -} diff --git a/src/dev.ts b/src/dev.ts deleted file mode 100644 index 1d1245c..0000000 --- a/src/dev.ts +++ /dev/null @@ -1,6 +0,0 @@ -import { Store } from './index'; -import { join } from 'path'; - - -const store = Store.create('435'); - diff --git a/src/file-state.ts b/src/file-state.ts deleted file mode 100644 index 4f96e8e..0000000 --- a/src/file-state.ts +++ /dev/null @@ -1,54 +0,0 @@ -export enum FileState { - /** - * File is not yet registered. - */ - unloaded = 'unloaded', - - /** - * File has been registered but not read into memory. - */ - loaded = 'loaded', - - /** - * File `data` is encrypted.
- * Encryption formats: xtea - */ - encrypted = 'encrypted', - - /** - * File `data` has been decrypted.
- * Encryption formats: xtea - */ - decrypted = 'decrypted', - - /** - * File `data` is compressed.
- * Compression formats: bzip, gzip - */ - compressed = 'compressed', - - /** - * File `data` is encoded for the JS5 format. - */ - encoded = 'encoded', - - /** - * File `data` is in raw binary form, uncompressed and unencrypted. - */ - raw = 'raw', - - /** - * The file was found, but is empty. - */ - empty = 'empty', - - /** - * The file was found, but is corrupted. - */ - corrupt = 'corrupt', - - /** - * The file was not found. - */ - missing = 'missing' -} diff --git a/src/file-system/file-base.ts b/src/file-system/file-base.ts index 305be70..0b71085 100644 --- a/src/file-system/file-base.ts +++ b/src/file-system/file-base.ts @@ -1,6 +1,6 @@ -import { IndexEntity } from './db/index.entity'; +import { IndexEntity } from '../db/index.entity'; import { FileStore } from './file-store'; -import { FileType } from './file-type'; +import { FileType } from '../config/file-type'; export class FileBase { diff --git a/src/file-system/file-store.ts b/src/file-system/file-store.ts index 0702739..efb3234 100644 --- a/src/file-system/file-store.ts +++ b/src/file-system/file-store.ts @@ -2,7 +2,7 @@ import { join } from 'path'; import JSON5 from 'json5'; import { existsSync, readFileSync } from 'graceful-fs'; import { logger } from '@runejs/common'; -import { IndexDatabase } from './db/index-database'; +import { IndexDatabase } from '../db/index-database'; import { ArchiveConfig } from '../config'; import { JS5 } from './js5'; import { Archive } from './archive'; diff --git a/src/file-system/flat-file.ts b/src/file-system/flat-file.ts index a33a273..2b2e705 100644 --- a/src/file-system/flat-file.ts +++ b/src/file-system/flat-file.ts @@ -1,6 +1,6 @@ import { FileBase } from './file-base'; import { FileStore } from './file-store'; -import { FileType } from './file-type'; +import { FileType } from '../config/file-type'; import { Archive } from './archive'; import { Group } from './group'; diff --git a/src/flat-file.ts b/src/flat-file.ts deleted file mode 100644 index 272a4f3..0000000 --- a/src/flat-file.ts +++ /dev/null @@ -1,124 +0,0 @@ -import { join } from 'path'; -import { existsSync, readFileSync } from 'graceful-fs'; -import { ByteBuffer, logger } from '@runejs/common'; - -import { FileIndexEntity } from './db'; -import { FileBreadcrumb, IndexedFile } from './indexed-file'; -import { FileState } from './file-state'; -import { isSet } from './util'; - - -export class FlatFile extends IndexedFile { - - public stripes: number[] = []; - public stripeCount: number = 1; - - public constructor(index: FileIndexEntity, breadcrumb?: Partial) { - super(index, breadcrumb); - - if(isSet(index.stripes)) { - this.stripes = index.stripes.split(',').map(n => Number(n)); - } - if(isSet(index.stripeCount)) { - this.stripeCount = index.stripeCount; - } - if(isSet(index.version)) { - this.version = index.version; - } - if(isSet(index.nameHash)) { - this.nameHash = index.nameHash; - } - } - - public override read(compress: boolean = false): ByteBuffer | null | Promise { - if(!this.group) { - throw new Error(`Flat file ${this.key} could not be read as it does not belong to any known groups.`); - } - - const filePath = this.path; - - if(!existsSync(filePath)) { - logger.error(`File not found: ${filePath}`); - this.setState(FileState.missing); - return null; - } - - let data: Buffer | null = null; - - try { - data = readFileSync(filePath); - } catch(error) { - logger.error(`Error reading file ${this.name || this.key } at ${filePath}:`, error); - data = null; - } - - if(!data) { - this.setState(FileState.corrupt); - } else if(!data.length) { - this.setState(FileState.empty); - } else { - const fileData = new ByteBuffer(data); - - if(!this.name) { - this.name = this.group.name || this.key; - } - - if(!this.nameHash) { - this.nameHash = this.group.nameHash || undefined; - } - - if(!this.stripes) { - const stripeStr = this.index.stripes; - if(stripeStr?.length) { - this.stripes = stripeStr.split(',')?.map(s => Number(s)) ?? undefined; - } - } - - if(!this.crc32) { - this.crc32 = this.index.crc32 ?? 0; - } - - if(!this.sha256) { - this.sha256 = this.index.sha256 ?? undefined; - } - - this.setData(fileData, FileState.raw); - - if(this.size !== this.index.size || this.sha256 !== this.generateSha256()) { - this._modified = true; - } - - return fileData; - } - - logger.error(`Error reading file data: ${filePath}`); - return null; - } - - public override get path(): string { - const groupPath = this.group?.path || null; - if(!groupPath) { - throw new Error(`Error generating file path; File ${this.key} has not been added to a group.`); - } - - if(this.group.fileCount === 1 || this.archive?.config?.flatten) { - return groupPath + this.type; - } else { - return join(groupPath, String(this.name || this.key)) + this.type; - } - } - - public override get outputPath(): string { - const groupOutputPath = this.group?.outputPath || null; - if(!groupOutputPath) { - throw new Error(`Error generating file output path; File ${this.key} has not been added to a group.`); - } - - if(this.group.fileCount === 1 || this.archive?.config?.flatten) { - return groupOutputPath + this.type; - } else { - return join(groupOutputPath, String(this.name || this.key) + this.type); - } - } - -} diff --git a/src/group.ts b/src/group.ts deleted file mode 100644 index 4c9331d..0000000 --- a/src/group.ts +++ /dev/null @@ -1,344 +0,0 @@ -import { join } from 'path'; -import { existsSync, mkdirSync, readdirSync, rmSync, statSync } from 'graceful-fs'; -import { ByteBuffer, logger } from '@runejs/common'; - -import { FlatFile } from './flat-file'; -import { GroupIndexEntity } from './db'; -import { FileBreadcrumb, IndexedFile } from './indexed-file'; -import { FileState } from './file-state'; -import { isSet } from './util'; - - -export class Group extends IndexedFile { - - public readonly files: Map = new Map(); - public readonly fileSizes: Map = new Map(); - - public stripes: number[] = []; - public stripeCount: number = 1; - - private _fileCount: number = 0; - - public constructor(index: GroupIndexEntity, breadcrumb?: Partial) { - super(index, breadcrumb); - - if(isSet(index.stripes)) { - this.stripes = index.stripes.split(',').map(n => Number(n)); - } - if(isSet(index.stripeCount)) { - this.stripeCount = index.stripeCount; - } - if(isSet(index.version)) { - this.version = index.version; - } - if(isSet(index.nameHash)) { - this.nameHash = index.nameHash; - } - - if(this.archive.config.groupNames) { - const nameEntries = Object.entries(this.archive.config.groupNames); - const namedEntry = nameEntries.find(entry => entry[1] === this.numericKey) || null; - if(namedEntry) { - this.name = namedEntry[0]; - } - } - } - - public override decode(): ByteBuffer | null { - this.unpack(); - this.decompress(); - - if(this._fileCount === 1) { - const flatFile: FlatFile = Array.from(this.files.values())[0]; - flatFile.name = this.name; - flatFile.nameHash = this.nameHash; - flatFile.sha256 = this.sha256; - flatFile.crc32 = this.crc32; - flatFile.encryption = this.encryption; - flatFile.setData(this._data, this.state); - } else { - const dataLength = this._data?.length || 0; - - if(!dataLength || dataLength <= 0) { - logger.error(`Error decoding group ${this.key}`); - return null; - } - - this._data.readerIndex = (dataLength - 1); // EOF - - this.stripeCount = this._data.get('byte', 'unsigned'); - - this._data.readerIndex = (dataLength - 1 - this.stripeCount * this.files.size * 4); // Stripe data footer - - if(this._data.readerIndex < 0) { - logger.error(`Invalid reader index of ${this._data.readerIndex} for group ${this.archive.name}:${this.key}.`); - return null; - } - - for(let stripe = 0; stripe < this.stripeCount; stripe++) { - let currentLength = 0; - for(const [ fileIndex, file ] of this.files) { - const delta = this._data.get('int'); - currentLength += delta; - - if(!file.stripes?.length) { - file.stripes = new Array(this.stripeCount); - } - - let size = 0; - if(!this.fileSizes.has(fileIndex)) { - this.fileSizes.set(fileIndex, 0); - } else { - size = this.fileSizes.get(fileIndex); - } - - file.stripes[stripe] = currentLength; - this.fileSizes.set(fileIndex, size + currentLength); - } - } - - for(const [ fileIndex, file ] of this.files) { - const fileSize = this.fileSizes.get(fileIndex) || 0; - file.setData(new ByteBuffer(fileSize), FileState.raw); - file.size = fileSize; - } - - this._data.readerIndex = 0; - - for(let stripe = 0; stripe < this.stripeCount; stripe++) { - for(const [ , file ] of this.files) { - let stripeLength = file.stripes[stripe]; - let sourceEnd: number = this._data.readerIndex + stripeLength; - - if(this._data.readerIndex + stripeLength >= this._data.length) { - sourceEnd = this._data.length; - stripeLength = (this._data.readerIndex + stripeLength) - this._data.length; - } - - const stripeData = this._data.getSlice(this._data.readerIndex, stripeLength); - - file.data.putBytes(stripeData); - - this._data.readerIndex = sourceEnd; - } - } - - this.files.forEach(file => file.generateSha256()); - } - - this.setData(this._data, FileState.raw); - this._fileCount = this.files.size; - return this._data ?? null; - } - - public override encode(): ByteBuffer | null { - // Single-file group - if(this._fileCount === 1) { - const flatFile = Array.from(this.files.values())[0]; - this.setData(flatFile.data ?? new ByteBuffer([]), FileState.encoded); - return this._data; - } - - // Multi-file group - const fileData: ByteBuffer[] = Array.from(this.files.values()).map(file => file?.data ?? new ByteBuffer(0)); - const fileSizes = fileData.map(data => data.length); - const fileCount = this._fileCount; - const stripeCount = this.stripes?.length ?? 1; - - if(!stripeCount) { - return null; - } - - // Size of all individual files + 1 int per file containing it's size - // + 1 at the end for the total group stripe count - const groupSize = fileSizes.reduce((a, c) => a + c) + (stripeCount * fileCount * 4) + 1; - const groupBuffer = new ByteBuffer(groupSize); - - fileData.forEach(data => data.readerIndex = 0); - - // Write file content stripes - for(let stripe = 0; stripe < stripeCount; stripe++) { - for(const [ , file ] of this.files) { - if(!file?.data?.length) { - continue; - } - - const stripeSize = file.stripes[stripe]; - - if(stripeSize) { - const stripeData = file.data.getSlice(file.data.readerIndex, stripeSize); - file.data.readerIndex = file.data.readerIndex + stripeSize; - groupBuffer.putBytes(stripeData); - } - } - } - - for(let stripe = 0; stripe < stripeCount; stripe++) { - let prevSize = 0; - for(const [ , file ] of this.files) { - if(!file?.data?.length) { - continue; - } - - const stripeSize = file.stripes[stripe] ?? 0; - groupBuffer.put(stripeSize - prevSize, 'int'); - prevSize = stripeSize; - } - } - - groupBuffer.put(this.stripes?.length ?? 1, 'byte'); - - this.setData(groupBuffer.flipWriter(), FileState.encoded); - return this._data; - } - - public override async read(compress: boolean = false, readDiskFiles: boolean = true): Promise { - if(!this.index) { - logger.error(`Error reading group ${this.name} files: Group is not indexed, please re-index the ` + - `${this.archive.name} archive.`); - return null; - } - - if(this.index.data?.length) { - this.setData(this.index.data, FileState.compressed); - } - - let indexedFiles = await this.index.files; - - if(!indexedFiles?.length) { - // Single file indexes are not stored to save on DB space and read/write times - // So if a group has no children, assume it is a single-file group and create a single index for it - const { name, nameHash, version, size, crc32, sha256, stripes, stripeCount, archive, state } = this; - indexedFiles = this.index.files = [ this.indexService.validateFile({ - numericKey: 0, name, nameHash, version, size, crc32, sha256, stripes, stripeCount, - group: this, archive - }) ]; - } - - let childFileCount = 1; - - const groupPath = this.path; - - if(this.archive.versioned) { - this.version = this.index.version; - } - - if(existsSync(groupPath) && statSync(groupPath).isDirectory()) { - childFileCount = readdirSync(groupPath).length ?? 1; - } - - if(indexedFiles.length !== childFileCount) { - this._modified = true; - } - - this._fileCount = childFileCount; - - this.files.clear(); - this.fileSizes.clear(); - - // Load the group's files - for(const fileIndexData of indexedFiles) { - const file = new FlatFile(fileIndexData, { - group: this, archive: this.archive, store: this.store - }); - - this.files.set(file.key, file); - this.fileSizes.set(file.key, fileIndexData.size); - } - - this.stripeCount = (this.index as GroupIndexEntity).stripeCount || 1; - - // Read the content of each file within the group - if(readDiskFiles) { - Array.from(this.files.values()).forEach(file => file.read(compress)); - - if(this._fileCount === 1) { - // Single file group, set the group data to match the flat file data - const file = this.files.get('0'); - this.setData(file.data, file.state); - } - } - - this.encode(); - - const originalDigest = this.sha256; - this.generateSha256(); - - if(this.sha256 && originalDigest !== this.sha256) { - // logger.info(`Detected changes in file ${this.archive.name}:${groupName}.`); - this.index.sha256 = this.sha256; - this._modified = true; - } - - if(compress && this.state !== FileState.compressed) { - this.compress(); - } - - return this._data; - } - - public override write(): void { - if(!this._fileCount) { - logger.error(`Error writing group ${this.name}: Group is empty.`); - return; - } - - // logger.info(`Writing group ${this.name}...`); - - const groupPath = this.outputPath; - - if(existsSync(groupPath)) { - rmSync(groupPath, { recursive: true, force: true }); - } - - if(this.files.size > 1 && !this.archive.config.flatten) { - mkdirSync(groupPath, { recursive: true }); - } - - if(!this.archive.config.flatten) { - Array.from(this.files.values()).forEach(file => file.write()); - } else { - super.write(); - } - } - - public has(fileIndex: string | number): boolean { - return this.files.has(String(fileIndex)); - } - - public get(fileIndex: string | number): FlatFile | null { - return this.files.get(String(fileIndex)) ?? null; - } - - public set(fileIndex: string | number, file: FlatFile): void { - this.files.set(String(fileIndex), file); - this._fileCount = this.files.size; - } - - public override get path(): string { - const archivePath = this.archive?.path || null; - if(!archivePath) { - throw new Error(`Error generating group path; Archive path not provided to group ${this.key}.`); - } - - return join(archivePath, String(this.name || this.key)); - } - - public override get outputPath(): string { - const archiveOutputPath = this.archive?.outputPath || null; - if(!archiveOutputPath) { - throw new Error(`Error generating group output path; Archive output path not provided to group ${this.key}.`); - } - - const groupPath = join(archiveOutputPath, String(this.name || this.key)); - return this.archive.config.flatten ? groupPath + this.type : groupPath; - } - - public get fileCount(): number { - return this._fileCount; - } - - public get type(): string { - return this.archive?.config?.contentType ?? ''; - } -} diff --git a/src/index.ts b/src/index.ts deleted file mode 100644 index 00d2b30..0000000 --- a/src/index.ts +++ /dev/null @@ -1,10 +0,0 @@ -export * from './indexed-file'; -export * from './flat-file'; -export * from './group'; -export * from './archive'; -export * from './store'; -export * from './file-state'; -export * from './util'; -export * from './db'; -export * from './config'; - diff --git a/src/indexed-file.ts b/src/indexed-file.ts deleted file mode 100644 index 0cd75d4..0000000 --- a/src/indexed-file.ts +++ /dev/null @@ -1,502 +0,0 @@ -import { writeFileSync } from 'graceful-fs'; -import { createHash } from 'crypto'; -import { ByteBuffer, logger } from '@runejs/common'; -import { Bzip2, CompressionMethod, getCompressionMethod, Gzip } from '@runejs/common/compress'; -import { EncryptionMethod, Xtea, XteaKeys } from '@runejs/common/encrypt'; -import { Crc32 } from '@runejs/common/crc32'; - -import { IndexEntity, IndexService } from './db'; -import { Store } from './store'; -import { Archive } from './archive'; -import { Group } from './group'; -import { isSet } from './util'; -import { FileState } from './file-state'; - - -export interface FileBreadcrumb { - store: Store; - archive: Archive; - group: Group; -} - - -export abstract class IndexedFile { - - public readonly key: string; - public readonly store: Store; - public readonly archive: Archive; - public readonly group: Group; - - public index: T; - public name: string = ''; - public nameHash: number = -1; - public version: number = 0; - public size: number = 0; - public crc32: number = -1; - public sha256: string = ''; - public encryption: EncryptionMethod | [ EncryptionMethod, string ] = 'none'; - public compression: CompressionMethod = 'none'; - public state: FileState = FileState.unloaded; - - protected _data: ByteBuffer | null = null; - protected _modified: boolean = false; - - protected constructor(index: T, breadcrumb?: Partial) { - this.index = index; - this.key = String(index.key); - - if(isSet(index.name)) { - this.name = index.name; - } - if(isSet(index.size)) { - this.size = index.size; - } - if(isSet(index.crc32)) { - this.crc32 = index.crc32; - } - if(isSet(index.sha256)) { - this.sha256 = index.sha256; - } - if(isSet(index['state'])) { - this.state = FileState[index['state']]; - } - - if(breadcrumb) { - const { store, archive, group } = breadcrumb; - - if(isSet(store)) { - this.store = store; - } - if(isSet(archive)) { - this.archive = archive; - } - if(isSet(group)) { - this.group = group; - } - } - - // Attempt to infer the archive or store that this file belongs to, if not provided in the options - - if(!this.archive) { - if(this.group?.archive) { - this.archive = this.group.archive; - } - } - - if(!this.store) { - if(this.archive?.store) { - this.store = this.archive.store; - } else if(this.group?.store) { - this.store = this.group.store; - } - } - - this.encryption = this.archive?.config?.encryption || 'none'; - this.compression = this.archive?.config?.compression || 'none'; - } - - public unpack(): ByteBuffer | null { - const archiveKey: number = this.archive ? this.archive.numericKey : 255; - const fileKey = this.numericKey; - const archiveName: string = this.archive ? this.archive.name : 'main'; - const indexChannel: ByteBuffer = archiveKey !== 255 ? - this.store.js5ArchiveIndexes.get(String(archiveKey)) : this.store.js5MainIndex; - - if(archiveKey === 255 && fileKey === 255) { - return null; - } - - const indexDataLength = 6; - const dataChannel = this.store.js5MainArchiveData; - - indexChannel.readerIndex = 0; - dataChannel.readerIndex = 0; - - let pointer = fileKey * indexDataLength; - - if(pointer < 0 || pointer >= indexChannel.length) { - logger.error(`File ${fileKey} was not found within the ${archiveName} archive index file.`); - return null; - } - - const fileIndexData = new ByteBuffer(indexDataLength); - indexChannel.copy(fileIndexData, 0, pointer, pointer + indexDataLength); - - if(fileIndexData.readable !== indexDataLength) { - logger.error(`Error extracting JS5 file ${fileKey}: the end of the data stream was reached.`); - return null; - } - - this.size = fileIndexData.get('int24', 'unsigned'); - const stripeCount = fileIndexData.get('int24', 'unsigned'); - - if(this.size <= 0) { - logger.warn(`Extracted JS5 file ${fileKey} has a recorded size of 0, no file data will be extracted.`); - return null; - } - - const data = new ByteBuffer(this.size); - const stripeDataLength = 512; - const stripeLength = 520; - - let stripe = 0; - let remaining = this.size; - pointer = stripeCount * stripeLength; - - do { - const temp = new ByteBuffer(stripeLength); - dataChannel.copy(temp, 0, pointer, pointer + stripeLength); - - if(temp.readable !== stripeLength) { - logger.error(`Error reading stripe for packed file ${fileKey}, the end of the data stream was reached.`); - return null; - } - - const stripeFileIndex = temp.get('short', 'unsigned'); - const currentStripe = temp.get('short', 'unsigned'); - const nextStripe = temp.get('int24', 'unsigned'); - const stripeArchiveIndex = temp.get('byte', 'unsigned'); - const stripeData = new ByteBuffer(stripeDataLength); - temp.copy(stripeData, 0, temp.readerIndex, temp.readerIndex + stripeDataLength); - - if(remaining > stripeDataLength) { - stripeData.copy(data, data.writerIndex, 0, stripeDataLength); - data.writerIndex = (data.writerIndex + stripeDataLength); - remaining -= stripeDataLength; - - if(stripeArchiveIndex !== archiveKey) { - logger.error(`Archive index mismatch, expected archive ${archiveKey} but found archive ${stripeFileIndex}`); - return null; - } - - if(stripeFileIndex !== fileKey) { - logger.error(`File index mismatch, expected ${fileKey} but found ${stripeFileIndex}.`); - return null; - } - - if(currentStripe !== stripe++) { - logger.error(`Error extracting JS5 file ${fileKey}, file data is corrupted.`); - return null; - } - - pointer = nextStripe * stripeLength; - } else { - stripeData.copy(data, data.writerIndex, 0, remaining); - data.writerIndex = (data.writerIndex + remaining); - remaining = 0; - } - } while(remaining > 0); - - if(data?.length) { - this.setData(data, FileState.compressed); - } else { - this.setData(null, FileState.missing); - } - - return data?.length ? data : null; - } - - public pack(): ByteBuffer | null { - return this._data; // @TODO needed for full re-packing of the data file - } - - public decode(): ByteBuffer | null { - return this._data; // stubbed - } - - public encode(): ByteBuffer | null { - return this._data; // stubbed - } - - public decompress(): ByteBuffer | null { - if(!this._data?.length) { - return null; - } - - this._data.readerIndex = 0; - - this.compression = getCompressionMethod(this._data.get('byte', 'unsigned')); - const compressedLength = this._data.get('int', 'unsigned'); - - const readerIndex = this._data.readerIndex; - - const compressedData = this.decrypt(); - compressedData.readerIndex = readerIndex; - let data: ByteBuffer; - - if(this.compression === 'none') { - // Uncompressed file - data = new ByteBuffer(compressedLength); - compressedData.copy(data, 0, compressedData.readerIndex, compressedLength); - compressedData.readerIndex = (compressedData.readerIndex + compressedLength); - } else { - // BZIP or GZIP compressed file - const decompressedLength = compressedData.get('int', 'unsigned'); - if(decompressedLength < 0) { - logger.error(this.encryption === 'xtea' ? `Missing or invalid XTEA key.` : - `Invalid decompressed file length: ${decompressedLength}`); - } else { - const decompressedData = new ByteBuffer(this.compression === 'bzip' ? - decompressedLength : (compressedData.length - compressedData.readerIndex + 2)); - - compressedData.copy(decompressedData, 0, compressedData.readerIndex); - - try { - data = this.compression === 'bzip' ? Bzip2.decompress(decompressedData) : Gzip.decompress(decompressedData); - - compressedData.readerIndex = compressedData.readerIndex + compressedLength; - - if(data.length !== decompressedLength) { - logger.error(`Compression length mismatch.`); - data = null; - } - } catch(error) { - if(this.state === FileState.encrypted) { - logger.error(`Unable to decrypt file ${this.name || this.key}.`); - this.archive?.incrementMissingEncryptionKeys(); - } else { - logger.error(`Unable to decompress file ${this.name || this.key}: ${error?.message ?? error}`); - } - data = null; - } - } - } - - // Read the file footer, if it has one - if(compressedData.readable >= 2) { - this.version = compressedData.get('short', 'unsigned'); - } - - if((data?.length ?? 0) > 0) { - this.setData(data, FileState.encoded); - } - - return this._data ?? null; - } - - public compress(): ByteBuffer | null { - if(!this._data?.length) { - return null; - } - - const decompressedData = this._data; - let data: ByteBuffer; - - if(this.compression === 'none') { - // uncompressed files - data = new ByteBuffer(decompressedData.length + 5); - - // indicate that no file compression is applied - data.put(0); - - // write the uncompressed file length - data.put(decompressedData.length, 'int'); - - // write the uncompressed file data - data.putBytes(decompressedData); - } else { - // compressed Bzip2 or Gzip file - - const compressedData: ByteBuffer = this.compression === 'bzip' ? - Bzip2.compress(decompressedData) : Gzip.compress(decompressedData); - - const compressedLength: number = compressedData.length; - - data = new ByteBuffer(compressedData.length + 9); - - // indicate which type of file compression was used (1 or 2) - data.put(this.compression === 'bzip' ? 1 : 2); - - // write the compressed file length - data.put(compressedLength, 'int'); - - // write the uncompressed file length - data.put(decompressedData.length, 'int'); - - // write the compressed file data - data.putBytes(compressedData); - } - - return this.setData(data, FileState.compressed); - } - - public decrypt(): ByteBuffer { - if(this.state === FileState.encrypted) { - // if(this.archive?.config?.encryption) { - // File name must match the given pattern to be encrypted - if(!this.name) { - throw new Error(`Error decrypting file ${this.key}: File name not found.`); - } - - if(Array.isArray(this.archive.config.encryption)) { - const [ encryption, pattern ] = this.archive.config.encryption; - const patternRegex = new RegExp(pattern); - - // Only XTEA encryption is supported for v1.0.0 - if(encryption !== 'xtea' || !patternRegex.test(this.name)) { - // File name does not match the pattern, data should be unencrypted - this.setState(FileState.decrypted); - return this._data; - } - } else if(this.archive.config.encryption !== 'xtea') { - // Only XTEA encryption is supported for v1.0.0 - this.setState(FileState.decrypted); - return this._data; - } - } - - const gameBuild = this.store.gameBuild ?? null; - - // XTEA requires that we know which game build is running so that we pick the correct keystore file - if(!gameBuild) { - if(this.store && !this.store.gameBuildMissing) { - this.store.setGameBuildMissing(); - logger.warn(`Game build must be supplied to decompress XTEA encrypted files.`, - `Please provide the game build using the --build argument.`); - } - - this.setState(FileState.decrypted); - return this._data; - } - - let keySets: XteaKeys[] = []; - - const loadedKeys = this.store.getEncryptionKeys(this.name); - if(loadedKeys) { - if(!Array.isArray(loadedKeys)) { - keySets = [ loadedKeys ]; - } else { - keySets = loadedKeys; - } - } - - this._data.readerIndex = 0; - - this.compression = getCompressionMethod(this._data.get('byte', 'unsigned')); - const compressedLength = this._data.get('int', 'unsigned'); - - const readerIndex = this._data.readerIndex; - - const keySet = keySets.find(keySet => keySet.gameBuild === gameBuild); - - if(Xtea.validKeys(keySet?.key)) { - const dataCopy = this._data.clone(); - dataCopy.readerIndex = readerIndex; - - let lengthOffset = readerIndex; - if(dataCopy.length - (compressedLength + readerIndex + 4) >= 2) { - lengthOffset += 2; - } - - const decryptedData = Xtea.decrypt(dataCopy, keySet.key, dataCopy.length - lengthOffset); - - if(decryptedData?.length) { - decryptedData.copy(dataCopy, readerIndex, 0); - dataCopy.readerIndex = readerIndex; - this.setState(FileState.decrypted); - return dataCopy; - } else { - logger.warn(`Invalid XTEA decryption keys found for file ${this.name || this.key} using game build ${gameBuild}.`); - } - } else { - // logger.warn(`No XTEA decryption keys found for file ${this.name || this.fileKey} using game build ${gameBuild}.`); - } - - return this._data; - } - - public read(compress?: boolean): ByteBuffer | null | Promise { - if(this.state === FileState.unloaded) { - this.setState(FileState.loaded); - } - - return this._data; - } - - public write(): void { - if(this._data?.length) { - writeFileSync(this.outputPath, Buffer.from(this._data)); - } - } - - public setData(data: Buffer, state: FileState): ByteBuffer | null; - public setData(data: ByteBuffer, state: FileState): ByteBuffer | null; - public setData(data: ByteBuffer | Buffer, state: FileState): ByteBuffer | null; - public setData(data: ByteBuffer | Buffer, state: FileState): ByteBuffer | null { - this.size = data?.length ?? 0; - - if(this.size) { - this._data = new ByteBuffer(data); - this._data.readerIndex = 0; - this._data.writerIndex = 0; - } else { - this._data = null; - } - - this.state = state; - - if(state === FileState.compressed) { - this.generateCrc32(); - } else if(state === FileState.raw || state === FileState.encoded) { - this.generateSha256(); - } - - return this._data; - } - - public generateCrc32(): number { - this.crc32 = this._data?.length ? Crc32.update(0, this.size, Buffer.from(this._data)) : -1; - return this.crc32; - } - - public generateSha256(): string { - this.sha256 = this._data?.length ? createHash('sha256') - .update(Buffer.from(this._data)).digest('hex') : ''; - return this.sha256; - } - - public setState(fileState: FileState): void { - this.state = fileState; - } - - public abstract get path(): string; - - public abstract get outputPath(): string; - - public get numericKey(): number { - return Number(this.key); - } - - public get named(): boolean { - if(!this.name) { - return false; - } - - return !/^\d+$/.test(this.name); - } - - public get hasNameHash(): boolean { - return isSet(this.nameHash) && !isNaN(this.nameHash) && this.nameHash !== -1; - } - - public get data(): ByteBuffer { - return this._data; - } - - public get indexService(): IndexService { - return this.store.indexService; - } - - public get type(): string { - return this.archive?.config?.contentType ?? ''; - } - - public get empty(): boolean { - return (this._data?.length ?? 0) !== 0; - } - - public get modified(): boolean { - return this.index.crc32 !== this.crc32 || this.index.sha256 !== this.sha256 || this.index.size !== this.size; - } - -} diff --git a/src/scripts/dev.ts b/src/scripts/dev.ts new file mode 100644 index 0000000..49118d7 --- /dev/null +++ b/src/scripts/dev.ts @@ -0,0 +1 @@ +// currently empty diff --git a/src/store.ts b/src/store.ts deleted file mode 100644 index a1656fc..0000000 --- a/src/store.ts +++ /dev/null @@ -1,526 +0,0 @@ -import { existsSync, mkdirSync, readdirSync, readFileSync, rmSync, statSync } from 'graceful-fs'; -import { join } from 'path'; -import JSON5 from 'json5'; -import { ByteBuffer, logger } from '@runejs/common'; -import { Xtea, XteaKeys } from '@runejs/common/encrypt'; -import { Crc32 } from '@runejs/common/crc32'; - -import { Archive, FileState } from './index'; -import { ArchiveIndexEntity, IndexService, StoreIndexEntity } from './db'; -import { ArchiveConfig } from './config'; - - -export type StoreFormat = 'unpacked' | 'packed'; - -export interface StoreOptions { - outputPath?: string | undefined; -} - - -export class Store { - - public readonly archives: Map = new Map(); - public readonly fileNameHashes: Map = new Map(); - public readonly indexService: IndexService; - - private _js5MainIndex: ByteBuffer; - private _js5ArchiveIndexes: Map; - private _js5MainArchiveData: ByteBuffer; - - private _index: StoreIndexEntity; - private _mainArchive: Archive; - private _data: ByteBuffer; - private _compressed: boolean = false; - private _js5Encoded: boolean = false; - private _path: string; - private _outputPath: string; - private _archiveConfig: { [key: string]: ArchiveConfig }; - private _encryptionKeys: Map; - private _gameBuild: string; - private _gameBuildMissing: boolean; - - protected constructor(gameBuild: string, path: string, outputPath?: string) { - this._gameBuild = gameBuild; - this._path = path; - this._outputPath = outputPath ? outputPath : join(path, 'unpacked'); - this.indexService = new IndexService(this); - this.loadArchiveConfig(); - Crc32.init(); - } - - public static async create(gameBuild: string, path: string = './', options?: StoreOptions): Promise { - const store = new Store(gameBuild, path, options?.outputPath); - - await store.indexService.load(); - - store._index = await store.indexService.getStoreIndex(); - - if(!store._index) { - store._index = new StoreIndexEntity(); - store._index.gameBuild = gameBuild; - } - - store.loadEncryptionKeys(); - store.loadFileNames(); - - store.archives.clear(); - - const archiveConfigs = Object.entries(store.archiveConfig); - const mainArchiveConfig = Array.from(Object.values(store.archiveConfig)).find(a => a.index === 255); - - if(!mainArchiveConfig) { - throw new Error(`Main archive (index 255) configuration was not found. ` + - `Please configure the main archive using the archives.json5 file within the store config directory.`) - } - - const mainArchiveIndex = new ArchiveIndexEntity(); - mainArchiveIndex.key = 255; - mainArchiveIndex.gameBuild = gameBuild; - mainArchiveIndex.name = 'main'; - store._mainArchive = new Archive(mainArchiveIndex, mainArchiveConfig, { store }); - - let archiveIndexes = await store.indexService.getArchiveIndexes(); - - if(!archiveIndexes?.length) { - archiveIndexes = new Array(archiveConfigs.length); - } - - for(const [ name, config ] of archiveConfigs) { - if(config.index === 255) { - continue; - } - - if(config.build) { - let revision: number; - if(gameBuild.includes('_')) { - revision = Number(gameBuild.substring(gameBuild.indexOf('_') + 1)); - } else { - revision = Number(gameBuild); - } - if(revision < config.build) { - // logger.info(`Skipping archive ${name} as it is not available in this game build.`); - continue; - } - } - - let archiveIndex = archiveIndexes.find(a => a?.key === config.index); - if(!archiveIndex) { - archiveIndex = store.indexService.validateArchive({ - numericKey: config.index, - name, - nameHash: store.hashFileName(name), - config - }); - } - - const archive = new Archive(archiveIndex, config, { - store, archive: store._mainArchive - }); - - store.archives.set(archive.key, archive); - - // Bulk-fetch the archive's groups - const groups = archiveIndex.groups = await store.indexService.getGroupIndexes(archiveIndex); - - // Bulk-fetch the archive's files and sort them into the appropriate groups - const archiveFileIndexes = await store.indexService.getFileIndexes(archiveIndex); - for(const fileIndex of archiveFileIndexes) { - const group = groups.find(group => group.key === fileIndex.groupKey); - if(!group) { - continue; - } - - if(!Array.isArray(group.files) || !group.files?.length) { - group.files = [ fileIndex ]; - } else { - group.files.push(fileIndex); - } - } - } - - return store; - } - - public loadPackedStore(): void { - const js5StorePath = join(this.path, 'packed'); - - if(!existsSync(js5StorePath)) { - throw new Error(`${js5StorePath} could not be found.`); - } - - const stats = statSync(js5StorePath); - if(!stats?.isDirectory()) { - throw new Error(`${js5StorePath} is not a valid directory.`); - } - - const storeFileNames = readdirSync(js5StorePath); - const dataFile = 'main_file_cache.dat2'; - const mainIndexFile = 'main_file_cache.idx255'; - - if(storeFileNames.indexOf(dataFile) === -1) { - throw new Error(`The main ${dataFile} data file could not be found.`); - } - - if(storeFileNames.indexOf(mainIndexFile) === -1) { - throw new Error(`The main ${mainIndexFile} index file could not be found.`); - } - - const indexFilePrefix = 'main_file_cache.idx'; - const dataFilePath = join(js5StorePath, dataFile); - const mainIndexFilePath = join(js5StorePath, mainIndexFile); - - this._js5MainArchiveData = new ByteBuffer(readFileSync(dataFilePath)); - this._js5MainIndex = new ByteBuffer(readFileSync(mainIndexFilePath)); - this._js5ArchiveIndexes = new Map(); - - for(const fileName of storeFileNames) { - if(!fileName?.length || fileName === mainIndexFile || fileName === dataFile) { - continue; - } - - if(!fileName.startsWith(indexFilePrefix)) { - continue; - } - - const index = fileName.substring(fileName.indexOf('.idx') + 4); - const numericIndex = Number(index); - - if(isNaN(numericIndex)) { - logger.error(`Index file ${fileName} does not have a valid extension.`); - } - - if(!this.has(index)) { - logger.warn(`Archive ${index} was found within the JS5 store, but is not configured for flat file store use.`, - `Please add the archive to the archives.json5 configuration file to load it properly.`); - continue; - } - - this._js5ArchiveIndexes.set(index, new ByteBuffer(readFileSync(join(js5StorePath, fileName)))); - } - - logger.info(`Packed store loaded for game build ${this.gameBuild}.`); - } - - public pack(): void { - // @TODO - } - - public decode(decodeGroups: boolean = true): ByteBuffer | null { - this.archives.forEach(archive => archive.decode(decodeGroups)); - return null; - } - - public encode(encodeGroups: boolean = true): ByteBuffer { - const fileSize = 4 * this.archiveCount; - - this._data = new ByteBuffer(fileSize + 31); - - this._data.put(0); - this._data.put(fileSize, 'int'); - - for(let archiveIndex = 0; archiveIndex < this.archiveCount; archiveIndex++) { - this._data.put(this.get(archiveIndex).index.crc32, 'int'); - } - - this.mainArchive.setData(this._data, FileState.encoded); - this.mainArchive.index.data = this.index.data = this._data.toNodeBuffer(); - - if(encodeGroups) { - this.archives.forEach(archive => archive.encode(true)); - } - - return this.data; - } - - public compress(compressGroups: boolean = true): ByteBuffer | null { - this.archives.forEach(archive => archive.compress(compressGroups)); - - this._compressed = true; - return this._data; - } - - public async read(compress: boolean = false, readDiskFiles: boolean = true): Promise { - this._js5Encoded = false; - this._compressed = false; - - for(const [ , archive ] of this.archives) { - await archive.read(false, readDiskFiles); - } - - if(compress) { - this.compress(); - } - - return this.encode(); - } - - public write(): void { - if(!this.archives.size) { - throw new Error(`Archives not loaded, please load a flat file store or a JS5 store.`); - } - - const start = Date.now(); - logger.info(`Writing flat file store...`); - - try { - if(existsSync(this.outputPath)) { - rmSync(this.outputPath, { recursive: true, force: true }); - } - - mkdirSync(this.outputPath, { recursive: true }); - } catch(error) { - logger.error(`Error clearing file store output path (${this.outputPath}):`, error); - return; - } - - try { - logger.info(`Writing archive contents to disk...`); - this.archives.forEach(archive => archive.write()); - logger.info(`Archives written.`); - } catch(error) { - logger.error(`Error writing archives:`, error); - return; - } - - const end = Date.now(); - logger.info(`Flat file store written in ${(end - start) / 1000} seconds.`); - } - - public async saveIndexData(saveArchives: boolean = true, saveGroups: boolean = true, saveFiles: boolean = true): Promise { - try { - await this.indexService.saveStoreIndex(); - logger.info(`File store index saved.`); - } catch(error) { - logger.error(`Error indexing file store:`, error); - return; - } - - if(saveArchives) { - logger.info(`Indexing archives...`); - - for(const [ , archive ] of this.archives) { - try { - await archive.saveIndexData(false); - } catch(error) { - logger.error(`Error indexing archive:`, error); - return; - } - } - } - - if(saveGroups) { - logger.info(`Indexing archive groups...`); - - for(const [ , archive ] of this.archives) { - try { - await archive.saveGroupIndexes(false); - } catch(error) { - logger.error(`Error indexing group:`, error); - return; - } - } - } - - if(saveFiles) { - logger.info(`Indexing archive files...`); - - for(const [ , archive ] of this.archives) { - try { - await archive.saveFlatFileIndexes(); - } catch(error) { - logger.error(`Error indexing flat file:`, error); - return; - } - } - } - } - - public find(archiveName: string): Archive { - return Array.from(this.archives.values()).find(child => child?.name === archiveName) ?? null; - } - - public get(archiveKey: string): Archive | null; - public get(archiveKey: number): Archive | null; - public get(archiveKey: string | number): Archive | null; - public get(archiveKey: string | number): Archive | null { - return this.archives.get(String(archiveKey)) ?? null; - } - - public set(archiveKey: string, archive: Archive): void; - public set(archiveKey: number, archive: Archive): void; - public set(archiveKey: string | number, archive: Archive): void; - public set(archiveKey: string | number, archive: Archive): void { - this.archives.set(String(archiveKey), archive); - } - - public has(archiveKey: string | number): boolean { - return this.archives.has(String(archiveKey)); - } - - public loadArchiveConfig(): void { - const configPath = join(this.path, 'config', 'archives.json5'); - if(!existsSync(configPath)) { - logger.error(`Error loading store: ${configPath} was not found.`); - return; - } - - this._archiveConfig = JSON5.parse(readFileSync(configPath, 'utf-8')) as { [key: string]: ArchiveConfig }; - - if(!Object.values(this._archiveConfig)?.length) { - throw new Error(`Error reading archive configuration file. ` + - `Please ensure that the ${configPath} file exists and is valid.`); - } - } - - public getEncryptionKeys(fileName: string): XteaKeys | XteaKeys[] | null { - if(!this.encryptionKeys.size) { - this.loadEncryptionKeys(); - } - - const keySets = this.encryptionKeys.get(fileName); - if(!keySets) { - return null; - } - - if(this.gameBuild !== undefined) { - return keySets.find(keySet => keySet.gameBuild === this.gameBuild) ?? null; - } - - return keySets; - } - - public loadEncryptionKeys(): void { - const configPath = join(this.path, 'config', 'xtea'); - this._encryptionKeys = Xtea.loadKeys(configPath); - - if(!this.encryptionKeys.size) { - throw new Error(`Error reading encryption key lookup table. ` + - `Please ensure that the ${configPath} file exists and is valid.`); - } - } - - public hashFileName(fileName: string): number { - if(!fileName) { - return 0; - } - - let hash = 0; - for(let i = 0; i < fileName.length; i++) { - hash = fileName.charCodeAt(i) + ((hash << 5) - hash); - } - - return hash | 0; - } - - public findFileName(nameHash: string | number | undefined, defaultName?: string | undefined): string | undefined { - if(!this.fileNameHashes.size) { - this.loadFileNames(); - } - - if(nameHash === undefined || nameHash === null) { - return defaultName; - } - - if(typeof nameHash === 'string') { - nameHash = Number(nameHash); - } - - if(isNaN(nameHash) || nameHash === -1 || nameHash === 0) { - return defaultName; - } - - return this.fileNameHashes.get(nameHash) || defaultName; - } - - public loadFileNames(): void { - const configPath = join(this.path, 'config', 'name-hashes.json'); - if(!existsSync(configPath)) { - logger.error(`Error loading file names: ${configPath} was not found.`); - return; - } - - const nameTable = JSON.parse(readFileSync(configPath, 'utf-8')) as { [key: string]: string }; - Object.keys(nameTable).forEach(nameHash => this.fileNameHashes.set(Number(nameHash), nameTable[nameHash])); - - if(!this.fileNameHashes.size) { - throw new Error(`Error reading file name lookup table. ` + - `Please ensure that the ${configPath} file exists and is valid.`); - } - } - - public setGameBuildMissing(): void { - this._gameBuildMissing = true; - } - - public get archiveCount(): number { - return this.archives?.size || 0; - } - - public get js5MainIndex(): ByteBuffer { - if(!this._js5MainIndex?.length || !this._js5MainArchiveData?.length) { - this.decode(); - } - - return this._js5MainIndex; - } - - public get js5ArchiveIndexes(): Map { - if(!this._js5MainIndex?.length || !this._js5MainArchiveData?.length) { - this.decode(); - } - - return this._js5ArchiveIndexes; - } - - public get js5MainArchiveData(): ByteBuffer { - if(!this._js5MainIndex?.length || !this._js5MainArchiveData?.length) { - this.decode(); - } - - return this._js5MainArchiveData; - } - - public get index(): StoreIndexEntity { - return this._index; - } - - public get mainArchive(): Archive { - return this._mainArchive; - } - - public get data(): ByteBuffer { - return this._data; - } - - public get compressed(): boolean { - return this._compressed; - } - - public get js5Encoded(): boolean { - return this._js5Encoded; - } - - public get path(): string { - return this._path; - } - - public get outputPath(): string { - return this._outputPath; - } - - public get gameBuild(): string { - return this._gameBuild; - } - - public get archiveConfig(): { [p: string]: ArchiveConfig } { - return this._archiveConfig; - } - - public get encryptionKeys(): Map { - return this._encryptionKeys; - } - - public get gameBuildMissing(): boolean { - return this._gameBuildMissing; - } -} diff --git a/src/util/index.ts b/src/util/index.ts deleted file mode 100644 index 32737a8..0000000 --- a/src/util/index.ts +++ /dev/null @@ -1,2 +0,0 @@ -export const isSet = (variable: any): boolean => - variable !== undefined && variable !== null; From 108a79813b8750e2a3ae5e444b69483dbda2c71a Mon Sep 17 00:00:00 2001 From: Kikorono Date: Tue, 12 Jul 2022 18:34:46 -0500 Subject: [PATCH 10/32] Changing the packed/ dir to js5/ and the indexes/ dir to index/ --- .gitignore | 7 ++++--- {packed => js5}/.gitkeep | 0 src/db/index-database.ts | 2 +- src/db/{index.entity.ts => index-entity.ts} | 0 src/file-system/file-base.ts | 2 +- src/file-system/file-store.ts | 2 +- 6 files changed, 7 insertions(+), 6 deletions(-) rename {packed => js5}/.gitkeep (100%) rename src/db/{index.entity.ts => index-entity.ts} (100%) diff --git a/.gitignore b/.gitignore index 0fe1b64..00bd5a4 100644 --- a/.gitignore +++ b/.gitignore @@ -1,10 +1,11 @@ /unpacked/ -packed/* +js5/* /lib/ /temp/ /tmp/ config/xtea/* -/indexes/ +indexes/* +index/* /logs/ @@ -26,4 +27,4 @@ yarn-error.log* *.sln *.sw? -!packed/.gitkeep \ No newline at end of file +!js5/.gitkeep diff --git a/packed/.gitkeep b/js5/.gitkeep similarity index 100% rename from packed/.gitkeep rename to js5/.gitkeep diff --git a/src/db/index-database.ts b/src/db/index-database.ts index da6e29d..2700c8a 100644 --- a/src/db/index-database.ts +++ b/src/db/index-database.ts @@ -1,7 +1,7 @@ import { Connection, createConnection, LoggerOptions, Repository } from 'typeorm'; import { join } from 'path'; import { existsSync, mkdirSync } from 'graceful-fs'; -import { IndexEntity } from './index.entity'; +import { IndexEntity } from './index-entity'; import { FileType } from '../config/file-type'; diff --git a/src/db/index.entity.ts b/src/db/index-entity.ts similarity index 100% rename from src/db/index.entity.ts rename to src/db/index-entity.ts diff --git a/src/file-system/file-base.ts b/src/file-system/file-base.ts index 0b71085..09c8864 100644 --- a/src/file-system/file-base.ts +++ b/src/file-system/file-base.ts @@ -1,4 +1,4 @@ -import { IndexEntity } from '../db/index.entity'; +import { IndexEntity } from '../db/index-entity'; import { FileStore } from './file-store'; import { FileType } from '../config/file-type'; diff --git a/src/file-system/file-store.ts b/src/file-system/file-store.ts index efb3234..47d127a 100644 --- a/src/file-system/file-store.ts +++ b/src/file-system/file-store.ts @@ -127,7 +127,7 @@ export class FileStore { } async openDatabase(): Promise { - this._database = new IndexDatabase(this.gameBuild, join(this.fileStorePath, 'indexes')); + this._database = new IndexDatabase(this.gameBuild, join(this.fileStorePath, 'index')); await this._database.openConnection(); return this._database; } From 9fbe8eb8864ed0225e2e1fb17690db90ae1fe72b Mon Sep 17 00:00:00 2001 From: Kikorono Date: Wed, 13 Jul 2022 19:55:55 -0500 Subject: [PATCH 11/32] Getting the basic js5 indexing system working and writing to SQLite DB --- config/archives.json5 | 66 +++++++++---------- src/config/archive-config.ts | 2 +- src/db/index-database.ts | 5 ++ src/db/index-entity.ts | 4 +- src/file-system/archive.ts | 7 ++- src/file-system/file-base.ts | 6 ++ src/file-system/file-store.ts | 39 ++++++++++-- src/file-system/js5.ts | 115 +++++++++++++++++++++------------- src/scripts/dev.ts | 74 +++++++++++++++++++++- 9 files changed, 231 insertions(+), 87 deletions(-) diff --git a/config/archives.json5 b/config/archives.json5 index 226ef3f..4bdc993 100644 --- a/config/archives.json5 +++ b/config/archives.json5 @@ -1,25 +1,25 @@ { // The main index of indexes main: { - index: 255, - compression: 'uncompressed', + key: 255, + compression: 'none', versioned: false }, // Regular indexes anims: { - index: 0, + key: 0, compression: 'bzip', versioned: true, flatten: true }, bases: { - index: 1, + key: 1, compression: 'gzip', versioned: true }, config: { - index: 2, + key: 2, compression: 'bzip', versioned: true, groupNames: { @@ -44,62 +44,62 @@ } }, interfaces: { - index: 3, + key: 3, compression: 'bzip', versioned: true, flatten: true }, synth_sounds: { - index: 4, + key: 4, compression: 'gzip', versioned: true, contentType: '.wav' }, maps: { - index: 5, + key: 5, compression: 'gzip', encryption: [ 'xtea', '^l[0-9]{1,3}_[0-9]{1,3}$' ], versioned: true, filesNamed: true }, midi_songs: { - index: 6, + key: 6, compression: 'gzip', versioned: false, contentType: '.mid', filesNamed: true }, models: { - index: 7, + key: 7, compression: 'gzip', versioned: true, contentType: '.dat' }, sprites: { - index: 8, + key: 8, compression: 'gzip', versioned: true, filesNamed: true }, textures: { - index: 9, + key: 9, compression: 'gzip', versioned: true }, binary: { - index: 10, - compression: 'uncompressed', + key: 10, + compression: 'none', versioned: true, filesNamed: true }, midi_jingles: { - index: 11, + key: 11, compression: 'gzip', versioned: true, contentType: '.mid' }, clientscripts: { - index: 12, + key: 12, compression: 'gzip', versioned: true, contentType: '.cs2', @@ -107,104 +107,104 @@ build: 435 }, fontmetrics: { - index: 13, + key: 13, compression: 'gzip', versioned: false, filesNamed: true, build: 443 }, vorbis: { - index: 14, + key: 14, compression: 'gzip', versioned: true, build: 451 }, midi_instruments: { - index: 15, + key: 15, compression: 'gzip', versioned: false, build: 451 }, config_loc: { - index: 16, + key: 16, compression: 'gzip', versioned: false, build: 489 }, config_enum: { - index: 17, + key: 17, compression: 'gzip', versioned: false, build: 489 }, config_npc: { - index: 18, + key: 18, compression: 'gzip', versioned: false, build: 489 }, config_obj: { - index: 19, + key: 19, compression: 'gzip', versioned: false, build: 489 }, config_seq: { - index: 20, + key: 20, compression: 'gzip', versioned: false, build: 489 }, config_spot: { - index: 21, + key: 21, compression: 'gzip', versioned: false, build: 489 }, config_var_bit: { - index: 22, + key: 22, compression: 'gzip', versioned: false, build: 489 }, worldmapdata: { - index: 23, + key: 23, compression: 'gzip', versioned: false, build: 493 }, quickchat: { - index: 24, + key: 24, compression: 'gzip', versioned: false, build: 498 }, quickchat_global: { - index: 25, + key: 25, compression: 'gzip', versioned: false, build: 498 }, materials: { - index: 26, + key: 26, compression: 'gzip', versioned: false, build: 500 }, config_particle: { - index: 27, + key: 27, compression: 'gzip', versioned: false, build: 523 }, defaults: { - index: 28, + key: 28, compression: 'gzip', versioned: false, build: 537 }, billboards: { - index: 29, + key: 29, compression: 'gzip', versioned: false, build: 582 diff --git a/src/config/archive-config.ts b/src/config/archive-config.ts index d2a5694..9c3ef47 100644 --- a/src/config/archive-config.ts +++ b/src/config/archive-config.ts @@ -3,7 +3,7 @@ import { EncryptionMethod } from '@runejs/common/encrypt'; export interface ArchiveConfig { - index: number; + key: number; name: string; versioned?: boolean; compression?: CompressionMethod; diff --git a/src/db/index-database.ts b/src/db/index-database.ts index 2700c8a..a8143b1 100644 --- a/src/db/index-database.ts +++ b/src/db/index-database.ts @@ -40,6 +40,11 @@ export class IndexDatabase { return this._connection; } + // @todo bulk save - 07/13/22 - Kiko + async saveIndex(indexEntity: IndexEntity): Promise { + return await this.repository.save(indexEntity); + } + async getIndex(fileType: FileType, key: number, parentKey: number): Promise { return await this.repository.findOne({ where: { fileType, key, parentKey } diff --git a/src/db/index-entity.ts b/src/db/index-entity.ts index ca130cc..ce4545b 100644 --- a/src/db/index-entity.ts +++ b/src/db/index-entity.ts @@ -67,8 +67,8 @@ export class IndexEntity { @Column('boolean', { nullable: true, default: false }) encrypted: boolean = false; - @Column('integer', { name: 'stripe_count', nullable: false, default: 1 }) - stripeCount: number = 1; + @Column('integer', { name: 'stripe_count', nullable: true, default: null }) + stripeCount: number = null; @Column('text', { nullable: true, default: null }) stripes: string | null = null; diff --git a/src/file-system/archive.ts b/src/file-system/archive.ts index a030c38..3310822 100644 --- a/src/file-system/archive.ts +++ b/src/file-system/archive.ts @@ -1,6 +1,7 @@ import { FileBase } from './file-base'; import { FileStore } from './file-store'; import { Group } from './group'; +import { CompressionMethod } from '@runejs/common/compress'; export class Archive extends FileBase { @@ -10,8 +11,12 @@ export class Archive extends FileBase { constructor( fileStore: FileStore, key: number, + name: string, + indexFileCompressionMethod: CompressionMethod = 'none', ) { - super(fileStore, key, -1, 'ARCHIVE'); + super(fileStore, key, 255, 'ARCHIVE'); + this.index.name = name; + this.index.compressionMethod = indexFileCompressionMethod; this.groups = new Map(); } diff --git a/src/file-system/file-base.ts b/src/file-system/file-base.ts index 09c8864..af74d9a 100644 --- a/src/file-system/file-base.ts +++ b/src/file-system/file-base.ts @@ -17,11 +17,17 @@ export class FileBase { ) { this.fileStore = fileStore; this.index = new IndexEntity(); + this.index.gameBuild = fileStore.gameBuild; this.index.fileType = fileType; this.index.key = key; this.index.parentKey = parentKey; } + async saveIndex(): Promise { + this.index = await this.fileStore.database.saveIndex(this.index); + return this.index; + } + async loadIndex(): Promise { const indexEntity = await this.fileStore.database.getIndex( this.index.fileType, this.index.key, this.index.parentKey diff --git a/src/file-system/file-store.ts b/src/file-system/file-store.ts index 47d127a..b482401 100644 --- a/src/file-system/file-store.ts +++ b/src/file-system/file-store.ts @@ -11,6 +11,7 @@ import { Archive } from './archive'; export class FileStore { readonly gameBuild: string; + readonly vanillaBuild: boolean; readonly fileStorePath: string; readonly fileNameHashes: Map; readonly js5: JS5; @@ -19,8 +20,10 @@ export class FileStore { private _archiveConfig: { [key: string]: ArchiveConfig }; private _database: IndexDatabase; - constructor(gameBuild: string, storePath: string = './') { - this.gameBuild = gameBuild; + constructor(gameBuild: string | number, storePath: string = './') { + this.gameBuild = String(gameBuild); + // @todo make `vanillaBuild` obsolete via auto-detection - 07/13/22 - Kiko + this.vanillaBuild = typeof gameBuild === 'number'; this.fileStorePath = storePath; this.fileNameHashes = new Map(); this.archives = new Map(); @@ -30,13 +33,33 @@ export class FileStore { } async load(): Promise { + await this.openDatabase(); + const archiveNames = Object.keys(this._archiveConfig); for (const archiveName of archiveNames) { const archiveConfig = this._archiveConfig[archiveName]; - if (!this.archives.has(archiveConfig.index)) { - const archive = new Archive(this, archiveConfig.index); - this.archives.set(archiveConfig.index, archive); + + if (archiveConfig.key === 255) { + continue; + } + + if (this.vanillaBuild && archiveConfig.build) { + const buildNumber = Number(this.gameBuild); + + if (buildNumber < archiveConfig.build) { + continue; + } + } + + if (!this.archives.has(archiveConfig.key)) { + const archive = new Archive( + this, + archiveConfig.key, + archiveName, + archiveConfig.compression || 'none' + ); + this.archives.set(archiveConfig.key, archive); await archive.loadIndex(); } } @@ -127,7 +150,11 @@ export class FileStore { } async openDatabase(): Promise { - this._database = new IndexDatabase(this.gameBuild, join(this.fileStorePath, 'index')); + this._database = new IndexDatabase( + this.gameBuild, + join(this.fileStorePath, 'index'), + [ 'error', 'warn' ], + ); await this._database.openConnection(); return this._database; } diff --git a/src/file-system/js5.ts b/src/file-system/js5.ts index 28b9255..88f8648 100644 --- a/src/file-system/js5.ts +++ b/src/file-system/js5.ts @@ -1,26 +1,29 @@ +import { join } from 'path'; +import { existsSync, readdirSync, readFileSync, statSync } from 'graceful-fs'; import { ByteBuffer, logger } from '@runejs/common'; import { Bzip2, getCompressionMethod, Gzip } from '@runejs/common/compress'; import { Xtea, XteaKeys } from '@runejs/common/encrypt'; +import { archiveFlags } from '../config/archive-flags'; import { Group } from './group'; import { Archive } from './archive'; import { FileStore } from './file-store'; import { ArchiveFormat } from '../config'; -import { archiveFlags } from '../config/archive-flags'; import { FlatFile } from './flat-file'; -import { join } from 'path'; -import { existsSync, readdirSync, readFileSync, statSync } from 'graceful-fs'; export class JS5 { readonly fileStore: FileStore; + encryptionKeys: Map; + private mainIndex: ByteBuffer; private archiveIndexes: Map; private mainArchiveData: ByteBuffer; constructor(fileStore: FileStore) { this.fileStore = fileStore; + this.loadEncryptionKeys(); } loadJS5Store(): void { @@ -112,10 +115,10 @@ export class JS5 { } fileDetails.fileSize = fileIndexData.get('int24', 'unsigned'); - fileDetails.stripeCount = fileIndexData.get('int24', 'unsigned'); + const stripeCount = fileIndexData.get('int24', 'unsigned'); if (fileDetails.fileSize <= 0) { - logger.warn(`Extracted JS5 file ${fileKey} has a recorded size of 0, no file data will be extracted.`); + logger.warn(`JS5 file ${fileKey} is empty or has been removed.`); return null; } @@ -125,13 +128,13 @@ export class JS5 { let stripe = 0; let remaining = fileDetails.fileSize; - pointer = fileDetails.stripeCount * stripeLength; + pointer = stripeCount * stripeLength; do { const temp = new ByteBuffer(stripeLength); dataChannel.copy(temp, 0, pointer, pointer + stripeLength); - if(temp.readable !== stripeLength) { + if (temp.readable !== stripeLength) { logger.error(`Error reading stripe for packed file ${fileKey}, the end of the data stream was reached.`); return null; } @@ -143,22 +146,22 @@ export class JS5 { const stripeData = new ByteBuffer(stripeDataLength); temp.copy(stripeData, 0, temp.readerIndex, temp.readerIndex + stripeDataLength); - if(remaining > stripeDataLength) { + if (remaining > stripeDataLength) { stripeData.copy(data, data.writerIndex, 0, stripeDataLength); data.writerIndex = (data.writerIndex + stripeDataLength); remaining -= stripeDataLength; - if(stripeArchiveIndex !== archiveKey) { + if (stripeArchiveIndex !== archiveKey) { logger.error(`Archive index mismatch, expected archive ${archiveKey} but found archive ${stripeFileIndex}`); return null; } - if(stripeFileIndex !== fileKey) { + if (stripeFileIndex !== fileKey) { logger.error(`File index mismatch, expected ${fileKey} but found ${stripeFileIndex}.`); return null; } - if(currentStripe !== stripe++) { + if (currentStripe !== stripe++) { logger.error(`Error extracting JS5 file ${fileKey}, file data is corrupted.`); return null; } @@ -212,27 +215,23 @@ export class JS5 { return null; } - if (!fileDetails.encrypted) { + // @todo move to JS5.decodeArchive + const archiveName = file instanceof Archive ? 'main' : file.archive.index.name; + const archiveConfig = this.fileStore.archiveConfig[archiveName]; + + if (archiveConfig.encryption) { + const [ encryption, pattern ] = archiveConfig.encryption; + const patternRegex = new RegExp(pattern); + + // Only XTEA encryption is supported at this time + if(encryption !== 'xtea' || !patternRegex.test(fileName)) { + // FileBase name does not match the pattern, data should be unencrypted + return fileDetails.compressedData; + } + } else { return fileDetails.compressedData; } - // @todo move to JS5.decodeArchive - // const archiveName = file instanceof Archive ? 'main' : file.archive.index.name; - // const archiveConfig = this.fileStore.archiveConfig[archiveName]; - // - // if (archiveConfig.encryption) { - // const [ encryption, pattern ] = archiveConfig.encryption; - // const patternRegex = new RegExp(pattern); - // - // // Only XTEA encryption is supported at this time - // if(encryption !== 'xtea' || !patternRegex.test(fileName)) { - // // FileBase name does not match the pattern, data should be unencrypted - // return fileDetails.compressedData; - // } - // } else { - // return fileDetails.compressedData; - // } - const gameBuild = this.fileStore.gameBuild; let keySets: XteaKeys[] = []; @@ -265,6 +264,7 @@ export class JS5 { dataCopy.readerIndex = readerIndex; fileDetails.compressedData = dataCopy.toNodeBuffer(); fileDetails.encrypted = false; + return fileDetails.compressedData; } else { logger.warn(`Invalid XTEA decryption keys found for file ` + `${fileName || fileDetails.key} using game build ${ gameBuild }.`); @@ -276,7 +276,7 @@ export class JS5 { fileDetails.fileError = 'MISSING_ENCRYPTION_KEYS'; } - return fileDetails.compressedData; + return null; } decompress(file: Group | Archive): Buffer | null { @@ -286,7 +286,8 @@ export class JS5 { return null; } - if (fileDetails.encrypted && !this.decrypt(file)) { + if (!this.decrypt(file)) { + fileDetails.encrypted = true; return null; } @@ -330,19 +331,19 @@ export class JS5 { data = null; } } catch (error) { - logger.error(`Error decompressing file ${fileDetails.name || fileDetails.key}:`, - error?.message ?? error); + logger.error(`Error decompressing file ${fileDetails.name || fileDetails.key}: ` + + `${error?.message ?? error}`); data = null; } } } - // Read the file footer, if it has one - if(compressedData.readable >= 2) { - fileDetails.version = compressedData.get('short', 'unsigned'); - } - if (data?.length) { + // Read the file footer, if it has one + if(compressedData.readable >= 2) { + fileDetails.version = compressedData.get('short', 'unsigned'); + } + fileDetails.data = data?.toNodeBuffer(); } @@ -484,7 +485,8 @@ export class JS5 { // Load group database indexes, or create them if they don't yet exist // @todo batch load all archive groups at once for (const group of groups) { - await group.loadIndex(); + // @todo removed for faster unpacking testing - 07/13/22 - Kiko + // await group.loadIndex(); } // Group name hashes @@ -553,7 +555,8 @@ export class JS5 { // @todo batch load all grouped files at once for (const group of groups) { for (const [ , flatFile ] of group.files) { - await flatFile.loadIndex(); + // @todo removed for faster unpacking testing - 07/13/22 - Kiko + // await flatFile.loadIndex(); } } @@ -571,6 +574,10 @@ export class JS5 { } } + decodeMainIndex(): Buffer | null { + return null; // @todo stub + } + pack(file: Group | Archive): Buffer | null { return null; // @todo stub } @@ -656,9 +663,31 @@ export class JS5 { return this.mainIndex; } - getEncryptionKeys(fileName: string): any[] { - // @todo pull key files from openrs2.org - return []; // @todo stub + getEncryptionKeys(fileName: string): XteaKeys | XteaKeys[] | null { + if(!this.encryptionKeys.size) { + this.loadEncryptionKeys(); + } + + const keySets = this.encryptionKeys.get(fileName); + if(!keySets) { + return null; + } + + if(this.fileStore.gameBuild !== undefined) { + return keySets.find(keySet => keySet.gameBuild === this.fileStore.gameBuild) ?? null; + } + + return keySets; + } + + loadEncryptionKeys(): void { + const configPath = join(this.fileStore.fileStorePath, 'config', 'xtea'); + this.encryptionKeys = Xtea.loadKeys(configPath); + + if(!this.encryptionKeys.size) { + throw new Error(`Error reading encryption key lookup table. ` + + `Please ensure that the ${configPath} file exists and is valid.`); + } } } diff --git a/src/scripts/dev.ts b/src/scripts/dev.ts index 49118d7..d2fe83c 100644 --- a/src/scripts/dev.ts +++ b/src/scripts/dev.ts @@ -1 +1,73 @@ -// currently empty +import { FileStore } from '../file-system/file-store'; +import { logger } from '@runejs/common'; + + +const dev = async () => { + const fileStore = new FileStore(435); + await fileStore.load(); + fileStore.js5Load(); + + logger.info(`Unpacking archives from JS5 store...`); + + for (const [ , archive ] of fileStore.archives) { + fileStore.js5.unpack(archive); + } + + logger.info(`Decoding JS5 archives...`); + + for (const [ , archive ] of fileStore.archives) { + await fileStore.js5.decodeArchive(archive); + } + + logger.info(`Saving archive indexes...`); + + for (const [ , archive ] of fileStore.archives) { + await archive.saveIndex(); + } + + logger.info(`Unpacking groups from JS5 store...`); + + for (const [ , archive ] of fileStore.archives) { + for (const [ , group ] of archive.groups) { + fileStore.js5.unpack(group); + } + + logger.info(`Finished unpacking archive ${archive.index.name} groups.`); + } + + logger.info(`Decoding JS5 groups...`); + + for (const [ , archive ] of fileStore.archives) { + for (const [ , group ] of archive.groups) { + await fileStore.js5.decodeGroup(group); + } + + logger.info(`Finished decoding archive ${archive.index.name} groups.`); + } + + logger.info(`Saving group indexes...`); + + for (const [ , archive ] of fileStore.archives) { + for (const [ , group ] of archive.groups) { + await group.saveIndex(); + } + } + + // logger.info(`Saving flat file indexes...`); + // + // for (const [ , archive ] of fileStore.archives) { + // for (const [ , group ] of archive.groups) { + // if (group.files.size <= 1) { + // continue; + // } + // + // for (const [ , flatFile ] of group.files) { + // await flatFile.saveIndex(); + // } + // } + // } + + logger.info(`Complete!`); +}; + +dev().catch(console.error); From abf81987b7d2fd3bf65bd2b90088a56ffd766404 Mon Sep 17 00:00:00 2001 From: Kikorono Date: Wed, 13 Jul 2022 21:10:41 -0500 Subject: [PATCH 12/32] Implementing filestore index loading and fixing some JS5 decode issues --- src/db/index-database.ts | 31 ++++++++++++++++++++++++++++--- src/db/index-entity.ts | 11 +++++++---- src/file-system/archive.ts | 33 ++++++++++++++++++++++++++++----- src/file-system/file-base.ts | 8 +++++--- src/file-system/file-store.ts | 7 ++++++- src/file-system/flat-file.ts | 3 +-- src/file-system/group.ts | 35 ++++++++++++++++++++++++++++++----- src/file-system/js5.ts | 6 +++--- src/scripts/dev.ts | 26 ++++++++++---------------- 9 files changed, 118 insertions(+), 42 deletions(-) diff --git a/src/db/index-database.ts b/src/db/index-database.ts index a8143b1..3999a63 100644 --- a/src/db/index-database.ts +++ b/src/db/index-database.ts @@ -40,14 +40,39 @@ export class IndexDatabase { return this._connection; } - // @todo bulk save - 07/13/22 - Kiko + async upsertIndexes(indexEntities: IndexEntity[]): Promise { + const chunkSize = 100; + for (let i = 0; i < indexEntities.length; i += chunkSize) { + const chunk = indexEntities.slice(i, i + chunkSize); + await this.repository.upsert(chunk, { + conflictPaths: [ 'fileType', 'gameBuild', 'key', 'archiveKey', 'groupKey' ], + skipUpdateIfNoValuesChanged: true, + }); + } + } + + async saveIndexes(indexEntities: IndexEntity[]): Promise { + await this.repository.save(indexEntities, { + chunk: 500, + transaction: false, + reload: false, + listeners: false, + }); + } + async saveIndex(indexEntity: IndexEntity): Promise { return await this.repository.save(indexEntity); } - async getIndex(fileType: FileType, key: number, parentKey: number): Promise { + async getIndexes(fileType: FileType, archiveKey: number, groupKey: number = -1): Promise { + return await this.repository.find({ + where: { fileType, archiveKey, groupKey, gameBuild: this.gameBuild } + }); + } + + async getIndex(fileType: FileType, key: number, archiveKey: number, groupKey: number = -1): Promise { return await this.repository.findOne({ - where: { fileType, key, parentKey } + where: { fileType, key, archiveKey, groupKey, gameBuild: this.gameBuild } }) || null; } diff --git a/src/db/index-entity.ts b/src/db/index-entity.ts index ce4545b..29319cb 100644 --- a/src/db/index-entity.ts +++ b/src/db/index-entity.ts @@ -4,9 +4,9 @@ import { CompressionMethod } from '@runejs/common/compress'; import { FileError } from '../config/file-error'; -@Entity('file') +@Entity('file_index') @Index('index_identifier', [ - 'fileType', 'gameBuild', 'key', 'parentKey' + 'fileType', 'gameBuild', 'key', 'archiveKey', 'groupKey' ], { unique: true }) export class IndexEntity { @@ -19,8 +19,11 @@ export class IndexEntity { @PrimaryColumn('integer', { nullable: false, unique: false }) key: number; - @PrimaryColumn('integer', { name: 'parent_key', nullable: false, unique: false, default: -1 }) - parentKey: number = -1; + @PrimaryColumn('integer', { name: 'archive_key', nullable: false, unique: false, default: -1 }) + archiveKey: number = -1; + + @PrimaryColumn('integer', { name: 'group_key', nullable: false, unique: false, default: -1 }) + groupKey: number = -1; @Column('text', { nullable: true, default: null }) name: string = null; diff --git a/src/file-system/archive.ts b/src/file-system/archive.ts index 3310822..90fd15d 100644 --- a/src/file-system/archive.ts +++ b/src/file-system/archive.ts @@ -6,7 +6,7 @@ import { CompressionMethod } from '@runejs/common/compress'; export class Archive extends FileBase { - readonly groups: Map; + readonly groups: Map; constructor( fileStore: FileStore, @@ -14,10 +14,33 @@ export class Archive extends FileBase { name: string, indexFileCompressionMethod: CompressionMethod = 'none', ) { - super(fileStore, key, 255, 'ARCHIVE'); + super(fileStore, key, 255, -1, 'ARCHIVE'); this.index.name = name; this.index.compressionMethod = indexFileCompressionMethod; - this.groups = new Map(); + this.groups = new Map(); + } + + async upsertGroupIndexes(): Promise { + const groupIndexes = Array.from(this.groups.values()).map(group => group.index); + await this.fileStore.database.upsertIndexes(groupIndexes); + } + + async loadGroupIndexes(): Promise { + const groupIndexes = await this.fileStore.database.getIndexes('GROUP', this.index.key); + + if (!groupIndexes?.length) { + return; + } + + for (const groupIndex of groupIndexes) { + const groupKey = groupIndex.key; + + if (!this.groups.has(groupKey)) { + const group = new Group(this.fileStore, groupKey, this); + group.index = groupIndex; + this.groups.set(groupKey, group); + } + } } js5Unpack(): Buffer | null { @@ -45,11 +68,11 @@ export class Archive extends FileBase { } getGroup(groupIndex: number): Group | null { - return this.groups.get(String(groupIndex)) || null; + return this.groups.get(groupIndex) || null; } setGroup(groupIndex: number, group: Group): void { - this.groups.set(String(groupIndex), group); + this.groups.set(groupIndex, group); } findGroup(groupName: string): Group | null { diff --git a/src/file-system/file-base.ts b/src/file-system/file-base.ts index af74d9a..1b03ee9 100644 --- a/src/file-system/file-base.ts +++ b/src/file-system/file-base.ts @@ -12,7 +12,8 @@ export class FileBase { constructor( fileStore: FileStore, key: number, - parentKey: number, + archiveKey: number, + groupKey: number, fileType: FileType, ) { this.fileStore = fileStore; @@ -20,7 +21,8 @@ export class FileBase { this.index.gameBuild = fileStore.gameBuild; this.index.fileType = fileType; this.index.key = key; - this.index.parentKey = parentKey; + this.index.groupKey = groupKey; + this.index.archiveKey = archiveKey; } async saveIndex(): Promise { @@ -30,7 +32,7 @@ export class FileBase { async loadIndex(): Promise { const indexEntity = await this.fileStore.database.getIndex( - this.index.fileType, this.index.key, this.index.parentKey + this.index.fileType, this.index.key, this.index.archiveKey ); if (indexEntity) { diff --git a/src/file-system/file-store.ts b/src/file-system/file-store.ts index b482401..5a4f537 100644 --- a/src/file-system/file-store.ts +++ b/src/file-system/file-store.ts @@ -60,11 +60,16 @@ export class FileStore { archiveConfig.compression || 'none' ); this.archives.set(archiveConfig.key, archive); - await archive.loadIndex(); } } } + async loadArchiveIndexes(): Promise { + for (const [ , archive ] of this.archives) { + await archive.loadIndex(); + } + } + js5Load(): void { this.js5.loadJS5Store(); } diff --git a/src/file-system/flat-file.ts b/src/file-system/flat-file.ts index 2b2e705..0facad6 100644 --- a/src/file-system/flat-file.ts +++ b/src/file-system/flat-file.ts @@ -1,6 +1,5 @@ import { FileBase } from './file-base'; import { FileStore } from './file-store'; -import { FileType } from '../config/file-type'; import { Archive } from './archive'; import { Group } from './group'; @@ -15,7 +14,7 @@ export class FlatFile extends FileBase { key: number, group: Group, ) { - super(fileStore, key, group.index.key, 'FILE'); + super(fileStore, key, group.archive.index.key, group.index.key, 'FILE'); this.archive = group.archive; this.group = group; } diff --git a/src/file-system/group.ts b/src/file-system/group.ts index a36dde8..02e171c 100644 --- a/src/file-system/group.ts +++ b/src/file-system/group.ts @@ -7,16 +7,41 @@ import { FlatFile } from './flat-file'; export class Group extends FileBase { readonly archive: Archive; - readonly files: Map; + readonly files: Map; constructor( fileStore: FileStore, key: number, archive: Archive, ) { - super(fileStore, key, archive.index.key, 'GROUP'); + super(fileStore, key, archive.index.key, -1, 'GROUP'); this.archive = archive; - this.files = new Map(); + this.files = new Map(); + } + + async upsertFileIndexes(): Promise { + const fileIndexes = Array.from(this.files.values()).map(file => file.index); + await this.fileStore.database.upsertIndexes(fileIndexes); + } + + async loadFileIndexes(): Promise { + const fileIndexes = await this.fileStore.database.getIndexes( + 'FILE', this.archive.index.key, this.index.key + ); + + if (!fileIndexes?.length) { + return; + } + + for (const fileIndex of fileIndexes) { + const fileKey = fileIndex.key; + + if (!this.files.has(fileKey)) { + const file = new FlatFile(this.fileStore, fileKey, this); + file.index = fileIndex; + this.files.set(fileKey, file); + } + } } js5Unpack(): Buffer | null { @@ -44,11 +69,11 @@ export class Group extends FileBase { } getFile(fileIndex: number): FlatFile | null { - return this.files.get(String(fileIndex)) || null; + return this.files.get(fileIndex) || null; } setFile(fileIndex: number, file: FlatFile): void { - this.files.set(String(fileIndex), file); + this.files.set(fileIndex, file); } findFile(fileName: string): FlatFile | null { diff --git a/src/file-system/js5.ts b/src/file-system/js5.ts index 88f8648..6eac2f8 100644 --- a/src/file-system/js5.ts +++ b/src/file-system/js5.ts @@ -383,9 +383,9 @@ export class JS5 { return; } - const fileSizeMap = new Map(); - const fileStripeMap = new Map(); - const fileDataMap = new Map(); + const fileSizeMap = new Map(); + const fileStripeMap = new Map(); + const fileDataMap = new Map(); for (const [ flatFileKey, ] of files) { fileSizeMap.set(flatFileKey, 0); diff --git a/src/scripts/dev.ts b/src/scripts/dev.ts index d2fe83c..b8b3a3e 100644 --- a/src/scripts/dev.ts +++ b/src/scripts/dev.ts @@ -3,6 +3,7 @@ import { logger } from '@runejs/common'; const dev = async () => { + const start = Date.now(); const fileStore = new FileStore(435); await fileStore.load(); fileStore.js5Load(); @@ -47,27 +48,20 @@ const dev = async () => { logger.info(`Saving group indexes...`); + for (const [ , archive ] of fileStore.archives) { + await archive.upsertGroupIndexes(); + } + + logger.info(`Saving flat file indexes...`); + for (const [ , archive ] of fileStore.archives) { for (const [ , group ] of archive.groups) { - await group.saveIndex(); + await group.upsertFileIndexes(); } } - // logger.info(`Saving flat file indexes...`); - // - // for (const [ , archive ] of fileStore.archives) { - // for (const [ , group ] of archive.groups) { - // if (group.files.size <= 1) { - // continue; - // } - // - // for (const [ , flatFile ] of group.files) { - // await flatFile.saveIndex(); - // } - // } - // } - - logger.info(`Complete!`); + const end = Date.now(); + logger.info(`Operations completed in ${(end - start) / 1000} seconds.`); }; dev().catch(console.error); From 8cebf959fad7db62af4388964c0ff71d78ded32c Mon Sep 17 00:00:00 2001 From: Kikorono Date: Sun, 17 Jul 2022 18:26:21 -0500 Subject: [PATCH 13/32] Adding validation for checksum, shaDigest, fileSize, compressed versions of the same variables, name, and nameHash within the index --- src/db/index-entity.ts | 1 + src/file-system/archive.ts | 20 ++++++++ src/file-system/file-base.ts | 93 +++++++++++++++++++++++++++++++++++ src/file-system/file-store.ts | 2 + src/file-system/group.ts | 20 ++++++++ src/file-system/js5.ts | 25 +++------- 6 files changed, 144 insertions(+), 17 deletions(-) diff --git a/src/db/index-entity.ts b/src/db/index-entity.ts index 29319cb..dbbdef4 100644 --- a/src/db/index-entity.ts +++ b/src/db/index-entity.ts @@ -2,6 +2,7 @@ import { Column, CreateDateColumn, Entity, Index, PrimaryColumn, UpdateDateColum import { FileType } from '../config/file-type'; import { CompressionMethod } from '@runejs/common/compress'; import { FileError } from '../config/file-error'; +import { Buffer } from 'buffer'; @Entity('file_index') diff --git a/src/file-system/archive.ts b/src/file-system/archive.ts index 90fd15d..1ccda5b 100644 --- a/src/file-system/archive.ts +++ b/src/file-system/archive.ts @@ -2,6 +2,7 @@ import { FileBase } from './file-base'; import { FileStore } from './file-store'; import { Group } from './group'; import { CompressionMethod } from '@runejs/common/compress'; +import { logger } from '@runejs/common'; export class Archive extends FileBase { @@ -20,6 +21,25 @@ export class Archive extends FileBase { this.groups = new Map(); } + override validate(trackChanges: boolean = true): void { + super.validate(trackChanges); + + let archiveModified: boolean = false; + + const { childCount } = this.index; + const newChildCount = this.groups.size; + + if (childCount !== newChildCount) { + this.index.childCount = newChildCount; + archiveModified = true; + } + + if (archiveModified && trackChanges) { + logger.info(`Archive ${this.index.name || this.index.key} child count has changed.`); + this.index.version++; + } + } + async upsertGroupIndexes(): Promise { const groupIndexes = Array.from(this.groups.values()).map(group => group.index); await this.fileStore.database.upsertIndexes(groupIndexes); diff --git a/src/file-system/file-base.ts b/src/file-system/file-base.ts index 1b03ee9..d5f86ee 100644 --- a/src/file-system/file-base.ts +++ b/src/file-system/file-base.ts @@ -1,6 +1,10 @@ import { IndexEntity } from '../db/index-entity'; import { FileStore } from './file-store'; import { FileType } from '../config/file-type'; +import { Crc32 } from '@runejs/common/crc32'; +import { createHash } from 'crypto'; +import { Buffer } from 'buffer'; +import { logger } from '@runejs/common'; export class FileBase { @@ -25,7 +29,80 @@ export class FileBase { this.index.archiveKey = archiveKey; } + validate(trackChanges: boolean = true): void { + const { + data, compressedData, + checksum, shaDigest, fileSize, + compressedChecksum, compressedShaDigest, compressedFileSize, + name, nameHash, + } = this.index; + let fileModified: boolean = false; + + const currentChecksum = this.generateChecksum(data); + const currentShaDigest = this.generateShaDigest(data); + const currentFileSize = data?.length || 0; + + const currentCompressedChecksum = this.generateChecksum(compressedData); + const currentCompressedShaDigest = this.generateShaDigest(compressedData); + const currentCompressedFileSize = compressedData?.length || 0; + + if (name && nameHash === -1) { + // nameHash not set + this.index.nameHash = this.fileStore.hashFileName(name); + } + + if (nameHash !== -1 && !name) { + // name not set + const lookupTableName = this.fileStore.findFileName(nameHash); + if (lookupTableName) { + this.index.name = lookupTableName; + } + } + + if (checksum !== currentChecksum) { + // uncompressed crc32 mismatch + this.index.checksum = currentChecksum; + fileModified = true; + } + + if (shaDigest !== currentShaDigest) { + // uncompressed sha256 mismatch + this.index.shaDigest = currentShaDigest; + fileModified = true; + } + + if (fileSize !== currentFileSize) { + // uncompressed file size mismatch + this.index.fileSize = currentFileSize; + fileModified = true; + } + + if (compressedChecksum !== currentCompressedChecksum) { + // compressed crc32 mismatch + this.index.compressedChecksum = currentCompressedChecksum; + fileModified = true; + } + + if (compressedShaDigest !== currentCompressedShaDigest) { + // compressed sha256 mismatch + this.index.compressedShaDigest = currentCompressedShaDigest; + fileModified = true; + } + + if (compressedFileSize !== currentCompressedFileSize) { + // compressed file size mismatch + this.index.compressedFileSize = currentCompressedFileSize; + fileModified = true; + } + + if (fileModified && trackChanges) { + logger.info(`File ${this.index.name || this.index.key} has been modified.`); + this.index.version++; + } + } + async saveIndex(): Promise { + this.validate(); this.index = await this.fileStore.database.saveIndex(this.index); return this.index; } @@ -42,6 +119,22 @@ export class FileBase { return this.index; } + generateChecksum(data: Buffer): number { + if (!data?.length) { + return -1; + } + + return Crc32.update(0, data.length, data); + } + + generateShaDigest(data: Buffer): string { + if (!data?.length) { + return null; + } + + return createHash('sha256').update(data).digest('hex'); + } + get stripes(): number[] { if (!this.index?.stripes) { return []; diff --git a/src/file-system/file-store.ts b/src/file-system/file-store.ts index 5a4f537..b5295b8 100644 --- a/src/file-system/file-store.ts +++ b/src/file-system/file-store.ts @@ -6,6 +6,7 @@ import { IndexDatabase } from '../db/index-database'; import { ArchiveConfig } from '../config'; import { JS5 } from './js5'; import { Archive } from './archive'; +import { Crc32 } from '@runejs/common/crc32'; export class FileStore { @@ -30,6 +31,7 @@ export class FileStore { this.loadArchiveConfig(); this.loadFileNames(); this.js5 = new JS5(this); + Crc32.init(); } async load(): Promise { diff --git a/src/file-system/group.ts b/src/file-system/group.ts index 02e171c..44566c7 100644 --- a/src/file-system/group.ts +++ b/src/file-system/group.ts @@ -2,6 +2,7 @@ import { FileBase } from './file-base'; import { FileStore } from './file-store'; import { Archive } from './archive'; import { FlatFile } from './flat-file'; +import { logger } from '@runejs/common'; export class Group extends FileBase { @@ -19,6 +20,25 @@ export class Group extends FileBase { this.files = new Map(); } + override validate(trackChanges: boolean = true): void { + super.validate(trackChanges); + + let groupModified: boolean = false; + + const { childCount } = this.index; + const newChildCount = this.files.size; + + if (childCount !== newChildCount) { + this.index.childCount = newChildCount; + groupModified = true; + } + + if (groupModified && trackChanges) { + logger.info(`Group ${this.index.name || this.index.key} child count has changed.`); + this.index.version++; + } + } + async upsertFileIndexes(): Promise { const fileIndexes = Array.from(this.files.values()).map(file => file.index); await this.fileStore.database.upsertIndexes(fileIndexes); diff --git a/src/file-system/js5.ts b/src/file-system/js5.ts index 6eac2f8..c2a7ea3 100644 --- a/src/file-system/js5.ts +++ b/src/file-system/js5.ts @@ -359,7 +359,9 @@ export class JS5 { this.decompress(group); if (!groupDetails.data) { - logger.error(`Unable to decode group ${groupName || groupKey}.`); + if (!groupDetails.fileError) { + logger.warn(`Unable to decode group ${ groupName || groupKey }.`); + } return; } } @@ -437,7 +439,10 @@ export class JS5 { for (const [ fileIndex, file ] of files) { file.index.data = fileDataMap.get(fileIndex).toNodeBuffer(); + file.validate(false); } + + group.validate(false); } async decodeArchive(archive: Archive): Promise { @@ -482,13 +487,6 @@ export class JS5 { archive.setGroup(groupKey, group); } - // Load group database indexes, or create them if they don't yet exist - // @todo batch load all archive groups at once - for (const group of groups) { - // @todo removed for faster unpacking testing - 07/13/22 - Kiko - // await group.loadIndex(); - } - // Group name hashes if (flags.groupNames) { for (const group of groups) { @@ -551,15 +549,6 @@ export class JS5 { } } - // Load flat file database indexes, or create them if they don't yet exist - // @todo batch load all grouped files at once - for (const group of groups) { - for (const [ , flatFile ] of group.files) { - // @todo removed for faster unpacking testing - 07/13/22 - Kiko - // await flatFile.loadIndex(); - } - } - // Grouped file names if (flags.groupNames) { for (const group of groups) { @@ -572,6 +561,8 @@ export class JS5 { } } } + + archive.validate(false); } decodeMainIndex(): Buffer | null { From 95e7e9e4de64e0938a3998fc00090c598f30978f Mon Sep 17 00:00:00 2001 From: Kikorono Date: Sun, 17 Jul 2022 18:33:37 -0500 Subject: [PATCH 14/32] Fixing childCount database column name --- src/db/index-entity.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/db/index-entity.ts b/src/db/index-entity.ts index dbbdef4..0d22145 100644 --- a/src/db/index-entity.ts +++ b/src/db/index-entity.ts @@ -35,7 +35,7 @@ export class IndexEntity { @Column('integer', { nullable: false, default: 0 }) version: number = 0; - @Column('integer', { name: 'childCount', nullable: false, default: 0 }) + @Column('integer', { name: 'child_count', nullable: false, default: 0 }) childCount: number = 0; @Column('integer', { nullable: false, default: -1 }) From 98185690f1ed6859a041b3e981d474b237f7da46 Mon Sep 17 00:00:00 2001 From: Kikorono Date: Mon, 18 Jul 2022 18:35:11 -0500 Subject: [PATCH 15/32] Working on adding support for Jag Cache stores (builds 234-399) --- config/jag-cache-archives.json5 | 26 +++ config/{archives.json5 => js5-archives.json5} | 81 +------- src/config/archive-config.ts | 12 +- src/config/djb2.ts | 74 +++++++ src/config/index.ts | 4 + src/db/index-database.ts | 2 +- src/db/index-entity.ts | 3 +- src/file-system/file-store-base.ts | 90 +++++++++ src/file-system/file-store.ts | 191 ------------------ src/file-system/flat-file.ts | 22 -- .../{file-base.ts => indexed-file-base.ts} | 29 ++- src/file-system/jag/jag-archive.ts | 12 ++ src/file-system/jag/jag-file.ts | 6 + src/file-system/jag/jag-index.ts | 6 + src/file-system/jag/jag-store.ts | 35 ++++ src/file-system/jag/jag.ts | 67 ++++++ src/file-system/{ => js5}/archive.ts | 19 +- src/file-system/js5/flat-file.ts | 22 ++ src/file-system/{ => js5}/group.ts | 12 +- src/file-system/js5/js5-file-store.ts | 50 +++++ src/file-system/{ => js5}/js5.ts | 55 +++-- src/scripts/dev.ts | 6 +- src/scripts/index.ts | 1 - src/scripts/indexer.ts | 184 ++++++++++++----- src/scripts/script-executor.ts | 13 +- 25 files changed, 603 insertions(+), 419 deletions(-) create mode 100644 config/jag-cache-archives.json5 rename config/{archives.json5 => js5-archives.json5} (56%) create mode 100644 src/config/djb2.ts create mode 100644 src/file-system/file-store-base.ts delete mode 100644 src/file-system/file-store.ts delete mode 100644 src/file-system/flat-file.ts rename src/file-system/{file-base.ts => indexed-file-base.ts} (89%) create mode 100644 src/file-system/jag/jag-archive.ts create mode 100644 src/file-system/jag/jag-file.ts create mode 100644 src/file-system/jag/jag-index.ts create mode 100644 src/file-system/jag/jag-store.ts create mode 100644 src/file-system/jag/jag.ts rename src/file-system/{ => js5}/archive.ts (84%) create mode 100644 src/file-system/js5/flat-file.ts rename src/file-system/{ => js5}/group.ts (90%) create mode 100644 src/file-system/js5/js5-file-store.ts rename src/file-system/{ => js5}/js5.ts (93%) delete mode 100644 src/scripts/index.ts diff --git a/config/jag-cache-archives.json5 b/config/jag-cache-archives.json5 new file mode 100644 index 0000000..88068cb --- /dev/null +++ b/config/jag-cache-archives.json5 @@ -0,0 +1,26 @@ +{ + title: { + key: 1 + }, + config: { + key: 2 + }, + interface: { + key: 3 + }, + media: { + key: 4 + }, + versionlist: { + key: 5 + }, + textures: { + key: 6 + }, + wordenc: { + key: 7 + }, + sounds: { + key: 8 + } +} diff --git a/config/archives.json5 b/config/js5-archives.json5 similarity index 56% rename from config/archives.json5 rename to config/js5-archives.json5 index 4bdc993..28a7630 100644 --- a/config/archives.json5 +++ b/config/js5-archives.json5 @@ -1,27 +1,19 @@ { // The main index of indexes main: { - key: 255, - compression: 'none', - versioned: false + key: 255 }, // Regular indexes anims: { key: 0, - compression: 'bzip', - versioned: true, flatten: true }, bases: { - key: 1, - compression: 'gzip', - versioned: true + key: 1 }, config: { key: 2, - compression: 'bzip', - versioned: true, groupNames: { '.flu': 1, // Floor Underlays // 2? @@ -45,168 +37,109 @@ }, interfaces: { key: 3, - compression: 'bzip', - versioned: true, flatten: true }, synth_sounds: { key: 4, - compression: 'gzip', - versioned: true, contentType: '.wav' }, maps: { key: 5, - compression: 'gzip', - encryption: [ 'xtea', '^l[0-9]{1,3}_[0-9]{1,3}$' ], - versioned: true, - filesNamed: true + encryption: [ 'xtea', '^l[0-9]{1,3}_[0-9]{1,3}$' ] }, midi_songs: { key: 6, - compression: 'gzip', - versioned: false, - contentType: '.mid', - filesNamed: true + contentType: '.mid' }, models: { key: 7, - compression: 'gzip', - versioned: true, contentType: '.dat' }, sprites: { - key: 8, - compression: 'gzip', - versioned: true, - filesNamed: true + key: 8 }, textures: { - key: 9, - compression: 'gzip', - versioned: true + key: 9 }, binary: { - key: 10, - compression: 'none', - versioned: true, - filesNamed: true + key: 10 }, midi_jingles: { key: 11, - compression: 'gzip', - versioned: true, contentType: '.mid' }, clientscripts: { key: 12, compression: 'gzip', - versioned: true, contentType: '.cs2', - filesNamed: true, build: 435 }, fontmetrics: { key: 13, - compression: 'gzip', - versioned: false, - filesNamed: true, build: 443 }, vorbis: { key: 14, - compression: 'gzip', - versioned: true, build: 451 }, midi_instruments: { key: 15, - compression: 'gzip', - versioned: false, build: 451 }, config_loc: { key: 16, - compression: 'gzip', - versioned: false, build: 489 }, config_enum: { key: 17, - compression: 'gzip', - versioned: false, build: 489 }, config_npc: { key: 18, - compression: 'gzip', - versioned: false, build: 489 }, config_obj: { key: 19, - compression: 'gzip', - versioned: false, build: 489 }, config_seq: { key: 20, - compression: 'gzip', - versioned: false, build: 489 }, config_spot: { key: 21, - compression: 'gzip', - versioned: false, build: 489 }, config_var_bit: { key: 22, - compression: 'gzip', - versioned: false, build: 489 }, worldmapdata: { key: 23, - compression: 'gzip', - versioned: false, build: 493 }, quickchat: { key: 24, - compression: 'gzip', - versioned: false, build: 498 }, quickchat_global: { key: 25, - compression: 'gzip', - versioned: false, build: 498 }, materials: { key: 26, - compression: 'gzip', - versioned: false, build: 500 }, config_particle: { key: 27, - compression: 'gzip', - versioned: false, build: 523 }, defaults: { key: 28, - compression: 'gzip', - versioned: false, build: 537 }, billboards: { key: 29, - compression: 'gzip', - versioned: false, build: 582 } } diff --git a/src/config/archive-config.ts b/src/config/archive-config.ts index 9c3ef47..636880d 100644 --- a/src/config/archive-config.ts +++ b/src/config/archive-config.ts @@ -1,16 +1,16 @@ -import { CompressionMethod } from '@runejs/common/compress'; import { EncryptionMethod } from '@runejs/common/encrypt'; -export interface ArchiveConfig { +export interface JS5ArchiveConfig { key: number; - name: string; - versioned?: boolean; - compression?: CompressionMethod; encryption?: [ EncryptionMethod, string ]; contentType?: string; - filesNamed?: boolean; flatten?: boolean; groupNames?: { [key: string]: number }; build?: number; } + + +export interface JagCacheArchiveConfig { + key: number; +} diff --git a/src/config/djb2.ts b/src/config/djb2.ts new file mode 100644 index 0000000..a1cdddb --- /dev/null +++ b/src/config/djb2.ts @@ -0,0 +1,74 @@ +import { join } from 'path'; +import { existsSync, readFileSync } from 'graceful-fs'; +import { logger } from '@runejs/common'; + +export class Djb2 { + + readonly configPath: string; + readonly fileNameHashes: Map; + + constructor(configPath: string) { + this.configPath = configPath; + this.fileNameHashes = new Map(); + this.loadFileNames(); + } + + hashFileName(fileName: string): number { + if (!fileName) { + return 0; + } + + let hash = 0; + for (let i = 0; i < fileName.length; i++) { + hash = fileName.charCodeAt(i) + ((hash << 5) - hash); + } + + const nameHash = hash | 0; + + this.fileNameHashes.set(nameHash, fileName); + + return nameHash; + } + + findFileName(nameHash: string | number | undefined, defaultName?: string | undefined): string | undefined { + if (!this.fileNameHashes.size) { + this.loadFileNames(); + } + + if (nameHash === undefined || nameHash === null) { + return defaultName; + } + + if (typeof nameHash === 'string') { + nameHash = Number(nameHash); + } + + if (isNaN(nameHash) || nameHash === -1 || nameHash === 0) { + return defaultName; + } + + return this.fileNameHashes.get(nameHash) || defaultName; + } + + loadFileNames(): void { + const configPath = join(this.configPath, 'name-hashes.json'); + if (!existsSync(configPath)) { + logger.error(`Error loading file names: ${configPath} was not found.`); + return; + } + + const nameTable = JSON.parse( + readFileSync(configPath, 'utf-8') + ) as { [key: string]: string }; + + Object.keys(nameTable).forEach( + nameHash => this.fileNameHashes.set(Number(nameHash), nameTable[nameHash]) + ); + + if(!this.fileNameHashes.size) { + logger.error(`Error reading file name lookup table. ` + + `Please ensure that the ${configPath} file exists and is valid.`); + } + } + +} diff --git a/src/config/index.ts b/src/config/index.ts index 76f19cd..e0116fa 100644 --- a/src/config/index.ts +++ b/src/config/index.ts @@ -1,2 +1,6 @@ export * from './archive-config'; export * from './archive-format'; +export * from './file-error'; +export * from './file-type'; +export * from './archive-flags'; +export * from './djb2'; diff --git a/src/db/index-database.ts b/src/db/index-database.ts index 3999a63..b33b640 100644 --- a/src/db/index-database.ts +++ b/src/db/index-database.ts @@ -2,7 +2,7 @@ import { Connection, createConnection, LoggerOptions, Repository } from 'typeorm import { join } from 'path'; import { existsSync, mkdirSync } from 'graceful-fs'; import { IndexEntity } from './index-entity'; -import { FileType } from '../config/file-type'; +import { FileType } from '../config'; export class IndexDatabase { diff --git a/src/db/index-entity.ts b/src/db/index-entity.ts index 0d22145..b0b7497 100644 --- a/src/db/index-entity.ts +++ b/src/db/index-entity.ts @@ -1,7 +1,6 @@ import { Column, CreateDateColumn, Entity, Index, PrimaryColumn, UpdateDateColumn } from 'typeorm'; -import { FileType } from '../config/file-type'; import { CompressionMethod } from '@runejs/common/compress'; -import { FileError } from '../config/file-error'; +import { FileError, FileType } from '../config'; import { Buffer } from 'buffer'; diff --git a/src/file-system/file-store-base.ts b/src/file-system/file-store-base.ts new file mode 100644 index 0000000..1371610 --- /dev/null +++ b/src/file-system/file-store-base.ts @@ -0,0 +1,90 @@ +import { Djb2 } from '../config'; +import { IndexDatabase } from '../db/index-database'; +import { join } from 'path'; +import { Crc32 } from '@runejs/common/crc32'; +import { existsSync, readFileSync } from 'graceful-fs'; +import { logger } from '@runejs/common'; +import JSON5 from 'json5'; +import { IndexedFileBase } from './indexed-file-base'; + + +export abstract class FileStoreBase, C> { + + readonly gameBuild: string; + readonly fileStorePath: string; + readonly archiveConfigFileName: string; + readonly archives: Map; + readonly djb2: Djb2; + + private _archiveConfig: { [key: string]: C }; + private _database: IndexDatabase; + + protected constructor(gameBuild: string | number, storePath: string, archiveConfigFileName: string) { + this.gameBuild = String(gameBuild); + this.archiveConfigFileName = archiveConfigFileName; + this.fileStorePath = storePath; + this.archives = new Map(); + this.loadArchiveConfig(); + this.djb2 = new Djb2(join(storePath, 'config')); + Crc32.init(); + } + + abstract load(): void | Promise; + + async loadArchiveIndexes(): Promise { + for (const [ , archive ] of this.archives) { + await archive.loadIndex(); + } + } + + loadArchiveConfig(): void { + const configPath = join(this.fileStorePath, 'config', this.archiveConfigFileName + '.json5'); + + if (!existsSync(configPath)) { + logger.error(`Error loading file store: ${configPath} was not found.`); + return; + } + + this._archiveConfig = JSON5.parse( + readFileSync(configPath, 'utf-8') + ) as { [key: string]: C }; + + if (!Object.values(this._archiveConfig)?.length) { + throw new Error(`Error reading archive configuration file. ` + + `Please ensure that the ${configPath} file exists and is valid.`); + } + } + + async openDatabase(): Promise { + this._database = new IndexDatabase( + this.gameBuild, + join(this.fileStorePath, 'index'), + [ 'error', 'warn' ], + ); + await this._database.openConnection(); + return this._database; + } + + getArchive(archiveKey: number): A | null { + return this.archives.get(archiveKey) || null; + } + + setArchive(archiveKey: number, archive: A): void { + this.archives.set(archiveKey, archive); + } + + findArchive(archiveName: string): A | null { + return Array.from(this.archives.values()).find( + a => a?.index?.name === archiveName + ) || null; + } + + get archiveConfig(): { [key: string]: C } { + return this._archiveConfig; + } + + get database(): IndexDatabase { + return this._database; + } + +} diff --git a/src/file-system/file-store.ts b/src/file-system/file-store.ts deleted file mode 100644 index b5295b8..0000000 --- a/src/file-system/file-store.ts +++ /dev/null @@ -1,191 +0,0 @@ -import { join } from 'path'; -import JSON5 from 'json5'; -import { existsSync, readFileSync } from 'graceful-fs'; -import { logger } from '@runejs/common'; -import { IndexDatabase } from '../db/index-database'; -import { ArchiveConfig } from '../config'; -import { JS5 } from './js5'; -import { Archive } from './archive'; -import { Crc32 } from '@runejs/common/crc32'; - - -export class FileStore { - - readonly gameBuild: string; - readonly vanillaBuild: boolean; - readonly fileStorePath: string; - readonly fileNameHashes: Map; - readonly js5: JS5; - readonly archives: Map; - - private _archiveConfig: { [key: string]: ArchiveConfig }; - private _database: IndexDatabase; - - constructor(gameBuild: string | number, storePath: string = './') { - this.gameBuild = String(gameBuild); - // @todo make `vanillaBuild` obsolete via auto-detection - 07/13/22 - Kiko - this.vanillaBuild = typeof gameBuild === 'number'; - this.fileStorePath = storePath; - this.fileNameHashes = new Map(); - this.archives = new Map(); - this.loadArchiveConfig(); - this.loadFileNames(); - this.js5 = new JS5(this); - Crc32.init(); - } - - async load(): Promise { - await this.openDatabase(); - - const archiveNames = Object.keys(this._archiveConfig); - - for (const archiveName of archiveNames) { - const archiveConfig = this._archiveConfig[archiveName]; - - if (archiveConfig.key === 255) { - continue; - } - - if (this.vanillaBuild && archiveConfig.build) { - const buildNumber = Number(this.gameBuild); - - if (buildNumber < archiveConfig.build) { - continue; - } - } - - if (!this.archives.has(archiveConfig.key)) { - const archive = new Archive( - this, - archiveConfig.key, - archiveName, - archiveConfig.compression || 'none' - ); - this.archives.set(archiveConfig.key, archive); - } - } - } - - async loadArchiveIndexes(): Promise { - for (const [ , archive ] of this.archives) { - await archive.loadIndex(); - } - } - - js5Load(): void { - this.js5.loadJS5Store(); - } - - js5EncodeMainIndex(): Buffer { - return this.js5.encodeMainIndex().toNodeBuffer(); - } - - hashFileName(fileName: string): number { - if (!fileName) { - return 0; - } - - let hash = 0; - for (let i = 0; i < fileName.length; i++) { - hash = fileName.charCodeAt(i) + ((hash << 5) - hash); - } - - const nameHash = hash | 0; - - this.fileNameHashes.set(nameHash, fileName); - - return nameHash; - } - - findFileName(nameHash: string | number | undefined, defaultName?: string | undefined): string | undefined { - if (!this.fileNameHashes.size) { - this.loadFileNames(); - } - - if (nameHash === undefined || nameHash === null) { - return defaultName; - } - - if (typeof nameHash === 'string') { - nameHash = Number(nameHash); - } - - if (isNaN(nameHash) || nameHash === -1 || nameHash === 0) { - return defaultName; - } - - return this.fileNameHashes.get(nameHash) || defaultName; - } - - loadFileNames(): void { - const configPath = join(this.fileStorePath, 'config', 'name-hashes.json'); - if (!existsSync(configPath)) { - logger.error(`Error loading file names: ${configPath} was not found.`); - return; - } - - const nameTable = JSON.parse( - readFileSync(configPath, 'utf-8') - ) as { [key: string]: string }; - - Object.keys(nameTable).forEach( - nameHash => this.fileNameHashes.set(Number(nameHash), nameTable[nameHash]) - ); - - if(!this.fileNameHashes.size) { - logger.error(`Error reading file name lookup table. ` + - `Please ensure that the ${configPath} file exists and is valid.`); - } - } - - loadArchiveConfig(): void { - const configPath = join(this.fileStorePath, 'config', 'archives.json5'); - - if (!existsSync(configPath)) { - logger.error(`Error loading file store: ${configPath} was not found.`); - return; - } - - this._archiveConfig = JSON5.parse( - readFileSync(configPath, 'utf-8') - ) as { [key: string]: ArchiveConfig }; - - if (!Object.values(this._archiveConfig)?.length) { - throw new Error(`Error reading archive configuration file. ` + - `Please ensure that the ${configPath} file exists and is valid.`); - } - } - - async openDatabase(): Promise { - this._database = new IndexDatabase( - this.gameBuild, - join(this.fileStorePath, 'index'), - [ 'error', 'warn' ], - ); - await this._database.openConnection(); - return this._database; - } - - getArchive(archiveKey: number): Archive | null { - return this.archives.get(archiveKey) || null; - } - - setArchive(archiveKey: number, archive: Archive): void { - this.archives.set(archiveKey, archive); - } - - findArchive(archiveName: string): Archive | null { - return Array.from(this.archives.values()).find( - a => a?.index?.name === archiveName - ) || null; - } - - get archiveConfig(): { [key: string]: ArchiveConfig } { - return this._archiveConfig; - } - - get database(): IndexDatabase { - return this._database; - } - -} diff --git a/src/file-system/flat-file.ts b/src/file-system/flat-file.ts deleted file mode 100644 index 0facad6..0000000 --- a/src/file-system/flat-file.ts +++ /dev/null @@ -1,22 +0,0 @@ -import { FileBase } from './file-base'; -import { FileStore } from './file-store'; -import { Archive } from './archive'; -import { Group } from './group'; - - -export class FlatFile extends FileBase { - - readonly archive: Archive; - readonly group: Group; - - constructor( - fileStore: FileStore, - key: number, - group: Group, - ) { - super(fileStore, key, group.archive.index.key, group.index.key, 'FILE'); - this.archive = group.archive; - this.group = group; - } - -} diff --git a/src/file-system/file-base.ts b/src/file-system/indexed-file-base.ts similarity index 89% rename from src/file-system/file-base.ts rename to src/file-system/indexed-file-base.ts index d5f86ee..b6bfd2a 100644 --- a/src/file-system/file-base.ts +++ b/src/file-system/indexed-file-base.ts @@ -1,32 +1,31 @@ import { IndexEntity } from '../db/index-entity'; -import { FileStore } from './file-store'; -import { FileType } from '../config/file-type'; +import { FileStoreBase } from './file-store-base'; +import { FileType } from '../config'; +import { logger } from '@runejs/common'; +import { Buffer } from 'buffer'; import { Crc32 } from '@runejs/common/crc32'; import { createHash } from 'crypto'; -import { Buffer } from 'buffer'; -import { logger } from '@runejs/common'; - -export class FileBase { - readonly fileStore: FileStore; +export abstract class IndexedFileBase> { + readonly fileStore: S; index: IndexEntity; - constructor( - fileStore: FileStore, - key: number, - archiveKey: number, - groupKey: number, + protected constructor( + fileStore: S, fileType: FileType, + key: number, + archiveKey: number = -1, + groupKey: number = -1, ) { this.fileStore = fileStore; this.index = new IndexEntity(); this.index.gameBuild = fileStore.gameBuild; this.index.fileType = fileType; this.index.key = key; - this.index.groupKey = groupKey; this.index.archiveKey = archiveKey; + this.index.groupKey = groupKey; } validate(trackChanges: boolean = true): void { @@ -48,12 +47,12 @@ export class FileBase { if (name && nameHash === -1) { // nameHash not set - this.index.nameHash = this.fileStore.hashFileName(name); + this.index.nameHash = this.fileStore.djb2.hashFileName(name); } if (nameHash !== -1 && !name) { // name not set - const lookupTableName = this.fileStore.findFileName(nameHash); + const lookupTableName = this.fileStore.djb2.findFileName(nameHash); if (lookupTableName) { this.index.name = lookupTableName; } diff --git a/src/file-system/jag/jag-archive.ts b/src/file-system/jag/jag-archive.ts new file mode 100644 index 0000000..157becb --- /dev/null +++ b/src/file-system/jag/jag-archive.ts @@ -0,0 +1,12 @@ +import { JagStore } from './jag-store'; +import { IndexedFileBase } from '../indexed-file-base'; + + +export class JagArchive extends IndexedFileBase { + + constructor(jagStore: JagStore, archiveKey: number, archiveName: string) { + super(jagStore, 'ARCHIVE', archiveKey, -1, -1); + this.index.name = archiveName; + } + +} diff --git a/src/file-system/jag/jag-file.ts b/src/file-system/jag/jag-file.ts new file mode 100644 index 0000000..ce6dbfb --- /dev/null +++ b/src/file-system/jag/jag-file.ts @@ -0,0 +1,6 @@ +import { IndexedFileBase } from '../indexed-file-base'; + + +export class JagFile extends IndexedFileBase { + +} diff --git a/src/file-system/jag/jag-index.ts b/src/file-system/jag/jag-index.ts new file mode 100644 index 0000000..ae5c58e --- /dev/null +++ b/src/file-system/jag/jag-index.ts @@ -0,0 +1,6 @@ +import { IndexedFileBase } from '../indexed-file-base'; + + +export class JagIndex extends IndexedFileBase { + +} diff --git a/src/file-system/jag/jag-store.ts b/src/file-system/jag/jag-store.ts new file mode 100644 index 0000000..6ac4dc0 --- /dev/null +++ b/src/file-system/jag/jag-store.ts @@ -0,0 +1,35 @@ +import { JagCacheArchiveConfig } from '../../config'; +import { JagArchive } from './jag-archive'; +import { FileStoreBase } from '../file-store-base'; +import { Jag } from './jag'; + + +export class JagStore extends FileStoreBase { + + readonly jag: Jag; + + constructor(gameBuild: string | number, storePath: string = './') { + super(gameBuild, storePath, 'jag-cache-archives'); + this.jag = new Jag(this); + } + + override async load(): Promise { + await this.openDatabase(); + + const archiveNames = Object.keys(this.archiveConfig); + + for (const archiveName of archiveNames) { + const archiveConfig = this.archiveConfig[archiveName]; + + if (!this.archives.has(archiveConfig.key)) { + const archive = new JagArchive( + this, + archiveConfig.key, + archiveName, + ); + this.archives.set(archiveConfig.key, archive); + } + } + } + +} diff --git a/src/file-system/jag/jag.ts b/src/file-system/jag/jag.ts new file mode 100644 index 0000000..a1ed0bf --- /dev/null +++ b/src/file-system/jag/jag.ts @@ -0,0 +1,67 @@ +import { join } from 'path'; +import { existsSync, readdirSync, readFileSync, statSync } from 'graceful-fs'; +import { ByteBuffer, logger } from '@runejs/common'; +import { JagStore } from './jag-store'; + + +export class Jag { + + readonly jagStore: JagStore; + + private indexFiles: Map; + private dataFile: ByteBuffer; + + constructor(jagStore: JagStore) { + this.jagStore = jagStore; + } + + loadJagFiles(): void { + const jagStorePath = join(this.jagStore.fileStorePath, 'jag'); + + if (!existsSync(jagStorePath)) { + throw new Error(`${jagStorePath} could not be found.`); + } + + const stats = statSync(jagStorePath); + if (!stats?.isDirectory()) { + throw new Error(`${jagStorePath} is not a valid directory.`); + } + + const storeFileNames = readdirSync(jagStorePath); + const dataFileName = 'main_file_cache.dat'; + + if (storeFileNames.indexOf(dataFileName) === -1) { + throw new Error(`The main ${dataFileName} data file could not be found.`); + } + + const indexFilePrefix = 'main_file_cache.idx'; + const dataFilePath = join(jagStorePath, dataFileName); + + this.dataFile = new ByteBuffer(readFileSync(dataFilePath)); + this.indexFiles = new Map(); + + for (const fileName of storeFileNames) { + if (!fileName?.length || fileName === dataFileName) { + continue; + } + + if (!fileName.startsWith(indexFilePrefix)) { + continue; + } + + const index = fileName.substring(fileName.indexOf('.idx') + 4); + const numericIndex = Number(index); + + if (isNaN(numericIndex)) { + logger.error(`Index file ${fileName} does not have a valid extension.`); + } + + this.indexFiles.set(numericIndex, new ByteBuffer(readFileSync(join(jagStorePath, fileName)))); + } + + logger.info(`Jag store files loaded for game build ${this.jagStore.gameBuild}.`); + } + + // @todo 18/07/22 - Kiko + +} diff --git a/src/file-system/archive.ts b/src/file-system/js5/archive.ts similarity index 84% rename from src/file-system/archive.ts rename to src/file-system/js5/archive.ts index 1ccda5b..dd1b725 100644 --- a/src/file-system/archive.ts +++ b/src/file-system/js5/archive.ts @@ -1,23 +1,20 @@ -import { FileBase } from './file-base'; -import { FileStore } from './file-store'; +import { JS5FileStore } from './js5-file-store'; import { Group } from './group'; -import { CompressionMethod } from '@runejs/common/compress'; import { logger } from '@runejs/common'; +import { IndexedFileBase } from '../indexed-file-base'; -export class Archive extends FileBase { +export class Archive extends IndexedFileBase { readonly groups: Map; constructor( - fileStore: FileStore, - key: number, - name: string, - indexFileCompressionMethod: CompressionMethod = 'none', + fileStore: JS5FileStore, + archiveKey: number, + archiveName: string, ) { - super(fileStore, key, 255, -1, 'ARCHIVE'); - this.index.name = name; - this.index.compressionMethod = indexFileCompressionMethod; + super(fileStore, 'ARCHIVE', archiveKey, 255, -1); + this.index.name = archiveName; this.groups = new Map(); } diff --git a/src/file-system/js5/flat-file.ts b/src/file-system/js5/flat-file.ts new file mode 100644 index 0000000..41dddeb --- /dev/null +++ b/src/file-system/js5/flat-file.ts @@ -0,0 +1,22 @@ +import { JS5FileStore } from './js5-file-store'; +import { Archive } from './archive'; +import { Group } from './group'; +import { IndexedFileBase } from '../indexed-file-base'; + + +export class FlatFile extends IndexedFileBase { + + readonly archive: Archive; + readonly group: Group; + + constructor( + fileStore: JS5FileStore, + fileKey: number, + group: Group, + ) { + super(fileStore, 'FILE', fileKey, group.archive.index.key, group.index.key); + this.archive = group.archive; + this.group = group; + } + +} diff --git a/src/file-system/group.ts b/src/file-system/js5/group.ts similarity index 90% rename from src/file-system/group.ts rename to src/file-system/js5/group.ts index 44566c7..201a19b 100644 --- a/src/file-system/group.ts +++ b/src/file-system/js5/group.ts @@ -1,21 +1,21 @@ -import { FileBase } from './file-base'; -import { FileStore } from './file-store'; +import { JS5FileStore } from './js5-file-store'; import { Archive } from './archive'; import { FlatFile } from './flat-file'; import { logger } from '@runejs/common'; +import { IndexedFileBase } from '../indexed-file-base'; -export class Group extends FileBase { +export class Group extends IndexedFileBase { readonly archive: Archive; readonly files: Map; constructor( - fileStore: FileStore, - key: number, + fileStore: JS5FileStore, + groupKey: number, archive: Archive, ) { - super(fileStore, key, archive.index.key, -1, 'GROUP'); + super(fileStore, 'GROUP', groupKey, archive.index.key); this.archive = archive; this.files = new Map(); } diff --git a/src/file-system/js5/js5-file-store.ts b/src/file-system/js5/js5-file-store.ts new file mode 100644 index 0000000..494b6e5 --- /dev/null +++ b/src/file-system/js5/js5-file-store.ts @@ -0,0 +1,50 @@ +import { JS5ArchiveConfig } from '../../config'; +import { JS5 } from './js5'; +import { Archive } from './archive'; +import { FileStoreBase } from '../file-store-base'; + + +export class JS5FileStore extends FileStoreBase{ + + readonly vanillaBuild: boolean; + readonly js5: JS5; + + constructor(gameBuild: string | number, storePath: string = './') { + super(gameBuild, storePath, 'js5-archives'); + // @todo make `vanillaBuild` obsolete via auto-detection - 07/13/22 - Kiko + this.vanillaBuild = typeof gameBuild === 'number'; + this.js5 = new JS5(this); + } + + override async load(): Promise { + await this.openDatabase(); + + const archiveNames = Object.keys(this.archiveConfig); + + for (const archiveName of archiveNames) { + const archiveConfig = this.archiveConfig[archiveName]; + + if (archiveConfig.key === 255) { + continue; + } + + if (this.vanillaBuild && archiveConfig.build) { + const buildNumber = Number(this.gameBuild); + + if (buildNumber < archiveConfig.build) { + continue; + } + } + + if (!this.archives.has(archiveConfig.key)) { + const archive = new Archive( + this, + archiveConfig.key, + archiveName, + ); + this.archives.set(archiveConfig.key, archive); + } + } + } + +} diff --git a/src/file-system/js5.ts b/src/file-system/js5/js5.ts similarity index 93% rename from src/file-system/js5.ts rename to src/file-system/js5/js5.ts index c2a7ea3..0169f6e 100644 --- a/src/file-system/js5.ts +++ b/src/file-system/js5/js5.ts @@ -3,30 +3,30 @@ import { existsSync, readdirSync, readFileSync, statSync } from 'graceful-fs'; import { ByteBuffer, logger } from '@runejs/common'; import { Bzip2, getCompressionMethod, Gzip } from '@runejs/common/compress'; import { Xtea, XteaKeys } from '@runejs/common/encrypt'; -import { archiveFlags } from '../config/archive-flags'; +import { archiveFlags } from '../../config'; import { Group } from './group'; import { Archive } from './archive'; -import { FileStore } from './file-store'; -import { ArchiveFormat } from '../config'; +import { JS5FileStore } from './js5-file-store'; +import { ArchiveFormat } from '../../config'; import { FlatFile } from './flat-file'; export class JS5 { - readonly fileStore: FileStore; + readonly fileStore: JS5FileStore; encryptionKeys: Map; - private mainIndex: ByteBuffer; - private archiveIndexes: Map; - private mainArchiveData: ByteBuffer; + private mainIndexFile: ByteBuffer; + private indexFiles: Map; + private dataFile: ByteBuffer; - constructor(fileStore: FileStore) { + constructor(fileStore: JS5FileStore) { this.fileStore = fileStore; this.loadEncryptionKeys(); } - loadJS5Store(): void { + loadJS5Files(): void { const js5StorePath = join(this.fileStore.fileStorePath, 'js5'); if (!existsSync(js5StorePath)) { @@ -39,11 +39,11 @@ export class JS5 { } const storeFileNames = readdirSync(js5StorePath); - const dataFile = 'main_file_cache.dat2'; + const dataFileName = 'main_file_cache.dat2'; const mainIndexFile = 'main_file_cache.idx255'; - if (storeFileNames.indexOf(dataFile) === -1) { - throw new Error(`The main ${dataFile} data file could not be found.`); + if (storeFileNames.indexOf(dataFileName) === -1) { + throw new Error(`The main ${dataFileName} data file could not be found.`); } if (storeFileNames.indexOf(mainIndexFile) === -1) { @@ -51,15 +51,15 @@ export class JS5 { } const indexFilePrefix = 'main_file_cache.idx'; - const dataFilePath = join(js5StorePath, dataFile); + const dataFilePath = join(js5StorePath, dataFileName); const mainIndexFilePath = join(js5StorePath, mainIndexFile); - this.mainArchiveData = new ByteBuffer(readFileSync(dataFilePath)); - this.mainIndex = new ByteBuffer(readFileSync(mainIndexFilePath)); - this.archiveIndexes = new Map(); + this.dataFile = new ByteBuffer(readFileSync(dataFilePath)); + this.mainIndexFile = new ByteBuffer(readFileSync(mainIndexFilePath)); + this.indexFiles = new Map(); for (const fileName of storeFileNames) { - if (!fileName?.length || fileName === mainIndexFile || fileName === dataFile) { + if (!fileName?.length || fileName === mainIndexFile || fileName === dataFileName) { continue; } @@ -74,10 +74,10 @@ export class JS5 { logger.error(`Index file ${fileName} does not have a valid extension.`); } - this.archiveIndexes.set(index, new ByteBuffer(readFileSync(join(js5StorePath, fileName)))); + this.indexFiles.set(numericIndex, new ByteBuffer(readFileSync(join(js5StorePath, fileName)))); } - logger.info(`JS5 store loaded for game build ${this.fileStore.gameBuild}.`); + logger.info(`JS5 store file loaded for game build ${this.fileStore.gameBuild}.`); } unpack(file: Group | Archive): Buffer | null { @@ -87,14 +87,14 @@ export class JS5 { const archiveName: string = file instanceof Archive ? 'main' : file.archive.index.name; const indexChannel: ByteBuffer = archiveKey !== 255 ? - this.archiveIndexes.get(String(archiveKey)) : this.mainIndex; + this.indexFiles.get(archiveKey) : this.mainIndexFile; if (archiveKey === 255 && fileKey === 255) { return null; } const indexDataLength = 6; - const dataChannel = this.mainArchiveData; + const dataChannel = this.dataFile; indexChannel.readerIndex = 0; dataChannel.readerIndex = 0; @@ -184,10 +184,7 @@ export class JS5 { return fileDetails.compressedData; } - readCompressedFileHeader(file: Group | Archive): { - compressedLength: number; - readerIndex: number; - } { + readCompressedFileHeader(file: Group | Archive): { compressedLength: number, readerIndex: number } { const fileDetails = file.index; if (!fileDetails.compressedData?.length) { @@ -491,7 +488,7 @@ export class JS5 { if (flags.groupNames) { for (const group of groups) { group.index.nameHash = archiveData.get('int'); - group.index.name = this.fileStore.findFileName( + group.index.name = this.fileStore.djb2.findFileName( group.index.nameHash, group.index.name || String(group.index.nameHash) || String(group.index.key) ); @@ -554,7 +551,7 @@ export class JS5 { for (const group of groups) { for (const [ , flatFile ] of group.files) { flatFile.index.nameHash = archiveData.get('int'); - flatFile.index.name = this.fileStore.findFileName( + flatFile.index.name = this.fileStore.djb2.findFileName( flatFile.index.nameHash, flatFile.index.name || String(flatFile.index.nameHash) || String(flatFile.index.key) ); @@ -650,8 +647,8 @@ export class JS5 { data.put(this.fileStore.archives.get(archiveIndex).index.checksum, 'int'); } - this.mainIndex = data; - return this.mainIndex; + this.mainIndexFile = data; + return this.mainIndexFile; } getEncryptionKeys(fileName: string): XteaKeys | XteaKeys[] | null { diff --git a/src/scripts/dev.ts b/src/scripts/dev.ts index b8b3a3e..0c4c54f 100644 --- a/src/scripts/dev.ts +++ b/src/scripts/dev.ts @@ -1,12 +1,12 @@ -import { FileStore } from '../file-system/file-store'; +import { JS5FileStore } from '../file-system/js5/js5-file-store'; import { logger } from '@runejs/common'; const dev = async () => { const start = Date.now(); - const fileStore = new FileStore(435); + const fileStore = new JS5FileStore(435); await fileStore.load(); - fileStore.js5Load(); + fileStore.js5.loadJS5Files(); logger.info(`Unpacking archives from JS5 store...`); diff --git a/src/scripts/index.ts b/src/scripts/index.ts deleted file mode 100644 index 5848b19..0000000 --- a/src/scripts/index.ts +++ /dev/null @@ -1 +0,0 @@ -export * from './script-executor'; diff --git a/src/scripts/indexer.ts b/src/scripts/indexer.ts index 114e786..4e6796b 100644 --- a/src/scripts/indexer.ts +++ b/src/scripts/indexer.ts @@ -1,14 +1,16 @@ import { join } from 'path'; import { existsSync, mkdirSync } from 'graceful-fs'; import { logger } from '@runejs/common'; - -import { Store, StoreFormat } from '../index'; -import { ScriptExecutor, ArgumentOptions } from './index'; +import { ScriptExecutor, ArgumentOptions } from './script-executor'; +import { JS5FileStore } from '../file-system/js5/js5-file-store'; +import { JagStore } from '../file-system/jag/jag-store'; +import { FileStoreBase } from '../file-system/file-store-base'; +import { Archive } from '../file-system/js5/archive'; interface IndexerOptions { dir: string; - format: StoreFormat | 'flat' | 'js5'; + format: 'flat' | 'js5' | 'jag'; archive: string; build: string; } @@ -20,7 +22,10 @@ const indexerArgumentOptions: ArgumentOptions = { description: `The store root directory. Defaults to the current location.` }, format: { - alias: 'f', type: 'string', default: 'unpacked', choices: [ 'unpacked', 'packed', 'flat', 'js5' ], + alias: 'f', + type: 'string', + default: 'unpacked', + choices: [ 'unpacked', 'packed', 'flat', 'js5' ], description: `The format of the store to index, either 'unpacked' (flat files) or 'packed' (JS5 format). Defaults to 'unpacked'.` }, archive: { @@ -34,84 +39,155 @@ const indexerArgumentOptions: ArgumentOptions = { }; -async function indexFiles(store: Store, args: IndexerOptions): Promise { - const argDebugString = args ? Array.from(Object.entries(args)) - .map(([ key, val ]) => `${key} = ${val}`).join(', ') : ''; +const indexJS5Store = async (store: JS5FileStore) => { + logger.info(`Unpacking archives from JS5 store...`); + + for (const [ , archive ] of store.archives) { + store.js5.unpack(archive); + } + + logger.info(`Decoding JS5 archives...`); + + for (const [ , archive ] of store.archives) { + await store.js5.decodeArchive(archive); + } - const { archive: archiveName } = args; + logger.info(`Saving archive indexes...`); - let format = args.format; - if(format === 'js5') { - format = 'packed'; - } else if(format === 'flat') { - format = 'unpacked'; + for (const [ , archive ] of store.archives) { + await archive.saveIndex(); } - if(format === 'packed') { - store.loadPackedStore(); - } else { - const outputDir = store.outputPath; - if(!existsSync(outputDir)) { - mkdirSync(outputDir, { recursive: true }); + logger.info(`Unpacking groups from JS5 store...`); + + for (const [ , archive ] of store.archives) { + for (const [ , group ] of archive.groups) { + store.js5.unpack(group); } + + logger.info(`Finished unpacking archive ${ archive.index.name } groups.`); } - if(archiveName === 'main') { - logger.info(`Indexing ${format} store with arguments:`, argDebugString); + logger.info(`Decoding JS5 groups...`); - if(format === 'unpacked') { - await store.read(); - } else if(format === 'packed') { - store.decode(true); + for (const [ , archive ] of store.archives) { + for (const [ , group ] of archive.groups) { + await store.js5.decodeGroup(group); } - store.encode(true); - store.compress(true); + logger.info(`Finished decoding archive ${ archive.index.name } groups.`); + } - await store.saveIndexData(true, true, true); - } else { - logger.info(`Indexing ${format} archive ${archiveName} with arguments:`, argDebugString); + logger.info(`Saving group indexes...`); + + for (const [ , archive ] of store.archives) { + await archive.upsertGroupIndexes(); + } - const archive = store.find(archiveName); + logger.info(`Saving flat file indexes...`); - if(format === 'unpacked') { - await archive.read(false); - } else if(format === 'packed') { - archive.decode(); + for (const [ , archive ] of store.archives) { + for (const [ , group ] of archive.groups) { + await group.upsertFileIndexes(); } + } +}; + - archive.encode(true); - archive.compress(true); +const indexJS5Archive = async (store: JS5FileStore, archiveName: string) => { + const archive = store.findArchive(archiveName); - await store.saveIndexData(false, false, false); - await archive.saveIndexData(true, true); + if (!archive) { + logger.error(`Archive ${ archiveName } was not found.`); + return; } -} + logger.info(`Unpacking archive ${ archiveName } from JS5 store...`); -new ScriptExecutor().executeScript(indexerArgumentOptions, async (terminal, args) => { - const start = Date.now(); - logger.info(`Indexing store...`); + store.js5.unpack(archive); - const { build, dir } = args; + logger.info(`Decoding archive ${ archiveName }...`); - const logDir = join(dir, 'logs'); + await store.js5.decodeArchive(archive); + + logger.info(`Saving archive ${ archiveName } index...`); + + await archive.saveIndex(); + + logger.info(`Unpacking groups from archive ${ archiveName }...`); + + for (const [ , group ] of archive.groups) { + store.js5.unpack(group); + } - if(!existsSync(logDir)) { + logger.info(`Decoding archive ${ archiveName } groups...`); + + for (const [ , group ] of archive.groups) { + await store.js5.decodeGroup(group); + } + + logger.info(`Saving group indexes...`); + + await archive.upsertGroupIndexes(); + + logger.info(`Saving flat file indexes...`); + + for (const [ , group ] of archive.groups) { + await group.upsertFileIndexes(); + } +}; + + +const indexJagStore = async (store: JagStore) => { + // @todo 18/07/22 - Kiko +}; + + +const indexJagArchive = async (store: JagStore, archiveName: string) => { + // @todo 18/07/22 - Kiko +}; + + +const indexerScript = async ( + terminal, + { build, dir, format, archive: archiveName } +) => { + const start = Date.now(); + const logDir = join(dir, 'logs'); + const numericBuildNumber: boolean = /^\d+$/.test(build); + + if (!existsSync(logDir)) { mkdirSync(logDir, { recursive: true }); } - logger.destination(join(logDir, `index_${build}.log`)); + logger.destination(join(logDir, `index-${ format }-${ build }.log`)); - const store = await Store.create(build, dir); + logger.info(`Indexing ${ format } file store...`); - await indexFiles(store, args); + if (format === 'js5') { + const store = new JS5FileStore(numericBuildNumber ? Number(build) : build, dir); + await store.load(); + store.js5.loadJS5Files(); + if (archiveName === 'main') { + await indexJS5Store(store); + } else { + await indexJS5Archive(store, archiveName); + } + } else if (format === 'jag') { + const store = new JagStore(numericBuildNumber ? Number(build) : build, dir); + await store.load(); + store.jag.loadJagFiles(); + + // @todo 18/07/22 - Kiko + } else if (format === 'flat') { + // @todo 18/07/22 - Kiko + } + + logger.info(`Indexing completed in ${ (Date.now() - start) / 1000 } seconds.`); logger.boom.flushSync(); logger.boom.end(); +}; - const end = Date.now(); - logger.info(`Indexing completed in ${(end - start) / 1000} seconds.`); - process.exit(0); -}); +new ScriptExecutor().executeScript(indexerArgumentOptions, indexerScript); diff --git a/src/scripts/script-executor.ts b/src/scripts/script-executor.ts index 29796c5..ae83274 100644 --- a/src/scripts/script-executor.ts +++ b/src/scripts/script-executor.ts @@ -1,5 +1,6 @@ import yargs from 'yargs/yargs'; import { Options } from 'yargs'; +import { logger } from '@runejs/common'; export type ArgumentOptions = { [key: string]: Options }; @@ -11,11 +12,15 @@ export class ScriptExecutor { return yargs(process.argv.slice(2)).options(argumentOptions).argv as any as T; } - public executeScript(argumentOptions: ArgumentOptions, - executor: (terminalInterface: ScriptExecutor, args: T) => Promise): void { + public executeScript( + argumentOptions: ArgumentOptions, + script: (terminalInterface: ScriptExecutor, args: T + ) => Promise): void { (async function(terminal: ScriptExecutor, args: T) { - await executor(terminal, args); - }(this, this.getArguments(argumentOptions))); + await script(terminal, args); + }(this, this.getArguments(argumentOptions))) + .catch(logger.error) + .finally(() => process.exit(0)); } } From 4b1355456382410887229f54caf7b5db3d7b60bc Mon Sep 17 00:00:00 2001 From: Kikorono Date: Wed, 20 Jul 2022 12:38:46 -0500 Subject: [PATCH 16/32] Adding the ability to pull data/caches from OpenRS2.org if they're found there --- config/jag-cache-indexes.json5 | 17 +++ package-lock.json | 185 ++++++++++++++++++++++++++++- package.json | 10 +- src/db/index-database.ts | 4 + src/file-system/file-store-base.ts | 4 + src/file-system/jag/jag.ts | 2 +- src/file-system/js5/js5.ts | 68 +++++++++-- src/openrs2/index.ts | 1 + src/openrs2/openrs2.ts | 146 +++++++++++++++++++++++ src/scripts/builds.ts | 70 +++++++++++ src/scripts/dev.ts | 2 +- src/scripts/indexer.ts | 80 +++++++++---- src/scripts/script-executor.ts | 10 +- 13 files changed, 551 insertions(+), 48 deletions(-) create mode 100644 config/jag-cache-indexes.json5 create mode 100644 src/openrs2/index.ts create mode 100644 src/openrs2/openrs2.ts create mode 100644 src/scripts/builds.ts diff --git a/config/jag-cache-indexes.json5 b/config/jag-cache-indexes.json5 new file mode 100644 index 0000000..58d0cb1 --- /dev/null +++ b/config/jag-cache-indexes.json5 @@ -0,0 +1,17 @@ +{ + archives: { + key: 0 + }, + models: { + key: 1 + }, + animations: { + key: 2 + }, + midi: { + key: 3 + }, + maps: { + key: 4 + } +} diff --git a/package-lock.json b/package-lock.json index 6ff18f1..811fe2a 100644 --- a/package-lock.json +++ b/package-lock.json @@ -1,25 +1,28 @@ { - "name": "@runejs/store", - "version": "1.0.0-beta.1", + "name": "@runejs/filestore", + "version": "1.0.0-next.0", "lockfileVersion": 2, "requires": true, "packages": { "": { - "name": "@runejs/store", - "version": "1.0.0-beta.1", + "name": "@runejs/filestore", + "version": "1.0.0-next.0", "license": "GPL-3.0", "dependencies": { "@runejs/common": "2.0.2-beta.2", - "graceful-fs": "^4.2.0", + "adm-zip": "^0.5.9", + "axios": "^0.27.2", + "graceful-fs": ">=4.2.0", "json5": "^2.2.0", "reflect-metadata": "^0.1.13", "sqlite": "^4.0.25", - "tslib": "^2.3.1", + "tslib": ">=2.3.0", "typeorm": "^0.2.44", "yargs": "^17.3.1" }, "devDependencies": { "@runejs/eslint-config": "^1.1.0", + "@types/adm-zip": "^0.5.0", "@types/graceful-fs": "^4.1.5", "@types/node": "^16.11.26", "@types/yargs": "^17.0.9", @@ -201,6 +204,15 @@ "resolved": "https://registry.npmjs.org/@sqltools/formatter/-/formatter-1.2.3.tgz", "integrity": "sha512-O3uyB/JbkAEMZaP3YqyHH7TMnex7tWyCbCI4EfJdOCoN6HIhqdJBWTM6aCCiWQ/5f5wxjgU735QAIpJbjDvmzg==" }, + "node_modules/@types/adm-zip": { + "version": "0.5.0", + "resolved": "https://registry.npmjs.org/@types/adm-zip/-/adm-zip-0.5.0.tgz", + "integrity": "sha512-FCJBJq9ODsQZUNURo5ILAQueuA8WJhRvuihS3ke2iI25mJlfV2LK8jG2Qj2z2AWg8U0FtWWqBHVRetceLskSaw==", + "dev": true, + "dependencies": { + "@types/node": "*" + } + }, "node_modules/@types/graceful-fs": { "version": "4.1.5", "resolved": "https://registry.npmjs.org/@types/graceful-fs/-/graceful-fs-4.1.5.tgz", @@ -459,6 +471,14 @@ "acorn": "^6.0.0 || ^7.0.0 || ^8.0.0" } }, + "node_modules/adm-zip": { + "version": "0.5.9", + "resolved": "https://registry.npmjs.org/adm-zip/-/adm-zip-0.5.9.tgz", + "integrity": "sha512-s+3fXLkeeLjZ2kLjCBwQufpI5fuN+kIGBxu6530nVQZGVol0d7Y/M88/xw9HGGUcJjKf8LutN3VPRUBq6N7Ajg==", + "engines": { + "node": ">=6.0" + } + }, "node_modules/ajv": { "version": "6.12.6", "resolved": "https://registry.npmjs.org/ajv/-/ajv-6.12.6.tgz", @@ -595,6 +615,11 @@ "node": ">=8" } }, + "node_modules/asynckit": { + "version": "0.4.0", + "resolved": "https://registry.npmjs.org/asynckit/-/asynckit-0.4.0.tgz", + "integrity": "sha512-Oei9OH4tRh0YqU3GxhX79dM/mwVgvbZJaSNaRk+bshkj0S5cfHcgYakreBjrHwatXKbz+IoIdYLxrKim2MjW0Q==" + }, "node_modules/atomic-sleep": { "version": "1.0.0", "resolved": "https://registry.npmjs.org/atomic-sleep/-/atomic-sleep-1.0.0.tgz", @@ -603,6 +628,15 @@ "node": ">=8.0.0" } }, + "node_modules/axios": { + "version": "0.27.2", + "resolved": "https://registry.npmjs.org/axios/-/axios-0.27.2.tgz", + "integrity": "sha512-t+yRIyySRTp/wua5xEr+z1q60QmLq8ABsS5O9Me1AsE5dfKqgnCFzwiCZZ/cGNd1lq4/7akDWMxdhVlucjmnOQ==", + "dependencies": { + "follow-redirects": "^1.14.9", + "form-data": "^4.0.0" + } + }, "node_modules/balanced-match": { "version": "1.0.0", "resolved": "https://registry.npmjs.org/balanced-match/-/balanced-match-1.0.0.tgz", @@ -1003,6 +1037,17 @@ "resolved": "https://registry.npmjs.org/color-name/-/color-name-1.1.3.tgz", "integrity": "sha1-p9BVi9icQveV3UIyj3QIMcpTvCU=" }, + "node_modules/combined-stream": { + "version": "1.0.8", + "resolved": "https://registry.npmjs.org/combined-stream/-/combined-stream-1.0.8.tgz", + "integrity": "sha512-FQN4MRfuJeHf7cBbBMJFXhKSDq+2kAArBlmRBvcvFE5BB1HZKXtSFASDhdlz9zOYwxh8lDdnvmMOe/+5cdoEdg==", + "dependencies": { + "delayed-stream": "~1.0.0" + }, + "engines": { + "node": ">= 0.8" + } + }, "node_modules/commander": { "version": "2.8.1", "resolved": "https://registry.npmjs.org/commander/-/commander-2.8.1.tgz", @@ -1207,6 +1252,14 @@ "integrity": "sha512-oIPzksmTg4/MriiaYGO+okXDT7ztn/w3Eptv/+gSIdMdKsJo0u4CfYNFJPy+4SKMuCqGw2wxnA+URMg3t8a/bQ==", "dev": true }, + "node_modules/delayed-stream": { + "version": "1.0.0", + "resolved": "https://registry.npmjs.org/delayed-stream/-/delayed-stream-1.0.0.tgz", + "integrity": "sha512-ZySD7Nf91aLB0RxL4KGrKHBXl7Eds1DAmEdcoVawXnLD7SDhpNgtuII2aAkg7a7QS41jxPSZ17p4VdGnMHk3MQ==", + "engines": { + "node": ">=0.4.0" + } + }, "node_modules/delegates": { "version": "1.0.0", "resolved": "https://registry.npmjs.org/delegates/-/delegates-1.0.0.tgz", @@ -1683,6 +1736,38 @@ "integrity": "sha512-WIWGi2L3DyTUvUrwRKgGi9TwxQMUEqPOPQBVi71R96jZXJdFskXEmf54BoZaS1kknGODoIGASGEzBUYdyMCBJg==", "dev": true }, + "node_modules/follow-redirects": { + "version": "1.15.1", + "resolved": "https://registry.npmjs.org/follow-redirects/-/follow-redirects-1.15.1.tgz", + "integrity": "sha512-yLAMQs+k0b2m7cVxpS1VKJVvoz7SS9Td1zss3XRwXj+ZDH00RJgnuLx7E44wx02kQLrdM3aOOy+FpzS7+8OizA==", + "funding": [ + { + "type": "individual", + "url": "https://github.com/sponsors/RubenVerborgh" + } + ], + "engines": { + "node": ">=4.0" + }, + "peerDependenciesMeta": { + "debug": { + "optional": true + } + } + }, + "node_modules/form-data": { + "version": "4.0.0", + "resolved": "https://registry.npmjs.org/form-data/-/form-data-4.0.0.tgz", + "integrity": "sha512-ETEklSGi5t0QMZuiXoA/Q6vcnxcLQP5vdugSpuAyi6SVGi2clPPp+xgEhuMaHC+zGgn31Kd235W35f7Hykkaww==", + "dependencies": { + "asynckit": "^0.4.0", + "combined-stream": "^1.0.8", + "mime-types": "^2.1.12" + }, + "engines": { + "node": ">= 6" + } + }, "node_modules/fs-constants": { "version": "1.0.0", "resolved": "https://registry.npmjs.org/fs-constants/-/fs-constants-1.0.0.tgz", @@ -2132,6 +2217,25 @@ "node": ">=8.6" } }, + "node_modules/mime-db": { + "version": "1.52.0", + "resolved": "https://registry.npmjs.org/mime-db/-/mime-db-1.52.0.tgz", + "integrity": "sha512-sPU4uV7dYlvtWJxwwxHD0PuihVNiE7TyAbQ5SWxDCB9mUYvOgroQOwYQQOKPJ8CIbE+1ETVlOoK1UC2nU3gYvg==", + "engines": { + "node": ">= 0.6" + } + }, + "node_modules/mime-types": { + "version": "2.1.35", + "resolved": "https://registry.npmjs.org/mime-types/-/mime-types-2.1.35.tgz", + "integrity": "sha512-ZDY+bPm5zTTF+YpCrAU9nK0UgICYPT0QtT1NZWFv4s++TNkcgVaT0g6+4R2uI4MjQjzysHB1zxuWL50hzaeXiw==", + "dependencies": { + "mime-db": "1.52.0" + }, + "engines": { + "node": ">= 0.6" + } + }, "node_modules/mimic-response": { "version": "3.1.0", "resolved": "https://registry.npmjs.org/mimic-response/-/mimic-response-3.1.0.tgz", @@ -3773,6 +3877,15 @@ "resolved": "https://registry.npmjs.org/@sqltools/formatter/-/formatter-1.2.3.tgz", "integrity": "sha512-O3uyB/JbkAEMZaP3YqyHH7TMnex7tWyCbCI4EfJdOCoN6HIhqdJBWTM6aCCiWQ/5f5wxjgU735QAIpJbjDvmzg==" }, + "@types/adm-zip": { + "version": "0.5.0", + "resolved": "https://registry.npmjs.org/@types/adm-zip/-/adm-zip-0.5.0.tgz", + "integrity": "sha512-FCJBJq9ODsQZUNURo5ILAQueuA8WJhRvuihS3ke2iI25mJlfV2LK8jG2Qj2z2AWg8U0FtWWqBHVRetceLskSaw==", + "dev": true, + "requires": { + "@types/node": "*" + } + }, "@types/graceful-fs": { "version": "4.1.5", "resolved": "https://registry.npmjs.org/@types/graceful-fs/-/graceful-fs-4.1.5.tgz", @@ -3934,6 +4047,11 @@ "dev": true, "requires": {} }, + "adm-zip": { + "version": "0.5.9", + "resolved": "https://registry.npmjs.org/adm-zip/-/adm-zip-0.5.9.tgz", + "integrity": "sha512-s+3fXLkeeLjZ2kLjCBwQufpI5fuN+kIGBxu6530nVQZGVol0d7Y/M88/xw9HGGUcJjKf8LutN3VPRUBq6N7Ajg==" + }, "ajv": { "version": "6.12.6", "resolved": "https://registry.npmjs.org/ajv/-/ajv-6.12.6.tgz", @@ -4044,11 +4162,25 @@ "integrity": "sha512-HGyxoOTYUyCM6stUe6EJgnd4EoewAI7zMdfqO+kGjnlZmBDz/cR5pf8r/cR4Wq60sL/p0IkcjUEEPwS3GFrIyw==", "dev": true }, + "asynckit": { + "version": "0.4.0", + "resolved": "https://registry.npmjs.org/asynckit/-/asynckit-0.4.0.tgz", + "integrity": "sha512-Oei9OH4tRh0YqU3GxhX79dM/mwVgvbZJaSNaRk+bshkj0S5cfHcgYakreBjrHwatXKbz+IoIdYLxrKim2MjW0Q==" + }, "atomic-sleep": { "version": "1.0.0", "resolved": "https://registry.npmjs.org/atomic-sleep/-/atomic-sleep-1.0.0.tgz", "integrity": "sha512-kNOjDqAh7px0XWNI+4QbzoiR/nTkHAWNud2uvnJquD1/x5a7EQZMJT0AczqK0Qn67oY/TTQ1LbUKajZpp3I9tQ==" }, + "axios": { + "version": "0.27.2", + "resolved": "https://registry.npmjs.org/axios/-/axios-0.27.2.tgz", + "integrity": "sha512-t+yRIyySRTp/wua5xEr+z1q60QmLq8ABsS5O9Me1AsE5dfKqgnCFzwiCZZ/cGNd1lq4/7akDWMxdhVlucjmnOQ==", + "requires": { + "follow-redirects": "^1.14.9", + "form-data": "^4.0.0" + } + }, "balanced-match": { "version": "1.0.0", "resolved": "https://registry.npmjs.org/balanced-match/-/balanced-match-1.0.0.tgz", @@ -4341,6 +4473,14 @@ "resolved": "https://registry.npmjs.org/color-name/-/color-name-1.1.3.tgz", "integrity": "sha1-p9BVi9icQveV3UIyj3QIMcpTvCU=" }, + "combined-stream": { + "version": "1.0.8", + "resolved": "https://registry.npmjs.org/combined-stream/-/combined-stream-1.0.8.tgz", + "integrity": "sha512-FQN4MRfuJeHf7cBbBMJFXhKSDq+2kAArBlmRBvcvFE5BB1HZKXtSFASDhdlz9zOYwxh8lDdnvmMOe/+5cdoEdg==", + "requires": { + "delayed-stream": "~1.0.0" + } + }, "commander": { "version": "2.8.1", "resolved": "https://registry.npmjs.org/commander/-/commander-2.8.1.tgz", @@ -4496,6 +4636,11 @@ "integrity": "sha512-oIPzksmTg4/MriiaYGO+okXDT7ztn/w3Eptv/+gSIdMdKsJo0u4CfYNFJPy+4SKMuCqGw2wxnA+URMg3t8a/bQ==", "dev": true }, + "delayed-stream": { + "version": "1.0.0", + "resolved": "https://registry.npmjs.org/delayed-stream/-/delayed-stream-1.0.0.tgz", + "integrity": "sha512-ZySD7Nf91aLB0RxL4KGrKHBXl7Eds1DAmEdcoVawXnLD7SDhpNgtuII2aAkg7a7QS41jxPSZ17p4VdGnMHk3MQ==" + }, "delegates": { "version": "1.0.0", "resolved": "https://registry.npmjs.org/delegates/-/delegates-1.0.0.tgz", @@ -4865,6 +5010,21 @@ "integrity": "sha512-WIWGi2L3DyTUvUrwRKgGi9TwxQMUEqPOPQBVi71R96jZXJdFskXEmf54BoZaS1kknGODoIGASGEzBUYdyMCBJg==", "dev": true }, + "follow-redirects": { + "version": "1.15.1", + "resolved": "https://registry.npmjs.org/follow-redirects/-/follow-redirects-1.15.1.tgz", + "integrity": "sha512-yLAMQs+k0b2m7cVxpS1VKJVvoz7SS9Td1zss3XRwXj+ZDH00RJgnuLx7E44wx02kQLrdM3aOOy+FpzS7+8OizA==" + }, + "form-data": { + "version": "4.0.0", + "resolved": "https://registry.npmjs.org/form-data/-/form-data-4.0.0.tgz", + "integrity": "sha512-ETEklSGi5t0QMZuiXoA/Q6vcnxcLQP5vdugSpuAyi6SVGi2clPPp+xgEhuMaHC+zGgn31Kd235W35f7Hykkaww==", + "requires": { + "asynckit": "^0.4.0", + "combined-stream": "^1.0.8", + "mime-types": "^2.1.12" + } + }, "fs-constants": { "version": "1.0.0", "resolved": "https://registry.npmjs.org/fs-constants/-/fs-constants-1.0.0.tgz", @@ -5200,6 +5360,19 @@ "picomatch": "^2.2.3" } }, + "mime-db": { + "version": "1.52.0", + "resolved": "https://registry.npmjs.org/mime-db/-/mime-db-1.52.0.tgz", + "integrity": "sha512-sPU4uV7dYlvtWJxwwxHD0PuihVNiE7TyAbQ5SWxDCB9mUYvOgroQOwYQQOKPJ8CIbE+1ETVlOoK1UC2nU3gYvg==" + }, + "mime-types": { + "version": "2.1.35", + "resolved": "https://registry.npmjs.org/mime-types/-/mime-types-2.1.35.tgz", + "integrity": "sha512-ZDY+bPm5zTTF+YpCrAU9nK0UgICYPT0QtT1NZWFv4s++TNkcgVaT0g6+4R2uI4MjQjzysHB1zxuWL50hzaeXiw==", + "requires": { + "mime-db": "1.52.0" + } + }, "mimic-response": { "version": "3.1.0", "resolved": "https://registry.npmjs.org/mimic-response/-/mimic-response-3.1.0.tgz", diff --git a/package.json b/package.json index 1955dda..c34f4d9 100644 --- a/package.json +++ b/package.json @@ -19,9 +19,8 @@ "lint": "eslint --ext .ts src", "lint:fix": "eslint --ext .ts src --fix", "unpack": "ts-node-dev --max-old-space-size=2048 src/scripts/unpacker.ts", - "unpacker": "npm run unpack", "index": "ts-node-dev --max-old-space-size=2048 src/scripts/indexer.ts", - "indexer": "npm run index", + "builds": "ts-node-dev src/scripts/builds.ts", "copy-documents": "copyfiles package.json README.md .npmignore LICENSE lib", "package": "rimraf lib && npm i && npm run build && npm run copy-documents && cd lib && npm publish --dry-run", "publish:next": "npm run package && cd lib && npm publish -tag next", @@ -55,16 +54,19 @@ }, "dependencies": { "@runejs/common": "2.0.2-beta.2", - "graceful-fs": "^4.2.0", + "adm-zip": "^0.5.9", + "axios": "^0.27.2", + "graceful-fs": ">=4.2.0", "json5": "^2.2.0", "reflect-metadata": "^0.1.13", "sqlite": "^4.0.25", - "tslib": "^2.3.1", + "tslib": ">=2.3.0", "typeorm": "^0.2.44", "yargs": "^17.3.1" }, "devDependencies": { "@runejs/eslint-config": "^1.1.0", + "@types/adm-zip": "^0.5.0", "@types/graceful-fs": "^4.1.5", "@types/node": "^16.11.26", "@types/yargs": "^17.0.9", diff --git a/src/db/index-database.ts b/src/db/index-database.ts index b33b640..b363e54 100644 --- a/src/db/index-database.ts +++ b/src/db/index-database.ts @@ -40,6 +40,10 @@ export class IndexDatabase { return this._connection; } + async closeConnection(): Promise { + await this._connection.close(); + } + async upsertIndexes(indexEntities: IndexEntity[]): Promise { const chunkSize = 100; for (let i = 0; i < indexEntities.length; i += chunkSize) { diff --git a/src/file-system/file-store-base.ts b/src/file-system/file-store-base.ts index 1371610..b3030f4 100644 --- a/src/file-system/file-store-base.ts +++ b/src/file-system/file-store-base.ts @@ -65,6 +65,10 @@ export abstract class FileStoreBase, C> { return this._database; } + async closeDatabase(): Promise { + await this._database.closeConnection(); + } + getArchive(archiveKey: number): A | null { return this.archives.get(archiveKey) || null; } diff --git a/src/file-system/jag/jag.ts b/src/file-system/jag/jag.ts index a1ed0bf..f0486a1 100644 --- a/src/file-system/jag/jag.ts +++ b/src/file-system/jag/jag.ts @@ -15,7 +15,7 @@ export class Jag { this.jagStore = jagStore; } - loadJagFiles(): void { + loadLocalJagFiles(): void { const jagStorePath = join(this.jagStore.fileStorePath, 'jag'); if (!existsSync(jagStorePath)) { diff --git a/src/file-system/js5/js5.ts b/src/file-system/js5/js5.ts index 0169f6e..8314ef7 100644 --- a/src/file-system/js5/js5.ts +++ b/src/file-system/js5/js5.ts @@ -9,6 +9,12 @@ import { Archive } from './archive'; import { JS5FileStore } from './js5-file-store'; import { ArchiveFormat } from '../../config'; import { FlatFile } from './flat-file'; +import { OpenRS2CacheFile } from '../../openrs2'; + + +const dataFileName = 'main_file_cache.dat2'; +const indexFileNamePrefix = 'main_file_cache.idx'; +const mainIndexFileName = `${ indexFileNamePrefix }255`; export class JS5 { @@ -23,10 +29,55 @@ export class JS5 { constructor(fileStore: JS5FileStore) { this.fileStore = fileStore; + this.indexFiles = new Map(); this.loadEncryptionKeys(); } - loadJS5Files(): void { + loadOpenRS2CacheFiles(cacheFiles: OpenRS2CacheFile[]): void { + const dataFileBuffer = cacheFiles.find(file => file.name === dataFileName)?.data || null; + if (!dataFileBuffer?.length) { + throw new Error(`The main ${dataFileName} data file could not be found.`); + } + + const mainIndexFileBuffer = cacheFiles.find(file => file.name === mainIndexFileName)?.data || null; + if (!mainIndexFileBuffer?.length) { + throw new Error(`The main ${mainIndexFileName} index file could not be found.`); + } + + this.dataFile = new ByteBuffer(dataFileBuffer); + this.mainIndexFile = new ByteBuffer(mainIndexFileBuffer); + this.indexFiles.clear(); + + for (const cacheFile of cacheFiles) { + const fileName = cacheFile?.name; + + if (!fileName?.length || fileName === mainIndexFileName || fileName === dataFileName) { + continue; + } + + if (!fileName.startsWith(indexFileNamePrefix)) { + continue; + } + + if (!cacheFile?.data?.length) { + logger.error(`Index file ${fileName} is empty!`); + continue; + } + + const index = fileName.substring(fileName.indexOf('.idx') + 4); + const numericIndex = Number(index); + + if (isNaN(numericIndex)) { + logger.error(`Index file ${fileName} does not have a valid extension.`); + } + + this.indexFiles.set(numericIndex, new ByteBuffer(cacheFile.data)); + } + + logger.info(`JS5 store file loaded for game build ${this.fileStore.gameBuild}.`); + } + + loadLocalCacheFiles(): void { const js5StorePath = join(this.fileStore.fileStorePath, 'js5'); if (!existsSync(js5StorePath)) { @@ -39,31 +90,28 @@ export class JS5 { } const storeFileNames = readdirSync(js5StorePath); - const dataFileName = 'main_file_cache.dat2'; - const mainIndexFile = 'main_file_cache.idx255'; if (storeFileNames.indexOf(dataFileName) === -1) { throw new Error(`The main ${dataFileName} data file could not be found.`); } - if (storeFileNames.indexOf(mainIndexFile) === -1) { - throw new Error(`The main ${mainIndexFile} index file could not be found.`); + if (storeFileNames.indexOf(mainIndexFileName) === -1) { + throw new Error(`The main ${mainIndexFileName} index file could not be found.`); } - const indexFilePrefix = 'main_file_cache.idx'; const dataFilePath = join(js5StorePath, dataFileName); - const mainIndexFilePath = join(js5StorePath, mainIndexFile); + const mainIndexFilePath = join(js5StorePath, mainIndexFileName); this.dataFile = new ByteBuffer(readFileSync(dataFilePath)); this.mainIndexFile = new ByteBuffer(readFileSync(mainIndexFilePath)); - this.indexFiles = new Map(); + this.indexFiles.clear(); for (const fileName of storeFileNames) { - if (!fileName?.length || fileName === mainIndexFile || fileName === dataFileName) { + if (!fileName?.length || fileName === mainIndexFileName || fileName === dataFileName) { continue; } - if (!fileName.startsWith(indexFilePrefix)) { + if (!fileName.startsWith(indexFileNamePrefix)) { continue; } diff --git a/src/openrs2/index.ts b/src/openrs2/index.ts new file mode 100644 index 0000000..227d6fa --- /dev/null +++ b/src/openrs2/index.ts @@ -0,0 +1 @@ +export * from './openrs2'; diff --git a/src/openrs2/openrs2.ts b/src/openrs2/openrs2.ts new file mode 100644 index 0000000..e2b8dd2 --- /dev/null +++ b/src/openrs2/openrs2.ts @@ -0,0 +1,146 @@ +import axios from 'axios'; +import AdmZip, { IZipEntry } from 'adm-zip'; +import { Buffer } from 'buffer'; +import { logger } from '@runejs/common'; + + +const openRS2Endpoint = 'https://archive.openrs2.org'; + + +export interface OpenRS2BuildNumber { + major: number; + minor: number | null; +} + + +export interface OpenRS2Cache { + id: number; + scope: 'runescape' | string; + game: 'runescape' | 'darkscape' | string; + environment: 'live' | 'beta' | string; + language: 'en' | 'de' | 'fr' | 'pt' | string; + builds: OpenRS2BuildNumber[]; + timestamp: Date; // ISO 8601 format + sources: string[]; + valid_indexes: number | null; + indexes: number | null; + valid_groups: number | null; + groups: number | null; + valid_keys: number | null; + keys: number | null; + size: number | null; + blocks: number | null; + disk_store_valid: boolean | null; +} + + +export interface OpenRS2CacheFile { + name: string; + data: Buffer; +} + + +export const getCacheFormat = (files: OpenRS2CacheFile[]): 'js5' | 'jag' => { + return files.find( + file => file.name === 'main_file_cache.dat2' || file.name === 'main_file_cache.idx255' + ) !== null ? 'js5' : 'jag'; +}; + + +export const getOpenRS2CacheList = async (): Promise => { + const response = await axios.get( + `${ openRS2Endpoint }/caches.json` + ); + return response.data; +}; + + +export const getAvailableBuilds = async ( + scope: string = 'runescape', + game: string = 'runescape' +): Promise => { + const cacheList = (await getOpenRS2CacheList()) + .filter(cacheDetails => + // Filter out caches by desired scope and game + cacheDetails.scope === scope && cacheDetails.game === game + ) + .map(cacheDetails => { + // Map cache list to a list of build numbers + + if (!cacheDetails?.builds?.length) { + return [ -1 ]; + } + + // Map the build number arrays to actual numbers + return cacheDetails.builds.map(build => { + if (!build) { + return -1; + } + + if (build.minor !== null && build.minor > 0) { + return parseFloat(build.major + '.' + build.minor); + } else { + return build.major; + } + }); + }) + // Flatten the array + .flat() + // Filter out anything less than or equal to 0 + .filter(buildNumber => buildNumber > 0) + // Sort the list of numbers + .sort((a, b) => a - b); + + // Remove any duplicates + return [ ...new Set(cacheList) ]; +}; + + +export const getOpenRS2CacheById = async ( + id: number, + scope: string = 'runescape' +): Promise => { + const response = await axios.get( + `${ openRS2Endpoint }/caches/${ scope }/${ id }/disk.zip`, + { + responseType: 'arraybuffer' + } + ); + + const zip = new AdmZip(Buffer.from(response.data, 'binary')); + return zip.getEntries().map(entry => ({ + name: entry.name, + data: entry.getData() + })); +}; + + +export const getOpenRS2CacheByBuild = async ( + build: number +): Promise => { + logger.info(`Searching OpenRS2 for build ${ build }...`); + + const cacheList = (await getOpenRS2CacheList()) + .filter(c => c.scope === 'runescape' && c.game === 'runescape'); + + const desiredCacheInfo = cacheList.find(cacheDetails => { + for (const b of cacheDetails.builds) { + if (b.major === build) { + return true; + } + } + + return false; + }); + + if (desiredCacheInfo) { + logger.info(`Build ${ build } was found within the OpenRS2 archive, fetching data...`); + + const cacheFiles = await getOpenRS2CacheById(desiredCacheInfo.id); + return cacheFiles?.length ? cacheFiles : null; + } else { + logger.error(`Build ${ build } was not found within the OpenRS2.org archive.`); + } + + return null; +}; diff --git a/src/scripts/builds.ts b/src/scripts/builds.ts new file mode 100644 index 0000000..4d8a021 --- /dev/null +++ b/src/scripts/builds.ts @@ -0,0 +1,70 @@ +import { ScriptExecutor, ArgumentOptions } from './script-executor'; +import { + getAvailableBuilds +} from '../openrs2'; +import { logger } from '@runejs/common'; + + +interface BuildsOptions { + scope: string; + game: string; + range?: number; +} + + +const buildsArgumentOptions: ArgumentOptions = { + scope: { + alias: 's', + type: 'string', + default: 'runescape', + description: `The scope to search for available builds for within OpenRS2.org. Defaults to 'runescape'.` + }, + game: { + alias: 'g', + type: 'string', + default: 'runescape', + description: `The game to search for available builds for within OpenRS2.org. Defaults to 'runescape'.` + }, + range: { + alias: 'r', + type: 'number', + description: `The lower number range of builds to search for, ie '400' to return builds between 400-499. Returns all build ranges by default.` + }, +}; + + +const buildsScript = async ( + { scope, game, range } +) => { + if (range && range % 100 !== 0) { + logger.error(`Invalid range of ${ range }: Range must be a multiple of 100.`); + return; + } + + const availableBuilds = await getAvailableBuilds(scope, game); + let msg = `Available builds on OpenRS2.org with scope ${ scope } and game ${ game }`; + + if (range) { + msg += ` for build range ${ range } - ${ range + 99 }`; + } + + logger.info(`${ msg }:`); + + let lastHundred = 0; + + for (const buildNumber of availableBuilds) { + const lower100 = Math.floor(buildNumber / 100) * 100; + + if (!range || range === lower100) { + if (lastHundred !== lower100) { + logger.info(`[ ${ lower100 } - ${ lower100 + 99 } ]`); + lastHundred = lower100; + } + + logger.info(String(buildNumber)); + } + } +}; + + +new ScriptExecutor().executeScript(buildsArgumentOptions, buildsScript); diff --git a/src/scripts/dev.ts b/src/scripts/dev.ts index 0c4c54f..9b1d2c9 100644 --- a/src/scripts/dev.ts +++ b/src/scripts/dev.ts @@ -6,7 +6,7 @@ const dev = async () => { const start = Date.now(); const fileStore = new JS5FileStore(435); await fileStore.load(); - fileStore.js5.loadJS5Files(); + fileStore.js5.loadLocalCacheFiles(); logger.info(`Unpacking archives from JS5 store...`); diff --git a/src/scripts/indexer.ts b/src/scripts/indexer.ts index 4e6796b..2a6867f 100644 --- a/src/scripts/indexer.ts +++ b/src/scripts/indexer.ts @@ -4,8 +4,10 @@ import { logger } from '@runejs/common'; import { ScriptExecutor, ArgumentOptions } from './script-executor'; import { JS5FileStore } from '../file-system/js5/js5-file-store'; import { JagStore } from '../file-system/jag/jag-store'; -import { FileStoreBase } from '../file-system/file-store-base'; -import { Archive } from '../file-system/js5/archive'; +import { + getOpenRS2CacheByBuild, + OpenRS2CacheFile +} from '../openrs2'; interface IndexerOptions { @@ -17,25 +19,38 @@ interface IndexerOptions { const indexerArgumentOptions: ArgumentOptions = { + archive: { + alias: 'a', + type: 'string', + default: 'main', + description: `The archive to index. Defaults to 'main', which will index all store archives one by one. Specify an archive name to index a single archive.` + }, + build: { + alias: 'b', + type: 'string', + default: '435', + description: `The game build (revision) that the store should belong to, also known as the game build number. Defaults to '435', a game build from late October, 2006.` + }, dir: { - alias: 'd', type: 'string', default: './', + alias: 'd', + type: 'string', + default: './', description: `The store root directory. Defaults to the current location.` }, format: { alias: 'f', type: 'string', - default: 'unpacked', - choices: [ 'unpacked', 'packed', 'flat', 'js5' ], - description: `The format of the store to index, either 'unpacked' (flat files) or 'packed' (JS5 format). Defaults to 'unpacked'.` + default: 'js5', + choices: [ 'jag', 'js5', 'flat' ], + description: `The format of the store to index, either 'js5' (400+ JS5 format), 'jag' (234-399 .jag format), or 'flat' (flat files). Defaults to 'js5'.` }, - archive: { - alias: 'a', type: 'string', default: 'main', - description: `The archive to index. Defaults to 'main', which will index all store archives one by one. Specify an archive name to index a single archive.` + source: { + alias: 's', + type: 'string', + default: 'openrs2', + choices: [ 'openrs2', 'local' ], + description: `The store source location - either 'openrs2' to pull cache and xtea files from OpenRS2.org or 'local' for caches and xtea files stored locally.` }, - build: { - alias: 'b', type: 'string', default: '435', - description: `The game build (revision) that the store should belong to, also known as the game build number. Defaults to '435', a game build from late October, 2006.` - } }; @@ -149,37 +164,60 @@ const indexJagArchive = async (store: JagStore, archiveName: string) => { const indexerScript = async ( - terminal, - { build, dir, format, archive: archiveName } + { build, dir, format, archive: archiveName, source } ) => { const start = Date.now(); const logDir = join(dir, 'logs'); - const numericBuildNumber: boolean = /^\d+$/.test(build); - + const numericBuildNumber: number = /^\d+$/.test(build) ? parseInt(build, 10) : -1; + let cacheFiles: OpenRS2CacheFile[] | 'local' = 'local'; + if (!existsSync(logDir)) { mkdirSync(logDir, { recursive: true }); } logger.destination(join(logDir, `index-${ format }-${ build }.log`)); + if (source === 'openrs2') { + if (numericBuildNumber) { + const openRS2CacheFiles = await getOpenRS2CacheByBuild(numericBuildNumber); + if (!openRS2CacheFiles?.length) { + return; + } + + cacheFiles = openRS2CacheFiles; + } else { + logger.error(`A numeric build number must be used in order to pull cache information from OpenRS2.org.`); + return; + } + } + logger.info(`Indexing ${ format } file store...`); if (format === 'js5') { - const store = new JS5FileStore(numericBuildNumber ? Number(build) : build, dir); + const store = new JS5FileStore(numericBuildNumber !== -1 ? numericBuildNumber : build, dir); await store.load(); - store.js5.loadJS5Files(); + + if (cacheFiles === 'local') { + store.js5.loadLocalCacheFiles(); + } else { + store.js5.loadOpenRS2CacheFiles(cacheFiles); + } if (archiveName === 'main') { await indexJS5Store(store); } else { await indexJS5Archive(store, archiveName); } + + await store.closeDatabase(); } else if (format === 'jag') { - const store = new JagStore(numericBuildNumber ? Number(build) : build, dir); + const store = new JagStore(numericBuildNumber !== -1 ? numericBuildNumber : build, dir); await store.load(); - store.jag.loadJagFiles(); + store.jag.loadLocalJagFiles(); // @todo 18/07/22 - Kiko + + await store.closeDatabase(); } else if (format === 'flat') { // @todo 18/07/22 - Kiko } diff --git a/src/scripts/script-executor.ts b/src/scripts/script-executor.ts index ae83274..a13c322 100644 --- a/src/scripts/script-executor.ts +++ b/src/scripts/script-executor.ts @@ -14,11 +14,11 @@ export class ScriptExecutor { public executeScript( argumentOptions: ArgumentOptions, - script: (terminalInterface: ScriptExecutor, args: T - ) => Promise): void { - (async function(terminal: ScriptExecutor, args: T) { - await script(terminal, args); - }(this, this.getArguments(argumentOptions))) + script: (args: T) => Promise + ): void { + (async function(args: T) { + await script(args); + }(this.getArguments(argumentOptions))) .catch(logger.error) .finally(() => process.exit(0)); } From 12e244be600b346e20d5768b41ec116ce9e1846b Mon Sep 17 00:00:00 2001 From: Kikorono Date: Wed, 20 Jul 2022 14:06:41 -0500 Subject: [PATCH 17/32] XTEA keys will now be fetched from OpenRS2.org, if none are found then local files will be used --- src/file-system/js5/js5-file-store.ts | 1 + src/file-system/js5/js5.ts | 114 +++++++++++++++----------- src/openrs2/openrs2.ts | 51 ++++++++++-- src/scripts/builds.ts | 2 +- src/scripts/indexer.ts | 6 +- src/scripts/script-executor.ts | 6 +- 6 files changed, 118 insertions(+), 62 deletions(-) diff --git a/src/file-system/js5/js5-file-store.ts b/src/file-system/js5/js5-file-store.ts index 494b6e5..39f25b9 100644 --- a/src/file-system/js5/js5-file-store.ts +++ b/src/file-system/js5/js5-file-store.ts @@ -17,6 +17,7 @@ export class JS5FileStore extends FileStoreBase{ } override async load(): Promise { + await this.js5.loadEncryptionKeys(); await this.openDatabase(); const archiveNames = Object.keys(this.archiveConfig); diff --git a/src/file-system/js5/js5.ts b/src/file-system/js5/js5.ts index 8314ef7..e3ce355 100644 --- a/src/file-system/js5/js5.ts +++ b/src/file-system/js5/js5.ts @@ -9,7 +9,8 @@ import { Archive } from './archive'; import { JS5FileStore } from './js5-file-store'; import { ArchiveFormat } from '../../config'; import { FlatFile } from './flat-file'; -import { OpenRS2CacheFile } from '../../openrs2'; +import { getXteaKeysByBuild, OpenRS2CacheFile } from '../../openrs2'; +import { XteaConfig } from '@runejs/common/encrypt/xtea'; const dataFileName = 'main_file_cache.dat2'; @@ -21,7 +22,8 @@ export class JS5 { readonly fileStore: JS5FileStore; - encryptionKeys: Map; + localEncryptionKeys: Map; + openRS2EncryptionKeys: XteaConfig[]; private mainIndexFile: ByteBuffer; private indexFiles: Map; @@ -30,18 +32,17 @@ export class JS5 { constructor(fileStore: JS5FileStore) { this.fileStore = fileStore; this.indexFiles = new Map(); - this.loadEncryptionKeys(); } loadOpenRS2CacheFiles(cacheFiles: OpenRS2CacheFile[]): void { const dataFileBuffer = cacheFiles.find(file => file.name === dataFileName)?.data || null; if (!dataFileBuffer?.length) { - throw new Error(`The main ${dataFileName} data file could not be found.`); + throw new Error(`The main ${ dataFileName } data file could not be found.`); } const mainIndexFileBuffer = cacheFiles.find(file => file.name === mainIndexFileName)?.data || null; if (!mainIndexFileBuffer?.length) { - throw new Error(`The main ${mainIndexFileName} index file could not be found.`); + throw new Error(`The main ${ mainIndexFileName } index file could not be found.`); } this.dataFile = new ByteBuffer(dataFileBuffer); @@ -60,7 +61,7 @@ export class JS5 { } if (!cacheFile?.data?.length) { - logger.error(`Index file ${fileName} is empty!`); + logger.error(`Index file ${ fileName } is empty!`); continue; } @@ -68,35 +69,35 @@ export class JS5 { const numericIndex = Number(index); if (isNaN(numericIndex)) { - logger.error(`Index file ${fileName} does not have a valid extension.`); + logger.error(`Index file ${ fileName } does not have a valid extension.`); } this.indexFiles.set(numericIndex, new ByteBuffer(cacheFile.data)); } - logger.info(`JS5 store file loaded for game build ${this.fileStore.gameBuild}.`); + logger.info(`JS5 store file loaded for game build ${ this.fileStore.gameBuild }.`); } loadLocalCacheFiles(): void { const js5StorePath = join(this.fileStore.fileStorePath, 'js5'); if (!existsSync(js5StorePath)) { - throw new Error(`${js5StorePath} could not be found.`); + throw new Error(`${ js5StorePath } could not be found.`); } const stats = statSync(js5StorePath); if (!stats?.isDirectory()) { - throw new Error(`${js5StorePath} is not a valid directory.`); + throw new Error(`${ js5StorePath } is not a valid directory.`); } const storeFileNames = readdirSync(js5StorePath); if (storeFileNames.indexOf(dataFileName) === -1) { - throw new Error(`The main ${dataFileName} data file could not be found.`); + throw new Error(`The main ${ dataFileName } data file could not be found.`); } if (storeFileNames.indexOf(mainIndexFileName) === -1) { - throw new Error(`The main ${mainIndexFileName} index file could not be found.`); + throw new Error(`The main ${ mainIndexFileName } index file could not be found.`); } const dataFilePath = join(js5StorePath, dataFileName); @@ -119,13 +120,13 @@ export class JS5 { const numericIndex = Number(index); if (isNaN(numericIndex)) { - logger.error(`Index file ${fileName} does not have a valid extension.`); + logger.error(`Index file ${ fileName } does not have a valid extension.`); } this.indexFiles.set(numericIndex, new ByteBuffer(readFileSync(join(js5StorePath, fileName)))); } - logger.info(`JS5 store file loaded for game build ${this.fileStore.gameBuild}.`); + logger.info(`JS5 store file loaded for game build ${ this.fileStore.gameBuild }.`); } unpack(file: Group | Archive): Buffer | null { @@ -150,7 +151,7 @@ export class JS5 { let pointer = fileKey * indexDataLength; if (pointer < 0 || pointer >= indexChannel.length) { - logger.error(`File ${fileKey} was not found within the ${archiveName} archive index file.`); + logger.error(`File ${ fileKey } was not found within the ${ archiveName } archive index file.`); return null; } @@ -158,7 +159,7 @@ export class JS5 { indexChannel.copy(fileIndexData, 0, pointer, pointer + indexDataLength); if (fileIndexData.readable !== indexDataLength) { - logger.error(`Error extracting JS5 file ${fileKey}: the end of the data stream was reached.`); + logger.error(`Error extracting JS5 file ${ fileKey }: the end of the data stream was reached.`); return null; } @@ -166,7 +167,7 @@ export class JS5 { const stripeCount = fileIndexData.get('int24', 'unsigned'); if (fileDetails.fileSize <= 0) { - logger.warn(`JS5 file ${fileKey} is empty or has been removed.`); + logger.warn(`JS5 file ${ fileKey } is empty or has been removed.`); return null; } @@ -183,7 +184,7 @@ export class JS5 { dataChannel.copy(temp, 0, pointer, pointer + stripeLength); if (temp.readable !== stripeLength) { - logger.error(`Error reading stripe for packed file ${fileKey}, the end of the data stream was reached.`); + logger.error(`Error reading stripe for packed file ${ fileKey }, the end of the data stream was reached.`); return null; } @@ -200,17 +201,17 @@ export class JS5 { remaining -= stripeDataLength; if (stripeArchiveIndex !== archiveKey) { - logger.error(`Archive index mismatch, expected archive ${archiveKey} but found archive ${stripeFileIndex}`); + logger.error(`Archive index mismatch, expected archive ${ archiveKey } but found archive ${ stripeFileIndex }`); return null; } if (stripeFileIndex !== fileKey) { - logger.error(`File index mismatch, expected ${fileKey} but found ${stripeFileIndex}.`); + logger.error(`File index mismatch, expected ${ fileKey } but found ${ stripeFileIndex }.`); return null; } if (currentStripe !== stripe++) { - logger.error(`Error extracting JS5 file ${fileKey}, file data is corrupted.`); + logger.error(`Error extracting JS5 file ${ fileKey }, file data is corrupted.`); return null; } @@ -255,7 +256,7 @@ export class JS5 { const fileName = fileDetails.name; if (!fileDetails.compressedData?.length) { - logger.error(`Error decrypting file ${fileName || fileDetails.key}, file data not found.`, + logger.error(`Error decrypting file ${ fileName || fileDetails.key }, file data not found.`, `Please ensure that the file has been unpacked from an existing JS5 file store using JS5.unpack(file);`); return null; } @@ -269,7 +270,7 @@ export class JS5 { const patternRegex = new RegExp(pattern); // Only XTEA encryption is supported at this time - if(encryption !== 'xtea' || !patternRegex.test(fileName)) { + if (encryption !== 'xtea' || !patternRegex.test(fileName)) { // FileBase name does not match the pattern, data should be unencrypted return fileDetails.compressedData; } @@ -282,7 +283,7 @@ export class JS5 { const loadedKeys = this.getEncryptionKeys(fileName); if (loadedKeys) { - if(!Array.isArray(loadedKeys)) { + if (!Array.isArray(loadedKeys)) { keySets = [ loadedKeys ]; } else { keySets = loadedKeys; @@ -298,7 +299,7 @@ export class JS5 { dataCopy.readerIndex = readerIndex; let lengthOffset = readerIndex; - if(dataCopy.length - (compressedLength + readerIndex + 4) >= 2) { + if (dataCopy.length - (compressedLength + readerIndex + 4) >= 2) { lengthOffset += 2; } @@ -312,12 +313,12 @@ export class JS5 { return fileDetails.compressedData; } else { logger.warn(`Invalid XTEA decryption keys found for file ` + - `${fileName || fileDetails.key} using game build ${ gameBuild }.`); + `${ fileName || fileDetails.key } using game build ${ gameBuild }.`); fileDetails.fileError = 'MISSING_ENCRYPTION_KEYS'; } } else { logger.warn(`No XTEA decryption keys found for file ` + - `${fileName || fileDetails.key} using game build ${gameBuild}.`); + `${ fileName || fileDetails.key } using game build ${ gameBuild }.`); fileDetails.fileError = 'MISSING_ENCRYPTION_KEYS'; } @@ -352,11 +353,11 @@ export class JS5 { // BZIP or GZIP compressed file const decompressedLength = compressedData.get('int', 'unsigned'); if (decompressedLength < 0) { - const errorPrefix = `Unable to decompress file ${fileDetails.name || fileDetails.key}:`; + const errorPrefix = `Unable to decompress file ${ fileDetails.name || fileDetails.key }:`; if (fileDetails.fileError === 'FILE_MISSING') { - logger.error(`${errorPrefix} Missing file data.`); + logger.error(`${ errorPrefix } Missing file data.`); } else { - logger.error(`${errorPrefix} Missing or invalid XTEA key.`); + logger.error(`${ errorPrefix } Missing or invalid XTEA key.`); fileDetails.fileError = 'MISSING_ENCRYPTION_KEYS'; } } else { @@ -376,8 +377,8 @@ export class JS5 { data = null; } } catch (error) { - logger.error(`Error decompressing file ${fileDetails.name || fileDetails.key}: ` + - `${error?.message ?? error}`); + logger.error(`Error decompressing file ${ fileDetails.name || fileDetails.key }: ` + + `${ error?.message ?? error }`); data = null; } } @@ -385,7 +386,7 @@ export class JS5 { if (data?.length) { // Read the file footer, if it has one - if(compressedData.readable >= 2) { + if (compressedData.readable >= 2) { fileDetails.version = compressedData.get('short', 'unsigned'); } @@ -413,7 +414,7 @@ export class JS5 { const data = new ByteBuffer(groupDetails.data); - if(groupDetails.childCount === 1) { + if (groupDetails.childCount === 1) { return; } @@ -425,8 +426,8 @@ export class JS5 { groupDetails.childCount * 4); // Stripe data footer if (data.readerIndex < 0) { - logger.error(`Invalid reader index of ${data.readerIndex} for group ` + - `${groupName || groupKey}.`); + logger.error(`Invalid reader index of ${ data.readerIndex } for group ` + + `${ groupName || groupKey }.`); return; } @@ -499,13 +500,13 @@ export class JS5 { const archiveName = archiveDetails.name; - logger.info(`Decoding archive ${archiveName}...`); + logger.info(`Decoding archive ${ archiveName }...`); if (!archiveDetails.data) { this.decompress(archive); if (!archiveDetails.data) { - logger.error(`Unable to decode archive ${archiveName}.`); + logger.error(`Unable to decode archive ${ archiveName }.`); return; } } @@ -522,7 +523,7 @@ export class JS5 { let missingEncryptionKeys = 0; let accumulator = 0; - logger.info(`${groupCount} groups were found within the ${archiveName} archive.`); + logger.info(`${ groupCount } groups were found within the ${ archiveName } archive.`); // Group index keys for (let i = 0; i < groupCount; i++) { @@ -700,29 +701,46 @@ export class JS5 { } getEncryptionKeys(fileName: string): XteaKeys | XteaKeys[] | null { - if(!this.encryptionKeys.size) { - this.loadEncryptionKeys(); + if (this.openRS2EncryptionKeys?.length) { + const fileKeys = this.openRS2EncryptionKeys.find(file => file.name === fileName); + if (fileKeys) { + return { + gameBuild: this.fileStore.gameBuild, + key: fileKeys.key + }; + } } - const keySets = this.encryptionKeys.get(fileName); - if(!keySets) { + const keySets = this.localEncryptionKeys?.get(fileName); + if (!keySets) { return null; } - if(this.fileStore.gameBuild !== undefined) { + if (this.fileStore.gameBuild !== undefined) { return keySets.find(keySet => keySet.gameBuild === this.fileStore.gameBuild) ?? null; } return keySets; } - loadEncryptionKeys(): void { + async loadEncryptionKeys(): Promise { + if (/^\d+$/.test(this.fileStore.gameBuild)) { + const openRS2Keys = await getXteaKeysByBuild(parseInt(this.fileStore.gameBuild, 10)); + + if (openRS2Keys?.length) { + logger.info(`XTEA keys found for build ${ this.fileStore.gameBuild } on OpenRS2.org.`); + this.openRS2EncryptionKeys = openRS2Keys; + return; + } + } + + logger.warn(`XTEA keys not found for build ${ this.fileStore.gameBuild } on OpenRS2.org, using local XTEA key files instead.`); const configPath = join(this.fileStore.fileStorePath, 'config', 'xtea'); - this.encryptionKeys = Xtea.loadKeys(configPath); + this.localEncryptionKeys = Xtea.loadKeys(configPath); - if(!this.encryptionKeys.size) { + if (!this.localEncryptionKeys.size) { throw new Error(`Error reading encryption key lookup table. ` + - `Please ensure that the ${configPath} file exists and is valid.`); + `Please ensure that the ${ configPath } file exists and is valid.`); } } diff --git a/src/openrs2/openrs2.ts b/src/openrs2/openrs2.ts index e2b8dd2..2bbb933 100644 --- a/src/openrs2/openrs2.ts +++ b/src/openrs2/openrs2.ts @@ -2,6 +2,8 @@ import axios from 'axios'; import AdmZip, { IZipEntry } from 'adm-zip'; import { Buffer } from 'buffer'; import { logger } from '@runejs/common'; +import { XteaKeys } from '@runejs/common/encrypt'; +import { XteaConfig } from '@runejs/common/encrypt/xtea'; const openRS2Endpoint = 'https://archive.openrs2.org'; @@ -96,17 +98,28 @@ export const getAvailableBuilds = async ( }; -export const getOpenRS2CacheById = async ( +export const getOpenRS2CacheDetailsByBuild = async ( + build: number, +): Promise => { + return (await getOpenRS2CacheList())?.find( + c => c.scope === 'runescape' && c.game === 'runescape' && c.builds.find(b => b.major === build) + ) || null; +}; + + +export const getOpenRS2CacheFilesById = async ( id: number, scope: string = 'runescape' ): Promise => { const response = await axios.get( `${ openRS2Endpoint }/caches/${ scope }/${ id }/disk.zip`, - { - responseType: 'arraybuffer' - } + { responseType: 'arraybuffer' } ); + if (!response?.data) { + return []; + } + const zip = new AdmZip(Buffer.from(response.data, 'binary')); return zip.getEntries().map(entry => ({ name: entry.name, @@ -115,13 +128,13 @@ export const getOpenRS2CacheById = async ( }; -export const getOpenRS2CacheByBuild = async ( +export const getOpenRS2CacheFilesByBuild = async ( build: number ): Promise => { logger.info(`Searching OpenRS2 for build ${ build }...`); const cacheList = (await getOpenRS2CacheList()) - .filter(c => c.scope === 'runescape' && c.game === 'runescape'); + ?.filter(c => c.scope === 'runescape' && c.game === 'runescape') || []; const desiredCacheInfo = cacheList.find(cacheDetails => { for (const b of cacheDetails.builds) { @@ -136,7 +149,7 @@ export const getOpenRS2CacheByBuild = async ( if (desiredCacheInfo) { logger.info(`Build ${ build } was found within the OpenRS2 archive, fetching data...`); - const cacheFiles = await getOpenRS2CacheById(desiredCacheInfo.id); + const cacheFiles = await getOpenRS2CacheFilesById(desiredCacheInfo.id); return cacheFiles?.length ? cacheFiles : null; } else { logger.error(`Build ${ build } was not found within the OpenRS2.org archive.`); @@ -144,3 +157,27 @@ export const getOpenRS2CacheByBuild = async ( return null; }; + + +export const getXteaKeysById = async ( + id: number, + scope: string = 'runescape' +): Promise => { + const response = await axios.get( + `${ openRS2Endpoint }/caches/${ scope }/${ id }/keys.json` + ); + + return response?.data || []; +}; + + +export const getXteaKeysByBuild = async ( + build: number +): Promise => { + const cacheDetails = await getOpenRS2CacheDetailsByBuild(build); + if (!cacheDetails) { + return []; + } + + return await getXteaKeysById(cacheDetails.id, cacheDetails.scope); +}; diff --git a/src/scripts/builds.ts b/src/scripts/builds.ts index 4d8a021..9900fc4 100644 --- a/src/scripts/builds.ts +++ b/src/scripts/builds.ts @@ -40,7 +40,7 @@ const buildsScript = async ( logger.error(`Invalid range of ${ range }: Range must be a multiple of 100.`); return; } - + const availableBuilds = await getAvailableBuilds(scope, game); let msg = `Available builds on OpenRS2.org with scope ${ scope } and game ${ game }`; diff --git a/src/scripts/indexer.ts b/src/scripts/indexer.ts index 2a6867f..543a2ed 100644 --- a/src/scripts/indexer.ts +++ b/src/scripts/indexer.ts @@ -5,7 +5,7 @@ import { ScriptExecutor, ArgumentOptions } from './script-executor'; import { JS5FileStore } from '../file-system/js5/js5-file-store'; import { JagStore } from '../file-system/jag/jag-store'; import { - getOpenRS2CacheByBuild, + getOpenRS2CacheFilesByBuild, OpenRS2CacheFile } from '../openrs2'; @@ -179,7 +179,7 @@ const indexerScript = async ( if (source === 'openrs2') { if (numericBuildNumber) { - const openRS2CacheFiles = await getOpenRS2CacheByBuild(numericBuildNumber); + const openRS2CacheFiles = await getOpenRS2CacheFilesByBuild(numericBuildNumber); if (!openRS2CacheFiles?.length) { return; } @@ -191,7 +191,7 @@ const indexerScript = async ( } } - logger.info(`Indexing ${ format } file store...`); + logger.info(`Indexing ${ format === 'flat' ? format : format.toUpperCase() } file store...`); if (format === 'js5') { const store = new JS5FileStore(numericBuildNumber !== -1 ? numericBuildNumber : build, dir); diff --git a/src/scripts/script-executor.ts b/src/scripts/script-executor.ts index a13c322..6407435 100644 --- a/src/scripts/script-executor.ts +++ b/src/scripts/script-executor.ts @@ -18,9 +18,9 @@ export class ScriptExecutor { ): void { (async function(args: T) { await script(args); - }(this.getArguments(argumentOptions))) - .catch(logger.error) - .finally(() => process.exit(0)); + }(this.getArguments(argumentOptions))); + // .catch(logger.error); + // .finally(() => process.exit(0)); } } From ddd98bf478098625187606a6f05016b0f5132d79 Mon Sep 17 00:00:00 2001 From: Kikorono Date: Thu, 21 Jul 2022 10:53:25 -0500 Subject: [PATCH 18/32] Porting over JS5 archive and group encoding in preparation for flat file store support. Also removed a lot of redundant old code, added in proper todos for stubbed methods, added interface file names to the JS5 archive config, and made everything just a bit cleaner. --- config/js5-archives.json5 | 527 ++++++++++++++++-- src/config/archive-config.ts | 14 +- src/file-system/file-store-base.ts | 64 ++- src/file-system/jag/jag-archive.ts | 11 +- src/file-system/jag/jag-file.ts | 3 +- src/file-system/jag/jag-index.ts | 3 +- src/file-system/jag/jag-store.ts | 20 +- src/file-system/jag/jag.ts | 17 +- .../js5/{archive.ts => js5-archive.ts} | 63 +-- src/file-system/js5/js5-file-store.ts | 36 +- .../js5/{flat-file.ts => js5-file.ts} | 12 +- .../js5/{group.ts => js5-group.ts} | 63 +-- src/file-system/js5/js5.ts | 214 +++++-- src/scripts/dev.ts | 2 +- src/scripts/indexer.ts | 8 +- src/scripts/script-executor.ts | 5 +- 16 files changed, 811 insertions(+), 251 deletions(-) rename src/file-system/js5/{archive.ts => js5-archive.ts} (55%) rename src/file-system/js5/{flat-file.ts => js5-file.ts} (60%) rename src/file-system/js5/{group.ts => js5-group.ts} (55%) diff --git a/config/js5-archives.json5 b/config/js5-archives.json5 index 28a7630..4675b11 100644 --- a/config/js5-archives.json5 +++ b/config/js5-archives.json5 @@ -7,7 +7,7 @@ // Regular indexes anims: { key: 0, - flatten: true + flattenGroups: true }, bases: { key: 1 @@ -37,7 +37,476 @@ }, interfaces: { key: 3, - flatten: true + flattenGroups: true, + groupNames: { + '100guide_eggs_overlay': 0, + '100guide_flour_overlay': 1, + '100guide_inv_flour': 2, + '100guide_milk_overlay': 3, + 'tog_water_bowl': 4, + 'agilityarena_overlay': 5, + 'agilityarena_trade': 6, + 'ahoy_blackout': 7, + 'ahoy_islandmap': 8, + 'ahoy_runedraw': 9, + 'ahoy_windspeed': 10, + 'bank_deposit_box': 11, + 'bankpin_main': 13, + 'bankpin_settings': 14, + 'banner_padlock_keys': 15, + 'banner_anti_virus': 16, + 'banner_scamming': 21, + 'banner_security': 22, + 'banner_xmas': 23, + 'barrows_overlay': 24, + 'barrows_puzzle': 25, + 'blast_furnace_bar_stock': 28, + 'blast_furnace_plan_scroll': 29, + 'blast_furnace_temp_gauge': 30, + 'boardgames_challenge': 31, + 'boardgames_draughts': 32, + 'boardgames_draughts_options': 33, + 'boardgames_draughts_overlay': 34, + 'boardgames_draughts_view': 35, + 'boardgames_runelink': 36, + 'boardgames_runelink_options': 37, + 'boardgames_runelink_overlay': 38, + 'boardgames_runelink_view': 39, + 'boardgames_runesquares': 40, + 'boardgames_runesquares_options': 41, + 'boardgames_runesquares_overlay': 42, + 'boardgames_runesquares_view': 43, + 'boardgames_runeversi': 44, + 'boardgames_runeversi_options': 45, + 'boardgames_runeversi_overlay': 46, + 'boardgames_runeversi_view': 47, + 'bob_locator_amulet': 48, + 'burgh_map': 51, + 'canoe': 52, + 'canoe_stations_map': 53, + 'castlewars_catapult': 54, + 'castlewars_score': 55, + 'castlewars_shopside': 56, + 'castlewars_status_overlay': 57, + 'castlewars_status_overlay_saradomin': 58, + 'castlewars_status_overlay_zamorak': 59, + 'castlewars_trade': 60, + 'cat_naming': 61, + 'cave_goblin_markers': 62, + 'champions_scroll': 63, + 'chat1': 64, + 'chat2': 65, + 'chat3': 66, + 'chat4': 67, + 'chat_np1': 68, + 'chat_np2': 69, + 'chat_np3': 70, + 'chat_np4': 71, + 'confirm_destroy': 94, + 'sailing_transport_world_map': 95, + 'darkness_dark': 96, + 'darkness_light': 97, + 'darkness_medium': 98, + 'death_dice': 99, + 'deep_blue': 100, + 'pattern_next': 103, + 'stockmarket': 105, + 'stockside': 107, + 'stockcollect': 109, + 'dwarf_rock_book': 111, + 'dwarf_rock_cannon': 112, + 'dwarf_rock_schematics': 113, + 'dwarf_rock_schematics_control': 114, + 'enakh_film': 117, + 'enakh_smoke_overlay': 118, + 'fade_to_black': 120, + 'fade_to_light_blue': 121, + 'fade_to_white': 122, + 'fairy_certificate': 123, + 'farming_tools': 125, + 'farming_tools_side': 126, + 'favour_keyring': 127, + 'doubleobjbox': 131, + 'garden_list': 132, + 'garden_quiz': 133, + 'aide_compass': 135, + 'chatdefault': 137, + 'glidermap': 138, + 'gnomeball': 139, + 'horror_metaldoor': 142, + 'hauntedmine_controls': 144, + 'inventory': 149, + 'keldagrim_map': 150, + 'keldagrim_story1': 151, + 'keldagrim_titles': 152, + 'aide_death': 153, + 'leather_crafting': 154, + 'legends_mirror': 155, + 'quickchat_tutorial': 157, + 'rocko_kittens': 160, + 'smki_assignment': 161, + 'smki_learn': 163, + 'smki_buy': 164, + 'smki_smoke_overlay': 165, + 'rocko_cannonball': 166, + 'rocko_underwater': 167, + 'rocko_seagull': 168, + 'fade_from_black': 170, + 'chatlarge': 173, + 'wom_scroll': 174, + 'heat_overlay': 175, + 'fade_from_white': 177, + 'light2': 178, + 'light_blue': 179, + 'lightning_flash': 180, + 'logout': 182, + 'macro_combilock': 185, + 'macro_evil_bob': 186, + 'music_v3': 187, + 'macro_mime_emotes': 188, + 'macro_quizshow': 191, + 'magic': 192, + 'magic_zaros': 193, + 'magictraining_grave': 196, + 'magictraining_shop': 197, + 'magictraining_tele': 198, + 'mazetimer': 209, + 'message1': 210, + 'message2': 211, + 'message3': 212, + 'message4': 213, + 'message5': 214, + 'message_np1': 215, + 'message_np2': 216, + 'message_np3': 217, + 'message_np4': 218, + 'message_np5': 219, + 'messagescroll': 220, + 'messagescroll2': 221, + 'peng_emote': 223, + 'misc_shipjourney': 224, + 'mm_message': 225, + 'mole_mud': 226, + 'mourning_deathalter_list': 227, + 'multi2': 228, + 'multi2_chat': 229, + 'multi3': 230, + 'multi3_chat': 231, + 'multi4': 232, + 'multi4_chat': 233, + 'multi5': 234, + 'multi5_chat': 235, + 'multivar2': 236, + 'multivar4': 237, + 'multivar5': 238, + 'npcchat1': 241, + 'npcchat2': 242, + 'npcchat3': 243, + 'npcchat4': 244, + 'npcchat_np1': 245, + 'npcchat_np2': 246, + 'npcchat_np3': 247, + 'npcchat_np4': 248, + 'surok_letter1': 249, + 'surok_letter2': 250, + 'olaf2_lock_gate': 252, + 'olaf2_skull_puzzle': 253, + 'olaf2_treasuremap': 254, + 'area_task': 259, + 'dream_armour': 260, + 'options': 261, + 'pinball_interface': 263, + 'paint_cannon': 264, + 'clanwars_overlay': 265, + 'pest_rewards': 267, + 'prayer': 271, + 'priestperil_gravemonument': 272, + 'prisonpete': 273, + 'questjournal_v2': 274, + 'questjournal_scroll': 275, + 'ratcatcher_flute': 282, + 'ratcatcher_flute_music': 283, + 'ratcatcher_overlay': 284, + 'rd_combolock': 285, + 'regicide_still': 286, + 'roguesden_dials': 290, + 'roguesden_gear_select': 291, + 'roguesden_gears': 292, + 'roguesden_puzzle': 293, + 'rum_deal_title': 295, + 'sandstorm': 296, + 'seer_combolock': 298, + 'smithing_new': 300, + 'skill_cookmany': 307, + 'lotr_scroll': 308, + 'skill_multi1': 309, + 'slayer_staff_spells': 310, + 'smelting': 311, + 'smokeoverlay': 313, + 'soulbane_angerbar': 315, + 'soulbane_cut': 316, + 'soulbane_darkness': 317, + 'soulbane_scare': 318, + 'staff_spells': 319, + 'stats': 320, + 'swamp_boatjourney': 321, + 'swamp_boatjourney_back': 322, + 'tanner': 324, + 'target': 325, + 'teleport_other': 326, + 'cws_warning_11': 327, + 'temple_screen': 328, + 'templetrek_map': 329, + 'tradeconfirm': 334, + 'trademain': 335, + 'tradeside': 336, + 'trail_cluelong': 345, + 'trail_map01': 346, + 'trail_map02': 347, + 'trail_map03': 348, + 'trail_map04': 349, + 'trail_map05': 350, + 'trail_map06': 351, + 'trail_map07': 352, + 'trail_map08': 353, + 'trail_map09': 354, + 'trail_map10': 355, + 'trail_map11': 356, + 'trail_map12': 357, + 'trail_map13': 358, + 'trail_map14': 359, + 'trail_map15': 360, + 'trail_map16': 361, + 'trail_map17': 362, + 'trail_puzzle': 363, + 'trail_reward': 364, + 'trail_sextant': 365, + 'trawler_overlay': 366, + 'trawler_reward': 367, + 'trawler_start': 368, + 'tutorial_progress': 371, + 'tutorial_text': 372, + 'tzhaar_fightpit': 373, + 'welcome_screen': 378, + 'werewolf_goal': 379, + 'welcome_final_tex': 380, + 'wilderness_overlay': 381, + 'wilderness_warning': 382, + 'wom_telescope': 386, + 'wornitems': 387, + 'objdialog': 389, + 'poh_hangman': 393, + 'poh_house_options': 398, + 'poh_magic_tablets': 400, + 'poh_ranging': 401, + 'poh_sawmill': 403, + 'poh_scrying_pool': 404, + 'banner_poh': 405, + 'pest_mace': 406, + 'pest_lander_overlay': 407, + 'pest_status_overlay': 408, + 'mcannon_interface': 409, + 'warguild_defence': 410, + 'warguild_defence_mini': 411, + 'warguild_dummy': 412, + 'brew_telescope_overlay': 413, + 'brew_game_over': 414, + 'brew_overlay': 415, + 'brew_worktable': 416, + 'brew_tools': 417, + 'tutorial_text2': 421, + 'fairy2_certificate_broken': 423, + 'fairy2_message': 424, + 'scroll_godfather': 427, + 'ntk_overlay': 428, + 'magic_lunar': 430, + 'xbows_enchant_bolt': 432, + 'xbows_pouch': 433, + 'crafting_silver_casting': 438, + 'openurl': 445, + 'crafting_gold': 446, + 'banner_chatheads': 447, + 'brain_water_overlay': 448, + 'multivar3': 451, + 'myq3_statue_painting': 452, + 'myq3_blackout': 453, + 'myq3_boat_map': 454, + 'crafting_spinning': 459, + 'seaslug_boat_travel': 461, + 'kings_letter_v2': 463, + 'emotes': 464, + 'zep_balloon_map': 469, + 'zep_interface': 470, + 'zep_interface_side': 471, + 'anma_rgb': 480, + 'barbassault_horn': 484, + 'barbassault_over_att': 485, + 'barbassault_over_col': 486, + 'barbassault_over_def': 487, + 'barbassault_over_heal': 488, + 'barbassault_playerstat': 490, + 'barbassault_reward_shop': 491, + 'barbassault_timer_overlay': 494, + 'barbassault_turret': 495, + 'barbassault_tutorial': 496, + 'barbassault_wavecomplete': 497, + 'contact_scroll_blood': 498, + 'skill_guide_v2': 499, + 'poh_hangman_german': 507, + 'tol_homonculus_overlay': 508, + 'tol_cage_puzzle': 509, + 'tol_pressure_machine': 510, + 'tol_pipe_machine': 511, + 'inventory_ranging': 512, + 'inventory_wear': 513, + 'brain_cat_overlay': 514, + 'brain_gas_overlay': 515, + 'objbox': 519, + 'dream_cyrisus': 521, + 'dream_monster_stat': 522, + 'dream_player_stats': 523, + 'dream_title': 524, + 'vm_museum_map': 527, + 'vm_digsite': 528, + 'vm_kudos': 532, + 'vm_timeline': 534, + 'grim_tasklist': 538, + 'noexit': 539, + 'crafting_glass': 542, + 'dragon_slayer_qip_clouds': 543, + 'dragon_slayer_qip_cr_journey': 544, + 'dragon_slayer_qip_elvarg': 545, + 'dragon_slayer_qip_elvarg_fly': 546, + 'dragon_slayer_qip_map': 547, + 'toplevel': 548, + 'toplevel_full': 549, + 'friends2': 550, + 'ignore2': 551, + 'snapshot_main': 553, + 'multi2_mes': 557, + 'skill_multi6': 558, + 'pattern_cards': 559, + 'cws_warning_9': 560, + 'cws_warning_21': 561, + 'cws_warning_2': 562, + 'cws_warning_5': 563, + 'cws_warning_23': 564, + 'cws_warning_10': 565, + 'cws_warning_6': 566, + 'cws_warning_14': 567, + 'cws_warning_3': 568, + 'cws_warning_16': 569, + 'cws_warning_17': 570, + 'cws_warning_19': 571, + 'cws_warning_13': 572, + 'cws_warning_12': 573, + 'cws_warning_1': 574, + 'cws_warning_8': 576, + 'cws_warning_18': 577, + 'cws_warning_15': 578, + 'cws_warning_4': 579, + 'cws_warning_20': 580, + 'cws_warning_24': 581, + 'skill_multi1_small': 582, + 'cws_doomsayer': 583, + 'kr_king_statue': 584, + 'kr_pyramid_escape': 585, + 'kr_picklock': 588, + 'clanjoin': 589, + 'clansetup': 590, + 'cws_warning_25': 600, + 'godwars_overlay': 601, + 'shop_template': 620, + 'shop_template_side': 621, + 'banner_group': 622, + 'banner_group_assist': 623, + 'cws_warning_26': 627, + 'duel2_side': 628, + 'duel2_scoreboard': 632, + 'duel2_challengeoverlay': 638, + 'duel2_select_type': 640, + 'exchange_history': 643, + 'exchange_sets_side': 644, + 'exchange_itemsets': 645, + 'exchange_sand_timer': 646, + 'clanwars_viewing_orb': 649, + 'cws_warning_30': 650, + 'gravestone_shop': 652, + 'bounty_overlay': 653, + 'bounty_overlay_waiting': 656, + 'bounty_warning': 657, + 'lore_stats_side': 662, + 'lore_cats_side': 663, + 'lore_bank_side': 665, + 'equip_screen2': 667, + 'inventory_wear2': 670, + 'lore_bank': 671, + 'tutorial2_mesbox': 674, + 'cws_warning_27': 676, + 'cws_warning_28': 677, + 'cws_warning_29': 678, + 'banner_summoning': 679, + 'rabbit_shop': 686, + 'rabbit_overlay': 689, + 'banner_easter08': 715, + 'summoning_side': 722, + 'carpet_info': 723, + 'carpet_runesquares': 724, + 'carpet_runelink': 725, + 'carpet_runeversi': 726, + 'carpet_draughts': 727, + 'carpet_main': 728, + 'carpet_ticket': 729, + 'graphics_options': 742, + 'sound_options': 743, + 'loginscreen': 744, + 'statusicons': 745, + 'toplevel_fullscreen': 746, + 'topstat_lore': 747, + 'topstat_hitpoints': 748, + 'topstat_prayer': 749, + 'topstat_run': 750, + 'filterbuttons': 751, + 'chattop': 752, + 'pmchat': 754, + 'worldmap': 755, + 'boardgames_options': 756, + 'npcchatlarge': 757, + 'canoe_travel': 758, + 'tutorial2_objbox': 760, + 'bank_v2_main': 762, + 'bank_v2_side': 763, + 'tutorial2_switch_task': 765, + 'tutorial2_death': 766, + 'bank_v2_help': 767, + 'tutorial2_text': 769, + 'rcguild_side': 778, + 'rcguild_rewards': 779, + 'rcguild_map': 780, + 'rcguild_overlay': 781, + 'crcs_tightrope': 783, + 'crcs_side': 784, + 'crcs_rewards': 785, + 'crcs_equipment': 786, + 'crcs_scoreboard': 788, + 'clanwars_end': 790, + 'clanwars_setup': 791, + 'clanwars_setup_side': 792, + 'banner_halloween': 800, + 'quickchat_locked': 801, + 'sc_tutorial_overlay': 802, + 'sc_scores_border': 803, + 'sc_item_transfer': 805, + 'sc_remaining_clay': 806, + 'sc_scores': 810, + 'sc_reward_shop': 811, + 'sc_processing': 813, + 'luc2_chapter2': 814, + 'luc2_telescope_view': 816, + 'luc2_chapter3': 822, + 'luc2_chapter1': 826, + 'luc2_lucien_projectiles': 827 + } }, synth_sounds: { key: 4, @@ -71,75 +540,57 @@ clientscripts: { key: 12, compression: 'gzip', - contentType: '.cs2', - build: 435 + contentType: '.cs2' }, fontmetrics: { - key: 13, - build: 443 + key: 13 }, vorbis: { - key: 14, - build: 451 + key: 14 }, midi_instruments: { - key: 15, - build: 451 + key: 15 }, config_loc: { - key: 16, - build: 489 + key: 16 }, config_enum: { - key: 17, - build: 489 + key: 17 }, config_npc: { - key: 18, - build: 489 + key: 18 }, config_obj: { - key: 19, - build: 489 + key: 19 }, config_seq: { - key: 20, - build: 489 + key: 20 }, config_spot: { - key: 21, - build: 489 + key: 21 }, config_var_bit: { - key: 22, - build: 489 + key: 22 }, worldmapdata: { - key: 23, - build: 493 + key: 23 }, quickchat: { - key: 24, - build: 498 + key: 24 }, quickchat_global: { - key: 25, - build: 498 + key: 25 }, materials: { - key: 26, - build: 500 + key: 26 }, config_particle: { - key: 27, - build: 523 + key: 27 }, defaults: { - key: 28, - build: 537 + key: 28 }, billboards: { - key: 29, - build: 582 + key: 29 } } diff --git a/src/config/archive-config.ts b/src/config/archive-config.ts index 636880d..d775549 100644 --- a/src/config/archive-config.ts +++ b/src/config/archive-config.ts @@ -1,16 +1,14 @@ import { EncryptionMethod } from '@runejs/common/encrypt'; -export interface JS5ArchiveConfig { +export interface ArchiveConfig { key: number; - encryption?: [ EncryptionMethod, string ]; - contentType?: string; - flatten?: boolean; - groupNames?: { [key: string]: number }; - build?: number; } -export interface JagCacheArchiveConfig { - key: number; +export interface JS5ArchiveConfig extends ArchiveConfig { + encryption?: [ EncryptionMethod, string ]; + contentType?: string; + flattenGroups?: boolean; + groupNames?: { [key: string]: number }; } diff --git a/src/file-system/file-store-base.ts b/src/file-system/file-store-base.ts index b3030f4..f0fc0e5 100644 --- a/src/file-system/file-store-base.ts +++ b/src/file-system/file-store-base.ts @@ -1,14 +1,15 @@ -import { Djb2 } from '../config'; -import { IndexDatabase } from '../db/index-database'; +import JSON5 from 'json5'; import { join } from 'path'; -import { Crc32 } from '@runejs/common/crc32'; import { existsSync, readFileSync } from 'graceful-fs'; +import { Crc32 } from '@runejs/common/crc32'; import { logger } from '@runejs/common'; -import JSON5 from 'json5'; + +import { ArchiveConfig, Djb2 } from '../config'; +import { IndexDatabase } from '../db/index-database'; import { IndexedFileBase } from './indexed-file-base'; -export abstract class FileStoreBase, C> { +export abstract class FileStoreBase, C extends ArchiveConfig = ArchiveConfig> { readonly gameBuild: string; readonly fileStorePath: string; @@ -69,18 +70,55 @@ export abstract class FileStoreBase, C> { await this._database.closeConnection(); } - getArchive(archiveKey: number): A | null { - return this.archives.get(archiveKey) || null; + getArchive(archiveKey: number): A | null; + getArchive(archiveName: string): A | null; + getArchive(archiveKeyOrName: number | string): A | null; + getArchive(archiveKeyOrName: number | string): A | null { + if (typeof archiveKeyOrName === 'string') { + return Array.from(this.archives.values()).find( + a => a?.index?.name === archiveKeyOrName + ) || null; + } else { + return this.archives.get(archiveKeyOrName) || null; + } } - setArchive(archiveKey: number, archive: A): void { - this.archives.set(archiveKey, archive); + setArchive(archiveKey: number, archive: A): void; + setArchive(archiveName: string, archive: A): void; + setArchive(archiveKeyOrName: number | string, archive: A): void; + setArchive(archiveKeyOrName: number | string, archive: A): void { + if (typeof archiveKeyOrName === 'string') { + const archiveConfig = this.getArchiveConfig(archiveKeyOrName); + if (archiveConfig) { + this.archives.set(archiveConfig.key, archive); + } else { + logger.error(`Archive ${ archiveKeyOrName } configuration was not found.`); + } + } else { + this.archives.set(archiveKeyOrName, archive); + } } - findArchive(archiveName: string): A | null { - return Array.from(this.archives.values()).find( - a => a?.index?.name === archiveName - ) || null; + getArchiveConfig(archiveKey: number): C | null; + getArchiveConfig(archiveName: string): C | null; + getArchiveConfig(archiveKeyOrName: number | string): C | null; + getArchiveConfig(archiveKeyOrName: number | string): C | null { + if (typeof archiveKeyOrName === 'string') { + return this._archiveConfig[archiveKeyOrName] || null; + } else { + return Object.values(this._archiveConfig).find(c => c.key === archiveKeyOrName) || null; + } + } + + getArchiveName(archiveKey: number): string | null { + const archiveEntries = Object.entries(this._archiveConfig); + for (const [ name, config ] of archiveEntries) { + if (config.key === archiveKey) { + return name; + } + } + + return null; } get archiveConfig(): { [key: string]: C } { diff --git a/src/file-system/jag/jag-archive.ts b/src/file-system/jag/jag-archive.ts index 157becb..3f60818 100644 --- a/src/file-system/jag/jag-archive.ts +++ b/src/file-system/jag/jag-archive.ts @@ -1,12 +1,19 @@ import { JagStore } from './jag-store'; import { IndexedFileBase } from '../indexed-file-base'; +import { ArchiveConfig } from '../../config'; export class JagArchive extends IndexedFileBase { - constructor(jagStore: JagStore, archiveKey: number, archiveName: string) { + readonly config: ArchiveConfig; + + constructor( + jagStore: JagStore, + archiveKey: number, + ) { super(jagStore, 'ARCHIVE', archiveKey, -1, -1); - this.index.name = archiveName; + this.config = jagStore.getArchiveConfig(archiveKey); + this.index.name = jagStore.getArchiveName(archiveKey); } } diff --git a/src/file-system/jag/jag-file.ts b/src/file-system/jag/jag-file.ts index ce6dbfb..890a742 100644 --- a/src/file-system/jag/jag-file.ts +++ b/src/file-system/jag/jag-file.ts @@ -1,6 +1,7 @@ import { IndexedFileBase } from '../indexed-file-base'; +import { JagStore } from './jag-store'; -export class JagFile extends IndexedFileBase { +export class JagFile extends IndexedFileBase { } diff --git a/src/file-system/jag/jag-index.ts b/src/file-system/jag/jag-index.ts index ae5c58e..fad72e5 100644 --- a/src/file-system/jag/jag-index.ts +++ b/src/file-system/jag/jag-index.ts @@ -1,6 +1,7 @@ import { IndexedFileBase } from '../indexed-file-base'; +import { JagStore } from './jag-store'; -export class JagIndex extends IndexedFileBase { +export class JagIndex extends IndexedFileBase { } diff --git a/src/file-system/jag/jag-store.ts b/src/file-system/jag/jag-store.ts index 6ac4dc0..2926b27 100644 --- a/src/file-system/jag/jag-store.ts +++ b/src/file-system/jag/jag-store.ts @@ -1,10 +1,9 @@ -import { JagCacheArchiveConfig } from '../../config'; import { JagArchive } from './jag-archive'; import { FileStoreBase } from '../file-store-base'; import { Jag } from './jag'; -export class JagStore extends FileStoreBase { +export class JagStore extends FileStoreBase { readonly jag: Jag; @@ -15,21 +14,10 @@ export class JagStore extends FileStoreBase { override async load(): Promise { await this.openDatabase(); + } - const archiveNames = Object.keys(this.archiveConfig); - - for (const archiveName of archiveNames) { - const archiveConfig = this.archiveConfig[archiveName]; - - if (!this.archives.has(archiveConfig.key)) { - const archive = new JagArchive( - this, - archiveConfig.key, - archiveName, - ); - this.archives.set(archiveConfig.key, archive); - } - } + createArchive(archiveKey: number): void { + this.setArchive(archiveKey, new JagArchive(this, archiveKey)); } } diff --git a/src/file-system/jag/jag.ts b/src/file-system/jag/jag.ts index f0486a1..64dd019 100644 --- a/src/file-system/jag/jag.ts +++ b/src/file-system/jag/jag.ts @@ -2,6 +2,7 @@ import { join } from 'path'; import { existsSync, readdirSync, readFileSync, statSync } from 'graceful-fs'; import { ByteBuffer, logger } from '@runejs/common'; import { JagStore } from './jag-store'; +import { OpenRS2CacheFile } from '../../openrs2'; export class Jag { @@ -15,7 +16,11 @@ export class Jag { this.jagStore = jagStore; } - loadLocalJagFiles(): void { + // @todo stubbed - 21/07/22 - Kiko + readOpenRS2CacheFiles(cacheFiles: OpenRS2CacheFile[]): void { + } + + readLocalJagFiles(): void { const jagStorePath = join(this.jagStore.fileStorePath, 'jag'); if (!existsSync(jagStorePath)) { @@ -49,17 +54,17 @@ export class Jag { continue; } - const index = fileName.substring(fileName.indexOf('.idx') + 4); - const numericIndex = Number(index); + const indexString = fileName.substring(fileName.indexOf('.idx') + 4); + const indexKey = Number(indexString); - if (isNaN(numericIndex)) { + if (isNaN(indexKey)) { logger.error(`Index file ${fileName} does not have a valid extension.`); } - this.indexFiles.set(numericIndex, new ByteBuffer(readFileSync(join(jagStorePath, fileName)))); + this.indexFiles.set(indexKey, new ByteBuffer(readFileSync(join(jagStorePath, fileName)))); } - logger.info(`Jag store files loaded for game build ${this.jagStore.gameBuild}.`); + logger.info(`JAG store files loaded for game build ${this.jagStore.gameBuild}.`); } // @todo 18/07/22 - Kiko diff --git a/src/file-system/js5/archive.ts b/src/file-system/js5/js5-archive.ts similarity index 55% rename from src/file-system/js5/archive.ts rename to src/file-system/js5/js5-archive.ts index dd1b725..86b005b 100644 --- a/src/file-system/js5/archive.ts +++ b/src/file-system/js5/js5-archive.ts @@ -1,21 +1,23 @@ import { JS5FileStore } from './js5-file-store'; -import { Group } from './group'; +import { JS5Group } from './js5-group'; import { logger } from '@runejs/common'; import { IndexedFileBase } from '../indexed-file-base'; +import { JS5ArchiveConfig } from '../../config'; -export class Archive extends IndexedFileBase { +export class JS5Archive extends IndexedFileBase { - readonly groups: Map; + readonly groups: Map; + readonly config: JS5ArchiveConfig; constructor( fileStore: JS5FileStore, archiveKey: number, - archiveName: string, ) { super(fileStore, 'ARCHIVE', archiveKey, 255, -1); - this.index.name = archiveName; - this.groups = new Map(); + this.config = fileStore.getArchiveConfig(archiveKey); + this.index.name = fileStore.getArchiveName(archiveKey); + this.groups = new Map(); } override validate(trackChanges: boolean = true): void { @@ -53,49 +55,28 @@ export class Archive extends IndexedFileBase { const groupKey = groupIndex.key; if (!this.groups.has(groupKey)) { - const group = new Group(this.fileStore, groupKey, this); + const group = new JS5Group(this.fileStore, groupKey, this); group.index = groupIndex; this.groups.set(groupKey, group); } } } - js5Unpack(): Buffer | null { - return this.fileStore.js5.unpack(this); - } - - js5Decompress(): Buffer | null { - return this.fileStore.js5.decompress(this); - } - - async js5Decode(): Promise { - await this.fileStore.js5.decodeArchive(this); - } - - js5Pack(): Buffer | null { - return this.fileStore.js5.pack(this); - } - - js5Compress(): Buffer | null { - return this.fileStore.js5.compress(this); - } - - js5Encode(): Buffer | null { - return this.fileStore.js5.encodeArchive(this); - } - - getGroup(groupIndex: number): Group | null { - return this.groups.get(groupIndex) || null; - } - - setGroup(groupIndex: number, group: Group): void { - this.groups.set(groupIndex, group); + getGroup(groupKey: number): JS5Group | null; + getGroup(groupName: string): JS5Group | null; + getGroup(groupKeyOrName: number | string): JS5Group | null; + getGroup(groupKeyOrName: number | string): JS5Group | null { + if (typeof groupKeyOrName === 'string') { + return Array.from(this.groups.values()).find( + group => group?.index?.name === groupKeyOrName + ) || null; + } else { + return this.groups.get(groupKeyOrName) || null; + } } - findGroup(groupName: string): Group | null { - return Array.from(this.groups.values()).find( - group => group?.index?.name === groupName - ) || null; + setGroup(groupKey: number, group: JS5Group): void { + this.groups.set(groupKey, group); } } diff --git a/src/file-system/js5/js5-file-store.ts b/src/file-system/js5/js5-file-store.ts index 39f25b9..3726fe3 100644 --- a/src/file-system/js5/js5-file-store.ts +++ b/src/file-system/js5/js5-file-store.ts @@ -1,51 +1,25 @@ import { JS5ArchiveConfig } from '../../config'; import { JS5 } from './js5'; -import { Archive } from './archive'; +import { JS5Archive } from './js5-archive'; import { FileStoreBase } from '../file-store-base'; -export class JS5FileStore extends FileStoreBase{ +export class JS5FileStore extends FileStoreBase{ - readonly vanillaBuild: boolean; readonly js5: JS5; constructor(gameBuild: string | number, storePath: string = './') { super(gameBuild, storePath, 'js5-archives'); - // @todo make `vanillaBuild` obsolete via auto-detection - 07/13/22 - Kiko - this.vanillaBuild = typeof gameBuild === 'number'; this.js5 = new JS5(this); } override async load(): Promise { await this.js5.loadEncryptionKeys(); await this.openDatabase(); + } - const archiveNames = Object.keys(this.archiveConfig); - - for (const archiveName of archiveNames) { - const archiveConfig = this.archiveConfig[archiveName]; - - if (archiveConfig.key === 255) { - continue; - } - - if (this.vanillaBuild && archiveConfig.build) { - const buildNumber = Number(this.gameBuild); - - if (buildNumber < archiveConfig.build) { - continue; - } - } - - if (!this.archives.has(archiveConfig.key)) { - const archive = new Archive( - this, - archiveConfig.key, - archiveName, - ); - this.archives.set(archiveConfig.key, archive); - } - } + createArchive(archiveKey: number): void { + this.setArchive(archiveKey, new JS5Archive(this, archiveKey)); } } diff --git a/src/file-system/js5/flat-file.ts b/src/file-system/js5/js5-file.ts similarity index 60% rename from src/file-system/js5/flat-file.ts rename to src/file-system/js5/js5-file.ts index 41dddeb..bd25e39 100644 --- a/src/file-system/js5/flat-file.ts +++ b/src/file-system/js5/js5-file.ts @@ -1,18 +1,18 @@ import { JS5FileStore } from './js5-file-store'; -import { Archive } from './archive'; -import { Group } from './group'; +import { JS5Archive } from './js5-archive'; +import { JS5Group } from './js5-group'; import { IndexedFileBase } from '../indexed-file-base'; -export class FlatFile extends IndexedFileBase { +export class JS5File extends IndexedFileBase { - readonly archive: Archive; - readonly group: Group; + readonly archive: JS5Archive; + readonly group: JS5Group; constructor( fileStore: JS5FileStore, fileKey: number, - group: Group, + group: JS5Group, ) { super(fileStore, 'FILE', fileKey, group.archive.index.key, group.index.key); this.archive = group.archive; diff --git a/src/file-system/js5/group.ts b/src/file-system/js5/js5-group.ts similarity index 55% rename from src/file-system/js5/group.ts rename to src/file-system/js5/js5-group.ts index 201a19b..0fb97e4 100644 --- a/src/file-system/js5/group.ts +++ b/src/file-system/js5/js5-group.ts @@ -1,23 +1,23 @@ import { JS5FileStore } from './js5-file-store'; -import { Archive } from './archive'; -import { FlatFile } from './flat-file'; +import { JS5Archive } from './js5-archive'; +import { JS5File } from './js5-file'; import { logger } from '@runejs/common'; import { IndexedFileBase } from '../indexed-file-base'; -export class Group extends IndexedFileBase { +export class JS5Group extends IndexedFileBase { - readonly archive: Archive; - readonly files: Map; + readonly archive: JS5Archive; + readonly files: Map; constructor( fileStore: JS5FileStore, groupKey: number, - archive: Archive, + archive: JS5Archive, ) { super(fileStore, 'GROUP', groupKey, archive.index.key); this.archive = archive; - this.files = new Map(); + this.files = new Map(); } override validate(trackChanges: boolean = true): void { @@ -57,49 +57,28 @@ export class Group extends IndexedFileBase { const fileKey = fileIndex.key; if (!this.files.has(fileKey)) { - const file = new FlatFile(this.fileStore, fileKey, this); + const file = new JS5File(this.fileStore, fileKey, this); file.index = fileIndex; this.files.set(fileKey, file); } } } - js5Unpack(): Buffer | null { - return this.fileStore.js5.unpack(this); - } - - js5Decompress(): Buffer | null { - return this.fileStore.js5.decompress(this); - } - - async js5Decode(): Promise { - await this.fileStore.js5.decodeGroup(this); - } - - js5Pack(): Buffer | null { - return this.fileStore.js5.pack(this); - } - - js5Compress(): Buffer | null { - return this.fileStore.js5.compress(this); - } - - js5Encode(): Buffer | null { - return this.fileStore.js5.encodeGroup(this); - } - - getFile(fileIndex: number): FlatFile | null { - return this.files.get(fileIndex) || null; - } - - setFile(fileIndex: number, file: FlatFile): void { - this.files.set(fileIndex, file); + getFile(fileKey: number): JS5File | null; + getFile(fileName: string): JS5File | null; + getFile(fileKeyOrName: number | string): JS5File | null; + getFile(fileKeyOrName: number | string): JS5File | null { + if (typeof fileKeyOrName === 'string') { + return Array.from(this.files.values()).find( + file => file?.index?.name === fileKeyOrName + ) || null; + } else { + return this.files.get(fileKeyOrName) || null; + } } - findFile(fileName: string): FlatFile | null { - return Array.from(this.files.values()).find( - file => file?.index?.name === fileName - ) || null; + setFile(fileKey: number, file: JS5File): void { + this.files.set(fileKey, file); } } diff --git a/src/file-system/js5/js5.ts b/src/file-system/js5/js5.ts index e3ce355..b2bd605 100644 --- a/src/file-system/js5/js5.ts +++ b/src/file-system/js5/js5.ts @@ -4,11 +4,11 @@ import { ByteBuffer, logger } from '@runejs/common'; import { Bzip2, getCompressionMethod, Gzip } from '@runejs/common/compress'; import { Xtea, XteaKeys } from '@runejs/common/encrypt'; import { archiveFlags } from '../../config'; -import { Group } from './group'; -import { Archive } from './archive'; +import { JS5Group } from './js5-group'; +import { JS5Archive } from './js5-archive'; import { JS5FileStore } from './js5-file-store'; import { ArchiveFormat } from '../../config'; -import { FlatFile } from './flat-file'; +import { JS5File } from './js5-file'; import { getXteaKeysByBuild, OpenRS2CacheFile } from '../../openrs2'; import { XteaConfig } from '@runejs/common/encrypt/xtea'; @@ -34,7 +34,7 @@ export class JS5 { this.indexFiles = new Map(); } - loadOpenRS2CacheFiles(cacheFiles: OpenRS2CacheFile[]): void { + readOpenRS2CacheFiles(cacheFiles: OpenRS2CacheFile[]): void { const dataFileBuffer = cacheFiles.find(file => file.name === dataFileName)?.data || null; if (!dataFileBuffer?.length) { throw new Error(`The main ${ dataFileName } data file could not be found.`); @@ -65,20 +65,21 @@ export class JS5 { continue; } - const index = fileName.substring(fileName.indexOf('.idx') + 4); - const numericIndex = Number(index); + const indexString = fileName.substring(fileName.indexOf('.idx') + 4); + const archiveKey = Number(indexString); - if (isNaN(numericIndex)) { + if (isNaN(archiveKey)) { logger.error(`Index file ${ fileName } does not have a valid extension.`); } - this.indexFiles.set(numericIndex, new ByteBuffer(cacheFile.data)); + this.indexFiles.set(archiveKey, new ByteBuffer(cacheFile.data)); + this.fileStore.createArchive(archiveKey); } logger.info(`JS5 store file loaded for game build ${ this.fileStore.gameBuild }.`); } - loadLocalCacheFiles(): void { + readLocalCacheFiles(): void { const js5StorePath = join(this.fileStore.fileStorePath, 'js5'); if (!existsSync(js5StorePath)) { @@ -116,24 +117,25 @@ export class JS5 { continue; } - const index = fileName.substring(fileName.indexOf('.idx') + 4); - const numericIndex = Number(index); + const indexString = fileName.substring(fileName.indexOf('.idx') + 4); + const archiveKey = Number(indexString); - if (isNaN(numericIndex)) { + if (isNaN(archiveKey)) { logger.error(`Index file ${ fileName } does not have a valid extension.`); } - this.indexFiles.set(numericIndex, new ByteBuffer(readFileSync(join(js5StorePath, fileName)))); + this.indexFiles.set(archiveKey, new ByteBuffer(readFileSync(join(js5StorePath, fileName)))); + this.fileStore.createArchive(archiveKey); } logger.info(`JS5 store file loaded for game build ${ this.fileStore.gameBuild }.`); } - unpack(file: Group | Archive): Buffer | null { + unpack(file: JS5Group | JS5Archive): Buffer | null { const fileDetails = file.index; const fileKey = fileDetails.key; - const archiveKey: number = file instanceof Archive ? 255 : file.archive.index.key; - const archiveName: string = file instanceof Archive ? 'main' : file.archive.index.name; + const archiveKey: number = file instanceof JS5Archive ? 255 : file.archive.index.key; + const archiveName: string = file instanceof JS5Archive ? 'main' : file.archive.index.name; const indexChannel: ByteBuffer = archiveKey !== 255 ? this.indexFiles.get(archiveKey) : this.mainIndexFile; @@ -233,7 +235,7 @@ export class JS5 { return fileDetails.compressedData; } - readCompressedFileHeader(file: Group | Archive): { compressedLength: number, readerIndex: number } { + readCompressedFileHeader(file: JS5Group | JS5Archive): { compressedLength: number, readerIndex: number } { const fileDetails = file.index; if (!fileDetails.compressedData?.length) { @@ -251,7 +253,7 @@ export class JS5 { return { compressedLength, readerIndex }; } - decrypt(file: Group | Archive): Buffer { + decrypt(file: JS5Group | JS5Archive): Buffer { const fileDetails = file.index; const fileName = fileDetails.name; @@ -262,7 +264,7 @@ export class JS5 { } // @todo move to JS5.decodeArchive - const archiveName = file instanceof Archive ? 'main' : file.archive.index.name; + const archiveName = file instanceof JS5Archive ? 'main' : file.archive.index.name; const archiveConfig = this.fileStore.archiveConfig[archiveName]; if (archiveConfig.encryption) { @@ -325,7 +327,7 @@ export class JS5 { return null; } - decompress(file: Group | Archive): Buffer | null { + decompress(file: JS5Group | JS5Archive): Buffer | null { const fileDetails = file.index; if (!fileDetails.compressedData?.length) { @@ -396,7 +398,7 @@ export class JS5 { return fileDetails.data; } - async decodeGroup(group: Group): Promise { + async decodeGroup(group: JS5Group): Promise { const groupDetails = group.index; const { key: groupKey, name: groupName } = groupDetails; const files = group.files; @@ -491,7 +493,7 @@ export class JS5 { group.validate(false); } - async decodeArchive(archive: Archive): Promise { + async decodeArchive(archive: JS5Archive): Promise { const archiveDetails = archive.index; if (archiveDetails.key === 255) { @@ -511,16 +513,13 @@ export class JS5 { } } - // logger.info(`Archive ${archiveName} checksum: ${this.crc32}`); - const archiveData = new ByteBuffer(archiveDetails.data); const format = archiveDetails.archiveFormat = archiveData.get('byte', 'unsigned'); const mainDataType = format >= ArchiveFormat.smart ? 'smart_int' : 'short'; archiveDetails.version = format >= ArchiveFormat.versioned ? archiveData.get('int') : 0; const flags = archiveFlags(archiveData.get('byte', 'unsigned')); const groupCount = archiveData.get(mainDataType, 'unsigned'); - const groups: Group[] = new Array(groupCount); - let missingEncryptionKeys = 0; + const groups: JS5Group[] = new Array(groupCount); let accumulator = 0; logger.info(`${ groupCount } groups were found within the ${ archiveName } archive.`); @@ -529,7 +528,7 @@ export class JS5 { for (let i = 0; i < groupCount; i++) { const delta = archiveData.get(mainDataType, 'unsigned'); const groupKey = accumulator += delta; - const group = groups[i] = new Group(this.fileStore, groupKey, archive); + const group = groups[i] = new JS5Group(this.fileStore, groupKey, archive); archive.setGroup(groupKey, group); } @@ -590,7 +589,7 @@ export class JS5 { for (let i = 0; i < fileCount; i++) { const delta = archiveData.get(mainDataType, 'unsigned'); const childFileIndex = accumulator += delta; - const flatFile = new FlatFile(this.fileStore, childFileIndex, group); + const flatFile = new JS5File(this.fileStore, childFileIndex, group); group.setFile(childFileIndex, flatFile); } } @@ -611,19 +610,22 @@ export class JS5 { archive.validate(false); } + // @todo stubbed - 21/07/22 - Kiko decodeMainIndex(): Buffer | null { - return null; // @todo stub + return null; } - pack(file: Group | Archive): Buffer | null { - return null; // @todo stub + // @todo stubbed - 21/07/22 - Kiko + pack(file: JS5Group | JS5Archive): Buffer | null { + return null; } - encrypt(file: Group | Archive): Buffer | null { - return null; // @todo stub + // @todo stubbed - 21/07/22 - Kiko + encrypt(file: JS5Group | JS5Archive): Buffer | null { + return null; } - compress(file: Group | Archive): Buffer | null { + compress(file: JS5Group | JS5Archive): Buffer | null { const fileDetails = file.index; if (!fileDetails.data?.length) { @@ -675,12 +677,148 @@ export class JS5 { return fileDetails.compressedData; } - encodeGroup(group: Group): Buffer { - return null; // @todo stub + encodeGroup(group: JS5Group): Buffer | null { + const { files: fileMap, index, stripes } = group; + const fileCount = fileMap.size; + + // Single-file group + if (fileCount <= 1) { + index.data = fileMap.get(0)?.index?.data || null; + return index.data; + } + + // Multi-file group + const files = Array.from(fileMap.values()); + const fileSizes = files.map(file => file.index.fileSize); + const stripeCount = stripes?.length ?? 1; + + // Size of all individual files + 1 int per file containing it's size + // + 1 at the end for the total group stripe count + const groupSize = fileSizes.reduce( + (a, c) => a + c) + (stripeCount * fileCount * 4 + ) + 1; + + const groupBuffer = new ByteBuffer(groupSize); + + // Write child file data + for (let stripe = 0; stripe < stripeCount; stripe++) { + files.forEach(file => { + const fileData = file.index.data; + if (!fileData?.length) { + return; + } + + const fileBuffer = new ByteBuffer(fileData); + const stripeSize = file.stripes?.[stripe] ?? 0; + + if (stripeSize) { + const stripeData = fileBuffer.getSlice(fileBuffer.readerIndex, stripeSize); + fileBuffer.readerIndex = fileBuffer.readerIndex + stripeSize; + groupBuffer.putBytes(stripeData); + } + }); + } + + // Write child file stripe lengths + for (let stripe = 0; stripe < stripeCount; stripe++) { + let prevSize = 0; + files.forEach(file => { + const fileData = file.index.data; + if (!fileData?.length) { + return; + } + + const stripeLength = file.stripes?.[stripe] ?? 0; + groupBuffer.put(stripeLength - prevSize, 'int'); + prevSize = stripeLength; + }); + } + + // Write group child file stripe count + groupBuffer.put(stripeCount, 'byte'); + + if (groupBuffer.length) { + index.data = groupBuffer.toNodeBuffer(); + } else { + index.data = null; + } + + return index.data; } - encodeArchive(archive: Archive): Buffer { - return null; // @todo stub + // @todo support newer archive fields & formats - 21/07/22 - Kiko + encodeArchive(archive: JS5Archive): Buffer | null { + const { groups: groupMap, index } = archive; + const groups = Array.from(groupMap.values()); + const groupCount = groups.length; + const filesNamed = groups.filter(group => group.index.name !== null).length !== 0; + let lastFileKey = 0; + + // @todo calculate the proper size instead of using a set amount here - 21/07/22 - Kiko + const buffer = new ByteBuffer(1000 * 1000); + + // Write archive index file header + buffer.put(index.archiveFormat ?? ArchiveFormat.original); + buffer.put(filesNamed ? 1 : 0); + buffer.put(groupCount, 'short'); + + // Write group keys + groups.forEach(group => { + buffer.put(group.index.key - lastFileKey, 'short'); + lastFileKey = group.index.key; + }); + + // Write group names (if applicable) + if (filesNamed) { + groups.forEach(group => buffer.put(group.index.nameHash ?? -1, 'int')); + } + + // Write uncompressed crc values + groups.forEach(group => buffer.put(group.index.checksum ?? -1, 'int')); + + // Write group version numbers + groups.forEach(group => buffer.put(group.index.version, 'int')); + + // Write group child file counts + groups.forEach(group => buffer.put(group.files?.size ?? 1, 'short')); + + // Write group child file keys + groups.forEach(group => { + if (group.files.size > 1) { + lastFileKey = 0; + + group.files.forEach(file => { + buffer.put(file.index.key - lastFileKey, 'short'); + lastFileKey = file.index.key; + }); + } else { + buffer.put(0, 'short'); + } + }); + + // Write group child file names (if applicable) + if (filesNamed) { + groups.forEach(group => { + if (group.files.size > 1) { + lastFileKey = 0; + + group.files.forEach(file => + buffer.put(file.index.nameHash ?? -1, 'int')); + } else { + buffer.put(0, 'int'); + } + }); + } + + const archiveIndexData = buffer?.flipWriter(); + + if (archiveIndexData?.length) { + index.data = archiveIndexData.toNodeBuffer(); + } else { + index.data = null; + } + + return index.data; } encodeMainIndex(): ByteBuffer { diff --git a/src/scripts/dev.ts b/src/scripts/dev.ts index 9b1d2c9..c00a729 100644 --- a/src/scripts/dev.ts +++ b/src/scripts/dev.ts @@ -6,7 +6,7 @@ const dev = async () => { const start = Date.now(); const fileStore = new JS5FileStore(435); await fileStore.load(); - fileStore.js5.loadLocalCacheFiles(); + fileStore.js5.readLocalCacheFiles(); logger.info(`Unpacking archives from JS5 store...`); diff --git a/src/scripts/indexer.ts b/src/scripts/indexer.ts index 543a2ed..af293a5 100644 --- a/src/scripts/indexer.ts +++ b/src/scripts/indexer.ts @@ -110,7 +110,7 @@ const indexJS5Store = async (store: JS5FileStore) => { const indexJS5Archive = async (store: JS5FileStore, archiveName: string) => { - const archive = store.findArchive(archiveName); + const archive = store.getArchive(archiveName); if (!archive) { logger.error(`Archive ${ archiveName } was not found.`); @@ -198,9 +198,9 @@ const indexerScript = async ( await store.load(); if (cacheFiles === 'local') { - store.js5.loadLocalCacheFiles(); + store.js5.readLocalCacheFiles(); } else { - store.js5.loadOpenRS2CacheFiles(cacheFiles); + store.js5.readOpenRS2CacheFiles(cacheFiles); } if (archiveName === 'main') { @@ -213,7 +213,7 @@ const indexerScript = async ( } else if (format === 'jag') { const store = new JagStore(numericBuildNumber !== -1 ? numericBuildNumber : build, dir); await store.load(); - store.jag.loadLocalJagFiles(); + store.jag.readLocalJagFiles(); // @todo 18/07/22 - Kiko diff --git a/src/scripts/script-executor.ts b/src/scripts/script-executor.ts index 6407435..9b2e8e4 100644 --- a/src/scripts/script-executor.ts +++ b/src/scripts/script-executor.ts @@ -18,9 +18,8 @@ export class ScriptExecutor { ): void { (async function(args: T) { await script(args); - }(this.getArguments(argumentOptions))); - // .catch(logger.error); - // .finally(() => process.exit(0)); + }(this.getArguments(argumentOptions))) + .finally(() => process.exit(0)); } } From f70e4ad3479ee0d2c102cdf050887cc5f0a2b673 Mon Sep 17 00:00:00 2001 From: Kikorono Date: Tue, 2 Aug 2022 16:22:00 -0500 Subject: [PATCH 19/32] Working on JAG format cache decoding and indexing - about 50% completed --- config/jag-cache-indexes.json5 | 17 - package-lock.json | 666 ++++----------------------- package.json | 6 +- src/compress/bzip2.ts | 21 + src/compress/gzip.ts | 12 + src/compress/index.ts | 2 + src/config/file-type.ts | 3 +- src/db/index-database.ts | 2 +- src/db/index-entity.ts | 7 +- src/file-system/indexed-file-base.ts | 2 + src/file-system/jag/index.ts | 5 + src/file-system/jag/jag-archive.ts | 11 +- src/file-system/jag/jag-file.ts | 4 + src/file-system/jag/jag-index.ts | 58 +++ src/file-system/jag/jag-store.ts | 39 +- src/file-system/jag/jag.ts | 213 ++++++++- src/file-system/js5/index.ts | 5 + src/file-system/js5/js5.ts | 115 ++--- src/openrs2/openrs2.ts | 5 +- src/scripts/indexer.ts | 75 ++- src/scripts/script-executor.ts | 4 +- 21 files changed, 594 insertions(+), 678 deletions(-) delete mode 100644 config/jag-cache-indexes.json5 create mode 100644 src/compress/bzip2.ts create mode 100644 src/compress/gzip.ts create mode 100644 src/compress/index.ts create mode 100644 src/file-system/jag/index.ts create mode 100644 src/file-system/js5/index.ts diff --git a/config/jag-cache-indexes.json5 b/config/jag-cache-indexes.json5 deleted file mode 100644 index 58d0cb1..0000000 --- a/config/jag-cache-indexes.json5 +++ /dev/null @@ -1,17 +0,0 @@ -{ - archives: { - key: 0 - }, - models: { - key: 1 - }, - animations: { - key: 2 - }, - midi: { - key: 3 - }, - maps: { - key: 4 - } -} diff --git a/package-lock.json b/package-lock.json index 811fe2a..01bf1f5 100644 --- a/package-lock.json +++ b/package-lock.json @@ -9,16 +9,18 @@ "version": "1.0.0-next.0", "license": "GPL-3.0", "dependencies": { - "@runejs/common": "2.0.2-beta.2", + "@runejs/common": "file:../common", "adm-zip": "^0.5.9", "axios": "^0.27.2", + "compressjs": "^1.0.3", "graceful-fs": ">=4.2.0", "json5": "^2.2.0", "reflect-metadata": "^0.1.13", "sqlite": "^4.0.25", "tslib": ">=2.3.0", "typeorm": "^0.2.44", - "yargs": "^17.3.1" + "yargs": "^17.3.1", + "zlib": "^1.0.5" }, "devDependencies": { "@runejs/eslint-config": "^1.1.0", @@ -43,6 +45,34 @@ "typescript": ">=4.5.0" } }, + "../common": { + "version": "3.0.0-beta.3", + "license": "GPL-3.0", + "dependencies": { + "buffer": "^6.0.3", + "compressjs": "^1.0.3", + "dotenv": "^16.0.0", + "path": "^0.12.7", + "pino": "^7.10.0", + "pino-pretty": "^7.6.1", + "zlib": "^1.0.5" + }, + "devDependencies": { + "@runejs/eslint-config": "^2.0.0", + "@types/node": "^16.11.26", + "@typescript-eslint/eslint-plugin": "^5.14.0", + "@typescript-eslint/parser": "^5.14.0", + "copyfiles": "^2.4.1", + "eslint": "^8.11.0", + "rimraf": "^3.0.2", + "ts-node-dev": "^1.1.8", + "tslib": "^2.3.1", + "typescript": "^4.6.3" + }, + "peerDependencies": { + "tslib": ">=2.3.0" + } + }, "../common/lib": { "name": "@runejs/common", "version": "2.0.2-beta.2", @@ -111,11 +141,6 @@ "js-yaml": "bin/js-yaml.js" } }, - "node_modules/@hapi/bourne": { - "version": "2.0.0", - "resolved": "https://registry.npmjs.org/@hapi/bourne/-/bourne-2.0.0.tgz", - "integrity": "sha512-WEezM1FWztfbzqIUbsDzFRVMxSoLy3HugVcux6KDDtTqzPsLE8NDRHfXvev66aH1i2oOKKar3/XDjbvh/OUBdg==" - }, "node_modules/@humanwhocodes/config-array": { "version": "0.9.5", "resolved": "https://registry.npmjs.org/@humanwhocodes/config-array/-/config-array-0.9.5.tgz", @@ -172,21 +197,8 @@ } }, "node_modules/@runejs/common": { - "version": "2.0.2-beta.2", - "resolved": "https://registry.npmjs.org/@runejs/common/-/common-2.0.2-beta.2.tgz", - "integrity": "sha512-Xd9/Ws3ByAdI1R08Ye9fWqlS8SImjQw05Bsw0w46zcGyDx8N6jmeClf9Y/s4Akh9VZQuoAMFg2ANKsRK0MucNw==", - "dependencies": { - "compressjs": "^1.0.3", - "js-yaml": "^3.14.1", - "pino": "^6.14.0", - "pino-pretty": "^4.8.0", - "sonic-boom": "^2.6.0", - "tslib": "^2.3.1" - }, - "peerDependencies": { - "tslib": ">=2.3.0", - "typescript": ">=4.5.0" - } + "resolved": "../common", + "link": true }, "node_modules/@runejs/eslint-config": { "version": "1.1.0", @@ -512,17 +524,6 @@ "node": ">=0.10.0" } }, - "node_modules/ansi-styles": { - "version": "3.2.1", - "resolved": "https://registry.npmjs.org/ansi-styles/-/ansi-styles-3.2.1.tgz", - "integrity": "sha512-VT0ZI6kZRdTh8YyJw3SMbYm/u+NqfsAxEpWO0Pf9sq8/e94WxxOpPKx9FR1FlyCtOVDNOQ+8ntlqFxiRc+r5qA==", - "dependencies": { - "color-convert": "^1.9.0" - }, - "engines": { - "node": ">=4" - } - }, "node_modules/any-promise": { "version": "1.3.0", "resolved": "https://registry.npmjs.org/any-promise/-/any-promise-1.3.0.tgz", @@ -571,41 +572,6 @@ "integrity": "sha512-58S9QDqG0Xx27YwPSt9fJxivjYl432YCwfDMfZ+71RAqUrZef7LrKQZ3LHLOwCS4FLNBplP533Zx895SeOCHvA==", "dev": true }, - "node_modules/argparse": { - "version": "1.0.10", - "resolved": "https://registry.npmjs.org/argparse/-/argparse-1.0.10.tgz", - "integrity": "sha512-o5Roy6tNG4SL/FOkCAN6RzjiakZS25RLYFrcMttJqbdd8BWrnA+fGz57iN5Pb06pvBGvl5gQ0B48dJlslXvoTg==", - "dependencies": { - "sprintf-js": "~1.0.2" - } - }, - "node_modules/args": { - "version": "5.0.1", - "resolved": "https://registry.npmjs.org/args/-/args-5.0.1.tgz", - "integrity": "sha512-1kqmFCFsPffavQFGt8OxJdIcETti99kySRUPMpOhaGjL6mRJn8HFU1OxKY5bMqfZKUwTQc1mZkAjmGYaVOHFtQ==", - "dependencies": { - "camelcase": "5.0.0", - "chalk": "2.4.2", - "leven": "2.1.0", - "mri": "1.1.4" - }, - "engines": { - "node": ">= 6.0.0" - } - }, - "node_modules/args/node_modules/chalk": { - "version": "2.4.2", - "resolved": "https://registry.npmjs.org/chalk/-/chalk-2.4.2.tgz", - "integrity": "sha512-Mti+f9lpJNcwF4tWV8/OrTTtF1gZi+f8FqlyAdouralcFWFQWF2+NgCHShjkCb+IFBLq9buZwE1xckQU4peSuQ==", - "dependencies": { - "ansi-styles": "^3.2.1", - "escape-string-regexp": "^1.0.5", - "supports-color": "^5.3.0" - }, - "engines": { - "node": ">=4" - } - }, "node_modules/array-union": { "version": "2.1.0", "resolved": "https://registry.npmjs.org/array-union/-/array-union-2.1.0.tgz", @@ -620,14 +586,6 @@ "resolved": "https://registry.npmjs.org/asynckit/-/asynckit-0.4.0.tgz", "integrity": "sha512-Oei9OH4tRh0YqU3GxhX79dM/mwVgvbZJaSNaRk+bshkj0S5cfHcgYakreBjrHwatXKbz+IoIdYLxrKim2MjW0Q==" }, - "node_modules/atomic-sleep": { - "version": "1.0.0", - "resolved": "https://registry.npmjs.org/atomic-sleep/-/atomic-sleep-1.0.0.tgz", - "integrity": "sha512-kNOjDqAh7px0XWNI+4QbzoiR/nTkHAWNud2uvnJquD1/x5a7EQZMJT0AczqK0Qn67oY/TTQ1LbUKajZpp3I9tQ==", - "engines": { - "node": ">=8.0.0" - } - }, "node_modules/axios": { "version": "0.27.2", "resolved": "https://registry.npmjs.org/axios/-/axios-0.27.2.tgz", @@ -775,14 +733,6 @@ "node": ">=6" } }, - "node_modules/camelcase": { - "version": "5.0.0", - "resolved": "https://registry.npmjs.org/camelcase/-/camelcase-5.0.0.tgz", - "integrity": "sha512-faqwZqnWxbxn+F1d399ygeamQNy3lPp/H9H6rNrqYh4FSVCtcY+3cub1MxA8o9mDd55mM8Aghuu/kuyYA6VTsA==", - "engines": { - "node": ">=6" - } - }, "node_modules/chalk": { "version": "4.1.0", "resolved": "https://registry.npmjs.org/chalk/-/chalk-4.1.0.tgz", @@ -1024,19 +974,6 @@ "node": ">=0.10.0" } }, - "node_modules/color-convert": { - "version": "1.9.3", - "resolved": "https://registry.npmjs.org/color-convert/-/color-convert-1.9.3.tgz", - "integrity": "sha512-QfAUtd+vFdAtFQcC8CCyYt1fYWxSqAiK2cSD6zDB8N3cpsEBAvRxp9zOGg6G/SHHJYAT88/az/IuDGALsNVbGg==", - "dependencies": { - "color-name": "1.1.3" - } - }, - "node_modules/color-name": { - "version": "1.1.3", - "resolved": "https://registry.npmjs.org/color-name/-/color-name-1.1.3.tgz", - "integrity": "sha1-p9BVi9icQveV3UIyj3QIMcpTvCU=" - }, "node_modules/combined-stream": { "version": "1.0.8", "resolved": "https://registry.npmjs.org/combined-stream/-/combined-stream-1.0.8.tgz", @@ -1062,7 +999,7 @@ "node_modules/compressjs": { "version": "1.0.3", "resolved": "https://registry.npmjs.org/compressjs/-/compressjs-1.0.3.tgz", - "integrity": "sha1-ldt03VuQOM+AvKMhqw7eJxtJWbY=", + "integrity": "sha512-jpKJjBTretQACTGLNuvnozP1JdP2ZLrjdGdBgk/tz1VfXlUcBhhSZW6vEsuThmeot/yjvSrPQKEgfF3X2Lpi8Q==", "dependencies": { "amdefine": "~1.0.0", "commander": "~2.8.1" @@ -1198,14 +1135,6 @@ "node": ">= 8" } }, - "node_modules/dateformat": { - "version": "4.6.3", - "resolved": "https://registry.npmjs.org/dateformat/-/dateformat-4.6.3.tgz", - "integrity": "sha512-2P0p0pFGzHS5EMnhdxQi7aJN+iMheud0UhG4dlE1DLAlvL8JHjJJTX/CSm4JXwV0Ka5nGk3zC5mcb5bUQUxxMA==", - "engines": { - "node": "*" - } - }, "node_modules/debug": { "version": "4.3.2", "resolved": "https://registry.npmjs.org/debug/-/debug-4.3.2.tgz", @@ -1325,6 +1254,7 @@ "version": "1.4.4", "resolved": "https://registry.npmjs.org/end-of-stream/-/end-of-stream-1.4.4.tgz", "integrity": "sha512-+uw1inIHVPQoaVuHzRyXd21icM+cnt4CzD5rW+NC1wjOUSTOs+Te7FOv7AhN7vS9x/oIyhLP5PR1H+phQAHu5Q==", + "devOptional": true, "dependencies": { "once": "^1.4.0" } @@ -1337,14 +1267,6 @@ "node": ">=6" } }, - "node_modules/escape-string-regexp": { - "version": "1.0.5", - "resolved": "https://registry.npmjs.org/escape-string-regexp/-/escape-string-regexp-1.0.5.tgz", - "integrity": "sha1-G2HAViGQqN/2rjuyzwIAyhMLhtQ=", - "engines": { - "node": ">=0.8.0" - } - }, "node_modules/eslint": { "version": "8.11.0", "resolved": "https://registry.npmjs.org/eslint/-/eslint-8.11.0.tgz", @@ -1545,18 +1467,6 @@ "node": "^12.22.0 || ^14.17.0 || >=16.0.0" } }, - "node_modules/esprima": { - "version": "4.0.1", - "resolved": "https://registry.npmjs.org/esprima/-/esprima-4.0.1.tgz", - "integrity": "sha512-eGuFFw7Upda+g4p+QHvnW0RyTX/SVeJBDM/gCtMARO0cLuT2HcEKnTPvhjV6aGeqrCB/sbNop0Kszm0jsaWU4A==", - "bin": { - "esparse": "bin/esparse.js", - "esvalidate": "bin/esvalidate.js" - }, - "engines": { - "node": ">=4" - } - }, "node_modules/esquery": { "version": "1.4.0", "resolved": "https://registry.npmjs.org/esquery/-/esquery-1.4.0.tgz", @@ -1660,19 +1570,6 @@ "integrity": "sha1-PYpcZog6FqMMqGQ+hR8Zuqd5eRc=", "dev": true }, - "node_modules/fast-redact": { - "version": "3.1.1", - "resolved": "https://registry.npmjs.org/fast-redact/-/fast-redact-3.1.1.tgz", - "integrity": "sha512-odVmjC8x8jNeMZ3C+rPMESzXVSEU8tSWSHv9HFxP2mm89G/1WwqhrerJDQm9Zus8X6aoRgQDThKqptdNA6bt+A==", - "engines": { - "node": ">=6" - } - }, - "node_modules/fast-safe-stringify": { - "version": "2.1.1", - "resolved": "https://registry.npmjs.org/fast-safe-stringify/-/fast-safe-stringify-2.1.1.tgz", - "integrity": "sha512-W+KJc2dmILlPplD/H4K9l9LcAHAfPtP6BY84uVLXQ6Evcz9Lcg33Y2z1IVblT6xdY54PXYVHEv+0Wpq8Io6zkA==" - }, "node_modules/fastq": { "version": "1.13.0", "resolved": "https://registry.npmjs.org/fastq/-/fastq-1.13.0.tgz", @@ -1725,11 +1622,6 @@ "node": "^10.12.0 || >=12.0.0" } }, - "node_modules/flatstr": { - "version": "1.0.12", - "resolved": "https://registry.npmjs.org/flatstr/-/flatstr-1.0.12.tgz", - "integrity": "sha512-4zPxDyhCyiN2wIAtSLI6gc82/EjqZc1onI4Mz/l0pWrAlsSfYH/2ZIcU+e3oA2wDwbzIWNKwa23F8rh6+DRWkw==" - }, "node_modules/flatted": { "version": "3.2.5", "resolved": "https://registry.npmjs.org/flatted/-/flatted-3.2.5.tgz", @@ -1923,14 +1815,6 @@ "node": ">= 0.4.0" } }, - "node_modules/has-flag": { - "version": "3.0.0", - "resolved": "https://registry.npmjs.org/has-flag/-/has-flag-3.0.0.tgz", - "integrity": "sha1-tdRU3CGZriJWmfNGfloH87lVuv0=", - "engines": { - "node": ">=4" - } - }, "node_modules/has-unicode": { "version": "2.0.1", "resolved": "https://registry.npmjs.org/has-unicode/-/has-unicode-2.0.1.tgz", @@ -2096,34 +1980,6 @@ "integrity": "sha1-6PvzdNxVb/iUehDcsFctYz8s+hA=", "dev": true }, - "node_modules/jmespath": { - "version": "0.15.0", - "resolved": "https://registry.npmjs.org/jmespath/-/jmespath-0.15.0.tgz", - "integrity": "sha1-o/Iiqarp+Wb10nx5ZRDigJF2Qhc=", - "engines": { - "node": ">= 0.6.0" - } - }, - "node_modules/joycon": { - "version": "2.2.5", - "resolved": "https://registry.npmjs.org/joycon/-/joycon-2.2.5.tgz", - "integrity": "sha512-YqvUxoOcVPnCp0VU1/56f+iKSdvIRJYPznH22BdXV3xMk75SFXhWeJkZ8C9XxUWt1b5x2X1SxuFygW1U0FmkEQ==", - "engines": { - "node": ">=6" - } - }, - "node_modules/js-yaml": { - "version": "3.14.1", - "resolved": "https://registry.npmjs.org/js-yaml/-/js-yaml-3.14.1.tgz", - "integrity": "sha512-okMH7OXXJ7YrN9Ok3/SXrnu4iX9yOk+25nqX4imS2npuvTYDmo/QEZoqwZkYaIDk3jVvBOTOIEgEhaLOynBS9g==", - "dependencies": { - "argparse": "^1.0.7", - "esprima": "^4.0.0" - }, - "bin": { - "js-yaml": "bin/js-yaml.js" - } - }, "node_modules/json-schema-traverse": { "version": "0.4.1", "resolved": "https://registry.npmjs.org/json-schema-traverse/-/json-schema-traverse-0.4.1.tgz", @@ -2150,14 +2006,6 @@ "node": ">=6" } }, - "node_modules/leven": { - "version": "2.1.0", - "resolved": "https://registry.npmjs.org/leven/-/leven-2.1.0.tgz", - "integrity": "sha1-wuep93IJTe6dNCAq6KzORoeHVYA=", - "engines": { - "node": ">=0.10.0" - } - }, "node_modules/levn": { "version": "0.4.1", "resolved": "https://registry.npmjs.org/levn/-/levn-0.4.1.tgz", @@ -2281,14 +2129,6 @@ "integrity": "sha512-gKLcREMhtuZRwRAfqP3RFW+TK4JqApVBtOIftVgjuABpAtpxhPGaDcfvbhNvD0B8iD1oUr/txX35NjcaY6Ns/A==", "devOptional": true }, - "node_modules/mri": { - "version": "1.1.4", - "resolved": "https://registry.npmjs.org/mri/-/mri-1.1.4.tgz", - "integrity": "sha512-6y7IjGPm8AzlvoUrwAaw1tLnUBudaS3752vcd8JtrpGGQn+rXIe63LFVHm/YMwtqAuh+LJPCFdlLYPWM1nYn6w==", - "engines": { - "node": ">=4" - } - }, "node_modules/ms": { "version": "2.1.2", "resolved": "https://registry.npmjs.org/ms/-/ms-2.1.2.tgz", @@ -2499,72 +2339,6 @@ "url": "https://github.com/sponsors/jonschlinkert" } }, - "node_modules/pino": { - "version": "6.14.0", - "resolved": "https://registry.npmjs.org/pino/-/pino-6.14.0.tgz", - "integrity": "sha512-iuhEDel3Z3hF9Jfe44DPXR8l07bhjuFY3GMHIXbjnY9XcafbyDDwl2sN2vw2GjMPf5Nkoe+OFao7ffn9SXaKDg==", - "dependencies": { - "fast-redact": "^3.0.0", - "fast-safe-stringify": "^2.0.8", - "flatstr": "^1.0.12", - "pino-std-serializers": "^3.1.0", - "process-warning": "^1.0.0", - "quick-format-unescaped": "^4.0.3", - "sonic-boom": "^1.0.2" - }, - "bin": { - "pino": "bin.js" - } - }, - "node_modules/pino-pretty": { - "version": "4.8.0", - "resolved": "https://registry.npmjs.org/pino-pretty/-/pino-pretty-4.8.0.tgz", - "integrity": "sha512-mhQfHG4rw5ZFpWL44m0Utjo4GC2+HMfdNvxyA8lLw0sIqn6fCf7uQe6dPckUcW/obly+OQHD7B/MTso6LNizYw==", - "dependencies": { - "@hapi/bourne": "^2.0.0", - "args": "^5.0.1", - "chalk": "^4.0.0", - "dateformat": "^4.5.1", - "fast-safe-stringify": "^2.0.7", - "jmespath": "^0.15.0", - "joycon": "^2.2.5", - "pump": "^3.0.0", - "readable-stream": "^3.6.0", - "rfdc": "^1.3.0", - "split2": "^3.1.1", - "strip-json-comments": "^3.1.1" - }, - "bin": { - "pino-pretty": "bin.js" - } - }, - "node_modules/pino-pretty/node_modules/readable-stream": { - "version": "3.6.0", - "resolved": "https://registry.npmjs.org/readable-stream/-/readable-stream-3.6.0.tgz", - "integrity": "sha512-BViHy7LKeTz4oNnkcLJ+lVSL6vpiFeX6/d3oSH8zCW7UxP2onchk+vTGB143xuFjHS3deTgkKoXXymXqymiIdA==", - "dependencies": { - "inherits": "^2.0.3", - "string_decoder": "^1.1.1", - "util-deprecate": "^1.0.1" - }, - "engines": { - "node": ">= 6" - } - }, - "node_modules/pino-std-serializers": { - "version": "3.2.0", - "resolved": "https://registry.npmjs.org/pino-std-serializers/-/pino-std-serializers-3.2.0.tgz", - "integrity": "sha512-EqX4pwDPrt3MuOAAUBMU0Tk5kR/YcCM5fNPEzgCO2zJ5HfX0vbiH9HbJglnyeQsN96Kznae6MWD47pZB5avTrg==" - }, - "node_modules/pino/node_modules/sonic-boom": { - "version": "1.4.1", - "resolved": "https://registry.npmjs.org/sonic-boom/-/sonic-boom-1.4.1.tgz", - "integrity": "sha512-LRHh/A8tpW7ru89lrlkU4AszXt1dbwSjVWguGrmlxE7tawVmDBlI1PILMkXAxJTwqhgsEeTHzj36D5CmHgQmNg==", - "dependencies": { - "atomic-sleep": "^1.0.0", - "flatstr": "^1.0.12" - } - }, "node_modules/prebuild-install": { "version": "7.0.1", "resolved": "https://registry.npmjs.org/prebuild-install/-/prebuild-install-7.0.1.tgz", @@ -2616,15 +2390,11 @@ "integrity": "sha512-3ouUOpQhtgrbOa17J7+uxOTpITYWaGP7/AhoR3+A+/1e9skrzelGi/dXzEYyvbxubEF6Wn2ypscTKiKJFFn1ag==", "devOptional": true }, - "node_modules/process-warning": { - "version": "1.0.0", - "resolved": "https://registry.npmjs.org/process-warning/-/process-warning-1.0.0.tgz", - "integrity": "sha512-du4wfLyj4yCZq1VupnVSZmRsPJsNuxoDQFdCFHLaYiEbFBD7QE0a+I4D7hOxrVnh78QE/YipFAj9lXHiXocV+Q==" - }, "node_modules/pump": { "version": "3.0.0", "resolved": "https://registry.npmjs.org/pump/-/pump-3.0.0.tgz", "integrity": "sha512-LwZy+p3SFs1Pytd/jYct4wpv49HiYCqd9Rlc5ZVdk0V+8Yzv6jR5Blk3TRmPL1ft69TxP0IMZGJ+WPFU2BFhww==", + "devOptional": true, "dependencies": { "end-of-stream": "^1.1.0", "once": "^1.3.1" @@ -2659,11 +2429,6 @@ } ] }, - "node_modules/quick-format-unescaped": { - "version": "4.0.4", - "resolved": "https://registry.npmjs.org/quick-format-unescaped/-/quick-format-unescaped-4.0.4.tgz", - "integrity": "sha512-tYC1Q1hgyRuHgloV/YXs2w15unPVh8qfu/qCTfhTYamaw7fyhumKa2yGpdSo87vY32rIclj+4fWYQXUMs9EHvg==" - }, "node_modules/rc": { "version": "1.2.8", "resolved": "https://registry.npmjs.org/rc/-/rc-1.2.8.tgz", @@ -2776,11 +2541,6 @@ "node": ">=0.10.0" } }, - "node_modules/rfdc": { - "version": "1.3.0", - "resolved": "https://registry.npmjs.org/rfdc/-/rfdc-1.3.0.tgz", - "integrity": "sha512-V2hovdzFbOi77/WajaSMXk2OLm+xNIeQdMMuB7icj7bk6zi2F8GGAxigcnDFpJHbNyNcgyJDiP+8nOrY5cZGrA==" - }, "node_modules/rimraf": { "version": "3.0.2", "resolved": "https://registry.npmjs.org/rimraf/-/rimraf-3.0.2.tgz", @@ -2943,14 +2703,6 @@ "node": ">=8" } }, - "node_modules/sonic-boom": { - "version": "2.6.0", - "resolved": "https://registry.npmjs.org/sonic-boom/-/sonic-boom-2.6.0.tgz", - "integrity": "sha512-6xYZFRmDEtxGqfOKcDQ4cPLrNa0SPEDI+wlzDAHowXE6YV42NeXqg9mP2KkiM8JVu3lHfZ2iQKYlGOz+kTpphg==", - "dependencies": { - "atomic-sleep": "^1.0.0" - } - }, "node_modules/source-map": { "version": "0.6.1", "resolved": "https://registry.npmjs.org/source-map/-/source-map-0.6.1.tgz", @@ -2970,32 +2722,6 @@ "source-map": "^0.6.0" } }, - "node_modules/split2": { - "version": "3.2.2", - "resolved": "https://registry.npmjs.org/split2/-/split2-3.2.2.tgz", - "integrity": "sha512-9NThjpgZnifTkJpzTZ7Eue85S49QwpNhZTq6GRJwObb6jnLFNGB7Qm73V5HewTROPyxD0C29xqmaI68bQtV+hg==", - "dependencies": { - "readable-stream": "^3.0.0" - } - }, - "node_modules/split2/node_modules/readable-stream": { - "version": "3.6.0", - "resolved": "https://registry.npmjs.org/readable-stream/-/readable-stream-3.6.0.tgz", - "integrity": "sha512-BViHy7LKeTz4oNnkcLJ+lVSL6vpiFeX6/d3oSH8zCW7UxP2onchk+vTGB143xuFjHS3deTgkKoXXymXqymiIdA==", - "dependencies": { - "inherits": "^2.0.3", - "string_decoder": "^1.1.1", - "util-deprecate": "^1.0.1" - }, - "engines": { - "node": ">= 6" - } - }, - "node_modules/sprintf-js": { - "version": "1.0.3", - "resolved": "https://registry.npmjs.org/sprintf-js/-/sprintf-js-1.0.3.tgz", - "integrity": "sha1-BOaSb2YolTVPPdAVIDYzuFcpfiw=" - }, "node_modules/sqlite": { "version": "4.0.25", "resolved": "https://registry.npmjs.org/sqlite/-/sqlite-4.0.25.tgz", @@ -3005,6 +2731,7 @@ "version": "1.1.1", "resolved": "https://registry.npmjs.org/string_decoder/-/string_decoder-1.1.1.tgz", "integrity": "sha512-n/ShnvDi6FHbbVfviro+WojiFzv+s8MPMHBczVePfUpDJLwoLT0ht1l4YwBCbi8pJAveEEdnkHyPyTP/mzRfwg==", + "devOptional": true, "dependencies": { "safe-buffer": "~5.1.0" } @@ -3048,6 +2775,7 @@ "version": "3.1.1", "resolved": "https://registry.npmjs.org/strip-json-comments/-/strip-json-comments-3.1.1.tgz", "integrity": "sha512-6fPc+R4ihwqP6N/aIv2f1gMH8lOVtWQHoqC4yK6oSDVVocumAsfCqjkXnqiYMhmMwS/mEHLp7Vehlt3ql6lEig==", + "dev": true, "engines": { "node": ">=8" }, @@ -3055,17 +2783,6 @@ "url": "https://github.com/sponsors/sindresorhus" } }, - "node_modules/supports-color": { - "version": "5.5.0", - "resolved": "https://registry.npmjs.org/supports-color/-/supports-color-5.5.0.tgz", - "integrity": "sha512-QjVjwdXIt408MIiAqCX4oUKsgU2EqAGzs2Ppkm4aQYbjm+ZEWEcW4SfFNTr4uMNZma0ey4f5lgLrkB0aX0QMow==", - "dependencies": { - "has-flag": "^3.0.0" - }, - "engines": { - "node": ">=4" - } - }, "node_modules/supports-preserve-symlinks-flag": { "version": "1.0.0", "resolved": "https://registry.npmjs.org/supports-preserve-symlinks-flag/-/supports-preserve-symlinks-flag-1.0.0.tgz", @@ -3468,6 +3185,7 @@ "version": "4.5.5", "resolved": "https://registry.npmjs.org/typescript/-/typescript-4.5.5.tgz", "integrity": "sha512-TCTIul70LyWe6IJWT8QSYeA54WQe8EjQFU4wY52Fasj5UKx88LNYKCgBEHcOMOrFF1rKGbD8v/xcNWVUq9SymA==", + "dev": true, "bin": { "tsc": "bin/tsc", "tsserver": "bin/tsserver" @@ -3497,7 +3215,8 @@ "node_modules/util-deprecate": { "version": "1.0.2", "resolved": "https://registry.npmjs.org/util-deprecate/-/util-deprecate-1.0.2.tgz", - "integrity": "sha1-RQ1Nyfpw3nMnYvvS1KKJgUGaDM8=" + "integrity": "sha1-RQ1Nyfpw3nMnYvvS1KKJgUGaDM8=", + "devOptional": true }, "node_modules/uuid": { "version": "8.3.2", @@ -3767,6 +3486,15 @@ "@types/zen-observable": "0.8.3", "zen-observable": "0.8.15" } + }, + "node_modules/zlib": { + "version": "1.0.5", + "resolved": "https://registry.npmjs.org/zlib/-/zlib-1.0.5.tgz", + "integrity": "sha512-40fpE2II+Cd3k8HWTWONfeKE2jL+P42iWJ1zzps5W51qcTsOUKM5Q5m2PFb0CLxlmFAaUuUdJGc3OfZy947v0w==", + "hasInstallScript": true, + "engines": { + "node": ">=0.2.0" + } } }, "dependencies": { @@ -3804,11 +3532,6 @@ } } }, - "@hapi/bourne": { - "version": "2.0.0", - "resolved": "https://registry.npmjs.org/@hapi/bourne/-/bourne-2.0.0.tgz", - "integrity": "sha512-WEezM1FWztfbzqIUbsDzFRVMxSoLy3HugVcux6KDDtTqzPsLE8NDRHfXvev66aH1i2oOKKar3/XDjbvh/OUBdg==" - }, "@humanwhocodes/config-array": { "version": "0.9.5", "resolved": "https://registry.npmjs.org/@humanwhocodes/config-array/-/config-array-0.9.5.tgz", @@ -3853,16 +3576,25 @@ } }, "@runejs/common": { - "version": "2.0.2-beta.2", - "resolved": "https://registry.npmjs.org/@runejs/common/-/common-2.0.2-beta.2.tgz", - "integrity": "sha512-Xd9/Ws3ByAdI1R08Ye9fWqlS8SImjQw05Bsw0w46zcGyDx8N6jmeClf9Y/s4Akh9VZQuoAMFg2ANKsRK0MucNw==", + "version": "file:../common", "requires": { + "@runejs/eslint-config": "^2.0.0", + "@types/node": "^16.11.26", + "@typescript-eslint/eslint-plugin": "^5.14.0", + "@typescript-eslint/parser": "^5.14.0", + "buffer": "^6.0.3", "compressjs": "^1.0.3", - "js-yaml": "^3.14.1", - "pino": "^6.14.0", - "pino-pretty": "^4.8.0", - "sonic-boom": "^2.6.0", - "tslib": "^2.3.1" + "copyfiles": "^2.4.1", + "dotenv": "^16.0.0", + "eslint": "^8.11.0", + "path": "^0.12.7", + "pino": "^7.10.0", + "pino-pretty": "^7.6.1", + "rimraf": "^3.0.2", + "ts-node-dev": "^1.1.8", + "tslib": "^2.3.1", + "typescript": "^4.6.3", + "zlib": "^1.0.5" } }, "@runejs/eslint-config": { @@ -4075,14 +3807,6 @@ "integrity": "sha1-w7M6te42DYbg5ijwRorn7yfWVN8=", "devOptional": true }, - "ansi-styles": { - "version": "3.2.1", - "resolved": "https://registry.npmjs.org/ansi-styles/-/ansi-styles-3.2.1.tgz", - "integrity": "sha512-VT0ZI6kZRdTh8YyJw3SMbYm/u+NqfsAxEpWO0Pf9sq8/e94WxxOpPKx9FR1FlyCtOVDNOQ+8ntlqFxiRc+r5qA==", - "requires": { - "color-convert": "^1.9.0" - } - }, "any-promise": { "version": "1.3.0", "resolved": "https://registry.npmjs.org/any-promise/-/any-promise-1.3.0.tgz", @@ -4125,37 +3849,6 @@ "integrity": "sha512-58S9QDqG0Xx27YwPSt9fJxivjYl432YCwfDMfZ+71RAqUrZef7LrKQZ3LHLOwCS4FLNBplP533Zx895SeOCHvA==", "dev": true }, - "argparse": { - "version": "1.0.10", - "resolved": "https://registry.npmjs.org/argparse/-/argparse-1.0.10.tgz", - "integrity": "sha512-o5Roy6tNG4SL/FOkCAN6RzjiakZS25RLYFrcMttJqbdd8BWrnA+fGz57iN5Pb06pvBGvl5gQ0B48dJlslXvoTg==", - "requires": { - "sprintf-js": "~1.0.2" - } - }, - "args": { - "version": "5.0.1", - "resolved": "https://registry.npmjs.org/args/-/args-5.0.1.tgz", - "integrity": "sha512-1kqmFCFsPffavQFGt8OxJdIcETti99kySRUPMpOhaGjL6mRJn8HFU1OxKY5bMqfZKUwTQc1mZkAjmGYaVOHFtQ==", - "requires": { - "camelcase": "5.0.0", - "chalk": "2.4.2", - "leven": "2.1.0", - "mri": "1.1.4" - }, - "dependencies": { - "chalk": { - "version": "2.4.2", - "resolved": "https://registry.npmjs.org/chalk/-/chalk-2.4.2.tgz", - "integrity": "sha512-Mti+f9lpJNcwF4tWV8/OrTTtF1gZi+f8FqlyAdouralcFWFQWF2+NgCHShjkCb+IFBLq9buZwE1xckQU4peSuQ==", - "requires": { - "ansi-styles": "^3.2.1", - "escape-string-regexp": "^1.0.5", - "supports-color": "^5.3.0" - } - } - } - }, "array-union": { "version": "2.1.0", "resolved": "https://registry.npmjs.org/array-union/-/array-union-2.1.0.tgz", @@ -4167,11 +3860,6 @@ "resolved": "https://registry.npmjs.org/asynckit/-/asynckit-0.4.0.tgz", "integrity": "sha512-Oei9OH4tRh0YqU3GxhX79dM/mwVgvbZJaSNaRk+bshkj0S5cfHcgYakreBjrHwatXKbz+IoIdYLxrKim2MjW0Q==" }, - "atomic-sleep": { - "version": "1.0.0", - "resolved": "https://registry.npmjs.org/atomic-sleep/-/atomic-sleep-1.0.0.tgz", - "integrity": "sha512-kNOjDqAh7px0XWNI+4QbzoiR/nTkHAWNud2uvnJquD1/x5a7EQZMJT0AczqK0Qn67oY/TTQ1LbUKajZpp3I9tQ==" - }, "axios": { "version": "0.27.2", "resolved": "https://registry.npmjs.org/axios/-/axios-0.27.2.tgz", @@ -4280,11 +3968,6 @@ "integrity": "sha512-P8BjAsXvZS+VIDUI11hHCQEv74YT67YUi5JJFNWIqL235sBmjX4+qx9Muvls5ivyNENctx46xQLQ3aTuE7ssaQ==", "dev": true }, - "camelcase": { - "version": "5.0.0", - "resolved": "https://registry.npmjs.org/camelcase/-/camelcase-5.0.0.tgz", - "integrity": "sha512-faqwZqnWxbxn+F1d399ygeamQNy3lPp/H9H6rNrqYh4FSVCtcY+3cub1MxA8o9mDd55mM8Aghuu/kuyYA6VTsA==" - }, "chalk": { "version": "4.1.0", "resolved": "https://registry.npmjs.org/chalk/-/chalk-4.1.0.tgz", @@ -4460,19 +4143,6 @@ "integrity": "sha1-DQcLTQQ6W+ozovGkDi7bPZpMz3c=", "devOptional": true }, - "color-convert": { - "version": "1.9.3", - "resolved": "https://registry.npmjs.org/color-convert/-/color-convert-1.9.3.tgz", - "integrity": "sha512-QfAUtd+vFdAtFQcC8CCyYt1fYWxSqAiK2cSD6zDB8N3cpsEBAvRxp9zOGg6G/SHHJYAT88/az/IuDGALsNVbGg==", - "requires": { - "color-name": "1.1.3" - } - }, - "color-name": { - "version": "1.1.3", - "resolved": "https://registry.npmjs.org/color-name/-/color-name-1.1.3.tgz", - "integrity": "sha1-p9BVi9icQveV3UIyj3QIMcpTvCU=" - }, "combined-stream": { "version": "1.0.8", "resolved": "https://registry.npmjs.org/combined-stream/-/combined-stream-1.0.8.tgz", @@ -4492,7 +4162,7 @@ "compressjs": { "version": "1.0.3", "resolved": "https://registry.npmjs.org/compressjs/-/compressjs-1.0.3.tgz", - "integrity": "sha1-ldt03VuQOM+AvKMhqw7eJxtJWbY=", + "integrity": "sha512-jpKJjBTretQACTGLNuvnozP1JdP2ZLrjdGdBgk/tz1VfXlUcBhhSZW6vEsuThmeot/yjvSrPQKEgfF3X2Lpi8Q==", "requires": { "amdefine": "~1.0.0", "commander": "~2.8.1" @@ -4602,11 +4272,6 @@ "which": "^2.0.1" } }, - "dateformat": { - "version": "4.6.3", - "resolved": "https://registry.npmjs.org/dateformat/-/dateformat-4.6.3.tgz", - "integrity": "sha512-2P0p0pFGzHS5EMnhdxQi7aJN+iMheud0UhG4dlE1DLAlvL8JHjJJTX/CSm4JXwV0Ka5nGk3zC5mcb5bUQUxxMA==" - }, "debug": { "version": "4.3.2", "resolved": "https://registry.npmjs.org/debug/-/debug-4.3.2.tgz", @@ -4694,6 +4359,7 @@ "version": "1.4.4", "resolved": "https://registry.npmjs.org/end-of-stream/-/end-of-stream-1.4.4.tgz", "integrity": "sha512-+uw1inIHVPQoaVuHzRyXd21icM+cnt4CzD5rW+NC1wjOUSTOs+Te7FOv7AhN7vS9x/oIyhLP5PR1H+phQAHu5Q==", + "devOptional": true, "requires": { "once": "^1.4.0" } @@ -4703,11 +4369,6 @@ "resolved": "https://registry.npmjs.org/escalade/-/escalade-3.1.1.tgz", "integrity": "sha512-k0er2gUkLf8O0zKJiAhmkTnJlTvINGv7ygDNPbeIsX/TJjGJZHuh9B2UxbsaEkmlEo9MfhrSzmhIlhRlI2GXnw==" }, - "escape-string-regexp": { - "version": "1.0.5", - "resolved": "https://registry.npmjs.org/escape-string-regexp/-/escape-string-regexp-1.0.5.tgz", - "integrity": "sha1-G2HAViGQqN/2rjuyzwIAyhMLhtQ=" - }, "eslint": { "version": "8.11.0", "resolved": "https://registry.npmjs.org/eslint/-/eslint-8.11.0.tgz", @@ -4858,11 +4519,6 @@ "eslint-visitor-keys": "^3.3.0" } }, - "esprima": { - "version": "4.0.1", - "resolved": "https://registry.npmjs.org/esprima/-/esprima-4.0.1.tgz", - "integrity": "sha512-eGuFFw7Upda+g4p+QHvnW0RyTX/SVeJBDM/gCtMARO0cLuT2HcEKnTPvhjV6aGeqrCB/sbNop0Kszm0jsaWU4A==" - }, "esquery": { "version": "1.4.0", "resolved": "https://registry.npmjs.org/esquery/-/esquery-1.4.0.tgz", @@ -4946,16 +4602,6 @@ "integrity": "sha1-PYpcZog6FqMMqGQ+hR8Zuqd5eRc=", "dev": true }, - "fast-redact": { - "version": "3.1.1", - "resolved": "https://registry.npmjs.org/fast-redact/-/fast-redact-3.1.1.tgz", - "integrity": "sha512-odVmjC8x8jNeMZ3C+rPMESzXVSEU8tSWSHv9HFxP2mm89G/1WwqhrerJDQm9Zus8X6aoRgQDThKqptdNA6bt+A==" - }, - "fast-safe-stringify": { - "version": "2.1.1", - "resolved": "https://registry.npmjs.org/fast-safe-stringify/-/fast-safe-stringify-2.1.1.tgz", - "integrity": "sha512-W+KJc2dmILlPplD/H4K9l9LcAHAfPtP6BY84uVLXQ6Evcz9Lcg33Y2z1IVblT6xdY54PXYVHEv+0Wpq8Io6zkA==" - }, "fastq": { "version": "1.13.0", "resolved": "https://registry.npmjs.org/fastq/-/fastq-1.13.0.tgz", @@ -4999,11 +4645,6 @@ "rimraf": "^3.0.2" } }, - "flatstr": { - "version": "1.0.12", - "resolved": "https://registry.npmjs.org/flatstr/-/flatstr-1.0.12.tgz", - "integrity": "sha512-4zPxDyhCyiN2wIAtSLI6gc82/EjqZc1onI4Mz/l0pWrAlsSfYH/2ZIcU+e3oA2wDwbzIWNKwa23F8rh6+DRWkw==" - }, "flatted": { "version": "3.2.5", "resolved": "https://registry.npmjs.org/flatted/-/flatted-3.2.5.tgz", @@ -5146,11 +4787,6 @@ "function-bind": "^1.1.1" } }, - "has-flag": { - "version": "3.0.0", - "resolved": "https://registry.npmjs.org/has-flag/-/has-flag-3.0.0.tgz", - "integrity": "sha1-tdRU3CGZriJWmfNGfloH87lVuv0=" - }, "has-unicode": { "version": "2.0.1", "resolved": "https://registry.npmjs.org/has-unicode/-/has-unicode-2.0.1.tgz", @@ -5269,25 +4905,6 @@ "integrity": "sha1-6PvzdNxVb/iUehDcsFctYz8s+hA=", "dev": true }, - "jmespath": { - "version": "0.15.0", - "resolved": "https://registry.npmjs.org/jmespath/-/jmespath-0.15.0.tgz", - "integrity": "sha1-o/Iiqarp+Wb10nx5ZRDigJF2Qhc=" - }, - "joycon": { - "version": "2.2.5", - "resolved": "https://registry.npmjs.org/joycon/-/joycon-2.2.5.tgz", - "integrity": "sha512-YqvUxoOcVPnCp0VU1/56f+iKSdvIRJYPznH22BdXV3xMk75SFXhWeJkZ8C9XxUWt1b5x2X1SxuFygW1U0FmkEQ==" - }, - "js-yaml": { - "version": "3.14.1", - "resolved": "https://registry.npmjs.org/js-yaml/-/js-yaml-3.14.1.tgz", - "integrity": "sha512-okMH7OXXJ7YrN9Ok3/SXrnu4iX9yOk+25nqX4imS2npuvTYDmo/QEZoqwZkYaIDk3jVvBOTOIEgEhaLOynBS9g==", - "requires": { - "argparse": "^1.0.7", - "esprima": "^4.0.0" - } - }, "json-schema-traverse": { "version": "0.4.1", "resolved": "https://registry.npmjs.org/json-schema-traverse/-/json-schema-traverse-0.4.1.tgz", @@ -5308,11 +4925,6 @@ "minimist": "^1.2.5" } }, - "leven": { - "version": "2.1.0", - "resolved": "https://registry.npmjs.org/leven/-/leven-2.1.0.tgz", - "integrity": "sha1-wuep93IJTe6dNCAq6KzORoeHVYA=" - }, "levn": { "version": "0.4.1", "resolved": "https://registry.npmjs.org/levn/-/levn-0.4.1.tgz", @@ -5403,11 +5015,6 @@ "integrity": "sha512-gKLcREMhtuZRwRAfqP3RFW+TK4JqApVBtOIftVgjuABpAtpxhPGaDcfvbhNvD0B8iD1oUr/txX35NjcaY6Ns/A==", "devOptional": true }, - "mri": { - "version": "1.1.4", - "resolved": "https://registry.npmjs.org/mri/-/mri-1.1.4.tgz", - "integrity": "sha512-6y7IjGPm8AzlvoUrwAaw1tLnUBudaS3752vcd8JtrpGGQn+rXIe63LFVHm/YMwtqAuh+LJPCFdlLYPWM1nYn6w==" - }, "ms": { "version": "2.1.2", "resolved": "https://registry.npmjs.org/ms/-/ms-2.1.2.tgz", @@ -5589,67 +5196,6 @@ "integrity": "sha512-JU3teHTNjmE2VCGFzuY8EXzCDVwEqB2a8fsIvwaStHhAWJEeVd1o1QD80CU6+ZdEXXSLbSsuLwJjkCBWqRQUVA==", "dev": true }, - "pino": { - "version": "6.14.0", - "resolved": "https://registry.npmjs.org/pino/-/pino-6.14.0.tgz", - "integrity": "sha512-iuhEDel3Z3hF9Jfe44DPXR8l07bhjuFY3GMHIXbjnY9XcafbyDDwl2sN2vw2GjMPf5Nkoe+OFao7ffn9SXaKDg==", - "requires": { - "fast-redact": "^3.0.0", - "fast-safe-stringify": "^2.0.8", - "flatstr": "^1.0.12", - "pino-std-serializers": "^3.1.0", - "process-warning": "^1.0.0", - "quick-format-unescaped": "^4.0.3", - "sonic-boom": "^1.0.2" - }, - "dependencies": { - "sonic-boom": { - "version": "1.4.1", - "resolved": "https://registry.npmjs.org/sonic-boom/-/sonic-boom-1.4.1.tgz", - "integrity": "sha512-LRHh/A8tpW7ru89lrlkU4AszXt1dbwSjVWguGrmlxE7tawVmDBlI1PILMkXAxJTwqhgsEeTHzj36D5CmHgQmNg==", - "requires": { - "atomic-sleep": "^1.0.0", - "flatstr": "^1.0.12" - } - } - } - }, - "pino-pretty": { - "version": "4.8.0", - "resolved": "https://registry.npmjs.org/pino-pretty/-/pino-pretty-4.8.0.tgz", - "integrity": "sha512-mhQfHG4rw5ZFpWL44m0Utjo4GC2+HMfdNvxyA8lLw0sIqn6fCf7uQe6dPckUcW/obly+OQHD7B/MTso6LNizYw==", - "requires": { - "@hapi/bourne": "^2.0.0", - "args": "^5.0.1", - "chalk": "^4.0.0", - "dateformat": "^4.5.1", - "fast-safe-stringify": "^2.0.7", - "jmespath": "^0.15.0", - "joycon": "^2.2.5", - "pump": "^3.0.0", - "readable-stream": "^3.6.0", - "rfdc": "^1.3.0", - "split2": "^3.1.1", - "strip-json-comments": "^3.1.1" - }, - "dependencies": { - "readable-stream": { - "version": "3.6.0", - "resolved": "https://registry.npmjs.org/readable-stream/-/readable-stream-3.6.0.tgz", - "integrity": "sha512-BViHy7LKeTz4oNnkcLJ+lVSL6vpiFeX6/d3oSH8zCW7UxP2onchk+vTGB143xuFjHS3deTgkKoXXymXqymiIdA==", - "requires": { - "inherits": "^2.0.3", - "string_decoder": "^1.1.1", - "util-deprecate": "^1.0.1" - } - } - } - }, - "pino-std-serializers": { - "version": "3.2.0", - "resolved": "https://registry.npmjs.org/pino-std-serializers/-/pino-std-serializers-3.2.0.tgz", - "integrity": "sha512-EqX4pwDPrt3MuOAAUBMU0Tk5kR/YcCM5fNPEzgCO2zJ5HfX0vbiH9HbJglnyeQsN96Kznae6MWD47pZB5avTrg==" - }, "prebuild-install": { "version": "7.0.1", "resolved": "https://registry.npmjs.org/prebuild-install/-/prebuild-install-7.0.1.tgz", @@ -5691,15 +5237,11 @@ "integrity": "sha512-3ouUOpQhtgrbOa17J7+uxOTpITYWaGP7/AhoR3+A+/1e9skrzelGi/dXzEYyvbxubEF6Wn2ypscTKiKJFFn1ag==", "devOptional": true }, - "process-warning": { - "version": "1.0.0", - "resolved": "https://registry.npmjs.org/process-warning/-/process-warning-1.0.0.tgz", - "integrity": "sha512-du4wfLyj4yCZq1VupnVSZmRsPJsNuxoDQFdCFHLaYiEbFBD7QE0a+I4D7hOxrVnh78QE/YipFAj9lXHiXocV+Q==" - }, "pump": { "version": "3.0.0", "resolved": "https://registry.npmjs.org/pump/-/pump-3.0.0.tgz", "integrity": "sha512-LwZy+p3SFs1Pytd/jYct4wpv49HiYCqd9Rlc5ZVdk0V+8Yzv6jR5Blk3TRmPL1ft69TxP0IMZGJ+WPFU2BFhww==", + "devOptional": true, "requires": { "end-of-stream": "^1.1.0", "once": "^1.3.1" @@ -5717,11 +5259,6 @@ "integrity": "sha512-NuaNSa6flKT5JaSYQzJok04JzTL1CA6aGhv5rfLW3PgqA+M2ChpZQnAC8h8i4ZFkBS8X5RqkDBHA7r4hej3K9A==", "dev": true }, - "quick-format-unescaped": { - "version": "4.0.4", - "resolved": "https://registry.npmjs.org/quick-format-unescaped/-/quick-format-unescaped-4.0.4.tgz", - "integrity": "sha512-tYC1Q1hgyRuHgloV/YXs2w15unPVh8qfu/qCTfhTYamaw7fyhumKa2yGpdSo87vY32rIclj+4fWYQXUMs9EHvg==" - }, "rc": { "version": "1.2.8", "resolved": "https://registry.npmjs.org/rc/-/rc-1.2.8.tgz", @@ -5805,11 +5342,6 @@ "integrity": "sha512-U9nH88a3fc/ekCF1l0/UP1IosiuIjyTh7hBvXVMHYgVcfGvt897Xguj2UOLDeI5BG2m7/uwyaLVT6fbtCwTyzw==", "dev": true }, - "rfdc": { - "version": "1.3.0", - "resolved": "https://registry.npmjs.org/rfdc/-/rfdc-1.3.0.tgz", - "integrity": "sha512-V2hovdzFbOi77/WajaSMXk2OLm+xNIeQdMMuB7icj7bk6zi2F8GGAxigcnDFpJHbNyNcgyJDiP+8nOrY5cZGrA==" - }, "rimraf": { "version": "3.0.2", "resolved": "https://registry.npmjs.org/rimraf/-/rimraf-3.0.2.tgz", @@ -5906,14 +5438,6 @@ "integrity": "sha512-g9Q1haeby36OSStwb4ntCGGGaKsaVSjQ68fBxoQcutl5fS1vuY18H3wSt3jFyFtrkx+Kz0V1G85A4MyAdDMi2Q==", "dev": true }, - "sonic-boom": { - "version": "2.6.0", - "resolved": "https://registry.npmjs.org/sonic-boom/-/sonic-boom-2.6.0.tgz", - "integrity": "sha512-6xYZFRmDEtxGqfOKcDQ4cPLrNa0SPEDI+wlzDAHowXE6YV42NeXqg9mP2KkiM8JVu3lHfZ2iQKYlGOz+kTpphg==", - "requires": { - "atomic-sleep": "^1.0.0" - } - }, "source-map": { "version": "0.6.1", "resolved": "https://registry.npmjs.org/source-map/-/source-map-0.6.1.tgz", @@ -5930,31 +5454,6 @@ "source-map": "^0.6.0" } }, - "split2": { - "version": "3.2.2", - "resolved": "https://registry.npmjs.org/split2/-/split2-3.2.2.tgz", - "integrity": "sha512-9NThjpgZnifTkJpzTZ7Eue85S49QwpNhZTq6GRJwObb6jnLFNGB7Qm73V5HewTROPyxD0C29xqmaI68bQtV+hg==", - "requires": { - "readable-stream": "^3.0.0" - }, - "dependencies": { - "readable-stream": { - "version": "3.6.0", - "resolved": "https://registry.npmjs.org/readable-stream/-/readable-stream-3.6.0.tgz", - "integrity": "sha512-BViHy7LKeTz4oNnkcLJ+lVSL6vpiFeX6/d3oSH8zCW7UxP2onchk+vTGB143xuFjHS3deTgkKoXXymXqymiIdA==", - "requires": { - "inherits": "^2.0.3", - "string_decoder": "^1.1.1", - "util-deprecate": "^1.0.1" - } - } - } - }, - "sprintf-js": { - "version": "1.0.3", - "resolved": "https://registry.npmjs.org/sprintf-js/-/sprintf-js-1.0.3.tgz", - "integrity": "sha1-BOaSb2YolTVPPdAVIDYzuFcpfiw=" - }, "sqlite": { "version": "4.0.25", "resolved": "https://registry.npmjs.org/sqlite/-/sqlite-4.0.25.tgz", @@ -5964,6 +5463,7 @@ "version": "1.1.1", "resolved": "https://registry.npmjs.org/string_decoder/-/string_decoder-1.1.1.tgz", "integrity": "sha512-n/ShnvDi6FHbbVfviro+WojiFzv+s8MPMHBczVePfUpDJLwoLT0ht1l4YwBCbi8pJAveEEdnkHyPyTP/mzRfwg==", + "devOptional": true, "requires": { "safe-buffer": "~5.1.0" } @@ -5997,15 +5497,8 @@ "strip-json-comments": { "version": "3.1.1", "resolved": "https://registry.npmjs.org/strip-json-comments/-/strip-json-comments-3.1.1.tgz", - "integrity": "sha512-6fPc+R4ihwqP6N/aIv2f1gMH8lOVtWQHoqC4yK6oSDVVocumAsfCqjkXnqiYMhmMwS/mEHLp7Vehlt3ql6lEig==" - }, - "supports-color": { - "version": "5.5.0", - "resolved": "https://registry.npmjs.org/supports-color/-/supports-color-5.5.0.tgz", - "integrity": "sha512-QjVjwdXIt408MIiAqCX4oUKsgU2EqAGzs2Ppkm4aQYbjm+ZEWEcW4SfFNTr4uMNZma0ey4f5lgLrkB0aX0QMow==", - "requires": { - "has-flag": "^3.0.0" - } + "integrity": "sha512-6fPc+R4ihwqP6N/aIv2f1gMH8lOVtWQHoqC4yK6oSDVVocumAsfCqjkXnqiYMhmMwS/mEHLp7Vehlt3ql6lEig==", + "dev": true }, "supports-preserve-symlinks-flag": { "version": "1.0.0", @@ -6258,7 +5751,8 @@ "typescript": { "version": "4.5.5", "resolved": "https://registry.npmjs.org/typescript/-/typescript-4.5.5.tgz", - "integrity": "sha512-TCTIul70LyWe6IJWT8QSYeA54WQe8EjQFU4wY52Fasj5UKx88LNYKCgBEHcOMOrFF1rKGbD8v/xcNWVUq9SymA==" + "integrity": "sha512-TCTIul70LyWe6IJWT8QSYeA54WQe8EjQFU4wY52Fasj5UKx88LNYKCgBEHcOMOrFF1rKGbD8v/xcNWVUq9SymA==", + "dev": true }, "untildify": { "version": "4.0.0", @@ -6278,7 +5772,8 @@ "util-deprecate": { "version": "1.0.2", "resolved": "https://registry.npmjs.org/util-deprecate/-/util-deprecate-1.0.2.tgz", - "integrity": "sha1-RQ1Nyfpw3nMnYvvS1KKJgUGaDM8=" + "integrity": "sha1-RQ1Nyfpw3nMnYvvS1KKJgUGaDM8=", + "devOptional": true }, "uuid": { "version": "8.3.2", @@ -6480,6 +5975,11 @@ "@types/zen-observable": "0.8.3", "zen-observable": "0.8.15" } + }, + "zlib": { + "version": "1.0.5", + "resolved": "https://registry.npmjs.org/zlib/-/zlib-1.0.5.tgz", + "integrity": "sha512-40fpE2II+Cd3k8HWTWONfeKE2jL+P42iWJ1zzps5W51qcTsOUKM5Q5m2PFb0CLxlmFAaUuUdJGc3OfZy947v0w==" } } } diff --git a/package.json b/package.json index c34f4d9..d009188 100644 --- a/package.json +++ b/package.json @@ -53,16 +53,18 @@ "typescript": ">=4.5.0" }, "dependencies": { - "@runejs/common": "2.0.2-beta.2", + "@runejs/common": "file:../common", "adm-zip": "^0.5.9", "axios": "^0.27.2", + "compressjs": "^1.0.3", "graceful-fs": ">=4.2.0", "json5": "^2.2.0", "reflect-metadata": "^0.1.13", "sqlite": "^4.0.25", "tslib": ">=2.3.0", "typeorm": "^0.2.44", - "yargs": "^17.3.1" + "yargs": "^17.3.1", + "zlib": "^1.0.5" }, "devDependencies": { "@runejs/eslint-config": "^1.1.0", diff --git a/src/compress/bzip2.ts b/src/compress/bzip2.ts new file mode 100644 index 0000000..2cffd8c --- /dev/null +++ b/src/compress/bzip2.ts @@ -0,0 +1,21 @@ +import { Buffer } from 'buffer'; +import * as compressjs from 'compressjs'; +import { ByteBuffer } from '@runejs/common'; +const bzip2 = compressjs.Bzip2; + + +export const compressHeadlessBzip2 = (data: Buffer | ByteBuffer): Buffer => { + const compressedData = bzip2.compressFile(data, undefined, 1); + return Buffer.from(compressedData.slice(4, compressedData.length)); +}; + +export const decompressHeadlessBzip2 = (data: Buffer | ByteBuffer): Buffer => { + const buffer = Buffer.alloc(data.length + 4); + data.copy(buffer, 4); + buffer[0] = 'B'.charCodeAt(0); + buffer[1] = 'Z'.charCodeAt(0); + buffer[2] = 'h'.charCodeAt(0); + buffer[3] = '1'.charCodeAt(0); + + return Buffer.from(bzip2.decompressFile(buffer)); +}; diff --git a/src/compress/gzip.ts b/src/compress/gzip.ts new file mode 100644 index 0000000..95be7c3 --- /dev/null +++ b/src/compress/gzip.ts @@ -0,0 +1,12 @@ +import { Buffer } from 'buffer'; +import { gunzipSync, gzipSync } from 'zlib'; +import { ByteBuffer } from '@runejs/common'; + + +export const compressGzip = (data: Buffer | ByteBuffer): Buffer => { + return Buffer.from(gzipSync(data)); +}; + +export const decompressGzip = (data: Buffer | ByteBuffer): Buffer => { + return Buffer.from(gunzipSync(data)); +}; diff --git a/src/compress/index.ts b/src/compress/index.ts new file mode 100644 index 0000000..f626245 --- /dev/null +++ b/src/compress/index.ts @@ -0,0 +1,2 @@ +export * from './bzip2'; +export * from './gzip'; diff --git a/src/config/file-type.ts b/src/config/file-type.ts index 0e89e7a..dc6e3f2 100644 --- a/src/config/file-type.ts +++ b/src/config/file-type.ts @@ -2,4 +2,5 @@ export type FileType = 'FILE' | 'GROUP' | 'ARCHIVE' | - 'STORE'; + 'STORE' | + 'INDEX'; diff --git a/src/db/index-database.ts b/src/db/index-database.ts index b363e54..48b022f 100644 --- a/src/db/index-database.ts +++ b/src/db/index-database.ts @@ -49,7 +49,7 @@ export class IndexDatabase { for (let i = 0; i < indexEntities.length; i += chunkSize) { const chunk = indexEntities.slice(i, i + chunkSize); await this.repository.upsert(chunk, { - conflictPaths: [ 'fileType', 'gameBuild', 'key', 'archiveKey', 'groupKey' ], + conflictPaths: [ 'fileType', 'gameBuild', 'key', 'archiveKey', 'groupKey', 'indexKey' ], skipUpdateIfNoValuesChanged: true, }); } diff --git a/src/db/index-entity.ts b/src/db/index-entity.ts index b0b7497..17573d3 100644 --- a/src/db/index-entity.ts +++ b/src/db/index-entity.ts @@ -6,7 +6,7 @@ import { Buffer } from 'buffer'; @Entity('file_index') @Index('index_identifier', [ - 'fileType', 'gameBuild', 'key', 'archiveKey', 'groupKey' + 'fileType', 'gameBuild', 'key', 'archiveKey', 'groupKey', 'indexKey' ], { unique: true }) export class IndexEntity { @@ -22,6 +22,11 @@ export class IndexEntity { @PrimaryColumn('integer', { name: 'archive_key', nullable: false, unique: false, default: -1 }) archiveKey: number = -1; + // For JAG format files + @PrimaryColumn('integer', { name: 'index_key', nullable: false, unique: false, default: -1 }) + indexKey: number = -1; + + // For JS5 format files @PrimaryColumn('integer', { name: 'group_key', nullable: false, unique: false, default: -1 }) groupKey: number = -1; diff --git a/src/file-system/indexed-file-base.ts b/src/file-system/indexed-file-base.ts index b6bfd2a..0127d0c 100644 --- a/src/file-system/indexed-file-base.ts +++ b/src/file-system/indexed-file-base.ts @@ -18,6 +18,7 @@ export abstract class IndexedFileBase> { key: number, archiveKey: number = -1, groupKey: number = -1, + indexKey: number = -1, ) { this.fileStore = fileStore; this.index = new IndexEntity(); @@ -26,6 +27,7 @@ export abstract class IndexedFileBase> { this.index.key = key; this.index.archiveKey = archiveKey; this.index.groupKey = groupKey; + this.index.indexKey = indexKey; } validate(trackChanges: boolean = true): void { diff --git a/src/file-system/jag/index.ts b/src/file-system/jag/index.ts new file mode 100644 index 0000000..8e9f7db --- /dev/null +++ b/src/file-system/jag/index.ts @@ -0,0 +1,5 @@ +export * from './jag'; +export * from './jag-store'; +export * from './jag-archive'; +export * from './jag-file'; +export * from './jag-index'; diff --git a/src/file-system/jag/jag-archive.ts b/src/file-system/jag/jag-archive.ts index 3f60818..3e4bdbb 100644 --- a/src/file-system/jag/jag-archive.ts +++ b/src/file-system/jag/jag-archive.ts @@ -1,19 +1,28 @@ import { JagStore } from './jag-store'; import { IndexedFileBase } from '../indexed-file-base'; import { ArchiveConfig } from '../../config'; +import { indexes } from './jag'; +import { JagFile } from './jag-file'; export class JagArchive extends IndexedFileBase { readonly config: ArchiveConfig; + readonly files: Map; constructor( jagStore: JagStore, archiveKey: number, ) { - super(jagStore, 'ARCHIVE', archiveKey, -1, -1); + super(jagStore, 'ARCHIVE', archiveKey, -1, -1, indexes.archives); this.config = jagStore.getArchiveConfig(archiveKey); this.index.name = jagStore.getArchiveName(archiveKey); + this.files = new Map(); + } + + async upsertFileIndexes(): Promise { + const fileIndexes = Array.from(this.files.values()).map(file => file.index); + await this.fileStore.database.upsertIndexes(fileIndexes); } } diff --git a/src/file-system/jag/jag-file.ts b/src/file-system/jag/jag-file.ts index 890a742..cc0b07e 100644 --- a/src/file-system/jag/jag-file.ts +++ b/src/file-system/jag/jag-file.ts @@ -4,4 +4,8 @@ import { JagStore } from './jag-store'; export class JagFile extends IndexedFileBase { + constructor(jagStore: JagStore, fileKey: number, indexKey: number, archiveKey: number = -1) { + super(jagStore, 'FILE', fileKey, archiveKey, -1, indexKey); + } + } diff --git a/src/file-system/jag/jag-index.ts b/src/file-system/jag/jag-index.ts index fad72e5..8e18747 100644 --- a/src/file-system/jag/jag-index.ts +++ b/src/file-system/jag/jag-index.ts @@ -1,7 +1,65 @@ import { IndexedFileBase } from '../indexed-file-base'; import { JagStore } from './jag-store'; +import { JagFileIndex, indexes } from './jag'; +import { JagArchive } from './jag-archive'; +import { JagFile } from './jag-file'; +import { JS5File } from '../js5'; export class JagIndex extends IndexedFileBase { + readonly files: Map; + + fileIndexes: JagFileIndex[]; + + constructor(jagStore: JagStore, indexKey: number) { + super(jagStore, 'INDEX', indexKey); + const indexNames = Object.keys(indexes); + for (const name of indexNames) { + if (indexes[name] === indexKey) { + this.index.name = name; + } + } + this.files = new Map(); + } + + async upsertFileIndexes(): Promise { + const fileIndexes = Array.from(this.files.values()).map(file => file.index); + await this.fileStore.database.upsertIndexes(fileIndexes); + } + + getArchive(archiveKey: number): JagArchive | null; + getArchive(archiveName: string): JagArchive | null; + getArchive(archiveKeyOrName: number | string): JagArchive | null; + getArchive(archiveKeyOrName: number | string): JagArchive | null { + let archive: JagFile | JagArchive | null; + + if (typeof archiveKeyOrName === 'string') { + archive = Array.from(this.files.values()).find( + file => file?.index?.name === archiveKeyOrName + ) || null; + } else { + archive = this.files.get(archiveKeyOrName) || null; + } + + return (archive && archive instanceof JagArchive) ? archive : null; + } + + getFile(fileKey: number): JagFile | null; + getFile(fileName: string): JagFile | null; + getFile(fileKeyOrName: number | string): JagFile | null; + getFile(fileKeyOrName: number | string): JagFile | null { + let file: JagFile | JagArchive | null; + + if (typeof fileKeyOrName === 'string') { + file = Array.from(this.files.values()).find( + file => file?.index?.name === fileKeyOrName + ) || null; + } else { + file = this.files.get(fileKeyOrName) || null; + } + + return (file && file instanceof JagFile) ? file : null; + } + } diff --git a/src/file-system/jag/jag-store.ts b/src/file-system/jag/jag-store.ts index 2926b27..84a9f62 100644 --- a/src/file-system/jag/jag-store.ts +++ b/src/file-system/jag/jag-store.ts @@ -1,23 +1,56 @@ import { JagArchive } from './jag-archive'; import { FileStoreBase } from '../file-store-base'; -import { Jag } from './jag'; +import { Jag, indexes } from './jag'; +import { JagIndex } from './jag-index'; export class JagStore extends FileStoreBase { readonly jag: Jag; + readonly indexes: Map; constructor(gameBuild: string | number, storePath: string = './') { super(gameBuild, storePath, 'jag-cache-archives'); this.jag = new Jag(this); + this.indexes = new Map(); } override async load(): Promise { await this.openDatabase(); } - createArchive(archiveKey: number): void { - this.setArchive(archiveKey, new JagArchive(this, archiveKey)); + createArchive(archiveKey: number): JagArchive { + const archive = new JagArchive(this, archiveKey); + this.setArchive(archiveKey, archive); + return archive; + } + + createIndex(indexKey: number): void { + this.setIndex(indexKey, new JagIndex(this, indexKey)); + } + + getIndex(indexKey: number): JagIndex | null; + getIndex(indexName: string): JagIndex | null; + getIndex(indexKeyOrName: number | string): JagIndex | null; + getIndex(indexKeyOrName: number | string): JagIndex | null { + if (typeof indexKeyOrName === 'string') { + return Array.from(this.indexes.values()).find( + i => i?.index?.name === indexKeyOrName + ) || null; + } else { + return this.indexes.get(indexKeyOrName) || null; + } + } + + setIndex(indexKey: number, index: JagIndex): void; + setIndex(indexName: string, index: JagIndex): void; + setIndex(indexKeyOrName: number | string, index: JagIndex): void; + setIndex(indexKeyOrName: number | string, index: JagIndex): void { + if (typeof indexKeyOrName === 'string') { + this.indexes.set(indexes[indexKeyOrName], index); + } else { + this.indexes.set(indexKeyOrName, index); + } } } diff --git a/src/file-system/jag/jag.ts b/src/file-system/jag/jag.ts index 64dd019..ec5d76c 100644 --- a/src/file-system/jag/jag.ts +++ b/src/file-system/jag/jag.ts @@ -3,6 +3,38 @@ import { existsSync, readdirSync, readFileSync, statSync } from 'graceful-fs'; import { ByteBuffer, logger } from '@runejs/common'; import { JagStore } from './jag-store'; import { OpenRS2CacheFile } from '../../openrs2'; +import { IndexedFileBase } from '../indexed-file-base'; +import { JagFile } from './jag-file'; +import { Buffer } from 'buffer'; +import { JagArchive } from './jag-archive'; +import { decompressHeadlessBzip2 } from '../../compress'; + + +const dataFileName = 'main_file_cache.dat'; +const indexFileNamePrefix = 'main_file_cache.idx'; + + +export const indexes = { + archives: 0, + models: 1, + animations: 2, + midi: 3, + maps: 4, +}; + + +export interface JagFileIndex { + fileSize: number; + sectorNumber: number; +} + + +export interface JagSectorHeader { + fileKey: number; + filePartNumber: number; + sectorNumber: number; + indexKey: number; +} export class Jag { @@ -14,13 +46,49 @@ export class Jag { constructor(jagStore: JagStore) { this.jagStore = jagStore; + this.indexFiles = new Map(); } - // @todo stubbed - 21/07/22 - Kiko readOpenRS2CacheFiles(cacheFiles: OpenRS2CacheFile[]): void { + const dataFileBuffer = cacheFiles.find(file => file.name === dataFileName)?.data || null; + if (!dataFileBuffer?.length) { + throw new Error(`The main ${ dataFileName } data file could not be found.`); + } + + this.dataFile = new ByteBuffer(dataFileBuffer); + this.indexFiles.clear(); + + for (const cacheFile of cacheFiles) { + const fileName = cacheFile?.name; + + if (!fileName?.length || fileName === dataFileName) { + continue; + } + + if (!fileName.startsWith(indexFileNamePrefix)) { + continue; + } + + if (!cacheFile?.data?.length) { + logger.error(`Index file ${ fileName } is empty!`); + continue; + } + + const indexString = fileName.substring(fileName.indexOf('.idx') + 4); + const indexKey = Number(indexString); + + if (isNaN(indexKey)) { + logger.error(`Index file ${ fileName } does not have a valid extension.`); + } + + this.indexFiles.set(indexKey, new ByteBuffer(cacheFile.data)); + this.jagStore.createIndex(indexKey); + } + + logger.info(`JAG store files loaded for game build ${this.jagStore.gameBuild}.`); } - readLocalJagFiles(): void { + readLocalCacheFiles(): void { const jagStorePath = join(this.jagStore.fileStorePath, 'jag'); if (!existsSync(jagStorePath)) { @@ -62,11 +130,152 @@ export class Jag { } this.indexFiles.set(indexKey, new ByteBuffer(readFileSync(join(jagStorePath, fileName)))); + this.jagStore.createIndex(indexKey); } logger.info(`JAG store files loaded for game build ${this.jagStore.gameBuild}.`); } + decodeIndex(indexName: string): void { + logger.info(`Decoding JAG index ${indexName}...`); + + const indexKey = indexes[indexName]; + const indexFile = this.indexFiles.get(indexKey); + const fileCount = indexFile.length / 6; + + logger.info(`${fileCount} file indexes found.`); + + const index = this.jagStore.getIndex(indexKey); + index.fileIndexes = new Array(fileCount); + + for (let fileKey = 0; fileKey < fileCount; fileKey++) { + const fileSize = indexFile.get('int24', 'unsigned'); + const sectorPos = indexFile.get('int24', 'unsigned'); + index.fileIndexes[fileKey] = { + fileSize, sectorNumber: sectorPos + }; + + let file: IndexedFileBase; + + if (indexName === 'archives') { + file = this.jagStore.createArchive(fileKey); + } else { + file = new JagFile(this.jagStore, fileKey, indexKey); + } + + index.files.set(fileKey, file); + } + + logger.info(`Index ${indexName} has been loaded.`); + } + + unpack(file: JagArchive | JagFile): Buffer | null { + const fileIndexData = this.jagStore.getIndex(file.index.indexKey); + const { fileSize, sectorNumber } = fileIndexData.fileIndexes[file.index.key]; + const fileData = new ByteBuffer(fileSize); + const sectorDataLength = 512; + const sectorLength = 520; + let remainingData = fileSize; + let currentSectorNumber = sectorNumber; + let cycles = 0; + + while (remainingData > 0) { + let readableSectorData = sectorLength; + let remaining = this.dataFile.readable - currentSectorNumber * sectorLength; + + if (remaining < sectorLength) { + readableSectorData = remaining; + } + + const block = this.dataFile.getSlice(currentSectorNumber * sectorLength, readableSectorData); + + const sectorFileKey = block.get('short', 'unsigned');//readUnsignedShortBE(); + const sectorFilePartNumber = block.get('short', 'unsigned');//readUnsignedShortBE(); + const sectorNumber = block.get('int24', 'unsigned');//(block.readUnsignedByte() << 16) + // | (block.readUnsignedByte() << 8) | block.readUnsignedByte(); + const sectorIndexKey = block.get('byte', 'unsigned'); + + readableSectorData -= 8; + + let bytesThisCycle = remainingData; + + if (bytesThisCycle > sectorDataLength) { + bytesThisCycle = sectorDataLength; + } + + block.copy( + fileData, + fileData.writerIndex, + block.readerIndex, + block.readerIndex + readableSectorData + ); + + fileData.writerIndex = fileData.writerIndex + bytesThisCycle; + remainingData -= bytesThisCycle; + + if (cycles !== sectorFilePartNumber) { + logger.error(`Error extracting JAG file ${ file.index.key }, file data is corrupted.`); + return null; + } + + if (remainingData > 0) { + // saved index keys have 1 added to them for some reason + if (sectorIndexKey !== file.index.indexKey + 1) { + logger.error(`Index key mismatch, expected index ${ file.index.indexKey } but found ${ sectorIndexKey }`); + return null; + } + + if (sectorFileKey !== file.index.key) { + logger.error(`File index mismatch, expected ${ file.index.key } but found ${ sectorFileKey }.`); + return null; + } + } + + cycles++; + currentSectorNumber = sectorNumber; + } + + if (fileData.length) { + file.index.compressedData = fileData.toNodeBuffer(); + } else { + file.index.compressedData = null; + file.index.fileError = 'FILE_MISSING'; + } + + file.validate(false); + + return file.index.compressedData; + } + // @todo 18/07/22 - Kiko + decodeArchive(archive: JagArchive): Buffer | null { + let archiveData = new ByteBuffer(archive.index.compressedData); + + const uncompressed = archiveData.get('int24', 'unsigned'); + const compressed = archiveData.get('int24', 'unsigned'); + + if (uncompressed !== compressed) { + const compressedData = archiveData.getSlice(archiveData.readerIndex, archiveData.length - archiveData.readerIndex); + archiveData = new ByteBuffer(decompressHeadlessBzip2(compressedData)); + } + + const fileCount = archiveData.get('short', 'unsigned'); + let fileDataOffset = archiveData.readerIndex + fileCount * 10; + + for (let fileKey = 0; fileKey < fileCount; fileKey++) { + const fileNameHash = archiveData.get('int'); + const decompressedFileLength = archiveData.get('int24', 'unsigned'); + const compressedFileLength = archiveData.get('int24', 'unsigned'); + fileDataOffset += compressedFileLength; + } + + // @todo read file data and decompress files if needed - 18/07/22 - Kiko + + if (archiveData.length) { + archive.index.data = archiveData.toNodeBuffer(); + } + + return archive.index.data; + } } diff --git a/src/file-system/js5/index.ts b/src/file-system/js5/index.ts new file mode 100644 index 0000000..75f732a --- /dev/null +++ b/src/file-system/js5/index.ts @@ -0,0 +1,5 @@ +export * from './js5'; +export * from './js5-file-store'; +export * from './js5-archive'; +export * from './js5-group'; +export * from './js5-file'; diff --git a/src/file-system/js5/js5.ts b/src/file-system/js5/js5.ts index b2bd605..05685bd 100644 --- a/src/file-system/js5/js5.ts +++ b/src/file-system/js5/js5.ts @@ -1,16 +1,20 @@ import { join } from 'path'; +import { Buffer } from 'buffer'; import { existsSync, readdirSync, readFileSync, statSync } from 'graceful-fs'; + import { ByteBuffer, logger } from '@runejs/common'; -import { Bzip2, getCompressionMethod, Gzip } from '@runejs/common/compress'; -import { Xtea, XteaKeys } from '@runejs/common/encrypt'; -import { archiveFlags } from '../../config'; -import { JS5Group } from './js5-group'; -import { JS5Archive } from './js5-archive'; -import { JS5FileStore } from './js5-file-store'; -import { ArchiveFormat } from '../../config'; -import { JS5File } from './js5-file'; +import { getCompressionMethod } from '@runejs/common/compress'; +import { Xtea, XteaKeys, XteaConfig } from '@runejs/common/encrypt'; + +import { JS5FileStore, JS5Archive, JS5Group, JS5File } from '.'; +import { archiveFlags, ArchiveFormat } from '../../config'; import { getXteaKeysByBuild, OpenRS2CacheFile } from '../../openrs2'; -import { XteaConfig } from '@runejs/common/encrypt/xtea'; +import { + compressHeadlessBzip2, + decompressHeadlessBzip2, + compressGzip, + decompressGzip +} from '../../compress'; const dataFileName = 'main_file_cache.dat2'; @@ -132,8 +136,8 @@ export class JS5 { } unpack(file: JS5Group | JS5Archive): Buffer | null { - const fileDetails = file.index; - const fileKey = fileDetails.key; + const fileIndex = file.index; + const fileKey = fileIndex.key; const archiveKey: number = file instanceof JS5Archive ? 255 : file.archive.index.key; const archiveName: string = file instanceof JS5Archive ? 'main' : file.archive.index.name; @@ -165,74 +169,75 @@ export class JS5 { return null; } - fileDetails.fileSize = fileIndexData.get('int24', 'unsigned'); - const stripeCount = fileIndexData.get('int24', 'unsigned'); + fileIndex.fileSize = fileIndexData.get('int24', 'unsigned'); + const sectorNumber = fileIndexData.get('int24', 'unsigned'); - if (fileDetails.fileSize <= 0) { + if (fileIndex.fileSize <= 0) { logger.warn(`JS5 file ${ fileKey } is empty or has been removed.`); return null; } - const data = new ByteBuffer(fileDetails.fileSize); - const stripeDataLength = 512; - const stripeLength = 520; + const data = new ByteBuffer(fileIndex.fileSize); + const sectorDataLength = 512; + const sectorLength = 520; - let stripe = 0; - let remaining = fileDetails.fileSize; - pointer = stripeCount * stripeLength; + let sector = 0; + let remainingData = fileIndex.fileSize; + pointer = sectorNumber * sectorLength; do { - const temp = new ByteBuffer(stripeLength); - dataChannel.copy(temp, 0, pointer, pointer + stripeLength); + const temp = new ByteBuffer(sectorLength); + dataChannel.copy(temp, 0, pointer, pointer + sectorLength); - if (temp.readable !== stripeLength) { + if (temp.readable !== sectorLength) { logger.error(`Error reading stripe for packed file ${ fileKey }, the end of the data stream was reached.`); return null; } - const stripeFileIndex = temp.get('short', 'unsigned'); - const currentStripe = temp.get('short', 'unsigned'); - const nextStripe = temp.get('int24', 'unsigned'); - const stripeArchiveIndex = temp.get('byte', 'unsigned'); - const stripeData = new ByteBuffer(stripeDataLength); - temp.copy(stripeData, 0, temp.readerIndex, temp.readerIndex + stripeDataLength); + const sectorFileKey = temp.get('short', 'unsigned'); + const sectorFilePartNumber = temp.get('short', 'unsigned'); + const sectorNumber = temp.get('int24', 'unsigned'); + const sectorIndexKey = temp.get('byte', 'unsigned'); + + const sectorData = new ByteBuffer(sectorDataLength); + temp.copy(sectorData, 0, temp.readerIndex, temp.readerIndex + sectorDataLength); - if (remaining > stripeDataLength) { - stripeData.copy(data, data.writerIndex, 0, stripeDataLength); - data.writerIndex = (data.writerIndex + stripeDataLength); - remaining -= stripeDataLength; + if (remainingData > sectorDataLength) { + sectorData.copy(data, data.writerIndex, 0, sectorDataLength); + data.writerIndex = (data.writerIndex + sectorDataLength); + remainingData -= sectorDataLength; - if (stripeArchiveIndex !== archiveKey) { - logger.error(`Archive index mismatch, expected archive ${ archiveKey } but found archive ${ stripeFileIndex }`); + if (sectorIndexKey !== archiveKey) { + logger.error(`Archive index mismatch, expected archive ${ archiveKey } but found ${ sectorFileKey }`); return null; } - if (stripeFileIndex !== fileKey) { - logger.error(`File index mismatch, expected ${ fileKey } but found ${ stripeFileIndex }.`); + if (sectorFileKey !== fileKey) { + logger.error(`File index mismatch, expected ${ fileKey } but found ${ sectorFileKey }.`); return null; } - if (currentStripe !== stripe++) { + if (sectorFilePartNumber !== sector++) { logger.error(`Error extracting JS5 file ${ fileKey }, file data is corrupted.`); return null; } - pointer = nextStripe * stripeLength; + pointer = sectorNumber * sectorLength; } else { - stripeData.copy(data, data.writerIndex, 0, remaining); - data.writerIndex = (data.writerIndex + remaining); - remaining = 0; + sectorData.copy(data, data.writerIndex, 0, remainingData); + data.writerIndex = (data.writerIndex + remainingData); + remainingData = 0; } - } while (remaining > 0); + } while (remainingData > 0); - if (data?.length) { - fileDetails.compressedData = data.toNodeBuffer(); + if (data.length) { + fileIndex.compressedData = data.toNodeBuffer(); } else { - fileDetails.compressedData = null; - fileDetails.fileError = 'FILE_MISSING'; + fileIndex.compressedData = null; + fileIndex.fileError = 'FILE_MISSING'; } - return fileDetails.compressedData; + return fileIndex.compressedData; } readCompressedFileHeader(file: JS5Group | JS5Archive): { compressedLength: number, readerIndex: number } { @@ -344,11 +349,11 @@ export class JS5 { // JS5.decrypt will set compressedData to the new decrypted data after completion const compressedData = new ByteBuffer(fileDetails.compressedData); compressedData.readerIndex = readerIndex; - let data: ByteBuffer; + let data: Buffer; if (fileDetails.compressionMethod === 'none') { // Uncompressed file - data = new ByteBuffer(compressedLength); + data = Buffer.alloc(compressedLength); compressedData.copy(data, 0, compressedData.readerIndex, compressedLength); compressedData.readerIndex = (compressedData.readerIndex + compressedLength); } else { @@ -370,7 +375,8 @@ export class JS5 { try { data = fileDetails.compressionMethod === 'bzip' ? - Bzip2.decompress(decompressedData) : Gzip.decompress(decompressedData); + decompressHeadlessBzip2(decompressedData) : + decompressGzip(decompressedData); compressedData.readerIndex = compressedData.readerIndex + compressedLength; @@ -392,7 +398,7 @@ export class JS5 { fileDetails.version = compressedData.get('short', 'unsigned'); } - fileDetails.data = data?.toNodeBuffer(); + fileDetails.data = data; } return fileDetails.data; @@ -650,8 +656,9 @@ export class JS5 { } else { // compressed Bzip2 or Gzip file - const compressedData: ByteBuffer = fileDetails.compressionMethod === 'bzip' ? - Bzip2.compress(decompressedData) : Gzip.compress(decompressedData); + const compressedData: Buffer = fileDetails.compressionMethod === 'bzip' ? + compressHeadlessBzip2(decompressedData) : + compressGzip(decompressedData); const compressedLength: number = compressedData.length; diff --git a/src/openrs2/openrs2.ts b/src/openrs2/openrs2.ts index 2bbb933..99e3ffe 100644 --- a/src/openrs2/openrs2.ts +++ b/src/openrs2/openrs2.ts @@ -1,9 +1,8 @@ import axios from 'axios'; -import AdmZip, { IZipEntry } from 'adm-zip'; +import AdmZip from 'adm-zip'; import { Buffer } from 'buffer'; import { logger } from '@runejs/common'; -import { XteaKeys } from '@runejs/common/encrypt'; -import { XteaConfig } from '@runejs/common/encrypt/xtea'; +import { XteaConfig } from '@runejs/common/encrypt'; const openRS2Endpoint = 'https://archive.openrs2.org'; diff --git a/src/scripts/indexer.ts b/src/scripts/indexer.ts index af293a5..41040fe 100644 --- a/src/scripts/indexer.ts +++ b/src/scripts/indexer.ts @@ -2,12 +2,13 @@ import { join } from 'path'; import { existsSync, mkdirSync } from 'graceful-fs'; import { logger } from '@runejs/common'; import { ScriptExecutor, ArgumentOptions } from './script-executor'; -import { JS5FileStore } from '../file-system/js5/js5-file-store'; -import { JagStore } from '../file-system/jag/jag-store'; +import { JS5FileStore } from '../file-system/js5'; +import { indexes, JagArchive, JagStore } from '../file-system/jag'; import { getOpenRS2CacheFilesByBuild, OpenRS2CacheFile } from '../openrs2'; +import { fileTarget, prettyPrintTarget } from '../../../common/src'; interface IndexerOptions { @@ -154,7 +155,52 @@ const indexJS5Archive = async (store: JS5FileStore, archiveName: string) => { const indexJagStore = async (store: JagStore) => { - // @todo 18/07/22 - Kiko + logger.info(`Decoding JAG store indexes...`); + + const indexNames = Object.keys(indexes); + for (const indexName of indexNames) { + store.jag.decodeIndex(indexName); + } + + logger.info(`Saving indexes...`); + + for (const [ , indexFile ] of store.indexes) { + await indexFile.saveIndex(); + } + + for (const [, indexFile ] of store.indexes) { + logger.info(`Unpacking JAG files for index ${indexFile.index.name}...`); + + for (const [ , file ] of indexFile.files) { + store.jag.unpack(file); + } + } + + logger.info(`Decoding JAG archives...`); + + const archiveIndex = store.getIndex(indexes.archives); + + // @todo unfinished - 02/08/22 - Kiko + for (const [ , archive ] of archiveIndex.files) { + if (archive instanceof JagArchive) { + store.jag.decodeArchive(archive); + } + } + + logger.info(`Saving JAG file indexes...`); + + for (const [, index ] of store.indexes) { + await index.upsertFileIndexes(); + } + + logger.info(`Saving JAG archive file indexes...`); + + // @todo unfinished - 02/08/22 - Kiko + for (const [ , archive ] of archiveIndex.files) { + if (archive instanceof JagArchive) { + await archive.upsertFileIndexes(); + } + } }; @@ -175,7 +221,11 @@ const indexerScript = async ( mkdirSync(logDir, { recursive: true }); } - logger.destination(join(logDir, `index-${ format }-${ build }.log`)); + // logger.destination(join(logDir, `index-${ format }-${ build }.log`)); + logger.setTargets([ + prettyPrintTarget(), + fileTarget(join(logDir, `index-${ format }-${ build }.log`)) + ]); if (source === 'openrs2') { if (numericBuildNumber) { @@ -213,9 +263,18 @@ const indexerScript = async ( } else if (format === 'jag') { const store = new JagStore(numericBuildNumber !== -1 ? numericBuildNumber : build, dir); await store.load(); - store.jag.readLocalJagFiles(); - // @todo 18/07/22 - Kiko + if (cacheFiles === 'local') { + store.jag.readLocalCacheFiles(); + } else { + store.jag.readOpenRS2CacheFiles(cacheFiles); + } + + if (archiveName === 'main') { + await indexJagStore(store); + } else { + await indexJagArchive(store, archiveName); + } await store.closeDatabase(); } else if (format === 'flat') { @@ -223,8 +282,8 @@ const indexerScript = async ( } logger.info(`Indexing completed in ${ (Date.now() - start) / 1000 } seconds.`); - logger.boom.flushSync(); - logger.boom.end(); + // logger.boom.flushSync(); + // logger.boom.end(); }; diff --git a/src/scripts/script-executor.ts b/src/scripts/script-executor.ts index 9b2e8e4..c33aa20 100644 --- a/src/scripts/script-executor.ts +++ b/src/scripts/script-executor.ts @@ -18,8 +18,8 @@ export class ScriptExecutor { ): void { (async function(args: T) { await script(args); - }(this.getArguments(argumentOptions))) - .finally(() => process.exit(0)); + }(this.getArguments(argumentOptions))); + //.finally(() => process.exit(0)); } } From 74969f2f0e7f56cd287b6011fcdd60b7f3eaca4e Mon Sep 17 00:00:00 2001 From: Kikorono Date: Mon, 8 Aug 2022 16:02:39 -0500 Subject: [PATCH 20/32] Moving Jag format index entities to their own table structure and refactoring all of the indexing code to support the two differing formats Also adding in all of the cache file names+hashes for Jag format caches --- config/jag-cache-archives.json5 | 26 - config/name-hashes.json | 180 +- package-lock.json | 3333 +++++++++-------- package.json | 5 +- src/config/archive-config.ts | 2 +- src/config/file-type.ts | 7 +- src/config/index.ts | 2 +- src/config/{djb2.ts => name-hasher.ts} | 28 +- src/db/index-database.ts | 92 +- src/db/jag-database.ts | 36 + src/db/jag-index-entity.ts | 76 + src/db/js5-database.ts | 36 + .../{index-entity.ts => js5-index-entity.ts} | 19 +- src/file-system/cache/cache-file.ts | 7 + src/file-system/cache/cache-format.ts | 12 + src/file-system/cache/index.ts | 2 + src/file-system/file-store-base.ts | 115 +- src/file-system/jag/index.ts | 2 +- src/file-system/jag/jag-archive.ts | 22 +- src/file-system/jag/jag-file-base.ts | 141 + .../jag/{jag-store.ts => jag-file-store.ts} | 25 +- src/file-system/jag/jag-file.ts | 10 +- src/file-system/jag/jag-index.ts | 23 +- src/file-system/jag/jag.ts | 127 +- src/file-system/js5/js5-archive.ts | 17 +- .../js5-file-base.ts} | 38 +- src/file-system/js5/js5-file-store.ts | 109 +- src/file-system/js5/js5-file.ts | 12 +- src/file-system/js5/js5-group.ts | 22 +- src/file-system/js5/js5.ts | 42 +- src/openrs2/openrs2.ts | 18 +- src/scripts/dev.ts | 4 +- src/scripts/indexer.ts | 72 +- src/scripts/name-hasher.ts | 167 + 34 files changed, 2890 insertions(+), 1939 deletions(-) delete mode 100644 config/jag-cache-archives.json5 rename src/config/{djb2.ts => name-hasher.ts} (73%) create mode 100644 src/db/jag-database.ts create mode 100644 src/db/jag-index-entity.ts create mode 100644 src/db/js5-database.ts rename src/db/{index-entity.ts => js5-index-entity.ts} (85%) create mode 100644 src/file-system/cache/cache-file.ts create mode 100644 src/file-system/cache/cache-format.ts create mode 100644 src/file-system/cache/index.ts create mode 100644 src/file-system/jag/jag-file-base.ts rename src/file-system/jag/{jag-store.ts => jag-file-store.ts} (71%) rename src/file-system/{indexed-file-base.ts => js5/js5-file-base.ts} (82%) create mode 100644 src/scripts/name-hasher.ts diff --git a/config/jag-cache-archives.json5 b/config/jag-cache-archives.json5 deleted file mode 100644 index 88068cb..0000000 --- a/config/jag-cache-archives.json5 +++ /dev/null @@ -1,26 +0,0 @@ -{ - title: { - key: 1 - }, - config: { - key: 2 - }, - interface: { - key: 3 - }, - media: { - key: 4 - }, - versionlist: { - key: 5 - }, - textures: { - key: 6 - }, - wordenc: { - key: 7 - }, - sounds: { - key: 8 - } -} diff --git a/config/name-hashes.json b/config/name-hashes.json index a23a8e4..64a9d63 100644 --- a/config/name-hashes.json +++ b/config/name-hashes.json @@ -169062,5 +169062,183 @@ "-1081813035": "m199_198", "-2824623370": "l199_198", "-1081813034": "m199_199", - "-2824623369": "l199_199" + "-2824623369": "l199_199", + "6133252": "45.dat", + "8297314": "data", + "19979093": "46.dat", + "33824934": "47.dat", + "47670775": "48.dat", + "53973365": "combaticons.dat", + "61516616": "49.dat", + "125902192": "backbase1.dat", + "139748033": "backbase2.dat", + "150819851": "idk.dat", + "204062206": "q8_full.dat", + "224847211": "0.dat", + "232787039": "sounds.dat", + "238693052": "1.dat", + "252137566": "model_version", + "252538893": "2.dat", + "266384734": "3.dat", + "280230575": "4.dat", + "294076416": "5.dat", + "305236077": "prayeroff.dat", + "307922257": "6.dat", + "321768098": "7.dat", + "335613939": "8.dat", + "349459780": "9.dat", + "383739196": "varp.dat", + "392041951": "prayeron.dat", + "449541346": "mod_icons.dat", + "529843337": "cross.dat", + "612871759": "mapdots.dat", + "661178691": "magicoff.dat", + "661681639": "staticons.dat", + "682978269": "loc.dat", + "682997061": "loc.idx", + "715169772": "anim_index", + "839488367": "mapscene.dat", + "886159288": "seq.dat", + "1018124075": "headicons_hint.dat", + "1043559214": "steelborder.dat", + "1152574301": "wornicons.dat", + "1165431679": "number_button.dat", + "1354546316": "backleft1.dat", + "1362520410": "mapedge.dat", + "1368392157": "backleft2.dat", + "1442199444": "rightarrow.dat", + "1464846521": "backvmid1.dat", + "1478692362": "backvmid2.dat", + "1489108188": "npc.dat", + "1489126980": "npc.idx", + "1492538203": "backvmid3.dat", + "1644583778": "mapback.dat", + "1648736955": "badenc.txt", + "1654911043": "p11_full.dat", + "1694123055": "prayerglow.dat", + "1694783164": "domainenc.txt", + "1698082440": "10.dat", + "1711928281": "11.dat", + "1725774122": "12.dat", + "1727594325": "magicoff2.dat", + "1739619963": "13.dat", + "1753465804": "14.dat", + "1758274153": "staticons2.dat", + "1766681864": "chatback.dat", + "1767311645": "15.dat", + "1781157486": "16.dat", + "1795003327": "17.dat", + "1808849168": "18.dat", + "1822695009": "19.dat", + "1889496696": "sideicons.dat", + "1915414053": "map_crc", + "1922934081": "leftarrow.dat", + "1955686745": "titlebutton.dat", + "1955804455": "mapmarker.dat", + "1986120039": "keys.dat", + "1987120305": "map_index", + "2025126712": "overlay_multiway.dat", + "2038060091": "headicons_pk.dat", + "2081559868": "miscgraphics.dat", + "-1929337337": "index.dat", + "-227242592": "p12_full.dat", + "-1124181286": "b12_full.dat", + "-566502255": "title.dat", + "-1752651416": "logo.dat", + "-1891508522": "titlebox.dat", + "-1668775416": "runes.dat", + "-1569261396": "flo.dat", + "-1667617738": "obj.dat", + "-1667598946": "obj.idx", + "-955170442": "spotanim.dat", + "-514869585": "varbit.dat", + "-1568083395": "invback.dat", + "-1623648789": "backhmid1.dat", + "-427405255": "compass.dat", + "-1204854137": "mapfunction.dat", + "-1502153170": "hitmarks.dat", + "-288954319": "headicons.dat", + "-1337835461": "headicons_prayer.dat", + "-1571073093": "scrollbar.dat", + "-1392068576": "redstone1.dat", + "-1378222735": "redstone2.dat", + "-1364376894": "redstone3.dat", + "-1593819477": "backright1.dat", + "-1579973636": "backright2.dat", + "-1102299012": "backtop1.dat", + "-1609802948": "backhmid2.dat", + "-1000916878": "tradebacking.dat", + "-716997548": "steelborder2.dat", + "-1868599050": "combatboxes.dat", + "-884827257": "sworddecor.dat", + "-952192193": "combaticons2.dat", + "-938346352": "combaticons3.dat", + "-1809621253": "miscgraphics3.dat", + "-869490323": "magicon.dat", + "-1448902313": "magicon2.dat", + "-1823467094": "miscgraphics2.dat", + "-416634290": "chest.dat", + "-58065069": "coins.dat", + "-351562801": "tex_brown.dat", + "-1811229622": "tex_red.dat", + "-797498902": "anim_version", + "-945480188": "midi_version", + "-923525801": "map_version", + "-1761598724": "model_crc", + "-40228664": "anim_crc", + "-1121254206": "midi_crc", + "-706585152": "model_index", + "-1691482954": "midi_index", + "-1752288555": "20.dat", + "-1738442714": "21.dat", + "-1724596873": "22.dat", + "-1710751032": "23.dat", + "-1696905191": "24.dat", + "-1683059350": "25.dat", + "-1669213509": "26.dat", + "-1655367668": "27.dat", + "-1641521827": "28.dat", + "-1627675986": "29.dat", + "-907692254": "30.dat", + "-893846413": "31.dat", + "-880000572": "32.dat", + "-866154731": "33.dat", + "-852308890": "34.dat", + "-838463049": "35.dat", + "-824617208": "36.dat", + "-810771367": "37.dat", + "-796925526": "38.dat", + "-783079685": "39.dat", + "-63095953": "40.dat", + "-49250112": "41.dat", + "-35404271": "42.dat", + "-21558430": "43.dat", + "-7712589": "44.dat", + "-573349193": "fragmentsenc.txt", + "-840867198": "tldlist.txt", + "150838643": "idk.idx", + "383757988": "varp.idx", + "886178080": "seq.idx", + "-955151650": "spotanim.idx", + "-514850793": "varbit.idx", + "-1569242604": "flo.idx", + "182685561": "mesanim.dat", + "182704353": "mesanim.idx", + "1029250116": "mes.dat", + "1029268908": "mes.idx", + "-1818025236": "param.dat", + "-1818006444": "param.idx", + "22834782": "gnomeball_buttons.dat", + "450862262": "overlay_duel.dat", + "523617556": "rightarrow_small.dat", + "819035239": "letter.dat", + "902321338": "pen.dat", + "1150791544": "key.dat", + "1451391714": "button_brown.dat", + "2004158547": "startgame.dat", + "-1004178375": "leftarrow_small.dat", + "-1857300557": "blackmark.dat", + "-888498683": "button_red.dat", + "-384541308": "titlescroll.dat", + "-90207845": "button_brown_big.dat" } diff --git a/package-lock.json b/package-lock.json index 01bf1f5..a61dc45 100644 --- a/package-lock.json +++ b/package-lock.json @@ -9,7 +9,7 @@ "version": "1.0.0-next.0", "license": "GPL-3.0", "dependencies": { - "@runejs/common": "file:../common", + "@runejs/common": "2.0.2-beta.2", "adm-zip": "^0.5.9", "axios": "^0.27.2", "compressjs": "^1.0.3", @@ -35,7 +35,7 @@ "eslint": "^8.11.0", "rimraf": "^3.0.2", "source-map-support": "^0.5.21", - "ts-node-dev": "^1.1.8", + "ts-node-dev": "^2.0.0", "typescript": "^4.5.5" }, "peerDependencies": { @@ -45,106 +45,47 @@ "typescript": ">=4.5.0" } }, - "../common": { - "version": "3.0.0-beta.3", - "license": "GPL-3.0", - "dependencies": { - "buffer": "^6.0.3", - "compressjs": "^1.0.3", - "dotenv": "^16.0.0", - "path": "^0.12.7", - "pino": "^7.10.0", - "pino-pretty": "^7.6.1", - "zlib": "^1.0.5" - }, - "devDependencies": { - "@runejs/eslint-config": "^2.0.0", - "@types/node": "^16.11.26", - "@typescript-eslint/eslint-plugin": "^5.14.0", - "@typescript-eslint/parser": "^5.14.0", - "copyfiles": "^2.4.1", - "eslint": "^8.11.0", - "rimraf": "^3.0.2", - "ts-node-dev": "^1.1.8", - "tslib": "^2.3.1", - "typescript": "^4.6.3" - }, - "peerDependencies": { - "tslib": ">=2.3.0" - } - }, - "../common/lib": { - "name": "@runejs/common", - "version": "2.0.2-beta.2", - "extraneous": true, - "license": "GPL-3.0", + "node_modules/@cspotcode/source-map-support": { + "version": "0.8.1", + "resolved": "https://registry.npmjs.org/@cspotcode/source-map-support/-/source-map-support-0.8.1.tgz", + "integrity": "sha512-IchNf6dN4tHoMFIn/7OE8LWZ19Y6q/67Bmf6vnGREv8RSbBVb9LPJxEcnwrcwX6ixSvaiGoomAUvu4YSxXrVgw==", + "dev": true, "dependencies": { - "compressjs": "^1.0.3", - "js-yaml": "^3.14.1", - "pino": "^6.14.0", - "pino-pretty": "^4.8.0", - "sonic-boom": "^2.6.0", - "tslib": "^2.3.1" - }, - "devDependencies": { - "@runejs/eslint-config": "^1.0.0", - "@types/node": "^16.11.26", - "@types/pino": "^6.3.12", - "@typescript-eslint/eslint-plugin": "^4.33.0", - "@typescript-eslint/parser": "^4.33.0", - "copyfiles": "^2.4.1", - "eslint": "^7.32.0", - "rimraf": "^3.0.2", - "ts-node-dev": "^1.1.8", - "typescript": "^4.5.5" + "@jridgewell/trace-mapping": "0.3.9" }, - "peerDependencies": { - "tslib": ">=2.3.0", - "typescript": ">=4.5.0" + "engines": { + "node": ">=12" } }, "node_modules/@eslint/eslintrc": { - "version": "1.2.1", - "resolved": "https://registry.npmjs.org/@eslint/eslintrc/-/eslintrc-1.2.1.tgz", - "integrity": "sha512-bxvbYnBPN1Gibwyp6NrpnFzA3YtRL3BBAyEAFVIpNTm2Rn4Vy87GA5M4aSn3InRrlsbX5N0GW7XIx+U4SAEKdQ==", + "version": "1.3.0", + "resolved": "https://registry.npmjs.org/@eslint/eslintrc/-/eslintrc-1.3.0.tgz", + "integrity": "sha512-UWW0TMTmk2d7hLcWD1/e2g5HDM/HQ3csaLSqXCfqwh4uNDuNqlaKWXmEsL4Cs41Z0KnILNvwbHAah3C2yt06kw==", "dev": true, "dependencies": { "ajv": "^6.12.4", "debug": "^4.3.2", - "espree": "^9.3.1", - "globals": "^13.9.0", + "espree": "^9.3.2", + "globals": "^13.15.0", "ignore": "^5.2.0", "import-fresh": "^3.2.1", "js-yaml": "^4.1.0", - "minimatch": "^3.0.4", + "minimatch": "^3.1.2", "strip-json-comments": "^3.1.1" }, "engines": { "node": "^12.22.0 || ^14.17.0 || >=16.0.0" } }, - "node_modules/@eslint/eslintrc/node_modules/argparse": { - "version": "2.0.1", - "resolved": "https://registry.npmjs.org/argparse/-/argparse-2.0.1.tgz", - "integrity": "sha512-8+9WqebbFzpX9OR+Wa6O29asIogeRMzcGtAINdpMHHyAg10f05aSFVBbcEqGf/PXw1EjAZ+q2/bEBg3DvurK3Q==", - "dev": true - }, - "node_modules/@eslint/eslintrc/node_modules/js-yaml": { - "version": "4.1.0", - "resolved": "https://registry.npmjs.org/js-yaml/-/js-yaml-4.1.0.tgz", - "integrity": "sha512-wpxZs9NoxZaJESJGIZTyDEaYpl0FKSA+FB9aJiyemKhMwkxQg63h4T1KJgUGHpTqPDNRcmmYLugrRjJlBtWvRA==", - "dev": true, - "dependencies": { - "argparse": "^2.0.1" - }, - "bin": { - "js-yaml": "bin/js-yaml.js" - } + "node_modules/@hapi/bourne": { + "version": "2.1.0", + "resolved": "https://registry.npmjs.org/@hapi/bourne/-/bourne-2.1.0.tgz", + "integrity": "sha512-i1BpaNDVLJdRBEKeJWkVO6tYX6DMFBuwMhSuWqLsY4ufeTKGVuV5rBsUhxPayXqnnWHgXUAmWK16H/ykO5Wj4Q==" }, "node_modules/@humanwhocodes/config-array": { - "version": "0.9.5", - "resolved": "https://registry.npmjs.org/@humanwhocodes/config-array/-/config-array-0.9.5.tgz", - "integrity": "sha512-ObyMyWxZiCu/yTisA7uzx81s40xR2fD5Cg/2Kq7G02ajkNubJf6BopgDTmDyc3U7sXpNKM8cYOw7s7Tyr+DnCw==", + "version": "0.10.4", + "resolved": "https://registry.npmjs.org/@humanwhocodes/config-array/-/config-array-0.10.4.tgz", + "integrity": "sha512-mXAIHxZT3Vcpg83opl1wGlVZ9xydbfZO3r5YfRSH6Gpp2J/PfdBP0wbDa2sO6/qRbcalpoevVyW6A/fI6LfeMw==", "dev": true, "dependencies": { "@humanwhocodes/object-schema": "^1.2.1", @@ -155,12 +96,47 @@ "node": ">=10.10.0" } }, + "node_modules/@humanwhocodes/gitignore-to-minimatch": { + "version": "1.0.2", + "resolved": "https://registry.npmjs.org/@humanwhocodes/gitignore-to-minimatch/-/gitignore-to-minimatch-1.0.2.tgz", + "integrity": "sha512-rSqmMJDdLFUsyxR6FMtD00nfQKKLFb1kv+qBbOVKqErvloEIJLo5bDTJTQNTYgeyp78JsA7u/NPi5jT1GR/MuA==", + "dev": true, + "funding": { + "type": "github", + "url": "https://github.com/sponsors/nzakas" + } + }, "node_modules/@humanwhocodes/object-schema": { "version": "1.2.1", "resolved": "https://registry.npmjs.org/@humanwhocodes/object-schema/-/object-schema-1.2.1.tgz", "integrity": "sha512-ZnQMnLV4e7hDlUvw8H+U8ASL02SS2Gn6+9Ac3wGGLIe7+je2AeAOxPY+izIPJDfFDb7eDjev0Us8MO1iFRN8hA==", "dev": true }, + "node_modules/@jridgewell/resolve-uri": { + "version": "3.1.0", + "resolved": "https://registry.npmjs.org/@jridgewell/resolve-uri/-/resolve-uri-3.1.0.tgz", + "integrity": "sha512-F2msla3tad+Mfht5cJq7LSXcdudKTWCVYUgw6pLFOOHSTtZlj6SWNYAp+AhuqLmWdBO2X5hPrLcu8cVP8fy28w==", + "dev": true, + "engines": { + "node": ">=6.0.0" + } + }, + "node_modules/@jridgewell/sourcemap-codec": { + "version": "1.4.14", + "resolved": "https://registry.npmjs.org/@jridgewell/sourcemap-codec/-/sourcemap-codec-1.4.14.tgz", + "integrity": "sha512-XPSJHWmi394fuUuzDnGz1wiKqWfo1yXecHQMRf2l6hztTO+nPru658AyDngaBe7isIxEkRsPR3FZh+s7iVa4Uw==", + "dev": true + }, + "node_modules/@jridgewell/trace-mapping": { + "version": "0.3.9", + "resolved": "https://registry.npmjs.org/@jridgewell/trace-mapping/-/trace-mapping-0.3.9.tgz", + "integrity": "sha512-3Belt6tdc8bPgAtbcmdtNJlirVoTmEb5e2gC94PnkwEW9jI6CAHUeoG85tjWP5WquqfavoMtMwiG4P926ZKKuQ==", + "dev": true, + "dependencies": { + "@jridgewell/resolve-uri": "^3.0.3", + "@jridgewell/sourcemap-codec": "^1.4.10" + } + }, "node_modules/@nodelib/fs.scandir": { "version": "2.1.5", "resolved": "https://registry.npmjs.org/@nodelib/fs.scandir/-/fs.scandir-2.1.5.tgz", @@ -197,8 +173,41 @@ } }, "node_modules/@runejs/common": { - "resolved": "../common", - "link": true + "version": "2.0.2-beta.2", + "resolved": "https://registry.npmjs.org/@runejs/common/-/common-2.0.2-beta.2.tgz", + "integrity": "sha512-Xd9/Ws3ByAdI1R08Ye9fWqlS8SImjQw05Bsw0w46zcGyDx8N6jmeClf9Y/s4Akh9VZQuoAMFg2ANKsRK0MucNw==", + "dependencies": { + "compressjs": "^1.0.3", + "js-yaml": "^3.14.1", + "pino": "^6.14.0", + "pino-pretty": "^4.8.0", + "sonic-boom": "^2.6.0", + "tslib": "^2.3.1" + }, + "peerDependencies": { + "tslib": ">=2.3.0", + "typescript": ">=4.5.0" + } + }, + "node_modules/@runejs/common/node_modules/argparse": { + "version": "1.0.10", + "resolved": "https://registry.npmjs.org/argparse/-/argparse-1.0.10.tgz", + "integrity": "sha512-o5Roy6tNG4SL/FOkCAN6RzjiakZS25RLYFrcMttJqbdd8BWrnA+fGz57iN5Pb06pvBGvl5gQ0B48dJlslXvoTg==", + "dependencies": { + "sprintf-js": "~1.0.2" + } + }, + "node_modules/@runejs/common/node_modules/js-yaml": { + "version": "3.14.1", + "resolved": "https://registry.npmjs.org/js-yaml/-/js-yaml-3.14.1.tgz", + "integrity": "sha512-okMH7OXXJ7YrN9Ok3/SXrnu4iX9yOk+25nqX4imS2npuvTYDmo/QEZoqwZkYaIDk3jVvBOTOIEgEhaLOynBS9g==", + "dependencies": { + "argparse": "^1.0.7", + "esprima": "^4.0.0" + }, + "bin": { + "js-yaml": "bin/js-yaml.js" + } }, "node_modules/@runejs/eslint-config": { "version": "1.1.0", @@ -216,6 +225,30 @@ "resolved": "https://registry.npmjs.org/@sqltools/formatter/-/formatter-1.2.3.tgz", "integrity": "sha512-O3uyB/JbkAEMZaP3YqyHH7TMnex7tWyCbCI4EfJdOCoN6HIhqdJBWTM6aCCiWQ/5f5wxjgU735QAIpJbjDvmzg==" }, + "node_modules/@tsconfig/node10": { + "version": "1.0.9", + "resolved": "https://registry.npmjs.org/@tsconfig/node10/-/node10-1.0.9.tgz", + "integrity": "sha512-jNsYVVxU8v5g43Erja32laIDHXeoNvFEpX33OK4d6hljo3jDhCBDhx5dhCCTMWUojscpAagGiRkBKxpdl9fxqA==", + "dev": true + }, + "node_modules/@tsconfig/node12": { + "version": "1.0.11", + "resolved": "https://registry.npmjs.org/@tsconfig/node12/-/node12-1.0.11.tgz", + "integrity": "sha512-cqefuRsh12pWyGsIoBKJA9luFu3mRxCA+ORZvA4ktLSzIuCUtWVxGIuXigEwO5/ywWFMZ2QEGKWvkZG1zDMTag==", + "dev": true + }, + "node_modules/@tsconfig/node14": { + "version": "1.0.3", + "resolved": "https://registry.npmjs.org/@tsconfig/node14/-/node14-1.0.3.tgz", + "integrity": "sha512-ysT8mhdixWK6Hw3i1V2AeRqZ5WfXg1G43mqoYlM2nc6388Fq5jcXyr5mRsqViLx/GJYdoL0bfXD8nmF+Zn/Iow==", + "dev": true + }, + "node_modules/@tsconfig/node16": { + "version": "1.0.3", + "resolved": "https://registry.npmjs.org/@tsconfig/node16/-/node16-1.0.3.tgz", + "integrity": "sha512-yOlFc+7UtL/89t2ZhjPvvB/DeAr3r+Dq58IgzsFkOAvVC6NMJXmCGjbptdXdR9qsX7pKcTL+s87FtYREi2dEEQ==", + "dev": true + }, "node_modules/@types/adm-zip": { "version": "0.5.0", "resolved": "https://registry.npmjs.org/@types/adm-zip/-/adm-zip-0.5.0.tgz", @@ -235,21 +268,21 @@ } }, "node_modules/@types/json-schema": { - "version": "7.0.9", - "resolved": "https://registry.npmjs.org/@types/json-schema/-/json-schema-7.0.9.tgz", - "integrity": "sha512-qcUXuemtEu+E5wZSJHNxUXeCZhAfXKQ41D+duX+VYPde7xyEVZci+/oXKJL13tnRs9lR2pr4fod59GT6/X1/yQ==", + "version": "7.0.11", + "resolved": "https://registry.npmjs.org/@types/json-schema/-/json-schema-7.0.11.tgz", + "integrity": "sha512-wOuvG1SN4Us4rez+tylwwwCV1psiNVOkJeM3AUWUNWg/jDQY2+HE/444y5gc+jBmRqASOm2Oeh5c1axHobwRKQ==", "dev": true }, "node_modules/@types/node": { - "version": "16.11.26", - "resolved": "https://registry.npmjs.org/@types/node/-/node-16.11.26.tgz", - "integrity": "sha512-GZ7bu5A6+4DtG7q9GsoHXy3ALcgeIHP4NnL0Vv2wu0uUB/yQex26v0tf6/na1mm0+bS9Uw+0DFex7aaKr2qawQ==", + "version": "16.11.47", + "resolved": "https://registry.npmjs.org/@types/node/-/node-16.11.47.tgz", + "integrity": "sha512-fpP+jk2zJ4VW66+wAMFoBJlx1bxmBKx4DUFf68UHgdGCOuyUTDlLWqsaNPJh7xhNDykyJ9eIzAygilP/4WoN8g==", "dev": true }, "node_modules/@types/strip-bom": { "version": "3.0.0", "resolved": "https://registry.npmjs.org/@types/strip-bom/-/strip-bom-3.0.0.tgz", - "integrity": "sha1-FKjsOVbC6B7bdSB5CuzyHCkK69I=", + "integrity": "sha512-xevGOReSYGM7g/kUBZzPqCrR/KYAo+F0yiPc85WFTJa0MSLtyFTVTU6cJu/aV4mid7IffDIWqo69THF2o4JiEQ==", "dev": true }, "node_modules/@types/strip-json-comments": { @@ -259,18 +292,18 @@ "dev": true }, "node_modules/@types/yargs": { - "version": "17.0.9", - "resolved": "https://registry.npmjs.org/@types/yargs/-/yargs-17.0.9.tgz", - "integrity": "sha512-Ci8+4/DOtkHRylcisKmVMtmVO5g7weUVCKcsu1sJvF1bn0wExTmbHmhFKj7AnEm0de800iovGhdSKzYnzbaHpg==", + "version": "17.0.11", + "resolved": "https://registry.npmjs.org/@types/yargs/-/yargs-17.0.11.tgz", + "integrity": "sha512-aB4y9UDUXTSMxmM4MH+YnuR0g5Cph3FLQBoWoMB21DSvFVAxRVEHEMx3TLh+zUZYMCQtKiqazz0Q4Rre31f/OA==", "dev": true, "dependencies": { "@types/yargs-parser": "*" } }, "node_modules/@types/yargs-parser": { - "version": "20.2.1", - "resolved": "https://registry.npmjs.org/@types/yargs-parser/-/yargs-parser-20.2.1.tgz", - "integrity": "sha512-7tFImggNeNBVMsn0vLrpn1H1uPrUBdnARPTpZoitY37ZrdJREzf7I16tMrlK3hen349gr1NYh8CmZQa7CTG6Aw==", + "version": "21.0.0", + "resolved": "https://registry.npmjs.org/@types/yargs-parser/-/yargs-parser-21.0.0.tgz", + "integrity": "sha512-iO9ZQHkZxHn4mSakYV0vFHAVDyEOIJQrV2uZ06HxEPcx+mt8swXoZHIbaaJ2crJYFfErySgktuTZ3BeLz+XmFA==", "dev": true }, "node_modules/@types/zen-observable": { @@ -279,19 +312,19 @@ "integrity": "sha512-fbF6oTd4sGGy0xjHPKAt+eS2CrxJ3+6gQ3FGcBoIJR2TLAyCkCyI8JqZNy+FeON0AhVgNJoUumVoZQjBFUqHkw==" }, "node_modules/@typescript-eslint/eslint-plugin": { - "version": "5.14.0", - "resolved": "https://registry.npmjs.org/@typescript-eslint/eslint-plugin/-/eslint-plugin-5.14.0.tgz", - "integrity": "sha512-ir0wYI4FfFUDfLcuwKzIH7sMVA+db7WYen47iRSaCGl+HMAZI9fpBwfDo45ZALD3A45ZGyHWDNLhbg8tZrMX4w==", + "version": "5.33.0", + "resolved": "https://registry.npmjs.org/@typescript-eslint/eslint-plugin/-/eslint-plugin-5.33.0.tgz", + "integrity": "sha512-jHvZNSW2WZ31OPJ3enhLrEKvAZNyAFWZ6rx9tUwaessTc4sx9KmgMNhVcqVAl1ETnT5rU5fpXTLmY9YvC1DCNg==", "dev": true, "dependencies": { - "@typescript-eslint/scope-manager": "5.14.0", - "@typescript-eslint/type-utils": "5.14.0", - "@typescript-eslint/utils": "5.14.0", - "debug": "^4.3.2", + "@typescript-eslint/scope-manager": "5.33.0", + "@typescript-eslint/type-utils": "5.33.0", + "@typescript-eslint/utils": "5.33.0", + "debug": "^4.3.4", "functional-red-black-tree": "^1.0.1", - "ignore": "^5.1.8", + "ignore": "^5.2.0", "regexpp": "^3.2.0", - "semver": "^7.3.5", + "semver": "^7.3.7", "tsutils": "^3.21.0" }, "engines": { @@ -312,15 +345,15 @@ } }, "node_modules/@typescript-eslint/parser": { - "version": "5.14.0", - "resolved": "https://registry.npmjs.org/@typescript-eslint/parser/-/parser-5.14.0.tgz", - "integrity": "sha512-aHJN8/FuIy1Zvqk4U/gcO/fxeMKyoSv/rS46UXMXOJKVsLQ+iYPuXNbpbH7cBLcpSbmyyFbwrniLx5+kutu1pw==", + "version": "5.33.0", + "resolved": "https://registry.npmjs.org/@typescript-eslint/parser/-/parser-5.33.0.tgz", + "integrity": "sha512-cgM5cJrWmrDV2KpvlcSkelTBASAs1mgqq+IUGKJvFxWrapHpaRy5EXPQz9YaKF3nZ8KY18ILTiVpUtbIac86/w==", "dev": true, "dependencies": { - "@typescript-eslint/scope-manager": "5.14.0", - "@typescript-eslint/types": "5.14.0", - "@typescript-eslint/typescript-estree": "5.14.0", - "debug": "^4.3.2" + "@typescript-eslint/scope-manager": "5.33.0", + "@typescript-eslint/types": "5.33.0", + "@typescript-eslint/typescript-estree": "5.33.0", + "debug": "^4.3.4" }, "engines": { "node": "^12.22.0 || ^14.17.0 || >=16.0.0" @@ -339,13 +372,13 @@ } }, "node_modules/@typescript-eslint/scope-manager": { - "version": "5.14.0", - "resolved": "https://registry.npmjs.org/@typescript-eslint/scope-manager/-/scope-manager-5.14.0.tgz", - "integrity": "sha512-LazdcMlGnv+xUc5R4qIlqH0OWARyl2kaP8pVCS39qSL3Pd1F7mI10DbdXeARcE62sVQE4fHNvEqMWsypWO+yEw==", + "version": "5.33.0", + "resolved": "https://registry.npmjs.org/@typescript-eslint/scope-manager/-/scope-manager-5.33.0.tgz", + "integrity": "sha512-/Jta8yMNpXYpRDl8EwF/M8It2A9sFJTubDo0ATZefGXmOqlaBffEw0ZbkbQ7TNDK6q55NPHFshGBPAZvZkE8Pw==", "dev": true, "dependencies": { - "@typescript-eslint/types": "5.14.0", - "@typescript-eslint/visitor-keys": "5.14.0" + "@typescript-eslint/types": "5.33.0", + "@typescript-eslint/visitor-keys": "5.33.0" }, "engines": { "node": "^12.22.0 || ^14.17.0 || >=16.0.0" @@ -356,13 +389,13 @@ } }, "node_modules/@typescript-eslint/type-utils": { - "version": "5.14.0", - "resolved": "https://registry.npmjs.org/@typescript-eslint/type-utils/-/type-utils-5.14.0.tgz", - "integrity": "sha512-d4PTJxsqaUpv8iERTDSQBKUCV7Q5yyXjqXUl3XF7Sd9ogNLuKLkxz82qxokqQ4jXdTPZudWpmNtr/JjbbvUixw==", + "version": "5.33.0", + "resolved": "https://registry.npmjs.org/@typescript-eslint/type-utils/-/type-utils-5.33.0.tgz", + "integrity": "sha512-2zB8uEn7hEH2pBeyk3NpzX1p3lF9dKrEbnXq1F7YkpZ6hlyqb2yZujqgRGqXgRBTHWIUG3NGx/WeZk224UKlIA==", "dev": true, "dependencies": { - "@typescript-eslint/utils": "5.14.0", - "debug": "^4.3.2", + "@typescript-eslint/utils": "5.33.0", + "debug": "^4.3.4", "tsutils": "^3.21.0" }, "engines": { @@ -382,9 +415,9 @@ } }, "node_modules/@typescript-eslint/types": { - "version": "5.14.0", - "resolved": "https://registry.npmjs.org/@typescript-eslint/types/-/types-5.14.0.tgz", - "integrity": "sha512-BR6Y9eE9360LNnW3eEUqAg6HxS9Q35kSIs4rp4vNHRdfg0s+/PgHgskvu5DFTM7G5VKAVjuyaN476LCPrdA7Mw==", + "version": "5.33.0", + "resolved": "https://registry.npmjs.org/@typescript-eslint/types/-/types-5.33.0.tgz", + "integrity": "sha512-nIMt96JngB4MYFYXpZ/3ZNU4GWPNdBbcB5w2rDOCpXOVUkhtNlG2mmm8uXhubhidRZdwMaMBap7Uk8SZMU/ppw==", "dev": true, "engines": { "node": "^12.22.0 || ^14.17.0 || >=16.0.0" @@ -395,17 +428,17 @@ } }, "node_modules/@typescript-eslint/typescript-estree": { - "version": "5.14.0", - "resolved": "https://registry.npmjs.org/@typescript-eslint/typescript-estree/-/typescript-estree-5.14.0.tgz", - "integrity": "sha512-QGnxvROrCVtLQ1724GLTHBTR0lZVu13izOp9njRvMkCBgWX26PKvmMP8k82nmXBRD3DQcFFq2oj3cKDwr0FaUA==", + "version": "5.33.0", + "resolved": "https://registry.npmjs.org/@typescript-eslint/typescript-estree/-/typescript-estree-5.33.0.tgz", + "integrity": "sha512-tqq3MRLlggkJKJUrzM6wltk8NckKyyorCSGMq4eVkyL5sDYzJJcMgZATqmF8fLdsWrW7OjjIZ1m9v81vKcaqwQ==", "dev": true, "dependencies": { - "@typescript-eslint/types": "5.14.0", - "@typescript-eslint/visitor-keys": "5.14.0", - "debug": "^4.3.2", - "globby": "^11.0.4", + "@typescript-eslint/types": "5.33.0", + "@typescript-eslint/visitor-keys": "5.33.0", + "debug": "^4.3.4", + "globby": "^11.1.0", "is-glob": "^4.0.3", - "semver": "^7.3.5", + "semver": "^7.3.7", "tsutils": "^3.21.0" }, "engines": { @@ -422,15 +455,15 @@ } }, "node_modules/@typescript-eslint/utils": { - "version": "5.14.0", - "resolved": "https://registry.npmjs.org/@typescript-eslint/utils/-/utils-5.14.0.tgz", - "integrity": "sha512-EHwlII5mvUA0UsKYnVzySb/5EE/t03duUTweVy8Zqt3UQXBrpEVY144OTceFKaOe4xQXZJrkptCf7PjEBeGK4w==", + "version": "5.33.0", + "resolved": "https://registry.npmjs.org/@typescript-eslint/utils/-/utils-5.33.0.tgz", + "integrity": "sha512-JxOAnXt9oZjXLIiXb5ZIcZXiwVHCkqZgof0O8KPgz7C7y0HS42gi75PdPlqh1Tf109M0fyUw45Ao6JLo7S5AHw==", "dev": true, "dependencies": { "@types/json-schema": "^7.0.9", - "@typescript-eslint/scope-manager": "5.14.0", - "@typescript-eslint/types": "5.14.0", - "@typescript-eslint/typescript-estree": "5.14.0", + "@typescript-eslint/scope-manager": "5.33.0", + "@typescript-eslint/types": "5.33.0", + "@typescript-eslint/typescript-estree": "5.33.0", "eslint-scope": "^5.1.1", "eslint-utils": "^3.0.0" }, @@ -446,13 +479,13 @@ } }, "node_modules/@typescript-eslint/visitor-keys": { - "version": "5.14.0", - "resolved": "https://registry.npmjs.org/@typescript-eslint/visitor-keys/-/visitor-keys-5.14.0.tgz", - "integrity": "sha512-yL0XxfzR94UEkjBqyymMLgCBdojzEuy/eim7N9/RIcTNxpJudAcqsU8eRyfzBbcEzGoPWfdM3AGak3cN08WOIw==", + "version": "5.33.0", + "resolved": "https://registry.npmjs.org/@typescript-eslint/visitor-keys/-/visitor-keys-5.33.0.tgz", + "integrity": "sha512-/XsqCzD4t+Y9p5wd9HZiptuGKBlaZO5showwqODii5C0nZawxWLF+Q6k5wYHBrQv96h6GYKyqqMHCSTqta8Kiw==", "dev": true, "dependencies": { - "@typescript-eslint/types": "5.14.0", - "eslint-visitor-keys": "^3.0.0" + "@typescript-eslint/types": "5.33.0", + "eslint-visitor-keys": "^3.3.0" }, "engines": { "node": "^12.22.0 || ^14.17.0 || >=16.0.0" @@ -463,9 +496,9 @@ } }, "node_modules/acorn": { - "version": "8.7.0", - "resolved": "https://registry.npmjs.org/acorn/-/acorn-8.7.0.tgz", - "integrity": "sha512-V/LGr1APy+PXIwKebEWrkZPwoeoF+w1jiOBUmuxuiUIaOHtob8Qc9BTrYo7VuI5fR8tqsy+buA2WFooR5olqvQ==", + "version": "8.8.0", + "resolved": "https://registry.npmjs.org/acorn/-/acorn-8.8.0.tgz", + "integrity": "sha512-QOxyigPVrpZ2GXT+PFyZTl6TtOFc5egxHIP9IlQ+RbupQuX4RkT/Bee4/kQuC02Xkzg84JcT7oLYtDIQxp+v7w==", "dev": true, "bin": { "acorn": "bin/acorn" @@ -483,6 +516,15 @@ "acorn": "^6.0.0 || ^7.0.0 || ^8.0.0" } }, + "node_modules/acorn-walk": { + "version": "8.2.0", + "resolved": "https://registry.npmjs.org/acorn-walk/-/acorn-walk-8.2.0.tgz", + "integrity": "sha512-k+iyHEuPgSw6SbuDpGQM+06HQUa04DZ3o+F6CSzXMvvI5KMvnaEqXe+YVe555R9nn6GPt404fos4wcgpw12SDA==", + "dev": true, + "engines": { + "node": ">=0.4.0" + } + }, "node_modules/adm-zip": { "version": "0.5.9", "resolved": "https://registry.npmjs.org/adm-zip/-/adm-zip-0.5.9.tgz", @@ -510,24 +552,37 @@ "node_modules/amdefine": { "version": "1.0.1", "resolved": "https://registry.npmjs.org/amdefine/-/amdefine-1.0.1.tgz", - "integrity": "sha1-SlKCrBZHKek2Gbz9OtFR+BfOkfU=", + "integrity": "sha512-S2Hw0TtNkMJhIabBwIojKL9YHO5T0n5eNqWJ7Lrlel/zDbftQpxpapi8tZs3X1HWa+u+QeydGmzzNU0m09+Rcg==", "engines": { "node": ">=0.4.2" } }, "node_modules/ansi-regex": { - "version": "2.1.1", - "resolved": "https://registry.npmjs.org/ansi-regex/-/ansi-regex-2.1.1.tgz", - "integrity": "sha1-w7M6te42DYbg5ijwRorn7yfWVN8=", - "devOptional": true, + "version": "5.0.1", + "resolved": "https://registry.npmjs.org/ansi-regex/-/ansi-regex-5.0.1.tgz", + "integrity": "sha512-quJQXlTSUGL2LH9SUXo8VwsY4soanhgo6LNSm84E1LBcE8s3O0wpdiRzyR9z/ZZJMlMWv37qOOb9pdJlMUEKFQ==", "engines": { - "node": ">=0.10.0" + "node": ">=8" + } + }, + "node_modules/ansi-styles": { + "version": "4.3.0", + "resolved": "https://registry.npmjs.org/ansi-styles/-/ansi-styles-4.3.0.tgz", + "integrity": "sha512-zbB9rCJAT1rbjiVDb2hqKFHNYLxgtk8NURxZ3IZwD3F6NtxbXZQCnnSi1Lkx+IDohdPlFp222wVALIheZJQSEg==", + "dependencies": { + "color-convert": "^2.0.1" + }, + "engines": { + "node": ">=8" + }, + "funding": { + "url": "https://github.com/chalk/ansi-styles?sponsor=1" } }, "node_modules/any-promise": { "version": "1.3.0", "resolved": "https://registry.npmjs.org/any-promise/-/any-promise-1.3.0.tgz", - "integrity": "sha1-q8av7tzqUugJzcA3au0845Y10X8=" + "integrity": "sha512-7UvmKalWRt1wgjL1RrGxoSJW/0QZFIegpeGvZG9kjp8vrRu55XTHbwnqq2GpXm9uLbcuhxm3IqX9OB4MZR1b2A==" }, "node_modules/anymatch": { "version": "3.1.2", @@ -550,28 +605,95 @@ "node": ">= 6.0.0" } }, - "node_modules/aproba": { - "version": "1.2.0", - "resolved": "https://registry.npmjs.org/aproba/-/aproba-1.2.0.tgz", - "integrity": "sha512-Y9J6ZjXtoYh8RnXVCMOU/ttDmk1aBjunq9vO0ta5x85WDQiQfUF9sIPBITdbiiIVcBo03Hi3jMxigBtsddlXRw==", - "devOptional": true - }, - "node_modules/are-we-there-yet": { - "version": "1.1.5", - "resolved": "https://registry.npmjs.org/are-we-there-yet/-/are-we-there-yet-1.1.5.tgz", - "integrity": "sha512-5hYdAkZlcG8tOLujVDTgCT+uPX0VnpAH28gWsLfzpXYm7wP6mp5Q/gYyR7YQ0cKVJcXJnl3j2kpBan13PtQf6w==", - "devOptional": true, - "dependencies": { - "delegates": "^1.0.0", - "readable-stream": "^2.0.6" - } - }, "node_modules/arg": { "version": "4.1.3", "resolved": "https://registry.npmjs.org/arg/-/arg-4.1.3.tgz", "integrity": "sha512-58S9QDqG0Xx27YwPSt9fJxivjYl432YCwfDMfZ+71RAqUrZef7LrKQZ3LHLOwCS4FLNBplP533Zx895SeOCHvA==", "dev": true }, + "node_modules/argparse": { + "version": "2.0.1", + "resolved": "https://registry.npmjs.org/argparse/-/argparse-2.0.1.tgz", + "integrity": "sha512-8+9WqebbFzpX9OR+Wa6O29asIogeRMzcGtAINdpMHHyAg10f05aSFVBbcEqGf/PXw1EjAZ+q2/bEBg3DvurK3Q==" + }, + "node_modules/args": { + "version": "5.0.3", + "resolved": "https://registry.npmjs.org/args/-/args-5.0.3.tgz", + "integrity": "sha512-h6k/zfFgusnv3i5TU08KQkVKuCPBtL/PWQbWkHUxvJrZ2nAyeaUupneemcrgn1xmqxPQsPIzwkUhOpoqPDRZuA==", + "dependencies": { + "camelcase": "5.0.0", + "chalk": "2.4.2", + "leven": "2.1.0", + "mri": "1.1.4" + }, + "engines": { + "node": ">= 6.0.0" + } + }, + "node_modules/args/node_modules/ansi-styles": { + "version": "3.2.1", + "resolved": "https://registry.npmjs.org/ansi-styles/-/ansi-styles-3.2.1.tgz", + "integrity": "sha512-VT0ZI6kZRdTh8YyJw3SMbYm/u+NqfsAxEpWO0Pf9sq8/e94WxxOpPKx9FR1FlyCtOVDNOQ+8ntlqFxiRc+r5qA==", + "dependencies": { + "color-convert": "^1.9.0" + }, + "engines": { + "node": ">=4" + } + }, + "node_modules/args/node_modules/chalk": { + "version": "2.4.2", + "resolved": "https://registry.npmjs.org/chalk/-/chalk-2.4.2.tgz", + "integrity": "sha512-Mti+f9lpJNcwF4tWV8/OrTTtF1gZi+f8FqlyAdouralcFWFQWF2+NgCHShjkCb+IFBLq9buZwE1xckQU4peSuQ==", + "dependencies": { + "ansi-styles": "^3.2.1", + "escape-string-regexp": "^1.0.5", + "supports-color": "^5.3.0" + }, + "engines": { + "node": ">=4" + } + }, + "node_modules/args/node_modules/color-convert": { + "version": "1.9.3", + "resolved": "https://registry.npmjs.org/color-convert/-/color-convert-1.9.3.tgz", + "integrity": "sha512-QfAUtd+vFdAtFQcC8CCyYt1fYWxSqAiK2cSD6zDB8N3cpsEBAvRxp9zOGg6G/SHHJYAT88/az/IuDGALsNVbGg==", + "dependencies": { + "color-name": "1.1.3" + } + }, + "node_modules/args/node_modules/color-name": { + "version": "1.1.3", + "resolved": "https://registry.npmjs.org/color-name/-/color-name-1.1.3.tgz", + "integrity": "sha512-72fSenhMw2HZMTVHeCA9KCmpEIbzWiQsjN+BHcBbS9vr1mtt+vJjPdksIBNUmKAW8TFUDPJK5SUU3QhE9NEXDw==" + }, + "node_modules/args/node_modules/escape-string-regexp": { + "version": "1.0.5", + "resolved": "https://registry.npmjs.org/escape-string-regexp/-/escape-string-regexp-1.0.5.tgz", + "integrity": "sha512-vbRorB5FUQWvla16U8R/qgaFIya2qGzwDrNmCZuYKrbdSUMG6I1ZCGQRefkRVhuOkIGVne7BQ35DSfo1qvJqFg==", + "engines": { + "node": ">=0.8.0" + } + }, + "node_modules/args/node_modules/has-flag": { + "version": "3.0.0", + "resolved": "https://registry.npmjs.org/has-flag/-/has-flag-3.0.0.tgz", + "integrity": "sha512-sKJf1+ceQBr4SMkvQnBDNDtf4TXpVhVGateu0t918bl30FnbE2m4vNLX+VWe/dpjlb+HugGYzW7uQXH98HPEYw==", + "engines": { + "node": ">=4" + } + }, + "node_modules/args/node_modules/supports-color": { + "version": "5.5.0", + "resolved": "https://registry.npmjs.org/supports-color/-/supports-color-5.5.0.tgz", + "integrity": "sha512-QjVjwdXIt408MIiAqCX4oUKsgU2EqAGzs2Ppkm4aQYbjm+ZEWEcW4SfFNTr4uMNZma0ey4f5lgLrkB0aX0QMow==", + "dependencies": { + "has-flag": "^3.0.0" + }, + "engines": { + "node": ">=4" + } + }, "node_modules/array-union": { "version": "2.1.0", "resolved": "https://registry.npmjs.org/array-union/-/array-union-2.1.0.tgz", @@ -586,6 +708,14 @@ "resolved": "https://registry.npmjs.org/asynckit/-/asynckit-0.4.0.tgz", "integrity": "sha512-Oei9OH4tRh0YqU3GxhX79dM/mwVgvbZJaSNaRk+bshkj0S5cfHcgYakreBjrHwatXKbz+IoIdYLxrKim2MjW0Q==" }, + "node_modules/atomic-sleep": { + "version": "1.0.0", + "resolved": "https://registry.npmjs.org/atomic-sleep/-/atomic-sleep-1.0.0.tgz", + "integrity": "sha512-kNOjDqAh7px0XWNI+4QbzoiR/nTkHAWNud2uvnJquD1/x5a7EQZMJT0AczqK0Qn67oY/TTQ1LbUKajZpp3I9tQ==", + "engines": { + "node": ">=8.0.0" + } + }, "node_modules/axios": { "version": "0.27.2", "resolved": "https://registry.npmjs.org/axios/-/axios-0.27.2.tgz", @@ -596,9 +726,9 @@ } }, "node_modules/balanced-match": { - "version": "1.0.0", - "resolved": "https://registry.npmjs.org/balanced-match/-/balanced-match-1.0.0.tgz", - "integrity": "sha1-ibTRmasr7kneFk6gK4nORi1xt2c=" + "version": "1.0.2", + "resolved": "https://registry.npmjs.org/balanced-match/-/balanced-match-1.0.2.tgz", + "integrity": "sha512-3oSeUO0TMV67hN1AmbXsK4yaqU7tjiHlbxRDZOpH0KW9+CeX4bRAaX0Anxt0tx2MrpRpWwQaPwIlISEJhYU5Pw==" }, "node_modules/base64-js": { "version": "1.5.1", @@ -620,14 +750,14 @@ ] }, "node_modules/better-sqlite3": { - "version": "7.5.0", - "resolved": "https://registry.npmjs.org/better-sqlite3/-/better-sqlite3-7.5.0.tgz", - "integrity": "sha512-6FdG9DoytYGDhLW7VWW1vxjEz7xHkqK6LnaUQYA8d6GHNgZhu9PFX2xwKEEnSBRoT1J4PjTUPeg217ShxNmuPg==", + "version": "7.6.2", + "resolved": "https://registry.npmjs.org/better-sqlite3/-/better-sqlite3-7.6.2.tgz", + "integrity": "sha512-S5zIU1Hink2AH4xPsN0W43T1/AJ5jrPh7Oy07ocuW/AKYYY02GWzz9NH0nbSMn/gw6fDZ5jZ1QsHt1BXAwJ6Lg==", "devOptional": true, "hasInstallScript": true, "dependencies": { "bindings": "^1.5.0", - "prebuild-install": "^7.0.0" + "prebuild-install": "^7.1.0" } }, "node_modules/binary-extensions": { @@ -673,6 +803,15 @@ "node": ">= 6" } }, + "node_modules/bl/node_modules/string_decoder": { + "version": "1.3.0", + "resolved": "https://registry.npmjs.org/string_decoder/-/string_decoder-1.3.0.tgz", + "integrity": "sha512-hkRX8U1WjJFd8LsDJ2yQ/wWWxaopEsABU1XfkM8A+j0+85JAGppt16cr1Whg6KIbb4okU6Mql6BOj+uup/wKeA==", + "devOptional": true, + "dependencies": { + "safe-buffer": "~5.2.0" + } + }, "node_modules/brace-expansion": { "version": "1.1.11", "resolved": "https://registry.npmjs.org/brace-expansion/-/brace-expansion-1.1.11.tgz", @@ -733,10 +872,18 @@ "node": ">=6" } }, + "node_modules/camelcase": { + "version": "5.0.0", + "resolved": "https://registry.npmjs.org/camelcase/-/camelcase-5.0.0.tgz", + "integrity": "sha512-faqwZqnWxbxn+F1d399ygeamQNy3lPp/H9H6rNrqYh4FSVCtcY+3cub1MxA8o9mDd55mM8Aghuu/kuyYA6VTsA==", + "engines": { + "node": ">=6" + } + }, "node_modules/chalk": { - "version": "4.1.0", - "resolved": "https://registry.npmjs.org/chalk/-/chalk-4.1.0.tgz", - "integrity": "sha512-qwx12AxXe2Q5xQ43Ac//I6v5aXTipYrSESdOgzrN+9XjgEpyjpKuvSGaN4qE93f7TQTlerQQ8S+EQ0EyDoVL1A==", + "version": "4.1.2", + "resolved": "https://registry.npmjs.org/chalk/-/chalk-4.1.2.tgz", + "integrity": "sha512-oKnbhFyRIXpUuez8iBMmyEa4nbj4IOQyuhc/wy9kY7/WVPcwIO9VA668Pu8RkO7+0G76SLROeyw9CpQ061i4mA==", "dependencies": { "ansi-styles": "^4.1.0", "supports-color": "^7.1.0" @@ -748,55 +895,6 @@ "url": "https://github.com/chalk/chalk?sponsor=1" } }, - "node_modules/chalk/node_modules/ansi-styles": { - "version": "4.3.0", - "resolved": "https://registry.npmjs.org/ansi-styles/-/ansi-styles-4.3.0.tgz", - "integrity": "sha512-zbB9rCJAT1rbjiVDb2hqKFHNYLxgtk8NURxZ3IZwD3F6NtxbXZQCnnSi1Lkx+IDohdPlFp222wVALIheZJQSEg==", - "dependencies": { - "color-convert": "^2.0.1" - }, - "engines": { - "node": ">=8" - }, - "funding": { - "url": "https://github.com/chalk/ansi-styles?sponsor=1" - } - }, - "node_modules/chalk/node_modules/color-convert": { - "version": "2.0.1", - "resolved": "https://registry.npmjs.org/color-convert/-/color-convert-2.0.1.tgz", - "integrity": "sha512-RRECPsj7iu/xb5oKYcsFHSppFNnsj/52OVTRKb4zP5onXwVF3zVmmToNcOfGC+CRDpfK/U584fMg38ZHCaElKQ==", - "dependencies": { - "color-name": "~1.1.4" - }, - "engines": { - "node": ">=7.0.0" - } - }, - "node_modules/chalk/node_modules/color-name": { - "version": "1.1.4", - "resolved": "https://registry.npmjs.org/color-name/-/color-name-1.1.4.tgz", - "integrity": "sha512-dOy+3AuW3a2wNbZHIuMZpTcgjGuLU/uBL/ubcZF9OXbDo8ff4O8yVp5Bf0efS8uEoYo5q4Fx7dY9OgQGXgAsQA==" - }, - "node_modules/chalk/node_modules/has-flag": { - "version": "4.0.0", - "resolved": "https://registry.npmjs.org/has-flag/-/has-flag-4.0.0.tgz", - "integrity": "sha512-EykJT/Q1KjTWctppgIAgfSO0tKVuZUjhgMr17kqTumMl6Afv3EISleU7qZUzoXDFTAHTDC4NOoG/ZxU3EvlMPQ==", - "engines": { - "node": ">=8" - } - }, - "node_modules/chalk/node_modules/supports-color": { - "version": "7.2.0", - "resolved": "https://registry.npmjs.org/supports-color/-/supports-color-7.2.0.tgz", - "integrity": "sha512-qpCAvRl9stuOHveKsn7HncJRvv501qIacKzQlO/+Lwxc9+0q2wLyv4Dfvt80/DPn2pqOBsJdDiogXGR9+OvwRw==", - "dependencies": { - "has-flag": "^4.0.0" - }, - "engines": { - "node": ">=8" - } - }, "node_modules/chokidar": { "version": "3.5.3", "resolved": "https://registry.npmjs.org/chokidar/-/chokidar-3.5.3.tgz", @@ -824,15 +922,27 @@ "fsevents": "~2.3.2" } }, - "node_modules/chownr": { - "version": "1.1.4", - "resolved": "https://registry.npmjs.org/chownr/-/chownr-1.1.4.tgz", - "integrity": "sha512-jJ0bqzaylmJtVnNgzTeSOs8DPavpbYgEr/b0YL8/2GO3xJEhInFmhKMUnEJQjZumK7KXGFhUy89PrsJWlakBVg==", - "devOptional": true - }, - "node_modules/cli-highlight": { - "version": "2.1.11", - "resolved": "https://registry.npmjs.org/cli-highlight/-/cli-highlight-2.1.11.tgz", + "node_modules/chokidar/node_modules/glob-parent": { + "version": "5.1.2", + "resolved": "https://registry.npmjs.org/glob-parent/-/glob-parent-5.1.2.tgz", + "integrity": "sha512-AOIgSQCepiJYwP3ARnGx+5VnTu2HBYdzbGP45eLw1vr3zB3vZLeyed1sC9hnbcOc9/SrMyM5RPQrkGz4aS9Zow==", + "dev": true, + "dependencies": { + "is-glob": "^4.0.1" + }, + "engines": { + "node": ">= 6" + } + }, + "node_modules/chownr": { + "version": "1.1.4", + "resolved": "https://registry.npmjs.org/chownr/-/chownr-1.1.4.tgz", + "integrity": "sha512-jJ0bqzaylmJtVnNgzTeSOs8DPavpbYgEr/b0YL8/2GO3xJEhInFmhKMUnEJQjZumK7KXGFhUy89PrsJWlakBVg==", + "devOptional": true + }, + "node_modules/cli-highlight": { + "version": "2.1.11", + "resolved": "https://registry.npmjs.org/cli-highlight/-/cli-highlight-2.1.11.tgz", "integrity": "sha512-9KDcoEVwyUXrjcJNvHD0NFc/hiwe/WPVYIleQh2O1N2Zro5gWJZ/K+3DGn8w8P/F6FxOgzyC5bxDyHIgCSPhGg==", "dependencies": { "chalk": "^4.0.0", @@ -850,46 +960,6 @@ "npm": ">=5.0.0" } }, - "node_modules/cli-highlight/node_modules/ansi-regex": { - "version": "5.0.1", - "resolved": "https://registry.npmjs.org/ansi-regex/-/ansi-regex-5.0.1.tgz", - "integrity": "sha512-quJQXlTSUGL2LH9SUXo8VwsY4soanhgo6LNSm84E1LBcE8s3O0wpdiRzyR9z/ZZJMlMWv37qOOb9pdJlMUEKFQ==", - "engines": { - "node": ">=8" - } - }, - "node_modules/cli-highlight/node_modules/is-fullwidth-code-point": { - "version": "3.0.0", - "resolved": "https://registry.npmjs.org/is-fullwidth-code-point/-/is-fullwidth-code-point-3.0.0.tgz", - "integrity": "sha512-zymm5+u+sCsSWyD9qNaejV3DFvhCKclKdizYaJUuHA83RLjb7nSuGnddCHGv0hk+KY7BMAlsWeK4Ueg6EV6XQg==", - "engines": { - "node": ">=8" - } - }, - "node_modules/cli-highlight/node_modules/string-width": { - "version": "4.2.3", - "resolved": "https://registry.npmjs.org/string-width/-/string-width-4.2.3.tgz", - "integrity": "sha512-wKyQRQpjJ0sIp62ErSZdGsjMJWsap5oRNihHhu6G7JVO/9jIB6UyevL+tXuOqrng8j/cxKTWyWUwvSTriiZz/g==", - "dependencies": { - "emoji-regex": "^8.0.0", - "is-fullwidth-code-point": "^3.0.0", - "strip-ansi": "^6.0.1" - }, - "engines": { - "node": ">=8" - } - }, - "node_modules/cli-highlight/node_modules/strip-ansi": { - "version": "6.0.1", - "resolved": "https://registry.npmjs.org/strip-ansi/-/strip-ansi-6.0.1.tgz", - "integrity": "sha512-Y38VPSHcqkFrCpFnQ9vuSXmquuv5oXOKpGeT6aGrr3o3Gc9AlVa6JBfUSOCnbxGGZF+/0ooI7KrPuUSztUdU5A==", - "dependencies": { - "ansi-regex": "^5.0.1" - }, - "engines": { - "node": ">=8" - } - }, "node_modules/cli-highlight/node_modules/yargs": { "version": "16.2.0", "resolved": "https://registry.npmjs.org/yargs/-/yargs-16.2.0.tgz", @@ -925,54 +995,21 @@ "wrap-ansi": "^7.0.0" } }, - "node_modules/cliui/node_modules/ansi-regex": { - "version": "5.0.1", - "resolved": "https://registry.npmjs.org/ansi-regex/-/ansi-regex-5.0.1.tgz", - "integrity": "sha512-quJQXlTSUGL2LH9SUXo8VwsY4soanhgo6LNSm84E1LBcE8s3O0wpdiRzyR9z/ZZJMlMWv37qOOb9pdJlMUEKFQ==", - "engines": { - "node": ">=8" - } - }, - "node_modules/cliui/node_modules/is-fullwidth-code-point": { - "version": "3.0.0", - "resolved": "https://registry.npmjs.org/is-fullwidth-code-point/-/is-fullwidth-code-point-3.0.0.tgz", - "integrity": "sha512-zymm5+u+sCsSWyD9qNaejV3DFvhCKclKdizYaJUuHA83RLjb7nSuGnddCHGv0hk+KY7BMAlsWeK4Ueg6EV6XQg==", - "engines": { - "node": ">=8" - } - }, - "node_modules/cliui/node_modules/string-width": { - "version": "4.2.3", - "resolved": "https://registry.npmjs.org/string-width/-/string-width-4.2.3.tgz", - "integrity": "sha512-wKyQRQpjJ0sIp62ErSZdGsjMJWsap5oRNihHhu6G7JVO/9jIB6UyevL+tXuOqrng8j/cxKTWyWUwvSTriiZz/g==", - "dependencies": { - "emoji-regex": "^8.0.0", - "is-fullwidth-code-point": "^3.0.0", - "strip-ansi": "^6.0.1" - }, - "engines": { - "node": ">=8" - } - }, - "node_modules/cliui/node_modules/strip-ansi": { - "version": "6.0.1", - "resolved": "https://registry.npmjs.org/strip-ansi/-/strip-ansi-6.0.1.tgz", - "integrity": "sha512-Y38VPSHcqkFrCpFnQ9vuSXmquuv5oXOKpGeT6aGrr3o3Gc9AlVa6JBfUSOCnbxGGZF+/0ooI7KrPuUSztUdU5A==", + "node_modules/color-convert": { + "version": "2.0.1", + "resolved": "https://registry.npmjs.org/color-convert/-/color-convert-2.0.1.tgz", + "integrity": "sha512-RRECPsj7iu/xb5oKYcsFHSppFNnsj/52OVTRKb4zP5onXwVF3zVmmToNcOfGC+CRDpfK/U584fMg38ZHCaElKQ==", "dependencies": { - "ansi-regex": "^5.0.1" + "color-name": "~1.1.4" }, "engines": { - "node": ">=8" + "node": ">=7.0.0" } }, - "node_modules/code-point-at": { - "version": "1.1.0", - "resolved": "https://registry.npmjs.org/code-point-at/-/code-point-at-1.1.0.tgz", - "integrity": "sha1-DQcLTQQ6W+ozovGkDi7bPZpMz3c=", - "devOptional": true, - "engines": { - "node": ">=0.10.0" - } + "node_modules/color-name": { + "version": "1.1.4", + "resolved": "https://registry.npmjs.org/color-name/-/color-name-1.1.4.tgz", + "integrity": "sha512-dOy+3AuW3a2wNbZHIuMZpTcgjGuLU/uBL/ubcZF9OXbDo8ff4O8yVp5Bf0efS8uEoYo5q4Fx7dY9OgQGXgAsQA==" }, "node_modules/combined-stream": { "version": "1.0.8", @@ -988,7 +1025,7 @@ "node_modules/commander": { "version": "2.8.1", "resolved": "https://registry.npmjs.org/commander/-/commander-2.8.1.tgz", - "integrity": "sha1-Br42f+v9oMMwqh4qBy09yXYkJdQ=", + "integrity": "sha512-+pJLBFVk+9ZZdlAOB5WuIElVPPth47hILFkmGym57aq8kwxsowvByvB0DHs1vQAhyMZzdcpTtF0VDKGkSDR4ZQ==", "dependencies": { "graceful-readlink": ">= 1.0.0" }, @@ -1011,13 +1048,7 @@ "node_modules/concat-map": { "version": "0.0.1", "resolved": "https://registry.npmjs.org/concat-map/-/concat-map-0.0.1.tgz", - "integrity": "sha1-2Klr13/Wjfd5OnMDajug1UBdR3s=" - }, - "node_modules/console-control-strings": { - "version": "1.1.0", - "resolved": "https://registry.npmjs.org/console-control-strings/-/console-control-strings-1.1.0.tgz", - "integrity": "sha1-PXz0Rk22RG6mRL9LOVB/mFEAjo4=", - "devOptional": true + "integrity": "sha512-/Srv4dswyQNBfohGpz9o6Yb3Gz3SrUDqBH5rTuhGR7ahtlbYKnVxw2bCFMRljaA7EXHaXZ8wsHdodFvbkhKmqg==" }, "node_modules/copyfiles": { "version": "2.4.1", @@ -1038,50 +1069,6 @@ "copyup": "copyfiles" } }, - "node_modules/copyfiles/node_modules/ansi-regex": { - "version": "5.0.1", - "resolved": "https://registry.npmjs.org/ansi-regex/-/ansi-regex-5.0.1.tgz", - "integrity": "sha512-quJQXlTSUGL2LH9SUXo8VwsY4soanhgo6LNSm84E1LBcE8s3O0wpdiRzyR9z/ZZJMlMWv37qOOb9pdJlMUEKFQ==", - "dev": true, - "engines": { - "node": ">=8" - } - }, - "node_modules/copyfiles/node_modules/is-fullwidth-code-point": { - "version": "3.0.0", - "resolved": "https://registry.npmjs.org/is-fullwidth-code-point/-/is-fullwidth-code-point-3.0.0.tgz", - "integrity": "sha512-zymm5+u+sCsSWyD9qNaejV3DFvhCKclKdizYaJUuHA83RLjb7nSuGnddCHGv0hk+KY7BMAlsWeK4Ueg6EV6XQg==", - "dev": true, - "engines": { - "node": ">=8" - } - }, - "node_modules/copyfiles/node_modules/string-width": { - "version": "4.2.3", - "resolved": "https://registry.npmjs.org/string-width/-/string-width-4.2.3.tgz", - "integrity": "sha512-wKyQRQpjJ0sIp62ErSZdGsjMJWsap5oRNihHhu6G7JVO/9jIB6UyevL+tXuOqrng8j/cxKTWyWUwvSTriiZz/g==", - "dev": true, - "dependencies": { - "emoji-regex": "^8.0.0", - "is-fullwidth-code-point": "^3.0.0", - "strip-ansi": "^6.0.1" - }, - "engines": { - "node": ">=8" - } - }, - "node_modules/copyfiles/node_modules/strip-ansi": { - "version": "6.0.1", - "resolved": "https://registry.npmjs.org/strip-ansi/-/strip-ansi-6.0.1.tgz", - "integrity": "sha512-Y38VPSHcqkFrCpFnQ9vuSXmquuv5oXOKpGeT6aGrr3o3Gc9AlVa6JBfUSOCnbxGGZF+/0ooI7KrPuUSztUdU5A==", - "dev": true, - "dependencies": { - "ansi-regex": "^5.0.1" - }, - "engines": { - "node": ">=8" - } - }, "node_modules/copyfiles/node_modules/yargs": { "version": "16.2.0", "resolved": "https://registry.npmjs.org/yargs/-/yargs-16.2.0.tgz", @@ -1110,10 +1097,10 @@ } }, "node_modules/core-util-is": { - "version": "1.0.2", - "resolved": "https://registry.npmjs.org/core-util-is/-/core-util-is-1.0.2.tgz", - "integrity": "sha1-tf1UIgqivFq1eqtxQMlAdUUDwac=", - "devOptional": true + "version": "1.0.3", + "resolved": "https://registry.npmjs.org/core-util-is/-/core-util-is-1.0.3.tgz", + "integrity": "sha512-ZQBvi1DcpJ4GDqanjucZ2Hj3wEO5pZDS89BWbkcrvdxksJorwUDDZamX9ldFkp9aw2lmBDLgkObEA4DWNJ9FYQ==", + "dev": true }, "node_modules/create-require": { "version": "1.1.1", @@ -1135,10 +1122,18 @@ "node": ">= 8" } }, + "node_modules/dateformat": { + "version": "4.6.3", + "resolved": "https://registry.npmjs.org/dateformat/-/dateformat-4.6.3.tgz", + "integrity": "sha512-2P0p0pFGzHS5EMnhdxQi7aJN+iMheud0UhG4dlE1DLAlvL8JHjJJTX/CSm4JXwV0Ka5nGk3zC5mcb5bUQUxxMA==", + "engines": { + "node": "*" + } + }, "node_modules/debug": { - "version": "4.3.2", - "resolved": "https://registry.npmjs.org/debug/-/debug-4.3.2.tgz", - "integrity": "sha512-mOp8wKcvj7XxC78zLgw/ZA+6TSgkoE2C/ienthhRD298T7UNwAg9diBpLRxC0mOezLl4B0xV7M0cCO6P/O0Xhw==", + "version": "4.3.4", + "resolved": "https://registry.npmjs.org/debug/-/debug-4.3.4.tgz", + "integrity": "sha512-PRWFHuSU3eDtQJPvnNY7Jcket1j0t5OuOsFzPPzsekD52Zl8qUfFIPEiswXqIvHWGVHOgX+7G/vCNNhehwxfkQ==", "dependencies": { "ms": "2.1.2" }, @@ -1189,11 +1184,14 @@ "node": ">=0.4.0" } }, - "node_modules/delegates": { - "version": "1.0.0", - "resolved": "https://registry.npmjs.org/delegates/-/delegates-1.0.0.tgz", - "integrity": "sha1-hMbhWbgZBP3KWaDvRM2HDTElD5o=", - "devOptional": true + "node_modules/detect-libc": { + "version": "2.0.1", + "resolved": "https://registry.npmjs.org/detect-libc/-/detect-libc-2.0.1.tgz", + "integrity": "sha512-463v3ZeIrcWtdgIg6vI6XUncguvr2TnGl4SzDXinkt9mSLpBJKXT3mW6xT3VQdDN11+WVs29pgvivTc4Lp8v+w==", + "devOptional": true, + "engines": { + "node": ">=8" + } }, "node_modules/diff": { "version": "4.0.2", @@ -1239,7 +1237,7 @@ "node_modules/dynamic-dedupe": { "version": "0.3.0", "resolved": "https://registry.npmjs.org/dynamic-dedupe/-/dynamic-dedupe-0.3.0.tgz", - "integrity": "sha1-BuRMIj9eTpTXjvnbI6ZRXOL5YqE=", + "integrity": "sha512-ssuANeD+z97meYOqd50e04Ze5qp4bPqo8cCkI4TRjZkzAUgIDTrXV1R8QCdINpiI+hw14+rYazvTRdQrz0/rFQ==", "dev": true, "dependencies": { "xtend": "^4.0.0" @@ -1254,7 +1252,6 @@ "version": "1.4.4", "resolved": "https://registry.npmjs.org/end-of-stream/-/end-of-stream-1.4.4.tgz", "integrity": "sha512-+uw1inIHVPQoaVuHzRyXd21icM+cnt4CzD5rW+NC1wjOUSTOs+Te7FOv7AhN7vS9x/oIyhLP5PR1H+phQAHu5Q==", - "devOptional": true, "dependencies": { "once": "^1.4.0" } @@ -1267,14 +1264,27 @@ "node": ">=6" } }, + "node_modules/escape-string-regexp": { + "version": "4.0.0", + "resolved": "https://registry.npmjs.org/escape-string-regexp/-/escape-string-regexp-4.0.0.tgz", + "integrity": "sha512-TtpcNJ3XAzx3Gq8sWRzJaVajRs0uVxA2YAkdb1jm2YkPz4G6egUFAyA3n5vtEIZefPk5Wa4UXbKuS5fKkJWdgA==", + "dev": true, + "engines": { + "node": ">=10" + }, + "funding": { + "url": "https://github.com/sponsors/sindresorhus" + } + }, "node_modules/eslint": { - "version": "8.11.0", - "resolved": "https://registry.npmjs.org/eslint/-/eslint-8.11.0.tgz", - "integrity": "sha512-/KRpd9mIRg2raGxHRGwW9ZywYNAClZrHjdueHcrVDuO3a6bj83eoTirCCk0M0yPwOjWYKHwRVRid+xK4F/GHgA==", + "version": "8.21.0", + "resolved": "https://registry.npmjs.org/eslint/-/eslint-8.21.0.tgz", + "integrity": "sha512-/XJ1+Qurf1T9G2M5IHrsjp+xrGT73RZf23xA1z5wB1ZzzEAWSZKvRwhWxTFp1rvkvCfwcvAUNAP31bhKTTGfDA==", "dev": true, "dependencies": { - "@eslint/eslintrc": "^1.2.1", - "@humanwhocodes/config-array": "^0.9.2", + "@eslint/eslintrc": "^1.3.0", + "@humanwhocodes/config-array": "^0.10.4", + "@humanwhocodes/gitignore-to-minimatch": "^1.0.2", "ajv": "^6.10.0", "chalk": "^4.0.0", "cross-spawn": "^7.0.2", @@ -1284,14 +1294,17 @@ "eslint-scope": "^7.1.1", "eslint-utils": "^3.0.0", "eslint-visitor-keys": "^3.3.0", - "espree": "^9.3.1", + "espree": "^9.3.3", "esquery": "^1.4.0", "esutils": "^2.0.2", "fast-deep-equal": "^3.1.3", "file-entry-cache": "^6.0.1", + "find-up": "^5.0.0", "functional-red-black-tree": "^1.0.1", "glob-parent": "^6.0.1", - "globals": "^13.6.0", + "globals": "^13.15.0", + "globby": "^11.1.0", + "grapheme-splitter": "^1.0.4", "ignore": "^5.2.0", "import-fresh": "^3.0.0", "imurmurhash": "^0.1.4", @@ -1300,7 +1313,7 @@ "json-stable-stringify-without-jsonify": "^1.0.1", "levn": "^0.4.1", "lodash.merge": "^4.6.2", - "minimatch": "^3.0.4", + "minimatch": "^3.1.2", "natural-compare": "^1.4.0", "optionator": "^0.9.1", "regexpp": "^3.2.0", @@ -1368,33 +1381,6 @@ "node": "^12.22.0 || ^14.17.0 || >=16.0.0" } }, - "node_modules/eslint/node_modules/ansi-regex": { - "version": "5.0.1", - "resolved": "https://registry.npmjs.org/ansi-regex/-/ansi-regex-5.0.1.tgz", - "integrity": "sha512-quJQXlTSUGL2LH9SUXo8VwsY4soanhgo6LNSm84E1LBcE8s3O0wpdiRzyR9z/ZZJMlMWv37qOOb9pdJlMUEKFQ==", - "dev": true, - "engines": { - "node": ">=8" - } - }, - "node_modules/eslint/node_modules/argparse": { - "version": "2.0.1", - "resolved": "https://registry.npmjs.org/argparse/-/argparse-2.0.1.tgz", - "integrity": "sha512-8+9WqebbFzpX9OR+Wa6O29asIogeRMzcGtAINdpMHHyAg10f05aSFVBbcEqGf/PXw1EjAZ+q2/bEBg3DvurK3Q==", - "dev": true - }, - "node_modules/eslint/node_modules/escape-string-regexp": { - "version": "4.0.0", - "resolved": "https://registry.npmjs.org/escape-string-regexp/-/escape-string-regexp-4.0.0.tgz", - "integrity": "sha512-TtpcNJ3XAzx3Gq8sWRzJaVajRs0uVxA2YAkdb1jm2YkPz4G6egUFAyA3n5vtEIZefPk5Wa4UXbKuS5fKkJWdgA==", - "dev": true, - "engines": { - "node": ">=10" - }, - "funding": { - "url": "https://github.com/sponsors/sindresorhus" - } - }, "node_modules/eslint/node_modules/eslint-scope": { "version": "7.1.1", "resolved": "https://registry.npmjs.org/eslint-scope/-/eslint-scope-7.1.1.tgz", @@ -1417,54 +1403,33 @@ "node": ">=4.0" } }, - "node_modules/eslint/node_modules/glob-parent": { - "version": "6.0.2", - "resolved": "https://registry.npmjs.org/glob-parent/-/glob-parent-6.0.2.tgz", - "integrity": "sha512-XxwI8EOhVQgWp6iDL+3b0r86f4d6AX6zSU55HfB4ydCEuXLXc5FcYeOu+nnGftS4TEju/11rt4KJPTMgbfmv4A==", + "node_modules/espree": { + "version": "9.3.3", + "resolved": "https://registry.npmjs.org/espree/-/espree-9.3.3.tgz", + "integrity": "sha512-ORs1Rt/uQTqUKjDdGCyrtYxbazf5umATSf/K4qxjmZHORR6HJk+2s/2Pqe+Kk49HHINC/xNIrGfgh8sZcll0ng==", "dev": true, "dependencies": { - "is-glob": "^4.0.3" + "acorn": "^8.8.0", + "acorn-jsx": "^5.3.2", + "eslint-visitor-keys": "^3.3.0" }, "engines": { - "node": ">=10.13.0" - } - }, - "node_modules/eslint/node_modules/js-yaml": { - "version": "4.1.0", - "resolved": "https://registry.npmjs.org/js-yaml/-/js-yaml-4.1.0.tgz", - "integrity": "sha512-wpxZs9NoxZaJESJGIZTyDEaYpl0FKSA+FB9aJiyemKhMwkxQg63h4T1KJgUGHpTqPDNRcmmYLugrRjJlBtWvRA==", - "dev": true, - "dependencies": { - "argparse": "^2.0.1" - }, - "bin": { - "js-yaml": "bin/js-yaml.js" - } - }, - "node_modules/eslint/node_modules/strip-ansi": { - "version": "6.0.1", - "resolved": "https://registry.npmjs.org/strip-ansi/-/strip-ansi-6.0.1.tgz", - "integrity": "sha512-Y38VPSHcqkFrCpFnQ9vuSXmquuv5oXOKpGeT6aGrr3o3Gc9AlVa6JBfUSOCnbxGGZF+/0ooI7KrPuUSztUdU5A==", - "dev": true, - "dependencies": { - "ansi-regex": "^5.0.1" + "node": "^12.22.0 || ^14.17.0 || >=16.0.0" }, - "engines": { - "node": ">=8" + "funding": { + "url": "https://opencollective.com/eslint" } }, - "node_modules/espree": { - "version": "9.3.1", - "resolved": "https://registry.npmjs.org/espree/-/espree-9.3.1.tgz", - "integrity": "sha512-bvdyLmJMfwkV3NCRl5ZhJf22zBFo1y8bYh3VYb+bfzqNB4Je68P2sSuXyuFquzWLebHpNd2/d5uv7yoP9ISnGQ==", - "dev": true, - "dependencies": { - "acorn": "^8.7.0", - "acorn-jsx": "^5.3.1", - "eslint-visitor-keys": "^3.3.0" + "node_modules/esprima": { + "version": "4.0.1", + "resolved": "https://registry.npmjs.org/esprima/-/esprima-4.0.1.tgz", + "integrity": "sha512-eGuFFw7Upda+g4p+QHvnW0RyTX/SVeJBDM/gCtMARO0cLuT2HcEKnTPvhjV6aGeqrCB/sbNop0Kszm0jsaWU4A==", + "bin": { + "esparse": "bin/esparse.js", + "esvalidate": "bin/esvalidate.js" }, "engines": { - "node": "^12.22.0 || ^14.17.0 || >=16.0.0" + "node": ">=4" } }, "node_modules/esquery": { @@ -1558,6 +1523,18 @@ "node": ">=8.6.0" } }, + "node_modules/fast-glob/node_modules/glob-parent": { + "version": "5.1.2", + "resolved": "https://registry.npmjs.org/glob-parent/-/glob-parent-5.1.2.tgz", + "integrity": "sha512-AOIgSQCepiJYwP3ARnGx+5VnTu2HBYdzbGP45eLw1vr3zB3vZLeyed1sC9hnbcOc9/SrMyM5RPQrkGz4aS9Zow==", + "dev": true, + "dependencies": { + "is-glob": "^4.0.1" + }, + "engines": { + "node": ">= 6" + } + }, "node_modules/fast-json-stable-stringify": { "version": "2.1.0", "resolved": "https://registry.npmjs.org/fast-json-stable-stringify/-/fast-json-stable-stringify-2.1.0.tgz", @@ -1567,9 +1544,22 @@ "node_modules/fast-levenshtein": { "version": "2.0.6", "resolved": "https://registry.npmjs.org/fast-levenshtein/-/fast-levenshtein-2.0.6.tgz", - "integrity": "sha1-PYpcZog6FqMMqGQ+hR8Zuqd5eRc=", + "integrity": "sha512-DCXu6Ifhqcks7TZKY3Hxp3y6qphY5SJZmrWMDrKcERSOXWQdMhU9Ig/PYrzyw/ul9jOIyh0N4M0tbC5hodg8dw==", "dev": true }, + "node_modules/fast-redact": { + "version": "3.1.1", + "resolved": "https://registry.npmjs.org/fast-redact/-/fast-redact-3.1.1.tgz", + "integrity": "sha512-odVmjC8x8jNeMZ3C+rPMESzXVSEU8tSWSHv9HFxP2mm89G/1WwqhrerJDQm9Zus8X6aoRgQDThKqptdNA6bt+A==", + "engines": { + "node": ">=6" + } + }, + "node_modules/fast-safe-stringify": { + "version": "2.1.1", + "resolved": "https://registry.npmjs.org/fast-safe-stringify/-/fast-safe-stringify-2.1.1.tgz", + "integrity": "sha512-W+KJc2dmILlPplD/H4K9l9LcAHAfPtP6BY84uVLXQ6Evcz9Lcg33Y2z1IVblT6xdY54PXYVHEv+0Wpq8Io6zkA==" + }, "node_modules/fastq": { "version": "1.13.0", "resolved": "https://registry.npmjs.org/fastq/-/fastq-1.13.0.tgz", @@ -1609,10 +1599,26 @@ "node": ">=8" } }, - "node_modules/flat-cache": { - "version": "3.0.4", - "resolved": "https://registry.npmjs.org/flat-cache/-/flat-cache-3.0.4.tgz", - "integrity": "sha512-dm9s5Pw7Jc0GvMYbshN6zchCA9RgQlzzEZX3vylR9IqFfS8XciblUXOKfW6SiuJ0e13eDYZoZV5wdrev7P3Nwg==", + "node_modules/find-up": { + "version": "5.0.0", + "resolved": "https://registry.npmjs.org/find-up/-/find-up-5.0.0.tgz", + "integrity": "sha512-78/PXT1wlLLDgTzDs7sjq9hzz0vXD+zn+7wypEe4fXQxCmdmqfGsEPQxmiCSQI3ajFV91bVSsvNtrJRiW6nGng==", + "dev": true, + "dependencies": { + "locate-path": "^6.0.0", + "path-exists": "^4.0.0" + }, + "engines": { + "node": ">=10" + }, + "funding": { + "url": "https://github.com/sponsors/sindresorhus" + } + }, + "node_modules/flat-cache": { + "version": "3.0.4", + "resolved": "https://registry.npmjs.org/flat-cache/-/flat-cache-3.0.4.tgz", + "integrity": "sha512-dm9s5Pw7Jc0GvMYbshN6zchCA9RgQlzzEZX3vylR9IqFfS8XciblUXOKfW6SiuJ0e13eDYZoZV5wdrev7P3Nwg==", "dev": true, "dependencies": { "flatted": "^3.1.0", @@ -1622,10 +1628,15 @@ "node": "^10.12.0 || >=12.0.0" } }, + "node_modules/flatstr": { + "version": "1.0.12", + "resolved": "https://registry.npmjs.org/flatstr/-/flatstr-1.0.12.tgz", + "integrity": "sha512-4zPxDyhCyiN2wIAtSLI6gc82/EjqZc1onI4Mz/l0pWrAlsSfYH/2ZIcU+e3oA2wDwbzIWNKwa23F8rh6+DRWkw==" + }, "node_modules/flatted": { - "version": "3.2.5", - "resolved": "https://registry.npmjs.org/flatted/-/flatted-3.2.5.tgz", - "integrity": "sha512-WIWGi2L3DyTUvUrwRKgGi9TwxQMUEqPOPQBVi71R96jZXJdFskXEmf54BoZaS1kknGODoIGASGEzBUYdyMCBJg==", + "version": "3.2.6", + "resolved": "https://registry.npmjs.org/flatted/-/flatted-3.2.6.tgz", + "integrity": "sha512-0sQoMh9s0BYsm+12Huy/rkKxVu4R1+r96YX5cG44rHV0pQ6iC3Q+mkoMFaGWObMFYQxCVT+ssG1ksneA2MI9KQ==", "dev": true }, "node_modules/follow-redirects": { @@ -1669,7 +1680,7 @@ "node_modules/fs.realpath": { "version": "1.0.0", "resolved": "https://registry.npmjs.org/fs.realpath/-/fs.realpath-1.0.0.tgz", - "integrity": "sha1-FQStJSMVjKpA20onh8sBQRmU6k8=" + "integrity": "sha512-OO0pH2lK6a0hZnAdau5ItzHPI6pUlvI7jMVnxUQRtw4owF2wk8lOSabtGDCTP4Ggrg2MbGnWO9X8K1t4+fGMDw==" }, "node_modules/fsevents": { "version": "2.3.2", @@ -1694,25 +1705,9 @@ "node_modules/functional-red-black-tree": { "version": "1.0.1", "resolved": "https://registry.npmjs.org/functional-red-black-tree/-/functional-red-black-tree-1.0.1.tgz", - "integrity": "sha1-GwqzvVU7Kg1jmdKcDj6gslIHgyc=", + "integrity": "sha512-dsKNQNdj6xA3T+QlADDA7mOSlX0qiMINjn0cgr+eGHGsbSHzTabcIogz2+p/iqP1Xs6EP/sS2SbqH+brGTbq0g==", "dev": true }, - "node_modules/gauge": { - "version": "2.7.4", - "resolved": "https://registry.npmjs.org/gauge/-/gauge-2.7.4.tgz", - "integrity": "sha1-LANAXHU4w51+s3sxcCLjJfsBi/c=", - "devOptional": true, - "dependencies": { - "aproba": "^1.0.3", - "console-control-strings": "^1.0.0", - "has-unicode": "^2.0.0", - "object-assign": "^4.1.0", - "signal-exit": "^3.0.0", - "string-width": "^1.0.1", - "strip-ansi": "^3.0.1", - "wide-align": "^1.1.0" - } - }, "node_modules/get-caller-file": { "version": "2.0.5", "resolved": "https://registry.npmjs.org/get-caller-file/-/get-caller-file-2.0.5.tgz", @@ -1724,18 +1719,18 @@ "node_modules/github-from-package": { "version": "0.0.0", "resolved": "https://registry.npmjs.org/github-from-package/-/github-from-package-0.0.0.tgz", - "integrity": "sha1-l/tdlr/eiXMxPyDoKI75oWf6ZM4=", + "integrity": "sha512-SyHy3T1v2NUXn29OsWdxmK6RwHD+vkj3v8en8AOBZ1wBQ/hCAQ5bAQTD02kW4W9tUp/3Qh6J8r9EvntiyCmOOw==", "devOptional": true }, "node_modules/glob": { - "version": "7.1.6", - "resolved": "https://registry.npmjs.org/glob/-/glob-7.1.6.tgz", - "integrity": "sha512-LwaxwyZ72Lk7vZINtNNrywX0ZuLyStrdDtabefZKAY5ZGJhVtgdznluResxNmPitE0SAO+O26sWTHeKSI2wMBA==", + "version": "7.2.3", + "resolved": "https://registry.npmjs.org/glob/-/glob-7.2.3.tgz", + "integrity": "sha512-nFR0zLpU2YCaRxwoCJvL6UvCH2JFyFVIvwTLsIf21AuHlMskA1hhTdk+LlYJtOlYt9v6dvszD2BGRqBL+iQK9Q==", "dependencies": { "fs.realpath": "^1.0.0", "inflight": "^1.0.4", "inherits": "2", - "minimatch": "^3.0.4", + "minimatch": "^3.1.1", "once": "^1.3.0", "path-is-absolute": "^1.0.0" }, @@ -1747,21 +1742,21 @@ } }, "node_modules/glob-parent": { - "version": "5.1.2", - "resolved": "https://registry.npmjs.org/glob-parent/-/glob-parent-5.1.2.tgz", - "integrity": "sha512-AOIgSQCepiJYwP3ARnGx+5VnTu2HBYdzbGP45eLw1vr3zB3vZLeyed1sC9hnbcOc9/SrMyM5RPQrkGz4aS9Zow==", + "version": "6.0.2", + "resolved": "https://registry.npmjs.org/glob-parent/-/glob-parent-6.0.2.tgz", + "integrity": "sha512-XxwI8EOhVQgWp6iDL+3b0r86f4d6AX6zSU55HfB4ydCEuXLXc5FcYeOu+nnGftS4TEju/11rt4KJPTMgbfmv4A==", "dev": true, "dependencies": { - "is-glob": "^4.0.1" + "is-glob": "^4.0.3" }, "engines": { - "node": ">= 6" + "node": ">=10.13.0" } }, "node_modules/globals": { - "version": "13.12.1", - "resolved": "https://registry.npmjs.org/globals/-/globals-13.12.1.tgz", - "integrity": "sha512-317dFlgY2pdJZ9rspXDks7073GpDmXdfbM3vYYp0HAMKGDh1FfWPleI2ljVNLQX5M5lXcAslTcPTrOrMEFOjyw==", + "version": "13.17.0", + "resolved": "https://registry.npmjs.org/globals/-/globals-13.17.0.tgz", + "integrity": "sha512-1C+6nQRb1GwGMKm2dH/E7enFAMxGTmGI7/dEdhy/DNelv85w9B72t3uc5frtMNXIbzrarJJ/lTCjcaZwbLJmyw==", "dev": true, "dependencies": { "type-fest": "^0.20.2" @@ -1794,14 +1789,20 @@ } }, "node_modules/graceful-fs": { - "version": "4.2.9", - "resolved": "https://registry.npmjs.org/graceful-fs/-/graceful-fs-4.2.9.tgz", - "integrity": "sha512-NtNxqUcXgpW2iMrfqSfR73Glt39K+BLwWsPs94yR63v45T0Wbej7eRmL5cWfwEgqXnmjQp3zaJTshdRW/qC2ZQ==" + "version": "4.2.10", + "resolved": "https://registry.npmjs.org/graceful-fs/-/graceful-fs-4.2.10.tgz", + "integrity": "sha512-9ByhssR2fPVsNZj478qUUbKfmL0+t5BDVyjShtyZZLiK7ZDAArFFfopyOTj0M05wE2tJPisA4iTnnXl2YoPvOA==" }, "node_modules/graceful-readlink": { "version": "1.0.1", "resolved": "https://registry.npmjs.org/graceful-readlink/-/graceful-readlink-1.0.1.tgz", - "integrity": "sha1-TK+tdrxi8C+gObL5Tpo906ORpyU=" + "integrity": "sha512-8tLu60LgxF6XpdbK8OW3FA+IfTNBn1ZHGHKF4KQbEeSkajYw5PlYJcKluntgegDPTg8UkHjpet1T82vk6TQ68w==" + }, + "node_modules/grapheme-splitter": { + "version": "1.0.4", + "resolved": "https://registry.npmjs.org/grapheme-splitter/-/grapheme-splitter-1.0.4.tgz", + "integrity": "sha512-bzh50DW9kTPM00T8y4o8vQg89Di9oLJVLW/KaOGIXJWP/iqCN6WKYkbNOF04vFLJhwcpYUh9ydh/+5vpOqV4YQ==", + "dev": true }, "node_modules/has": { "version": "1.0.3", @@ -1815,11 +1816,13 @@ "node": ">= 0.4.0" } }, - "node_modules/has-unicode": { - "version": "2.0.1", - "resolved": "https://registry.npmjs.org/has-unicode/-/has-unicode-2.0.1.tgz", - "integrity": "sha1-4Ob+aijPUROIVeCG0Wkedx3iqLk=", - "devOptional": true + "node_modules/has-flag": { + "version": "4.0.0", + "resolved": "https://registry.npmjs.org/has-flag/-/has-flag-4.0.0.tgz", + "integrity": "sha512-EykJT/Q1KjTWctppgIAgfSO0tKVuZUjhgMr17kqTumMl6Afv3EISleU7qZUzoXDFTAHTDC4NOoG/ZxU3EvlMPQ==", + "engines": { + "node": ">=8" + } }, "node_modules/highlight.js": { "version": "10.7.3", @@ -1876,7 +1879,7 @@ "node_modules/imurmurhash": { "version": "0.1.4", "resolved": "https://registry.npmjs.org/imurmurhash/-/imurmurhash-0.1.4.tgz", - "integrity": "sha1-khi5srkoojixPcT7a21XbyMUU+o=", + "integrity": "sha512-JmXMZ6wuvDmLiHEml9ykzqO6lwFbof0GG4IkcGaENdCRDDmMVnny7s5HsIgHCbaq0w2MyPhDqkhTUgS2LU2PHA==", "dev": true, "engines": { "node": ">=0.8.19" @@ -1885,7 +1888,7 @@ "node_modules/inflight": { "version": "1.0.6", "resolved": "https://registry.npmjs.org/inflight/-/inflight-1.0.6.tgz", - "integrity": "sha1-Sb1jMdfQLQwJvJEKEHW6gWW1bfk=", + "integrity": "sha512-k92I/b08q4wvFscXCLvqfsHCrjrF7yiXsQuIVvVE7N82W3+aqpzuUdBbfhWcy/FZR3/4IgflMgKLOsvPDrGCJA==", "dependencies": { "once": "^1.3.0", "wrappy": "1" @@ -1915,9 +1918,9 @@ } }, "node_modules/is-core-module": { - "version": "2.8.1", - "resolved": "https://registry.npmjs.org/is-core-module/-/is-core-module-2.8.1.tgz", - "integrity": "sha512-SdNCUs284hr40hFTFP6l0IfZ/RSrMXF3qgoRHd3/79unUTvrFO/JoXwkGm+5J/Oe3E/b5GsnG330uUNgRpu1PA==", + "version": "2.10.0", + "resolved": "https://registry.npmjs.org/is-core-module/-/is-core-module-2.10.0.tgz", + "integrity": "sha512-Erxj2n/LDAZ7H8WNJXd9tw38GYM3dv8rk8Zcs+jJuxYTW7sozH+SS8NtrSjVL1/vpLvWi1hxy96IzjJ3EHTJJg==", "dev": true, "dependencies": { "has": "^1.0.3" @@ -1929,22 +1932,18 @@ "node_modules/is-extglob": { "version": "2.1.1", "resolved": "https://registry.npmjs.org/is-extglob/-/is-extglob-2.1.1.tgz", - "integrity": "sha1-qIwCU1eR8C7TfHahueqXc8gz+MI=", + "integrity": "sha512-SbKbANkN603Vi4jEZv49LeVJMn4yGwsbzZworEoyEiutsN3nJYdbO36zfhGJ6QEDpOZIFkDtnq5JRxmvl3jsoQ==", "dev": true, "engines": { "node": ">=0.10.0" } }, "node_modules/is-fullwidth-code-point": { - "version": "1.0.0", - "resolved": "https://registry.npmjs.org/is-fullwidth-code-point/-/is-fullwidth-code-point-1.0.0.tgz", - "integrity": "sha1-754xOG8DGn8NZDr4L95QxFfvAMs=", - "devOptional": true, - "dependencies": { - "number-is-nan": "^1.0.0" - }, + "version": "3.0.0", + "resolved": "https://registry.npmjs.org/is-fullwidth-code-point/-/is-fullwidth-code-point-3.0.0.tgz", + "integrity": "sha512-zymm5+u+sCsSWyD9qNaejV3DFvhCKclKdizYaJUuHA83RLjb7nSuGnddCHGv0hk+KY7BMAlsWeK4Ueg6EV6XQg==", "engines": { - "node": ">=0.10.0" + "node": ">=8" } }, "node_modules/is-glob": { @@ -1969,17 +1968,44 @@ } }, "node_modules/isarray": { - "version": "1.0.0", - "resolved": "https://registry.npmjs.org/isarray/-/isarray-1.0.0.tgz", - "integrity": "sha1-u5NdSFgsuhaMBoNJV6VKPgcSTxE=", - "devOptional": true + "version": "0.0.1", + "resolved": "https://registry.npmjs.org/isarray/-/isarray-0.0.1.tgz", + "integrity": "sha512-D2S+3GLxWH+uhrNEcoh/fnmYeP8E8/zHl644d/jdA0g2uyXvy3sb0qxotE+ne0LtccHknQzWwZEzhak7oJ0COQ==", + "dev": true }, "node_modules/isexe": { "version": "2.0.0", "resolved": "https://registry.npmjs.org/isexe/-/isexe-2.0.0.tgz", - "integrity": "sha1-6PvzdNxVb/iUehDcsFctYz8s+hA=", + "integrity": "sha512-RHxMLp9lnKHGHRng9QFhRCMbYAcVpn69smSGcq3f36xjgVVWThj4qqLbTLlq7Ssj8B+fIQ1EuCEGI2lKsyQeIw==", "dev": true }, + "node_modules/jmespath": { + "version": "0.15.0", + "resolved": "https://registry.npmjs.org/jmespath/-/jmespath-0.15.0.tgz", + "integrity": "sha512-+kHj8HXArPfpPEKGLZ+kB5ONRTCiGQXo8RQYL0hH8t6pWXUBBK5KkkQmTNOwKK4LEsd0yTsgtjJVm4UBSZea4w==", + "engines": { + "node": ">= 0.6.0" + } + }, + "node_modules/joycon": { + "version": "2.2.5", + "resolved": "https://registry.npmjs.org/joycon/-/joycon-2.2.5.tgz", + "integrity": "sha512-YqvUxoOcVPnCp0VU1/56f+iKSdvIRJYPznH22BdXV3xMk75SFXhWeJkZ8C9XxUWt1b5x2X1SxuFygW1U0FmkEQ==", + "engines": { + "node": ">=6" + } + }, + "node_modules/js-yaml": { + "version": "4.1.0", + "resolved": "https://registry.npmjs.org/js-yaml/-/js-yaml-4.1.0.tgz", + "integrity": "sha512-wpxZs9NoxZaJESJGIZTyDEaYpl0FKSA+FB9aJiyemKhMwkxQg63h4T1KJgUGHpTqPDNRcmmYLugrRjJlBtWvRA==", + "dependencies": { + "argparse": "^2.0.1" + }, + "bin": { + "js-yaml": "bin/js-yaml.js" + } + }, "node_modules/json-schema-traverse": { "version": "0.4.1", "resolved": "https://registry.npmjs.org/json-schema-traverse/-/json-schema-traverse-0.4.1.tgz", @@ -1989,16 +2015,13 @@ "node_modules/json-stable-stringify-without-jsonify": { "version": "1.0.1", "resolved": "https://registry.npmjs.org/json-stable-stringify-without-jsonify/-/json-stable-stringify-without-jsonify-1.0.1.tgz", - "integrity": "sha1-nbe1lJatPzz+8wp1FC0tkwrXJlE=", + "integrity": "sha512-Bdboy+l7tA3OGW6FjyFHWkP5LuByj1Tk33Ljyq0axyzdk9//JSi2u3fP1QSmd1KNwq6VOKYGlAu87CisVir6Pw==", "dev": true }, "node_modules/json5": { - "version": "2.2.0", - "resolved": "https://registry.npmjs.org/json5/-/json5-2.2.0.tgz", - "integrity": "sha512-f+8cldu7X/y7RAJurMEJmdoKXGB/X550w2Nr3tTbezL6RwEE/iMcm+tZnXeoZtKuOq6ft8+CqzEkrIgx1fPoQA==", - "dependencies": { - "minimist": "^1.2.5" - }, + "version": "2.2.1", + "resolved": "https://registry.npmjs.org/json5/-/json5-2.2.1.tgz", + "integrity": "sha512-1hqLFMSrGHRHxav9q9gNjJ5EXznIxGVO09xQRrwplcS8qs28pZ8s8hupZAmqDwZUmVZ2Qb2jnyPOWcDH8m8dlA==", "bin": { "json5": "lib/cli.js" }, @@ -2006,6 +2029,14 @@ "node": ">=6" } }, + "node_modules/leven": { + "version": "2.1.0", + "resolved": "https://registry.npmjs.org/leven/-/leven-2.1.0.tgz", + "integrity": "sha512-nvVPLpIHUxCUoRLrFqTgSxXJ614d8AgQoWl7zPe/2VadE8+1dpU3LBhowRuBAcuwruWtOdD8oYC9jDNJjXDPyA==", + "engines": { + "node": ">=0.10.0" + } + }, "node_modules/levn": { "version": "0.4.1", "resolved": "https://registry.npmjs.org/levn/-/levn-0.4.1.tgz", @@ -2019,6 +2050,21 @@ "node": ">= 0.8.0" } }, + "node_modules/locate-path": { + "version": "6.0.0", + "resolved": "https://registry.npmjs.org/locate-path/-/locate-path-6.0.0.tgz", + "integrity": "sha512-iPZK6eYjbxRu3uB4/WZ3EsEIMJFMqAoopl3R+zuq0UjcAm/MO6KCweDgPfP3elTztoKP3KtnVHxTn2NHBSDVUw==", + "dev": true, + "dependencies": { + "p-locate": "^5.0.0" + }, + "engines": { + "node": ">=10" + }, + "funding": { + "url": "https://github.com/sponsors/sindresorhus" + } + }, "node_modules/lodash.merge": { "version": "4.6.2", "resolved": "https://registry.npmjs.org/lodash.merge/-/lodash.merge-4.6.2.tgz", @@ -2053,13 +2099,13 @@ } }, "node_modules/micromatch": { - "version": "4.0.4", - "resolved": "https://registry.npmjs.org/micromatch/-/micromatch-4.0.4.tgz", - "integrity": "sha512-pRmzw/XUcwXGpD9aI9q/0XOwLNygjETJ8y0ao0wdqprrzDa4YnxLcz7fQRZr8voh8V10kGhABbNcHVk5wHgWwg==", + "version": "4.0.5", + "resolved": "https://registry.npmjs.org/micromatch/-/micromatch-4.0.5.tgz", + "integrity": "sha512-DMy+ERcEW2q8Z2Po+WNXuw3c5YaUSFjAO5GsJqfEl7UjvtIuFKO6ZrKvcItdy98dwFI2N1tg3zNIdKaQT+aNdA==", "dev": true, "dependencies": { - "braces": "^3.0.1", - "picomatch": "^2.2.3" + "braces": "^3.0.2", + "picomatch": "^2.3.1" }, "engines": { "node": ">=8.6" @@ -2097,9 +2143,9 @@ } }, "node_modules/minimatch": { - "version": "3.0.4", - "resolved": "https://registry.npmjs.org/minimatch/-/minimatch-3.0.4.tgz", - "integrity": "sha512-yJHVQEhyqPLUTgt9B83PXu6W3rx4MvvHvSUvToogpwoGDOUQ+yDrR0HRot+yOCdCO7u4hX3pWft6kWBBcqh0UA==", + "version": "3.1.2", + "resolved": "https://registry.npmjs.org/minimatch/-/minimatch-3.1.2.tgz", + "integrity": "sha512-J7p63hRiAjw1NDEww1W7i37+ByIrOWO5XQQAzZ3VOcL0PNybwpfmV/N05zFAzwQ9USyEcX6t3UO+K5aqBQOIHw==", "dependencies": { "brace-expansion": "^1.1.7" }, @@ -2110,7 +2156,8 @@ "node_modules/minimist": { "version": "1.2.6", "resolved": "https://registry.npmjs.org/minimist/-/minimist-1.2.6.tgz", - "integrity": "sha512-Jsjnk4bw3YJqYzbdyBiNsPWHPfO++UGG749Cxs6peCu5Xg4nrena6OVxOYxrQTqww0Jmwt+Ref8rggumkTLz9Q==" + "integrity": "sha512-Jsjnk4bw3YJqYzbdyBiNsPWHPfO++UGG749Cxs6peCu5Xg4nrena6OVxOYxrQTqww0Jmwt+Ref8rggumkTLz9Q==", + "devOptional": true }, "node_modules/mkdirp": { "version": "1.0.4", @@ -2129,6 +2176,14 @@ "integrity": "sha512-gKLcREMhtuZRwRAfqP3RFW+TK4JqApVBtOIftVgjuABpAtpxhPGaDcfvbhNvD0B8iD1oUr/txX35NjcaY6Ns/A==", "devOptional": true }, + "node_modules/mri": { + "version": "1.1.4", + "resolved": "https://registry.npmjs.org/mri/-/mri-1.1.4.tgz", + "integrity": "sha512-6y7IjGPm8AzlvoUrwAaw1tLnUBudaS3752vcd8JtrpGGQn+rXIe63LFVHm/YMwtqAuh+LJPCFdlLYPWM1nYn6w==", + "engines": { + "node": ">=4" + } + }, "node_modules/ms": { "version": "2.1.2", "resolved": "https://registry.npmjs.org/ms/-/ms-2.1.2.tgz", @@ -2153,13 +2208,13 @@ "node_modules/natural-compare": { "version": "1.4.0", "resolved": "https://registry.npmjs.org/natural-compare/-/natural-compare-1.4.0.tgz", - "integrity": "sha1-Sr6/7tdUHywnrPspvbvRXI1bpPc=", + "integrity": "sha512-OWND8ei3VtNC9h7V60qff3SVobHr996CTwgxubgyQYEpg290h9J0buyECNNJexkFm5sOajh5G116RYA1c8ZMSw==", "dev": true }, "node_modules/node-abi": { - "version": "3.8.0", - "resolved": "https://registry.npmjs.org/node-abi/-/node-abi-3.8.0.tgz", - "integrity": "sha512-tzua9qWWi7iW4I42vUPKM+SfaF0vQSLAm4yO5J83mSwB7GeoWrDKC/K+8YCnYNwqP5duwazbw2X9l4m8SC2cUw==", + "version": "3.24.0", + "resolved": "https://registry.npmjs.org/node-abi/-/node-abi-3.24.0.tgz", + "integrity": "sha512-YPG3Co0luSu6GwOBsmIdGW6Wx0NyNDLg/hriIyDllVsNwnI6UeqaWShxC3lbH4LtEQUgoLP3XR1ndXiDAWvmRw==", "devOptional": true, "dependencies": { "semver": "^7.3.5" @@ -2171,37 +2226,13 @@ "node_modules/noms": { "version": "0.0.0", "resolved": "https://registry.npmjs.org/noms/-/noms-0.0.0.tgz", - "integrity": "sha1-2o69nzr51nYJGbJ9nNyAkqczKFk=", + "integrity": "sha512-lNDU9VJaOPxUmXcLb+HQFeUgQQPtMI24Gt6hgfuMHRJgMRHMF/qZ4HJD3GDru4sSw9IQl2jPjAYnQrdIeLbwow==", "dev": true, "dependencies": { "inherits": "^2.0.1", "readable-stream": "~1.0.31" } }, - "node_modules/noms/node_modules/isarray": { - "version": "0.0.1", - "resolved": "https://registry.npmjs.org/isarray/-/isarray-0.0.1.tgz", - "integrity": "sha1-ihis/Kmo9Bd+Cav8YDiTmwXR7t8=", - "dev": true - }, - "node_modules/noms/node_modules/readable-stream": { - "version": "1.0.34", - "resolved": "https://registry.npmjs.org/readable-stream/-/readable-stream-1.0.34.tgz", - "integrity": "sha1-Elgg40vIQtLyqq+v5MKRbuMsFXw=", - "dev": true, - "dependencies": { - "core-util-is": "~1.0.0", - "inherits": "~2.0.1", - "isarray": "0.0.1", - "string_decoder": "~0.10.x" - } - }, - "node_modules/noms/node_modules/string_decoder": { - "version": "0.10.31", - "resolved": "https://registry.npmjs.org/string_decoder/-/string_decoder-0.10.31.tgz", - "integrity": "sha1-YuIDvEF2bGwoyfyEMB2rHFMQ+pQ=", - "dev": true - }, "node_modules/normalize-path": { "version": "3.0.0", "resolved": "https://registry.npmjs.org/normalize-path/-/normalize-path-3.0.0.tgz", @@ -2211,31 +2242,10 @@ "node": ">=0.10.0" } }, - "node_modules/npmlog": { - "version": "4.1.2", - "resolved": "https://registry.npmjs.org/npmlog/-/npmlog-4.1.2.tgz", - "integrity": "sha512-2uUqazuKlTaSI/dC8AzicUck7+IrEaOnN/e0jd3Xtt1KcGpwx30v50mL7oPyr/h9bL3E4aZccVwpwP+5W9Vjkg==", - "devOptional": true, - "dependencies": { - "are-we-there-yet": "~1.1.2", - "console-control-strings": "~1.1.0", - "gauge": "~2.7.3", - "set-blocking": "~2.0.0" - } - }, - "node_modules/number-is-nan": { - "version": "1.0.1", - "resolved": "https://registry.npmjs.org/number-is-nan/-/number-is-nan-1.0.1.tgz", - "integrity": "sha1-CXtgK1NCKlIsGvuHkDGDNpQaAR0=", - "devOptional": true, - "engines": { - "node": ">=0.10.0" - } - }, "node_modules/object-assign": { "version": "4.1.1", "resolved": "https://registry.npmjs.org/object-assign/-/object-assign-4.1.1.tgz", - "integrity": "sha1-IQmtx5ZYh8/AXLvUQsrIv7s2CGM=", + "integrity": "sha512-rJgTQnkUnH1sFw8yT6VSU3zD3sWmu6sZhIseY8VX+GRu3P6F7Fu+JNDoXfklElbLJSnc3FUQHVe4cU5hj+BcUg==", "engines": { "node": ">=0.10.0" } @@ -2243,7 +2253,7 @@ "node_modules/once": { "version": "1.4.0", "resolved": "https://registry.npmjs.org/once/-/once-1.4.0.tgz", - "integrity": "sha1-WDsap3WWHUsROsF9nFC6753Xa9E=", + "integrity": "sha512-lNaJgI+2Q5URQBkccEKHTQOPaXdUxnZZElQTZY0MFUAuaEqe1E+Nyvgdz/aIyNi6Z9MzO5dv1H8n58/GELp3+w==", "dependencies": { "wrappy": "1" } @@ -2265,6 +2275,36 @@ "node": ">= 0.8.0" } }, + "node_modules/p-limit": { + "version": "3.1.0", + "resolved": "https://registry.npmjs.org/p-limit/-/p-limit-3.1.0.tgz", + "integrity": "sha512-TYOanM3wGwNGsZN2cVTYPArw454xnXj5qmWF1bEoAc4+cU/ol7GVh7odevjp1FNHduHc3KZMcFduxU5Xc6uJRQ==", + "dev": true, + "dependencies": { + "yocto-queue": "^0.1.0" + }, + "engines": { + "node": ">=10" + }, + "funding": { + "url": "https://github.com/sponsors/sindresorhus" + } + }, + "node_modules/p-locate": { + "version": "5.0.0", + "resolved": "https://registry.npmjs.org/p-locate/-/p-locate-5.0.0.tgz", + "integrity": "sha512-LaNjtRWUBY++zB5nE/NwcaoMylSPk+S+ZHNB1TzdbMJMny6dynpAGt7X/tl/QYq3TIeE6nxHppbo2LGymrG5Pw==", + "dev": true, + "dependencies": { + "p-limit": "^3.0.2" + }, + "engines": { + "node": ">=10" + }, + "funding": { + "url": "https://github.com/sponsors/sindresorhus" + } + }, "node_modules/parent-module": { "version": "1.0.1", "resolved": "https://registry.npmjs.org/parent-module/-/parent-module-1.0.1.tgz", @@ -2295,10 +2335,19 @@ "resolved": "https://registry.npmjs.org/parse5/-/parse5-6.0.1.tgz", "integrity": "sha512-Ofn/CTFzRGTTxwpNEs9PP93gXShHcTq255nzRYSKe8AkVpZY7e1fpmTfOyoIvjP5HG7Z2ZM7VS9PPhQGW2pOpw==" }, + "node_modules/path-exists": { + "version": "4.0.0", + "resolved": "https://registry.npmjs.org/path-exists/-/path-exists-4.0.0.tgz", + "integrity": "sha512-ak9Qy5Q7jYb2Wwcey5Fpvg2KoAc/ZIhLSLOSBmRmygPsGwkVVt0fZa0qrtMz+m6tJTAHfZQ8FnmB4MG4LWy7/w==", + "dev": true, + "engines": { + "node": ">=8" + } + }, "node_modules/path-is-absolute": { "version": "1.0.1", "resolved": "https://registry.npmjs.org/path-is-absolute/-/path-is-absolute-1.0.1.tgz", - "integrity": "sha1-F0uSaHNVNP+8es5r9TpanhtcX18=", + "integrity": "sha512-AVbw3UJ2e9bq64vSaS9Am0fje1Pa8pbGqTTsmXfaIiMpnr5DlDhfJOuLj9Sf95ZPVDAUerDfEk88MPmPe7UCQg==", "engines": { "node": ">=0.10.0" } @@ -2339,10 +2388,84 @@ "url": "https://github.com/sponsors/jonschlinkert" } }, + "node_modules/pino": { + "version": "6.14.0", + "resolved": "https://registry.npmjs.org/pino/-/pino-6.14.0.tgz", + "integrity": "sha512-iuhEDel3Z3hF9Jfe44DPXR8l07bhjuFY3GMHIXbjnY9XcafbyDDwl2sN2vw2GjMPf5Nkoe+OFao7ffn9SXaKDg==", + "dependencies": { + "fast-redact": "^3.0.0", + "fast-safe-stringify": "^2.0.8", + "flatstr": "^1.0.12", + "pino-std-serializers": "^3.1.0", + "process-warning": "^1.0.0", + "quick-format-unescaped": "^4.0.3", + "sonic-boom": "^1.0.2" + }, + "bin": { + "pino": "bin.js" + } + }, + "node_modules/pino-pretty": { + "version": "4.8.0", + "resolved": "https://registry.npmjs.org/pino-pretty/-/pino-pretty-4.8.0.tgz", + "integrity": "sha512-mhQfHG4rw5ZFpWL44m0Utjo4GC2+HMfdNvxyA8lLw0sIqn6fCf7uQe6dPckUcW/obly+OQHD7B/MTso6LNizYw==", + "dependencies": { + "@hapi/bourne": "^2.0.0", + "args": "^5.0.1", + "chalk": "^4.0.0", + "dateformat": "^4.5.1", + "fast-safe-stringify": "^2.0.7", + "jmespath": "^0.15.0", + "joycon": "^2.2.5", + "pump": "^3.0.0", + "readable-stream": "^3.6.0", + "rfdc": "^1.3.0", + "split2": "^3.1.1", + "strip-json-comments": "^3.1.1" + }, + "bin": { + "pino-pretty": "bin.js" + } + }, + "node_modules/pino-pretty/node_modules/readable-stream": { + "version": "3.6.0", + "resolved": "https://registry.npmjs.org/readable-stream/-/readable-stream-3.6.0.tgz", + "integrity": "sha512-BViHy7LKeTz4oNnkcLJ+lVSL6vpiFeX6/d3oSH8zCW7UxP2onchk+vTGB143xuFjHS3deTgkKoXXymXqymiIdA==", + "dependencies": { + "inherits": "^2.0.3", + "string_decoder": "^1.1.1", + "util-deprecate": "^1.0.1" + }, + "engines": { + "node": ">= 6" + } + }, + "node_modules/pino-pretty/node_modules/string_decoder": { + "version": "1.3.0", + "resolved": "https://registry.npmjs.org/string_decoder/-/string_decoder-1.3.0.tgz", + "integrity": "sha512-hkRX8U1WjJFd8LsDJ2yQ/wWWxaopEsABU1XfkM8A+j0+85JAGppt16cr1Whg6KIbb4okU6Mql6BOj+uup/wKeA==", + "dependencies": { + "safe-buffer": "~5.2.0" + } + }, + "node_modules/pino-std-serializers": { + "version": "3.2.0", + "resolved": "https://registry.npmjs.org/pino-std-serializers/-/pino-std-serializers-3.2.0.tgz", + "integrity": "sha512-EqX4pwDPrt3MuOAAUBMU0Tk5kR/YcCM5fNPEzgCO2zJ5HfX0vbiH9HbJglnyeQsN96Kznae6MWD47pZB5avTrg==" + }, + "node_modules/pino/node_modules/sonic-boom": { + "version": "1.4.1", + "resolved": "https://registry.npmjs.org/sonic-boom/-/sonic-boom-1.4.1.tgz", + "integrity": "sha512-LRHh/A8tpW7ru89lrlkU4AszXt1dbwSjVWguGrmlxE7tawVmDBlI1PILMkXAxJTwqhgsEeTHzj36D5CmHgQmNg==", + "dependencies": { + "atomic-sleep": "^1.0.0", + "flatstr": "^1.0.12" + } + }, "node_modules/prebuild-install": { - "version": "7.0.1", - "resolved": "https://registry.npmjs.org/prebuild-install/-/prebuild-install-7.0.1.tgz", - "integrity": "sha512-QBSab31WqkyxpnMWQxubYAHR5S9B2+r81ucocew34Fkl98FhvKIF50jIJnNOBmAZfyNV7vE5T6gd3hTVWgY6tg==", + "version": "7.1.1", + "resolved": "https://registry.npmjs.org/prebuild-install/-/prebuild-install-7.1.1.tgz", + "integrity": "sha512-jAXscXWMcCK8GgCoHOfIr0ODh5ai8mj63L2nWrjuAgXE6tDyYGnx4/8o/rCgU+B4JSyZBKbeZqzhtwtC3ovxjw==", "devOptional": true, "dependencies": { "detect-libc": "^2.0.0", @@ -2352,7 +2475,6 @@ "mkdirp-classic": "^0.5.3", "napi-build-utils": "^1.0.1", "node-abi": "^3.3.0", - "npmlog": "^4.0.1", "pump": "^3.0.0", "rc": "^1.2.7", "simple-get": "^4.0.0", @@ -2366,15 +2488,6 @@ "node": ">=10" } }, - "node_modules/prebuild-install/node_modules/detect-libc": { - "version": "2.0.1", - "resolved": "https://registry.npmjs.org/detect-libc/-/detect-libc-2.0.1.tgz", - "integrity": "sha512-463v3ZeIrcWtdgIg6vI6XUncguvr2TnGl4SzDXinkt9mSLpBJKXT3mW6xT3VQdDN11+WVs29pgvivTc4Lp8v+w==", - "devOptional": true, - "engines": { - "node": ">=8" - } - }, "node_modules/prelude-ls": { "version": "1.2.1", "resolved": "https://registry.npmjs.org/prelude-ls/-/prelude-ls-1.2.1.tgz", @@ -2388,13 +2501,17 @@ "version": "2.0.1", "resolved": "https://registry.npmjs.org/process-nextick-args/-/process-nextick-args-2.0.1.tgz", "integrity": "sha512-3ouUOpQhtgrbOa17J7+uxOTpITYWaGP7/AhoR3+A+/1e9skrzelGi/dXzEYyvbxubEF6Wn2ypscTKiKJFFn1ag==", - "devOptional": true + "dev": true + }, + "node_modules/process-warning": { + "version": "1.0.0", + "resolved": "https://registry.npmjs.org/process-warning/-/process-warning-1.0.0.tgz", + "integrity": "sha512-du4wfLyj4yCZq1VupnVSZmRsPJsNuxoDQFdCFHLaYiEbFBD7QE0a+I4D7hOxrVnh78QE/YipFAj9lXHiXocV+Q==" }, "node_modules/pump": { "version": "3.0.0", "resolved": "https://registry.npmjs.org/pump/-/pump-3.0.0.tgz", "integrity": "sha512-LwZy+p3SFs1Pytd/jYct4wpv49HiYCqd9Rlc5ZVdk0V+8Yzv6jR5Blk3TRmPL1ft69TxP0IMZGJ+WPFU2BFhww==", - "devOptional": true, "dependencies": { "end-of-stream": "^1.1.0", "once": "^1.3.1" @@ -2429,6 +2546,11 @@ } ] }, + "node_modules/quick-format-unescaped": { + "version": "4.0.4", + "resolved": "https://registry.npmjs.org/quick-format-unescaped/-/quick-format-unescaped-4.0.4.tgz", + "integrity": "sha512-tYC1Q1hgyRuHgloV/YXs2w15unPVh8qfu/qCTfhTYamaw7fyhumKa2yGpdSo87vY32rIclj+4fWYQXUMs9EHvg==" + }, "node_modules/rc": { "version": "1.2.8", "resolved": "https://registry.npmjs.org/rc/-/rc-1.2.8.tgz", @@ -2447,25 +2569,22 @@ "node_modules/rc/node_modules/strip-json-comments": { "version": "2.0.1", "resolved": "https://registry.npmjs.org/strip-json-comments/-/strip-json-comments-2.0.1.tgz", - "integrity": "sha1-PFMZQukIwml8DsNEhYwobHygpgo=", + "integrity": "sha512-4gB8na07fecVVkOI6Rs4e7T6NOTki5EmL7TUduTs6bu3EdnSycntVJ4re8kgZA+wx9IueI2Y11bfbgwtzuE0KQ==", "devOptional": true, "engines": { "node": ">=0.10.0" } }, "node_modules/readable-stream": { - "version": "2.3.7", - "resolved": "https://registry.npmjs.org/readable-stream/-/readable-stream-2.3.7.tgz", - "integrity": "sha512-Ebho8K4jIbHAxnuxi7o42OrZgF/ZTNcsZj6nRKyUmkhLFq8CHItp/fy6hQZuZmP/n3yZ9VBUbp4zz/mX8hmYPw==", - "devOptional": true, + "version": "1.0.34", + "resolved": "https://registry.npmjs.org/readable-stream/-/readable-stream-1.0.34.tgz", + "integrity": "sha512-ok1qVCJuRkNmvebYikljxJA/UEsKwLl2nI1OmaqAu4/UE+h0wKCHok4XkL/gvi39OacXvw59RJUOFUkDib2rHg==", + "dev": true, "dependencies": { "core-util-is": "~1.0.0", - "inherits": "~2.0.3", - "isarray": "~1.0.0", - "process-nextick-args": "~2.0.0", - "safe-buffer": "~5.1.1", - "string_decoder": "~1.1.1", - "util-deprecate": "~1.0.1" + "inherits": "~2.0.1", + "isarray": "0.0.1", + "string_decoder": "~0.10.x" } }, "node_modules/readdirp": { @@ -2500,18 +2619,18 @@ "node_modules/require-directory": { "version": "2.1.1", "resolved": "https://registry.npmjs.org/require-directory/-/require-directory-2.1.1.tgz", - "integrity": "sha1-jGStX9MNqxyXbiNE/+f3kqam30I=", + "integrity": "sha512-fGxEI7+wsG9xrvdjsrlmL22OMTTiHRwAMroiEeMgq8gzoLC/PQr7RsRDSTLUg/bZAZtF+TVIkHc6/4RIKrui+Q==", "engines": { "node": ">=0.10.0" } }, "node_modules/resolve": { - "version": "1.22.0", - "resolved": "https://registry.npmjs.org/resolve/-/resolve-1.22.0.tgz", - "integrity": "sha512-Hhtrw0nLeSrFQ7phPp4OOcVjLPIeMnRlr5mcnVuMe7M/7eBn98A3hmFRLoFo3DLZkivSYwhRUJTyPyWAk56WLw==", + "version": "1.22.1", + "resolved": "https://registry.npmjs.org/resolve/-/resolve-1.22.1.tgz", + "integrity": "sha512-nBpuuYuY5jFsli/JIs1oldw6fOQCBioohqWZg/2hiaOybXOft4lonv85uDOKXdf8rhyK159cxU5cDcK/NKk8zw==", "dev": true, "dependencies": { - "is-core-module": "^2.8.1", + "is-core-module": "^2.9.0", "path-parse": "^1.0.7", "supports-preserve-symlinks-flag": "^1.0.0" }, @@ -2541,6 +2660,11 @@ "node": ">=0.10.0" } }, + "node_modules/rfdc": { + "version": "1.3.0", + "resolved": "https://registry.npmjs.org/rfdc/-/rfdc-1.3.0.tgz", + "integrity": "sha512-V2hovdzFbOi77/WajaSMXk2OLm+xNIeQdMMuB7icj7bk6zi2F8GGAxigcnDFpJHbNyNcgyJDiP+8nOrY5cZGrA==" + }, "node_modules/rimraf": { "version": "3.0.2", "resolved": "https://registry.npmjs.org/rimraf/-/rimraf-3.0.2.tgz", @@ -2580,9 +2704,23 @@ } }, "node_modules/safe-buffer": { - "version": "5.1.2", - "resolved": "https://registry.npmjs.org/safe-buffer/-/safe-buffer-5.1.2.tgz", - "integrity": "sha512-Gd2UZBJDkXlY7GbJxfsE8/nvKkUEU1G38c1siN6QP6a9PT9MmHB8GnpscSmMJSoF8LOIrt8ud/wPtojys4G6+g==" + "version": "5.2.1", + "resolved": "https://registry.npmjs.org/safe-buffer/-/safe-buffer-5.2.1.tgz", + "integrity": "sha512-rp3So07KcdmmKbGvgaNxQSJr7bGVSVk5S9Eq1F+ppbRo70+YeaDxkw5Dd8NPN+GD6bjnYm2VuPuCXmpuYvmCXQ==", + "funding": [ + { + "type": "github", + "url": "https://github.com/sponsors/feross" + }, + { + "type": "patreon", + "url": "https://www.patreon.com/feross" + }, + { + "type": "consulting", + "url": "https://feross.org/support" + } + ] }, "node_modules/sax": { "version": "1.2.4", @@ -2590,9 +2728,9 @@ "integrity": "sha512-NqVDv9TpANUjFm0N8uM5GxL36UgKi9/atZw+x7YFnQ8ckwFGKrl4xX4yWtrey3UJm5nP1kUbnYgLopqWNSRhWw==" }, "node_modules/semver": { - "version": "7.3.5", - "resolved": "https://registry.npmjs.org/semver/-/semver-7.3.5.tgz", - "integrity": "sha512-PoeGJYh8HK4BTO/a9Tf6ZG3veo/A7ZVsYrSA6J8ny9nb3B1VrpkuN+z9OE5wfE5p6H4LchYZsegiQgbJD94ZFQ==", + "version": "7.3.7", + "resolved": "https://registry.npmjs.org/semver/-/semver-7.3.7.tgz", + "integrity": "sha512-QlYTucUYOews+WeEujDoEGziz4K6c47V/Bd+LjSSYcA94p+DmINdf7ncaUinThfvZyu13lN9OY1XDxt8C0Tw0g==", "devOptional": true, "dependencies": { "lru-cache": "^6.0.0" @@ -2604,12 +2742,6 @@ "node": ">=10" } }, - "node_modules/set-blocking": { - "version": "2.0.0", - "resolved": "https://registry.npmjs.org/set-blocking/-/set-blocking-2.0.0.tgz", - "integrity": "sha1-BF+XgtARrppoA93TgrJDkrPYkPc=", - "devOptional": true - }, "node_modules/sha.js": { "version": "2.4.11", "resolved": "https://registry.npmjs.org/sha.js/-/sha.js-2.4.11.tgz", @@ -2643,12 +2775,6 @@ "node": ">=8" } }, - "node_modules/signal-exit": { - "version": "3.0.3", - "resolved": "https://registry.npmjs.org/signal-exit/-/signal-exit-3.0.3.tgz", - "integrity": "sha512-VUJ49FC8U1OxwZLxIbTTrDvLnf/6TDgxZcK8wxR8zs13xpx7xbG60ndBlhNrFi2EMuFRoeDoJO7wthSLq42EjA==", - "devOptional": true - }, "node_modules/simple-concat": { "version": "1.0.1", "resolved": "https://registry.npmjs.org/simple-concat/-/simple-concat-1.0.1.tgz", @@ -2703,6 +2829,14 @@ "node": ">=8" } }, + "node_modules/sonic-boom": { + "version": "2.8.0", + "resolved": "https://registry.npmjs.org/sonic-boom/-/sonic-boom-2.8.0.tgz", + "integrity": "sha512-kuonw1YOYYNOve5iHdSahXPOK49GqwA+LZhI6Wz/l0rP57iKyXXIHaRagOBHAPmGwJC6od2Z9zgvZ5loSgMlVg==", + "dependencies": { + "atomic-sleep": "^1.0.0" + } + }, "node_modules/source-map": { "version": "0.6.1", "resolved": "https://registry.npmjs.org/source-map/-/source-map-0.6.1.tgz", @@ -2722,50 +2856,79 @@ "source-map": "^0.6.0" } }, - "node_modules/sqlite": { - "version": "4.0.25", - "resolved": "https://registry.npmjs.org/sqlite/-/sqlite-4.0.25.tgz", - "integrity": "sha512-gqCEcLF8FOTeW/na3SRYWLQkw2jZXgVj1DdgRJbm0jvrhnUgBIuNDUUm649AnBNDNHhI5XskwT8dvc8vearRLQ==" + "node_modules/split2": { + "version": "3.2.2", + "resolved": "https://registry.npmjs.org/split2/-/split2-3.2.2.tgz", + "integrity": "sha512-9NThjpgZnifTkJpzTZ7Eue85S49QwpNhZTq6GRJwObb6jnLFNGB7Qm73V5HewTROPyxD0C29xqmaI68bQtV+hg==", + "dependencies": { + "readable-stream": "^3.0.0" + } }, - "node_modules/string_decoder": { - "version": "1.1.1", - "resolved": "https://registry.npmjs.org/string_decoder/-/string_decoder-1.1.1.tgz", - "integrity": "sha512-n/ShnvDi6FHbbVfviro+WojiFzv+s8MPMHBczVePfUpDJLwoLT0ht1l4YwBCbi8pJAveEEdnkHyPyTP/mzRfwg==", - "devOptional": true, + "node_modules/split2/node_modules/readable-stream": { + "version": "3.6.0", + "resolved": "https://registry.npmjs.org/readable-stream/-/readable-stream-3.6.0.tgz", + "integrity": "sha512-BViHy7LKeTz4oNnkcLJ+lVSL6vpiFeX6/d3oSH8zCW7UxP2onchk+vTGB143xuFjHS3deTgkKoXXymXqymiIdA==", "dependencies": { - "safe-buffer": "~5.1.0" + "inherits": "^2.0.3", + "string_decoder": "^1.1.1", + "util-deprecate": "^1.0.1" + }, + "engines": { + "node": ">= 6" + } + }, + "node_modules/split2/node_modules/string_decoder": { + "version": "1.3.0", + "resolved": "https://registry.npmjs.org/string_decoder/-/string_decoder-1.3.0.tgz", + "integrity": "sha512-hkRX8U1WjJFd8LsDJ2yQ/wWWxaopEsABU1XfkM8A+j0+85JAGppt16cr1Whg6KIbb4okU6Mql6BOj+uup/wKeA==", + "dependencies": { + "safe-buffer": "~5.2.0" } }, + "node_modules/sprintf-js": { + "version": "1.0.3", + "resolved": "https://registry.npmjs.org/sprintf-js/-/sprintf-js-1.0.3.tgz", + "integrity": "sha512-D9cPgkvLlV3t3IzL0D0YLvGA9Ahk4PcvVwUbN0dSGr1aP0Nrt4AEnTUbuGvquEC0mA64Gqt1fzirlRs5ibXx8g==" + }, + "node_modules/sqlite": { + "version": "4.1.2", + "resolved": "https://registry.npmjs.org/sqlite/-/sqlite-4.1.2.tgz", + "integrity": "sha512-FlBG51gHbux5vPjwnoqFEghNGvnTMTbHyiI09U3qFTQs9AtWuwd4i++6+WCusCXKrVdIDLzfdGekrolr3m4U4A==" + }, + "node_modules/string_decoder": { + "version": "0.10.31", + "resolved": "https://registry.npmjs.org/string_decoder/-/string_decoder-0.10.31.tgz", + "integrity": "sha512-ev2QzSzWPYmy9GuqfIVildA4OdcGLeFZQrq5ys6RtiuF+RQQiZWr8TZNyAcuVXyQRYfEO+MsoB/1BuQVhOJuoQ==", + "dev": true + }, "node_modules/string-width": { - "version": "1.0.2", - "resolved": "https://registry.npmjs.org/string-width/-/string-width-1.0.2.tgz", - "integrity": "sha1-EYvfW4zcUaKn5w0hHgfisLmxB9M=", - "devOptional": true, + "version": "4.2.3", + "resolved": "https://registry.npmjs.org/string-width/-/string-width-4.2.3.tgz", + "integrity": "sha512-wKyQRQpjJ0sIp62ErSZdGsjMJWsap5oRNihHhu6G7JVO/9jIB6UyevL+tXuOqrng8j/cxKTWyWUwvSTriiZz/g==", "dependencies": { - "code-point-at": "^1.0.0", - "is-fullwidth-code-point": "^1.0.0", - "strip-ansi": "^3.0.0" + "emoji-regex": "^8.0.0", + "is-fullwidth-code-point": "^3.0.0", + "strip-ansi": "^6.0.1" }, "engines": { - "node": ">=0.10.0" + "node": ">=8" } }, "node_modules/strip-ansi": { - "version": "3.0.1", - "resolved": "https://registry.npmjs.org/strip-ansi/-/strip-ansi-3.0.1.tgz", - "integrity": "sha1-ajhfuIU9lS1f8F0Oiq+UJ43GPc8=", - "devOptional": true, + "version": "6.0.1", + "resolved": "https://registry.npmjs.org/strip-ansi/-/strip-ansi-6.0.1.tgz", + "integrity": "sha512-Y38VPSHcqkFrCpFnQ9vuSXmquuv5oXOKpGeT6aGrr3o3Gc9AlVa6JBfUSOCnbxGGZF+/0ooI7KrPuUSztUdU5A==", "dependencies": { - "ansi-regex": "^2.0.0" + "ansi-regex": "^5.0.1" }, "engines": { - "node": ">=0.10.0" + "node": ">=8" } }, "node_modules/strip-bom": { "version": "3.0.0", "resolved": "https://registry.npmjs.org/strip-bom/-/strip-bom-3.0.0.tgz", - "integrity": "sha1-IzTBjpx1n3vdVv3vfprj1YjmjtM=", + "integrity": "sha512-vavAMRXOgBVNF6nyEEmL3DBK19iRpDcoIwW+swQ+CbGiu7lju6t+JklA1MHweoWtadgt4ISVUsXLyDq34ddcwA==", "dev": true, "engines": { "node": ">=4" @@ -2775,7 +2938,6 @@ "version": "3.1.1", "resolved": "https://registry.npmjs.org/strip-json-comments/-/strip-json-comments-3.1.1.tgz", "integrity": "sha512-6fPc+R4ihwqP6N/aIv2f1gMH8lOVtWQHoqC4yK6oSDVVocumAsfCqjkXnqiYMhmMwS/mEHLp7Vehlt3ql6lEig==", - "dev": true, "engines": { "node": ">=8" }, @@ -2783,6 +2945,17 @@ "url": "https://github.com/sponsors/sindresorhus" } }, + "node_modules/supports-color": { + "version": "7.2.0", + "resolved": "https://registry.npmjs.org/supports-color/-/supports-color-7.2.0.tgz", + "integrity": "sha512-qpCAvRl9stuOHveKsn7HncJRvv501qIacKzQlO/+Lwxc9+0q2wLyv4Dfvt80/DPn2pqOBsJdDiogXGR9+OvwRw==", + "dependencies": { + "has-flag": "^4.0.0" + }, + "engines": { + "node": ">=8" + } + }, "node_modules/supports-preserve-symlinks-flag": { "version": "1.0.0", "resolved": "https://registry.npmjs.org/supports-preserve-symlinks-flag/-/supports-preserve-symlinks-flag-1.0.0.tgz", @@ -2837,10 +3010,19 @@ "node": ">= 6" } }, + "node_modules/tar-stream/node_modules/string_decoder": { + "version": "1.3.0", + "resolved": "https://registry.npmjs.org/string_decoder/-/string_decoder-1.3.0.tgz", + "integrity": "sha512-hkRX8U1WjJFd8LsDJ2yQ/wWWxaopEsABU1XfkM8A+j0+85JAGppt16cr1Whg6KIbb4okU6Mql6BOj+uup/wKeA==", + "devOptional": true, + "dependencies": { + "safe-buffer": "~5.2.0" + } + }, "node_modules/text-table": { "version": "0.2.0", "resolved": "https://registry.npmjs.org/text-table/-/text-table-0.2.0.tgz", - "integrity": "sha1-f17oI66AUgfACvLfSoTsP8+lcLQ=", + "integrity": "sha512-N+8UisAXDGk8PFXP4HAzVR9nbfmVJ3zYLAWiTIoqC5v5isinhr+r5uaO8+7r3BMfuNIufIsA7RdpVgacC2cSpw==", "dev": true }, "node_modules/thenify": { @@ -2854,7 +3036,7 @@ "node_modules/thenify-all": { "version": "1.6.0", "resolved": "https://registry.npmjs.org/thenify-all/-/thenify-all-1.6.0.tgz", - "integrity": "sha1-GhkY1ALY/D+Y+/I02wvMjMEOlyY=", + "integrity": "sha512-RNxQH/qI8/t3thXJDwcstUO4zeqo64+Uy/+sNVRBx4Xn2OX+OZ9oP+iJnNFqplFra2ZUVeKCSa2oVWi3T4uVmA==", "dependencies": { "thenify": ">= 3.1.0 < 4" }, @@ -2872,6 +3054,42 @@ "xtend": "~4.0.1" } }, + "node_modules/through2/node_modules/isarray": { + "version": "1.0.0", + "resolved": "https://registry.npmjs.org/isarray/-/isarray-1.0.0.tgz", + "integrity": "sha512-VLghIWNM6ELQzo7zwmcg0NmTVyWKYjvIeM83yjp0wRDTmUnrM678fQbcKBo6n2CJEF0szoG//ytg+TKla89ALQ==", + "dev": true + }, + "node_modules/through2/node_modules/readable-stream": { + "version": "2.3.7", + "resolved": "https://registry.npmjs.org/readable-stream/-/readable-stream-2.3.7.tgz", + "integrity": "sha512-Ebho8K4jIbHAxnuxi7o42OrZgF/ZTNcsZj6nRKyUmkhLFq8CHItp/fy6hQZuZmP/n3yZ9VBUbp4zz/mX8hmYPw==", + "dev": true, + "dependencies": { + "core-util-is": "~1.0.0", + "inherits": "~2.0.3", + "isarray": "~1.0.0", + "process-nextick-args": "~2.0.0", + "safe-buffer": "~5.1.1", + "string_decoder": "~1.1.1", + "util-deprecate": "~1.0.1" + } + }, + "node_modules/through2/node_modules/safe-buffer": { + "version": "5.1.2", + "resolved": "https://registry.npmjs.org/safe-buffer/-/safe-buffer-5.1.2.tgz", + "integrity": "sha512-Gd2UZBJDkXlY7GbJxfsE8/nvKkUEU1G38c1siN6QP6a9PT9MmHB8GnpscSmMJSoF8LOIrt8ud/wPtojys4G6+g==", + "dev": true + }, + "node_modules/through2/node_modules/string_decoder": { + "version": "1.1.1", + "resolved": "https://registry.npmjs.org/string_decoder/-/string_decoder-1.1.1.tgz", + "integrity": "sha512-n/ShnvDi6FHbbVfviro+WojiFzv+s8MPMHBczVePfUpDJLwoLT0ht1l4YwBCbi8pJAveEEdnkHyPyTP/mzRfwg==", + "dev": true, + "dependencies": { + "safe-buffer": "~5.1.0" + } + }, "node_modules/to-regex-range": { "version": "5.0.1", "resolved": "https://registry.npmjs.org/to-regex-range/-/to-regex-range-5.0.1.tgz", @@ -2893,21 +3111,64 @@ "tree-kill": "cli.js" } }, + "node_modules/ts-node": { + "version": "10.9.1", + "resolved": "https://registry.npmjs.org/ts-node/-/ts-node-10.9.1.tgz", + "integrity": "sha512-NtVysVPkxxrwFGUUxGYhfux8k78pQB3JqYBXlLRZgdGUqTO5wU/UyHop5p70iEbGhB7q5KmiZiU0Y3KlJrScEw==", + "dev": true, + "dependencies": { + "@cspotcode/source-map-support": "^0.8.0", + "@tsconfig/node10": "^1.0.7", + "@tsconfig/node12": "^1.0.7", + "@tsconfig/node14": "^1.0.0", + "@tsconfig/node16": "^1.0.2", + "acorn": "^8.4.1", + "acorn-walk": "^8.1.1", + "arg": "^4.1.0", + "create-require": "^1.1.0", + "diff": "^4.0.1", + "make-error": "^1.1.1", + "v8-compile-cache-lib": "^3.0.1", + "yn": "3.1.1" + }, + "bin": { + "ts-node": "dist/bin.js", + "ts-node-cwd": "dist/bin-cwd.js", + "ts-node-esm": "dist/bin-esm.js", + "ts-node-script": "dist/bin-script.js", + "ts-node-transpile-only": "dist/bin-transpile.js", + "ts-script": "dist/bin-script-deprecated.js" + }, + "peerDependencies": { + "@swc/core": ">=1.2.50", + "@swc/wasm": ">=1.2.50", + "@types/node": "*", + "typescript": ">=2.7" + }, + "peerDependenciesMeta": { + "@swc/core": { + "optional": true + }, + "@swc/wasm": { + "optional": true + } + } + }, "node_modules/ts-node-dev": { - "version": "1.1.8", - "resolved": "https://registry.npmjs.org/ts-node-dev/-/ts-node-dev-1.1.8.tgz", - "integrity": "sha512-Q/m3vEwzYwLZKmV6/0VlFxcZzVV/xcgOt+Tx/VjaaRHyiBcFlV0541yrT09QjzzCxlDZ34OzKjrFAynlmtflEg==", + "version": "2.0.0", + "resolved": "https://registry.npmjs.org/ts-node-dev/-/ts-node-dev-2.0.0.tgz", + "integrity": "sha512-ywMrhCfH6M75yftYvrvNarLEY+SUXtUvU8/0Z6llrHQVBx12GiFk5sStF8UdfE/yfzk9IAq7O5EEbTQsxlBI8w==", "dev": true, "dependencies": { "chokidar": "^3.5.1", "dynamic-dedupe": "^0.3.0", - "minimist": "^1.2.5", + "minimist": "^1.2.6", "mkdirp": "^1.0.4", "resolve": "^1.0.0", "rimraf": "^2.6.1", "source-map-support": "^0.5.12", "tree-kill": "^1.2.2", - "ts-node": "^9.0.0", + "ts-node": "^10.4.0", "tsconfig": "^7.0.0" }, "bin": { @@ -2939,32 +3200,6 @@ "rimraf": "bin.js" } }, - "node_modules/ts-node-dev/node_modules/ts-node": { - "version": "9.1.1", - "resolved": "https://registry.npmjs.org/ts-node/-/ts-node-9.1.1.tgz", - "integrity": "sha512-hPlt7ZACERQGf03M253ytLY3dHbGNGrAq9qIHWUY9XHYl1z7wYngSr3OQ5xmui8o2AaxsONxIzjafLUiWBo1Fg==", - "dev": true, - "dependencies": { - "arg": "^4.1.0", - "create-require": "^1.1.0", - "diff": "^4.0.1", - "make-error": "^1.1.1", - "source-map-support": "^0.5.17", - "yn": "3.1.1" - }, - "bin": { - "ts-node": "dist/bin.js", - "ts-node-script": "dist/bin-script.js", - "ts-node-transpile-only": "dist/bin-transpile.js", - "ts-script": "dist/bin-script-deprecated.js" - }, - "engines": { - "node": ">=10.0.0" - }, - "peerDependencies": { - "typescript": ">=2.7" - } - }, "node_modules/tsconfig": { "version": "7.0.0", "resolved": "https://registry.npmjs.org/tsconfig/-/tsconfig-7.0.0.tgz", @@ -2980,16 +3215,16 @@ "node_modules/tsconfig/node_modules/strip-json-comments": { "version": "2.0.1", "resolved": "https://registry.npmjs.org/strip-json-comments/-/strip-json-comments-2.0.1.tgz", - "integrity": "sha1-PFMZQukIwml8DsNEhYwobHygpgo=", + "integrity": "sha512-4gB8na07fecVVkOI6Rs4e7T6NOTki5EmL7TUduTs6bu3EdnSycntVJ4re8kgZA+wx9IueI2Y11bfbgwtzuE0KQ==", "dev": true, "engines": { "node": ">=0.10.0" } }, "node_modules/tslib": { - "version": "2.3.1", - "resolved": "https://registry.npmjs.org/tslib/-/tslib-2.3.1.tgz", - "integrity": "sha512-77EbyPPpMz+FRFRuAFlWMtmgUWGe9UOG2Z25NqCwiIjRhOf5iKGuzSe5P2w1laq+FkRy4p+PCuVkJSGkzTEKVw==" + "version": "2.4.0", + "resolved": "https://registry.npmjs.org/tslib/-/tslib-2.4.0.tgz", + "integrity": "sha512-d6xOpEDfsi2CZVlPQzGeux8XMwLT9hssAsaPYExaQMuYskwb+x1x7J371tWlbBdWHroy99KnVB6qIkUbs5X3UQ==" }, "node_modules/tsutils": { "version": "3.21.0", @@ -3015,7 +3250,7 @@ "node_modules/tunnel-agent": { "version": "0.6.0", "resolved": "https://registry.npmjs.org/tunnel-agent/-/tunnel-agent-0.6.0.tgz", - "integrity": "sha1-J6XeoGs2sEoKmWZ3SykIaPD8QP0=", + "integrity": "sha512-McnNiV1l8RYeY8tBgEpuodCC1mLUdbSN+CYBL7kJsJNInOP8UjDDEwdk6Mw60vdLLrr5NHKZhMAOSrR2NZuQ+w==", "devOptional": true, "dependencies": { "safe-buffer": "^5.0.1" @@ -3049,9 +3284,9 @@ } }, "node_modules/typeorm": { - "version": "0.2.44", - "resolved": "https://registry.npmjs.org/typeorm/-/typeorm-0.2.44.tgz", - "integrity": "sha512-yFyb9Ts73vGaS/O06TvLpzvT5U/ngO31GeciNc0eoH7P1QcG8kVZdOy9FHJqkTeDmIljMRgWjbYUoMw53ZY7Xw==", + "version": "0.2.45", + "resolved": "https://registry.npmjs.org/typeorm/-/typeorm-0.2.45.tgz", + "integrity": "sha512-c0rCO8VMJ3ER7JQ73xfk0zDnVv0WDjpsP6Q1m6CVKul7DB9iVdWLRjPzc8v2eaeBuomsbZ2+gTaYr8k1gm3bYA==", "dependencies": { "@sqltools/formatter": "^1.2.2", "app-root-path": "^3.0.0", @@ -3142,11 +3377,6 @@ } } }, - "node_modules/typeorm/node_modules/argparse": { - "version": "2.0.1", - "resolved": "https://registry.npmjs.org/argparse/-/argparse-2.0.1.tgz", - "integrity": "sha512-8+9WqebbFzpX9OR+Wa6O29asIogeRMzcGtAINdpMHHyAg10f05aSFVBbcEqGf/PXw1EjAZ+q2/bEBg3DvurK3Q==" - }, "node_modules/typeorm/node_modules/buffer": { "version": "6.0.3", "resolved": "https://registry.npmjs.org/buffer/-/buffer-6.0.3.tgz", @@ -3170,22 +3400,10 @@ "ieee754": "^1.2.1" } }, - "node_modules/typeorm/node_modules/js-yaml": { - "version": "4.1.0", - "resolved": "https://registry.npmjs.org/js-yaml/-/js-yaml-4.1.0.tgz", - "integrity": "sha512-wpxZs9NoxZaJESJGIZTyDEaYpl0FKSA+FB9aJiyemKhMwkxQg63h4T1KJgUGHpTqPDNRcmmYLugrRjJlBtWvRA==", - "dependencies": { - "argparse": "^2.0.1" - }, - "bin": { - "js-yaml": "bin/js-yaml.js" - } - }, "node_modules/typescript": { - "version": "4.5.5", - "resolved": "https://registry.npmjs.org/typescript/-/typescript-4.5.5.tgz", - "integrity": "sha512-TCTIul70LyWe6IJWT8QSYeA54WQe8EjQFU4wY52Fasj5UKx88LNYKCgBEHcOMOrFF1rKGbD8v/xcNWVUq9SymA==", - "dev": true, + "version": "4.7.4", + "resolved": "https://registry.npmjs.org/typescript/-/typescript-4.7.4.tgz", + "integrity": "sha512-C0WQT0gezHuw6AdY1M2jxUO83Rjf0HP7Sk1DtXj6j1EwkQNZrHAg2XPWlq62oqEhYvONq5pkC2Y9oPljWToLmQ==", "bin": { "tsc": "bin/tsc", "tsserver": "bin/tsserver" @@ -3215,8 +3433,7 @@ "node_modules/util-deprecate": { "version": "1.0.2", "resolved": "https://registry.npmjs.org/util-deprecate/-/util-deprecate-1.0.2.tgz", - "integrity": "sha1-RQ1Nyfpw3nMnYvvS1KKJgUGaDM8=", - "devOptional": true + "integrity": "sha512-EPD5q1uXyFxJpCrLnCc1nHnq3gOa6DZBocAIiI2TaSCA7VCJ1UJDMagCzIkXNsUYfD1daK//LTEQ8xiIbrHtcw==" }, "node_modules/uuid": { "version": "8.3.2", @@ -3232,6 +3449,12 @@ "integrity": "sha512-l8lCEmLcLYZh4nbunNZvQCJc5pv7+RCwa8q/LdUx8u7lsWvPDKmpodJAJNwkAhJC//dFY48KuIEmjtd4RViDrA==", "dev": true }, + "node_modules/v8-compile-cache-lib": { + "version": "3.0.1", + "resolved": "https://registry.npmjs.org/v8-compile-cache-lib/-/v8-compile-cache-lib-3.0.1.tgz", + "integrity": "sha512-wa7YjyUGfNZngI/vtK0UHAN+lgDCxBPCylVXGp0zu59Fz5aiGtNXaq3DhIov063MorB+VfufLh3JlF2KdTK3xg==", + "dev": true + }, "node_modules/which": { "version": "2.0.2", "resolved": "https://registry.npmjs.org/which/-/which-2.0.2.tgz", @@ -3247,15 +3470,6 @@ "node": ">= 8" } }, - "node_modules/wide-align": { - "version": "1.1.3", - "resolved": "https://registry.npmjs.org/wide-align/-/wide-align-1.1.3.tgz", - "integrity": "sha512-QGkOQc8XL6Bt5PwnsExKBPuMKBxnGxWWW3fU55Xt4feHozMUhdUMaBCk290qpm/wG5u/RSKzwdAC4i51YigihA==", - "devOptional": true, - "dependencies": { - "string-width": "^1.0.2 || 2" - } - }, "node_modules/word-wrap": { "version": "1.2.3", "resolved": "https://registry.npmjs.org/word-wrap/-/word-wrap-1.2.3.tgz", @@ -3281,80 +3495,10 @@ "url": "https://github.com/chalk/wrap-ansi?sponsor=1" } }, - "node_modules/wrap-ansi/node_modules/ansi-regex": { - "version": "5.0.1", - "resolved": "https://registry.npmjs.org/ansi-regex/-/ansi-regex-5.0.1.tgz", - "integrity": "sha512-quJQXlTSUGL2LH9SUXo8VwsY4soanhgo6LNSm84E1LBcE8s3O0wpdiRzyR9z/ZZJMlMWv37qOOb9pdJlMUEKFQ==", - "engines": { - "node": ">=8" - } - }, - "node_modules/wrap-ansi/node_modules/ansi-styles": { - "version": "4.3.0", - "resolved": "https://registry.npmjs.org/ansi-styles/-/ansi-styles-4.3.0.tgz", - "integrity": "sha512-zbB9rCJAT1rbjiVDb2hqKFHNYLxgtk8NURxZ3IZwD3F6NtxbXZQCnnSi1Lkx+IDohdPlFp222wVALIheZJQSEg==", - "dependencies": { - "color-convert": "^2.0.1" - }, - "engines": { - "node": ">=8" - }, - "funding": { - "url": "https://github.com/chalk/ansi-styles?sponsor=1" - } - }, - "node_modules/wrap-ansi/node_modules/color-convert": { - "version": "2.0.1", - "resolved": "https://registry.npmjs.org/color-convert/-/color-convert-2.0.1.tgz", - "integrity": "sha512-RRECPsj7iu/xb5oKYcsFHSppFNnsj/52OVTRKb4zP5onXwVF3zVmmToNcOfGC+CRDpfK/U584fMg38ZHCaElKQ==", - "dependencies": { - "color-name": "~1.1.4" - }, - "engines": { - "node": ">=7.0.0" - } - }, - "node_modules/wrap-ansi/node_modules/color-name": { - "version": "1.1.4", - "resolved": "https://registry.npmjs.org/color-name/-/color-name-1.1.4.tgz", - "integrity": "sha512-dOy+3AuW3a2wNbZHIuMZpTcgjGuLU/uBL/ubcZF9OXbDo8ff4O8yVp5Bf0efS8uEoYo5q4Fx7dY9OgQGXgAsQA==" - }, - "node_modules/wrap-ansi/node_modules/is-fullwidth-code-point": { - "version": "3.0.0", - "resolved": "https://registry.npmjs.org/is-fullwidth-code-point/-/is-fullwidth-code-point-3.0.0.tgz", - "integrity": "sha512-zymm5+u+sCsSWyD9qNaejV3DFvhCKclKdizYaJUuHA83RLjb7nSuGnddCHGv0hk+KY7BMAlsWeK4Ueg6EV6XQg==", - "engines": { - "node": ">=8" - } - }, - "node_modules/wrap-ansi/node_modules/string-width": { - "version": "4.2.3", - "resolved": "https://registry.npmjs.org/string-width/-/string-width-4.2.3.tgz", - "integrity": "sha512-wKyQRQpjJ0sIp62ErSZdGsjMJWsap5oRNihHhu6G7JVO/9jIB6UyevL+tXuOqrng8j/cxKTWyWUwvSTriiZz/g==", - "dependencies": { - "emoji-regex": "^8.0.0", - "is-fullwidth-code-point": "^3.0.0", - "strip-ansi": "^6.0.1" - }, - "engines": { - "node": ">=8" - } - }, - "node_modules/wrap-ansi/node_modules/strip-ansi": { - "version": "6.0.1", - "resolved": "https://registry.npmjs.org/strip-ansi/-/strip-ansi-6.0.1.tgz", - "integrity": "sha512-Y38VPSHcqkFrCpFnQ9vuSXmquuv5oXOKpGeT6aGrr3o3Gc9AlVa6JBfUSOCnbxGGZF+/0ooI7KrPuUSztUdU5A==", - "dependencies": { - "ansi-regex": "^5.0.1" - }, - "engines": { - "node": ">=8" - } - }, "node_modules/wrappy": { "version": "1.0.2", "resolved": "https://registry.npmjs.org/wrappy/-/wrappy-1.0.2.tgz", - "integrity": "sha1-tSQ9jz7BqjXxNkYFvA0QNuMKtp8=" + "integrity": "sha512-l4Sp/DRseor9wL6EvV2+TuQn63dMkPjZ/sp9XkghTEbV9KlPS1xUsZ3u7/IQO4wxtcFB4bgpQPRcR3QCvezPcQ==" }, "node_modules/xml2js": { "version": "0.4.23", @@ -3400,9 +3544,9 @@ "devOptional": true }, "node_modules/yargs": { - "version": "17.3.1", - "resolved": "https://registry.npmjs.org/yargs/-/yargs-17.3.1.tgz", - "integrity": "sha512-WUANQeVgjLbNsEmGk20f+nlHgOqzRFpiGWVaBrYGYIGANIIu3lWjoyi0fNlFmJkvfhCZ6BXINe7/W2O2bV4iaA==", + "version": "17.5.1", + "resolved": "https://registry.npmjs.org/yargs/-/yargs-17.5.1.tgz", + "integrity": "sha512-t6YAJcxDkNX7NFYiVtKvWUz8l+PaKTLiL63mJYWR2GnHq2gjEWISzsLp9wg3aY36dY1j+gfIEL3pIF+XlJJfbA==", "dependencies": { "cliui": "^7.0.2", "escalade": "^3.1.1", @@ -3417,53 +3561,13 @@ } }, "node_modules/yargs-parser": { - "version": "21.0.1", - "resolved": "https://registry.npmjs.org/yargs-parser/-/yargs-parser-21.0.1.tgz", - "integrity": "sha512-9BK1jFpLzJROCI5TzwZL/TU4gqjK5xiHV/RfWLOahrjAko/e4DJkRDZQXfvqAsiZzzYhgAzbgz6lg48jcm4GLg==", + "version": "21.1.1", + "resolved": "https://registry.npmjs.org/yargs-parser/-/yargs-parser-21.1.1.tgz", + "integrity": "sha512-tVpsJW7DdjecAiFpbIB1e3qxIQsE6NoPc5/eTdrbbIC4h0LVsWhnoa3g+m2HclBIujHzsxZ4VJVA+GUuc2/LBw==", "engines": { "node": ">=12" } }, - "node_modules/yargs/node_modules/ansi-regex": { - "version": "5.0.1", - "resolved": "https://registry.npmjs.org/ansi-regex/-/ansi-regex-5.0.1.tgz", - "integrity": "sha512-quJQXlTSUGL2LH9SUXo8VwsY4soanhgo6LNSm84E1LBcE8s3O0wpdiRzyR9z/ZZJMlMWv37qOOb9pdJlMUEKFQ==", - "engines": { - "node": ">=8" - } - }, - "node_modules/yargs/node_modules/is-fullwidth-code-point": { - "version": "3.0.0", - "resolved": "https://registry.npmjs.org/is-fullwidth-code-point/-/is-fullwidth-code-point-3.0.0.tgz", - "integrity": "sha512-zymm5+u+sCsSWyD9qNaejV3DFvhCKclKdizYaJUuHA83RLjb7nSuGnddCHGv0hk+KY7BMAlsWeK4Ueg6EV6XQg==", - "engines": { - "node": ">=8" - } - }, - "node_modules/yargs/node_modules/string-width": { - "version": "4.2.3", - "resolved": "https://registry.npmjs.org/string-width/-/string-width-4.2.3.tgz", - "integrity": "sha512-wKyQRQpjJ0sIp62ErSZdGsjMJWsap5oRNihHhu6G7JVO/9jIB6UyevL+tXuOqrng8j/cxKTWyWUwvSTriiZz/g==", - "dependencies": { - "emoji-regex": "^8.0.0", - "is-fullwidth-code-point": "^3.0.0", - "strip-ansi": "^6.0.1" - }, - "engines": { - "node": ">=8" - } - }, - "node_modules/yargs/node_modules/strip-ansi": { - "version": "6.0.1", - "resolved": "https://registry.npmjs.org/strip-ansi/-/strip-ansi-6.0.1.tgz", - "integrity": "sha512-Y38VPSHcqkFrCpFnQ9vuSXmquuv5oXOKpGeT6aGrr3o3Gc9AlVa6JBfUSOCnbxGGZF+/0ooI7KrPuUSztUdU5A==", - "dependencies": { - "ansi-regex": "^5.0.1" - }, - "engines": { - "node": ">=8" - } - }, "node_modules/yn": { "version": "3.1.1", "resolved": "https://registry.npmjs.org/yn/-/yn-3.1.1.tgz", @@ -3473,6 +3577,18 @@ "node": ">=6" } }, + "node_modules/yocto-queue": { + "version": "0.1.0", + "resolved": "https://registry.npmjs.org/yocto-queue/-/yocto-queue-0.1.0.tgz", + "integrity": "sha512-rVksvsnNCdJ/ohGc6xgPwyN8eheCxsiLM8mxuE/t/mOVqJewPuO1miLpTHQiRgTKCLexL4MeAFVagts7HmNZ2Q==", + "dev": true, + "engines": { + "node": ">=10" + }, + "funding": { + "url": "https://github.com/sponsors/sindresorhus" + } + }, "node_modules/zen-observable": { "version": "0.8.15", "resolved": "https://registry.npmjs.org/zen-observable/-/zen-observable-0.8.15.tgz", @@ -3498,44 +3614,41 @@ } }, "dependencies": { + "@cspotcode/source-map-support": { + "version": "0.8.1", + "resolved": "https://registry.npmjs.org/@cspotcode/source-map-support/-/source-map-support-0.8.1.tgz", + "integrity": "sha512-IchNf6dN4tHoMFIn/7OE8LWZ19Y6q/67Bmf6vnGREv8RSbBVb9LPJxEcnwrcwX6ixSvaiGoomAUvu4YSxXrVgw==", + "dev": true, + "requires": { + "@jridgewell/trace-mapping": "0.3.9" + } + }, "@eslint/eslintrc": { - "version": "1.2.1", - "resolved": "https://registry.npmjs.org/@eslint/eslintrc/-/eslintrc-1.2.1.tgz", - "integrity": "sha512-bxvbYnBPN1Gibwyp6NrpnFzA3YtRL3BBAyEAFVIpNTm2Rn4Vy87GA5M4aSn3InRrlsbX5N0GW7XIx+U4SAEKdQ==", + "version": "1.3.0", + "resolved": "https://registry.npmjs.org/@eslint/eslintrc/-/eslintrc-1.3.0.tgz", + "integrity": "sha512-UWW0TMTmk2d7hLcWD1/e2g5HDM/HQ3csaLSqXCfqwh4uNDuNqlaKWXmEsL4Cs41Z0KnILNvwbHAah3C2yt06kw==", "dev": true, "requires": { "ajv": "^6.12.4", "debug": "^4.3.2", - "espree": "^9.3.1", - "globals": "^13.9.0", + "espree": "^9.3.2", + "globals": "^13.15.0", "ignore": "^5.2.0", "import-fresh": "^3.2.1", "js-yaml": "^4.1.0", - "minimatch": "^3.0.4", + "minimatch": "^3.1.2", "strip-json-comments": "^3.1.1" - }, - "dependencies": { - "argparse": { - "version": "2.0.1", - "resolved": "https://registry.npmjs.org/argparse/-/argparse-2.0.1.tgz", - "integrity": "sha512-8+9WqebbFzpX9OR+Wa6O29asIogeRMzcGtAINdpMHHyAg10f05aSFVBbcEqGf/PXw1EjAZ+q2/bEBg3DvurK3Q==", - "dev": true - }, - "js-yaml": { - "version": "4.1.0", - "resolved": "https://registry.npmjs.org/js-yaml/-/js-yaml-4.1.0.tgz", - "integrity": "sha512-wpxZs9NoxZaJESJGIZTyDEaYpl0FKSA+FB9aJiyemKhMwkxQg63h4T1KJgUGHpTqPDNRcmmYLugrRjJlBtWvRA==", - "dev": true, - "requires": { - "argparse": "^2.0.1" - } - } } }, + "@hapi/bourne": { + "version": "2.1.0", + "resolved": "https://registry.npmjs.org/@hapi/bourne/-/bourne-2.1.0.tgz", + "integrity": "sha512-i1BpaNDVLJdRBEKeJWkVO6tYX6DMFBuwMhSuWqLsY4ufeTKGVuV5rBsUhxPayXqnnWHgXUAmWK16H/ykO5Wj4Q==" + }, "@humanwhocodes/config-array": { - "version": "0.9.5", - "resolved": "https://registry.npmjs.org/@humanwhocodes/config-array/-/config-array-0.9.5.tgz", - "integrity": "sha512-ObyMyWxZiCu/yTisA7uzx81s40xR2fD5Cg/2Kq7G02ajkNubJf6BopgDTmDyc3U7sXpNKM8cYOw7s7Tyr+DnCw==", + "version": "0.10.4", + "resolved": "https://registry.npmjs.org/@humanwhocodes/config-array/-/config-array-0.10.4.tgz", + "integrity": "sha512-mXAIHxZT3Vcpg83opl1wGlVZ9xydbfZO3r5YfRSH6Gpp2J/PfdBP0wbDa2sO6/qRbcalpoevVyW6A/fI6LfeMw==", "dev": true, "requires": { "@humanwhocodes/object-schema": "^1.2.1", @@ -3543,12 +3656,40 @@ "minimatch": "^3.0.4" } }, + "@humanwhocodes/gitignore-to-minimatch": { + "version": "1.0.2", + "resolved": "https://registry.npmjs.org/@humanwhocodes/gitignore-to-minimatch/-/gitignore-to-minimatch-1.0.2.tgz", + "integrity": "sha512-rSqmMJDdLFUsyxR6FMtD00nfQKKLFb1kv+qBbOVKqErvloEIJLo5bDTJTQNTYgeyp78JsA7u/NPi5jT1GR/MuA==", + "dev": true + }, "@humanwhocodes/object-schema": { "version": "1.2.1", "resolved": "https://registry.npmjs.org/@humanwhocodes/object-schema/-/object-schema-1.2.1.tgz", "integrity": "sha512-ZnQMnLV4e7hDlUvw8H+U8ASL02SS2Gn6+9Ac3wGGLIe7+je2AeAOxPY+izIPJDfFDb7eDjev0Us8MO1iFRN8hA==", "dev": true }, + "@jridgewell/resolve-uri": { + "version": "3.1.0", + "resolved": "https://registry.npmjs.org/@jridgewell/resolve-uri/-/resolve-uri-3.1.0.tgz", + "integrity": "sha512-F2msla3tad+Mfht5cJq7LSXcdudKTWCVYUgw6pLFOOHSTtZlj6SWNYAp+AhuqLmWdBO2X5hPrLcu8cVP8fy28w==", + "dev": true + }, + "@jridgewell/sourcemap-codec": { + "version": "1.4.14", + "resolved": "https://registry.npmjs.org/@jridgewell/sourcemap-codec/-/sourcemap-codec-1.4.14.tgz", + "integrity": "sha512-XPSJHWmi394fuUuzDnGz1wiKqWfo1yXecHQMRf2l6hztTO+nPru658AyDngaBe7isIxEkRsPR3FZh+s7iVa4Uw==", + "dev": true + }, + "@jridgewell/trace-mapping": { + "version": "0.3.9", + "resolved": "https://registry.npmjs.org/@jridgewell/trace-mapping/-/trace-mapping-0.3.9.tgz", + "integrity": "sha512-3Belt6tdc8bPgAtbcmdtNJlirVoTmEb5e2gC94PnkwEW9jI6CAHUeoG85tjWP5WquqfavoMtMwiG4P926ZKKuQ==", + "dev": true, + "requires": { + "@jridgewell/resolve-uri": "^3.0.3", + "@jridgewell/sourcemap-codec": "^1.4.10" + } + }, "@nodelib/fs.scandir": { "version": "2.1.5", "resolved": "https://registry.npmjs.org/@nodelib/fs.scandir/-/fs.scandir-2.1.5.tgz", @@ -3576,25 +3717,35 @@ } }, "@runejs/common": { - "version": "file:../common", + "version": "2.0.2-beta.2", + "resolved": "https://registry.npmjs.org/@runejs/common/-/common-2.0.2-beta.2.tgz", + "integrity": "sha512-Xd9/Ws3ByAdI1R08Ye9fWqlS8SImjQw05Bsw0w46zcGyDx8N6jmeClf9Y/s4Akh9VZQuoAMFg2ANKsRK0MucNw==", "requires": { - "@runejs/eslint-config": "^2.0.0", - "@types/node": "^16.11.26", - "@typescript-eslint/eslint-plugin": "^5.14.0", - "@typescript-eslint/parser": "^5.14.0", - "buffer": "^6.0.3", "compressjs": "^1.0.3", - "copyfiles": "^2.4.1", - "dotenv": "^16.0.0", - "eslint": "^8.11.0", - "path": "^0.12.7", - "pino": "^7.10.0", - "pino-pretty": "^7.6.1", - "rimraf": "^3.0.2", - "ts-node-dev": "^1.1.8", - "tslib": "^2.3.1", - "typescript": "^4.6.3", - "zlib": "^1.0.5" + "js-yaml": "^3.14.1", + "pino": "^6.14.0", + "pino-pretty": "^4.8.0", + "sonic-boom": "^2.6.0", + "tslib": "^2.3.1" + }, + "dependencies": { + "argparse": { + "version": "1.0.10", + "resolved": "https://registry.npmjs.org/argparse/-/argparse-1.0.10.tgz", + "integrity": "sha512-o5Roy6tNG4SL/FOkCAN6RzjiakZS25RLYFrcMttJqbdd8BWrnA+fGz57iN5Pb06pvBGvl5gQ0B48dJlslXvoTg==", + "requires": { + "sprintf-js": "~1.0.2" + } + }, + "js-yaml": { + "version": "3.14.1", + "resolved": "https://registry.npmjs.org/js-yaml/-/js-yaml-3.14.1.tgz", + "integrity": "sha512-okMH7OXXJ7YrN9Ok3/SXrnu4iX9yOk+25nqX4imS2npuvTYDmo/QEZoqwZkYaIDk3jVvBOTOIEgEhaLOynBS9g==", + "requires": { + "argparse": "^1.0.7", + "esprima": "^4.0.0" + } + } } }, "@runejs/eslint-config": { @@ -3609,6 +3760,30 @@ "resolved": "https://registry.npmjs.org/@sqltools/formatter/-/formatter-1.2.3.tgz", "integrity": "sha512-O3uyB/JbkAEMZaP3YqyHH7TMnex7tWyCbCI4EfJdOCoN6HIhqdJBWTM6aCCiWQ/5f5wxjgU735QAIpJbjDvmzg==" }, + "@tsconfig/node10": { + "version": "1.0.9", + "resolved": "https://registry.npmjs.org/@tsconfig/node10/-/node10-1.0.9.tgz", + "integrity": "sha512-jNsYVVxU8v5g43Erja32laIDHXeoNvFEpX33OK4d6hljo3jDhCBDhx5dhCCTMWUojscpAagGiRkBKxpdl9fxqA==", + "dev": true + }, + "@tsconfig/node12": { + "version": "1.0.11", + "resolved": "https://registry.npmjs.org/@tsconfig/node12/-/node12-1.0.11.tgz", + "integrity": "sha512-cqefuRsh12pWyGsIoBKJA9luFu3mRxCA+ORZvA4ktLSzIuCUtWVxGIuXigEwO5/ywWFMZ2QEGKWvkZG1zDMTag==", + "dev": true + }, + "@tsconfig/node14": { + "version": "1.0.3", + "resolved": "https://registry.npmjs.org/@tsconfig/node14/-/node14-1.0.3.tgz", + "integrity": "sha512-ysT8mhdixWK6Hw3i1V2AeRqZ5WfXg1G43mqoYlM2nc6388Fq5jcXyr5mRsqViLx/GJYdoL0bfXD8nmF+Zn/Iow==", + "dev": true + }, + "@tsconfig/node16": { + "version": "1.0.3", + "resolved": "https://registry.npmjs.org/@tsconfig/node16/-/node16-1.0.3.tgz", + "integrity": "sha512-yOlFc+7UtL/89t2ZhjPvvB/DeAr3r+Dq58IgzsFkOAvVC6NMJXmCGjbptdXdR9qsX7pKcTL+s87FtYREi2dEEQ==", + "dev": true + }, "@types/adm-zip": { "version": "0.5.0", "resolved": "https://registry.npmjs.org/@types/adm-zip/-/adm-zip-0.5.0.tgz", @@ -3628,21 +3803,21 @@ } }, "@types/json-schema": { - "version": "7.0.9", - "resolved": "https://registry.npmjs.org/@types/json-schema/-/json-schema-7.0.9.tgz", - "integrity": "sha512-qcUXuemtEu+E5wZSJHNxUXeCZhAfXKQ41D+duX+VYPde7xyEVZci+/oXKJL13tnRs9lR2pr4fod59GT6/X1/yQ==", + "version": "7.0.11", + "resolved": "https://registry.npmjs.org/@types/json-schema/-/json-schema-7.0.11.tgz", + "integrity": "sha512-wOuvG1SN4Us4rez+tylwwwCV1psiNVOkJeM3AUWUNWg/jDQY2+HE/444y5gc+jBmRqASOm2Oeh5c1axHobwRKQ==", "dev": true }, "@types/node": { - "version": "16.11.26", - "resolved": "https://registry.npmjs.org/@types/node/-/node-16.11.26.tgz", - "integrity": "sha512-GZ7bu5A6+4DtG7q9GsoHXy3ALcgeIHP4NnL0Vv2wu0uUB/yQex26v0tf6/na1mm0+bS9Uw+0DFex7aaKr2qawQ==", + "version": "16.11.47", + "resolved": "https://registry.npmjs.org/@types/node/-/node-16.11.47.tgz", + "integrity": "sha512-fpP+jk2zJ4VW66+wAMFoBJlx1bxmBKx4DUFf68UHgdGCOuyUTDlLWqsaNPJh7xhNDykyJ9eIzAygilP/4WoN8g==", "dev": true }, "@types/strip-bom": { "version": "3.0.0", "resolved": "https://registry.npmjs.org/@types/strip-bom/-/strip-bom-3.0.0.tgz", - "integrity": "sha1-FKjsOVbC6B7bdSB5CuzyHCkK69I=", + "integrity": "sha512-xevGOReSYGM7g/kUBZzPqCrR/KYAo+F0yiPc85WFTJa0MSLtyFTVTU6cJu/aV4mid7IffDIWqo69THF2o4JiEQ==", "dev": true }, "@types/strip-json-comments": { @@ -3652,18 +3827,18 @@ "dev": true }, "@types/yargs": { - "version": "17.0.9", - "resolved": "https://registry.npmjs.org/@types/yargs/-/yargs-17.0.9.tgz", - "integrity": "sha512-Ci8+4/DOtkHRylcisKmVMtmVO5g7weUVCKcsu1sJvF1bn0wExTmbHmhFKj7AnEm0de800iovGhdSKzYnzbaHpg==", + "version": "17.0.11", + "resolved": "https://registry.npmjs.org/@types/yargs/-/yargs-17.0.11.tgz", + "integrity": "sha512-aB4y9UDUXTSMxmM4MH+YnuR0g5Cph3FLQBoWoMB21DSvFVAxRVEHEMx3TLh+zUZYMCQtKiqazz0Q4Rre31f/OA==", "dev": true, "requires": { "@types/yargs-parser": "*" } }, "@types/yargs-parser": { - "version": "20.2.1", - "resolved": "https://registry.npmjs.org/@types/yargs-parser/-/yargs-parser-20.2.1.tgz", - "integrity": "sha512-7tFImggNeNBVMsn0vLrpn1H1uPrUBdnARPTpZoitY37ZrdJREzf7I16tMrlK3hen349gr1NYh8CmZQa7CTG6Aw==", + "version": "21.0.0", + "resolved": "https://registry.npmjs.org/@types/yargs-parser/-/yargs-parser-21.0.0.tgz", + "integrity": "sha512-iO9ZQHkZxHn4mSakYV0vFHAVDyEOIJQrV2uZ06HxEPcx+mt8swXoZHIbaaJ2crJYFfErySgktuTZ3BeLz+XmFA==", "dev": true }, "@types/zen-observable": { @@ -3672,104 +3847,104 @@ "integrity": "sha512-fbF6oTd4sGGy0xjHPKAt+eS2CrxJ3+6gQ3FGcBoIJR2TLAyCkCyI8JqZNy+FeON0AhVgNJoUumVoZQjBFUqHkw==" }, "@typescript-eslint/eslint-plugin": { - "version": "5.14.0", - "resolved": "https://registry.npmjs.org/@typescript-eslint/eslint-plugin/-/eslint-plugin-5.14.0.tgz", - "integrity": "sha512-ir0wYI4FfFUDfLcuwKzIH7sMVA+db7WYen47iRSaCGl+HMAZI9fpBwfDo45ZALD3A45ZGyHWDNLhbg8tZrMX4w==", + "version": "5.33.0", + "resolved": "https://registry.npmjs.org/@typescript-eslint/eslint-plugin/-/eslint-plugin-5.33.0.tgz", + "integrity": "sha512-jHvZNSW2WZ31OPJ3enhLrEKvAZNyAFWZ6rx9tUwaessTc4sx9KmgMNhVcqVAl1ETnT5rU5fpXTLmY9YvC1DCNg==", "dev": true, "requires": { - "@typescript-eslint/scope-manager": "5.14.0", - "@typescript-eslint/type-utils": "5.14.0", - "@typescript-eslint/utils": "5.14.0", - "debug": "^4.3.2", + "@typescript-eslint/scope-manager": "5.33.0", + "@typescript-eslint/type-utils": "5.33.0", + "@typescript-eslint/utils": "5.33.0", + "debug": "^4.3.4", "functional-red-black-tree": "^1.0.1", - "ignore": "^5.1.8", + "ignore": "^5.2.0", "regexpp": "^3.2.0", - "semver": "^7.3.5", + "semver": "^7.3.7", "tsutils": "^3.21.0" } }, "@typescript-eslint/parser": { - "version": "5.14.0", - "resolved": "https://registry.npmjs.org/@typescript-eslint/parser/-/parser-5.14.0.tgz", - "integrity": "sha512-aHJN8/FuIy1Zvqk4U/gcO/fxeMKyoSv/rS46UXMXOJKVsLQ+iYPuXNbpbH7cBLcpSbmyyFbwrniLx5+kutu1pw==", + "version": "5.33.0", + "resolved": "https://registry.npmjs.org/@typescript-eslint/parser/-/parser-5.33.0.tgz", + "integrity": "sha512-cgM5cJrWmrDV2KpvlcSkelTBASAs1mgqq+IUGKJvFxWrapHpaRy5EXPQz9YaKF3nZ8KY18ILTiVpUtbIac86/w==", "dev": true, "requires": { - "@typescript-eslint/scope-manager": "5.14.0", - "@typescript-eslint/types": "5.14.0", - "@typescript-eslint/typescript-estree": "5.14.0", - "debug": "^4.3.2" + "@typescript-eslint/scope-manager": "5.33.0", + "@typescript-eslint/types": "5.33.0", + "@typescript-eslint/typescript-estree": "5.33.0", + "debug": "^4.3.4" } }, "@typescript-eslint/scope-manager": { - "version": "5.14.0", - "resolved": "https://registry.npmjs.org/@typescript-eslint/scope-manager/-/scope-manager-5.14.0.tgz", - "integrity": "sha512-LazdcMlGnv+xUc5R4qIlqH0OWARyl2kaP8pVCS39qSL3Pd1F7mI10DbdXeARcE62sVQE4fHNvEqMWsypWO+yEw==", + "version": "5.33.0", + "resolved": "https://registry.npmjs.org/@typescript-eslint/scope-manager/-/scope-manager-5.33.0.tgz", + "integrity": "sha512-/Jta8yMNpXYpRDl8EwF/M8It2A9sFJTubDo0ATZefGXmOqlaBffEw0ZbkbQ7TNDK6q55NPHFshGBPAZvZkE8Pw==", "dev": true, "requires": { - "@typescript-eslint/types": "5.14.0", - "@typescript-eslint/visitor-keys": "5.14.0" + "@typescript-eslint/types": "5.33.0", + "@typescript-eslint/visitor-keys": "5.33.0" } }, "@typescript-eslint/type-utils": { - "version": "5.14.0", - "resolved": "https://registry.npmjs.org/@typescript-eslint/type-utils/-/type-utils-5.14.0.tgz", - "integrity": "sha512-d4PTJxsqaUpv8iERTDSQBKUCV7Q5yyXjqXUl3XF7Sd9ogNLuKLkxz82qxokqQ4jXdTPZudWpmNtr/JjbbvUixw==", + "version": "5.33.0", + "resolved": "https://registry.npmjs.org/@typescript-eslint/type-utils/-/type-utils-5.33.0.tgz", + "integrity": "sha512-2zB8uEn7hEH2pBeyk3NpzX1p3lF9dKrEbnXq1F7YkpZ6hlyqb2yZujqgRGqXgRBTHWIUG3NGx/WeZk224UKlIA==", "dev": true, "requires": { - "@typescript-eslint/utils": "5.14.0", - "debug": "^4.3.2", + "@typescript-eslint/utils": "5.33.0", + "debug": "^4.3.4", "tsutils": "^3.21.0" } }, "@typescript-eslint/types": { - "version": "5.14.0", - "resolved": "https://registry.npmjs.org/@typescript-eslint/types/-/types-5.14.0.tgz", - "integrity": "sha512-BR6Y9eE9360LNnW3eEUqAg6HxS9Q35kSIs4rp4vNHRdfg0s+/PgHgskvu5DFTM7G5VKAVjuyaN476LCPrdA7Mw==", + "version": "5.33.0", + "resolved": "https://registry.npmjs.org/@typescript-eslint/types/-/types-5.33.0.tgz", + "integrity": "sha512-nIMt96JngB4MYFYXpZ/3ZNU4GWPNdBbcB5w2rDOCpXOVUkhtNlG2mmm8uXhubhidRZdwMaMBap7Uk8SZMU/ppw==", "dev": true }, "@typescript-eslint/typescript-estree": { - "version": "5.14.0", - "resolved": "https://registry.npmjs.org/@typescript-eslint/typescript-estree/-/typescript-estree-5.14.0.tgz", - "integrity": "sha512-QGnxvROrCVtLQ1724GLTHBTR0lZVu13izOp9njRvMkCBgWX26PKvmMP8k82nmXBRD3DQcFFq2oj3cKDwr0FaUA==", + "version": "5.33.0", + "resolved": "https://registry.npmjs.org/@typescript-eslint/typescript-estree/-/typescript-estree-5.33.0.tgz", + "integrity": "sha512-tqq3MRLlggkJKJUrzM6wltk8NckKyyorCSGMq4eVkyL5sDYzJJcMgZATqmF8fLdsWrW7OjjIZ1m9v81vKcaqwQ==", "dev": true, "requires": { - "@typescript-eslint/types": "5.14.0", - "@typescript-eslint/visitor-keys": "5.14.0", - "debug": "^4.3.2", - "globby": "^11.0.4", + "@typescript-eslint/types": "5.33.0", + "@typescript-eslint/visitor-keys": "5.33.0", + "debug": "^4.3.4", + "globby": "^11.1.0", "is-glob": "^4.0.3", - "semver": "^7.3.5", + "semver": "^7.3.7", "tsutils": "^3.21.0" } }, "@typescript-eslint/utils": { - "version": "5.14.0", - "resolved": "https://registry.npmjs.org/@typescript-eslint/utils/-/utils-5.14.0.tgz", - "integrity": "sha512-EHwlII5mvUA0UsKYnVzySb/5EE/t03duUTweVy8Zqt3UQXBrpEVY144OTceFKaOe4xQXZJrkptCf7PjEBeGK4w==", + "version": "5.33.0", + "resolved": "https://registry.npmjs.org/@typescript-eslint/utils/-/utils-5.33.0.tgz", + "integrity": "sha512-JxOAnXt9oZjXLIiXb5ZIcZXiwVHCkqZgof0O8KPgz7C7y0HS42gi75PdPlqh1Tf109M0fyUw45Ao6JLo7S5AHw==", "dev": true, "requires": { "@types/json-schema": "^7.0.9", - "@typescript-eslint/scope-manager": "5.14.0", - "@typescript-eslint/types": "5.14.0", - "@typescript-eslint/typescript-estree": "5.14.0", + "@typescript-eslint/scope-manager": "5.33.0", + "@typescript-eslint/types": "5.33.0", + "@typescript-eslint/typescript-estree": "5.33.0", "eslint-scope": "^5.1.1", "eslint-utils": "^3.0.0" } }, "@typescript-eslint/visitor-keys": { - "version": "5.14.0", - "resolved": "https://registry.npmjs.org/@typescript-eslint/visitor-keys/-/visitor-keys-5.14.0.tgz", - "integrity": "sha512-yL0XxfzR94UEkjBqyymMLgCBdojzEuy/eim7N9/RIcTNxpJudAcqsU8eRyfzBbcEzGoPWfdM3AGak3cN08WOIw==", + "version": "5.33.0", + "resolved": "https://registry.npmjs.org/@typescript-eslint/visitor-keys/-/visitor-keys-5.33.0.tgz", + "integrity": "sha512-/XsqCzD4t+Y9p5wd9HZiptuGKBlaZO5showwqODii5C0nZawxWLF+Q6k5wYHBrQv96h6GYKyqqMHCSTqta8Kiw==", "dev": true, "requires": { - "@typescript-eslint/types": "5.14.0", - "eslint-visitor-keys": "^3.0.0" + "@typescript-eslint/types": "5.33.0", + "eslint-visitor-keys": "^3.3.0" } }, "acorn": { - "version": "8.7.0", - "resolved": "https://registry.npmjs.org/acorn/-/acorn-8.7.0.tgz", - "integrity": "sha512-V/LGr1APy+PXIwKebEWrkZPwoeoF+w1jiOBUmuxuiUIaOHtob8Qc9BTrYo7VuI5fR8tqsy+buA2WFooR5olqvQ==", + "version": "8.8.0", + "resolved": "https://registry.npmjs.org/acorn/-/acorn-8.8.0.tgz", + "integrity": "sha512-QOxyigPVrpZ2GXT+PFyZTl6TtOFc5egxHIP9IlQ+RbupQuX4RkT/Bee4/kQuC02Xkzg84JcT7oLYtDIQxp+v7w==", "dev": true }, "acorn-jsx": { @@ -3779,6 +3954,12 @@ "dev": true, "requires": {} }, + "acorn-walk": { + "version": "8.2.0", + "resolved": "https://registry.npmjs.org/acorn-walk/-/acorn-walk-8.2.0.tgz", + "integrity": "sha512-k+iyHEuPgSw6SbuDpGQM+06HQUa04DZ3o+F6CSzXMvvI5KMvnaEqXe+YVe555R9nn6GPt404fos4wcgpw12SDA==", + "dev": true + }, "adm-zip": { "version": "0.5.9", "resolved": "https://registry.npmjs.org/adm-zip/-/adm-zip-0.5.9.tgz", @@ -3799,18 +3980,25 @@ "amdefine": { "version": "1.0.1", "resolved": "https://registry.npmjs.org/amdefine/-/amdefine-1.0.1.tgz", - "integrity": "sha1-SlKCrBZHKek2Gbz9OtFR+BfOkfU=" + "integrity": "sha512-S2Hw0TtNkMJhIabBwIojKL9YHO5T0n5eNqWJ7Lrlel/zDbftQpxpapi8tZs3X1HWa+u+QeydGmzzNU0m09+Rcg==" }, "ansi-regex": { - "version": "2.1.1", - "resolved": "https://registry.npmjs.org/ansi-regex/-/ansi-regex-2.1.1.tgz", - "integrity": "sha1-w7M6te42DYbg5ijwRorn7yfWVN8=", - "devOptional": true + "version": "5.0.1", + "resolved": "https://registry.npmjs.org/ansi-regex/-/ansi-regex-5.0.1.tgz", + "integrity": "sha512-quJQXlTSUGL2LH9SUXo8VwsY4soanhgo6LNSm84E1LBcE8s3O0wpdiRzyR9z/ZZJMlMWv37qOOb9pdJlMUEKFQ==" + }, + "ansi-styles": { + "version": "4.3.0", + "resolved": "https://registry.npmjs.org/ansi-styles/-/ansi-styles-4.3.0.tgz", + "integrity": "sha512-zbB9rCJAT1rbjiVDb2hqKFHNYLxgtk8NURxZ3IZwD3F6NtxbXZQCnnSi1Lkx+IDohdPlFp222wVALIheZJQSEg==", + "requires": { + "color-convert": "^2.0.1" + } }, "any-promise": { "version": "1.3.0", "resolved": "https://registry.npmjs.org/any-promise/-/any-promise-1.3.0.tgz", - "integrity": "sha1-q8av7tzqUugJzcA3au0845Y10X8=" + "integrity": "sha512-7UvmKalWRt1wgjL1RrGxoSJW/0QZFIegpeGvZG9kjp8vrRu55XTHbwnqq2GpXm9uLbcuhxm3IqX9OB4MZR1b2A==" }, "anymatch": { "version": "3.1.2", @@ -3827,28 +4015,79 @@ "resolved": "https://registry.npmjs.org/app-root-path/-/app-root-path-3.0.0.tgz", "integrity": "sha512-qMcx+Gy2UZynHjOHOIXPNvpf+9cjvk3cWrBBK7zg4gH9+clobJRb9NGzcT7mQTcV/6Gm/1WelUtqxVXnNlrwcw==" }, - "aproba": { - "version": "1.2.0", - "resolved": "https://registry.npmjs.org/aproba/-/aproba-1.2.0.tgz", - "integrity": "sha512-Y9J6ZjXtoYh8RnXVCMOU/ttDmk1aBjunq9vO0ta5x85WDQiQfUF9sIPBITdbiiIVcBo03Hi3jMxigBtsddlXRw==", - "devOptional": true - }, - "are-we-there-yet": { - "version": "1.1.5", - "resolved": "https://registry.npmjs.org/are-we-there-yet/-/are-we-there-yet-1.1.5.tgz", - "integrity": "sha512-5hYdAkZlcG8tOLujVDTgCT+uPX0VnpAH28gWsLfzpXYm7wP6mp5Q/gYyR7YQ0cKVJcXJnl3j2kpBan13PtQf6w==", - "devOptional": true, - "requires": { - "delegates": "^1.0.0", - "readable-stream": "^2.0.6" - } - }, "arg": { "version": "4.1.3", "resolved": "https://registry.npmjs.org/arg/-/arg-4.1.3.tgz", "integrity": "sha512-58S9QDqG0Xx27YwPSt9fJxivjYl432YCwfDMfZ+71RAqUrZef7LrKQZ3LHLOwCS4FLNBplP533Zx895SeOCHvA==", "dev": true }, + "argparse": { + "version": "2.0.1", + "resolved": "https://registry.npmjs.org/argparse/-/argparse-2.0.1.tgz", + "integrity": "sha512-8+9WqebbFzpX9OR+Wa6O29asIogeRMzcGtAINdpMHHyAg10f05aSFVBbcEqGf/PXw1EjAZ+q2/bEBg3DvurK3Q==" + }, + "args": { + "version": "5.0.3", + "resolved": "https://registry.npmjs.org/args/-/args-5.0.3.tgz", + "integrity": "sha512-h6k/zfFgusnv3i5TU08KQkVKuCPBtL/PWQbWkHUxvJrZ2nAyeaUupneemcrgn1xmqxPQsPIzwkUhOpoqPDRZuA==", + "requires": { + "camelcase": "5.0.0", + "chalk": "2.4.2", + "leven": "2.1.0", + "mri": "1.1.4" + }, + "dependencies": { + "ansi-styles": { + "version": "3.2.1", + "resolved": "https://registry.npmjs.org/ansi-styles/-/ansi-styles-3.2.1.tgz", + "integrity": "sha512-VT0ZI6kZRdTh8YyJw3SMbYm/u+NqfsAxEpWO0Pf9sq8/e94WxxOpPKx9FR1FlyCtOVDNOQ+8ntlqFxiRc+r5qA==", + "requires": { + "color-convert": "^1.9.0" + } + }, + "chalk": { + "version": "2.4.2", + "resolved": "https://registry.npmjs.org/chalk/-/chalk-2.4.2.tgz", + "integrity": "sha512-Mti+f9lpJNcwF4tWV8/OrTTtF1gZi+f8FqlyAdouralcFWFQWF2+NgCHShjkCb+IFBLq9buZwE1xckQU4peSuQ==", + "requires": { + "ansi-styles": "^3.2.1", + "escape-string-regexp": "^1.0.5", + "supports-color": "^5.3.0" + } + }, + "color-convert": { + "version": "1.9.3", + "resolved": "https://registry.npmjs.org/color-convert/-/color-convert-1.9.3.tgz", + "integrity": "sha512-QfAUtd+vFdAtFQcC8CCyYt1fYWxSqAiK2cSD6zDB8N3cpsEBAvRxp9zOGg6G/SHHJYAT88/az/IuDGALsNVbGg==", + "requires": { + "color-name": "1.1.3" + } + }, + "color-name": { + "version": "1.1.3", + "resolved": "https://registry.npmjs.org/color-name/-/color-name-1.1.3.tgz", + "integrity": "sha512-72fSenhMw2HZMTVHeCA9KCmpEIbzWiQsjN+BHcBbS9vr1mtt+vJjPdksIBNUmKAW8TFUDPJK5SUU3QhE9NEXDw==" + }, + "escape-string-regexp": { + "version": "1.0.5", + "resolved": "https://registry.npmjs.org/escape-string-regexp/-/escape-string-regexp-1.0.5.tgz", + "integrity": "sha512-vbRorB5FUQWvla16U8R/qgaFIya2qGzwDrNmCZuYKrbdSUMG6I1ZCGQRefkRVhuOkIGVne7BQ35DSfo1qvJqFg==" + }, + "has-flag": { + "version": "3.0.0", + "resolved": "https://registry.npmjs.org/has-flag/-/has-flag-3.0.0.tgz", + "integrity": "sha512-sKJf1+ceQBr4SMkvQnBDNDtf4TXpVhVGateu0t918bl30FnbE2m4vNLX+VWe/dpjlb+HugGYzW7uQXH98HPEYw==" + }, + "supports-color": { + "version": "5.5.0", + "resolved": "https://registry.npmjs.org/supports-color/-/supports-color-5.5.0.tgz", + "integrity": "sha512-QjVjwdXIt408MIiAqCX4oUKsgU2EqAGzs2Ppkm4aQYbjm+ZEWEcW4SfFNTr4uMNZma0ey4f5lgLrkB0aX0QMow==", + "requires": { + "has-flag": "^3.0.0" + } + } + } + }, "array-union": { "version": "2.1.0", "resolved": "https://registry.npmjs.org/array-union/-/array-union-2.1.0.tgz", @@ -3860,6 +4099,11 @@ "resolved": "https://registry.npmjs.org/asynckit/-/asynckit-0.4.0.tgz", "integrity": "sha512-Oei9OH4tRh0YqU3GxhX79dM/mwVgvbZJaSNaRk+bshkj0S5cfHcgYakreBjrHwatXKbz+IoIdYLxrKim2MjW0Q==" }, + "atomic-sleep": { + "version": "1.0.0", + "resolved": "https://registry.npmjs.org/atomic-sleep/-/atomic-sleep-1.0.0.tgz", + "integrity": "sha512-kNOjDqAh7px0XWNI+4QbzoiR/nTkHAWNud2uvnJquD1/x5a7EQZMJT0AczqK0Qn67oY/TTQ1LbUKajZpp3I9tQ==" + }, "axios": { "version": "0.27.2", "resolved": "https://registry.npmjs.org/axios/-/axios-0.27.2.tgz", @@ -3870,9 +4114,9 @@ } }, "balanced-match": { - "version": "1.0.0", - "resolved": "https://registry.npmjs.org/balanced-match/-/balanced-match-1.0.0.tgz", - "integrity": "sha1-ibTRmasr7kneFk6gK4nORi1xt2c=" + "version": "1.0.2", + "resolved": "https://registry.npmjs.org/balanced-match/-/balanced-match-1.0.2.tgz", + "integrity": "sha512-3oSeUO0TMV67hN1AmbXsK4yaqU7tjiHlbxRDZOpH0KW9+CeX4bRAaX0Anxt0tx2MrpRpWwQaPwIlISEJhYU5Pw==" }, "base64-js": { "version": "1.5.1", @@ -3880,13 +4124,13 @@ "integrity": "sha512-AKpaYlHn8t4SVbOHCy+b5+KKgvR4vrsD8vbvrbiQJps7fKDTkjkDry6ji0rUJjC0kzbNePLwzxq8iypo41qeWA==" }, "better-sqlite3": { - "version": "7.5.0", - "resolved": "https://registry.npmjs.org/better-sqlite3/-/better-sqlite3-7.5.0.tgz", - "integrity": "sha512-6FdG9DoytYGDhLW7VWW1vxjEz7xHkqK6LnaUQYA8d6GHNgZhu9PFX2xwKEEnSBRoT1J4PjTUPeg217ShxNmuPg==", + "version": "7.6.2", + "resolved": "https://registry.npmjs.org/better-sqlite3/-/better-sqlite3-7.6.2.tgz", + "integrity": "sha512-S5zIU1Hink2AH4xPsN0W43T1/AJ5jrPh7Oy07ocuW/AKYYY02GWzz9NH0nbSMn/gw6fDZ5jZ1QsHt1BXAwJ6Lg==", "devOptional": true, "requires": { "bindings": "^1.5.0", - "prebuild-install": "^7.0.0" + "prebuild-install": "^7.1.0" } }, "binary-extensions": { @@ -3925,6 +4169,15 @@ "string_decoder": "^1.1.1", "util-deprecate": "^1.0.1" } + }, + "string_decoder": { + "version": "1.3.0", + "resolved": "https://registry.npmjs.org/string_decoder/-/string_decoder-1.3.0.tgz", + "integrity": "sha512-hkRX8U1WjJFd8LsDJ2yQ/wWWxaopEsABU1XfkM8A+j0+85JAGppt16cr1Whg6KIbb4okU6Mql6BOj+uup/wKeA==", + "devOptional": true, + "requires": { + "safe-buffer": "~5.2.0" + } } } }, @@ -3968,49 +4221,18 @@ "integrity": "sha512-P8BjAsXvZS+VIDUI11hHCQEv74YT67YUi5JJFNWIqL235sBmjX4+qx9Muvls5ivyNENctx46xQLQ3aTuE7ssaQ==", "dev": true }, + "camelcase": { + "version": "5.0.0", + "resolved": "https://registry.npmjs.org/camelcase/-/camelcase-5.0.0.tgz", + "integrity": "sha512-faqwZqnWxbxn+F1d399ygeamQNy3lPp/H9H6rNrqYh4FSVCtcY+3cub1MxA8o9mDd55mM8Aghuu/kuyYA6VTsA==" + }, "chalk": { - "version": "4.1.0", - "resolved": "https://registry.npmjs.org/chalk/-/chalk-4.1.0.tgz", - "integrity": "sha512-qwx12AxXe2Q5xQ43Ac//I6v5aXTipYrSESdOgzrN+9XjgEpyjpKuvSGaN4qE93f7TQTlerQQ8S+EQ0EyDoVL1A==", + "version": "4.1.2", + "resolved": "https://registry.npmjs.org/chalk/-/chalk-4.1.2.tgz", + "integrity": "sha512-oKnbhFyRIXpUuez8iBMmyEa4nbj4IOQyuhc/wy9kY7/WVPcwIO9VA668Pu8RkO7+0G76SLROeyw9CpQ061i4mA==", "requires": { "ansi-styles": "^4.1.0", "supports-color": "^7.1.0" - }, - "dependencies": { - "ansi-styles": { - "version": "4.3.0", - "resolved": "https://registry.npmjs.org/ansi-styles/-/ansi-styles-4.3.0.tgz", - "integrity": "sha512-zbB9rCJAT1rbjiVDb2hqKFHNYLxgtk8NURxZ3IZwD3F6NtxbXZQCnnSi1Lkx+IDohdPlFp222wVALIheZJQSEg==", - "requires": { - "color-convert": "^2.0.1" - } - }, - "color-convert": { - "version": "2.0.1", - "resolved": "https://registry.npmjs.org/color-convert/-/color-convert-2.0.1.tgz", - "integrity": "sha512-RRECPsj7iu/xb5oKYcsFHSppFNnsj/52OVTRKb4zP5onXwVF3zVmmToNcOfGC+CRDpfK/U584fMg38ZHCaElKQ==", - "requires": { - "color-name": "~1.1.4" - } - }, - "color-name": { - "version": "1.1.4", - "resolved": "https://registry.npmjs.org/color-name/-/color-name-1.1.4.tgz", - "integrity": "sha512-dOy+3AuW3a2wNbZHIuMZpTcgjGuLU/uBL/ubcZF9OXbDo8ff4O8yVp5Bf0efS8uEoYo5q4Fx7dY9OgQGXgAsQA==" - }, - "has-flag": { - "version": "4.0.0", - "resolved": "https://registry.npmjs.org/has-flag/-/has-flag-4.0.0.tgz", - "integrity": "sha512-EykJT/Q1KjTWctppgIAgfSO0tKVuZUjhgMr17kqTumMl6Afv3EISleU7qZUzoXDFTAHTDC4NOoG/ZxU3EvlMPQ==" - }, - "supports-color": { - "version": "7.2.0", - "resolved": "https://registry.npmjs.org/supports-color/-/supports-color-7.2.0.tgz", - "integrity": "sha512-qpCAvRl9stuOHveKsn7HncJRvv501qIacKzQlO/+Lwxc9+0q2wLyv4Dfvt80/DPn2pqOBsJdDiogXGR9+OvwRw==", - "requires": { - "has-flag": "^4.0.0" - } - } } }, "chokidar": { @@ -4027,6 +4249,17 @@ "is-glob": "~4.0.1", "normalize-path": "~3.0.0", "readdirp": "~3.6.0" + }, + "dependencies": { + "glob-parent": { + "version": "5.1.2", + "resolved": "https://registry.npmjs.org/glob-parent/-/glob-parent-5.1.2.tgz", + "integrity": "sha512-AOIgSQCepiJYwP3ARnGx+5VnTu2HBYdzbGP45eLw1vr3zB3vZLeyed1sC9hnbcOc9/SrMyM5RPQrkGz4aS9Zow==", + "dev": true, + "requires": { + "is-glob": "^4.0.1" + } + } } }, "chownr": { @@ -4048,34 +4281,6 @@ "yargs": "^16.0.0" }, "dependencies": { - "ansi-regex": { - "version": "5.0.1", - "resolved": "https://registry.npmjs.org/ansi-regex/-/ansi-regex-5.0.1.tgz", - "integrity": "sha512-quJQXlTSUGL2LH9SUXo8VwsY4soanhgo6LNSm84E1LBcE8s3O0wpdiRzyR9z/ZZJMlMWv37qOOb9pdJlMUEKFQ==" - }, - "is-fullwidth-code-point": { - "version": "3.0.0", - "resolved": "https://registry.npmjs.org/is-fullwidth-code-point/-/is-fullwidth-code-point-3.0.0.tgz", - "integrity": "sha512-zymm5+u+sCsSWyD9qNaejV3DFvhCKclKdizYaJUuHA83RLjb7nSuGnddCHGv0hk+KY7BMAlsWeK4Ueg6EV6XQg==" - }, - "string-width": { - "version": "4.2.3", - "resolved": "https://registry.npmjs.org/string-width/-/string-width-4.2.3.tgz", - "integrity": "sha512-wKyQRQpjJ0sIp62ErSZdGsjMJWsap5oRNihHhu6G7JVO/9jIB6UyevL+tXuOqrng8j/cxKTWyWUwvSTriiZz/g==", - "requires": { - "emoji-regex": "^8.0.0", - "is-fullwidth-code-point": "^3.0.0", - "strip-ansi": "^6.0.1" - } - }, - "strip-ansi": { - "version": "6.0.1", - "resolved": "https://registry.npmjs.org/strip-ansi/-/strip-ansi-6.0.1.tgz", - "integrity": "sha512-Y38VPSHcqkFrCpFnQ9vuSXmquuv5oXOKpGeT6aGrr3o3Gc9AlVa6JBfUSOCnbxGGZF+/0ooI7KrPuUSztUdU5A==", - "requires": { - "ansi-regex": "^5.0.1" - } - }, "yargs": { "version": "16.2.0", "resolved": "https://registry.npmjs.org/yargs/-/yargs-16.2.0.tgz", @@ -4105,43 +4310,20 @@ "string-width": "^4.2.0", "strip-ansi": "^6.0.0", "wrap-ansi": "^7.0.0" - }, - "dependencies": { - "ansi-regex": { - "version": "5.0.1", - "resolved": "https://registry.npmjs.org/ansi-regex/-/ansi-regex-5.0.1.tgz", - "integrity": "sha512-quJQXlTSUGL2LH9SUXo8VwsY4soanhgo6LNSm84E1LBcE8s3O0wpdiRzyR9z/ZZJMlMWv37qOOb9pdJlMUEKFQ==" - }, - "is-fullwidth-code-point": { - "version": "3.0.0", - "resolved": "https://registry.npmjs.org/is-fullwidth-code-point/-/is-fullwidth-code-point-3.0.0.tgz", - "integrity": "sha512-zymm5+u+sCsSWyD9qNaejV3DFvhCKclKdizYaJUuHA83RLjb7nSuGnddCHGv0hk+KY7BMAlsWeK4Ueg6EV6XQg==" - }, - "string-width": { - "version": "4.2.3", - "resolved": "https://registry.npmjs.org/string-width/-/string-width-4.2.3.tgz", - "integrity": "sha512-wKyQRQpjJ0sIp62ErSZdGsjMJWsap5oRNihHhu6G7JVO/9jIB6UyevL+tXuOqrng8j/cxKTWyWUwvSTriiZz/g==", - "requires": { - "emoji-regex": "^8.0.0", - "is-fullwidth-code-point": "^3.0.0", - "strip-ansi": "^6.0.1" - } - }, - "strip-ansi": { - "version": "6.0.1", - "resolved": "https://registry.npmjs.org/strip-ansi/-/strip-ansi-6.0.1.tgz", - "integrity": "sha512-Y38VPSHcqkFrCpFnQ9vuSXmquuv5oXOKpGeT6aGrr3o3Gc9AlVa6JBfUSOCnbxGGZF+/0ooI7KrPuUSztUdU5A==", - "requires": { - "ansi-regex": "^5.0.1" - } - } } }, - "code-point-at": { - "version": "1.1.0", - "resolved": "https://registry.npmjs.org/code-point-at/-/code-point-at-1.1.0.tgz", - "integrity": "sha1-DQcLTQQ6W+ozovGkDi7bPZpMz3c=", - "devOptional": true + "color-convert": { + "version": "2.0.1", + "resolved": "https://registry.npmjs.org/color-convert/-/color-convert-2.0.1.tgz", + "integrity": "sha512-RRECPsj7iu/xb5oKYcsFHSppFNnsj/52OVTRKb4zP5onXwVF3zVmmToNcOfGC+CRDpfK/U584fMg38ZHCaElKQ==", + "requires": { + "color-name": "~1.1.4" + } + }, + "color-name": { + "version": "1.1.4", + "resolved": "https://registry.npmjs.org/color-name/-/color-name-1.1.4.tgz", + "integrity": "sha512-dOy+3AuW3a2wNbZHIuMZpTcgjGuLU/uBL/ubcZF9OXbDo8ff4O8yVp5Bf0efS8uEoYo5q4Fx7dY9OgQGXgAsQA==" }, "combined-stream": { "version": "1.0.8", @@ -4154,7 +4336,7 @@ "commander": { "version": "2.8.1", "resolved": "https://registry.npmjs.org/commander/-/commander-2.8.1.tgz", - "integrity": "sha1-Br42f+v9oMMwqh4qBy09yXYkJdQ=", + "integrity": "sha512-+pJLBFVk+9ZZdlAOB5WuIElVPPth47hILFkmGym57aq8kwxsowvByvB0DHs1vQAhyMZzdcpTtF0VDKGkSDR4ZQ==", "requires": { "graceful-readlink": ">= 1.0.0" } @@ -4171,13 +4353,7 @@ "concat-map": { "version": "0.0.1", "resolved": "https://registry.npmjs.org/concat-map/-/concat-map-0.0.1.tgz", - "integrity": "sha1-2Klr13/Wjfd5OnMDajug1UBdR3s=" - }, - "console-control-strings": { - "version": "1.1.0", - "resolved": "https://registry.npmjs.org/console-control-strings/-/console-control-strings-1.1.0.tgz", - "integrity": "sha1-PXz0Rk22RG6mRL9LOVB/mFEAjo4=", - "devOptional": true + "integrity": "sha512-/Srv4dswyQNBfohGpz9o6Yb3Gz3SrUDqBH5rTuhGR7ahtlbYKnVxw2bCFMRljaA7EXHaXZ8wsHdodFvbkhKmqg==" }, "copyfiles": { "version": "2.4.1", @@ -4194,38 +4370,6 @@ "yargs": "^16.1.0" }, "dependencies": { - "ansi-regex": { - "version": "5.0.1", - "resolved": "https://registry.npmjs.org/ansi-regex/-/ansi-regex-5.0.1.tgz", - "integrity": "sha512-quJQXlTSUGL2LH9SUXo8VwsY4soanhgo6LNSm84E1LBcE8s3O0wpdiRzyR9z/ZZJMlMWv37qOOb9pdJlMUEKFQ==", - "dev": true - }, - "is-fullwidth-code-point": { - "version": "3.0.0", - "resolved": "https://registry.npmjs.org/is-fullwidth-code-point/-/is-fullwidth-code-point-3.0.0.tgz", - "integrity": "sha512-zymm5+u+sCsSWyD9qNaejV3DFvhCKclKdizYaJUuHA83RLjb7nSuGnddCHGv0hk+KY7BMAlsWeK4Ueg6EV6XQg==", - "dev": true - }, - "string-width": { - "version": "4.2.3", - "resolved": "https://registry.npmjs.org/string-width/-/string-width-4.2.3.tgz", - "integrity": "sha512-wKyQRQpjJ0sIp62ErSZdGsjMJWsap5oRNihHhu6G7JVO/9jIB6UyevL+tXuOqrng8j/cxKTWyWUwvSTriiZz/g==", - "dev": true, - "requires": { - "emoji-regex": "^8.0.0", - "is-fullwidth-code-point": "^3.0.0", - "strip-ansi": "^6.0.1" - } - }, - "strip-ansi": { - "version": "6.0.1", - "resolved": "https://registry.npmjs.org/strip-ansi/-/strip-ansi-6.0.1.tgz", - "integrity": "sha512-Y38VPSHcqkFrCpFnQ9vuSXmquuv5oXOKpGeT6aGrr3o3Gc9AlVa6JBfUSOCnbxGGZF+/0ooI7KrPuUSztUdU5A==", - "dev": true, - "requires": { - "ansi-regex": "^5.0.1" - } - }, "yargs": { "version": "16.2.0", "resolved": "https://registry.npmjs.org/yargs/-/yargs-16.2.0.tgz", @@ -4250,10 +4394,10 @@ } }, "core-util-is": { - "version": "1.0.2", - "resolved": "https://registry.npmjs.org/core-util-is/-/core-util-is-1.0.2.tgz", - "integrity": "sha1-tf1UIgqivFq1eqtxQMlAdUUDwac=", - "devOptional": true + "version": "1.0.3", + "resolved": "https://registry.npmjs.org/core-util-is/-/core-util-is-1.0.3.tgz", + "integrity": "sha512-ZQBvi1DcpJ4GDqanjucZ2Hj3wEO5pZDS89BWbkcrvdxksJorwUDDZamX9ldFkp9aw2lmBDLgkObEA4DWNJ9FYQ==", + "dev": true }, "create-require": { "version": "1.1.1", @@ -4272,10 +4416,15 @@ "which": "^2.0.1" } }, + "dateformat": { + "version": "4.6.3", + "resolved": "https://registry.npmjs.org/dateformat/-/dateformat-4.6.3.tgz", + "integrity": "sha512-2P0p0pFGzHS5EMnhdxQi7aJN+iMheud0UhG4dlE1DLAlvL8JHjJJTX/CSm4JXwV0Ka5nGk3zC5mcb5bUQUxxMA==" + }, "debug": { - "version": "4.3.2", - "resolved": "https://registry.npmjs.org/debug/-/debug-4.3.2.tgz", - "integrity": "sha512-mOp8wKcvj7XxC78zLgw/ZA+6TSgkoE2C/ienthhRD298T7UNwAg9diBpLRxC0mOezLl4B0xV7M0cCO6P/O0Xhw==", + "version": "4.3.4", + "resolved": "https://registry.npmjs.org/debug/-/debug-4.3.4.tgz", + "integrity": "sha512-PRWFHuSU3eDtQJPvnNY7Jcket1j0t5OuOsFzPPzsekD52Zl8qUfFIPEiswXqIvHWGVHOgX+7G/vCNNhehwxfkQ==", "requires": { "ms": "2.1.2" } @@ -4306,10 +4455,10 @@ "resolved": "https://registry.npmjs.org/delayed-stream/-/delayed-stream-1.0.0.tgz", "integrity": "sha512-ZySD7Nf91aLB0RxL4KGrKHBXl7Eds1DAmEdcoVawXnLD7SDhpNgtuII2aAkg7a7QS41jxPSZ17p4VdGnMHk3MQ==" }, - "delegates": { - "version": "1.0.0", - "resolved": "https://registry.npmjs.org/delegates/-/delegates-1.0.0.tgz", - "integrity": "sha1-hMbhWbgZBP3KWaDvRM2HDTElD5o=", + "detect-libc": { + "version": "2.0.1", + "resolved": "https://registry.npmjs.org/detect-libc/-/detect-libc-2.0.1.tgz", + "integrity": "sha512-463v3ZeIrcWtdgIg6vI6XUncguvr2TnGl4SzDXinkt9mSLpBJKXT3mW6xT3VQdDN11+WVs29pgvivTc4Lp8v+w==", "devOptional": true }, "diff": { @@ -4344,7 +4493,7 @@ "dynamic-dedupe": { "version": "0.3.0", "resolved": "https://registry.npmjs.org/dynamic-dedupe/-/dynamic-dedupe-0.3.0.tgz", - "integrity": "sha1-BuRMIj9eTpTXjvnbI6ZRXOL5YqE=", + "integrity": "sha512-ssuANeD+z97meYOqd50e04Ze5qp4bPqo8cCkI4TRjZkzAUgIDTrXV1R8QCdINpiI+hw14+rYazvTRdQrz0/rFQ==", "dev": true, "requires": { "xtend": "^4.0.0" @@ -4359,7 +4508,6 @@ "version": "1.4.4", "resolved": "https://registry.npmjs.org/end-of-stream/-/end-of-stream-1.4.4.tgz", "integrity": "sha512-+uw1inIHVPQoaVuHzRyXd21icM+cnt4CzD5rW+NC1wjOUSTOs+Te7FOv7AhN7vS9x/oIyhLP5PR1H+phQAHu5Q==", - "devOptional": true, "requires": { "once": "^1.4.0" } @@ -4369,14 +4517,21 @@ "resolved": "https://registry.npmjs.org/escalade/-/escalade-3.1.1.tgz", "integrity": "sha512-k0er2gUkLf8O0zKJiAhmkTnJlTvINGv7ygDNPbeIsX/TJjGJZHuh9B2UxbsaEkmlEo9MfhrSzmhIlhRlI2GXnw==" }, + "escape-string-regexp": { + "version": "4.0.0", + "resolved": "https://registry.npmjs.org/escape-string-regexp/-/escape-string-regexp-4.0.0.tgz", + "integrity": "sha512-TtpcNJ3XAzx3Gq8sWRzJaVajRs0uVxA2YAkdb1jm2YkPz4G6egUFAyA3n5vtEIZefPk5Wa4UXbKuS5fKkJWdgA==", + "dev": true + }, "eslint": { - "version": "8.11.0", - "resolved": "https://registry.npmjs.org/eslint/-/eslint-8.11.0.tgz", - "integrity": "sha512-/KRpd9mIRg2raGxHRGwW9ZywYNAClZrHjdueHcrVDuO3a6bj83eoTirCCk0M0yPwOjWYKHwRVRid+xK4F/GHgA==", + "version": "8.21.0", + "resolved": "https://registry.npmjs.org/eslint/-/eslint-8.21.0.tgz", + "integrity": "sha512-/XJ1+Qurf1T9G2M5IHrsjp+xrGT73RZf23xA1z5wB1ZzzEAWSZKvRwhWxTFp1rvkvCfwcvAUNAP31bhKTTGfDA==", "dev": true, "requires": { - "@eslint/eslintrc": "^1.2.1", - "@humanwhocodes/config-array": "^0.9.2", + "@eslint/eslintrc": "^1.3.0", + "@humanwhocodes/config-array": "^0.10.4", + "@humanwhocodes/gitignore-to-minimatch": "^1.0.2", "ajv": "^6.10.0", "chalk": "^4.0.0", "cross-spawn": "^7.0.2", @@ -4386,14 +4541,17 @@ "eslint-scope": "^7.1.1", "eslint-utils": "^3.0.0", "eslint-visitor-keys": "^3.3.0", - "espree": "^9.3.1", + "espree": "^9.3.3", "esquery": "^1.4.0", "esutils": "^2.0.2", "fast-deep-equal": "^3.1.3", "file-entry-cache": "^6.0.1", + "find-up": "^5.0.0", "functional-red-black-tree": "^1.0.1", "glob-parent": "^6.0.1", - "globals": "^13.6.0", + "globals": "^13.15.0", + "globby": "^11.1.0", + "grapheme-splitter": "^1.0.4", "ignore": "^5.2.0", "import-fresh": "^3.0.0", "imurmurhash": "^0.1.4", @@ -4402,7 +4560,7 @@ "json-stable-stringify-without-jsonify": "^1.0.1", "levn": "^0.4.1", "lodash.merge": "^4.6.2", - "minimatch": "^3.0.4", + "minimatch": "^3.1.2", "natural-compare": "^1.4.0", "optionator": "^0.9.1", "regexpp": "^3.2.0", @@ -4412,24 +4570,6 @@ "v8-compile-cache": "^2.0.3" }, "dependencies": { - "ansi-regex": { - "version": "5.0.1", - "resolved": "https://registry.npmjs.org/ansi-regex/-/ansi-regex-5.0.1.tgz", - "integrity": "sha512-quJQXlTSUGL2LH9SUXo8VwsY4soanhgo6LNSm84E1LBcE8s3O0wpdiRzyR9z/ZZJMlMWv37qOOb9pdJlMUEKFQ==", - "dev": true - }, - "argparse": { - "version": "2.0.1", - "resolved": "https://registry.npmjs.org/argparse/-/argparse-2.0.1.tgz", - "integrity": "sha512-8+9WqebbFzpX9OR+Wa6O29asIogeRMzcGtAINdpMHHyAg10f05aSFVBbcEqGf/PXw1EjAZ+q2/bEBg3DvurK3Q==", - "dev": true - }, - "escape-string-regexp": { - "version": "4.0.0", - "resolved": "https://registry.npmjs.org/escape-string-regexp/-/escape-string-regexp-4.0.0.tgz", - "integrity": "sha512-TtpcNJ3XAzx3Gq8sWRzJaVajRs0uVxA2YAkdb1jm2YkPz4G6egUFAyA3n5vtEIZefPk5Wa4UXbKuS5fKkJWdgA==", - "dev": true - }, "eslint-scope": { "version": "7.1.1", "resolved": "https://registry.npmjs.org/eslint-scope/-/eslint-scope-7.1.1.tgz", @@ -4445,33 +4585,6 @@ "resolved": "https://registry.npmjs.org/estraverse/-/estraverse-5.3.0.tgz", "integrity": "sha512-MMdARuVEQziNTeJD8DgMqmhwR11BRQ/cBP+pLtYdSTnf3MIO8fFeiINEbX36ZdNlfU/7A9f3gUw49B3oQsvwBA==", "dev": true - }, - "glob-parent": { - "version": "6.0.2", - "resolved": "https://registry.npmjs.org/glob-parent/-/glob-parent-6.0.2.tgz", - "integrity": "sha512-XxwI8EOhVQgWp6iDL+3b0r86f4d6AX6zSU55HfB4ydCEuXLXc5FcYeOu+nnGftS4TEju/11rt4KJPTMgbfmv4A==", - "dev": true, - "requires": { - "is-glob": "^4.0.3" - } - }, - "js-yaml": { - "version": "4.1.0", - "resolved": "https://registry.npmjs.org/js-yaml/-/js-yaml-4.1.0.tgz", - "integrity": "sha512-wpxZs9NoxZaJESJGIZTyDEaYpl0FKSA+FB9aJiyemKhMwkxQg63h4T1KJgUGHpTqPDNRcmmYLugrRjJlBtWvRA==", - "dev": true, - "requires": { - "argparse": "^2.0.1" - } - }, - "strip-ansi": { - "version": "6.0.1", - "resolved": "https://registry.npmjs.org/strip-ansi/-/strip-ansi-6.0.1.tgz", - "integrity": "sha512-Y38VPSHcqkFrCpFnQ9vuSXmquuv5oXOKpGeT6aGrr3o3Gc9AlVa6JBfUSOCnbxGGZF+/0ooI7KrPuUSztUdU5A==", - "dev": true, - "requires": { - "ansi-regex": "^5.0.1" - } } } }, @@ -4509,16 +4622,21 @@ "dev": true }, "espree": { - "version": "9.3.1", - "resolved": "https://registry.npmjs.org/espree/-/espree-9.3.1.tgz", - "integrity": "sha512-bvdyLmJMfwkV3NCRl5ZhJf22zBFo1y8bYh3VYb+bfzqNB4Je68P2sSuXyuFquzWLebHpNd2/d5uv7yoP9ISnGQ==", + "version": "9.3.3", + "resolved": "https://registry.npmjs.org/espree/-/espree-9.3.3.tgz", + "integrity": "sha512-ORs1Rt/uQTqUKjDdGCyrtYxbazf5umATSf/K4qxjmZHORR6HJk+2s/2Pqe+Kk49HHINC/xNIrGfgh8sZcll0ng==", "dev": true, "requires": { - "acorn": "^8.7.0", - "acorn-jsx": "^5.3.1", + "acorn": "^8.8.0", + "acorn-jsx": "^5.3.2", "eslint-visitor-keys": "^3.3.0" } }, + "esprima": { + "version": "4.0.1", + "resolved": "https://registry.npmjs.org/esprima/-/esprima-4.0.1.tgz", + "integrity": "sha512-eGuFFw7Upda+g4p+QHvnW0RyTX/SVeJBDM/gCtMARO0cLuT2HcEKnTPvhjV6aGeqrCB/sbNop0Kszm0jsaWU4A==" + }, "esquery": { "version": "1.4.0", "resolved": "https://registry.npmjs.org/esquery/-/esquery-1.4.0.tgz", @@ -4588,6 +4706,17 @@ "glob-parent": "^5.1.2", "merge2": "^1.3.0", "micromatch": "^4.0.4" + }, + "dependencies": { + "glob-parent": { + "version": "5.1.2", + "resolved": "https://registry.npmjs.org/glob-parent/-/glob-parent-5.1.2.tgz", + "integrity": "sha512-AOIgSQCepiJYwP3ARnGx+5VnTu2HBYdzbGP45eLw1vr3zB3vZLeyed1sC9hnbcOc9/SrMyM5RPQrkGz4aS9Zow==", + "dev": true, + "requires": { + "is-glob": "^4.0.1" + } + } } }, "fast-json-stable-stringify": { @@ -4599,9 +4728,19 @@ "fast-levenshtein": { "version": "2.0.6", "resolved": "https://registry.npmjs.org/fast-levenshtein/-/fast-levenshtein-2.0.6.tgz", - "integrity": "sha1-PYpcZog6FqMMqGQ+hR8Zuqd5eRc=", + "integrity": "sha512-DCXu6Ifhqcks7TZKY3Hxp3y6qphY5SJZmrWMDrKcERSOXWQdMhU9Ig/PYrzyw/ul9jOIyh0N4M0tbC5hodg8dw==", "dev": true }, + "fast-redact": { + "version": "3.1.1", + "resolved": "https://registry.npmjs.org/fast-redact/-/fast-redact-3.1.1.tgz", + "integrity": "sha512-odVmjC8x8jNeMZ3C+rPMESzXVSEU8tSWSHv9HFxP2mm89G/1WwqhrerJDQm9Zus8X6aoRgQDThKqptdNA6bt+A==" + }, + "fast-safe-stringify": { + "version": "2.1.1", + "resolved": "https://registry.npmjs.org/fast-safe-stringify/-/fast-safe-stringify-2.1.1.tgz", + "integrity": "sha512-W+KJc2dmILlPplD/H4K9l9LcAHAfPtP6BY84uVLXQ6Evcz9Lcg33Y2z1IVblT6xdY54PXYVHEv+0Wpq8Io6zkA==" + }, "fastq": { "version": "1.13.0", "resolved": "https://registry.npmjs.org/fastq/-/fastq-1.13.0.tgz", @@ -4635,6 +4774,16 @@ "to-regex-range": "^5.0.1" } }, + "find-up": { + "version": "5.0.0", + "resolved": "https://registry.npmjs.org/find-up/-/find-up-5.0.0.tgz", + "integrity": "sha512-78/PXT1wlLLDgTzDs7sjq9hzz0vXD+zn+7wypEe4fXQxCmdmqfGsEPQxmiCSQI3ajFV91bVSsvNtrJRiW6nGng==", + "dev": true, + "requires": { + "locate-path": "^6.0.0", + "path-exists": "^4.0.0" + } + }, "flat-cache": { "version": "3.0.4", "resolved": "https://registry.npmjs.org/flat-cache/-/flat-cache-3.0.4.tgz", @@ -4645,10 +4794,15 @@ "rimraf": "^3.0.2" } }, + "flatstr": { + "version": "1.0.12", + "resolved": "https://registry.npmjs.org/flatstr/-/flatstr-1.0.12.tgz", + "integrity": "sha512-4zPxDyhCyiN2wIAtSLI6gc82/EjqZc1onI4Mz/l0pWrAlsSfYH/2ZIcU+e3oA2wDwbzIWNKwa23F8rh6+DRWkw==" + }, "flatted": { - "version": "3.2.5", - "resolved": "https://registry.npmjs.org/flatted/-/flatted-3.2.5.tgz", - "integrity": "sha512-WIWGi2L3DyTUvUrwRKgGi9TwxQMUEqPOPQBVi71R96jZXJdFskXEmf54BoZaS1kknGODoIGASGEzBUYdyMCBJg==", + "version": "3.2.6", + "resolved": "https://registry.npmjs.org/flatted/-/flatted-3.2.6.tgz", + "integrity": "sha512-0sQoMh9s0BYsm+12Huy/rkKxVu4R1+r96YX5cG44rHV0pQ6iC3Q+mkoMFaGWObMFYQxCVT+ssG1ksneA2MI9KQ==", "dev": true }, "follow-redirects": { @@ -4675,7 +4829,7 @@ "fs.realpath": { "version": "1.0.0", "resolved": "https://registry.npmjs.org/fs.realpath/-/fs.realpath-1.0.0.tgz", - "integrity": "sha1-FQStJSMVjKpA20onh8sBQRmU6k8=" + "integrity": "sha512-OO0pH2lK6a0hZnAdau5ItzHPI6pUlvI7jMVnxUQRtw4owF2wk8lOSabtGDCTP4Ggrg2MbGnWO9X8K1t4+fGMDw==" }, "fsevents": { "version": "2.3.2", @@ -4693,25 +4847,9 @@ "functional-red-black-tree": { "version": "1.0.1", "resolved": "https://registry.npmjs.org/functional-red-black-tree/-/functional-red-black-tree-1.0.1.tgz", - "integrity": "sha1-GwqzvVU7Kg1jmdKcDj6gslIHgyc=", + "integrity": "sha512-dsKNQNdj6xA3T+QlADDA7mOSlX0qiMINjn0cgr+eGHGsbSHzTabcIogz2+p/iqP1Xs6EP/sS2SbqH+brGTbq0g==", "dev": true }, - "gauge": { - "version": "2.7.4", - "resolved": "https://registry.npmjs.org/gauge/-/gauge-2.7.4.tgz", - "integrity": "sha1-LANAXHU4w51+s3sxcCLjJfsBi/c=", - "devOptional": true, - "requires": { - "aproba": "^1.0.3", - "console-control-strings": "^1.0.0", - "has-unicode": "^2.0.0", - "object-assign": "^4.1.0", - "signal-exit": "^3.0.0", - "string-width": "^1.0.1", - "strip-ansi": "^3.0.1", - "wide-align": "^1.1.0" - } - }, "get-caller-file": { "version": "2.0.5", "resolved": "https://registry.npmjs.org/get-caller-file/-/get-caller-file-2.0.5.tgz", @@ -4720,35 +4858,35 @@ "github-from-package": { "version": "0.0.0", "resolved": "https://registry.npmjs.org/github-from-package/-/github-from-package-0.0.0.tgz", - "integrity": "sha1-l/tdlr/eiXMxPyDoKI75oWf6ZM4=", + "integrity": "sha512-SyHy3T1v2NUXn29OsWdxmK6RwHD+vkj3v8en8AOBZ1wBQ/hCAQ5bAQTD02kW4W9tUp/3Qh6J8r9EvntiyCmOOw==", "devOptional": true }, "glob": { - "version": "7.1.6", - "resolved": "https://registry.npmjs.org/glob/-/glob-7.1.6.tgz", - "integrity": "sha512-LwaxwyZ72Lk7vZINtNNrywX0ZuLyStrdDtabefZKAY5ZGJhVtgdznluResxNmPitE0SAO+O26sWTHeKSI2wMBA==", + "version": "7.2.3", + "resolved": "https://registry.npmjs.org/glob/-/glob-7.2.3.tgz", + "integrity": "sha512-nFR0zLpU2YCaRxwoCJvL6UvCH2JFyFVIvwTLsIf21AuHlMskA1hhTdk+LlYJtOlYt9v6dvszD2BGRqBL+iQK9Q==", "requires": { "fs.realpath": "^1.0.0", "inflight": "^1.0.4", "inherits": "2", - "minimatch": "^3.0.4", + "minimatch": "^3.1.1", "once": "^1.3.0", "path-is-absolute": "^1.0.0" } }, "glob-parent": { - "version": "5.1.2", - "resolved": "https://registry.npmjs.org/glob-parent/-/glob-parent-5.1.2.tgz", - "integrity": "sha512-AOIgSQCepiJYwP3ARnGx+5VnTu2HBYdzbGP45eLw1vr3zB3vZLeyed1sC9hnbcOc9/SrMyM5RPQrkGz4aS9Zow==", + "version": "6.0.2", + "resolved": "https://registry.npmjs.org/glob-parent/-/glob-parent-6.0.2.tgz", + "integrity": "sha512-XxwI8EOhVQgWp6iDL+3b0r86f4d6AX6zSU55HfB4ydCEuXLXc5FcYeOu+nnGftS4TEju/11rt4KJPTMgbfmv4A==", "dev": true, "requires": { - "is-glob": "^4.0.1" + "is-glob": "^4.0.3" } }, "globals": { - "version": "13.12.1", - "resolved": "https://registry.npmjs.org/globals/-/globals-13.12.1.tgz", - "integrity": "sha512-317dFlgY2pdJZ9rspXDks7073GpDmXdfbM3vYYp0HAMKGDh1FfWPleI2ljVNLQX5M5lXcAslTcPTrOrMEFOjyw==", + "version": "13.17.0", + "resolved": "https://registry.npmjs.org/globals/-/globals-13.17.0.tgz", + "integrity": "sha512-1C+6nQRb1GwGMKm2dH/E7enFAMxGTmGI7/dEdhy/DNelv85w9B72t3uc5frtMNXIbzrarJJ/lTCjcaZwbLJmyw==", "dev": true, "requires": { "type-fest": "^0.20.2" @@ -4769,14 +4907,20 @@ } }, "graceful-fs": { - "version": "4.2.9", - "resolved": "https://registry.npmjs.org/graceful-fs/-/graceful-fs-4.2.9.tgz", - "integrity": "sha512-NtNxqUcXgpW2iMrfqSfR73Glt39K+BLwWsPs94yR63v45T0Wbej7eRmL5cWfwEgqXnmjQp3zaJTshdRW/qC2ZQ==" + "version": "4.2.10", + "resolved": "https://registry.npmjs.org/graceful-fs/-/graceful-fs-4.2.10.tgz", + "integrity": "sha512-9ByhssR2fPVsNZj478qUUbKfmL0+t5BDVyjShtyZZLiK7ZDAArFFfopyOTj0M05wE2tJPisA4iTnnXl2YoPvOA==" }, "graceful-readlink": { "version": "1.0.1", "resolved": "https://registry.npmjs.org/graceful-readlink/-/graceful-readlink-1.0.1.tgz", - "integrity": "sha1-TK+tdrxi8C+gObL5Tpo906ORpyU=" + "integrity": "sha512-8tLu60LgxF6XpdbK8OW3FA+IfTNBn1ZHGHKF4KQbEeSkajYw5PlYJcKluntgegDPTg8UkHjpet1T82vk6TQ68w==" + }, + "grapheme-splitter": { + "version": "1.0.4", + "resolved": "https://registry.npmjs.org/grapheme-splitter/-/grapheme-splitter-1.0.4.tgz", + "integrity": "sha512-bzh50DW9kTPM00T8y4o8vQg89Di9oLJVLW/KaOGIXJWP/iqCN6WKYkbNOF04vFLJhwcpYUh9ydh/+5vpOqV4YQ==", + "dev": true }, "has": { "version": "1.0.3", @@ -4787,11 +4931,10 @@ "function-bind": "^1.1.1" } }, - "has-unicode": { - "version": "2.0.1", - "resolved": "https://registry.npmjs.org/has-unicode/-/has-unicode-2.0.1.tgz", - "integrity": "sha1-4Ob+aijPUROIVeCG0Wkedx3iqLk=", - "devOptional": true + "has-flag": { + "version": "4.0.0", + "resolved": "https://registry.npmjs.org/has-flag/-/has-flag-4.0.0.tgz", + "integrity": "sha512-EykJT/Q1KjTWctppgIAgfSO0tKVuZUjhgMr17kqTumMl6Afv3EISleU7qZUzoXDFTAHTDC4NOoG/ZxU3EvlMPQ==" }, "highlight.js": { "version": "10.7.3", @@ -4822,13 +4965,13 @@ "imurmurhash": { "version": "0.1.4", "resolved": "https://registry.npmjs.org/imurmurhash/-/imurmurhash-0.1.4.tgz", - "integrity": "sha1-khi5srkoojixPcT7a21XbyMUU+o=", + "integrity": "sha512-JmXMZ6wuvDmLiHEml9ykzqO6lwFbof0GG4IkcGaENdCRDDmMVnny7s5HsIgHCbaq0w2MyPhDqkhTUgS2LU2PHA==", "dev": true }, "inflight": { "version": "1.0.6", "resolved": "https://registry.npmjs.org/inflight/-/inflight-1.0.6.tgz", - "integrity": "sha1-Sb1jMdfQLQwJvJEKEHW6gWW1bfk=", + "integrity": "sha512-k92I/b08q4wvFscXCLvqfsHCrjrF7yiXsQuIVvVE7N82W3+aqpzuUdBbfhWcy/FZR3/4IgflMgKLOsvPDrGCJA==", "requires": { "once": "^1.3.0", "wrappy": "1" @@ -4855,9 +4998,9 @@ } }, "is-core-module": { - "version": "2.8.1", - "resolved": "https://registry.npmjs.org/is-core-module/-/is-core-module-2.8.1.tgz", - "integrity": "sha512-SdNCUs284hr40hFTFP6l0IfZ/RSrMXF3qgoRHd3/79unUTvrFO/JoXwkGm+5J/Oe3E/b5GsnG330uUNgRpu1PA==", + "version": "2.10.0", + "resolved": "https://registry.npmjs.org/is-core-module/-/is-core-module-2.10.0.tgz", + "integrity": "sha512-Erxj2n/LDAZ7H8WNJXd9tw38GYM3dv8rk8Zcs+jJuxYTW7sozH+SS8NtrSjVL1/vpLvWi1hxy96IzjJ3EHTJJg==", "dev": true, "requires": { "has": "^1.0.3" @@ -4866,17 +5009,13 @@ "is-extglob": { "version": "2.1.1", "resolved": "https://registry.npmjs.org/is-extglob/-/is-extglob-2.1.1.tgz", - "integrity": "sha1-qIwCU1eR8C7TfHahueqXc8gz+MI=", + "integrity": "sha512-SbKbANkN603Vi4jEZv49LeVJMn4yGwsbzZworEoyEiutsN3nJYdbO36zfhGJ6QEDpOZIFkDtnq5JRxmvl3jsoQ==", "dev": true }, "is-fullwidth-code-point": { - "version": "1.0.0", - "resolved": "https://registry.npmjs.org/is-fullwidth-code-point/-/is-fullwidth-code-point-1.0.0.tgz", - "integrity": "sha1-754xOG8DGn8NZDr4L95QxFfvAMs=", - "devOptional": true, - "requires": { - "number-is-nan": "^1.0.0" - } + "version": "3.0.0", + "resolved": "https://registry.npmjs.org/is-fullwidth-code-point/-/is-fullwidth-code-point-3.0.0.tgz", + "integrity": "sha512-zymm5+u+sCsSWyD9qNaejV3DFvhCKclKdizYaJUuHA83RLjb7nSuGnddCHGv0hk+KY7BMAlsWeK4Ueg6EV6XQg==" }, "is-glob": { "version": "4.0.3", @@ -4894,17 +5033,35 @@ "dev": true }, "isarray": { - "version": "1.0.0", - "resolved": "https://registry.npmjs.org/isarray/-/isarray-1.0.0.tgz", - "integrity": "sha1-u5NdSFgsuhaMBoNJV6VKPgcSTxE=", - "devOptional": true + "version": "0.0.1", + "resolved": "https://registry.npmjs.org/isarray/-/isarray-0.0.1.tgz", + "integrity": "sha512-D2S+3GLxWH+uhrNEcoh/fnmYeP8E8/zHl644d/jdA0g2uyXvy3sb0qxotE+ne0LtccHknQzWwZEzhak7oJ0COQ==", + "dev": true }, "isexe": { "version": "2.0.0", "resolved": "https://registry.npmjs.org/isexe/-/isexe-2.0.0.tgz", - "integrity": "sha1-6PvzdNxVb/iUehDcsFctYz8s+hA=", + "integrity": "sha512-RHxMLp9lnKHGHRng9QFhRCMbYAcVpn69smSGcq3f36xjgVVWThj4qqLbTLlq7Ssj8B+fIQ1EuCEGI2lKsyQeIw==", "dev": true }, + "jmespath": { + "version": "0.15.0", + "resolved": "https://registry.npmjs.org/jmespath/-/jmespath-0.15.0.tgz", + "integrity": "sha512-+kHj8HXArPfpPEKGLZ+kB5ONRTCiGQXo8RQYL0hH8t6pWXUBBK5KkkQmTNOwKK4LEsd0yTsgtjJVm4UBSZea4w==" + }, + "joycon": { + "version": "2.2.5", + "resolved": "https://registry.npmjs.org/joycon/-/joycon-2.2.5.tgz", + "integrity": "sha512-YqvUxoOcVPnCp0VU1/56f+iKSdvIRJYPznH22BdXV3xMk75SFXhWeJkZ8C9XxUWt1b5x2X1SxuFygW1U0FmkEQ==" + }, + "js-yaml": { + "version": "4.1.0", + "resolved": "https://registry.npmjs.org/js-yaml/-/js-yaml-4.1.0.tgz", + "integrity": "sha512-wpxZs9NoxZaJESJGIZTyDEaYpl0FKSA+FB9aJiyemKhMwkxQg63h4T1KJgUGHpTqPDNRcmmYLugrRjJlBtWvRA==", + "requires": { + "argparse": "^2.0.1" + } + }, "json-schema-traverse": { "version": "0.4.1", "resolved": "https://registry.npmjs.org/json-schema-traverse/-/json-schema-traverse-0.4.1.tgz", @@ -4914,16 +5071,18 @@ "json-stable-stringify-without-jsonify": { "version": "1.0.1", "resolved": "https://registry.npmjs.org/json-stable-stringify-without-jsonify/-/json-stable-stringify-without-jsonify-1.0.1.tgz", - "integrity": "sha1-nbe1lJatPzz+8wp1FC0tkwrXJlE=", + "integrity": "sha512-Bdboy+l7tA3OGW6FjyFHWkP5LuByj1Tk33Ljyq0axyzdk9//JSi2u3fP1QSmd1KNwq6VOKYGlAu87CisVir6Pw==", "dev": true }, "json5": { - "version": "2.2.0", - "resolved": "https://registry.npmjs.org/json5/-/json5-2.2.0.tgz", - "integrity": "sha512-f+8cldu7X/y7RAJurMEJmdoKXGB/X550w2Nr3tTbezL6RwEE/iMcm+tZnXeoZtKuOq6ft8+CqzEkrIgx1fPoQA==", - "requires": { - "minimist": "^1.2.5" - } + "version": "2.2.1", + "resolved": "https://registry.npmjs.org/json5/-/json5-2.2.1.tgz", + "integrity": "sha512-1hqLFMSrGHRHxav9q9gNjJ5EXznIxGVO09xQRrwplcS8qs28pZ8s8hupZAmqDwZUmVZ2Qb2jnyPOWcDH8m8dlA==" + }, + "leven": { + "version": "2.1.0", + "resolved": "https://registry.npmjs.org/leven/-/leven-2.1.0.tgz", + "integrity": "sha512-nvVPLpIHUxCUoRLrFqTgSxXJ614d8AgQoWl7zPe/2VadE8+1dpU3LBhowRuBAcuwruWtOdD8oYC9jDNJjXDPyA==" }, "levn": { "version": "0.4.1", @@ -4935,6 +5094,15 @@ "type-check": "~0.4.0" } }, + "locate-path": { + "version": "6.0.0", + "resolved": "https://registry.npmjs.org/locate-path/-/locate-path-6.0.0.tgz", + "integrity": "sha512-iPZK6eYjbxRu3uB4/WZ3EsEIMJFMqAoopl3R+zuq0UjcAm/MO6KCweDgPfP3elTztoKP3KtnVHxTn2NHBSDVUw==", + "dev": true, + "requires": { + "p-locate": "^5.0.0" + } + }, "lodash.merge": { "version": "4.6.2", "resolved": "https://registry.npmjs.org/lodash.merge/-/lodash.merge-4.6.2.tgz", @@ -4963,13 +5131,13 @@ "dev": true }, "micromatch": { - "version": "4.0.4", - "resolved": "https://registry.npmjs.org/micromatch/-/micromatch-4.0.4.tgz", - "integrity": "sha512-pRmzw/XUcwXGpD9aI9q/0XOwLNygjETJ8y0ao0wdqprrzDa4YnxLcz7fQRZr8voh8V10kGhABbNcHVk5wHgWwg==", + "version": "4.0.5", + "resolved": "https://registry.npmjs.org/micromatch/-/micromatch-4.0.5.tgz", + "integrity": "sha512-DMy+ERcEW2q8Z2Po+WNXuw3c5YaUSFjAO5GsJqfEl7UjvtIuFKO6ZrKvcItdy98dwFI2N1tg3zNIdKaQT+aNdA==", "dev": true, "requires": { - "braces": "^3.0.1", - "picomatch": "^2.2.3" + "braces": "^3.0.2", + "picomatch": "^2.3.1" } }, "mime-db": { @@ -4992,9 +5160,9 @@ "devOptional": true }, "minimatch": { - "version": "3.0.4", - "resolved": "https://registry.npmjs.org/minimatch/-/minimatch-3.0.4.tgz", - "integrity": "sha512-yJHVQEhyqPLUTgt9B83PXu6W3rx4MvvHvSUvToogpwoGDOUQ+yDrR0HRot+yOCdCO7u4hX3pWft6kWBBcqh0UA==", + "version": "3.1.2", + "resolved": "https://registry.npmjs.org/minimatch/-/minimatch-3.1.2.tgz", + "integrity": "sha512-J7p63hRiAjw1NDEww1W7i37+ByIrOWO5XQQAzZ3VOcL0PNybwpfmV/N05zFAzwQ9USyEcX6t3UO+K5aqBQOIHw==", "requires": { "brace-expansion": "^1.1.7" } @@ -5002,7 +5170,8 @@ "minimist": { "version": "1.2.6", "resolved": "https://registry.npmjs.org/minimist/-/minimist-1.2.6.tgz", - "integrity": "sha512-Jsjnk4bw3YJqYzbdyBiNsPWHPfO++UGG749Cxs6peCu5Xg4nrena6OVxOYxrQTqww0Jmwt+Ref8rggumkTLz9Q==" + "integrity": "sha512-Jsjnk4bw3YJqYzbdyBiNsPWHPfO++UGG749Cxs6peCu5Xg4nrena6OVxOYxrQTqww0Jmwt+Ref8rggumkTLz9Q==", + "devOptional": true }, "mkdirp": { "version": "1.0.4", @@ -5015,6 +5184,11 @@ "integrity": "sha512-gKLcREMhtuZRwRAfqP3RFW+TK4JqApVBtOIftVgjuABpAtpxhPGaDcfvbhNvD0B8iD1oUr/txX35NjcaY6Ns/A==", "devOptional": true }, + "mri": { + "version": "1.1.4", + "resolved": "https://registry.npmjs.org/mri/-/mri-1.1.4.tgz", + "integrity": "sha512-6y7IjGPm8AzlvoUrwAaw1tLnUBudaS3752vcd8JtrpGGQn+rXIe63LFVHm/YMwtqAuh+LJPCFdlLYPWM1nYn6w==" + }, "ms": { "version": "2.1.2", "resolved": "https://registry.npmjs.org/ms/-/ms-2.1.2.tgz", @@ -5039,13 +5213,13 @@ "natural-compare": { "version": "1.4.0", "resolved": "https://registry.npmjs.org/natural-compare/-/natural-compare-1.4.0.tgz", - "integrity": "sha1-Sr6/7tdUHywnrPspvbvRXI1bpPc=", + "integrity": "sha512-OWND8ei3VtNC9h7V60qff3SVobHr996CTwgxubgyQYEpg290h9J0buyECNNJexkFm5sOajh5G116RYA1c8ZMSw==", "dev": true }, "node-abi": { - "version": "3.8.0", - "resolved": "https://registry.npmjs.org/node-abi/-/node-abi-3.8.0.tgz", - "integrity": "sha512-tzua9qWWi7iW4I42vUPKM+SfaF0vQSLAm4yO5J83mSwB7GeoWrDKC/K+8YCnYNwqP5duwazbw2X9l4m8SC2cUw==", + "version": "3.24.0", + "resolved": "https://registry.npmjs.org/node-abi/-/node-abi-3.24.0.tgz", + "integrity": "sha512-YPG3Co0luSu6GwOBsmIdGW6Wx0NyNDLg/hriIyDllVsNwnI6UeqaWShxC3lbH4LtEQUgoLP3XR1ndXiDAWvmRw==", "devOptional": true, "requires": { "semver": "^7.3.5" @@ -5054,37 +5228,11 @@ "noms": { "version": "0.0.0", "resolved": "https://registry.npmjs.org/noms/-/noms-0.0.0.tgz", - "integrity": "sha1-2o69nzr51nYJGbJ9nNyAkqczKFk=", + "integrity": "sha512-lNDU9VJaOPxUmXcLb+HQFeUgQQPtMI24Gt6hgfuMHRJgMRHMF/qZ4HJD3GDru4sSw9IQl2jPjAYnQrdIeLbwow==", "dev": true, "requires": { "inherits": "^2.0.1", "readable-stream": "~1.0.31" - }, - "dependencies": { - "isarray": { - "version": "0.0.1", - "resolved": "https://registry.npmjs.org/isarray/-/isarray-0.0.1.tgz", - "integrity": "sha1-ihis/Kmo9Bd+Cav8YDiTmwXR7t8=", - "dev": true - }, - "readable-stream": { - "version": "1.0.34", - "resolved": "https://registry.npmjs.org/readable-stream/-/readable-stream-1.0.34.tgz", - "integrity": "sha1-Elgg40vIQtLyqq+v5MKRbuMsFXw=", - "dev": true, - "requires": { - "core-util-is": "~1.0.0", - "inherits": "~2.0.1", - "isarray": "0.0.1", - "string_decoder": "~0.10.x" - } - }, - "string_decoder": { - "version": "0.10.31", - "resolved": "https://registry.npmjs.org/string_decoder/-/string_decoder-0.10.31.tgz", - "integrity": "sha1-YuIDvEF2bGwoyfyEMB2rHFMQ+pQ=", - "dev": true - } } }, "normalize-path": { @@ -5093,33 +5241,15 @@ "integrity": "sha512-6eZs5Ls3WtCisHWp9S2GUy8dqkpGi4BVSz3GaqiE6ezub0512ESztXUwUB6C6IKbQkY2Pnb/mD4WYojCRwcwLA==", "dev": true }, - "npmlog": { - "version": "4.1.2", - "resolved": "https://registry.npmjs.org/npmlog/-/npmlog-4.1.2.tgz", - "integrity": "sha512-2uUqazuKlTaSI/dC8AzicUck7+IrEaOnN/e0jd3Xtt1KcGpwx30v50mL7oPyr/h9bL3E4aZccVwpwP+5W9Vjkg==", - "devOptional": true, - "requires": { - "are-we-there-yet": "~1.1.2", - "console-control-strings": "~1.1.0", - "gauge": "~2.7.3", - "set-blocking": "~2.0.0" - } - }, - "number-is-nan": { - "version": "1.0.1", - "resolved": "https://registry.npmjs.org/number-is-nan/-/number-is-nan-1.0.1.tgz", - "integrity": "sha1-CXtgK1NCKlIsGvuHkDGDNpQaAR0=", - "devOptional": true - }, "object-assign": { "version": "4.1.1", "resolved": "https://registry.npmjs.org/object-assign/-/object-assign-4.1.1.tgz", - "integrity": "sha1-IQmtx5ZYh8/AXLvUQsrIv7s2CGM=" + "integrity": "sha512-rJgTQnkUnH1sFw8yT6VSU3zD3sWmu6sZhIseY8VX+GRu3P6F7Fu+JNDoXfklElbLJSnc3FUQHVe4cU5hj+BcUg==" }, "once": { "version": "1.4.0", "resolved": "https://registry.npmjs.org/once/-/once-1.4.0.tgz", - "integrity": "sha1-WDsap3WWHUsROsF9nFC6753Xa9E=", + "integrity": "sha512-lNaJgI+2Q5URQBkccEKHTQOPaXdUxnZZElQTZY0MFUAuaEqe1E+Nyvgdz/aIyNi6Z9MzO5dv1H8n58/GELp3+w==", "requires": { "wrappy": "1" } @@ -5138,6 +5268,24 @@ "word-wrap": "^1.2.3" } }, + "p-limit": { + "version": "3.1.0", + "resolved": "https://registry.npmjs.org/p-limit/-/p-limit-3.1.0.tgz", + "integrity": "sha512-TYOanM3wGwNGsZN2cVTYPArw454xnXj5qmWF1bEoAc4+cU/ol7GVh7odevjp1FNHduHc3KZMcFduxU5Xc6uJRQ==", + "dev": true, + "requires": { + "yocto-queue": "^0.1.0" + } + }, + "p-locate": { + "version": "5.0.0", + "resolved": "https://registry.npmjs.org/p-locate/-/p-locate-5.0.0.tgz", + "integrity": "sha512-LaNjtRWUBY++zB5nE/NwcaoMylSPk+S+ZHNB1TzdbMJMny6dynpAGt7X/tl/QYq3TIeE6nxHppbo2LGymrG5Pw==", + "dev": true, + "requires": { + "p-limit": "^3.0.2" + } + }, "parent-module": { "version": "1.0.1", "resolved": "https://registry.npmjs.org/parent-module/-/parent-module-1.0.1.tgz", @@ -5167,10 +5315,16 @@ } } }, + "path-exists": { + "version": "4.0.0", + "resolved": "https://registry.npmjs.org/path-exists/-/path-exists-4.0.0.tgz", + "integrity": "sha512-ak9Qy5Q7jYb2Wwcey5Fpvg2KoAc/ZIhLSLOSBmRmygPsGwkVVt0fZa0qrtMz+m6tJTAHfZQ8FnmB4MG4LWy7/w==", + "dev": true + }, "path-is-absolute": { "version": "1.0.1", "resolved": "https://registry.npmjs.org/path-is-absolute/-/path-is-absolute-1.0.1.tgz", - "integrity": "sha1-F0uSaHNVNP+8es5r9TpanhtcX18=" + "integrity": "sha512-AVbw3UJ2e9bq64vSaS9Am0fje1Pa8pbGqTTsmXfaIiMpnr5DlDhfJOuLj9Sf95ZPVDAUerDfEk88MPmPe7UCQg==" }, "path-key": { "version": "3.1.1", @@ -5196,10 +5350,79 @@ "integrity": "sha512-JU3teHTNjmE2VCGFzuY8EXzCDVwEqB2a8fsIvwaStHhAWJEeVd1o1QD80CU6+ZdEXXSLbSsuLwJjkCBWqRQUVA==", "dev": true }, + "pino": { + "version": "6.14.0", + "resolved": "https://registry.npmjs.org/pino/-/pino-6.14.0.tgz", + "integrity": "sha512-iuhEDel3Z3hF9Jfe44DPXR8l07bhjuFY3GMHIXbjnY9XcafbyDDwl2sN2vw2GjMPf5Nkoe+OFao7ffn9SXaKDg==", + "requires": { + "fast-redact": "^3.0.0", + "fast-safe-stringify": "^2.0.8", + "flatstr": "^1.0.12", + "pino-std-serializers": "^3.1.0", + "process-warning": "^1.0.0", + "quick-format-unescaped": "^4.0.3", + "sonic-boom": "^1.0.2" + }, + "dependencies": { + "sonic-boom": { + "version": "1.4.1", + "resolved": "https://registry.npmjs.org/sonic-boom/-/sonic-boom-1.4.1.tgz", + "integrity": "sha512-LRHh/A8tpW7ru89lrlkU4AszXt1dbwSjVWguGrmlxE7tawVmDBlI1PILMkXAxJTwqhgsEeTHzj36D5CmHgQmNg==", + "requires": { + "atomic-sleep": "^1.0.0", + "flatstr": "^1.0.12" + } + } + } + }, + "pino-pretty": { + "version": "4.8.0", + "resolved": "https://registry.npmjs.org/pino-pretty/-/pino-pretty-4.8.0.tgz", + "integrity": "sha512-mhQfHG4rw5ZFpWL44m0Utjo4GC2+HMfdNvxyA8lLw0sIqn6fCf7uQe6dPckUcW/obly+OQHD7B/MTso6LNizYw==", + "requires": { + "@hapi/bourne": "^2.0.0", + "args": "^5.0.1", + "chalk": "^4.0.0", + "dateformat": "^4.5.1", + "fast-safe-stringify": "^2.0.7", + "jmespath": "^0.15.0", + "joycon": "^2.2.5", + "pump": "^3.0.0", + "readable-stream": "^3.6.0", + "rfdc": "^1.3.0", + "split2": "^3.1.1", + "strip-json-comments": "^3.1.1" + }, + "dependencies": { + "readable-stream": { + "version": "3.6.0", + "resolved": "https://registry.npmjs.org/readable-stream/-/readable-stream-3.6.0.tgz", + "integrity": "sha512-BViHy7LKeTz4oNnkcLJ+lVSL6vpiFeX6/d3oSH8zCW7UxP2onchk+vTGB143xuFjHS3deTgkKoXXymXqymiIdA==", + "requires": { + "inherits": "^2.0.3", + "string_decoder": "^1.1.1", + "util-deprecate": "^1.0.1" + } + }, + "string_decoder": { + "version": "1.3.0", + "resolved": "https://registry.npmjs.org/string_decoder/-/string_decoder-1.3.0.tgz", + "integrity": "sha512-hkRX8U1WjJFd8LsDJ2yQ/wWWxaopEsABU1XfkM8A+j0+85JAGppt16cr1Whg6KIbb4okU6Mql6BOj+uup/wKeA==", + "requires": { + "safe-buffer": "~5.2.0" + } + } + } + }, + "pino-std-serializers": { + "version": "3.2.0", + "resolved": "https://registry.npmjs.org/pino-std-serializers/-/pino-std-serializers-3.2.0.tgz", + "integrity": "sha512-EqX4pwDPrt3MuOAAUBMU0Tk5kR/YcCM5fNPEzgCO2zJ5HfX0vbiH9HbJglnyeQsN96Kznae6MWD47pZB5avTrg==" + }, "prebuild-install": { - "version": "7.0.1", - "resolved": "https://registry.npmjs.org/prebuild-install/-/prebuild-install-7.0.1.tgz", - "integrity": "sha512-QBSab31WqkyxpnMWQxubYAHR5S9B2+r81ucocew34Fkl98FhvKIF50jIJnNOBmAZfyNV7vE5T6gd3hTVWgY6tg==", + "version": "7.1.1", + "resolved": "https://registry.npmjs.org/prebuild-install/-/prebuild-install-7.1.1.tgz", + "integrity": "sha512-jAXscXWMcCK8GgCoHOfIr0ODh5ai8mj63L2nWrjuAgXE6tDyYGnx4/8o/rCgU+B4JSyZBKbeZqzhtwtC3ovxjw==", "devOptional": true, "requires": { "detect-libc": "^2.0.0", @@ -5209,20 +5432,11 @@ "mkdirp-classic": "^0.5.3", "napi-build-utils": "^1.0.1", "node-abi": "^3.3.0", - "npmlog": "^4.0.1", "pump": "^3.0.0", "rc": "^1.2.7", "simple-get": "^4.0.0", "tar-fs": "^2.0.0", "tunnel-agent": "^0.6.0" - }, - "dependencies": { - "detect-libc": { - "version": "2.0.1", - "resolved": "https://registry.npmjs.org/detect-libc/-/detect-libc-2.0.1.tgz", - "integrity": "sha512-463v3ZeIrcWtdgIg6vI6XUncguvr2TnGl4SzDXinkt9mSLpBJKXT3mW6xT3VQdDN11+WVs29pgvivTc4Lp8v+w==", - "devOptional": true - } } }, "prelude-ls": { @@ -5235,13 +5449,17 @@ "version": "2.0.1", "resolved": "https://registry.npmjs.org/process-nextick-args/-/process-nextick-args-2.0.1.tgz", "integrity": "sha512-3ouUOpQhtgrbOa17J7+uxOTpITYWaGP7/AhoR3+A+/1e9skrzelGi/dXzEYyvbxubEF6Wn2ypscTKiKJFFn1ag==", - "devOptional": true + "dev": true + }, + "process-warning": { + "version": "1.0.0", + "resolved": "https://registry.npmjs.org/process-warning/-/process-warning-1.0.0.tgz", + "integrity": "sha512-du4wfLyj4yCZq1VupnVSZmRsPJsNuxoDQFdCFHLaYiEbFBD7QE0a+I4D7hOxrVnh78QE/YipFAj9lXHiXocV+Q==" }, "pump": { "version": "3.0.0", "resolved": "https://registry.npmjs.org/pump/-/pump-3.0.0.tgz", "integrity": "sha512-LwZy+p3SFs1Pytd/jYct4wpv49HiYCqd9Rlc5ZVdk0V+8Yzv6jR5Blk3TRmPL1ft69TxP0IMZGJ+WPFU2BFhww==", - "devOptional": true, "requires": { "end-of-stream": "^1.1.0", "once": "^1.3.1" @@ -5259,6 +5477,11 @@ "integrity": "sha512-NuaNSa6flKT5JaSYQzJok04JzTL1CA6aGhv5rfLW3PgqA+M2ChpZQnAC8h8i4ZFkBS8X5RqkDBHA7r4hej3K9A==", "dev": true }, + "quick-format-unescaped": { + "version": "4.0.4", + "resolved": "https://registry.npmjs.org/quick-format-unescaped/-/quick-format-unescaped-4.0.4.tgz", + "integrity": "sha512-tYC1Q1hgyRuHgloV/YXs2w15unPVh8qfu/qCTfhTYamaw7fyhumKa2yGpdSo87vY32rIclj+4fWYQXUMs9EHvg==" + }, "rc": { "version": "1.2.8", "resolved": "https://registry.npmjs.org/rc/-/rc-1.2.8.tgz", @@ -5274,24 +5497,21 @@ "strip-json-comments": { "version": "2.0.1", "resolved": "https://registry.npmjs.org/strip-json-comments/-/strip-json-comments-2.0.1.tgz", - "integrity": "sha1-PFMZQukIwml8DsNEhYwobHygpgo=", + "integrity": "sha512-4gB8na07fecVVkOI6Rs4e7T6NOTki5EmL7TUduTs6bu3EdnSycntVJ4re8kgZA+wx9IueI2Y11bfbgwtzuE0KQ==", "devOptional": true } } }, "readable-stream": { - "version": "2.3.7", - "resolved": "https://registry.npmjs.org/readable-stream/-/readable-stream-2.3.7.tgz", - "integrity": "sha512-Ebho8K4jIbHAxnuxi7o42OrZgF/ZTNcsZj6nRKyUmkhLFq8CHItp/fy6hQZuZmP/n3yZ9VBUbp4zz/mX8hmYPw==", - "devOptional": true, + "version": "1.0.34", + "resolved": "https://registry.npmjs.org/readable-stream/-/readable-stream-1.0.34.tgz", + "integrity": "sha512-ok1qVCJuRkNmvebYikljxJA/UEsKwLl2nI1OmaqAu4/UE+h0wKCHok4XkL/gvi39OacXvw59RJUOFUkDib2rHg==", + "dev": true, "requires": { "core-util-is": "~1.0.0", - "inherits": "~2.0.3", - "isarray": "~1.0.0", - "process-nextick-args": "~2.0.0", - "safe-buffer": "~5.1.1", - "string_decoder": "~1.1.1", - "util-deprecate": "~1.0.1" + "inherits": "~2.0.1", + "isarray": "0.0.1", + "string_decoder": "~0.10.x" } }, "readdirp": { @@ -5317,15 +5537,15 @@ "require-directory": { "version": "2.1.1", "resolved": "https://registry.npmjs.org/require-directory/-/require-directory-2.1.1.tgz", - "integrity": "sha1-jGStX9MNqxyXbiNE/+f3kqam30I=" + "integrity": "sha512-fGxEI7+wsG9xrvdjsrlmL22OMTTiHRwAMroiEeMgq8gzoLC/PQr7RsRDSTLUg/bZAZtF+TVIkHc6/4RIKrui+Q==" }, "resolve": { - "version": "1.22.0", - "resolved": "https://registry.npmjs.org/resolve/-/resolve-1.22.0.tgz", - "integrity": "sha512-Hhtrw0nLeSrFQ7phPp4OOcVjLPIeMnRlr5mcnVuMe7M/7eBn98A3hmFRLoFo3DLZkivSYwhRUJTyPyWAk56WLw==", + "version": "1.22.1", + "resolved": "https://registry.npmjs.org/resolve/-/resolve-1.22.1.tgz", + "integrity": "sha512-nBpuuYuY5jFsli/JIs1oldw6fOQCBioohqWZg/2hiaOybXOft4lonv85uDOKXdf8rhyK159cxU5cDcK/NKk8zw==", "dev": true, "requires": { - "is-core-module": "^2.8.1", + "is-core-module": "^2.9.0", "path-parse": "^1.0.7", "supports-preserve-symlinks-flag": "^1.0.0" } @@ -5342,6 +5562,11 @@ "integrity": "sha512-U9nH88a3fc/ekCF1l0/UP1IosiuIjyTh7hBvXVMHYgVcfGvt897Xguj2UOLDeI5BG2m7/uwyaLVT6fbtCwTyzw==", "dev": true }, + "rfdc": { + "version": "1.3.0", + "resolved": "https://registry.npmjs.org/rfdc/-/rfdc-1.3.0.tgz", + "integrity": "sha512-V2hovdzFbOi77/WajaSMXk2OLm+xNIeQdMMuB7icj7bk6zi2F8GGAxigcnDFpJHbNyNcgyJDiP+8nOrY5cZGrA==" + }, "rimraf": { "version": "3.0.2", "resolved": "https://registry.npmjs.org/rimraf/-/rimraf-3.0.2.tgz", @@ -5361,9 +5586,9 @@ } }, "safe-buffer": { - "version": "5.1.2", - "resolved": "https://registry.npmjs.org/safe-buffer/-/safe-buffer-5.1.2.tgz", - "integrity": "sha512-Gd2UZBJDkXlY7GbJxfsE8/nvKkUEU1G38c1siN6QP6a9PT9MmHB8GnpscSmMJSoF8LOIrt8ud/wPtojys4G6+g==" + "version": "5.2.1", + "resolved": "https://registry.npmjs.org/safe-buffer/-/safe-buffer-5.2.1.tgz", + "integrity": "sha512-rp3So07KcdmmKbGvgaNxQSJr7bGVSVk5S9Eq1F+ppbRo70+YeaDxkw5Dd8NPN+GD6bjnYm2VuPuCXmpuYvmCXQ==" }, "sax": { "version": "1.2.4", @@ -5371,20 +5596,14 @@ "integrity": "sha512-NqVDv9TpANUjFm0N8uM5GxL36UgKi9/atZw+x7YFnQ8ckwFGKrl4xX4yWtrey3UJm5nP1kUbnYgLopqWNSRhWw==" }, "semver": { - "version": "7.3.5", - "resolved": "https://registry.npmjs.org/semver/-/semver-7.3.5.tgz", - "integrity": "sha512-PoeGJYh8HK4BTO/a9Tf6ZG3veo/A7ZVsYrSA6J8ny9nb3B1VrpkuN+z9OE5wfE5p6H4LchYZsegiQgbJD94ZFQ==", + "version": "7.3.7", + "resolved": "https://registry.npmjs.org/semver/-/semver-7.3.7.tgz", + "integrity": "sha512-QlYTucUYOews+WeEujDoEGziz4K6c47V/Bd+LjSSYcA94p+DmINdf7ncaUinThfvZyu13lN9OY1XDxt8C0Tw0g==", "devOptional": true, "requires": { "lru-cache": "^6.0.0" } }, - "set-blocking": { - "version": "2.0.0", - "resolved": "https://registry.npmjs.org/set-blocking/-/set-blocking-2.0.0.tgz", - "integrity": "sha1-BF+XgtARrppoA93TgrJDkrPYkPc=", - "devOptional": true - }, "sha.js": { "version": "2.4.11", "resolved": "https://registry.npmjs.org/sha.js/-/sha.js-2.4.11.tgz", @@ -5409,12 +5628,6 @@ "integrity": "sha512-7++dFhtcx3353uBaq8DDR4NuxBetBzC7ZQOhmTQInHEd6bSrXdiEyzCvG07Z44UYdLShWUyXt5M/yhz8ekcb1A==", "dev": true }, - "signal-exit": { - "version": "3.0.3", - "resolved": "https://registry.npmjs.org/signal-exit/-/signal-exit-3.0.3.tgz", - "integrity": "sha512-VUJ49FC8U1OxwZLxIbTTrDvLnf/6TDgxZcK8wxR8zs13xpx7xbG60ndBlhNrFi2EMuFRoeDoJO7wthSLq42EjA==", - "devOptional": true - }, "simple-concat": { "version": "1.0.1", "resolved": "https://registry.npmjs.org/simple-concat/-/simple-concat-1.0.1.tgz", @@ -5438,6 +5651,14 @@ "integrity": "sha512-g9Q1haeby36OSStwb4ntCGGGaKsaVSjQ68fBxoQcutl5fS1vuY18H3wSt3jFyFtrkx+Kz0V1G85A4MyAdDMi2Q==", "dev": true }, + "sonic-boom": { + "version": "2.8.0", + "resolved": "https://registry.npmjs.org/sonic-boom/-/sonic-boom-2.8.0.tgz", + "integrity": "sha512-kuonw1YOYYNOve5iHdSahXPOK49GqwA+LZhI6Wz/l0rP57iKyXXIHaRagOBHAPmGwJC6od2Z9zgvZ5loSgMlVg==", + "requires": { + "atomic-sleep": "^1.0.0" + } + }, "source-map": { "version": "0.6.1", "resolved": "https://registry.npmjs.org/source-map/-/source-map-0.6.1.tgz", @@ -5454,51 +5675,86 @@ "source-map": "^0.6.0" } }, + "split2": { + "version": "3.2.2", + "resolved": "https://registry.npmjs.org/split2/-/split2-3.2.2.tgz", + "integrity": "sha512-9NThjpgZnifTkJpzTZ7Eue85S49QwpNhZTq6GRJwObb6jnLFNGB7Qm73V5HewTROPyxD0C29xqmaI68bQtV+hg==", + "requires": { + "readable-stream": "^3.0.0" + }, + "dependencies": { + "readable-stream": { + "version": "3.6.0", + "resolved": "https://registry.npmjs.org/readable-stream/-/readable-stream-3.6.0.tgz", + "integrity": "sha512-BViHy7LKeTz4oNnkcLJ+lVSL6vpiFeX6/d3oSH8zCW7UxP2onchk+vTGB143xuFjHS3deTgkKoXXymXqymiIdA==", + "requires": { + "inherits": "^2.0.3", + "string_decoder": "^1.1.1", + "util-deprecate": "^1.0.1" + } + }, + "string_decoder": { + "version": "1.3.0", + "resolved": "https://registry.npmjs.org/string_decoder/-/string_decoder-1.3.0.tgz", + "integrity": "sha512-hkRX8U1WjJFd8LsDJ2yQ/wWWxaopEsABU1XfkM8A+j0+85JAGppt16cr1Whg6KIbb4okU6Mql6BOj+uup/wKeA==", + "requires": { + "safe-buffer": "~5.2.0" + } + } + } + }, + "sprintf-js": { + "version": "1.0.3", + "resolved": "https://registry.npmjs.org/sprintf-js/-/sprintf-js-1.0.3.tgz", + "integrity": "sha512-D9cPgkvLlV3t3IzL0D0YLvGA9Ahk4PcvVwUbN0dSGr1aP0Nrt4AEnTUbuGvquEC0mA64Gqt1fzirlRs5ibXx8g==" + }, "sqlite": { - "version": "4.0.25", - "resolved": "https://registry.npmjs.org/sqlite/-/sqlite-4.0.25.tgz", - "integrity": "sha512-gqCEcLF8FOTeW/na3SRYWLQkw2jZXgVj1DdgRJbm0jvrhnUgBIuNDUUm649AnBNDNHhI5XskwT8dvc8vearRLQ==" + "version": "4.1.2", + "resolved": "https://registry.npmjs.org/sqlite/-/sqlite-4.1.2.tgz", + "integrity": "sha512-FlBG51gHbux5vPjwnoqFEghNGvnTMTbHyiI09U3qFTQs9AtWuwd4i++6+WCusCXKrVdIDLzfdGekrolr3m4U4A==" }, "string_decoder": { - "version": "1.1.1", - "resolved": "https://registry.npmjs.org/string_decoder/-/string_decoder-1.1.1.tgz", - "integrity": "sha512-n/ShnvDi6FHbbVfviro+WojiFzv+s8MPMHBczVePfUpDJLwoLT0ht1l4YwBCbi8pJAveEEdnkHyPyTP/mzRfwg==", - "devOptional": true, - "requires": { - "safe-buffer": "~5.1.0" - } + "version": "0.10.31", + "resolved": "https://registry.npmjs.org/string_decoder/-/string_decoder-0.10.31.tgz", + "integrity": "sha512-ev2QzSzWPYmy9GuqfIVildA4OdcGLeFZQrq5ys6RtiuF+RQQiZWr8TZNyAcuVXyQRYfEO+MsoB/1BuQVhOJuoQ==", + "dev": true }, "string-width": { - "version": "1.0.2", - "resolved": "https://registry.npmjs.org/string-width/-/string-width-1.0.2.tgz", - "integrity": "sha1-EYvfW4zcUaKn5w0hHgfisLmxB9M=", - "devOptional": true, + "version": "4.2.3", + "resolved": "https://registry.npmjs.org/string-width/-/string-width-4.2.3.tgz", + "integrity": "sha512-wKyQRQpjJ0sIp62ErSZdGsjMJWsap5oRNihHhu6G7JVO/9jIB6UyevL+tXuOqrng8j/cxKTWyWUwvSTriiZz/g==", "requires": { - "code-point-at": "^1.0.0", - "is-fullwidth-code-point": "^1.0.0", - "strip-ansi": "^3.0.0" + "emoji-regex": "^8.0.0", + "is-fullwidth-code-point": "^3.0.0", + "strip-ansi": "^6.0.1" } }, "strip-ansi": { - "version": "3.0.1", - "resolved": "https://registry.npmjs.org/strip-ansi/-/strip-ansi-3.0.1.tgz", - "integrity": "sha1-ajhfuIU9lS1f8F0Oiq+UJ43GPc8=", - "devOptional": true, + "version": "6.0.1", + "resolved": "https://registry.npmjs.org/strip-ansi/-/strip-ansi-6.0.1.tgz", + "integrity": "sha512-Y38VPSHcqkFrCpFnQ9vuSXmquuv5oXOKpGeT6aGrr3o3Gc9AlVa6JBfUSOCnbxGGZF+/0ooI7KrPuUSztUdU5A==", "requires": { - "ansi-regex": "^2.0.0" + "ansi-regex": "^5.0.1" } }, "strip-bom": { "version": "3.0.0", "resolved": "https://registry.npmjs.org/strip-bom/-/strip-bom-3.0.0.tgz", - "integrity": "sha1-IzTBjpx1n3vdVv3vfprj1YjmjtM=", + "integrity": "sha512-vavAMRXOgBVNF6nyEEmL3DBK19iRpDcoIwW+swQ+CbGiu7lju6t+JklA1MHweoWtadgt4ISVUsXLyDq34ddcwA==", "dev": true }, "strip-json-comments": { "version": "3.1.1", "resolved": "https://registry.npmjs.org/strip-json-comments/-/strip-json-comments-3.1.1.tgz", - "integrity": "sha512-6fPc+R4ihwqP6N/aIv2f1gMH8lOVtWQHoqC4yK6oSDVVocumAsfCqjkXnqiYMhmMwS/mEHLp7Vehlt3ql6lEig==", - "dev": true + "integrity": "sha512-6fPc+R4ihwqP6N/aIv2f1gMH8lOVtWQHoqC4yK6oSDVVocumAsfCqjkXnqiYMhmMwS/mEHLp7Vehlt3ql6lEig==" + }, + "supports-color": { + "version": "7.2.0", + "resolved": "https://registry.npmjs.org/supports-color/-/supports-color-7.2.0.tgz", + "integrity": "sha512-qpCAvRl9stuOHveKsn7HncJRvv501qIacKzQlO/+Lwxc9+0q2wLyv4Dfvt80/DPn2pqOBsJdDiogXGR9+OvwRw==", + "requires": { + "has-flag": "^4.0.0" + } }, "supports-preserve-symlinks-flag": { "version": "1.0.0", @@ -5541,13 +5797,22 @@ "string_decoder": "^1.1.1", "util-deprecate": "^1.0.1" } + }, + "string_decoder": { + "version": "1.3.0", + "resolved": "https://registry.npmjs.org/string_decoder/-/string_decoder-1.3.0.tgz", + "integrity": "sha512-hkRX8U1WjJFd8LsDJ2yQ/wWWxaopEsABU1XfkM8A+j0+85JAGppt16cr1Whg6KIbb4okU6Mql6BOj+uup/wKeA==", + "devOptional": true, + "requires": { + "safe-buffer": "~5.2.0" + } } } }, "text-table": { "version": "0.2.0", "resolved": "https://registry.npmjs.org/text-table/-/text-table-0.2.0.tgz", - "integrity": "sha1-f17oI66AUgfACvLfSoTsP8+lcLQ=", + "integrity": "sha512-N+8UisAXDGk8PFXP4HAzVR9nbfmVJ3zYLAWiTIoqC5v5isinhr+r5uaO8+7r3BMfuNIufIsA7RdpVgacC2cSpw==", "dev": true }, "thenify": { @@ -5561,7 +5826,7 @@ "thenify-all": { "version": "1.6.0", "resolved": "https://registry.npmjs.org/thenify-all/-/thenify-all-1.6.0.tgz", - "integrity": "sha1-GhkY1ALY/D+Y+/I02wvMjMEOlyY=", + "integrity": "sha512-RNxQH/qI8/t3thXJDwcstUO4zeqo64+Uy/+sNVRBx4Xn2OX+OZ9oP+iJnNFqplFra2ZUVeKCSa2oVWi3T4uVmA==", "requires": { "thenify": ">= 3.1.0 < 4" } @@ -5574,6 +5839,44 @@ "requires": { "readable-stream": "~2.3.6", "xtend": "~4.0.1" + }, + "dependencies": { + "isarray": { + "version": "1.0.0", + "resolved": "https://registry.npmjs.org/isarray/-/isarray-1.0.0.tgz", + "integrity": "sha512-VLghIWNM6ELQzo7zwmcg0NmTVyWKYjvIeM83yjp0wRDTmUnrM678fQbcKBo6n2CJEF0szoG//ytg+TKla89ALQ==", + "dev": true + }, + "readable-stream": { + "version": "2.3.7", + "resolved": "https://registry.npmjs.org/readable-stream/-/readable-stream-2.3.7.tgz", + "integrity": "sha512-Ebho8K4jIbHAxnuxi7o42OrZgF/ZTNcsZj6nRKyUmkhLFq8CHItp/fy6hQZuZmP/n3yZ9VBUbp4zz/mX8hmYPw==", + "dev": true, + "requires": { + "core-util-is": "~1.0.0", + "inherits": "~2.0.3", + "isarray": "~1.0.0", + "process-nextick-args": "~2.0.0", + "safe-buffer": "~5.1.1", + "string_decoder": "~1.1.1", + "util-deprecate": "~1.0.1" + } + }, + "safe-buffer": { + "version": "5.1.2", + "resolved": "https://registry.npmjs.org/safe-buffer/-/safe-buffer-5.1.2.tgz", + "integrity": "sha512-Gd2UZBJDkXlY7GbJxfsE8/nvKkUEU1G38c1siN6QP6a9PT9MmHB8GnpscSmMJSoF8LOIrt8ud/wPtojys4G6+g==", + "dev": true + }, + "string_decoder": { + "version": "1.1.1", + "resolved": "https://registry.npmjs.org/string_decoder/-/string_decoder-1.1.1.tgz", + "integrity": "sha512-n/ShnvDi6FHbbVfviro+WojiFzv+s8MPMHBczVePfUpDJLwoLT0ht1l4YwBCbi8pJAveEEdnkHyPyTP/mzRfwg==", + "dev": true, + "requires": { + "safe-buffer": "~5.1.0" + } + } } }, "to-regex-range": { @@ -5591,21 +5894,42 @@ "integrity": "sha512-L0Orpi8qGpRG//Nd+H90vFB+3iHnue1zSSGmNOOCh1GLJ7rUKVwV2HvijphGQS2UmhUZewS9VgvxYIdgr+fG1A==", "dev": true }, + "ts-node": { + "version": "10.9.1", + "resolved": "https://registry.npmjs.org/ts-node/-/ts-node-10.9.1.tgz", + "integrity": "sha512-NtVysVPkxxrwFGUUxGYhfux8k78pQB3JqYBXlLRZgdGUqTO5wU/UyHop5p70iEbGhB7q5KmiZiU0Y3KlJrScEw==", + "dev": true, + "requires": { + "@cspotcode/source-map-support": "^0.8.0", + "@tsconfig/node10": "^1.0.7", + "@tsconfig/node12": "^1.0.7", + "@tsconfig/node14": "^1.0.0", + "@tsconfig/node16": "^1.0.2", + "acorn": "^8.4.1", + "acorn-walk": "^8.1.1", + "arg": "^4.1.0", + "create-require": "^1.1.0", + "diff": "^4.0.1", + "make-error": "^1.1.1", + "v8-compile-cache-lib": "^3.0.1", + "yn": "3.1.1" + } + }, "ts-node-dev": { - "version": "1.1.8", - "resolved": "https://registry.npmjs.org/ts-node-dev/-/ts-node-dev-1.1.8.tgz", - "integrity": "sha512-Q/m3vEwzYwLZKmV6/0VlFxcZzVV/xcgOt+Tx/VjaaRHyiBcFlV0541yrT09QjzzCxlDZ34OzKjrFAynlmtflEg==", + "version": "2.0.0", + "resolved": "https://registry.npmjs.org/ts-node-dev/-/ts-node-dev-2.0.0.tgz", + "integrity": "sha512-ywMrhCfH6M75yftYvrvNarLEY+SUXtUvU8/0Z6llrHQVBx12GiFk5sStF8UdfE/yfzk9IAq7O5EEbTQsxlBI8w==", "dev": true, "requires": { "chokidar": "^3.5.1", "dynamic-dedupe": "^0.3.0", - "minimist": "^1.2.5", + "minimist": "^1.2.6", "mkdirp": "^1.0.4", "resolve": "^1.0.0", "rimraf": "^2.6.1", "source-map-support": "^0.5.12", "tree-kill": "^1.2.2", - "ts-node": "^9.0.0", + "ts-node": "^10.4.0", "tsconfig": "^7.0.0" }, "dependencies": { @@ -5617,20 +5941,6 @@ "requires": { "glob": "^7.1.3" } - }, - "ts-node": { - "version": "9.1.1", - "resolved": "https://registry.npmjs.org/ts-node/-/ts-node-9.1.1.tgz", - "integrity": "sha512-hPlt7ZACERQGf03M253ytLY3dHbGNGrAq9qIHWUY9XHYl1z7wYngSr3OQ5xmui8o2AaxsONxIzjafLUiWBo1Fg==", - "dev": true, - "requires": { - "arg": "^4.1.0", - "create-require": "^1.1.0", - "diff": "^4.0.1", - "make-error": "^1.1.1", - "source-map-support": "^0.5.17", - "yn": "3.1.1" - } } } }, @@ -5649,15 +5959,15 @@ "strip-json-comments": { "version": "2.0.1", "resolved": "https://registry.npmjs.org/strip-json-comments/-/strip-json-comments-2.0.1.tgz", - "integrity": "sha1-PFMZQukIwml8DsNEhYwobHygpgo=", + "integrity": "sha512-4gB8na07fecVVkOI6Rs4e7T6NOTki5EmL7TUduTs6bu3EdnSycntVJ4re8kgZA+wx9IueI2Y11bfbgwtzuE0KQ==", "dev": true } } }, "tslib": { - "version": "2.3.1", - "resolved": "https://registry.npmjs.org/tslib/-/tslib-2.3.1.tgz", - "integrity": "sha512-77EbyPPpMz+FRFRuAFlWMtmgUWGe9UOG2Z25NqCwiIjRhOf5iKGuzSe5P2w1laq+FkRy4p+PCuVkJSGkzTEKVw==" + "version": "2.4.0", + "resolved": "https://registry.npmjs.org/tslib/-/tslib-2.4.0.tgz", + "integrity": "sha512-d6xOpEDfsi2CZVlPQzGeux8XMwLT9hssAsaPYExaQMuYskwb+x1x7J371tWlbBdWHroy99KnVB6qIkUbs5X3UQ==" }, "tsutils": { "version": "3.21.0", @@ -5679,7 +5989,7 @@ "tunnel-agent": { "version": "0.6.0", "resolved": "https://registry.npmjs.org/tunnel-agent/-/tunnel-agent-0.6.0.tgz", - "integrity": "sha1-J6XeoGs2sEoKmWZ3SykIaPD8QP0=", + "integrity": "sha512-McnNiV1l8RYeY8tBgEpuodCC1mLUdbSN+CYBL7kJsJNInOP8UjDDEwdk6Mw60vdLLrr5NHKZhMAOSrR2NZuQ+w==", "devOptional": true, "requires": { "safe-buffer": "^5.0.1" @@ -5701,9 +6011,9 @@ "dev": true }, "typeorm": { - "version": "0.2.44", - "resolved": "https://registry.npmjs.org/typeorm/-/typeorm-0.2.44.tgz", - "integrity": "sha512-yFyb9Ts73vGaS/O06TvLpzvT5U/ngO31GeciNc0eoH7P1QcG8kVZdOy9FHJqkTeDmIljMRgWjbYUoMw53ZY7Xw==", + "version": "0.2.45", + "resolved": "https://registry.npmjs.org/typeorm/-/typeorm-0.2.45.tgz", + "integrity": "sha512-c0rCO8VMJ3ER7JQ73xfk0zDnVv0WDjpsP6Q1m6CVKul7DB9iVdWLRjPzc8v2eaeBuomsbZ2+gTaYr8k1gm3bYA==", "requires": { "@sqltools/formatter": "^1.2.2", "app-root-path": "^3.0.0", @@ -5724,11 +6034,6 @@ "zen-observable-ts": "^1.0.0" }, "dependencies": { - "argparse": { - "version": "2.0.1", - "resolved": "https://registry.npmjs.org/argparse/-/argparse-2.0.1.tgz", - "integrity": "sha512-8+9WqebbFzpX9OR+Wa6O29asIogeRMzcGtAINdpMHHyAg10f05aSFVBbcEqGf/PXw1EjAZ+q2/bEBg3DvurK3Q==" - }, "buffer": { "version": "6.0.3", "resolved": "https://registry.npmjs.org/buffer/-/buffer-6.0.3.tgz", @@ -5737,22 +6042,13 @@ "base64-js": "^1.3.1", "ieee754": "^1.2.1" } - }, - "js-yaml": { - "version": "4.1.0", - "resolved": "https://registry.npmjs.org/js-yaml/-/js-yaml-4.1.0.tgz", - "integrity": "sha512-wpxZs9NoxZaJESJGIZTyDEaYpl0FKSA+FB9aJiyemKhMwkxQg63h4T1KJgUGHpTqPDNRcmmYLugrRjJlBtWvRA==", - "requires": { - "argparse": "^2.0.1" - } } } }, "typescript": { - "version": "4.5.5", - "resolved": "https://registry.npmjs.org/typescript/-/typescript-4.5.5.tgz", - "integrity": "sha512-TCTIul70LyWe6IJWT8QSYeA54WQe8EjQFU4wY52Fasj5UKx88LNYKCgBEHcOMOrFF1rKGbD8v/xcNWVUq9SymA==", - "dev": true + "version": "4.7.4", + "resolved": "https://registry.npmjs.org/typescript/-/typescript-4.7.4.tgz", + "integrity": "sha512-C0WQT0gezHuw6AdY1M2jxUO83Rjf0HP7Sk1DtXj6j1EwkQNZrHAg2XPWlq62oqEhYvONq5pkC2Y9oPljWToLmQ==" }, "untildify": { "version": "4.0.0", @@ -5772,8 +6068,7 @@ "util-deprecate": { "version": "1.0.2", "resolved": "https://registry.npmjs.org/util-deprecate/-/util-deprecate-1.0.2.tgz", - "integrity": "sha1-RQ1Nyfpw3nMnYvvS1KKJgUGaDM8=", - "devOptional": true + "integrity": "sha512-EPD5q1uXyFxJpCrLnCc1nHnq3gOa6DZBocAIiI2TaSCA7VCJ1UJDMagCzIkXNsUYfD1daK//LTEQ8xiIbrHtcw==" }, "uuid": { "version": "8.3.2", @@ -5786,6 +6081,12 @@ "integrity": "sha512-l8lCEmLcLYZh4nbunNZvQCJc5pv7+RCwa8q/LdUx8u7lsWvPDKmpodJAJNwkAhJC//dFY48KuIEmjtd4RViDrA==", "dev": true }, + "v8-compile-cache-lib": { + "version": "3.0.1", + "resolved": "https://registry.npmjs.org/v8-compile-cache-lib/-/v8-compile-cache-lib-3.0.1.tgz", + "integrity": "sha512-wa7YjyUGfNZngI/vtK0UHAN+lgDCxBPCylVXGp0zu59Fz5aiGtNXaq3DhIov063MorB+VfufLh3JlF2KdTK3xg==", + "dev": true + }, "which": { "version": "2.0.2", "resolved": "https://registry.npmjs.org/which/-/which-2.0.2.tgz", @@ -5795,15 +6096,6 @@ "isexe": "^2.0.0" } }, - "wide-align": { - "version": "1.1.3", - "resolved": "https://registry.npmjs.org/wide-align/-/wide-align-1.1.3.tgz", - "integrity": "sha512-QGkOQc8XL6Bt5PwnsExKBPuMKBxnGxWWW3fU55Xt4feHozMUhdUMaBCk290qpm/wG5u/RSKzwdAC4i51YigihA==", - "devOptional": true, - "requires": { - "string-width": "^1.0.2 || 2" - } - }, "word-wrap": { "version": "1.2.3", "resolved": "https://registry.npmjs.org/word-wrap/-/word-wrap-1.2.3.tgz", @@ -5818,63 +6110,12 @@ "ansi-styles": "^4.0.0", "string-width": "^4.1.0", "strip-ansi": "^6.0.0" - }, - "dependencies": { - "ansi-regex": { - "version": "5.0.1", - "resolved": "https://registry.npmjs.org/ansi-regex/-/ansi-regex-5.0.1.tgz", - "integrity": "sha512-quJQXlTSUGL2LH9SUXo8VwsY4soanhgo6LNSm84E1LBcE8s3O0wpdiRzyR9z/ZZJMlMWv37qOOb9pdJlMUEKFQ==" - }, - "ansi-styles": { - "version": "4.3.0", - "resolved": "https://registry.npmjs.org/ansi-styles/-/ansi-styles-4.3.0.tgz", - "integrity": "sha512-zbB9rCJAT1rbjiVDb2hqKFHNYLxgtk8NURxZ3IZwD3F6NtxbXZQCnnSi1Lkx+IDohdPlFp222wVALIheZJQSEg==", - "requires": { - "color-convert": "^2.0.1" - } - }, - "color-convert": { - "version": "2.0.1", - "resolved": "https://registry.npmjs.org/color-convert/-/color-convert-2.0.1.tgz", - "integrity": "sha512-RRECPsj7iu/xb5oKYcsFHSppFNnsj/52OVTRKb4zP5onXwVF3zVmmToNcOfGC+CRDpfK/U584fMg38ZHCaElKQ==", - "requires": { - "color-name": "~1.1.4" - } - }, - "color-name": { - "version": "1.1.4", - "resolved": "https://registry.npmjs.org/color-name/-/color-name-1.1.4.tgz", - "integrity": "sha512-dOy+3AuW3a2wNbZHIuMZpTcgjGuLU/uBL/ubcZF9OXbDo8ff4O8yVp5Bf0efS8uEoYo5q4Fx7dY9OgQGXgAsQA==" - }, - "is-fullwidth-code-point": { - "version": "3.0.0", - "resolved": "https://registry.npmjs.org/is-fullwidth-code-point/-/is-fullwidth-code-point-3.0.0.tgz", - "integrity": "sha512-zymm5+u+sCsSWyD9qNaejV3DFvhCKclKdizYaJUuHA83RLjb7nSuGnddCHGv0hk+KY7BMAlsWeK4Ueg6EV6XQg==" - }, - "string-width": { - "version": "4.2.3", - "resolved": "https://registry.npmjs.org/string-width/-/string-width-4.2.3.tgz", - "integrity": "sha512-wKyQRQpjJ0sIp62ErSZdGsjMJWsap5oRNihHhu6G7JVO/9jIB6UyevL+tXuOqrng8j/cxKTWyWUwvSTriiZz/g==", - "requires": { - "emoji-regex": "^8.0.0", - "is-fullwidth-code-point": "^3.0.0", - "strip-ansi": "^6.0.1" - } - }, - "strip-ansi": { - "version": "6.0.1", - "resolved": "https://registry.npmjs.org/strip-ansi/-/strip-ansi-6.0.1.tgz", - "integrity": "sha512-Y38VPSHcqkFrCpFnQ9vuSXmquuv5oXOKpGeT6aGrr3o3Gc9AlVa6JBfUSOCnbxGGZF+/0ooI7KrPuUSztUdU5A==", - "requires": { - "ansi-regex": "^5.0.1" - } - } } }, "wrappy": { "version": "1.0.2", "resolved": "https://registry.npmjs.org/wrappy/-/wrappy-1.0.2.tgz", - "integrity": "sha1-tSQ9jz7BqjXxNkYFvA0QNuMKtp8=" + "integrity": "sha512-l4Sp/DRseor9wL6EvV2+TuQn63dMkPjZ/sp9XkghTEbV9KlPS1xUsZ3u7/IQO4wxtcFB4bgpQPRcR3QCvezPcQ==" }, "xml2js": { "version": "0.4.23", @@ -5908,9 +6149,9 @@ "devOptional": true }, "yargs": { - "version": "17.3.1", - "resolved": "https://registry.npmjs.org/yargs/-/yargs-17.3.1.tgz", - "integrity": "sha512-WUANQeVgjLbNsEmGk20f+nlHgOqzRFpiGWVaBrYGYIGANIIu3lWjoyi0fNlFmJkvfhCZ6BXINe7/W2O2bV4iaA==", + "version": "17.5.1", + "resolved": "https://registry.npmjs.org/yargs/-/yargs-17.5.1.tgz", + "integrity": "sha512-t6YAJcxDkNX7NFYiVtKvWUz8l+PaKTLiL63mJYWR2GnHq2gjEWISzsLp9wg3aY36dY1j+gfIEL3pIF+XlJJfbA==", "requires": { "cliui": "^7.0.2", "escalade": "^3.1.1", @@ -5919,42 +6160,12 @@ "string-width": "^4.2.3", "y18n": "^5.0.5", "yargs-parser": "^21.0.0" - }, - "dependencies": { - "ansi-regex": { - "version": "5.0.1", - "resolved": "https://registry.npmjs.org/ansi-regex/-/ansi-regex-5.0.1.tgz", - "integrity": "sha512-quJQXlTSUGL2LH9SUXo8VwsY4soanhgo6LNSm84E1LBcE8s3O0wpdiRzyR9z/ZZJMlMWv37qOOb9pdJlMUEKFQ==" - }, - "is-fullwidth-code-point": { - "version": "3.0.0", - "resolved": "https://registry.npmjs.org/is-fullwidth-code-point/-/is-fullwidth-code-point-3.0.0.tgz", - "integrity": "sha512-zymm5+u+sCsSWyD9qNaejV3DFvhCKclKdizYaJUuHA83RLjb7nSuGnddCHGv0hk+KY7BMAlsWeK4Ueg6EV6XQg==" - }, - "string-width": { - "version": "4.2.3", - "resolved": "https://registry.npmjs.org/string-width/-/string-width-4.2.3.tgz", - "integrity": "sha512-wKyQRQpjJ0sIp62ErSZdGsjMJWsap5oRNihHhu6G7JVO/9jIB6UyevL+tXuOqrng8j/cxKTWyWUwvSTriiZz/g==", - "requires": { - "emoji-regex": "^8.0.0", - "is-fullwidth-code-point": "^3.0.0", - "strip-ansi": "^6.0.1" - } - }, - "strip-ansi": { - "version": "6.0.1", - "resolved": "https://registry.npmjs.org/strip-ansi/-/strip-ansi-6.0.1.tgz", - "integrity": "sha512-Y38VPSHcqkFrCpFnQ9vuSXmquuv5oXOKpGeT6aGrr3o3Gc9AlVa6JBfUSOCnbxGGZF+/0ooI7KrPuUSztUdU5A==", - "requires": { - "ansi-regex": "^5.0.1" - } - } } }, "yargs-parser": { - "version": "21.0.1", - "resolved": "https://registry.npmjs.org/yargs-parser/-/yargs-parser-21.0.1.tgz", - "integrity": "sha512-9BK1jFpLzJROCI5TzwZL/TU4gqjK5xiHV/RfWLOahrjAko/e4DJkRDZQXfvqAsiZzzYhgAzbgz6lg48jcm4GLg==" + "version": "21.1.1", + "resolved": "https://registry.npmjs.org/yargs-parser/-/yargs-parser-21.1.1.tgz", + "integrity": "sha512-tVpsJW7DdjecAiFpbIB1e3qxIQsE6NoPc5/eTdrbbIC4h0LVsWhnoa3g+m2HclBIujHzsxZ4VJVA+GUuc2/LBw==" }, "yn": { "version": "3.1.1", @@ -5962,6 +6173,12 @@ "integrity": "sha512-Ux4ygGWsu2c7isFWe8Yu1YluJmqVhxqK2cLXNQA5AcC3QfbGNpM7fu0Y8b/z16pXLnFxZYvWhd3fhBY9DLmC6Q==", "dev": true }, + "yocto-queue": { + "version": "0.1.0", + "resolved": "https://registry.npmjs.org/yocto-queue/-/yocto-queue-0.1.0.tgz", + "integrity": "sha512-rVksvsnNCdJ/ohGc6xgPwyN8eheCxsiLM8mxuE/t/mOVqJewPuO1miLpTHQiRgTKCLexL4MeAFVagts7HmNZ2Q==", + "dev": true + }, "zen-observable": { "version": "0.8.15", "resolved": "https://registry.npmjs.org/zen-observable/-/zen-observable-0.8.15.tgz", diff --git a/package.json b/package.json index d009188..fc1ad1d 100644 --- a/package.json +++ b/package.json @@ -18,6 +18,7 @@ "start": "ts-node-dev src/scripts/dev.ts", "lint": "eslint --ext .ts src", "lint:fix": "eslint --ext .ts src --fix", + "namehash": "ts-node-dev --max-old-space-size=2048 src/scripts/name-hasher.ts", "unpack": "ts-node-dev --max-old-space-size=2048 src/scripts/unpacker.ts", "index": "ts-node-dev --max-old-space-size=2048 src/scripts/indexer.ts", "builds": "ts-node-dev src/scripts/builds.ts", @@ -53,7 +54,7 @@ "typescript": ">=4.5.0" }, "dependencies": { - "@runejs/common": "file:../common", + "@runejs/common": "2.0.2-beta.2", "adm-zip": "^0.5.9", "axios": "^0.27.2", "compressjs": "^1.0.3", @@ -79,7 +80,7 @@ "eslint": "^8.11.0", "rimraf": "^3.0.2", "source-map-support": "^0.5.21", - "ts-node-dev": "^1.1.8", + "ts-node-dev": "^2.0.0", "typescript": "^4.5.5" } } diff --git a/src/config/archive-config.ts b/src/config/archive-config.ts index d775549..e714e44 100644 --- a/src/config/archive-config.ts +++ b/src/config/archive-config.ts @@ -6,7 +6,7 @@ export interface ArchiveConfig { } -export interface JS5ArchiveConfig extends ArchiveConfig { +export interface Js5ArchiveConfig extends ArchiveConfig { encryption?: [ EncryptionMethod, string ]; contentType?: string; flattenGroups?: boolean; diff --git a/src/config/file-type.ts b/src/config/file-type.ts index dc6e3f2..6d34143 100644 --- a/src/config/file-type.ts +++ b/src/config/file-type.ts @@ -1,6 +1,11 @@ -export type FileType = +export type Js5FileType = 'FILE' | 'GROUP' | 'ARCHIVE' | + 'STORE'; + +export type JagFileType = + 'FILE' | + 'ARCHIVE' | 'STORE' | 'INDEX'; diff --git a/src/config/index.ts b/src/config/index.ts index e0116fa..9fcabeb 100644 --- a/src/config/index.ts +++ b/src/config/index.ts @@ -3,4 +3,4 @@ export * from './archive-format'; export * from './file-error'; export * from './file-type'; export * from './archive-flags'; -export * from './djb2'; +export * from './name-hasher'; diff --git a/src/config/djb2.ts b/src/config/name-hasher.ts similarity index 73% rename from src/config/djb2.ts rename to src/config/name-hasher.ts index a1cdddb..ba1d7b8 100644 --- a/src/config/djb2.ts +++ b/src/config/name-hasher.ts @@ -2,7 +2,8 @@ import { join } from 'path'; import { existsSync, readFileSync } from 'graceful-fs'; import { logger } from '@runejs/common'; -export class Djb2 { + +export class NameHasher { readonly configPath: string; readonly fileNameHashes: Map; @@ -13,7 +14,30 @@ export class Djb2 { this.loadFileNames(); } - hashFileName(fileName: string): number { + hashJagFileName(fileName: string): number { + const INT_MAX = 2147483648; + let hash = 0; + fileName = fileName.toUpperCase(); + + for (let i = 0; i < fileName.length; i++) { + hash = hash * 61 + fileName.charCodeAt(i) - 32; + + // Emulate Java's INT overflow-wrapping + while (hash > INT_MAX) { + const diff = hash - INT_MAX; + hash = -INT_MAX + diff; + } + + while (hash < -INT_MAX) { + const diff = Math.abs(hash) - INT_MAX; + hash = INT_MAX - diff; + } + } + + return hash; + } + + hashJs5FileName(fileName: string): number { if (!fileName) { return 0; } diff --git a/src/db/index-database.ts b/src/db/index-database.ts index 48b022f..9530e77 100644 --- a/src/db/index-database.ts +++ b/src/db/index-database.ts @@ -1,26 +1,58 @@ import { Connection, createConnection, LoggerOptions, Repository } from 'typeorm'; import { join } from 'path'; import { existsSync, mkdirSync } from 'graceful-fs'; -import { IndexEntity } from './index-entity'; -import { FileType } from '../config'; -export class IndexDatabase { +export abstract class IndexDatabase { - private readonly gameBuild: string; - private readonly databasePath: string; - private readonly loggerOptions: LoggerOptions; + protected readonly gameBuild: string; + protected readonly databasePath: string; + protected readonly loggerOptions: LoggerOptions; + protected readonly entityType: new () => ENTITY; - private _connection: Connection; - private _repository: Repository; + protected _connection: Connection; + protected _repository: Repository; - constructor(gameBuild: string, databasePath: string, loggerOptions: LoggerOptions = 'all') { + protected constructor( + gameBuild: string, + databasePath: string, + entityType: new () => ENTITY, + loggerOptions: LoggerOptions = 'all' + ) { this.gameBuild = gameBuild; this.databasePath = databasePath; + this.entityType = entityType; // [ 'error', 'warn' ], 'all', etc... this.loggerOptions = loggerOptions; } + abstract upsertIndexes(indexEntities: ENTITY[]): Promise; + + async getIndexes(where: WHERE): Promise { + return await this.repository.find({ + where: { ...where, gameBuild: this.gameBuild } + }) || []; + } + + async getIndex(where: WHERE): Promise { + return await this.repository.findOne({ + where: { ...where, gameBuild: this.gameBuild } + }) || null; + } + + async saveIndexes(indexEntities: ENTITY[]): Promise { + await this.repository.save(indexEntities as any, { + chunk: 500, + transaction: false, + reload: false, + listeners: false, + }); + } + + async saveIndex(indexEntity: ENTITY): Promise { + return await this.repository.save(indexEntity as any); + } + async openConnection(): Promise { if(!existsSync(this.databasePath)) { mkdirSync(this.databasePath, { recursive: true }); @@ -29,13 +61,13 @@ export class IndexDatabase { this._connection = await createConnection({ type: 'better-sqlite3', database: join(this.databasePath, `${this.gameBuild}.index.sqlite3`), - entities: [ IndexEntity ], + entities: [ this.entityType ], synchronize: true, logging: this.loggerOptions, name: 'index-repository' }); - this._repository = this._connection.getRepository(IndexEntity); + this._repository = this._connection.getRepository(this.entityType); return this._connection; } @@ -44,47 +76,11 @@ export class IndexDatabase { await this._connection.close(); } - async upsertIndexes(indexEntities: IndexEntity[]): Promise { - const chunkSize = 100; - for (let i = 0; i < indexEntities.length; i += chunkSize) { - const chunk = indexEntities.slice(i, i + chunkSize); - await this.repository.upsert(chunk, { - conflictPaths: [ 'fileType', 'gameBuild', 'key', 'archiveKey', 'groupKey', 'indexKey' ], - skipUpdateIfNoValuesChanged: true, - }); - } - } - - async saveIndexes(indexEntities: IndexEntity[]): Promise { - await this.repository.save(indexEntities, { - chunk: 500, - transaction: false, - reload: false, - listeners: false, - }); - } - - async saveIndex(indexEntity: IndexEntity): Promise { - return await this.repository.save(indexEntity); - } - - async getIndexes(fileType: FileType, archiveKey: number, groupKey: number = -1): Promise { - return await this.repository.find({ - where: { fileType, archiveKey, groupKey, gameBuild: this.gameBuild } - }); - } - - async getIndex(fileType: FileType, key: number, archiveKey: number, groupKey: number = -1): Promise { - return await this.repository.findOne({ - where: { fileType, key, archiveKey, groupKey, gameBuild: this.gameBuild } - }) || null; - } - get connection(): Connection { return this._connection; } - get repository(): Repository { + get repository(): Repository { return this._repository; } diff --git a/src/db/jag-database.ts b/src/db/jag-database.ts new file mode 100644 index 0000000..f960f79 --- /dev/null +++ b/src/db/jag-database.ts @@ -0,0 +1,36 @@ +import { IndexDatabase } from './index-database'; +import { JagIndexEntity } from './jag-index-entity'; +import { LoggerOptions } from 'typeorm'; +import { JagFileType } from '../config'; + + +export interface JagIndexEntityWhere { + fileType?: JagFileType; + key?: number; + indexKey?: number; + archiveKey?: number; +} + + +export class JagDatabase extends IndexDatabase { + + constructor( + gameBuild: string, + databasePath: string, + loggerOptions: LoggerOptions = 'all' + ) { + super(gameBuild, databasePath, JagIndexEntity, loggerOptions); + } + + override async upsertIndexes(indexEntities: JagIndexEntity[]): Promise { + const chunkSize = 100; + for (let i = 0; i < indexEntities.length; i += chunkSize) { + const chunk = indexEntities.slice(i, i + chunkSize); + await this.repository.upsert(chunk, { + conflictPaths: [ 'fileType', 'gameBuild', 'key', 'indexKey', 'archiveKey' ], + skipUpdateIfNoValuesChanged: true, + }); + } + } + +} diff --git a/src/db/jag-index-entity.ts b/src/db/jag-index-entity.ts new file mode 100644 index 0000000..18bd0b9 --- /dev/null +++ b/src/db/jag-index-entity.ts @@ -0,0 +1,76 @@ +import { Column, CreateDateColumn, Entity, Index, PrimaryColumn, UpdateDateColumn } from 'typeorm'; +import { CompressionMethod } from '@runejs/common/compress'; +import { FileError, JagFileType } from '../config'; +import { Buffer } from 'buffer'; + + +@Entity('jag_index') +@Index('index_identifier', [ + 'fileType', 'gameBuild', 'key', 'indexKey', 'archiveKey' +], { unique: true }) +export class JagIndexEntity { + + @PrimaryColumn('text', { name: 'file_type', nullable: false, unique: false }) + fileType: JagFileType; + + @PrimaryColumn('text', { name: 'game_build', nullable: false, unique: false }) + gameBuild: string; + + @PrimaryColumn('integer', { nullable: false, unique: false }) + key: number; + + @PrimaryColumn('integer', { name: 'index_key', nullable: false, unique: false }) + indexKey: number; + + @PrimaryColumn('integer', { name: 'archive_key', nullable: false, unique: false, default: -1 }) + archiveKey: number = -1; + + @Column('text', { nullable: true, default: null }) + name: string = null; + + @Column('integer', { name: 'name_hash', nullable: true, default: -1 }) + nameHash: number = -1; + + @Column('integer', { nullable: false, default: -1 }) + version: number = -1; + + @Column('integer', { name: 'child_count', nullable: false, default: 0 }) + childCount: number = 0; + + @Column('integer', { nullable: false, default: -1 }) + checksum: number = -1; + + @Column('text', { name: 'sha_digest', nullable: true, default: null }) + shaDigest: string = null; + + @Column('integer', { name: 'file_size', nullable: false, default: 0 }) + fileSize: number = 0; + + @Column('blob', { name: 'data', nullable: true, default: null }) + data: Buffer = null; + + @Column('text', { name: 'compression_method', nullable: true, default: null }) + compressionMethod: CompressionMethod = null; + + @Column('integer', { name: 'compressed_checksum', nullable: false, default: -1 }) + compressedChecksum: number = -1; + + @Column('text', { name: 'compressed_sha_digest', nullable: true, default: null }) + compressedShaDigest: string = null; + + @Column('integer', { name: 'compressed_file_size', nullable: false, default: 0 }) + compressedFileSize: number = 0; + + @Column('blob', { name: 'compressed_data', nullable: true, default: null }) + compressedData: Buffer = null; + + @Column('text', { name: 'file_error', nullable: true, default: null }) + fileError: FileError = null; + + @CreateDateColumn() + created?: Date; + + @UpdateDateColumn() + updated?: Date; + +} diff --git a/src/db/js5-database.ts b/src/db/js5-database.ts new file mode 100644 index 0000000..06596f3 --- /dev/null +++ b/src/db/js5-database.ts @@ -0,0 +1,36 @@ +import { IndexDatabase } from './index-database'; +import { Js5IndexEntity } from './js5-index-entity'; +import { LoggerOptions } from 'typeorm'; +import { Js5FileType } from '../config'; + + +export interface Js5IndexEntityWhere { + fileType?: Js5FileType; + key?: number; + archiveKey?: number; + groupKey?: number; +} + + +export class Js5Database extends IndexDatabase { + + constructor( + gameBuild: string, + databasePath: string, + loggerOptions: LoggerOptions = 'all' + ) { + super(gameBuild, databasePath, Js5IndexEntity, loggerOptions); + } + + override async upsertIndexes(indexEntities: Js5IndexEntity[]): Promise { + const chunkSize = 100; + for (let i = 0; i < indexEntities.length; i += chunkSize) { + const chunk = indexEntities.slice(i, i + chunkSize); + await this.repository.upsert(chunk, { + conflictPaths: [ 'fileType', 'gameBuild', 'key', 'archiveKey', 'groupKey' ], + skipUpdateIfNoValuesChanged: true, + }); + } + } + +} diff --git a/src/db/index-entity.ts b/src/db/js5-index-entity.ts similarity index 85% rename from src/db/index-entity.ts rename to src/db/js5-index-entity.ts index 17573d3..0cb9cc9 100644 --- a/src/db/index-entity.ts +++ b/src/db/js5-index-entity.ts @@ -1,17 +1,17 @@ import { Column, CreateDateColumn, Entity, Index, PrimaryColumn, UpdateDateColumn } from 'typeorm'; import { CompressionMethod } from '@runejs/common/compress'; -import { FileError, FileType } from '../config'; +import { FileError, Js5FileType } from '../config'; import { Buffer } from 'buffer'; -@Entity('file_index') +@Entity('js5_index') @Index('index_identifier', [ - 'fileType', 'gameBuild', 'key', 'archiveKey', 'groupKey', 'indexKey' + 'fileType', 'gameBuild', 'key', 'archiveKey', 'groupKey' ], { unique: true }) -export class IndexEntity { +export class Js5IndexEntity { @PrimaryColumn('text', { name: 'file_type', nullable: false, unique: false }) - fileType: FileType; + fileType: Js5FileType; @PrimaryColumn('text', { name: 'game_build', nullable: false, unique: false }) gameBuild: string; @@ -22,11 +22,6 @@ export class IndexEntity { @PrimaryColumn('integer', { name: 'archive_key', nullable: false, unique: false, default: -1 }) archiveKey: number = -1; - // For JAG format files - @PrimaryColumn('integer', { name: 'index_key', nullable: false, unique: false, default: -1 }) - indexKey: number = -1; - - // For JS5 format files @PrimaryColumn('integer', { name: 'group_key', nullable: false, unique: false, default: -1 }) groupKey: number = -1; @@ -36,8 +31,8 @@ export class IndexEntity { @Column('integer', { name: 'name_hash', nullable: true, default: -1 }) nameHash: number = -1; - @Column('integer', { nullable: false, default: 0 }) - version: number = 0; + @Column('integer', { nullable: false, default: -1 }) + version: number = -1; @Column('integer', { name: 'child_count', nullable: false, default: 0 }) childCount: number = 0; diff --git a/src/file-system/cache/cache-file.ts b/src/file-system/cache/cache-file.ts new file mode 100644 index 0000000..c0da70c --- /dev/null +++ b/src/file-system/cache/cache-file.ts @@ -0,0 +1,7 @@ +import { Buffer } from 'buffer'; + + +export interface CacheFile { + name: string; + data: Buffer; +} diff --git a/src/file-system/cache/cache-format.ts b/src/file-system/cache/cache-format.ts new file mode 100644 index 0000000..0725bd2 --- /dev/null +++ b/src/file-system/cache/cache-format.ts @@ -0,0 +1,12 @@ +import { CacheFile } from './cache-file'; + + +export const getCacheFormat = (cacheFiles: CacheFile[]): 'jag' | 'js5' | 'unknown' => { + if (cacheFiles.find(file => file.name === 'main_file_cache.idx255')) { + return 'js5'; + } else if (cacheFiles.find(file => file.name === 'main_file_cache.dat')) { + return 'jag'; + } + + return 'unknown'; +}; diff --git a/src/file-system/cache/index.ts b/src/file-system/cache/index.ts new file mode 100644 index 0000000..611793c --- /dev/null +++ b/src/file-system/cache/index.ts @@ -0,0 +1,2 @@ +export * from './cache-format'; +export * from './cache-file'; diff --git a/src/file-system/file-store-base.ts b/src/file-system/file-store-base.ts index f0fc0e5..850ef49 100644 --- a/src/file-system/file-store-base.ts +++ b/src/file-system/file-store-base.ts @@ -1,131 +1,34 @@ -import JSON5 from 'json5'; import { join } from 'path'; -import { existsSync, readFileSync } from 'graceful-fs'; import { Crc32 } from '@runejs/common/crc32'; -import { logger } from '@runejs/common'; -import { ArchiveConfig, Djb2 } from '../config'; +import { NameHasher } from '../config'; import { IndexDatabase } from '../db/index-database'; -import { IndexedFileBase } from './indexed-file-base'; -export abstract class FileStoreBase, C extends ArchiveConfig = ArchiveConfig> { +export abstract class FileStoreBase> { readonly gameBuild: string; readonly fileStorePath: string; - readonly archiveConfigFileName: string; - readonly archives: Map; - readonly djb2: Djb2; + readonly nameHasher: NameHasher; - private _archiveConfig: { [key: string]: C }; - private _database: IndexDatabase; + protected _database: DATABASE; - protected constructor(gameBuild: string | number, storePath: string, archiveConfigFileName: string) { + protected constructor(gameBuild: string | number, storePath: string) { this.gameBuild = String(gameBuild); - this.archiveConfigFileName = archiveConfigFileName; this.fileStorePath = storePath; - this.archives = new Map(); - this.loadArchiveConfig(); - this.djb2 = new Djb2(join(storePath, 'config')); + this.nameHasher = new NameHasher(join(storePath, 'config')); Crc32.init(); } - abstract load(): void | Promise; - - async loadArchiveIndexes(): Promise { - for (const [ , archive ] of this.archives) { - await archive.loadIndex(); - } - } - - loadArchiveConfig(): void { - const configPath = join(this.fileStorePath, 'config', this.archiveConfigFileName + '.json5'); - - if (!existsSync(configPath)) { - logger.error(`Error loading file store: ${configPath} was not found.`); - return; - } - - this._archiveConfig = JSON5.parse( - readFileSync(configPath, 'utf-8') - ) as { [key: string]: C }; + abstract openDatabase(): Promise; - if (!Object.values(this._archiveConfig)?.length) { - throw new Error(`Error reading archive configuration file. ` + - `Please ensure that the ${configPath} file exists and is valid.`); - } - } - - async openDatabase(): Promise { - this._database = new IndexDatabase( - this.gameBuild, - join(this.fileStorePath, 'index'), - [ 'error', 'warn' ], - ); - await this._database.openConnection(); - return this._database; - } + abstract load(): void | Promise; async closeDatabase(): Promise { await this._database.closeConnection(); } - getArchive(archiveKey: number): A | null; - getArchive(archiveName: string): A | null; - getArchive(archiveKeyOrName: number | string): A | null; - getArchive(archiveKeyOrName: number | string): A | null { - if (typeof archiveKeyOrName === 'string') { - return Array.from(this.archives.values()).find( - a => a?.index?.name === archiveKeyOrName - ) || null; - } else { - return this.archives.get(archiveKeyOrName) || null; - } - } - - setArchive(archiveKey: number, archive: A): void; - setArchive(archiveName: string, archive: A): void; - setArchive(archiveKeyOrName: number | string, archive: A): void; - setArchive(archiveKeyOrName: number | string, archive: A): void { - if (typeof archiveKeyOrName === 'string') { - const archiveConfig = this.getArchiveConfig(archiveKeyOrName); - if (archiveConfig) { - this.archives.set(archiveConfig.key, archive); - } else { - logger.error(`Archive ${ archiveKeyOrName } configuration was not found.`); - } - } else { - this.archives.set(archiveKeyOrName, archive); - } - } - - getArchiveConfig(archiveKey: number): C | null; - getArchiveConfig(archiveName: string): C | null; - getArchiveConfig(archiveKeyOrName: number | string): C | null; - getArchiveConfig(archiveKeyOrName: number | string): C | null { - if (typeof archiveKeyOrName === 'string') { - return this._archiveConfig[archiveKeyOrName] || null; - } else { - return Object.values(this._archiveConfig).find(c => c.key === archiveKeyOrName) || null; - } - } - - getArchiveName(archiveKey: number): string | null { - const archiveEntries = Object.entries(this._archiveConfig); - for (const [ name, config ] of archiveEntries) { - if (config.key === archiveKey) { - return name; - } - } - - return null; - } - - get archiveConfig(): { [key: string]: C } { - return this._archiveConfig; - } - - get database(): IndexDatabase { + get database(): DATABASE { return this._database; } diff --git a/src/file-system/jag/index.ts b/src/file-system/jag/index.ts index 8e9f7db..c9c5fcf 100644 --- a/src/file-system/jag/index.ts +++ b/src/file-system/jag/index.ts @@ -1,5 +1,5 @@ export * from './jag'; -export * from './jag-store'; +export * from './jag-file-store'; export * from './jag-archive'; export * from './jag-file'; export * from './jag-index'; diff --git a/src/file-system/jag/jag-archive.ts b/src/file-system/jag/jag-archive.ts index 3e4bdbb..1455923 100644 --- a/src/file-system/jag/jag-archive.ts +++ b/src/file-system/jag/jag-archive.ts @@ -1,22 +1,24 @@ -import { JagStore } from './jag-store'; -import { IndexedFileBase } from '../indexed-file-base'; -import { ArchiveConfig } from '../../config'; -import { indexes } from './jag'; +import { JagFileStore } from './jag-file-store'; +import { archives, indexes } from './jag'; import { JagFile } from './jag-file'; +import { JagFileBase } from './jag-file-base'; -export class JagArchive extends IndexedFileBase { +export class JagArchive extends JagFileBase { - readonly config: ArchiveConfig; readonly files: Map; constructor( - jagStore: JagStore, + jagStore: JagFileStore, archiveKey: number, ) { - super(jagStore, 'ARCHIVE', archiveKey, -1, -1, indexes.archives); - this.config = jagStore.getArchiveConfig(archiveKey); - this.index.name = jagStore.getArchiveName(archiveKey); + super(jagStore, 'ARCHIVE', archiveKey, indexes.archives, -1); + const archiveNames = Object.keys(archives); + for (const name of archiveNames) { + if (archives[name] === archiveKey) { + this.index.name = name; + } + } this.files = new Map(); } diff --git a/src/file-system/jag/jag-file-base.ts b/src/file-system/jag/jag-file-base.ts new file mode 100644 index 0000000..b86b39e --- /dev/null +++ b/src/file-system/jag/jag-file-base.ts @@ -0,0 +1,141 @@ +import { JagFileType } from '../../config'; +import { logger } from '@runejs/common'; +import { Buffer } from 'buffer'; +import { Crc32 } from '@runejs/common/crc32'; +import { createHash } from 'crypto'; +import { JagFileStore } from './jag-file-store'; +import { JagIndexEntity } from '../../db/jag-index-entity'; + + +export abstract class JagFileBase { + + readonly fileStore: JagFileStore; + + index: JagIndexEntity; + + protected constructor( + fileStore: JagFileStore, + fileType: JagFileType, + key: number, + indexKey: number = -1, + archiveKey: number = -1, + ) { + this.fileStore = fileStore; + this.index = new JagIndexEntity(); + this.index.gameBuild = fileStore.gameBuild; + this.index.fileType = fileType; + this.index.key = key; + this.index.indexKey = indexKey; + this.index.archiveKey = archiveKey; + } + + validate(trackChanges: boolean = true): void { + const { + data, compressedData, + checksum, shaDigest, fileSize, + compressedChecksum, compressedShaDigest, compressedFileSize, + name, nameHash, + } = this.index; + let fileModified: boolean = false; + + const currentChecksum = this.generateChecksum(data); + const currentShaDigest = this.generateShaDigest(data); + const currentFileSize = data?.length || 0; + + const currentCompressedChecksum = this.generateChecksum(compressedData); + const currentCompressedShaDigest = this.generateShaDigest(compressedData); + const currentCompressedFileSize = compressedData?.length || 0; + + if (name && nameHash === -1) { + // nameHash not set + this.index.nameHash = this.fileStore.nameHasher.hashJs5FileName(name); + } + + if (nameHash !== -1 && !name) { + // name not set + const lookupTableName = this.fileStore.nameHasher.findFileName(nameHash); + if (lookupTableName) { + this.index.name = lookupTableName; + } + } + + if (checksum !== currentChecksum) { + // uncompressed crc32 mismatch + this.index.checksum = currentChecksum; + fileModified = true; + } + + if (shaDigest !== currentShaDigest) { + // uncompressed sha256 mismatch + this.index.shaDigest = currentShaDigest; + fileModified = true; + } + + if (fileSize !== currentFileSize) { + // uncompressed file size mismatch + this.index.fileSize = currentFileSize; + fileModified = true; + } + + if (compressedChecksum !== currentCompressedChecksum) { + // compressed crc32 mismatch + this.index.compressedChecksum = currentCompressedChecksum; + fileModified = true; + } + + if (compressedShaDigest !== currentCompressedShaDigest) { + // compressed sha256 mismatch + this.index.compressedShaDigest = currentCompressedShaDigest; + fileModified = true; + } + + if (compressedFileSize !== currentCompressedFileSize) { + // compressed file size mismatch + this.index.compressedFileSize = currentCompressedFileSize; + fileModified = true; + } + + if (fileModified && trackChanges) { + logger.info(`File ${this.index.name || this.index.key} has been modified.`); + this.index.version++; + } + } + + async saveIndex(): Promise { + this.validate(); + this.index = await this.fileStore.database.saveIndex(this.index); + return this.index; + } + + async loadIndex(): Promise { + const indexEntity = await this.fileStore.database.getIndex({ + fileType: this.index.fileType, + key: this.index.key, + indexKey: this.index.indexKey, + archiveKey: this.index.archiveKey, + }); + + if (indexEntity) { + this.index = indexEntity; + } + + return this.index; + } + + generateChecksum(data: Buffer): number { + if (!data?.length) { + return -1; + } + + return Crc32.update(0, data.length, data); + } + + generateShaDigest(data: Buffer): string { + if (!data?.length) { + return null; + } + + return createHash('sha256').update(data).digest('hex'); + } + +} diff --git a/src/file-system/jag/jag-store.ts b/src/file-system/jag/jag-file-store.ts similarity index 71% rename from src/file-system/jag/jag-store.ts rename to src/file-system/jag/jag-file-store.ts index 84a9f62..ba47ee3 100644 --- a/src/file-system/jag/jag-store.ts +++ b/src/file-system/jag/jag-file-store.ts @@ -1,28 +1,35 @@ -import { JagArchive } from './jag-archive'; import { FileStoreBase } from '../file-store-base'; import { Jag, indexes } from './jag'; import { JagIndex } from './jag-index'; +import { JagIndexEntity } from '../../db/jag-index-entity'; +import { IndexDatabase } from '../../db/index-database'; +import { join } from 'path'; +import { JagDatabase } from '../../db/jag-database'; -export class JagStore extends FileStoreBase { +export class JagFileStore extends FileStoreBase { readonly jag: Jag; readonly indexes: Map; constructor(gameBuild: string | number, storePath: string = './') { - super(gameBuild, storePath, 'jag-cache-archives'); + super(gameBuild, storePath); this.jag = new Jag(this); this.indexes = new Map(); } - override async load(): Promise { - await this.openDatabase(); + override async openDatabase(): Promise> { + this._database = new JagDatabase( + this.gameBuild, + join(this.fileStorePath, 'index'), + [ 'error', 'warn' ], + ); + await this._database.openConnection(); + return this._database; } - createArchive(archiveKey: number): JagArchive { - const archive = new JagArchive(this, archiveKey); - this.setArchive(archiveKey, archive); - return archive; + override async load(): Promise { + await this.openDatabase(); } createIndex(indexKey: number): void { diff --git a/src/file-system/jag/jag-file.ts b/src/file-system/jag/jag-file.ts index cc0b07e..737f311 100644 --- a/src/file-system/jag/jag-file.ts +++ b/src/file-system/jag/jag-file.ts @@ -1,11 +1,11 @@ -import { IndexedFileBase } from '../indexed-file-base'; -import { JagStore } from './jag-store'; +import { JagFileStore } from './jag-file-store'; +import { JagFileBase } from './jag-file-base'; -export class JagFile extends IndexedFileBase { +export class JagFile extends JagFileBase { - constructor(jagStore: JagStore, fileKey: number, indexKey: number, archiveKey: number = -1) { - super(jagStore, 'FILE', fileKey, archiveKey, -1, indexKey); + constructor(jagStore: JagFileStore, fileKey: number, indexKey: number, archiveKey: number = -1) { + super(jagStore, 'FILE', fileKey, indexKey, archiveKey); } } diff --git a/src/file-system/jag/jag-index.ts b/src/file-system/jag/jag-index.ts index 8e18747..212d923 100644 --- a/src/file-system/jag/jag-index.ts +++ b/src/file-system/jag/jag-index.ts @@ -1,18 +1,17 @@ -import { IndexedFileBase } from '../indexed-file-base'; -import { JagStore } from './jag-store'; +import { JagFileStore } from './jag-file-store'; import { JagFileIndex, indexes } from './jag'; import { JagArchive } from './jag-archive'; import { JagFile } from './jag-file'; -import { JS5File } from '../js5'; +import { JagFileBase } from './jag-file-base'; -export class JagIndex extends IndexedFileBase { +export class JagIndex extends JagFileBase { readonly files: Map; fileIndexes: JagFileIndex[]; - constructor(jagStore: JagStore, indexKey: number) { + constructor(jagStore: JagFileStore, indexKey: number) { super(jagStore, 'INDEX', indexKey); const indexNames = Object.keys(indexes); for (const name of indexNames) { @@ -28,6 +27,12 @@ export class JagIndex extends IndexedFileBase { await this.fileStore.database.upsertIndexes(fileIndexes); } + createArchive(archiveKey: number): JagArchive { + const archive = new JagArchive(this.fileStore, archiveKey); + this.setArchive(archiveKey, archive); + return archive; + } + getArchive(archiveKey: number): JagArchive | null; getArchive(archiveName: string): JagArchive | null; getArchive(archiveKeyOrName: number | string): JagArchive | null; @@ -45,6 +50,10 @@ export class JagIndex extends IndexedFileBase { return (archive && archive instanceof JagArchive) ? archive : null; } + setArchive(archiveKey: number, archive: JagArchive): void { + this.files.set(archiveKey, archive); + } + getFile(fileKey: number): JagFile | null; getFile(fileName: string): JagFile | null; getFile(fileKeyOrName: number | string): JagFile | null; @@ -62,4 +71,8 @@ export class JagIndex extends IndexedFileBase { return (file && file instanceof JagFile) ? file : null; } + setFile(fileKey: number, file: JagFile): void { + this.files.set(fileKey, file); + } + } diff --git a/src/file-system/jag/jag.ts b/src/file-system/jag/jag.ts index ec5d76c..60d98b4 100644 --- a/src/file-system/jag/jag.ts +++ b/src/file-system/jag/jag.ts @@ -1,13 +1,13 @@ import { join } from 'path'; import { existsSync, readdirSync, readFileSync, statSync } from 'graceful-fs'; import { ByteBuffer, logger } from '@runejs/common'; -import { JagStore } from './jag-store'; -import { OpenRS2CacheFile } from '../../openrs2'; -import { IndexedFileBase } from '../indexed-file-base'; +import { JagFileStore } from './jag-file-store'; import { JagFile } from './jag-file'; import { Buffer } from 'buffer'; import { JagArchive } from './jag-archive'; import { decompressHeadlessBzip2 } from '../../compress'; +import { JagFileBase } from './jag-file-base'; +import { CacheFile } from '../cache'; const dataFileName = 'main_file_cache.dat'; @@ -23,6 +23,19 @@ export const indexes = { }; +export const archives = { + 'empty.jag': 0, + 'title.jag': 1, + 'config.jag': 2, + 'interface.jag': 3, + 'media.jag': 4, + 'versionlist.jag': 5, + 'textures.jag': 6, + 'wordenc.jag': 7, + 'sounds.jag': 8, +}; + + export interface JagFileIndex { fileSize: number; sectorNumber: number; @@ -39,17 +52,17 @@ export interface JagSectorHeader { export class Jag { - readonly jagStore: JagStore; + readonly jagStore: JagFileStore; private indexFiles: Map; private dataFile: ByteBuffer; - constructor(jagStore: JagStore) { + constructor(jagStore: JagFileStore) { this.jagStore = jagStore; this.indexFiles = new Map(); } - readOpenRS2CacheFiles(cacheFiles: OpenRS2CacheFile[]): void { + readOpenRS2CacheFiles(cacheFiles: CacheFile[]): void { const dataFileBuffer = cacheFiles.find(file => file.name === dataFileName)?.data || null; if (!dataFileBuffer?.length) { throw new Error(`The main ${ dataFileName } data file could not be found.`); @@ -155,10 +168,10 @@ export class Jag { fileSize, sectorNumber: sectorPos }; - let file: IndexedFileBase; + let file: JagFileBase; if (indexName === 'archives') { - file = this.jagStore.createArchive(fileKey); + file = new JagArchive(this.jagStore, fileKey); } else { file = new JagFile(this.jagStore, fileKey, indexKey); } @@ -189,10 +202,9 @@ export class Jag { const block = this.dataFile.getSlice(currentSectorNumber * sectorLength, readableSectorData); - const sectorFileKey = block.get('short', 'unsigned');//readUnsignedShortBE(); - const sectorFilePartNumber = block.get('short', 'unsigned');//readUnsignedShortBE(); - const sectorNumber = block.get('int24', 'unsigned');//(block.readUnsignedByte() << 16) - // | (block.readUnsignedByte() << 8) | block.readUnsignedByte(); + const sectorFileKey = block.get('short', 'unsigned'); + const sectorFilePartNumber = block.get('short', 'unsigned'); + const sectorNumber = block.get('int24', 'unsigned'); const sectorIndexKey = block.get('byte', 'unsigned'); readableSectorData -= 8; @@ -247,35 +259,84 @@ export class Jag { return file.index.compressedData; } - // @todo 18/07/22 - Kiko decodeArchive(archive: JagArchive): Buffer | null { - let archiveData = new ByteBuffer(archive.index.compressedData); + if (!archive.index.compressedData?.length) { + return null; + } - const uncompressed = archiveData.get('int24', 'unsigned'); - const compressed = archiveData.get('int24', 'unsigned'); + try { + let archiveData = new ByteBuffer(archive.index.compressedData); - if (uncompressed !== compressed) { - const compressedData = archiveData.getSlice(archiveData.readerIndex, archiveData.length - archiveData.readerIndex); - archiveData = new ByteBuffer(decompressHeadlessBzip2(compressedData)); - } + const uncompressed = archiveData.get('int24', 'unsigned'); + const compressed = archiveData.get('int24', 'unsigned'); + + if (uncompressed !== compressed) { + const compressedData = archiveData.getSlice(archiveData.readerIndex, archiveData.length - archiveData.readerIndex); + archiveData = new ByteBuffer(decompressHeadlessBzip2(compressedData)); + } - const fileCount = archiveData.get('short', 'unsigned'); - let fileDataOffset = archiveData.readerIndex + fileCount * 10; + const fileCount = archiveData.get('short', 'unsigned'); + const fileDataOffsets: number[] = new Array(fileCount); + archive.files.clear(); - for (let fileKey = 0; fileKey < fileCount; fileKey++) { - const fileNameHash = archiveData.get('int'); - const decompressedFileLength = archiveData.get('int24', 'unsigned'); - const compressedFileLength = archiveData.get('int24', 'unsigned'); - fileDataOffset += compressedFileLength; - } + let fileDataOffset = archiveData.readerIndex + fileCount * 10; - // @todo read file data and decompress files if needed - 18/07/22 - Kiko + // Read archive file headers + for (let fileKey = 0; fileKey < fileCount; fileKey++) { + const fileNameHash = archiveData.get('int'); + const fileName = this.jagStore.nameHasher.findFileName(fileNameHash); + const decompressedFileLength = archiveData.get('int24', 'unsigned'); + const compressedFileLength = archiveData.get('int24', 'unsigned'); + fileDataOffset += compressedFileLength; - if (archiveData.length) { - archive.index.data = archiveData.toNodeBuffer(); - } + fileDataOffsets[fileKey] = fileDataOffset; + + const file = new JagFile(this.jagStore, fileKey, archive.index.indexKey, archive.index.key); + file.index.nameHash = fileNameHash; + file.index.name = fileName; + file.index.fileSize = decompressedFileLength; + file.index.compressedFileSize = compressedFileLength; + + archive.files.set(fileKey, file); + } + + // Read archive file data + for (const [ fileKey, file ] of archive.files) { + try { + const fileDataOffset = fileDataOffsets[fileKey]; + const fileData = Buffer.alloc(file.index.compressedFileSize); + archiveData.copy(fileData, 0, fileDataOffset); + } catch (error) { + logger.error(`Error reading archive ${archive.index.name } file ${fileKey}`, error); + return null; + } + } + + // Decompress archive file data (if needed) + for (const [ fileKey, file ] of archive.files) { + try { + const { compressedFileSize, fileSize, compressedData } = file.index; + if (compressedData?.length && compressedFileSize !== fileSize) { + file.index.data = decompressHeadlessBzip2(file.index.compressedData); + if (file.index.data.length !== fileSize) { + file.index.fileSize = file.index.data.length; + } + } + } catch (error) { + logger.error(`Error decompressing archive ${archive.index.name } file ${fileKey}`, error); + return null; + } + } - return archive.index.data; + if (archiveData.length) { + archive.index.data = archiveData.toNodeBuffer(); + } + + return archive.index.data; + } catch (error) { + logger.error(`Error decoding archive ${archive.index.name }`, error); + return null; + } } } diff --git a/src/file-system/js5/js5-archive.ts b/src/file-system/js5/js5-archive.ts index 86b005b..55366cc 100644 --- a/src/file-system/js5/js5-archive.ts +++ b/src/file-system/js5/js5-archive.ts @@ -1,17 +1,17 @@ -import { JS5FileStore } from './js5-file-store'; +import { Js5FileStore } from './js5-file-store'; import { JS5Group } from './js5-group'; import { logger } from '@runejs/common'; -import { IndexedFileBase } from '../indexed-file-base'; -import { JS5ArchiveConfig } from '../../config'; +import { Js5FileBase } from './js5-file-base'; +import { Js5ArchiveConfig } from '../../config'; -export class JS5Archive extends IndexedFileBase { +export class Js5Archive extends Js5FileBase { readonly groups: Map; - readonly config: JS5ArchiveConfig; + readonly config: Js5ArchiveConfig; constructor( - fileStore: JS5FileStore, + fileStore: Js5FileStore, archiveKey: number, ) { super(fileStore, 'ARCHIVE', archiveKey, 255, -1); @@ -45,7 +45,10 @@ export class JS5Archive extends IndexedFileBase { } async loadGroupIndexes(): Promise { - const groupIndexes = await this.fileStore.database.getIndexes('GROUP', this.index.key); + const groupIndexes = await this.fileStore.database.getIndexes({ + fileType: 'GROUP', + archiveKey: this.index.key, + }); if (!groupIndexes?.length) { return; diff --git a/src/file-system/indexed-file-base.ts b/src/file-system/js5/js5-file-base.ts similarity index 82% rename from src/file-system/indexed-file-base.ts rename to src/file-system/js5/js5-file-base.ts index 0127d0c..112c80e 100644 --- a/src/file-system/indexed-file-base.ts +++ b/src/file-system/js5/js5-file-base.ts @@ -1,33 +1,32 @@ -import { IndexEntity } from '../db/index-entity'; -import { FileStoreBase } from './file-store-base'; -import { FileType } from '../config'; +import { Js5IndexEntity } from '../../db/js5-index-entity'; +import { Js5FileType } from '../../config'; import { logger } from '@runejs/common'; import { Buffer } from 'buffer'; import { Crc32 } from '@runejs/common/crc32'; import { createHash } from 'crypto'; +import { Js5FileStore } from './js5-file-store'; -export abstract class IndexedFileBase> { +export abstract class Js5FileBase { - readonly fileStore: S; - index: IndexEntity; + readonly fileStore: Js5FileStore; + + index: Js5IndexEntity; protected constructor( - fileStore: S, - fileType: FileType, + fileStore: Js5FileStore, + fileType: Js5FileType, key: number, archiveKey: number = -1, groupKey: number = -1, - indexKey: number = -1, ) { this.fileStore = fileStore; - this.index = new IndexEntity(); + this.index = new Js5IndexEntity(); this.index.gameBuild = fileStore.gameBuild; this.index.fileType = fileType; this.index.key = key; this.index.archiveKey = archiveKey; this.index.groupKey = groupKey; - this.index.indexKey = indexKey; } validate(trackChanges: boolean = true): void { @@ -49,12 +48,12 @@ export abstract class IndexedFileBase> { if (name && nameHash === -1) { // nameHash not set - this.index.nameHash = this.fileStore.djb2.hashFileName(name); + this.index.nameHash = this.fileStore.nameHasher.hashJs5FileName(name); } if (nameHash !== -1 && !name) { // name not set - const lookupTableName = this.fileStore.djb2.findFileName(nameHash); + const lookupTableName = this.fileStore.nameHasher.findFileName(nameHash); if (lookupTableName) { this.index.name = lookupTableName; } @@ -102,16 +101,19 @@ export abstract class IndexedFileBase> { } } - async saveIndex(): Promise { + async saveIndex(): Promise { this.validate(); this.index = await this.fileStore.database.saveIndex(this.index); return this.index; } - async loadIndex(): Promise { - const indexEntity = await this.fileStore.database.getIndex( - this.index.fileType, this.index.key, this.index.archiveKey - ); + async loadIndex(): Promise { + const indexEntity = await this.fileStore.database.getIndex({ + fileType: this.index.fileType, + key: this.index.key, + archiveKey: this.index.archiveKey, + groupKey: this.index.groupKey, + }); if (indexEntity) { this.index = indexEntity; diff --git a/src/file-system/js5/js5-file-store.ts b/src/file-system/js5/js5-file-store.ts index 3726fe3..1a92dec 100644 --- a/src/file-system/js5/js5-file-store.ts +++ b/src/file-system/js5/js5-file-store.ts @@ -1,25 +1,124 @@ -import { JS5ArchiveConfig } from '../../config'; +import JSON5 from 'json5'; +import { join } from 'path'; +import { existsSync, readFileSync } from 'graceful-fs'; +import { Js5ArchiveConfig } from '../../config'; import { JS5 } from './js5'; -import { JS5Archive } from './js5-archive'; +import { Js5Archive } from './js5-archive'; import { FileStoreBase } from '../file-store-base'; +import { logger } from '../../../../common'; +import { Js5Database } from '../../db/js5-database'; -export class JS5FileStore extends FileStoreBase{ +export class Js5FileStore extends FileStoreBase{ readonly js5: JS5; + readonly archives: Map; + + private _archiveConfig: { [key: string]: Js5ArchiveConfig }; constructor(gameBuild: string | number, storePath: string = './') { - super(gameBuild, storePath, 'js5-archives'); + super(gameBuild, storePath); + this.archives = new Map(); + this.loadArchiveConfig(); this.js5 = new JS5(this); } + override async openDatabase(): Promise { + this._database = new Js5Database( + this.gameBuild, + join(this.fileStorePath, 'index'), + [ 'error', 'warn' ], + ); + await this._database.openConnection(); + return this._database; + } + override async load(): Promise { await this.js5.loadEncryptionKeys(); await this.openDatabase(); } + async loadArchiveIndexes(): Promise { + for (const [ , archive ] of this.archives) { + await archive.loadIndex(); + } + } + + loadArchiveConfig(): void { + const configPath = join(this.fileStorePath, 'config', 'js5-archives.json5'); + + if (!existsSync(configPath)) { + logger.error(`Error loading file store: ${configPath} was not found.`); + return; + } + + this._archiveConfig = JSON5.parse( + readFileSync(configPath, 'utf-8') + ) as { [key: string]: Js5ArchiveConfig }; + + if (!Object.values(this._archiveConfig)?.length) { + throw new Error(`Error reading archive configuration file. ` + + `Please ensure that the ${configPath} file exists and is valid.`); + } + } + createArchive(archiveKey: number): void { - this.setArchive(archiveKey, new JS5Archive(this, archiveKey)); + this.setArchive(archiveKey, new Js5Archive(this, archiveKey)); + } + + getArchive(archiveKey: number): Js5Archive | null; + getArchive(archiveName: string): Js5Archive | null; + getArchive(archiveKeyOrName: number | string): Js5Archive | null; + getArchive(archiveKeyOrName: number | string): Js5Archive | null { + if (typeof archiveKeyOrName === 'string') { + return Array.from(this.archives.values()).find( + a => a?.index?.name === archiveKeyOrName + ) || null; + } else { + return this.archives.get(archiveKeyOrName) || null; + } + } + + setArchive(archiveKey: number, archive: Js5Archive): void; + setArchive(archiveName: string, archive: Js5Archive): void; + setArchive(archiveKeyOrName: number | string, archive: Js5Archive): void; + setArchive(archiveKeyOrName: number | string, archive: Js5Archive): void { + if (typeof archiveKeyOrName === 'string') { + const archiveConfig = this.getArchiveConfig(archiveKeyOrName); + if (archiveConfig) { + this.archives.set(archiveConfig.key, archive); + } else { + logger.error(`Archive ${ archiveKeyOrName } configuration was not found.`); + } + } else { + this.archives.set(archiveKeyOrName, archive); + } + } + + getArchiveConfig(archiveKey: number): Js5ArchiveConfig | null; + getArchiveConfig(archiveName: string): Js5ArchiveConfig | null; + getArchiveConfig(archiveKeyOrName: number | string): Js5ArchiveConfig | null; + getArchiveConfig(archiveKeyOrName: number | string): Js5ArchiveConfig | null { + if (typeof archiveKeyOrName === 'string') { + return this._archiveConfig[archiveKeyOrName] || null; + } else { + return Object.values(this._archiveConfig).find(c => c.key === archiveKeyOrName) || null; + } + } + + getArchiveName(archiveKey: number): string | null { + const archiveEntries = Object.entries(this._archiveConfig); + for (const [ name, config ] of archiveEntries) { + if (config.key === archiveKey) { + return name; + } + } + + return null; + } + + get archiveConfig(): { [key: string]: Js5ArchiveConfig } { + return this._archiveConfig; } } diff --git a/src/file-system/js5/js5-file.ts b/src/file-system/js5/js5-file.ts index bd25e39..2fc7b9f 100644 --- a/src/file-system/js5/js5-file.ts +++ b/src/file-system/js5/js5-file.ts @@ -1,16 +1,16 @@ -import { JS5FileStore } from './js5-file-store'; -import { JS5Archive } from './js5-archive'; +import { Js5FileStore } from './js5-file-store'; +import { Js5Archive } from './js5-archive'; import { JS5Group } from './js5-group'; -import { IndexedFileBase } from '../indexed-file-base'; +import { Js5FileBase } from './js5-file-base'; -export class JS5File extends IndexedFileBase { +export class JS5File extends Js5FileBase { - readonly archive: JS5Archive; + readonly archive: Js5Archive; readonly group: JS5Group; constructor( - fileStore: JS5FileStore, + fileStore: Js5FileStore, fileKey: number, group: JS5Group, ) { diff --git a/src/file-system/js5/js5-group.ts b/src/file-system/js5/js5-group.ts index 0fb97e4..3e7a451 100644 --- a/src/file-system/js5/js5-group.ts +++ b/src/file-system/js5/js5-group.ts @@ -1,19 +1,19 @@ -import { JS5FileStore } from './js5-file-store'; -import { JS5Archive } from './js5-archive'; +import { Js5FileStore } from './js5-file-store'; +import { Js5Archive } from './js5-archive'; import { JS5File } from './js5-file'; import { logger } from '@runejs/common'; -import { IndexedFileBase } from '../indexed-file-base'; +import { Js5FileBase } from './js5-file-base'; -export class JS5Group extends IndexedFileBase { +export class JS5Group extends Js5FileBase { - readonly archive: JS5Archive; + readonly archive: Js5Archive; readonly files: Map; constructor( - fileStore: JS5FileStore, + fileStore: Js5FileStore, groupKey: number, - archive: JS5Archive, + archive: Js5Archive, ) { super(fileStore, 'GROUP', groupKey, archive.index.key); this.archive = archive; @@ -45,9 +45,11 @@ export class JS5Group extends IndexedFileBase { } async loadFileIndexes(): Promise { - const fileIndexes = await this.fileStore.database.getIndexes( - 'FILE', this.archive.index.key, this.index.key - ); + const fileIndexes = await this.fileStore.database.getIndexes({ + fileType: 'FILE', + archiveKey: this.archive.index.key, + groupKey: this.index.key, + }); if (!fileIndexes?.length) { return; diff --git a/src/file-system/js5/js5.ts b/src/file-system/js5/js5.ts index 05685bd..102e92a 100644 --- a/src/file-system/js5/js5.ts +++ b/src/file-system/js5/js5.ts @@ -6,15 +6,16 @@ import { ByteBuffer, logger } from '@runejs/common'; import { getCompressionMethod } from '@runejs/common/compress'; import { Xtea, XteaKeys, XteaConfig } from '@runejs/common/encrypt'; -import { JS5FileStore, JS5Archive, JS5Group, JS5File } from '.'; +import { Js5FileStore, Js5Archive, JS5Group, JS5File } from '.'; import { archiveFlags, ArchiveFormat } from '../../config'; -import { getXteaKeysByBuild, OpenRS2CacheFile } from '../../openrs2'; +import { getXteaKeysByBuild } from '../../openrs2'; import { compressHeadlessBzip2, decompressHeadlessBzip2, compressGzip, decompressGzip } from '../../compress'; +import { CacheFile } from '../cache'; const dataFileName = 'main_file_cache.dat2'; @@ -24,7 +25,7 @@ const mainIndexFileName = `${ indexFileNamePrefix }255`; export class JS5 { - readonly fileStore: JS5FileStore; + readonly fileStore: Js5FileStore; localEncryptionKeys: Map; openRS2EncryptionKeys: XteaConfig[]; @@ -33,12 +34,12 @@ export class JS5 { private indexFiles: Map; private dataFile: ByteBuffer; - constructor(fileStore: JS5FileStore) { + constructor(fileStore: Js5FileStore) { this.fileStore = fileStore; this.indexFiles = new Map(); } - readOpenRS2CacheFiles(cacheFiles: OpenRS2CacheFile[]): void { + readOpenRS2CacheFiles(cacheFiles: CacheFile[]): void { const dataFileBuffer = cacheFiles.find(file => file.name === dataFileName)?.data || null; if (!dataFileBuffer?.length) { throw new Error(`The main ${ dataFileName } data file could not be found.`); @@ -135,11 +136,11 @@ export class JS5 { logger.info(`JS5 store file loaded for game build ${ this.fileStore.gameBuild }.`); } - unpack(file: JS5Group | JS5Archive): Buffer | null { + unpack(file: JS5Group | Js5Archive): Buffer | null { const fileIndex = file.index; const fileKey = fileIndex.key; - const archiveKey: number = file instanceof JS5Archive ? 255 : file.archive.index.key; - const archiveName: string = file instanceof JS5Archive ? 'main' : file.archive.index.name; + const archiveKey: number = file instanceof Js5Archive ? 255 : file.archive.index.key; + const archiveName: string = file instanceof Js5Archive ? 'main' : file.archive.index.name; const indexChannel: ByteBuffer = archiveKey !== 255 ? this.indexFiles.get(archiveKey) : this.mainIndexFile; @@ -240,7 +241,7 @@ export class JS5 { return fileIndex.compressedData; } - readCompressedFileHeader(file: JS5Group | JS5Archive): { compressedLength: number, readerIndex: number } { + readCompressedFileHeader(file: JS5Group | Js5Archive): { compressedLength: number, readerIndex: number } { const fileDetails = file.index; if (!fileDetails.compressedData?.length) { @@ -258,7 +259,7 @@ export class JS5 { return { compressedLength, readerIndex }; } - decrypt(file: JS5Group | JS5Archive): Buffer { + decrypt(file: JS5Group | Js5Archive): Buffer { const fileDetails = file.index; const fileName = fileDetails.name; @@ -269,7 +270,7 @@ export class JS5 { } // @todo move to JS5.decodeArchive - const archiveName = file instanceof JS5Archive ? 'main' : file.archive.index.name; + const archiveName = file instanceof Js5Archive ? 'main' : file.archive.index.name; const archiveConfig = this.fileStore.archiveConfig[archiveName]; if (archiveConfig.encryption) { @@ -302,6 +303,9 @@ export class JS5 { const keySet = keySets.find(keySet => keySet.gameBuild === gameBuild); if (Xtea.validKeys(keySet?.key)) { + logger.info(`XTEA decryption keys found for file ` + + `${ fileName || fileDetails.key }.`); + const dataCopy = encryptedData.clone(); dataCopy.readerIndex = readerIndex; @@ -332,7 +336,7 @@ export class JS5 { return null; } - decompress(file: JS5Group | JS5Archive): Buffer | null { + decompress(file: JS5Group | Js5Archive): Buffer | null { const fileDetails = file.index; if (!fileDetails.compressedData?.length) { @@ -499,7 +503,7 @@ export class JS5 { group.validate(false); } - async decodeArchive(archive: JS5Archive): Promise { + async decodeArchive(archive: Js5Archive): Promise { const archiveDetails = archive.index; if (archiveDetails.key === 255) { @@ -542,7 +546,7 @@ export class JS5 { if (flags.groupNames) { for (const group of groups) { group.index.nameHash = archiveData.get('int'); - group.index.name = this.fileStore.djb2.findFileName( + group.index.name = this.fileStore.nameHasher.findFileName( group.index.nameHash, group.index.name || String(group.index.nameHash) || String(group.index.key) ); @@ -605,7 +609,7 @@ export class JS5 { for (const group of groups) { for (const [ , flatFile ] of group.files) { flatFile.index.nameHash = archiveData.get('int'); - flatFile.index.name = this.fileStore.djb2.findFileName( + flatFile.index.name = this.fileStore.nameHasher.findFileName( flatFile.index.nameHash, flatFile.index.name || String(flatFile.index.nameHash) || String(flatFile.index.key) ); @@ -622,16 +626,16 @@ export class JS5 { } // @todo stubbed - 21/07/22 - Kiko - pack(file: JS5Group | JS5Archive): Buffer | null { + pack(file: JS5Group | Js5Archive): Buffer | null { return null; } // @todo stubbed - 21/07/22 - Kiko - encrypt(file: JS5Group | JS5Archive): Buffer | null { + encrypt(file: JS5Group | Js5Archive): Buffer | null { return null; } - compress(file: JS5Group | JS5Archive): Buffer | null { + compress(file: JS5Group | Js5Archive): Buffer | null { const fileDetails = file.index; if (!fileDetails.data?.length) { @@ -754,7 +758,7 @@ export class JS5 { } // @todo support newer archive fields & formats - 21/07/22 - Kiko - encodeArchive(archive: JS5Archive): Buffer | null { + encodeArchive(archive: Js5Archive): Buffer | null { const { groups: groupMap, index } = archive; const groups = Array.from(groupMap.values()); const groupCount = groups.length; diff --git a/src/openrs2/openrs2.ts b/src/openrs2/openrs2.ts index 99e3ffe..1c5639d 100644 --- a/src/openrs2/openrs2.ts +++ b/src/openrs2/openrs2.ts @@ -3,6 +3,7 @@ import AdmZip from 'adm-zip'; import { Buffer } from 'buffer'; import { logger } from '@runejs/common'; import { XteaConfig } from '@runejs/common/encrypt'; +import { CacheFile } from '../file-system/cache'; const openRS2Endpoint = 'https://archive.openrs2.org'; @@ -35,19 +36,6 @@ export interface OpenRS2Cache { } -export interface OpenRS2CacheFile { - name: string; - data: Buffer; -} - - -export const getCacheFormat = (files: OpenRS2CacheFile[]): 'js5' | 'jag' => { - return files.find( - file => file.name === 'main_file_cache.dat2' || file.name === 'main_file_cache.idx255' - ) !== null ? 'js5' : 'jag'; -}; - - export const getOpenRS2CacheList = async (): Promise => { const response = await axios.get( `${ openRS2Endpoint }/caches.json` @@ -109,7 +97,7 @@ export const getOpenRS2CacheDetailsByBuild = async ( export const getOpenRS2CacheFilesById = async ( id: number, scope: string = 'runescape' -): Promise => { +): Promise => { const response = await axios.get( `${ openRS2Endpoint }/caches/${ scope }/${ id }/disk.zip`, { responseType: 'arraybuffer' } @@ -129,7 +117,7 @@ export const getOpenRS2CacheFilesById = async ( export const getOpenRS2CacheFilesByBuild = async ( build: number -): Promise => { +): Promise => { logger.info(`Searching OpenRS2 for build ${ build }...`); const cacheList = (await getOpenRS2CacheList()) diff --git a/src/scripts/dev.ts b/src/scripts/dev.ts index c00a729..079bd74 100644 --- a/src/scripts/dev.ts +++ b/src/scripts/dev.ts @@ -1,10 +1,10 @@ -import { JS5FileStore } from '../file-system/js5/js5-file-store'; +import { Js5FileStore } from '../file-system/js5/js5-file-store'; import { logger } from '@runejs/common'; const dev = async () => { const start = Date.now(); - const fileStore = new JS5FileStore(435); + const fileStore = new Js5FileStore(435); await fileStore.load(); fileStore.js5.readLocalCacheFiles(); diff --git a/src/scripts/indexer.ts b/src/scripts/indexer.ts index 41040fe..b1531b9 100644 --- a/src/scripts/indexer.ts +++ b/src/scripts/indexer.ts @@ -2,18 +2,18 @@ import { join } from 'path'; import { existsSync, mkdirSync } from 'graceful-fs'; import { logger } from '@runejs/common'; import { ScriptExecutor, ArgumentOptions } from './script-executor'; -import { JS5FileStore } from '../file-system/js5'; -import { indexes, JagArchive, JagStore } from '../file-system/jag'; +import { Js5FileStore } from '../file-system/js5'; +import { indexes, JagArchive, JagFileStore } from '../file-system/jag'; import { getOpenRS2CacheFilesByBuild, - OpenRS2CacheFile } from '../openrs2'; import { fileTarget, prettyPrintTarget } from '../../../common/src'; +import { CacheFile, getCacheFormat } from '../file-system/cache'; interface IndexerOptions { dir: string; - format: 'flat' | 'js5' | 'jag'; + format: 'flat' | 'packed'; archive: string; build: string; } @@ -41,9 +41,9 @@ const indexerArgumentOptions: ArgumentOptions = { format: { alias: 'f', type: 'string', - default: 'js5', - choices: [ 'jag', 'js5', 'flat' ], - description: `The format of the store to index, either 'js5' (400+ JS5 format), 'jag' (234-399 .jag format), or 'flat' (flat files). Defaults to 'js5'.` + default: 'packed', + choices: [ 'packed', 'flat' ], + description: `The format of the store to index, either 'packed' (JAG or JS5 format) or 'flat' (flat files). Defaults to 'packed'.` }, source: { alias: 's', @@ -55,7 +55,7 @@ const indexerArgumentOptions: ArgumentOptions = { }; -const indexJS5Store = async (store: JS5FileStore) => { +const indexJS5Store = async (store: Js5FileStore) => { logger.info(`Unpacking archives from JS5 store...`); for (const [ , archive ] of store.archives) { @@ -65,13 +65,13 @@ const indexJS5Store = async (store: JS5FileStore) => { logger.info(`Decoding JS5 archives...`); for (const [ , archive ] of store.archives) { - await store.js5.decodeArchive(archive); + await store.js5.decodeArchive(archive).catch(e => logger.error(e)); } logger.info(`Saving archive indexes...`); for (const [ , archive ] of store.archives) { - await archive.saveIndex(); + await archive.saveIndex().catch(e => logger.error(e)); } logger.info(`Unpacking groups from JS5 store...`); @@ -88,7 +88,7 @@ const indexJS5Store = async (store: JS5FileStore) => { for (const [ , archive ] of store.archives) { for (const [ , group ] of archive.groups) { - await store.js5.decodeGroup(group); + await store.js5.decodeGroup(group).catch(e => logger.error(e)); } logger.info(`Finished decoding archive ${ archive.index.name } groups.`); @@ -97,20 +97,20 @@ const indexJS5Store = async (store: JS5FileStore) => { logger.info(`Saving group indexes...`); for (const [ , archive ] of store.archives) { - await archive.upsertGroupIndexes(); + await archive.upsertGroupIndexes().catch(e => logger.error(e)); } logger.info(`Saving flat file indexes...`); for (const [ , archive ] of store.archives) { for (const [ , group ] of archive.groups) { - await group.upsertFileIndexes(); + await group.upsertFileIndexes().catch(e => logger.error(e)); } } }; -const indexJS5Archive = async (store: JS5FileStore, archiveName: string) => { +const indexJS5Archive = async (store: Js5FileStore, archiveName: string) => { const archive = store.getArchive(archiveName); if (!archive) { @@ -124,11 +124,11 @@ const indexJS5Archive = async (store: JS5FileStore, archiveName: string) => { logger.info(`Decoding archive ${ archiveName }...`); - await store.js5.decodeArchive(archive); + await store.js5.decodeArchive(archive).catch(e => logger.error(e)); logger.info(`Saving archive ${ archiveName } index...`); - await archive.saveIndex(); + await archive.saveIndex().catch(e => logger.error(e)); logger.info(`Unpacking groups from archive ${ archiveName }...`); @@ -139,22 +139,22 @@ const indexJS5Archive = async (store: JS5FileStore, archiveName: string) => { logger.info(`Decoding archive ${ archiveName } groups...`); for (const [ , group ] of archive.groups) { - await store.js5.decodeGroup(group); + await store.js5.decodeGroup(group).catch(e => logger.error(e)); } logger.info(`Saving group indexes...`); - await archive.upsertGroupIndexes(); + await archive.upsertGroupIndexes().catch(e => logger.error(e)); logger.info(`Saving flat file indexes...`); for (const [ , group ] of archive.groups) { - await group.upsertFileIndexes(); + await group.upsertFileIndexes().catch(e => logger.error(e)); } }; -const indexJagStore = async (store: JagStore) => { +const indexJagStore = async (store: JagFileStore) => { logger.info(`Decoding JAG store indexes...`); const indexNames = Object.keys(indexes); @@ -165,7 +165,7 @@ const indexJagStore = async (store: JagStore) => { logger.info(`Saving indexes...`); for (const [ , indexFile ] of store.indexes) { - await indexFile.saveIndex(); + await indexFile.saveIndex().catch(e => logger.error(e)); } for (const [, indexFile ] of store.indexes) { @@ -180,9 +180,9 @@ const indexJagStore = async (store: JagStore) => { const archiveIndex = store.getIndex(indexes.archives); - // @todo unfinished - 02/08/22 - Kiko for (const [ , archive ] of archiveIndex.files) { if (archive instanceof JagArchive) { + logger.info(`Decoding archive ${archive.index.name}...`); store.jag.decodeArchive(archive); } } @@ -190,21 +190,20 @@ const indexJagStore = async (store: JagStore) => { logger.info(`Saving JAG file indexes...`); for (const [, index ] of store.indexes) { - await index.upsertFileIndexes(); + await index.upsertFileIndexes().catch(e => logger.error(e)); } logger.info(`Saving JAG archive file indexes...`); - // @todo unfinished - 02/08/22 - Kiko for (const [ , archive ] of archiveIndex.files) { if (archive instanceof JagArchive) { - await archive.upsertFileIndexes(); + await archive.upsertFileIndexes().catch(e => logger.error(e)); } } }; -const indexJagArchive = async (store: JagStore, archiveName: string) => { +const indexJagArchive = async (store: JagFileStore, archiveName: string) => { // @todo 18/07/22 - Kiko }; @@ -215,17 +214,16 @@ const indexerScript = async ( const start = Date.now(); const logDir = join(dir, 'logs'); const numericBuildNumber: number = /^\d+$/.test(build) ? parseInt(build, 10) : -1; - let cacheFiles: OpenRS2CacheFile[] | 'local' = 'local'; + let cacheFiles: CacheFile[] | 'local' = 'local'; if (!existsSync(logDir)) { mkdirSync(logDir, { recursive: true }); } - // logger.destination(join(logDir, `index-${ format }-${ build }.log`)); - logger.setTargets([ + /*logger.setTargets([ prettyPrintTarget(), - fileTarget(join(logDir, `index-${ format }-${ build }.log`)) - ]); + fileTarget(join(logDir, `index-${ build }.log`)) + ]);*/ if (source === 'openrs2') { if (numericBuildNumber) { @@ -241,10 +239,12 @@ const indexerScript = async ( } } - logger.info(`Indexing ${ format === 'flat' ? format : format.toUpperCase() } file store...`); + const storeType = cacheFiles !== 'local' ? getCacheFormat(cacheFiles) : 'flat'; - if (format === 'js5') { - const store = new JS5FileStore(numericBuildNumber !== -1 ? numericBuildNumber : build, dir); + logger.info(`Indexing ${ storeType === 'flat' ? storeType : storeType.toUpperCase() } file store...`); + + if (storeType === 'js5') { + const store = new Js5FileStore(numericBuildNumber !== -1 ? numericBuildNumber : build, dir); await store.load(); if (cacheFiles === 'local') { @@ -260,8 +260,8 @@ const indexerScript = async ( } await store.closeDatabase(); - } else if (format === 'jag') { - const store = new JagStore(numericBuildNumber !== -1 ? numericBuildNumber : build, dir); + } else if (storeType === 'jag') { + const store = new JagFileStore(numericBuildNumber !== -1 ? numericBuildNumber : build, dir); await store.load(); if (cacheFiles === 'local') { diff --git a/src/scripts/name-hasher.ts b/src/scripts/name-hasher.ts new file mode 100644 index 0000000..5cad08b --- /dev/null +++ b/src/scripts/name-hasher.ts @@ -0,0 +1,167 @@ +import { logger } from '@runejs/common'; +import { NameHasher } from '../config'; +import { join } from 'path'; + + +// @todo optimize this thing - 08/08/22 - Kiko +const bruteForcer = async () => { + const hashes = [ + 22834782, + -1857300557, + ]; + + const validChars = [ + 'A', 'B', 'C', 'D', 'E', 'F', 'G', 'H', 'I', + 'J', 'K', 'L', 'M', 'N', 'O', 'P', 'Q', 'R', + 'S', 'T', 'U', 'V', 'W', 'X', 'Y', 'Z', + '0', '1', '2', '3', '4', '5', '6', '7', '8', '9', + '_', '-' + ]; + + const validCharCodes = validChars.map(s => s.charCodeAt(0)); + + const INT_MAX = 2147483648; + + // Emulate Java's INT overflow-wrapping + const int32 = (value: number): number => { + while (value > INT_MAX) { + const diff = value - INT_MAX; + value = -INT_MAX + diff; + } + + while (value < -INT_MAX) { + const diff = Math.abs(value) - INT_MAX; + value = INT_MAX - diff; + } + + return value; + } + + const addToHash = (s: string, hash: number): number => { + for (let j = 0; j < s.length; j++) { + hash = int32((hash * 61 + s.charCodeAt(j)) - 32); + } + + return hash; + }; + + const getMatch = (hash: number): number => { + for (let i = 0; i < hashes.length; i++) { + if (hashes[i] === hash) { + return i; + } + } + + return -1; + }; + + const getHashForName = (dataName: string): number => { + let dataNameHash = 0; + for (let j = 0; j < dataName.length; j++) { + dataNameHash = int32((dataNameHash * 61 + dataName.charCodeAt(j)) - 32); + } + return dataNameHash; + }; + + const createString = (...charCodes: number[]): string => { + let result = ''; + for (const charCode of charCodes) { + const i = validCharCodes.indexOf(charCode); + result += validChars[i]; + } + + return result; + }; + + const bruteForceHash = (): void => { + let l1hash; + let l2hash; + let l3hash; + let l4hash; + let l5hash; + let l6hash; + let l7hash; + let l8hash; + let l9hash; + let hash; + + for (const c1 of validCharCodes) { + l1hash = c1 - 32; + for (const c2 of validCharCodes) { + l2hash = int32(l1hash * 61 + c2 - 32); + for (const c3 of validCharCodes) { + l3hash = int32(l2hash * 61 + c3 - 32); + console.log('First 3 characters (out of 9): ' + c1 + c2 + c3); + for (const c4 of validCharCodes) { + l4hash = int32(l3hash * 61 + c4 - 32); + for (const c5 of validCharCodes) { + l5hash = int32(l4hash * 61 + c5 - 32); + for (const c6 of validCharCodes) { + l6hash = int32(l5hash * 61 + c6 - 32); + for (const c7 of validCharCodes) { + l7hash = int32(l6hash * 61 + c7 - 32); + for (const c8 of validCharCodes) { + l8hash = int32(l7hash * 61 + c8 - 32); + for (const c9 of validCharCodes) { + l9hash = int32(l8hash * 61 + c9 - 32); + hash = addToHash(".DAT", l9hash); + const resultString = createString(c1, c2, c3, c4, c5, c6, c7, c8, c9); + if (getMatch(hash) !== -1) { + logger.info(resultString + '.DAT : ' + hash); + } + + hash = addToHash(".IDX", l9hash); + if (getMatch(hash) !== -1) { + logger.info(resultString + '.IDX : ' + hash); + } + if (getMatch(l9hash) !== -1) { + logger.info(resultString + ' : ' + hash); + } + } + } + } + } + } + } + } + } + } + } + + bruteForceHash(); +}; + + +const nameHasher = async () => { + const start = Date.now(); + + const hasher = new NameHasher(join('.', 'config')); + + const fileNames = [ + 'leftarrow_small.dat', + 'rightarrow_small.dat', + 'blackmark.dat', + 'button_brown.dat', + 'button_red.dat', + 'key.dat', + 'pen.dat', + 'startgame.dat', + 'titlescroll.dat', + 'letter.dat', + 'button_brown_big.dat', + 'overlay_duel.dat' + ]; + + const keyValueMap: { [key: string]: string } = {}; + + for (const fileName of fileNames) { + keyValueMap[String(hasher.hashJagFileName(fileName))] = fileName; + } + + console.log(JSON.stringify(keyValueMap, null, 4)); + + const end = Date.now(); + logger.info(`Operations completed in ${(end - start) / 1000} seconds.`); +}; + +nameHasher().catch(console.error); From ba0bcccf339f6f374bcc78129d46374a708c52b1 Mon Sep 17 00:00:00 2001 From: Kikorono Date: Mon, 8 Aug 2022 16:46:59 -0500 Subject: [PATCH 21/32] Adding missing file name/namehashes for caches 274 and 289 - hunt.dat + hunt.idx --- config/name-hashes.json | 4 +- src/scripts/name-hasher.ts | 86 +++++++++++++++++++------------------- 2 files changed, 47 insertions(+), 43 deletions(-) diff --git a/config/name-hashes.json b/config/name-hashes.json index 64a9d63..866426a 100644 --- a/config/name-hashes.json +++ b/config/name-hashes.json @@ -169240,5 +169240,7 @@ "-1857300557": "blackmark.dat", "-888498683": "button_red.dat", "-384541308": "titlescroll.dat", - "-90207845": "button_brown_big.dat" + "-90207845": "button_brown_big.dat", + "216316762": "hunt.dat", + "216335554": "hunt.idx" } diff --git a/src/scripts/name-hasher.ts b/src/scripts/name-hasher.ts index 5cad08b..e352606 100644 --- a/src/scripts/name-hasher.ts +++ b/src/scripts/name-hasher.ts @@ -6,8 +6,8 @@ import { join } from 'path'; // @todo optimize this thing - 08/08/22 - Kiko const bruteForcer = async () => { const hashes = [ - 22834782, - -1857300557, + 216335554, + 216316762, ]; const validChars = [ @@ -91,37 +91,49 @@ const bruteForcer = async () => { l2hash = int32(l1hash * 61 + c2 - 32); for (const c3 of validCharCodes) { l3hash = int32(l2hash * 61 + c3 - 32); - console.log('First 3 characters (out of 9): ' + c1 + c2 + c3); for (const c4 of validCharCodes) { l4hash = int32(l3hash * 61 + c4 - 32); - for (const c5 of validCharCodes) { - l5hash = int32(l4hash * 61 + c5 - 32); - for (const c6 of validCharCodes) { - l6hash = int32(l5hash * 61 + c6 - 32); - for (const c7 of validCharCodes) { - l7hash = int32(l6hash * 61 + c7 - 32); - for (const c8 of validCharCodes) { - l8hash = int32(l7hash * 61 + c8 - 32); - for (const c9 of validCharCodes) { - l9hash = int32(l8hash * 61 + c9 - 32); - hash = addToHash(".DAT", l9hash); - const resultString = createString(c1, c2, c3, c4, c5, c6, c7, c8, c9); - if (getMatch(hash) !== -1) { - logger.info(resultString + '.DAT : ' + hash); - } - - hash = addToHash(".IDX", l9hash); - if (getMatch(hash) !== -1) { - logger.info(resultString + '.IDX : ' + hash); - } - if (getMatch(l9hash) !== -1) { - logger.info(resultString + ' : ' + hash); - } - } - } - } - } + const resultString = createString(c1, c2, c3, c4); + hash = addToHash(".DAT", l4hash); + if (getMatch(hash) !== -1) { + logger.info(resultString + '.DAT : ' + hash); } + + hash = addToHash(".IDX", l4hash); + if (getMatch(hash) !== -1) { + logger.info(resultString + '.IDX : ' + hash); + } + if (getMatch(l4hash) !== -1) { + logger.info(resultString + ' : ' + hash); + } + // for (const c5 of validCharCodes) { + // l5hash = int32(l4hash * 61 + c5 - 32); + // for (const c6 of validCharCodes) { + // l6hash = int32(l5hash * 61 + c6 - 32); + // for (const c7 of validCharCodes) { + // l7hash = int32(l6hash * 61 + c7 - 32); + // for (const c8 of validCharCodes) { + // l8hash = int32(l7hash * 61 + c8 - 32); + // for (const c9 of validCharCodes) { + // l9hash = int32(l8hash * 61 + c9 - 32); + // hash = addToHash(".DAT", l9hash); + // const resultString = createString(c1, c2, c3, c4, c5, c6, c7, c8, c9); + // if (getMatch(hash) !== -1) { + // logger.info(resultString + '.DAT : ' + hash); + // } + // + // hash = addToHash(".IDX", l9hash); + // if (getMatch(hash) !== -1) { + // logger.info(resultString + '.IDX : ' + hash); + // } + // if (getMatch(l9hash) !== -1) { + // logger.info(resultString + ' : ' + hash); + // } + // } + // } + // } + // } + // } } } } @@ -138,18 +150,8 @@ const nameHasher = async () => { const hasher = new NameHasher(join('.', 'config')); const fileNames = [ - 'leftarrow_small.dat', - 'rightarrow_small.dat', - 'blackmark.dat', - 'button_brown.dat', - 'button_red.dat', - 'key.dat', - 'pen.dat', - 'startgame.dat', - 'titlescroll.dat', - 'letter.dat', - 'button_brown_big.dat', - 'overlay_duel.dat' + 'hunt.dat', + 'hunt.idx', ]; const keyValueMap: { [key: string]: string } = {}; From d073b3db35602d8fc0fede9687dfff204ea1961d Mon Sep 17 00:00:00 2001 From: Kikorono Date: Mon, 8 Aug 2022 17:31:07 -0500 Subject: [PATCH 22/32] Finishing up JAG cache indexing by adding the remaining missing DB fields --- src/file-system/jag/jag.ts | 124 +++++++++++++++++++------------------ 1 file changed, 64 insertions(+), 60 deletions(-) diff --git a/src/file-system/jag/jag.ts b/src/file-system/jag/jag.ts index 60d98b4..8c000d2 100644 --- a/src/file-system/jag/jag.ts +++ b/src/file-system/jag/jag.ts @@ -264,79 +264,83 @@ export class Jag { return null; } - try { - let archiveData = new ByteBuffer(archive.index.compressedData); + let archiveData = new ByteBuffer(archive.index.compressedData); - const uncompressed = archiveData.get('int24', 'unsigned'); - const compressed = archiveData.get('int24', 'unsigned'); + const uncompressed = archiveData.get('int24', 'unsigned'); + const compressed = archiveData.get('int24', 'unsigned'); - if (uncompressed !== compressed) { - const compressedData = archiveData.getSlice(archiveData.readerIndex, archiveData.length - archiveData.readerIndex); - archiveData = new ByteBuffer(decompressHeadlessBzip2(compressedData)); - } - - const fileCount = archiveData.get('short', 'unsigned'); - const fileDataOffsets: number[] = new Array(fileCount); - archive.files.clear(); - - let fileDataOffset = archiveData.readerIndex + fileCount * 10; + if (uncompressed !== compressed) { + const compressedData = archiveData.getSlice(archiveData.readerIndex, archiveData.length - archiveData.readerIndex); + archiveData = new ByteBuffer(decompressHeadlessBzip2(compressedData)); + archive.index.compressionMethod = 'bzip'; + } else { + archive.index.compressionMethod = 'none'; + } - // Read archive file headers - for (let fileKey = 0; fileKey < fileCount; fileKey++) { - const fileNameHash = archiveData.get('int'); - const fileName = this.jagStore.nameHasher.findFileName(fileNameHash); - const decompressedFileLength = archiveData.get('int24', 'unsigned'); - const compressedFileLength = archiveData.get('int24', 'unsigned'); - fileDataOffset += compressedFileLength; + const fileCount = archiveData.get('short', 'unsigned'); + archive.index.childCount = fileCount; + archive.files.clear(); - fileDataOffsets[fileKey] = fileDataOffset; + const fileDataOffsets: number[] = new Array(fileCount); + let fileDataOffset = archiveData.readerIndex + fileCount * 10; - const file = new JagFile(this.jagStore, fileKey, archive.index.indexKey, archive.index.key); - file.index.nameHash = fileNameHash; - file.index.name = fileName; - file.index.fileSize = decompressedFileLength; - file.index.compressedFileSize = compressedFileLength; + // Read archive file headers + for (let fileKey = 0; fileKey < fileCount; fileKey++) { + const fileNameHash = archiveData.get('int'); + const fileName = this.jagStore.nameHasher.findFileName(fileNameHash); + const decompressedFileLength = archiveData.get('int24', 'unsigned'); + const compressedFileLength = archiveData.get('int24', 'unsigned'); + fileDataOffsets[fileKey] = fileDataOffset; + fileDataOffset += compressedFileLength; + + const file = new JagFile(this.jagStore, fileKey, archive.index.indexKey, archive.index.key); + file.index.nameHash = fileNameHash; + file.index.name = fileName; + file.index.fileSize = decompressedFileLength; + file.index.compressedFileSize = compressedFileLength; + + archive.files.set(fileKey, file); + } - archive.files.set(fileKey, file); - } - - // Read archive file data - for (const [ fileKey, file ] of archive.files) { - try { - const fileDataOffset = fileDataOffsets[fileKey]; - const fileData = Buffer.alloc(file.index.compressedFileSize); - archiveData.copy(fileData, 0, fileDataOffset); - } catch (error) { - logger.error(`Error reading archive ${archive.index.name } file ${fileKey}`, error); - return null; - } + // Read archive file data + for (const [ fileKey, file ] of archive.files) { + try { + const fileDataOffset = fileDataOffsets[fileKey]; + const fileData = Buffer.alloc(file.index.compressedFileSize); + archiveData.copy(fileData, 0, fileDataOffset); + file.index.compressedData = fileData; + } catch (error) { + logger.error(`Error reading archive ${archive.index.name } file ${fileKey}`, error); } + } - // Decompress archive file data (if needed) - for (const [ fileKey, file ] of archive.files) { - try { - const { compressedFileSize, fileSize, compressedData } = file.index; - if (compressedData?.length && compressedFileSize !== fileSize) { - file.index.data = decompressHeadlessBzip2(file.index.compressedData); - if (file.index.data.length !== fileSize) { - file.index.fileSize = file.index.data.length; - } - } - } catch (error) { - logger.error(`Error decompressing archive ${archive.index.name } file ${fileKey}`, error); - return null; + // Decompress archive file data (if needed) + for (const [ fileKey, file ] of archive.files) { + try { + const { compressedFileSize, fileSize, compressedData } = file.index; + if (compressedData?.length && compressedFileSize !== fileSize) { + file.index.data = decompressHeadlessBzip2(file.index.compressedData); + file.index.compressionMethod = 'bzip'; + } else { + file.index.compressionMethod = 'none'; } + } catch (error) { + logger.error(`Error decompressing archive ${archive.index.name } file ${fileKey}`, error); } + } - if (archiveData.length) { - archive.index.data = archiveData.toNodeBuffer(); - } + // Validate each file + for (const [ , file ] of archive.files) { + file.validate(false); + } - return archive.index.data; - } catch (error) { - logger.error(`Error decoding archive ${archive.index.name }`, error); - return null; + if (archiveData.length) { + archive.index.data = archiveData.toNodeBuffer(); } + + archive.validate(false); + + return archive.index.data; } } From 2a1943ee5230cf65e1164611bcf8bebe0221b285 Mon Sep 17 00:00:00 2001 From: Kikorono Date: Fri, 12 Aug 2022 13:54:40 -0500 Subject: [PATCH 23/32] Prototyping out some parsers & decoders for JAG format files --- .../jag/content/animations/animations.ts | 98 +++++++++++++++++++ .../content/archives/interfaces/interfaces.ts | 3 + src/file-system/jag/index.ts | 1 + src/file-system/jag/jag-archive.ts | 17 ++++ src/file-system/js5/index.ts | 1 + 5 files changed, 120 insertions(+) create mode 100644 src/file-system/jag/content/animations/animations.ts create mode 100644 src/file-system/jag/content/archives/interfaces/interfaces.ts diff --git a/src/file-system/jag/content/animations/animations.ts b/src/file-system/jag/content/animations/animations.ts new file mode 100644 index 0000000..d87c1cf --- /dev/null +++ b/src/file-system/jag/content/animations/animations.ts @@ -0,0 +1,98 @@ +import { JagFileStore } from '../../jag-file-store'; +import { Buffer } from 'buffer'; +import { ByteBuffer, logger } from '@runejs/common'; +import { JagIndex } from '../../jag-index'; + + +export interface AnimationFile { + key: number; + version: number; + checksum: number; + data?: Buffer; +} + + +export class Animations { + + readonly jagStore: JagFileStore; + readonly animations: Map; + readonly animationsIndex: JagIndex; + + versionListDecoded: boolean = false; + + constructor(jagStore: JagFileStore) { + this.jagStore = jagStore; + this.animations = new Map(); + this.animationsIndex = this.jagStore.getIndex('animations'); + } + + decodeVersionList(): void { + const archiveIndex = this.jagStore.getIndex('archives'); + if (!archiveIndex) { + throw new Error(`Archive Index is not loaded!`); + } + + const versionListArchive = archiveIndex.getArchive('versionlist.jag'); + if (!versionListArchive) { + throw new Error(`versionlist.jag archive is not loaded!`); + } + + const animVersionList = versionListArchive.getFile('anim_version'); + const animChecksumList = versionListArchive.getFile('anim_crc'); + const animIndexList = versionListArchive.getFile('anim_index'); + + if (!animVersionList?.index?.data) { + throw new Error(`anim_version file is not loaded!`); + } + if (!animChecksumList?.index?.data) { + throw new Error(`anim_crc file is not loaded!`); + } + if (!animIndexList?.index?.data) { + throw new Error(`anim_index file is not loaded!`); + } + + this.animations.clear(); + + const versionData = new ByteBuffer(animVersionList.index.data); + const checksumData = new ByteBuffer(animVersionList.index.data); + const indexData = new ByteBuffer(animVersionList.index.data); + const animCount = versionData.length / 2; + + for (let i = 0; i < animCount; i++) { + const version = versionData.get('short', 'unsigned', 'le'); + const checksum = checksumData.get('int'); + const key = indexData.get('short', 'unsigned', 'le'); + + this.animations.set(key, { + key, version, checksum + }); + } + + this.versionListDecoded = true; + } + + decodeAll(): void { + if (!this.versionListDecoded) { + this.decodeVersionList(); + } + + for (const [ animKey, ] of this.animationsIndex.files) { + this.decode(animKey); + } + } + + decode(animKey: number): AnimationFile | null { + const animFile = this.animationsIndex.getFile(animKey); + + if (!animFile?.index?.data) { + logger.warn(`Animation ${animKey} is empty or missing.`); + return null; + } + + const animData = new ByteBuffer(animFile.index.data); + + // @todo stopped here - 12/08/22 - Kiko + return null; + } + +} diff --git a/src/file-system/jag/content/archives/interfaces/interfaces.ts b/src/file-system/jag/content/archives/interfaces/interfaces.ts new file mode 100644 index 0000000..5a50e69 --- /dev/null +++ b/src/file-system/jag/content/archives/interfaces/interfaces.ts @@ -0,0 +1,3 @@ +export class Interfaces { + +} diff --git a/src/file-system/jag/index.ts b/src/file-system/jag/index.ts index c9c5fcf..6aa0930 100644 --- a/src/file-system/jag/index.ts +++ b/src/file-system/jag/index.ts @@ -1,4 +1,5 @@ export * from './jag'; +export * from './jag-file-base'; export * from './jag-file-store'; export * from './jag-archive'; export * from './jag-file'; diff --git a/src/file-system/jag/jag-archive.ts b/src/file-system/jag/jag-archive.ts index 1455923..ae93681 100644 --- a/src/file-system/jag/jag-archive.ts +++ b/src/file-system/jag/jag-archive.ts @@ -27,4 +27,21 @@ export class JagArchive extends JagFileBase { await this.fileStore.database.upsertIndexes(fileIndexes); } + getFile(fileKey: number): JagFile | null; + getFile(fileName: string): JagFile | null; + getFile(fileKeyOrName: number | string): JagFile | null; + getFile(fileKeyOrName: number | string): JagFile | null { + if (typeof fileKeyOrName === 'string') { + return Array.from(this.files.values()).find( + file => file?.index?.name === fileKeyOrName + ) || null; + } else { + return this.files.get(fileKeyOrName) || null; + } + } + + setFile(fileKey: number, file: JagFile): void { + this.files.set(fileKey, file); + } + } diff --git a/src/file-system/js5/index.ts b/src/file-system/js5/index.ts index 75f732a..0832e68 100644 --- a/src/file-system/js5/index.ts +++ b/src/file-system/js5/index.ts @@ -1,4 +1,5 @@ export * from './js5'; +export * from './js5-file-base'; export * from './js5-file-store'; export * from './js5-archive'; export * from './js5-group'; From 4c5728ad8f6ebddaf4053639b9ded2d1f1351ed7 Mon Sep 17 00:00:00 2001 From: Kikorono Date: Fri, 12 Aug 2022 14:09:58 -0500 Subject: [PATCH 24/32] Stubbing out some JAG game interface decoder code --- .../jag/content/animations/animations.ts | 2 +- .../archives/interfaces/interface-archive.ts | 27 +++++++++++++++++++ .../content/archives/interfaces/interfaces.ts | 3 --- 3 files changed, 28 insertions(+), 4 deletions(-) create mode 100644 src/file-system/jag/content/archives/interfaces/interface-archive.ts delete mode 100644 src/file-system/jag/content/archives/interfaces/interfaces.ts diff --git a/src/file-system/jag/content/animations/animations.ts b/src/file-system/jag/content/animations/animations.ts index d87c1cf..c3926d5 100644 --- a/src/file-system/jag/content/animations/animations.ts +++ b/src/file-system/jag/content/animations/animations.ts @@ -91,7 +91,7 @@ export class Animations { const animData = new ByteBuffer(animFile.index.data); - // @todo stopped here - 12/08/22 - Kiko + //@todo stopped here - 12/08/22 - Kiko return null; } diff --git a/src/file-system/jag/content/archives/interfaces/interface-archive.ts b/src/file-system/jag/content/archives/interfaces/interface-archive.ts new file mode 100644 index 0000000..8a61fc6 --- /dev/null +++ b/src/file-system/jag/content/archives/interfaces/interface-archive.ts @@ -0,0 +1,27 @@ +import { JagFileStore } from '../../../jag-file-store'; +import { JagArchive } from '../../../jag-archive'; + + +export interface JagGameInterface { + key: number; +} + + +export class InterfaceArchive { + + readonly jagStore: JagFileStore; + readonly interfaces: Map; + readonly archive: JagArchive; + + constructor(jagStore: JagFileStore) { + this.jagStore = jagStore; + this.interfaces = new Map(); + this.archive = this.jagStore.getIndex('archives') + .getArchive('interface.jag'); + } + + decodeAll(): void { + //@todo stopped here - 12/08/22 - Kiko + } + +} diff --git a/src/file-system/jag/content/archives/interfaces/interfaces.ts b/src/file-system/jag/content/archives/interfaces/interfaces.ts deleted file mode 100644 index 5a50e69..0000000 --- a/src/file-system/jag/content/archives/interfaces/interfaces.ts +++ /dev/null @@ -1,3 +0,0 @@ -export class Interfaces { - -} From c1037f709efa20b467748bae4ca1aa6d1d148b33 Mon Sep 17 00:00:00 2001 From: Kikorono Date: Sat, 13 Aug 2022 13:52:34 -0500 Subject: [PATCH 25/32] Prototyping JAG interface decoding. Also adds support for JAG index entity reading from the database. --- src/db/{ => jag}/jag-database.ts | 4 +- src/db/{ => jag}/jag-index-entity.ts | 2 +- src/db/{ => js5}/js5-database.ts | 4 +- src/db/{ => js5}/js5-index-entity.ts | 2 +- .../jag/content/animations/animations.ts | 4 +- .../archives/interfaces/interface-archive.ts | 275 +++++++++++++++++- src/file-system/jag/jag-archive.ts | 22 ++ src/file-system/jag/jag-file-base.ts | 2 +- src/file-system/jag/jag-file-store.ts | 10 +- src/file-system/jag/jag-index.ts | 28 ++ src/file-system/js5/js5-file-base.ts | 2 +- src/file-system/js5/js5-file-store.ts | 4 +- 12 files changed, 342 insertions(+), 17 deletions(-) rename src/db/{ => jag}/jag-database.ts (91%) rename src/db/{ => jag}/jag-index-entity.ts (97%) rename src/db/{ => js5}/js5-database.ts (91%) rename src/db/{ => js5}/js5-index-entity.ts (98%) diff --git a/src/db/jag-database.ts b/src/db/jag/jag-database.ts similarity index 91% rename from src/db/jag-database.ts rename to src/db/jag/jag-database.ts index f960f79..cb87659 100644 --- a/src/db/jag-database.ts +++ b/src/db/jag/jag-database.ts @@ -1,7 +1,7 @@ -import { IndexDatabase } from './index-database'; +import { IndexDatabase } from '../index-database'; import { JagIndexEntity } from './jag-index-entity'; import { LoggerOptions } from 'typeorm'; -import { JagFileType } from '../config'; +import { JagFileType } from '../../config'; export interface JagIndexEntityWhere { diff --git a/src/db/jag-index-entity.ts b/src/db/jag/jag-index-entity.ts similarity index 97% rename from src/db/jag-index-entity.ts rename to src/db/jag/jag-index-entity.ts index 18bd0b9..d2241b8 100644 --- a/src/db/jag-index-entity.ts +++ b/src/db/jag/jag-index-entity.ts @@ -1,6 +1,6 @@ import { Column, CreateDateColumn, Entity, Index, PrimaryColumn, UpdateDateColumn } from 'typeorm'; import { CompressionMethod } from '@runejs/common/compress'; -import { FileError, JagFileType } from '../config'; +import { FileError, JagFileType } from '../../config'; import { Buffer } from 'buffer'; diff --git a/src/db/js5-database.ts b/src/db/js5/js5-database.ts similarity index 91% rename from src/db/js5-database.ts rename to src/db/js5/js5-database.ts index 06596f3..6965163 100644 --- a/src/db/js5-database.ts +++ b/src/db/js5/js5-database.ts @@ -1,7 +1,7 @@ -import { IndexDatabase } from './index-database'; +import { IndexDatabase } from '../index-database'; import { Js5IndexEntity } from './js5-index-entity'; import { LoggerOptions } from 'typeorm'; -import { Js5FileType } from '../config'; +import { Js5FileType } from '../../config'; export interface Js5IndexEntityWhere { diff --git a/src/db/js5-index-entity.ts b/src/db/js5/js5-index-entity.ts similarity index 98% rename from src/db/js5-index-entity.ts rename to src/db/js5/js5-index-entity.ts index 0cb9cc9..85e0deb 100644 --- a/src/db/js5-index-entity.ts +++ b/src/db/js5/js5-index-entity.ts @@ -1,6 +1,6 @@ import { Column, CreateDateColumn, Entity, Index, PrimaryColumn, UpdateDateColumn } from 'typeorm'; import { CompressionMethod } from '@runejs/common/compress'; -import { FileError, Js5FileType } from '../config'; +import { FileError, Js5FileType } from '../../config'; import { Buffer } from 'buffer'; diff --git a/src/file-system/jag/content/animations/animations.ts b/src/file-system/jag/content/animations/animations.ts index c3926d5..b1db660 100644 --- a/src/file-system/jag/content/animations/animations.ts +++ b/src/file-system/jag/content/animations/animations.ts @@ -59,9 +59,9 @@ export class Animations { const animCount = versionData.length / 2; for (let i = 0; i < animCount; i++) { - const version = versionData.get('short', 'unsigned', 'le'); + const version = versionData.get('short', 'unsigned'); const checksum = checksumData.get('int'); - const key = indexData.get('short', 'unsigned', 'le'); + const key = indexData.get('short', 'unsigned'); this.animations.set(key, { key, version, checksum diff --git a/src/file-system/jag/content/archives/interfaces/interface-archive.ts b/src/file-system/jag/content/archives/interfaces/interface-archive.ts index 8a61fc6..87ff147 100644 --- a/src/file-system/jag/content/archives/interfaces/interface-archive.ts +++ b/src/file-system/jag/content/archives/interfaces/interface-archive.ts @@ -1,9 +1,65 @@ import { JagFileStore } from '../../../jag-file-store'; import { JagArchive } from '../../../jag-archive'; +import { ByteBuffer } from '@runejs/common'; -export interface JagGameInterface { - key: number; +export class JagGameInterface { + id: number; + parentId: number; + type: number; + actionType: number; + contentType: number; + width: number; + height: number; + alpha: number; + hoveredPopup: number; + conditionTypes: number[]; + conditionValues: number[]; + cs1Opcodes: number[][]; + scrollLimit: number; + hiddenUntilHovered: boolean; + children: number[]; + childrenX: number[]; + childrenY: number[]; + unknownServerAttribute1: number; + unknownServerAttribute2: boolean; + items: number[]; + itemAmounts: number[]; + itemsSwappable: boolean; + isInventory: boolean; + itemsUsable: boolean; + deleteDraggedItems: boolean; + itemSpritesPadX: number; + itemSpritesPadY: number; + images: string[]; + imagesX: number[]; + imagesY: number[]; + options: string[]; + filled: boolean; + textCentered: boolean; + fontType: number; + textShadowed: boolean; + disabledText: string; + enabledText: string; + disabledColor: number; + enabledColor: number; + disabledHoverColor: number; + enabledHoverColor: number; + disabledImage: string; + enabledImage: string; + disabledModelType: number; + disabledModelId: number; + enabledModelType: number; + enabledModelId: number; + disabledAnimationId: number; + enabledAnimationId: number; + modelZoom: number; + modelRotationX: number; + modelRotationY: number; + actionAdditions: string; + actionText: string; + actionAttributes: number; + tooltip: string; } @@ -20,8 +76,221 @@ export class InterfaceArchive { .getArchive('interface.jag'); } + decode(data: ByteBuffer): JagGameInterface { + const gameInterface = new JagGameInterface(); + gameInterface.id = data.get('short', 'unsigned'); + + if (gameInterface.id === 65535) { + gameInterface.parentId = data.get('short', 'unsigned'); + gameInterface.id = data.get('short', 'unsigned'); + } + + const type = gameInterface.type = data.get('byte', 'unsigned'); + gameInterface.actionType = data.get('byte', 'unsigned'); + gameInterface.contentType = data.get('short', 'unsigned'); + const width = gameInterface.width = data.get('short', 'unsigned'); + const height = gameInterface.height = data.get('short', 'unsigned'); + gameInterface.alpha = data.get('byte', 'unsigned'); + gameInterface.hoveredPopup = data.get('byte', 'unsigned'); + + if (gameInterface.hoveredPopup !== 0) { + gameInterface.hoveredPopup = (gameInterface.hoveredPopup - 1 << 8) + + data.get('byte', 'unsigned'); // why? + } else { + gameInterface.hoveredPopup = -1; + } + + const conditionCount = data.get('byte', 'unsigned'); + + if (conditionCount > 0) { + gameInterface.conditionTypes = new Array(conditionCount); + gameInterface.conditionValues = new Array(conditionCount); + + for (let i = 0; i < conditionCount; i++) { + gameInterface.conditionTypes[i] = data.get('byte', 'unsigned'); + gameInterface.conditionValues[i] = data.get('short', 'unsigned'); + } + } + + const cs1OpcodeCount = data.get('byte', 'unsigned'); + + if (cs1OpcodeCount > 0) { + gameInterface.cs1Opcodes = new Array(cs1OpcodeCount); + + for (let i = 0; i < cs1OpcodeCount; i++) { + const cs1BlockCount = data.get('short', 'unsigned'); + gameInterface.cs1Opcodes[i] = new Array(cs1BlockCount); + + for (let j = 0; j < cs1BlockCount; j++) { + gameInterface.cs1Opcodes[i][j] = data.get('short', 'unsigned'); + } + } + } + + if (type === 0) { + gameInterface.scrollLimit = data.get('short', 'unsigned'); + gameInterface.hiddenUntilHovered = data.get('byte', 'unsigned') === 1; + + const childCount = data.get('short', 'unsigned'); + + gameInterface.children = new Array(childCount); + gameInterface.childrenX = new Array(childCount); + gameInterface.childrenY = new Array(childCount); + + for (let i = 0; i < childCount; i++) { + gameInterface.children[i] = data.get('short', 'unsigned'); + gameInterface.childrenX[i] = data.get('short'); + gameInterface.childrenY[i] = data.get('short'); + } + } + + if (type === 1) { + gameInterface.unknownServerAttribute1 = data.get('short', 'unsigned'); + gameInterface.unknownServerAttribute2 = data.get('byte', 'unsigned') === 1; + } + + if (type === 2) { + gameInterface.items = new Array(width * height); + gameInterface.itemAmounts = new Array(width * height); + gameInterface.itemsSwappable = data.get('byte', 'unsigned') === 1; + gameInterface.isInventory = data.get('byte', 'unsigned') === 1; + gameInterface.itemsUsable = data.get('byte', 'unsigned') === 1; + gameInterface.deleteDraggedItems = data.get('byte', 'unsigned') === 1; + gameInterface.itemSpritesPadX = data.get('byte', 'unsigned'); + gameInterface.itemSpritesPadY = data.get('byte', 'unsigned'); + gameInterface.images = new Array(20); + gameInterface.imagesX = new Array(20); + gameInterface.imagesY = new Array(20); + + for (let i = 0; i < 20; i++) { + const hasSprite = data.get('byte', 'unsigned') === 1; + if (hasSprite) { + gameInterface.imagesX[i] = data.get('short'); + gameInterface.imagesY[i] = data.get('short'); + gameInterface.images[i] = data.getString(10); + } + } + + gameInterface.options = new Array(5); + + for (let i = 0; i < 5; i++) { + gameInterface.options[i] = data.getString(10); + } + } + + if (type === 3) { + gameInterface.filled = data.get('byte', 'unsigned') === 1; + } + + if (type === 4 || type === 1) { + gameInterface.textCentered = data.get('byte', 'unsigned') === 1; + gameInterface.fontType = data.get('byte', 'unsigned'); + gameInterface.textShadowed = data.get('byte', 'unsigned') === 1; + } + + if (type === 4) { + gameInterface.disabledText = data.getString(10); + gameInterface.enabledText = data.getString(10); + } + + if (gameInterface.type === 1 || gameInterface.type === 3 || gameInterface.type === 4) { + gameInterface.disabledColor = data.get('int'); + } + + if (gameInterface.type === 3 || gameInterface.type === 4) { + gameInterface.enabledColor = data.get('int'); + gameInterface.disabledHoverColor = data.get('int'); + gameInterface.enabledHoverColor = data.get('int'); + } + + if (gameInterface.type === 5) { + gameInterface.disabledImage = data.getString(10); + gameInterface.enabledImage= data.getString(10); + } + + if (gameInterface.type === 6) { + let identifier = data.get('byte', 'unsigned'); + + if (identifier !== 0) { + gameInterface.disabledModelType = 1; + gameInterface.disabledModelId = (identifier - 1 << 8) + data.get('byte', 'unsigned'); + } + + identifier = data.get('byte', 'unsigned'); + + if (identifier !== 0) { + gameInterface.enabledModelType = 1; + gameInterface.enabledModelId = (identifier - 1 << 8) + data.get('byte', 'unsigned'); + } + + identifier = data.get('byte', 'unsigned'); + + if (identifier !== 0) { + gameInterface.disabledAnimationId = (identifier - 1 << 8) + data.get('byte', 'unsigned'); + } else { + gameInterface.disabledAnimationId = -1; + } + + identifier = data.get('byte', 'unsigned'); + + if (identifier !== 0) { + gameInterface.enabledAnimationId = (identifier - 1 << 8) + data.get('byte', 'unsigned'); + } else { + gameInterface.enabledAnimationId = -1; + } + + gameInterface.modelZoom = data.get('short', 'unsigned'); + gameInterface.modelRotationX = data.get('short', 'unsigned'); + gameInterface.modelRotationY = data.get('short', 'unsigned'); + } + + if (gameInterface.type === 7) { + gameInterface.items = new Array(width * height); + gameInterface.itemAmounts = new Array(width * height); + gameInterface.textCentered = data.get('byte', 'unsigned') === 1; + gameInterface.fontType = data.get('byte', 'unsigned'); + gameInterface.textShadowed = data.get('byte', 'unsigned') === 1; + gameInterface.disabledColor = data.get('int'); + gameInterface.itemSpritesPadX = data.get('short'); + gameInterface.itemSpritesPadY = data.get('short'); + gameInterface.isInventory = data.get('byte', 'unsigned') === 1; + gameInterface.options = new Array(5); + + for (let i = 0; i < 5; i++) { + gameInterface.options[i] = data.getString(10); + } + } + + if (gameInterface.type === 8) { + gameInterface.disabledText = data.getString(10); + } + + if (gameInterface.actionType === 2 || gameInterface.type === 2) { + gameInterface.actionAdditions = data.getString(10); + gameInterface.actionText = data.getString(10); + gameInterface.actionAttributes = data.get('short', 'unsigned'); + } + + if (gameInterface.actionType === 1 || gameInterface.actionType === 4 || gameInterface.actionType === 5 || gameInterface.actionType === 6) { + gameInterface.tooltip = data.getString(10); + } + + return gameInterface; + } + decodeAll(): void { - //@todo stopped here - 12/08/22 - Kiko + const dataFile = this.archive.getFile('data'); + if (!dataFile?.index?.data) { + throw new Error('interface.jag data file is not loaded!'); + } + + const data = new ByteBuffer(dataFile.index.data); + this.interfaces.clear(); + + while (data.readerIndex < data.length) { + const gameInterface = this.decode(data); + this.interfaces.set(gameInterface.id, gameInterface); + } } } diff --git a/src/file-system/jag/jag-archive.ts b/src/file-system/jag/jag-archive.ts index ae93681..d42b5a5 100644 --- a/src/file-system/jag/jag-archive.ts +++ b/src/file-system/jag/jag-archive.ts @@ -27,6 +27,28 @@ export class JagArchive extends JagFileBase { await this.fileStore.database.upsertIndexes(fileIndexes); } + async loadFileIndexes(): Promise { + const fileIndexes = await this.fileStore.database.getIndexes({ + fileType: 'FILE', + indexKey: this.index.indexKey, + archiveKey: this.index.key, + }); + + if (!fileIndexes?.length) { + return; + } + + for (const fileIndex of fileIndexes) { + const fileKey = fileIndex.key; + + if (!this.files.has(fileKey)) { + const file = new JagFile(this.fileStore, fileKey, this.index.indexKey, this.index.key); + file.index = fileIndex; + this.files.set(fileKey, file); + } + } + } + getFile(fileKey: number): JagFile | null; getFile(fileName: string): JagFile | null; getFile(fileKeyOrName: number | string): JagFile | null; diff --git a/src/file-system/jag/jag-file-base.ts b/src/file-system/jag/jag-file-base.ts index b86b39e..f0c50ac 100644 --- a/src/file-system/jag/jag-file-base.ts +++ b/src/file-system/jag/jag-file-base.ts @@ -4,7 +4,7 @@ import { Buffer } from 'buffer'; import { Crc32 } from '@runejs/common/crc32'; import { createHash } from 'crypto'; import { JagFileStore } from './jag-file-store'; -import { JagIndexEntity } from '../../db/jag-index-entity'; +import { JagIndexEntity } from '../../db/jag/jag-index-entity'; export abstract class JagFileBase { diff --git a/src/file-system/jag/jag-file-store.ts b/src/file-system/jag/jag-file-store.ts index ba47ee3..a3372b9 100644 --- a/src/file-system/jag/jag-file-store.ts +++ b/src/file-system/jag/jag-file-store.ts @@ -1,10 +1,10 @@ import { FileStoreBase } from '../file-store-base'; import { Jag, indexes } from './jag'; import { JagIndex } from './jag-index'; -import { JagIndexEntity } from '../../db/jag-index-entity'; +import { JagIndexEntity } from '../../db/jag/jag-index-entity'; import { IndexDatabase } from '../../db/index-database'; import { join } from 'path'; -import { JagDatabase } from '../../db/jag-database'; +import { JagDatabase } from '../../db/jag/jag-database'; export class JagFileStore extends FileStoreBase { @@ -28,6 +28,12 @@ export class JagFileStore extends FileStoreBase { return this._database; } + async loadIndexEntities(): Promise { + for (const [ , index ] of this.indexes) { + await index.loadIndex(); + } + } + override async load(): Promise { await this.openDatabase(); } diff --git a/src/file-system/jag/jag-index.ts b/src/file-system/jag/jag-index.ts index 212d923..1fc4235 100644 --- a/src/file-system/jag/jag-index.ts +++ b/src/file-system/jag/jag-index.ts @@ -27,6 +27,34 @@ export class JagIndex extends JagFileBase { await this.fileStore.database.upsertIndexes(fileIndexes); } + async loadFileIndexes(): Promise { + const fileIndexes = await this.fileStore.database.getIndexes({ + indexKey: this.index.key, + archiveKey: -1, + }); + + if (!fileIndexes?.length) { + return; + } + + for (const fileIndex of fileIndexes) { + const fileKey = fileIndex.key; + + if (!this.files.has(fileKey)) { + let file: JagFileBase; + + if (fileIndex.fileType === 'ARCHIVE') { + file = new JagArchive(this.fileStore, fileKey); + } else { + file = new JagFile(this.fileStore, fileKey, this.index.key, -1); + } + + file.index = fileIndex; + this.files.set(fileKey, file); + } + } + } + createArchive(archiveKey: number): JagArchive { const archive = new JagArchive(this.fileStore, archiveKey); this.setArchive(archiveKey, archive); diff --git a/src/file-system/js5/js5-file-base.ts b/src/file-system/js5/js5-file-base.ts index 112c80e..cb0518f 100644 --- a/src/file-system/js5/js5-file-base.ts +++ b/src/file-system/js5/js5-file-base.ts @@ -1,4 +1,4 @@ -import { Js5IndexEntity } from '../../db/js5-index-entity'; +import { Js5IndexEntity } from '../../db/js5/js5-index-entity'; import { Js5FileType } from '../../config'; import { logger } from '@runejs/common'; import { Buffer } from 'buffer'; diff --git a/src/file-system/js5/js5-file-store.ts b/src/file-system/js5/js5-file-store.ts index 1a92dec..63f694e 100644 --- a/src/file-system/js5/js5-file-store.ts +++ b/src/file-system/js5/js5-file-store.ts @@ -6,7 +6,7 @@ import { JS5 } from './js5'; import { Js5Archive } from './js5-archive'; import { FileStoreBase } from '../file-store-base'; import { logger } from '../../../../common'; -import { Js5Database } from '../../db/js5-database'; +import { Js5Database } from '../../db/js5/js5-database'; export class Js5FileStore extends FileStoreBase{ @@ -38,7 +38,7 @@ export class Js5FileStore extends FileStoreBase{ await this.openDatabase(); } - async loadArchiveIndexes(): Promise { + async loadArchiveEntities(): Promise { for (const [ , archive ] of this.archives) { await archive.loadIndex(); } From e1b775f5276972a25c0201774cab14eddcbb4421 Mon Sep 17 00:00:00 2001 From: Kikorono Date: Mon, 15 Aug 2022 11:29:45 -0500 Subject: [PATCH 26/32] Implementing JAG interface decoding and saving within JAG databases --- src/db/index-database.ts | 24 +-- .../jag/content/jag-game-interface-entity.ts | 174 ++++++++++++++++++ src/db/jag/index.ts | 3 + src/db/jag/jag-database.ts | 55 +++++- src/db/js5/index.ts | 2 + src/db/js5/js5-database.ts | 25 ++- ...ce-archive.ts => jag-interface-archive.ts} | 112 +++++------ src/file-system/jag/jag-file-store.ts | 4 +- src/file-system/jag/jag.ts | 2 + src/scripts/dev.ts | 62 +++---- 10 files changed, 325 insertions(+), 138 deletions(-) create mode 100644 src/db/jag/content/jag-game-interface-entity.ts create mode 100644 src/db/jag/index.ts create mode 100644 src/db/js5/index.ts rename src/file-system/jag/content/archives/interfaces/{interface-archive.ts => jag-interface-archive.ts} (81%) diff --git a/src/db/index-database.ts b/src/db/index-database.ts index 9530e77..e54b09d 100644 --- a/src/db/index-database.ts +++ b/src/db/index-database.ts @@ -8,7 +8,6 @@ export abstract class IndexDatabase { protected readonly gameBuild: string; protected readonly databasePath: string; protected readonly loggerOptions: LoggerOptions; - protected readonly entityType: new () => ENTITY; protected _connection: Connection; protected _repository: Repository; @@ -16,16 +15,16 @@ export abstract class IndexDatabase { protected constructor( gameBuild: string, databasePath: string, - entityType: new () => ENTITY, loggerOptions: LoggerOptions = 'all' ) { this.gameBuild = gameBuild; this.databasePath = databasePath; - this.entityType = entityType; // [ 'error', 'warn' ], 'all', etc... this.loggerOptions = loggerOptions; } + abstract openConnection(): Promise; + abstract upsertIndexes(indexEntities: ENTITY[]): Promise; async getIndexes(where: WHERE): Promise { @@ -53,25 +52,6 @@ export abstract class IndexDatabase { return await this.repository.save(indexEntity as any); } - async openConnection(): Promise { - if(!existsSync(this.databasePath)) { - mkdirSync(this.databasePath, { recursive: true }); - } - - this._connection = await createConnection({ - type: 'better-sqlite3', - database: join(this.databasePath, `${this.gameBuild}.index.sqlite3`), - entities: [ this.entityType ], - synchronize: true, - logging: this.loggerOptions, - name: 'index-repository' - }); - - this._repository = this._connection.getRepository(this.entityType); - - return this._connection; - } - async closeConnection(): Promise { await this._connection.close(); } diff --git a/src/db/jag/content/jag-game-interface-entity.ts b/src/db/jag/content/jag-game-interface-entity.ts new file mode 100644 index 0000000..8080683 --- /dev/null +++ b/src/db/jag/content/jag-game-interface-entity.ts @@ -0,0 +1,174 @@ +import { Column, Entity, PrimaryColumn } from 'typeorm'; + + +@Entity('jag_game_interface') +export class JagGameInterfaceEntity { + + @PrimaryColumn('integer', { nullable: false, unique: true }) + id: number; + + @Column('integer', { name: 'parent_id', nullable: false, default: -1 }) + parentId: number = -1; + + @Column('integer', { nullable: false }) + type: number; + + @Column('integer', { nullable: false }) + actionType: number; + + @Column('integer', { nullable: false }) + contentType: number; + + @Column('integer', { nullable: false }) + width: number; + + @Column('integer', { nullable: false }) + height: number; + + @Column('integer', { nullable: false }) + alpha: number; + + @Column('integer', { nullable: false }) + hoveredPopup: number; + + @Column('simple-json', { nullable: true }) + conditionTypes?: number[]; + + @Column('simple-json', { nullable: true }) + conditionValues?: number[]; + + @Column('simple-json', { nullable: true }) + cs1Opcodes?: number[][]; + + @Column('integer', { nullable: true }) + scrollLimit?: number; + + @Column('boolean', { nullable: true }) + hiddenUntilHovered?: boolean; + + @Column('simple-json', { nullable: true }) + children?: number[]; + + @Column('simple-json', { nullable: true }) + childrenX?: number[]; + + @Column('simple-json', { nullable: true }) + childrenY?: number[]; + + @Column('integer', { nullable: true }) + unknownServerAttribute1?: number; + + @Column('boolean', { nullable: true }) + unknownServerAttribute2?: boolean; + + @Column('simple-json', { nullable: true }) + items?: number[]; + + @Column('simple-json', { nullable: true }) + itemAmounts?: number[]; + + @Column('boolean', { nullable: true }) + itemsSwappable?: boolean; + + @Column('boolean', { nullable: true }) + isInventory?: boolean; + + @Column('boolean', { nullable: true }) + itemsUsable?: boolean; + + @Column('boolean', { nullable: true }) + deleteDraggedItems?: boolean; + + @Column('integer', { nullable: true }) + itemSpritesPadX?: number; + + @Column('integer', { nullable: true }) + itemSpritesPadY?: number; + + @Column('simple-json', { nullable: true }) + images?: string[]; + + @Column('simple-json', { nullable: true }) + imagesX?: number[]; + + @Column('simple-json', { nullable: true }) + imagesY?: number[]; + + @Column('simple-json', { nullable: true }) + options?: string[]; + + @Column('boolean', { nullable: true }) + filled?: boolean; + + @Column('boolean', { nullable: true }) + textCentered?: boolean; + + @Column('integer', { nullable: true }) + fontType?: number; + + @Column('boolean', { nullable: true }) + textShadowed?: boolean; + + @Column('text', { nullable: true }) + disabledText?: string; + + @Column('text', { nullable: true }) + enabledText?: string; + + @Column('integer', { nullable: true }) + disabledColor?: number; + + @Column('integer', { nullable: true }) + enabledColor?: number; + + @Column('integer', { nullable: true }) + disabledHoverColor?: number; + + @Column('integer', { nullable: true }) + enabledHoverColor?: number; + + @Column('text', { nullable: true }) + disabledImage?: string; + + @Column('text', { nullable: true }) + enabledImage?: string; + + @Column('integer', { nullable: true }) + disabledModelType?: number; + + @Column('integer', { nullable: true }) + disabledModelId?: number; + + @Column('integer', { nullable: true }) + enabledModelType?: number; + + @Column('integer', { nullable: true }) + enabledModelId?: number; + + @Column('integer', { nullable: true }) + disabledAnimationId?: number; + + @Column('integer', { nullable: true }) + enabledAnimationId?: number; + + @Column('integer', { nullable: true }) + modelZoom?: number; + + @Column('integer', { nullable: true }) + modelRotationX?: number; + + @Column('integer', { nullable: true }) + modelRotationY?: number; + + @Column('text', { nullable: true }) + actionAdditions?: string; + + @Column('text', { nullable: true }) + actionText?: string; + + @Column('integer', { nullable: true }) + actionAttributes?: number; + + @Column('text', { nullable: true }) + tooltip?: string; +} diff --git a/src/db/jag/index.ts b/src/db/jag/index.ts new file mode 100644 index 0000000..9fd3ba5 --- /dev/null +++ b/src/db/jag/index.ts @@ -0,0 +1,3 @@ +export * from './jag-database'; +export * from './jag-index-entity'; +export * from './content/jag-game-interface-entity'; diff --git a/src/db/jag/jag-database.ts b/src/db/jag/jag-database.ts index cb87659..e9c2cfa 100644 --- a/src/db/jag/jag-database.ts +++ b/src/db/jag/jag-database.ts @@ -1,7 +1,10 @@ import { IndexDatabase } from '../index-database'; import { JagIndexEntity } from './jag-index-entity'; -import { LoggerOptions } from 'typeorm'; +import { Connection, createConnection, LoggerOptions, Repository } from 'typeorm'; import { JagFileType } from '../../config'; +import { JagGameInterfaceEntity } from './content/jag-game-interface-entity'; +import { existsSync, mkdirSync } from 'graceful-fs'; +import { join } from 'path'; export interface JagIndexEntityWhere { @@ -14,12 +17,37 @@ export interface JagIndexEntityWhere { export class JagDatabase extends IndexDatabase { + private _interfaceRepo: Repository; + constructor( gameBuild: string, databasePath: string, loggerOptions: LoggerOptions = 'all' ) { - super(gameBuild, databasePath, JagIndexEntity, loggerOptions); + super(gameBuild, databasePath, loggerOptions); + } + + override async openConnection(): Promise { + if(!existsSync(this.databasePath)) { + mkdirSync(this.databasePath, { recursive: true }); + } + + this._connection = await createConnection({ + type: 'better-sqlite3', + database: join(this.databasePath, `${this.gameBuild}.index.sqlite3`), + entities: [ + JagIndexEntity, + JagGameInterfaceEntity + ], + synchronize: true, + logging: this.loggerOptions, + name: 'jag-index-repository' + }); + + this._repository = this._connection.getRepository(JagIndexEntity); + this._interfaceRepo = this._connection.getRepository(JagGameInterfaceEntity); + + return this._connection; } override async upsertIndexes(indexEntities: JagIndexEntity[]): Promise { @@ -33,4 +61,27 @@ export class JagDatabase extends IndexDatabase { + await this.interfaceRepo.save(entities, { + reload: false, + listeners: false, + transaction: false, + chunk: 100, + }); + } + + async saveInterface(entity: JagGameInterfaceEntity): Promise { + return await this.interfaceRepo.save(entity); + } + + async getInterface(id: number): Promise { + return await this.interfaceRepo.findOne({ + where: { id } + }); + } + + get interfaceRepo(): Repository { + return this._interfaceRepo; + } + } diff --git a/src/db/js5/index.ts b/src/db/js5/index.ts new file mode 100644 index 0000000..7f472c9 --- /dev/null +++ b/src/db/js5/index.ts @@ -0,0 +1,2 @@ +export * from './js5-database'; +export * from './js5-index-entity'; diff --git a/src/db/js5/js5-database.ts b/src/db/js5/js5-database.ts index 6965163..07a344a 100644 --- a/src/db/js5/js5-database.ts +++ b/src/db/js5/js5-database.ts @@ -1,7 +1,9 @@ import { IndexDatabase } from '../index-database'; import { Js5IndexEntity } from './js5-index-entity'; -import { LoggerOptions } from 'typeorm'; +import { Connection, createConnection, LoggerOptions } from 'typeorm'; import { Js5FileType } from '../../config'; +import { existsSync, mkdirSync } from 'graceful-fs'; +import { join } from 'path'; export interface Js5IndexEntityWhere { @@ -19,7 +21,26 @@ export class Js5Database extends IndexDatabase { + if(!existsSync(this.databasePath)) { + mkdirSync(this.databasePath, { recursive: true }); + } + + this._connection = await createConnection({ + type: 'better-sqlite3', + database: join(this.databasePath, `${this.gameBuild}.index.sqlite3`), + entities: [ Js5IndexEntity ], + synchronize: true, + logging: this.loggerOptions, + name: 'js5-index-repository' + }); + + this._repository = this._connection.getRepository(Js5IndexEntity); + + return this._connection; } override async upsertIndexes(indexEntities: Js5IndexEntity[]): Promise { diff --git a/src/file-system/jag/content/archives/interfaces/interface-archive.ts b/src/file-system/jag/content/archives/interfaces/jag-interface-archive.ts similarity index 81% rename from src/file-system/jag/content/archives/interfaces/interface-archive.ts rename to src/file-system/jag/content/archives/interfaces/jag-interface-archive.ts index 87ff147..7af2444 100644 --- a/src/file-system/jag/content/archives/interfaces/interface-archive.ts +++ b/src/file-system/jag/content/archives/interfaces/jag-interface-archive.ts @@ -1,83 +1,21 @@ -import { JagFileStore } from '../../../jag-file-store'; -import { JagArchive } from '../../../jag-archive'; +import { Buffer } from 'buffer'; import { ByteBuffer } from '@runejs/common'; +import { JagFileStore } from '../../../jag-file-store'; +import { JagGameInterfaceEntity } from '../../../../../db/jag'; -export class JagGameInterface { - id: number; - parentId: number; - type: number; - actionType: number; - contentType: number; - width: number; - height: number; - alpha: number; - hoveredPopup: number; - conditionTypes: number[]; - conditionValues: number[]; - cs1Opcodes: number[][]; - scrollLimit: number; - hiddenUntilHovered: boolean; - children: number[]; - childrenX: number[]; - childrenY: number[]; - unknownServerAttribute1: number; - unknownServerAttribute2: boolean; - items: number[]; - itemAmounts: number[]; - itemsSwappable: boolean; - isInventory: boolean; - itemsUsable: boolean; - deleteDraggedItems: boolean; - itemSpritesPadX: number; - itemSpritesPadY: number; - images: string[]; - imagesX: number[]; - imagesY: number[]; - options: string[]; - filled: boolean; - textCentered: boolean; - fontType: number; - textShadowed: boolean; - disabledText: string; - enabledText: string; - disabledColor: number; - enabledColor: number; - disabledHoverColor: number; - enabledHoverColor: number; - disabledImage: string; - enabledImage: string; - disabledModelType: number; - disabledModelId: number; - enabledModelType: number; - enabledModelId: number; - disabledAnimationId: number; - enabledAnimationId: number; - modelZoom: number; - modelRotationX: number; - modelRotationY: number; - actionAdditions: string; - actionText: string; - actionAttributes: number; - tooltip: string; -} - - -export class InterfaceArchive { +export class JagInterfaceArchive { readonly jagStore: JagFileStore; - readonly interfaces: Map; - readonly archive: JagArchive; + readonly interfaces: Map; constructor(jagStore: JagFileStore) { this.jagStore = jagStore; - this.interfaces = new Map(); - this.archive = this.jagStore.getIndex('archives') - .getArchive('interface.jag'); + this.interfaces = new Map(); } - decode(data: ByteBuffer): JagGameInterface { - const gameInterface = new JagGameInterface(); + decode(data: ByteBuffer): JagGameInterfaceEntity { + const gameInterface = new JagGameInterfaceEntity(); gameInterface.id = data.get('short', 'unsigned'); if (gameInterface.id === 65535) { @@ -279,7 +217,14 @@ export class InterfaceArchive { } decodeAll(): void { - const dataFile = this.archive.getFile('data'); + const archive = this.jagStore.getIndex('archives') + .getArchive('interface.jag'); + + if (!archive) { + throw new Error('interface.jag archive is not loaded!'); + } + + const dataFile = archive.getFile('data'); if (!dataFile?.index?.data) { throw new Error('interface.jag data file is not loaded!'); } @@ -293,4 +238,29 @@ export class InterfaceArchive { } } + encode(gameInterface: JagGameInterfaceEntity): Buffer | null { + // @todo stubbed - 15/08/22 - Kiko + return null; + } + + encodeAll(): Buffer | null { + // @todo stubbed - 15/08/22 - Kiko + return null; + } + + toJS5(gameInterface: JagGameInterfaceEntity): null { + // @todo stubbed - 15/08/22 - Kiko + return null; + } + + async loadAll(): Promise { + const entities = (await this.jagStore.database.interfaceRepo.find()) + .sort((a, b) => a.id - b.id); + entities.forEach(entity => this.interfaces.set(entity.id, entity)); + } + + async saveAll(): Promise { + await this.jagStore.database.saveInterfaces(Array.from(this.interfaces.values())); + } + } diff --git a/src/file-system/jag/jag-file-store.ts b/src/file-system/jag/jag-file-store.ts index a3372b9..96555ca 100644 --- a/src/file-system/jag/jag-file-store.ts +++ b/src/file-system/jag/jag-file-store.ts @@ -1,8 +1,6 @@ import { FileStoreBase } from '../file-store-base'; import { Jag, indexes } from './jag'; import { JagIndex } from './jag-index'; -import { JagIndexEntity } from '../../db/jag/jag-index-entity'; -import { IndexDatabase } from '../../db/index-database'; import { join } from 'path'; import { JagDatabase } from '../../db/jag/jag-database'; @@ -18,7 +16,7 @@ export class JagFileStore extends FileStoreBase { this.indexes = new Map(); } - override async openDatabase(): Promise> { + override async openDatabase(): Promise { this._database = new JagDatabase( this.gameBuild, join(this.fileStorePath, 'index'), diff --git a/src/file-system/jag/jag.ts b/src/file-system/jag/jag.ts index 8c000d2..9fd7293 100644 --- a/src/file-system/jag/jag.ts +++ b/src/file-system/jag/jag.ts @@ -322,6 +322,8 @@ export class Jag { file.index.data = decompressHeadlessBzip2(file.index.compressedData); file.index.compressionMethod = 'bzip'; } else { + file.index.data = file.index.compressedData; + file.index.compressedData = null; file.index.compressionMethod = 'none'; } } catch (error) { diff --git a/src/scripts/dev.ts b/src/scripts/dev.ts index 079bd74..f9953ef 100644 --- a/src/scripts/dev.ts +++ b/src/scripts/dev.ts @@ -1,64 +1,50 @@ -import { Js5FileStore } from '../file-system/js5/js5-file-store'; import { logger } from '@runejs/common'; +import { indexes, JagArchive, JagFileStore } from '../file-system/jag'; +import { JagInterfaceArchive } from '../file-system/jag/content/archives/interfaces/jag-interface-archive'; const dev = async () => { const start = Date.now(); - const fileStore = new Js5FileStore(435); - await fileStore.load(); - fileStore.js5.readLocalCacheFiles(); - logger.info(`Unpacking archives from JS5 store...`); + const store = new JagFileStore(317); - for (const [ , archive ] of fileStore.archives) { - fileStore.js5.unpack(archive); - } - - logger.info(`Decoding JS5 archives...`); + logger.info(`Loading JAG store for build ${store.gameBuild}...`); - for (const [ , archive ] of fileStore.archives) { - await fileStore.js5.decodeArchive(archive); - } + await store.load(); - logger.info(`Saving archive indexes...`); + logger.info(`Loading index entities...`); - for (const [ , archive ] of fileStore.archives) { - await archive.saveIndex(); + const indexNames = Object.keys(indexes); + for (const indexName of indexNames) { + store.createIndex(indexes[indexName]); } - logger.info(`Unpacking groups from JS5 store...`); + await store.loadIndexEntities(); - for (const [ , archive ] of fileStore.archives) { - for (const [ , group ] of archive.groups) { - fileStore.js5.unpack(group); - } + logger.info(`Loading index file entities...`); - logger.info(`Finished unpacking archive ${archive.index.name} groups.`); + for (const [ , indexFile ] of store.indexes) { + await indexFile.loadFileIndexes(); } - logger.info(`Decoding JS5 groups...`); + logger.info(`Loading archive file entities...`); - for (const [ , archive ] of fileStore.archives) { - for (const [ , group ] of archive.groups) { - await fileStore.js5.decodeGroup(group); - } + const archiveIndex = store.getIndex('archives'); - logger.info(`Finished decoding archive ${archive.index.name} groups.`); + for (const [ , file ] of archiveIndex.files) { + const archive = file as JagArchive; + await archive.loadFileIndexes(); } - logger.info(`Saving group indexes...`); + logger.info(`Decoding game interfaces...`); - for (const [ , archive ] of fileStore.archives) { - await archive.upsertGroupIndexes(); - } + const interfaceArchive = new JagInterfaceArchive(store); - logger.info(`Saving flat file indexes...`); + interfaceArchive.decodeAll(); - for (const [ , archive ] of fileStore.archives) { - for (const [ , group ] of archive.groups) { - await group.upsertFileIndexes(); - } - } + logger.info(`${interfaceArchive.interfaces.size} interfaces decoded. Saving interface entities...`); + + await interfaceArchive.saveAll(); const end = Date.now(); logger.info(`Operations completed in ${(end - start) / 1000} seconds.`); From 3a33c2210a7f1f242f1dedc1bec31f176e72391d Mon Sep 17 00:00:00 2001 From: Kikorono Date: Mon, 22 Aug 2022 18:29:25 -0500 Subject: [PATCH 27/32] Reimplementing Js5 http server support for the webclient project --- package-lock.json | 1657 ++++++++++++++--- package.json | 3 + src/config/file-type.ts | 2 +- src/db/jag/jag-database.ts | 4 +- src/db/jag/jag-index-entity.ts | 6 +- src/file-system/cache/index.ts | 2 - .../jag/content/animations/animations.ts | 8 +- .../interfaces/jag-interface-archive.ts | 8 +- src/file-system/jag/index.ts | 2 +- src/file-system/jag/jag-archive.ts | 8 +- .../jag/{jag-index.ts => jag-cache.ts} | 14 +- src/file-system/jag/jag-file-base.ts | 6 +- src/file-system/jag/jag-file-store.ts | 103 +- src/file-system/jag/jag-file.ts | 4 +- src/file-system/jag/jag.ts | 38 +- src/file-system/js5/js5-archive.ts | 18 +- src/file-system/js5/js5-file-store.ts | 67 +- src/file-system/js5/js5-file.ts | 8 +- src/file-system/js5/js5-group.ts | 20 +- src/file-system/js5/js5.ts | 48 +- src/file-system/packed/index.ts | 2 + .../packed-cache-file.ts} | 2 +- .../packed-cache-format.ts} | 4 +- src/http/js5-server.ts | 202 ++ src/openrs2/openrs2.ts | 6 +- src/scripts/dev.ts | 68 +- src/scripts/indexer.ts | 25 +- src/scripts/script-executor.ts | 2 +- 28 files changed, 1930 insertions(+), 407 deletions(-) delete mode 100644 src/file-system/cache/index.ts rename src/file-system/jag/{jag-index.ts => jag-cache.ts} (90%) create mode 100644 src/file-system/packed/index.ts rename src/file-system/{cache/cache-file.ts => packed/packed-cache-file.ts} (67%) rename src/file-system/{cache/cache-format.ts => packed/packed-cache-format.ts} (59%) create mode 100644 src/http/js5-server.ts diff --git a/package-lock.json b/package-lock.json index a61dc45..029d484 100644 --- a/package-lock.json +++ b/package-lock.json @@ -13,6 +13,7 @@ "adm-zip": "^0.5.9", "axios": "^0.27.2", "compressjs": "^1.0.3", + "express": "^4.18.1", "graceful-fs": ">=4.2.0", "json5": "^2.2.0", "reflect-metadata": "^0.1.13", @@ -25,6 +26,7 @@ "devDependencies": { "@runejs/eslint-config": "^1.1.0", "@types/adm-zip": "^0.5.0", + "@types/express": "^4.17.13", "@types/graceful-fs": "^4.1.5", "@types/node": "^16.11.26", "@types/yargs": "^17.0.9", @@ -77,6 +79,24 @@ "node": "^12.22.0 || ^14.17.0 || >=16.0.0" } }, + "node_modules/@eslint/eslintrc/node_modules/argparse": { + "version": "2.0.1", + "resolved": "https://registry.npmjs.org/argparse/-/argparse-2.0.1.tgz", + "integrity": "sha512-8+9WqebbFzpX9OR+Wa6O29asIogeRMzcGtAINdpMHHyAg10f05aSFVBbcEqGf/PXw1EjAZ+q2/bEBg3DvurK3Q==", + "dev": true + }, + "node_modules/@eslint/eslintrc/node_modules/js-yaml": { + "version": "4.1.0", + "resolved": "https://registry.npmjs.org/js-yaml/-/js-yaml-4.1.0.tgz", + "integrity": "sha512-wpxZs9NoxZaJESJGIZTyDEaYpl0FKSA+FB9aJiyemKhMwkxQg63h4T1KJgUGHpTqPDNRcmmYLugrRjJlBtWvRA==", + "dev": true, + "dependencies": { + "argparse": "^2.0.1" + }, + "bin": { + "js-yaml": "bin/js-yaml.js" + } + }, "node_modules/@hapi/bourne": { "version": "2.1.0", "resolved": "https://registry.npmjs.org/@hapi/bourne/-/bourne-2.1.0.tgz", @@ -189,26 +209,6 @@ "typescript": ">=4.5.0" } }, - "node_modules/@runejs/common/node_modules/argparse": { - "version": "1.0.10", - "resolved": "https://registry.npmjs.org/argparse/-/argparse-1.0.10.tgz", - "integrity": "sha512-o5Roy6tNG4SL/FOkCAN6RzjiakZS25RLYFrcMttJqbdd8BWrnA+fGz57iN5Pb06pvBGvl5gQ0B48dJlslXvoTg==", - "dependencies": { - "sprintf-js": "~1.0.2" - } - }, - "node_modules/@runejs/common/node_modules/js-yaml": { - "version": "3.14.1", - "resolved": "https://registry.npmjs.org/js-yaml/-/js-yaml-3.14.1.tgz", - "integrity": "sha512-okMH7OXXJ7YrN9Ok3/SXrnu4iX9yOk+25nqX4imS2npuvTYDmo/QEZoqwZkYaIDk3jVvBOTOIEgEhaLOynBS9g==", - "dependencies": { - "argparse": "^1.0.7", - "esprima": "^4.0.0" - }, - "bin": { - "js-yaml": "bin/js-yaml.js" - } - }, "node_modules/@runejs/eslint-config": { "version": "1.1.0", "resolved": "https://registry.npmjs.org/@runejs/eslint-config/-/eslint-config-1.1.0.tgz", @@ -258,6 +258,48 @@ "@types/node": "*" } }, + "node_modules/@types/body-parser": { + "version": "1.19.2", + "resolved": "https://registry.npmjs.org/@types/body-parser/-/body-parser-1.19.2.tgz", + "integrity": "sha512-ALYone6pm6QmwZoAgeyNksccT9Q4AWZQ6PvfwR37GT6r6FWUPguq6sUmNGSMV2Wr761oQoBxwGGa6DR5o1DC9g==", + "dev": true, + "dependencies": { + "@types/connect": "*", + "@types/node": "*" + } + }, + "node_modules/@types/connect": { + "version": "3.4.35", + "resolved": "https://registry.npmjs.org/@types/connect/-/connect-3.4.35.tgz", + "integrity": "sha512-cdeYyv4KWoEgpBISTxWvqYsVy444DOqehiF3fM3ne10AmJ62RSyNkUnxMJXHQWRQQX2eR94m5y1IZyDwBjV9FQ==", + "dev": true, + "dependencies": { + "@types/node": "*" + } + }, + "node_modules/@types/express": { + "version": "4.17.13", + "resolved": "https://registry.npmjs.org/@types/express/-/express-4.17.13.tgz", + "integrity": "sha512-6bSZTPaTIACxn48l50SR+axgrqm6qXFIxrdAKaG6PaJk3+zuUr35hBlgT7vOmJcum+OEaIBLtHV/qloEAFITeA==", + "dev": true, + "dependencies": { + "@types/body-parser": "*", + "@types/express-serve-static-core": "^4.17.18", + "@types/qs": "*", + "@types/serve-static": "*" + } + }, + "node_modules/@types/express-serve-static-core": { + "version": "4.17.30", + "resolved": "https://registry.npmjs.org/@types/express-serve-static-core/-/express-serve-static-core-4.17.30.tgz", + "integrity": "sha512-gstzbTWro2/nFed1WXtf+TtrpwxH7Ggs4RLYTLbeVgIkUQOI3WG/JKjgeOU1zXDvezllupjrf8OPIdvTbIaVOQ==", + "dev": true, + "dependencies": { + "@types/node": "*", + "@types/qs": "*", + "@types/range-parser": "*" + } + }, "node_modules/@types/graceful-fs": { "version": "4.1.5", "resolved": "https://registry.npmjs.org/@types/graceful-fs/-/graceful-fs-4.1.5.tgz", @@ -273,12 +315,40 @@ "integrity": "sha512-wOuvG1SN4Us4rez+tylwwwCV1psiNVOkJeM3AUWUNWg/jDQY2+HE/444y5gc+jBmRqASOm2Oeh5c1axHobwRKQ==", "dev": true }, + "node_modules/@types/mime": { + "version": "3.0.1", + "resolved": "https://registry.npmjs.org/@types/mime/-/mime-3.0.1.tgz", + "integrity": "sha512-Y4XFY5VJAuw0FgAqPNd6NNoV44jbq9Bz2L7Rh/J6jLTiHBSBJa9fxqQIvkIld4GsoDOcCbvzOUAbLPsSKKg+uA==", + "dev": true + }, "node_modules/@types/node": { - "version": "16.11.47", - "resolved": "https://registry.npmjs.org/@types/node/-/node-16.11.47.tgz", - "integrity": "sha512-fpP+jk2zJ4VW66+wAMFoBJlx1bxmBKx4DUFf68UHgdGCOuyUTDlLWqsaNPJh7xhNDykyJ9eIzAygilP/4WoN8g==", + "version": "16.11.52", + "resolved": "https://registry.npmjs.org/@types/node/-/node-16.11.52.tgz", + "integrity": "sha512-GnstYouCa9kbYokBCWEVrszJ1P2rGAQpKrqACHKuixkaT8XGu8nsqHvEUIGqDs5vwtsJ7LrYqnPDKRD1V+M39A==", + "dev": true + }, + "node_modules/@types/qs": { + "version": "6.9.7", + "resolved": "https://registry.npmjs.org/@types/qs/-/qs-6.9.7.tgz", + "integrity": "sha512-FGa1F62FT09qcrueBA6qYTrJPVDzah9a+493+o2PCXsesWHIn27G98TsSMs3WPNbZIEj4+VJf6saSFpvD+3Zsw==", "dev": true }, + "node_modules/@types/range-parser": { + "version": "1.2.4", + "resolved": "https://registry.npmjs.org/@types/range-parser/-/range-parser-1.2.4.tgz", + "integrity": "sha512-EEhsLsD6UsDM1yFhAvy0Cjr6VwmpMWqFBCb9w07wVugF7w9nfajxLuVmngTIpgS6svCnm6Vaw+MZhoDCKnOfsw==", + "dev": true + }, + "node_modules/@types/serve-static": { + "version": "1.15.0", + "resolved": "https://registry.npmjs.org/@types/serve-static/-/serve-static-1.15.0.tgz", + "integrity": "sha512-z5xyF6uh8CbjAu9760KDKsH2FcDxZ2tFCsA4HIMWE6IkiYMXfVoa+4f9KX+FN0ZLsaMw1WNG2ETLA6N+/YA+cg==", + "dev": true, + "dependencies": { + "@types/mime": "*", + "@types/node": "*" + } + }, "node_modules/@types/strip-bom": { "version": "3.0.0", "resolved": "https://registry.npmjs.org/@types/strip-bom/-/strip-bom-3.0.0.tgz", @@ -312,14 +382,14 @@ "integrity": "sha512-fbF6oTd4sGGy0xjHPKAt+eS2CrxJ3+6gQ3FGcBoIJR2TLAyCkCyI8JqZNy+FeON0AhVgNJoUumVoZQjBFUqHkw==" }, "node_modules/@typescript-eslint/eslint-plugin": { - "version": "5.33.0", - "resolved": "https://registry.npmjs.org/@typescript-eslint/eslint-plugin/-/eslint-plugin-5.33.0.tgz", - "integrity": "sha512-jHvZNSW2WZ31OPJ3enhLrEKvAZNyAFWZ6rx9tUwaessTc4sx9KmgMNhVcqVAl1ETnT5rU5fpXTLmY9YvC1DCNg==", + "version": "5.33.1", + "resolved": "https://registry.npmjs.org/@typescript-eslint/eslint-plugin/-/eslint-plugin-5.33.1.tgz", + "integrity": "sha512-S1iZIxrTvKkU3+m63YUOxYPKaP+yWDQrdhxTglVDVEVBf+aCSw85+BmJnyUaQQsk5TXFG/LpBu9fa+LrAQ91fQ==", "dev": true, "dependencies": { - "@typescript-eslint/scope-manager": "5.33.0", - "@typescript-eslint/type-utils": "5.33.0", - "@typescript-eslint/utils": "5.33.0", + "@typescript-eslint/scope-manager": "5.33.1", + "@typescript-eslint/type-utils": "5.33.1", + "@typescript-eslint/utils": "5.33.1", "debug": "^4.3.4", "functional-red-black-tree": "^1.0.1", "ignore": "^5.2.0", @@ -345,14 +415,14 @@ } }, "node_modules/@typescript-eslint/parser": { - "version": "5.33.0", - "resolved": "https://registry.npmjs.org/@typescript-eslint/parser/-/parser-5.33.0.tgz", - "integrity": "sha512-cgM5cJrWmrDV2KpvlcSkelTBASAs1mgqq+IUGKJvFxWrapHpaRy5EXPQz9YaKF3nZ8KY18ILTiVpUtbIac86/w==", + "version": "5.33.1", + "resolved": "https://registry.npmjs.org/@typescript-eslint/parser/-/parser-5.33.1.tgz", + "integrity": "sha512-IgLLtW7FOzoDlmaMoXdxG8HOCByTBXrB1V2ZQYSEV1ggMmJfAkMWTwUjjzagS6OkfpySyhKFkBw7A9jYmcHpZA==", "dev": true, "dependencies": { - "@typescript-eslint/scope-manager": "5.33.0", - "@typescript-eslint/types": "5.33.0", - "@typescript-eslint/typescript-estree": "5.33.0", + "@typescript-eslint/scope-manager": "5.33.1", + "@typescript-eslint/types": "5.33.1", + "@typescript-eslint/typescript-estree": "5.33.1", "debug": "^4.3.4" }, "engines": { @@ -372,13 +442,13 @@ } }, "node_modules/@typescript-eslint/scope-manager": { - "version": "5.33.0", - "resolved": "https://registry.npmjs.org/@typescript-eslint/scope-manager/-/scope-manager-5.33.0.tgz", - "integrity": "sha512-/Jta8yMNpXYpRDl8EwF/M8It2A9sFJTubDo0ATZefGXmOqlaBffEw0ZbkbQ7TNDK6q55NPHFshGBPAZvZkE8Pw==", + "version": "5.33.1", + "resolved": "https://registry.npmjs.org/@typescript-eslint/scope-manager/-/scope-manager-5.33.1.tgz", + "integrity": "sha512-8ibcZSqy4c5m69QpzJn8XQq9NnqAToC8OdH/W6IXPXv83vRyEDPYLdjAlUx8h/rbusq6MkW4YdQzURGOqsn3CA==", "dev": true, "dependencies": { - "@typescript-eslint/types": "5.33.0", - "@typescript-eslint/visitor-keys": "5.33.0" + "@typescript-eslint/types": "5.33.1", + "@typescript-eslint/visitor-keys": "5.33.1" }, "engines": { "node": "^12.22.0 || ^14.17.0 || >=16.0.0" @@ -389,12 +459,12 @@ } }, "node_modules/@typescript-eslint/type-utils": { - "version": "5.33.0", - "resolved": "https://registry.npmjs.org/@typescript-eslint/type-utils/-/type-utils-5.33.0.tgz", - "integrity": "sha512-2zB8uEn7hEH2pBeyk3NpzX1p3lF9dKrEbnXq1F7YkpZ6hlyqb2yZujqgRGqXgRBTHWIUG3NGx/WeZk224UKlIA==", + "version": "5.33.1", + "resolved": "https://registry.npmjs.org/@typescript-eslint/type-utils/-/type-utils-5.33.1.tgz", + "integrity": "sha512-X3pGsJsD8OiqhNa5fim41YtlnyiWMF/eKsEZGsHID2HcDqeSC5yr/uLOeph8rNF2/utwuI0IQoAK3fpoxcLl2g==", "dev": true, "dependencies": { - "@typescript-eslint/utils": "5.33.0", + "@typescript-eslint/utils": "5.33.1", "debug": "^4.3.4", "tsutils": "^3.21.0" }, @@ -415,9 +485,9 @@ } }, "node_modules/@typescript-eslint/types": { - "version": "5.33.0", - "resolved": "https://registry.npmjs.org/@typescript-eslint/types/-/types-5.33.0.tgz", - "integrity": "sha512-nIMt96JngB4MYFYXpZ/3ZNU4GWPNdBbcB5w2rDOCpXOVUkhtNlG2mmm8uXhubhidRZdwMaMBap7Uk8SZMU/ppw==", + "version": "5.33.1", + "resolved": "https://registry.npmjs.org/@typescript-eslint/types/-/types-5.33.1.tgz", + "integrity": "sha512-7K6MoQPQh6WVEkMrMW5QOA5FO+BOwzHSNd0j3+BlBwd6vtzfZceJ8xJ7Um2XDi/O3umS8/qDX6jdy2i7CijkwQ==", "dev": true, "engines": { "node": "^12.22.0 || ^14.17.0 || >=16.0.0" @@ -428,13 +498,13 @@ } }, "node_modules/@typescript-eslint/typescript-estree": { - "version": "5.33.0", - "resolved": "https://registry.npmjs.org/@typescript-eslint/typescript-estree/-/typescript-estree-5.33.0.tgz", - "integrity": "sha512-tqq3MRLlggkJKJUrzM6wltk8NckKyyorCSGMq4eVkyL5sDYzJJcMgZATqmF8fLdsWrW7OjjIZ1m9v81vKcaqwQ==", + "version": "5.33.1", + "resolved": "https://registry.npmjs.org/@typescript-eslint/typescript-estree/-/typescript-estree-5.33.1.tgz", + "integrity": "sha512-JOAzJ4pJ+tHzA2pgsWQi4804XisPHOtbvwUyqsuuq8+y5B5GMZs7lI1xDWs6V2d7gE/Ez5bTGojSK12+IIPtXA==", "dev": true, "dependencies": { - "@typescript-eslint/types": "5.33.0", - "@typescript-eslint/visitor-keys": "5.33.0", + "@typescript-eslint/types": "5.33.1", + "@typescript-eslint/visitor-keys": "5.33.1", "debug": "^4.3.4", "globby": "^11.1.0", "is-glob": "^4.0.3", @@ -455,15 +525,15 @@ } }, "node_modules/@typescript-eslint/utils": { - "version": "5.33.0", - "resolved": "https://registry.npmjs.org/@typescript-eslint/utils/-/utils-5.33.0.tgz", - "integrity": "sha512-JxOAnXt9oZjXLIiXb5ZIcZXiwVHCkqZgof0O8KPgz7C7y0HS42gi75PdPlqh1Tf109M0fyUw45Ao6JLo7S5AHw==", + "version": "5.33.1", + "resolved": "https://registry.npmjs.org/@typescript-eslint/utils/-/utils-5.33.1.tgz", + "integrity": "sha512-uphZjkMaZ4fE8CR4dU7BquOV6u0doeQAr8n6cQenl/poMaIyJtBu8eys5uk6u5HiDH01Mj5lzbJ5SfeDz7oqMQ==", "dev": true, "dependencies": { "@types/json-schema": "^7.0.9", - "@typescript-eslint/scope-manager": "5.33.0", - "@typescript-eslint/types": "5.33.0", - "@typescript-eslint/typescript-estree": "5.33.0", + "@typescript-eslint/scope-manager": "5.33.1", + "@typescript-eslint/types": "5.33.1", + "@typescript-eslint/typescript-estree": "5.33.1", "eslint-scope": "^5.1.1", "eslint-utils": "^3.0.0" }, @@ -479,12 +549,12 @@ } }, "node_modules/@typescript-eslint/visitor-keys": { - "version": "5.33.0", - "resolved": "https://registry.npmjs.org/@typescript-eslint/visitor-keys/-/visitor-keys-5.33.0.tgz", - "integrity": "sha512-/XsqCzD4t+Y9p5wd9HZiptuGKBlaZO5showwqODii5C0nZawxWLF+Q6k5wYHBrQv96h6GYKyqqMHCSTqta8Kiw==", + "version": "5.33.1", + "resolved": "https://registry.npmjs.org/@typescript-eslint/visitor-keys/-/visitor-keys-5.33.1.tgz", + "integrity": "sha512-nwIxOK8Z2MPWltLKMLOEZwmfBZReqUdbEoHQXeCpa+sRVARe5twpJGHCB4dk9903Yaf0nMAlGbQfaAH92F60eg==", "dev": true, "dependencies": { - "@typescript-eslint/types": "5.33.0", + "@typescript-eslint/types": "5.33.1", "eslint-visitor-keys": "^3.3.0" }, "engines": { @@ -495,6 +565,18 @@ "url": "https://opencollective.com/typescript-eslint" } }, + "node_modules/accepts": { + "version": "1.3.8", + "resolved": "https://registry.npmjs.org/accepts/-/accepts-1.3.8.tgz", + "integrity": "sha512-PYAthTa2m2VKxuvSD3DPC/Gy+U+sOA1LAuT8mkmRuvw+NACSaeXEQ+NHcVF7rONl6qcaxV3Uuemwawk+7+SJLw==", + "dependencies": { + "mime-types": "~2.1.34", + "negotiator": "0.6.3" + }, + "engines": { + "node": ">= 0.6" + } + }, "node_modules/acorn": { "version": "8.8.0", "resolved": "https://registry.npmjs.org/acorn/-/acorn-8.8.0.tgz", @@ -598,9 +680,9 @@ } }, "node_modules/app-root-path": { - "version": "3.0.0", - "resolved": "https://registry.npmjs.org/app-root-path/-/app-root-path-3.0.0.tgz", - "integrity": "sha512-qMcx+Gy2UZynHjOHOIXPNvpf+9cjvk3cWrBBK7zg4gH9+clobJRb9NGzcT7mQTcV/6Gm/1WelUtqxVXnNlrwcw==", + "version": "3.1.0", + "resolved": "https://registry.npmjs.org/app-root-path/-/app-root-path-3.1.0.tgz", + "integrity": "sha512-biN3PwB2gUtjaYy/isrU3aNWI5w+fAfvHkSvCKeQGxhmYpwKFUxudR3Yya+KqVRHBmEDYh+/lTozYCFbmzX4nA==", "engines": { "node": ">= 6.0.0" } @@ -612,9 +694,12 @@ "dev": true }, "node_modules/argparse": { - "version": "2.0.1", - "resolved": "https://registry.npmjs.org/argparse/-/argparse-2.0.1.tgz", - "integrity": "sha512-8+9WqebbFzpX9OR+Wa6O29asIogeRMzcGtAINdpMHHyAg10f05aSFVBbcEqGf/PXw1EjAZ+q2/bEBg3DvurK3Q==" + "version": "1.0.10", + "resolved": "https://registry.npmjs.org/argparse/-/argparse-1.0.10.tgz", + "integrity": "sha512-o5Roy6tNG4SL/FOkCAN6RzjiakZS25RLYFrcMttJqbdd8BWrnA+fGz57iN5Pb06pvBGvl5gQ0B48dJlslXvoTg==", + "dependencies": { + "sprintf-js": "~1.0.2" + } }, "node_modules/args": { "version": "5.0.3", @@ -694,6 +779,11 @@ "node": ">=4" } }, + "node_modules/array-flatten": { + "version": "1.1.1", + "resolved": "https://registry.npmjs.org/array-flatten/-/array-flatten-1.1.1.tgz", + "integrity": "sha512-PCVAQswWemu6UdxsDFFX/+gVeYqKAod3D3UVm91jHwynguOwAvYPhx8nNlM++NqRcK6CxxpUafjmhIdKiHibqg==" + }, "node_modules/array-union": { "version": "2.1.0", "resolved": "https://registry.npmjs.org/array-union/-/array-union-2.1.0.tgz", @@ -812,6 +902,42 @@ "safe-buffer": "~5.2.0" } }, + "node_modules/body-parser": { + "version": "1.20.0", + "resolved": "https://registry.npmjs.org/body-parser/-/body-parser-1.20.0.tgz", + "integrity": "sha512-DfJ+q6EPcGKZD1QWUjSpqp+Q7bDQTsQIF4zfUAtZ6qk+H/3/QRhg9CEp39ss+/T2vw0+HaidC0ecJj/DRLIaKg==", + "dependencies": { + "bytes": "3.1.2", + "content-type": "~1.0.4", + "debug": "2.6.9", + "depd": "2.0.0", + "destroy": "1.2.0", + "http-errors": "2.0.0", + "iconv-lite": "0.4.24", + "on-finished": "2.4.1", + "qs": "6.10.3", + "raw-body": "2.5.1", + "type-is": "~1.6.18", + "unpipe": "1.0.0" + }, + "engines": { + "node": ">= 0.8", + "npm": "1.2.8000 || >= 1.4.16" + } + }, + "node_modules/body-parser/node_modules/debug": { + "version": "2.6.9", + "resolved": "https://registry.npmjs.org/debug/-/debug-2.6.9.tgz", + "integrity": "sha512-bC7ElrdJaJnPbAP+1EotYvqZsb3ecl5wi6Bfi6BJTUcNowp6cvspg0jXznRTKDjm/E7AdgFBVeAPVMNcKGsHMA==", + "dependencies": { + "ms": "2.0.0" + } + }, + "node_modules/body-parser/node_modules/ms": { + "version": "2.0.0", + "resolved": "https://registry.npmjs.org/ms/-/ms-2.0.0.tgz", + "integrity": "sha512-Tpp60P6IUJDTuOq/5Z8cdskzJujfwqfOTkrwIwj7IRISpnkJnT6SyJ4PCPnGMoFjC9ddhal5KVIYtAt97ix05A==" + }, "node_modules/brace-expansion": { "version": "1.1.11", "resolved": "https://registry.npmjs.org/brace-expansion/-/brace-expansion-1.1.11.tgz", @@ -863,6 +989,26 @@ "integrity": "sha512-E+XQCRwSbaaiChtv6k6Dwgc+bx+Bs6vuKJHHl5kox/BaKbhiXzqQOwK4cO22yElGp2OCmjwVhT3HmxgyPGnJfQ==", "dev": true }, + "node_modules/bytes": { + "version": "3.1.2", + "resolved": "https://registry.npmjs.org/bytes/-/bytes-3.1.2.tgz", + "integrity": "sha512-/Nf7TyzTx6S3yRJObOAV7956r8cr2+Oj8AC5dt8wSP3BQAoeX58NoHyCU8P8zGkNXStjTSi6fzO6F0pBdcYbEg==", + "engines": { + "node": ">= 0.8" + } + }, + "node_modules/call-bind": { + "version": "1.0.2", + "resolved": "https://registry.npmjs.org/call-bind/-/call-bind-1.0.2.tgz", + "integrity": "sha512-7O+FbCihrB5WGbFYesctwmTKae6rOiIzmz1icreWJ+0aA7LJfuqhEso2T9ncpcFtzMQtzXf2QGGueWJGTYsqrA==", + "dependencies": { + "function-bind": "^1.1.1", + "get-intrinsic": "^1.0.2" + }, + "funding": { + "url": "https://github.com/sponsors/ljharb" + } + }, "node_modules/callsites": { "version": "3.1.0", "resolved": "https://registry.npmjs.org/callsites/-/callsites-3.1.0.tgz", @@ -1050,6 +1196,38 @@ "resolved": "https://registry.npmjs.org/concat-map/-/concat-map-0.0.1.tgz", "integrity": "sha512-/Srv4dswyQNBfohGpz9o6Yb3Gz3SrUDqBH5rTuhGR7ahtlbYKnVxw2bCFMRljaA7EXHaXZ8wsHdodFvbkhKmqg==" }, + "node_modules/content-disposition": { + "version": "0.5.4", + "resolved": "https://registry.npmjs.org/content-disposition/-/content-disposition-0.5.4.tgz", + "integrity": "sha512-FveZTNuGw04cxlAiWbzi6zTAL/lhehaWbTtgluJh4/E95DqMwTmha3KZN1aAWA8cFIhHzMZUvLevkw5Rqk+tSQ==", + "dependencies": { + "safe-buffer": "5.2.1" + }, + "engines": { + "node": ">= 0.6" + } + }, + "node_modules/content-type": { + "version": "1.0.4", + "resolved": "https://registry.npmjs.org/content-type/-/content-type-1.0.4.tgz", + "integrity": "sha512-hIP3EEPs8tB9AT1L+NUqtwOAps4mk2Zob89MWXMHjHWg9milF/j4osnnQLXBCBFBk/tvIG/tUc9mOUJiPBhPXA==", + "engines": { + "node": ">= 0.6" + } + }, + "node_modules/cookie": { + "version": "0.5.0", + "resolved": "https://registry.npmjs.org/cookie/-/cookie-0.5.0.tgz", + "integrity": "sha512-YZ3GUyn/o8gfKJlnlX7g7xq4gyO6OSuhGPKaaGssGB2qgDUS0gPgtTvoyZLTt9Ab6dC4hfc9dV5arkvc/OCmrw==", + "engines": { + "node": ">= 0.6" + } + }, + "node_modules/cookie-signature": { + "version": "1.0.6", + "resolved": "https://registry.npmjs.org/cookie-signature/-/cookie-signature-1.0.6.tgz", + "integrity": "sha512-QADzlaHc8icV8I7vbaJXJwod9HWYp8uCqf1xa4OfNu1T7JVxQIrUgOWtHdNDtPiywmFbiS12VjotIXLrKM3orQ==" + }, "node_modules/copyfiles": { "version": "2.4.1", "resolved": "https://registry.npmjs.org/copyfiles/-/copyfiles-2.4.1.tgz", @@ -1184,6 +1362,23 @@ "node": ">=0.4.0" } }, + "node_modules/depd": { + "version": "2.0.0", + "resolved": "https://registry.npmjs.org/depd/-/depd-2.0.0.tgz", + "integrity": "sha512-g7nH6P6dyDioJogAAGprGpCtVImJhpPk/roCzdb3fIh61/s/nPsfR6onyMwkCAR/OlC3yBC0lESvUoQEAssIrw==", + "engines": { + "node": ">= 0.8" + } + }, + "node_modules/destroy": { + "version": "1.2.0", + "resolved": "https://registry.npmjs.org/destroy/-/destroy-1.2.0.tgz", + "integrity": "sha512-2sJGJTaXIIaR1w4iJSNoN0hnMY7Gpc/n8D4qSCJw8QqFWXf7cuAgnEHxBpweaVcPevC2l3KpjYCx3NypQQgaJg==", + "engines": { + "node": ">= 0.8", + "npm": "1.2.8000 || >= 1.4.16" + } + }, "node_modules/detect-libc": { "version": "2.0.1", "resolved": "https://registry.npmjs.org/detect-libc/-/detect-libc-2.0.1.tgz", @@ -1243,11 +1438,24 @@ "xtend": "^4.0.0" } }, + "node_modules/ee-first": { + "version": "1.1.1", + "resolved": "https://registry.npmjs.org/ee-first/-/ee-first-1.1.1.tgz", + "integrity": "sha512-WMwm9LhRUo+WUaRN+vRuETqG89IgZphVSNkdFgeb6sS/E4OrDIN7t48CAewSHXc6C8lefD8KKfr5vY61brQlow==" + }, "node_modules/emoji-regex": { "version": "8.0.0", "resolved": "https://registry.npmjs.org/emoji-regex/-/emoji-regex-8.0.0.tgz", "integrity": "sha512-MSjYzcWNOA0ewAHpz0MxpYFvwg6yjy1NG3xteoqz644VCo/RPgnr1/GGt+ic3iJTzQ8Eu3TdM14SawnVUmGE6A==" }, + "node_modules/encodeurl": { + "version": "1.0.2", + "resolved": "https://registry.npmjs.org/encodeurl/-/encodeurl-1.0.2.tgz", + "integrity": "sha512-TPJXq8JqFaVYm2CWmPvnP2Iyo4ZSM7/QKcSmuMLDObfpH5fi7RUGmd/rTDf+rut/saiDiQEeVTNgAmJEdAOx0w==", + "engines": { + "node": ">= 0.8" + } + }, "node_modules/end-of-stream": { "version": "1.4.4", "resolved": "https://registry.npmjs.org/end-of-stream/-/end-of-stream-1.4.4.tgz", @@ -1264,6 +1472,11 @@ "node": ">=6" } }, + "node_modules/escape-html": { + "version": "1.0.3", + "resolved": "https://registry.npmjs.org/escape-html/-/escape-html-1.0.3.tgz", + "integrity": "sha512-NiSupZ4OeuGwr68lGIeym/ksIZMJodUGOSCZ/FSnTxcrekbvqrgdUxlJOMpijaKZVjAJrWrGs/6Jy8OMuyj9ow==" + }, "node_modules/escape-string-regexp": { "version": "4.0.0", "resolved": "https://registry.npmjs.org/escape-string-regexp/-/escape-string-regexp-4.0.0.tgz", @@ -1277,9 +1490,9 @@ } }, "node_modules/eslint": { - "version": "8.21.0", - "resolved": "https://registry.npmjs.org/eslint/-/eslint-8.21.0.tgz", - "integrity": "sha512-/XJ1+Qurf1T9G2M5IHrsjp+xrGT73RZf23xA1z5wB1ZzzEAWSZKvRwhWxTFp1rvkvCfwcvAUNAP31bhKTTGfDA==", + "version": "8.22.0", + "resolved": "https://registry.npmjs.org/eslint/-/eslint-8.22.0.tgz", + "integrity": "sha512-ci4t0sz6vSRKdmkOGmprBo6fmI4PrphDFMy5JEq/fNS0gQkJM3rLmrqcp8ipMcdobH3KtUP40KniAE9W19S4wA==", "dev": true, "dependencies": { "@eslint/eslintrc": "^1.3.0", @@ -1381,6 +1594,12 @@ "node": "^12.22.0 || ^14.17.0 || >=16.0.0" } }, + "node_modules/eslint/node_modules/argparse": { + "version": "2.0.1", + "resolved": "https://registry.npmjs.org/argparse/-/argparse-2.0.1.tgz", + "integrity": "sha512-8+9WqebbFzpX9OR+Wa6O29asIogeRMzcGtAINdpMHHyAg10f05aSFVBbcEqGf/PXw1EjAZ+q2/bEBg3DvurK3Q==", + "dev": true + }, "node_modules/eslint/node_modules/eslint-scope": { "version": "7.1.1", "resolved": "https://registry.npmjs.org/eslint-scope/-/eslint-scope-7.1.1.tgz", @@ -1403,6 +1622,18 @@ "node": ">=4.0" } }, + "node_modules/eslint/node_modules/js-yaml": { + "version": "4.1.0", + "resolved": "https://registry.npmjs.org/js-yaml/-/js-yaml-4.1.0.tgz", + "integrity": "sha512-wpxZs9NoxZaJESJGIZTyDEaYpl0FKSA+FB9aJiyemKhMwkxQg63h4T1KJgUGHpTqPDNRcmmYLugrRjJlBtWvRA==", + "dev": true, + "dependencies": { + "argparse": "^2.0.1" + }, + "bin": { + "js-yaml": "bin/js-yaml.js" + } + }, "node_modules/espree": { "version": "9.3.3", "resolved": "https://registry.npmjs.org/espree/-/espree-9.3.3.tgz", @@ -1492,6 +1723,14 @@ "node": ">=0.10.0" } }, + "node_modules/etag": { + "version": "1.8.1", + "resolved": "https://registry.npmjs.org/etag/-/etag-1.8.1.tgz", + "integrity": "sha512-aIL5Fx7mawVa300al2BnEE4iNvo1qETxLrPI/o05L7z6go7fCw1J6EQmbK4FmJ2AS7kgVF/KEZWufBfdClMcPg==", + "engines": { + "node": ">= 0.6" + } + }, "node_modules/expand-template": { "version": "2.0.3", "resolved": "https://registry.npmjs.org/expand-template/-/expand-template-2.0.3.tgz", @@ -1501,6 +1740,60 @@ "node": ">=6" } }, + "node_modules/express": { + "version": "4.18.1", + "resolved": "https://registry.npmjs.org/express/-/express-4.18.1.tgz", + "integrity": "sha512-zZBcOX9TfehHQhtupq57OF8lFZ3UZi08Y97dwFCkD8p9d/d2Y3M+ykKcwaMDEL+4qyUolgBDX6AblpR3fL212Q==", + "dependencies": { + "accepts": "~1.3.8", + "array-flatten": "1.1.1", + "body-parser": "1.20.0", + "content-disposition": "0.5.4", + "content-type": "~1.0.4", + "cookie": "0.5.0", + "cookie-signature": "1.0.6", + "debug": "2.6.9", + "depd": "2.0.0", + "encodeurl": "~1.0.2", + "escape-html": "~1.0.3", + "etag": "~1.8.1", + "finalhandler": "1.2.0", + "fresh": "0.5.2", + "http-errors": "2.0.0", + "merge-descriptors": "1.0.1", + "methods": "~1.1.2", + "on-finished": "2.4.1", + "parseurl": "~1.3.3", + "path-to-regexp": "0.1.7", + "proxy-addr": "~2.0.7", + "qs": "6.10.3", + "range-parser": "~1.2.1", + "safe-buffer": "5.2.1", + "send": "0.18.0", + "serve-static": "1.15.0", + "setprototypeof": "1.2.0", + "statuses": "2.0.1", + "type-is": "~1.6.18", + "utils-merge": "1.0.1", + "vary": "~1.1.2" + }, + "engines": { + "node": ">= 0.10.0" + } + }, + "node_modules/express/node_modules/debug": { + "version": "2.6.9", + "resolved": "https://registry.npmjs.org/debug/-/debug-2.6.9.tgz", + "integrity": "sha512-bC7ElrdJaJnPbAP+1EotYvqZsb3ecl5wi6Bfi6BJTUcNowp6cvspg0jXznRTKDjm/E7AdgFBVeAPVMNcKGsHMA==", + "dependencies": { + "ms": "2.0.0" + } + }, + "node_modules/express/node_modules/ms": { + "version": "2.0.0", + "resolved": "https://registry.npmjs.org/ms/-/ms-2.0.0.tgz", + "integrity": "sha512-Tpp60P6IUJDTuOq/5Z8cdskzJujfwqfOTkrwIwj7IRISpnkJnT6SyJ4PCPnGMoFjC9ddhal5KVIYtAt97ix05A==" + }, "node_modules/fast-deep-equal": { "version": "3.1.3", "resolved": "https://registry.npmjs.org/fast-deep-equal/-/fast-deep-equal-3.1.3.tgz", @@ -1548,9 +1841,9 @@ "dev": true }, "node_modules/fast-redact": { - "version": "3.1.1", - "resolved": "https://registry.npmjs.org/fast-redact/-/fast-redact-3.1.1.tgz", - "integrity": "sha512-odVmjC8x8jNeMZ3C+rPMESzXVSEU8tSWSHv9HFxP2mm89G/1WwqhrerJDQm9Zus8X6aoRgQDThKqptdNA6bt+A==", + "version": "3.1.2", + "resolved": "https://registry.npmjs.org/fast-redact/-/fast-redact-3.1.2.tgz", + "integrity": "sha512-+0em+Iya9fKGfEQGcd62Yv6onjBmmhV1uh86XVfOU8VwAe6kaFdQCWI9s0/Nnugx5Vd9tdbZ7e6gE2tR9dzXdw==", "engines": { "node": ">=6" } @@ -1599,6 +1892,36 @@ "node": ">=8" } }, + "node_modules/finalhandler": { + "version": "1.2.0", + "resolved": "https://registry.npmjs.org/finalhandler/-/finalhandler-1.2.0.tgz", + "integrity": "sha512-5uXcUVftlQMFnWC9qu/svkWv3GTd2PfUhK/3PLkYNAe7FbqJMt3515HaxE6eRL74GdsriiwujiawdaB1BpEISg==", + "dependencies": { + "debug": "2.6.9", + "encodeurl": "~1.0.2", + "escape-html": "~1.0.3", + "on-finished": "2.4.1", + "parseurl": "~1.3.3", + "statuses": "2.0.1", + "unpipe": "~1.0.0" + }, + "engines": { + "node": ">= 0.8" + } + }, + "node_modules/finalhandler/node_modules/debug": { + "version": "2.6.9", + "resolved": "https://registry.npmjs.org/debug/-/debug-2.6.9.tgz", + "integrity": "sha512-bC7ElrdJaJnPbAP+1EotYvqZsb3ecl5wi6Bfi6BJTUcNowp6cvspg0jXznRTKDjm/E7AdgFBVeAPVMNcKGsHMA==", + "dependencies": { + "ms": "2.0.0" + } + }, + "node_modules/finalhandler/node_modules/ms": { + "version": "2.0.0", + "resolved": "https://registry.npmjs.org/ms/-/ms-2.0.0.tgz", + "integrity": "sha512-Tpp60P6IUJDTuOq/5Z8cdskzJujfwqfOTkrwIwj7IRISpnkJnT6SyJ4PCPnGMoFjC9ddhal5KVIYtAt97ix05A==" + }, "node_modules/find-up": { "version": "5.0.0", "resolved": "https://registry.npmjs.org/find-up/-/find-up-5.0.0.tgz", @@ -1634,9 +1957,9 @@ "integrity": "sha512-4zPxDyhCyiN2wIAtSLI6gc82/EjqZc1onI4Mz/l0pWrAlsSfYH/2ZIcU+e3oA2wDwbzIWNKwa23F8rh6+DRWkw==" }, "node_modules/flatted": { - "version": "3.2.6", - "resolved": "https://registry.npmjs.org/flatted/-/flatted-3.2.6.tgz", - "integrity": "sha512-0sQoMh9s0BYsm+12Huy/rkKxVu4R1+r96YX5cG44rHV0pQ6iC3Q+mkoMFaGWObMFYQxCVT+ssG1ksneA2MI9KQ==", + "version": "3.2.7", + "resolved": "https://registry.npmjs.org/flatted/-/flatted-3.2.7.tgz", + "integrity": "sha512-5nqDSxl8nn5BSNxyR3n4I6eDmbolI6WT+QqR547RwxQapgjQBmtktdP+HTBb/a/zLsbzERTONyUB5pefh5TtjQ==", "dev": true }, "node_modules/follow-redirects": { @@ -1671,6 +1994,22 @@ "node": ">= 6" } }, + "node_modules/forwarded": { + "version": "0.2.0", + "resolved": "https://registry.npmjs.org/forwarded/-/forwarded-0.2.0.tgz", + "integrity": "sha512-buRG0fpBtRHSTCOASe6hD258tEubFoRLb4ZNA6NxMVHNw2gOcwHo9wyablzMzOA5z9xA9L1KNjk/Nt6MT9aYow==", + "engines": { + "node": ">= 0.6" + } + }, + "node_modules/fresh": { + "version": "0.5.2", + "resolved": "https://registry.npmjs.org/fresh/-/fresh-0.5.2.tgz", + "integrity": "sha512-zJ2mQYM18rEFOudeV4GShTGIQ7RbzA7ozbU9I/XBpm7kqgMywgmylMwXHxZJmkVoYkna9d2pVXVXPdYTP9ej8Q==", + "engines": { + "node": ">= 0.6" + } + }, "node_modules/fs-constants": { "version": "1.0.0", "resolved": "https://registry.npmjs.org/fs-constants/-/fs-constants-1.0.0.tgz", @@ -1699,8 +2038,7 @@ "node_modules/function-bind": { "version": "1.1.1", "resolved": "https://registry.npmjs.org/function-bind/-/function-bind-1.1.1.tgz", - "integrity": "sha512-yIovAzMX49sF8Yl58fSCWJ5svSLuaibPxXQJFLmBObTuCr0Mf1KiPopGM9NiFjiYBCbfaa2Fh6breQ6ANVTI0A==", - "dev": true + "integrity": "sha512-yIovAzMX49sF8Yl58fSCWJ5svSLuaibPxXQJFLmBObTuCr0Mf1KiPopGM9NiFjiYBCbfaa2Fh6breQ6ANVTI0A==" }, "node_modules/functional-red-black-tree": { "version": "1.0.1", @@ -1716,6 +2054,19 @@ "node": "6.* || 8.* || >= 10.*" } }, + "node_modules/get-intrinsic": { + "version": "1.1.2", + "resolved": "https://registry.npmjs.org/get-intrinsic/-/get-intrinsic-1.1.2.tgz", + "integrity": "sha512-Jfm3OyCxHh9DJyc28qGk+JmfkpO41A4XkneDSujN9MDXrm4oDKdHvndhZ2dN94+ERNfkYJWDclW6k2L/ZGHjXA==", + "dependencies": { + "function-bind": "^1.1.1", + "has": "^1.0.3", + "has-symbols": "^1.0.3" + }, + "funding": { + "url": "https://github.com/sponsors/ljharb" + } + }, "node_modules/github-from-package": { "version": "0.0.0", "resolved": "https://registry.npmjs.org/github-from-package/-/github-from-package-0.0.0.tgz", @@ -1808,7 +2159,6 @@ "version": "1.0.3", "resolved": "https://registry.npmjs.org/has/-/has-1.0.3.tgz", "integrity": "sha512-f2dvO0VU6Oej7RkWJGrehjbzMAjFp5/VKPp5tTpWIV4JHHZK1/BxbFRtf/siA2SWTe09caDmVtYYzWEIbBS4zw==", - "dev": true, "dependencies": { "function-bind": "^1.1.1" }, @@ -1824,6 +2174,17 @@ "node": ">=8" } }, + "node_modules/has-symbols": { + "version": "1.0.3", + "resolved": "https://registry.npmjs.org/has-symbols/-/has-symbols-1.0.3.tgz", + "integrity": "sha512-l3LCuF6MgDNwTDKkdYGEihYjt5pRPbEg46rtlmnSPlUbgmB8LOIrKJbYYFBSbnPaJexMKtiPO8hmeRjRz2Td+A==", + "engines": { + "node": ">= 0.4" + }, + "funding": { + "url": "https://github.com/sponsors/ljharb" + } + }, "node_modules/highlight.js": { "version": "10.7.3", "resolved": "https://registry.npmjs.org/highlight.js/-/highlight.js-10.7.3.tgz", @@ -1832,6 +2193,32 @@ "node": "*" } }, + "node_modules/http-errors": { + "version": "2.0.0", + "resolved": "https://registry.npmjs.org/http-errors/-/http-errors-2.0.0.tgz", + "integrity": "sha512-FtwrG/euBzaEjYeRqOgly7G0qviiXoJWnvEH2Z1plBdXgbyjv34pHTSb9zoeHMyDy33+DWy5Wt9Wo+TURtOYSQ==", + "dependencies": { + "depd": "2.0.0", + "inherits": "2.0.4", + "setprototypeof": "1.2.0", + "statuses": "2.0.1", + "toidentifier": "1.0.1" + }, + "engines": { + "node": ">= 0.8" + } + }, + "node_modules/iconv-lite": { + "version": "0.4.24", + "resolved": "https://registry.npmjs.org/iconv-lite/-/iconv-lite-0.4.24.tgz", + "integrity": "sha512-v3MXnZAcvnywkTUEZomIActle7RXXeedOR31wwl7VlyoXO4Qi9arvSenNQWne1TcRwhCL1HwLI21bEqdpj8/rA==", + "dependencies": { + "safer-buffer": ">= 2.1.2 < 3" + }, + "engines": { + "node": ">=0.10.0" + } + }, "node_modules/ieee754": { "version": "1.2.1", "resolved": "https://registry.npmjs.org/ieee754/-/ieee754-1.2.1.tgz", @@ -1905,6 +2292,14 @@ "integrity": "sha512-JV/yugV2uzW5iMRSiZAyDtQd+nxtUnjeLt0acNdw98kKLrvuRVyB80tsREOE7yvGVgalhZ6RNXCmEHkUKBKxew==", "devOptional": true }, + "node_modules/ipaddr.js": { + "version": "1.9.1", + "resolved": "https://registry.npmjs.org/ipaddr.js/-/ipaddr.js-1.9.1.tgz", + "integrity": "sha512-0KI/607xoxSToH7GjN1FfSbLoU0+btTicjsQSWQlh/hZykN8KpmMf7uYwPW3R+akZ6R/w18ZlXSHBYXiYUPO3g==", + "engines": { + "node": ">= 0.10" + } + }, "node_modules/is-binary-path": { "version": "2.1.0", "resolved": "https://registry.npmjs.org/is-binary-path/-/is-binary-path-2.1.0.tgz", @@ -1996,11 +2391,12 @@ } }, "node_modules/js-yaml": { - "version": "4.1.0", - "resolved": "https://registry.npmjs.org/js-yaml/-/js-yaml-4.1.0.tgz", - "integrity": "sha512-wpxZs9NoxZaJESJGIZTyDEaYpl0FKSA+FB9aJiyemKhMwkxQg63h4T1KJgUGHpTqPDNRcmmYLugrRjJlBtWvRA==", + "version": "3.14.1", + "resolved": "https://registry.npmjs.org/js-yaml/-/js-yaml-3.14.1.tgz", + "integrity": "sha512-okMH7OXXJ7YrN9Ok3/SXrnu4iX9yOk+25nqX4imS2npuvTYDmo/QEZoqwZkYaIDk3jVvBOTOIEgEhaLOynBS9g==", "dependencies": { - "argparse": "^2.0.1" + "argparse": "^1.0.7", + "esprima": "^4.0.0" }, "bin": { "js-yaml": "bin/js-yaml.js" @@ -2089,6 +2485,19 @@ "integrity": "sha512-s8UhlNe7vPKomQhC1qFelMokr/Sc3AgNbso3n74mVPA5LTZwkB9NlXf4XPamLxJE8h0gh73rM94xvwRT2CVInw==", "dev": true }, + "node_modules/media-typer": { + "version": "0.3.0", + "resolved": "https://registry.npmjs.org/media-typer/-/media-typer-0.3.0.tgz", + "integrity": "sha512-dq+qelQ9akHpcOl/gUVRTxVIOkAJ1wR3QAvb4RsVjS8oVoFjDGTc679wJYmUmknUF5HwMLOgb5O+a3KxfWapPQ==", + "engines": { + "node": ">= 0.6" + } + }, + "node_modules/merge-descriptors": { + "version": "1.0.1", + "resolved": "https://registry.npmjs.org/merge-descriptors/-/merge-descriptors-1.0.1.tgz", + "integrity": "sha512-cCi6g3/Zr1iqQi6ySbseM1Xvooa98N0w31jzUYrXPX2xqObmFGHJ0tQ5u74H3mVh7wLouTseZyYIq39g8cNp1w==" + }, "node_modules/merge2": { "version": "1.4.1", "resolved": "https://registry.npmjs.org/merge2/-/merge2-1.4.1.tgz", @@ -2098,6 +2507,14 @@ "node": ">= 8" } }, + "node_modules/methods": { + "version": "1.1.2", + "resolved": "https://registry.npmjs.org/methods/-/methods-1.1.2.tgz", + "integrity": "sha512-iclAHeNqNm68zFtnZ0e+1L2yUIdvzNoauKU4WBA3VvH/vPFieF7qfRlwUZU+DA9P9bPXIS90ulxoUoCH23sV2w==", + "engines": { + "node": ">= 0.6" + } + }, "node_modules/micromatch": { "version": "4.0.5", "resolved": "https://registry.npmjs.org/micromatch/-/micromatch-4.0.5.tgz", @@ -2111,6 +2528,17 @@ "node": ">=8.6" } }, + "node_modules/mime": { + "version": "1.6.0", + "resolved": "https://registry.npmjs.org/mime/-/mime-1.6.0.tgz", + "integrity": "sha512-x0Vn8spI+wuJ1O6S7gnbaQg8Pxh4NNHb7KSINmEWKiPE4RKOplvijn+NkmYmmRgP68mc70j2EbeTFRsrswaQeg==", + "bin": { + "mime": "cli.js" + }, + "engines": { + "node": ">=4" + } + }, "node_modules/mime-db": { "version": "1.52.0", "resolved": "https://registry.npmjs.org/mime-db/-/mime-db-1.52.0.tgz", @@ -2211,6 +2639,14 @@ "integrity": "sha512-OWND8ei3VtNC9h7V60qff3SVobHr996CTwgxubgyQYEpg290h9J0buyECNNJexkFm5sOajh5G116RYA1c8ZMSw==", "dev": true }, + "node_modules/negotiator": { + "version": "0.6.3", + "resolved": "https://registry.npmjs.org/negotiator/-/negotiator-0.6.3.tgz", + "integrity": "sha512-+EUsqGPLsM+j/zdChZjsnX51g4XrHFOIXwfnCVPGlQk/k5giakcKsuxCObBRu6DSm9opw/O6slWbJdghQM4bBg==", + "engines": { + "node": ">= 0.6" + } + }, "node_modules/node-abi": { "version": "3.24.0", "resolved": "https://registry.npmjs.org/node-abi/-/node-abi-3.24.0.tgz", @@ -2250,6 +2686,25 @@ "node": ">=0.10.0" } }, + "node_modules/object-inspect": { + "version": "1.12.2", + "resolved": "https://registry.npmjs.org/object-inspect/-/object-inspect-1.12.2.tgz", + "integrity": "sha512-z+cPxW0QGUp0mcqcsgQyLVRDoXFQbXOwBaqyF7VIgI4TWNQsDHrBpUQslRmIfAoYWdYzs6UlKJtB2XJpTaNSpQ==", + "funding": { + "url": "https://github.com/sponsors/ljharb" + } + }, + "node_modules/on-finished": { + "version": "2.4.1", + "resolved": "https://registry.npmjs.org/on-finished/-/on-finished-2.4.1.tgz", + "integrity": "sha512-oVlzkg3ENAhCk2zdv7IJwd/QUD4z2RxRwpkcGY8psCVcCYZNq4wYnVWALHM+brtuJjePWiYF/ClmuDr8Ch5+kg==", + "dependencies": { + "ee-first": "1.1.1" + }, + "engines": { + "node": ">= 0.8" + } + }, "node_modules/once": { "version": "1.4.0", "resolved": "https://registry.npmjs.org/once/-/once-1.4.0.tgz", @@ -2335,6 +2790,14 @@ "resolved": "https://registry.npmjs.org/parse5/-/parse5-6.0.1.tgz", "integrity": "sha512-Ofn/CTFzRGTTxwpNEs9PP93gXShHcTq255nzRYSKe8AkVpZY7e1fpmTfOyoIvjP5HG7Z2ZM7VS9PPhQGW2pOpw==" }, + "node_modules/parseurl": { + "version": "1.3.3", + "resolved": "https://registry.npmjs.org/parseurl/-/parseurl-1.3.3.tgz", + "integrity": "sha512-CiyeOxFT/JZyN5m0z9PfXw4SCBJ6Sygz1Dpl0wqjlhDEGGBP1GnsUVEL0p63hoG1fcj3fHynXi9NYO4nWOL+qQ==", + "engines": { + "node": ">= 0.8" + } + }, "node_modules/path-exists": { "version": "4.0.0", "resolved": "https://registry.npmjs.org/path-exists/-/path-exists-4.0.0.tgz", @@ -2367,6 +2830,11 @@ "integrity": "sha512-LDJzPVEEEPR+y48z93A0Ed0yXb8pAByGWo/k5YYdYgpY2/2EsOsksJrq7lOHxryrVOn1ejG6oAp8ahvOIQD8sw==", "dev": true }, + "node_modules/path-to-regexp": { + "version": "0.1.7", + "resolved": "https://registry.npmjs.org/path-to-regexp/-/path-to-regexp-0.1.7.tgz", + "integrity": "sha512-5DFkuoqlv1uYQKxy8omFBeJPQcdoE07Kv2sferDCrAq1ohOU+MSDswDIbnx3YAM60qIOnYa53wBhXW0EbMonrQ==" + }, "node_modules/path-type": { "version": "4.0.0", "resolved": "https://registry.npmjs.org/path-type/-/path-type-4.0.0.tgz", @@ -2508,6 +2976,18 @@ "resolved": "https://registry.npmjs.org/process-warning/-/process-warning-1.0.0.tgz", "integrity": "sha512-du4wfLyj4yCZq1VupnVSZmRsPJsNuxoDQFdCFHLaYiEbFBD7QE0a+I4D7hOxrVnh78QE/YipFAj9lXHiXocV+Q==" }, + "node_modules/proxy-addr": { + "version": "2.0.7", + "resolved": "https://registry.npmjs.org/proxy-addr/-/proxy-addr-2.0.7.tgz", + "integrity": "sha512-llQsMLSUDUPT44jdrU/O37qlnifitDP+ZwrmmZcoSKyLKvtZxpyV0n2/bD/N4tBAAZ/gJEdZU7KMraoK1+XYAg==", + "dependencies": { + "forwarded": "0.2.0", + "ipaddr.js": "1.9.1" + }, + "engines": { + "node": ">= 0.10" + } + }, "node_modules/pump": { "version": "3.0.0", "resolved": "https://registry.npmjs.org/pump/-/pump-3.0.0.tgz", @@ -2526,6 +3006,20 @@ "node": ">=6" } }, + "node_modules/qs": { + "version": "6.10.3", + "resolved": "https://registry.npmjs.org/qs/-/qs-6.10.3.tgz", + "integrity": "sha512-wr7M2E0OFRfIfJZjKGieI8lBKb7fRCH4Fv5KNPEs7gJ8jadvotdsS08PzOKR7opXhZ/Xkjtt3WF9g38drmyRqQ==", + "dependencies": { + "side-channel": "^1.0.4" + }, + "engines": { + "node": ">=0.6" + }, + "funding": { + "url": "https://github.com/sponsors/ljharb" + } + }, "node_modules/queue-microtask": { "version": "1.2.3", "resolved": "https://registry.npmjs.org/queue-microtask/-/queue-microtask-1.2.3.tgz", @@ -2551,6 +3045,28 @@ "resolved": "https://registry.npmjs.org/quick-format-unescaped/-/quick-format-unescaped-4.0.4.tgz", "integrity": "sha512-tYC1Q1hgyRuHgloV/YXs2w15unPVh8qfu/qCTfhTYamaw7fyhumKa2yGpdSo87vY32rIclj+4fWYQXUMs9EHvg==" }, + "node_modules/range-parser": { + "version": "1.2.1", + "resolved": "https://registry.npmjs.org/range-parser/-/range-parser-1.2.1.tgz", + "integrity": "sha512-Hrgsx+orqoygnmhFbKaHE6c296J+HTAQXoxEF6gNupROmmGJRoyzfG3ccAveqCBrwr/2yxQ5BVd/GTl5agOwSg==", + "engines": { + "node": ">= 0.6" + } + }, + "node_modules/raw-body": { + "version": "2.5.1", + "resolved": "https://registry.npmjs.org/raw-body/-/raw-body-2.5.1.tgz", + "integrity": "sha512-qqJBtEyVgS0ZmPGdCFPWJ3FreoqvG4MVQln/kCgF7Olq95IbOp0/BWyMwbdtn4VTvkM8Y7khCQ2Xgk/tcrCXig==", + "dependencies": { + "bytes": "3.1.2", + "http-errors": "2.0.0", + "iconv-lite": "0.4.24", + "unpipe": "1.0.0" + }, + "engines": { + "node": ">= 0.8" + } + }, "node_modules/rc": { "version": "1.2.8", "resolved": "https://registry.npmjs.org/rc/-/rc-1.2.8.tgz", @@ -2722,6 +3238,11 @@ } ] }, + "node_modules/safer-buffer": { + "version": "2.1.2", + "resolved": "https://registry.npmjs.org/safer-buffer/-/safer-buffer-2.1.2.tgz", + "integrity": "sha512-YZo3K82SD7Riyi0E1EQPojLz7kpepnSQI9IyPbHHg1XXXevb5dJI7tpyN2ADxGcQbHG7vcyRHk0cbwqcQriUtg==" + }, "node_modules/sax": { "version": "1.2.4", "resolved": "https://registry.npmjs.org/sax/-/sax-1.2.4.tgz", @@ -2742,6 +3263,66 @@ "node": ">=10" } }, + "node_modules/send": { + "version": "0.18.0", + "resolved": "https://registry.npmjs.org/send/-/send-0.18.0.tgz", + "integrity": "sha512-qqWzuOjSFOuqPjFe4NOsMLafToQQwBSOEpS+FwEt3A2V3vKubTquT3vmLTQpFgMXp8AlFWFuP1qKaJZOtPpVXg==", + "dependencies": { + "debug": "2.6.9", + "depd": "2.0.0", + "destroy": "1.2.0", + "encodeurl": "~1.0.2", + "escape-html": "~1.0.3", + "etag": "~1.8.1", + "fresh": "0.5.2", + "http-errors": "2.0.0", + "mime": "1.6.0", + "ms": "2.1.3", + "on-finished": "2.4.1", + "range-parser": "~1.2.1", + "statuses": "2.0.1" + }, + "engines": { + "node": ">= 0.8.0" + } + }, + "node_modules/send/node_modules/debug": { + "version": "2.6.9", + "resolved": "https://registry.npmjs.org/debug/-/debug-2.6.9.tgz", + "integrity": "sha512-bC7ElrdJaJnPbAP+1EotYvqZsb3ecl5wi6Bfi6BJTUcNowp6cvspg0jXznRTKDjm/E7AdgFBVeAPVMNcKGsHMA==", + "dependencies": { + "ms": "2.0.0" + } + }, + "node_modules/send/node_modules/debug/node_modules/ms": { + "version": "2.0.0", + "resolved": "https://registry.npmjs.org/ms/-/ms-2.0.0.tgz", + "integrity": "sha512-Tpp60P6IUJDTuOq/5Z8cdskzJujfwqfOTkrwIwj7IRISpnkJnT6SyJ4PCPnGMoFjC9ddhal5KVIYtAt97ix05A==" + }, + "node_modules/send/node_modules/ms": { + "version": "2.1.3", + "resolved": "https://registry.npmjs.org/ms/-/ms-2.1.3.tgz", + "integrity": "sha512-6FlzubTLZG3J2a/NVCAleEhjzq5oxgHyaCU9yYXvcLsvoVaHJq/s5xXI6/XXP6tz7R9xAOtHnSO/tXtF3WRTlA==" + }, + "node_modules/serve-static": { + "version": "1.15.0", + "resolved": "https://registry.npmjs.org/serve-static/-/serve-static-1.15.0.tgz", + "integrity": "sha512-XGuRDNjXUijsUL0vl6nSD7cwURuzEgglbOaFuZM9g3kwDXOWVTck0jLzjPzGD+TazWbboZYu52/9/XPdUgne9g==", + "dependencies": { + "encodeurl": "~1.0.2", + "escape-html": "~1.0.3", + "parseurl": "~1.3.3", + "send": "0.18.0" + }, + "engines": { + "node": ">= 0.8.0" + } + }, + "node_modules/setprototypeof": { + "version": "1.2.0", + "resolved": "https://registry.npmjs.org/setprototypeof/-/setprototypeof-1.2.0.tgz", + "integrity": "sha512-E5LDX7Wrp85Kil5bhZv46j8jOeboKq5JMmYM3gVGdGH8xFpPWXUMsNrlODCrkoxMEeNi/XZIwuRvY4XNwYMJpw==" + }, "node_modules/sha.js": { "version": "2.4.11", "resolved": "https://registry.npmjs.org/sha.js/-/sha.js-2.4.11.tgz", @@ -2775,6 +3356,19 @@ "node": ">=8" } }, + "node_modules/side-channel": { + "version": "1.0.4", + "resolved": "https://registry.npmjs.org/side-channel/-/side-channel-1.0.4.tgz", + "integrity": "sha512-q5XPytqFEIKHkGdiMIrY10mvLRvnQh42/+GoBlFW3b2LXLE2xxJpZFdm94we0BaoV3RwJyGqg5wS7epxTv0Zvw==", + "dependencies": { + "call-bind": "^1.0.0", + "get-intrinsic": "^1.0.2", + "object-inspect": "^1.9.0" + }, + "funding": { + "url": "https://github.com/sponsors/ljharb" + } + }, "node_modules/simple-concat": { "version": "1.0.1", "resolved": "https://registry.npmjs.org/simple-concat/-/simple-concat-1.0.1.tgz", @@ -2895,6 +3489,14 @@ "resolved": "https://registry.npmjs.org/sqlite/-/sqlite-4.1.2.tgz", "integrity": "sha512-FlBG51gHbux5vPjwnoqFEghNGvnTMTbHyiI09U3qFTQs9AtWuwd4i++6+WCusCXKrVdIDLzfdGekrolr3m4U4A==" }, + "node_modules/statuses": { + "version": "2.0.1", + "resolved": "https://registry.npmjs.org/statuses/-/statuses-2.0.1.tgz", + "integrity": "sha512-RwNA9Z/7PrK06rYLIzFMlaF+l73iwpzsqRIFgbMLbTcLD6cOao82TaWefPXQvB2fOC4AjuYSEndS7N/mTCbkdQ==", + "engines": { + "node": ">= 0.8" + } + }, "node_modules/string_decoder": { "version": "0.10.31", "resolved": "https://registry.npmjs.org/string_decoder/-/string_decoder-0.10.31.tgz", @@ -3102,6 +3704,14 @@ "node": ">=8.0" } }, + "node_modules/toidentifier": { + "version": "1.0.1", + "resolved": "https://registry.npmjs.org/toidentifier/-/toidentifier-1.0.1.tgz", + "integrity": "sha512-o5sSPKEkg/DIQNmH43V0/uerLrpzVedkUh8tGNvaeXpfpuwjKenlSox/2O/BTlZUtEe+JG7s5YhEz608PlAHRA==", + "engines": { + "node": ">=0.6" + } + }, "node_modules/tree-kill": { "version": "1.2.2", "resolved": "https://registry.npmjs.org/tree-kill/-/tree-kill-1.2.2.tgz", @@ -3111,49 +3721,6 @@ "tree-kill": "cli.js" } }, - "node_modules/ts-node": { - "version": "10.9.1", - "resolved": "https://registry.npmjs.org/ts-node/-/ts-node-10.9.1.tgz", - "integrity": "sha512-NtVysVPkxxrwFGUUxGYhfux8k78pQB3JqYBXlLRZgdGUqTO5wU/UyHop5p70iEbGhB7q5KmiZiU0Y3KlJrScEw==", - "dev": true, - "dependencies": { - "@cspotcode/source-map-support": "^0.8.0", - "@tsconfig/node10": "^1.0.7", - "@tsconfig/node12": "^1.0.7", - "@tsconfig/node14": "^1.0.0", - "@tsconfig/node16": "^1.0.2", - "acorn": "^8.4.1", - "acorn-walk": "^8.1.1", - "arg": "^4.1.0", - "create-require": "^1.1.0", - "diff": "^4.0.1", - "make-error": "^1.1.1", - "v8-compile-cache-lib": "^3.0.1", - "yn": "3.1.1" - }, - "bin": { - "ts-node": "dist/bin.js", - "ts-node-cwd": "dist/bin-cwd.js", - "ts-node-esm": "dist/bin-esm.js", - "ts-node-script": "dist/bin-script.js", - "ts-node-transpile-only": "dist/bin-transpile.js", - "ts-script": "dist/bin-script-deprecated.js" - }, - "peerDependencies": { - "@swc/core": ">=1.2.50", - "@swc/wasm": ">=1.2.50", - "@types/node": "*", - "typescript": ">=2.7" - }, - "peerDependenciesMeta": { - "@swc/core": { - "optional": true - }, - "@swc/wasm": { - "optional": true - } - } - }, "node_modules/ts-node-dev": { "version": "2.0.0", "resolved": "https://registry.npmjs.org/ts-node-dev/-/ts-node-dev-2.0.0.tgz", @@ -3200,6 +3767,49 @@ "rimraf": "bin.js" } }, + "node_modules/ts-node-dev/node_modules/ts-node": { + "version": "10.9.1", + "resolved": "https://registry.npmjs.org/ts-node/-/ts-node-10.9.1.tgz", + "integrity": "sha512-NtVysVPkxxrwFGUUxGYhfux8k78pQB3JqYBXlLRZgdGUqTO5wU/UyHop5p70iEbGhB7q5KmiZiU0Y3KlJrScEw==", + "dev": true, + "dependencies": { + "@cspotcode/source-map-support": "^0.8.0", + "@tsconfig/node10": "^1.0.7", + "@tsconfig/node12": "^1.0.7", + "@tsconfig/node14": "^1.0.0", + "@tsconfig/node16": "^1.0.2", + "acorn": "^8.4.1", + "acorn-walk": "^8.1.1", + "arg": "^4.1.0", + "create-require": "^1.1.0", + "diff": "^4.0.1", + "make-error": "^1.1.1", + "v8-compile-cache-lib": "^3.0.1", + "yn": "3.1.1" + }, + "bin": { + "ts-node": "dist/bin.js", + "ts-node-cwd": "dist/bin-cwd.js", + "ts-node-esm": "dist/bin-esm.js", + "ts-node-script": "dist/bin-script.js", + "ts-node-transpile-only": "dist/bin-transpile.js", + "ts-script": "dist/bin-script-deprecated.js" + }, + "peerDependencies": { + "@swc/core": ">=1.2.50", + "@swc/wasm": ">=1.2.50", + "@types/node": "*", + "typescript": ">=2.7" + }, + "peerDependenciesMeta": { + "@swc/core": { + "optional": true + }, + "@swc/wasm": { + "optional": true + } + } + }, "node_modules/tsconfig": { "version": "7.0.0", "resolved": "https://registry.npmjs.org/tsconfig/-/tsconfig-7.0.0.tgz", @@ -3283,6 +3893,18 @@ "url": "https://github.com/sponsors/sindresorhus" } }, + "node_modules/type-is": { + "version": "1.6.18", + "resolved": "https://registry.npmjs.org/type-is/-/type-is-1.6.18.tgz", + "integrity": "sha512-TkRKr9sUTxEH8MdfuCSP7VizJyzRNMjj2J2do2Jr3Kym598JVdEksuzPQCnlFPW4ky9Q+iA+ma9BGm06XQBy8g==", + "dependencies": { + "media-typer": "0.3.0", + "mime-types": "~2.1.24" + }, + "engines": { + "node": ">= 0.6" + } + }, "node_modules/typeorm": { "version": "0.2.45", "resolved": "https://registry.npmjs.org/typeorm/-/typeorm-0.2.45.tgz", @@ -3377,6 +3999,11 @@ } } }, + "node_modules/typeorm/node_modules/argparse": { + "version": "2.0.1", + "resolved": "https://registry.npmjs.org/argparse/-/argparse-2.0.1.tgz", + "integrity": "sha512-8+9WqebbFzpX9OR+Wa6O29asIogeRMzcGtAINdpMHHyAg10f05aSFVBbcEqGf/PXw1EjAZ+q2/bEBg3DvurK3Q==" + }, "node_modules/typeorm/node_modules/buffer": { "version": "6.0.3", "resolved": "https://registry.npmjs.org/buffer/-/buffer-6.0.3.tgz", @@ -3400,6 +4027,17 @@ "ieee754": "^1.2.1" } }, + "node_modules/typeorm/node_modules/js-yaml": { + "version": "4.1.0", + "resolved": "https://registry.npmjs.org/js-yaml/-/js-yaml-4.1.0.tgz", + "integrity": "sha512-wpxZs9NoxZaJESJGIZTyDEaYpl0FKSA+FB9aJiyemKhMwkxQg63h4T1KJgUGHpTqPDNRcmmYLugrRjJlBtWvRA==", + "dependencies": { + "argparse": "^2.0.1" + }, + "bin": { + "js-yaml": "bin/js-yaml.js" + } + }, "node_modules/typescript": { "version": "4.7.4", "resolved": "https://registry.npmjs.org/typescript/-/typescript-4.7.4.tgz", @@ -3412,6 +4050,14 @@ "node": ">=4.2.0" } }, + "node_modules/unpipe": { + "version": "1.0.0", + "resolved": "https://registry.npmjs.org/unpipe/-/unpipe-1.0.0.tgz", + "integrity": "sha512-pjy2bYhSsufwWlKwPc+l3cN7+wuJlK6uz0YdJEOlQDbl6jo/YlPi4mb8agUkVC8BF7V8NuzeyPNqRksA3hztKQ==", + "engines": { + "node": ">= 0.8" + } + }, "node_modules/untildify": { "version": "4.0.0", "resolved": "https://registry.npmjs.org/untildify/-/untildify-4.0.0.tgz", @@ -3435,6 +4081,14 @@ "resolved": "https://registry.npmjs.org/util-deprecate/-/util-deprecate-1.0.2.tgz", "integrity": "sha512-EPD5q1uXyFxJpCrLnCc1nHnq3gOa6DZBocAIiI2TaSCA7VCJ1UJDMagCzIkXNsUYfD1daK//LTEQ8xiIbrHtcw==" }, + "node_modules/utils-merge": { + "version": "1.0.1", + "resolved": "https://registry.npmjs.org/utils-merge/-/utils-merge-1.0.1.tgz", + "integrity": "sha512-pMZTvIkT1d+TFGvDOqodOclx0QWkkgi6Tdoa8gC8ffGAAqz9pzPTZWAybbsHHoED/ztMtkv/VoYTYyShUn81hA==", + "engines": { + "node": ">= 0.4.0" + } + }, "node_modules/uuid": { "version": "8.3.2", "resolved": "https://registry.npmjs.org/uuid/-/uuid-8.3.2.tgz", @@ -3455,6 +4109,14 @@ "integrity": "sha512-wa7YjyUGfNZngI/vtK0UHAN+lgDCxBPCylVXGp0zu59Fz5aiGtNXaq3DhIov063MorB+VfufLh3JlF2KdTK3xg==", "dev": true }, + "node_modules/vary": { + "version": "1.1.2", + "resolved": "https://registry.npmjs.org/vary/-/vary-1.1.2.tgz", + "integrity": "sha512-BNGbWLfd0eUPabhkXUVm0j8uuvREyTh5ovRa/dyow/BqAbZJyC+5fU+IzQOzmAKzYqYRAISoRhdQr3eIZ/PXqg==", + "engines": { + "node": ">= 0.8" + } + }, "node_modules/which": { "version": "2.0.2", "resolved": "https://registry.npmjs.org/which/-/which-2.0.2.tgz", @@ -3638,6 +4300,23 @@ "js-yaml": "^4.1.0", "minimatch": "^3.1.2", "strip-json-comments": "^3.1.1" + }, + "dependencies": { + "argparse": { + "version": "2.0.1", + "resolved": "https://registry.npmjs.org/argparse/-/argparse-2.0.1.tgz", + "integrity": "sha512-8+9WqebbFzpX9OR+Wa6O29asIogeRMzcGtAINdpMHHyAg10f05aSFVBbcEqGf/PXw1EjAZ+q2/bEBg3DvurK3Q==", + "dev": true + }, + "js-yaml": { + "version": "4.1.0", + "resolved": "https://registry.npmjs.org/js-yaml/-/js-yaml-4.1.0.tgz", + "integrity": "sha512-wpxZs9NoxZaJESJGIZTyDEaYpl0FKSA+FB9aJiyemKhMwkxQg63h4T1KJgUGHpTqPDNRcmmYLugrRjJlBtWvRA==", + "dev": true, + "requires": { + "argparse": "^2.0.1" + } + } } }, "@hapi/bourne": { @@ -3727,25 +4406,6 @@ "pino-pretty": "^4.8.0", "sonic-boom": "^2.6.0", "tslib": "^2.3.1" - }, - "dependencies": { - "argparse": { - "version": "1.0.10", - "resolved": "https://registry.npmjs.org/argparse/-/argparse-1.0.10.tgz", - "integrity": "sha512-o5Roy6tNG4SL/FOkCAN6RzjiakZS25RLYFrcMttJqbdd8BWrnA+fGz57iN5Pb06pvBGvl5gQ0B48dJlslXvoTg==", - "requires": { - "sprintf-js": "~1.0.2" - } - }, - "js-yaml": { - "version": "3.14.1", - "resolved": "https://registry.npmjs.org/js-yaml/-/js-yaml-3.14.1.tgz", - "integrity": "sha512-okMH7OXXJ7YrN9Ok3/SXrnu4iX9yOk+25nqX4imS2npuvTYDmo/QEZoqwZkYaIDk3jVvBOTOIEgEhaLOynBS9g==", - "requires": { - "argparse": "^1.0.7", - "esprima": "^4.0.0" - } - } } }, "@runejs/eslint-config": { @@ -3793,6 +4453,48 @@ "@types/node": "*" } }, + "@types/body-parser": { + "version": "1.19.2", + "resolved": "https://registry.npmjs.org/@types/body-parser/-/body-parser-1.19.2.tgz", + "integrity": "sha512-ALYone6pm6QmwZoAgeyNksccT9Q4AWZQ6PvfwR37GT6r6FWUPguq6sUmNGSMV2Wr761oQoBxwGGa6DR5o1DC9g==", + "dev": true, + "requires": { + "@types/connect": "*", + "@types/node": "*" + } + }, + "@types/connect": { + "version": "3.4.35", + "resolved": "https://registry.npmjs.org/@types/connect/-/connect-3.4.35.tgz", + "integrity": "sha512-cdeYyv4KWoEgpBISTxWvqYsVy444DOqehiF3fM3ne10AmJ62RSyNkUnxMJXHQWRQQX2eR94m5y1IZyDwBjV9FQ==", + "dev": true, + "requires": { + "@types/node": "*" + } + }, + "@types/express": { + "version": "4.17.13", + "resolved": "https://registry.npmjs.org/@types/express/-/express-4.17.13.tgz", + "integrity": "sha512-6bSZTPaTIACxn48l50SR+axgrqm6qXFIxrdAKaG6PaJk3+zuUr35hBlgT7vOmJcum+OEaIBLtHV/qloEAFITeA==", + "dev": true, + "requires": { + "@types/body-parser": "*", + "@types/express-serve-static-core": "^4.17.18", + "@types/qs": "*", + "@types/serve-static": "*" + } + }, + "@types/express-serve-static-core": { + "version": "4.17.30", + "resolved": "https://registry.npmjs.org/@types/express-serve-static-core/-/express-serve-static-core-4.17.30.tgz", + "integrity": "sha512-gstzbTWro2/nFed1WXtf+TtrpwxH7Ggs4RLYTLbeVgIkUQOI3WG/JKjgeOU1zXDvezllupjrf8OPIdvTbIaVOQ==", + "dev": true, + "requires": { + "@types/node": "*", + "@types/qs": "*", + "@types/range-parser": "*" + } + }, "@types/graceful-fs": { "version": "4.1.5", "resolved": "https://registry.npmjs.org/@types/graceful-fs/-/graceful-fs-4.1.5.tgz", @@ -3808,12 +4510,40 @@ "integrity": "sha512-wOuvG1SN4Us4rez+tylwwwCV1psiNVOkJeM3AUWUNWg/jDQY2+HE/444y5gc+jBmRqASOm2Oeh5c1axHobwRKQ==", "dev": true }, + "@types/mime": { + "version": "3.0.1", + "resolved": "https://registry.npmjs.org/@types/mime/-/mime-3.0.1.tgz", + "integrity": "sha512-Y4XFY5VJAuw0FgAqPNd6NNoV44jbq9Bz2L7Rh/J6jLTiHBSBJa9fxqQIvkIld4GsoDOcCbvzOUAbLPsSKKg+uA==", + "dev": true + }, "@types/node": { - "version": "16.11.47", - "resolved": "https://registry.npmjs.org/@types/node/-/node-16.11.47.tgz", - "integrity": "sha512-fpP+jk2zJ4VW66+wAMFoBJlx1bxmBKx4DUFf68UHgdGCOuyUTDlLWqsaNPJh7xhNDykyJ9eIzAygilP/4WoN8g==", + "version": "16.11.52", + "resolved": "https://registry.npmjs.org/@types/node/-/node-16.11.52.tgz", + "integrity": "sha512-GnstYouCa9kbYokBCWEVrszJ1P2rGAQpKrqACHKuixkaT8XGu8nsqHvEUIGqDs5vwtsJ7LrYqnPDKRD1V+M39A==", + "dev": true + }, + "@types/qs": { + "version": "6.9.7", + "resolved": "https://registry.npmjs.org/@types/qs/-/qs-6.9.7.tgz", + "integrity": "sha512-FGa1F62FT09qcrueBA6qYTrJPVDzah9a+493+o2PCXsesWHIn27G98TsSMs3WPNbZIEj4+VJf6saSFpvD+3Zsw==", + "dev": true + }, + "@types/range-parser": { + "version": "1.2.4", + "resolved": "https://registry.npmjs.org/@types/range-parser/-/range-parser-1.2.4.tgz", + "integrity": "sha512-EEhsLsD6UsDM1yFhAvy0Cjr6VwmpMWqFBCb9w07wVugF7w9nfajxLuVmngTIpgS6svCnm6Vaw+MZhoDCKnOfsw==", "dev": true }, + "@types/serve-static": { + "version": "1.15.0", + "resolved": "https://registry.npmjs.org/@types/serve-static/-/serve-static-1.15.0.tgz", + "integrity": "sha512-z5xyF6uh8CbjAu9760KDKsH2FcDxZ2tFCsA4HIMWE6IkiYMXfVoa+4f9KX+FN0ZLsaMw1WNG2ETLA6N+/YA+cg==", + "dev": true, + "requires": { + "@types/mime": "*", + "@types/node": "*" + } + }, "@types/strip-bom": { "version": "3.0.0", "resolved": "https://registry.npmjs.org/@types/strip-bom/-/strip-bom-3.0.0.tgz", @@ -3847,14 +4577,14 @@ "integrity": "sha512-fbF6oTd4sGGy0xjHPKAt+eS2CrxJ3+6gQ3FGcBoIJR2TLAyCkCyI8JqZNy+FeON0AhVgNJoUumVoZQjBFUqHkw==" }, "@typescript-eslint/eslint-plugin": { - "version": "5.33.0", - "resolved": "https://registry.npmjs.org/@typescript-eslint/eslint-plugin/-/eslint-plugin-5.33.0.tgz", - "integrity": "sha512-jHvZNSW2WZ31OPJ3enhLrEKvAZNyAFWZ6rx9tUwaessTc4sx9KmgMNhVcqVAl1ETnT5rU5fpXTLmY9YvC1DCNg==", + "version": "5.33.1", + "resolved": "https://registry.npmjs.org/@typescript-eslint/eslint-plugin/-/eslint-plugin-5.33.1.tgz", + "integrity": "sha512-S1iZIxrTvKkU3+m63YUOxYPKaP+yWDQrdhxTglVDVEVBf+aCSw85+BmJnyUaQQsk5TXFG/LpBu9fa+LrAQ91fQ==", "dev": true, "requires": { - "@typescript-eslint/scope-manager": "5.33.0", - "@typescript-eslint/type-utils": "5.33.0", - "@typescript-eslint/utils": "5.33.0", + "@typescript-eslint/scope-manager": "5.33.1", + "@typescript-eslint/type-utils": "5.33.1", + "@typescript-eslint/utils": "5.33.1", "debug": "^4.3.4", "functional-red-black-tree": "^1.0.1", "ignore": "^5.2.0", @@ -3864,52 +4594,52 @@ } }, "@typescript-eslint/parser": { - "version": "5.33.0", - "resolved": "https://registry.npmjs.org/@typescript-eslint/parser/-/parser-5.33.0.tgz", - "integrity": "sha512-cgM5cJrWmrDV2KpvlcSkelTBASAs1mgqq+IUGKJvFxWrapHpaRy5EXPQz9YaKF3nZ8KY18ILTiVpUtbIac86/w==", + "version": "5.33.1", + "resolved": "https://registry.npmjs.org/@typescript-eslint/parser/-/parser-5.33.1.tgz", + "integrity": "sha512-IgLLtW7FOzoDlmaMoXdxG8HOCByTBXrB1V2ZQYSEV1ggMmJfAkMWTwUjjzagS6OkfpySyhKFkBw7A9jYmcHpZA==", "dev": true, "requires": { - "@typescript-eslint/scope-manager": "5.33.0", - "@typescript-eslint/types": "5.33.0", - "@typescript-eslint/typescript-estree": "5.33.0", + "@typescript-eslint/scope-manager": "5.33.1", + "@typescript-eslint/types": "5.33.1", + "@typescript-eslint/typescript-estree": "5.33.1", "debug": "^4.3.4" } }, "@typescript-eslint/scope-manager": { - "version": "5.33.0", - "resolved": "https://registry.npmjs.org/@typescript-eslint/scope-manager/-/scope-manager-5.33.0.tgz", - "integrity": "sha512-/Jta8yMNpXYpRDl8EwF/M8It2A9sFJTubDo0ATZefGXmOqlaBffEw0ZbkbQ7TNDK6q55NPHFshGBPAZvZkE8Pw==", + "version": "5.33.1", + "resolved": "https://registry.npmjs.org/@typescript-eslint/scope-manager/-/scope-manager-5.33.1.tgz", + "integrity": "sha512-8ibcZSqy4c5m69QpzJn8XQq9NnqAToC8OdH/W6IXPXv83vRyEDPYLdjAlUx8h/rbusq6MkW4YdQzURGOqsn3CA==", "dev": true, "requires": { - "@typescript-eslint/types": "5.33.0", - "@typescript-eslint/visitor-keys": "5.33.0" + "@typescript-eslint/types": "5.33.1", + "@typescript-eslint/visitor-keys": "5.33.1" } }, "@typescript-eslint/type-utils": { - "version": "5.33.0", - "resolved": "https://registry.npmjs.org/@typescript-eslint/type-utils/-/type-utils-5.33.0.tgz", - "integrity": "sha512-2zB8uEn7hEH2pBeyk3NpzX1p3lF9dKrEbnXq1F7YkpZ6hlyqb2yZujqgRGqXgRBTHWIUG3NGx/WeZk224UKlIA==", + "version": "5.33.1", + "resolved": "https://registry.npmjs.org/@typescript-eslint/type-utils/-/type-utils-5.33.1.tgz", + "integrity": "sha512-X3pGsJsD8OiqhNa5fim41YtlnyiWMF/eKsEZGsHID2HcDqeSC5yr/uLOeph8rNF2/utwuI0IQoAK3fpoxcLl2g==", "dev": true, "requires": { - "@typescript-eslint/utils": "5.33.0", + "@typescript-eslint/utils": "5.33.1", "debug": "^4.3.4", "tsutils": "^3.21.0" } }, "@typescript-eslint/types": { - "version": "5.33.0", - "resolved": "https://registry.npmjs.org/@typescript-eslint/types/-/types-5.33.0.tgz", - "integrity": "sha512-nIMt96JngB4MYFYXpZ/3ZNU4GWPNdBbcB5w2rDOCpXOVUkhtNlG2mmm8uXhubhidRZdwMaMBap7Uk8SZMU/ppw==", + "version": "5.33.1", + "resolved": "https://registry.npmjs.org/@typescript-eslint/types/-/types-5.33.1.tgz", + "integrity": "sha512-7K6MoQPQh6WVEkMrMW5QOA5FO+BOwzHSNd0j3+BlBwd6vtzfZceJ8xJ7Um2XDi/O3umS8/qDX6jdy2i7CijkwQ==", "dev": true }, "@typescript-eslint/typescript-estree": { - "version": "5.33.0", - "resolved": "https://registry.npmjs.org/@typescript-eslint/typescript-estree/-/typescript-estree-5.33.0.tgz", - "integrity": "sha512-tqq3MRLlggkJKJUrzM6wltk8NckKyyorCSGMq4eVkyL5sDYzJJcMgZATqmF8fLdsWrW7OjjIZ1m9v81vKcaqwQ==", + "version": "5.33.1", + "resolved": "https://registry.npmjs.org/@typescript-eslint/typescript-estree/-/typescript-estree-5.33.1.tgz", + "integrity": "sha512-JOAzJ4pJ+tHzA2pgsWQi4804XisPHOtbvwUyqsuuq8+y5B5GMZs7lI1xDWs6V2d7gE/Ez5bTGojSK12+IIPtXA==", "dev": true, "requires": { - "@typescript-eslint/types": "5.33.0", - "@typescript-eslint/visitor-keys": "5.33.0", + "@typescript-eslint/types": "5.33.1", + "@typescript-eslint/visitor-keys": "5.33.1", "debug": "^4.3.4", "globby": "^11.1.0", "is-glob": "^4.0.3", @@ -3918,29 +4648,38 @@ } }, "@typescript-eslint/utils": { - "version": "5.33.0", - "resolved": "https://registry.npmjs.org/@typescript-eslint/utils/-/utils-5.33.0.tgz", - "integrity": "sha512-JxOAnXt9oZjXLIiXb5ZIcZXiwVHCkqZgof0O8KPgz7C7y0HS42gi75PdPlqh1Tf109M0fyUw45Ao6JLo7S5AHw==", + "version": "5.33.1", + "resolved": "https://registry.npmjs.org/@typescript-eslint/utils/-/utils-5.33.1.tgz", + "integrity": "sha512-uphZjkMaZ4fE8CR4dU7BquOV6u0doeQAr8n6cQenl/poMaIyJtBu8eys5uk6u5HiDH01Mj5lzbJ5SfeDz7oqMQ==", "dev": true, "requires": { "@types/json-schema": "^7.0.9", - "@typescript-eslint/scope-manager": "5.33.0", - "@typescript-eslint/types": "5.33.0", - "@typescript-eslint/typescript-estree": "5.33.0", + "@typescript-eslint/scope-manager": "5.33.1", + "@typescript-eslint/types": "5.33.1", + "@typescript-eslint/typescript-estree": "5.33.1", "eslint-scope": "^5.1.1", "eslint-utils": "^3.0.0" } }, "@typescript-eslint/visitor-keys": { - "version": "5.33.0", - "resolved": "https://registry.npmjs.org/@typescript-eslint/visitor-keys/-/visitor-keys-5.33.0.tgz", - "integrity": "sha512-/XsqCzD4t+Y9p5wd9HZiptuGKBlaZO5showwqODii5C0nZawxWLF+Q6k5wYHBrQv96h6GYKyqqMHCSTqta8Kiw==", + "version": "5.33.1", + "resolved": "https://registry.npmjs.org/@typescript-eslint/visitor-keys/-/visitor-keys-5.33.1.tgz", + "integrity": "sha512-nwIxOK8Z2MPWltLKMLOEZwmfBZReqUdbEoHQXeCpa+sRVARe5twpJGHCB4dk9903Yaf0nMAlGbQfaAH92F60eg==", "dev": true, "requires": { - "@typescript-eslint/types": "5.33.0", + "@typescript-eslint/types": "5.33.1", "eslint-visitor-keys": "^3.3.0" } }, + "accepts": { + "version": "1.3.8", + "resolved": "https://registry.npmjs.org/accepts/-/accepts-1.3.8.tgz", + "integrity": "sha512-PYAthTa2m2VKxuvSD3DPC/Gy+U+sOA1LAuT8mkmRuvw+NACSaeXEQ+NHcVF7rONl6qcaxV3Uuemwawk+7+SJLw==", + "requires": { + "mime-types": "~2.1.34", + "negotiator": "0.6.3" + } + }, "acorn": { "version": "8.8.0", "resolved": "https://registry.npmjs.org/acorn/-/acorn-8.8.0.tgz", @@ -4011,9 +4750,9 @@ } }, "app-root-path": { - "version": "3.0.0", - "resolved": "https://registry.npmjs.org/app-root-path/-/app-root-path-3.0.0.tgz", - "integrity": "sha512-qMcx+Gy2UZynHjOHOIXPNvpf+9cjvk3cWrBBK7zg4gH9+clobJRb9NGzcT7mQTcV/6Gm/1WelUtqxVXnNlrwcw==" + "version": "3.1.0", + "resolved": "https://registry.npmjs.org/app-root-path/-/app-root-path-3.1.0.tgz", + "integrity": "sha512-biN3PwB2gUtjaYy/isrU3aNWI5w+fAfvHkSvCKeQGxhmYpwKFUxudR3Yya+KqVRHBmEDYh+/lTozYCFbmzX4nA==" }, "arg": { "version": "4.1.3", @@ -4022,9 +4761,12 @@ "dev": true }, "argparse": { - "version": "2.0.1", - "resolved": "https://registry.npmjs.org/argparse/-/argparse-2.0.1.tgz", - "integrity": "sha512-8+9WqebbFzpX9OR+Wa6O29asIogeRMzcGtAINdpMHHyAg10f05aSFVBbcEqGf/PXw1EjAZ+q2/bEBg3DvurK3Q==" + "version": "1.0.10", + "resolved": "https://registry.npmjs.org/argparse/-/argparse-1.0.10.tgz", + "integrity": "sha512-o5Roy6tNG4SL/FOkCAN6RzjiakZS25RLYFrcMttJqbdd8BWrnA+fGz57iN5Pb06pvBGvl5gQ0B48dJlslXvoTg==", + "requires": { + "sprintf-js": "~1.0.2" + } }, "args": { "version": "5.0.3", @@ -4088,6 +4830,11 @@ } } }, + "array-flatten": { + "version": "1.1.1", + "resolved": "https://registry.npmjs.org/array-flatten/-/array-flatten-1.1.1.tgz", + "integrity": "sha512-PCVAQswWemu6UdxsDFFX/+gVeYqKAod3D3UVm91jHwynguOwAvYPhx8nNlM++NqRcK6CxxpUafjmhIdKiHibqg==" + }, "array-union": { "version": "2.1.0", "resolved": "https://registry.npmjs.org/array-union/-/array-union-2.1.0.tgz", @@ -4181,6 +4928,40 @@ } } }, + "body-parser": { + "version": "1.20.0", + "resolved": "https://registry.npmjs.org/body-parser/-/body-parser-1.20.0.tgz", + "integrity": "sha512-DfJ+q6EPcGKZD1QWUjSpqp+Q7bDQTsQIF4zfUAtZ6qk+H/3/QRhg9CEp39ss+/T2vw0+HaidC0ecJj/DRLIaKg==", + "requires": { + "bytes": "3.1.2", + "content-type": "~1.0.4", + "debug": "2.6.9", + "depd": "2.0.0", + "destroy": "1.2.0", + "http-errors": "2.0.0", + "iconv-lite": "0.4.24", + "on-finished": "2.4.1", + "qs": "6.10.3", + "raw-body": "2.5.1", + "type-is": "~1.6.18", + "unpipe": "1.0.0" + }, + "dependencies": { + "debug": { + "version": "2.6.9", + "resolved": "https://registry.npmjs.org/debug/-/debug-2.6.9.tgz", + "integrity": "sha512-bC7ElrdJaJnPbAP+1EotYvqZsb3ecl5wi6Bfi6BJTUcNowp6cvspg0jXznRTKDjm/E7AdgFBVeAPVMNcKGsHMA==", + "requires": { + "ms": "2.0.0" + } + }, + "ms": { + "version": "2.0.0", + "resolved": "https://registry.npmjs.org/ms/-/ms-2.0.0.tgz", + "integrity": "sha512-Tpp60P6IUJDTuOq/5Z8cdskzJujfwqfOTkrwIwj7IRISpnkJnT6SyJ4PCPnGMoFjC9ddhal5KVIYtAt97ix05A==" + } + } + }, "brace-expansion": { "version": "1.1.11", "resolved": "https://registry.npmjs.org/brace-expansion/-/brace-expansion-1.1.11.tgz", @@ -4215,6 +4996,20 @@ "integrity": "sha512-E+XQCRwSbaaiChtv6k6Dwgc+bx+Bs6vuKJHHl5kox/BaKbhiXzqQOwK4cO22yElGp2OCmjwVhT3HmxgyPGnJfQ==", "dev": true }, + "bytes": { + "version": "3.1.2", + "resolved": "https://registry.npmjs.org/bytes/-/bytes-3.1.2.tgz", + "integrity": "sha512-/Nf7TyzTx6S3yRJObOAV7956r8cr2+Oj8AC5dt8wSP3BQAoeX58NoHyCU8P8zGkNXStjTSi6fzO6F0pBdcYbEg==" + }, + "call-bind": { + "version": "1.0.2", + "resolved": "https://registry.npmjs.org/call-bind/-/call-bind-1.0.2.tgz", + "integrity": "sha512-7O+FbCihrB5WGbFYesctwmTKae6rOiIzmz1icreWJ+0aA7LJfuqhEso2T9ncpcFtzMQtzXf2QGGueWJGTYsqrA==", + "requires": { + "function-bind": "^1.1.1", + "get-intrinsic": "^1.0.2" + } + }, "callsites": { "version": "3.1.0", "resolved": "https://registry.npmjs.org/callsites/-/callsites-3.1.0.tgz", @@ -4355,6 +5150,29 @@ "resolved": "https://registry.npmjs.org/concat-map/-/concat-map-0.0.1.tgz", "integrity": "sha512-/Srv4dswyQNBfohGpz9o6Yb3Gz3SrUDqBH5rTuhGR7ahtlbYKnVxw2bCFMRljaA7EXHaXZ8wsHdodFvbkhKmqg==" }, + "content-disposition": { + "version": "0.5.4", + "resolved": "https://registry.npmjs.org/content-disposition/-/content-disposition-0.5.4.tgz", + "integrity": "sha512-FveZTNuGw04cxlAiWbzi6zTAL/lhehaWbTtgluJh4/E95DqMwTmha3KZN1aAWA8cFIhHzMZUvLevkw5Rqk+tSQ==", + "requires": { + "safe-buffer": "5.2.1" + } + }, + "content-type": { + "version": "1.0.4", + "resolved": "https://registry.npmjs.org/content-type/-/content-type-1.0.4.tgz", + "integrity": "sha512-hIP3EEPs8tB9AT1L+NUqtwOAps4mk2Zob89MWXMHjHWg9milF/j4osnnQLXBCBFBk/tvIG/tUc9mOUJiPBhPXA==" + }, + "cookie": { + "version": "0.5.0", + "resolved": "https://registry.npmjs.org/cookie/-/cookie-0.5.0.tgz", + "integrity": "sha512-YZ3GUyn/o8gfKJlnlX7g7xq4gyO6OSuhGPKaaGssGB2qgDUS0gPgtTvoyZLTt9Ab6dC4hfc9dV5arkvc/OCmrw==" + }, + "cookie-signature": { + "version": "1.0.6", + "resolved": "https://registry.npmjs.org/cookie-signature/-/cookie-signature-1.0.6.tgz", + "integrity": "sha512-QADzlaHc8icV8I7vbaJXJwod9HWYp8uCqf1xa4OfNu1T7JVxQIrUgOWtHdNDtPiywmFbiS12VjotIXLrKM3orQ==" + }, "copyfiles": { "version": "2.4.1", "resolved": "https://registry.npmjs.org/copyfiles/-/copyfiles-2.4.1.tgz", @@ -4455,6 +5273,16 @@ "resolved": "https://registry.npmjs.org/delayed-stream/-/delayed-stream-1.0.0.tgz", "integrity": "sha512-ZySD7Nf91aLB0RxL4KGrKHBXl7Eds1DAmEdcoVawXnLD7SDhpNgtuII2aAkg7a7QS41jxPSZ17p4VdGnMHk3MQ==" }, + "depd": { + "version": "2.0.0", + "resolved": "https://registry.npmjs.org/depd/-/depd-2.0.0.tgz", + "integrity": "sha512-g7nH6P6dyDioJogAAGprGpCtVImJhpPk/roCzdb3fIh61/s/nPsfR6onyMwkCAR/OlC3yBC0lESvUoQEAssIrw==" + }, + "destroy": { + "version": "1.2.0", + "resolved": "https://registry.npmjs.org/destroy/-/destroy-1.2.0.tgz", + "integrity": "sha512-2sJGJTaXIIaR1w4iJSNoN0hnMY7Gpc/n8D4qSCJw8QqFWXf7cuAgnEHxBpweaVcPevC2l3KpjYCx3NypQQgaJg==" + }, "detect-libc": { "version": "2.0.1", "resolved": "https://registry.npmjs.org/detect-libc/-/detect-libc-2.0.1.tgz", @@ -4499,11 +5327,21 @@ "xtend": "^4.0.0" } }, + "ee-first": { + "version": "1.1.1", + "resolved": "https://registry.npmjs.org/ee-first/-/ee-first-1.1.1.tgz", + "integrity": "sha512-WMwm9LhRUo+WUaRN+vRuETqG89IgZphVSNkdFgeb6sS/E4OrDIN7t48CAewSHXc6C8lefD8KKfr5vY61brQlow==" + }, "emoji-regex": { "version": "8.0.0", "resolved": "https://registry.npmjs.org/emoji-regex/-/emoji-regex-8.0.0.tgz", "integrity": "sha512-MSjYzcWNOA0ewAHpz0MxpYFvwg6yjy1NG3xteoqz644VCo/RPgnr1/GGt+ic3iJTzQ8Eu3TdM14SawnVUmGE6A==" }, + "encodeurl": { + "version": "1.0.2", + "resolved": "https://registry.npmjs.org/encodeurl/-/encodeurl-1.0.2.tgz", + "integrity": "sha512-TPJXq8JqFaVYm2CWmPvnP2Iyo4ZSM7/QKcSmuMLDObfpH5fi7RUGmd/rTDf+rut/saiDiQEeVTNgAmJEdAOx0w==" + }, "end-of-stream": { "version": "1.4.4", "resolved": "https://registry.npmjs.org/end-of-stream/-/end-of-stream-1.4.4.tgz", @@ -4517,6 +5355,11 @@ "resolved": "https://registry.npmjs.org/escalade/-/escalade-3.1.1.tgz", "integrity": "sha512-k0er2gUkLf8O0zKJiAhmkTnJlTvINGv7ygDNPbeIsX/TJjGJZHuh9B2UxbsaEkmlEo9MfhrSzmhIlhRlI2GXnw==" }, + "escape-html": { + "version": "1.0.3", + "resolved": "https://registry.npmjs.org/escape-html/-/escape-html-1.0.3.tgz", + "integrity": "sha512-NiSupZ4OeuGwr68lGIeym/ksIZMJodUGOSCZ/FSnTxcrekbvqrgdUxlJOMpijaKZVjAJrWrGs/6Jy8OMuyj9ow==" + }, "escape-string-regexp": { "version": "4.0.0", "resolved": "https://registry.npmjs.org/escape-string-regexp/-/escape-string-regexp-4.0.0.tgz", @@ -4524,9 +5367,9 @@ "dev": true }, "eslint": { - "version": "8.21.0", - "resolved": "https://registry.npmjs.org/eslint/-/eslint-8.21.0.tgz", - "integrity": "sha512-/XJ1+Qurf1T9G2M5IHrsjp+xrGT73RZf23xA1z5wB1ZzzEAWSZKvRwhWxTFp1rvkvCfwcvAUNAP31bhKTTGfDA==", + "version": "8.22.0", + "resolved": "https://registry.npmjs.org/eslint/-/eslint-8.22.0.tgz", + "integrity": "sha512-ci4t0sz6vSRKdmkOGmprBo6fmI4PrphDFMy5JEq/fNS0gQkJM3rLmrqcp8ipMcdobH3KtUP40KniAE9W19S4wA==", "dev": true, "requires": { "@eslint/eslintrc": "^1.3.0", @@ -4570,6 +5413,12 @@ "v8-compile-cache": "^2.0.3" }, "dependencies": { + "argparse": { + "version": "2.0.1", + "resolved": "https://registry.npmjs.org/argparse/-/argparse-2.0.1.tgz", + "integrity": "sha512-8+9WqebbFzpX9OR+Wa6O29asIogeRMzcGtAINdpMHHyAg10f05aSFVBbcEqGf/PXw1EjAZ+q2/bEBg3DvurK3Q==", + "dev": true + }, "eslint-scope": { "version": "7.1.1", "resolved": "https://registry.npmjs.org/eslint-scope/-/eslint-scope-7.1.1.tgz", @@ -4585,6 +5434,15 @@ "resolved": "https://registry.npmjs.org/estraverse/-/estraverse-5.3.0.tgz", "integrity": "sha512-MMdARuVEQziNTeJD8DgMqmhwR11BRQ/cBP+pLtYdSTnf3MIO8fFeiINEbX36ZdNlfU/7A9f3gUw49B3oQsvwBA==", "dev": true + }, + "js-yaml": { + "version": "4.1.0", + "resolved": "https://registry.npmjs.org/js-yaml/-/js-yaml-4.1.0.tgz", + "integrity": "sha512-wpxZs9NoxZaJESJGIZTyDEaYpl0FKSA+FB9aJiyemKhMwkxQg63h4T1KJgUGHpTqPDNRcmmYLugrRjJlBtWvRA==", + "dev": true, + "requires": { + "argparse": "^2.0.1" + } } } }, @@ -4683,12 +5541,70 @@ "integrity": "sha512-kVscqXk4OCp68SZ0dkgEKVi6/8ij300KBWTJq32P/dYeWTSwK41WyTxalN1eRmA5Z9UU/LX9D7FWSmV9SAYx6g==", "dev": true }, + "etag": { + "version": "1.8.1", + "resolved": "https://registry.npmjs.org/etag/-/etag-1.8.1.tgz", + "integrity": "sha512-aIL5Fx7mawVa300al2BnEE4iNvo1qETxLrPI/o05L7z6go7fCw1J6EQmbK4FmJ2AS7kgVF/KEZWufBfdClMcPg==" + }, "expand-template": { "version": "2.0.3", "resolved": "https://registry.npmjs.org/expand-template/-/expand-template-2.0.3.tgz", "integrity": "sha512-XYfuKMvj4O35f/pOXLObndIRvyQ+/+6AhODh+OKWj9S9498pHHn/IMszH+gt0fBCRWMNfk1ZSp5x3AifmnI2vg==", "devOptional": true }, + "express": { + "version": "4.18.1", + "resolved": "https://registry.npmjs.org/express/-/express-4.18.1.tgz", + "integrity": "sha512-zZBcOX9TfehHQhtupq57OF8lFZ3UZi08Y97dwFCkD8p9d/d2Y3M+ykKcwaMDEL+4qyUolgBDX6AblpR3fL212Q==", + "requires": { + "accepts": "~1.3.8", + "array-flatten": "1.1.1", + "body-parser": "1.20.0", + "content-disposition": "0.5.4", + "content-type": "~1.0.4", + "cookie": "0.5.0", + "cookie-signature": "1.0.6", + "debug": "2.6.9", + "depd": "2.0.0", + "encodeurl": "~1.0.2", + "escape-html": "~1.0.3", + "etag": "~1.8.1", + "finalhandler": "1.2.0", + "fresh": "0.5.2", + "http-errors": "2.0.0", + "merge-descriptors": "1.0.1", + "methods": "~1.1.2", + "on-finished": "2.4.1", + "parseurl": "~1.3.3", + "path-to-regexp": "0.1.7", + "proxy-addr": "~2.0.7", + "qs": "6.10.3", + "range-parser": "~1.2.1", + "safe-buffer": "5.2.1", + "send": "0.18.0", + "serve-static": "1.15.0", + "setprototypeof": "1.2.0", + "statuses": "2.0.1", + "type-is": "~1.6.18", + "utils-merge": "1.0.1", + "vary": "~1.1.2" + }, + "dependencies": { + "debug": { + "version": "2.6.9", + "resolved": "https://registry.npmjs.org/debug/-/debug-2.6.9.tgz", + "integrity": "sha512-bC7ElrdJaJnPbAP+1EotYvqZsb3ecl5wi6Bfi6BJTUcNowp6cvspg0jXznRTKDjm/E7AdgFBVeAPVMNcKGsHMA==", + "requires": { + "ms": "2.0.0" + } + }, + "ms": { + "version": "2.0.0", + "resolved": "https://registry.npmjs.org/ms/-/ms-2.0.0.tgz", + "integrity": "sha512-Tpp60P6IUJDTuOq/5Z8cdskzJujfwqfOTkrwIwj7IRISpnkJnT6SyJ4PCPnGMoFjC9ddhal5KVIYtAt97ix05A==" + } + } + }, "fast-deep-equal": { "version": "3.1.3", "resolved": "https://registry.npmjs.org/fast-deep-equal/-/fast-deep-equal-3.1.3.tgz", @@ -4732,9 +5648,9 @@ "dev": true }, "fast-redact": { - "version": "3.1.1", - "resolved": "https://registry.npmjs.org/fast-redact/-/fast-redact-3.1.1.tgz", - "integrity": "sha512-odVmjC8x8jNeMZ3C+rPMESzXVSEU8tSWSHv9HFxP2mm89G/1WwqhrerJDQm9Zus8X6aoRgQDThKqptdNA6bt+A==" + "version": "3.1.2", + "resolved": "https://registry.npmjs.org/fast-redact/-/fast-redact-3.1.2.tgz", + "integrity": "sha512-+0em+Iya9fKGfEQGcd62Yv6onjBmmhV1uh86XVfOU8VwAe6kaFdQCWI9s0/Nnugx5Vd9tdbZ7e6gE2tR9dzXdw==" }, "fast-safe-stringify": { "version": "2.1.1", @@ -4774,6 +5690,35 @@ "to-regex-range": "^5.0.1" } }, + "finalhandler": { + "version": "1.2.0", + "resolved": "https://registry.npmjs.org/finalhandler/-/finalhandler-1.2.0.tgz", + "integrity": "sha512-5uXcUVftlQMFnWC9qu/svkWv3GTd2PfUhK/3PLkYNAe7FbqJMt3515HaxE6eRL74GdsriiwujiawdaB1BpEISg==", + "requires": { + "debug": "2.6.9", + "encodeurl": "~1.0.2", + "escape-html": "~1.0.3", + "on-finished": "2.4.1", + "parseurl": "~1.3.3", + "statuses": "2.0.1", + "unpipe": "~1.0.0" + }, + "dependencies": { + "debug": { + "version": "2.6.9", + "resolved": "https://registry.npmjs.org/debug/-/debug-2.6.9.tgz", + "integrity": "sha512-bC7ElrdJaJnPbAP+1EotYvqZsb3ecl5wi6Bfi6BJTUcNowp6cvspg0jXznRTKDjm/E7AdgFBVeAPVMNcKGsHMA==", + "requires": { + "ms": "2.0.0" + } + }, + "ms": { + "version": "2.0.0", + "resolved": "https://registry.npmjs.org/ms/-/ms-2.0.0.tgz", + "integrity": "sha512-Tpp60P6IUJDTuOq/5Z8cdskzJujfwqfOTkrwIwj7IRISpnkJnT6SyJ4PCPnGMoFjC9ddhal5KVIYtAt97ix05A==" + } + } + }, "find-up": { "version": "5.0.0", "resolved": "https://registry.npmjs.org/find-up/-/find-up-5.0.0.tgz", @@ -4800,9 +5745,9 @@ "integrity": "sha512-4zPxDyhCyiN2wIAtSLI6gc82/EjqZc1onI4Mz/l0pWrAlsSfYH/2ZIcU+e3oA2wDwbzIWNKwa23F8rh6+DRWkw==" }, "flatted": { - "version": "3.2.6", - "resolved": "https://registry.npmjs.org/flatted/-/flatted-3.2.6.tgz", - "integrity": "sha512-0sQoMh9s0BYsm+12Huy/rkKxVu4R1+r96YX5cG44rHV0pQ6iC3Q+mkoMFaGWObMFYQxCVT+ssG1ksneA2MI9KQ==", + "version": "3.2.7", + "resolved": "https://registry.npmjs.org/flatted/-/flatted-3.2.7.tgz", + "integrity": "sha512-5nqDSxl8nn5BSNxyR3n4I6eDmbolI6WT+QqR547RwxQapgjQBmtktdP+HTBb/a/zLsbzERTONyUB5pefh5TtjQ==", "dev": true }, "follow-redirects": { @@ -4820,6 +5765,16 @@ "mime-types": "^2.1.12" } }, + "forwarded": { + "version": "0.2.0", + "resolved": "https://registry.npmjs.org/forwarded/-/forwarded-0.2.0.tgz", + "integrity": "sha512-buRG0fpBtRHSTCOASe6hD258tEubFoRLb4ZNA6NxMVHNw2gOcwHo9wyablzMzOA5z9xA9L1KNjk/Nt6MT9aYow==" + }, + "fresh": { + "version": "0.5.2", + "resolved": "https://registry.npmjs.org/fresh/-/fresh-0.5.2.tgz", + "integrity": "sha512-zJ2mQYM18rEFOudeV4GShTGIQ7RbzA7ozbU9I/XBpm7kqgMywgmylMwXHxZJmkVoYkna9d2pVXVXPdYTP9ej8Q==" + }, "fs-constants": { "version": "1.0.0", "resolved": "https://registry.npmjs.org/fs-constants/-/fs-constants-1.0.0.tgz", @@ -4841,8 +5796,7 @@ "function-bind": { "version": "1.1.1", "resolved": "https://registry.npmjs.org/function-bind/-/function-bind-1.1.1.tgz", - "integrity": "sha512-yIovAzMX49sF8Yl58fSCWJ5svSLuaibPxXQJFLmBObTuCr0Mf1KiPopGM9NiFjiYBCbfaa2Fh6breQ6ANVTI0A==", - "dev": true + "integrity": "sha512-yIovAzMX49sF8Yl58fSCWJ5svSLuaibPxXQJFLmBObTuCr0Mf1KiPopGM9NiFjiYBCbfaa2Fh6breQ6ANVTI0A==" }, "functional-red-black-tree": { "version": "1.0.1", @@ -4855,6 +5809,16 @@ "resolved": "https://registry.npmjs.org/get-caller-file/-/get-caller-file-2.0.5.tgz", "integrity": "sha512-DyFP3BM/3YHTQOCUL/w0OZHR0lpKeGrxotcHWcqNEdnltqFwXVfhEBQ94eIo34AfQpo0rGki4cyIiftY06h2Fg==" }, + "get-intrinsic": { + "version": "1.1.2", + "resolved": "https://registry.npmjs.org/get-intrinsic/-/get-intrinsic-1.1.2.tgz", + "integrity": "sha512-Jfm3OyCxHh9DJyc28qGk+JmfkpO41A4XkneDSujN9MDXrm4oDKdHvndhZ2dN94+ERNfkYJWDclW6k2L/ZGHjXA==", + "requires": { + "function-bind": "^1.1.1", + "has": "^1.0.3", + "has-symbols": "^1.0.3" + } + }, "github-from-package": { "version": "0.0.0", "resolved": "https://registry.npmjs.org/github-from-package/-/github-from-package-0.0.0.tgz", @@ -4926,7 +5890,6 @@ "version": "1.0.3", "resolved": "https://registry.npmjs.org/has/-/has-1.0.3.tgz", "integrity": "sha512-f2dvO0VU6Oej7RkWJGrehjbzMAjFp5/VKPp5tTpWIV4JHHZK1/BxbFRtf/siA2SWTe09caDmVtYYzWEIbBS4zw==", - "dev": true, "requires": { "function-bind": "^1.1.1" } @@ -4936,11 +5899,36 @@ "resolved": "https://registry.npmjs.org/has-flag/-/has-flag-4.0.0.tgz", "integrity": "sha512-EykJT/Q1KjTWctppgIAgfSO0tKVuZUjhgMr17kqTumMl6Afv3EISleU7qZUzoXDFTAHTDC4NOoG/ZxU3EvlMPQ==" }, + "has-symbols": { + "version": "1.0.3", + "resolved": "https://registry.npmjs.org/has-symbols/-/has-symbols-1.0.3.tgz", + "integrity": "sha512-l3LCuF6MgDNwTDKkdYGEihYjt5pRPbEg46rtlmnSPlUbgmB8LOIrKJbYYFBSbnPaJexMKtiPO8hmeRjRz2Td+A==" + }, "highlight.js": { "version": "10.7.3", "resolved": "https://registry.npmjs.org/highlight.js/-/highlight.js-10.7.3.tgz", "integrity": "sha512-tzcUFauisWKNHaRkN4Wjl/ZA07gENAjFl3J/c480dprkGTg5EQstgaNFqBfUqCq54kZRIEcreTsAgF/m2quD7A==" }, + "http-errors": { + "version": "2.0.0", + "resolved": "https://registry.npmjs.org/http-errors/-/http-errors-2.0.0.tgz", + "integrity": "sha512-FtwrG/euBzaEjYeRqOgly7G0qviiXoJWnvEH2Z1plBdXgbyjv34pHTSb9zoeHMyDy33+DWy5Wt9Wo+TURtOYSQ==", + "requires": { + "depd": "2.0.0", + "inherits": "2.0.4", + "setprototypeof": "1.2.0", + "statuses": "2.0.1", + "toidentifier": "1.0.1" + } + }, + "iconv-lite": { + "version": "0.4.24", + "resolved": "https://registry.npmjs.org/iconv-lite/-/iconv-lite-0.4.24.tgz", + "integrity": "sha512-v3MXnZAcvnywkTUEZomIActle7RXXeedOR31wwl7VlyoXO4Qi9arvSenNQWne1TcRwhCL1HwLI21bEqdpj8/rA==", + "requires": { + "safer-buffer": ">= 2.1.2 < 3" + } + }, "ieee754": { "version": "1.2.1", "resolved": "https://registry.npmjs.org/ieee754/-/ieee754-1.2.1.tgz", @@ -4988,6 +5976,11 @@ "integrity": "sha512-JV/yugV2uzW5iMRSiZAyDtQd+nxtUnjeLt0acNdw98kKLrvuRVyB80tsREOE7yvGVgalhZ6RNXCmEHkUKBKxew==", "devOptional": true }, + "ipaddr.js": { + "version": "1.9.1", + "resolved": "https://registry.npmjs.org/ipaddr.js/-/ipaddr.js-1.9.1.tgz", + "integrity": "sha512-0KI/607xoxSToH7GjN1FfSbLoU0+btTicjsQSWQlh/hZykN8KpmMf7uYwPW3R+akZ6R/w18ZlXSHBYXiYUPO3g==" + }, "is-binary-path": { "version": "2.1.0", "resolved": "https://registry.npmjs.org/is-binary-path/-/is-binary-path-2.1.0.tgz", @@ -5055,11 +6048,12 @@ "integrity": "sha512-YqvUxoOcVPnCp0VU1/56f+iKSdvIRJYPznH22BdXV3xMk75SFXhWeJkZ8C9XxUWt1b5x2X1SxuFygW1U0FmkEQ==" }, "js-yaml": { - "version": "4.1.0", - "resolved": "https://registry.npmjs.org/js-yaml/-/js-yaml-4.1.0.tgz", - "integrity": "sha512-wpxZs9NoxZaJESJGIZTyDEaYpl0FKSA+FB9aJiyemKhMwkxQg63h4T1KJgUGHpTqPDNRcmmYLugrRjJlBtWvRA==", + "version": "3.14.1", + "resolved": "https://registry.npmjs.org/js-yaml/-/js-yaml-3.14.1.tgz", + "integrity": "sha512-okMH7OXXJ7YrN9Ok3/SXrnu4iX9yOk+25nqX4imS2npuvTYDmo/QEZoqwZkYaIDk3jVvBOTOIEgEhaLOynBS9g==", "requires": { - "argparse": "^2.0.1" + "argparse": "^1.0.7", + "esprima": "^4.0.0" } }, "json-schema-traverse": { @@ -5124,12 +6118,27 @@ "integrity": "sha512-s8UhlNe7vPKomQhC1qFelMokr/Sc3AgNbso3n74mVPA5LTZwkB9NlXf4XPamLxJE8h0gh73rM94xvwRT2CVInw==", "dev": true }, + "media-typer": { + "version": "0.3.0", + "resolved": "https://registry.npmjs.org/media-typer/-/media-typer-0.3.0.tgz", + "integrity": "sha512-dq+qelQ9akHpcOl/gUVRTxVIOkAJ1wR3QAvb4RsVjS8oVoFjDGTc679wJYmUmknUF5HwMLOgb5O+a3KxfWapPQ==" + }, + "merge-descriptors": { + "version": "1.0.1", + "resolved": "https://registry.npmjs.org/merge-descriptors/-/merge-descriptors-1.0.1.tgz", + "integrity": "sha512-cCi6g3/Zr1iqQi6ySbseM1Xvooa98N0w31jzUYrXPX2xqObmFGHJ0tQ5u74H3mVh7wLouTseZyYIq39g8cNp1w==" + }, "merge2": { "version": "1.4.1", "resolved": "https://registry.npmjs.org/merge2/-/merge2-1.4.1.tgz", "integrity": "sha512-8q7VEgMJW4J8tcfVPy8g09NcQwZdbwFEqhe/WZkoIzjn/3TGDwtOCYtXGxA3O8tPzpczCCDgv+P2P5y00ZJOOg==", "dev": true }, + "methods": { + "version": "1.1.2", + "resolved": "https://registry.npmjs.org/methods/-/methods-1.1.2.tgz", + "integrity": "sha512-iclAHeNqNm68zFtnZ0e+1L2yUIdvzNoauKU4WBA3VvH/vPFieF7qfRlwUZU+DA9P9bPXIS90ulxoUoCH23sV2w==" + }, "micromatch": { "version": "4.0.5", "resolved": "https://registry.npmjs.org/micromatch/-/micromatch-4.0.5.tgz", @@ -5140,6 +6149,11 @@ "picomatch": "^2.3.1" } }, + "mime": { + "version": "1.6.0", + "resolved": "https://registry.npmjs.org/mime/-/mime-1.6.0.tgz", + "integrity": "sha512-x0Vn8spI+wuJ1O6S7gnbaQg8Pxh4NNHb7KSINmEWKiPE4RKOplvijn+NkmYmmRgP68mc70j2EbeTFRsrswaQeg==" + }, "mime-db": { "version": "1.52.0", "resolved": "https://registry.npmjs.org/mime-db/-/mime-db-1.52.0.tgz", @@ -5216,6 +6230,11 @@ "integrity": "sha512-OWND8ei3VtNC9h7V60qff3SVobHr996CTwgxubgyQYEpg290h9J0buyECNNJexkFm5sOajh5G116RYA1c8ZMSw==", "dev": true }, + "negotiator": { + "version": "0.6.3", + "resolved": "https://registry.npmjs.org/negotiator/-/negotiator-0.6.3.tgz", + "integrity": "sha512-+EUsqGPLsM+j/zdChZjsnX51g4XrHFOIXwfnCVPGlQk/k5giakcKsuxCObBRu6DSm9opw/O6slWbJdghQM4bBg==" + }, "node-abi": { "version": "3.24.0", "resolved": "https://registry.npmjs.org/node-abi/-/node-abi-3.24.0.tgz", @@ -5246,6 +6265,19 @@ "resolved": "https://registry.npmjs.org/object-assign/-/object-assign-4.1.1.tgz", "integrity": "sha512-rJgTQnkUnH1sFw8yT6VSU3zD3sWmu6sZhIseY8VX+GRu3P6F7Fu+JNDoXfklElbLJSnc3FUQHVe4cU5hj+BcUg==" }, + "object-inspect": { + "version": "1.12.2", + "resolved": "https://registry.npmjs.org/object-inspect/-/object-inspect-1.12.2.tgz", + "integrity": "sha512-z+cPxW0QGUp0mcqcsgQyLVRDoXFQbXOwBaqyF7VIgI4TWNQsDHrBpUQslRmIfAoYWdYzs6UlKJtB2XJpTaNSpQ==" + }, + "on-finished": { + "version": "2.4.1", + "resolved": "https://registry.npmjs.org/on-finished/-/on-finished-2.4.1.tgz", + "integrity": "sha512-oVlzkg3ENAhCk2zdv7IJwd/QUD4z2RxRwpkcGY8psCVcCYZNq4wYnVWALHM+brtuJjePWiYF/ClmuDr8Ch5+kg==", + "requires": { + "ee-first": "1.1.1" + } + }, "once": { "version": "1.4.0", "resolved": "https://registry.npmjs.org/once/-/once-1.4.0.tgz", @@ -5315,6 +6347,11 @@ } } }, + "parseurl": { + "version": "1.3.3", + "resolved": "https://registry.npmjs.org/parseurl/-/parseurl-1.3.3.tgz", + "integrity": "sha512-CiyeOxFT/JZyN5m0z9PfXw4SCBJ6Sygz1Dpl0wqjlhDEGGBP1GnsUVEL0p63hoG1fcj3fHynXi9NYO4nWOL+qQ==" + }, "path-exists": { "version": "4.0.0", "resolved": "https://registry.npmjs.org/path-exists/-/path-exists-4.0.0.tgz", @@ -5338,6 +6375,11 @@ "integrity": "sha512-LDJzPVEEEPR+y48z93A0Ed0yXb8pAByGWo/k5YYdYgpY2/2EsOsksJrq7lOHxryrVOn1ejG6oAp8ahvOIQD8sw==", "dev": true }, + "path-to-regexp": { + "version": "0.1.7", + "resolved": "https://registry.npmjs.org/path-to-regexp/-/path-to-regexp-0.1.7.tgz", + "integrity": "sha512-5DFkuoqlv1uYQKxy8omFBeJPQcdoE07Kv2sferDCrAq1ohOU+MSDswDIbnx3YAM60qIOnYa53wBhXW0EbMonrQ==" + }, "path-type": { "version": "4.0.0", "resolved": "https://registry.npmjs.org/path-type/-/path-type-4.0.0.tgz", @@ -5456,6 +6498,15 @@ "resolved": "https://registry.npmjs.org/process-warning/-/process-warning-1.0.0.tgz", "integrity": "sha512-du4wfLyj4yCZq1VupnVSZmRsPJsNuxoDQFdCFHLaYiEbFBD7QE0a+I4D7hOxrVnh78QE/YipFAj9lXHiXocV+Q==" }, + "proxy-addr": { + "version": "2.0.7", + "resolved": "https://registry.npmjs.org/proxy-addr/-/proxy-addr-2.0.7.tgz", + "integrity": "sha512-llQsMLSUDUPT44jdrU/O37qlnifitDP+ZwrmmZcoSKyLKvtZxpyV0n2/bD/N4tBAAZ/gJEdZU7KMraoK1+XYAg==", + "requires": { + "forwarded": "0.2.0", + "ipaddr.js": "1.9.1" + } + }, "pump": { "version": "3.0.0", "resolved": "https://registry.npmjs.org/pump/-/pump-3.0.0.tgz", @@ -5471,6 +6522,14 @@ "integrity": "sha512-XRsRjdf+j5ml+y/6GKHPZbrF/8p2Yga0JPtdqTIY2Xe5ohJPD9saDJJLPvp9+NSBprVvevdXZybnj2cv8OEd0A==", "dev": true }, + "qs": { + "version": "6.10.3", + "resolved": "https://registry.npmjs.org/qs/-/qs-6.10.3.tgz", + "integrity": "sha512-wr7M2E0OFRfIfJZjKGieI8lBKb7fRCH4Fv5KNPEs7gJ8jadvotdsS08PzOKR7opXhZ/Xkjtt3WF9g38drmyRqQ==", + "requires": { + "side-channel": "^1.0.4" + } + }, "queue-microtask": { "version": "1.2.3", "resolved": "https://registry.npmjs.org/queue-microtask/-/queue-microtask-1.2.3.tgz", @@ -5482,6 +6541,22 @@ "resolved": "https://registry.npmjs.org/quick-format-unescaped/-/quick-format-unescaped-4.0.4.tgz", "integrity": "sha512-tYC1Q1hgyRuHgloV/YXs2w15unPVh8qfu/qCTfhTYamaw7fyhumKa2yGpdSo87vY32rIclj+4fWYQXUMs9EHvg==" }, + "range-parser": { + "version": "1.2.1", + "resolved": "https://registry.npmjs.org/range-parser/-/range-parser-1.2.1.tgz", + "integrity": "sha512-Hrgsx+orqoygnmhFbKaHE6c296J+HTAQXoxEF6gNupROmmGJRoyzfG3ccAveqCBrwr/2yxQ5BVd/GTl5agOwSg==" + }, + "raw-body": { + "version": "2.5.1", + "resolved": "https://registry.npmjs.org/raw-body/-/raw-body-2.5.1.tgz", + "integrity": "sha512-qqJBtEyVgS0ZmPGdCFPWJ3FreoqvG4MVQln/kCgF7Olq95IbOp0/BWyMwbdtn4VTvkM8Y7khCQ2Xgk/tcrCXig==", + "requires": { + "bytes": "3.1.2", + "http-errors": "2.0.0", + "iconv-lite": "0.4.24", + "unpipe": "1.0.0" + } + }, "rc": { "version": "1.2.8", "resolved": "https://registry.npmjs.org/rc/-/rc-1.2.8.tgz", @@ -5590,6 +6665,11 @@ "resolved": "https://registry.npmjs.org/safe-buffer/-/safe-buffer-5.2.1.tgz", "integrity": "sha512-rp3So07KcdmmKbGvgaNxQSJr7bGVSVk5S9Eq1F+ppbRo70+YeaDxkw5Dd8NPN+GD6bjnYm2VuPuCXmpuYvmCXQ==" }, + "safer-buffer": { + "version": "2.1.2", + "resolved": "https://registry.npmjs.org/safer-buffer/-/safer-buffer-2.1.2.tgz", + "integrity": "sha512-YZo3K82SD7Riyi0E1EQPojLz7kpepnSQI9IyPbHHg1XXXevb5dJI7tpyN2ADxGcQbHG7vcyRHk0cbwqcQriUtg==" + }, "sax": { "version": "1.2.4", "resolved": "https://registry.npmjs.org/sax/-/sax-1.2.4.tgz", @@ -5604,6 +6684,64 @@ "lru-cache": "^6.0.0" } }, + "send": { + "version": "0.18.0", + "resolved": "https://registry.npmjs.org/send/-/send-0.18.0.tgz", + "integrity": "sha512-qqWzuOjSFOuqPjFe4NOsMLafToQQwBSOEpS+FwEt3A2V3vKubTquT3vmLTQpFgMXp8AlFWFuP1qKaJZOtPpVXg==", + "requires": { + "debug": "2.6.9", + "depd": "2.0.0", + "destroy": "1.2.0", + "encodeurl": "~1.0.2", + "escape-html": "~1.0.3", + "etag": "~1.8.1", + "fresh": "0.5.2", + "http-errors": "2.0.0", + "mime": "1.6.0", + "ms": "2.1.3", + "on-finished": "2.4.1", + "range-parser": "~1.2.1", + "statuses": "2.0.1" + }, + "dependencies": { + "debug": { + "version": "2.6.9", + "resolved": "https://registry.npmjs.org/debug/-/debug-2.6.9.tgz", + "integrity": "sha512-bC7ElrdJaJnPbAP+1EotYvqZsb3ecl5wi6Bfi6BJTUcNowp6cvspg0jXznRTKDjm/E7AdgFBVeAPVMNcKGsHMA==", + "requires": { + "ms": "2.0.0" + }, + "dependencies": { + "ms": { + "version": "2.0.0", + "resolved": "https://registry.npmjs.org/ms/-/ms-2.0.0.tgz", + "integrity": "sha512-Tpp60P6IUJDTuOq/5Z8cdskzJujfwqfOTkrwIwj7IRISpnkJnT6SyJ4PCPnGMoFjC9ddhal5KVIYtAt97ix05A==" + } + } + }, + "ms": { + "version": "2.1.3", + "resolved": "https://registry.npmjs.org/ms/-/ms-2.1.3.tgz", + "integrity": "sha512-6FlzubTLZG3J2a/NVCAleEhjzq5oxgHyaCU9yYXvcLsvoVaHJq/s5xXI6/XXP6tz7R9xAOtHnSO/tXtF3WRTlA==" + } + } + }, + "serve-static": { + "version": "1.15.0", + "resolved": "https://registry.npmjs.org/serve-static/-/serve-static-1.15.0.tgz", + "integrity": "sha512-XGuRDNjXUijsUL0vl6nSD7cwURuzEgglbOaFuZM9g3kwDXOWVTck0jLzjPzGD+TazWbboZYu52/9/XPdUgne9g==", + "requires": { + "encodeurl": "~1.0.2", + "escape-html": "~1.0.3", + "parseurl": "~1.3.3", + "send": "0.18.0" + } + }, + "setprototypeof": { + "version": "1.2.0", + "resolved": "https://registry.npmjs.org/setprototypeof/-/setprototypeof-1.2.0.tgz", + "integrity": "sha512-E5LDX7Wrp85Kil5bhZv46j8jOeboKq5JMmYM3gVGdGH8xFpPWXUMsNrlODCrkoxMEeNi/XZIwuRvY4XNwYMJpw==" + }, "sha.js": { "version": "2.4.11", "resolved": "https://registry.npmjs.org/sha.js/-/sha.js-2.4.11.tgz", @@ -5628,6 +6766,16 @@ "integrity": "sha512-7++dFhtcx3353uBaq8DDR4NuxBetBzC7ZQOhmTQInHEd6bSrXdiEyzCvG07Z44UYdLShWUyXt5M/yhz8ekcb1A==", "dev": true }, + "side-channel": { + "version": "1.0.4", + "resolved": "https://registry.npmjs.org/side-channel/-/side-channel-1.0.4.tgz", + "integrity": "sha512-q5XPytqFEIKHkGdiMIrY10mvLRvnQh42/+GoBlFW3b2LXLE2xxJpZFdm94we0BaoV3RwJyGqg5wS7epxTv0Zvw==", + "requires": { + "call-bind": "^1.0.0", + "get-intrinsic": "^1.0.2", + "object-inspect": "^1.9.0" + } + }, "simple-concat": { "version": "1.0.1", "resolved": "https://registry.npmjs.org/simple-concat/-/simple-concat-1.0.1.tgz", @@ -5713,6 +6861,11 @@ "resolved": "https://registry.npmjs.org/sqlite/-/sqlite-4.1.2.tgz", "integrity": "sha512-FlBG51gHbux5vPjwnoqFEghNGvnTMTbHyiI09U3qFTQs9AtWuwd4i++6+WCusCXKrVdIDLzfdGekrolr3m4U4A==" }, + "statuses": { + "version": "2.0.1", + "resolved": "https://registry.npmjs.org/statuses/-/statuses-2.0.1.tgz", + "integrity": "sha512-RwNA9Z/7PrK06rYLIzFMlaF+l73iwpzsqRIFgbMLbTcLD6cOao82TaWefPXQvB2fOC4AjuYSEndS7N/mTCbkdQ==" + }, "string_decoder": { "version": "0.10.31", "resolved": "https://registry.npmjs.org/string_decoder/-/string_decoder-0.10.31.tgz", @@ -5888,33 +7041,17 @@ "is-number": "^7.0.0" } }, + "toidentifier": { + "version": "1.0.1", + "resolved": "https://registry.npmjs.org/toidentifier/-/toidentifier-1.0.1.tgz", + "integrity": "sha512-o5sSPKEkg/DIQNmH43V0/uerLrpzVedkUh8tGNvaeXpfpuwjKenlSox/2O/BTlZUtEe+JG7s5YhEz608PlAHRA==" + }, "tree-kill": { "version": "1.2.2", "resolved": "https://registry.npmjs.org/tree-kill/-/tree-kill-1.2.2.tgz", "integrity": "sha512-L0Orpi8qGpRG//Nd+H90vFB+3iHnue1zSSGmNOOCh1GLJ7rUKVwV2HvijphGQS2UmhUZewS9VgvxYIdgr+fG1A==", "dev": true }, - "ts-node": { - "version": "10.9.1", - "resolved": "https://registry.npmjs.org/ts-node/-/ts-node-10.9.1.tgz", - "integrity": "sha512-NtVysVPkxxrwFGUUxGYhfux8k78pQB3JqYBXlLRZgdGUqTO5wU/UyHop5p70iEbGhB7q5KmiZiU0Y3KlJrScEw==", - "dev": true, - "requires": { - "@cspotcode/source-map-support": "^0.8.0", - "@tsconfig/node10": "^1.0.7", - "@tsconfig/node12": "^1.0.7", - "@tsconfig/node14": "^1.0.0", - "@tsconfig/node16": "^1.0.2", - "acorn": "^8.4.1", - "acorn-walk": "^8.1.1", - "arg": "^4.1.0", - "create-require": "^1.1.0", - "diff": "^4.0.1", - "make-error": "^1.1.1", - "v8-compile-cache-lib": "^3.0.1", - "yn": "3.1.1" - } - }, "ts-node-dev": { "version": "2.0.0", "resolved": "https://registry.npmjs.org/ts-node-dev/-/ts-node-dev-2.0.0.tgz", @@ -5941,6 +7078,27 @@ "requires": { "glob": "^7.1.3" } + }, + "ts-node": { + "version": "10.9.1", + "resolved": "https://registry.npmjs.org/ts-node/-/ts-node-10.9.1.tgz", + "integrity": "sha512-NtVysVPkxxrwFGUUxGYhfux8k78pQB3JqYBXlLRZgdGUqTO5wU/UyHop5p70iEbGhB7q5KmiZiU0Y3KlJrScEw==", + "dev": true, + "requires": { + "@cspotcode/source-map-support": "^0.8.0", + "@tsconfig/node10": "^1.0.7", + "@tsconfig/node12": "^1.0.7", + "@tsconfig/node14": "^1.0.0", + "@tsconfig/node16": "^1.0.2", + "acorn": "^8.4.1", + "acorn-walk": "^8.1.1", + "arg": "^4.1.0", + "create-require": "^1.1.0", + "diff": "^4.0.1", + "make-error": "^1.1.1", + "v8-compile-cache-lib": "^3.0.1", + "yn": "3.1.1" + } } } }, @@ -6010,6 +7168,15 @@ "integrity": "sha512-Ne+eE4r0/iWnpAxD852z3A+N0Bt5RN//NjJwRd2VFHEmrywxf5vsZlh4R6lixl6B+wz/8d+maTSAkN1FIkI3LQ==", "dev": true }, + "type-is": { + "version": "1.6.18", + "resolved": "https://registry.npmjs.org/type-is/-/type-is-1.6.18.tgz", + "integrity": "sha512-TkRKr9sUTxEH8MdfuCSP7VizJyzRNMjj2J2do2Jr3Kym598JVdEksuzPQCnlFPW4ky9Q+iA+ma9BGm06XQBy8g==", + "requires": { + "media-typer": "0.3.0", + "mime-types": "~2.1.24" + } + }, "typeorm": { "version": "0.2.45", "resolved": "https://registry.npmjs.org/typeorm/-/typeorm-0.2.45.tgz", @@ -6034,6 +7201,11 @@ "zen-observable-ts": "^1.0.0" }, "dependencies": { + "argparse": { + "version": "2.0.1", + "resolved": "https://registry.npmjs.org/argparse/-/argparse-2.0.1.tgz", + "integrity": "sha512-8+9WqebbFzpX9OR+Wa6O29asIogeRMzcGtAINdpMHHyAg10f05aSFVBbcEqGf/PXw1EjAZ+q2/bEBg3DvurK3Q==" + }, "buffer": { "version": "6.0.3", "resolved": "https://registry.npmjs.org/buffer/-/buffer-6.0.3.tgz", @@ -6042,6 +7214,14 @@ "base64-js": "^1.3.1", "ieee754": "^1.2.1" } + }, + "js-yaml": { + "version": "4.1.0", + "resolved": "https://registry.npmjs.org/js-yaml/-/js-yaml-4.1.0.tgz", + "integrity": "sha512-wpxZs9NoxZaJESJGIZTyDEaYpl0FKSA+FB9aJiyemKhMwkxQg63h4T1KJgUGHpTqPDNRcmmYLugrRjJlBtWvRA==", + "requires": { + "argparse": "^2.0.1" + } } } }, @@ -6050,6 +7230,11 @@ "resolved": "https://registry.npmjs.org/typescript/-/typescript-4.7.4.tgz", "integrity": "sha512-C0WQT0gezHuw6AdY1M2jxUO83Rjf0HP7Sk1DtXj6j1EwkQNZrHAg2XPWlq62oqEhYvONq5pkC2Y9oPljWToLmQ==" }, + "unpipe": { + "version": "1.0.0", + "resolved": "https://registry.npmjs.org/unpipe/-/unpipe-1.0.0.tgz", + "integrity": "sha512-pjy2bYhSsufwWlKwPc+l3cN7+wuJlK6uz0YdJEOlQDbl6jo/YlPi4mb8agUkVC8BF7V8NuzeyPNqRksA3hztKQ==" + }, "untildify": { "version": "4.0.0", "resolved": "https://registry.npmjs.org/untildify/-/untildify-4.0.0.tgz", @@ -6070,6 +7255,11 @@ "resolved": "https://registry.npmjs.org/util-deprecate/-/util-deprecate-1.0.2.tgz", "integrity": "sha512-EPD5q1uXyFxJpCrLnCc1nHnq3gOa6DZBocAIiI2TaSCA7VCJ1UJDMagCzIkXNsUYfD1daK//LTEQ8xiIbrHtcw==" }, + "utils-merge": { + "version": "1.0.1", + "resolved": "https://registry.npmjs.org/utils-merge/-/utils-merge-1.0.1.tgz", + "integrity": "sha512-pMZTvIkT1d+TFGvDOqodOclx0QWkkgi6Tdoa8gC8ffGAAqz9pzPTZWAybbsHHoED/ztMtkv/VoYTYyShUn81hA==" + }, "uuid": { "version": "8.3.2", "resolved": "https://registry.npmjs.org/uuid/-/uuid-8.3.2.tgz", @@ -6087,6 +7277,11 @@ "integrity": "sha512-wa7YjyUGfNZngI/vtK0UHAN+lgDCxBPCylVXGp0zu59Fz5aiGtNXaq3DhIov063MorB+VfufLh3JlF2KdTK3xg==", "dev": true }, + "vary": { + "version": "1.1.2", + "resolved": "https://registry.npmjs.org/vary/-/vary-1.1.2.tgz", + "integrity": "sha512-BNGbWLfd0eUPabhkXUVm0j8uuvREyTh5ovRa/dyow/BqAbZJyC+5fU+IzQOzmAKzYqYRAISoRhdQr3eIZ/PXqg==" + }, "which": { "version": "2.0.2", "resolved": "https://registry.npmjs.org/which/-/which-2.0.2.tgz", diff --git a/package.json b/package.json index fc1ad1d..b21daae 100644 --- a/package.json +++ b/package.json @@ -18,6 +18,7 @@ "start": "ts-node-dev src/scripts/dev.ts", "lint": "eslint --ext .ts src", "lint:fix": "eslint --ext .ts src --fix", + "http:js5": "ts-node-dev --max-old-space-size=2048 src/http/js5-server.ts", "namehash": "ts-node-dev --max-old-space-size=2048 src/scripts/name-hasher.ts", "unpack": "ts-node-dev --max-old-space-size=2048 src/scripts/unpacker.ts", "index": "ts-node-dev --max-old-space-size=2048 src/scripts/indexer.ts", @@ -58,6 +59,7 @@ "adm-zip": "^0.5.9", "axios": "^0.27.2", "compressjs": "^1.0.3", + "express": "^4.18.1", "graceful-fs": ">=4.2.0", "json5": "^2.2.0", "reflect-metadata": "^0.1.13", @@ -70,6 +72,7 @@ "devDependencies": { "@runejs/eslint-config": "^1.1.0", "@types/adm-zip": "^0.5.0", + "@types/express": "^4.17.13", "@types/graceful-fs": "^4.1.5", "@types/node": "^16.11.26", "@types/yargs": "^17.0.9", diff --git a/src/config/file-type.ts b/src/config/file-type.ts index 6d34143..199e6d5 100644 --- a/src/config/file-type.ts +++ b/src/config/file-type.ts @@ -8,4 +8,4 @@ export type JagFileType = 'FILE' | 'ARCHIVE' | 'STORE' | - 'INDEX'; + 'CACHE'; diff --git a/src/db/jag/jag-database.ts b/src/db/jag/jag-database.ts index e9c2cfa..32b5760 100644 --- a/src/db/jag/jag-database.ts +++ b/src/db/jag/jag-database.ts @@ -10,7 +10,7 @@ import { join } from 'path'; export interface JagIndexEntityWhere { fileType?: JagFileType; key?: number; - indexKey?: number; + cacheKey?: number; archiveKey?: number; } @@ -55,7 +55,7 @@ export class JagDatabase extends IndexDatabase; - readonly animationsIndex: JagIndex; + readonly animationsIndex: JagCache; versionListDecoded: boolean = false; constructor(jagStore: JagFileStore) { this.jagStore = jagStore; this.animations = new Map(); - this.animationsIndex = this.jagStore.getIndex('animations'); + this.animationsIndex = this.jagStore.getCache('animations'); } decodeVersionList(): void { - const archiveIndex = this.jagStore.getIndex('archives'); + const archiveIndex = this.jagStore.getCache('archives'); if (!archiveIndex) { throw new Error(`Archive Index is not loaded!`); } diff --git a/src/file-system/jag/content/archives/interfaces/jag-interface-archive.ts b/src/file-system/jag/content/archives/interfaces/jag-interface-archive.ts index 7af2444..e2dda1b 100644 --- a/src/file-system/jag/content/archives/interfaces/jag-interface-archive.ts +++ b/src/file-system/jag/content/archives/interfaces/jag-interface-archive.ts @@ -29,8 +29,10 @@ export class JagInterfaceArchive { const width = gameInterface.width = data.get('short', 'unsigned'); const height = gameInterface.height = data.get('short', 'unsigned'); gameInterface.alpha = data.get('byte', 'unsigned'); - gameInterface.hoveredPopup = data.get('byte', 'unsigned'); + // hoveredPopup = u_short, but only a single u_byte is written if there is no hovered popup + // use u_smart_short ? + gameInterface.hoveredPopup = data.get('byte', 'unsigned'); if (gameInterface.hoveredPopup !== 0) { gameInterface.hoveredPopup = (gameInterface.hoveredPopup - 1 << 8) + data.get('byte', 'unsigned'); // why? @@ -143,7 +145,7 @@ export class JagInterfaceArchive { if (gameInterface.type === 5) { gameInterface.disabledImage = data.getString(10); - gameInterface.enabledImage= data.getString(10); + gameInterface.enabledImage = data.getString(10); } if (gameInterface.type === 6) { @@ -217,7 +219,7 @@ export class JagInterfaceArchive { } decodeAll(): void { - const archive = this.jagStore.getIndex('archives') + const archive = this.jagStore.getCache('archives') .getArchive('interface.jag'); if (!archive) { diff --git a/src/file-system/jag/index.ts b/src/file-system/jag/index.ts index 6aa0930..33001e8 100644 --- a/src/file-system/jag/index.ts +++ b/src/file-system/jag/index.ts @@ -3,4 +3,4 @@ export * from './jag-file-base'; export * from './jag-file-store'; export * from './jag-archive'; export * from './jag-file'; -export * from './jag-index'; +export * from './jag-cache'; diff --git a/src/file-system/jag/jag-archive.ts b/src/file-system/jag/jag-archive.ts index d42b5a5..f23076b 100644 --- a/src/file-system/jag/jag-archive.ts +++ b/src/file-system/jag/jag-archive.ts @@ -1,5 +1,5 @@ import { JagFileStore } from './jag-file-store'; -import { archives, indexes } from './jag'; +import { archives, caches } from './jag'; import { JagFile } from './jag-file'; import { JagFileBase } from './jag-file-base'; @@ -12,7 +12,7 @@ export class JagArchive extends JagFileBase { jagStore: JagFileStore, archiveKey: number, ) { - super(jagStore, 'ARCHIVE', archiveKey, indexes.archives, -1); + super(jagStore, 'ARCHIVE', archiveKey, caches.archives, -1); const archiveNames = Object.keys(archives); for (const name of archiveNames) { if (archives[name] === archiveKey) { @@ -30,7 +30,7 @@ export class JagArchive extends JagFileBase { async loadFileIndexes(): Promise { const fileIndexes = await this.fileStore.database.getIndexes({ fileType: 'FILE', - indexKey: this.index.indexKey, + cacheKey: this.index.cacheKey, archiveKey: this.index.key, }); @@ -42,7 +42,7 @@ export class JagArchive extends JagFileBase { const fileKey = fileIndex.key; if (!this.files.has(fileKey)) { - const file = new JagFile(this.fileStore, fileKey, this.index.indexKey, this.index.key); + const file = new JagFile(this.fileStore, fileKey, this.index.cacheKey, this.index.key); file.index = fileIndex; this.files.set(fileKey, file); } diff --git a/src/file-system/jag/jag-index.ts b/src/file-system/jag/jag-cache.ts similarity index 90% rename from src/file-system/jag/jag-index.ts rename to src/file-system/jag/jag-cache.ts index 1fc4235..4ee5bd4 100644 --- a/src/file-system/jag/jag-index.ts +++ b/src/file-system/jag/jag-cache.ts @@ -1,21 +1,21 @@ import { JagFileStore } from './jag-file-store'; -import { JagFileIndex, indexes } from './jag'; +import { JagFileIndex, caches } from './jag'; import { JagArchive } from './jag-archive'; import { JagFile } from './jag-file'; import { JagFileBase } from './jag-file-base'; -export class JagIndex extends JagFileBase { +export class JagCache extends JagFileBase { readonly files: Map; fileIndexes: JagFileIndex[]; - constructor(jagStore: JagFileStore, indexKey: number) { - super(jagStore, 'INDEX', indexKey); - const indexNames = Object.keys(indexes); + constructor(jagStore: JagFileStore, cacheKey: number) { + super(jagStore, 'CACHE', cacheKey); + const indexNames = Object.keys(caches); for (const name of indexNames) { - if (indexes[name] === indexKey) { + if (caches[name] === cacheKey) { this.index.name = name; } } @@ -29,7 +29,7 @@ export class JagIndex extends JagFileBase { async loadFileIndexes(): Promise { const fileIndexes = await this.fileStore.database.getIndexes({ - indexKey: this.index.key, + cacheKey: this.index.key, archiveKey: -1, }); diff --git a/src/file-system/jag/jag-file-base.ts b/src/file-system/jag/jag-file-base.ts index f0c50ac..c77ce64 100644 --- a/src/file-system/jag/jag-file-base.ts +++ b/src/file-system/jag/jag-file-base.ts @@ -17,7 +17,7 @@ export abstract class JagFileBase { fileStore: JagFileStore, fileType: JagFileType, key: number, - indexKey: number = -1, + cacheKey: number = -1, archiveKey: number = -1, ) { this.fileStore = fileStore; @@ -25,7 +25,7 @@ export abstract class JagFileBase { this.index.gameBuild = fileStore.gameBuild; this.index.fileType = fileType; this.index.key = key; - this.index.indexKey = indexKey; + this.index.cacheKey = cacheKey; this.index.archiveKey = archiveKey; } @@ -111,7 +111,7 @@ export abstract class JagFileBase { const indexEntity = await this.fileStore.database.getIndex({ fileType: this.index.fileType, key: this.index.key, - indexKey: this.index.indexKey, + cacheKey: this.index.cacheKey, archiveKey: this.index.archiveKey, }); diff --git a/src/file-system/jag/jag-file-store.ts b/src/file-system/jag/jag-file-store.ts index 96555ca..501013e 100644 --- a/src/file-system/jag/jag-file-store.ts +++ b/src/file-system/jag/jag-file-store.ts @@ -1,19 +1,21 @@ -import { FileStoreBase } from '../file-store-base'; -import { Jag, indexes } from './jag'; -import { JagIndex } from './jag-index'; import { join } from 'path'; -import { JagDatabase } from '../../db/jag/jag-database'; +import { logger } from '@runejs/common'; +import { FileStoreBase } from '../file-store-base'; +import { Jag, caches } from './jag'; +import { JagCache } from './jag-cache'; +import { JagDatabase } from '../../db/jag'; +import { JagArchive } from './jag-archive'; export class JagFileStore extends FileStoreBase { readonly jag: Jag; - readonly indexes: Map; + readonly caches: Map; constructor(gameBuild: string | number, storePath: string = './') { super(gameBuild, storePath); this.jag = new Jag(this); - this.indexes = new Map(); + this.caches = new Map(); } override async openDatabase(): Promise { @@ -26,41 +28,86 @@ export class JagFileStore extends FileStoreBase { return this._database; } - async loadIndexEntities(): Promise { - for (const [ , index ] of this.indexes) { - await index.loadIndex(); + override async load( + loadCacheEntities: boolean = false, + loadCacheChildEntities: boolean = false, + loadArchiveChildEntities: boolean = false, + ): Promise { + logger.info(`Loading JAG store for build ${this.gameBuild}...`); + await this.openDatabase(); + + if (loadCacheEntities) { + await this.loadCacheEntities(loadCacheChildEntities, loadArchiveChildEntities); } } - override async load(): Promise { - await this.openDatabase(); + /** + * Load all cache entities for this file store. + * @param loadCacheChildEntities Whether or not to load cache child file entities. + * Defaults to `false`. + * @param loadArchiveChildEntities Whether or not to load archive child file entities. + * Only works if `loadCacheChildEntities` is also set to `true`. Defaults to `false`. + */ + async loadCacheEntities( + loadCacheChildEntities: boolean = false, + loadArchiveChildEntities: boolean = false, + ): Promise { + if (!this.caches.size) { + const cacheNames = Object.keys(caches); + for (const cacheName of cacheNames) { + this.createCache(caches[cacheName]); + } + } + + for (const [ , cache ] of this.caches) { + await cache.loadIndex(); + } + + if (loadCacheChildEntities) { + logger.info(`Loading cache file entities...`); + + for (const [ , cache ] of this.caches) { + await cache.loadFileIndexes(); + } + + if (loadArchiveChildEntities) { + logger.info(`Loading archive file entities...`); + + const archiveIndex = this.getCache('archives'); + + for (const [ , file ] of archiveIndex.files) { + const archive = file as JagArchive; + await archive.loadFileIndexes(); + } + } + } } - createIndex(indexKey: number): void { - this.setIndex(indexKey, new JagIndex(this, indexKey)); + createCache(cacheKey: number): void { + this.setCache(cacheKey, new JagCache(this, cacheKey)); } - getIndex(indexKey: number): JagIndex | null; - getIndex(indexName: string): JagIndex | null; - getIndex(indexKeyOrName: number | string): JagIndex | null; - getIndex(indexKeyOrName: number | string): JagIndex | null { - if (typeof indexKeyOrName === 'string') { - return Array.from(this.indexes.values()).find( - i => i?.index?.name === indexKeyOrName + getCache(cacheKey: number): JagCache | null; + getCache(cacheName: string): JagCache | null; + getCache(cacheKeyOrName: number | string): JagCache | null; + getCache(cacheKeyOrName: number | string): JagCache | null { + if (typeof cacheKeyOrName === 'string') { + return Array.from(this.caches.values()).find( + i => i?.index?.name === cacheKeyOrName ) || null; } else { - return this.indexes.get(indexKeyOrName) || null; + return this.caches.get(cacheKeyOrName) || null; } } - setIndex(indexKey: number, index: JagIndex): void; - setIndex(indexName: string, index: JagIndex): void; - setIndex(indexKeyOrName: number | string, index: JagIndex): void; - setIndex(indexKeyOrName: number | string, index: JagIndex): void { - if (typeof indexKeyOrName === 'string') { - this.indexes.set(indexes[indexKeyOrName], index); + setCache(cacheKey: number, cache: JagCache): void; + setCache(cacheName: string, cache: JagCache): void; + setCache(cacheKeyOrName: number | string, cache: JagCache): void; + setCache(cacheKeyOrName: number | string, cache: JagCache): void { + if (typeof cacheKeyOrName === 'string') { + this.caches.set(caches[cacheKeyOrName], cache); } else { - this.indexes.set(indexKeyOrName, index); + this.caches.set(cacheKeyOrName, cache); } } diff --git a/src/file-system/jag/jag-file.ts b/src/file-system/jag/jag-file.ts index 737f311..bc7fff5 100644 --- a/src/file-system/jag/jag-file.ts +++ b/src/file-system/jag/jag-file.ts @@ -4,8 +4,8 @@ import { JagFileBase } from './jag-file-base'; export class JagFile extends JagFileBase { - constructor(jagStore: JagFileStore, fileKey: number, indexKey: number, archiveKey: number = -1) { - super(jagStore, 'FILE', fileKey, indexKey, archiveKey); + constructor(jagStore: JagFileStore, fileKey: number, cacheKey: number, archiveKey: number = -1) { + super(jagStore, 'FILE', fileKey, cacheKey, archiveKey); } } diff --git a/src/file-system/jag/jag.ts b/src/file-system/jag/jag.ts index 9fd7293..66abf28 100644 --- a/src/file-system/jag/jag.ts +++ b/src/file-system/jag/jag.ts @@ -7,14 +7,14 @@ import { Buffer } from 'buffer'; import { JagArchive } from './jag-archive'; import { decompressHeadlessBzip2 } from '../../compress'; import { JagFileBase } from './jag-file-base'; -import { CacheFile } from '../cache'; +import { PackedCacheFile } from '../packed'; const dataFileName = 'main_file_cache.dat'; const indexFileNamePrefix = 'main_file_cache.idx'; -export const indexes = { +export const caches = { archives: 0, models: 1, animations: 2, @@ -46,7 +46,7 @@ export interface JagSectorHeader { fileKey: number; filePartNumber: number; sectorNumber: number; - indexKey: number; + cacheKey: number; } @@ -62,7 +62,7 @@ export class Jag { this.indexFiles = new Map(); } - readOpenRS2CacheFiles(cacheFiles: CacheFile[]): void { + readOpenRS2PackedCacheFiles(cacheFiles: PackedCacheFile[]): void { const dataFileBuffer = cacheFiles.find(file => file.name === dataFileName)?.data || null; if (!dataFileBuffer?.length) { throw new Error(`The main ${ dataFileName } data file could not be found.`); @@ -95,13 +95,13 @@ export class Jag { } this.indexFiles.set(indexKey, new ByteBuffer(cacheFile.data)); - this.jagStore.createIndex(indexKey); + this.jagStore.createCache(indexKey); } logger.info(`JAG store files loaded for game build ${this.jagStore.gameBuild}.`); } - readLocalCacheFiles(): void { + readLocalPackedCacheFiles(): void { const jagStorePath = join(this.jagStore.fileStorePath, 'jag'); if (!existsSync(jagStorePath)) { @@ -143,22 +143,22 @@ export class Jag { } this.indexFiles.set(indexKey, new ByteBuffer(readFileSync(join(jagStorePath, fileName)))); - this.jagStore.createIndex(indexKey); + this.jagStore.createCache(indexKey); } logger.info(`JAG store files loaded for game build ${this.jagStore.gameBuild}.`); } - decodeIndex(indexName: string): void { - logger.info(`Decoding JAG index ${indexName}...`); + decodeCache(indexName: string): void { + logger.info(`Decoding JAG cache index ${indexName}...`); - const indexKey = indexes[indexName]; - const indexFile = this.indexFiles.get(indexKey); + const cacheKey = caches[indexName]; + const indexFile = this.indexFiles.get(cacheKey); const fileCount = indexFile.length / 6; logger.info(`${fileCount} file indexes found.`); - const index = this.jagStore.getIndex(indexKey); + const index = this.jagStore.getCache(cacheKey); index.fileIndexes = new Array(fileCount); for (let fileKey = 0; fileKey < fileCount; fileKey++) { @@ -173,17 +173,17 @@ export class Jag { if (indexName === 'archives') { file = new JagArchive(this.jagStore, fileKey); } else { - file = new JagFile(this.jagStore, fileKey, indexKey); + file = new JagFile(this.jagStore, fileKey, cacheKey); } index.files.set(fileKey, file); } - logger.info(`Index ${indexName} has been loaded.`); + logger.info(`Cache index ${indexName} has been decoded.`); } unpack(file: JagArchive | JagFile): Buffer | null { - const fileIndexData = this.jagStore.getIndex(file.index.indexKey); + const fileIndexData = this.jagStore.getCache(file.index.cacheKey); const { fileSize, sectorNumber } = fileIndexData.fileIndexes[file.index.key]; const fileData = new ByteBuffer(fileSize); const sectorDataLength = 512; @@ -205,7 +205,7 @@ export class Jag { const sectorFileKey = block.get('short', 'unsigned'); const sectorFilePartNumber = block.get('short', 'unsigned'); const sectorNumber = block.get('int24', 'unsigned'); - const sectorIndexKey = block.get('byte', 'unsigned'); + const sectorCacheKey = block.get('byte', 'unsigned'); readableSectorData -= 8; @@ -232,8 +232,8 @@ export class Jag { if (remainingData > 0) { // saved index keys have 1 added to them for some reason - if (sectorIndexKey !== file.index.indexKey + 1) { - logger.error(`Index key mismatch, expected index ${ file.index.indexKey } but found ${ sectorIndexKey }`); + if (sectorCacheKey !== file.index.cacheKey + 1) { + logger.error(`Index key mismatch, expected cache ${ file.index.cacheKey } but found ${ sectorCacheKey }`); return null; } @@ -293,7 +293,7 @@ export class Jag { fileDataOffsets[fileKey] = fileDataOffset; fileDataOffset += compressedFileLength; - const file = new JagFile(this.jagStore, fileKey, archive.index.indexKey, archive.index.key); + const file = new JagFile(this.jagStore, fileKey, archive.index.cacheKey, archive.index.key); file.index.nameHash = fileNameHash; file.index.name = fileName; file.index.fileSize = decompressedFileLength; diff --git a/src/file-system/js5/js5-archive.ts b/src/file-system/js5/js5-archive.ts index 55366cc..e0aa5e1 100644 --- a/src/file-system/js5/js5-archive.ts +++ b/src/file-system/js5/js5-archive.ts @@ -1,5 +1,5 @@ import { Js5FileStore } from './js5-file-store'; -import { JS5Group } from './js5-group'; +import { Js5Group } from './js5-group'; import { logger } from '@runejs/common'; import { Js5FileBase } from './js5-file-base'; import { Js5ArchiveConfig } from '../../config'; @@ -7,7 +7,7 @@ import { Js5ArchiveConfig } from '../../config'; export class Js5Archive extends Js5FileBase { - readonly groups: Map; + readonly groups: Map; readonly config: Js5ArchiveConfig; constructor( @@ -17,7 +17,7 @@ export class Js5Archive extends Js5FileBase { super(fileStore, 'ARCHIVE', archiveKey, 255, -1); this.config = fileStore.getArchiveConfig(archiveKey); this.index.name = fileStore.getArchiveName(archiveKey); - this.groups = new Map(); + this.groups = new Map(); } override validate(trackChanges: boolean = true): void { @@ -58,17 +58,17 @@ export class Js5Archive extends Js5FileBase { const groupKey = groupIndex.key; if (!this.groups.has(groupKey)) { - const group = new JS5Group(this.fileStore, groupKey, this); + const group = new Js5Group(this.fileStore, groupKey, this); group.index = groupIndex; this.groups.set(groupKey, group); } } } - getGroup(groupKey: number): JS5Group | null; - getGroup(groupName: string): JS5Group | null; - getGroup(groupKeyOrName: number | string): JS5Group | null; - getGroup(groupKeyOrName: number | string): JS5Group | null { + getGroup(groupKey: number): Js5Group | null; + getGroup(groupName: string): Js5Group | null; + getGroup(groupKeyOrName: number | string): Js5Group | null; + getGroup(groupKeyOrName: number | string): Js5Group | null { if (typeof groupKeyOrName === 'string') { return Array.from(this.groups.values()).find( group => group?.index?.name === groupKeyOrName @@ -78,7 +78,7 @@ export class Js5Archive extends Js5FileBase { } } - setGroup(groupKey: number, group: JS5Group): void { + setGroup(groupKey: number, group: Js5Group): void { this.groups.set(groupKey, group); } diff --git a/src/file-system/js5/js5-file-store.ts b/src/file-system/js5/js5-file-store.ts index 63f694e..bf3d414 100644 --- a/src/file-system/js5/js5-file-store.ts +++ b/src/file-system/js5/js5-file-store.ts @@ -6,7 +6,8 @@ import { JS5 } from './js5'; import { Js5Archive } from './js5-archive'; import { FileStoreBase } from '../file-store-base'; import { logger } from '../../../../common'; -import { Js5Database } from '../../db/js5/js5-database'; +import { Js5Database } from '../../db/js5'; +import { Js5File } from './js5-file'; export class Js5FileStore extends FileStoreBase{ @@ -33,14 +34,64 @@ export class Js5FileStore extends FileStoreBase{ return this._database; } - override async load(): Promise { + override async load( + loadArchiveEntities: boolean = false, + loadArchiveChildEntities: boolean = false, + loadGroupChildEntities: boolean = false, + ): Promise { await this.js5.loadEncryptionKeys(); await this.openDatabase(); + + if (loadArchiveEntities) { + await this.loadArchiveEntities(loadArchiveChildEntities, loadGroupChildEntities); + } } - async loadArchiveEntities(): Promise { - for (const [ , archive ] of this.archives) { - await archive.loadIndex(); + /** + * Load all archive entities for this file store. + * @param loadArchiveChildEntities Whether or not to load group entities under each archive. + * Defaults to `false`. + * @param loadGroupChildEntities Whether or not to load flat file entities under each group. + * Only works if `loadArchiveChildEntities` is also set to `true`. Defaults to `false`. + */ + async loadArchiveEntities( + loadArchiveChildEntities: boolean = false, + loadGroupChildEntities: boolean = false, + ): Promise { + if (!this.archives.size) { + const archiveEntities = await this.database.getIndexes({ + fileType: 'ARCHIVE' + }); + + for (const entity of archiveEntities) { + const archive = this.createArchive(entity.key); + archive.index = entity; + } + } else { + for (const [ , archive ] of this.archives) { + await archive.loadIndex(); + } + } + + if (loadArchiveChildEntities) { + for (const [ , archive ] of this.archives) { + await archive.loadGroupIndexes(); + } + + if (loadGroupChildEntities) { + // Bulk load grouped files to save computing time + const files = await this.database.getIndexes({ + fileType: 'FILE', + }); + + for (const fileEntity of files) { + const archive = this.getArchive(fileEntity.archiveKey); + const group = archive.getGroup(fileEntity.groupKey); + const file = new Js5File(this, fileEntity.key, group); + file.index = fileEntity; + group.setFile(fileEntity.key, file); + } + } } } @@ -62,8 +113,10 @@ export class Js5FileStore extends FileStoreBase{ } } - createArchive(archiveKey: number): void { - this.setArchive(archiveKey, new Js5Archive(this, archiveKey)); + createArchive(archiveKey: number): Js5Archive { + const archive = new Js5Archive(this, archiveKey); + this.setArchive(archiveKey, archive); + return archive; } getArchive(archiveKey: number): Js5Archive | null; diff --git a/src/file-system/js5/js5-file.ts b/src/file-system/js5/js5-file.ts index 2fc7b9f..c4fd8e1 100644 --- a/src/file-system/js5/js5-file.ts +++ b/src/file-system/js5/js5-file.ts @@ -1,18 +1,18 @@ import { Js5FileStore } from './js5-file-store'; import { Js5Archive } from './js5-archive'; -import { JS5Group } from './js5-group'; +import { Js5Group } from './js5-group'; import { Js5FileBase } from './js5-file-base'; -export class JS5File extends Js5FileBase { +export class Js5File extends Js5FileBase { readonly archive: Js5Archive; - readonly group: JS5Group; + readonly group: Js5Group; constructor( fileStore: Js5FileStore, fileKey: number, - group: JS5Group, + group: Js5Group, ) { super(fileStore, 'FILE', fileKey, group.archive.index.key, group.index.key); this.archive = group.archive; diff --git a/src/file-system/js5/js5-group.ts b/src/file-system/js5/js5-group.ts index 3e7a451..447f4d1 100644 --- a/src/file-system/js5/js5-group.ts +++ b/src/file-system/js5/js5-group.ts @@ -1,14 +1,14 @@ import { Js5FileStore } from './js5-file-store'; import { Js5Archive } from './js5-archive'; -import { JS5File } from './js5-file'; +import { Js5File } from './js5-file'; import { logger } from '@runejs/common'; import { Js5FileBase } from './js5-file-base'; -export class JS5Group extends Js5FileBase { +export class Js5Group extends Js5FileBase { readonly archive: Js5Archive; - readonly files: Map; + readonly files: Map; constructor( fileStore: Js5FileStore, @@ -17,7 +17,7 @@ export class JS5Group extends Js5FileBase { ) { super(fileStore, 'GROUP', groupKey, archive.index.key); this.archive = archive; - this.files = new Map(); + this.files = new Map(); } override validate(trackChanges: boolean = true): void { @@ -59,17 +59,17 @@ export class JS5Group extends Js5FileBase { const fileKey = fileIndex.key; if (!this.files.has(fileKey)) { - const file = new JS5File(this.fileStore, fileKey, this); + const file = new Js5File(this.fileStore, fileKey, this); file.index = fileIndex; this.files.set(fileKey, file); } } } - getFile(fileKey: number): JS5File | null; - getFile(fileName: string): JS5File | null; - getFile(fileKeyOrName: number | string): JS5File | null; - getFile(fileKeyOrName: number | string): JS5File | null { + getFile(fileKey: number): Js5File | null; + getFile(fileName: string): Js5File | null; + getFile(fileKeyOrName: number | string): Js5File | null; + getFile(fileKeyOrName: number | string): Js5File | null { if (typeof fileKeyOrName === 'string') { return Array.from(this.files.values()).find( file => file?.index?.name === fileKeyOrName @@ -79,7 +79,7 @@ export class JS5Group extends Js5FileBase { } } - setFile(fileKey: number, file: JS5File): void { + setFile(fileKey: number, file: Js5File): void { this.files.set(fileKey, file); } diff --git a/src/file-system/js5/js5.ts b/src/file-system/js5/js5.ts index 102e92a..dc59868 100644 --- a/src/file-system/js5/js5.ts +++ b/src/file-system/js5/js5.ts @@ -6,7 +6,7 @@ import { ByteBuffer, logger } from '@runejs/common'; import { getCompressionMethod } from '@runejs/common/compress'; import { Xtea, XteaKeys, XteaConfig } from '@runejs/common/encrypt'; -import { Js5FileStore, Js5Archive, JS5Group, JS5File } from '.'; +import { Js5FileStore, Js5Archive, Js5Group, Js5File } from '.'; import { archiveFlags, ArchiveFormat } from '../../config'; import { getXteaKeysByBuild } from '../../openrs2'; import { @@ -15,7 +15,7 @@ import { compressGzip, decompressGzip } from '../../compress'; -import { CacheFile } from '../cache'; +import { PackedCacheFile } from '../packed'; const dataFileName = 'main_file_cache.dat2'; @@ -39,7 +39,7 @@ export class JS5 { this.indexFiles = new Map(); } - readOpenRS2CacheFiles(cacheFiles: CacheFile[]): void { + readOpenRS2CacheFiles(cacheFiles: PackedCacheFile[]): void { const dataFileBuffer = cacheFiles.find(file => file.name === dataFileName)?.data || null; if (!dataFileBuffer?.length) { throw new Error(`The main ${ dataFileName } data file could not be found.`); @@ -136,7 +136,7 @@ export class JS5 { logger.info(`JS5 store file loaded for game build ${ this.fileStore.gameBuild }.`); } - unpack(file: JS5Group | Js5Archive): Buffer | null { + unpack(file: Js5Group | Js5Archive): Buffer | null { const fileIndex = file.index; const fileKey = fileIndex.key; const archiveKey: number = file instanceof Js5Archive ? 255 : file.archive.index.key; @@ -241,7 +241,7 @@ export class JS5 { return fileIndex.compressedData; } - readCompressedFileHeader(file: JS5Group | Js5Archive): { compressedLength: number, readerIndex: number } { + readCompressedFileHeader(file: Js5Group | Js5Archive): { compressedLength: number, readerIndex: number } { const fileDetails = file.index; if (!fileDetails.compressedData?.length) { @@ -259,7 +259,7 @@ export class JS5 { return { compressedLength, readerIndex }; } - decrypt(file: JS5Group | Js5Archive): Buffer { + decrypt(file: Js5Group | Js5Archive): Buffer { const fileDetails = file.index; const fileName = fileDetails.name; @@ -269,7 +269,6 @@ export class JS5 { return null; } - // @todo move to JS5.decodeArchive const archiveName = file instanceof Js5Archive ? 'main' : file.archive.index.name; const archiveConfig = this.fileStore.archiveConfig[archiveName]; @@ -321,6 +320,7 @@ export class JS5 { dataCopy.readerIndex = readerIndex; fileDetails.compressedData = dataCopy.toNodeBuffer(); fileDetails.encrypted = false; + file.validate(false); return fileDetails.compressedData; } else { logger.warn(`Invalid XTEA decryption keys found for file ` + @@ -336,7 +336,7 @@ export class JS5 { return null; } - decompress(file: JS5Group | Js5Archive): Buffer | null { + decompress(file: Js5Group | Js5Archive): Buffer | null { const fileDetails = file.index; if (!fileDetails.compressedData?.length) { @@ -372,15 +372,20 @@ export class JS5 { fileDetails.fileError = 'MISSING_ENCRYPTION_KEYS'; } } else { - const decompressedData = new ByteBuffer(fileDetails.compressionMethod === 'bzip' ? - decompressedLength : (compressedData.length - compressedData.readerIndex + 2)); + const fileData = new ByteBuffer(compressedLength); - compressedData.copy(decompressedData, 0, compressedData.readerIndex); + logger.info(`Decompress ${fileDetails.compressionMethod}, ` + + `compressed len = ${fileDetails.compressedData.length}, ` + + `recorded compressed len = ${compressedLength}, ` + + `compressed data buffer len = ${fileData.length}, ` + + `decompressed len = ${decompressedLength}`); + + compressedData.copy(fileData, 0, compressedData.readerIndex); try { data = fileDetails.compressionMethod === 'bzip' ? - decompressHeadlessBzip2(decompressedData) : - decompressGzip(decompressedData); + decompressHeadlessBzip2(fileData) : + decompressGzip(fileData); compressedData.readerIndex = compressedData.readerIndex + compressedLength; @@ -405,10 +410,11 @@ export class JS5 { fileDetails.data = data; } + file.validate(false); return fileDetails.data; } - async decodeGroup(group: JS5Group): Promise { + async decodeGroup(group: Js5Group): Promise { const groupDetails = group.index; const { key: groupKey, name: groupName } = groupDetails; const files = group.files; @@ -529,7 +535,7 @@ export class JS5 { archiveDetails.version = format >= ArchiveFormat.versioned ? archiveData.get('int') : 0; const flags = archiveFlags(archiveData.get('byte', 'unsigned')); const groupCount = archiveData.get(mainDataType, 'unsigned'); - const groups: JS5Group[] = new Array(groupCount); + const groups: Js5Group[] = new Array(groupCount); let accumulator = 0; logger.info(`${ groupCount } groups were found within the ${ archiveName } archive.`); @@ -538,7 +544,7 @@ export class JS5 { for (let i = 0; i < groupCount; i++) { const delta = archiveData.get(mainDataType, 'unsigned'); const groupKey = accumulator += delta; - const group = groups[i] = new JS5Group(this.fileStore, groupKey, archive); + const group = groups[i] = new Js5Group(this.fileStore, groupKey, archive); archive.setGroup(groupKey, group); } @@ -599,7 +605,7 @@ export class JS5 { for (let i = 0; i < fileCount; i++) { const delta = archiveData.get(mainDataType, 'unsigned'); const childFileIndex = accumulator += delta; - const flatFile = new JS5File(this.fileStore, childFileIndex, group); + const flatFile = new Js5File(this.fileStore, childFileIndex, group); group.setFile(childFileIndex, flatFile); } } @@ -626,16 +632,16 @@ export class JS5 { } // @todo stubbed - 21/07/22 - Kiko - pack(file: JS5Group | Js5Archive): Buffer | null { + pack(file: Js5Group | Js5Archive): Buffer | null { return null; } // @todo stubbed - 21/07/22 - Kiko - encrypt(file: JS5Group | Js5Archive): Buffer | null { + encrypt(file: Js5Group | Js5Archive): Buffer | null { return null; } - compress(file: JS5Group | Js5Archive): Buffer | null { + compress(file: Js5Group | Js5Archive): Buffer | null { const fileDetails = file.index; if (!fileDetails.data?.length) { @@ -688,7 +694,7 @@ export class JS5 { return fileDetails.compressedData; } - encodeGroup(group: JS5Group): Buffer | null { + encodeGroup(group: Js5Group): Buffer | null { const { files: fileMap, index, stripes } = group; const fileCount = fileMap.size; diff --git a/src/file-system/packed/index.ts b/src/file-system/packed/index.ts new file mode 100644 index 0000000..6c17315 --- /dev/null +++ b/src/file-system/packed/index.ts @@ -0,0 +1,2 @@ +export * from './packed-cache-format'; +export * from './packed-cache-file'; diff --git a/src/file-system/cache/cache-file.ts b/src/file-system/packed/packed-cache-file.ts similarity index 67% rename from src/file-system/cache/cache-file.ts rename to src/file-system/packed/packed-cache-file.ts index c0da70c..128c341 100644 --- a/src/file-system/cache/cache-file.ts +++ b/src/file-system/packed/packed-cache-file.ts @@ -1,7 +1,7 @@ import { Buffer } from 'buffer'; -export interface CacheFile { +export interface PackedCacheFile { name: string; data: Buffer; } diff --git a/src/file-system/cache/cache-format.ts b/src/file-system/packed/packed-cache-format.ts similarity index 59% rename from src/file-system/cache/cache-format.ts rename to src/file-system/packed/packed-cache-format.ts index 0725bd2..a07c955 100644 --- a/src/file-system/cache/cache-format.ts +++ b/src/file-system/packed/packed-cache-format.ts @@ -1,7 +1,7 @@ -import { CacheFile } from './cache-file'; +import { PackedCacheFile } from './packed-cache-file'; -export const getCacheFormat = (cacheFiles: CacheFile[]): 'jag' | 'js5' | 'unknown' => { +export const getPackedCacheFormat = (cacheFiles: PackedCacheFile[]): 'jag' | 'js5' | 'unknown' => { if (cacheFiles.find(file => file.name === 'main_file_cache.idx255')) { return 'js5'; } else if (cacheFiles.find(file => file.name === 'main_file_cache.dat')) { diff --git a/src/http/js5-server.ts b/src/http/js5-server.ts new file mode 100644 index 0000000..196ab0d --- /dev/null +++ b/src/http/js5-server.ts @@ -0,0 +1,202 @@ +import express, { Response } from 'express'; +import { logger } from '@runejs/common'; +import { Js5FileStore } from '../file-system/js5'; +import { ArgumentOptions, ScriptExecutor } from '../scripts/script-executor'; +import { constants as http2 } from 'http2'; +import { Js5IndexEntity } from '../db/js5'; + + +interface ServerOptions { + build: string; + dir: string; + port: number; +} + + +const serverArgumentOptions: ArgumentOptions = { + build: { + alias: 'b', + type: 'string', + default: '435', + description: `The game build (revision) that the store should belong to, also known as the game build number. Defaults to '435', a game build from late October, 2006.` + }, + dir: { + alias: 'd', + type: 'string', + default: './', + description: `The store root directory. Defaults to the current location.` + }, + port: { + alias: 'p', + type: 'number', + default: 8080, + description: `The port from which the HTTP server will be accessed. Defaults to 8080.` + }, +}; + + +const handleError = (res: Response, status: number, msg: string) => { + logger.error(msg); + res.status(status).send(); +}; + + +const keyOrName = (input: string): string | number => { + if (/^\d+$/.test(input)) { + return Number(input); + } else { + return input; + } +}; + + +const writeFile = (res: Response, index: Js5IndexEntity, compressed: boolean = true) => { + res.writeHead(200, { + 'Content-Type': 'arraybuffer', + 'Content-Length': compressed ? index.compressedData.length : index.data.length, + 'Content-disposition': `attachment; filename=${index.name || index.key}` + }); + + res.end(compressed ? index.compressedData : index.data); +}; + + +const app = express(); +let store: Js5FileStore; + + +app.get('/archives/:archiveKey/groups/:groupKey/files/:fileKey', (req, res, next) => { + const { archiveKey, groupKey, fileKey } = req.params; + logger.info(`Request /archives/${archiveKey}/groups/${groupKey}/files/${fileKey}`); + + const archive = store.getArchive(keyOrName(archiveKey)); + if (!archive) { + handleError(res, http2.HTTP_STATUS_NOT_FOUND, `Archive ${archiveKey} was not found.`); + return; + } + + const group = archive.getGroup(keyOrName(groupKey)); + if (!group) { + handleError(res, http2.HTTP_STATUS_NOT_FOUND, `Group ${groupKey} was not found within Archive ${archiveKey}.`); + return; + } + + const file = group.getFile(keyOrName(fileKey)); + if (!file || !file.index.data?.length) { + handleError(res, http2.HTTP_STATUS_NOT_FOUND, `File ${groupKey}:${fileKey} was not found within Archive ${archiveKey}.`); + return; + } + + writeFile(res, file.index, false); +}); + +app.get('/archives/:archiveKey/groups/:groupKey', (req, res, next) => { + const { archiveKey, groupKey } = req.params; + logger.info(`Request /archives/${archiveKey}/groups/${groupKey}`); + + const archive = store.getArchive(keyOrName(archiveKey)); + if (!archive) { + handleError(res, http2.HTTP_STATUS_NOT_FOUND, `Archive ${archiveKey} was not found.`); + return; + } + + const group = archive.getGroup(keyOrName(groupKey)); + if (!group || !group.index.compressedData?.length) { + handleError(res, http2.HTTP_STATUS_NOT_FOUND, `Group ${groupKey} was not found within Archive ${archiveKey}.`); + return; + } + + writeFile(res, group.index); +}); + +app.get('/archives/:archiveKey/groups', (req, res, next) => { + const { archiveKey } = req.params; + logger.info(`Request /archives/${archiveKey}/groups`); + + const archive = store.getArchive(keyOrName(archiveKey)); + if (!archive) { + handleError(res, http2.HTTP_STATUS_NOT_FOUND, `Archive ${archiveKey} was not found.`); + return; + } + + const groupDetails = Array.from(archive.groups.values()).map(group => ({ + key: group.index.key, + name: group.index.name, + nameHash: group.index.nameHash, + childCount: group.index.childCount, + children: Array.from(group.files.values()).map(file => ({ + key: file.index.key, + name: file.index.name, + nameHash: file.index.nameHash, + })) + })); + + res.send(groupDetails); +}); + +app.get('/archives/:archiveKey', (req, res, next) => { + const { archiveKey } = req.params; + + logger.info(`Request /archives/${archiveKey}`); + + const archive = store.getArchive(keyOrName(archiveKey)); + if (!archive || !archive.index.compressedData?.length) { + handleError(res, http2.HTTP_STATUS_NOT_FOUND, `Archive ${archiveKey} was not found.`); + return; + } + + writeFile(res, archive.index); +}); + +app.get('/archives', (req, res) => { + logger.info(`Request /archives`); + + const archives = Array.from(store.archives.values()); + + const archiveList: Record[] = []; + + for (const archive of archives) { + archiveList.push({ + key: archive.index.key, + name: archive.index.name, + childCount: archive.index.childCount, + loadedChildCount: archive.groups.size, + }); + } + + res.send(archiveList); +}); + +app.get('/config', (req, res, next) => { + logger.info(`Request /config`); + res.status(200).contentType('json').send(store.archiveConfig); +}); + + +new ScriptExecutor().executeScript(serverArgumentOptions, async ( + { build, dir, port } +) => { + app.use((req, res, next) => { + res.header('Access-Control-Allow-Origin', '*'); + res.header('Access-Control-Allow-Headers', 'Origin, X-Requested-With, Content-Type, Accept'); + next(); + }); + + app.set('json spaces', 4); + app.set('port', port); + app.set('dir', dir); + app.set('build', build); + + app.listen(port, async () => { + const start = Date.now(); + + store = new Js5FileStore(build, dir); + await store.load(true, true, true); + + const end = Date.now(); + + logger.info(`File Store loaded in ${(end - start) / 1000} seconds.`); + logger.info('HTTP server online.'); + }); +}); + diff --git a/src/openrs2/openrs2.ts b/src/openrs2/openrs2.ts index 1c5639d..5c716c3 100644 --- a/src/openrs2/openrs2.ts +++ b/src/openrs2/openrs2.ts @@ -3,7 +3,7 @@ import AdmZip from 'adm-zip'; import { Buffer } from 'buffer'; import { logger } from '@runejs/common'; import { XteaConfig } from '@runejs/common/encrypt'; -import { CacheFile } from '../file-system/cache'; +import { PackedCacheFile } from '../file-system/packed'; const openRS2Endpoint = 'https://archive.openrs2.org'; @@ -97,7 +97,7 @@ export const getOpenRS2CacheDetailsByBuild = async ( export const getOpenRS2CacheFilesById = async ( id: number, scope: string = 'runescape' -): Promise => { +): Promise => { const response = await axios.get( `${ openRS2Endpoint }/caches/${ scope }/${ id }/disk.zip`, { responseType: 'arraybuffer' } @@ -117,7 +117,7 @@ export const getOpenRS2CacheFilesById = async ( export const getOpenRS2CacheFilesByBuild = async ( build: number -): Promise => { +): Promise => { logger.info(`Searching OpenRS2 for build ${ build }...`); const cacheList = (await getOpenRS2CacheList()) diff --git a/src/scripts/dev.ts b/src/scripts/dev.ts index f9953ef..a77ea07 100644 --- a/src/scripts/dev.ts +++ b/src/scripts/dev.ts @@ -1,50 +1,66 @@ +import { writeFileSync, existsSync, mkdirSync } from 'fs'; import { logger } from '@runejs/common'; -import { indexes, JagArchive, JagFileStore } from '../file-system/jag'; +import { JagFileStore } from '../file-system/jag'; import { JagInterfaceArchive } from '../file-system/jag/content/archives/interfaces/jag-interface-archive'; +import { join } from 'path'; +import { Js5FileStore } from '../file-system/js5'; -const dev = async () => { - const start = Date.now(); +const saveInterfaces = async (store: JagFileStore) => { + logger.info(`Decoding game interfaces...`); - const store = new JagFileStore(317); + const interfaceArchive = new JagInterfaceArchive(store); - logger.info(`Loading JAG store for build ${store.gameBuild}...`); + interfaceArchive.decodeAll(); - await store.load(); + logger.info(`${interfaceArchive.interfaces.size} interfaces decoded. Saving interface entities...`); - logger.info(`Loading index entities...`); + await interfaceArchive.saveAll(); +}; - const indexNames = Object.keys(indexes); - for (const indexName of indexNames) { - store.createIndex(indexes[indexName]); - } - await store.loadIndexEntities(); +const dumpInterfaceFile = (store: JagFileStore) => { + const archive = store.getCache('archives') + .getArchive('interface.jag'); - logger.info(`Loading index file entities...`); + if (!archive) { + throw new Error('interface.jag archive is not loaded!'); + } - for (const [ , indexFile ] of store.indexes) { - await indexFile.loadFileIndexes(); + const dataFile = archive.getFile('data'); + const binaryData = dataFile?.index?.data; + if (!binaryData) { + throw new Error('interface.jag data file is not loaded!'); } - logger.info(`Loading archive file entities...`); + const outputDir = join('.', 'unpacked', 'jag', 'interface.jag'); + if (!existsSync(outputDir)) { + mkdirSync(outputDir, { recursive: true }); + } - const archiveIndex = store.getIndex('archives'); + const outputFile = join(outputDir, 'data'); - for (const [ , file ] of archiveIndex.files) { - const archive = file as JagArchive; - await archive.loadFileIndexes(); - } + logger.info(`Writing file ${outputFile}`); - logger.info(`Decoding game interfaces...`); + writeFileSync(outputFile, binaryData); +}; - const interfaceArchive = new JagInterfaceArchive(store); - interfaceArchive.decodeAll(); +const dev = async () => { + const start = Date.now(); - logger.info(`${interfaceArchive.interfaces.size} interfaces decoded. Saving interface entities...`); + const store = new Js5FileStore(435); + await store.load(true, true, false); - await interfaceArchive.saveAll(); + const fileNames = [ + 'compass', + 'mapback' + ]; + + fileNames.forEach(name => { + const spriteFile = store.getArchive('sprites').getGroup(name); + store.js5.decompress(spriteFile); + }); const end = Date.now(); logger.info(`Operations completed in ${(end - start) / 1000} seconds.`); diff --git a/src/scripts/indexer.ts b/src/scripts/indexer.ts index b1531b9..474a619 100644 --- a/src/scripts/indexer.ts +++ b/src/scripts/indexer.ts @@ -3,12 +3,11 @@ import { existsSync, mkdirSync } from 'graceful-fs'; import { logger } from '@runejs/common'; import { ScriptExecutor, ArgumentOptions } from './script-executor'; import { Js5FileStore } from '../file-system/js5'; -import { indexes, JagArchive, JagFileStore } from '../file-system/jag'; +import { caches, JagArchive, JagFileStore } from '../file-system/jag'; import { getOpenRS2CacheFilesByBuild, } from '../openrs2'; -import { fileTarget, prettyPrintTarget } from '../../../common/src'; -import { CacheFile, getCacheFormat } from '../file-system/cache'; +import { PackedCacheFile, getPackedCacheFormat } from '../file-system/packed'; interface IndexerOptions { @@ -157,18 +156,18 @@ const indexJS5Archive = async (store: Js5FileStore, archiveName: string) => { const indexJagStore = async (store: JagFileStore) => { logger.info(`Decoding JAG store indexes...`); - const indexNames = Object.keys(indexes); + const indexNames = Object.keys(caches); for (const indexName of indexNames) { - store.jag.decodeIndex(indexName); + store.jag.decodeCache(indexName); } logger.info(`Saving indexes...`); - for (const [ , indexFile ] of store.indexes) { + for (const [ , indexFile ] of store.caches) { await indexFile.saveIndex().catch(e => logger.error(e)); } - for (const [, indexFile ] of store.indexes) { + for (const [, indexFile ] of store.caches) { logger.info(`Unpacking JAG files for index ${indexFile.index.name}...`); for (const [ , file ] of indexFile.files) { @@ -178,7 +177,7 @@ const indexJagStore = async (store: JagFileStore) => { logger.info(`Decoding JAG archives...`); - const archiveIndex = store.getIndex(indexes.archives); + const archiveIndex = store.getCache(caches.archives); for (const [ , archive ] of archiveIndex.files) { if (archive instanceof JagArchive) { @@ -189,7 +188,7 @@ const indexJagStore = async (store: JagFileStore) => { logger.info(`Saving JAG file indexes...`); - for (const [, index ] of store.indexes) { + for (const [, index ] of store.caches) { await index.upsertFileIndexes().catch(e => logger.error(e)); } @@ -214,7 +213,7 @@ const indexerScript = async ( const start = Date.now(); const logDir = join(dir, 'logs'); const numericBuildNumber: number = /^\d+$/.test(build) ? parseInt(build, 10) : -1; - let cacheFiles: CacheFile[] | 'local' = 'local'; + let cacheFiles: PackedCacheFile[] | 'local' = 'local'; if (!existsSync(logDir)) { mkdirSync(logDir, { recursive: true }); @@ -239,7 +238,7 @@ const indexerScript = async ( } } - const storeType = cacheFiles !== 'local' ? getCacheFormat(cacheFiles) : 'flat'; + const storeType = cacheFiles !== 'local' ? getPackedCacheFormat(cacheFiles) : 'flat'; logger.info(`Indexing ${ storeType === 'flat' ? storeType : storeType.toUpperCase() } file store...`); @@ -265,9 +264,9 @@ const indexerScript = async ( await store.load(); if (cacheFiles === 'local') { - store.jag.readLocalCacheFiles(); + store.jag.readLocalPackedCacheFiles(); } else { - store.jag.readOpenRS2CacheFiles(cacheFiles); + store.jag.readOpenRS2PackedCacheFiles(cacheFiles); } if (archiveName === 'main') { diff --git a/src/scripts/script-executor.ts b/src/scripts/script-executor.ts index c33aa20..8383de4 100644 --- a/src/scripts/script-executor.ts +++ b/src/scripts/script-executor.ts @@ -14,7 +14,7 @@ export class ScriptExecutor { public executeScript( argumentOptions: ArgumentOptions, - script: (args: T) => Promise + script: (args: T) => Promise ): void { (async function(args: T) { await script(args); From 072eeb9196c3aeb18523930eec6ff98cdb970a78 Mon Sep 17 00:00:00 2001 From: Kikorono Date: Wed, 24 Aug 2022 13:03:40 -0500 Subject: [PATCH 28/32] Updating to the latest /common 3.0.0 beta version and applying fixes Also adds the skeleton for JS5 game item file decoding --- package-lock.json | 331 +++++++++++------- package.json | 12 +- src/compress/bzip2.ts | 21 -- src/compress/gzip.ts | 12 - src/compress/index.ts | 2 - src/db/jag/jag-index-entity.ts | 4 +- src/db/js5/js5-index-entity.ts | 4 +- src/file-system/jag/jag.ts | 10 +- .../js5/content/config/js5-game-items.ts | 37 ++ src/file-system/js5/js5.ts | 151 ++++---- src/http/jag-routes.ts | 0 src/http/js5-routes.ts | 0 src/http/{js5-server.ts => server.ts} | 0 src/scripts/dev.ts | 7 +- src/scripts/indexer.ts | 40 +-- src/scripts/script-executor.ts | 2 - 16 files changed, 339 insertions(+), 294 deletions(-) delete mode 100644 src/compress/bzip2.ts delete mode 100644 src/compress/gzip.ts delete mode 100644 src/compress/index.ts create mode 100644 src/file-system/js5/content/config/js5-game-items.ts create mode 100644 src/http/jag-routes.ts create mode 100644 src/http/js5-routes.ts rename src/http/{js5-server.ts => server.ts} (100%) diff --git a/package-lock.json b/package-lock.json index 029d484..0ec4080 100644 --- a/package-lock.json +++ b/package-lock.json @@ -9,7 +9,7 @@ "version": "1.0.0-next.0", "license": "GPL-3.0", "dependencies": { - "@runejs/common": "2.0.2-beta.2", + "@runejs/common": "3.0.0-beta.8", "adm-zip": "^0.5.9", "axios": "^0.27.2", "compressjs": "^1.0.3", @@ -18,7 +18,7 @@ "json5": "^2.2.0", "reflect-metadata": "^0.1.13", "sqlite": "^4.0.25", - "tslib": ">=2.3.0", + "tslib": ">=2.4.0", "typeorm": "^0.2.44", "yargs": "^17.3.1", "zlib": "^1.0.5" @@ -38,12 +38,12 @@ "rimraf": "^3.0.2", "source-map-support": "^0.5.21", "ts-node-dev": "^2.0.0", - "typescript": "^4.5.5" + "typescript": "^4.7.4" }, "peerDependencies": { - "@runejs/common": "2.0.2-beta.2", + "@runejs/common": "3.0.0-beta.8", "graceful-fs": ">=4.2.0", - "tslib": ">=2.3.0", + "tslib": ">=2.4.0", "typescript": ">=4.5.0" } }, @@ -157,6 +157,19 @@ "@jridgewell/sourcemap-codec": "^1.4.10" } }, + "node_modules/@ledgerhq/compressjs": { + "version": "1.3.2", + "resolved": "https://registry.npmjs.org/@ledgerhq/compressjs/-/compressjs-1.3.2.tgz", + "integrity": "sha512-gonFwAifRkSYDO7rt3NIBlvjvY8Nw+NM6LT1SuOBppuvoKbYtBViNh3EBPbP86+3Y4ux7DLUsNiUlqOgubJsdA==", + "dependencies": { + "commander": "^2.20.0" + } + }, + "node_modules/@ledgerhq/compressjs/node_modules/commander": { + "version": "2.20.3", + "resolved": "https://registry.npmjs.org/commander/-/commander-2.20.3.tgz", + "integrity": "sha512-GpVkmM8vF2vQUkj2LvZmD35JxeJOLCwJ9cUkugyk2nuhbv3+mJvpLYYt+0+USMxE+oj+ey/lJEnhZw75x/OMcQ==" + }, "node_modules/@nodelib/fs.scandir": { "version": "2.1.5", "resolved": "https://registry.npmjs.org/@nodelib/fs.scandir/-/fs.scandir-2.1.5.tgz", @@ -193,20 +206,51 @@ } }, "node_modules/@runejs/common": { - "version": "2.0.2-beta.2", - "resolved": "https://registry.npmjs.org/@runejs/common/-/common-2.0.2-beta.2.tgz", - "integrity": "sha512-Xd9/Ws3ByAdI1R08Ye9fWqlS8SImjQw05Bsw0w46zcGyDx8N6jmeClf9Y/s4Akh9VZQuoAMFg2ANKsRK0MucNw==", + "version": "3.0.0-beta.8", + "resolved": "https://registry.npmjs.org/@runejs/common/-/common-3.0.0-beta.8.tgz", + "integrity": "sha512-S/SqQIcOdmvv3XEQmDR8jM5FMpP2RZP3ovOmqACSetRmjJIPObJXMI1lTA4Z5I0rylHM/5OiMiKIl8grCfZTGg==", "dependencies": { - "compressjs": "^1.0.3", - "js-yaml": "^3.14.1", + "@ledgerhq/compressjs": "^1.3.2", + "buffer": "^6.0.3", + "dotenv": "^16.0.0", + "path": "^0.12.7", "pino": "^6.14.0", - "pino-pretty": "^4.8.0", - "sonic-boom": "^2.6.0", - "tslib": "^2.3.1" + "pino-pretty": "^6.0.0", + "zlib": "^1.0.5" }, "peerDependencies": { - "tslib": ">=2.3.0", - "typescript": ">=4.5.0" + "tslib": ">=2.4.0" + } + }, + "node_modules/@runejs/common/node_modules/buffer": { + "version": "6.0.3", + "resolved": "https://registry.npmjs.org/buffer/-/buffer-6.0.3.tgz", + "integrity": "sha512-FTiCpNxtwiZZHEZbcbTIcZjERVICn9yq/pDFkTl95/AxzD1naBctN7YO68riM/gLSDY7sdrMby8hofADYuuqOA==", + "funding": [ + { + "type": "github", + "url": "https://github.com/sponsors/feross" + }, + { + "type": "patreon", + "url": "https://www.patreon.com/feross" + }, + { + "type": "consulting", + "url": "https://feross.org/support" + } + ], + "dependencies": { + "base64-js": "^1.3.1", + "ieee754": "^1.2.1" + } + }, + "node_modules/@runejs/common/node_modules/dotenv": { + "version": "16.0.1", + "resolved": "https://registry.npmjs.org/dotenv/-/dotenv-16.0.1.tgz", + "integrity": "sha512-1K6hR6wtk2FviQ4kEiSjFiH5rpzEVi8WW0x96aztHVMhEspNpc4DVOUTEHtEva5VThQ8IaBX1Pe4gSzpVVUsKQ==", + "engines": { + "node": ">=12" } }, "node_modules/@runejs/eslint-config": { @@ -693,14 +737,6 @@ "integrity": "sha512-58S9QDqG0Xx27YwPSt9fJxivjYl432YCwfDMfZ+71RAqUrZef7LrKQZ3LHLOwCS4FLNBplP533Zx895SeOCHvA==", "dev": true }, - "node_modules/argparse": { - "version": "1.0.10", - "resolved": "https://registry.npmjs.org/argparse/-/argparse-1.0.10.tgz", - "integrity": "sha512-o5Roy6tNG4SL/FOkCAN6RzjiakZS25RLYFrcMttJqbdd8BWrnA+fGz57iN5Pb06pvBGvl5gQ0B48dJlslXvoTg==", - "dependencies": { - "sprintf-js": "~1.0.2" - } - }, "node_modules/args": { "version": "5.0.3", "resolved": "https://registry.npmjs.org/args/-/args-5.0.3.tgz", @@ -1157,6 +1193,11 @@ "resolved": "https://registry.npmjs.org/color-name/-/color-name-1.1.4.tgz", "integrity": "sha512-dOy+3AuW3a2wNbZHIuMZpTcgjGuLU/uBL/ubcZF9OXbDo8ff4O8yVp5Bf0efS8uEoYo5q4Fx7dY9OgQGXgAsQA==" }, + "node_modules/colorette": { + "version": "1.4.0", + "resolved": "https://registry.npmjs.org/colorette/-/colorette-1.4.0.tgz", + "integrity": "sha512-Y2oEozpomLn7Q3HFP7dpww7AtMJplbM9lGZP6RDfHqmbeRjiwRg4n6VM6j4KLmRke85uWEI7JqF17f3pqdRA0g==" + }, "node_modules/combined-stream": { "version": "1.0.8", "resolved": "https://registry.npmjs.org/combined-stream/-/combined-stream-1.0.8.tgz", @@ -1651,18 +1692,6 @@ "url": "https://opencollective.com/eslint" } }, - "node_modules/esprima": { - "version": "4.0.1", - "resolved": "https://registry.npmjs.org/esprima/-/esprima-4.0.1.tgz", - "integrity": "sha512-eGuFFw7Upda+g4p+QHvnW0RyTX/SVeJBDM/gCtMARO0cLuT2HcEKnTPvhjV6aGeqrCB/sbNop0Kszm0jsaWU4A==", - "bin": { - "esparse": "bin/esparse.js", - "esvalidate": "bin/esvalidate.js" - }, - "engines": { - "node": ">=4" - } - }, "node_modules/esquery": { "version": "1.4.0", "resolved": "https://registry.npmjs.org/esquery/-/esquery-1.4.0.tgz", @@ -2383,23 +2412,11 @@ } }, "node_modules/joycon": { - "version": "2.2.5", - "resolved": "https://registry.npmjs.org/joycon/-/joycon-2.2.5.tgz", - "integrity": "sha512-YqvUxoOcVPnCp0VU1/56f+iKSdvIRJYPznH22BdXV3xMk75SFXhWeJkZ8C9XxUWt1b5x2X1SxuFygW1U0FmkEQ==", + "version": "3.1.1", + "resolved": "https://registry.npmjs.org/joycon/-/joycon-3.1.1.tgz", + "integrity": "sha512-34wB/Y7MW7bzjKRjUKTa46I2Z7eV62Rkhva+KkopW7Qvv/OSWBqvkSY7vusOPrNuZcUG3tApvdVgNB8POj3SPw==", "engines": { - "node": ">=6" - } - }, - "node_modules/js-yaml": { - "version": "3.14.1", - "resolved": "https://registry.npmjs.org/js-yaml/-/js-yaml-3.14.1.tgz", - "integrity": "sha512-okMH7OXXJ7YrN9Ok3/SXrnu4iX9yOk+25nqX4imS2npuvTYDmo/QEZoqwZkYaIDk3jVvBOTOIEgEhaLOynBS9g==", - "dependencies": { - "argparse": "^1.0.7", - "esprima": "^4.0.0" - }, - "bin": { - "js-yaml": "bin/js-yaml.js" + "node": ">=10" } }, "node_modules/json-schema-traverse": { @@ -2798,6 +2815,15 @@ "node": ">= 0.8" } }, + "node_modules/path": { + "version": "0.12.7", + "resolved": "https://registry.npmjs.org/path/-/path-0.12.7.tgz", + "integrity": "sha512-aXXC6s+1w7otVF9UletFkFcDsJeO7lSZBPUQhtb5O0xJe8LtYhj/GxldoL09bBj9+ZmE2hNoHqQSFMN5fikh4Q==", + "dependencies": { + "process": "^0.11.1", + "util": "^0.10.3" + } + }, "node_modules/path-exists": { "version": "4.0.0", "resolved": "https://registry.npmjs.org/path-exists/-/path-exists-4.0.0.tgz", @@ -2874,17 +2900,17 @@ } }, "node_modules/pino-pretty": { - "version": "4.8.0", - "resolved": "https://registry.npmjs.org/pino-pretty/-/pino-pretty-4.8.0.tgz", - "integrity": "sha512-mhQfHG4rw5ZFpWL44m0Utjo4GC2+HMfdNvxyA8lLw0sIqn6fCf7uQe6dPckUcW/obly+OQHD7B/MTso6LNizYw==", + "version": "6.0.0", + "resolved": "https://registry.npmjs.org/pino-pretty/-/pino-pretty-6.0.0.tgz", + "integrity": "sha512-jyeR2fXXWc68st1DTTM5NhkHlx8p+1fKZMfm84Jwq+jSw08IwAjNaZBZR6ts69hhPOfOjg/NiE1HYW7vBRPL3A==", "dependencies": { "@hapi/bourne": "^2.0.0", "args": "^5.0.1", - "chalk": "^4.0.0", + "colorette": "^1.3.0", "dateformat": "^4.5.1", "fast-safe-stringify": "^2.0.7", "jmespath": "^0.15.0", - "joycon": "^2.2.5", + "joycon": "^3.0.0", "pump": "^3.0.0", "readable-stream": "^3.6.0", "rfdc": "^1.3.0", @@ -2921,15 +2947,6 @@ "resolved": "https://registry.npmjs.org/pino-std-serializers/-/pino-std-serializers-3.2.0.tgz", "integrity": "sha512-EqX4pwDPrt3MuOAAUBMU0Tk5kR/YcCM5fNPEzgCO2zJ5HfX0vbiH9HbJglnyeQsN96Kznae6MWD47pZB5avTrg==" }, - "node_modules/pino/node_modules/sonic-boom": { - "version": "1.4.1", - "resolved": "https://registry.npmjs.org/sonic-boom/-/sonic-boom-1.4.1.tgz", - "integrity": "sha512-LRHh/A8tpW7ru89lrlkU4AszXt1dbwSjVWguGrmlxE7tawVmDBlI1PILMkXAxJTwqhgsEeTHzj36D5CmHgQmNg==", - "dependencies": { - "atomic-sleep": "^1.0.0", - "flatstr": "^1.0.12" - } - }, "node_modules/prebuild-install": { "version": "7.1.1", "resolved": "https://registry.npmjs.org/prebuild-install/-/prebuild-install-7.1.1.tgz", @@ -2965,6 +2982,14 @@ "node": ">= 0.8.0" } }, + "node_modules/process": { + "version": "0.11.10", + "resolved": "https://registry.npmjs.org/process/-/process-0.11.10.tgz", + "integrity": "sha512-cdGef/drWFoydD1JsMzuFf8100nZl+GT+yacc2bEced5f9Rjk4z+WtFUTBu9PhOi9j/jfmBPu0mMEY4wIdAF8A==", + "engines": { + "node": ">= 0.6.0" + } + }, "node_modules/process-nextick-args": { "version": "2.0.1", "resolved": "https://registry.npmjs.org/process-nextick-args/-/process-nextick-args-2.0.1.tgz", @@ -3424,11 +3449,12 @@ } }, "node_modules/sonic-boom": { - "version": "2.8.0", - "resolved": "https://registry.npmjs.org/sonic-boom/-/sonic-boom-2.8.0.tgz", - "integrity": "sha512-kuonw1YOYYNOve5iHdSahXPOK49GqwA+LZhI6Wz/l0rP57iKyXXIHaRagOBHAPmGwJC6od2Z9zgvZ5loSgMlVg==", + "version": "1.4.1", + "resolved": "https://registry.npmjs.org/sonic-boom/-/sonic-boom-1.4.1.tgz", + "integrity": "sha512-LRHh/A8tpW7ru89lrlkU4AszXt1dbwSjVWguGrmlxE7tawVmDBlI1PILMkXAxJTwqhgsEeTHzj36D5CmHgQmNg==", "dependencies": { - "atomic-sleep": "^1.0.0" + "atomic-sleep": "^1.0.0", + "flatstr": "^1.0.12" } }, "node_modules/source-map": { @@ -3479,11 +3505,6 @@ "safe-buffer": "~5.2.0" } }, - "node_modules/sprintf-js": { - "version": "1.0.3", - "resolved": "https://registry.npmjs.org/sprintf-js/-/sprintf-js-1.0.3.tgz", - "integrity": "sha512-D9cPgkvLlV3t3IzL0D0YLvGA9Ahk4PcvVwUbN0dSGr1aP0Nrt4AEnTUbuGvquEC0mA64Gqt1fzirlRs5ibXx8g==" - }, "node_modules/sqlite": { "version": "4.1.2", "resolved": "https://registry.npmjs.org/sqlite/-/sqlite-4.1.2.tgz", @@ -4042,6 +4063,7 @@ "version": "4.7.4", "resolved": "https://registry.npmjs.org/typescript/-/typescript-4.7.4.tgz", "integrity": "sha512-C0WQT0gezHuw6AdY1M2jxUO83Rjf0HP7Sk1DtXj6j1EwkQNZrHAg2XPWlq62oqEhYvONq5pkC2Y9oPljWToLmQ==", + "dev": true, "bin": { "tsc": "bin/tsc", "tsserver": "bin/tsserver" @@ -4076,11 +4098,24 @@ "punycode": "^2.1.0" } }, + "node_modules/util": { + "version": "0.10.4", + "resolved": "https://registry.npmjs.org/util/-/util-0.10.4.tgz", + "integrity": "sha512-0Pm9hTQ3se5ll1XihRic3FDIku70C+iHUdT/W926rSgHV5QgXsYbKZN8MSC3tJtSkhuROzvsQjAaFENRXr+19A==", + "dependencies": { + "inherits": "2.0.3" + } + }, "node_modules/util-deprecate": { "version": "1.0.2", "resolved": "https://registry.npmjs.org/util-deprecate/-/util-deprecate-1.0.2.tgz", "integrity": "sha512-EPD5q1uXyFxJpCrLnCc1nHnq3gOa6DZBocAIiI2TaSCA7VCJ1UJDMagCzIkXNsUYfD1daK//LTEQ8xiIbrHtcw==" }, + "node_modules/util/node_modules/inherits": { + "version": "2.0.3", + "resolved": "https://registry.npmjs.org/inherits/-/inherits-2.0.3.tgz", + "integrity": "sha512-x00IRNXNy63jwGkJmzPigoySHbaqpNuzKbBOmzK+g2OdZpQ9w+sxCN+VSB3ja7IAge2OP2qpfxTjeNcyjmW1uw==" + }, "node_modules/utils-merge": { "version": "1.0.1", "resolved": "https://registry.npmjs.org/utils-merge/-/utils-merge-1.0.1.tgz", @@ -4369,6 +4404,21 @@ "@jridgewell/sourcemap-codec": "^1.4.10" } }, + "@ledgerhq/compressjs": { + "version": "1.3.2", + "resolved": "https://registry.npmjs.org/@ledgerhq/compressjs/-/compressjs-1.3.2.tgz", + "integrity": "sha512-gonFwAifRkSYDO7rt3NIBlvjvY8Nw+NM6LT1SuOBppuvoKbYtBViNh3EBPbP86+3Y4ux7DLUsNiUlqOgubJsdA==", + "requires": { + "commander": "^2.20.0" + }, + "dependencies": { + "commander": { + "version": "2.20.3", + "resolved": "https://registry.npmjs.org/commander/-/commander-2.20.3.tgz", + "integrity": "sha512-GpVkmM8vF2vQUkj2LvZmD35JxeJOLCwJ9cUkugyk2nuhbv3+mJvpLYYt+0+USMxE+oj+ey/lJEnhZw75x/OMcQ==" + } + } + }, "@nodelib/fs.scandir": { "version": "2.1.5", "resolved": "https://registry.npmjs.org/@nodelib/fs.scandir/-/fs.scandir-2.1.5.tgz", @@ -4396,16 +4446,33 @@ } }, "@runejs/common": { - "version": "2.0.2-beta.2", - "resolved": "https://registry.npmjs.org/@runejs/common/-/common-2.0.2-beta.2.tgz", - "integrity": "sha512-Xd9/Ws3ByAdI1R08Ye9fWqlS8SImjQw05Bsw0w46zcGyDx8N6jmeClf9Y/s4Akh9VZQuoAMFg2ANKsRK0MucNw==", + "version": "3.0.0-beta.8", + "resolved": "https://registry.npmjs.org/@runejs/common/-/common-3.0.0-beta.8.tgz", + "integrity": "sha512-S/SqQIcOdmvv3XEQmDR8jM5FMpP2RZP3ovOmqACSetRmjJIPObJXMI1lTA4Z5I0rylHM/5OiMiKIl8grCfZTGg==", "requires": { - "compressjs": "^1.0.3", - "js-yaml": "^3.14.1", + "@ledgerhq/compressjs": "^1.3.2", + "buffer": "^6.0.3", + "dotenv": "^16.0.0", + "path": "^0.12.7", "pino": "^6.14.0", - "pino-pretty": "^4.8.0", - "sonic-boom": "^2.6.0", - "tslib": "^2.3.1" + "pino-pretty": "^6.0.0", + "zlib": "^1.0.5" + }, + "dependencies": { + "buffer": { + "version": "6.0.3", + "resolved": "https://registry.npmjs.org/buffer/-/buffer-6.0.3.tgz", + "integrity": "sha512-FTiCpNxtwiZZHEZbcbTIcZjERVICn9yq/pDFkTl95/AxzD1naBctN7YO68riM/gLSDY7sdrMby8hofADYuuqOA==", + "requires": { + "base64-js": "^1.3.1", + "ieee754": "^1.2.1" + } + }, + "dotenv": { + "version": "16.0.1", + "resolved": "https://registry.npmjs.org/dotenv/-/dotenv-16.0.1.tgz", + "integrity": "sha512-1K6hR6wtk2FviQ4kEiSjFiH5rpzEVi8WW0x96aztHVMhEspNpc4DVOUTEHtEva5VThQ8IaBX1Pe4gSzpVVUsKQ==" + } } }, "@runejs/eslint-config": { @@ -4760,14 +4827,6 @@ "integrity": "sha512-58S9QDqG0Xx27YwPSt9fJxivjYl432YCwfDMfZ+71RAqUrZef7LrKQZ3LHLOwCS4FLNBplP533Zx895SeOCHvA==", "dev": true }, - "argparse": { - "version": "1.0.10", - "resolved": "https://registry.npmjs.org/argparse/-/argparse-1.0.10.tgz", - "integrity": "sha512-o5Roy6tNG4SL/FOkCAN6RzjiakZS25RLYFrcMttJqbdd8BWrnA+fGz57iN5Pb06pvBGvl5gQ0B48dJlslXvoTg==", - "requires": { - "sprintf-js": "~1.0.2" - } - }, "args": { "version": "5.0.3", "resolved": "https://registry.npmjs.org/args/-/args-5.0.3.tgz", @@ -5120,6 +5179,11 @@ "resolved": "https://registry.npmjs.org/color-name/-/color-name-1.1.4.tgz", "integrity": "sha512-dOy+3AuW3a2wNbZHIuMZpTcgjGuLU/uBL/ubcZF9OXbDo8ff4O8yVp5Bf0efS8uEoYo5q4Fx7dY9OgQGXgAsQA==" }, + "colorette": { + "version": "1.4.0", + "resolved": "https://registry.npmjs.org/colorette/-/colorette-1.4.0.tgz", + "integrity": "sha512-Y2oEozpomLn7Q3HFP7dpww7AtMJplbM9lGZP6RDfHqmbeRjiwRg4n6VM6j4KLmRke85uWEI7JqF17f3pqdRA0g==" + }, "combined-stream": { "version": "1.0.8", "resolved": "https://registry.npmjs.org/combined-stream/-/combined-stream-1.0.8.tgz", @@ -5490,11 +5554,6 @@ "eslint-visitor-keys": "^3.3.0" } }, - "esprima": { - "version": "4.0.1", - "resolved": "https://registry.npmjs.org/esprima/-/esprima-4.0.1.tgz", - "integrity": "sha512-eGuFFw7Upda+g4p+QHvnW0RyTX/SVeJBDM/gCtMARO0cLuT2HcEKnTPvhjV6aGeqrCB/sbNop0Kszm0jsaWU4A==" - }, "esquery": { "version": "1.4.0", "resolved": "https://registry.npmjs.org/esquery/-/esquery-1.4.0.tgz", @@ -6043,18 +6102,9 @@ "integrity": "sha512-+kHj8HXArPfpPEKGLZ+kB5ONRTCiGQXo8RQYL0hH8t6pWXUBBK5KkkQmTNOwKK4LEsd0yTsgtjJVm4UBSZea4w==" }, "joycon": { - "version": "2.2.5", - "resolved": "https://registry.npmjs.org/joycon/-/joycon-2.2.5.tgz", - "integrity": "sha512-YqvUxoOcVPnCp0VU1/56f+iKSdvIRJYPznH22BdXV3xMk75SFXhWeJkZ8C9XxUWt1b5x2X1SxuFygW1U0FmkEQ==" - }, - "js-yaml": { - "version": "3.14.1", - "resolved": "https://registry.npmjs.org/js-yaml/-/js-yaml-3.14.1.tgz", - "integrity": "sha512-okMH7OXXJ7YrN9Ok3/SXrnu4iX9yOk+25nqX4imS2npuvTYDmo/QEZoqwZkYaIDk3jVvBOTOIEgEhaLOynBS9g==", - "requires": { - "argparse": "^1.0.7", - "esprima": "^4.0.0" - } + "version": "3.1.1", + "resolved": "https://registry.npmjs.org/joycon/-/joycon-3.1.1.tgz", + "integrity": "sha512-34wB/Y7MW7bzjKRjUKTa46I2Z7eV62Rkhva+KkopW7Qvv/OSWBqvkSY7vusOPrNuZcUG3tApvdVgNB8POj3SPw==" }, "json-schema-traverse": { "version": "0.4.1", @@ -6352,6 +6402,15 @@ "resolved": "https://registry.npmjs.org/parseurl/-/parseurl-1.3.3.tgz", "integrity": "sha512-CiyeOxFT/JZyN5m0z9PfXw4SCBJ6Sygz1Dpl0wqjlhDEGGBP1GnsUVEL0p63hoG1fcj3fHynXi9NYO4nWOL+qQ==" }, + "path": { + "version": "0.12.7", + "resolved": "https://registry.npmjs.org/path/-/path-0.12.7.tgz", + "integrity": "sha512-aXXC6s+1w7otVF9UletFkFcDsJeO7lSZBPUQhtb5O0xJe8LtYhj/GxldoL09bBj9+ZmE2hNoHqQSFMN5fikh4Q==", + "requires": { + "process": "^0.11.1", + "util": "^0.10.3" + } + }, "path-exists": { "version": "4.0.0", "resolved": "https://registry.npmjs.org/path-exists/-/path-exists-4.0.0.tgz", @@ -6404,31 +6463,20 @@ "process-warning": "^1.0.0", "quick-format-unescaped": "^4.0.3", "sonic-boom": "^1.0.2" - }, - "dependencies": { - "sonic-boom": { - "version": "1.4.1", - "resolved": "https://registry.npmjs.org/sonic-boom/-/sonic-boom-1.4.1.tgz", - "integrity": "sha512-LRHh/A8tpW7ru89lrlkU4AszXt1dbwSjVWguGrmlxE7tawVmDBlI1PILMkXAxJTwqhgsEeTHzj36D5CmHgQmNg==", - "requires": { - "atomic-sleep": "^1.0.0", - "flatstr": "^1.0.12" - } - } } }, "pino-pretty": { - "version": "4.8.0", - "resolved": "https://registry.npmjs.org/pino-pretty/-/pino-pretty-4.8.0.tgz", - "integrity": "sha512-mhQfHG4rw5ZFpWL44m0Utjo4GC2+HMfdNvxyA8lLw0sIqn6fCf7uQe6dPckUcW/obly+OQHD7B/MTso6LNizYw==", + "version": "6.0.0", + "resolved": "https://registry.npmjs.org/pino-pretty/-/pino-pretty-6.0.0.tgz", + "integrity": "sha512-jyeR2fXXWc68st1DTTM5NhkHlx8p+1fKZMfm84Jwq+jSw08IwAjNaZBZR6ts69hhPOfOjg/NiE1HYW7vBRPL3A==", "requires": { "@hapi/bourne": "^2.0.0", "args": "^5.0.1", - "chalk": "^4.0.0", + "colorette": "^1.3.0", "dateformat": "^4.5.1", "fast-safe-stringify": "^2.0.7", "jmespath": "^0.15.0", - "joycon": "^2.2.5", + "joycon": "^3.0.0", "pump": "^3.0.0", "readable-stream": "^3.6.0", "rfdc": "^1.3.0", @@ -6487,6 +6535,11 @@ "integrity": "sha512-vkcDPrRZo1QZLbn5RLGPpg/WmIQ65qoWWhcGKf/b5eplkkarX0m9z8ppCat4mlOqUsWpyNuYgO3VRyrYHSzX5g==", "dev": true }, + "process": { + "version": "0.11.10", + "resolved": "https://registry.npmjs.org/process/-/process-0.11.10.tgz", + "integrity": "sha512-cdGef/drWFoydD1JsMzuFf8100nZl+GT+yacc2bEced5f9Rjk4z+WtFUTBu9PhOi9j/jfmBPu0mMEY4wIdAF8A==" + }, "process-nextick-args": { "version": "2.0.1", "resolved": "https://registry.npmjs.org/process-nextick-args/-/process-nextick-args-2.0.1.tgz", @@ -6800,11 +6853,12 @@ "dev": true }, "sonic-boom": { - "version": "2.8.0", - "resolved": "https://registry.npmjs.org/sonic-boom/-/sonic-boom-2.8.0.tgz", - "integrity": "sha512-kuonw1YOYYNOve5iHdSahXPOK49GqwA+LZhI6Wz/l0rP57iKyXXIHaRagOBHAPmGwJC6od2Z9zgvZ5loSgMlVg==", + "version": "1.4.1", + "resolved": "https://registry.npmjs.org/sonic-boom/-/sonic-boom-1.4.1.tgz", + "integrity": "sha512-LRHh/A8tpW7ru89lrlkU4AszXt1dbwSjVWguGrmlxE7tawVmDBlI1PILMkXAxJTwqhgsEeTHzj36D5CmHgQmNg==", "requires": { - "atomic-sleep": "^1.0.0" + "atomic-sleep": "^1.0.0", + "flatstr": "^1.0.12" } }, "source-map": { @@ -6851,11 +6905,6 @@ } } }, - "sprintf-js": { - "version": "1.0.3", - "resolved": "https://registry.npmjs.org/sprintf-js/-/sprintf-js-1.0.3.tgz", - "integrity": "sha512-D9cPgkvLlV3t3IzL0D0YLvGA9Ahk4PcvVwUbN0dSGr1aP0Nrt4AEnTUbuGvquEC0mA64Gqt1fzirlRs5ibXx8g==" - }, "sqlite": { "version": "4.1.2", "resolved": "https://registry.npmjs.org/sqlite/-/sqlite-4.1.2.tgz", @@ -7228,7 +7277,8 @@ "typescript": { "version": "4.7.4", "resolved": "https://registry.npmjs.org/typescript/-/typescript-4.7.4.tgz", - "integrity": "sha512-C0WQT0gezHuw6AdY1M2jxUO83Rjf0HP7Sk1DtXj6j1EwkQNZrHAg2XPWlq62oqEhYvONq5pkC2Y9oPljWToLmQ==" + "integrity": "sha512-C0WQT0gezHuw6AdY1M2jxUO83Rjf0HP7Sk1DtXj6j1EwkQNZrHAg2XPWlq62oqEhYvONq5pkC2Y9oPljWToLmQ==", + "dev": true }, "unpipe": { "version": "1.0.0", @@ -7250,6 +7300,21 @@ "punycode": "^2.1.0" } }, + "util": { + "version": "0.10.4", + "resolved": "https://registry.npmjs.org/util/-/util-0.10.4.tgz", + "integrity": "sha512-0Pm9hTQ3se5ll1XihRic3FDIku70C+iHUdT/W926rSgHV5QgXsYbKZN8MSC3tJtSkhuROzvsQjAaFENRXr+19A==", + "requires": { + "inherits": "2.0.3" + }, + "dependencies": { + "inherits": { + "version": "2.0.3", + "resolved": "https://registry.npmjs.org/inherits/-/inherits-2.0.3.tgz", + "integrity": "sha512-x00IRNXNy63jwGkJmzPigoySHbaqpNuzKbBOmzK+g2OdZpQ9w+sxCN+VSB3ja7IAge2OP2qpfxTjeNcyjmW1uw==" + } + } + }, "util-deprecate": { "version": "1.0.2", "resolved": "https://registry.npmjs.org/util-deprecate/-/util-deprecate-1.0.2.tgz", diff --git a/package.json b/package.json index b21daae..b52de7f 100644 --- a/package.json +++ b/package.json @@ -18,7 +18,7 @@ "start": "ts-node-dev src/scripts/dev.ts", "lint": "eslint --ext .ts src", "lint:fix": "eslint --ext .ts src --fix", - "http:js5": "ts-node-dev --max-old-space-size=2048 src/http/js5-server.ts", + "http:js5": "ts-node-dev --max-old-space-size=2048 src/http/server.ts", "namehash": "ts-node-dev --max-old-space-size=2048 src/scripts/name-hasher.ts", "unpack": "ts-node-dev --max-old-space-size=2048 src/scripts/unpacker.ts", "index": "ts-node-dev --max-old-space-size=2048 src/scripts/indexer.ts", @@ -49,13 +49,13 @@ }, "homepage": "https://github.com/runejs/filestore#readme", "peerDependencies": { - "@runejs/common": "2.0.2-beta.2", + "@runejs/common": "3.0.0-beta.8", "graceful-fs": ">=4.2.0", - "tslib": ">=2.3.0", + "tslib": ">=2.4.0", "typescript": ">=4.5.0" }, "dependencies": { - "@runejs/common": "2.0.2-beta.2", + "@runejs/common": "3.0.0-beta.8", "adm-zip": "^0.5.9", "axios": "^0.27.2", "compressjs": "^1.0.3", @@ -64,7 +64,7 @@ "json5": "^2.2.0", "reflect-metadata": "^0.1.13", "sqlite": "^4.0.25", - "tslib": ">=2.3.0", + "tslib": ">=2.4.0", "typeorm": "^0.2.44", "yargs": "^17.3.1", "zlib": "^1.0.5" @@ -84,6 +84,6 @@ "rimraf": "^3.0.2", "source-map-support": "^0.5.21", "ts-node-dev": "^2.0.0", - "typescript": "^4.5.5" + "typescript": "^4.7.4" } } diff --git a/src/compress/bzip2.ts b/src/compress/bzip2.ts deleted file mode 100644 index 2cffd8c..0000000 --- a/src/compress/bzip2.ts +++ /dev/null @@ -1,21 +0,0 @@ -import { Buffer } from 'buffer'; -import * as compressjs from 'compressjs'; -import { ByteBuffer } from '@runejs/common'; -const bzip2 = compressjs.Bzip2; - - -export const compressHeadlessBzip2 = (data: Buffer | ByteBuffer): Buffer => { - const compressedData = bzip2.compressFile(data, undefined, 1); - return Buffer.from(compressedData.slice(4, compressedData.length)); -}; - -export const decompressHeadlessBzip2 = (data: Buffer | ByteBuffer): Buffer => { - const buffer = Buffer.alloc(data.length + 4); - data.copy(buffer, 4); - buffer[0] = 'B'.charCodeAt(0); - buffer[1] = 'Z'.charCodeAt(0); - buffer[2] = 'h'.charCodeAt(0); - buffer[3] = '1'.charCodeAt(0); - - return Buffer.from(bzip2.decompressFile(buffer)); -}; diff --git a/src/compress/gzip.ts b/src/compress/gzip.ts deleted file mode 100644 index 95be7c3..0000000 --- a/src/compress/gzip.ts +++ /dev/null @@ -1,12 +0,0 @@ -import { Buffer } from 'buffer'; -import { gunzipSync, gzipSync } from 'zlib'; -import { ByteBuffer } from '@runejs/common'; - - -export const compressGzip = (data: Buffer | ByteBuffer): Buffer => { - return Buffer.from(gzipSync(data)); -}; - -export const decompressGzip = (data: Buffer | ByteBuffer): Buffer => { - return Buffer.from(gunzipSync(data)); -}; diff --git a/src/compress/index.ts b/src/compress/index.ts deleted file mode 100644 index f626245..0000000 --- a/src/compress/index.ts +++ /dev/null @@ -1,2 +0,0 @@ -export * from './bzip2'; -export * from './gzip'; diff --git a/src/db/jag/jag-index-entity.ts b/src/db/jag/jag-index-entity.ts index 71f6ce6..6477b66 100644 --- a/src/db/jag/jag-index-entity.ts +++ b/src/db/jag/jag-index-entity.ts @@ -49,8 +49,8 @@ export class JagIndexEntity { @Column('blob', { name: 'data', nullable: true, default: null }) data: Buffer = null; - @Column('text', { name: 'compression_method', nullable: true, default: null }) - compressionMethod: CompressionMethod = null; + @Column('text', { name: 'compression_method', nullable: true, default: 'none' }) + compressionMethod: CompressionMethod = 'none'; @Column('integer', { name: 'compressed_checksum', nullable: false, default: -1 }) compressedChecksum: number = -1; diff --git a/src/db/js5/js5-index-entity.ts b/src/db/js5/js5-index-entity.ts index 85e0deb..efb98fd 100644 --- a/src/db/js5/js5-index-entity.ts +++ b/src/db/js5/js5-index-entity.ts @@ -52,8 +52,8 @@ export class Js5IndexEntity { @Column('blob', { name: 'data', nullable: true, default: null }) data: Buffer = null; - @Column('text', { name: 'compression_method', nullable: true, default: null }) - compressionMethod: CompressionMethod = null; + @Column('text', { name: 'compression_method', nullable: true, default: 'none' }) + compressionMethod: CompressionMethod = 'none'; @Column('integer', { name: 'compressed_checksum', nullable: false, default: -1 }) compressedChecksum: number = -1; diff --git a/src/file-system/jag/jag.ts b/src/file-system/jag/jag.ts index 66abf28..286bca1 100644 --- a/src/file-system/jag/jag.ts +++ b/src/file-system/jag/jag.ts @@ -5,9 +5,9 @@ import { JagFileStore } from './jag-file-store'; import { JagFile } from './jag-file'; import { Buffer } from 'buffer'; import { JagArchive } from './jag-archive'; -import { decompressHeadlessBzip2 } from '../../compress'; import { JagFileBase } from './jag-file-base'; import { PackedCacheFile } from '../packed'; +import { Bzip2 } from '@runejs/common/compress'; const dataFileName = 'main_file_cache.dat'; @@ -271,8 +271,8 @@ export class Jag { if (uncompressed !== compressed) { const compressedData = archiveData.getSlice(archiveData.readerIndex, archiveData.length - archiveData.readerIndex); - archiveData = new ByteBuffer(decompressHeadlessBzip2(compressedData)); - archive.index.compressionMethod = 'bzip'; + archiveData = new ByteBuffer(Bzip2.decompress(compressedData)); + archive.index.compressionMethod = 'bzip2'; } else { archive.index.compressionMethod = 'none'; } @@ -319,8 +319,8 @@ export class Jag { try { const { compressedFileSize, fileSize, compressedData } = file.index; if (compressedData?.length && compressedFileSize !== fileSize) { - file.index.data = decompressHeadlessBzip2(file.index.compressedData); - file.index.compressionMethod = 'bzip'; + file.index.data = Bzip2.decompress(file.index.compressedData); + file.index.compressionMethod = 'bzip2'; } else { file.index.data = file.index.compressedData; file.index.compressedData = null; diff --git a/src/file-system/js5/content/config/js5-game-items.ts b/src/file-system/js5/content/config/js5-game-items.ts new file mode 100644 index 0000000..7d658d4 --- /dev/null +++ b/src/file-system/js5/content/config/js5-game-items.ts @@ -0,0 +1,37 @@ +import { Js5FileStore } from '../../js5-file-store'; +import { ByteBuffer } from '@runejs/common'; + + +export class Js5GameItem { + id: number; +} + +export class Js5GameItems { + + readonly js5Store: Js5FileStore; + readonly items: Map; + + constructor(js5Store: Js5FileStore) { + this.js5Store = js5Store; + this.items = new Map(); + } + + decode(id: number, data: ByteBuffer): Js5GameItem { + const item = new Js5GameItem(); + item.id = id; + + while (true) { + const opcode = data.get('byte', 'unsigned'); + if (opcode === 0) { + break; + } + } + + return item; + } + + decodeAll(): void { + + } + +} diff --git a/src/file-system/js5/js5.ts b/src/file-system/js5/js5.ts index dc59868..8779f42 100644 --- a/src/file-system/js5/js5.ts +++ b/src/file-system/js5/js5.ts @@ -3,18 +3,12 @@ import { Buffer } from 'buffer'; import { existsSync, readdirSync, readFileSync, statSync } from 'graceful-fs'; import { ByteBuffer, logger } from '@runejs/common'; -import { getCompressionMethod } from '@runejs/common/compress'; +import { getCompressionMethod, Gzip, Bzip2 } from '@runejs/common/compress'; import { Xtea, XteaKeys, XteaConfig } from '@runejs/common/encrypt'; import { Js5FileStore, Js5Archive, Js5Group, Js5File } from '.'; import { archiveFlags, ArchiveFormat } from '../../config'; import { getXteaKeysByBuild } from '../../openrs2'; -import { - compressHeadlessBzip2, - decompressHeadlessBzip2, - compressGzip, - decompressGzip -} from '../../compress'; import { PackedCacheFile } from '../packed'; @@ -297,40 +291,41 @@ export class JS5 { } } - const { compressedLength, readerIndex } = this.readCompressedFileHeader(file); - const encryptedData = new ByteBuffer(fileDetails.compressedData); - const keySet = keySets.find(keySet => keySet.gameBuild === gameBuild); + try { + const { compressedLength, readerIndex } = this.readCompressedFileHeader(file); + const encryptedData = new ByteBuffer(fileDetails.compressedData); + const keySet = keySets.find(keySet => keySet.gameBuild === gameBuild); - if (Xtea.validKeys(keySet?.key)) { - logger.info(`XTEA decryption keys found for file ` + - `${ fileName || fileDetails.key }.`); - - const dataCopy = encryptedData.clone(); - dataCopy.readerIndex = readerIndex; + if (Xtea.validKeys(keySet?.key)) { + const dataCopy = encryptedData.clone(); + dataCopy.readerIndex = readerIndex; - let lengthOffset = readerIndex; - if (dataCopy.length - (compressedLength + readerIndex + 4) >= 2) { - lengthOffset += 2; - } + let lengthOffset = readerIndex; + if (dataCopy.length - (compressedLength + readerIndex + 4) >= 2) { + lengthOffset += 2; + } - const decryptedData = Xtea.decrypt(dataCopy, keySet.key, dataCopy.length - lengthOffset); + const decryptedData = Xtea.decrypt(dataCopy, keySet.key, dataCopy.length - lengthOffset); - if (decryptedData?.length) { - decryptedData.copy(dataCopy, readerIndex, 0); - dataCopy.readerIndex = readerIndex; - fileDetails.compressedData = dataCopy.toNodeBuffer(); - fileDetails.encrypted = false; - file.validate(false); - return fileDetails.compressedData; + if (decryptedData?.length) { + decryptedData.copy(dataCopy, readerIndex, 0); + dataCopy.readerIndex = readerIndex; + fileDetails.compressedData = dataCopy.toNodeBuffer(); + fileDetails.encrypted = false; + file.validate(false); + return fileDetails.compressedData; + } else { + logger.warn(`Invalid XTEA decryption keys found for file ` + + `${ fileName || fileDetails.key } using game build ${ gameBuild }.`); + fileDetails.fileError = 'MISSING_ENCRYPTION_KEYS'; + } } else { - logger.warn(`Invalid XTEA decryption keys found for file ` + + logger.warn(`No XTEA decryption keys found for file ` + `${ fileName || fileDetails.key } using game build ${ gameBuild }.`); fileDetails.fileError = 'MISSING_ENCRYPTION_KEYS'; } - } else { - logger.warn(`No XTEA decryption keys found for file ` + - `${ fileName || fileDetails.key } using game build ${ gameBuild }.`); - fileDetails.fileError = 'MISSING_ENCRYPTION_KEYS'; + } catch (err) { + logger.error(err); } return null; @@ -348,44 +343,39 @@ export class JS5 { return null; } - const { compressedLength, readerIndex } = this.readCompressedFileHeader(file); - - // JS5.decrypt will set compressedData to the new decrypted data after completion - const compressedData = new ByteBuffer(fileDetails.compressedData); - compressedData.readerIndex = readerIndex; let data: Buffer; - if (fileDetails.compressionMethod === 'none') { - // Uncompressed file - data = Buffer.alloc(compressedLength); - compressedData.copy(data, 0, compressedData.readerIndex, compressedLength); - compressedData.readerIndex = (compressedData.readerIndex + compressedLength); - } else { - // BZIP or GZIP compressed file - const decompressedLength = compressedData.get('int', 'unsigned'); - if (decompressedLength < 0) { - const errorPrefix = `Unable to decompress file ${ fileDetails.name || fileDetails.key }:`; - if (fileDetails.fileError === 'FILE_MISSING') { - logger.error(`${ errorPrefix } Missing file data.`); - } else { - logger.error(`${ errorPrefix } Missing or invalid XTEA key.`); - fileDetails.fileError = 'MISSING_ENCRYPTION_KEYS'; - } - } else { - const fileData = new ByteBuffer(compressedLength); + try { + const { compressedLength, readerIndex } = this.readCompressedFileHeader(file); - logger.info(`Decompress ${fileDetails.compressionMethod}, ` + - `compressed len = ${fileDetails.compressedData.length}, ` + - `recorded compressed len = ${compressedLength}, ` + - `compressed data buffer len = ${fileData.length}, ` + - `decompressed len = ${decompressedLength}`); + // JS5.decrypt will set compressedData to the new decrypted data after completion + const compressedData = new ByteBuffer(fileDetails.compressedData); + compressedData.readerIndex = readerIndex; - compressedData.copy(fileData, 0, compressedData.readerIndex); + if (fileDetails.compressionMethod === 'none') { + // Uncompressed file + data = Buffer.alloc(compressedLength); + compressedData.copy(data, 0, compressedData.readerIndex, compressedLength); + compressedData.readerIndex = (compressedData.readerIndex + compressedLength); + } else { + // BZIP or GZIP compressed file + const decompressedLength = compressedData.get('int', 'unsigned'); + if (decompressedLength < 0) { + const errorPrefix = `Unable to decompress file ${ fileDetails.name || fileDetails.key }:`; + if (fileDetails.fileError === 'FILE_MISSING') { + logger.error(`${ errorPrefix } Missing file data.`); + } else { + logger.error(`${ errorPrefix } Missing or invalid XTEA key.`); + fileDetails.fileError = 'MISSING_ENCRYPTION_KEYS'; + } + } else { + const fileData = new ByteBuffer(compressedLength); + + compressedData.copy(fileData, 0, compressedData.readerIndex); - try { - data = fileDetails.compressionMethod === 'bzip' ? - decompressHeadlessBzip2(fileData) : - decompressGzip(fileData); + data = fileDetails.compressionMethod === 'bzip2' ? + Bzip2.decompress(fileData) : + Gzip.decompress(fileData); compressedData.readerIndex = compressedData.readerIndex + compressedLength; @@ -393,23 +383,22 @@ export class JS5 { logger.error(`Compression length mismatch.`); data = null; } - } catch (error) { - logger.error(`Error decompressing file ${ fileDetails.name || fileDetails.key }: ` + - `${ error?.message ?? error }`); - data = null; } } - } - if (data?.length) { - // Read the file footer, if it has one - if (compressedData.readable >= 2) { - fileDetails.version = compressedData.get('short', 'unsigned'); + if (data?.length) { + // Read the file footer, if it has one + if (compressedData.readable >= 2) { + fileDetails.version = compressedData.get('short', 'unsigned'); + } } - - fileDetails.data = data; + } catch (error) { + logger.error(`Error decompressing file ${ fileDetails.name || fileDetails.key }: ` + + `${ error?.message ?? error }`); + data = null; } + fileDetails.data = data; file.validate(false); return fileDetails.data; } @@ -666,16 +655,16 @@ export class JS5 { } else { // compressed Bzip2 or Gzip file - const compressedData: Buffer = fileDetails.compressionMethod === 'bzip' ? - compressHeadlessBzip2(decompressedData) : - compressGzip(decompressedData); + const compressedData: Buffer = fileDetails.compressionMethod === 'bzip2' ? + Bzip2.compress(decompressedData) : + Gzip.compress(decompressedData); const compressedLength: number = compressedData.length; data = new ByteBuffer(compressedData.length + 9); // indicate which type of file compression was used (1 or 2) - data.put(fileDetails.compressionMethod === 'bzip' ? 1 : 2); + data.put(fileDetails.compressionMethod === 'bzip2' ? 1 : 2); // write the compressed file length data.put(compressedLength, 'int'); diff --git a/src/http/jag-routes.ts b/src/http/jag-routes.ts new file mode 100644 index 0000000..e69de29 diff --git a/src/http/js5-routes.ts b/src/http/js5-routes.ts new file mode 100644 index 0000000..e69de29 diff --git a/src/http/js5-server.ts b/src/http/server.ts similarity index 100% rename from src/http/js5-server.ts rename to src/http/server.ts diff --git a/src/scripts/dev.ts b/src/scripts/dev.ts index a77ea07..3e14933 100644 --- a/src/scripts/dev.ts +++ b/src/scripts/dev.ts @@ -1,11 +1,16 @@ import { writeFileSync, existsSync, mkdirSync } from 'fs'; -import { logger } from '@runejs/common'; +import { logger, prettyPrintTarget } from '@runejs/common'; import { JagFileStore } from '../file-system/jag'; import { JagInterfaceArchive } from '../file-system/jag/content/archives/interfaces/jag-interface-archive'; import { join } from 'path'; import { Js5FileStore } from '../file-system/js5'; +logger.setTargets([ + prettyPrintTarget() +]); + + const saveInterfaces = async (store: JagFileStore) => { logger.info(`Decoding game interfaces...`); diff --git a/src/scripts/indexer.ts b/src/scripts/indexer.ts index 474a619..f5640de 100644 --- a/src/scripts/indexer.ts +++ b/src/scripts/indexer.ts @@ -1,5 +1,3 @@ -import { join } from 'path'; -import { existsSync, mkdirSync } from 'graceful-fs'; import { logger } from '@runejs/common'; import { ScriptExecutor, ArgumentOptions } from './script-executor'; import { Js5FileStore } from '../file-system/js5'; @@ -64,13 +62,13 @@ const indexJS5Store = async (store: Js5FileStore) => { logger.info(`Decoding JS5 archives...`); for (const [ , archive ] of store.archives) { - await store.js5.decodeArchive(archive).catch(e => logger.error(e)); + await store.js5.decodeArchive(archive); } logger.info(`Saving archive indexes...`); for (const [ , archive ] of store.archives) { - await archive.saveIndex().catch(e => logger.error(e)); + await archive.saveIndex(); } logger.info(`Unpacking groups from JS5 store...`); @@ -87,7 +85,7 @@ const indexJS5Store = async (store: Js5FileStore) => { for (const [ , archive ] of store.archives) { for (const [ , group ] of archive.groups) { - await store.js5.decodeGroup(group).catch(e => logger.error(e)); + await store.js5.decodeGroup(group); } logger.info(`Finished decoding archive ${ archive.index.name } groups.`); @@ -96,14 +94,14 @@ const indexJS5Store = async (store: Js5FileStore) => { logger.info(`Saving group indexes...`); for (const [ , archive ] of store.archives) { - await archive.upsertGroupIndexes().catch(e => logger.error(e)); + await archive.upsertGroupIndexes(); } logger.info(`Saving flat file indexes...`); for (const [ , archive ] of store.archives) { for (const [ , group ] of archive.groups) { - await group.upsertFileIndexes().catch(e => logger.error(e)); + await group.upsertFileIndexes(); } } }; @@ -123,11 +121,11 @@ const indexJS5Archive = async (store: Js5FileStore, archiveName: string) => { logger.info(`Decoding archive ${ archiveName }...`); - await store.js5.decodeArchive(archive).catch(e => logger.error(e)); + await store.js5.decodeArchive(archive); logger.info(`Saving archive ${ archiveName } index...`); - await archive.saveIndex().catch(e => logger.error(e)); + await archive.saveIndex(); logger.info(`Unpacking groups from archive ${ archiveName }...`); @@ -138,17 +136,17 @@ const indexJS5Archive = async (store: Js5FileStore, archiveName: string) => { logger.info(`Decoding archive ${ archiveName } groups...`); for (const [ , group ] of archive.groups) { - await store.js5.decodeGroup(group).catch(e => logger.error(e)); + await store.js5.decodeGroup(group); } logger.info(`Saving group indexes...`); - await archive.upsertGroupIndexes().catch(e => logger.error(e)); + await archive.upsertGroupIndexes(); logger.info(`Saving flat file indexes...`); for (const [ , group ] of archive.groups) { - await group.upsertFileIndexes().catch(e => logger.error(e)); + await group.upsertFileIndexes(); } }; @@ -164,7 +162,7 @@ const indexJagStore = async (store: JagFileStore) => { logger.info(`Saving indexes...`); for (const [ , indexFile ] of store.caches) { - await indexFile.saveIndex().catch(e => logger.error(e)); + await indexFile.saveIndex(); } for (const [, indexFile ] of store.caches) { @@ -189,14 +187,14 @@ const indexJagStore = async (store: JagFileStore) => { logger.info(`Saving JAG file indexes...`); for (const [, index ] of store.caches) { - await index.upsertFileIndexes().catch(e => logger.error(e)); + await index.upsertFileIndexes(); } logger.info(`Saving JAG archive file indexes...`); for (const [ , archive ] of archiveIndex.files) { if (archive instanceof JagArchive) { - await archive.upsertFileIndexes().catch(e => logger.error(e)); + await archive.upsertFileIndexes(); } } }; @@ -211,19 +209,9 @@ const indexerScript = async ( { build, dir, format, archive: archiveName, source } ) => { const start = Date.now(); - const logDir = join(dir, 'logs'); const numericBuildNumber: number = /^\d+$/.test(build) ? parseInt(build, 10) : -1; let cacheFiles: PackedCacheFile[] | 'local' = 'local'; - if (!existsSync(logDir)) { - mkdirSync(logDir, { recursive: true }); - } - - /*logger.setTargets([ - prettyPrintTarget(), - fileTarget(join(logDir, `index-${ build }.log`)) - ]);*/ - if (source === 'openrs2') { if (numericBuildNumber) { const openRS2CacheFiles = await getOpenRS2CacheFilesByBuild(numericBuildNumber); @@ -281,8 +269,6 @@ const indexerScript = async ( } logger.info(`Indexing completed in ${ (Date.now() - start) / 1000 } seconds.`); - // logger.boom.flushSync(); - // logger.boom.end(); }; diff --git a/src/scripts/script-executor.ts b/src/scripts/script-executor.ts index 8383de4..3ffc161 100644 --- a/src/scripts/script-executor.ts +++ b/src/scripts/script-executor.ts @@ -1,6 +1,5 @@ import yargs from 'yargs/yargs'; import { Options } from 'yargs'; -import { logger } from '@runejs/common'; export type ArgumentOptions = { [key: string]: Options }; @@ -19,7 +18,6 @@ export class ScriptExecutor { (async function(args: T) { await script(args); }(this.getArguments(argumentOptions))); - //.finally(() => process.exit(0)); } } From c8a75e7c34123e4933dc2e5f8ca9d6e21dd3f85a Mon Sep 17 00:00:00 2001 From: Kikorono Date: Mon, 29 Aug 2022 11:59:44 -0500 Subject: [PATCH 29/32] Improving the HTTP server using decorators with full support for all JS5 builds --- package-lock.json | 39 ++++ package.json | 11 +- src/db/js5/js5-database.ts | 1 + src/file-system/index.ts | 4 + .../js5/content/config/config-file.ts | 0 .../js5/content/config/js5-game-items.ts | 129 ++++++++++++- src/file-system/js5/js5-archive.ts | 48 ++++- src/file-system/js5/js5-file-store.ts | 58 ++++-- src/file-system/js5/js5-group.ts | 50 ++++- src/http/config.ts | 4 + src/http/index.ts | 1 + src/http/js5-routes.ts | 7 + src/http/js5/js5.controller.ts | 124 +++++++++++++ src/http/js5/js5.service.ts | 174 ++++++++++++++++++ src/http/server.ts | 174 ++---------------- src/scripts/dev.ts | 7 +- src/scripts/indexer.ts | 2 +- src/scripts/unpacker.ts | 21 +-- 18 files changed, 636 insertions(+), 218 deletions(-) create mode 100644 src/file-system/index.ts create mode 100644 src/file-system/js5/content/config/config-file.ts create mode 100644 src/http/config.ts create mode 100644 src/http/index.ts create mode 100644 src/http/js5/js5.controller.ts create mode 100644 src/http/js5/js5.service.ts diff --git a/package-lock.json b/package-lock.json index 0ec4080..903f302 100644 --- a/package-lock.json +++ b/package-lock.json @@ -9,6 +9,8 @@ "version": "1.0.0-next.0", "license": "GPL-3.0", "dependencies": { + "@decorators/di": "^1.0.3", + "@decorators/express": "^2.6.0", "@runejs/common": "3.0.0-beta.8", "adm-zip": "^0.5.9", "axios": "^0.27.2", @@ -59,6 +61,29 @@ "node": ">=12" } }, + "node_modules/@decorators/di": { + "version": "1.0.3", + "resolved": "https://registry.npmjs.org/@decorators/di/-/di-1.0.3.tgz", + "integrity": "sha512-wafyQo5lqGABT+Lh7Od9/qULg7DG/kZFU3mLUKZFuiV/KATYlnv198yQxaZUZerhUDoTl/cZKu9t4mJa0rZK4Q==", + "dependencies": { + "reflect-metadata": "0.1.13" + }, + "engines": { + "node": ">=7.1.0" + } + }, + "node_modules/@decorators/express": { + "version": "2.6.0", + "resolved": "https://registry.npmjs.org/@decorators/express/-/express-2.6.0.tgz", + "integrity": "sha512-th/LqSAHAAW1ob2CU6hxtsfnryGThmz0AmGP45rA3Yoi0Q7+FHAZVfH7NJSYHITai/uXjjOtZPPW/V0gFfbgUw==", + "engines": { + "node": ">=7.1.0" + }, + "peerDependencies": { + "@decorators/di": ">=1.0.2", + "express": ">=4.16.3" + } + }, "node_modules/@eslint/eslintrc": { "version": "1.3.0", "resolved": "https://registry.npmjs.org/@eslint/eslintrc/-/eslintrc-1.3.0.tgz", @@ -4320,6 +4345,20 @@ "@jridgewell/trace-mapping": "0.3.9" } }, + "@decorators/di": { + "version": "1.0.3", + "resolved": "https://registry.npmjs.org/@decorators/di/-/di-1.0.3.tgz", + "integrity": "sha512-wafyQo5lqGABT+Lh7Od9/qULg7DG/kZFU3mLUKZFuiV/KATYlnv198yQxaZUZerhUDoTl/cZKu9t4mJa0rZK4Q==", + "requires": { + "reflect-metadata": "0.1.13" + } + }, + "@decorators/express": { + "version": "2.6.0", + "resolved": "https://registry.npmjs.org/@decorators/express/-/express-2.6.0.tgz", + "integrity": "sha512-th/LqSAHAAW1ob2CU6hxtsfnryGThmz0AmGP45rA3Yoi0Q7+FHAZVfH7NJSYHITai/uXjjOtZPPW/V0gFfbgUw==", + "requires": {} + }, "@eslint/eslintrc": { "version": "1.3.0", "resolved": "https://registry.npmjs.org/@eslint/eslintrc/-/eslintrc-1.3.0.tgz", diff --git a/package.json b/package.json index b52de7f..ce23a79 100644 --- a/package.json +++ b/package.json @@ -8,17 +8,16 @@ ".": "./index.js", "./config": "./config/index.js", "./db": "./db/index.js", - "./scripts": "./scripts/index.js", - "./util": "./util/index.js", - "./indexer": "./scripts/indexer.js", - "./unpacker": "./scripts/unpacker.js" + "./file-system": "./file-system/index.js", + "./http": "./http/index.js", + "./openrs2": "./openrs2/index.js" }, "scripts": { "build": "tsc", "start": "ts-node-dev src/scripts/dev.ts", "lint": "eslint --ext .ts src", "lint:fix": "eslint --ext .ts src --fix", - "http:js5": "ts-node-dev --max-old-space-size=2048 src/http/server.ts", + "http": "ts-node-dev --max-old-space-size=2048 src/http/server.ts", "namehash": "ts-node-dev --max-old-space-size=2048 src/scripts/name-hasher.ts", "unpack": "ts-node-dev --max-old-space-size=2048 src/scripts/unpacker.ts", "index": "ts-node-dev --max-old-space-size=2048 src/scripts/indexer.ts", @@ -55,6 +54,8 @@ "typescript": ">=4.5.0" }, "dependencies": { + "@decorators/di": "^1.0.3", + "@decorators/express": "^2.6.0", "@runejs/common": "3.0.0-beta.8", "adm-zip": "^0.5.9", "axios": "^0.27.2", diff --git a/src/db/js5/js5-database.ts b/src/db/js5/js5-database.ts index 07a344a..83e972b 100644 --- a/src/db/js5/js5-database.ts +++ b/src/db/js5/js5-database.ts @@ -9,6 +9,7 @@ import { join } from 'path'; export interface Js5IndexEntityWhere { fileType?: Js5FileType; key?: number; + name?: string; archiveKey?: number; groupKey?: number; } diff --git a/src/file-system/index.ts b/src/file-system/index.ts new file mode 100644 index 0000000..5802b7a --- /dev/null +++ b/src/file-system/index.ts @@ -0,0 +1,4 @@ +export * from './file-store-base'; +export * from './packed'; +export * from './jag'; +export * from './js5'; diff --git a/src/file-system/js5/content/config/config-file.ts b/src/file-system/js5/content/config/config-file.ts new file mode 100644 index 0000000..e69de29 diff --git a/src/file-system/js5/content/config/js5-game-items.ts b/src/file-system/js5/content/config/js5-game-items.ts index 7d658d4..f76c0f7 100644 --- a/src/file-system/js5/content/config/js5-game-items.ts +++ b/src/file-system/js5/content/config/js5-game-items.ts @@ -4,6 +4,44 @@ import { ByteBuffer } from '@runejs/common'; export class Js5GameItem { id: number; + inventoryModelId: number; + name: string; + modelZoom: number; + modelRotationX: number; + modelRotationY: number; + modelOffsetX: number; + modelOffsetY: number; + unknownServerAttribute1: number; + stackable: boolean = false; + value: number; + members: boolean = false; + maleModelId1: number; + maleModelOffset: number; + maleModelId2: number; + femaleModelId1: number; + femaleModelOffset: number; + femaleModelId2: number; + groundOptions: string[]; + interfaceOptions: string[]; + originalModelColors: number[]; + modifiedModelColors: number[]; + maleModelId3: number; + femaleModelId3: number; + maleDialogueModelId1: number; + femaleDialogueModelId1: number; + maleDialogueModelId2: number; + femaleDialogueModelId2: number; + modelRotationZ: number; + noteId: number; + noteTemplateId: number; + stackIds: number[]; + stackAmounts: number[]; + modelScaleX: number; + modelScaleY: number; + modelScaleZ: number; + lightingModifier: number; + shadowModifier: number; + teamIndex: number; } export class Js5GameItems { @@ -21,10 +59,93 @@ export class Js5GameItems { item.id = id; while (true) { - const opcode = data.get('byte', 'unsigned'); - if (opcode === 0) { - break; - } + const o = data.get('byte', 'u'); + + if (o === 0) { + break; // eof + } else if (o === 1) { + item.inventoryModelId = data.get('short', 'u'); + } else if (o === 2) { + item.name = data.getString(); + } else if (o === 4) { + // case 3 was item description prior to build 400 + item.modelZoom = data.get('short', 'u'); + } else if (o === 5) { + item.modelRotationX = data.get('short', 'u'); + } else if (o === 6) { + item.modelRotationY = data.get('short', 'u'); + } else if (o === 7) { + // client subtracts 65536 if the value is over 32767 + // so this should be a SIGNED short then, right?... + item.modelOffsetX = data.get('short', 'u'); + } else if (o === 8) { + // client subtracts 65536 if the value is over 32767 + // so this should be a SIGNED short then, right?... + item.modelOffsetY = data.get('short', 'u'); + } else if (o === 10) { + // case 9 missing - what was it (if anything)? + item.unknownServerAttribute1 = data.get('short', 'u'); + } else if (o === 11) { + item.stackable = true; + } else if (o === 12) { + item.value = data.get('int'); + } else if (o === 16) { + // cases 13-15 missing - what were they (if anything)? + item.members = true; + } else if (o === 23) { + // cases 17-22 missing - what were they (if anything)? + item.maleModelId1 = data.get('short', 'u'); + item.maleModelOffset = data.get('byte'); + } else if (o === 24) { + item.maleModelId2 = data.get('short', 'u'); + } else if (o === 25) { + item.femaleModelId1 = data.get('short', 'u'); + item.femaleModelOffset = data.get('byte'); + } else if (o === 26) { + item.femaleModelId2 = data.get('short', 'u'); + } else if (o >= 30 && o < 35) { + // cases 27-29 missing - what were they (if anything)? + if (!item.groundOptions) { + item.groundOptions = new Array(5); + } + + item.groundOptions[o - 30] = data.getString(); + } else if (o >= 35 && o < 40) { + if (!item.interfaceOptions) { + item.interfaceOptions = new Array(5); + } + + item.interfaceOptions[o - 35] = data.getString(); + } else if (o === 40) { + const colorCount = data.get('byte', 'u'); + item.originalModelColors = new Array(colorCount); + item.modifiedModelColors = new Array(colorCount); + for (let i = 0; i < colorCount; i++) { + item.originalModelColors[i] = data.get('short', 'u'); + item.modifiedModelColors[i] = data.get('short', 'u'); + } + } else if (o === 78) { + // cases 41-77 missing - what were they (if anything)? + item.maleModelId3 = data.get('short', 'u'); + } else if (o === 79) { + item.femaleModelId3 = data.get('short', 'u'); + } else if (o === 90) { + item.maleDialogueModelId1 = data.get('short', 'u'); + } else if (o === 91) { + item.femaleDialogueModelId1 = data.get('short', 'u'); + } else if (o === 92) { + item.maleDialogueModelId2 = data.get('short', 'u'); + } else if (o === 93) { + item.femaleDialogueModelId2 = data.get('short', 'u'); + } else if (o === 95) { + // case 94 missing - what was it (if anything)? + item.modelRotationZ = data.get('short', 'u'); + } else if (o === 97) { + // case 96 missing - what was it (if anything)? + item.noteId = data.get('short', 'u'); + } else if (o === 98) { + item.noteTemplateId = data.get('short', 'u'); + } // @todo stopped here - 24/08/22 - Kiko } return item; diff --git a/src/file-system/js5/js5-archive.ts b/src/file-system/js5/js5-archive.ts index e0aa5e1..44b2935 100644 --- a/src/file-system/js5/js5-archive.ts +++ b/src/file-system/js5/js5-archive.ts @@ -3,6 +3,7 @@ import { Js5Group } from './js5-group'; import { logger } from '@runejs/common'; import { Js5FileBase } from './js5-file-base'; import { Js5ArchiveConfig } from '../../config'; +import { Js5IndexEntity } from '../../db/js5'; export class Js5Archive extends Js5FileBase { @@ -65,17 +66,48 @@ export class Js5Archive extends Js5FileBase { } } - getGroup(groupKey: number): Js5Group | null; - getGroup(groupName: string): Js5Group | null; - getGroup(groupKeyOrName: number | string): Js5Group | null; - getGroup(groupKeyOrName: number | string): Js5Group | null { - if (typeof groupKeyOrName === 'string') { - return Array.from(this.groups.values()).find( - group => group?.index?.name === groupKeyOrName + async getGroup(groupKey: number): Promise; + async getGroup(groupName: string): Promise; + async getGroup(groupIdentifier: number | string): Promise; + async getGroup(groupIdentifier: number | string): Promise { + let group: Js5Group; + + if (typeof groupIdentifier === 'string') { + group = Array.from(this.groups.values()).find( + group => group?.index?.name === groupIdentifier ) || null; } else { - return this.groups.get(groupKeyOrName) || null; + group = this.groups.get(groupIdentifier) || null; } + + if (!group?.index) { + let groupEntity: Js5IndexEntity; + + if (typeof groupIdentifier === 'number' || /^\d*$/.test(groupIdentifier)) { + const groupKey = typeof groupIdentifier === 'string' ? parseInt(groupIdentifier, 10) : groupIdentifier; + groupEntity = await this.fileStore.database.getIndex({ + fileType: 'GROUP', + archiveKey: this.index.key, + key: groupKey + }); + } else { + groupEntity = await this.fileStore.database.getIndex({ + fileType: 'GROUP', + archiveKey: this.index.key, + name: String(groupIdentifier) + }); + } + + if (!group) { + group = new Js5Group(this.fileStore, groupEntity.key, this); + group.index = groupEntity; + this.groups.set(groupEntity.key, group); + } else { + group.index = groupEntity; + } + } + + return group; } setGroup(groupKey: number, group: Js5Group): void { diff --git a/src/file-system/js5/js5-file-store.ts b/src/file-system/js5/js5-file-store.ts index bf3d414..bfd9c3d 100644 --- a/src/file-system/js5/js5-file-store.ts +++ b/src/file-system/js5/js5-file-store.ts @@ -6,7 +6,7 @@ import { JS5 } from './js5'; import { Js5Archive } from './js5-archive'; import { FileStoreBase } from '../file-store-base'; import { logger } from '../../../../common'; -import { Js5Database } from '../../db/js5'; +import { Js5Database, Js5IndexEntity } from '../../db/js5'; import { Js5File } from './js5-file'; @@ -16,6 +16,7 @@ export class Js5FileStore extends FileStoreBase{ readonly archives: Map; private _archiveConfig: { [key: string]: Js5ArchiveConfig }; + private _loaded: boolean = false; constructor(gameBuild: string | number, storePath: string = './') { super(gameBuild, storePath); @@ -39,12 +40,18 @@ export class Js5FileStore extends FileStoreBase{ loadArchiveChildEntities: boolean = false, loadGroupChildEntities: boolean = false, ): Promise { + if (this._loaded) { + return; + } + await this.js5.loadEncryptionKeys(); await this.openDatabase(); if (loadArchiveEntities) { await this.loadArchiveEntities(loadArchiveChildEntities, loadGroupChildEntities); } + + this._loaded = true; } /** @@ -85,8 +92,8 @@ export class Js5FileStore extends FileStoreBase{ }); for (const fileEntity of files) { - const archive = this.getArchive(fileEntity.archiveKey); - const group = archive.getGroup(fileEntity.groupKey); + const archive = await this.getArchive(fileEntity.archiveKey); + const group = await archive.getGroup(fileEntity.groupKey); const file = new Js5File(this, fileEntity.key, group); file.index = fileEntity; group.setFile(fileEntity.key, file); @@ -119,17 +126,46 @@ export class Js5FileStore extends FileStoreBase{ return archive; } - getArchive(archiveKey: number): Js5Archive | null; - getArchive(archiveName: string): Js5Archive | null; - getArchive(archiveKeyOrName: number | string): Js5Archive | null; - getArchive(archiveKeyOrName: number | string): Js5Archive | null { - if (typeof archiveKeyOrName === 'string') { - return Array.from(this.archives.values()).find( - a => a?.index?.name === archiveKeyOrName + async getArchive(archiveKey: number): Promise; + async getArchive(archiveName: string): Promise; + async getArchive(archiveIdentifier: number | string): Promise; + async getArchive(archiveIdentifier: number | string): Promise { + let archive: Js5Archive; + + if (typeof archiveIdentifier === 'string') { + archive = Array.from(this.archives.values()).find( + a => a?.index?.name === archiveIdentifier ) || null; } else { - return this.archives.get(archiveKeyOrName) || null; + archive = this.archives.get(archiveIdentifier) || null; + } + + if (!archive?.index) { + let archiveEntity: Js5IndexEntity; + + if (typeof archiveIdentifier === 'number' || /^\d*$/.test(archiveIdentifier)) { + const archiveKey = typeof archiveIdentifier === 'string' ? parseInt(archiveIdentifier, 10) : archiveIdentifier; + archiveEntity = await this.database.getIndex({ + fileType: 'ARCHIVE', + key: archiveKey + }); + } else { + archiveEntity = await this.database.getIndex({ + fileType: 'ARCHIVE', + name: String(archiveIdentifier) + }); + } + + if (!archive) { + archive = new Js5Archive(this, archiveEntity.key); + archive.index = archiveEntity; + this.archives.set(archiveEntity.key, archive); + } else { + archive.index = archiveEntity; + } } + + return archive; } setArchive(archiveKey: number, archive: Js5Archive): void; diff --git a/src/file-system/js5/js5-group.ts b/src/file-system/js5/js5-group.ts index 447f4d1..ddc6ff1 100644 --- a/src/file-system/js5/js5-group.ts +++ b/src/file-system/js5/js5-group.ts @@ -3,6 +3,7 @@ import { Js5Archive } from './js5-archive'; import { Js5File } from './js5-file'; import { logger } from '@runejs/common'; import { Js5FileBase } from './js5-file-base'; +import { Js5IndexEntity } from '../../db/js5'; export class Js5Group extends Js5FileBase { @@ -66,17 +67,50 @@ export class Js5Group extends Js5FileBase { } } - getFile(fileKey: number): Js5File | null; - getFile(fileName: string): Js5File | null; - getFile(fileKeyOrName: number | string): Js5File | null; - getFile(fileKeyOrName: number | string): Js5File | null { - if (typeof fileKeyOrName === 'string') { - return Array.from(this.files.values()).find( - file => file?.index?.name === fileKeyOrName + async getFile(fileKey: number): Promise; + async getFile(fileName: string): Promise; + async getFile(fileIdentifier: number | string): Promise; + async getFile(fileIdentifier: number | string): Promise { + let file: Js5File; + + if (typeof fileIdentifier === 'string') { + file = Array.from(this.files.values()).find( + file => file?.index?.name === fileIdentifier ) || null; } else { - return this.files.get(fileKeyOrName) || null; + file = this.files.get(fileIdentifier) || null; } + + if (!file?.index) { + let fileEntity: Js5IndexEntity; + + if (typeof fileIdentifier === 'number' || /^\d*$/.test(fileIdentifier)) { + const fileKey = typeof fileIdentifier === 'string' ? parseInt(fileIdentifier, 10) : fileIdentifier; + fileEntity = await this.fileStore.database.getIndex({ + fileType: 'GROUP', + archiveKey: this.index.archiveKey, + groupKey: this.index.key, + key: fileKey + }); + } else { + fileEntity = await this.fileStore.database.getIndex({ + fileType: 'GROUP', + archiveKey: this.index.archiveKey, + groupKey: this.index.key, + name: String(fileIdentifier) + }); + } + + if (!file) { + file = new Js5File(this.fileStore, fileEntity.key, this); + file.index = fileEntity; + this.files.set(fileEntity.key, file); + } else { + file.index = fileEntity; + } + } + + return file; } setFile(fileKey: number, file: Js5File): void { diff --git a/src/http/config.ts b/src/http/config.ts new file mode 100644 index 0000000..712c0ae --- /dev/null +++ b/src/http/config.ts @@ -0,0 +1,4 @@ +import { InjectionToken } from '@decorators/di'; + + +export const FILESTORE_DIR = new InjectionToken('FILESTORE_DIR'); diff --git a/src/http/index.ts b/src/http/index.ts new file mode 100644 index 0000000..0ce5251 --- /dev/null +++ b/src/http/index.ts @@ -0,0 +1 @@ +export * from './server'; diff --git a/src/http/js5-routes.ts b/src/http/js5-routes.ts index e69de29..c276a4d 100644 --- a/src/http/js5-routes.ts +++ b/src/http/js5-routes.ts @@ -0,0 +1,7 @@ +import { Request, Response, Router } from 'express'; + +export const js5Router = Router(); + +js5Router.get('/archives', (req: Request, res: Response) => { + +}); diff --git a/src/http/js5/js5.controller.ts b/src/http/js5/js5.controller.ts new file mode 100644 index 0000000..0fcb0aa --- /dev/null +++ b/src/http/js5/js5.controller.ts @@ -0,0 +1,124 @@ +import { Response as ExpressResponse } from 'express'; +import { Controller, Get, Params, Response } from '@decorators/express'; +import { Js5Service } from './js5.service'; +import { Inject } from '@decorators/di'; + + +@Controller('/js5') +export class Js5Controller { + + constructor(@Inject(Js5Service) private js5Service: Js5Service) { + } + + @Get('/:gameBuild/archives/:archiveIdentifier/groups/:groupIdentifier/files/:fileIdentifier/data') + async getArchiveGroupFileData( + @Response() res: ExpressResponse, + @Params('gameBuild') gameBuild: string | number, + @Params('archiveIdentifier') archiveIdentifier: string | number, + @Params('groupIdentifier') groupIdentifier: string | number, + @Params('fileIdentifier') fileIdentifier: string | number + ) { + const data = await this.js5Service.getArchiveGroupFileData(gameBuild, archiveIdentifier, groupIdentifier, fileIdentifier); + + res.writeHead(200, { + 'Content-Type': 'arraybuffer', + 'Content-Length': data.length, + 'Content-disposition': `attachment; filename=${fileIdentifier}` + }); + + res.end(data); + } + + @Get('/:gameBuild/archives/:archiveIdentifier/groups/:groupIdentifier/files/:fileIdentifier') + async getArchiveGroupFile( + @Response() res: ExpressResponse, + @Params('gameBuild') gameBuild: string | number, + @Params('archiveIdentifier') archiveIdentifier: string | number, + @Params('groupIdentifier') groupIdentifier: string | number, + @Params('fileIdentifier') fileIdentifier: string | number + ) { + res.send(await this.js5Service.getArchiveGroupFile(gameBuild, archiveIdentifier, groupIdentifier, fileIdentifier)); + } + + @Get('/:gameBuild/archives/:archiveIdentifier/groups/:groupIdentifier/files') + async getArchiveGroupFileList( + @Response() res: ExpressResponse, + @Params('gameBuild') gameBuild: string | number, + @Params('archiveIdentifier') archiveIdentifier: string | number, + @Params('groupIdentifier') groupIdentifier: string | number + ) { + res.send(await this.js5Service.getArchiveGroupFileList(gameBuild, archiveIdentifier, groupIdentifier)); + } + + @Get('/:gameBuild/archives/:archiveIdentifier/groups/:groupIdentifier/data') + async getArchiveGroupData( + @Response() res: ExpressResponse, + @Params('gameBuild') gameBuild: string | number, + @Params('archiveIdentifier') archiveIdentifier: string | number, + @Params('groupIdentifier') groupIdentifier: string | number + ) { + const data = await this.js5Service.getArchiveGroupData(gameBuild, archiveIdentifier, groupIdentifier); + + res.writeHead(200, { + 'Content-Type': 'arraybuffer', + 'Content-Length': data.length, + 'Content-disposition': `attachment; filename=${groupIdentifier}` + }); + + res.end(data); + } + + @Get('/:gameBuild/archives/:archiveIdentifier/groups/:groupIdentifier') + async getArchiveGroup( + @Response() res: ExpressResponse, + @Params('gameBuild') gameBuild: string | number, + @Params('archiveIdentifier') archiveIdentifier: string | number, + @Params('groupIdentifier') groupIdentifier: string | number + ) { + res.send(await this.js5Service.getArchiveGroup(gameBuild, archiveIdentifier, groupIdentifier)); + } + + @Get('/:gameBuild/archives/:archiveIdentifier/groups') + async getArchiveGroupList( + @Response() res: ExpressResponse, + @Params('gameBuild') gameBuild: string | number, + @Params('archiveIdentifier') archiveIdentifier: string | number + ) { + res.send(await this.js5Service.getArchiveGroupList(gameBuild, archiveIdentifier)); + } + + @Get('/:gameBuild/archives/:archiveIdentifier/data') + async getArchiveData( + @Response() res: ExpressResponse, + @Params('gameBuild') gameBuild: string | number, + @Params('archiveIdentifier') archiveIdentifier: string | number + ) { + const data = await this.js5Service.getArchiveData(gameBuild, archiveIdentifier); + + res.writeHead(200, { + 'Content-Type': 'arraybuffer', + 'Content-Length': data.length, + 'Content-disposition': `attachment; filename=${archiveIdentifier}` + }); + + res.end(data); + } + + @Get('/:gameBuild/archives/:archiveIdentifier') + async getArchive( + @Response() res: ExpressResponse, + @Params('gameBuild') gameBuild: string | number, + @Params('archiveIdentifier') archiveIdentifier: string | number + ) { + res.send(await this.js5Service.getArchive(gameBuild, archiveIdentifier)); + } + + @Get('/:gameBuild/archives') + async getArchiveList( + @Response() res: ExpressResponse, + @Params('gameBuild') gameBuild: string | number + ) { + res.send(await this.js5Service.getArchiveList(gameBuild)); + } + +} diff --git a/src/http/js5/js5.service.ts b/src/http/js5/js5.service.ts new file mode 100644 index 0000000..8323780 --- /dev/null +++ b/src/http/js5/js5.service.ts @@ -0,0 +1,174 @@ +import { Inject, Injectable } from '@decorators/di'; +import { FILESTORE_DIR } from '../config'; +import { Js5FileStore } from '../../file-system'; +import { Js5IndexEntity } from '../../db/js5'; +import { logger } from '@runejs/common'; +import { Buffer } from 'buffer'; + + +@Injectable() +export class Js5Service { + + readonly stores: Map; + + constructor( + @Inject(FILESTORE_DIR) private fileStoreDir: string + ) { + this.stores = new Map(); + logger.info('Js5Service initialized'); + } + + async getArchiveGroupFileData( + gameBuild: string | number, + archiveIdentifier: string | number, + groupIdentifier: string | number, + fileIdentifier: string | number + ): Promise { + const fileStore = await this.getFileStore(gameBuild); + const archive = await fileStore.getArchive(archiveIdentifier); + const group = await archive.getGroup(groupIdentifier); + const file = await group.getFile(fileIdentifier); + return file.index.compressedData; + } + + async getArchiveGroupFile( + gameBuild: string | number, + archiveIdentifier: string | number, + groupIdentifier: string | number, + fileIdentifier: string | number + ): Promise { + const fileStore = await this.getFileStore(gameBuild); + const archive = await fileStore.getArchive(archiveIdentifier); + const group = await archive.getGroup(groupIdentifier); + const file = await group.getFile(fileIdentifier); + + const index = { ...file.index }; + delete index.data; + delete index.compressedData; + + return index; + } + + async getArchiveGroupFileList( + gameBuild: string | number, + archiveIdentifier: string | number, + groupIdentifier: string | number + ): Promise { + const fileStore = await this.getFileStore(gameBuild); + const archive = await fileStore.getArchive(archiveIdentifier); + const group = await archive.getGroup(groupIdentifier); + + await group.loadFileIndexes(); + + const files = Array.from(group.files.values()); + const results: Js5IndexEntity[] = new Array(files.length); + + for (let i = 0; i < files.length; i++) { + results[i] = { ...files[i].index }; + delete results[i].data; + delete results[i].compressedData; + } + + return results; + } + + async getArchiveGroupData( + gameBuild: string | number, + archiveIdentifier: string | number, + groupIdentifier: string | number + ): Promise { + const fileStore = await this.getFileStore(gameBuild); + const archive = await fileStore.getArchive(archiveIdentifier); + const group = await archive.getGroup(groupIdentifier); + return group.index.compressedData; + } + + async getArchiveGroup( + gameBuild: string | number, + archiveIdentifier: string | number, + groupIdentifier: string | number + ): Promise { + const fileStore = await this.getFileStore(gameBuild); + const archive = await fileStore.getArchive(archiveIdentifier); + const group = await archive.getGroup(groupIdentifier); + + const index = { ...group.index }; + delete index.data; + delete index.compressedData; + + return index; + } + + async getArchiveGroupList( + gameBuild: string | number, + archiveIdentifier: string | number + ): Promise { + const fileStore = await this.getFileStore(gameBuild); + const archive = await fileStore.getArchive(archiveIdentifier); + + await archive.loadGroupIndexes(); + + const groups = Array.from(archive.groups.values()); + const results: Js5IndexEntity[] = new Array(groups.length); + + for (let i = 0; i < groups.length; i++) { + results[i] = { ...groups[i].index }; + delete results[i].data; + delete results[i].compressedData; + } + + return results; + } + + async getArchiveData( + gameBuild: string | number, + archiveIdentifier: string | number + ): Promise { + const fileStore = await this.getFileStore(gameBuild); + const archive = await fileStore.getArchive(archiveIdentifier); + return archive.index.compressedData; + } + + async getArchive( + gameBuild: string | number, + archiveIdentifier: string | number + ): Promise { + const fileStore = await this.getFileStore(gameBuild); + const archive = await fileStore.getArchive(archiveIdentifier); + + const index = { ...archive.index }; + delete index.data; + delete index.compressedData; + + return index; + } + + async getArchiveList(gameBuild: string | number): Promise { + const fileStore = await this.getFileStore(gameBuild); + + const archives = Array.from(fileStore.archives.values()); + const results: Js5IndexEntity[] = new Array(archives.length); + + for (let i = 0; i < archives.length; i++) { + results[i] = { ...archives[i].index }; + delete results[i].data; + delete results[i].compressedData; + } + + return results; + } + + async getFileStore(gameBuild: string | number): Promise { + if (this.stores.has(gameBuild)) { + return this.stores.get(gameBuild); + } + + const fileStore = new Js5FileStore(gameBuild, this.fileStoreDir); + await fileStore.load(true); + + this.stores.set(gameBuild, fileStore); + + return fileStore; + } + +} diff --git a/src/http/server.ts b/src/http/server.ts index 196ab0d..f0618fd 100644 --- a/src/http/server.ts +++ b/src/http/server.ts @@ -1,25 +1,19 @@ -import express, { Response } from 'express'; +import express from 'express'; import { logger } from '@runejs/common'; -import { Js5FileStore } from '../file-system/js5'; import { ArgumentOptions, ScriptExecutor } from '../scripts/script-executor'; -import { constants as http2 } from 'http2'; -import { Js5IndexEntity } from '../db/js5'; +import { Container } from '@decorators/di'; +import { FILESTORE_DIR } from './config'; +import { attachControllers } from '@decorators/express'; +import { Js5Controller } from './js5/js5.controller'; interface ServerOptions { - build: string; dir: string; port: number; } const serverArgumentOptions: ArgumentOptions = { - build: { - alias: 'b', - type: 'string', - default: '435', - description: `The game build (revision) that the store should belong to, also known as the game build number. Defaults to '435', a game build from late October, 2006.` - }, dir: { alias: 'd', type: 'string', @@ -35,147 +29,11 @@ const serverArgumentOptions: ArgumentOptions = { }; -const handleError = (res: Response, status: number, msg: string) => { - logger.error(msg); - res.status(status).send(); -}; - - -const keyOrName = (input: string): string | number => { - if (/^\d+$/.test(input)) { - return Number(input); - } else { - return input; - } -}; - - -const writeFile = (res: Response, index: Js5IndexEntity, compressed: boolean = true) => { - res.writeHead(200, { - 'Content-Type': 'arraybuffer', - 'Content-Length': compressed ? index.compressedData.length : index.data.length, - 'Content-disposition': `attachment; filename=${index.name || index.key}` - }); - - res.end(compressed ? index.compressedData : index.data); -}; - - -const app = express(); -let store: Js5FileStore; - - -app.get('/archives/:archiveKey/groups/:groupKey/files/:fileKey', (req, res, next) => { - const { archiveKey, groupKey, fileKey } = req.params; - logger.info(`Request /archives/${archiveKey}/groups/${groupKey}/files/${fileKey}`); - - const archive = store.getArchive(keyOrName(archiveKey)); - if (!archive) { - handleError(res, http2.HTTP_STATUS_NOT_FOUND, `Archive ${archiveKey} was not found.`); - return; - } - - const group = archive.getGroup(keyOrName(groupKey)); - if (!group) { - handleError(res, http2.HTTP_STATUS_NOT_FOUND, `Group ${groupKey} was not found within Archive ${archiveKey}.`); - return; - } - - const file = group.getFile(keyOrName(fileKey)); - if (!file || !file.index.data?.length) { - handleError(res, http2.HTTP_STATUS_NOT_FOUND, `File ${groupKey}:${fileKey} was not found within Archive ${archiveKey}.`); - return; - } - - writeFile(res, file.index, false); -}); - -app.get('/archives/:archiveKey/groups/:groupKey', (req, res, next) => { - const { archiveKey, groupKey } = req.params; - logger.info(`Request /archives/${archiveKey}/groups/${groupKey}`); - - const archive = store.getArchive(keyOrName(archiveKey)); - if (!archive) { - handleError(res, http2.HTTP_STATUS_NOT_FOUND, `Archive ${archiveKey} was not found.`); - return; - } - - const group = archive.getGroup(keyOrName(groupKey)); - if (!group || !group.index.compressedData?.length) { - handleError(res, http2.HTTP_STATUS_NOT_FOUND, `Group ${groupKey} was not found within Archive ${archiveKey}.`); - return; - } - - writeFile(res, group.index); -}); - -app.get('/archives/:archiveKey/groups', (req, res, next) => { - const { archiveKey } = req.params; - logger.info(`Request /archives/${archiveKey}/groups`); - - const archive = store.getArchive(keyOrName(archiveKey)); - if (!archive) { - handleError(res, http2.HTTP_STATUS_NOT_FOUND, `Archive ${archiveKey} was not found.`); - return; - } - - const groupDetails = Array.from(archive.groups.values()).map(group => ({ - key: group.index.key, - name: group.index.name, - nameHash: group.index.nameHash, - childCount: group.index.childCount, - children: Array.from(group.files.values()).map(file => ({ - key: file.index.key, - name: file.index.name, - nameHash: file.index.nameHash, - })) - })); - - res.send(groupDetails); -}); - -app.get('/archives/:archiveKey', (req, res, next) => { - const { archiveKey } = req.params; - - logger.info(`Request /archives/${archiveKey}`); - - const archive = store.getArchive(keyOrName(archiveKey)); - if (!archive || !archive.index.compressedData?.length) { - handleError(res, http2.HTTP_STATUS_NOT_FOUND, `Archive ${archiveKey} was not found.`); - return; - } - - writeFile(res, archive.index); -}); - -app.get('/archives', (req, res) => { - logger.info(`Request /archives`); - - const archives = Array.from(store.archives.values()); - - const archiveList: Record[] = []; - - for (const archive of archives) { - archiveList.push({ - key: archive.index.key, - name: archive.index.name, - childCount: archive.index.childCount, - loadedChildCount: archive.groups.size, - }); - } - - res.send(archiveList); -}); - -app.get('/config', (req, res, next) => { - logger.info(`Request /config`); - res.status(200).contentType('json').send(store.archiveConfig); -}); - - new ScriptExecutor().executeScript(serverArgumentOptions, async ( - { build, dir, port } + { dir, port } ) => { + const app = express(); + app.use((req, res, next) => { res.header('Access-Control-Allow-Origin', '*'); res.header('Access-Control-Allow-Headers', 'Origin, X-Requested-With, Content-Type, Accept'); @@ -184,19 +42,15 @@ new ScriptExecutor().executeScript(serverArgumentOptions, async ( app.set('json spaces', 4); app.set('port', port); - app.set('dir', dir); - app.set('build', build); - app.listen(port, async () => { - const start = Date.now(); - - store = new Js5FileStore(build, dir); - await store.load(true, true, true); + Container.provide([ + { provide: FILESTORE_DIR, useValue: dir }, + ]); - const end = Date.now(); + attachControllers(app, [ Js5Controller ]); - logger.info(`File Store loaded in ${(end - start) / 1000} seconds.`); - logger.info('HTTP server online.'); + app.listen(port, async () => { + logger.info(`HTTP server listening at port ${port}.`); }); }); diff --git a/src/scripts/dev.ts b/src/scripts/dev.ts index 3e14933..a77ea07 100644 --- a/src/scripts/dev.ts +++ b/src/scripts/dev.ts @@ -1,16 +1,11 @@ import { writeFileSync, existsSync, mkdirSync } from 'fs'; -import { logger, prettyPrintTarget } from '@runejs/common'; +import { logger } from '@runejs/common'; import { JagFileStore } from '../file-system/jag'; import { JagInterfaceArchive } from '../file-system/jag/content/archives/interfaces/jag-interface-archive'; import { join } from 'path'; import { Js5FileStore } from '../file-system/js5'; -logger.setTargets([ - prettyPrintTarget() -]); - - const saveInterfaces = async (store: JagFileStore) => { logger.info(`Decoding game interfaces...`); diff --git a/src/scripts/indexer.ts b/src/scripts/indexer.ts index f5640de..af81733 100644 --- a/src/scripts/indexer.ts +++ b/src/scripts/indexer.ts @@ -108,7 +108,7 @@ const indexJS5Store = async (store: Js5FileStore) => { const indexJS5Archive = async (store: Js5FileStore, archiveName: string) => { - const archive = store.getArchive(archiveName); + const archive = await store.getArchive(archiveName); if (!archive) { logger.error(`Archive ${ archiveName } was not found.`); diff --git a/src/scripts/unpacker.ts b/src/scripts/unpacker.ts index 3783fdc..388e29b 100644 --- a/src/scripts/unpacker.ts +++ b/src/scripts/unpacker.ts @@ -1,9 +1,7 @@ import { join } from 'path'; import { existsSync, readdirSync, statSync, mkdirSync } from 'graceful-fs'; import { logger } from '@runejs/common'; - -import { Store } from '../index'; -import { ScriptExecutor, ArgumentOptions } from './index'; +import { ArgumentOptions, ScriptExecutor } from './script-executor'; interface UnpackOptions { @@ -34,7 +32,7 @@ const unpackerArgumentOptions: ArgumentOptions = { }; -async function unpackFiles(store: Store, args: UnpackOptions): Promise { +/*async function unpackFiles(store: Store, args: UnpackOptions): Promise { const argDebugString = args ? Array.from(Object.entries(args)) .map(([ key, val ]) => `${key} = ${val}`).join(', ') : ''; @@ -83,10 +81,10 @@ async function unpackFiles(store: Store, args: UnpackOptions): Promise { await a.saveIndexData(true, true); } -} +}*/ -new ScriptExecutor().executeScript(unpackerArgumentOptions, async (terminal, args) => { +new ScriptExecutor().executeScript(unpackerArgumentOptions, async (args) => { const start = Date.now(); logger.info(`Unpacking JS5 store...`); @@ -98,9 +96,7 @@ new ScriptExecutor().executeScript(unpackerArgumentOptions, async mkdirSync(logDir, { recursive: true }); } - logger.destination(join(logDir, `unpack_${ build }.log`)); - - const store = await Store.create(build, dir); + /*const store = await Store.create(build, dir); const js5Dir = join(dir, 'packed'); @@ -118,11 +114,6 @@ new ScriptExecutor().executeScript(unpackerArgumentOptions, async } } - logger.boom.flushSync(); - logger.boom.end(); - const end = Date.now(); - logger.info(`Unpacking completed in ${(end - start) / 1000} seconds.`); - - process.exit(0); + logger.info(`Unpacking completed in ${(end - start) / 1000} seconds.`);*/ }); From 907d7ec235cd91197736f035b4ba74259279269a Mon Sep 17 00:00:00 2001 From: Kikorono Date: Mon, 29 Aug 2022 13:00:12 -0500 Subject: [PATCH 30/32] Moving JS5 uncompressed and compressed file data into separate tables to speed up index load times and only pull/store blobs on-demand --- src/db/js5/index.ts | 2 + src/db/js5/js5-compressed-data-entity.ts | 30 ++++++ src/db/js5/js5-database.ts | 95 ++++++++++++++++- src/db/js5/js5-index-entity.ts | 8 +- src/db/js5/js5-uncompressed-data-entity.ts | 30 ++++++ src/file-system/js5/js5-archive.ts | 12 +++ src/file-system/js5/js5-file-base.ts | 115 +++++++++++++++++++-- src/file-system/js5/js5-group.ts | 12 +++ src/file-system/js5/js5.ts | 72 ++++++------- src/http/js5/js5.service.ts | 67 +++--------- src/scripts/indexer.ts | 36 +++++++ 11 files changed, 372 insertions(+), 107 deletions(-) create mode 100644 src/db/js5/js5-compressed-data-entity.ts create mode 100644 src/db/js5/js5-uncompressed-data-entity.ts diff --git a/src/db/js5/index.ts b/src/db/js5/index.ts index 7f472c9..72f4893 100644 --- a/src/db/js5/index.ts +++ b/src/db/js5/index.ts @@ -1,2 +1,4 @@ export * from './js5-database'; export * from './js5-index-entity'; +export * from './js5-uncompressed-data-entity'; +export * from './js5-compressed-data-entity'; diff --git a/src/db/js5/js5-compressed-data-entity.ts b/src/db/js5/js5-compressed-data-entity.ts new file mode 100644 index 0000000..0ae7db5 --- /dev/null +++ b/src/db/js5/js5-compressed-data-entity.ts @@ -0,0 +1,30 @@ +import { Column, Entity, Index, PrimaryColumn } from 'typeorm'; +import { Js5FileType } from '../../config'; +import { Buffer } from 'buffer'; + + +@Entity('js5_compressed_data') +@Index('compressed_data_identifier', [ + 'fileType', 'gameBuild', 'key', 'archiveKey', 'groupKey' +], { unique: true }) +export class Js5CompressedDataEntity { + + @PrimaryColumn('text', { name: 'file_type', nullable: false, unique: false }) + fileType: Js5FileType; + + @PrimaryColumn('text', { name: 'game_build', nullable: false, unique: false }) + gameBuild: string; + + @PrimaryColumn('integer', { nullable: false, unique: false }) + key: number; + + @PrimaryColumn('integer', { name: 'archive_key', nullable: false, unique: false, default: -1 }) + archiveKey: number = -1; + + @PrimaryColumn('integer', { name: 'group_key', nullable: false, unique: false, default: -1 }) + groupKey: number = -1; + + @Column('blob', { name: 'buffer', nullable: true, default: null }) + buffer: Buffer = null; + +} diff --git a/src/db/js5/js5-database.ts b/src/db/js5/js5-database.ts index 83e972b..f42d514 100644 --- a/src/db/js5/js5-database.ts +++ b/src/db/js5/js5-database.ts @@ -1,9 +1,11 @@ import { IndexDatabase } from '../index-database'; import { Js5IndexEntity } from './js5-index-entity'; -import { Connection, createConnection, LoggerOptions } from 'typeorm'; +import { Connection, createConnection, LoggerOptions, Repository } from 'typeorm'; import { Js5FileType } from '../../config'; import { existsSync, mkdirSync } from 'graceful-fs'; import { join } from 'path'; +import { Js5UncompressedDataEntity } from './js5-uncompressed-data-entity'; +import { Js5CompressedDataEntity } from './js5-compressed-data-entity'; export interface Js5IndexEntityWhere { @@ -17,6 +19,9 @@ export interface Js5IndexEntityWhere { export class Js5Database extends IndexDatabase { + private _uncompressedDataRepo: Repository; + private _compressedDataRepo: Repository; + constructor( gameBuild: string, databasePath: string, @@ -33,17 +38,103 @@ export class Js5Database extends IndexDatabase { + return this._uncompressedDataRepo.findOne({ + where: { + gameBuild: this.gameBuild, + ...where + } + }); + } + + async getAllUncompressedData(where: Js5IndexEntityWhere): Promise { + return this._uncompressedDataRepo.find({ + where: { + gameBuild: this.gameBuild, + ...where + } + }); + } + + async saveUncompressedData(uncompressedDataEntity: Js5UncompressedDataEntity): Promise { + return this._uncompressedDataRepo.save(uncompressedDataEntity); + } + + async saveAllUncompressedData(uncompressedDataEntities: Js5UncompressedDataEntity[]): Promise { + await this._uncompressedDataRepo.save(uncompressedDataEntities, { + chunk: 500, + transaction: false, + reload: false, + listeners: false, + }); + } + + async upsertAllUncompressedData(uncompressedDataEntities: Js5UncompressedDataEntity[]): Promise { + const chunkSize = 100; + for (let i = 0; i < uncompressedDataEntities.length; i += chunkSize) { + const chunk = uncompressedDataEntities.slice(i, i + chunkSize); + await this._uncompressedDataRepo.upsert(chunk, { + conflictPaths: [ 'fileType', 'gameBuild', 'key', 'archiveKey', 'groupKey' ], + skipUpdateIfNoValuesChanged: true, + }); + } + } + + async getCompressedData(where: Js5IndexEntityWhere): Promise { + return this._compressedDataRepo.findOne({ + where: { + gameBuild: this.gameBuild, + ...where + } + }); + } + + async getAllCompressedData(where: Js5IndexEntityWhere): Promise { + return this._compressedDataRepo.find({ + where: { + gameBuild: this.gameBuild, + ...where + } + }); + } + + async saveCompressedData(compressedDataEntity: Js5CompressedDataEntity): Promise { + return this._compressedDataRepo.save(compressedDataEntity); + } + + async saveAllCompressedData(compressedDataEntities: Js5CompressedDataEntity[]): Promise { + await this._compressedDataRepo.save(compressedDataEntities, { + chunk: 500, + transaction: false, + reload: false, + listeners: false, + }); + } + + async upsertAllCompressedData(compressedDataEntities: Js5CompressedDataEntity[]): Promise { + const chunkSize = 100; + for (let i = 0; i < compressedDataEntities.length; i += chunkSize) { + const chunk = compressedDataEntities.slice(i, i + chunkSize); + await this._compressedDataRepo.upsert(chunk, { + conflictPaths: [ 'fileType', 'gameBuild', 'key', 'archiveKey', 'groupKey' ], + skipUpdateIfNoValuesChanged: true, + }); + } + } + override async upsertIndexes(indexEntities: Js5IndexEntity[]): Promise { const chunkSize = 100; for (let i = 0; i < indexEntities.length; i += chunkSize) { diff --git a/src/db/js5/js5-index-entity.ts b/src/db/js5/js5-index-entity.ts index efb98fd..5f0e7d6 100644 --- a/src/db/js5/js5-index-entity.ts +++ b/src/db/js5/js5-index-entity.ts @@ -49,8 +49,8 @@ export class Js5IndexEntity { @Column('integer', { name: 'file_size', nullable: false, default: 0 }) fileSize: number = 0; - @Column('blob', { name: 'data', nullable: true, default: null }) - data: Buffer = null; + // @Column('blob', { name: 'data', nullable: true, default: null }) + // data: Buffer = null; @Column('text', { name: 'compression_method', nullable: true, default: 'none' }) compressionMethod: CompressionMethod = 'none'; @@ -64,8 +64,8 @@ export class Js5IndexEntity { @Column('integer', { name: 'compressed_file_size', nullable: false, default: 0 }) compressedFileSize: number = 0; - @Column('blob', { name: 'compressed_data', nullable: true, default: null }) - compressedData: Buffer = null; + // @Column('blob', { name: 'compressed_data', nullable: true, default: null }) + // compressedData: Buffer = null; @Column('boolean', { nullable: true, default: false }) encrypted: boolean = false; diff --git a/src/db/js5/js5-uncompressed-data-entity.ts b/src/db/js5/js5-uncompressed-data-entity.ts new file mode 100644 index 0000000..918859e --- /dev/null +++ b/src/db/js5/js5-uncompressed-data-entity.ts @@ -0,0 +1,30 @@ +import { Column, Entity, Index, PrimaryColumn } from 'typeorm'; +import { Js5FileType } from '../../config'; +import { Buffer } from 'buffer'; + + +@Entity('js5_uncompressed_data') +@Index('uncompressed_data_identifier', [ + 'fileType', 'gameBuild', 'key', 'archiveKey', 'groupKey' +], { unique: true }) +export class Js5UncompressedDataEntity { + + @PrimaryColumn('text', { name: 'file_type', nullable: false, unique: false }) + fileType: Js5FileType; + + @PrimaryColumn('text', { name: 'game_build', nullable: false, unique: false }) + gameBuild: string; + + @PrimaryColumn('integer', { nullable: false, unique: false }) + key: number; + + @PrimaryColumn('integer', { name: 'archive_key', nullable: false, unique: false, default: -1 }) + archiveKey: number = -1; + + @PrimaryColumn('integer', { name: 'group_key', nullable: false, unique: false, default: -1 }) + groupKey: number = -1; + + @Column('blob', { name: 'buffer', nullable: true, default: null }) + buffer: Buffer = null; + +} diff --git a/src/file-system/js5/js5-archive.ts b/src/file-system/js5/js5-archive.ts index 44b2935..cb0357f 100644 --- a/src/file-system/js5/js5-archive.ts +++ b/src/file-system/js5/js5-archive.ts @@ -66,6 +66,18 @@ export class Js5Archive extends Js5FileBase { } } + async upsertGroupData(): Promise { + const groups = Array.from(this.groups.values()); + const uncompressed = groups.map(group => group.uncompressedData).filter(data => data?.buffer && data?.buffer?.length !== 0); + const compressed = groups.map(group => group.compressedData).filter(data => data?.buffer && data?.buffer?.length !== 0); + if (uncompressed.length) { + await this.fileStore.database.upsertAllUncompressedData(uncompressed); + } + if (compressed.length) { + await this.fileStore.database.upsertAllCompressedData(compressed); + } + } + async getGroup(groupKey: number): Promise; async getGroup(groupName: string): Promise; async getGroup(groupIdentifier: number | string): Promise; diff --git a/src/file-system/js5/js5-file-base.ts b/src/file-system/js5/js5-file-base.ts index cb0518f..ee5c481 100644 --- a/src/file-system/js5/js5-file-base.ts +++ b/src/file-system/js5/js5-file-base.ts @@ -1,10 +1,10 @@ -import { Js5IndexEntity } from '../../db/js5/js5-index-entity'; -import { Js5FileType } from '../../config'; -import { logger } from '@runejs/common'; import { Buffer } from 'buffer'; -import { Crc32 } from '@runejs/common/crc32'; import { createHash } from 'crypto'; +import { logger } from '@runejs/common'; +import { Crc32 } from '@runejs/common/crc32'; import { Js5FileStore } from './js5-file-store'; +import { Js5FileType } from '../../config'; +import { Js5IndexEntity, Js5CompressedDataEntity, Js5UncompressedDataEntity } from '../../db/js5'; export abstract class Js5FileBase { @@ -12,6 +12,8 @@ export abstract class Js5FileBase { readonly fileStore: Js5FileStore; index: Js5IndexEntity; + uncompressedData: Js5UncompressedDataEntity; + compressedData: Js5CompressedDataEntity; protected constructor( fileStore: Js5FileStore, @@ -22,16 +24,19 @@ export abstract class Js5FileBase { ) { this.fileStore = fileStore; this.index = new Js5IndexEntity(); - this.index.gameBuild = fileStore.gameBuild; - this.index.fileType = fileType; - this.index.key = key; - this.index.archiveKey = archiveKey; - this.index.groupKey = groupKey; + this.uncompressedData = new Js5UncompressedDataEntity(); + this.compressedData = new Js5CompressedDataEntity(); + this.uncompressedData.gameBuild = this.compressedData.gameBuild = this.index.gameBuild = fileStore.gameBuild; + this.uncompressedData.fileType = this.compressedData.fileType = this.index.fileType = fileType; + this.uncompressedData.key = this.compressedData.key = this.index.key = key; + this.uncompressedData.archiveKey = this.compressedData.archiveKey = this.index.archiveKey = archiveKey; + this.uncompressedData.groupKey = this.compressedData.groupKey = this.index.groupKey = groupKey; } validate(trackChanges: boolean = true): void { + const data = this.uncompressedData.buffer; + const compressedData = this.compressedData.buffer; const { - data, compressedData, checksum, shaDigest, fileSize, compressedChecksum, compressedShaDigest, compressedFileSize, name, nameHash, @@ -95,12 +100,102 @@ export abstract class Js5FileBase { fileModified = true; } + if (compressedFileSize === fileSize && this.compressedData?.buffer?.length) { + // File has no compression, clear the compressed data buffer so that we do not create a + // duplicate data entity record for it + this.compressedData.buffer = null; + } + if (fileModified && trackChanges) { logger.info(`File ${this.index.name || this.index.key} has been modified.`); this.index.version++; } } + async saveUncompressedData(): Promise { + if (!this.uncompressedData?.buffer?.length) { + // Do not save a record for files with missing or empty data + return null; + } + + this.uncompressedData = await this.fileStore.database.saveUncompressedData(this.uncompressedData); + return this.uncompressedData; + } + + async loadUncompressedData(): Promise { + const entity = await this.fileStore.database.getUncompressedData({ + fileType: this.index.fileType, + key: this.index.key, + archiveKey: this.index.archiveKey, + groupKey: this.index.groupKey, + }); + + if (entity) { + this.uncompressedData = entity; + } + + return this.uncompressedData; + } + + async getUncompressedData(): Promise { + if (this.uncompressedData?.buffer?.length) { + return this.uncompressedData.buffer; + } + + const uncompressedData = await this.loadUncompressedData(); + if (uncompressedData?.buffer?.length) { + return uncompressedData.buffer; + } + + return null; + } + + async saveCompressedData(): Promise { + if (!this.compressedData?.buffer?.length) { + // Do not save a record for files with missing or empty data + return null; + } + + this.compressedData = await this.fileStore.database.saveCompressedData(this.compressedData); + return this.compressedData; + } + + async loadCompressedData(): Promise { + const entity = await this.fileStore.database.getCompressedData({ + fileType: this.index.fileType, + key: this.index.key, + archiveKey: this.index.archiveKey, + groupKey: this.index.groupKey, + }); + + if (entity) { + this.compressedData = entity; + } + + return this.compressedData; + } + + async getCompressedData(): Promise { + if (!this.index) { + await this.loadIndex(); + } + + if (this.index.compressionMethod === 'none') { + return this.getUncompressedData(); + } + + if (this.compressedData?.buffer?.length) { + return this.compressedData.buffer; + } + + const compressedData = await this.loadCompressedData(); + if (compressedData?.buffer?.length) { + return compressedData.buffer; + } + + return null; + } + async saveIndex(): Promise { this.validate(); this.index = await this.fileStore.database.saveIndex(this.index); diff --git a/src/file-system/js5/js5-group.ts b/src/file-system/js5/js5-group.ts index ddc6ff1..8fa9472 100644 --- a/src/file-system/js5/js5-group.ts +++ b/src/file-system/js5/js5-group.ts @@ -67,6 +67,18 @@ export class Js5Group extends Js5FileBase { } } + async upsertFileData(): Promise { + const files = Array.from(this.files.values()); + const uncompressed = files.map(group => group.uncompressedData).filter(data => data?.buffer && data?.buffer?.length !== 0); + const compressed = files.map(group => group.compressedData).filter(data => data?.buffer && data?.buffer?.length !== 0); + if (uncompressed.length) { + await this.fileStore.database.upsertAllUncompressedData(uncompressed); + } + if (compressed.length) { + await this.fileStore.database.upsertAllCompressedData(compressed); + } + } + async getFile(fileKey: number): Promise; async getFile(fileName: string): Promise; async getFile(fileIdentifier: number | string): Promise; diff --git a/src/file-system/js5/js5.ts b/src/file-system/js5/js5.ts index 8779f42..63f3ff2 100644 --- a/src/file-system/js5/js5.ts +++ b/src/file-system/js5/js5.ts @@ -226,23 +226,23 @@ export class JS5 { } while (remainingData > 0); if (data.length) { - fileIndex.compressedData = data.toNodeBuffer(); + file.compressedData.buffer = data.toNodeBuffer(); } else { - fileIndex.compressedData = null; + file.compressedData.buffer = null; fileIndex.fileError = 'FILE_MISSING'; } - return fileIndex.compressedData; + return file.compressedData.buffer; } readCompressedFileHeader(file: Js5Group | Js5Archive): { compressedLength: number, readerIndex: number } { const fileDetails = file.index; - if (!fileDetails.compressedData?.length) { + if (!file.compressedData.buffer?.length) { return { compressedLength: 0, readerIndex: 0 }; } - const compressedData = new ByteBuffer(fileDetails.compressedData); + const compressedData = new ByteBuffer(file.compressedData.buffer); fileDetails.compressionMethod = getCompressionMethod( compressedData.get('byte', 'unsigned')); @@ -257,7 +257,7 @@ export class JS5 { const fileDetails = file.index; const fileName = fileDetails.name; - if (!fileDetails.compressedData?.length) { + if (!file.compressedData.buffer?.length) { logger.error(`Error decrypting file ${ fileName || fileDetails.key }, file data not found.`, `Please ensure that the file has been unpacked from an existing JS5 file store using JS5.unpack(file);`); return null; @@ -273,10 +273,10 @@ export class JS5 { // Only XTEA encryption is supported at this time if (encryption !== 'xtea' || !patternRegex.test(fileName)) { // FileBase name does not match the pattern, data should be unencrypted - return fileDetails.compressedData; + return file.compressedData.buffer; } } else { - return fileDetails.compressedData; + return file.compressedData.buffer; } const gameBuild = this.fileStore.gameBuild; @@ -293,7 +293,7 @@ export class JS5 { try { const { compressedLength, readerIndex } = this.readCompressedFileHeader(file); - const encryptedData = new ByteBuffer(fileDetails.compressedData); + const encryptedData = new ByteBuffer(file.compressedData.buffer); const keySet = keySets.find(keySet => keySet.gameBuild === gameBuild); if (Xtea.validKeys(keySet?.key)) { @@ -310,10 +310,10 @@ export class JS5 { if (decryptedData?.length) { decryptedData.copy(dataCopy, readerIndex, 0); dataCopy.readerIndex = readerIndex; - fileDetails.compressedData = dataCopy.toNodeBuffer(); + file.compressedData.buffer = dataCopy.toNodeBuffer(); fileDetails.encrypted = false; file.validate(false); - return fileDetails.compressedData; + return file.compressedData.buffer; } else { logger.warn(`Invalid XTEA decryption keys found for file ` + `${ fileName || fileDetails.key } using game build ${ gameBuild }.`); @@ -334,7 +334,7 @@ export class JS5 { decompress(file: Js5Group | Js5Archive): Buffer | null { const fileDetails = file.index; - if (!fileDetails.compressedData?.length) { + if (!file.compressedData.buffer?.length) { return null; } @@ -349,7 +349,7 @@ export class JS5 { const { compressedLength, readerIndex } = this.readCompressedFileHeader(file); // JS5.decrypt will set compressedData to the new decrypted data after completion - const compressedData = new ByteBuffer(fileDetails.compressedData); + const compressedData = new ByteBuffer(file.compressedData.buffer); compressedData.readerIndex = readerIndex; if (fileDetails.compressionMethod === 'none') { @@ -398,9 +398,9 @@ export class JS5 { data = null; } - fileDetails.data = data; + file.uncompressedData.buffer = data; file.validate(false); - return fileDetails.data; + return file.uncompressedData.buffer; } async decodeGroup(group: Js5Group): Promise { @@ -408,10 +408,10 @@ export class JS5 { const { key: groupKey, name: groupName } = groupDetails; const files = group.files; - if (!groupDetails.data) { + if (!group.uncompressedData.buffer) { this.decompress(group); - if (!groupDetails.data) { + if (!group.uncompressedData.buffer) { if (!groupDetails.fileError) { logger.warn(`Unable to decode group ${ groupName || groupKey }.`); } @@ -419,7 +419,7 @@ export class JS5 { } } - const data = new ByteBuffer(groupDetails.data); + const data = new ByteBuffer(group.uncompressedData.buffer); if (groupDetails.childCount === 1) { return; @@ -491,7 +491,7 @@ export class JS5 { } for (const [ fileIndex, file ] of files) { - file.index.data = fileDataMap.get(fileIndex).toNodeBuffer(); + file.uncompressedData.buffer = fileDataMap.get(fileIndex).toNodeBuffer(); file.validate(false); } @@ -509,16 +509,16 @@ export class JS5 { logger.info(`Decoding archive ${ archiveName }...`); - if (!archiveDetails.data) { + if (!archive.uncompressedData.buffer) { this.decompress(archive); - if (!archiveDetails.data) { + if (!archive.uncompressedData.buffer) { logger.error(`Unable to decode archive ${ archiveName }.`); return; } } - const archiveData = new ByteBuffer(archiveDetails.data); + const archiveData = new ByteBuffer(archive.uncompressedData.buffer); const format = archiveDetails.archiveFormat = archiveData.get('byte', 'unsigned'); const mainDataType = format >= ArchiveFormat.smart ? 'smart_int' : 'short'; archiveDetails.version = format >= ArchiveFormat.versioned ? archiveData.get('int') : 0; @@ -633,11 +633,11 @@ export class JS5 { compress(file: Js5Group | Js5Archive): Buffer | null { const fileDetails = file.index; - if (!fileDetails.data?.length) { + if (!file.uncompressedData.buffer?.length) { return null; } - const decompressedData = new ByteBuffer(fileDetails.data); + const decompressedData = new ByteBuffer(file.uncompressedData.buffer); let data: ByteBuffer; if (fileDetails.compressionMethod === 'none') { @@ -677,10 +677,10 @@ export class JS5 { } if (data?.length) { - fileDetails.compressedData = data.toNodeBuffer(); + file.compressedData.buffer = data.toNodeBuffer(); } - return fileDetails.compressedData; + return file.compressedData.buffer; } encodeGroup(group: Js5Group): Buffer | null { @@ -689,8 +689,8 @@ export class JS5 { // Single-file group if (fileCount <= 1) { - index.data = fileMap.get(0)?.index?.data || null; - return index.data; + group.uncompressedData.buffer = fileMap.get(0)?.uncompressedData?.buffer || null; + return group.uncompressedData.buffer; } // Multi-file group @@ -709,7 +709,7 @@ export class JS5 { // Write child file data for (let stripe = 0; stripe < stripeCount; stripe++) { files.forEach(file => { - const fileData = file.index.data; + const fileData = file.uncompressedData.buffer; if (!fileData?.length) { return; } @@ -729,7 +729,7 @@ export class JS5 { for (let stripe = 0; stripe < stripeCount; stripe++) { let prevSize = 0; files.forEach(file => { - const fileData = file.index.data; + const fileData = file.uncompressedData.buffer; if (!fileData?.length) { return; } @@ -744,12 +744,12 @@ export class JS5 { groupBuffer.put(stripeCount, 'byte'); if (groupBuffer.length) { - index.data = groupBuffer.toNodeBuffer(); + group.uncompressedData.buffer = groupBuffer.toNodeBuffer(); } else { - index.data = null; + group.uncompressedData.buffer = null; } - return index.data; + return group.uncompressedData.buffer; } // @todo support newer archive fields & formats - 21/07/22 - Kiko @@ -819,12 +819,12 @@ export class JS5 { const archiveIndexData = buffer?.flipWriter(); if (archiveIndexData?.length) { - index.data = archiveIndexData.toNodeBuffer(); + archive.uncompressedData.buffer = archiveIndexData.toNodeBuffer(); } else { - index.data = null; + archive.uncompressedData.buffer = null; } - return index.data; + return archive.uncompressedData.buffer; } encodeMainIndex(): ByteBuffer { diff --git a/src/http/js5/js5.service.ts b/src/http/js5/js5.service.ts index 8323780..1dfadcd 100644 --- a/src/http/js5/js5.service.ts +++ b/src/http/js5/js5.service.ts @@ -23,12 +23,12 @@ export class Js5Service { archiveIdentifier: string | number, groupIdentifier: string | number, fileIdentifier: string | number - ): Promise { + ): Promise { const fileStore = await this.getFileStore(gameBuild); const archive = await fileStore.getArchive(archiveIdentifier); const group = await archive.getGroup(groupIdentifier); const file = await group.getFile(fileIdentifier); - return file.index.compressedData; + return file.getCompressedData(); } async getArchiveGroupFile( @@ -41,12 +41,7 @@ export class Js5Service { const archive = await fileStore.getArchive(archiveIdentifier); const group = await archive.getGroup(groupIdentifier); const file = await group.getFile(fileIdentifier); - - const index = { ...file.index }; - delete index.data; - delete index.compressedData; - - return index; + return file.index; } async getArchiveGroupFileList( @@ -60,27 +55,18 @@ export class Js5Service { await group.loadFileIndexes(); - const files = Array.from(group.files.values()); - const results: Js5IndexEntity[] = new Array(files.length); - - for (let i = 0; i < files.length; i++) { - results[i] = { ...files[i].index }; - delete results[i].data; - delete results[i].compressedData; - } - - return results; + return Array.from(group.files.values()).map(file => file.index); } async getArchiveGroupData( gameBuild: string | number, archiveIdentifier: string | number, groupIdentifier: string | number - ): Promise { + ): Promise { const fileStore = await this.getFileStore(gameBuild); const archive = await fileStore.getArchive(archiveIdentifier); const group = await archive.getGroup(groupIdentifier); - return group.index.compressedData; + return group.getCompressedData(); } async getArchiveGroup( @@ -91,12 +77,7 @@ export class Js5Service { const fileStore = await this.getFileStore(gameBuild); const archive = await fileStore.getArchive(archiveIdentifier); const group = await archive.getGroup(groupIdentifier); - - const index = { ...group.index }; - delete index.data; - delete index.compressedData; - - return index; + return group.index; } async getArchiveGroupList( @@ -108,25 +89,16 @@ export class Js5Service { await archive.loadGroupIndexes(); - const groups = Array.from(archive.groups.values()); - const results: Js5IndexEntity[] = new Array(groups.length); - - for (let i = 0; i < groups.length; i++) { - results[i] = { ...groups[i].index }; - delete results[i].data; - delete results[i].compressedData; - } - - return results; + return Array.from(archive.groups.values()).map(group => group.index); } async getArchiveData( gameBuild: string | number, archiveIdentifier: string | number - ): Promise { + ): Promise { const fileStore = await this.getFileStore(gameBuild); const archive = await fileStore.getArchive(archiveIdentifier); - return archive.index.compressedData; + return archive.getCompressedData(); } async getArchive( @@ -135,27 +107,12 @@ export class Js5Service { ): Promise { const fileStore = await this.getFileStore(gameBuild); const archive = await fileStore.getArchive(archiveIdentifier); - - const index = { ...archive.index }; - delete index.data; - delete index.compressedData; - - return index; + return archive.index; } async getArchiveList(gameBuild: string | number): Promise { const fileStore = await this.getFileStore(gameBuild); - - const archives = Array.from(fileStore.archives.values()); - const results: Js5IndexEntity[] = new Array(archives.length); - - for (let i = 0; i < archives.length; i++) { - results[i] = { ...archives[i].index }; - delete results[i].data; - delete results[i].compressedData; - } - - return results; + return Array.from(fileStore.archives.values()).map(archive => archive.index); } async getFileStore(gameBuild: string | number): Promise { diff --git a/src/scripts/indexer.ts b/src/scripts/indexer.ts index af81733..3b61dae 100644 --- a/src/scripts/indexer.ts +++ b/src/scripts/indexer.ts @@ -104,6 +104,27 @@ const indexJS5Store = async (store: Js5FileStore) => { await group.upsertFileIndexes(); } } + + logger.info(`Saving archive data...`); + + for (const [ , archive ] of store.archives) { + await archive.saveUncompressedData(); + await archive.saveCompressedData(); + } + + logger.info(`Saving group data...`); + + for (const [ , archive ] of store.archives) { + await archive.upsertGroupData(); + } + + logger.info(`Saving flat file data...`); + + for (const [ , archive ] of store.archives) { + for (const [ , group ] of archive.groups) { + await group.upsertFileData(); + } + } }; @@ -148,6 +169,21 @@ const indexJS5Archive = async (store: Js5FileStore, archiveName: string) => { for (const [ , group ] of archive.groups) { await group.upsertFileIndexes(); } + + logger.info(`Saving archive ${ archiveName } data...`); + + await archive.saveUncompressedData(); + await archive.saveCompressedData(); + + logger.info(`Saving group data...`); + + await archive.upsertGroupData(); + + logger.info(`Saving flat file data...`); + + for (const [ , group ] of archive.groups) { + await group.upsertFileData(); + } }; From 1e0b45e2b8f1bab56af5ddcc4c97c271aec06830 Mon Sep 17 00:00:00 2001 From: Kikorono Date: Thu, 1 Sep 2022 12:59:40 -0500 Subject: [PATCH 31/32] Merging uncompressed and compressed data tables + adding archive-config route to the js5 http server --- package-lock.json | 634 ++++++++---------- package.json | 8 +- src/db/js5/index.ts | 3 +- ...ssed-data-entity.ts => js5-data-entity.ts} | 13 +- src/db/js5/js5-database.ts | 69 +- src/db/js5/js5-index-entity.ts | 6 - src/db/js5/js5-uncompressed-data-entity.ts | 30 - src/file-system/jag/jag.ts | 7 +- src/file-system/js5/js5-archive.ts | 2 +- src/file-system/js5/js5-file-base.ts | 49 +- src/file-system/js5/js5-group.ts | 2 +- src/file-system/js5/js5.ts | 45 +- src/http/index.ts | 2 + src/http/jag-routes.ts | 0 src/http/js5-routes.ts | 7 - src/http/js5/js5.controller.ts | 8 + src/http/js5/js5.service.ts | 6 + src/http/server.ts | 4 +- src/scripts/dev.ts | 9 +- 19 files changed, 395 insertions(+), 509 deletions(-) rename src/db/js5/{js5-compressed-data-entity.ts => js5-data-entity.ts} (74%) delete mode 100644 src/db/js5/js5-uncompressed-data-entity.ts delete mode 100644 src/http/jag-routes.ts delete mode 100644 src/http/js5-routes.ts diff --git a/package-lock.json b/package-lock.json index 903f302..6845958 100644 --- a/package-lock.json +++ b/package-lock.json @@ -11,7 +11,7 @@ "dependencies": { "@decorators/di": "^1.0.3", "@decorators/express": "^2.6.0", - "@runejs/common": "3.0.0-beta.8", + "@runejs/common": "3.0.0-beta.9", "adm-zip": "^0.5.9", "axios": "^0.27.2", "compressjs": "^1.0.3", @@ -41,12 +41,6 @@ "source-map-support": "^0.5.21", "ts-node-dev": "^2.0.0", "typescript": "^4.7.4" - }, - "peerDependencies": { - "@runejs/common": "3.0.0-beta.8", - "graceful-fs": ">=4.2.0", - "tslib": ">=2.4.0", - "typescript": ">=4.5.0" } }, "node_modules/@cspotcode/source-map-support": { @@ -85,14 +79,14 @@ } }, "node_modules/@eslint/eslintrc": { - "version": "1.3.0", - "resolved": "https://registry.npmjs.org/@eslint/eslintrc/-/eslintrc-1.3.0.tgz", - "integrity": "sha512-UWW0TMTmk2d7hLcWD1/e2g5HDM/HQ3csaLSqXCfqwh4uNDuNqlaKWXmEsL4Cs41Z0KnILNvwbHAah3C2yt06kw==", + "version": "1.3.1", + "resolved": "https://registry.npmjs.org/@eslint/eslintrc/-/eslintrc-1.3.1.tgz", + "integrity": "sha512-OhSY22oQQdw3zgPOOwdoj01l/Dzl1Z+xyUP33tkSN+aqyEhymJCcPHyXt+ylW8FSe0TfRC2VG+ROQOapD0aZSQ==", "dev": true, "dependencies": { "ajv": "^6.12.4", "debug": "^4.3.2", - "espree": "^9.3.2", + "espree": "^9.4.0", "globals": "^13.15.0", "ignore": "^5.2.0", "import-fresh": "^3.2.1", @@ -102,24 +96,9 @@ }, "engines": { "node": "^12.22.0 || ^14.17.0 || >=16.0.0" - } - }, - "node_modules/@eslint/eslintrc/node_modules/argparse": { - "version": "2.0.1", - "resolved": "https://registry.npmjs.org/argparse/-/argparse-2.0.1.tgz", - "integrity": "sha512-8+9WqebbFzpX9OR+Wa6O29asIogeRMzcGtAINdpMHHyAg10f05aSFVBbcEqGf/PXw1EjAZ+q2/bEBg3DvurK3Q==", - "dev": true - }, - "node_modules/@eslint/eslintrc/node_modules/js-yaml": { - "version": "4.1.0", - "resolved": "https://registry.npmjs.org/js-yaml/-/js-yaml-4.1.0.tgz", - "integrity": "sha512-wpxZs9NoxZaJESJGIZTyDEaYpl0FKSA+FB9aJiyemKhMwkxQg63h4T1KJgUGHpTqPDNRcmmYLugrRjJlBtWvRA==", - "dev": true, - "dependencies": { - "argparse": "^2.0.1" }, - "bin": { - "js-yaml": "bin/js-yaml.js" + "funding": { + "url": "https://opencollective.com/eslint" } }, "node_modules/@hapi/bourne": { @@ -151,6 +130,19 @@ "url": "https://github.com/sponsors/nzakas" } }, + "node_modules/@humanwhocodes/module-importer": { + "version": "1.0.1", + "resolved": "https://registry.npmjs.org/@humanwhocodes/module-importer/-/module-importer-1.0.1.tgz", + "integrity": "sha512-bxveV4V8v5Yb4ncFTT3rPSgZBOpCkjfK0y4oVVVJwIuDVBRMDXrPyXRL988i5ap9m9bnyEEjWfm5WkBmtffLfA==", + "dev": true, + "engines": { + "node": ">=12.22" + }, + "funding": { + "type": "github", + "url": "https://github.com/sponsors/nzakas" + } + }, "node_modules/@humanwhocodes/object-schema": { "version": "1.2.1", "resolved": "https://registry.npmjs.org/@humanwhocodes/object-schema/-/object-schema-1.2.1.tgz", @@ -190,11 +182,6 @@ "commander": "^2.20.0" } }, - "node_modules/@ledgerhq/compressjs/node_modules/commander": { - "version": "2.20.3", - "resolved": "https://registry.npmjs.org/commander/-/commander-2.20.3.tgz", - "integrity": "sha512-GpVkmM8vF2vQUkj2LvZmD35JxeJOLCwJ9cUkugyk2nuhbv3+mJvpLYYt+0+USMxE+oj+ey/lJEnhZw75x/OMcQ==" - }, "node_modules/@nodelib/fs.scandir": { "version": "2.1.5", "resolved": "https://registry.npmjs.org/@nodelib/fs.scandir/-/fs.scandir-2.1.5.tgz", @@ -231,9 +218,9 @@ } }, "node_modules/@runejs/common": { - "version": "3.0.0-beta.8", - "resolved": "https://registry.npmjs.org/@runejs/common/-/common-3.0.0-beta.8.tgz", - "integrity": "sha512-S/SqQIcOdmvv3XEQmDR8jM5FMpP2RZP3ovOmqACSetRmjJIPObJXMI1lTA4Z5I0rylHM/5OiMiKIl8grCfZTGg==", + "version": "3.0.0-beta.9", + "resolved": "https://registry.npmjs.org/@runejs/common/-/common-3.0.0-beta.9.tgz", + "integrity": "sha512-KjiIhnyq0PxJ0HFHSIXV6NpCqrYgZGfbt4fDHFEhOQTmGgs3UPT6i9Hcd/tgsmThcpjSVTltIEnb9ZAG1VclzA==", "dependencies": { "@ledgerhq/compressjs": "^1.3.2", "buffer": "^6.0.3", @@ -247,37 +234,6 @@ "tslib": ">=2.4.0" } }, - "node_modules/@runejs/common/node_modules/buffer": { - "version": "6.0.3", - "resolved": "https://registry.npmjs.org/buffer/-/buffer-6.0.3.tgz", - "integrity": "sha512-FTiCpNxtwiZZHEZbcbTIcZjERVICn9yq/pDFkTl95/AxzD1naBctN7YO68riM/gLSDY7sdrMby8hofADYuuqOA==", - "funding": [ - { - "type": "github", - "url": "https://github.com/sponsors/feross" - }, - { - "type": "patreon", - "url": "https://www.patreon.com/feross" - }, - { - "type": "consulting", - "url": "https://feross.org/support" - } - ], - "dependencies": { - "base64-js": "^1.3.1", - "ieee754": "^1.2.1" - } - }, - "node_modules/@runejs/common/node_modules/dotenv": { - "version": "16.0.1", - "resolved": "https://registry.npmjs.org/dotenv/-/dotenv-16.0.1.tgz", - "integrity": "sha512-1K6hR6wtk2FviQ4kEiSjFiH5rpzEVi8WW0x96aztHVMhEspNpc4DVOUTEHtEva5VThQ8IaBX1Pe4gSzpVVUsKQ==", - "engines": { - "node": ">=12" - } - }, "node_modules/@runejs/eslint-config": { "version": "1.1.0", "resolved": "https://registry.npmjs.org/@runejs/eslint-config/-/eslint-config-1.1.0.tgz", @@ -391,9 +347,9 @@ "dev": true }, "node_modules/@types/node": { - "version": "16.11.52", - "resolved": "https://registry.npmjs.org/@types/node/-/node-16.11.52.tgz", - "integrity": "sha512-GnstYouCa9kbYokBCWEVrszJ1P2rGAQpKrqACHKuixkaT8XGu8nsqHvEUIGqDs5vwtsJ7LrYqnPDKRD1V+M39A==", + "version": "16.11.56", + "resolved": "https://registry.npmjs.org/@types/node/-/node-16.11.56.tgz", + "integrity": "sha512-aFcUkv7EddxxOa/9f74DINReQ/celqH8DiB3fRYgVDM2Xm5QJL8sl80QKuAnGvwAsMn+H3IFA6WCrQh1CY7m1A==", "dev": true }, "node_modules/@types/qs": { @@ -431,9 +387,9 @@ "dev": true }, "node_modules/@types/yargs": { - "version": "17.0.11", - "resolved": "https://registry.npmjs.org/@types/yargs/-/yargs-17.0.11.tgz", - "integrity": "sha512-aB4y9UDUXTSMxmM4MH+YnuR0g5Cph3FLQBoWoMB21DSvFVAxRVEHEMx3TLh+zUZYMCQtKiqazz0Q4Rre31f/OA==", + "version": "17.0.12", + "resolved": "https://registry.npmjs.org/@types/yargs/-/yargs-17.0.12.tgz", + "integrity": "sha512-Nz4MPhecOFArtm81gFQvQqdV7XYCrWKx5uUt6GNHredFHn1i2mtWqXTON7EPXMtNi1qjtjEM/VCHDhcHsAMLXQ==", "dev": true, "dependencies": { "@types/yargs-parser": "*" @@ -451,14 +407,14 @@ "integrity": "sha512-fbF6oTd4sGGy0xjHPKAt+eS2CrxJ3+6gQ3FGcBoIJR2TLAyCkCyI8JqZNy+FeON0AhVgNJoUumVoZQjBFUqHkw==" }, "node_modules/@typescript-eslint/eslint-plugin": { - "version": "5.33.1", - "resolved": "https://registry.npmjs.org/@typescript-eslint/eslint-plugin/-/eslint-plugin-5.33.1.tgz", - "integrity": "sha512-S1iZIxrTvKkU3+m63YUOxYPKaP+yWDQrdhxTglVDVEVBf+aCSw85+BmJnyUaQQsk5TXFG/LpBu9fa+LrAQ91fQ==", + "version": "5.36.1", + "resolved": "https://registry.npmjs.org/@typescript-eslint/eslint-plugin/-/eslint-plugin-5.36.1.tgz", + "integrity": "sha512-iC40UK8q1tMepSDwiLbTbMXKDxzNy+4TfPWgIL661Ym0sD42vRcQU93IsZIrmi+x292DBr60UI/gSwfdVYexCA==", "dev": true, "dependencies": { - "@typescript-eslint/scope-manager": "5.33.1", - "@typescript-eslint/type-utils": "5.33.1", - "@typescript-eslint/utils": "5.33.1", + "@typescript-eslint/scope-manager": "5.36.1", + "@typescript-eslint/type-utils": "5.36.1", + "@typescript-eslint/utils": "5.36.1", "debug": "^4.3.4", "functional-red-black-tree": "^1.0.1", "ignore": "^5.2.0", @@ -484,14 +440,14 @@ } }, "node_modules/@typescript-eslint/parser": { - "version": "5.33.1", - "resolved": "https://registry.npmjs.org/@typescript-eslint/parser/-/parser-5.33.1.tgz", - "integrity": "sha512-IgLLtW7FOzoDlmaMoXdxG8HOCByTBXrB1V2ZQYSEV1ggMmJfAkMWTwUjjzagS6OkfpySyhKFkBw7A9jYmcHpZA==", + "version": "5.36.1", + "resolved": "https://registry.npmjs.org/@typescript-eslint/parser/-/parser-5.36.1.tgz", + "integrity": "sha512-/IsgNGOkBi7CuDfUbwt1eOqUXF9WGVBW9dwEe1pi+L32XrTsZIgmDFIi2RxjzsvB/8i+MIf5JIoTEH8LOZ368A==", "dev": true, "dependencies": { - "@typescript-eslint/scope-manager": "5.33.1", - "@typescript-eslint/types": "5.33.1", - "@typescript-eslint/typescript-estree": "5.33.1", + "@typescript-eslint/scope-manager": "5.36.1", + "@typescript-eslint/types": "5.36.1", + "@typescript-eslint/typescript-estree": "5.36.1", "debug": "^4.3.4" }, "engines": { @@ -511,13 +467,13 @@ } }, "node_modules/@typescript-eslint/scope-manager": { - "version": "5.33.1", - "resolved": "https://registry.npmjs.org/@typescript-eslint/scope-manager/-/scope-manager-5.33.1.tgz", - "integrity": "sha512-8ibcZSqy4c5m69QpzJn8XQq9NnqAToC8OdH/W6IXPXv83vRyEDPYLdjAlUx8h/rbusq6MkW4YdQzURGOqsn3CA==", + "version": "5.36.1", + "resolved": "https://registry.npmjs.org/@typescript-eslint/scope-manager/-/scope-manager-5.36.1.tgz", + "integrity": "sha512-pGC2SH3/tXdu9IH3ItoqciD3f3RRGCh7hb9zPdN2Drsr341zgd6VbhP5OHQO/reUqihNltfPpMpTNihFMarP2w==", "dev": true, "dependencies": { - "@typescript-eslint/types": "5.33.1", - "@typescript-eslint/visitor-keys": "5.33.1" + "@typescript-eslint/types": "5.36.1", + "@typescript-eslint/visitor-keys": "5.36.1" }, "engines": { "node": "^12.22.0 || ^14.17.0 || >=16.0.0" @@ -528,12 +484,13 @@ } }, "node_modules/@typescript-eslint/type-utils": { - "version": "5.33.1", - "resolved": "https://registry.npmjs.org/@typescript-eslint/type-utils/-/type-utils-5.33.1.tgz", - "integrity": "sha512-X3pGsJsD8OiqhNa5fim41YtlnyiWMF/eKsEZGsHID2HcDqeSC5yr/uLOeph8rNF2/utwuI0IQoAK3fpoxcLl2g==", + "version": "5.36.1", + "resolved": "https://registry.npmjs.org/@typescript-eslint/type-utils/-/type-utils-5.36.1.tgz", + "integrity": "sha512-xfZhfmoQT6m3lmlqDvDzv9TiCYdw22cdj06xY0obSznBsT///GK5IEZQdGliXpAOaRL34o8phEvXzEo/VJx13Q==", "dev": true, "dependencies": { - "@typescript-eslint/utils": "5.33.1", + "@typescript-eslint/typescript-estree": "5.36.1", + "@typescript-eslint/utils": "5.36.1", "debug": "^4.3.4", "tsutils": "^3.21.0" }, @@ -554,9 +511,9 @@ } }, "node_modules/@typescript-eslint/types": { - "version": "5.33.1", - "resolved": "https://registry.npmjs.org/@typescript-eslint/types/-/types-5.33.1.tgz", - "integrity": "sha512-7K6MoQPQh6WVEkMrMW5QOA5FO+BOwzHSNd0j3+BlBwd6vtzfZceJ8xJ7Um2XDi/O3umS8/qDX6jdy2i7CijkwQ==", + "version": "5.36.1", + "resolved": "https://registry.npmjs.org/@typescript-eslint/types/-/types-5.36.1.tgz", + "integrity": "sha512-jd93ShpsIk1KgBTx9E+hCSEuLCUFwi9V/urhjOWnOaksGZFbTOxAT47OH2d4NLJnLhkVD+wDbB48BuaycZPLBg==", "dev": true, "engines": { "node": "^12.22.0 || ^14.17.0 || >=16.0.0" @@ -567,13 +524,13 @@ } }, "node_modules/@typescript-eslint/typescript-estree": { - "version": "5.33.1", - "resolved": "https://registry.npmjs.org/@typescript-eslint/typescript-estree/-/typescript-estree-5.33.1.tgz", - "integrity": "sha512-JOAzJ4pJ+tHzA2pgsWQi4804XisPHOtbvwUyqsuuq8+y5B5GMZs7lI1xDWs6V2d7gE/Ez5bTGojSK12+IIPtXA==", + "version": "5.36.1", + "resolved": "https://registry.npmjs.org/@typescript-eslint/typescript-estree/-/typescript-estree-5.36.1.tgz", + "integrity": "sha512-ih7V52zvHdiX6WcPjsOdmADhYMDN15SylWRZrT2OMy80wzKbc79n8wFW0xpWpU0x3VpBz/oDgTm2xwDAnFTl+g==", "dev": true, "dependencies": { - "@typescript-eslint/types": "5.33.1", - "@typescript-eslint/visitor-keys": "5.33.1", + "@typescript-eslint/types": "5.36.1", + "@typescript-eslint/visitor-keys": "5.36.1", "debug": "^4.3.4", "globby": "^11.1.0", "is-glob": "^4.0.3", @@ -594,15 +551,15 @@ } }, "node_modules/@typescript-eslint/utils": { - "version": "5.33.1", - "resolved": "https://registry.npmjs.org/@typescript-eslint/utils/-/utils-5.33.1.tgz", - "integrity": "sha512-uphZjkMaZ4fE8CR4dU7BquOV6u0doeQAr8n6cQenl/poMaIyJtBu8eys5uk6u5HiDH01Mj5lzbJ5SfeDz7oqMQ==", + "version": "5.36.1", + "resolved": "https://registry.npmjs.org/@typescript-eslint/utils/-/utils-5.36.1.tgz", + "integrity": "sha512-lNj4FtTiXm5c+u0pUehozaUWhh7UYKnwryku0nxJlYUEWetyG92uw2pr+2Iy4M/u0ONMKzfrx7AsGBTCzORmIg==", "dev": true, "dependencies": { "@types/json-schema": "^7.0.9", - "@typescript-eslint/scope-manager": "5.33.1", - "@typescript-eslint/types": "5.33.1", - "@typescript-eslint/typescript-estree": "5.33.1", + "@typescript-eslint/scope-manager": "5.36.1", + "@typescript-eslint/types": "5.36.1", + "@typescript-eslint/typescript-estree": "5.36.1", "eslint-scope": "^5.1.1", "eslint-utils": "^3.0.0" }, @@ -618,12 +575,12 @@ } }, "node_modules/@typescript-eslint/visitor-keys": { - "version": "5.33.1", - "resolved": "https://registry.npmjs.org/@typescript-eslint/visitor-keys/-/visitor-keys-5.33.1.tgz", - "integrity": "sha512-nwIxOK8Z2MPWltLKMLOEZwmfBZReqUdbEoHQXeCpa+sRVARe5twpJGHCB4dk9903Yaf0nMAlGbQfaAH92F60eg==", + "version": "5.36.1", + "resolved": "https://registry.npmjs.org/@typescript-eslint/visitor-keys/-/visitor-keys-5.36.1.tgz", + "integrity": "sha512-ojB9aRyRFzVMN3b5joSYni6FAS10BBSCAfKJhjJAV08t/a95aM6tAhz+O1jF+EtgxktuSO3wJysp2R+Def/IWQ==", "dev": true, "dependencies": { - "@typescript-eslint/types": "5.33.1", + "@typescript-eslint/types": "5.36.1", "eslint-visitor-keys": "^3.3.0" }, "engines": { @@ -762,6 +719,11 @@ "integrity": "sha512-58S9QDqG0Xx27YwPSt9fJxivjYl432YCwfDMfZ+71RAqUrZef7LrKQZ3LHLOwCS4FLNBplP533Zx895SeOCHvA==", "dev": true }, + "node_modules/argparse": { + "version": "2.0.1", + "resolved": "https://registry.npmjs.org/argparse/-/argparse-2.0.1.tgz", + "integrity": "sha512-8+9WqebbFzpX9OR+Wa6O29asIogeRMzcGtAINdpMHHyAg10f05aSFVBbcEqGf/PXw1EjAZ+q2/bEBg3DvurK3Q==" + }, "node_modules/args": { "version": "5.0.3", "resolved": "https://registry.npmjs.org/args/-/args-5.0.3.tgz", @@ -940,6 +902,30 @@ "readable-stream": "^3.4.0" } }, + "node_modules/bl/node_modules/buffer": { + "version": "5.7.1", + "resolved": "https://registry.npmjs.org/buffer/-/buffer-5.7.1.tgz", + "integrity": "sha512-EHcyIPBQ4BSGlvjB16k5KgAJ27CIsHY/2JBmCRReo48y9rQ3MaUzWX3KVlBa4U7MyX02HdVj0K7C3WaB3ju7FQ==", + "devOptional": true, + "funding": [ + { + "type": "github", + "url": "https://github.com/sponsors/feross" + }, + { + "type": "patreon", + "url": "https://www.patreon.com/feross" + }, + { + "type": "consulting", + "url": "https://feross.org/support" + } + ], + "dependencies": { + "base64-js": "^1.3.1", + "ieee754": "^1.1.13" + } + }, "node_modules/bl/node_modules/readable-stream": { "version": "3.6.0", "resolved": "https://registry.npmjs.org/readable-stream/-/readable-stream-3.6.0.tgz", @@ -1021,10 +1007,9 @@ } }, "node_modules/buffer": { - "version": "5.7.1", - "resolved": "https://registry.npmjs.org/buffer/-/buffer-5.7.1.tgz", - "integrity": "sha512-EHcyIPBQ4BSGlvjB16k5KgAJ27CIsHY/2JBmCRReo48y9rQ3MaUzWX3KVlBa4U7MyX02HdVj0K7C3WaB3ju7FQ==", - "devOptional": true, + "version": "6.0.3", + "resolved": "https://registry.npmjs.org/buffer/-/buffer-6.0.3.tgz", + "integrity": "sha512-FTiCpNxtwiZZHEZbcbTIcZjERVICn9yq/pDFkTl95/AxzD1naBctN7YO68riM/gLSDY7sdrMby8hofADYuuqOA==", "funding": [ { "type": "github", @@ -1041,7 +1026,7 @@ ], "dependencies": { "base64-js": "^1.3.1", - "ieee754": "^1.1.13" + "ieee754": "^1.2.1" } }, "node_modules/buffer-from": { @@ -1235,15 +1220,9 @@ } }, "node_modules/commander": { - "version": "2.8.1", - "resolved": "https://registry.npmjs.org/commander/-/commander-2.8.1.tgz", - "integrity": "sha512-+pJLBFVk+9ZZdlAOB5WuIElVPPth47hILFkmGym57aq8kwxsowvByvB0DHs1vQAhyMZzdcpTtF0VDKGkSDR4ZQ==", - "dependencies": { - "graceful-readlink": ">= 1.0.0" - }, - "engines": { - "node": ">= 0.6.x" - } + "version": "2.20.3", + "resolved": "https://registry.npmjs.org/commander/-/commander-2.20.3.tgz", + "integrity": "sha512-GpVkmM8vF2vQUkj2LvZmD35JxeJOLCwJ9cUkugyk2nuhbv3+mJvpLYYt+0+USMxE+oj+ey/lJEnhZw75x/OMcQ==" }, "node_modules/compressjs": { "version": "1.0.3", @@ -1257,6 +1236,17 @@ "compressjs": "bin/compressjs" } }, + "node_modules/compressjs/node_modules/commander": { + "version": "2.8.1", + "resolved": "https://registry.npmjs.org/commander/-/commander-2.8.1.tgz", + "integrity": "sha512-+pJLBFVk+9ZZdlAOB5WuIElVPPth47hILFkmGym57aq8kwxsowvByvB0DHs1vQAhyMZzdcpTtF0VDKGkSDR4ZQ==", + "dependencies": { + "graceful-readlink": ">= 1.0.0" + }, + "engines": { + "node": ">= 0.6.x" + } + }, "node_modules/concat-map": { "version": "0.0.1", "resolved": "https://registry.npmjs.org/concat-map/-/concat-map-0.0.1.tgz", @@ -1488,11 +1478,11 @@ } }, "node_modules/dotenv": { - "version": "8.6.0", - "resolved": "https://registry.npmjs.org/dotenv/-/dotenv-8.6.0.tgz", - "integrity": "sha512-IrPdXQsk2BbzvCBGBOTmmSH5SodmqZNt4ERAZDmW4CT+tL8VtvinqywuANaFu4bOMWki16nqf0e4oC0QIaDr/g==", + "version": "16.0.2", + "resolved": "https://registry.npmjs.org/dotenv/-/dotenv-16.0.2.tgz", + "integrity": "sha512-JvpYKUmzQhYoIFgK2MOnF3bciIZoItIIoryihy0rIA+H4Jy0FmgyKYAHCTN98P5ybGSJcIFbh6QKeJdtZd1qhA==", "engines": { - "node": ">=10" + "node": ">=12" } }, "node_modules/dynamic-dedupe": { @@ -1556,14 +1546,15 @@ } }, "node_modules/eslint": { - "version": "8.22.0", - "resolved": "https://registry.npmjs.org/eslint/-/eslint-8.22.0.tgz", - "integrity": "sha512-ci4t0sz6vSRKdmkOGmprBo6fmI4PrphDFMy5JEq/fNS0gQkJM3rLmrqcp8ipMcdobH3KtUP40KniAE9W19S4wA==", + "version": "8.23.0", + "resolved": "https://registry.npmjs.org/eslint/-/eslint-8.23.0.tgz", + "integrity": "sha512-pBG/XOn0MsJcKcTRLr27S5HpzQo4kLr+HjLQIyK4EiCsijDl/TB+h5uEuJU6bQ8Edvwz1XWOjpaP2qgnXGpTcA==", "dev": true, "dependencies": { - "@eslint/eslintrc": "^1.3.0", + "@eslint/eslintrc": "^1.3.1", "@humanwhocodes/config-array": "^0.10.4", "@humanwhocodes/gitignore-to-minimatch": "^1.0.2", + "@humanwhocodes/module-importer": "^1.0.1", "ajv": "^6.10.0", "chalk": "^4.0.0", "cross-spawn": "^7.0.2", @@ -1573,7 +1564,7 @@ "eslint-scope": "^7.1.1", "eslint-utils": "^3.0.0", "eslint-visitor-keys": "^3.3.0", - "espree": "^9.3.3", + "espree": "^9.4.0", "esquery": "^1.4.0", "esutils": "^2.0.2", "fast-deep-equal": "^3.1.3", @@ -1598,8 +1589,7 @@ "regexpp": "^3.2.0", "strip-ansi": "^6.0.1", "strip-json-comments": "^3.1.0", - "text-table": "^0.2.0", - "v8-compile-cache": "^2.0.3" + "text-table": "^0.2.0" }, "bin": { "eslint": "bin/eslint.js" @@ -1660,12 +1650,6 @@ "node": "^12.22.0 || ^14.17.0 || >=16.0.0" } }, - "node_modules/eslint/node_modules/argparse": { - "version": "2.0.1", - "resolved": "https://registry.npmjs.org/argparse/-/argparse-2.0.1.tgz", - "integrity": "sha512-8+9WqebbFzpX9OR+Wa6O29asIogeRMzcGtAINdpMHHyAg10f05aSFVBbcEqGf/PXw1EjAZ+q2/bEBg3DvurK3Q==", - "dev": true - }, "node_modules/eslint/node_modules/eslint-scope": { "version": "7.1.1", "resolved": "https://registry.npmjs.org/eslint-scope/-/eslint-scope-7.1.1.tgz", @@ -1688,22 +1672,10 @@ "node": ">=4.0" } }, - "node_modules/eslint/node_modules/js-yaml": { - "version": "4.1.0", - "resolved": "https://registry.npmjs.org/js-yaml/-/js-yaml-4.1.0.tgz", - "integrity": "sha512-wpxZs9NoxZaJESJGIZTyDEaYpl0FKSA+FB9aJiyemKhMwkxQg63h4T1KJgUGHpTqPDNRcmmYLugrRjJlBtWvRA==", - "dev": true, - "dependencies": { - "argparse": "^2.0.1" - }, - "bin": { - "js-yaml": "bin/js-yaml.js" - } - }, "node_modules/espree": { - "version": "9.3.3", - "resolved": "https://registry.npmjs.org/espree/-/espree-9.3.3.tgz", - "integrity": "sha512-ORs1Rt/uQTqUKjDdGCyrtYxbazf5umATSf/K4qxjmZHORR6HJk+2s/2Pqe+Kk49HHINC/xNIrGfgh8sZcll0ng==", + "version": "9.4.0", + "resolved": "https://registry.npmjs.org/espree/-/espree-9.4.0.tgz", + "integrity": "sha512-DQmnRpLj7f6TgN/NYb0MTzJXL+vJF9h3pHy4JhCIs3zwcgez8xmGg3sXHcEO97BrmO2OSvCwMdfdlyl+E9KjOw==", "dev": true, "dependencies": { "acorn": "^8.8.0", @@ -2444,6 +2416,17 @@ "node": ">=10" } }, + "node_modules/js-yaml": { + "version": "4.1.0", + "resolved": "https://registry.npmjs.org/js-yaml/-/js-yaml-4.1.0.tgz", + "integrity": "sha512-wpxZs9NoxZaJESJGIZTyDEaYpl0FKSA+FB9aJiyemKhMwkxQg63h4T1KJgUGHpTqPDNRcmmYLugrRjJlBtWvRA==", + "dependencies": { + "argparse": "^2.0.1" + }, + "bin": { + "js-yaml": "bin/js-yaml.js" + } + }, "node_modules/json-schema-traverse": { "version": "0.4.1", "resolved": "https://registry.npmjs.org/json-schema-traverse/-/json-schema-traverse-0.4.1.tgz", @@ -4045,49 +4028,18 @@ } } }, - "node_modules/typeorm/node_modules/argparse": { - "version": "2.0.1", - "resolved": "https://registry.npmjs.org/argparse/-/argparse-2.0.1.tgz", - "integrity": "sha512-8+9WqebbFzpX9OR+Wa6O29asIogeRMzcGtAINdpMHHyAg10f05aSFVBbcEqGf/PXw1EjAZ+q2/bEBg3DvurK3Q==" - }, - "node_modules/typeorm/node_modules/buffer": { - "version": "6.0.3", - "resolved": "https://registry.npmjs.org/buffer/-/buffer-6.0.3.tgz", - "integrity": "sha512-FTiCpNxtwiZZHEZbcbTIcZjERVICn9yq/pDFkTl95/AxzD1naBctN7YO68riM/gLSDY7sdrMby8hofADYuuqOA==", - "funding": [ - { - "type": "github", - "url": "https://github.com/sponsors/feross" - }, - { - "type": "patreon", - "url": "https://www.patreon.com/feross" - }, - { - "type": "consulting", - "url": "https://feross.org/support" - } - ], - "dependencies": { - "base64-js": "^1.3.1", - "ieee754": "^1.2.1" - } - }, - "node_modules/typeorm/node_modules/js-yaml": { - "version": "4.1.0", - "resolved": "https://registry.npmjs.org/js-yaml/-/js-yaml-4.1.0.tgz", - "integrity": "sha512-wpxZs9NoxZaJESJGIZTyDEaYpl0FKSA+FB9aJiyemKhMwkxQg63h4T1KJgUGHpTqPDNRcmmYLugrRjJlBtWvRA==", - "dependencies": { - "argparse": "^2.0.1" - }, - "bin": { - "js-yaml": "bin/js-yaml.js" + "node_modules/typeorm/node_modules/dotenv": { + "version": "8.6.0", + "resolved": "https://registry.npmjs.org/dotenv/-/dotenv-8.6.0.tgz", + "integrity": "sha512-IrPdXQsk2BbzvCBGBOTmmSH5SodmqZNt4ERAZDmW4CT+tL8VtvinqywuANaFu4bOMWki16nqf0e4oC0QIaDr/g==", + "engines": { + "node": ">=10" } }, "node_modules/typescript": { - "version": "4.7.4", - "resolved": "https://registry.npmjs.org/typescript/-/typescript-4.7.4.tgz", - "integrity": "sha512-C0WQT0gezHuw6AdY1M2jxUO83Rjf0HP7Sk1DtXj6j1EwkQNZrHAg2XPWlq62oqEhYvONq5pkC2Y9oPljWToLmQ==", + "version": "4.8.2", + "resolved": "https://registry.npmjs.org/typescript/-/typescript-4.8.2.tgz", + "integrity": "sha512-C0I1UsrrDHo2fYI5oaCGbSejwX4ch+9Y5jTQELvovfmFkK3HHSZJB8MSJcWLmCUBzQBchCrZ9rMRV6GuNrvGtw==", "dev": true, "bin": { "tsc": "bin/tsc", @@ -4157,12 +4109,6 @@ "uuid": "dist/bin/uuid" } }, - "node_modules/v8-compile-cache": { - "version": "2.3.0", - "resolved": "https://registry.npmjs.org/v8-compile-cache/-/v8-compile-cache-2.3.0.tgz", - "integrity": "sha512-l8lCEmLcLYZh4nbunNZvQCJc5pv7+RCwa8q/LdUx8u7lsWvPDKmpodJAJNwkAhJC//dFY48KuIEmjtd4RViDrA==", - "dev": true - }, "node_modules/v8-compile-cache-lib": { "version": "3.0.1", "resolved": "https://registry.npmjs.org/v8-compile-cache-lib/-/v8-compile-cache-lib-3.0.1.tgz", @@ -4360,37 +4306,20 @@ "requires": {} }, "@eslint/eslintrc": { - "version": "1.3.0", - "resolved": "https://registry.npmjs.org/@eslint/eslintrc/-/eslintrc-1.3.0.tgz", - "integrity": "sha512-UWW0TMTmk2d7hLcWD1/e2g5HDM/HQ3csaLSqXCfqwh4uNDuNqlaKWXmEsL4Cs41Z0KnILNvwbHAah3C2yt06kw==", + "version": "1.3.1", + "resolved": "https://registry.npmjs.org/@eslint/eslintrc/-/eslintrc-1.3.1.tgz", + "integrity": "sha512-OhSY22oQQdw3zgPOOwdoj01l/Dzl1Z+xyUP33tkSN+aqyEhymJCcPHyXt+ylW8FSe0TfRC2VG+ROQOapD0aZSQ==", "dev": true, "requires": { "ajv": "^6.12.4", "debug": "^4.3.2", - "espree": "^9.3.2", + "espree": "^9.4.0", "globals": "^13.15.0", "ignore": "^5.2.0", "import-fresh": "^3.2.1", "js-yaml": "^4.1.0", "minimatch": "^3.1.2", "strip-json-comments": "^3.1.1" - }, - "dependencies": { - "argparse": { - "version": "2.0.1", - "resolved": "https://registry.npmjs.org/argparse/-/argparse-2.0.1.tgz", - "integrity": "sha512-8+9WqebbFzpX9OR+Wa6O29asIogeRMzcGtAINdpMHHyAg10f05aSFVBbcEqGf/PXw1EjAZ+q2/bEBg3DvurK3Q==", - "dev": true - }, - "js-yaml": { - "version": "4.1.0", - "resolved": "https://registry.npmjs.org/js-yaml/-/js-yaml-4.1.0.tgz", - "integrity": "sha512-wpxZs9NoxZaJESJGIZTyDEaYpl0FKSA+FB9aJiyemKhMwkxQg63h4T1KJgUGHpTqPDNRcmmYLugrRjJlBtWvRA==", - "dev": true, - "requires": { - "argparse": "^2.0.1" - } - } } }, "@hapi/bourne": { @@ -4415,6 +4344,12 @@ "integrity": "sha512-rSqmMJDdLFUsyxR6FMtD00nfQKKLFb1kv+qBbOVKqErvloEIJLo5bDTJTQNTYgeyp78JsA7u/NPi5jT1GR/MuA==", "dev": true }, + "@humanwhocodes/module-importer": { + "version": "1.0.1", + "resolved": "https://registry.npmjs.org/@humanwhocodes/module-importer/-/module-importer-1.0.1.tgz", + "integrity": "sha512-bxveV4V8v5Yb4ncFTT3rPSgZBOpCkjfK0y4oVVVJwIuDVBRMDXrPyXRL988i5ap9m9bnyEEjWfm5WkBmtffLfA==", + "dev": true + }, "@humanwhocodes/object-schema": { "version": "1.2.1", "resolved": "https://registry.npmjs.org/@humanwhocodes/object-schema/-/object-schema-1.2.1.tgz", @@ -4449,13 +4384,6 @@ "integrity": "sha512-gonFwAifRkSYDO7rt3NIBlvjvY8Nw+NM6LT1SuOBppuvoKbYtBViNh3EBPbP86+3Y4ux7DLUsNiUlqOgubJsdA==", "requires": { "commander": "^2.20.0" - }, - "dependencies": { - "commander": { - "version": "2.20.3", - "resolved": "https://registry.npmjs.org/commander/-/commander-2.20.3.tgz", - "integrity": "sha512-GpVkmM8vF2vQUkj2LvZmD35JxeJOLCwJ9cUkugyk2nuhbv3+mJvpLYYt+0+USMxE+oj+ey/lJEnhZw75x/OMcQ==" - } } }, "@nodelib/fs.scandir": { @@ -4485,9 +4413,9 @@ } }, "@runejs/common": { - "version": "3.0.0-beta.8", - "resolved": "https://registry.npmjs.org/@runejs/common/-/common-3.0.0-beta.8.tgz", - "integrity": "sha512-S/SqQIcOdmvv3XEQmDR8jM5FMpP2RZP3ovOmqACSetRmjJIPObJXMI1lTA4Z5I0rylHM/5OiMiKIl8grCfZTGg==", + "version": "3.0.0-beta.9", + "resolved": "https://registry.npmjs.org/@runejs/common/-/common-3.0.0-beta.9.tgz", + "integrity": "sha512-KjiIhnyq0PxJ0HFHSIXV6NpCqrYgZGfbt4fDHFEhOQTmGgs3UPT6i9Hcd/tgsmThcpjSVTltIEnb9ZAG1VclzA==", "requires": { "@ledgerhq/compressjs": "^1.3.2", "buffer": "^6.0.3", @@ -4496,22 +4424,6 @@ "pino": "^6.14.0", "pino-pretty": "^6.0.0", "zlib": "^1.0.5" - }, - "dependencies": { - "buffer": { - "version": "6.0.3", - "resolved": "https://registry.npmjs.org/buffer/-/buffer-6.0.3.tgz", - "integrity": "sha512-FTiCpNxtwiZZHEZbcbTIcZjERVICn9yq/pDFkTl95/AxzD1naBctN7YO68riM/gLSDY7sdrMby8hofADYuuqOA==", - "requires": { - "base64-js": "^1.3.1", - "ieee754": "^1.2.1" - } - }, - "dotenv": { - "version": "16.0.1", - "resolved": "https://registry.npmjs.org/dotenv/-/dotenv-16.0.1.tgz", - "integrity": "sha512-1K6hR6wtk2FviQ4kEiSjFiH5rpzEVi8WW0x96aztHVMhEspNpc4DVOUTEHtEva5VThQ8IaBX1Pe4gSzpVVUsKQ==" - } } }, "@runejs/eslint-config": { @@ -4623,9 +4535,9 @@ "dev": true }, "@types/node": { - "version": "16.11.52", - "resolved": "https://registry.npmjs.org/@types/node/-/node-16.11.52.tgz", - "integrity": "sha512-GnstYouCa9kbYokBCWEVrszJ1P2rGAQpKrqACHKuixkaT8XGu8nsqHvEUIGqDs5vwtsJ7LrYqnPDKRD1V+M39A==", + "version": "16.11.56", + "resolved": "https://registry.npmjs.org/@types/node/-/node-16.11.56.tgz", + "integrity": "sha512-aFcUkv7EddxxOa/9f74DINReQ/celqH8DiB3fRYgVDM2Xm5QJL8sl80QKuAnGvwAsMn+H3IFA6WCrQh1CY7m1A==", "dev": true }, "@types/qs": { @@ -4663,9 +4575,9 @@ "dev": true }, "@types/yargs": { - "version": "17.0.11", - "resolved": "https://registry.npmjs.org/@types/yargs/-/yargs-17.0.11.tgz", - "integrity": "sha512-aB4y9UDUXTSMxmM4MH+YnuR0g5Cph3FLQBoWoMB21DSvFVAxRVEHEMx3TLh+zUZYMCQtKiqazz0Q4Rre31f/OA==", + "version": "17.0.12", + "resolved": "https://registry.npmjs.org/@types/yargs/-/yargs-17.0.12.tgz", + "integrity": "sha512-Nz4MPhecOFArtm81gFQvQqdV7XYCrWKx5uUt6GNHredFHn1i2mtWqXTON7EPXMtNi1qjtjEM/VCHDhcHsAMLXQ==", "dev": true, "requires": { "@types/yargs-parser": "*" @@ -4683,14 +4595,14 @@ "integrity": "sha512-fbF6oTd4sGGy0xjHPKAt+eS2CrxJ3+6gQ3FGcBoIJR2TLAyCkCyI8JqZNy+FeON0AhVgNJoUumVoZQjBFUqHkw==" }, "@typescript-eslint/eslint-plugin": { - "version": "5.33.1", - "resolved": "https://registry.npmjs.org/@typescript-eslint/eslint-plugin/-/eslint-plugin-5.33.1.tgz", - "integrity": "sha512-S1iZIxrTvKkU3+m63YUOxYPKaP+yWDQrdhxTglVDVEVBf+aCSw85+BmJnyUaQQsk5TXFG/LpBu9fa+LrAQ91fQ==", + "version": "5.36.1", + "resolved": "https://registry.npmjs.org/@typescript-eslint/eslint-plugin/-/eslint-plugin-5.36.1.tgz", + "integrity": "sha512-iC40UK8q1tMepSDwiLbTbMXKDxzNy+4TfPWgIL661Ym0sD42vRcQU93IsZIrmi+x292DBr60UI/gSwfdVYexCA==", "dev": true, "requires": { - "@typescript-eslint/scope-manager": "5.33.1", - "@typescript-eslint/type-utils": "5.33.1", - "@typescript-eslint/utils": "5.33.1", + "@typescript-eslint/scope-manager": "5.36.1", + "@typescript-eslint/type-utils": "5.36.1", + "@typescript-eslint/utils": "5.36.1", "debug": "^4.3.4", "functional-red-black-tree": "^1.0.1", "ignore": "^5.2.0", @@ -4700,52 +4612,53 @@ } }, "@typescript-eslint/parser": { - "version": "5.33.1", - "resolved": "https://registry.npmjs.org/@typescript-eslint/parser/-/parser-5.33.1.tgz", - "integrity": "sha512-IgLLtW7FOzoDlmaMoXdxG8HOCByTBXrB1V2ZQYSEV1ggMmJfAkMWTwUjjzagS6OkfpySyhKFkBw7A9jYmcHpZA==", + "version": "5.36.1", + "resolved": "https://registry.npmjs.org/@typescript-eslint/parser/-/parser-5.36.1.tgz", + "integrity": "sha512-/IsgNGOkBi7CuDfUbwt1eOqUXF9WGVBW9dwEe1pi+L32XrTsZIgmDFIi2RxjzsvB/8i+MIf5JIoTEH8LOZ368A==", "dev": true, "requires": { - "@typescript-eslint/scope-manager": "5.33.1", - "@typescript-eslint/types": "5.33.1", - "@typescript-eslint/typescript-estree": "5.33.1", + "@typescript-eslint/scope-manager": "5.36.1", + "@typescript-eslint/types": "5.36.1", + "@typescript-eslint/typescript-estree": "5.36.1", "debug": "^4.3.4" } }, "@typescript-eslint/scope-manager": { - "version": "5.33.1", - "resolved": "https://registry.npmjs.org/@typescript-eslint/scope-manager/-/scope-manager-5.33.1.tgz", - "integrity": "sha512-8ibcZSqy4c5m69QpzJn8XQq9NnqAToC8OdH/W6IXPXv83vRyEDPYLdjAlUx8h/rbusq6MkW4YdQzURGOqsn3CA==", + "version": "5.36.1", + "resolved": "https://registry.npmjs.org/@typescript-eslint/scope-manager/-/scope-manager-5.36.1.tgz", + "integrity": "sha512-pGC2SH3/tXdu9IH3ItoqciD3f3RRGCh7hb9zPdN2Drsr341zgd6VbhP5OHQO/reUqihNltfPpMpTNihFMarP2w==", "dev": true, "requires": { - "@typescript-eslint/types": "5.33.1", - "@typescript-eslint/visitor-keys": "5.33.1" + "@typescript-eslint/types": "5.36.1", + "@typescript-eslint/visitor-keys": "5.36.1" } }, "@typescript-eslint/type-utils": { - "version": "5.33.1", - "resolved": "https://registry.npmjs.org/@typescript-eslint/type-utils/-/type-utils-5.33.1.tgz", - "integrity": "sha512-X3pGsJsD8OiqhNa5fim41YtlnyiWMF/eKsEZGsHID2HcDqeSC5yr/uLOeph8rNF2/utwuI0IQoAK3fpoxcLl2g==", + "version": "5.36.1", + "resolved": "https://registry.npmjs.org/@typescript-eslint/type-utils/-/type-utils-5.36.1.tgz", + "integrity": "sha512-xfZhfmoQT6m3lmlqDvDzv9TiCYdw22cdj06xY0obSznBsT///GK5IEZQdGliXpAOaRL34o8phEvXzEo/VJx13Q==", "dev": true, "requires": { - "@typescript-eslint/utils": "5.33.1", + "@typescript-eslint/typescript-estree": "5.36.1", + "@typescript-eslint/utils": "5.36.1", "debug": "^4.3.4", "tsutils": "^3.21.0" } }, "@typescript-eslint/types": { - "version": "5.33.1", - "resolved": "https://registry.npmjs.org/@typescript-eslint/types/-/types-5.33.1.tgz", - "integrity": "sha512-7K6MoQPQh6WVEkMrMW5QOA5FO+BOwzHSNd0j3+BlBwd6vtzfZceJ8xJ7Um2XDi/O3umS8/qDX6jdy2i7CijkwQ==", + "version": "5.36.1", + "resolved": "https://registry.npmjs.org/@typescript-eslint/types/-/types-5.36.1.tgz", + "integrity": "sha512-jd93ShpsIk1KgBTx9E+hCSEuLCUFwi9V/urhjOWnOaksGZFbTOxAT47OH2d4NLJnLhkVD+wDbB48BuaycZPLBg==", "dev": true }, "@typescript-eslint/typescript-estree": { - "version": "5.33.1", - "resolved": "https://registry.npmjs.org/@typescript-eslint/typescript-estree/-/typescript-estree-5.33.1.tgz", - "integrity": "sha512-JOAzJ4pJ+tHzA2pgsWQi4804XisPHOtbvwUyqsuuq8+y5B5GMZs7lI1xDWs6V2d7gE/Ez5bTGojSK12+IIPtXA==", + "version": "5.36.1", + "resolved": "https://registry.npmjs.org/@typescript-eslint/typescript-estree/-/typescript-estree-5.36.1.tgz", + "integrity": "sha512-ih7V52zvHdiX6WcPjsOdmADhYMDN15SylWRZrT2OMy80wzKbc79n8wFW0xpWpU0x3VpBz/oDgTm2xwDAnFTl+g==", "dev": true, "requires": { - "@typescript-eslint/types": "5.33.1", - "@typescript-eslint/visitor-keys": "5.33.1", + "@typescript-eslint/types": "5.36.1", + "@typescript-eslint/visitor-keys": "5.36.1", "debug": "^4.3.4", "globby": "^11.1.0", "is-glob": "^4.0.3", @@ -4754,26 +4667,26 @@ } }, "@typescript-eslint/utils": { - "version": "5.33.1", - "resolved": "https://registry.npmjs.org/@typescript-eslint/utils/-/utils-5.33.1.tgz", - "integrity": "sha512-uphZjkMaZ4fE8CR4dU7BquOV6u0doeQAr8n6cQenl/poMaIyJtBu8eys5uk6u5HiDH01Mj5lzbJ5SfeDz7oqMQ==", + "version": "5.36.1", + "resolved": "https://registry.npmjs.org/@typescript-eslint/utils/-/utils-5.36.1.tgz", + "integrity": "sha512-lNj4FtTiXm5c+u0pUehozaUWhh7UYKnwryku0nxJlYUEWetyG92uw2pr+2Iy4M/u0ONMKzfrx7AsGBTCzORmIg==", "dev": true, "requires": { "@types/json-schema": "^7.0.9", - "@typescript-eslint/scope-manager": "5.33.1", - "@typescript-eslint/types": "5.33.1", - "@typescript-eslint/typescript-estree": "5.33.1", + "@typescript-eslint/scope-manager": "5.36.1", + "@typescript-eslint/types": "5.36.1", + "@typescript-eslint/typescript-estree": "5.36.1", "eslint-scope": "^5.1.1", "eslint-utils": "^3.0.0" } }, "@typescript-eslint/visitor-keys": { - "version": "5.33.1", - "resolved": "https://registry.npmjs.org/@typescript-eslint/visitor-keys/-/visitor-keys-5.33.1.tgz", - "integrity": "sha512-nwIxOK8Z2MPWltLKMLOEZwmfBZReqUdbEoHQXeCpa+sRVARe5twpJGHCB4dk9903Yaf0nMAlGbQfaAH92F60eg==", + "version": "5.36.1", + "resolved": "https://registry.npmjs.org/@typescript-eslint/visitor-keys/-/visitor-keys-5.36.1.tgz", + "integrity": "sha512-ojB9aRyRFzVMN3b5joSYni6FAS10BBSCAfKJhjJAV08t/a95aM6tAhz+O1jF+EtgxktuSO3wJysp2R+Def/IWQ==", "dev": true, "requires": { - "@typescript-eslint/types": "5.33.1", + "@typescript-eslint/types": "5.36.1", "eslint-visitor-keys": "^3.3.0" } }, @@ -4866,6 +4779,11 @@ "integrity": "sha512-58S9QDqG0Xx27YwPSt9fJxivjYl432YCwfDMfZ+71RAqUrZef7LrKQZ3LHLOwCS4FLNBplP533Zx895SeOCHvA==", "dev": true }, + "argparse": { + "version": "2.0.1", + "resolved": "https://registry.npmjs.org/argparse/-/argparse-2.0.1.tgz", + "integrity": "sha512-8+9WqebbFzpX9OR+Wa6O29asIogeRMzcGtAINdpMHHyAg10f05aSFVBbcEqGf/PXw1EjAZ+q2/bEBg3DvurK3Q==" + }, "args": { "version": "5.0.3", "resolved": "https://registry.npmjs.org/args/-/args-5.0.3.tgz", @@ -5004,6 +4922,16 @@ "readable-stream": "^3.4.0" }, "dependencies": { + "buffer": { + "version": "5.7.1", + "resolved": "https://registry.npmjs.org/buffer/-/buffer-5.7.1.tgz", + "integrity": "sha512-EHcyIPBQ4BSGlvjB16k5KgAJ27CIsHY/2JBmCRReo48y9rQ3MaUzWX3KVlBa4U7MyX02HdVj0K7C3WaB3ju7FQ==", + "devOptional": true, + "requires": { + "base64-js": "^1.3.1", + "ieee754": "^1.1.13" + } + }, "readable-stream": { "version": "3.6.0", "resolved": "https://registry.npmjs.org/readable-stream/-/readable-stream-3.6.0.tgz", @@ -5079,13 +5007,12 @@ } }, "buffer": { - "version": "5.7.1", - "resolved": "https://registry.npmjs.org/buffer/-/buffer-5.7.1.tgz", - "integrity": "sha512-EHcyIPBQ4BSGlvjB16k5KgAJ27CIsHY/2JBmCRReo48y9rQ3MaUzWX3KVlBa4U7MyX02HdVj0K7C3WaB3ju7FQ==", - "devOptional": true, + "version": "6.0.3", + "resolved": "https://registry.npmjs.org/buffer/-/buffer-6.0.3.tgz", + "integrity": "sha512-FTiCpNxtwiZZHEZbcbTIcZjERVICn9yq/pDFkTl95/AxzD1naBctN7YO68riM/gLSDY7sdrMby8hofADYuuqOA==", "requires": { "base64-js": "^1.3.1", - "ieee754": "^1.1.13" + "ieee754": "^1.2.1" } }, "buffer-from": { @@ -5232,12 +5159,9 @@ } }, "commander": { - "version": "2.8.1", - "resolved": "https://registry.npmjs.org/commander/-/commander-2.8.1.tgz", - "integrity": "sha512-+pJLBFVk+9ZZdlAOB5WuIElVPPth47hILFkmGym57aq8kwxsowvByvB0DHs1vQAhyMZzdcpTtF0VDKGkSDR4ZQ==", - "requires": { - "graceful-readlink": ">= 1.0.0" - } + "version": "2.20.3", + "resolved": "https://registry.npmjs.org/commander/-/commander-2.20.3.tgz", + "integrity": "sha512-GpVkmM8vF2vQUkj2LvZmD35JxeJOLCwJ9cUkugyk2nuhbv3+mJvpLYYt+0+USMxE+oj+ey/lJEnhZw75x/OMcQ==" }, "compressjs": { "version": "1.0.3", @@ -5246,6 +5170,16 @@ "requires": { "amdefine": "~1.0.0", "commander": "~2.8.1" + }, + "dependencies": { + "commander": { + "version": "2.8.1", + "resolved": "https://registry.npmjs.org/commander/-/commander-2.8.1.tgz", + "integrity": "sha512-+pJLBFVk+9ZZdlAOB5WuIElVPPth47hILFkmGym57aq8kwxsowvByvB0DHs1vQAhyMZzdcpTtF0VDKGkSDR4ZQ==", + "requires": { + "graceful-readlink": ">= 1.0.0" + } + } } }, "concat-map": { @@ -5417,9 +5351,9 @@ } }, "dotenv": { - "version": "8.6.0", - "resolved": "https://registry.npmjs.org/dotenv/-/dotenv-8.6.0.tgz", - "integrity": "sha512-IrPdXQsk2BbzvCBGBOTmmSH5SodmqZNt4ERAZDmW4CT+tL8VtvinqywuANaFu4bOMWki16nqf0e4oC0QIaDr/g==" + "version": "16.0.2", + "resolved": "https://registry.npmjs.org/dotenv/-/dotenv-16.0.2.tgz", + "integrity": "sha512-JvpYKUmzQhYoIFgK2MOnF3bciIZoItIIoryihy0rIA+H4Jy0FmgyKYAHCTN98P5ybGSJcIFbh6QKeJdtZd1qhA==" }, "dynamic-dedupe": { "version": "0.3.0", @@ -5470,14 +5404,15 @@ "dev": true }, "eslint": { - "version": "8.22.0", - "resolved": "https://registry.npmjs.org/eslint/-/eslint-8.22.0.tgz", - "integrity": "sha512-ci4t0sz6vSRKdmkOGmprBo6fmI4PrphDFMy5JEq/fNS0gQkJM3rLmrqcp8ipMcdobH3KtUP40KniAE9W19S4wA==", + "version": "8.23.0", + "resolved": "https://registry.npmjs.org/eslint/-/eslint-8.23.0.tgz", + "integrity": "sha512-pBG/XOn0MsJcKcTRLr27S5HpzQo4kLr+HjLQIyK4EiCsijDl/TB+h5uEuJU6bQ8Edvwz1XWOjpaP2qgnXGpTcA==", "dev": true, "requires": { - "@eslint/eslintrc": "^1.3.0", + "@eslint/eslintrc": "^1.3.1", "@humanwhocodes/config-array": "^0.10.4", "@humanwhocodes/gitignore-to-minimatch": "^1.0.2", + "@humanwhocodes/module-importer": "^1.0.1", "ajv": "^6.10.0", "chalk": "^4.0.0", "cross-spawn": "^7.0.2", @@ -5487,7 +5422,7 @@ "eslint-scope": "^7.1.1", "eslint-utils": "^3.0.0", "eslint-visitor-keys": "^3.3.0", - "espree": "^9.3.3", + "espree": "^9.4.0", "esquery": "^1.4.0", "esutils": "^2.0.2", "fast-deep-equal": "^3.1.3", @@ -5512,16 +5447,9 @@ "regexpp": "^3.2.0", "strip-ansi": "^6.0.1", "strip-json-comments": "^3.1.0", - "text-table": "^0.2.0", - "v8-compile-cache": "^2.0.3" + "text-table": "^0.2.0" }, "dependencies": { - "argparse": { - "version": "2.0.1", - "resolved": "https://registry.npmjs.org/argparse/-/argparse-2.0.1.tgz", - "integrity": "sha512-8+9WqebbFzpX9OR+Wa6O29asIogeRMzcGtAINdpMHHyAg10f05aSFVBbcEqGf/PXw1EjAZ+q2/bEBg3DvurK3Q==", - "dev": true - }, "eslint-scope": { "version": "7.1.1", "resolved": "https://registry.npmjs.org/eslint-scope/-/eslint-scope-7.1.1.tgz", @@ -5537,15 +5465,6 @@ "resolved": "https://registry.npmjs.org/estraverse/-/estraverse-5.3.0.tgz", "integrity": "sha512-MMdARuVEQziNTeJD8DgMqmhwR11BRQ/cBP+pLtYdSTnf3MIO8fFeiINEbX36ZdNlfU/7A9f3gUw49B3oQsvwBA==", "dev": true - }, - "js-yaml": { - "version": "4.1.0", - "resolved": "https://registry.npmjs.org/js-yaml/-/js-yaml-4.1.0.tgz", - "integrity": "sha512-wpxZs9NoxZaJESJGIZTyDEaYpl0FKSA+FB9aJiyemKhMwkxQg63h4T1KJgUGHpTqPDNRcmmYLugrRjJlBtWvRA==", - "dev": true, - "requires": { - "argparse": "^2.0.1" - } } } }, @@ -5583,9 +5502,9 @@ "dev": true }, "espree": { - "version": "9.3.3", - "resolved": "https://registry.npmjs.org/espree/-/espree-9.3.3.tgz", - "integrity": "sha512-ORs1Rt/uQTqUKjDdGCyrtYxbazf5umATSf/K4qxjmZHORR6HJk+2s/2Pqe+Kk49HHINC/xNIrGfgh8sZcll0ng==", + "version": "9.4.0", + "resolved": "https://registry.npmjs.org/espree/-/espree-9.4.0.tgz", + "integrity": "sha512-DQmnRpLj7f6TgN/NYb0MTzJXL+vJF9h3pHy4JhCIs3zwcgez8xmGg3sXHcEO97BrmO2OSvCwMdfdlyl+E9KjOw==", "dev": true, "requires": { "acorn": "^8.8.0", @@ -6145,6 +6064,14 @@ "resolved": "https://registry.npmjs.org/joycon/-/joycon-3.1.1.tgz", "integrity": "sha512-34wB/Y7MW7bzjKRjUKTa46I2Z7eV62Rkhva+KkopW7Qvv/OSWBqvkSY7vusOPrNuZcUG3tApvdVgNB8POj3SPw==" }, + "js-yaml": { + "version": "4.1.0", + "resolved": "https://registry.npmjs.org/js-yaml/-/js-yaml-4.1.0.tgz", + "integrity": "sha512-wpxZs9NoxZaJESJGIZTyDEaYpl0FKSA+FB9aJiyemKhMwkxQg63h4T1KJgUGHpTqPDNRcmmYLugrRjJlBtWvRA==", + "requires": { + "argparse": "^2.0.1" + } + }, "json-schema-traverse": { "version": "0.4.1", "resolved": "https://registry.npmjs.org/json-schema-traverse/-/json-schema-traverse-0.4.1.tgz", @@ -7289,34 +7216,17 @@ "zen-observable-ts": "^1.0.0" }, "dependencies": { - "argparse": { - "version": "2.0.1", - "resolved": "https://registry.npmjs.org/argparse/-/argparse-2.0.1.tgz", - "integrity": "sha512-8+9WqebbFzpX9OR+Wa6O29asIogeRMzcGtAINdpMHHyAg10f05aSFVBbcEqGf/PXw1EjAZ+q2/bEBg3DvurK3Q==" - }, - "buffer": { - "version": "6.0.3", - "resolved": "https://registry.npmjs.org/buffer/-/buffer-6.0.3.tgz", - "integrity": "sha512-FTiCpNxtwiZZHEZbcbTIcZjERVICn9yq/pDFkTl95/AxzD1naBctN7YO68riM/gLSDY7sdrMby8hofADYuuqOA==", - "requires": { - "base64-js": "^1.3.1", - "ieee754": "^1.2.1" - } - }, - "js-yaml": { - "version": "4.1.0", - "resolved": "https://registry.npmjs.org/js-yaml/-/js-yaml-4.1.0.tgz", - "integrity": "sha512-wpxZs9NoxZaJESJGIZTyDEaYpl0FKSA+FB9aJiyemKhMwkxQg63h4T1KJgUGHpTqPDNRcmmYLugrRjJlBtWvRA==", - "requires": { - "argparse": "^2.0.1" - } + "dotenv": { + "version": "8.6.0", + "resolved": "https://registry.npmjs.org/dotenv/-/dotenv-8.6.0.tgz", + "integrity": "sha512-IrPdXQsk2BbzvCBGBOTmmSH5SodmqZNt4ERAZDmW4CT+tL8VtvinqywuANaFu4bOMWki16nqf0e4oC0QIaDr/g==" } } }, "typescript": { - "version": "4.7.4", - "resolved": "https://registry.npmjs.org/typescript/-/typescript-4.7.4.tgz", - "integrity": "sha512-C0WQT0gezHuw6AdY1M2jxUO83Rjf0HP7Sk1DtXj6j1EwkQNZrHAg2XPWlq62oqEhYvONq5pkC2Y9oPljWToLmQ==", + "version": "4.8.2", + "resolved": "https://registry.npmjs.org/typescript/-/typescript-4.8.2.tgz", + "integrity": "sha512-C0I1UsrrDHo2fYI5oaCGbSejwX4ch+9Y5jTQELvovfmFkK3HHSZJB8MSJcWLmCUBzQBchCrZ9rMRV6GuNrvGtw==", "dev": true }, "unpipe": { @@ -7369,12 +7279,6 @@ "resolved": "https://registry.npmjs.org/uuid/-/uuid-8.3.2.tgz", "integrity": "sha512-+NYs2QeMWy+GWFOEm9xnn6HCDp0l7QBD7ml8zLUmJ+93Q5NF0NocErnwkTkXVFNiX3/fpC6afS8Dhb/gz7R7eg==" }, - "v8-compile-cache": { - "version": "2.3.0", - "resolved": "https://registry.npmjs.org/v8-compile-cache/-/v8-compile-cache-2.3.0.tgz", - "integrity": "sha512-l8lCEmLcLYZh4nbunNZvQCJc5pv7+RCwa8q/LdUx8u7lsWvPDKmpodJAJNwkAhJC//dFY48KuIEmjtd4RViDrA==", - "dev": true - }, "v8-compile-cache-lib": { "version": "3.0.1", "resolved": "https://registry.npmjs.org/v8-compile-cache-lib/-/v8-compile-cache-lib-3.0.1.tgz", diff --git a/package.json b/package.json index ce23a79..330fae0 100644 --- a/package.json +++ b/package.json @@ -47,16 +47,10 @@ "url": "https://github.com/runejs/filestore/issues" }, "homepage": "https://github.com/runejs/filestore#readme", - "peerDependencies": { - "@runejs/common": "3.0.0-beta.8", - "graceful-fs": ">=4.2.0", - "tslib": ">=2.4.0", - "typescript": ">=4.5.0" - }, "dependencies": { "@decorators/di": "^1.0.3", "@decorators/express": "^2.6.0", - "@runejs/common": "3.0.0-beta.8", + "@runejs/common": "3.0.0-beta.9", "adm-zip": "^0.5.9", "axios": "^0.27.2", "compressjs": "^1.0.3", diff --git a/src/db/js5/index.ts b/src/db/js5/index.ts index 72f4893..010d69c 100644 --- a/src/db/js5/index.ts +++ b/src/db/js5/index.ts @@ -1,4 +1,3 @@ export * from './js5-database'; export * from './js5-index-entity'; -export * from './js5-uncompressed-data-entity'; -export * from './js5-compressed-data-entity'; +export * from './js5-data-entity'; diff --git a/src/db/js5/js5-compressed-data-entity.ts b/src/db/js5/js5-data-entity.ts similarity index 74% rename from src/db/js5/js5-compressed-data-entity.ts rename to src/db/js5/js5-data-entity.ts index 0ae7db5..b779202 100644 --- a/src/db/js5/js5-compressed-data-entity.ts +++ b/src/db/js5/js5-data-entity.ts @@ -1,4 +1,4 @@ -import { Column, Entity, Index, PrimaryColumn } from 'typeorm'; +import { Column, CreateDateColumn, Entity, Index, PrimaryColumn, UpdateDateColumn } from 'typeorm'; import { Js5FileType } from '../../config'; import { Buffer } from 'buffer'; @@ -7,7 +7,7 @@ import { Buffer } from 'buffer'; @Index('compressed_data_identifier', [ 'fileType', 'gameBuild', 'key', 'archiveKey', 'groupKey' ], { unique: true }) -export class Js5CompressedDataEntity { +export class Js5DataEntity { @PrimaryColumn('text', { name: 'file_type', nullable: false, unique: false }) fileType: Js5FileType; @@ -27,4 +27,13 @@ export class Js5CompressedDataEntity { @Column('blob', { name: 'buffer', nullable: true, default: null }) buffer: Buffer = null; + @Column('boolean', { nullable: true, default: false }) + compressed: boolean = false; + + @CreateDateColumn() + created?: Date; + + @UpdateDateColumn() + updated?: Date; + } diff --git a/src/db/js5/js5-database.ts b/src/db/js5/js5-database.ts index f42d514..8fc337a 100644 --- a/src/db/js5/js5-database.ts +++ b/src/db/js5/js5-database.ts @@ -1,11 +1,10 @@ +import { join } from 'path'; +import { existsSync, mkdirSync } from 'graceful-fs'; +import { Connection, createConnection, LoggerOptions, Repository } from 'typeorm'; import { IndexDatabase } from '../index-database'; import { Js5IndexEntity } from './js5-index-entity'; -import { Connection, createConnection, LoggerOptions, Repository } from 'typeorm'; import { Js5FileType } from '../../config'; -import { existsSync, mkdirSync } from 'graceful-fs'; -import { join } from 'path'; -import { Js5UncompressedDataEntity } from './js5-uncompressed-data-entity'; -import { Js5CompressedDataEntity } from './js5-compressed-data-entity'; +import { Js5DataEntity } from './js5-data-entity'; export interface Js5IndexEntityWhere { @@ -19,8 +18,7 @@ export interface Js5IndexEntityWhere { export class Js5Database extends IndexDatabase { - private _uncompressedDataRepo: Repository; - private _compressedDataRepo: Repository; + private _dataRepo: Repository; constructor( gameBuild: string, @@ -38,43 +36,44 @@ export class Js5Database extends IndexDatabase { - return this._uncompressedDataRepo.findOne({ + async getUncompressedData(where: Js5IndexEntityWhere): Promise { + return this._dataRepo.findOne({ where: { gameBuild: this.gameBuild, + compressed: false, ...where } }); } - async getAllUncompressedData(where: Js5IndexEntityWhere): Promise { - return this._uncompressedDataRepo.find({ + async getAllUncompressedData(where: Js5IndexEntityWhere): Promise { + return this._dataRepo.find({ where: { gameBuild: this.gameBuild, + compressed: false, ...where } }); } - async saveUncompressedData(uncompressedDataEntity: Js5UncompressedDataEntity): Promise { - return this._uncompressedDataRepo.save(uncompressedDataEntity); + async saveUncompressedData(uncompressedDataEntity: Js5DataEntity): Promise { + return this._dataRepo.save({ ...uncompressedDataEntity, compressed: false }); } - async saveAllUncompressedData(uncompressedDataEntities: Js5UncompressedDataEntity[]): Promise { - await this._uncompressedDataRepo.save(uncompressedDataEntities, { + async saveAllUncompressedData(uncompressedDataEntities: Js5DataEntity[]): Promise { + await this._dataRepo.save({ ...uncompressedDataEntities, compressed: false }, { chunk: 500, transaction: false, reload: false, @@ -82,41 +81,43 @@ export class Js5Database extends IndexDatabase { + async upsertAllUncompressedData(uncompressedDataEntities: Js5DataEntity[]): Promise { const chunkSize = 100; for (let i = 0; i < uncompressedDataEntities.length; i += chunkSize) { - const chunk = uncompressedDataEntities.slice(i, i + chunkSize); - await this._uncompressedDataRepo.upsert(chunk, { + const chunk = uncompressedDataEntities.slice(i, i + chunkSize).map(d => ({ ...d, compressed: false })); + await this._dataRepo.upsert(chunk, { conflictPaths: [ 'fileType', 'gameBuild', 'key', 'archiveKey', 'groupKey' ], skipUpdateIfNoValuesChanged: true, }); } } - async getCompressedData(where: Js5IndexEntityWhere): Promise { - return this._compressedDataRepo.findOne({ + async getCompressedData(where: Js5IndexEntityWhere): Promise { + return this._dataRepo.findOne({ where: { gameBuild: this.gameBuild, + compressed: true, ...where } }); } - async getAllCompressedData(where: Js5IndexEntityWhere): Promise { - return this._compressedDataRepo.find({ + async getAllCompressedData(where: Js5IndexEntityWhere): Promise { + return this._dataRepo.find({ where: { gameBuild: this.gameBuild, + compressed: true, ...where } }); } - async saveCompressedData(compressedDataEntity: Js5CompressedDataEntity): Promise { - return this._compressedDataRepo.save(compressedDataEntity); + async saveCompressedData(compressedDataEntity: Js5DataEntity): Promise { + return this._dataRepo.save({ ...compressedDataEntity, compressed: true }); } - async saveAllCompressedData(compressedDataEntities: Js5CompressedDataEntity[]): Promise { - await this._compressedDataRepo.save(compressedDataEntities, { + async saveAllCompressedData(compressedDataEntities: Js5DataEntity[]): Promise { + await this._dataRepo.save({ ...compressedDataEntities, compressed: true }, { chunk: 500, transaction: false, reload: false, @@ -124,11 +125,11 @@ export class Js5Database extends IndexDatabase { + async upsertAllCompressedData(compressedDataEntities: Js5DataEntity[]): Promise { const chunkSize = 100; for (let i = 0; i < compressedDataEntities.length; i += chunkSize) { - const chunk = compressedDataEntities.slice(i, i + chunkSize); - await this._compressedDataRepo.upsert(chunk, { + const chunk = compressedDataEntities.slice(i, i + chunkSize).map(d => ({ ...d, compressed: true })); + await this._dataRepo.upsert(chunk, { conflictPaths: [ 'fileType', 'gameBuild', 'key', 'archiveKey', 'groupKey' ], skipUpdateIfNoValuesChanged: true, }); @@ -146,4 +147,8 @@ export class Js5Database extends IndexDatabase { + return this._dataRepo; + } + } diff --git a/src/db/js5/js5-index-entity.ts b/src/db/js5/js5-index-entity.ts index 5f0e7d6..db3bf9b 100644 --- a/src/db/js5/js5-index-entity.ts +++ b/src/db/js5/js5-index-entity.ts @@ -49,9 +49,6 @@ export class Js5IndexEntity { @Column('integer', { name: 'file_size', nullable: false, default: 0 }) fileSize: number = 0; - // @Column('blob', { name: 'data', nullable: true, default: null }) - // data: Buffer = null; - @Column('text', { name: 'compression_method', nullable: true, default: 'none' }) compressionMethod: CompressionMethod = 'none'; @@ -64,9 +61,6 @@ export class Js5IndexEntity { @Column('integer', { name: 'compressed_file_size', nullable: false, default: 0 }) compressedFileSize: number = 0; - // @Column('blob', { name: 'compressed_data', nullable: true, default: null }) - // compressedData: Buffer = null; - @Column('boolean', { nullable: true, default: false }) encrypted: boolean = false; diff --git a/src/db/js5/js5-uncompressed-data-entity.ts b/src/db/js5/js5-uncompressed-data-entity.ts deleted file mode 100644 index 918859e..0000000 --- a/src/db/js5/js5-uncompressed-data-entity.ts +++ /dev/null @@ -1,30 +0,0 @@ -import { Column, Entity, Index, PrimaryColumn } from 'typeorm'; -import { Js5FileType } from '../../config'; -import { Buffer } from 'buffer'; - - -@Entity('js5_uncompressed_data') -@Index('uncompressed_data_identifier', [ - 'fileType', 'gameBuild', 'key', 'archiveKey', 'groupKey' -], { unique: true }) -export class Js5UncompressedDataEntity { - - @PrimaryColumn('text', { name: 'file_type', nullable: false, unique: false }) - fileType: Js5FileType; - - @PrimaryColumn('text', { name: 'game_build', nullable: false, unique: false }) - gameBuild: string; - - @PrimaryColumn('integer', { nullable: false, unique: false }) - key: number; - - @PrimaryColumn('integer', { name: 'archive_key', nullable: false, unique: false, default: -1 }) - archiveKey: number = -1; - - @PrimaryColumn('integer', { name: 'group_key', nullable: false, unique: false, default: -1 }) - groupKey: number = -1; - - @Column('blob', { name: 'buffer', nullable: true, default: null }) - buffer: Buffer = null; - -} diff --git a/src/file-system/jag/jag.ts b/src/file-system/jag/jag.ts index 286bca1..d96a984 100644 --- a/src/file-system/jag/jag.ts +++ b/src/file-system/jag/jag.ts @@ -1,13 +1,14 @@ import { join } from 'path'; import { existsSync, readdirSync, readFileSync, statSync } from 'graceful-fs'; -import { ByteBuffer, logger } from '@runejs/common'; +import { Buffer } from 'buffer'; +import { logger } from '@runejs/common'; +import { ByteBuffer } from '@runejs/common/buffer'; +import { Bzip2 } from '@runejs/common/compress'; import { JagFileStore } from './jag-file-store'; import { JagFile } from './jag-file'; -import { Buffer } from 'buffer'; import { JagArchive } from './jag-archive'; import { JagFileBase } from './jag-file-base'; import { PackedCacheFile } from '../packed'; -import { Bzip2 } from '@runejs/common/compress'; const dataFileName = 'main_file_cache.dat'; diff --git a/src/file-system/js5/js5-archive.ts b/src/file-system/js5/js5-archive.ts index cb0357f..aa724e5 100644 --- a/src/file-system/js5/js5-archive.ts +++ b/src/file-system/js5/js5-archive.ts @@ -68,7 +68,7 @@ export class Js5Archive extends Js5FileBase { async upsertGroupData(): Promise { const groups = Array.from(this.groups.values()); - const uncompressed = groups.map(group => group.uncompressedData).filter(data => data?.buffer && data?.buffer?.length !== 0); + const uncompressed = groups.map(group => group.data).filter(data => data?.buffer && data?.buffer?.length !== 0); const compressed = groups.map(group => group.compressedData).filter(data => data?.buffer && data?.buffer?.length !== 0); if (uncompressed.length) { await this.fileStore.database.upsertAllUncompressedData(uncompressed); diff --git a/src/file-system/js5/js5-file-base.ts b/src/file-system/js5/js5-file-base.ts index ee5c481..c1e5f60 100644 --- a/src/file-system/js5/js5-file-base.ts +++ b/src/file-system/js5/js5-file-base.ts @@ -4,7 +4,7 @@ import { logger } from '@runejs/common'; import { Crc32 } from '@runejs/common/crc32'; import { Js5FileStore } from './js5-file-store'; import { Js5FileType } from '../../config'; -import { Js5IndexEntity, Js5CompressedDataEntity, Js5UncompressedDataEntity } from '../../db/js5'; +import { Js5IndexEntity, Js5DataEntity } from '../../db/js5'; export abstract class Js5FileBase { @@ -12,8 +12,8 @@ export abstract class Js5FileBase { readonly fileStore: Js5FileStore; index: Js5IndexEntity; - uncompressedData: Js5UncompressedDataEntity; - compressedData: Js5CompressedDataEntity; + data: Js5DataEntity; + compressedData: Js5DataEntity; protected constructor( fileStore: Js5FileStore, @@ -24,22 +24,22 @@ export abstract class Js5FileBase { ) { this.fileStore = fileStore; this.index = new Js5IndexEntity(); - this.uncompressedData = new Js5UncompressedDataEntity(); - this.compressedData = new Js5CompressedDataEntity(); - this.uncompressedData.gameBuild = this.compressedData.gameBuild = this.index.gameBuild = fileStore.gameBuild; - this.uncompressedData.fileType = this.compressedData.fileType = this.index.fileType = fileType; - this.uncompressedData.key = this.compressedData.key = this.index.key = key; - this.uncompressedData.archiveKey = this.compressedData.archiveKey = this.index.archiveKey = archiveKey; - this.uncompressedData.groupKey = this.compressedData.groupKey = this.index.groupKey = groupKey; + this.data = new Js5DataEntity(); + this.compressedData = new Js5DataEntity(); + this.data.gameBuild = this.compressedData.gameBuild = this.index.gameBuild = fileStore.gameBuild; + this.data.fileType = this.compressedData.fileType = this.index.fileType = fileType; + this.data.key = this.compressedData.key = this.index.key = key; + this.data.archiveKey = this.compressedData.archiveKey = this.index.archiveKey = archiveKey; + this.data.groupKey = this.compressedData.groupKey = this.index.groupKey = groupKey; } validate(trackChanges: boolean = true): void { - const data = this.uncompressedData.buffer; + const data = this.data.buffer; const compressedData = this.compressedData.buffer; const { checksum, shaDigest, fileSize, compressedChecksum, compressedShaDigest, compressedFileSize, - name, nameHash, + name, nameHash, compressionMethod } = this.index; let fileModified: boolean = false; @@ -100,7 +100,8 @@ export abstract class Js5FileBase { fileModified = true; } - if (compressedFileSize === fileSize && this.compressedData?.buffer?.length) { + if (compressionMethod === 'none' || + (compressedFileSize === fileSize && this.compressedData?.buffer?.length)) { // File has no compression, clear the compressed data buffer so that we do not create a // duplicate data entity record for it this.compressedData.buffer = null; @@ -112,17 +113,17 @@ export abstract class Js5FileBase { } } - async saveUncompressedData(): Promise { - if (!this.uncompressedData?.buffer?.length) { + async saveUncompressedData(): Promise { + if (!this.data?.buffer?.length) { // Do not save a record for files with missing or empty data return null; } - this.uncompressedData = await this.fileStore.database.saveUncompressedData(this.uncompressedData); - return this.uncompressedData; + this.data = await this.fileStore.database.saveUncompressedData(this.data); + return this.data; } - async loadUncompressedData(): Promise { + async loadUncompressedData(): Promise { const entity = await this.fileStore.database.getUncompressedData({ fileType: this.index.fileType, key: this.index.key, @@ -131,15 +132,15 @@ export abstract class Js5FileBase { }); if (entity) { - this.uncompressedData = entity; + this.data = entity; } - return this.uncompressedData; + return this.data; } async getUncompressedData(): Promise { - if (this.uncompressedData?.buffer?.length) { - return this.uncompressedData.buffer; + if (this.data?.buffer?.length) { + return this.data.buffer; } const uncompressedData = await this.loadUncompressedData(); @@ -150,7 +151,7 @@ export abstract class Js5FileBase { return null; } - async saveCompressedData(): Promise { + async saveCompressedData(): Promise { if (!this.compressedData?.buffer?.length) { // Do not save a record for files with missing or empty data return null; @@ -160,7 +161,7 @@ export abstract class Js5FileBase { return this.compressedData; } - async loadCompressedData(): Promise { + async loadCompressedData(): Promise { const entity = await this.fileStore.database.getCompressedData({ fileType: this.index.fileType, key: this.index.key, diff --git a/src/file-system/js5/js5-group.ts b/src/file-system/js5/js5-group.ts index 8fa9472..53ea6d5 100644 --- a/src/file-system/js5/js5-group.ts +++ b/src/file-system/js5/js5-group.ts @@ -69,7 +69,7 @@ export class Js5Group extends Js5FileBase { async upsertFileData(): Promise { const files = Array.from(this.files.values()); - const uncompressed = files.map(group => group.uncompressedData).filter(data => data?.buffer && data?.buffer?.length !== 0); + const uncompressed = files.map(group => group.data).filter(data => data?.buffer && data?.buffer?.length !== 0); const compressed = files.map(group => group.compressedData).filter(data => data?.buffer && data?.buffer?.length !== 0); if (uncompressed.length) { await this.fileStore.database.upsertAllUncompressedData(uncompressed); diff --git a/src/file-system/js5/js5.ts b/src/file-system/js5/js5.ts index 63f3ff2..60e24ad 100644 --- a/src/file-system/js5/js5.ts +++ b/src/file-system/js5/js5.ts @@ -2,7 +2,8 @@ import { join } from 'path'; import { Buffer } from 'buffer'; import { existsSync, readdirSync, readFileSync, statSync } from 'graceful-fs'; -import { ByteBuffer, logger } from '@runejs/common'; +import { logger } from '@runejs/common'; +import { ByteBuffer } from '@runejs/common/buffer'; import { getCompressionMethod, Gzip, Bzip2 } from '@runejs/common/compress'; import { Xtea, XteaKeys, XteaConfig } from '@runejs/common/encrypt'; @@ -398,9 +399,9 @@ export class JS5 { data = null; } - file.uncompressedData.buffer = data; + file.data.buffer = data; file.validate(false); - return file.uncompressedData.buffer; + return file.data.buffer; } async decodeGroup(group: Js5Group): Promise { @@ -408,10 +409,10 @@ export class JS5 { const { key: groupKey, name: groupName } = groupDetails; const files = group.files; - if (!group.uncompressedData.buffer) { + if (!group.data.buffer) { this.decompress(group); - if (!group.uncompressedData.buffer) { + if (!group.data.buffer) { if (!groupDetails.fileError) { logger.warn(`Unable to decode group ${ groupName || groupKey }.`); } @@ -419,7 +420,7 @@ export class JS5 { } } - const data = new ByteBuffer(group.uncompressedData.buffer); + const data = new ByteBuffer(group.data.buffer); if (groupDetails.childCount === 1) { return; @@ -491,7 +492,7 @@ export class JS5 { } for (const [ fileIndex, file ] of files) { - file.uncompressedData.buffer = fileDataMap.get(fileIndex).toNodeBuffer(); + file.data.buffer = fileDataMap.get(fileIndex).toNodeBuffer(); file.validate(false); } @@ -509,16 +510,16 @@ export class JS5 { logger.info(`Decoding archive ${ archiveName }...`); - if (!archive.uncompressedData.buffer) { + if (!archive.data.buffer) { this.decompress(archive); - if (!archive.uncompressedData.buffer) { + if (!archive.data.buffer) { logger.error(`Unable to decode archive ${ archiveName }.`); return; } } - const archiveData = new ByteBuffer(archive.uncompressedData.buffer); + const archiveData = new ByteBuffer(archive.data.buffer); const format = archiveDetails.archiveFormat = archiveData.get('byte', 'unsigned'); const mainDataType = format >= ArchiveFormat.smart ? 'smart_int' : 'short'; archiveDetails.version = format >= ArchiveFormat.versioned ? archiveData.get('int') : 0; @@ -633,11 +634,11 @@ export class JS5 { compress(file: Js5Group | Js5Archive): Buffer | null { const fileDetails = file.index; - if (!file.uncompressedData.buffer?.length) { + if (!file.data.buffer?.length) { return null; } - const decompressedData = new ByteBuffer(file.uncompressedData.buffer); + const decompressedData = new ByteBuffer(file.data.buffer); let data: ByteBuffer; if (fileDetails.compressionMethod === 'none') { @@ -689,8 +690,8 @@ export class JS5 { // Single-file group if (fileCount <= 1) { - group.uncompressedData.buffer = fileMap.get(0)?.uncompressedData?.buffer || null; - return group.uncompressedData.buffer; + group.data.buffer = fileMap.get(0)?.data?.buffer || null; + return group.data.buffer; } // Multi-file group @@ -709,7 +710,7 @@ export class JS5 { // Write child file data for (let stripe = 0; stripe < stripeCount; stripe++) { files.forEach(file => { - const fileData = file.uncompressedData.buffer; + const fileData = file.data.buffer; if (!fileData?.length) { return; } @@ -729,7 +730,7 @@ export class JS5 { for (let stripe = 0; stripe < stripeCount; stripe++) { let prevSize = 0; files.forEach(file => { - const fileData = file.uncompressedData.buffer; + const fileData = file.data.buffer; if (!fileData?.length) { return; } @@ -744,12 +745,12 @@ export class JS5 { groupBuffer.put(stripeCount, 'byte'); if (groupBuffer.length) { - group.uncompressedData.buffer = groupBuffer.toNodeBuffer(); + group.data.buffer = groupBuffer.toNodeBuffer(); } else { - group.uncompressedData.buffer = null; + group.data.buffer = null; } - return group.uncompressedData.buffer; + return group.data.buffer; } // @todo support newer archive fields & formats - 21/07/22 - Kiko @@ -819,12 +820,12 @@ export class JS5 { const archiveIndexData = buffer?.flipWriter(); if (archiveIndexData?.length) { - archive.uncompressedData.buffer = archiveIndexData.toNodeBuffer(); + archive.data.buffer = archiveIndexData.toNodeBuffer(); } else { - archive.uncompressedData.buffer = null; + archive.data.buffer = null; } - return archive.uncompressedData.buffer; + return archive.data.buffer; } encodeMainIndex(): ByteBuffer { diff --git a/src/http/index.ts b/src/http/index.ts index 0ce5251..34b746c 100644 --- a/src/http/index.ts +++ b/src/http/index.ts @@ -1 +1,3 @@ export * from './server'; +export * from './js5/js5.service'; +export * from './js5/js5.controller'; diff --git a/src/http/jag-routes.ts b/src/http/jag-routes.ts deleted file mode 100644 index e69de29..0000000 diff --git a/src/http/js5-routes.ts b/src/http/js5-routes.ts deleted file mode 100644 index c276a4d..0000000 --- a/src/http/js5-routes.ts +++ /dev/null @@ -1,7 +0,0 @@ -import { Request, Response, Router } from 'express'; - -export const js5Router = Router(); - -js5Router.get('/archives', (req: Request, res: Response) => { - -}); diff --git a/src/http/js5/js5.controller.ts b/src/http/js5/js5.controller.ts index 0fcb0aa..cf6a18a 100644 --- a/src/http/js5/js5.controller.ts +++ b/src/http/js5/js5.controller.ts @@ -121,4 +121,12 @@ export class Js5Controller { res.send(await this.js5Service.getArchiveList(gameBuild)); } + @Get('/:gameBuild/archive-config') + async getArchiveConfig( + @Response() res: ExpressResponse, + @Params('gameBuild') gameBuild: string | number + ) { + res.send(await this.js5Service.getArchiveConfig(gameBuild)); + } + } diff --git a/src/http/js5/js5.service.ts b/src/http/js5/js5.service.ts index 1dfadcd..4157911 100644 --- a/src/http/js5/js5.service.ts +++ b/src/http/js5/js5.service.ts @@ -4,6 +4,7 @@ import { Js5FileStore } from '../../file-system'; import { Js5IndexEntity } from '../../db/js5'; import { logger } from '@runejs/common'; import { Buffer } from 'buffer'; +import { Js5ArchiveConfig } from '../../config'; @Injectable() @@ -115,6 +116,11 @@ export class Js5Service { return Array.from(fileStore.archives.values()).map(archive => archive.index); } + async getArchiveConfig(gameBuild): Promise<{ [key: string]: Js5ArchiveConfig }> { + const fileStore = await this.getFileStore(gameBuild); + return fileStore.archiveConfig; + } + async getFileStore(gameBuild: string | number): Promise { if (this.stores.has(gameBuild)) { return this.stores.get(gameBuild); diff --git a/src/http/server.ts b/src/http/server.ts index f0618fd..beffcfb 100644 --- a/src/http/server.ts +++ b/src/http/server.ts @@ -1,9 +1,9 @@ import express from 'express'; +import { Container } from '@decorators/di'; +import { attachControllers } from '@decorators/express'; import { logger } from '@runejs/common'; import { ArgumentOptions, ScriptExecutor } from '../scripts/script-executor'; -import { Container } from '@decorators/di'; import { FILESTORE_DIR } from './config'; -import { attachControllers } from '@decorators/express'; import { Js5Controller } from './js5/js5.controller'; diff --git a/src/scripts/dev.ts b/src/scripts/dev.ts index a77ea07..ce232df 100644 --- a/src/scripts/dev.ts +++ b/src/scripts/dev.ts @@ -1,9 +1,8 @@ import { writeFileSync, existsSync, mkdirSync } from 'fs'; import { logger } from '@runejs/common'; -import { JagFileStore } from '../file-system/jag'; import { JagInterfaceArchive } from '../file-system/jag/content/archives/interfaces/jag-interface-archive'; import { join } from 'path'; -import { Js5FileStore } from '../file-system/js5'; +import { Js5FileStore, JagFileStore } from '../file-system'; const saveInterfaces = async (store: JagFileStore) => { @@ -57,10 +56,10 @@ const dev = async () => { 'mapback' ]; - fileNames.forEach(name => { - const spriteFile = store.getArchive('sprites').getGroup(name); + for (const name of fileNames) { + const spriteFile = await (await store.getArchive('sprites')).getGroup(name); store.js5.decompress(spriteFile); - }); + } const end = Date.now(); logger.info(`Operations completed in ${(end - start) / 1000} seconds.`); From d63f842a551dea8d9e54525b1f42219089aab529 Mon Sep 17 00:00:00 2001 From: Kikorono Date: Sun, 4 Sep 2022 18:12:40 -0500 Subject: [PATCH 32/32] Moving JAG file data out to another table like JS5 and fixing tons of JAG indexing bugs --- config/js5-archives.json5 | 1 - package-lock.json | 14 +- package.json | 2 +- src/db/jag/index.ts | 1 + src/db/jag/jag-data-entity.ts | 39 ++++ src/db/jag/jag-database.ts | 99 +++++++- src/db/jag/jag-index-entity.ts | 7 - src/db/js5/js5-data-entity.ts | 12 +- src/db/js5/js5-database.ts | 6 +- .../jag/content/animations/animations.ts | 16 +- .../interfaces/jag-interface-archive.ts | 214 +++++++++--------- src/file-system/jag/jag-archive.ts | 12 + src/file-system/jag/jag-cache.ts | 12 + src/file-system/jag/jag-file-base.ts | 116 +++++++++- src/file-system/jag/jag.ts | 59 +++-- src/file-system/js5/js5-file-base.ts | 12 +- src/file-system/js5/js5-file-store.ts | 27 +++ src/file-system/js5/js5.ts | 10 +- src/scripts/dev.ts | 20 +- src/scripts/indexer.ts | 52 ++++- 20 files changed, 531 insertions(+), 200 deletions(-) create mode 100644 src/db/jag/jag-data-entity.ts diff --git a/config/js5-archives.json5 b/config/js5-archives.json5 index 4675b11..977b5c9 100644 --- a/config/js5-archives.json5 +++ b/config/js5-archives.json5 @@ -539,7 +539,6 @@ }, clientscripts: { key: 12, - compression: 'gzip', contentType: '.cs2' }, fontmetrics: { diff --git a/package-lock.json b/package-lock.json index 6845958..2953e16 100644 --- a/package-lock.json +++ b/package-lock.json @@ -11,7 +11,7 @@ "dependencies": { "@decorators/di": "^1.0.3", "@decorators/express": "^2.6.0", - "@runejs/common": "3.0.0-beta.9", + "@runejs/common": "3.0.0-beta.10", "adm-zip": "^0.5.9", "axios": "^0.27.2", "compressjs": "^1.0.3", @@ -218,9 +218,9 @@ } }, "node_modules/@runejs/common": { - "version": "3.0.0-beta.9", - "resolved": "https://registry.npmjs.org/@runejs/common/-/common-3.0.0-beta.9.tgz", - "integrity": "sha512-KjiIhnyq0PxJ0HFHSIXV6NpCqrYgZGfbt4fDHFEhOQTmGgs3UPT6i9Hcd/tgsmThcpjSVTltIEnb9ZAG1VclzA==", + "version": "3.0.0-beta.10", + "resolved": "https://registry.npmjs.org/@runejs/common/-/common-3.0.0-beta.10.tgz", + "integrity": "sha512-4EnjDBlAfv26oHkSg6q6xAU7gfQj4QKKDUh/e0NJZ3DkwrQuimlNHEfprn7/Wl6wa4mzqJZl05MT2XRfGFhttQ==", "dependencies": { "@ledgerhq/compressjs": "^1.3.2", "buffer": "^6.0.3", @@ -4413,9 +4413,9 @@ } }, "@runejs/common": { - "version": "3.0.0-beta.9", - "resolved": "https://registry.npmjs.org/@runejs/common/-/common-3.0.0-beta.9.tgz", - "integrity": "sha512-KjiIhnyq0PxJ0HFHSIXV6NpCqrYgZGfbt4fDHFEhOQTmGgs3UPT6i9Hcd/tgsmThcpjSVTltIEnb9ZAG1VclzA==", + "version": "3.0.0-beta.10", + "resolved": "https://registry.npmjs.org/@runejs/common/-/common-3.0.0-beta.10.tgz", + "integrity": "sha512-4EnjDBlAfv26oHkSg6q6xAU7gfQj4QKKDUh/e0NJZ3DkwrQuimlNHEfprn7/Wl6wa4mzqJZl05MT2XRfGFhttQ==", "requires": { "@ledgerhq/compressjs": "^1.3.2", "buffer": "^6.0.3", diff --git a/package.json b/package.json index 330fae0..1676cee 100644 --- a/package.json +++ b/package.json @@ -50,7 +50,7 @@ "dependencies": { "@decorators/di": "^1.0.3", "@decorators/express": "^2.6.0", - "@runejs/common": "3.0.0-beta.9", + "@runejs/common": "3.0.0-beta.10", "adm-zip": "^0.5.9", "axios": "^0.27.2", "compressjs": "^1.0.3", diff --git a/src/db/jag/index.ts b/src/db/jag/index.ts index 9fd3ba5..f9bde8e 100644 --- a/src/db/jag/index.ts +++ b/src/db/jag/index.ts @@ -1,3 +1,4 @@ export * from './jag-database'; export * from './jag-index-entity'; +export * from './jag-data-entity'; export * from './content/jag-game-interface-entity'; diff --git a/src/db/jag/jag-data-entity.ts b/src/db/jag/jag-data-entity.ts new file mode 100644 index 0000000..23f2890 --- /dev/null +++ b/src/db/jag/jag-data-entity.ts @@ -0,0 +1,39 @@ +import { Column, CreateDateColumn, Entity, Index, PrimaryColumn, UpdateDateColumn } from 'typeorm'; +import { JagFileType } from '../../config'; +import { Buffer } from 'buffer'; + + +@Entity('jag_data') +@Index('data_identifier', [ + 'fileType', 'gameBuild', 'key', 'cacheKey', 'archiveKey', 'compressed' +], { unique: true }) +export class JagDataEntity { + + @PrimaryColumn('text', { name: 'file_type', nullable: false, unique: false }) + fileType: JagFileType; + + @PrimaryColumn('text', { name: 'game_build', nullable: false, unique: false }) + gameBuild: string; + + @PrimaryColumn('integer', { nullable: false, unique: false }) + key: number; + + @PrimaryColumn('integer', { name: 'cache_key', nullable: false, unique: false }) + cacheKey: number; + + @PrimaryColumn('integer', { name: 'archive_key', nullable: false, unique: false, default: -1 }) + archiveKey: number = -1; + + @PrimaryColumn('boolean', { nullable: false, default: false }) + compressed: boolean = false; + + @Column('blob', { name: 'buffer', nullable: true, default: null }) + buffer: Buffer = null; + + @CreateDateColumn() + created?: Date; + + @UpdateDateColumn() + updated?: Date; + +} diff --git a/src/db/jag/jag-database.ts b/src/db/jag/jag-database.ts index 32b5760..25bf72d 100644 --- a/src/db/jag/jag-database.ts +++ b/src/db/jag/jag-database.ts @@ -5,11 +5,13 @@ import { JagFileType } from '../../config'; import { JagGameInterfaceEntity } from './content/jag-game-interface-entity'; import { existsSync, mkdirSync } from 'graceful-fs'; import { join } from 'path'; +import { JagDataEntity } from './jag-data-entity'; export interface JagIndexEntityWhere { fileType?: JagFileType; key?: number; + name?: string; cacheKey?: number; archiveKey?: number; } @@ -18,6 +20,7 @@ export interface JagIndexEntityWhere { export class JagDatabase extends IndexDatabase { private _interfaceRepo: Repository; + private _dataRepo: Repository; constructor( gameBuild: string, @@ -37,19 +40,109 @@ export class JagDatabase extends IndexDatabase { + return this._dataRepo.findOne({ + where: { + gameBuild: this.gameBuild, + compressed: false, + ...where + } + }); + } + + async getAllUncompressedData(where: JagIndexEntityWhere): Promise { + return this._dataRepo.find({ + where: { + gameBuild: this.gameBuild, + compressed: false, + ...where + } + }); + } + + async saveUncompressedData(uncompressedDataEntity: JagDataEntity): Promise { + return this._dataRepo.save({ ...uncompressedDataEntity, compressed: false }); + } + + async saveAllUncompressedData(uncompressedDataEntities: JagDataEntity[]): Promise { + await this._dataRepo.save({ ...uncompressedDataEntities, compressed: false }, { + chunk: 500, + transaction: false, + reload: false, + listeners: false, + }); + } + + async upsertAllUncompressedData(uncompressedDataEntities: JagDataEntity[]): Promise { + const chunkSize = 100; + for (let i = 0; i < uncompressedDataEntities.length; i += chunkSize) { + const chunk = uncompressedDataEntities.slice(i, i + chunkSize).map(d => ({ ...d, compressed: false })); + await this._dataRepo.upsert(chunk, { + conflictPaths: [ 'fileType', 'gameBuild', 'key', 'cacheKey', 'archiveKey', 'compressed' ], + skipUpdateIfNoValuesChanged: true, + }); + } + } + + async getCompressedData(where: JagIndexEntityWhere): Promise { + return this._dataRepo.findOne({ + where: { + gameBuild: this.gameBuild, + compressed: true, + ...where + } + }); + } + + async getAllCompressedData(where: JagIndexEntityWhere): Promise { + return this._dataRepo.find({ + where: { + gameBuild: this.gameBuild, + compressed: true, + ...where + } + }); + } + + async saveCompressedData(compressedDataEntity: JagDataEntity): Promise { + return this._dataRepo.save({ ...compressedDataEntity, compressed: true }); + } + + async saveAllCompressedData(compressedDataEntities: JagDataEntity[]): Promise { + await this._dataRepo.save({ ...compressedDataEntities, compressed: true }, { + chunk: 500, + transaction: false, + reload: false, + listeners: false, + }); + } + + async upsertAllCompressedData(compressedDataEntities: JagDataEntity[]): Promise { + const chunkSize = 100; + for (let i = 0; i < compressedDataEntities.length; i += chunkSize) { + const chunk = compressedDataEntities.slice(i, i + chunkSize).map(d => ({ ...d, compressed: true })); + await this._dataRepo.upsert(chunk, { + conflictPaths: [ 'fileType', 'gameBuild', 'key', 'cacheKey', 'archiveKey', 'compressed' ], + skipUpdateIfNoValuesChanged: true, + }); + } + } + override async upsertIndexes(indexEntities: JagIndexEntity[]): Promise { const chunkSize = 100; for (let i = 0; i < indexEntities.length; i += chunkSize) { @@ -84,4 +177,8 @@ export class JagDatabase extends IndexDatabase { + return this._dataRepo; + } + } diff --git a/src/db/jag/jag-index-entity.ts b/src/db/jag/jag-index-entity.ts index 6477b66..2bfb18e 100644 --- a/src/db/jag/jag-index-entity.ts +++ b/src/db/jag/jag-index-entity.ts @@ -1,7 +1,6 @@ import { Column, CreateDateColumn, Entity, Index, PrimaryColumn, UpdateDateColumn } from 'typeorm'; import { CompressionMethod } from '@runejs/common/compress'; import { FileError, JagFileType } from '../../config'; -import { Buffer } from 'buffer'; @Entity('jag_index') @@ -46,9 +45,6 @@ export class JagIndexEntity { @Column('integer', { name: 'file_size', nullable: false, default: 0 }) fileSize: number = 0; - @Column('blob', { name: 'data', nullable: true, default: null }) - data: Buffer = null; - @Column('text', { name: 'compression_method', nullable: true, default: 'none' }) compressionMethod: CompressionMethod = 'none'; @@ -61,9 +57,6 @@ export class JagIndexEntity { @Column('integer', { name: 'compressed_file_size', nullable: false, default: 0 }) compressedFileSize: number = 0; - @Column('blob', { name: 'compressed_data', nullable: true, default: null }) - compressedData: Buffer = null; - @Column('text', { name: 'file_error', nullable: true, default: null }) fileError: FileError = null; diff --git a/src/db/js5/js5-data-entity.ts b/src/db/js5/js5-data-entity.ts index b779202..c770f4a 100644 --- a/src/db/js5/js5-data-entity.ts +++ b/src/db/js5/js5-data-entity.ts @@ -3,9 +3,9 @@ import { Js5FileType } from '../../config'; import { Buffer } from 'buffer'; -@Entity('js5_compressed_data') -@Index('compressed_data_identifier', [ - 'fileType', 'gameBuild', 'key', 'archiveKey', 'groupKey' +@Entity('js5_data') +@Index('data_identifier', [ + 'fileType', 'gameBuild', 'key', 'archiveKey', 'groupKey', 'compressed' ], { unique: true }) export class Js5DataEntity { @@ -24,12 +24,12 @@ export class Js5DataEntity { @PrimaryColumn('integer', { name: 'group_key', nullable: false, unique: false, default: -1 }) groupKey: number = -1; + @PrimaryColumn('boolean', { nullable: false, default: false }) + compressed: boolean = false; + @Column('blob', { name: 'buffer', nullable: true, default: null }) buffer: Buffer = null; - @Column('boolean', { nullable: true, default: false }) - compressed: boolean = false; - @CreateDateColumn() created?: Date; diff --git a/src/db/js5/js5-database.ts b/src/db/js5/js5-database.ts index 8fc337a..f1fc5d6 100644 --- a/src/db/js5/js5-database.ts +++ b/src/db/js5/js5-database.ts @@ -39,7 +39,7 @@ export class Js5Database extends IndexDatabase ({ ...d, compressed: false })); await this._dataRepo.upsert(chunk, { - conflictPaths: [ 'fileType', 'gameBuild', 'key', 'archiveKey', 'groupKey' ], + conflictPaths: [ 'fileType', 'gameBuild', 'key', 'archiveKey', 'groupKey', 'compressed' ], skipUpdateIfNoValuesChanged: true, }); } @@ -130,7 +130,7 @@ export class Js5Database extends IndexDatabase ({ ...d, compressed: true })); await this._dataRepo.upsert(chunk, { - conflictPaths: [ 'fileType', 'gameBuild', 'key', 'archiveKey', 'groupKey' ], + conflictPaths: [ 'fileType', 'gameBuild', 'key', 'archiveKey', 'groupKey', 'compressed' ], skipUpdateIfNoValuesChanged: true, }); } diff --git a/src/file-system/jag/content/animations/animations.ts b/src/file-system/jag/content/animations/animations.ts index 28bdf03..6cef9d4 100644 --- a/src/file-system/jag/content/animations/animations.ts +++ b/src/file-system/jag/content/animations/animations.ts @@ -41,21 +41,21 @@ export class Animations { const animChecksumList = versionListArchive.getFile('anim_crc'); const animIndexList = versionListArchive.getFile('anim_index'); - if (!animVersionList?.index?.data) { + if (!animVersionList?.data?.buffer?.length) { throw new Error(`anim_version file is not loaded!`); } - if (!animChecksumList?.index?.data) { + if (!animChecksumList?.data?.buffer?.length) { throw new Error(`anim_crc file is not loaded!`); } - if (!animIndexList?.index?.data) { + if (!animIndexList?.data?.buffer?.length) { throw new Error(`anim_index file is not loaded!`); } this.animations.clear(); - const versionData = new ByteBuffer(animVersionList.index.data); - const checksumData = new ByteBuffer(animVersionList.index.data); - const indexData = new ByteBuffer(animVersionList.index.data); + const versionData = new ByteBuffer(animVersionList.data.buffer); + const checksumData = new ByteBuffer(animVersionList.data.buffer); + const indexData = new ByteBuffer(animVersionList.data.buffer); const animCount = versionData.length / 2; for (let i = 0; i < animCount; i++) { @@ -84,12 +84,12 @@ export class Animations { decode(animKey: number): AnimationFile | null { const animFile = this.animationsIndex.getFile(animKey); - if (!animFile?.index?.data) { + if (!animFile?.data?.buffer?.length) { logger.warn(`Animation ${animKey} is empty or missing.`); return null; } - const animData = new ByteBuffer(animFile.index.data); + const animData = new ByteBuffer(animFile.data.buffer); //@todo stopped here - 12/08/22 - Kiko return null; diff --git a/src/file-system/jag/content/archives/interfaces/jag-interface-archive.ts b/src/file-system/jag/content/archives/interfaces/jag-interface-archive.ts index e2dda1b..37eff86 100644 --- a/src/file-system/jag/content/archives/interfaces/jag-interface-archive.ts +++ b/src/file-system/jag/content/archives/interfaces/jag-interface-archive.ts @@ -1,5 +1,5 @@ import { Buffer } from 'buffer'; -import { ByteBuffer } from '@runejs/common'; +import { ByteBuffer, logger } from '@runejs/common'; import { JagFileStore } from '../../../jag-file-store'; import { JagGameInterfaceEntity } from '../../../../../db/jag'; @@ -15,210 +15,210 @@ export class JagInterfaceArchive { } decode(data: ByteBuffer): JagGameInterfaceEntity { - const gameInterface = new JagGameInterfaceEntity(); - gameInterface.id = data.get('short', 'unsigned'); + const inter = new JagGameInterfaceEntity(); + inter.id = data.get('short', 'unsigned'); - if (gameInterface.id === 65535) { - gameInterface.parentId = data.get('short', 'unsigned'); - gameInterface.id = data.get('short', 'unsigned'); + if (inter.id === 65535) { + inter.parentId = data.get('short', 'unsigned'); + inter.id = data.get('short', 'unsigned'); } - const type = gameInterface.type = data.get('byte', 'unsigned'); - gameInterface.actionType = data.get('byte', 'unsigned'); - gameInterface.contentType = data.get('short', 'unsigned'); - const width = gameInterface.width = data.get('short', 'unsigned'); - const height = gameInterface.height = data.get('short', 'unsigned'); - gameInterface.alpha = data.get('byte', 'unsigned'); + const type = inter.type = data.get('byte', 'unsigned'); + inter.actionType = data.get('byte', 'unsigned'); + inter.contentType = data.get('short', 'unsigned'); + const width = inter.width = data.get('short', 'unsigned'); + const height = inter.height = data.get('short', 'unsigned'); + inter.alpha = data.get('byte', 'unsigned'); // hoveredPopup = u_short, but only a single u_byte is written if there is no hovered popup // use u_smart_short ? - gameInterface.hoveredPopup = data.get('byte', 'unsigned'); - if (gameInterface.hoveredPopup !== 0) { - gameInterface.hoveredPopup = (gameInterface.hoveredPopup - 1 << 8) + + inter.hoveredPopup = data.get('byte', 'unsigned'); + if (inter.hoveredPopup !== 0) { + inter.hoveredPopup = (inter.hoveredPopup - 1 << 8) + data.get('byte', 'unsigned'); // why? } else { - gameInterface.hoveredPopup = -1; + inter.hoveredPopup = -1; } const conditionCount = data.get('byte', 'unsigned'); if (conditionCount > 0) { - gameInterface.conditionTypes = new Array(conditionCount); - gameInterface.conditionValues = new Array(conditionCount); + inter.conditionTypes = new Array(conditionCount); + inter.conditionValues = new Array(conditionCount); for (let i = 0; i < conditionCount; i++) { - gameInterface.conditionTypes[i] = data.get('byte', 'unsigned'); - gameInterface.conditionValues[i] = data.get('short', 'unsigned'); + inter.conditionTypes[i] = data.get('byte', 'unsigned'); + inter.conditionValues[i] = data.get('short', 'unsigned'); } } const cs1OpcodeCount = data.get('byte', 'unsigned'); if (cs1OpcodeCount > 0) { - gameInterface.cs1Opcodes = new Array(cs1OpcodeCount); + inter.cs1Opcodes = new Array(cs1OpcodeCount); for (let i = 0; i < cs1OpcodeCount; i++) { const cs1BlockCount = data.get('short', 'unsigned'); - gameInterface.cs1Opcodes[i] = new Array(cs1BlockCount); + inter.cs1Opcodes[i] = new Array(cs1BlockCount); for (let j = 0; j < cs1BlockCount; j++) { - gameInterface.cs1Opcodes[i][j] = data.get('short', 'unsigned'); + inter.cs1Opcodes[i][j] = data.get('short', 'unsigned'); } } } if (type === 0) { - gameInterface.scrollLimit = data.get('short', 'unsigned'); - gameInterface.hiddenUntilHovered = data.get('byte', 'unsigned') === 1; + inter.scrollLimit = data.get('short', 'unsigned'); + inter.hiddenUntilHovered = data.get('byte', 'unsigned') === 1; const childCount = data.get('short', 'unsigned'); - gameInterface.children = new Array(childCount); - gameInterface.childrenX = new Array(childCount); - gameInterface.childrenY = new Array(childCount); + inter.children = new Array(childCount); + inter.childrenX = new Array(childCount); + inter.childrenY = new Array(childCount); for (let i = 0; i < childCount; i++) { - gameInterface.children[i] = data.get('short', 'unsigned'); - gameInterface.childrenX[i] = data.get('short'); - gameInterface.childrenY[i] = data.get('short'); + inter.children[i] = data.get('short', 'unsigned'); + inter.childrenX[i] = data.get('short'); + inter.childrenY[i] = data.get('short'); } } if (type === 1) { - gameInterface.unknownServerAttribute1 = data.get('short', 'unsigned'); - gameInterface.unknownServerAttribute2 = data.get('byte', 'unsigned') === 1; + inter.unknownServerAttribute1 = data.get('short', 'unsigned'); + inter.unknownServerAttribute2 = data.get('byte', 'unsigned') === 1; } if (type === 2) { - gameInterface.items = new Array(width * height); - gameInterface.itemAmounts = new Array(width * height); - gameInterface.itemsSwappable = data.get('byte', 'unsigned') === 1; - gameInterface.isInventory = data.get('byte', 'unsigned') === 1; - gameInterface.itemsUsable = data.get('byte', 'unsigned') === 1; - gameInterface.deleteDraggedItems = data.get('byte', 'unsigned') === 1; - gameInterface.itemSpritesPadX = data.get('byte', 'unsigned'); - gameInterface.itemSpritesPadY = data.get('byte', 'unsigned'); - gameInterface.images = new Array(20); - gameInterface.imagesX = new Array(20); - gameInterface.imagesY = new Array(20); + inter.items = new Array(width * height); + inter.itemAmounts = new Array(width * height); + inter.itemsSwappable = data.get('byte', 'unsigned') === 1; + inter.isInventory = data.get('byte', 'unsigned') === 1; + inter.itemsUsable = data.get('byte', 'unsigned') === 1; + inter.deleteDraggedItems = data.get('byte', 'unsigned') === 1; + inter.itemSpritesPadX = data.get('byte', 'unsigned'); + inter.itemSpritesPadY = data.get('byte', 'unsigned'); + inter.images = new Array(20); + inter.imagesX = new Array(20); + inter.imagesY = new Array(20); for (let i = 0; i < 20; i++) { const hasSprite = data.get('byte', 'unsigned') === 1; if (hasSprite) { - gameInterface.imagesX[i] = data.get('short'); - gameInterface.imagesY[i] = data.get('short'); - gameInterface.images[i] = data.getString(10); + inter.imagesX[i] = data.get('short'); + inter.imagesY[i] = data.get('short'); + inter.images[i] = data.getString(10); } } - gameInterface.options = new Array(5); + inter.options = new Array(5); for (let i = 0; i < 5; i++) { - gameInterface.options[i] = data.getString(10); + inter.options[i] = data.getString(10); } } if (type === 3) { - gameInterface.filled = data.get('byte', 'unsigned') === 1; + inter.filled = data.get('byte', 'unsigned') === 1; } if (type === 4 || type === 1) { - gameInterface.textCentered = data.get('byte', 'unsigned') === 1; - gameInterface.fontType = data.get('byte', 'unsigned'); - gameInterface.textShadowed = data.get('byte', 'unsigned') === 1; + inter.textCentered = data.get('byte', 'unsigned') === 1; + inter.fontType = data.get('byte', 'unsigned'); + inter.textShadowed = data.get('byte', 'unsigned') === 1; } if (type === 4) { - gameInterface.disabledText = data.getString(10); - gameInterface.enabledText = data.getString(10); + inter.disabledText = data.getString(10); + inter.enabledText = data.getString(10); } - if (gameInterface.type === 1 || gameInterface.type === 3 || gameInterface.type === 4) { - gameInterface.disabledColor = data.get('int'); + if (inter.type === 1 || inter.type === 3 || inter.type === 4) { + inter.disabledColor = data.get('int'); } - if (gameInterface.type === 3 || gameInterface.type === 4) { - gameInterface.enabledColor = data.get('int'); - gameInterface.disabledHoverColor = data.get('int'); - gameInterface.enabledHoverColor = data.get('int'); + if (inter.type === 3 || inter.type === 4) { + inter.enabledColor = data.get('int'); + inter.disabledHoverColor = data.get('int'); + inter.enabledHoverColor = data.get('int'); } - if (gameInterface.type === 5) { - gameInterface.disabledImage = data.getString(10); - gameInterface.enabledImage = data.getString(10); + if (inter.type === 5) { + inter.disabledImage = data.getString(10); + inter.enabledImage = data.getString(10); } - if (gameInterface.type === 6) { + if (inter.type === 6) { let identifier = data.get('byte', 'unsigned'); if (identifier !== 0) { - gameInterface.disabledModelType = 1; - gameInterface.disabledModelId = (identifier - 1 << 8) + data.get('byte', 'unsigned'); + inter.disabledModelType = 1; + inter.disabledModelId = (identifier - 1 << 8) + data.get('byte', 'unsigned'); } identifier = data.get('byte', 'unsigned'); if (identifier !== 0) { - gameInterface.enabledModelType = 1; - gameInterface.enabledModelId = (identifier - 1 << 8) + data.get('byte', 'unsigned'); + inter.enabledModelType = 1; + inter.enabledModelId = (identifier - 1 << 8) + data.get('byte', 'unsigned'); } identifier = data.get('byte', 'unsigned'); if (identifier !== 0) { - gameInterface.disabledAnimationId = (identifier - 1 << 8) + data.get('byte', 'unsigned'); + inter.disabledAnimationId = (identifier - 1 << 8) + data.get('byte', 'unsigned'); } else { - gameInterface.disabledAnimationId = -1; + inter.disabledAnimationId = -1; } identifier = data.get('byte', 'unsigned'); if (identifier !== 0) { - gameInterface.enabledAnimationId = (identifier - 1 << 8) + data.get('byte', 'unsigned'); + inter.enabledAnimationId = (identifier - 1 << 8) + data.get('byte', 'unsigned'); } else { - gameInterface.enabledAnimationId = -1; + inter.enabledAnimationId = -1; } - gameInterface.modelZoom = data.get('short', 'unsigned'); - gameInterface.modelRotationX = data.get('short', 'unsigned'); - gameInterface.modelRotationY = data.get('short', 'unsigned'); + inter.modelZoom = data.get('short', 'unsigned'); + inter.modelRotationX = data.get('short', 'unsigned'); + inter.modelRotationY = data.get('short', 'unsigned'); } - if (gameInterface.type === 7) { - gameInterface.items = new Array(width * height); - gameInterface.itemAmounts = new Array(width * height); - gameInterface.textCentered = data.get('byte', 'unsigned') === 1; - gameInterface.fontType = data.get('byte', 'unsigned'); - gameInterface.textShadowed = data.get('byte', 'unsigned') === 1; - gameInterface.disabledColor = data.get('int'); - gameInterface.itemSpritesPadX = data.get('short'); - gameInterface.itemSpritesPadY = data.get('short'); - gameInterface.isInventory = data.get('byte', 'unsigned') === 1; - gameInterface.options = new Array(5); + if (inter.type === 7) { + inter.items = new Array(width * height); + inter.itemAmounts = new Array(width * height); + inter.textCentered = data.get('byte', 'unsigned') === 1; + inter.fontType = data.get('byte', 'unsigned'); + inter.textShadowed = data.get('byte', 'unsigned') === 1; + inter.disabledColor = data.get('int'); + inter.itemSpritesPadX = data.get('short'); + inter.itemSpritesPadY = data.get('short'); + inter.isInventory = data.get('byte', 'unsigned') === 1; + inter.options = new Array(5); for (let i = 0; i < 5; i++) { - gameInterface.options[i] = data.getString(10); + inter.options[i] = data.getString(10); } } - if (gameInterface.type === 8) { - gameInterface.disabledText = data.getString(10); + if (inter.type === 8) { + inter.disabledText = data.getString(10); } - if (gameInterface.actionType === 2 || gameInterface.type === 2) { - gameInterface.actionAdditions = data.getString(10); - gameInterface.actionText = data.getString(10); - gameInterface.actionAttributes = data.get('short', 'unsigned'); + if (inter.actionType === 2 || inter.type === 2) { + inter.actionAdditions = data.getString(10); + inter.actionText = data.getString(10); + inter.actionAttributes = data.get('short', 'unsigned'); } - if (gameInterface.actionType === 1 || gameInterface.actionType === 4 || gameInterface.actionType === 5 || gameInterface.actionType === 6) { - gameInterface.tooltip = data.getString(10); + if (inter.actionType === 1 || inter.actionType === 4 || inter.actionType === 5 || inter.actionType === 6) { + inter.tooltip = data.getString(10); } - return gameInterface; + return inter; } - decodeAll(): void { + async decodeAll(): Promise { const archive = this.jagStore.getCache('archives') .getArchive('interface.jag'); @@ -227,16 +227,26 @@ export class JagInterfaceArchive { } const dataFile = archive.getFile('data'); - if (!dataFile?.index?.data) { + + await dataFile.loadUncompressedData(); + + if (!dataFile?.data?.buffer?.length) { throw new Error('interface.jag data file is not loaded!'); } - const data = new ByteBuffer(dataFile.index.data); + const data = new ByteBuffer(dataFile.data.buffer); this.interfaces.clear(); + data.get('short', 'unsigned'); // interface count + while (data.readerIndex < data.length) { - const gameInterface = this.decode(data); - this.interfaces.set(gameInterface.id, gameInterface); + try { + const gameInterface = this.decode(data); + this.interfaces.set(gameInterface.id, gameInterface); + } catch (e) { + logger.error(e); + break; + } } } diff --git a/src/file-system/jag/jag-archive.ts b/src/file-system/jag/jag-archive.ts index f23076b..93bef94 100644 --- a/src/file-system/jag/jag-archive.ts +++ b/src/file-system/jag/jag-archive.ts @@ -49,6 +49,18 @@ export class JagArchive extends JagFileBase { } } + async upsertFileData(): Promise { + const files = Array.from(this.files.values()); + const uncompressed = files.map(file => file.data).filter(data => data?.buffer && data?.buffer?.length !== 0); + const compressed = files.map(file => file.compressedData).filter(data => data?.buffer && data?.buffer?.length !== 0); + if (uncompressed.length) { + await this.fileStore.database.upsertAllUncompressedData(uncompressed); + } + if (compressed.length) { + await this.fileStore.database.upsertAllCompressedData(compressed); + } + } + getFile(fileKey: number): JagFile | null; getFile(fileName: string): JagFile | null; getFile(fileKeyOrName: number | string): JagFile | null; diff --git a/src/file-system/jag/jag-cache.ts b/src/file-system/jag/jag-cache.ts index 4ee5bd4..ed896d0 100644 --- a/src/file-system/jag/jag-cache.ts +++ b/src/file-system/jag/jag-cache.ts @@ -55,6 +55,18 @@ export class JagCache extends JagFileBase { } } + async upsertFileData(): Promise { + const files = Array.from(this.files.values()); + const uncompressed = files.map(file => file.data).filter(data => data?.buffer && data?.buffer?.length !== 0); + const compressed = files.map(file => file.compressedData).filter(data => data?.buffer && data?.buffer?.length !== 0); + if (uncompressed.length) { + await this.fileStore.database.upsertAllUncompressedData(uncompressed); + } + if (compressed.length) { + await this.fileStore.database.upsertAllCompressedData(compressed); + } + } + createArchive(archiveKey: number): JagArchive { const archive = new JagArchive(this.fileStore, archiveKey); this.setArchive(archiveKey, archive); diff --git a/src/file-system/jag/jag-file-base.ts b/src/file-system/jag/jag-file-base.ts index c77ce64..c91617e 100644 --- a/src/file-system/jag/jag-file-base.ts +++ b/src/file-system/jag/jag-file-base.ts @@ -4,7 +4,7 @@ import { Buffer } from 'buffer'; import { Crc32 } from '@runejs/common/crc32'; import { createHash } from 'crypto'; import { JagFileStore } from './jag-file-store'; -import { JagIndexEntity } from '../../db/jag/jag-index-entity'; +import { JagDataEntity, JagIndexEntity } from '../../db/jag'; export abstract class JagFileBase { @@ -12,6 +12,8 @@ export abstract class JagFileBase { readonly fileStore: JagFileStore; index: JagIndexEntity; + data: JagDataEntity; + compressedData: JagDataEntity; protected constructor( fileStore: JagFileStore, @@ -22,19 +24,23 @@ export abstract class JagFileBase { ) { this.fileStore = fileStore; this.index = new JagIndexEntity(); - this.index.gameBuild = fileStore.gameBuild; - this.index.fileType = fileType; - this.index.key = key; - this.index.cacheKey = cacheKey; - this.index.archiveKey = archiveKey; + this.data = new JagDataEntity(); + this.compressedData = new JagDataEntity(); + this.compressedData.compressed = true; + this.data.gameBuild = this.compressedData.gameBuild = this.index.gameBuild = fileStore.gameBuild; + this.data.fileType = this.compressedData.fileType = this.index.fileType = fileType; + this.data.key = this.compressedData.key = this.index.key = key; + this.data.cacheKey = this.compressedData.cacheKey = this.index.cacheKey = cacheKey; + this.data.archiveKey = this.compressedData.archiveKey = this.index.archiveKey = archiveKey; } validate(trackChanges: boolean = true): void { + const data = this.data.buffer; + const compressedData = this.compressedData.buffer; const { - data, compressedData, checksum, shaDigest, fileSize, compressedChecksum, compressedShaDigest, compressedFileSize, - name, nameHash, + name, nameHash, compressionMethod, } = this.index; let fileModified: boolean = false; @@ -95,12 +101,106 @@ export abstract class JagFileBase { fileModified = true; } + if ((!this.index.compressionMethod || this.index.compressionMethod === 'none' || this.index.compressedFileSize === fileSize) + && this.compressedData?.buffer?.length) { + // File has no compression, clear the compressed data buffer so that we do not create a + // duplicate data entity record for it + this.compressedData.buffer = null; + this.index.compressedFileSize = 0; + this.index.compressedChecksum = -1; + this.index.compressedShaDigest = null; + } + if (fileModified && trackChanges) { logger.info(`File ${this.index.name || this.index.key} has been modified.`); this.index.version++; } } + async saveUncompressedData(): Promise { + if (!this.data?.buffer?.length) { + // Do not save a record for files with missing or empty data + return null; + } + + this.data = await this.fileStore.database.saveUncompressedData(this.data); + return this.data; + } + + async loadUncompressedData(): Promise { + const entity = await this.fileStore.database.getUncompressedData({ + fileType: this.index.fileType, + key: this.index.key, + archiveKey: this.index.archiveKey, + cacheKey: this.index.cacheKey, + }); + + if (entity) { + this.data = entity; + } + + return this.data; + } + + async getUncompressedData(): Promise { + if (this.data?.buffer?.length) { + return this.data.buffer; + } + + const uncompressedData = await this.loadUncompressedData(); + if (uncompressedData?.buffer?.length) { + return uncompressedData.buffer; + } + + return null; + } + + async saveCompressedData(): Promise { + if (!this.compressedData?.buffer?.length) { + // Do not save a record for files with missing or empty data + return null; + } + + this.compressedData = await this.fileStore.database.saveCompressedData(this.compressedData); + return this.compressedData; + } + + async loadCompressedData(): Promise { + const entity = await this.fileStore.database.getCompressedData({ + fileType: this.index.fileType, + key: this.index.key, + archiveKey: this.index.archiveKey, + cacheKey: this.index.cacheKey, + }); + + if (entity) { + this.compressedData = entity; + } + + return this.compressedData; + } + + async getCompressedData(): Promise { + if (!this.index) { + await this.loadIndex(); + } + + if (!this.index.compressionMethod || this.index.compressionMethod === 'none') { + return this.getUncompressedData(); + } + + if (this.compressedData?.buffer?.length) { + return this.compressedData.buffer; + } + + const compressedData = await this.loadCompressedData(); + if (compressedData?.buffer?.length) { + return compressedData.buffer; + } + + return null; + } + async saveIndex(): Promise { this.validate(); this.index = await this.fileStore.database.saveIndex(this.index); diff --git a/src/file-system/jag/jag.ts b/src/file-system/jag/jag.ts index d96a984..d918a96 100644 --- a/src/file-system/jag/jag.ts +++ b/src/file-system/jag/jag.ts @@ -3,7 +3,7 @@ import { existsSync, readdirSync, readFileSync, statSync } from 'graceful-fs'; import { Buffer } from 'buffer'; import { logger } from '@runejs/common'; import { ByteBuffer } from '@runejs/common/buffer'; -import { Bzip2 } from '@runejs/common/compress'; +import { Bzip2, Gzip } from '@runejs/common/compress'; import { JagFileStore } from './jag-file-store'; import { JagFile } from './jag-file'; import { JagArchive } from './jag-archive'; @@ -159,13 +159,16 @@ export class Jag { logger.info(`${fileCount} file indexes found.`); - const index = this.jagStore.getCache(cacheKey); - index.fileIndexes = new Array(fileCount); + const cache = this.jagStore.getCache(cacheKey); + cache.fileIndexes = new Array(fileCount); + cache.index.childCount = fileCount; + cache.index.compressionMethod = 'none'; + cache.data.buffer = indexFile.toNodeBuffer(); for (let fileKey = 0; fileKey < fileCount; fileKey++) { const fileSize = indexFile.get('int24', 'unsigned'); const sectorPos = indexFile.get('int24', 'unsigned'); - index.fileIndexes[fileKey] = { + cache.fileIndexes[fileKey] = { fileSize, sectorNumber: sectorPos }; @@ -177,9 +180,11 @@ export class Jag { file = new JagFile(this.jagStore, fileKey, cacheKey); } - index.files.set(fileKey, file); + cache.files.set(fileKey, file); } + cache.validate(false); + logger.info(`Cache index ${indexName} has been decoded.`); } @@ -248,33 +253,46 @@ export class Jag { currentSectorNumber = sectorNumber; } + if (!(file instanceof JagArchive)) { + file.index.compressionMethod = 'gzip'; + } + if (fileData.length) { - file.index.compressedData = fileData.toNodeBuffer(); + file.compressedData.buffer = fileData.toNodeBuffer(); + + if (!(file instanceof JagArchive)) { + file.data.buffer = Gzip.decompress(fileData); + } } else { - file.index.compressedData = null; + file.compressedData.buffer = null; file.index.fileError = 'FILE_MISSING'; } - file.validate(false); + if (!(file instanceof JagArchive)) { + file.validate(false); + } - return file.index.compressedData; + return file.compressedData.buffer; } decodeArchive(archive: JagArchive): Buffer | null { - if (!archive.index.compressedData?.length) { + if (!archive.compressedData?.buffer?.length) { return null; } - let archiveData = new ByteBuffer(archive.index.compressedData); + let archiveData = new ByteBuffer(archive.compressedData.buffer); const uncompressed = archiveData.get('int24', 'unsigned'); const compressed = archiveData.get('int24', 'unsigned'); if (uncompressed !== compressed) { const compressedData = archiveData.getSlice(archiveData.readerIndex, archiveData.length - archiveData.readerIndex); - archiveData = new ByteBuffer(Bzip2.decompress(compressedData)); + const decompressedData = Bzip2.decompress(compressedData); + archiveData = new ByteBuffer(decompressedData); + archive.data.buffer = decompressedData; archive.index.compressionMethod = 'bzip2'; } else { + archive.data.buffer = archiveData.toNodeBuffer(); archive.index.compressionMethod = 'none'; } @@ -309,7 +327,7 @@ export class Jag { const fileDataOffset = fileDataOffsets[fileKey]; const fileData = Buffer.alloc(file.index.compressedFileSize); archiveData.copy(fileData, 0, fileDataOffset); - file.index.compressedData = fileData; + file.compressedData.buffer = fileData; } catch (error) { logger.error(`Error reading archive ${archive.index.name } file ${fileKey}`, error); } @@ -318,13 +336,12 @@ export class Jag { // Decompress archive file data (if needed) for (const [ fileKey, file ] of archive.files) { try { - const { compressedFileSize, fileSize, compressedData } = file.index; - if (compressedData?.length && compressedFileSize !== fileSize) { - file.index.data = Bzip2.decompress(file.index.compressedData); + const { compressedFileSize, fileSize } = file.index; + if (file.compressedData?.buffer?.length && compressedFileSize !== fileSize) { + file.data.buffer = Bzip2.decompress(file.compressedData.buffer); file.index.compressionMethod = 'bzip2'; } else { - file.index.data = file.index.compressedData; - file.index.compressedData = null; + file.data.buffer = file.compressedData.buffer; file.index.compressionMethod = 'none'; } } catch (error) { @@ -337,13 +354,9 @@ export class Jag { file.validate(false); } - if (archiveData.length) { - archive.index.data = archiveData.toNodeBuffer(); - } - archive.validate(false); - return archive.index.data; + return archive.data.buffer; } } diff --git a/src/file-system/js5/js5-file-base.ts b/src/file-system/js5/js5-file-base.ts index c1e5f60..a4cae9c 100644 --- a/src/file-system/js5/js5-file-base.ts +++ b/src/file-system/js5/js5-file-base.ts @@ -26,6 +26,7 @@ export abstract class Js5FileBase { this.index = new Js5IndexEntity(); this.data = new Js5DataEntity(); this.compressedData = new Js5DataEntity(); + this.compressedData.compressed = true; this.data.gameBuild = this.compressedData.gameBuild = this.index.gameBuild = fileStore.gameBuild; this.data.fileType = this.compressedData.fileType = this.index.fileType = fileType; this.data.key = this.compressedData.key = this.index.key = key; @@ -58,7 +59,7 @@ export abstract class Js5FileBase { if (nameHash !== -1 && !name) { // name not set - const lookupTableName = this.fileStore.nameHasher.findFileName(nameHash); + const lookupTableName = this.fileStore.findFileName(this); if (lookupTableName) { this.index.name = lookupTableName; } @@ -100,11 +101,14 @@ export abstract class Js5FileBase { fileModified = true; } - if (compressionMethod === 'none' || - (compressedFileSize === fileSize && this.compressedData?.buffer?.length)) { + if ((!this.index.compressionMethod || this.index.compressionMethod === 'none' || this.index.compressedFileSize === fileSize) + && this.compressedData?.buffer?.length) { // File has no compression, clear the compressed data buffer so that we do not create a // duplicate data entity record for it this.compressedData.buffer = null; + this.index.compressedFileSize = 0; + this.index.compressedChecksum = -1; + this.index.compressedShaDigest = null; } if (fileModified && trackChanges) { @@ -181,7 +185,7 @@ export abstract class Js5FileBase { await this.loadIndex(); } - if (this.index.compressionMethod === 'none') { + if (!this.index.compressionMethod || this.index.compressionMethod === 'none') { return this.getUncompressedData(); } diff --git a/src/file-system/js5/js5-file-store.ts b/src/file-system/js5/js5-file-store.ts index bfd9c3d..fd16a7f 100644 --- a/src/file-system/js5/js5-file-store.ts +++ b/src/file-system/js5/js5-file-store.ts @@ -8,6 +8,7 @@ import { FileStoreBase } from '../file-store-base'; import { logger } from '../../../../common'; import { Js5Database, Js5IndexEntity } from '../../db/js5'; import { Js5File } from './js5-file'; +import { Js5FileBase } from './js5-file-base'; export class Js5FileStore extends FileStoreBase{ @@ -120,6 +121,32 @@ export class Js5FileStore extends FileStoreBase{ } } + findFileName(file: Js5FileBase): string | null { + const nameHash = file.index.nameHash || -1; + + if (nameHash !== -1) { + return this.nameHasher.findFileName(nameHash, file.index.name || String(file.index.nameHash) || String(file.index.key)); + } + + const { key, groupKey, archiveKey, fileType } = file.index; + + const archiveConfig = this.getArchiveConfig(fileType === 'ARCHIVE' ? key : archiveKey); + + if (archiveConfig) { + if (fileType === 'GROUP' && archiveConfig.groupNames) { + const groupNames = Array.from(Object.entries(archiveConfig.groupNames)); + return groupNames.find(([ , k ]) => k === key)?.[0] || null; + } else if (fileType === 'ARCHIVE') { + const configs = Array.from(Object.entries(this.archiveConfig)); + return configs.find(([ , config ]) => config.key === archiveKey)?.[0] || null; + } else if (fileType === 'FILE') { + // @todo 2/9/22 - Kiko + } + } + + return null; + } + createArchive(archiveKey: number): Js5Archive { const archive = new Js5Archive(this, archiveKey); this.setArchive(archiveKey, archive); diff --git a/src/file-system/js5/js5.ts b/src/file-system/js5/js5.ts index 60e24ad..e9a209e 100644 --- a/src/file-system/js5/js5.ts +++ b/src/file-system/js5/js5.ts @@ -542,10 +542,7 @@ export class JS5 { if (flags.groupNames) { for (const group of groups) { group.index.nameHash = archiveData.get('int'); - group.index.name = this.fileStore.nameHasher.findFileName( - group.index.nameHash, - group.index.name || String(group.index.nameHash) || String(group.index.key) - ); + group.index.name = this.fileStore.findFileName(group); } } @@ -605,10 +602,7 @@ export class JS5 { for (const group of groups) { for (const [ , flatFile ] of group.files) { flatFile.index.nameHash = archiveData.get('int'); - flatFile.index.name = this.fileStore.nameHasher.findFileName( - flatFile.index.nameHash, - flatFile.index.name || String(flatFile.index.nameHash) || String(flatFile.index.key) - ); + flatFile.index.name = this.fileStore.findFileName(flatFile); } } } diff --git a/src/scripts/dev.ts b/src/scripts/dev.ts index ce232df..090d34d 100644 --- a/src/scripts/dev.ts +++ b/src/scripts/dev.ts @@ -10,7 +10,7 @@ const saveInterfaces = async (store: JagFileStore) => { const interfaceArchive = new JagInterfaceArchive(store); - interfaceArchive.decodeAll(); + await interfaceArchive.decodeAll(); logger.info(`${interfaceArchive.interfaces.size} interfaces decoded. Saving interface entities...`); @@ -27,8 +27,8 @@ const dumpInterfaceFile = (store: JagFileStore) => { } const dataFile = archive.getFile('data'); - const binaryData = dataFile?.index?.data; - if (!binaryData) { + const binaryData = dataFile?.data?.buffer; + if (!binaryData?.length) { throw new Error('interface.jag data file is not loaded!'); } @@ -48,18 +48,10 @@ const dumpInterfaceFile = (store: JagFileStore) => { const dev = async () => { const start = Date.now(); - const store = new Js5FileStore(435); - await store.load(true, true, false); + const store = new JagFileStore(327); + await store.load(true, true, true); - const fileNames = [ - 'compass', - 'mapback' - ]; - - for (const name of fileNames) { - const spriteFile = await (await store.getArchive('sprites')).getGroup(name); - store.js5.decompress(spriteFile); - } + await saveInterfaces(store); const end = Date.now(); logger.info(`Operations completed in ${(end - start) / 1000} seconds.`); diff --git a/src/scripts/indexer.ts b/src/scripts/indexer.ts index 3b61dae..e8ebb81 100644 --- a/src/scripts/indexer.ts +++ b/src/scripts/indexer.ts @@ -1,11 +1,14 @@ import { logger } from '@runejs/common'; import { ScriptExecutor, ArgumentOptions } from './script-executor'; import { Js5FileStore } from '../file-system/js5'; -import { caches, JagArchive, JagFileStore } from '../file-system/jag'; +import { archives, caches, JagArchive, JagFileStore } from '../file-system/jag'; import { getOpenRS2CacheFilesByBuild, } from '../openrs2'; import { PackedCacheFile, getPackedCacheFormat } from '../file-system/packed'; +import { + JagInterfaceArchive +} from '../file-system/jag/content/archives/interfaces/jag-interface-archive'; interface IndexerOptions { @@ -195,16 +198,16 @@ const indexJagStore = async (store: JagFileStore) => { store.jag.decodeCache(indexName); } - logger.info(`Saving indexes...`); + logger.info(`Saving JAG caches...`); - for (const [ , indexFile ] of store.caches) { - await indexFile.saveIndex(); + for (const [ , cache ] of store.caches) { + await cache.saveIndex(); } - for (const [, indexFile ] of store.caches) { - logger.info(`Unpacking JAG files for index ${indexFile.index.name}...`); + for (const [, cache ] of store.caches) { + logger.info(`Unpacking JAG files for index ${cache.index.name}...`); - for (const [ , file ] of indexFile.files) { + for (const [ , file ] of cache.files) { store.jag.unpack(file); } } @@ -233,6 +236,41 @@ const indexJagStore = async (store: JagFileStore) => { await archive.upsertFileIndexes(); } } + + logger.info(`Saving JAG cache data...`); + + for (const [ , cache ] of store.caches) { + await cache.saveCompressedData(); + await cache.saveUncompressedData(); + } + + logger.info(`Saving JAG cache file data...`); + + for (const [ , cache ] of store.caches) { + await cache.upsertFileData(); + } + + logger.info(`Saving JAG archive file data...`); + + for (const [ , archive ] of archiveIndex.files) { + if (archive instanceof JagArchive) { + await archive.upsertFileData(); + } + } + + const saveInterfaces = async (store: JagFileStore) => { + logger.info(`Decoding game interfaces...`); + + const interfaceArchive = new JagInterfaceArchive(store); + + await interfaceArchive.decodeAll(); + + logger.info(`${interfaceArchive.interfaces.size} interfaces decoded. Saving interface entities...`); + + await interfaceArchive.saveAll(); + }; + + await saveInterfaces(store); };