From 90919364ac2df91beb41d823c2488541c84371f6 Mon Sep 17 00:00:00 2001 From: Joan Gallego Girona Date: Fri, 29 Nov 2024 11:05:28 +0100 Subject: [PATCH 1/3] missingInDB files now also report lastUpdated date (#7507) --- app/api/files.v2/FilesHealthCheck.ts | 8 +++++++- .../files.v2/infrastructure/S3FileStorage.ts | 2 +- app/api/files.v2/model/StoredFile.ts | 5 ++++- .../files.v2/specs/FilesHealthCheck.spec.ts | 20 +++++++++---------- 4 files changed, 22 insertions(+), 13 deletions(-) diff --git a/app/api/files.v2/FilesHealthCheck.ts b/app/api/files.v2/FilesHealthCheck.ts index 674a847258..14f5ea8b39 100644 --- a/app/api/files.v2/FilesHealthCheck.ts +++ b/app/api/files.v2/FilesHealthCheck.ts @@ -2,6 +2,7 @@ import { FilesDataSource } from './contracts/FilesDataSource'; import { FileStorage } from './contracts/FileStorage'; import { StoredFile } from './model/StoredFile'; import { URLAttachment } from './model/URLAttachment'; + function filterFilesInStorage(files: StoredFile[]) { return files.filter( file => @@ -14,6 +15,7 @@ function filterFilesInStorage(files: StoredFile[]) { type missingInDBFileDTO = { filename: string; checksumMatchCount: number; + lastModified?: Date; }; export class FilesHealthCheck { @@ -69,7 +71,11 @@ export class FilesHealthCheck { if (checksumMatchCount > 1) { counters.missingInDbWithChecksumMatches += 1; } - this.onMissingInDBCB({ filename: storedFile.fullPath, checksumMatchCount }); + this.onMissingInDBCB({ + filename: storedFile.fullPath, + checksumMatchCount, + lastModified: storedFile.lastModified, + }); }); return { diff --git a/app/api/files.v2/infrastructure/S3FileStorage.ts b/app/api/files.v2/infrastructure/S3FileStorage.ts index 801b800a3e..d95fcf310d 100644 --- a/app/api/files.v2/infrastructure/S3FileStorage.ts +++ b/app/api/files.v2/infrastructure/S3FileStorage.ts @@ -53,6 +53,6 @@ export class S3FileStorage implements FileStorage { continuationToken = await requestNext(continuationToken); } - return objects.map(c => new StoredFile(c.Key!, c.ETag!)); + return objects.map(c => new StoredFile(c.Key!, c.LastModified, c.ETag!)); } } diff --git a/app/api/files.v2/model/StoredFile.ts b/app/api/files.v2/model/StoredFile.ts index 9fc7e23b16..1650503792 100644 --- a/app/api/files.v2/model/StoredFile.ts +++ b/app/api/files.v2/model/StoredFile.ts @@ -5,11 +5,14 @@ export class StoredFile { readonly fullPath: string; + readonly lastModified?: Date; + readonly checksum?: string; - constructor(fullPath: string, checksum?: string) { + constructor(fullPath: string, lastModified?: Date, checksum?: string) { this.filename = path.basename(fullPath); this.fullPath = fullPath; this.checksum = checksum; + this.lastModified = lastModified; } } diff --git a/app/api/files.v2/specs/FilesHealthCheck.spec.ts b/app/api/files.v2/specs/FilesHealthCheck.spec.ts index 551e17d6c0..c9f09b97c7 100644 --- a/app/api/files.v2/specs/FilesHealthCheck.spec.ts +++ b/app/api/files.v2/specs/FilesHealthCheck.spec.ts @@ -184,12 +184,12 @@ describe('FilesHealthCheck', () => { it('should emit the count of duplicated checksums for each file emited', async () => { testStorageFiles = [ - new StoredFile('document/file1', 'checksum1'), - new StoredFile('document/file2', 'checksum2'), + new StoredFile('document/file1', new Date(), 'checksum1'), + new StoredFile('document/file2', new Date(), 'checksum2'), new StoredFile('document/file3'), - new StoredFile('document/file4', 'checksum1'), - new StoredFile('document/file5', 'checksum1'), - new StoredFile('document/file6', 'checksum2'), + new StoredFile('document/file4', new Date(), 'checksum1'), + new StoredFile('document/file5', new Date(), 'checksum1'), + new StoredFile('document/file6', new Date(), 'checksum2'), ]; await testingEnvironment.setUp({ files: [] }); @@ -211,12 +211,12 @@ describe('FilesHealthCheck', () => { it('should emit in the summary the number of files which have a checksum match count > 1', async () => { testStorageFiles = [ - new StoredFile('document/file1', 'checksum1'), - new StoredFile('document/file2', 'checksum2'), + new StoredFile('document/file1', new Date(), 'checksum1'), + new StoredFile('document/file2', new Date(), 'checksum2'), new StoredFile('document/file3'), - new StoredFile('document/file4', 'checksum1'), - new StoredFile('document/file5', 'checksum1'), - new StoredFile('document/file6', 'checksum2'), + new StoredFile('document/file4', new Date(), 'checksum1'), + new StoredFile('document/file5', new Date(), 'checksum1'), + new StoredFile('document/file6', new Date(), 'checksum2'), ]; await testingEnvironment.setUp({ files: [] }); From e8102c047d82538bfaf95f96aca7eb1c4bf04717 Mon Sep 17 00:00:00 2001 From: Daneryl Date: Fri, 29 Nov 2024 11:05:34 +0100 Subject: [PATCH 2/3] Bump version --- package.json | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/package.json b/package.json index 7e1736cb5b..8d0e0da000 100644 --- a/package.json +++ b/package.json @@ -1,6 +1,6 @@ { "name": "uwazi", - "version": "1.191.10", + "version": "1.191.11", "description": "Uwazi is a free, open-source solution for organising, analysing and publishing your documents.", "keywords": [ "react" From c2546c492cbf63c387f85751627b0eb40ec5369c Mon Sep 17 00:00:00 2001 From: Daneryl Date: Fri, 29 Nov 2024 11:05:48 +0100 Subject: [PATCH 3/3] Merge back from production and Bump rc version --- package.json | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/package.json b/package.json index 902c89b4d9..d477108fdc 100644 --- a/package.json +++ b/package.json @@ -1,6 +1,6 @@ { "name": "uwazi", - "version": "1.192.0-rc10", + "version": "1.192.0-rc11", "description": "Uwazi is a free, open-source solution for organising, analysing and publishing your documents.", "keywords": [ "react"