Merge remote-tracking branch 'origin/development' into 7085-ix-status…

…es-revamp-dev

.github/workflows/ci_check_translations.yml

@@ -3,9 +3,7 @@ name: check translations
 on:
   push:
     branches:
-      - production
       - development
-      - staging
   pull_request:
 
 concurrency:

.github/workflows/ci_cypress_component.yml

@@ -3,10 +3,7 @@ name: Component testing
 on:
   push:
     branches:
-      - production
       - development
-      - staging
-
   pull_request:
 
 concurrency:

.github/workflows/ci_e2e_cypress_base.yml

@@ -3,9 +3,7 @@ name: e2e cypress BASE
 on:
   push:
     branches:
-      - production
       - development
-      - staging
   pull_request:
 
 concurrency:

.github/workflows/ci_e2e_cypress_pages.yml

@@ -3,9 +3,7 @@ name: e2e cypress PAGES
 on:
   push:
     branches:
-      - production
       - development
-      - staging
   pull_request:
 
 concurrency:

.github/workflows/ci_e2e_cypress_ssettings.yml

@@ -3,9 +3,7 @@ name: e2e cypress SETTINGS
 on:
   push:
     branches:
-      - production
       - development
-      - staging
   pull_request:
 
 concurrency:

.github/workflows/ci_e2e_puppeteer.yml

@@ -3,9 +3,7 @@ name: e2e puppeteer
 on:
   push:
     branches:
-      - production
       - development
-      - staging
   pull_request:
 
 concurrency:

.github/workflows/ci_eslint.yml

@@ -3,9 +3,7 @@ name: eslint
 on:
   push:
     branches:
-      - production
       - development
-      - staging
   pull_request:
 
 concurrency:

.github/workflows/ci_ts_emit_check.yml

@@ -3,9 +3,7 @@ name: check-emitted-types
 on:
   push:
     branches:
-      - production
       - development
-      - staging
   pull_request:
 
 concurrency:

.github/workflows/ci_ts_types.yml

@@ -3,9 +3,7 @@ name: check-types
 on:
   push:
     branches:
-      - production
       - development
-      - staging
   pull_request:
 
 concurrency:

.github/workflows/ci_unit_tests.yml

@@ -3,9 +3,7 @@ name: Unit tests
 on:
   push:
     branches:
-      - production
       - development
-      - staging
   pull_request:
 
 concurrency:

app/api/files.v2/FilesHealthCheck.ts

@@ -0,0 +1,71 @@
+import { FilesDataSource } from './contracts/FilesDataSource';
+import { FileStorage } from './contracts/FileStorage';
+import { URLAttachment } from './model/URLAttachment';
+
+function filterFilesInStorage(files: string[]) {
+  return files.filter(
+    file =>
+      !file.includes('/log/') && !file.includes('/segmentation/') && !file.includes('index.html')
+  );
+}
+
+export class FilesHealthCheck {
+  // eslint-disable-next-line class-methods-use-this
+  private onMissingInDBCB: (filename: string) => void = () => {};
+
+  // eslint-disable-next-line class-methods-use-this
+  private onMissingInStorageCB: (fileDTO: { _id: string; filename: string }) => void = () => {};
+
+  private fileStorage: FileStorage;
+
+  private filesDS: FilesDataSource;
+
+  constructor(fileStorage: FileStorage, filesDS: FilesDataSource) {
+    this.fileStorage = fileStorage;
+    this.filesDS = filesDS;
+  }
+
+  async execute() {
+    const allFilesInDb = await this.filesDS.getAll().all();
+    const allFilesInStorage = await this.fileStorage.list();
+    const filteredFilesInStorage = new Set(filterFilesInStorage(allFilesInStorage));
+    let missingInStorage = 0;
+    const missingInStorageList: string[] = [];
+    const missingInDbList: string[] = [];
+    const countInStorage = filteredFilesInStorage.size;
+    let countInDb = 0;
+
+    allFilesInDb.forEach(file => {
+      countInDb += 1;
+      const existsInStorage = filteredFilesInStorage.delete(this.fileStorage.getPath(file));
+
+      if (!existsInStorage && !(file instanceof URLAttachment)) {
+        missingInStorage += 1;
+        missingInStorageList.push(this.fileStorage.getPath(file));
+        this.onMissingInStorageCB({ _id: file.id, filename: file.filename });
+      }
+    });
+
+    filteredFilesInStorage.forEach(file => {
+      missingInDbList.push(file);
+      this.onMissingInDBCB(file);
+    });
+
+    return {
+      missingInStorageList,
+      missingInStorage,
+      missingInDbList,
+      missingInDb: filteredFilesInStorage.size,
+      countInDb,
+      countInStorage,
+    };
+  }
+
+  onMissingInDB(cb: (filename: string) => void) {
+    this.onMissingInDBCB = cb;
+  }
+
+  onMissingInStorage(cb: (fileDTO: { _id: string; filename: string }) => void) {
+    this.onMissingInStorageCB = cb;
+  }
+}

app/api/files.v2/contracts/FileStorage.ts

@@ -0,0 +1,6 @@
+import { UwaziFile } from '../model/UwaziFile';
+
+export interface FileStorage {
+  list(): Promise<string[]>;
+  getPath(file: UwaziFile): string;
+}

app/api/files.v2/contracts/FilesDataSource.ts

@@ -1,3 +1,7 @@
+import { ResultSet } from 'api/common.v2/contracts/ResultSet';
+import { UwaziFile } from '../model/UwaziFile';
+
 export interface FilesDataSource {
   filesExistForEntities(files: { entity: string; _id: string }[]): Promise<boolean>;
+  getAll(): ResultSet<UwaziFile>;
 }

app/api/files.v2/database/FilesMappers.ts

@@ -0,0 +1,53 @@
+// import { OptionalId } from 'mongodb';
+
+import { FileDBOType } from './schemas/filesTypes';
+import { UwaziFile } from '../model/UwaziFile';
+import { Document } from '../model/Document';
+import { URLAttachment } from '../model/URLAttachment';
+import { Attachment } from '../model/Attachment';
+import { CustomUpload } from '../model/CustomUpload';
+
+export const FileMappers = {
+  // toDBO(file: UwaziFile): OptionalId<FileDBOType> {
+  //   return {
+  //     filename: file.filename,
+  //     entity: file.entity,
+  //     type: 'document',
+  //     totalPages: file.totalPages,
+  //   };
+  // },
+
+  toModel(fileDBO: FileDBOType): UwaziFile {
+    if (fileDBO.type === 'attachment' && fileDBO.url) {
+      return new URLAttachment(
+        fileDBO._id.toString(),
+        fileDBO.entity,
+        fileDBO.totalPages,
+        fileDBO.url
+      );
+    }
+    if (fileDBO.type === 'attachment') {
+      return new Attachment(
+        fileDBO._id.toString(),
+        fileDBO.entity,
+        fileDBO.totalPages,
+        fileDBO.filename
+      );
+    }
+
+    if (fileDBO.type === 'custom') {
+      return new CustomUpload(
+        fileDBO._id.toString(),
+        fileDBO.entity,
+        fileDBO.totalPages,
+        fileDBO.filename
+      );
+    }
+    return new Document(
+      fileDBO._id.toString(),
+      fileDBO.entity,
+      fileDBO.totalPages,
+      fileDBO.filename
+    );
+  },
+};

app/api/files.v2/database/MongoFilesDataSource.ts

@@ -1,9 +1,19 @@
 import { MongoDataSource } from 'api/common.v2/database/MongoDataSource';
+import { MongoResultSet } from 'api/common.v2/database/MongoResultSet';
 import { ObjectId } from 'mongodb';
 import { FilesDataSource } from '../contracts/FilesDataSource';
+import { FileMappers } from './FilesMappers';
 import { FileDBOType } from './schemas/filesTypes';
+import { UwaziFile } from '../model/UwaziFile';
 
 export class MongoFilesDataSource extends MongoDataSource<FileDBOType> implements FilesDataSource {
+  getAll() {
+    return new MongoResultSet<FileDBOType, UwaziFile>(
+      this.getCollection().find({}, { projection: { fullText: 0 } }),
+      FileMappers.toModel
+    );
+  }
+
   protected collectionName = 'files';
 
   async filesExistForEntities(files: { entity: string; _id: string }[]) {

app/api/files.v2/database/schemas/filesTypes.ts

@@ -3,10 +3,12 @@ import { ObjectId } from 'mongodb';
 interface BaseFileDBOType {
   _id: ObjectId;
   entity: string;
+  filename: string;
+  url: string;
 }
 
 interface DocumentFileDBOType extends BaseFileDBOType {
-  type: 'document';
+  type: 'document' | 'attachment' | 'custom';
   totalPages: number;
 }
 

app/api/files.v2/database/specs/MongoFilesDataSource.spec.ts

@@ -8,9 +8,9 @@ const factory = getFixturesFactory();
 
 const fixtures = {
   files: [
-    factory.file('file1', 'entity1', 'document', 'file1.pdf'),
-    factory.file('file2', 'entity2', 'document', 'file2.pdf'),
-    factory.file('file3', 'entity3', 'document', 'file3.pdf'),
+    factory.fileDeprecated('file1', 'entity1', 'document', 'file1.pdf'),
+    factory.fileDeprecated('file2', 'entity2', 'document', 'file2.pdf'),
+    factory.fileDeprecated('file3', 'entity3', 'document', 'file3.pdf'),
   ],
 };
 

app/api/files.v2/infrastructure/S3FileStorage.ts

@@ -0,0 +1,57 @@
+import { _Object, ListObjectsV2Command, S3Client } from '@aws-sdk/client-s3';
+import { config } from 'api/config';
+import { Tenant } from 'api/tenants/tenantContext';
+import path from 'path';
+import { FileStorage } from '../contracts/FileStorage';
+import { Attachment } from '../model/Attachment';
+import { UwaziFile } from '../model/UwaziFile';
+import { URLAttachment } from '../model/URLAttachment';
+import { CustomUpload } from '../model/CustomUpload';
+
+export class S3FileStorage implements FileStorage {
+  private s3Client: S3Client;
+
+  private tenant: Tenant;
+
+  constructor(s3Client: S3Client, tenant: Tenant) {
+    this.s3Client = s3Client;
+    this.tenant = tenant;
+  }
+
+  getPath(file: UwaziFile): string {
+    if (file instanceof Attachment) {
+      return path.join(this.tenant.attachments, file.filename);
+    }
+    if (file instanceof CustomUpload) {
+      return path.join(this.tenant.customUploads, file.filename);
+    }
+    if (file instanceof URLAttachment) {
+      return 'not implemented';
+    }
+    return path.join(this.tenant.uploadedDocuments, file.filename);
+  }
+
+  async list(): Promise<string[]> {
+    const objects: _Object[] = [];
+    const requestNext = async (token?: string) => {
+      const response = await this.s3Client.send(
+        new ListObjectsV2Command({
+          Bucket: config.s3.bucket,
+          Prefix: `${this.tenant.name}/`,
+          ContinuationToken: token,
+          MaxKeys: config.s3.batchSize,
+        })
+      );
+      objects.push(...(response.Contents || []));
+      return response.NextContinuationToken;
+    };
+
+    let continuationToken = await requestNext();
+    while (continuationToken) {
+      // eslint-disable-next-line no-await-in-loop
+      continuationToken = await requestNext(continuationToken);
+    }
+
+    return objects.map(c => c.Key!);
+  }
+}