Configured elastic index to have replicas configurable by tenant (#7676)

* Configured elastic index to have replicas configurable by tenant

* Changed esUseReplicas to esReplicas: number

* Added flag for REINDEX_WITH_OPTIMIZATION

* Renamed wrongly named feature

* Upped version

app/api/config.ts

@@ -68,6 +68,7 @@ export const config = {
     activityLogs: ACTIVITY_LOGS_FOLDER || `${filesRootPath}/log/`,
     featureFlags: {
       s3Storage: false,
+      esReplicas: 0,
     },
   },
   externalServices: Boolean(process.env.EXTERNAL_SERVICES) || false,

app/api/search/entitiesIndex.js

@@ -10,11 +10,11 @@ import { MongoSettingsDataSource } from 'api/settings.v2/database/MongoSettingsD
 import { LanguageUtils } from 'shared/language';
 import { DefaultTransactionManager } from 'api/common.v2/database/data_source_defaults';
 import { otherLanguageSchema } from 'shared/language/availableLanguages';
-import elasticMapping from '../../../database/elastic_mapping/elastic_mapping';
+import { getTenantESMapping } from 'api/tenants/tenantESMapping';
 import elasticMapFactory from '../../../database/elastic_mapping/elasticMapFactory';
 import { elastic } from './elastic';
 
-export class IndexError extends Error {}
+class IndexError extends Error {}
 
 const preprocessEntitiesToIndex = async entitiesToIndex => {
   const db = getConnection();
@@ -183,9 +183,9 @@ const updateMapping = async tmpls => {
 
 const reindexAll = async (tmpls, searchInstance) => {
   await elastic.indices.delete();
-  await elastic.indices.create({ body: elasticMapping });
+  await elastic.indices.create({ body: getTenantESMapping() });
   await updateMapping(tmpls);
   return indexEntities({ query: {}, searchInstance });
 };
 
-export { bulkIndex, indexEntities, updateMapping, reindexAll };
+export { IndexError, bulkIndex, indexEntities, updateMapping, reindexAll };

app/api/tenants/specs/tenantESMapping.spec.ts

@@ -0,0 +1,29 @@
+import { tenants } from '../tenantContext';
+import { getTenantESMapping } from '../tenantESMapping';
+
+describe('tenantESMapping', () => {
+  describe('getTenantESMapping', () => {
+    it('should use the base elastic mapping', async () => {
+      tenants.add({
+        name: 'test-tenant',
+        dbName: 'test-tenant-db',
+      });
+
+      await tenants.run(async () => {
+        expect(getTenantESMapping().settings['index.number_of_replicas']).toBe(0);
+      }, 'test-tenant');
+    });
+
+    it('should use the append tenant specific configuration to base mapping', async () => {
+      tenants.add({
+        name: 'test-tenant',
+        dbName: 'test-tenant-db',
+        featureFlags: { esReplicas: 2 },
+      });
+
+      await tenants.run(async () => {
+        expect(getTenantESMapping().settings['index.number_of_replicas']).toBe(2);
+      }, 'test-tenant');
+    });
+  });
+});

app/api/tenants/specs/tenantsModel.spec.ts

@@ -51,6 +51,7 @@ describe('tenantsModel', () => {
         healthChecks: 'un-needed data',
         featureFlags: {
           s3Storage: false,
+          esReplicas: 1,
         },
       },
       {
@@ -82,6 +83,7 @@ describe('tenantsModel', () => {
         activityLogs: 'path',
         featureFlags: {
           s3Storage: false,
+          esReplicas: 1,
         },
       });
       expect(tenantTwo).toEqual({

app/api/tenants/tenantContext.ts

@@ -15,6 +15,7 @@ type Tenant = {
   activityLogs: string;
   featureFlags?: {
     s3Storage?: boolean;
+    esReplicas?: number;
     sync?: boolean;
     v1_transactions?: boolean;
   };

app/api/tenants/tenantESMapping.ts

@@ -0,0 +1,16 @@
+import { tenants } from './index';
+import elasticMapping from '../../../database/elastic_mapping/elastic_mapping';
+
+const getTenantESMapping = () => {
+  const tenantElasticMapping = {
+    settings: { ...elasticMapping.settings },
+    mappings: { ...elasticMapping.mappings },
+  };
+
+  tenantElasticMapping.settings['index.number_of_replicas'] =
+    tenants.current().featureFlags?.esReplicas || 0;
+
+  return tenantElasticMapping;
+};
+
+export { getTenantESMapping };

app/api/tenants/tenantsModel.ts

@@ -30,6 +30,7 @@ const mongoSchema = new mongoose.Schema({
   activityLogs: String,
   featureFlags: {
     s3Storage: Boolean,
+    esReplicas: Number,
     sync: Boolean,
     v1_transactions: Boolean,
   },

database/reindex_elastic.js

@@ -21,24 +21,40 @@ const headers = {
   'Content-Type': 'application/json',
 };
 
-const setReindexSettings = async (refreshInterval, numberOfReplicas, translogDurability) =>
-  fetch(`${getIndexUrl()}/_settings`, {
-    method: 'PUT',
-    headers,
-    body: {
-      index: {
-        refresh_interval: refreshInterval,
-        number_of_replicas: numberOfReplicas,
-        translog: {
-          durability: translogDurability,
-        },
+const setReindexSettings = async (refreshInterval, numberOfReplicas, translogDurability) => {
+  let body = {
+    index: {
+      refresh_interval: refreshInterval,
+      number_of_replicas: numberOfReplicas,
+      translog: {
+        durability: translogDurability,
       },
     },
+  };
+
+  if (process.env.REINDEX_WITH_OPTIMIZATION) {
+    body = JSON.stringify(body);
+  }
+
+  const result = await fetch(`${getIndexUrl()}/_settings`, {
+    method: 'PUT',
+    headers,
+    body,
   });
 
+  return result;
+};
+
 const restoreSettings = async () => {
   process.stdout.write('Restoring index settings...');
-  const result = setReindexSettings('1s', 0, 'request');
+
+  const tenantReplicas = tenants.current().featureFlags?.esReplicas || 0;
+
+  if (tenants.current().featureFlags?.esReplicas) {
+    process.stdout.write('restoring ES Replicas...');
+  }
+
+  const result = setReindexSettings('1s', tenantReplicas, 'request');
   process.stdout.write(' [done]\n');
   return result;
 };

package.json

@@ -1,6 +1,6 @@
 {
   "name": "uwazi",
-  "version": "1.201.0-rc3",
+  "version": "1.202.0-rc2",
   "description": "Uwazi is a free, open-source solution for organising, analysing and publishing your documents.",
   "keywords": [
     "react"