Merge pull request #2615 from huridocs/multidb

support easy yarn dump-db and restore-db & allow easy db switching via .env

README.md

@@ -102,10 +102,12 @@ $ yarn run-production
 
 ### Environment Variables
 
-Uwazi supports the following environment variables to customize its deployment:
+Uwazi supports the following environment variables to customize its deployment
+(`.env` is supported but not stored in the repository):
 
 * `DBHOST`: MongoDB hostname (default: `localhost`)
-* `DATABASE_NAME`: MongoDB instance name
+* `DATABASE_NAME`: MongoDB instance name (default: `uwazi_development`)
+* `INDEX_NAME`: Elastic search index name (default: `DATABASE_NAME`)
 * `ELASTICSEARCH_URL`: ElasticSearch connection URL (default: `http://localhost:9200`)
 * `UPLOADS_FOLDER`: Folder on local filesystem where uploaded PDF and other files are written to (_TODO temporarily or permanently?_)
 

app/api/config/database.js

@@ -1,8 +1,10 @@
 const { DBHOST } = process.env;
 const { DATABASE_NAME } = process.env;
+
+// Keep in sync with elasticindexes.js!
 export default {
   demo: 'mongodb://localhost/uwazi_demo',
-  development: DBHOST ? `mongodb://${DBHOST}/${DATABASE_NAME}` : 'mongodb://localhost/uwazi_development',
-  testing: DBHOST ? `mongodb://${DBHOST}/${DATABASE_NAME}` : 'mongodb://localhost/uwazi_testing',
-  production: DBHOST ? `mongodb://${DBHOST}/${DATABASE_NAME}` : 'mongodb://localhost/uwazi_development'
+  development: `mongodb://${DBHOST || 'localhost'}/${DATABASE_NAME || 'uwazi_development'}`,
+  testing: `mongodb://${DBHOST || 'localhost'}/${DATABASE_NAME || 'uwazi_testing'}`,
+  production: `mongodb://${DBHOST || 'localhost'}/${DATABASE_NAME || 'uwazi_development'}`
 };

app/api/config/elasticIndexes.js

@@ -1,8 +1,9 @@
-const { INDEX_NAME } = process.env;
+const { DATABASE_NAME, INDEX_NAME } = process.env;
 
+// Keep in sync with database.js!
 export default {
-  production: INDEX_NAME || 'uwazi_development',
-  testing: INDEX_NAME || 'testing',
-  development: INDEX_NAME || 'uwazi_development',
-  demo: INDEX_NAME || 'uwazi_demo'
+  demo: INDEX_NAME || DATABASE_NAME || 'uwazi_demo',
+  development: INDEX_NAME || DATABASE_NAME || 'uwazi_development',
+  testing: INDEX_NAME || DATABASE_NAME || 'testing',
+  production: INDEX_NAME || DATABASE_NAME || 'uwazi_development',
 };

app/server.js

@@ -1,4 +1,7 @@
+/** @format */
+
 /* eslint-disable no-console */
+
 import bodyParser from 'body-parser';
 import compression from 'compression';
 import express from 'express';
@@ -34,7 +37,7 @@ app.use(helmet());
 
 const http = Server(app);
 
-const uncaughtError = (error) => {
+const uncaughtError = error => {
   handleError(error, { uncaught: true });
   process.exit(1);
 };
@@ -64,7 +67,6 @@ app.use('/assets', express.static(paths.customUploads));
 // retained for backwards compatibility
 app.use('/uploaded_documents', express.static(paths.customUploads));
 
-
 apiRoutes(app, http);
 
 serverRenderingRoutes(app);
@@ -81,8 +83,8 @@ if (process.env.DBUSER) {
   };
 }
 
-mongoose.connect(dbConfig[app.get('env')], { ...dbAuth })
-.then(async () => {
+console.info('==> Connecting to', dbConfig[app.get('env')]);
+mongoose.connect(dbConfig[app.get('env')], { ...dbAuth }).then(async () => {
   console.info('==> Processing system keys...');
   await translations.processSystemKeys(systemKeys);
 
@@ -94,7 +96,7 @@ mongoose.connect(dbConfig[app.get('env')], { ...dbAuth })
 
   const port = ports[app.get('env')];
 
-  const bindAddress = ({ true: 'localhost' })[process.env.LOCALHOST_ONLY];
+  const bindAddress = { true: 'localhost' }[process.env.LOCALHOST_ONLY];
 
   semanticSearchManager.start();
 
@@ -104,13 +106,14 @@ mongoose.connect(dbConfig[app.get('env')], { ...dbAuth })
     const { evidencesVault } = await settings.get();
     if (evidencesVault && evidencesVault.token && evidencesVault.template) {
       console.info('==> 📥 evidences vault config detected, started sync ....');
-      repeater.start(
-        () => vaultSync.sync(evidencesVault.token, evidencesVault.template),
-        10000
-      );
+      repeater.start(() => vaultSync.sync(evidencesVault.token, evidencesVault.template), 10000);
     }
 
-    console.info('==> 🌎 Listening on port %s. Open up http://localhost:%s/ in your browser.', port, port);
+    console.info(
+      '==> 🌎 Listening on port %s. Open up http://localhost:%s/ in your browser.',
+      port,
+      port
+    );
     if (process.env.HOT) {
       console.info('');
       console.info('==> 📦 webpack is watching...');

database/blank_state.sh

@@ -1,5 +1,7 @@
 #!/bin/bash
 
+[[ -f ".env" ]] && source ".env"
+
 parent_path=$( cd "$(dirname "${BASH_SOURCE[0]}")" ; pwd -P )
 cd "$parent_path"
 
@@ -9,3 +11,7 @@ HOST=${2:-${DBHOST:-127.0.0.1}}
 echo -e "\n\nDeleting $DB database on $HOST"
 mongo -host $HOST $DB --eval "db.dropDatabase()"
 mongorestore -h $HOST blank_state/uwazi_development/ --db=$DB
+
+export DATABASE_NAME=$DB
+yarn migrate
+yarn reindex

database/dump_db.sh

@@ -0,0 +1,16 @@
+#!/bin/bash
+
+[[ -f ".env" ]] && source ".env"
+
+OLDWD=$( cd "$(dirname "${BASH_SOURCE[0]}")"/.. ; pwd -P )
+cd /tmp
+
+DB=${1:-${DATABASE_NAME:-uwazi_development}}
+HOST=${2:-${DBHOST:-127.0.0.1}}
+
+rm -rf $DB*
+mongodump -h $HOST --gzip --db $DB -o $DB
+cp -r $OLDWD/uploaded_documents $DB/
+tar czf $OLDWD/$DB.tar.gz $DB/
+
+echo "Wrote $DB.tar.gz."

database/restore_db.sh

@@ -0,0 +1,27 @@
+#!/bin/bash
+
+[[ -f ".env" ]] && source ".env"
+
+OLDWD=$( cd "$(dirname "${BASH_SOURCE[0]}")"/.. ; pwd -P )
+cd /tmp
+
+DB=${1:-${DATABASE_NAME:-uwazi_development}}
+HOST=${2:-${DBHOST:-127.0.0.1}}
+
+if [ ! -f "$OLDWD/$DB.tar.gz" ]; then
+  echo "Did not find expected $OLDWD/$DB.tar.gz - run restore-db <database name> to pick a different db."
+  exit -1
+fi
+
+rm -rf $DB*
+tar xzf $OLDWD/$DB.tar.gz
+mongo -host $HOST $DB --eval "db.dropDatabase()"
+mongorestore -h $HOST --gzip --db $DB $DB/$DB
+cp $DB/uploaded_documents/* $OLDWD/uploaded_documents
+
+echo "Restored $DB.tar.gz."
+
+cd $OLDWD
+export DATABASE_NAME=$DB
+yarn migrate
+yarn reindex

package.json

@@ -39,8 +39,10 @@
     "import-csv": "node run.js ./database/import_csv.js",
     "import-evidences": "node run.js ./database/import_evidences_vault.js",
     "vault-evidences-template": "node run.js ./database/vault_evidences_template.js",
-    "blank-state": "./database/blank_state.sh; yarn migrate; yarn reindex",
+    "blank-state": "./database/blank_state.sh",
     "admin-user": "./database/admin_user.sh",
+    "dump-db": "./database/dump_db.sh",
+    "restore-db": "./database/restore_db.sh",
     "fixtures": "cd uwazi-fixtures; ./restore.sh; yarn reindex",
     "add-migration": "./node_modules/.bin/plop --plopfile ./app/api/migrations/plopfile.js migration",
     "migrate": "node run.js ./app/api/migrations/migrate.js",
@@ -95,6 +97,7 @@
     "crypto-js": "^3.1.9-1",
     "csvtojson": "^2.0.8",
     "diacritics": "^1.3.0",
+    "dotenv": "^8.2.0",
     "elasticsearch": "15.1.1",
     "es6-promise": "4.2.4",
     "express": "^4.13.3",

run.js

@@ -1,3 +1,4 @@
+require('dotenv').config();
 require('@babel/register')({ extensions: ['.js', '.jsx', '.ts', '.tsx'] });
 
 process.env.ROOT_PATH = process.env.ROOT_PATH || __dirname;

server.js

@@ -1,3 +1,5 @@
+require('dotenv').config();
+
 process.env.ROOT_PATH = process.env.ROOT_PATH || __dirname;
 const { NODE_ENV } = process.env;
 

yarn.lock

@@ -4282,6 +4282,11 @@ dot-prop@^4.1.0:
   dependencies:
     is-obj "^1.0.0"
 
+dotenv@^8.2.0:
+  version "8.2.0"
+  resolved "https://registry.yarnpkg.com/dotenv/-/dotenv-8.2.0.tgz#97e619259ada750eea3e4ea3e26bceea5424b16a"
+  integrity sha512-8sJ78ElpbDJBHNeBzUbUVLsqKdccaa/BXF1uPTw3GrvQTBgrQrtObr2mUrE38vzYd8cEv+m/JBfDLioYcfXoaw==
+
 download@^4.0.0, download@^4.1.2:
   version "4.4.3"
   resolved "https://registry.yarnpkg.com/download/-/download-4.4.3.tgz#aa55fdad392d95d4b68e8c2be03e0c2aa21ba9ac"