From 89ded7beb9b564aa47185b7f775d16122a36b48f Mon Sep 17 00:00:00 2001 From: zonagit Date: Fri, 27 Oct 2023 13:25:57 -0700 Subject: [PATCH] Proj a testing to main v2.4.0 testing (#177) * preliminary removing of simple db connection source * Using linux epoch timestamp in name of worker - * decreasing starting connections to 1, in pool * splitting out CWS and Camunda to two different JDBC pools and also setting removeAbandoned timeout value to be 15 minutes * Update WorkerMonitorBackgroundThread.java * Merge pull request #70 from NASA-AMMOS/issue-41-hikaricp Switch connection pool from Tomcat to HikariCP * making pool half the size * Allow custom adaptation actions to be added to the actions dropdown * Add ability to invoke adaptation js code to the workers page (#121) Co-authored-by: zef * Allow customization of the workers modal * External Adaptation DB Support (#127) * add adaptation jdbc connectors and configs * adaptation installer and server xml update * installer and configure script edit * cleaned Cwsinstaller adapt configs * remove log * new server adaptation plugin resource * change pool name for external db server * add qualifier for jdbc adaptation template * add cws-engine and cws-ui jdbc adapt template * empty commit * empty commit 2 * clean up xml file and naming convention * clear webdriver cache - update to 5.5.0 (#179) --------- Co-authored-by: Galen A Hollins Co-authored-by: Galen Hollins <47542238+galenatjpl@users.noreply.github.com> Co-authored-by: zef Co-authored-by: Josh Haile <35245966+voxparcxls@users.noreply.github.com> --- create_server_dist.sh | 1 + .../main/java/jpl/cws/core/db/DbService.java | 13 ++- .../main/java/jpl/cws/task/CwsInstaller.java | 85 +++++++++++++++++++ cws-test/pom.xml | 2 +- .../test/java/jpl/cws/test/WebTestUtil.java | 2 +- .../webapp/js/adaptation-workers-modal.js | 8 ++ .../src/main/webapp/js/adaptation-workers.js | 1 - install/configure.sh | 69 +++++++++++++-- install/context.xml | 2 + install/cws-engine/applicationContext.xml | 4 + install/cws-ui/adaptation-workers-modal.ftl | 9 ++ install/cws-ui/applicationContext.xml | 4 + install/cws-ui/deployments.ftl | 8 +- install/tomcat_conf/server.xml | 1 + install/tomcat_conf/server_adaptation.xml | 15 ++++ 15 files changed, 208 insertions(+), 16 deletions(-) create mode 100644 cws-ui/src/main/webapp/js/adaptation-workers-modal.js create mode 100644 install/cws-ui/adaptation-workers-modal.ftl create mode 100644 install/tomcat_conf/server_adaptation.xml diff --git a/create_server_dist.sh b/create_server_dist.sh index cc757833..21f73f83 100755 --- a/create_server_dist.sh +++ b/create_server_dist.sh @@ -68,6 +68,7 @@ cp ${INSTALL_DIR}/tomcat_lib/css-jaas.cfg ${CONFIG_TEMPLATES cp ${INSTALL_DIR}/tomcat_bin/setenv.sh ${CONFIG_TEMPLATES_DIR}/tomcat_bin cp ${INSTALL_DIR}/tomcat_conf/bpm-platform.xml ${CONFIG_TEMPLATES_DIR}/tomcat_conf cp ${INSTALL_DIR}/tomcat_conf/server.xml ${CONFIG_TEMPLATES_DIR}/tomcat_conf +cp ${INSTALL_DIR}/tomcat_conf/server_adaptation.xml ${CONFIG_TEMPLATES_DIR}/tomcat_conf cp ${INSTALL_DIR}/tomcat_conf/web.xml ${CONFIG_TEMPLATES_DIR}/tomcat_conf cp ${INSTALL_DIR}/tomcat_conf/ldap_plugin_bean.xml ${CONFIG_TEMPLATES_DIR}/tomcat_conf cp ${INSTALL_DIR}/tomcat_conf/ldap_plugin_ref.xml ${CONFIG_TEMPLATES_DIR}/tomcat_conf diff --git a/cws-core/src/main/java/jpl/cws/core/db/DbService.java b/cws-core/src/main/java/jpl/cws/core/db/DbService.java index 5366bd04..59842a0a 100644 --- a/cws-core/src/main/java/jpl/cws/core/db/DbService.java +++ b/cws-core/src/main/java/jpl/cws/core/db/DbService.java @@ -9,6 +9,7 @@ import org.slf4j.Logger; import org.slf4j.LoggerFactory; import org.springframework.beans.factory.annotation.Autowired; +import org.springframework.beans.factory.annotation.Qualifier; import org.springframework.jdbc.core.JdbcTemplate; import org.springframework.jdbc.core.namedparam.MapSqlParameterSource; import org.springframework.jdbc.core.namedparam.NamedParameterJdbcTemplate; @@ -22,9 +23,15 @@ public class DbService { private static final Logger log = LoggerFactory.getLogger(DbService.class); - @Autowired protected JdbcTemplate jdbcTemplate; - @Autowired protected NamedParameterJdbcTemplate namedJdbcTemplate; - + @Autowired + protected JdbcTemplate jdbcTemplate; + + @Autowired + protected NamedParameterJdbcTemplate namedJdbcTemplate; + + @Autowired + protected JdbcTemplate jdbcAdaptationTemplate; + protected static final long SLOW_WARN_THRESHOLD = 100; protected static final int IN_CLAUSE_MAX_ELEMENTS = 100; diff --git a/cws-installer/src/main/java/jpl/cws/task/CwsInstaller.java b/cws-installer/src/main/java/jpl/cws/task/CwsInstaller.java index 0fd50f29..d97ea564 100644 --- a/cws-installer/src/main/java/jpl/cws/task/CwsInstaller.java +++ b/cws-installer/src/main/java/jpl/cws/task/CwsInstaller.java @@ -149,6 +149,16 @@ public class CwsInstaller { private static String cws_db_username; private static String cws_db_password; + private static String cws_adapt_use_shared_db; + private static String cws_adapt_db_type; + private static String cws_adapt_db_host; + private static String cws_adapt_db_port; + private static String cws_adapt_db_name; + private static String cws_adapt_db_url; + private static String cws_adapt_db_driver; + private static String cws_adapt_db_username; + private static String cws_adapt_db_password; + private static String cws_auth_scheme; private static String cws_user; private static String cws_user_firstname; @@ -1571,6 +1581,34 @@ private static void showInstallationInfo() { print("Database User = " + cws_db_username); print("Database Password = ****** (hidden) "); print("...................................................................................."); + + cws_adapt_use_shared_db = getPreset("adaptation_use_shared_db"); + cws_adapt_db_type = getPreset("adaptation_db_type"); + cws_adapt_db_host = getPreset("adaptation_db_host"); + cws_adapt_db_port = getPreset("adaptation_db_port"); + cws_adapt_db_name = getPreset("adaptation_db_name"); + cws_adapt_db_username = getPreset("adaptation_db_username"); + cws_adapt_db_password = getPreset("adaptation_db_password"); + cws_adapt_db_url = "jdbc:" + cws_adapt_db_type + "://" + cws_adapt_db_host + ":" + cws_adapt_db_port + "/" + cws_adapt_db_name + "?autoReconnect=true"; + + if (cws_adapt_use_shared_db != null) { + if (cws_adapt_use_shared_db.equalsIgnoreCase("y")) { + + if (cws_adapt_db_type.equals("mariadb")) { + cws_adapt_db_driver = "org.mariadb.jdbc.Driver"; + } else if (cws_adapt_db_type.equals("mysql")) { + cws_adapt_db_driver = "com.mysql.jdbc.Driver"; + } + print(" CWS Adapt Database "); + print(" Database Type = " + cws_adapt_db_type); + print(" Database URL = " + cws_adapt_db_url); + print(" Database Driver = " + cws_adapt_db_driver); + print(" Database User = " + cws_adapt_db_username); + print(" Database Password = ****** (hidden) "); + } + } + print("...................................................................................."); + if (cws_auth_scheme.equals("LDAP")) { print("LDAP User = " + cws_user); } @@ -2140,6 +2178,8 @@ private static void createFreshWorkDir() { Paths.get(config_work_dir + SEP + "tomcat_conf" + SEP + "bpm-platform.xml")); copy(Paths.get( config_templates_dir + SEP + "tomcat_conf" + SEP + "server.xml"), Paths.get(config_work_dir + SEP + "tomcat_conf" + SEP + "server.xml")); + copy(Paths.get( config_templates_dir + SEP + "tomcat_conf" + SEP + "server_adaptation.xml"), + Paths.get(config_work_dir + SEP + "tomcat_conf" + SEP + "server_adaptation.xml")); copy(Paths.get( config_templates_dir + SEP + "tomcat_conf" + SEP + "web.xml"), Paths.get(config_work_dir + SEP + "tomcat_conf" + SEP + "web.xml")); copy(Paths.get( config_templates_dir + SEP + "engine-rest_mods" + SEP + "web.xml"), @@ -2208,6 +2248,28 @@ private static void updateFiles() throws IOException { content = content.replace("__CWS_DB_DRIVER__", cws_db_driver); content = content.replace("__CWS_DB_USERNAME__", cws_db_username); content = content.replace("__CWS_DB_PASSWORD__", cws_db_password); + + writeToFile(filePath, content); + copy( + Paths.get(config_work_dir + SEP + "tomcat_conf" + SEP + "server_adaptation.xml"), + Paths.get(cws_tomcat_root + SEP + "conf" + SEP + "server_adaptation.xml")); + if (cws_adapt_db_url != null && cws_adapt_db_driver != null && + cws_adapt_db_username != null && cws_adapt_db_password != null) { + + // Fill in the __ADAPTATION_SERVER_RESOURCE__ + String serverAdaptationContent = getFileContents( + Paths.get(config_work_dir + SEP + "tomcat_conf" + SEP + "server_adaptation.xml")); + content = content.replace(""); + content = content.replace("__CWS_ADAPTATION_BEAN__", "\n" + + "\t\t\n" + + "\t"); + } else { + content = content.replace("__CWS_ADAPTATION_JNDI__", ""); + content = content.replace("__CWS_ADAPTATION_BEAN__", ""); + } + content = content.replace("__CWS_DB_DRIVER__", cws_db_driver); content = content.replace("__CWS_DB_URL__", cws_db_url); content = content.replace("__CWS_DB_USERNAME__", cws_db_username); @@ -2490,6 +2563,18 @@ private static void updateCwsUiConfig() throws IOException { print(" Updating cws-ui/applicationContext.xml..."); Path path = Paths.get(config_work_dir + SEP + "cws-ui" + SEP + "applicationContext.xml"); String content = getFileContents(path); + + if (cws_adapt_db_url != null && cws_adapt_db_driver != null && + cws_adapt_db_username != null && cws_adapt_db_password != null) { + content = content.replace("__CWS_ADAPTATION_JNDI__", ""); + content = content.replace("__CWS_ADAPTATION_BEAN__", "\n" + + "\t\t\n" + + "\t"); + } else { + content = content.replace("__CWS_ADAPTATION_JNDI__", ""); + content = content.replace("__CWS_ADAPTATION_BEAN__", ""); + } + content = content.replace("__CWS_DB_DRIVER__", cws_db_driver); content = content.replace("__CWS_DB_URL__", cws_db_url); content = content.replace("__CWS_DB_USERNAME__", cws_db_username); diff --git a/cws-test/pom.xml b/cws-test/pom.xml index 330979f7..c851db8f 100644 --- a/cws-test/pom.xml +++ b/cws-test/pom.xml @@ -114,7 +114,7 @@ io.github.bonigarcia webdrivermanager - 5.2.0 + 5.5.0 test diff --git a/cws-test/src/test/java/jpl/cws/test/WebTestUtil.java b/cws-test/src/test/java/jpl/cws/test/WebTestUtil.java index 67b636af..7fffe03f 100644 --- a/cws-test/src/test/java/jpl/cws/test/WebTestUtil.java +++ b/cws-test/src/test/java/jpl/cws/test/WebTestUtil.java @@ -123,7 +123,7 @@ protected void initChromeDriver() { chromeOptions.addArguments("--disable-gpu"); chromeOptions.addArguments("--disable-dev-shm-usage"); - WebDriverManager.chromedriver().setup(); + WebDriverManager.chromedriver().clearDriverCache().setup(); driver = new ChromeDriver(chromeOptions); log.info("Driver initialized: " + driver); diff --git a/cws-ui/src/main/webapp/js/adaptation-workers-modal.js b/cws-ui/src/main/webapp/js/adaptation-workers-modal.js new file mode 100644 index 00000000..a8870084 --- /dev/null +++ b/cws-ui/src/main/webapp/js/adaptation-workers-modal.js @@ -0,0 +1,8 @@ +// override with js code file in your adaptation project for example like this +// cp your-cws-adaptation/src/main/resources/cws-ui/adaptation-deployments.js common-workflow-service/cws-ui/src/web/main/webapp/js/adaptation-deployments.js +// This adaptation allows extra information to be added to each worker. The particular use case that +// drove this need is a request to add the aws scaling class each worker belongs to +function addAdaptationWorkersInfo(dataProcKey, listWorkers) { + return; // Common workflow behavior is to do nothing +} + diff --git a/cws-ui/src/main/webapp/js/adaptation-workers.js b/cws-ui/src/main/webapp/js/adaptation-workers.js index 48f770bc..10949ea1 100644 --- a/cws-ui/src/main/webapp/js/adaptation-workers.js +++ b/cws-ui/src/main/webapp/js/adaptation-workers.js @@ -3,4 +3,3 @@ function adaptationWorkersReady() { return; // CWS core behavior is to do nothing here } - diff --git a/install/configure.sh b/install/configure.sh index 2dbf3db2..ef0e7897 100755 --- a/install/configure.sh +++ b/install/configure.sh @@ -318,16 +318,66 @@ if [[ "${CWS_INSTALL_TYPE}" = "1" ]] || [[ "${CWS_INSTALL_TYPE}" = "2" ]]; then # Record the fact that tables have been created touch ${ROOT}/.databaseTablesCreated + # Create any adaptation tables, if provided - if [[ -f ${ROOT}/sql/cws/adaptation.sql ]]; then - print "Creating adaptation tables..." - mysql --defaults-file=${ROOT}/config/my.cnf ${DB_NAME} < ${ROOT}/sql/cws/adaptation.sql - if [[ $? -gt 0 ]]; then - print "ERROR: Problem creating adaptation tables." - print " Please check your database configuration and/or adaptation script '${ROOT}/sql/cws/adaptation.sql', and try again." - rm -rf ${ROOT}/config/my.cnf - exit 1 - fi + if [[ -f ${ROOT}/sql/cws/adaptation.sql && -f ${ROOT}/sql/cws/adaptation_core.sql && -f ${ROOT}/sql/cws/adaptation_external.sql ]]; then + print "Checking for Adaptation DB Settings..." + ADAPT_USE_SHARED_DB=`grep adaptation_use_shared_db ${CWS_INSTALLER_PRESET_FILE} | grep -v "^#" | cut -d"=" -f 2` + + if [[ "${ADAPT_USE_SHARED_DB}" == "y" ]]; then + ADAPT_DB_HOST=`grep adaptation_db_host ${CWS_INSTALLER_PRESET_FILE} | grep -v "^#" | cut -d"=" -f 2` + ADAPT_DB_NAME=`grep adaptation_db_name ${CWS_INSTALLER_PRESET_FILE} | grep -v "^#" | cut -d"=" -f 2` + ADAPT_DB_USER=`grep adaptation_db_username ${CWS_INSTALLER_PRESET_FILE} | grep -v "^#" | cut -d"=" -f 2` + ADAPT_DB_PASS=`grep adaptation_db_password ${CWS_INSTALLER_PRESET_FILE} | grep -v "^#" | cut -d"=" -f 2` + + echo "[mysql]" > ${ROOT}/config/myadapt.cnf + echo "host=${ADAPT_DB_HOST}" >> ${ROOT}/config/myadapt.cnf + echo "user=\"${ADAPT_DB_USER}\"" >> ${ROOT}/config/myadapt.cnf + echo "password=\"${ADAPT_DB_PASS}\"" >> ${ROOT}/config/myadapt.cnf + chmod 644 ${ROOT}/config/myadapt.cnf + + cat ${ROOT}/config/myadapt.cnf + + # ---------------------------------------- + print "Your adaptation database configuration is:" + print " DB HOST: ${ADAPT_DB_HOST}" + print " DB NAME: ${ADAPT_DB_NAME}" + print " DB USER: ${ADAPT_DB_USER}" + print "This script will now create the database necessary for CWS to function." + + + print "Checking whether database ${ADAPT_DB_NAME} already exists..." + RES=`mysql --defaults-file=${ROOT}/config/myadapt.cnf -e "SELECT SCHEMA_NAME FROM INFORMATION_SCHEMA.SCHEMATA WHERE SCHEMA_NAME = '${ADAPT_DB_NAME}'"` + + if [[ $? -gt 0 ]]; then + print "ERROR: Problem checking for database. " + print " Please check your database configuration, and try again." + rm -f ${ROOT}/config/myadapt.cnf + exit 1 + fi + print " Database ${ADAPT_DB_NAME} exists." + + print "Creating adaptation tables in External DB..." + mysql --defaults-file=${ROOT}/config/my.cnf ${DB_NAME} < ${ROOT}/sql/cws/adaptation_core.sql + mysql --defaults-file=${ROOT}/config/myadapt.cnf ${ADAPT_DB_NAME} < ${ROOT}/sql/cws/adaptation_external.sql + if [[ $? -gt 0 ]]; then + print "ERROR: Problem creating adaptation tables." + print " Please check your database configuration and/or adaptation script '${ROOT}/sql/cws/adaptation_external.sql', and try again." + rm -rf ${ROOT}/config/myadapt.cnf + exit 1 + fi + else + print "Creating adaptation tables in Default Core DB..." + if [[ -f ${ROOT}/sql/cws/adaptation.sql ]]; then + mysql --defaults-file=${ROOT}/config/my.cnf ${DB_NAME} < ${ROOT}/sql/cws/adaptation.sql + if [[ $? -gt 0 ]]; then + print "ERROR: Problem creating adaptation tables." + print " Please check your database configuration and/or adaptation script '${ROOT}/sql/cws/adaptation.sql', and try again." + rm -rf ${ROOT}/config/my.cnf + exit 1 + fi + fi + fi # Record the fact that we applied a database adaptation touch ${ROOT}/.adaptationTablesCreated @@ -346,5 +396,6 @@ if [[ "${CWS_INSTALL_TYPE}" = "1" ]] || [[ "${CWS_INSTALL_TYPE}" = "2" ]]; then fi rm -f ${ROOT}/config/my.cnf +rm -f ${ROOT}/config/myadapt.cnf print "Finished" diff --git a/install/context.xml b/install/context.xml index b5adf33b..3ff3c4e7 100644 --- a/install/context.xml +++ b/install/context.xml @@ -36,4 +36,6 @@ + + \ No newline at end of file diff --git a/install/cws-engine/applicationContext.xml b/install/cws-engine/applicationContext.xml index 3ddefccf..64388dc4 100644 --- a/install/cws-engine/applicationContext.xml +++ b/install/cws-engine/applicationContext.xml @@ -22,6 +22,8 @@ + __CWS_ADAPTATION_JNDI__ + @@ -30,6 +32,8 @@ + __CWS_ADAPTATION_BEAN__ + diff --git a/install/cws-ui/adaptation-workers-modal.ftl b/install/cws-ui/adaptation-workers-modal.ftl new file mode 100644 index 00000000..7c9e4e67 --- /dev/null +++ b/install/cws-ui/adaptation-workers-modal.ftl @@ -0,0 +1,9 @@ + + diff --git a/install/cws-ui/applicationContext.xml b/install/cws-ui/applicationContext.xml index 8f438875..f7e77707 100644 --- a/install/cws-ui/applicationContext.xml +++ b/install/cws-ui/applicationContext.xml @@ -25,6 +25,8 @@ + __CWS_ADAPTATION_JNDI__ + @@ -33,6 +35,8 @@ + __CWS_ADAPTATION_BEAN__ + diff --git a/install/cws-ui/deployments.ftl b/install/cws-ui/deployments.ftl index dc44ea71..f623134e 100644 --- a/install/cws-ui/deployments.ftl +++ b/install/cws-ui/deployments.ftl @@ -6,6 +6,8 @@ + +